├── .editorconfig ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── bin └── pdf-bot.js ├── examples ├── pdf-bot.config.js └── receiving-api.js ├── package.json ├── production ├── README.md ├── nginx.conf └── pm2.config.js ├── src ├── api.js ├── error.js ├── pdfGenerator.js ├── queue.js ├── storage │ └── s3.js ├── utils.js └── webhook.js ├── storage ├── db │ └── .gitignore └── pdf │ └── .gitignore └── test ├── api.test.js ├── error.test.js ├── pdfGenerator.test.js ├── queue.test.js ├── storage └── s3.test.js └── webhook.test.js /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | # Spaces in coffee 13 | [**.coffee] 14 | indent_style = space 15 | indent_size = 2 16 | 17 | [**.js] 18 | indent_style = space 19 | indent_size = 2 20 | 21 | [**.jsx] 22 | indent_style = space 23 | indent_size = 2 24 | 25 | # Tabs in less 26 | [**.less] 27 | indent_style = tab 28 | indent_size = 2 29 | 30 | [**.css] 31 | indent_style = tab 32 | indent_size = 2 33 | 34 | [**.php] 35 | indent_style = space 36 | indent_size = 4 37 | 38 | [**.html] 39 | indent_style = tab 40 | indent_size = 2 41 | 42 | [Makefile] 43 | indent_style = tab 44 | indent_size = 4 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | .DS_Store 61 | package-lock.json 62 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "node" 4 | script: 5 | - npm test 6 | after_success: 7 | - cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Esben Petersen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🤖 pdf-bot 2 | 3 | [![npm](https://img.shields.io/npm/v/pdf-bot.svg)](https://www.npmjs.com/package/pdf-bot) [![Build Status](https://travis-ci.org/esbenp/pdf-bot.svg?branch=master)](https://travis-ci.org/esbenp/pdf-bot) [![Coverage Status](https://coveralls.io/repos/github/esbenp/pdf-bot/badge.svg?branch=master)](https://coveralls.io/github/esbenp/pdf-bot?branch=master) 4 | 5 | Easily create a microservice for generating PDFs using headless Chrome. 6 | 7 | `pdf-bot` is installed on a server and will receive URLs to turn into PDFs through its API or CLI. `pdf-bot` will manage a queue of PDF jobs. Once a PDF job has run it will notify you using a webhook so you can fetch the API. `pdf-bot` supports storing PDFs on S3 out of the box. Failed PDF generations and Webhook pings will be retried after a configurable decaying schedule. 8 | 9 | ![How to use the pdf-bot CLI](http://imgur.com/aRHye2l.gif) 10 | 11 | `pdf-bot` uses [`html-pdf-chrome`](https://github.com/westy92/html-pdf-chrome) under the hood and supports all the settings that it supports. Major thanks to [@westy92](https://github.com/westy92/html-pdf-chrome) for making this possible. 12 | 13 | ## How does it work? 14 | 15 | Imagine you have an app that creates invoices. You want to save those invoices as PDF. You install `pdf-bot` on a server as an API. Your app server sends the URL of the invoice to the `pdf-bot` server. A cronjob on the `pdf-bot` server keeps checking for new jobs, generates a PDF using headless Chrome and sends the location back to the application server using a webhook. 16 | 17 | ## Prerequisites 18 | 19 | * Node.js v6 or later 20 | 21 | ## Installation 22 | 23 | ```bash 24 | $ npm install -g pdf-bot 25 | $ pdf-bot install 26 | ``` 27 | 28 | > Make sure the node path is in your $PATH 29 | 30 | `pdf-bot install` will prompt for some basic configurations and then create a storage folder where your database and pdf files will be saved. 31 | 32 | ### Configuration 33 | 34 | `pdf-bot` comes packaged with sensible defaults. At the very minimum you must have a config file in the same folder from which you are executing `pdf-bot` with a `storagePath` given. However, in reality what you probably want to do is use the `pdf-bot install` command to generate a configuration file and then use an alias `ALIAS pdf-bot = "pdf-bot -c /home/pdf-bot.config.js"` 35 | 36 | `pdf-bot.config.js` 37 | ```js 38 | var htmlPdf = require('html-pdf-chrome') 39 | 40 | module.exports = { 41 | api: { 42 | token: 'crazy-secret' 43 | }, 44 | generator: { 45 | completionTrigger: new htmlPdf.CompletionTrigger.Timer(1000) // 1 sec timeout 46 | }, 47 | storagePath: 'storage' 48 | } 49 | ``` 50 | 51 | ```bash 52 | $ pdf-bot -c ./pdf-bot.config.js push https://esbenp.github.io 53 | ``` 54 | 55 | [See a full list of the available configuration options.](#options) 56 | 57 | ## Usage guide 58 | 59 | ### Structure and concept 60 | 61 | `pdf-bot` is meant to be a microservice that runs a server to generate PDFs for you. That usually means you will send requests from your application server to the PDF server to request an url to be generated as a PDF. `pdf-bot` will manage a queue and retry failed generations. Once a job is successfully generated a path to it will be sent back to your application server. 62 | 63 | Let us check out the flow for an app that generates PDF invoices. 64 | 65 | ``` 66 | 1. (App server): An invoice is created ----> Send URL to invoice to pdf-bot server 67 | 2. (pdf-bot server): Put the URL in the queue 68 | 3. (pdf-bot server): PDF is generated using headless Chrome 69 | 4. (pdf-bot server): (if failed try again using 1 min, 3 min, 10 min, 30 min, 60 min delay) 70 | 5. (pdf-bot server): Upload PDF to storage (e.g. Amazon S3) 71 | 6. (pdf-bot server): Send S3 location of PDF back to the app server 72 | 7. (App server): Receive S3 location of PDF -> Check signature sum matches for security 73 | 8. (App server): Handle PDF however you see fit (move it, download it, save it etc.) 74 | ``` 75 | 76 | You can send meta data to the `pdf-bot` server that will be sent back to the application. This can help you identify what PDF you are receiving. 77 | 78 | ### Setup 79 | 80 | On your `pdf-bot` server start by creating a config file `pdf-bot.config.js`. [You can see an example file here](https://github.com/esbenp/pdf-bot/blob/master/examples/pdf-bot.config.js) 81 | 82 | `pdf-bot.config.js` 83 | ```js 84 | module.exports = { 85 | api: { 86 | port: 3000, 87 | token: 'api-token' 88 | }, 89 | storage: { 90 | 's3': createS3Config({ 91 | bucket: '', 92 | accessKeyId: '', 93 | region: '', 94 | secretAccessKey: '' 95 | }) 96 | }, 97 | webhook: { 98 | secret: '1234', 99 | url: 'http://localhost:3000/webhooks/pdf' 100 | } 101 | } 102 | ``` 103 | 104 | As a minimum you should configure an access token for your API. This will be used to authenticate jobs sent to your `pdf-bot` server. You also need to add a `webhook` configuration to have pdf notifications sent back to your application server. You should add a `secret` that will be used to generate a signature used to check that the request has not been tampered with during transfer. 105 | 106 | Start your API using 107 | 108 | `pdf-bot -c ./pdf-bot.config.js api` 109 | 110 | This will start an [express server](http://expressjs.com) that listens for new jobs on port `3000`. 111 | 112 | #### Setting up Chrome 113 | 114 | `pdf-bot` uses [html-pdf-chrome](https://github.com/westy92/html-pdf-chrome) which in turns uses [chrome-launcher](https://github.com/GoogleChrome/lighthouse/tree/master/chrome-launcher) to launch chrome. You should check out those two resources on how to properly setup Chrome. However, with `chrome-launcher` Chrome should be started automatically. Otherwise, `html-pdf-chrome` has a small guide on how to have it running as a process using `pm2`. 115 | 116 | You can install chrome on Ubuntu using 117 | 118 | ``` 119 | sudo apt-get update && apt-get install chromium-browser 120 | ``` 121 | 122 | If you are testing things on OSX or similar, `chrome-launcher` should be able to find and automatically startup Chrome for you. 123 | 124 | #### Setting up the receiving API 125 | 126 | In the [examples folder](https://github.com/esbenp/pdf-bot/blob/master/examples/receiving-api.js) there is a small example on how the application API could look. Basically, you just have to define an endpoint that will receive the webhook and check that the signature matches. 127 | 128 | ```javascript 129 | api.post('/hook', function (req, res) { 130 | var signature = req.get('X-PDF-Signature', 'sha1=') 131 | 132 | var bodyCrypted = require('crypto') 133 | .createHmac('sha1', '12345') 134 | .update(JSON.stringify(req.body)) 135 | .digest('hex') 136 | 137 | if (bodyCrypted !== signature) { 138 | res.status(401).send() 139 | return 140 | } 141 | 142 | console.log('PDF webhook received', JSON.stringify(req.body)) 143 | 144 | res.status(204).send() 145 | }) 146 | ``` 147 | 148 | ### Setup production environment 149 | 150 | [Follow the guide under `production/` to see how to setup `pdf-bot` using `pm2` and `nginx`](https://github.com/esbenp/pdf-bot/blob/master/production/README.md) 151 | 152 | ### Setup crontab 153 | 154 | We setup our crontab to continuously look for jobs that have not yet been completed. 155 | 156 | ```bash 157 | * * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js shift >> /var/log/pdfbot.log 2>&1 158 | * * * * * node $(npm bin -g)/pdf-bot -c ./pdf-bot.config.js ping:retry-failed >> /var/log/pdfbot.log 2>&1 159 | ``` 160 | 161 | ### Quick example using the CLI 162 | 163 | Let us assume I want to generate a PDF for `https://esbenp.github.io`. I can add the job using the `pdf-bot` CLI. 164 | 165 | ```bash 166 | $ pdf-bot -c ./pdf-bot.config.js push https://esbenp.github.io --meta '{"id":1}' 167 | ``` 168 | 169 | Next, if my crontab is not setup to run it automatically I can run it using the `shift` command 170 | 171 | ```bash 172 | $ pdf-bot -c ./pdf-bot.config.js shift 173 | ``` 174 | 175 | This will look for the oldest uncompleted job and run it. 176 | 177 | ### How can I generate PDFs for sites that use Javascript? 178 | 179 | This is a common issue with PDF generation. Luckily, `html-pdf-chrome` has a really awesome API for dealing with Javascript. You can specify a timeout in milliseconds, wait for elements or custom events. To add a wait simply configure the `generator` key in your configuration. Below are a few examples. 180 | 181 | **Wait for 5 seconds** 182 | 183 | ```javascript 184 | var htmlPdf = require('html-pdf-chrome') 185 | 186 | module.exports = { 187 | api: { 188 | token: 'api-token' 189 | }, 190 | // html-pdf-chrome options 191 | generator: { 192 | completionTrigger: new htmlPdf.CompletionTrigger.Timer(5000), // waits for 5 sec 193 | }, 194 | webhook: { 195 | secret: '1234', 196 | url: 'http://localhost:3000/webhooks/pdf' 197 | } 198 | } 199 | ``` 200 | 201 | **Wait for event** 202 | 203 | ```javascript 204 | var htmlPdf = require('html-pdf-chrome') 205 | 206 | module.exports = { 207 | api: { 208 | token: 'api-token' 209 | }, 210 | // html-pdf-chrome options 211 | generator: { 212 | completionTrigger: new htmlPdf.CompletionTrigger.Event( 213 | 'myEvent', // name of the event to listen for 214 | '#myElement', // optional DOM element CSS selector to listen on, defaults to body 215 | 5000 // optional timeout (milliseconds) 216 | ) 217 | }, 218 | webhook: { 219 | secret: '1234', 220 | url: 'http://localhost:3000/webhooks/pdf' 221 | } 222 | } 223 | ``` 224 | 225 | In your Javascript trigger the event when rendering is complete 226 | 227 | ```javascript 228 | document.getElementById('myElement').dispatchEvent(new CustomEvent('myEvent')); 229 | ``` 230 | 231 | **Wait for variable** 232 | 233 | ```javascript 234 | var htmlPdf = require('html-pdf-chrome') 235 | 236 | module.exports = { 237 | api: { 238 | token: 'api-token' 239 | }, 240 | // html-pdf-chrome options 241 | generator: { 242 | completionTrigger: new htmlPdf.CompletionTrigger.Variable( 243 | 'myVarName', // optional, name of the variable to wait for. Defaults to 'htmlPdfDone' 244 | 5000 // optional, timeout (milliseconds) 245 | ) 246 | }, 247 | webhook: { 248 | secret: '1234', 249 | url: 'http://localhost:3000/webhooks/pdf' 250 | } 251 | } 252 | ``` 253 | 254 | In your Javascript set the variable when the rendering is complete 255 | 256 | ```javascript 257 | window.myVarName = true; 258 | ``` 259 | 260 | [You can find more completion triggers in html-pdf-chrome's documentation](https://github.com/westy92/html-pdf-chrome#trigger-render-completion) 261 | 262 | ## API 263 | 264 | Below are given the endpoints that are exposed by `pdf-server`'s REST API 265 | 266 | ### Push URL to queue: POST / 267 | 268 | key | type | required | description 269 | --- | ---- | -------- | ----------- 270 | url | string | yes | The URL to generate a PDF from 271 | meta | object | | Optional meta data object to send back to the webhook url 272 | 273 | #### Example 274 | 275 | ```bash 276 | curl -X POST -H 'Authorization: Bearer api-token' -H 'Content-Type: application/json' http://pdf-bot.com/ -d ' 277 | { 278 | "url":"https://esbenp.github.io", 279 | "meta":{ 280 | "type":"invoice", 281 | "id":1 282 | } 283 | }' 284 | ``` 285 | 286 | ## Storage 287 | 288 | Currently `pdf-bot` comes bundled with build-in support for storing PDFs on Amazon S3. 289 | 290 | [Feel free to contribute a PR if you want to see other storage plugins in `pdf-bot`](https://github.com/esbenp/pdf-bot/compare)! 291 | 292 | ### Amazon S3 293 | 294 | To install S3 storage add a key to the `storage` configuration. Notice, you can add as many different locations you want by giving them different keys. 295 | 296 | ```javascript 297 | var createS3Config = require('pdf-bot/src/storage/s3') 298 | 299 | module.exports = { 300 | api: { 301 | token: 'api-token' 302 | }, 303 | storage: { 304 | 'my_s3': createS3Config({ 305 | bucket: '[YOUR BUCKET NAME]', 306 | accessKeyId: '[YOUR ACCESS KEY ID]', 307 | region: '[YOUR REGION]', 308 | secretAccessKey: '[YOUR SECRET ACCESS KEY]' 309 | }) 310 | }, 311 | webhook: { 312 | secret: '1234', 313 | url: 'http://localhost:3000/webhooks/pdf' 314 | } 315 | } 316 | 317 | ``` 318 | 319 | ## Options 320 | 321 | ```javascript 322 | var decaySchedule = [ 323 | 1000 * 60, // 1 minute 324 | 1000 * 60 * 3, // 3 minutes 325 | 1000 * 60 * 10, // 10 minutes 326 | 1000 * 60 * 30, // 30 minutes 327 | 1000 * 60 * 60 // 1 hour 328 | ]; 329 | 330 | module.exports = { 331 | // The settings of the API 332 | api: { 333 | // The port your express.js instance listens to requests from. (default: 3000) 334 | port: 3000, 335 | // The token used to validate requests to your API. Not required, but 100% recommended. 336 | token: 'api-token' 337 | }, 338 | // html-pdf-chrome 339 | generator: { 340 | // Triggers that specify when the PDF should be generated 341 | completionTrigger: new htmlPdf.CompletionTrigger.Timer(1000), // waits for 1 sec 342 | // The port to listen for Chrome (default: 9222) 343 | port: 9222 344 | }, 345 | queue: { 346 | // How frequent should pdf-bot retry failed generations? 347 | // (default: 1 min, 3 min, 10 min, 30 min, 60 min) 348 | generationRetryStrategy: function(job, retries) { 349 | return decaySchedule[retries - 1] ? decaySchedule[retries - 1] : 0 350 | }, 351 | // How many times should pdf-bot try to generate a PDF? 352 | // (default: 5) 353 | generationMaxTries: 5, 354 | // How frequent should pdf-bot retry failed webhook pings? 355 | // (default: 1 min, 3 min, 10 min, 30 min, 60 min) 356 | webhookRetryStrategy: function(job, retries) { 357 | return decaySchedule[retries - 1] ? decaySchedule[retries - 1] : 0 358 | }, 359 | // How many times should pdf-bot try to ping a webhook? 360 | // (default: 5) 361 | webhookMaxTries: 5, 362 | // In what path should the database be stored? 363 | path: 'storage/db/db.json', 364 | // pdf-bot uses lowdb. You can pass options to it here. 365 | lowDbOptions: { 366 | 367 | } 368 | }, 369 | storage: { 370 | 's3': createS3Config({ 371 | bucket: '', 372 | accessKeyId: '', 373 | region: '', 374 | secretAccessKey: '' 375 | }) 376 | }, 377 | webhook: { 378 | // The prefix to add to all pdf-bot headers on the webhook response. 379 | // I.e. X-PDF-Transaction and X-PDF-Signature. (default: X-PDF-) 380 | headerNamespace: 'X-PDF-', 381 | // Extra request options to add to the Webhook ping. 382 | requestOptions: { 383 | 384 | }, 385 | // The secret used to generate the hmac-sha1 signature hash. 386 | // !Not required, but should definitely be included! 387 | secret: '1234', 388 | // The endpoint to send PDF messages to. 389 | url: 'http://localhost:3000/webhooks/pdf' 390 | } 391 | } 392 | ``` 393 | 394 | ## CLI 395 | 396 | `pdf-bot` comes with a full CLI included! Use `-c` to pass a configuration to `pdf-bot`. You can also use `--help` to get a list of all commands. An example is given below. 397 | 398 | ```bash 399 | $ pdf-bot.js --config ./examples/pdf-bot.config.js --help 400 | 401 | 402 | Usage: pdf-bot [options] [command] 403 | 404 | 405 | Options: 406 | 407 | -V, --version output the version number 408 | -c, --config Path to configuration file 409 | -h, --help output usage information 410 | 411 | 412 | Commands: 413 | 414 | api Start the API 415 | install 416 | generate [jobID] Generate PDF for job 417 | jobs [options] List all completed jobs 418 | ping [jobID] Attempt to ping webhook for job 419 | ping:retry-failed 420 | pings [jobId] List pings for a job 421 | purge [options] Will remove all completed jobs 422 | push [options] [url] Push new job to the queue 423 | shift Run the next job in the queue 424 | ``` 425 | 426 | ## Debug mode 427 | 428 | `pdf-bot` uses `debug` for debug messages. You can turn on debugging by setting the environment variable `DEBUG=pdf:*` like so 429 | 430 | ```bash 431 | DEBUG=pdf:* pdf-bot jobs 432 | ``` 433 | 434 | ## Tests 435 | 436 | ```bash 437 | $ npm run test 438 | ``` 439 | 440 | ## Issues 441 | 442 | [Please report issues to the issue tracker](https://github.com/esbenp/pdf-bot/issues/new) 443 | 444 | ## License 445 | 446 | The MIT License (MIT). Please see [License File](https://github.com/esbenp/pdf-bot/blob/master/LICENSE) for more information. 447 | -------------------------------------------------------------------------------- /bin/pdf-bot.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var fs = require('fs') 4 | var path = require('path') 5 | var debug = require('debug')('pdf:cli') 6 | var Table = require('cli-table') 7 | var program = require('commander'); 8 | var merge = require('lodash.merge') 9 | var createPdfGenerator = require('../src/pdfGenerator') 10 | var createApi = require('../src/api') 11 | var error = require('../src/error') 12 | var createQueue = require('../src/queue') 13 | var webhook = require('../src/webhook') 14 | var pjson = require('../package.json') 15 | var execSync = require('child_process').execSync 16 | var prompt = require('prompt') 17 | 18 | program 19 | .version(pjson.version) 20 | .option('-c, --config ', 'Path to configuration file') 21 | 22 | var decaySchedule = [ 23 | 1000 * 60, // 1 minute 24 | 1000 * 60 * 3, // 3 minutes 25 | 1000 * 60 * 10, // 10 minutes 26 | 1000 * 60 * 30, // 30 minutes 27 | 1000 * 60 * 60 // 1 hour 28 | ]; 29 | 30 | var configuration, queue 31 | var defaultConfig = { 32 | api: { 33 | port: 3000, 34 | //token: 'api-token' 35 | }, 36 | // html-pdf-chrome options 37 | generator: { 38 | 39 | }, 40 | queue: { 41 | generationRetryStrategy: function(job, retries) { 42 | return decaySchedule[retries - 1] ? decaySchedule[retries - 1] : 0 43 | }, 44 | generationMaxTries: 5, 45 | webhookRetryStrategy: function(job, retries) { 46 | return decaySchedule[retries - 1] ? decaySchedule[retries - 1] : 0 47 | }, 48 | webhookMaxTries: 5, 49 | lowDbOptions: { 50 | 51 | } 52 | }, 53 | storage: { 54 | /* 55 | 's3': createS3Config({ 56 | bucket: '', 57 | accessKeyId: '', 58 | region: '', 59 | secretAccessKey: '' 60 | }) 61 | */ 62 | }, 63 | storagePath: 'storage', 64 | /*webhook: { 65 | headerNamespace: 'X-PDF-', 66 | requestOptions: { 67 | 68 | }, 69 | secret: '12345', 70 | url: 'http://localhost:3001/hook' 71 | }*/ 72 | } 73 | 74 | program 75 | .command('api') 76 | .description('Start the API') 77 | .action(function (options) { 78 | // We delay initiation of queue. This is because the API will load the DB in memory as 79 | // copy A. When we make changes through the CLI this creates copy B. But next time the 80 | // user pushes to the queue using the API copy A will be persisted again. 81 | var initiateQueue = openConfig(true) 82 | 83 | var apiOptions = configuration.api 84 | var port = apiOptions.port 85 | 86 | createApi(initiateQueue, { 87 | port: port, 88 | token: apiOptions.token 89 | }).listen(port, function() { 90 | debug('Listening to port %d', port) 91 | }) 92 | }) 93 | 94 | program 95 | .command('install') 96 | .action(function (options) { 97 | var configPath = program.config || path.join(process.cwd(), 'pdf-bot.config.js') 98 | 99 | function startPrompt() { 100 | prompt.start({noHandleSIGINT: true}) 101 | prompt.get([ 102 | { 103 | name: 'storagePath', 104 | description: 'Enter a path for storage', 105 | default: path.join(process.cwd(), 'pdf-storage'), 106 | required: true 107 | }, 108 | { 109 | name: 'token', 110 | description: 'An access token for your API', 111 | required: false 112 | }], function (err, result) { 113 | if (err) { 114 | process.exit(0) 115 | } 116 | var options = {} 117 | 118 | if (result.token) { 119 | options.api = {token: result.token} 120 | } 121 | 122 | options.storagePath = result.storagePath 123 | 124 | var configContents = "module.exports = " + JSON.stringify(options, null, 2) 125 | 126 | fs.writeFileSync(configPath, configContents) 127 | 128 | if (!fs.existsSync(options.storagePath)) { 129 | fs.mkdirSync(options.storagePath, '0775') 130 | fs.mkdirSync(path.join(options.storagePath, 'db'), '0775') 131 | fs.mkdirSync(path.join(options.storagePath, 'pdf'), '0775') 132 | } 133 | 134 | console.log('pdf-bot was installed successfully.') 135 | console.log('Config file is placed at ' + configPath + ' and contains') 136 | console.log(configContents) 137 | console.log('You should add ALIAS pdf-bot="pdf-bot -c ' + configPath + '" to your ~/.profile') 138 | }); 139 | } 140 | 141 | var existingConfigFileFound = fs.existsSync(configPath) 142 | if (existingConfigFileFound) { 143 | prompt.start({noHandleSIGINT: true}) 144 | prompt.get([ 145 | { 146 | name: 'replaceConfig', 147 | description: 'A config file already exists, are you sure you want to override (yes/no)' 148 | } 149 | ], function (err, result) { 150 | if (err) { 151 | process.exit(0) 152 | } 153 | if (result.replaceConfig !== 'yes') { 154 | process.exit(0) 155 | } else { 156 | startPrompt() 157 | } 158 | }) 159 | } else { 160 | startPrompt() 161 | } 162 | }) 163 | 164 | program 165 | .command('generate [jobID]') 166 | .description('Generate PDF for job') 167 | .action(function (jobId, options){ 168 | openConfig() 169 | 170 | var job = queue.getById(jobId) 171 | 172 | if (!job) { 173 | console.log('Job not found') 174 | return; 175 | } 176 | 177 | processJob(job, configuration) 178 | }) 179 | 180 | program 181 | .command('jobs') 182 | .description('List all completed jobs') 183 | .option('--completed', 'Show completed jobs') 184 | .option('--failed', 'Show failed jobs') 185 | .option('-l, --limit [limit]', 'Limit how many jobs to show') 186 | .action(function (options) { 187 | openConfig() 188 | 189 | listJobs(queue, options.failed, options.completed, options.limit) 190 | }) 191 | 192 | program 193 | .command('ping [jobID]') 194 | .description('Attempt to ping webhook for job') 195 | .action(function (jobId, options) { 196 | openConfig() 197 | 198 | var job = queue.getById(jobId) 199 | 200 | if (!job) { 201 | console.log('Job not found.') 202 | return; 203 | } 204 | 205 | ping(job, configuration.webhook) 206 | }) 207 | 208 | program 209 | .command('ping:retry-failed') 210 | .action(function() { 211 | openConfig() 212 | 213 | var maxTries = configuration.queue.webhookMaxTries 214 | var retryStrategy = configuration.queue.webhookRetryStrategy 215 | 216 | var next = queue.getNextWithoutSuccessfulPing(retryStrategy, maxTries) 217 | 218 | if (next) { 219 | ping(next, configuration.webhook) 220 | } 221 | }) 222 | 223 | program 224 | .command('pings [jobId]') 225 | .description('List pings for a job') 226 | .action(function (jobId, options) { 227 | openConfig() 228 | 229 | var job = queue.getById(jobId) 230 | 231 | if (!job) { 232 | console.log('Job not found') 233 | return; 234 | } 235 | 236 | var table = new Table({ 237 | head: ['ID', 'URL', 'Method', 'Status', 'Sent at', 'Response', 'Payload'], 238 | colWidths: [40, 40, 50, 20, 20, 20] 239 | }); 240 | 241 | for(var i in job.pings) { 242 | var ping = job.pings[i] 243 | 244 | table.push([ 245 | ping.id, 246 | ping.url, 247 | ping.method, 248 | ping.status, 249 | formatDate(ping.sent_at), 250 | JSON.stringify(ping.response), 251 | JSON.stringify(ping.payload) 252 | ]) 253 | } 254 | 255 | console.log(table.toString()) 256 | }) 257 | 258 | program 259 | .command('purge') 260 | .description('Will remove all completed jobs') 261 | .option('--failed', 'Remove all failed jobs') 262 | .option('--new', 'Remove all new jobs') 263 | .action(function (options) { 264 | openConfig() 265 | 266 | queue.purge(options.failed, options.new) 267 | 268 | console.log('The queue was purged.') 269 | }) 270 | 271 | program 272 | .command('push [url]') 273 | .description('Push new job to the queue') 274 | .option('-m, --meta [meta]', 'JSON string with meta data. Default: \'{}\'') 275 | .action(function (url, options) { 276 | openConfig() 277 | 278 | var response = queue.addToQueue({ 279 | url: url, 280 | meta: JSON.parse(options.meta || '{}') 281 | }) 282 | 283 | if (error.isError(response)) { 284 | console.error('Could not push to queue: %s', response.message) 285 | process.exit(1) 286 | } 287 | }) 288 | 289 | program 290 | .command('shift') 291 | .description('Run the next job in the queue') 292 | .action(function (url) { 293 | openConfig() 294 | 295 | var maxTries = configuration.queue.generationMaxTries 296 | var retryStrategy = configuration.queue.generationRetryStrategy 297 | 298 | var next = queue.getNext(retryStrategy, maxTries) 299 | 300 | if (next) { 301 | processJob(next, configuration) 302 | } 303 | }) 304 | 305 | program.parse(process.argv) 306 | 307 | if (!process.argv.slice(2).length) { 308 | program.outputHelp(); 309 | } 310 | 311 | function processJob(job, configuration) { 312 | var generatorOptions = configuration.generator 313 | var storagePlugins = configuration.storage 314 | 315 | var generator = createPdfGenerator(configuration.storagePath, generatorOptions, storagePlugins) 316 | 317 | queue.processJob(generator, job, configuration.webhook).then(response => { 318 | if (error.isError(response)) { 319 | console.error(response.message) 320 | process.exit(1) 321 | } else { 322 | console.log('Job ID ' + job.id + ' was processed.') 323 | process.exit(0) 324 | } 325 | }) 326 | } 327 | 328 | function openConfig(delayQueueCreation = false) { 329 | configuration = defaultConfig 330 | 331 | if (!program.config) { 332 | if (fs.existsSync(path.join(process.cwd(), 'pdf-bot.config.js'))) { 333 | program.config = 'pdf-bot.config.js' 334 | } else { 335 | throw new Error('You need to supply a config file') 336 | } 337 | } 338 | 339 | var configPath = path.join(process.cwd(), program.config) 340 | 341 | if (!fs.existsSync(configPath)) { 342 | throw new Error('No config file was found at ' + configPath) 343 | } 344 | 345 | debug('Creating CLI using config file %s', configPath) 346 | merge(configuration, require(configPath)) 347 | 348 | if (!fs.existsSync(configuration.storagePath)) { 349 | throw new Error('Whoops! Looks like your storage folder does not exist. You should run pdf-bot install.') 350 | } 351 | 352 | if (!fs.existsSync(path.join(configuration.storagePath, 'db'))) { 353 | throw new Error('There is no database folder in the storage folder. Create it: storage/db') 354 | } 355 | 356 | if (!fs.existsSync(path.join(configuration.storagePath, 'pdf'))) { 357 | throw new Error('There is no pdf folder in the storage folder. Create it: storage/pdf') 358 | } 359 | 360 | function initiateQueue() { 361 | var queueOptions = configuration.queue 362 | return createQueue(path.join(configuration.storagePath, 'db/db.json'), queueOptions.lowDbOptions) 363 | } 364 | 365 | if (delayQueueCreation) { 366 | return initiateQueue 367 | } else { 368 | queue = initiateQueue() 369 | } 370 | } 371 | 372 | function listJobs(queue, failed = false, limit) { 373 | var response = queue.getList( 374 | failed, 375 | limit 376 | ) 377 | 378 | var table = new Table({ 379 | head: ['ID', 'URL', 'Meta', 'PDF Gen. tries', 'Created at', 'Completed at'], 380 | colWidths: [40, 40, 50, 20, 20, 20] 381 | }); 382 | 383 | for(var i in response) { 384 | var job = response[i] 385 | 386 | table.push([ 387 | job.id, 388 | job.url, 389 | JSON.stringify(job.meta), 390 | job.generations.length, 391 | formatDate(job.created_at), 392 | formatDate(job.completed_at) 393 | ]) 394 | } 395 | 396 | console.log(table.toString()); 397 | } 398 | 399 | function ping(job, webhookConfiguration) { 400 | queue.attemptPing(job, webhookConfiguration || {}).then(response => { 401 | if (!response.error) { 402 | console.log('Ping succeeded: ' + JSON.stringify(response)) 403 | } else { 404 | console.error('Ping failed: ' + JSON.stringify(response)) 405 | } 406 | 407 | return response 408 | }) 409 | } 410 | 411 | function formatDate(input) { 412 | if (!input) { 413 | return '' 414 | } 415 | 416 | return (new Date(input)).toLocaleString() 417 | } 418 | -------------------------------------------------------------------------------- /examples/pdf-bot.config.js: -------------------------------------------------------------------------------- 1 | var htmlPdf = require('html-pdf-chrome') 2 | var createS3Config = require('../src/storage/s3') 3 | 4 | module.exports = { 5 | api: { 6 | token: 'api-token' 7 | }, 8 | // html-pdf-chrome options 9 | generator: { 10 | completionTrigger: new htmlPdf.CompletionTrigger.Timer(1000), // waits for 1 sec 11 | //port: 50 // chrome port 12 | }, 13 | queue: { 14 | 15 | }, 16 | storage: { 17 | /*'s3': createS3Config({ 18 | bucket: '', 19 | accessKeyId: '', 20 | region: '', 21 | secretAccessKey: '' 22 | })*/ 23 | }, 24 | // storagePath: '', 25 | webhook: { 26 | headerNamespace: 'X-PDF-', 27 | requestOptions: { 28 | 29 | }, 30 | secret: '1234', 31 | url: 'http://localhost:3000/webhooks/pdf' 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/receiving-api.js: -------------------------------------------------------------------------------- 1 | var express = require('express') 2 | var bodyParser = require('body-parser') 3 | 4 | var api = express() 5 | api.use(bodyParser.json()) 6 | 7 | api.post('/hook', function (req, res) { 8 | var signature = req.get('X-PDF-Signature', 'sha1=') 9 | 10 | var bodyCrypted = require('crypto') 11 | .createHmac('sha1', '12345') 12 | .update(JSON.stringify(req.body)) 13 | .digest('hex') 14 | 15 | if (bodyCrypted !== signature) { 16 | res.status(401).send() 17 | return 18 | } 19 | 20 | console.log('PDF webhook received', JSON.stringify(req.body)) 21 | 22 | res.status(204).send() 23 | }) 24 | 25 | api.listen(3001, function() { 26 | console.log('Listening to port 3001') 27 | }) 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pdf-bot", 3 | "version": "0.3.3", 4 | "author": "Esben Petersen ", 5 | "homepage": "https://github.com/esbenp/pdf-bot", 6 | "license": "MIT", 7 | "repository": { 8 | "type": "git", 9 | "url": "https://github.com/esbenp/pdf-bot.git" 10 | }, 11 | "engines": { 12 | "node": ">= 6" 13 | }, 14 | "description": "A Node queue API for generating PDFs using headless Chrome. Comes with a CLI, S3 storage and webhooks for notifying subscribers about generated PDFs", 15 | "main": "./src/index.js", 16 | "bin": "./bin/pdf-bot.js", 17 | "files": [ 18 | "bin/", 19 | "src/", 20 | "storage/" 21 | ], 22 | "scripts": { 23 | "example": "DEBUG=pdf:* node ./bin/pdf-bot.js --config ./examples/pdf-bot.config.js", 24 | "example:receiving-api": "DEBUG=pdf:* node ./examples/receiving-api.js", 25 | "test": "nyc --reporter=lcov --reporter=text mocha test/*.test.js --recursive --coverage", 26 | "test:watch": "mocha -w test/*.test.js --recursive" 27 | }, 28 | "dependencies": { 29 | "body-parser": "^1.17.2", 30 | "cli-table": "^0.3.1", 31 | "commander": "^2.11.0", 32 | "debug": "^2.6.8", 33 | "express": "^4.15.3", 34 | "html-pdf-chrome": "^0.2.0", 35 | "lodash.merge": "^4.6.0", 36 | "lowdb": "^0.16.2", 37 | "node-fetch": "^1.7.1", 38 | "prompt": "^1.0.0", 39 | "s3": "^4.4.0", 40 | "uuid": "^3.1.0" 41 | }, 42 | "devDependencies": { 43 | "assert": "^1.4.1", 44 | "coveralls": "^2.13.1", 45 | "mocha": "^3.5.0", 46 | "nyc": "^11.1.0", 47 | "proxyquire": "^1.8.0", 48 | "sinon": "^3.0.0", 49 | "supertest": "^3.0.0" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /production/README.md: -------------------------------------------------------------------------------- 1 | # Running pdf-bot in production 2 | 3 | ## Run pdf-bot using pm2 4 | 5 | It is recommended to use [pm2](https://github.com/Unitech/pm2) to run a pdf-bot process. 6 | 7 | First install `pm2` 8 | 9 | ``` 10 | npm install -g pm2 11 | ``` 12 | 13 | [Create a configuration file using the one in this repo as an example](https://github.com/esbenp/pdf-bot/blob/master/production/pm2.config.js) 14 | 15 | `pdf-bot-process.config.js` 16 | ```javascript 17 | module.exports = { 18 | apps : [{ 19 | name : "pdf-bot", 20 | script : "pdf-bot", 21 | args : "api -c ./pdf-bot.config.js", 22 | // Should be from whatever folder your pdf-bot.config.js is in 23 | // cwd : "/home/[user]/", 24 | env: { 25 | "DEBUG" : "pdf:*", 26 | "NODE_ENV": "production", 27 | }, 28 | }] 29 | } 30 | ``` 31 | 32 | Run in using `pm2 start pdf-bot-process.config.js` 33 | 34 | [Read more about starting the app on server restarts](http://pm2.keymetrics.io/docs/usage/startup/) 35 | 36 | ## Use nginx to proxy requests 37 | 38 | If you run `pdf-bot` on port 3000 or similar it is recommended to run it behind an nginx proxy. 39 | 40 | Create a site that listens to port 80 and uses the [config from the `production/` folder](https://github.com/esbenp/pdf-bot/blob/master/production/nginx.conf) 41 | -------------------------------------------------------------------------------- /production/nginx.conf: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80 default_server; 3 | listen [::]:80 default_server; 4 | 5 | server_name pdf-bot; 6 | 7 | location / { 8 | proxy_set_header X-Real-IP $remote_addr; 9 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 10 | proxy_set_header Host $http_host; 11 | proxy_set_header X-NginX-Proxy true; 12 | proxy_pass http://127.0.0.1:3000/; 13 | proxy_redirect off; 14 | proxy_http_version 1.1; 15 | proxy_set_header Upgrade $http_upgrade; 16 | proxy_set_header Connection "upgrade"; 17 | 18 | proxy_redirect off; 19 | proxy_set_header X-Forwarded-Proto $scheme; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /production/pm2.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | apps : [{ 3 | name : "pdf-bot", 4 | script : "pdf-bot", 5 | args : "api -c ./pdf-bot.config.js", 6 | // Should be from whatever folder your pdf-bot.config.js is in 7 | // cwd : "/home/[user]/", 8 | env: { 9 | "DEBUG" : "pdf:*", 10 | "NODE_ENV": "production", 11 | }, 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /src/api.js: -------------------------------------------------------------------------------- 1 | // these need to occur after dotenv 2 | var express = require('express') 3 | var bodyParser = require('body-parser') 4 | var debug = require('debug')('pdf:api') 5 | var error = require('./error') 6 | 7 | function createApi(createQueue, options = {}) { 8 | var api = express() 9 | api.use(bodyParser.json()) 10 | 11 | var token = options.token 12 | 13 | if (!token) { 14 | debug('Warning: The server should be protected using a token.') 15 | } 16 | 17 | api.post('/', function(req, res) { 18 | var queue = createQueue() 19 | var authHeader = req.get('Authorization') 20 | 21 | if (token && (!authHeader || authHeader.replace(/Bearer (.*)$/i, '$1') !== token)) { 22 | res.status(401).json(error.createErrorResponse(error.ERROR_INVALID_TOKEN)) 23 | return 24 | } 25 | 26 | var response = queue.addToQueue({ 27 | url: req.body.url, 28 | meta: req.body.meta || {} 29 | }) 30 | 31 | if (error.isError(response)) { 32 | res.status(422).json(response) 33 | return 34 | } 35 | 36 | res.status(201).json(response) 37 | }) 38 | 39 | return api 40 | } 41 | 42 | module.exports = createApi 43 | -------------------------------------------------------------------------------- /src/error.js: -------------------------------------------------------------------------------- 1 | function createErrorResponse (type) { 2 | return { 3 | code: errorCodes[type], 4 | error: true, 5 | message: errorMessages[type] 6 | } 7 | } 8 | 9 | function isError (response) { 10 | return response.error && response.code 11 | } 12 | 13 | function getErrorCode(type) { 14 | return errorCodes[type] 15 | } 16 | 17 | var ERROR_INVALID_TOKEN = 'ERROR_INVALID_TOKEN' 18 | var ERROR_INVALID_URL = 'ERROR_INVALID_URL' 19 | var ERROR_HTML_PDF_CHROME_ERROR = 'ERROR_HTML_PDF_CHROME_ERROR' 20 | var ERROR_META_IS_NOT_OBJECT = 'ERROR_META_IS_NOT_OBJECT' 21 | var ERROR_INVALID_JSON_RESPONSE = 'ERROR_INVALID_JSON_RESPONSE' 22 | 23 | var errorCodes = { 24 | [ERROR_INVALID_TOKEN]: '001', 25 | [ERROR_INVALID_URL]: '002', 26 | [ERROR_HTML_PDF_CHROME_ERROR]: '003', 27 | [ERROR_META_IS_NOT_OBJECT]: '004', 28 | [ERROR_INVALID_JSON_RESPONSE]: '005' 29 | } 30 | 31 | var errorMessages = { 32 | [ERROR_INVALID_TOKEN]: 'Invalid token.', 33 | [ERROR_INVALID_URL]: 'Invalid url.', 34 | [ERROR_HTML_PDF_CHROME_ERROR]: 'html-pdf-chrome error:', 35 | [ERROR_META_IS_NOT_OBJECT]: 'Meta data is not a valid object', 36 | [ERROR_INVALID_JSON_RESPONSE]: 'Invalid JSON response' 37 | } 38 | 39 | module.exports = { 40 | createErrorResponse: createErrorResponse, 41 | isError: isError, 42 | getErrorCode: getErrorCode, 43 | ERROR_INVALID_TOKEN: ERROR_INVALID_TOKEN, 44 | ERROR_INVALID_URL: ERROR_INVALID_URL, 45 | ERROR_HTML_PDF_CHROME_ERROR: ERROR_HTML_PDF_CHROME_ERROR, 46 | ERROR_META_IS_NOT_OBJECT: ERROR_META_IS_NOT_OBJECT, 47 | ERROR_INVALID_JSON_RESPONSE: ERROR_INVALID_JSON_RESPONSE 48 | } 49 | -------------------------------------------------------------------------------- /src/pdfGenerator.js: -------------------------------------------------------------------------------- 1 | var path = require('path') 2 | var htmlPdf = require('html-pdf-chrome') 3 | var uuid = require('uuid') 4 | var debug = require('debug')('pdf:generator') 5 | var error = require('./error') 6 | var uuid = require('uuid') 7 | var utils = require('./utils') 8 | 9 | function createPdfGenerator(storagePath, options = {}, storagePlugins = {}) { 10 | return function createPdf (url, job) { 11 | debug('Creating PDF for url %s with options %s', url, JSON.stringify(options)) 12 | 13 | var generationId = uuid() 14 | var generated_at = utils.getCurrentDateTimeAsString() 15 | 16 | function createResponseObject() { 17 | return { 18 | id: generationId, 19 | generated_at: generated_at 20 | } 21 | } 22 | 23 | return htmlPdf 24 | .create(url, options) 25 | .then((pdf) => { 26 | var pdfPath = path.join(storagePath, 'pdf', (uuid() + '.pdf')) 27 | 28 | debug('Saving PDF to %s', pdfPath) 29 | 30 | return pdf 31 | .toFile(pdfPath) 32 | .then(function(response){ 33 | var storage = { 34 | local: pdfPath 35 | } 36 | var storagePluginPromises = [] 37 | for (var i in storagePlugins) { 38 | // Because i will change before the promise is resolved 39 | // we use a self executing function to inject the variable 40 | // into a different scope 41 | var then = (function(type) { 42 | return function (response) { 43 | return Object.assign(response, { 44 | type: type 45 | }) 46 | } 47 | })(i) 48 | 49 | storagePluginPromises.push( 50 | storagePlugins[i](pdfPath, job).then(then) 51 | ) 52 | } 53 | 54 | return Promise.all(storagePluginPromises).then(responses => { 55 | for(var i in responses) { 56 | var response = responses[i] 57 | 58 | storage[response.type] = { 59 | path: response.path, 60 | meta: response.meta || {} 61 | } 62 | } 63 | 64 | return Object.assign( 65 | createResponseObject(), 66 | { 67 | storage: storage 68 | } 69 | ) 70 | }) 71 | }) 72 | }) 73 | .catch(msg => { 74 | var response = error.createErrorResponse(error.ERROR_HTML_PDF_CHROME_ERROR) 75 | 76 | response.message += ' ' + msg 77 | 78 | return Object.assign(createResponseObject(), response) 79 | }) 80 | } 81 | } 82 | 83 | module.exports = createPdfGenerator 84 | -------------------------------------------------------------------------------- /src/queue.js: -------------------------------------------------------------------------------- 1 | var low = require('lowdb') 2 | var uuid = require('uuid') 3 | var debug = require('debug')('pdf:db') 4 | var error = require('./error') 5 | var webhook = require('./webhook') 6 | var utils = require('./utils') 7 | 8 | function createQueue (path, options = {}, initialValue = []) { 9 | var db = low(path, options) 10 | 11 | db.defaults({ 12 | queue: initialValue 13 | }) 14 | .write() 15 | 16 | var createQueueMethod = function (func) { 17 | return function() { 18 | var args = Array.prototype.slice.call(arguments, 0) 19 | return func.apply(func, [db].concat(args)) 20 | } 21 | } 22 | 23 | return { 24 | addToQueue: createQueueMethod(addToQueue), 25 | attemptPing: createQueueMethod(attemptPing), 26 | getById: createQueueMethod(getById), 27 | getList: createQueueMethod(getList), 28 | getNext: createQueueMethod(getNext), 29 | getNextWithoutSuccessfulPing: createQueueMethod(getNextWithoutSuccessfulPing), 30 | processJob: createQueueMethod(processJob), 31 | purge: createQueueMethod(purge) 32 | } 33 | } 34 | 35 | function addToQueue (db, data) { 36 | var id = uuid() 37 | var createdAt = utils.getCurrentDateTimeAsString() 38 | 39 | var defaults = { 40 | meta: {} 41 | } 42 | 43 | if (!data.url || !utils.isValidUrl(data.url)) { 44 | return error.createErrorResponse(error.ERROR_INVALID_URL) 45 | } 46 | 47 | if (data.meta && typeof data.meta !== 'object') { 48 | return error.createErrorResponse(error.ERROR_META_IS_NOT_OBJECT) 49 | } 50 | 51 | data = Object.assign(defaults, data, { 52 | id: id, 53 | created_at: createdAt, 54 | completed_at: null, 55 | generations: [], 56 | pings: [], 57 | storage: {} 58 | }) 59 | 60 | debug('Pushing job to queue with data %s', JSON.stringify(data)) 61 | 62 | db 63 | .get('queue') 64 | .push(data) 65 | .write() 66 | 67 | return data 68 | } 69 | 70 | // ========= 71 | // RETRIEVAL 72 | // ========= 73 | 74 | function getList (db, failed = false, completed = false, limit) { 75 | var query = db.get('queue') 76 | 77 | query = query.filter(function (job) { 78 | // failed jobs 79 | if (!failed && job.completed_at === null && job.generations.length > 0) { 80 | return false 81 | } 82 | 83 | // completed jobs 84 | if (!completed && job.completed_at !== null) { 85 | return false 86 | } 87 | 88 | return true 89 | }) 90 | 91 | if (limit) { 92 | query = query.take(limit) 93 | } 94 | 95 | return query.value() 96 | } 97 | 98 | function getById (db, id) { 99 | return db 100 | .get('queue') 101 | .find({ id: id }) 102 | .value() 103 | } 104 | 105 | function getNext (db, shouldWait, maxTries = 5) { 106 | return db 107 | .get('queue') 108 | .filter(function (job) { 109 | if (job.completed_at !== null) { 110 | return false 111 | } 112 | 113 | var currentTries = job.generations.length 114 | 115 | if (currentTries === 0) { 116 | return true 117 | } 118 | 119 | if (currentTries < maxTries) { 120 | var lastRun = job.generations[currentTries - 1].generated_at 121 | 122 | if (_hasWaitedLongEnough(lastRun, shouldWait(job, currentTries))) { 123 | return true 124 | } 125 | } 126 | 127 | return false 128 | }) 129 | .take(1) 130 | .value()[0] 131 | } 132 | 133 | function getNextWithoutSuccessfulPing (db, shouldWait, maxTries = 5) { 134 | return db 135 | .get('queue') 136 | .filter(function (job) { 137 | var currentTries = job.pings.length 138 | 139 | if (job.completed_at === null) { 140 | return false 141 | } 142 | 143 | if (currentTries === 0) { 144 | return true 145 | } 146 | 147 | if (currentTries >= maxTries) { 148 | return false 149 | } 150 | 151 | var unsuccessfulPings = job.pings.filter(ping => ping.error) 152 | 153 | // There are some successful ping(s) 154 | if (unsuccessfulPings.length !== job.pings.length) { 155 | return false 156 | } 157 | 158 | var lastTry = unsuccessfulPings[unsuccessfulPings.length - 1].sent_at 159 | if (_hasWaitedLongEnough(lastTry, shouldWait(job, currentTries))) { 160 | return true 161 | } 162 | 163 | return false 164 | }) 165 | .take(1) 166 | .value()[0] 167 | } 168 | 169 | function purge (db, failed = false, pristine = false, maxTries = 5) { 170 | var query = db.get('queue').slice(0) 171 | 172 | query = query.filter(function (job) { 173 | // failed jobs 174 | if (failed && job.completed_at === null && job.generations.length >= maxTries) { 175 | return true 176 | } 177 | 178 | // new jobs 179 | if (pristine && job.completed_at === null && job.generations.length < maxTries) { 180 | return true 181 | } 182 | 183 | // completed jobs 184 | if (job.completed_at !== null) { 185 | return true 186 | } 187 | 188 | return false 189 | }) 190 | 191 | var queue = query.value() 192 | 193 | for(var i in queue) { 194 | db.get('queue').remove({ id: queue[i].id }).write() 195 | } 196 | } 197 | 198 | // ========== 199 | // PROCESSING 200 | // ========== 201 | 202 | function processJob (db, generator, job, webhookOptions) { 203 | return generator(job.url, job) 204 | .then(response => { 205 | _logGeneration(db, job.id, response) 206 | 207 | if (!error.isError(response)) { 208 | debug('Job %s was processed, marking job as complete.', job.id) 209 | 210 | _markAsCompleted(db, job.id) 211 | _setStorage(db, job.id, response.storage) 212 | 213 | if (webhookOptions) { 214 | // Important to return promise otherwise the npm cli process will exit early 215 | return attemptPing(db, job, webhookOptions) 216 | .then(function() { 217 | return response 218 | }) 219 | } 220 | } 221 | 222 | return response 223 | }) 224 | } 225 | 226 | // ======= 227 | // PINGING 228 | // ======= 229 | 230 | function attemptPing (db, job, webhookOptions) { 231 | if (!(typeof webhookOptions === 'object')) { 232 | throw new Error('No webhook is configured.') 233 | } 234 | 235 | return webhook.ping(job, webhookOptions) 236 | .then(response => { 237 | _logPing(db, job.id, response) 238 | 239 | return response 240 | }) 241 | } 242 | 243 | // =============== 244 | // PRIVATE METHODS 245 | // =============== 246 | 247 | function _hasWaitedLongEnough (logTimestamp, timeToWait) { 248 | var diff = (new Date() - new Date(logTimestamp)) 249 | return diff > timeToWait 250 | } 251 | 252 | function _logGeneration (db, id, response) { 253 | debug('Logging try for job ID %s', id) 254 | 255 | var job = getById(db, id) 256 | 257 | var generations = job.generations.slice(0) 258 | generations.push(response) 259 | 260 | return db 261 | .get('queue') 262 | .find({ id: id }) 263 | .assign({ generations: generations }) 264 | .write() 265 | } 266 | 267 | function _logPing (db, id, response) { 268 | debug('Logging ping for job ID %s', id) 269 | 270 | var job = getById(db, id) 271 | 272 | var pings = job.pings.slice(0) 273 | pings.push(response) 274 | 275 | return db 276 | .get('queue') 277 | .find({ id: id }) 278 | .assign({ pings: pings }) 279 | .write() 280 | } 281 | 282 | function _markAsCompleted (db, id) { 283 | var completed_at = utils.getCurrentDateTimeAsString() 284 | 285 | debug('Marking job ID %s as completed at %s', id, completed_at) 286 | 287 | return db 288 | .get('queue') 289 | .find({ id: id }) 290 | .assign({ completed_at: completed_at }) 291 | .write() 292 | } 293 | 294 | function _setStorage (db, id, storage) { 295 | return db 296 | .get('queue') 297 | .find({ id: id }) 298 | .assign({ storage: storage }) 299 | .write() 300 | } 301 | 302 | module.exports = createQueue 303 | -------------------------------------------------------------------------------- /src/storage/s3.js: -------------------------------------------------------------------------------- 1 | var debug = require('debug')('pdf:s3') 2 | var s3 = require('s3') 3 | var path = require('path') 4 | 5 | function createS3Storage(options = {}) { 6 | if (!options.accessKeyId) { 7 | throw new Error('S3: No access key given') 8 | } 9 | 10 | if (!options.secretAccessKey) { 11 | throw new Error('S3: No secret access key given') 12 | } 13 | 14 | if (!options.region) { 15 | throw new Error('S3: No region specified') 16 | } 17 | 18 | if (!options.bucket) { 19 | throw new Error('S3: No bucket was specified') 20 | } 21 | 22 | return function uploadToS3 (localPath, job) { 23 | return new Promise((resolve, reject) => { 24 | var client = s3.createClient( 25 | Object.assign(options.s3ClientOptions || {}, 26 | { 27 | s3Options: { 28 | accessKeyId: options.accessKeyId, 29 | secretAccessKey: options.secretAccessKey, 30 | region: options.region, 31 | } 32 | } 33 | ) 34 | ) 35 | 36 | var remotePath = (options.path || '') 37 | if (typeof options.path === 'function') { 38 | remotePath = options.path(localPath, job) 39 | } 40 | 41 | var pathSplitted = localPath.split('/') 42 | var fileName = pathSplitted[pathSplitted.length - 1] 43 | var fullRemotePath = path.join(remotePath, fileName) 44 | 45 | var uploadOptions = { 46 | localFile: localPath, 47 | 48 | s3Params: { 49 | Bucket: options.bucket, 50 | Key: fullRemotePath, 51 | }, 52 | } 53 | 54 | debug('Pushing job ID %s to S3 path: %s/%s', job.id, options.bucket, fileName) 55 | 56 | var uploader = client.uploadFile(uploadOptions); 57 | uploader.on('error', function(err) { 58 | reject(err) 59 | }); 60 | uploader.on('end', function(data) { 61 | resolve({ 62 | path: { 63 | bucket: uploadOptions.s3Params.Bucket, 64 | region: options.region, 65 | key: uploadOptions.s3Params.Key 66 | } 67 | }) 68 | }); 69 | }) 70 | } 71 | } 72 | 73 | module.exports = createS3Storage 74 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | var isValidUrl = function (url) { 2 | return url.match(/((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\-;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-_]*)?\??(?:[\-\+=&;%@\.\w_]*)#?(?:[\.\!\/\\\w]*))?)/) 3 | } 4 | 5 | function getCurrentDateTimeAsString() { 6 | return (new Date()).toUTCString() 7 | } 8 | 9 | module.exports = { 10 | isValidUrl: isValidUrl, 11 | getCurrentDateTimeAsString: getCurrentDateTimeAsString 12 | } 13 | -------------------------------------------------------------------------------- /src/webhook.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto') 2 | var debug = require('debug')('pdf:webhook') 3 | var fetch = require('node-fetch') 4 | var uuid = require('uuid') 5 | var error = require('./error') 6 | var utils = require('./utils') 7 | 8 | function ping (job, options) { 9 | if (!options.url || !utils.isValidUrl(options.url)) { 10 | throw new Error('Webhook is not valid url.') 11 | } 12 | 13 | if (!options.secret) { 14 | throw new Error('You need to supply a secret for your webhooks') 15 | } 16 | 17 | var requestOptions = options.requestOptions || {} 18 | 19 | var headerOptions = requestOptions.headers || {} 20 | 21 | requestOptions.method = 'POST' 22 | headerOptions['Content-Type'] = 'application/json' 23 | 24 | var bodyRaw = { 25 | id: job.id, 26 | url: job.url, 27 | meta: job.meta, 28 | storage: job.storage 29 | } 30 | var body = JSON.stringify(bodyRaw) 31 | 32 | var signature = generateSignature(body, options.secret) 33 | 34 | var requestId = uuid() 35 | var namespace = options.headerNamespace || 'X-PDF-' 36 | headerOptions[namespace + 'Transaction'] = requestId 37 | headerOptions[namespace + 'Signature'] = signature 38 | 39 | var headers = new fetch.Headers() 40 | for(var i in headerOptions) { 41 | headers.set(i, headerOptions[i]) 42 | } 43 | 44 | requestOptions.headers = headers 45 | requestOptions.body = body 46 | 47 | debug( 48 | 'Pinging job ID %s at URL %s with request options %s', 49 | job.id, 50 | options.url, 51 | JSON.stringify(requestOptions) 52 | ) 53 | 54 | var sent_at = utils.getCurrentDateTimeAsString() 55 | 56 | function createResponse (response, error) { 57 | var status = response.status 58 | 59 | return getContentBody(response).then(body => { 60 | return { 61 | id: requestId, 62 | status: response.status, 63 | method: requestOptions.method, 64 | payload: bodyRaw, 65 | response: body, 66 | url: options.url, 67 | sent_at: sent_at, 68 | error: !response.ok 69 | } 70 | }) 71 | } 72 | 73 | return fetch(options.url, requestOptions) 74 | .then(function (response) { 75 | return createResponse(response, !response.ok) 76 | }) 77 | .catch(function (response) { 78 | return createResponse(response, true) 79 | }) 80 | } 81 | 82 | module.exports = { 83 | generateSignature: generateSignature, 84 | ping: ping 85 | } 86 | 87 | function generateSignature (payload, key) { 88 | return crypto.createHmac('sha1', key).update(payload).digest('hex') 89 | } 90 | 91 | function getContentBody (response) { 92 | return new Promise(function(resolve){ 93 | var emptyCodes = [204, 205] 94 | if (emptyCodes.indexOf(response.status) !== -1) { 95 | resolve({}) 96 | } 97 | 98 | // Happens for instance on ECONNREFUSED 99 | if (!(response instanceof fetch.Response)) { 100 | resolve(response) 101 | } 102 | 103 | var contentType = response.headers.get('content-type'); 104 | if (contentType.indexOf('json') === -1) { 105 | return response.text().then(resolve) 106 | } 107 | 108 | return response.text().then(text => { 109 | if (!text) { 110 | return resolve({}); 111 | } 112 | try { 113 | return resolve(JSON.parse(text)) 114 | } catch (e) { 115 | return resolve( 116 | Object.assign( 117 | error.createErrorResponse(error.ERROR_INVALID_JSON_RESPONSE), 118 | { 119 | response: text 120 | } 121 | ) 122 | ) 123 | } 124 | }) 125 | }) 126 | } 127 | -------------------------------------------------------------------------------- /storage/db/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /storage/pdf/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /test/api.test.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon') 2 | var request = require('supertest') 3 | var createApi = require('../src/api') 4 | var error = require('../src/error') 5 | 6 | describe('api: POST /', function () { 7 | var api 8 | beforeEach(function(){ 9 | api = createApi(function(){}, { 10 | token: '1234' 11 | }) 12 | }) 13 | 14 | it('should return 401 if no token is given', function(done) { 15 | request(api) 16 | .post('/') 17 | .expect(401, done) 18 | }) 19 | 20 | it('should return 401 if invalid token is give', function (done) { 21 | request(api) 22 | .post('/') 23 | .set('Authorization', 'Bearer test') 24 | .expect(401, done) 25 | }) 26 | 27 | it('should return 422 on errorneous responses', function(done) { 28 | queue = function () { 29 | return { 30 | addToQueue: function() { 31 | return { 32 | code: '001', 33 | error: true 34 | } 35 | } 36 | } 37 | } 38 | var api = createApi(queue, { 39 | token: '1234' 40 | }) 41 | 42 | request(api) 43 | .post('/') 44 | .set('Authorization', 'Bearer 1234') 45 | .send({}) 46 | .expect(422, done) 47 | }) 48 | 49 | it('should run the queue with the correct params', function (done) { 50 | var meta = {id: 1} 51 | 52 | var addToQueue = sinon.stub() 53 | addToQueue.onCall(0).returns({ id: '1234' }) 54 | 55 | var queue = function() { 56 | return { 57 | addToQueue: addToQueue 58 | } 59 | } 60 | var api = createApi(queue, { 61 | token: '1234' 62 | }) 63 | 64 | request(api) 65 | .post('/') 66 | .set('Authorization', 'Bearer 1234') 67 | .send({ url: 'https://google.com', meta: meta }) 68 | .expect(201) 69 | .end(function (err, res) { 70 | if (err) return done(err) 71 | 72 | if (!addToQueue.calledWith({ url: 'https://google.com', meta: meta })) { 73 | throw new Error('Queue was not called with correct url') 74 | } 75 | 76 | done() 77 | }) 78 | }) 79 | }) 80 | -------------------------------------------------------------------------------- /test/error.test.js: -------------------------------------------------------------------------------- 1 | var errorUtils = require('../src/error') 2 | 3 | describe('Error utils', function() { 4 | it('should correctly determine if error response', function() { 5 | var notError1 = errorUtils.isError({ code: '001' }) 6 | var notError2 = errorUtils.isError({ error: true }) 7 | var error = errorUtils.isError({ code: '001', error: true }) 8 | 9 | if (notError1 === true || notError2 === true) { 10 | throw new Error('Wrongly determined error response') 11 | } 12 | 13 | if (!error) { 14 | throw new Error('Did not determine error response') 15 | } 16 | }) 17 | }) 18 | -------------------------------------------------------------------------------- /test/pdfGenerator.test.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon') 2 | var htmlPdf = require('html-pdf-chrome') 3 | var createGenerator = require('../src/pdfGenerator') 4 | var error = require('../src/error') 5 | 6 | describe('PDF Generator', function() { 7 | var generator 8 | var pdf 9 | var createStub 10 | beforeEach(function(){ 11 | pdf = { 12 | toFile: sinon.stub().returns(new Promise(function(resolve){ 13 | resolve() 14 | })) 15 | } 16 | createStub = sinon.stub(htmlPdf, 'create'); 17 | createStub.onCall(0).returns(new Promise((resolve) => resolve(pdf))) 18 | generator = createGenerator('storage') 19 | }) 20 | 21 | afterEach(function(){ 22 | createStub.restore() 23 | }) 24 | 25 | it('should call html-pdf-chrome with the correct options', function() { 26 | var options = {options: true} 27 | generator = createGenerator('storage', options)('url') 28 | 29 | if (!createStub.calledOnce || !createStub.calledWith('url', options)) { 30 | throw new Error('Correct options not passed') 31 | } 32 | }) 33 | 34 | it('should attempt to write pdf to storage', function(done) { 35 | generator('url').then(() => { 36 | if (!pdf.toFile.calledOnce || !pdf.toFile.args[0][0].match(/storage\/pdf\/(.+)\.pdf$/)) { 37 | throw new Error('PDF was not attempted to saved') 38 | } 39 | 40 | done() 41 | }) 42 | }) 43 | 44 | it('should apply all passed storage configurations', function(done) { 45 | var storage = { 46 | storage_1: function() { 47 | return new Promise((resolve) => resolve({ path: 'file_1' })) 48 | }, 49 | storage_2: function() { 50 | return new Promise((resolve) => resolve({ path: 'file_2' })) 51 | } 52 | } 53 | 54 | createGenerator('storage', {}, storage)('url').then(response => { 55 | var storage = response.storage 56 | 57 | if (storage.storage_1.path !== 'file_1' || storage.storage_2.path !== 'file_2') { 58 | throw new Error('Storage response not properly set') 59 | } 60 | 61 | done() 62 | }) 63 | }) 64 | 65 | it('should return error response thrown promises', function(done) { 66 | createStub.onCall(0).returns(new Promise((resolve, reject) => reject('error'))) 67 | 68 | createGenerator('storage', {}, {})('url').then(response => { 69 | if (!error.isError(response)) { 70 | throw new Exception('Generator rejection did not resolve in error promise') 71 | } 72 | 73 | done() 74 | }) 75 | }) 76 | }) 77 | -------------------------------------------------------------------------------- /test/queue.test.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var fs = require('fs') 3 | var path = require('path') 4 | var sinon = require('sinon') 5 | var baseCreateQueue = require('../src/queue') 6 | var error = require('../src/error') 7 | var webhook = require('../src/webhook') 8 | 9 | var queuePath = path.join(__dirname, 'db.json') 10 | 11 | function getQueue() { 12 | return JSON.parse(fs.readFileSync(queuePath, 'utf8')).queue 13 | } 14 | 15 | function deleteQueue() { 16 | fs.unlinkSync(queuePath) 17 | } 18 | 19 | function createQueue() { 20 | // Some times we want to create the queue in the test 21 | if (fs.existsSync(queuePath)) { 22 | deleteQueue() 23 | } 24 | return baseCreateQueue.apply(this, [queuePath].concat([].slice.call(arguments))) 25 | } 26 | 27 | var i = 0 28 | function createJob(completed = false, generationTries = 0, pingTries = 0) { 29 | i++ 30 | 31 | var generations = [] 32 | for(var k = 0; k < generationTries; k++) { 33 | generations.push({ id: 'xxx' }) 34 | } 35 | var pings = [] 36 | for(var k = 0; k < pingTries; k++) { 37 | pings.push({ id: 'xxx' }) 38 | } 39 | 40 | return { 41 | id: i, 42 | completed_at: (completed ? '2017-01-01' : null), 43 | generations: generations, 44 | pings: pings 45 | } 46 | } 47 | 48 | describe('queue : retrieval', function() { 49 | var queue 50 | beforeEach(function() { 51 | i = 0 52 | queue = createQueue() 53 | }) 54 | 55 | afterEach(function(){ 56 | deleteQueue() 57 | }) 58 | 59 | it('should create a default structure', function() { 60 | var queue = getQueue() 61 | assert(Array.isArray(queue)) 62 | }) 63 | 64 | it('should create error when passing invalid url', function() { 65 | var response = queue.addToQueue({ 66 | url: '$#$#@%@#' 67 | }) 68 | 69 | assert(response.error) 70 | assert.equal(response.code, error.getErrorCode(error.ERROR_INVALID_URL)) 71 | }) 72 | 73 | it('should create error when passing invalid meta', function() { 74 | var response = queue.addToQueue({ 75 | meta: 'not-object', 76 | url: 'http://localhost' 77 | }) 78 | 79 | assert(response.error) 80 | assert.equal(response.code, error.getErrorCode(error.ERROR_META_IS_NOT_OBJECT)) 81 | }) 82 | 83 | it('should save jobs to the queue', function() { 84 | queue.addToQueue({ 85 | meta: { 86 | hello: true 87 | }, 88 | url: 'http://localhost' 89 | }) 90 | 91 | var job = getQueue()[0] 92 | 93 | assert.equal(job.meta.hello, true) 94 | assert.equal(job.url, 'http://localhost') 95 | }) 96 | 97 | it('should return failed jobs when failed flag is passed', function(){ 98 | queue = createQueue({}, [ 99 | createJob(false, 2), 100 | createJob(true, 1), 101 | createJob(false, 0) // has not been run yet 102 | ]) 103 | 104 | var list = queue.getList(true) 105 | 106 | assert.equal(list.length, 2) 107 | assert.equal(list[0].id, 1) 108 | }) 109 | 110 | it('should return completed jobs', function() { 111 | queue = createQueue({}, [ 112 | createJob(true), 113 | createJob(true), 114 | createJob(false, 1) 115 | ]) 116 | 117 | var list = queue.getList(false, true) 118 | 119 | assert.equal(list.length, 2) 120 | assert.equal(list[1].id, 2) 121 | }) 122 | 123 | it('should return new jobs', function() { 124 | queue = createQueue({}, [ 125 | createJob(true), 126 | createJob(true), 127 | createJob(false, 1), // failed 128 | createJob(false, 0) // new 129 | ]) 130 | 131 | var list = queue.getList(false, false) 132 | 133 | assert.equal(list.length, 1) 134 | assert.equal(list[0].id, 4) 135 | }) 136 | 137 | it('should limit', function() { 138 | var jobs = [] 139 | for(var i = 0; i <= 20; i++) { 140 | jobs.push(createJob(false)) 141 | } 142 | 143 | queue = createQueue({}, jobs) 144 | 145 | var list = queue.getList(false, false, 10) 146 | 147 | assert.equal(list.length, 10) 148 | }) 149 | 150 | it('should return the correct job by id', function() { 151 | queue = createQueue({}, [ 152 | createJob(true), 153 | Object.assign(createJob(true), { meta: { correct: true } }), 154 | createJob(true) 155 | ]) 156 | 157 | var job = queue.getById(2) 158 | 159 | assert.equal(job.meta.correct, true) 160 | }) 161 | 162 | it('should return the next job if no tries were found', function() { 163 | queue = createQueue({}, [ 164 | createJob(true), 165 | createJob(false, 5), 166 | createJob(false), 167 | createJob(true) 168 | ]) 169 | 170 | var job = queue.getNext(function(){}, 5) 171 | 172 | assert.equal(job.id, 3) 173 | }) 174 | 175 | it('should return the next job that is within decay schedule', function() { 176 | var dateOne = inFiveMinutes() 177 | var dateTwo = fiveMinutesAgo() 178 | 179 | var jobWithManyGenerations = [] 180 | for(var k = 0; k < 10; k++) { 181 | jobWithManyGenerations.push({ id: k, generated_at: dateTwo }) 182 | } 183 | 184 | queue = createQueue({}, [ 185 | createJob(true), 186 | Object.assign(createJob(false), { generations: jobWithManyGenerations }), // should skip this due to generations 187 | Object.assign(createJob(false), { generations: [{id: 1, generated_at: dateOne }] }), // should skip due to decay 188 | Object.assign(createJob(false), { generations: [{id: 1, generated_at: dateTwo }] }) 189 | ]) 190 | 191 | var job = queue.getNext(function(){ return 1000 * 60 * 4 }, 5) 192 | 193 | assert.equal(job.id, 4) 194 | }) 195 | 196 | it('should get next with no pings', function() { 197 | var jobWithManyPings = [] 198 | for(var k = 0; k < 10; k++) { 199 | jobWithManyPings.push({ id: k, sent_at: fiveMinutesAgo(), error: true }) 200 | } 201 | 202 | queue = createQueue({}, [ 203 | Object.assign(createJob(true, 1, 5), {pings: jobWithManyPings}), // exceeds limit 204 | createJob(false, 1, 0), // should skip since it is not completed 205 | Object.assign(createJob(true, 1), {pings: [{id:55, sent_at: fiveMinutesAgo(), error: true }]}) 206 | ]) 207 | 208 | var job = queue.getNextWithoutSuccessfulPing(function(){ return 1000 * 60 * 4 }, 5) 209 | 210 | assert.equal(job.id, 3) 211 | }) 212 | 213 | it('should get next ping that is within decay schedule', function() { 214 | var dateOne = fiveMinutesAgo() 215 | var dateTwo = inFiveMinutes() 216 | 217 | var jobWithManyPings = [] 218 | for(var k = 0; k < 10; k++) { 219 | jobWithManyPings.push({ id: k, sent_at: dateTwo, error: true }) 220 | } 221 | 222 | queue = createQueue({}, [ 223 | // not within decay 224 | Object.assign(createJob(true), { pings: [{ id: 1, error: true, sent_at: dateTwo }, { id: 2, error: true, sent_at: dateTwo }] }), 225 | // too many pings 226 | Object.assign(createJob(true), { pings: jobWithManyPings }), 227 | // next 228 | Object.assign(createJob(true), { pings: [{ id: 4, error: true, sent_at: dateOne }] }), 229 | // after previous 230 | Object.assign(createJob(true), { pings: [{ id: 5, error: true, sent_at: dateOne }] }) 231 | ]) 232 | 233 | var job = queue.getNextWithoutSuccessfulPing(function() { return 1000 * 60 * 4 }, 5) 234 | 235 | assert.equal(job.id, 3) 236 | }) 237 | 238 | it('should purge queue for completed', function() { 239 | queue = createQueue({}, [ 240 | createJob(true), 241 | createJob(true), 242 | createJob(false), 243 | createJob(false, 1) 244 | ]) 245 | 246 | queue.purge(false, false, 5) 247 | 248 | var contents = getQueue() 249 | 250 | assert.equal(contents.length, 2) 251 | assert.equal(contents[0].id, 3) 252 | assert.equal(contents[1].id, 4) 253 | }) 254 | 255 | it('should purge queue for failed', function() { 256 | queue = createQueue({}, [ 257 | createJob(true), 258 | createJob(true), 259 | createJob(false), 260 | createJob(false, 1), 261 | createJob(false, 6) 262 | ]) 263 | 264 | queue.purge(true, false, 5) 265 | 266 | var contents = getQueue() 267 | 268 | assert.equal(contents.length, 2) 269 | assert.equal(contents[0].id, 3) 270 | assert.equal(contents[1].id, 4) 271 | }) 272 | 273 | it('should purge queue for new', function() { 274 | queue = createQueue({}, [ 275 | createJob(true), 276 | createJob(true), 277 | createJob(false), 278 | createJob(false, 1), 279 | createJob(false, 6) 280 | ]) 281 | 282 | queue.purge(false, true, 5) 283 | 284 | var contents = getQueue() 285 | 286 | assert.equal(contents.length, 1) 287 | assert.equal(contents[0].id, 5) 288 | }) 289 | }) 290 | 291 | describe('queue : processing', function() { 292 | beforeEach(function(){ 293 | i = 0 294 | queue = createQueue() 295 | }) 296 | 297 | afterEach(function(){ 298 | deleteQueue() 299 | }) 300 | 301 | it('should log generation', function(done) { 302 | var job = createJob(false) 303 | var errorGenerator = sinon.stub().returns(new Promise(resolve => resolve({ 304 | code: '001', 305 | error: true 306 | }))) 307 | var successGenerator = sinon.stub().returns(new Promise(resolve => resolve({ 308 | success: true 309 | }))) 310 | queue = createQueue({}, [ 311 | job 312 | ]) 313 | 314 | Promise.all([ 315 | queue.processJob(errorGenerator, job), 316 | queue.processJob(successGenerator, job) 317 | ]).then(function(responses) { 318 | assert(error.isError(responses[0])) 319 | assert(!error.isError(responses[1])) 320 | 321 | var dbJob = getQueue()[0] 322 | 323 | assert.equal(dbJob.generations.length, 2) 324 | assert.equal(dbJob.generations[0].code, '001') 325 | assert.equal(dbJob.generations[1].success, true) 326 | 327 | done() 328 | }) 329 | }) 330 | 331 | it('should mark as complete on success', function (done) { 332 | var job = createJob(false) 333 | queue = createQueue({}, [ 334 | job 335 | ]) 336 | 337 | var pingStub = sinon.stub(webhook, 'ping').returns(new Promise(resolve => resolve({ pinged: true }))) 338 | var generatorStub = sinon.stub().returns(new Promise(resolve => resolve({ 339 | completed: true, 340 | storage: { 341 | local: 'awesome' 342 | } 343 | }))) 344 | 345 | var webhookOptions = { url: 'http://localhost' } 346 | queue.processJob(generatorStub, job, webhookOptions).then(response => { 347 | assert.equal(response.completed, true) 348 | 349 | var dbJob = getQueue()[0] 350 | 351 | assert(dbJob.completed_at !== null) 352 | assert(dbJob.storage.local, 'awesome') 353 | assert(response.completed, true) 354 | 355 | var pingArgs = pingStub.args[0] 356 | assert.equal(pingArgs[0], job) 357 | assert.equal(pingArgs[1], webhookOptions) 358 | 359 | pingStub.restore() 360 | done() 361 | }) 362 | }) 363 | }) 364 | 365 | describe('queue : pinging', function() { 366 | beforeEach(function(){ 367 | i = 0 368 | queue = createQueue() 369 | }) 370 | 371 | afterEach(function(){ 372 | deleteQueue() 373 | }) 374 | 375 | it('should throw error if no webhook is configured', function() { 376 | var didThrow = false 377 | try { 378 | queue.attemptPing(createJob(true)) 379 | } catch (e) { 380 | if (e.toString() === 'Error: No webhook is configured.') { 381 | didThrow = true 382 | } 383 | } 384 | 385 | assert(didThrow) 386 | }) 387 | 388 | it('should attempt ping with correct parameters', function(done){ 389 | var job = createJob(true) 390 | queue = createQueue({}, [ 391 | job 392 | ]) 393 | 394 | pingStub = sinon.stub(webhook, 'ping').returns( 395 | new Promise((resolve) => resolve({ message: 'yay' })) 396 | ) 397 | 398 | var url = 'http://localhost'; 399 | queue.attemptPing(job, { 400 | url: url 401 | }).then(response => { 402 | assert.equal(response.message, 'yay') 403 | 404 | dbJob = getQueue()[0] 405 | 406 | assert.equal(dbJob.pings.length, 1) 407 | 408 | var ping = dbJob.pings[0] 409 | 410 | assert.equal(ping.message, 'yay') 411 | 412 | pingStub.restore() 413 | done() 414 | }) 415 | }) 416 | }) 417 | 418 | function inFiveMinutes() { 419 | var dateOne = new Date() 420 | dateOne.setTime(dateOne.getTime() + (1000 * 60 * 5)) // add 5 minutes 421 | return dateOne.toUTCString() 422 | } 423 | 424 | function fiveMinutesAgo() { 425 | var dateOne = new Date() 426 | dateOne.setTime(dateOne.getTime() - (1000 * 60 * 5)) // add 5 minutes 427 | return dateOne.toUTCString() 428 | } 429 | -------------------------------------------------------------------------------- /test/storage/s3.test.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon') 2 | var s3 = require('s3') 3 | var createS3Storage = require('../../src/storage/s3') 4 | 5 | var job = { 6 | id: 1 7 | } 8 | 9 | describe('storage:s3', function() { 10 | var createClientStub, uploadFileStub, onSpy 11 | beforeEach(function(){ 12 | onSpy = sinon.stub().callsFake(function(type, func) { 13 | if (type === 'end') { 14 | func({}) 15 | } 16 | }) 17 | uploadFileStub = sinon.stub().returns({ 18 | on: onSpy 19 | }) 20 | createClientStub = sinon.stub(s3, 'createClient').returns({ 21 | uploadFile: uploadFileStub 22 | }) 23 | }) 24 | 25 | afterEach(function(){ 26 | createClientStub.restore() 27 | }) 28 | 29 | it('should throw when access key id is not passed', function() { 30 | var didThrow = false 31 | try { 32 | createS3Storage({}) 33 | } catch(e) { 34 | if (e.toString() === 'Error: S3: No access key given') { 35 | didThrow = true 36 | } 37 | } 38 | if (!didThrow) { 39 | throw new Error('Error was not thrown when no access key id was given') 40 | } 41 | }) 42 | 43 | it('should throw when secret access key is not passed', function() { 44 | var didThrow = false 45 | try { 46 | createS3Storage({ accessKeyId: '1234' }) 47 | } catch(e) { 48 | if (e.toString() === 'Error: S3: No secret access key given') { 49 | didThrow = true 50 | } 51 | } 52 | if (!didThrow) { 53 | throw new Error('Error was not thrown when no access key id was given') 54 | } 55 | }) 56 | 57 | it('should throw when region is not passed', function() { 58 | var didThrow = false 59 | try { 60 | createS3Storage({ accessKeyId: '1234', secretAccessKey: '1234' }) 61 | } catch(e) { 62 | if (e.toString() === 'Error: S3: No region specified') { 63 | didThrow = true 64 | } 65 | } 66 | if (!didThrow) { 67 | throw new Error('Error was not thrown when no access key id was given') 68 | } 69 | }) 70 | 71 | it('should throw when bucket is not passed', function() { 72 | var didThrow = false 73 | try { 74 | createS3Storage({ accessKeyId: '1234', secretAccessKey: '1234', region: 'us-west-1' }) 75 | } catch(e) { 76 | if (e.toString() === 'Error: S3: No bucket was specified') { 77 | didThrow = true 78 | } 79 | } 80 | if (!didThrow) { 81 | throw new Error('Error was not thrown when no access key id was given') 82 | } 83 | }) 84 | 85 | it('create client with correct settings', function(done) { 86 | createS3Storage({ 87 | accessKeyId: '1234', 88 | secretAccessKey: '4321', 89 | region: 'us-west-1', 90 | bucket: 'bucket', 91 | s3ClientOptions: { 92 | test: true 93 | } 94 | })('path', job).then(() => { 95 | var expectedOptions = { 96 | s3Options: { 97 | accessKeyId: '1234', 98 | secretAccessKey: '4321', 99 | region: 'us-west-1' 100 | }, 101 | test: true 102 | } 103 | 104 | if (!createClientStub.calledOnce || !createClientStub.calledWith(expectedOptions)) { 105 | throw new Error('Client was not created with correct options') 106 | } 107 | 108 | done() 109 | }) 110 | }) 111 | 112 | it('should attempt to upload file with correct params', function(done){ 113 | createS3Storage({ 114 | accessKeyId: '1234', 115 | secretAccessKey: '4321', 116 | region: 'us-west-1', 117 | bucket: 'bucket', 118 | path: 'remote-folder', 119 | s3ClientOptions: { 120 | test: true 121 | } 122 | })('some/epic/path', job).then(() => { 123 | var expectedOptions = { 124 | localFile: 'some/epic/path', 125 | s3Params: { 126 | Bucket: 'bucket', 127 | Key: 'remote-folder/path' 128 | } 129 | } 130 | if (!uploadFileStub.calledOnce || !uploadFileStub.calledWith(expectedOptions)) { 131 | throw new Error('uploadFile was not called with correct options') 132 | } 133 | 134 | done() 135 | }) 136 | }) 137 | 138 | it('should call path if a function is passed', function(done) { 139 | var path = sinon.stub().returns('remote-path') 140 | createS3Storage({ 141 | accessKeyId: '1234', 142 | secretAccessKey: '4321', 143 | region: 'us-west-1', 144 | bucket: 'bucket', 145 | path: path, 146 | s3ClientOptions: { 147 | test: true 148 | } 149 | })('path', job).then(() => { 150 | if (!path.calledOnce || !path.calledWith('path', job)) { 151 | throw new Error('Path function was not called') 152 | } 153 | 154 | done() 155 | }) 156 | }) 157 | }) 158 | -------------------------------------------------------------------------------- /test/webhook.test.js: -------------------------------------------------------------------------------- 1 | var sinon = require('sinon') 2 | var proxyquire = require('proxyquire') 3 | var fetch = require('node-fetch') 4 | 5 | var job = { 6 | id: 1, 7 | url: 'http://localhost', 8 | meta: { 9 | id: 1 10 | }, 11 | storage: { 12 | local: 'something.pdf' 13 | } 14 | } 15 | 16 | describe('webhook', function() { 17 | var options = { 18 | secret: '1234', 19 | url: 'http://localhost/hook' 20 | } 21 | var fetchStub, promise, webhook 22 | 23 | beforeEach(function() { 24 | promise = new Promise(resolve => resolve({})) 25 | fetchStub = sinon.stub().returns(promise) 26 | webhook = proxyquire('../src/webhook', { 27 | 'node-fetch': fetchStub 28 | }) 29 | }) 30 | 31 | it('should throw error if no valid url is given', function() { 32 | var didThrow = false 33 | try { 34 | webhook.ping({}, { url: 'hello' }) 35 | } catch (e) { 36 | didThrow = true 37 | } 38 | 39 | if (!didThrow) { 40 | throw new Error('Did not throw on invalid URL') 41 | } 42 | }) 43 | 44 | it('should throw if no secret is given', function() { 45 | var didThrow = false 46 | try { 47 | webhook.ping({}, { url: 'http://localhost' }) 48 | } catch (e) { 49 | didThrow = true 50 | } 51 | 52 | if (!didThrow) { 53 | throw new Error('Did not throw on no secret') 54 | } 55 | }) 56 | 57 | it('should add passed request options to the request', function() { 58 | options.headerNamespace = 'X-Tests-' 59 | options.requestOptions = { 60 | headers: { 61 | 'X-Something': 'hello' 62 | }, 63 | method: 'GET' 64 | } 65 | 66 | webhook.ping(job, options) 67 | 68 | var fetchOptions = fetchStub.args[0][1] 69 | var headers = fetchOptions.headers.raw() 70 | 71 | if ( 72 | headers['content-type'][0] !== 'application/json' || 73 | !headers['x-tests-transaction'][0] || 74 | !headers['x-tests-signature'][0] || 75 | headers['x-something'][0] !== 'hello' 76 | ) { 77 | console.log(headers) 78 | throw new Error('Headers were not set correctly.') 79 | } 80 | 81 | if (fetchOptions.method !== 'POST') { 82 | throw new Error('Mehod was not POST.') 83 | } 84 | 85 | if (fetchOptions.body !== JSON.stringify(job)) { 86 | throw new Error('Body was not correct.') 87 | } 88 | }) 89 | 90 | it('should return empty response for 204 and 205', function(done) { 91 | var json = sinon.spy() 92 | fetchStub.returns( 93 | new Promise(function(resolve) { 94 | return resolve({ 95 | json: json, 96 | status: 204 97 | }) 98 | }) 99 | ) 100 | 101 | webhook.ping(job, options).then(function(response) { 102 | if (!json.notCalled) { 103 | throw new Error('json was called') 104 | } 105 | 106 | done() 107 | }) 108 | }) 109 | 110 | it('should be marked as error on bad response', function(done) { 111 | fetchStub.returns( 112 | new Promise(function(resolve) { 113 | return resolve({ 114 | status: 422 115 | }) 116 | }) 117 | ) 118 | 119 | webhook.ping(job, options).then(function(response) { 120 | if (!response.error) { 121 | throw new Error('it was not marked as error') 122 | } 123 | 124 | done() 125 | }) 126 | }) 127 | 128 | it('should return proper response on success', function (done) { 129 | fetchStub.returns( 130 | new Promise(function (resolve) { 131 | resolve(new fetch.Response(JSON.stringify('response'), { 132 | status: 200, 133 | headers: { 134 | 'content-type': 'application/json' 135 | } 136 | })) 137 | }) 138 | ) 139 | 140 | webhook.ping(job, options).then(function (response) { 141 | if (response.id !== fetchStub.args[0][1].headers.raw()['x-tests-transaction'][0] || 142 | response.method !== 'POST' || 143 | response.response !== 'response' || 144 | response.status !== 200) { 145 | console.log(response) 146 | throw new Error('Invalid response') 147 | } 148 | 149 | done() 150 | }) 151 | }) 152 | 153 | it('should generate proper HMAC signature', function() { 154 | var key = '12345' 155 | var body = 'awesome pdf generator' 156 | var signature = webhook.generateSignature(body, key) 157 | 158 | if (signature !== '6ff42a71ad26f83b76ea41defa22fb520716ddfb') { 159 | throw new Error('Generated signature was not correct') 160 | } 161 | }) 162 | }) 163 | --------------------------------------------------------------------------------