├── .eslintrc ├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json └── test ├── .eslintrc ├── mocha.opts └── test.js /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "extends": [ 4 | "jongleberry" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store* 3 | *.log 4 | *.gz 5 | 6 | node_modules 7 | coverage 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | node_js: 2 | - 4 3 | - 5 4 | language: node_js 5 | sudo: false 6 | script: npm run test-ci 7 | after_script: npm install coveralls@2 && cat ./coverage/lcov.info | coveralls 8 | env: 9 | global: 10 | - secure: Tqy9f+O8zudsSRlptkCDrE3As1mscrD8pfovR7By+WEwf9mIurCMd5wg+iMPiqtv1L8yaJEqNGTtR7b2a4bo+0r6DO9V6Ebf7hnyLlHq8fNnYDUjvirKe4efZ8a+wmTOB5GhlvkUc4/yt0casoMqreTCRDyKPEUJ4fOIIbMZtCE= 11 | - secure: GepSVeqIo7ZH/jHfr3a29IBzAtWWNTndpKlUFVGHrLN7v7MMU85HEaYlsNfXK0ZUsxcLkLx14W7NsPtdxYYu91aGePYFc+jfCd1a1fKQECj/G+8CORKaHLbPRY3qKEJZnS3XNJ7/vbURsNb3U0wtn67nABLkXX+IYavZ2KfeCoo= 12 | - secure: Tmx4PQvrrP1+HOC0Cwb3HrTY5t1XyaaBGMlVePzxrmpW2pThJAq/IknsHYnncYmHfE8CA7pKBm9Qen+JnAiOQ9C1wqjb2Q6zGFUiidtmf7d5pFcy7hpbMFeTFwdQWDi+eY9Eju4KxBaX3pTtWt/VMFY+0h/X0l7897FlTlwtCgo= 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | The MIT License (MIT) 3 | 4 | Copyright (c) 2015 Jonathan Ong me@jongleberry.com 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # s3-cache 3 | 4 | [![NPM version][npm-image]][npm-url] 5 | [![Build status][travis-image]][travis-url] 6 | [![Test coverage][coveralls-image]][coveralls-url] 7 | [![Dependency Status][david-image]][david-url] 8 | [![License][license-image]][license-url] 9 | [![Downloads][downloads-image]][downloads-url] 10 | 11 | > NOTE: MAINTAINER WANTED 12 | 13 | Caches and serves responses to and from S3. 14 | Similar to [koa-cash](https://github.com/koajs/cash), 15 | except it optimizes for the S3 use-case by streaming. 16 | 17 | Usage: 18 | 19 | ```js 20 | let cache = require('koa-s3-cache')({ 21 | // s3 credentials and stuff 22 | }) 23 | 24 | app.use(function* (next) { 25 | // served from cache 26 | if (yield* cache.get(this)) return 27 | 28 | // do some crazy computation 29 | this.body = new Buffer(1024 * 1024) 30 | 31 | // save it to s3 32 | yield* cache.put(this) 33 | }) 34 | ``` 35 | 36 | This is best for dynamically created content that is cached (i.e. thumbnails). 37 | Instead of caching yourself in the business logic, 38 | cache transparently with this module. 39 | 40 | ## API 41 | 42 | ### const cache = Cache(options) 43 | 44 | Create a `cache` instance. 45 | 46 | S3 options: 47 | 48 | - `key` 49 | - `secret` 50 | - `bucket` 51 | 52 | Other options: 53 | 54 | - `salt` - add a salt to namespace your `cache` instances 55 | 56 | ### app.use(cache) 57 | 58 | You can use the cache as middleware, 59 | which caches all downstream middleware. 60 | 61 | ```js 62 | app.use(cache) 63 | 64 | app.use(function* () { 65 | this.body = 'something computationally intensive' 66 | }) 67 | ``` 68 | 69 | ### app.use(cache.wrap( next => )) 70 | 71 | Wrap a middleware with the cache. 72 | Useful for conditional caching 73 | 74 | ```js 75 | app.use(cache.wrap(function* () { 76 | this.body = 'something computationally intensive' 77 | })) 78 | ``` 79 | 80 | ### const served = yield cache.get(this) 81 | 82 | Serve this request from the cache. 83 | Returns `served`, which is whether the response has been served from the cache. 84 | 85 | ### yield cache.put(this) 86 | 87 | Caches the current response. 88 | 89 | ## Notes 90 | 91 | - You should set an object lifecycle rule. 92 | Ex. delete all files in the bucket after 7 days. 93 | - Objects are stored with `REDUCED_REDUNDANCY`. 94 | - Only supports `200-2` status codes. 95 | - If the body is streaming, the stream is cached to the filesystem so that the S3 client knows its `content-length`. 96 | 97 | [npm-image]: https://img.shields.io/npm/v/koa-s3-cache.svg?style=flat-square 98 | [npm-url]: https://npmjs.org/package/koa-s3-cache 99 | [github-tag]: http://img.shields.io/github/tag/koajs/s3-cache.svg?style=flat-square 100 | [github-url]: https://github.com/koajs/s3-cache/tags 101 | [travis-image]: https://img.shields.io/travis/koajs/s3-cache.svg?style=flat-square 102 | [travis-url]: https://travis-ci.org/koajs/s3-cache 103 | [coveralls-image]: https://img.shields.io/coveralls/koajs/s3-cache.svg?style=flat-square 104 | [coveralls-url]: https://coveralls.io/r/koajs/s3-cache 105 | [david-image]: http://img.shields.io/david/koajs/s3-cache.svg?style=flat-square 106 | [david-url]: https://david-dm.org/koajs/s3-cache 107 | [license-image]: http://img.shields.io/npm/l/koa-s3-cache.svg?style=flat-square 108 | [license-url]: LICENSE 109 | [downloads-image]: http://img.shields.io/npm/dm/koa-s3-cache.svg?style=flat-square 110 | [downloads-url]: https://npmjs.org/package/koa-s3-cache 111 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const onFinished = require('on-finished') 4 | const temp = require('temp-path') 5 | const crypto = require('crypto') 6 | const knox = require('knox') 7 | const cp = require('fs-cp') 8 | const fs = require('fs') 9 | 10 | module.exports = Cache 11 | 12 | function Cache (options) { 13 | if (!(this instanceof Cache)) return new Cache(options) 14 | 15 | this.client = knox.createClient(options) 16 | 17 | // salt for hashing 18 | this.salt = options.salt || '' 19 | } 20 | 21 | // pretend it's a generator function so app.use() works 22 | // TODO: remove when we add async/await support 23 | Cache.prototype.constructor = function * () {}.constructor 24 | Cache.prototype.call = function * (ctx, next) { 25 | if (yield this.get(ctx)) return 26 | 27 | yield next 28 | 29 | yield this.put(ctx) 30 | } 31 | 32 | Cache.prototype.wrap = function (fn) { 33 | const self = this 34 | return function * (next) { 35 | if (yield self.get(this)) return 36 | 37 | yield fn.call(this, next) 38 | 39 | yield self.put(this) 40 | } 41 | } 42 | 43 | Cache.prototype.get = function * (ctx) { 44 | const key = this._key(ctx) 45 | const res = yield new Promise((resolve, reject) => { 46 | this.client.getFile(key, (err, res) => { 47 | /* istanbul ignore if */ 48 | if (err) return reject(err) 49 | resolve(res) 50 | }) 51 | }) 52 | 53 | if (res.statusCode !== 200) { 54 | res.resume() 55 | return false 56 | } 57 | 58 | ctx.body = res 59 | 60 | const keys = [ 61 | 'cache-control', 62 | 'content-disposition', 63 | 'content-encoding', 64 | 'content-language', 65 | 'content-length', 66 | 'content-type', 67 | 'etag', 68 | ] 69 | for (const key of keys) { 70 | if (res.headers[key]) { 71 | ctx.response.set(key, res.headers[key]) 72 | } 73 | } 74 | 75 | if (ctx.fresh) ctx.status = 304 76 | 77 | return true 78 | } 79 | 80 | Cache.prototype.set = 81 | Cache.prototype.put = function * (ctx) { 82 | switch (ctx.status) { 83 | case 200: 84 | case 201: 85 | case 202: 86 | break 87 | default: 88 | return 89 | } 90 | 91 | const client = this.client 92 | const response = ctx.response 93 | const res = ctx.res 94 | let body = response.body 95 | if (!body) return 96 | 97 | // if a stream, we save it to a file and serve from that 98 | // because amazon needs to know the content length prior -_- 99 | let filename 100 | if (typeof body.pipe === 'function') { 101 | yield cp(body, filename = temp()) 102 | body = null 103 | 104 | // re-serve the same response 105 | response.body = fs.createReadStream(filename) 106 | // always delete the file afterwards 107 | onFinished(res, () => { 108 | setImmediate(() => { 109 | fs.unlink(filename, noop) 110 | }) 111 | }) 112 | } 113 | 114 | const headers = { 115 | 'x-amz-storage-class': 'REDUCED_REDUNDANCY', 116 | } 117 | 118 | // note: capitalization required for Knox 119 | const keys = [ 120 | 'Cache-Control', 121 | 'Content-Disposition', 122 | 'Content-Encoding', 123 | 'Content-Language', 124 | 'Content-Type', 125 | ] 126 | for (const key of keys) { 127 | if (response.get(key)) { 128 | headers[key] = response.get(key) 129 | } 130 | } 131 | 132 | const key = this._key(ctx) 133 | 134 | const s3response = yield new Promise((resolve, reject) => { 135 | if (filename) { 136 | client.putFile(filename, key, headers, callback) 137 | } else { 138 | client.putBuffer(body, key, headers, callback) 139 | } 140 | 141 | function callback (err, res) { 142 | // istanbul ignore if */ 143 | if (err) return reject(err) 144 | resolve(res) 145 | 146 | // TODO: real error handling 147 | // istanbul ignore if */ 148 | if (res.statusCode !== 200) { 149 | res.setEncoding('utf8') 150 | res.on('data', (chunk) => { 151 | /* eslint no-console: 0 */ 152 | console.log(chunk) 153 | }) 154 | } 155 | } 156 | }) 157 | 158 | // dump the response because we don't care 159 | s3response.resume() 160 | ctx.assert(s3response.statusCode === 200, 'Did not get a 200 from uploading the file.') 161 | 162 | // use s3's headers 163 | response.etag = s3response.headers.etag 164 | 165 | if (ctx.fresh) ctx.status = 304 166 | } 167 | 168 | // hash the path so that it's always a simple string for s3 169 | // and to make it more cacheable by making the first few chars random 170 | Cache.prototype._key = function (ctx) { 171 | return crypto.createHash('sha256') 172 | .update(this.salt) 173 | .update('url:') 174 | .update(ctx.request.url) 175 | .digest('hex') 176 | } 177 | 178 | function noop () {} 179 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "koa-s3-cache", 3 | "description": "Koa middleware to cache and serve from S3", 4 | "version": "1.1.0", 5 | "author": "Jonathan Ong (http://jongleberry.com)", 6 | "license": "MIT", 7 | "repository": "koajs/s3-cache", 8 | "dependencies": { 9 | "fs-cp": "^1.3.0", 10 | "knox": "^0.9.2", 11 | "on-finished": "^2.2.0", 12 | "temp-path": "^1.0.0" 13 | }, 14 | "devDependencies": { 15 | "babel-eslint": "^6.0.0", 16 | "codecov": "^1.0.0", 17 | "eslint": "^2.1.0", 18 | "eslint-config-jongleberry": "^4.0.0", 19 | "eslint-plugin-babel": "^3.2.0", 20 | "eslint-plugin-promise": "^1.0.8", 21 | "eslint-plugin-standard": "^1.0.0", 22 | "istanbul": "^0.4.2", 23 | "koa": "^1.1.2", 24 | "mocha": "^2.4.5", 25 | "supertest": "^1.2.0" 26 | }, 27 | "scripts": { 28 | "lint": "eslint index.js test", 29 | "test": "mocha", 30 | "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- --reporter dot", 31 | "test-ci": "npm run lint && istanbul cover node_modules/mocha/bin/_mocha --report lcovonly -- --reporter dot" 32 | }, 33 | "keywords": [ 34 | "koa", 35 | "s3", 36 | "cache" 37 | ], 38 | "files": [ 39 | "index.js" 40 | ] 41 | } 42 | -------------------------------------------------------------------------------- /test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --timeout 10s 2 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const request = require('supertest') 4 | const assert = require('assert') 5 | const koa = require('koa') 6 | const fs = require('fs') 7 | 8 | const cache = require('..')({ 9 | key: process.env.KOA_S3_CACHE_AWS_ACCESS_KEY_ID || process.env.AWS_ACCESS_KEY_ID, 10 | secret: process.env.KOA_S3_CACHE_AWS_SECRET_ACCESS_KEY || process.env.AWS_SECRET_ACCESS_KEY, 11 | bucket: process.env.KOA_S3_CACHE_BUCKET, 12 | salt: random(), 13 | }) 14 | 15 | describe('string', () => { 16 | const app = koa() 17 | const path = '/' + random() 18 | const text = 'kljasldkfjlaksdjflkajsdf' 19 | 20 | app.use(function * () { 21 | if (yield cache.get(this)) return 22 | 23 | this.body = text 24 | 25 | yield cache.put(this) 26 | }) 27 | 28 | const server = app.listen() 29 | 30 | let etag 31 | 32 | it('should cache and serve the response', done => { 33 | request(server) 34 | .get(path) 35 | .expect('Content-Type', /text\/plain/) 36 | .expect(200) 37 | .expect(text, (err, res) => { 38 | if (err) return done(err) 39 | 40 | assert(etag = res.headers.etag) 41 | done() 42 | }) 43 | }) 44 | 45 | it('should serve the cached response', done => { 46 | request(server) 47 | .get(path) 48 | .expect('Content-Type', /text\/plain/) 49 | .expect('ETag', etag) 50 | .expect(200) 51 | .expect(text, (err, res) => { 52 | if (err) return done(err) 53 | 54 | assert(etag = res.headers.etag) 55 | done() 56 | }) 57 | }) 58 | 59 | it('should support caching', done => { 60 | request(server) 61 | .get(path) 62 | .set('if-none-match', etag) 63 | .expect(304, done) 64 | }) 65 | }) 66 | 67 | describe('stream', () => { 68 | const app = koa() 69 | const path = '/' + random() 70 | 71 | app.use(function * () { 72 | if (yield cache.get(this)) return 73 | 74 | this.type = 'text' 75 | this.body = fs.createReadStream(__filename) 76 | 77 | yield cache.put(this) 78 | }) 79 | 80 | const server = app.listen() 81 | 82 | let etag 83 | 84 | it('should cache and serve the response', done => { 85 | request(server) 86 | .get(path) 87 | .expect('Content-Type', /text\/plain/) 88 | .expect(200, (err, res) => { 89 | if (err) return done(err) 90 | 91 | assert(etag = res.headers.etag) 92 | assert(/laksjdflkajsldkfjalksdf/.test(res.text)) 93 | done() 94 | }) 95 | }) 96 | 97 | it('should serve the cached response', done => { 98 | request(server) 99 | .get(path) 100 | .expect('Content-Type', /text\/plain/) 101 | .expect('ETag', etag) 102 | .expect(200, (err, res) => { 103 | if (err) return done(err) 104 | 105 | assert(etag = res.headers.etag) 106 | assert(/klajsdfljasdfasdf/.test(res.text)) 107 | done() 108 | }) 109 | }) 110 | }) 111 | 112 | describe('404', () => { 113 | const app = koa() 114 | 115 | app.use(function * () { 116 | if (yield cache.get(this)) return 117 | 118 | yield cache.put(this) 119 | }) 120 | 121 | const server = app.listen() 122 | 123 | it('should not cache 404s', done => { 124 | request(server) 125 | .get('/') 126 | .expect(404) 127 | .expect('Not Found', (err, res) => { 128 | if (err) return done(err) 129 | 130 | assert(!res.headers.tag) 131 | done() 132 | }) 133 | }) 134 | }) 135 | 136 | describe('no content', () => { 137 | const app = koa() 138 | 139 | app.use(function * () { 140 | if (yield cache.get(this)) return 141 | 142 | this.body = '' 143 | 144 | yield cache.put(this) 145 | }) 146 | 147 | const server = app.listen() 148 | 149 | it('should not cache empty bodies', done => { 150 | request(server) 151 | .get('/') 152 | .expect(200) 153 | .expect('', (err, res) => { 154 | if (err) return done(err) 155 | 156 | assert(!res.headers.tag) 157 | done() 158 | }) 159 | }) 160 | }) 161 | 162 | describe('middleware', () => { 163 | const app = koa() 164 | const path = '/' + random() 165 | const body = random() 166 | 167 | app.use(cache) 168 | 169 | app.use(function * () { 170 | this.body = body 171 | }) 172 | 173 | const server = app.listen() 174 | 175 | it('should work as middleware', done => { 176 | request(server) 177 | .get(path) 178 | .expect(body) 179 | .expect(200, done) 180 | }) 181 | }) 182 | 183 | describe('.wrap()', () => { 184 | const app = koa() 185 | const path = '/' + random() 186 | const body = random() 187 | 188 | app.use(cache.wrap(function * () { 189 | this.body = body 190 | })) 191 | 192 | const server = app.listen() 193 | 194 | it('should wrap middleware', done => { 195 | request(server) 196 | .get(path) 197 | .expect(body) 198 | .expect(200, done) 199 | }) 200 | }) 201 | 202 | function random () { 203 | return Math.random().toString(36) 204 | } 205 | --------------------------------------------------------------------------------