├── index.js ├── .eslintrc ├── .gitignore ├── .editorconfig ├── .travis.yml ├── .vscode └── launch.json ├── test ├── auth.js ├── errors.js ├── headers.js ├── body.js ├── abort.js ├── setup.js ├── transforms.js ├── cache.js └── request-options.js ├── package.json ├── lib ├── request-options.js └── proxy.js └── README.md /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/proxy'); 2 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "eslint:recommended", 3 | "env": { 4 | "node": true 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | .DS_Store 4 | *.log 5 | .eslintcache 6 | .vscode 7 | .coveralls.yml 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "4" 4 | matrix: 5 | allow_failures: 6 | fast_finish: true 7 | script: "npm run test-travis && npm run lint" 8 | after_script: 9 | - echo "Generating .coverall.yml" 10 | - echo "repo_token: ${COVERALLS_REPO_TOKEN}" > .coveralls.yml 11 | - echo "Submitting code coverage to coveralls" 12 | - cat ./coverage/lcov.info | ./node_modules/.bin/coveralls 13 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Mocha Tests", 11 | "program": "${workspaceFolder}/node_modules/mocha/bin/_mocha", 12 | "args": [ 13 | "-u", 14 | "tdd", 15 | "--timeout", 16 | "999999", 17 | "--colors", 18 | "${workspaceFolder}/test/body.js", 19 | "-g", 20 | "posts to a wildcard route" 21 | ], 22 | "internalConsoleOptions": "openOnSessionStart" 23 | }, 24 | { 25 | "type": "node", 26 | "request": "launch", 27 | "name": "Launch Program", 28 | "program": "${workspaceFolder}/index.js" 29 | } 30 | ] 31 | } 32 | -------------------------------------------------------------------------------- /test/auth.js: -------------------------------------------------------------------------------- 1 | var supertest = require('supertest'); 2 | var proxy = require('..'); 3 | var setup = require('./setup'); 4 | 5 | describe('proxy authentication', function() { 6 | var self; 7 | 8 | beforeEach(setup.beforeEach); 9 | afterEach(setup.afterEach); 10 | 11 | beforeEach(function() { 12 | self = this; 13 | self.isAuthenticated = false; 14 | 15 | this.server.use(function(req, res, next) { 16 | req.ext.isAuthenticated = self.isAuthenticated; 17 | next(); 18 | }); 19 | 20 | self.proxyOptions.ensureAuthenticated = true; 21 | this.server.get('/proxy', proxy(self.proxyOptions)); 22 | 23 | this.server.use(setup.errorHandler); 24 | }); 25 | 26 | it('returns 401 when request not authenticated', function(done) { 27 | self.isAuthenticated = false; 28 | supertest(this.server) 29 | .get('/proxy') 30 | .expect(401, done); 31 | }); 32 | 33 | it('succeeds when request is authenticated', function(done) { 34 | this.isAuthenticated = true; 35 | supertest(this.server) 36 | .get('/proxy') 37 | .expect(200, done); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /test/errors.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var supertest = require('supertest'); 3 | var memoryCache = require('memory-cache-stream'); 4 | var proxy = require('..'); 5 | var setup = require('./setup'); 6 | 7 | describe('timeout', function() { 8 | beforeEach(setup.beforeEach); 9 | afterEach(setup.afterEach); 10 | 11 | var self; 12 | beforeEach(function() { 13 | self = this; 14 | 15 | this.server.get('/proxy', proxy(this.proxyOptions)); 16 | this.server.use(setup.errorHandler); 17 | }); 18 | 19 | it('returns error message', function(done) { 20 | this.apiResponseStatus = 400; 21 | this.apiResponse = {error: 'bad request'}; 22 | 23 | supertest(this.server).get('/proxy') 24 | .expect(400) 25 | .expect('Content-Type', /application\/json/) 26 | .expect(function(res) { 27 | assert.deepEqual(res.body, self.apiResponse); 28 | }) 29 | .end(done); 30 | }); 31 | 32 | it('api timeout returns 504', function(done) { 33 | this.apiLatency = 50; 34 | this.proxyOptions.timeout = 20; 35 | 36 | supertest(this.server) 37 | .get('/proxy') 38 | .expect(504, done); 39 | }); 40 | 41 | it('api returns 408 when instructed to cache', function(done) { 42 | this.apiLatency = 50; 43 | this.proxyOptions.timeout = 20; 44 | this.proxyOptions.cache = memoryCache(); 45 | 46 | supertest(this.server) 47 | .get('/proxy') 48 | .expect(408) 49 | .end(function(err, res) { 50 | self.proxyOptions.cache.exists(self.baseApiUrl, function(_err, exists) { 51 | assert.equal(0, exists); 52 | done(); 53 | }); 54 | }); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "express-request-proxy", 3 | "version": "2.2.2", 4 | "description": "Intelligent http proxy Express middleware", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha --reporter spec --bail --exit --check-leaks test/", 8 | "test-cov": "istanbul cover node_modules/mocha/bin/_mocha -- --reporter dot --check-leaks test/", 9 | "test-travis": "istanbul cover node_modules/mocha/bin/_mocha --report lcovonly -- --reporter spec --exit --check-leaks test/", 10 | "lint": "eslint --cache ./lib index.js" 11 | }, 12 | "repository": { 13 | "type": "git", 14 | "url": "https://github.com/dvonlehman/express-request-proxy" 15 | }, 16 | "keywords": [ 17 | "express", 18 | "middleware", 19 | "request", 20 | "api", 21 | "proxy" 22 | ], 23 | "author": "David Von Lehman (https://github.com/dvonlehman)", 24 | "license": "Apache-2.0", 25 | "bugs": { 26 | "url": "https://github.com/dvonlehman/express-request-proxy/issues" 27 | }, 28 | "homepage": "https://github.com/dvonlehman/express-request-proxy", 29 | "dependencies": { 30 | "async": "^2.6.1", 31 | "body-parser": "^1.18.3", 32 | "camel-case": "^3.0.0", 33 | "debug": "^3.1.0", 34 | "lodash": "^4.17.10", 35 | "lru-cache": "^4.1.3", 36 | "path-to-regexp": "^1.1.1", 37 | "request": "^2.87.0", 38 | "simple-errors": "^1.0.1", 39 | "through2": "^2.0.3", 40 | "type-is": "^1.6.16", 41 | "url-join": "4.0.0" 42 | }, 43 | "devDependencies": { 44 | "compression": "^1.7.2", 45 | "coveralls": "^3.0.2", 46 | "dash-assert": "^1.3.1", 47 | "eslint": "^4.19.1", 48 | "express": "^4.16.3", 49 | "istanbul": "^0.4.5", 50 | "memory-cache-stream": "^1.2.0", 51 | "mocha": "^5.2.0", 52 | "shortid": "^2.2.8", 53 | "sinon": "^5.0.10", 54 | "supertest": "^3.1.0" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /test/headers.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var supertest = require('supertest'); 3 | var proxy = require('..'); 4 | var setup = require('./setup'); 5 | 6 | describe('http headers', function() { 7 | var self; 8 | beforeEach(setup.beforeEach); 9 | afterEach(setup.afterEach); 10 | 11 | beforeEach(function() { 12 | self = this; 13 | this.server.all('/proxy', proxy(this.proxyOptions)); 14 | this.server.use(setup.errorHandler); 15 | }); 16 | 17 | it('passes through content-type', function(done) { 18 | supertest(this.server).get('/proxy') 19 | .expect(200) 20 | .expect('Content-Type', /^application\/json/) 21 | .end(done); 22 | }); 23 | 24 | it('passes correct Content-Length for post requests', function(done) { 25 | var postData = {foo: 1}; 26 | 27 | supertest(this.server).post('/proxy') 28 | .set('Content-Type', 'application/json') 29 | .send(postData) 30 | .expect(200) 31 | .expect('Content-Type', /^application\/json/) 32 | .expect(function(res) { 33 | assert.equal(parseInt(res.body.headers['content-length'], 10), 34 | JSON.stringify(postData).length); 35 | }) 36 | .end(done); 37 | }); 38 | 39 | it('uses correct user-agent', function(done) { 40 | supertest(this.server).get('/proxy') 41 | .expect(200) 42 | .expect(function(res) { 43 | assert.equal(res.body.headers['user-agent'], 'express-request-proxy'); 44 | }) 45 | .end(done); 46 | }); 47 | 48 | it('passes through gzipped response', function(done) { 49 | this.apiGzipped = true; 50 | this.apiResponse = {foo: 1, name: 'elmer'}; 51 | 52 | supertest(this.server).get('/proxy') 53 | .set('Accept-Encoding', 'gzip') 54 | .expect('Content-Encoding', 'gzip') 55 | .expect(function(res) { 56 | assert.deepEqual(res.body, self.apiResponse); 57 | }) 58 | .end(done); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /test/body.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var supertest = require('supertest'); 3 | var proxy = require('..'); 4 | var debug = require('debug')('express-request-proxy:test'); 5 | var querystring = require('querystring'); 6 | var setup = require('./setup'); 7 | 8 | describe('req body', function() { 9 | var self; 10 | 11 | beforeEach(setup.beforeEach); 12 | afterEach(setup.afterEach); 13 | 14 | beforeEach(function() { 15 | self = this; 16 | this.server.all('/proxy', proxy(this.proxyOptions)); 17 | this.server.use(setup.errorHandler); 18 | }); 19 | 20 | it('posts JSON body', function(done) { 21 | var postData = {foo: 1, arr: [1, 2, 3]}; 22 | 23 | supertest(this.server).post('/proxy') 24 | .set('Content-Type', 'application/json') 25 | .send(postData) 26 | .expect(200) 27 | .expect('Content-Type', /^application\/json/) 28 | .expect(function(res) { 29 | assert.deepEqual(res.body.body, postData); 30 | assert.equal(res.body.headers['content-type'], 'application/json'); 31 | }) 32 | .end(done); 33 | }); 34 | 35 | it('posts url-encoded form body', function(done) { 36 | var postData = {foo: 1, bar: 'hello'}; 37 | 38 | supertest(this.server).post('/proxy') 39 | .set('Content-Type', 'application/x-www-form-urlencoded') 40 | .send(querystring.stringify(postData)) 41 | .expect(200) 42 | .expect(function(res) { 43 | assert.deepEqual(res.body.body, postData); 44 | }) 45 | .end(done); 46 | }); 47 | 48 | it('posts to a wildcard route', function(done) { 49 | var postData = {key: 'asfasdfasdfasdf'}; 50 | 51 | this.proxyOptions.url = 'http://localhost:' + this.apiPort + '/api/*'; 52 | this.remoteApi.post('/api/v1/token', function(req, res, next) { 53 | res.json({path: req.path}); 54 | }); 55 | 56 | this.server.post('/api/auth/*', function(req, res, next) { 57 | debug('hit the auth endpoint'); 58 | proxy(self.proxyOptions)(req, res, next); 59 | }); 60 | 61 | supertest(this.server).post('/api/auth/v1/token') 62 | .send(postData) 63 | .expect(200) 64 | .expect(function(res) { 65 | assert.equal(res.body.path, '/api/v1/token'); 66 | }) 67 | .end(done); 68 | }); 69 | }); 70 | -------------------------------------------------------------------------------- /test/abort.js: -------------------------------------------------------------------------------- 1 | var supertest = require('supertest'); 2 | var request = require('request'); 3 | var assert = require('assert'); 4 | var proxy = require('..'); 5 | var setup = require('./setup'); 6 | 7 | describe('request abortion', function() { 8 | var self; 9 | var fullyExecuted = false; 10 | var postCount = 0; 11 | var remoteRequestDelay = 50; 12 | 13 | beforeEach(setup.beforeEach); 14 | afterEach(setup.afterEach); 15 | 16 | beforeEach(function() { 17 | this.remoteApi.get('/longRoute', function(req, res) { 18 | var closedBeforeSend = false; 19 | req.on('close', function() { 20 | closedBeforeSend = true; 21 | }) 22 | setTimeout(function() { 23 | fullyExecuted = !closedBeforeSend; 24 | res.send({status: true}); 25 | }, remoteRequestDelay); 26 | }); 27 | this.remoteApi.post('/postCounter', function(req, res) { 28 | postCount++; 29 | res.send({status: true}); 30 | }); 31 | this.server.get('/proxyLongRoute', proxy({ 32 | url: 'http://localhost:' + this.apiPort + '/longRoute', 33 | cache: false 34 | })); 35 | this.server.post('/proxyPostCounter', proxy({ 36 | url: 'http://localhost:' + this.apiPort + '/postCounter', 37 | cache: false 38 | })); 39 | }); 40 | 41 | it('should process the remote api request fully if client do not close prematurely connection', function(done) { 42 | fullyExecuted = false; 43 | supertest(this.server) 44 | .get('/proxyLongRoute') 45 | .expect(200) 46 | .end(function(err, res) { 47 | if (!fullyExecuted) return done(new Error('Not fully executed')); 48 | done(); 49 | }); 50 | }); 51 | 52 | it('should cancel the remote api request fully if client do not close prematurely connection', function(done) { 53 | var self = this; 54 | this.server.listen(this.apiPort + 1, function() { 55 | fullyExecuted = false; 56 | var req = request({ 57 | url: 'http://localhost:' + (self.apiPort + 1) + '/proxyLongRoute' 58 | }); 59 | // Abort main request after 250ms. 60 | setTimeout(function() { req.abort(); }, remoteRequestDelay / 2); 61 | setTimeout(function() { 62 | if (!fullyExecuted) return done(); 63 | done(new Error('Request has not been closed.')); 64 | }, remoteRequestDelay + 10); 65 | }); 66 | }); 67 | 68 | it('should call the remote api request only once when request has a body', function(done) { 69 | postCount = 0; 70 | supertest(this.server) 71 | .post('/proxyPostCounter') 72 | .set('Content-Type', 'application/json') 73 | .send({sample: 'content'}) 74 | .expect(200) 75 | .end(function(err, res) { 76 | setTimeout(() => { 77 | assert.equal(postCount, 1); 78 | done(); 79 | }, 50); 80 | }); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var http = require('http'); 3 | var urljoin = require('url-join'); 4 | var bodyParser = require('body-parser'); 5 | var debug = require('debug')('express-request-proxy'); 6 | var _ = require('lodash'); 7 | var compression = require('compression'); 8 | var is = require('type-is'); 9 | 10 | module.exports.beforeEach = function() { 11 | var self = this; 12 | this.apiLatency = 0; 13 | this.apiResponse = null; 14 | this.apiResponseStatus = 200; 15 | this.originHeaders = {}; 16 | this.apiGzipped = false; 17 | 18 | this.remoteApi = express(); 19 | this.apiPort = 5998; 20 | 21 | function maybeParseBody(req, res, next) { 22 | if (is.hasBody(req)) { 23 | switch (is(req, ['urlencoded', 'json'])) { 24 | case 'urlencoded': 25 | debug('parse api urlencoded body'); 26 | return bodyParser.urlencoded({extended: false})(req, res, next); 27 | case 'json': 28 | debug('parse api json body'); 29 | return bodyParser.json()(req, res, next); 30 | default: 31 | break; 32 | } 33 | } 34 | 35 | next(); 36 | } 37 | 38 | this.remoteApi.use(compression({ 39 | threshold: 0, // bottom-out the min bytes threshold to force everything to be compressed 40 | filter: function(req, res) { 41 | return self.apiGzipped === true; 42 | } 43 | })); 44 | 45 | this.remoteApi.all('/api', maybeParseBody, function(req, res, next) { 46 | setTimeout(function() { 47 | if (!self.originHeaders['Content-Type']) { 48 | self.originHeaders['Content-Type'] = 'application/json'; 49 | } 50 | 51 | _.each(self.originHeaders, function(value, key) { 52 | res.set(key, value); 53 | }); 54 | 55 | if (self.apiResponseStatus) { 56 | res.statusCode = self.apiResponseStatus; 57 | } 58 | 59 | if (self.apiResponse) { 60 | if (self.originHeaders['Content-Type'] === 'application/json') { 61 | debug('remote api json response'); 62 | return res.json(self.apiResponse); 63 | } 64 | res.send(self.apiResponse); 65 | } else { 66 | var context = _.pick(req, 'query', 'path', 'params', 'headers', 'method', 'body'); 67 | context.fullUrl = urljoin('http://localhost:' + self.apiPort, req.originalUrl); 68 | res.json(context); 69 | } 70 | }, self.apiLatency); 71 | }); 72 | 73 | this.baseApiUrl = 'http://localhost:' + this.apiPort + '/api'; 74 | this.apiServer = http.createServer(this.remoteApi).listen(this.apiPort); 75 | 76 | this.proxyOptions = { 77 | url: this.baseApiUrl, 78 | timeout: 3000 79 | }; 80 | 81 | this.server = express(); 82 | this.server.use(function(req, res, next) { 83 | req.ext = {}; 84 | next(); 85 | }); 86 | }; 87 | 88 | module.exports.errorHandler = function(err, req, res, next) { 89 | if (!err.status) err.status = 500; 90 | 91 | // if (err.status >= 500) 92 | process.stderr.write(err.message); 93 | res.status(err.status).send(err.message); 94 | }; 95 | 96 | module.exports.afterEach = function(done) { 97 | if (this.apiServer) this.apiServer.close(); 98 | 99 | if (this.proxyOptions.cache) { 100 | this.proxyOptions.cache.flushall(done); 101 | } else { 102 | done(); 103 | } 104 | }; 105 | -------------------------------------------------------------------------------- /test/transforms.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var memoryCache = require('memory-cache-stream'); 3 | var supertest = require('supertest'); 4 | var _ = require('lodash'); 5 | var through2 = require('through2'); 6 | var proxy = require('..'); 7 | var setup = require('./setup'); 8 | 9 | describe('response transforms', function() { 10 | beforeEach(setup.beforeEach); 11 | afterEach(setup.afterEach); 12 | 13 | beforeEach(function() { 14 | this.proxyOptions.transforms = [ 15 | appenderTransform('<>', 'text/html') 16 | ]; 17 | 18 | this.originHeaders = { 19 | 'Content-Type': 'text/plain' 20 | }; 21 | 22 | this.apiResponse = '1234'; 23 | 24 | this.server.get('/proxy', proxy(this.proxyOptions)); 25 | this.server.use(setup.errorHandler); 26 | }); 27 | 28 | it('performs transform', function(done) { 29 | supertest(this.server).get('/proxy') 30 | .expect(200) 31 | .expect('Content-Type', /^text\/html/) 32 | .expect(function(res) { 33 | assert.equal(res.text, '1234<>'); 34 | }) 35 | .end(done); 36 | }); 37 | 38 | it('recreates the transform stream between requests', function(done) { 39 | var self = this; 40 | supertest(self.server).get('/proxy') 41 | .end(function() { 42 | supertest(self.server).get('/proxy') 43 | .expect(200) 44 | .expect('Content-Type', /^text\/html/) 45 | .expect(function(res) { 46 | assert.equal(res.text, '1234<>'); 47 | }) 48 | .end(done); 49 | }); 50 | }); 51 | 52 | it('transformed response is stored in cache', function(done) { 53 | var self = this; 54 | 55 | _.extend(this.proxyOptions, { 56 | cache: memoryCache(), 57 | cacheMaxAge: 100 58 | }); 59 | 60 | supertest(this.server).get('/proxy') 61 | .expect(200) 62 | .expect('Express-Request-Proxy-Cache', 'miss') 63 | .expect(function(res) { 64 | assert.equal(res.text, '1234<>'); 65 | }) 66 | .end(function(err, res) { 67 | if (err) return done(err); 68 | 69 | supertest(self.server).get('/proxy') 70 | .expect(200) 71 | .expect('Content-Type', /^text\/html/) 72 | .expect('Express-Request-Proxy-Cache', 'hit') 73 | .expect('1234<>') 74 | .end(done); 75 | }); 76 | }); 77 | 78 | it('works with multiple transforms', function(done) { 79 | this.proxyOptions.transforms.push(appenderTransform('<>', 'text/html')); 80 | 81 | supertest(this.server).get('/proxy') 82 | .expect(200) 83 | .expect(function(res) { 84 | assert.equal(res.text, '1234<><>'); 85 | }) 86 | .end(done); 87 | }); 88 | 89 | it('allows transform to override content-type', function(done) { 90 | this.proxyOptions.transforms = [appenderTransform('XYZ', 'text/html')]; 91 | supertest(this.server) 92 | .get('/proxy') 93 | .expect(200) 94 | .expect('Content-Type', /^text\/html/) 95 | .expect(function(res) { 96 | assert.equal(res.text, '1234XYZ'); 97 | }) 98 | .end(done); 99 | }); 100 | 101 | function appenderTransform(appendText, contentType) { 102 | return { 103 | name: 'appender', 104 | contentType: contentType, 105 | transform: function() { 106 | return through2(function(chunk, enc, cb) { 107 | this.push(chunk + appendText); 108 | cb(); 109 | }); 110 | } 111 | }; 112 | } 113 | }); 114 | -------------------------------------------------------------------------------- /lib/request-options.js: -------------------------------------------------------------------------------- 1 | var parseUrl = require('url').parse; 2 | var formatUrl = require('url').format; 3 | var _ = require('lodash'); 4 | var querystring = require('querystring'); 5 | var pathToRegexp = require('path-to-regexp'); 6 | 7 | var BLOCK_HEADERS = ['host', 'cookie']; 8 | var CACHE_HEADERS = ['if-none-match', 'if-modified-since']; 9 | // var PASSTHROUGH_HEADERS = ['authorization', 'accepts', 'if-none-match', 'if-modified-since']; 10 | 11 | // Returns an options object that can be fed to the request module. 12 | module.exports = function(req, options, limits) { 13 | var requestOptions = _.pick(options, 14 | 'method', 'timeout', 'maxRedirects', 'proxy', 'followRedirect'); 15 | 16 | // If an explicit method was not specified on the options, then use the 17 | // method of the inbound request to the proxy. 18 | if (!requestOptions.method) { 19 | requestOptions.method = req.method; 20 | } 21 | 22 | // Ensure that passed in options for timeout and maxRedirects cannot exceed 23 | // the platform imposed limits (if defined). 24 | if (_.isObject(limits) === true) { 25 | if (_.isNumber(limits.timeout)) { 26 | if (_.isNumber(options.timeout) === false || options.timeout > limits.timeout) { 27 | requestOptions.timeout = limits.timeout; 28 | } 29 | } 30 | if (_.isNumber(limits.maxRedirects)) { 31 | if (_.isNumber(options.maxRedirects) === false || 32 | options.maxRedirects > limits.maxRedirects) { 33 | requestOptions.maxRedirects = limits.maxRedirects; 34 | } 35 | } 36 | } 37 | 38 | // Extend the incoming query with any additional parameters specified in the options 39 | if (_.isObject(options.query)) { 40 | _.extend(req.query, options.query); 41 | } 42 | 43 | var parsedUrl = parseUrl(options.url); 44 | 45 | // Compile the path expression of the originUrl 46 | var compiledPath = pathToRegexp.compile(parsedUrl.path); 47 | 48 | // Need to decode the path as splat params like 'path/*' will result in an encoded forward slash 49 | // like http://someapi.com/v1/path1%2Fpath2. 50 | var pathname = decodeURIComponent(compiledPath(_.extend({}, req.params, options.params || {}))); 51 | 52 | // Substitute the actual values using both those from the incoming 53 | // params as well as those configured in the options. Values in the 54 | // options take precedence. 55 | 56 | // If options.originalQuery is true, ignore the above and just 57 | // use the original raw querystring as the search 58 | 59 | requestOptions.url = formatUrl(_.extend({ 60 | protocol: parsedUrl.protocol, 61 | host: parsedUrl.host, 62 | pathname: pathname 63 | }, options.originalQuery ? 64 | {search: req.url.replace(/^.+\?/, '')} : 65 | {query: _.extend({}, querystring.parse(parsedUrl.query), req.query, options.query)} 66 | )); 67 | 68 | requestOptions.headers = {}; 69 | 70 | // Passthrough headers 71 | _.each(req.headers, function(value, key) { 72 | if (shouldPassthroughHeader(key)) { 73 | requestOptions.headers[key] = value; 74 | } 75 | }); 76 | 77 | // Forward the IP of the originating request. This is de-facto proxy behavior. 78 | if (req.ip) { 79 | requestOptions.headers['x-forwarded-for'] = req.ip; 80 | } 81 | 82 | if (req.headers && req.headers.host) { 83 | var hostSplit = req.headers.host.split(':'); 84 | var host = hostSplit[0]; 85 | var port = hostSplit[1]; 86 | 87 | if (port) { 88 | requestOptions.headers['x-forwarded-port'] = port; 89 | } 90 | 91 | requestOptions.headers['x-forwarded-host'] = host; 92 | } 93 | 94 | requestOptions.headers['x-forwarded-proto'] = req.secure ? 'https' : 'http'; 95 | 96 | // Default to accepting gzip encoding 97 | if (!requestOptions.headers['accept-encoding']) { 98 | requestOptions.headers['accept-encoding'] = 'gzip'; 99 | } 100 | 101 | // Inject additional headers from the options 102 | if (_.isObject(options.headers)) { 103 | _.extend(requestOptions.headers, options.headers); 104 | } 105 | 106 | // Override the user-agent 107 | if (options.userAgent) { 108 | requestOptions.headers['user-agent'] = options.userAgent; 109 | } 110 | 111 | return requestOptions; 112 | 113 | function shouldPassthroughHeader(header) { 114 | if (_.includes(BLOCK_HEADERS, header) === true) return false; 115 | if (options.cache && _.includes(CACHE_HEADERS, header) === true) return false; 116 | 117 | return true; 118 | } 119 | }; 120 | -------------------------------------------------------------------------------- /test/cache.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var supertest = require('supertest'); 3 | var _ = require('lodash'); 4 | var async = require('async'); 5 | var memoryCache = require('memory-cache-stream'); 6 | // var redis = require('redis'); 7 | var debug = require('debug')('express-request-proxy'); 8 | var proxy = require('..'); 9 | var setup = require('./setup'); 10 | var shortid = require('shortid'); 11 | 12 | require('dash-assert'); 13 | 14 | describe('proxy cache', function() { 15 | var self; 16 | 17 | beforeEach(setup.beforeEach); 18 | afterEach(setup.afterEach); 19 | 20 | beforeEach(function() { 21 | self = this; 22 | this.cache = memoryCache(); 23 | 24 | _.extend(this.proxyOptions, { 25 | cache: this.cache, 26 | cacheMaxAge: 100 27 | }); 28 | 29 | this.server.get('/proxy', function(req, res, next) { 30 | proxy(self.proxyOptions)(req, res, next); 31 | }); 32 | 33 | this.server.use(setup.errorHandler); 34 | }); 35 | 36 | it('writes api response to cache', function(done) { 37 | supertest(this.server).get('/proxy') 38 | .expect(200) 39 | .expect('Content-Type', /application\/json/) 40 | .expect('Cache-Control', 'max-age=' + this.proxyOptions.cacheMaxAge) 41 | .expect('Express-Request-Proxy-Cache', 'miss') 42 | .end(function(err, res) { 43 | if (err) return done(err); 44 | 45 | var cacheKey = res.body.fullUrl; 46 | async.each([cacheKey, cacheKey + '__headers'], function(key, cb) { 47 | self.cache.exists(cacheKey + '__headers', function(_err, exists) { 48 | if (_err) return cb(_err); 49 | assert.ok(exists); 50 | cb(); 51 | }); 52 | }, done); 53 | }); 54 | }); 55 | 56 | it('cache key includes the querystring args', function(done) { 57 | this.proxyOptions.query = { 58 | arg1: 'a', 59 | arg2: 'b' 60 | }; 61 | 62 | supertest(this.server).get('/proxy') 63 | .expect(200) 64 | .expect('Content-Type', /application\/json/) 65 | .expect('Cache-Control', 'max-age=' + this.proxyOptions.cacheMaxAge) 66 | .end(function(err, res) { 67 | if (err) return done(err); 68 | 69 | assert.isTrue(self.cache.exists(self.proxyOptions.url + '?arg1=a&arg2=b')); 70 | done(); 71 | }); 72 | }); 73 | 74 | it('reads api response from cache', function(done) { 75 | this.apiResponse = {name: 'foo'}; 76 | 77 | this.cache.setex(this.baseApiUrl, 78 | this.proxyOptions.cacheMaxAge, 79 | JSON.stringify(this.apiResponse)); 80 | 81 | this.cache.setex(this.baseApiUrl + '__headers', 82 | this.proxyOptions.cacheMaxAge, 83 | JSON.stringify({'content-type': 'application/json'})); 84 | 85 | supertest(this.server) 86 | .get('/proxy') 87 | .set('Accept', 'application/json') 88 | .expect(200) 89 | .expect('Content-Type', /^application\/json/) 90 | .expect('Cache-Control', /^max-age/) 91 | .expect('Express-Request-Proxy-Cache', 'hit') 92 | .expect(function(res) { 93 | assert.deepEqual(res.body, self.apiResponse); 94 | }) 95 | .end(done); 96 | }); 97 | 98 | it('bypasses cache for non-GET requests', function(done) { 99 | supertest(this.server).put('/proxy') 100 | .expect(200) 101 | .end(function(err, res) { 102 | self.cache.exists(self.baseApiUrl, function(_err, exists) { 103 | if (_err) return done(_err); 104 | assert.equal(exists, 0); 105 | done(); 106 | }); 107 | }); 108 | }); 109 | 110 | it('overrides Cache-Control from the origin API', function(done) { 111 | this.apiResponse = {id: shortid.generate()}; 112 | this.originHeaders = { 113 | 'Cache-Control': 'max-age=20' 114 | }; 115 | this.proxyOptions.cacheMaxAge = 200; 116 | 117 | supertest(this.server).get('/proxy') 118 | .expect(200) 119 | .expect('Cache-Control', 'max-age=' + self.proxyOptions.cacheMaxAge) 120 | .expect(function(res) { 121 | assert.deepEqual(res.body, self.apiResponse); 122 | }) 123 | .end(done); 124 | }); 125 | 126 | 127 | it('removes cache related headers', function(done) { 128 | self.apiResponse = {id: shortid.generate()}; 129 | 130 | this.originHeaders = { 131 | 'Last-Modified': new Date().toUTCString(), 132 | Expires: new Date().toUTCString(), 133 | Etag: '345345345345' 134 | }; 135 | 136 | this.proxyOptions.cacheMaxAge = 200; 137 | supertest(this.server).get('/proxy') 138 | .expect(200) 139 | .expect('Content-Type', /application\/json/) 140 | .expect(function(res) { 141 | assert.deepEqual(res.body, self.apiResponse); 142 | assert.ok(_.isUndefined(res.headers['last-modified'])); 143 | assert.ok(_.isUndefined(res.headers.expires)); 144 | assert.ok(_.isUndefined(res.headers.etag)); 145 | }) 146 | .end(done); 147 | }); 148 | 149 | it('original content-type preserved when request comes from cache', function(done) { 150 | this.apiResponse = shortid.generate(); 151 | this.originHeaders = { 152 | 'Content-Type': 'some/custom-type' 153 | }; 154 | 155 | supertest(this.server).get('/proxy') 156 | .expect(200) 157 | .expect('Express-Request-Proxy-Cache', 'miss') 158 | .expect('Content-Type', /^some\/custom-type/) 159 | .expect(self.apiResponse) 160 | .end(function(err, res) { 161 | supertest(self.server).get('/proxy') 162 | .expect('Express-Request-Proxy-Cache', 'hit') 163 | .expect(200) 164 | .expect(self.apiResponse) 165 | .expect('Content-Type', /^some\/custom-type/) 166 | .end(done); 167 | }); 168 | }); 169 | 170 | 171 | it('does not cache non-200 responses from remote API', function(done) { 172 | this.apiResponse = {message: 'not found'}; 173 | this.apiResponseStatus = 404; 174 | 175 | supertest(this.server) 176 | .get('/proxy') 177 | .expect(404) 178 | .expect('Express-Request-Proxy-Cache', 'miss') 179 | .end(function(res) { 180 | self.cache.exists(self.baseApiUrl, function(err, exists) { 181 | assert.ok(!exists); 182 | done(); 183 | }); 184 | }); 185 | }); 186 | 187 | it('does not send conditional get headers to origin', function(done) { 188 | this.originHeaders = { 189 | ETag: '2435345345', 190 | 'If-Modified-Since': (new Date()).toUTCString() 191 | }; 192 | 193 | supertest(this.server) 194 | .get('/proxy') 195 | .expect(function(res) { 196 | assert.isUndefined(res.body.headers.etag); 197 | assert.isUndefined(res.body.headers['if-modified-since']); 198 | }) 199 | .end(done); 200 | }); 201 | 202 | it('retains gzip encoding when served from cache', function(done) { 203 | this.apiGzipped = true; 204 | this.apiResponse = {foo: 1, name: 'elmer'}; 205 | 206 | async.series([ 207 | function(cb) { 208 | supertest(self.server).get('/proxy') 209 | .set('Accept-Encoding', 'gzip') 210 | .expect('Express-Request-Proxy-Cache', 'miss') 211 | .expect(function(res) { 212 | assert.deepEqual(res.body, self.apiResponse); 213 | }) 214 | .end(cb); 215 | }, 216 | function(cb) { 217 | // Check what's in the cache 218 | self.cache.get(self.baseApiUrl + '__headers', function(err, value) { 219 | assert.deepEqual({ 220 | 'content-type': 'application/json; charset=utf-8' 221 | }, JSON.parse(value)); 222 | cb(); 223 | }); 224 | }, 225 | function(cb) { 226 | supertest(self.server).get('/proxy') 227 | .set('Accept-Encoding', 'gzip') 228 | .expect('Express-Request-Proxy-Cache', 'hit') 229 | .expect(function(res) { 230 | assert.deepEqual(res.body, self.apiResponse); 231 | }) 232 | .end(cb); 233 | } 234 | ], done); 235 | }); 236 | }); 237 | -------------------------------------------------------------------------------- /test/request-options.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var sinon = require('sinon'); 3 | var parseUrl = require('url').parse; 4 | var formatUrl = require('url').format; 5 | var requestOptions = require('../lib/request-options'); 6 | 7 | require('dash-assert'); 8 | 9 | describe('requestOptions', function() { 10 | it('substitutes and extends query params', function() { 11 | var req = { 12 | method: 'get', 13 | query: { 14 | id: 5, 15 | apikey: 'should_be_overridden' 16 | } 17 | }; 18 | 19 | var endpointOptions = { 20 | url: 'http://someapi.com/foo', 21 | query: { 22 | apikey: '123' 23 | } 24 | }; 25 | 26 | var opts = requestOptions(req, endpointOptions); 27 | assert.ok(sinon.match(opts, { 28 | url: parseUrl(endpointOptions.url), 29 | qs: {apikey: '123', id: 5}, 30 | method: 'get' 31 | })); 32 | }); 33 | 34 | it('substitutes in path', function() { 35 | var req = { 36 | method: 'get', 37 | params: { 38 | resource: 'foo', 39 | id: '123', 40 | apikey: 'should_be_overridden' 41 | } 42 | }; 43 | 44 | var endpointOptions = { 45 | url: 'http://someapi.com/:apikey/:resource/:id', 46 | params: { 47 | apikey: 'secret' 48 | } 49 | }; 50 | 51 | var opts = requestOptions(req, endpointOptions); 52 | assert.equal(formatUrl(opts.url), 'http://someapi.com/secret/foo/123'); 53 | }); 54 | 55 | it('substitutes optional parameters', function(done) { 56 | var req = { 57 | method: 'get', 58 | params: { 59 | resource: 'foo', 60 | id: '123', 61 | apikey: 'should_be_overridden' 62 | } 63 | }; 64 | 65 | var endpointOptions = { 66 | url: 'http://someapi.com/:apikey/:resource/:id?', 67 | params: { 68 | apikey: 'secret' 69 | } 70 | }; 71 | 72 | var opts = requestOptions(req, endpointOptions); 73 | assert.equal(formatUrl(opts.url), 'http://someapi.com/secret/foo/123'); 74 | 75 | // Now clear out the id and make sure it is not passed 76 | req.params.resource = 'bar'; 77 | req.params.id = null; 78 | opts = requestOptions(req, endpointOptions); 79 | assert.equal(formatUrl(opts.url), 'http://someapi.com/secret/bar'); 80 | 81 | done(); 82 | }); 83 | 84 | it('builds origin URL with wildcard parameter', function(done) { 85 | var req = { 86 | method: 'get', 87 | params: { 88 | 0: 'path1/path2', 89 | version: 'v1' 90 | } 91 | }; 92 | 93 | var endpointOptions = { 94 | url: 'http://someapi.com/:version/*' 95 | }; 96 | 97 | var opts = requestOptions(req, endpointOptions); 98 | assert.deepEqual(opts.url, 'http://someapi.com/v1/path1/path2'); 99 | 100 | done(); 101 | }); 102 | 103 | it('wildcard route with no params', function(done) { 104 | var req = { 105 | method: 'post', 106 | params: { 107 | 0: 'v1/token' 108 | } 109 | }; 110 | 111 | var endpointOptions = { 112 | url: 'http://domain.com/api/auth/*' 113 | }; 114 | 115 | var opts = requestOptions(req, endpointOptions); 116 | assert.deepEqual(opts.url, 'http://domain.com/api/auth/v1/token'); 117 | 118 | done(); 119 | }); 120 | 121 | it('appends headers', function() { 122 | var req = { 123 | method: 'get', 124 | headers: { 125 | header1: 'a', 126 | header2: '2' 127 | } 128 | }; 129 | 130 | var endpointOptions = { 131 | url: 'http://someapi.com', 132 | headers: { 133 | header1: '1' 134 | } 135 | }; 136 | 137 | var opts = requestOptions(req, endpointOptions); 138 | assert.equal(opts.headers.header1, '1'); 139 | assert.equal(opts.headers.header2, '2'); 140 | }); 141 | 142 | it('does not passthrough blocked headers', function() { 143 | var req = { 144 | method: 'get', 145 | headers: { 146 | cookie: 'should_not_passthrough', 147 | 'if-none-match': '345345', 148 | header1: '1' 149 | } 150 | }; 151 | 152 | var endpointOptions = { 153 | url: 'http://someapi.com' 154 | }; 155 | 156 | var opts = requestOptions(req, endpointOptions); 157 | assert.isUndefined(opts.headers.cookie); 158 | }); 159 | 160 | it('does not passthrough certain headers when response to be cached', function() { 161 | var req = { 162 | method: 'get', 163 | headers: { 164 | cookie: 'should_not_passthrough', 165 | 'if-none-match': 'should_not_passthrough', 166 | 'if-modified-since': 'should_not_passthrough', 167 | header1: '1' 168 | } 169 | }; 170 | 171 | var endpointOptions = { 172 | url: 'http://someapi.com', 173 | cache: {} 174 | }; 175 | 176 | var opts = requestOptions(req, endpointOptions); 177 | assert.isUndefined(opts.headers['if-none-match']); 178 | assert.isUndefined(opts.headers['if-modified-since']); 179 | assert.equal(opts.headers.header1, '1'); 180 | }); 181 | 182 | it('default headers appended', function() { 183 | var req = { 184 | method: 'get', 185 | ip: '127.0.0.1', 186 | secure: true 187 | }; 188 | 189 | var endpointOptions = { 190 | url: 'http://someapi.com', 191 | cache: {} 192 | }; 193 | 194 | var opts = requestOptions(req, endpointOptions); 195 | assert.equal(opts.headers['x-forwarded-for'], req.ip); 196 | assert.equal(opts.headers['accept-encoding'], 'gzip'); 197 | assert.equal(opts.headers['x-forwarded-proto'], 'https'); 198 | 199 | req.secure = false; 200 | opts = requestOptions(req, endpointOptions); 201 | 202 | assert.equal(opts.headers['x-forwarded-proto'], 'http'); 203 | }); 204 | 205 | it('default headers appended host and port', function() { 206 | var req = { 207 | headers: { 208 | host: 'localhost:8080' 209 | } 210 | }; 211 | 212 | var endpointOptions = { 213 | url: 'http://someapi.com', 214 | cache: {} 215 | }; 216 | 217 | var opts = requestOptions(req, endpointOptions); 218 | assert.equal(opts.headers['x-forwarded-host'], 'localhost'); 219 | assert.equal(opts.headers['x-forwarded-port'], '8080'); 220 | }); 221 | 222 | it('cannot exceed limit options', function() { 223 | var req = { 224 | method: 'get', 225 | headers: { 226 | cookie: 'should_not_passthrough', 227 | 'if-none-match': 'should_not_passthrough', 228 | 'if-modified-since': 'should_not_passthrough', 229 | header1: '1' 230 | } 231 | }; 232 | 233 | var endpointOptions = { 234 | url: 'http://someapi.com', 235 | timeout: 30, 236 | maxRedirects: 5 237 | }; 238 | 239 | var limits = { 240 | timeout: 5, 241 | maxRedirects: 3 242 | }; 243 | 244 | var opts = requestOptions(req, endpointOptions, limits); 245 | assert.equal(opts.timeout, limits.timeout); 246 | assert.equal(opts.maxRedirects, limits.maxRedirects); 247 | }); 248 | 249 | it('uses method from the req', function() { 250 | var req = { 251 | method: 'post' 252 | }; 253 | 254 | var endpointOptions = { 255 | url: 'http://someapi.com' 256 | }; 257 | 258 | var opts = requestOptions(req, endpointOptions); 259 | assert.equal(opts.method, req.method); 260 | }); 261 | 262 | it('uses method from the options', function() { 263 | var req = { 264 | method: 'post' 265 | }; 266 | 267 | var endpointOptions = { 268 | url: 'http://someapi.com', 269 | method: 'put' 270 | }; 271 | 272 | var opts = requestOptions(req, endpointOptions); 273 | assert.equal(opts.method, endpointOptions.method); 274 | }); 275 | 276 | it('allows followRedirect from the options', function() { 277 | var req = { 278 | followRedirect: undefined, 279 | method: 'get' 280 | }; 281 | 282 | var endpointOptions = { 283 | url: 'http://someapi.com', 284 | followRedirect: true 285 | }; 286 | 287 | var opts = requestOptions(req, endpointOptions); 288 | assert.equal(opts.followRedirect, endpointOptions.followRedirect); 289 | 290 | endpointOptions.followRedirect = false; 291 | opts = requestOptions(req, endpointOptions); 292 | assert.equal(opts.followRedirect, endpointOptions.followRedirect); 293 | }); 294 | }); 295 | -------------------------------------------------------------------------------- /lib/proxy.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'), 2 | async = require('async'), 3 | debug = require('debug')('express-request-proxy:proxy'), 4 | // through2 = require('through2'), 5 | zlib = require('zlib'), 6 | request = require('request'), 7 | requestOptions = require('./request-options'), 8 | is = require('type-is'); 9 | 10 | require('simple-errors'); 11 | 12 | 13 | // Headers from the original API response that should be preserved and sent along 14 | // in cached responses. 15 | var headersToPreserveInCache = ['content-type']; 16 | 17 | /* eslint-disable consistent-return */ 18 | module.exports = function(options) { 19 | 20 | // Allow discarding response headers to be configurable 21 | var discardApiResponseHeaders = options.discardApiResponseHeaders || ['set-cookie', 'content-length']; 22 | 23 | options = _.defaults(options || {}, { 24 | ensureAuthenticated: false, 25 | cache: null, 26 | cacheMaxAge: 0, 27 | userAgent: 'express-request-proxy', 28 | cacheHttpHeader: 'Express-Request-Proxy-Cache', 29 | cacheKeyFn: null, 30 | timeout: 5000, 31 | maxRedirects: 5, 32 | gzip: true, 33 | originalQuery: false 34 | }); 35 | 36 | return function(req, res, next) { 37 | var method = req.method.toUpperCase(); 38 | 39 | // Allow for a global cache to be specified on the parent Express app 40 | if (!options.cache) { 41 | options.cache = req.app.settings.cache; 42 | } 43 | 44 | if (!req.ext) { 45 | req.ext = {}; 46 | } 47 | 48 | req.ext.requestHandler = 'express-request-proxy'; 49 | 50 | if (options.ensureAuthenticated === true) { 51 | // Look for the isAuthenticated function which PassportJS defines. Some other authentication 52 | // method can be used, but it needs to define the isAuthenicated function. 53 | if (req.ext.isAuthenticated !== true) { 54 | debug('user is not authenticated'); 55 | return next(Error.http(401, 'User must be authenticated to invoke this API endpoint')); 56 | } 57 | } 58 | 59 | if (method.toUpperCase() === 'GET' && options.cache && options.cacheMaxAge > 0) { 60 | if (!options.cache) return next(new Error('No cache provider configured')); 61 | 62 | return proxyViaCache(req, res, next); 63 | } 64 | 65 | makeApiCall(req, res, next); 66 | }; 67 | 68 | function makeApiCall(req, res, next) { 69 | var apiRequestOptions; 70 | try { 71 | apiRequestOptions = requestOptions(req, options); 72 | } catch (err) { 73 | debug('error building request options %s', err.stack); 74 | return next(Error.http(400, err.message)); 75 | } 76 | 77 | debug('making %s call to %s', apiRequestOptions.method, apiRequestOptions.url); 78 | 79 | // If the req has a body, pipe it into the proxy request 80 | var apiRequest; 81 | var originalApiRequest; 82 | if (is.hasBody(req)) { 83 | debug('piping req body to remote http endpoint'); 84 | originalApiRequest = request(apiRequestOptions); 85 | apiRequest = req.pipe(originalApiRequest); 86 | } else { 87 | apiRequest = request(apiRequestOptions); 88 | originalApiRequest = apiRequest; 89 | } 90 | 91 | // Abort api request if client close its connection 92 | req.on('close', function() { 93 | originalApiRequest.abort(); 94 | }); 95 | 96 | apiRequest.on('error', function(err) { 97 | unhandledApiError(err, next); 98 | }); 99 | 100 | // Defer piping the response until the response event so we can 101 | // check the status code. 102 | apiRequest.on('response', function(resp) { 103 | // Do not attempt to apply transforms to error responses 104 | if (resp.statusCode >= 400) { 105 | debug('Received error %s from %s', resp.statusCode, apiRequestOptions.url); 106 | return apiRequest.pipe(res); 107 | } 108 | 109 | if (_.isArray(options.transforms)) { 110 | apiRequest = applyTransforms(apiRequest, options.transforms, resp.headers); 111 | } 112 | 113 | // Need to explicitly passthrough headers, otherwise they will get lost 114 | // in the transforms pipe. 115 | for (var key in resp.headers) { 116 | if (_.includes(discardApiResponseHeaders, key) === false) { 117 | res.set(key, resp.headers[key]); 118 | } 119 | } 120 | 121 | apiRequest.pipe(res); 122 | }); 123 | } 124 | 125 | function proxyViaCache(req, res, next) { 126 | var apiRequestOptions; 127 | try { 128 | apiRequestOptions = requestOptions(req, options); 129 | } catch (err) { 130 | debug('error building request options %s', err.stack); 131 | return next(Error.http(400, err.message)); 132 | } 133 | 134 | var cacheKey; 135 | if (_.isFunction(options.cacheKeyFn)) { 136 | cacheKey = options.cacheKeyFn(req, apiRequestOptions); 137 | } else { 138 | cacheKey = apiRequestOptions.url; 139 | } 140 | 141 | // Try retrieving from the cache 142 | debug('checking if key %s exists in cache', cacheKey); 143 | options.cache.exists(cacheKey, function(err, exists) { 144 | if (err) return next(err); 145 | 146 | debug('api response exists in cache=%s', exists); 147 | if (exists) { 148 | debug('api response exists in cache'); 149 | return pipeToResponseFromCache(cacheKey, req, res, next); 150 | } 151 | 152 | debug('key %s not in cache', cacheKey); 153 | 154 | res.set('Cache-Control', 'max-age=' + options.cacheMaxAge); 155 | res.set(options.cacheHttpHeader, 'miss'); 156 | 157 | debug('making %s request to %s', apiRequestOptions.method, apiRequestOptions.url); 158 | 159 | var apiRequest = request(apiRequestOptions); 160 | apiRequest.on('error', function(_err) { 161 | debug('error making api call'); 162 | unhandledApiError(_err, next); 163 | }); 164 | 165 | // Defer piping the response until the response event so we can 166 | // check the status code. 167 | apiRequest.on('response', function(resp) { 168 | // Don't cache error responses. Just pipe the response right on out. 169 | if (resp.statusCode !== 200) { 170 | return apiRequest.pipe(res); 171 | } 172 | 173 | // If the api response is gzip encoded, unzip it before attempting 174 | // any transforms. 175 | if (resp.headers['content-encoding'] === 'gzip') { 176 | apiRequest = apiRequest.pipe(zlib.createGunzip()); 177 | } 178 | 179 | // Store the headers as a separate cache entry 180 | var headersToKeep = _.pick(resp.headers, headersToPreserveInCache); 181 | 182 | if (_.isArray(options.transforms)) { 183 | apiRequest = applyTransforms(apiRequest, options.transforms, headersToKeep); 184 | } 185 | 186 | // This needs to happen after the call to applyTransforms so transforms 187 | // have the opportunity to modify the contentType. 188 | _.forOwn(headersToKeep, function(value, key) { 189 | debug('setting header %s to %s', key, value); 190 | res.set(key, value); 191 | }); 192 | 193 | // Store the headers as a separate cache entry 194 | if (_.isEmpty(headersToKeep) === false) { 195 | debug('writing original headers to cache'); 196 | options.cache.setex(cacheKey + '__headers', 197 | options.cacheMaxAge, 198 | JSON.stringify(headersToKeep)); 199 | } 200 | 201 | debug('cache api response for %s seconds', options.cacheMaxAge); 202 | 203 | apiRequest.pipe(options.cache.writeThrough(cacheKey, options.cacheMaxAge)) 204 | .pipe(res); 205 | }); 206 | }); 207 | } 208 | 209 | function setHeadersFromCache(cacheKey, res, callback) { 210 | // get the ttl of the main object and fetch the headers in parallel. 211 | async.parallel({ 212 | ttl: function(cb) { 213 | options.cache.ttl(cacheKey, cb); 214 | }, 215 | headers: function(cb) { 216 | options.cache.get(cacheKey + '__headers', function(err, value) { 217 | // restore the headers 218 | var headers = {}; 219 | if (value) { 220 | try { 221 | headers = JSON.parse(value); 222 | } catch (jsonError) { 223 | debug('can\'t parse headers as json'); 224 | } 225 | } 226 | cb(null, headers); 227 | }); 228 | } 229 | }, function(err, results) { 230 | if (err) return callback(err); 231 | 232 | _.forOwn(results.headers, function(value, key) { 233 | res.set(key, value); 234 | }); 235 | 236 | // Set a custom header indicating that the response was served from cache. 237 | res.set(options.cacheHttpHeader, 'hit'); 238 | 239 | debug('setting max-age to remaining TTL of %s', results.ttl); 240 | res.set('Cache-Control', 'max-age=' + results.ttl); 241 | callback(); 242 | }); 243 | } 244 | 245 | function pipeToResponseFromCache(cacheKey, req, res, next) { 246 | debug('getting TTL of cached api response'); 247 | 248 | setHeadersFromCache(cacheKey, res, function(err) { 249 | if (err) return next(err); 250 | 251 | if (_.isFunction(options.cache.readStream)) { 252 | options.cache.readStream(cacheKey).pipe(res); 253 | return; 254 | } 255 | options.cache.get(cacheKey, function(_err, data) { 256 | if (_err) return next(_err); 257 | res.end(data); 258 | }); 259 | }); 260 | } 261 | 262 | function unhandledApiError(err, next) { 263 | debug('unhandled API error: %s', err.code); 264 | if (err.code === 'ETIMEDOUT' || err.code === 'ESOCKETTIMEDOUT') { 265 | return next(Error.http(504, 'API call timed out')); 266 | } 267 | return next(err); 268 | } 269 | 270 | // function apiErrorResponse(statusCode, apiRequest, next) { 271 | // var error = ''; 272 | // apiRequest.pipe(through2(function(chunk, enc, cb) { 273 | // error += chunk; 274 | // cb(); 275 | // }, function() { 276 | // return next(Error.http(statusCode, error)); 277 | // })); 278 | // } 279 | 280 | function applyTransforms(stream, transforms, headers) { 281 | // Pipe the stream through each transform in sequence 282 | transforms.forEach(function(transform) { 283 | debug('applying transform %s', transform.name); 284 | if (transform.contentType) { 285 | headers['Content-Type'] = transform.contentType; 286 | } 287 | 288 | stream = stream.pipe(transform.transform()); 289 | }); 290 | 291 | return stream; 292 | } 293 | }; 294 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # express-request-proxy 2 | 3 | [![NPM Version][npm-image]][npm-url] 4 | [![NPM Downloads][downloads-image]][downloads-url] 5 | [![Build Status][travis-image]][travis-url] 6 | [![Test Coverage][coveralls-image]][coveralls-url] 7 | 8 | High performance streaming http request reverse proxy for [Express](http://expressjs.com) based on the [request http client](https://www.npmjs.com/package/request). Supports caching, custom routes, server-side injection of sensitive keys, authentication, and response transformations. 9 | 10 | ## Usage 11 | 12 | #### Server Code 13 | 14 | ```js 15 | var redis = require("redis"); 16 | var requestProxy = require("express-request-proxy"); 17 | 18 | require("redis-streams")(redis); 19 | 20 | app.get( 21 | "/api/:resource/:id", 22 | requestProxy({ 23 | cache: redis.createClient(), 24 | cacheMaxAge: 60, 25 | url: "https://someapi.com/api/:resource/:id", 26 | query: { 27 | secret_key: process.env.SOMEAPI_SECRET_KEY 28 | }, 29 | headers: { 30 | "X-Custom-Header": process.env.SOMEAPI_CUSTOM_HEADER 31 | } 32 | }) 33 | ); 34 | ``` 35 | 36 | #### Client Code 37 | 38 | ```js 39 | $.ajax({ 40 | url: "/api/widgets/" + widgetId, 41 | statusCode: { 42 | 200: function(data) { 43 | console.log(data); 44 | }, 45 | 404: function() { 46 | console.log("cannot find widget"); 47 | } 48 | } 49 | }); 50 | ``` 51 | 52 | ### Options 53 | 54 | **`url`** 55 | 56 | String representing the url of the remote endpoint to proxy to. Can contain named route parameters. Query parameters can be appended with the `query` option. This is the only required option. 57 | 58 | **`params`** 59 | 60 | An object containing properties to substitute for named route parameters in the remote URL. Named parameters are declared using the same conventions as Express routes, i.e. `/pathname/:param1/:param2`. The path portion of the `url` is parsed using the [path-to-regexp](https://www.npmjs.com/package/path-to-regexp) module. Defaults to `{}`. 61 | 62 | **`query`** 63 | 64 | An object of parameters to be appended to the querystring of the specified `url`. This is a good way to append sensitive parameters that are stored as environment variables on the server avoiding the need to expose them to the client. Any parameters specified here will override an identically named parameter in the `url` or on the incoming request to the proxy. Defaults to `{}`. 65 | 66 | **`headers`** 67 | 68 | An object of HTTP headers to be appended to the request to the remote url. This is also a good way to inject sensitive keys stored in environment variables. See the [http basic auth](#http-basic-auth) section for example usage. 69 | 70 | **`cache`** 71 | 72 | Cache object that conforms to the [node_redis](https://www.npmjs.com/package/redis) API. Can set to `false` at an endpoint level to explicitly disable caching for certain APIs. See [Cache section](#caching) below for more details. 73 | 74 | **`cacheMaxAge`** 75 | 76 | The duration to cache the API response. If an API response is returned from the cache, a `max-age` http header will be included with the remaining TTL. 77 | 78 | **`ensureAuthenticated`** 79 | 80 | Ensure that there is a valid logged in user in order to invoke the proxy. See the [Ensure Authenticated](#ensure-authenticated) section below for details. 81 | 82 | **`userAgent`** 83 | 84 | The user agent string passed as a header in the http call to the remote API. Defaults to `"express-api-proxy"`. 85 | 86 | **`cacheHttpHeader`** 87 | 88 | Name of the http header returned in the proxy response with the value `"hit"` or `"miss"`. Defaults to `"Express-Request-Proxy-Cache"`. 89 | 90 | **`timeout`** 91 | 92 | The number of milliseconds to wait for a server to send response headers before aborting the request. See [request/request](https://github.com/request/request#user-content-requestoptions-callback) for more information. Default value `5000`. 93 | 94 | #### HTTP Basic Auth 95 | 96 | For http endpoints protected by [HTTP Basic authentication](http://en.wikipedia.org/wiki/Basic_access_authentication#Client_side), a username and password should be sent in the form `username:password` which is then base64 encoded. 97 | 98 | ```js 99 | var usernamePassword = 100 | process.env.SOMEAPI_USERNAME + ":" + process.env.SOMEAPI_PASSSWORD; 101 | 102 | app.post( 103 | "/api/:resource", 104 | requestProxy({ 105 | cache: redis.createClient(), 106 | cacheMaxAge: 60, 107 | url: "https://someapi.com/api/:resource", 108 | headers: { 109 | Authorization: "Basic " + new Buffer(usernamePassword).toString("base64") 110 | } 111 | }) 112 | ); 113 | ``` 114 | 115 | #### Logged-In User Properties 116 | 117 | Sometimes it's necessary to pass attributes of the current logged in user (on the server) into the request to the remote endpoint as headers, query params, etc. Rather than passing environment variables, simply specify the desired user properties. 118 | 119 | ```js 120 | app.all("/api/protected/:resource", function(req, res, next) { 121 | var proxy = requestProxy({ 122 | url: "http://remoteapi.com/api", 123 | query: { 124 | access_token: req.user.accessToken 125 | } 126 | }); 127 | proxy(req, res, next); 128 | }); 129 | ``` 130 | 131 | This assumes that prior middleware has set the `req.user` property, which was perhaps stored in [session state](https://www.npmjs.com/package/express-session). 132 | 133 | ### Caching 134 | 135 | For remote endpoints whose responses do not change frequently, it is often desirable to cache responses at the proxy level. This avoids repeated network round-trip latency and can skirt rate limits imposed by the API provider. 136 | 137 | The object provided to the `cache` option is expected to implement a subset of the [node_redis](https://github.com/mranney/node_redis) interface, specifically the [get](http://redis.io/commands/get), [set](http://redis.io/commands/set), [setex](http://redis.io/commands/setex), [exists](http://redis.io/commands/exists), [del](http://redis.io/commands/del), and [ttl](http://redis.io/commands/ttl) commands. The node_redis package can be used directly, other cache stores require a wrapper module that adapts to the redis interface. 138 | 139 | As an optimization, two additional functions, `readStream` and `writeThrough` must be implemented on the cache object to allow direct piping of the API responses into and out of the cache. This avoids buffering the entire API response in memory. For node_redis, the [redis-streams](https://www.npmjs.com/package/redis-streams) package augments the `RedisClient` with these two functions. Simply add the following line to your module before the proxy middleware is executed: 140 | 141 | ```js 142 | var redis = require("redis"); 143 | 144 | require("redis-streams")(redis); 145 | // After line above, calls to redis.createClient will return enhanced 146 | // object with readStream and writeThrough functions. 147 | 148 | app.get( 149 | "/proxy/:route", 150 | requestProxy({ 151 | cache: redis.createClient(), 152 | cacheMaxAge: 300, // cache responses for 5 minutes 153 | url: "https://someapi.com/:route" 154 | }) 155 | ); 156 | ``` 157 | 158 | Only `GET` requests are subject to caching, for all other methods the `cacheMaxAge` is ignored. 159 | 160 | #### Caching Headers 161 | 162 | If an API response is served from the cache, the `max-age` header will be set to the remaining TTL of the cached object. The proxy cache trumps any HTTP headers such as `Last-Modified`, `Expires`, or `ETag`, so these get discarded. Effectively the proxy takes over the caching behavior from the origin for the duration that it exists there. 163 | 164 | ### Ensure Authenticated 165 | 166 | It's possible restrict proxy calls to authenticated users via the `ensureAuthenticated` option. 167 | 168 | ```js 169 | app.all( 170 | "/proxy/protected", 171 | requestProxy({ 172 | url: "https://someapi.com/sensitive", 173 | ensureAuthenticated: true 174 | }) 175 | ); 176 | ``` 177 | 178 | The proxy does not perform authentication itself, that task is delegated to other middleware that executes earlier in the request pipeline which sets the property `req.ext.isAuthenticated`. If `ensureAuthenticated` is `true` and `req.ext.isAuthenticated !== true`, a 401 (Unauthorized) HTTP response is returned before ever executing the remote request. 179 | 180 | Note that this is different than authentication that might be enforced by the remote API itself. That's handled by injecting headers or query params as discussed above. 181 | 182 | ### Wildcard routes 183 | 184 | Sometimes you want to configure one catch-all proxy route that will forward on all path segments starting from the `*`. The example below will proxy a request to `GET /api/widgets/12345` to `GET https://remoteapi.com/api/v1/widgets/12345` and `POST /api/users` to `POST https://remoteapi.com/api/v1/users`. 185 | 186 | ```js 187 | app.all( 188 | "/api/*", 189 | requestProxy({ 190 | url: "https://remoteapi.com/api/v1/*", 191 | query: { 192 | apikey: "xxx" 193 | } 194 | }) 195 | ); 196 | ``` 197 | 198 | ### Transforms 199 | 200 | The proxy supports transforming the API response before piping it back to the caller. Transforms are functions which return a Node.js [transform stream](http://nodejs.org/api/stream.html#stream_class_stream_transform). The [through2](https://github.com/rvagg/through2) package provides a lightweight wrapper that makes transforms easier to implement. 201 | 202 | Here's a trivial transform function that simply appends some text 203 | 204 | ```js 205 | module.exports = function(options) { 206 | return { 207 | name: "appender", 208 | transform: function() { 209 | return through2( 210 | function(chunk, enc, cb) { 211 | this.push(chunk); 212 | cb(); 213 | }, 214 | function(cb) { 215 | this.push(options.appendText); 216 | cb(); 217 | } 218 | ); 219 | } 220 | }; 221 | }; 222 | ``` 223 | 224 | If the transform needs to change the `Content-Type` of the response, a `contentType` property can be declared on the transform function that the proxy will recognize and set the header accordingly. 225 | 226 | ```js 227 | module.exports = function() { 228 | return { 229 | name: 'appender', 230 | contentType: 'application/json', 231 | transform: function() { 232 | return through2(...) 233 | } 234 | }; 235 | } 236 | ``` 237 | 238 | See the [markdown-transform](https://github.com/4front/markdown-transform) for a real world example. For transforming HTML responses, the [trumpet package](https://www.npmjs.com/package/trumpet), with it's streaming capabilities, is a natural fit. 239 | 240 | Here's how you could request a GitHub README.md as html: 241 | 242 | ```js 243 | var markdownTransform = require("markdown-transform"); 244 | 245 | app.get( 246 | "/:repoOwner/:repoName/readme", 247 | requestProxy({ 248 | url: 249 | "https://raw.githubusercontent.com/:repoOwner/:repoName/master/README.md", 250 | transforms: [markdownTransform({ highlight: true })] 251 | }) 252 | ); 253 | ``` 254 | 255 | ## License 256 | 257 | Licensed under the [Apache License, Version 2.0](http://www.apache.org/licenses/LICENSE-2.0). 258 | 259 | [npm-image]: https://img.shields.io/npm/v/express-request-proxy.svg?style=flat 260 | [npm-url]: https://npmjs.org/package/express-request-proxy 261 | [travis-image]: https://img.shields.io/travis/dvonlehman/express-request-proxy.svg?style=flat 262 | [travis-url]: https://travis-ci.org/dvonlehman/express-request-proxy 263 | [coveralls-image]: https://img.shields.io/coveralls/dvonlehman/express-request-proxy.svg?style=flat 264 | [coveralls-url]: https://coveralls.io/r/dvonlehman/express-request-proxy?branch=master 265 | [downloads-image]: https://img.shields.io/npm/dm/express-request-proxy.svg?style=flat 266 | [downloads-url]: https://npmjs.org/package/express-request-proxy 267 | --------------------------------------------------------------------------------