├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── example ├── config.js ├── index.js ├── package.json ├── server.js └── updater.js ├── index.js ├── package.json ├── server.js ├── updater.js └── utils.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | example/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Anson Yu Wang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # simple-npm-cache-proxy 2 | a simple npm proxy cache package locally 3 | -------------------------------------------------------------------------------- /example/config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var path = require('path'); 3 | module.exports = { 4 | redis: 'redis://127.0.0.1:6379/5', 5 | updateInterval: 120000, 6 | tarballCacheDir: path.join(__dirname, 'cache'), 7 | upstreams: [ 8 | { 9 | name: 'private', 10 | urlExp: /^\/(-\/user\/|enjoy-).*$/i, 11 | proxyTo: 'http://localhost:4873', 12 | replace: [/http:\/\/localhost:4873/g, 'http://localhost:7000'], 13 | cache: false, 14 | }, { 15 | name: 'official', 16 | urlExp: /.*/, 17 | proxyTo: 'https://registry.npmjs.org', 18 | replace: [/https:\/\/registry\.npmjs\.org/g, 'http://localhost:7000'], 19 | cache: true, 20 | } 21 | ], 22 | stCacheOptions: { 23 | // refer to https://www.npmjs.com/package/st 24 | fd: { 25 | max: 1000, 26 | maxAge: 1000 * 60 * 60, 27 | }, 28 | stat: false, 29 | content: { 30 | max: 1024 * 1024 * 64, 31 | maxAge: 1000 * 60 * 10, 32 | cacheControl: 'public, max-age=600' 33 | }, 34 | }, 35 | }; 36 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var config = require('./config'); 3 | var Server = require('simple-npm-cache-proxy/server'); 4 | var server = Server(config); 5 | var Updater = require('simple-npm-cache-proxy/updater'); 6 | var updater = Updater(config); 7 | server.listen(7000); 8 | updater.run(); 9 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "example", 3 | "version": "1.0.0", 4 | "description": "simple-npm-cache-proxy example server", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "pm2 start -i 0 index.js --name simple-npm-cache-proxy", 8 | "stop": "pm2 stop simple-npm-cache-proxy", 9 | "reload": "pm2 reload simple-npm-cache-proxy", 10 | "test": "echo \"Error: no test specified\" && exit 1" 11 | }, 12 | "author": "undoZen ", 13 | "license": "ISC", 14 | "dependencies": { 15 | "simple-npm-cache-proxy": "^1.0.0" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /example/server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var config = require('./config'); 3 | var Server = require('simple-npm-cache-proxy/server'); 4 | var server = Server(config); 5 | server.listen(7000); 6 | -------------------------------------------------------------------------------- /example/updater.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | var config = require('./config'); 3 | var Updater = require('simple-npm-cache-proxy/updater'); 4 | var updater = Updater(config); 5 | updater.run(); 6 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | exports.Server = require('./server'); 3 | exports.Updater = require('./updater'); 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "simple-npm-cache-proxy", 3 | "version": "1.1.0", 4 | "description": "a simple npm proxy cache package locally", 5 | "main": "server.js", 6 | "scripts": { 7 | "test": "prove --exec iojs test/*.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/undoZen/simple-npm-cache-proxy.git" 12 | }, 13 | "keywords": [ 14 | "npm" 15 | ], 16 | "author": "undoZen ", 17 | "license": "MIT", 18 | "bugs": { 19 | "url": "https://github.com/undoZen/simple-npm-cache-proxy/issues" 20 | }, 21 | "homepage": "https://github.com/undoZen/simple-npm-cache-proxy", 22 | "dependencies": { 23 | "bluebird": "^2.10.2", 24 | "bunyan-hub": "^1.1.0", 25 | "bunyan-hub-logger": "^1.2.1", 26 | "co": "^4.5.1", 27 | "concat-stream": "^1.4.8", 28 | "config": "^1.12.0", 29 | "hiredis": "^0.4.1", 30 | "ioredis": "^1.15.1", 31 | "lodash": "^3.10.1", 32 | "mkdirp": "^0.5.0", 33 | "promisingagent": "^5.1.1", 34 | "rimraf": "^2.5.2", 35 | "simple-http-proxy": "^0.5.11", 36 | "st": "^1.1.0" 37 | }, 38 | "devDependencies": { 39 | "rimraf": "^2.3.2", 40 | "tape": "^4.0.0" 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | global.Promise = require('bluebird'); 3 | var Redis = require('ioredis'); 4 | var logger = require('bunyan-hub-logger'); 5 | logger.replaceDebug('simple-npm-cache-proxy'); 6 | var http = require('http'); 7 | http.globalAgent.maxSockets = Infinity; 8 | var url = require('url'); 9 | var path = require('path'); 10 | var fs = require('fs'); 11 | var _ = require('lodash'); 12 | var co = require('co'); 13 | //var superagent = require('superagent'); 14 | var mkdirp = require('mkdirp'); 15 | var concat = require('concat-stream'); 16 | var shp = require('simple-http-proxy'); 17 | var st = require('st'); 18 | var utils = require('./utils'); 19 | var xtend = utils.xtend; 20 | var pickHeaders = utils.pickHeaders; 21 | var replaceBodyRegistry = utils.replaceBodyRegistry; 22 | var matchUpstream = utils.matchUpstream; 23 | var defaultUpstream = utils.defaultUpstream; 24 | var randomInterval = utils.randomInterval; 25 | var log = logger({ 26 | app: 'simple-npm-cache-proxy', 27 | name: 'server', 28 | serializers: xtend(logger.stdSerializers, { 29 | response: logger.stdSerializers.res 30 | }), 31 | }); 32 | var rimraf = Promise.promisify(require('rimraf')); 33 | 34 | module.exports = function (config) { 35 | 36 | var db = new Redis(config.redis || void 0); 37 | var cachedRequest = {}; 38 | var mount = st(xtend({ 39 | cache: utils.defaultStCacheOptions, 40 | }, { 41 | cache: config.stCacheOptions || {}, 42 | }, { 43 | path: config.tarballCacheDir, 44 | index: false, 45 | passthrough: true, 46 | })); 47 | var interval = randomInterval(config); 48 | 49 | var proxy = co.wrap(function * (upstream, req, res) { 50 | var crkey = upstream.name + '|' + req.url; 51 | if (upstream.cache && req.method === 'GET' && !req.url.match(/^\/-\//)) { 52 | var cachedJson = yield db.get('cache||' + req.url); 53 | if (cachedJson) { 54 | var cache = JSON.parse(cachedJson); 55 | log.debug({ 56 | req: req, 57 | cachedObject: cache 58 | }); 59 | if ('if-none-match' in req.headers && req.headers['if-none-match'] === cache.etag) { 60 | cache.headers['content-length'] = 0; 61 | return { 62 | statusCode: 304, 63 | headers: cache.headers, 64 | body: new Buffer(0), 65 | }; 66 | } 67 | return cache; 68 | } 69 | var cr; 70 | if ((cr = cachedRequest[crkey])) { 71 | return cr; 72 | } 73 | } 74 | if ('if-none-match' in req.headers) { // make sure upstream return 200 instead of 304 75 | delete req.headers['if-none-match']; 76 | } 77 | if ('accept-encoding' in req.headers) { // remove gzip, get raw text body 78 | req.headers['accept-encoding'] = 'identity'; 79 | } 80 | var registryUrl, registryHost; 81 | if (Array.isArray(upstream.proxyTo)) { 82 | registryUrl = upstream.proxyTo[0]; 83 | registryHost = upstream.proxyTo[1]; 84 | } else { 85 | registryUrl = upstream.proxyTo; 86 | registryHost = url.parse(upstream.proxyTo).host; 87 | } 88 | req.headers.host = registryHost; 89 | 90 | var rp = new Promise(function(resolve, reject) { 91 | var request, response; 92 | var concatStream = concat(function(body) { 93 | if (!response && request) response = request.res; 94 | var headers = response.headers; 95 | // cache tarball 96 | if (upstream.cache && req.method === 'GET' && !headers.location && 97 | headers['content-type'] === 'application/octet-stream') { 98 | var filePath = path.resolve(config.tarballCacheDir, req.url.replace(/^\/+/, '')); 99 | mkdirp(path.dirname(filePath), function(err) { 100 | if (err) log.error(err); 101 | fs.writeFile(filePath, body, function(err) { 102 | if (err) log.error(err); 103 | }); 104 | }); 105 | } 106 | // cache json 107 | else if ((headers['content-type'] || '').match(/^application\/json/i)) { 108 | body = body.toString('utf8'); 109 | log.debug({ 110 | req: req, 111 | body: body, 112 | }); 113 | body = replaceBodyRegistry(upstream.replace, body); 114 | log.debug({ 115 | req: req, 116 | headers: headers, 117 | replacedBody: body, 118 | }); 119 | headers = xtend(headers, { 120 | 'content-length': Buffer.byteLength(body) 121 | }); 122 | if (upstream.cache && req.method === 'GET' && !req.url.match(/^\/-\//) && typeof headers['etag'] === 'string') { 123 | var b; 124 | try { 125 | var b = JSON.stringify(JSON.parse(body)); 126 | } catch (e) {} 127 | if (b) { 128 | headers['content-length'] = Buffer.byteLength(b); 129 | } 130 | db.set('cache||' + req.url, JSON.stringify({ 131 | statusCode: response.statusCode, 132 | headers: headers, 133 | body: body, 134 | etag: headers.etag, 135 | })); 136 | db.zadd('schedule', Date.now() + interval(), req.url); 137 | } 138 | } 139 | resolve({ 140 | statusCode: response.statusCode, 141 | headers: pickHeaders(headers), 142 | body: body, 143 | }); 144 | }); 145 | //if (['GET', 'HEAD'].indexOf(req.method) === -1) { 146 | shp(registryUrl, { 147 | timeout: 300000, 148 | onrequest: function(options, req) { 149 | options.headers.host = registryHost; 150 | }, 151 | onresponse: function(_response, res) { 152 | response = _response; 153 | response.pipe(concatStream); 154 | return true; 155 | }, 156 | })(req, res, reject); 157 | /* 158 | } else { 159 | request = superagent(req.method, registryUrl + req.url) 160 | .redirects(1) 161 | .set(req.headers); 162 | request.pipe(concatStream); 163 | request._callback = function(err) { 164 | if (err) return reject(err); 165 | }; 166 | } 167 | */ 168 | }); 169 | if (req.method === 'GET') { 170 | cachedRequest[crkey] = rp; 171 | rp.finally(function() { 172 | delete cachedRequest[crkey]; 173 | }); 174 | } 175 | return rp; 176 | }); 177 | 178 | function routeProxy(req, res) { 179 | var upstream = matchUpstream(config.upstreams, req.url); 180 | return proxy(upstream, req, res); 181 | } 182 | var server = http.createServer(); 183 | server.on('request', function(req, res) { 184 | req.url = req.url.replace(/\/+/g, '/').replace(/\?.*$/g, ''); 185 | res.status = function(code) { 186 | res.statusCode = code; 187 | }; 188 | req.res = res; 189 | res.req = req; 190 | log.debug({ 191 | req: req, 192 | res: res 193 | }); 194 | if (req.url.match(/\/__flush__\//)) { 195 | var flushUrl = req.url.substring('/__flush__'.length); 196 | return co(function * () { 197 | if (/\.tgz$/i.test(flushUrl)) { 198 | yield rimraf(path.join(config.tarballCacheDir, flushUrl)); 199 | } else { 200 | yield db.del('cache||' + flushUrl); 201 | } 202 | res.end('done\n'); 203 | }).catch(resError); 204 | } 205 | ((['GET', 'HEAD'].indexOf(req.method) === -1) ? 206 | routeProxy(req, res) : 207 | new Promise(function(resolve, reject) { 208 | req.on('close', resolve); 209 | req.on('finish', resolve); 210 | mount(req, res, function() { 211 | resolve(routeProxy(req, res)); 212 | }); 213 | }) 214 | ).then(function(cache) { 215 | res.writeHeader(cache.statusCode, cache.headers); 216 | res.end(cache.body); 217 | }).catch(resError); 218 | 219 | function resError(err) { 220 | if (err) log.error(err); 221 | if (err && !res.headersSent) { 222 | res.status(500); 223 | log.error(err); 224 | res.end('500 internal error.'); 225 | } 226 | } 227 | }); 228 | 229 | return server; 230 | }; 231 | -------------------------------------------------------------------------------- /updater.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | global.Promise = require('bluebird'); 3 | var Redis = require('ioredis'); 4 | var logger = require('bunyan-hub-logger'); 5 | logger.replaceDebug('simple-npm-cache-proxy'); 6 | var http = require('http'); 7 | http.globalAgent.maxSockets = Infinity; 8 | var urlParse = require('url').parse; 9 | var path = require('path'); 10 | var fs = require('fs'); 11 | var _ = require('lodash'); 12 | var co = require('co'); 13 | var request = require('promisingagent'); 14 | var mkdirp = require('mkdirp'); 15 | var concat = require('concat-stream'); 16 | var shp = require('simple-http-proxy'); 17 | var utils = require('./utils'); 18 | var matchUpstream = utils.matchUpstream; 19 | var replaceBodyRegistry = utils.replaceBodyRegistry; 20 | var xtend = utils.xtend; 21 | var sleep = utils.sleep; 22 | var pickHeaders = utils.pickHeaders; 23 | var log = logger({ 24 | app: 'simple-npm-cache-proxy', 25 | name: 'update', 26 | serializers: xtend(logger.stdSerializers, { 27 | response: logger.stdSerializers.res 28 | }), 29 | }); 30 | 31 | var up = co.wrap(function * (ctx) { 32 | var urls = yield ctx.db.zrangebyscore('schedule', 0, Date.now()); 33 | if (!urls.length) { 34 | return; 35 | } 36 | var url; 37 | while ((url = urls.shift())) { 38 | yield update(url, ctx); 39 | } 40 | }); 41 | 42 | module.exports = Updater; 43 | function Updater(config) { 44 | var db = new Redis(config.redis || void 0); 45 | var ctx = { 46 | run: run, 47 | config: config, 48 | db: db, 49 | interval: utils.randomInterval(config), 50 | }; 51 | return ctx; 52 | function run() { 53 | up(ctx).catch(log.error.bind(log)).then(setTimeout.bind(null, run, 100)); 54 | } 55 | } 56 | 57 | function update(url, ctx) { 58 | var db = ctx.db; 59 | var config = ctx.config; 60 | var interval = ctx.interval; 61 | return co(function * () { 62 | var upstream = matchUpstream(config.upstreams, url); 63 | db.zadd('schedule', Date.now() + interval(), url); 64 | var cachedJson = yield db.get('cache||' + url); 65 | var etag; 66 | if (cachedJson) { 67 | var cache = JSON.parse(cachedJson); 68 | etag = cache.etag; 69 | } 70 | log.trace({ 71 | job: 'update', 72 | url: url, 73 | }); 74 | var registryUrl, registryHost; 75 | if (Array.isArray(upstream.proxyTo)) { 76 | registryUrl = upstream.proxyTo[0]; 77 | registryHost = upstream.proxyTo[1]; 78 | } else { 79 | registryUrl = upstream.proxyTo; 80 | registryHost = urlParse(upstream.proxyTo).host; 81 | } 82 | var r = yield request.get(registryUrl + url, { 83 | headers: xtend({ 84 | host: registryHost, 85 | }, etag ? {'if-none-match': etag} : {}), 86 | }); 87 | log.trace({ 88 | job: 'update', 89 | url: url, 90 | etag: etag, 91 | response: r, 92 | rtext: r.text, 93 | }); 94 | if (r.statusCode !== 200 || !r.headers.etag) { 95 | return; 96 | } 97 | var headers = r.headers; 98 | if (r.text && headers.etag && headers.etag !== etag) { 99 | var body = replaceBodyRegistry(upstream.replace, r.text); 100 | var b; 101 | try { 102 | b = JSON.stringify(JSON.parse(body)); 103 | } catch (e) {} 104 | if (b) { 105 | var cacheObject = { 106 | statusCode: r.statusCode, 107 | headers: xtend(pickHeaders(r.headers), { 108 | 'content-length': Buffer.byteLength(b) 109 | }), 110 | etag: r.headers.etag, 111 | body: body, 112 | }; 113 | log.info({ 114 | updated: true, 115 | cacheObject: cacheObject, 116 | }); 117 | yield db.set('cache||' + url, JSON.stringify(cacheObject)); 118 | } 119 | } 120 | }).catch(log.error.bind(log)); 121 | } 122 | -------------------------------------------------------------------------------- /utils.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | global.Promise = require('bluebird'); 3 | var _ = require('lodash'); 4 | 5 | var defaultStCacheOptions = { 6 | fd: { 7 | max: 1000, 8 | maxAge: 1000 * 60 * 60, 9 | }, 10 | stat: false, 11 | content: { 12 | max: 1024 * 1024 * 64, 13 | maxAge: 1000 * 60 * 10, 14 | cacheControl: 'public, max-age=600' 15 | }, 16 | }; 17 | exports.defaultStCacheOptions = defaultStCacheOptions; 18 | 19 | exports.matchUpstream = matchUpstream; 20 | function matchUpstream(upstreams, url) { 21 | return upstreams.filter(upstream => { 22 | return upstream && 23 | upstream.urlExp instanceof RegExp && 24 | upstream.urlExp.exec(url); 25 | })[0]; 26 | } 27 | 28 | exports.replaceBodyRegistry = replaceBodyRegistry; 29 | function replaceBodyRegistry(replace, body) { 30 | return body.replace(replace[0], replace[1]); 31 | } 32 | 33 | exports.xtend = xtend; 34 | function xtend() { 35 | var args = Array.prototype.slice.call(arguments); 36 | args.unshift({}); 37 | return _.assign.apply(_, args); 38 | } 39 | 40 | exports.sleep = sleep; 41 | function sleep(ms) { 42 | return new Promise(function(resolve) { 43 | setTimeout(resolve, ms); 44 | }); 45 | } 46 | 47 | exports.pickHeaders = pickHeaders; 48 | function pickHeaders(headers) { 49 | return _.pick(headers, [ 50 | 'etag', 51 | 'content-type', 52 | 'content-length', 53 | ]); 54 | } 55 | 56 | exports.randomInterval = randomInterval; 57 | function randomInterval(config) { 58 | var interval = config.updateInterval || 2000; 59 | return function () { 60 | return interval * (Math.random() * 0.4 + 0.8) 61 | }; 62 | } 63 | --------------------------------------------------------------------------------