├── .gitignore ├── .jshintrc ├── .travis.yml ├── CHANGELOG.md ├── LICENSE.txt ├── README.md ├── TODO.md ├── bin └── npm-lazy-mirror ├── config └── config-defaults.json ├── example ├── package-blacklist.json └── server-config.json ├── lib ├── cache.js ├── config.js ├── constants.js ├── handlers.js ├── package.js └── registry.js ├── package.json ├── server.js └── test ├── helpers.js ├── npm.js └── runner.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.o 8 | *.so 9 | 10 | # Packages # 11 | ############ 12 | # it's better to unpack these files and commit the raw source 13 | # git has its own built in compression methods 14 | *.7z 15 | *.dmg 16 | *.gz 17 | *.iso 18 | *.jar 19 | *.rar 20 | *.tar 21 | *.zip 22 | 23 | # Logs and databases # 24 | ###################### 25 | *.log 26 | *.sql 27 | *.sqlite 28 | 29 | # OS generated files # 30 | ###################### 31 | .DS_Store 32 | .DS_Store? 33 | ._* 34 | .Spotlight-V100 35 | .Trashes 36 | Icon? 37 | ehthumbs.db 38 | Thumbs.db 39 | 40 | # node.js related 41 | node_modules 42 | *.seed 43 | *.log 44 | *.csv 45 | *.dat 46 | *.out 47 | *.pid 48 | *.gz 49 | 50 | pids 51 | logs 52 | results 53 | 54 | npm-debug.log 55 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | // JSHint Default Configuration File (as on JSHint website) 3 | // See http://jshint.com/docs/ for more details 4 | 5 | "maxerr" : 50, // {int} Maximum error before stopping 6 | 7 | // Enforcing 8 | "bitwise" : true, // true: Prohibit bitwise operators (&, |, ^, etc.) 9 | "camelcase" : true, // true: Identifiers must be in camelCase 10 | "curly" : true, // true: Require {} for every new block or scope 11 | "eqeqeq" : true, // true: Require triple equals (===) for comparison 12 | "forin" : true, // true: Require filtering for..in loops with obj.hasOwnProperty() 13 | "immed" : true, // true: Require immediate invocations to be wrapped in parens e.g. `(function () { } ());` 14 | "indent" : 4, // {int} Number of spaces to use for indentation 15 | "latedef" : true, // true: Require variables/functions to be defined before being used 16 | "newcap" : true, // true: Require capitalization of all constructor functions e.g. `new F()` 17 | "noarg" : true, // true: Prohibit use of `arguments.caller` and `arguments.callee` 18 | "noempty" : true, // true: Prohibit use of empty blocks 19 | "nonew" : false, // true: Prohibit use of constructors for side-effects (without assignment) 20 | "plusplus" : false, // true: Prohibit use of `++` & `--` 21 | "quotmark" : "single", // Quotation mark consistency: 22 | // false : do nothing (default) 23 | // true : ensure whatever is used is consistent 24 | // "single" : require single quotes 25 | // "double" : require double quotes 26 | "undef" : true, // true: Require all non-global variables to be declared (prevents global leaks) 27 | "unused" : true, // true: Require all defined variables be used 28 | "strict" : true, // true: Requires all functions run in ES5 Strict Mode 29 | "trailing" : true, // true: Prohibit trailing whitespaces 30 | "maxparams" : false, // {int} Max number of formal params allowed per function 31 | "maxdepth" : false, // {int} Max depth of nested blocks (within functions) 32 | "maxstatements" : false, // {int} Max number statements per function 33 | "maxcomplexity" : false, // {int} Max cyclomatic complexity per function 34 | "maxlen" : false, // {int} Max number of characters per line 35 | 36 | // Relaxing 37 | "asi" : false, // true: Tolerate Automatic Semicolon Insertion (no semicolons) 38 | "boss" : false, // true: Tolerate assignments where comparisons would be expected 39 | "debug" : false, // true: Allow debugger statements e.g. browser breakpoints. 40 | "eqnull" : false, // true: Tolerate use of `== null` 41 | "esnext" : true, // true: Allow ES.next (ES6) syntax (ex: `const`) 42 | "moz" : false, // true: Allow Mozilla specific syntax (extends and overrides esnext features) 43 | // (ex: `for each`, multiple try/catch, function expression…) 44 | "evil" : false, // true: Tolerate use of `eval` and `new Function()` 45 | "expr" : false, // true: Tolerate `ExpressionStatement` as Programs 46 | "funcscope" : false, // true: Tolerate defining variables inside control statements" 47 | "globalstrict" : true, // true: Allow global "use strict" (also enables 'strict') 48 | "iterator" : false, // true: Tolerate using the `__iterator__` property 49 | "lastsemic" : false, // true: Tolerate omitting a semicolon for the last statement of a 1-line block 50 | "laxbreak" : false, // true: Tolerate possibly unsafe line breakings 51 | "laxcomma" : false, // true: Tolerate comma-first style coding 52 | "loopfunc" : false, // true: Tolerate functions being defined in loops 53 | "multistr" : false, // true: Tolerate multi-line strings 54 | "proto" : false, // true: Tolerate using the `__proto__` property 55 | "scripturl" : false, // true: Tolerate script-targeted URLs 56 | "smarttabs" : false, // true: Tolerate mixed tabs/spaces when used for alignment 57 | "shadow" : false, // true: Allows re-define variables later in code e.g. `var x=1; x=2;` 58 | "sub" : false, // true: Tolerate using `[]` notation when it can still be expressed in dot notation 59 | "supernew" : false, // true: Tolerate `new function () { ... };` and `new Object;` 60 | "validthis" : false, // true: Tolerate using this in a non-constructor function 61 | 62 | // Environments 63 | "browser" : false, // Web Browser (window, document, etc) 64 | "couch" : false, // CouchDB 65 | "devel" : true, // Development/debugging (alert, confirm, etc) 66 | "dojo" : false, // Dojo Toolkit 67 | "jquery" : false, // jQuery 68 | "mootools" : false, // MooTools 69 | "node" : true, // Node.js 70 | "nonstandard" : false, // Widely adopted globals (escape, unescape, etc) 71 | "prototypejs" : false, // Prototype and Scriptaculous 72 | "rhino" : false, // Rhino 73 | "worker" : false, // Web Workers 74 | "wsh" : false, // Windows Scripting Host 75 | "yui" : false, // Yahoo User Interface 76 | 77 | // Legacy 78 | "nomen" : false, // true: Prohibit dangling `_` in variables 79 | "onevar" : false, // true: Allow only one `var` statement per function 80 | "passfail" : false, // true: Stop on first error 81 | "white" : false, // true: Check against strict whitespace and indentation rules 82 | 83 | // Custom Globals 84 | "globals" : {} // additional predefined global variables 85 | } -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.11" 4 | - "0.10" 5 | 6 | before_script: 7 | - node server.js & 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.3.2 4 | - Fixed a typo with stale resource config (#13) 5 | - Make HTTPS the default upstream registry transport (#12) 6 | 7 | ## 0.3.1 8 | - Fixed issue with config.json file not being found and issue where upstreamPort 9 | wasn't being honored ( #10 ) 10 | - Fixes an issue with Lactate cache expiry options leading to excessive CPU 11 | usage ( #11 ) 12 | 13 | ## 0.3.0 14 | - Large cleanup and refactor for new feature implementations. 15 | - Now have the ability to serve stale resources from the cache while they cannot 16 | be refreshed from the upstream registry. (`--permit_stale_resources`) 17 | - Support a list of blacklisted packages using semver ranges via a JSON 18 | configuration file. See an example configuration in the `examples/` folder. 19 | Enabled with `--package_blacklist ` 20 | - Added example server configuration in `examples/` 21 | - Fix a bug where non 200 upstream registry responses were written to disk. 22 | - Other minor bug fixes. 23 | 24 | ## 0.2.3 25 | - Fix packages with multiple versions not being routed to the appropriate 26 | handler. 27 | 28 | ## 0.2.2 29 | - Support version parsing when the package identifies itself with multiple 30 | semvers. 31 | 32 | ## 0.2.1 33 | 34 | - Ensure dots are parsed in the package name via HTTP routes. 35 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2013 ActiveState 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #npm-lazy-mirror 2 | --- 3 | 4 | A lazy mirroring local npm server. 5 | 6 | [![Build Status](https://travis-ci.org/ActiveState/npm-lazy-mirror.svg?branch=master)](https://travis-ci.org/ActiveState/npm-lazy-mirror) 7 | 8 | ## About 9 | 10 | This package provides a lazy mirroring option for those that: 11 | 12 | * Don't want to mirror the entire couchDB for npmjs.org 13 | * Don't want to setup cron jobs for those tasks 14 | * Want to use an in-memory caching server (allocation configurable) 15 | * Want to easily cache dist tarballs and package metadata to disk 16 | 17 | ## Install 18 | 19 | * `npm install -g npm-lazy-mirror` 20 | 21 | ## Run 22 | 23 | With CLI flags: 24 | 25 | * `npm-lazy-mirror -p -a -b --cache_dir /npm-data` 26 | 27 | With a JSON configuration: 28 | 29 | * `npm-lazy-mirror -C /path/to/config,json` 30 | 31 | See `example/server-config.json` for usage. 32 | 33 | **Note:** Your `remote_address` configuration is important, as it is the address used when re-writing 34 | tarball URLs in the metadata. It's certainly always best to use a DNS entry here, 35 | rather than an IP. 36 | 37 | ## Client configuration 38 | 39 | Simply point your local npm config to the lazy mirror (permanent): 40 | 41 | npm config set registry http://localhost:2000/ 42 | 43 | or per install: 44 | 45 | npm i --registry http://localhost:2000 supertest 46 | 47 | ## Help 48 | 49 | Run `npm-lazy-mirror -h` to see a full list of options. 50 | 51 | ## Features 52 | 53 | * Caches all tarball / JSON metadata to disk 54 | * Mirror serves files (200MB max by default) from memory, with a configurable LRU cache. 55 | * Ability to blacklist packages by semantic versioning specification 56 | * Option to serve stale resources while the upstream registry is offline 57 | * Upstream resources are fetched on the fly from the remote registry, the fetching, storing and serving to the client all happen in the same request. 58 | * Configurable with custom npm registries. 59 | * HTTP/S proxy support 60 | * It's Fast and stands up under load. Expect 5000+ req/s with one core. 61 | 62 | A cold run installing `express` takes ~12 seconds (fetching from upstream registry on-the-fly): 63 | 64 | npm install express 2.44s user 0.81s system 27% cpu 11.769 total 65 | 66 | A warm run after all `express` assets are locally cached takes ~3 seconds: 67 | 68 | npm install express 2.43s user 0.78s system 115% cpu 2.768 total 69 | 70 | ## Implementation Caveats 71 | 72 | You cannot use this mirror for publishing modules or user management, such 73 | requests will be forwarded to the upstream registry for processing. 74 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | ## Todo 2 | 3 | * Indexing for `/all/-/` requests 4 | * Automatic disk pruning of stale resources 5 | * Resource locking to prevent multiple requests to the upstream registry 6 | * Worker processes 7 | * Separate process / request log streams 8 | * Force pre-fetching pre-defined assets on startup 9 | 10 | -------------------------------------------------------------------------------- /bin/npm-lazy-mirror: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # 4 | # Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 5 | # 6 | 7 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 8 | node $DIR/../lib/node_modules/npm-lazy-mirror/server.js $@ 9 | 10 | -------------------------------------------------------------------------------- /config/config-defaults.json: -------------------------------------------------------------------------------- 1 | { 2 | "http_port": 2000, 3 | "server_address": "localhost", 4 | 5 | "upstream_host": "registry.npmjs.org", 6 | "upstream_port": 443, 7 | "upstream_use_https": true, 8 | "upstream_verify_ssl": true, 9 | 10 | "cache_dir": "/tmp/npm-lazy", 11 | "cache_mem": 200 12 | 13 | } 14 | -------------------------------------------------------------------------------- /example/package-blacklist.json: -------------------------------------------------------------------------------- 1 | { 2 | "cake": "0.1.1", 3 | "express": "*", 4 | "mocha": "1.0.x - 1.17.0", 5 | "coffee-script": "> 1.0.0" 6 | } 7 | -------------------------------------------------------------------------------- /example/server-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "http_port": 2000, 3 | "https_port": 2443, 4 | "http_enabled": true, 5 | "https_enabled": false, 6 | "https_key": "", 7 | "https_cert": "", 8 | 9 | "http_proxy": null, 10 | "https_proxy": null, 11 | 12 | "real_external_port": 2000, 13 | "server_address": "localhost", 14 | "bind_address": "127.0.0.1", 15 | 16 | "log_level": "info", 17 | 18 | "package_blacklist": "../example/package-blacklist", 19 | "permit_stale_resources": true, 20 | 21 | "upstream_host": "registry.npmjs.org", 22 | "upstream_port": "443", 23 | "upstream_use_https": true, 24 | "upstream_verify_ssl": true, 25 | "upstream_db_path": "/", 26 | 27 | "cache_expiry": 320000, 28 | "cache_dir": "/tmp/npm-lazy", 29 | "cache_mem": 512 30 | } 31 | -------------------------------------------------------------------------------- /lib/cache.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Async = require('async'); 8 | var Constants = require('./constants'); 9 | var Fs = require('fs'); 10 | var Mkdirp = require('mkdirp'); 11 | var Path = require('path'); 12 | var Url = require('url'); 13 | 14 | /** 15 | * Simple cache get/set operations for disk ops 16 | * 17 | * @constructor 18 | * @param {Object} opts - the main server config object 19 | * @returns {Cache} 20 | */ 21 | var Cache = function(opts) { 22 | 23 | if (!this instanceof Cache) { 24 | return new Cache(opts); 25 | } 26 | 27 | if (!opts) { return new Error('Options object not supplied'); } 28 | if (!opts.cacheDir) { return new Error('Cache directory not supplied'); } 29 | 30 | this.cacheDir = opts.cacheDir; 31 | this.config = opts; 32 | this.fsStats = opts.cache.fsStats; 33 | this.metaFolder = 'meta'; 34 | this.tgzFolder = 'tgz'; 35 | 36 | return this; 37 | }; 38 | 39 | /** 40 | * Checks the mtime of the file has not exceeded the configured maximum expiry 41 | * 42 | * @param {string} path - file path 43 | * @callback cb - (err, bool, fs.Stats) 44 | */ 45 | Cache.prototype.diskExpired = function(path, cb) { 46 | 47 | var cache = this; 48 | 49 | Fs.stat(path, function(err, stats) { 50 | 51 | if (err) { return cb (err); } 52 | var now = Date.now(); 53 | var mtime = stats.mtime.getTime(); 54 | 55 | if ( (now - mtime) >= cache.config.cacheExpiry ) { 56 | return cb(null, true, stats); 57 | } 58 | 59 | return cb(null, false, stats); 60 | }); 61 | }; 62 | 63 | /** 64 | * Checks if the requested resource exists on the disk. This is an expensive I/O 65 | * op so should be checked with AsyncCache first. 66 | * 67 | * @param {string} path - file path 68 | * @callback cb - (err, bool) 69 | */ 70 | Cache.prototype.existsDisk = function(path, cb) { 71 | 72 | Fs.exists(path, function(exists) { 73 | cb(null, exists); 74 | }); 75 | }; 76 | 77 | /** 78 | * Checks if the requested resource is valid 79 | * 80 | * @param {Object} opts - type, name, version 81 | * @callback cb - (err, bool) 82 | */ 83 | Cache.prototype.validate = function(opts, cb) { 84 | 85 | if (!opts.type) { return cb(new Error('Type not supplied')); } 86 | if (!opts.name) { return cb(new Error('Name not supplied')); } 87 | if (!opts.version) { opts.version = ''; } 88 | 89 | var cache = this, 90 | path; 91 | if (opts.type === Constants.TYPE_META) { 92 | path = cache.getMetaPath(opts.name); 93 | } else if (opts.type === Constants.TYPE_TGZ) { 94 | path = cache.getTarballPath(opts.name, opts.version); 95 | } else { 96 | return cb(new Error('Unknown type: ' + opts.type)); 97 | } 98 | cache.fsStats.get(path, cb); 99 | }; 100 | 101 | /** 102 | * Overrides all the dist/tarball keys in the package metadata to point 103 | * to this mirror. 104 | * 105 | * @param {Object} pkg_json - The package metadata 106 | * @callback cb - (err, new_json) 107 | */ 108 | Cache.prototype.overrrideTarballUrls = function(pkgJson, cb) { 109 | 110 | var cache = this; 111 | 112 | // If the server is running with HTTPS enabled, default to that protocol 113 | var protocol = cache.config.httpsEnabled ? 'https:' : 'http'; 114 | 115 | var changeDistUrl = function(ver, done) { 116 | ver = pkgJson.versions[ver]; 117 | if (ver.dist && ver.dist.tarball) { 118 | var distUrl = Url.parse(ver.dist.tarball); 119 | distUrl.host = null; 120 | distUrl.hostname = cache.config.serverAddress; 121 | distUrl.port = cache.config.realExternalPort; 122 | distUrl.protocol = protocol; 123 | ver.dist.tarball = Url.format(distUrl); 124 | done(); 125 | } else { 126 | done(); 127 | } 128 | }; 129 | 130 | if (!pkgJson.versions) { return cb(null, pkgJson); } 131 | 132 | Async.each(Object.keys(pkgJson.versions), changeDistUrl, function(err) { 133 | return cb(err, pkgJson); 134 | }); 135 | 136 | }; 137 | 138 | /** 139 | * Sets the metadata for a package 140 | * 141 | * @param {string} pkg - The package name 142 | * @param {Object} meta - The original package metadata 143 | * @callback cb - (err, pkg_json) 144 | */ 145 | Cache.prototype.setMeta = function(pkg, meta, cb) { 146 | 147 | var cache = this; 148 | var path = this.getMetaPath(pkg); 149 | var dir = Path.dirname(path); 150 | var pkgJson; 151 | 152 | Async.series([ 153 | function(done) { 154 | new Mkdirp(dir, done); 155 | }, 156 | function(done){ 157 | Fs.writeFile(path + '.orig', JSON.stringify(meta), done); 158 | }, 159 | function(done) { 160 | cache.overrrideTarballUrls(meta, function(err, json){ 161 | if (err) { return done(err); } 162 | pkgJson = json; 163 | done(); 164 | }); 165 | }, 166 | function(done) { 167 | Fs.writeFile(path, JSON.stringify(pkgJson), done); 168 | } 169 | ], function(err) { 170 | cb(err, pkgJson); 171 | }); 172 | }; 173 | 174 | 175 | /** 176 | * Sets the metadata for a specific package version 177 | * 178 | * @param {Object} pkg - The package (./lib/package) object 179 | * @param {Object} meta - The original package metadata 180 | * @callback cb - (err) 181 | */ 182 | Cache.prototype.setMetaVersion = function(pkg, meta, cb) { 183 | 184 | var path = this.getMetaVersionPath(pkg.name, pkg.version); 185 | var dir = Path.dirname(path); 186 | 187 | new Mkdirp(dir, function(err) { 188 | if (err) { return cb(err); } 189 | Fs.writeFile(path, JSON.stringify(meta), cb); 190 | }); 191 | 192 | }; 193 | 194 | /** 195 | * Gets the metadata filepath for a package 196 | * 197 | * @param {string} pkg - The package name 198 | * @returns {string} 199 | */ 200 | Cache.prototype.getMetaPath = function(name) { 201 | return Path.join(this.cacheDir, name, this.metaFolder, name + '.json'); 202 | }; 203 | 204 | /** 205 | * Gets the metadata filepath for a specific package version 206 | * 207 | * @param {string} pkg - The package name 208 | * @param {string} version - The package version 209 | * @returns {string} 210 | */ 211 | Cache.prototype.getMetaVersionPath = function(name, version) { 212 | return Path.join(this.cacheDir, name, this.metaFolder, name + '-' + version + '.json'); 213 | }; 214 | 215 | /** 216 | * Async wrapper for getMetaPath 217 | * 218 | * @param {string} pkg - The package name 219 | * @callback (err, path) 220 | */ 221 | Cache.prototype.getMeta = function(name, cb) { 222 | 223 | if (!name) { return cb(new Error('Name not supplied')); } 224 | return cb(null, this.getMetaPath(name)); 225 | }; 226 | 227 | /** 228 | * Async wrapper for getMetaVersionPath 229 | * 230 | * @param {string} pkg - The package name 231 | * @param {string} version - The package version 232 | * @callback (err, path) 233 | */ 234 | Cache.prototype.getMetaVersion = function(name, version, cb) { 235 | 236 | if (!name) { return cb(new Error('Name not supplied')); } 237 | if (!version) { return cb(new Error('Version not supplied')); } 238 | 239 | return cb(null, this.getMetaVersionPath(name, version)); 240 | }; 241 | 242 | 243 | /** 244 | * Gets the path to the tarball for a specific package version 245 | * 246 | * @param {string} pkg - The package name 247 | * @param {string} version - The package version 248 | * @returns {string} 249 | */ 250 | Cache.prototype.getTarballPath = function(name, version) { 251 | return Path.join(this.cacheDir, name, this.tgzFolder, name + '-' + version + '.tgz'); 252 | }; 253 | 254 | /** 255 | * Async wrapper for getTarballPath 256 | * 257 | * @param {string} pkg - The package name 258 | * @param {string} version - The package version 259 | * @callback (err, path) 260 | */ 261 | Cache.prototype.getTarball = function(name, version, cb) { 262 | 263 | if (!name) { return cb(new Error('Name not supplied')); } 264 | if (!version) { return cb(new Error('Version not supplied')); } 265 | 266 | return cb(null, this.getTarballPath(name, version)); 267 | }; 268 | 269 | module.exports = Cache; 270 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2014 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Config = require('../config/config-defaults'); 8 | var PackageJSON = require('../package'); 9 | var Lactate = require('lactate'); 10 | var Npm = require('npm'); 11 | var FS = require('fs'); 12 | 13 | var Optimist = require('optimist') 14 | .usage('Usage: $0 -p -a -c ') 15 | .describe({ 16 | server_address: 'The external DNS name this mirror should serve on [default: localhost]', 17 | bind_address: 'The local interface bind address [default: 127.0.0.1]', 18 | cache_dir: 'The full directory path the cache directory (no trailing slash) [default: /tmp/files]', 19 | http_port: 'The HTTP port for the lazy mirror to listen on [default: 2000]', 20 | config: 'The path to the JSON configuration file', 21 | cache_expiry: 'Expire the upstream registry cache assets on this value (ms) [default: 24 hours]', 22 | cache_mem: 'The maxmium size in MB of registry data to keep in memory. A larger allocation reduces disk hits and improves performance [default: 200]', 23 | http_enabled: 'Enable the HTTP server on -p, this is true if neither --http_enabled or --https_enabled are supplied', 24 | https_enabled: 'Enable the HTTPS server on --https_port', 25 | https_key: 'The path to the private SSL key', 26 | https_cert: 'The path to the private SSL certificate', 27 | https_port: 'The HTTPS port (requires --https_enabled) [default: 443]', 28 | http_proxy: 'Specify a HTTP proxy to traverse before making outbound requests', 29 | https_proxy: 'Specify a HTTPS proxy to traverse before making outbound requests', 30 | package_blacklist: 'Full path to a JSON configuration file of blacklisted packages', 31 | permit_stale_resources: 'Continue to serve stale resources if the upstream registry cannot be contacted', 32 | real_external_port: 'If this mirror is behind a proxy, set this flag to the real external port for client connections', 33 | upstream_host: 'The upstream registry host [default: registry.npmjs.org]', 34 | upstream_port: 'The default upstream registry port [default: 80]', 35 | upstream_db_path: 'The default upstream registry database path, default: none (requires a leading slash)', 36 | upstream_use_https: 'Use HTTPS only when proxying to the upstream registry [default: false]', 37 | upstream_verify_ssl: 'Verify the upstream registry\'s SSL certificates [default: false]', 38 | log_level: 'Logging level [default: info]', 39 | version: 'Print version and exit', 40 | help: 'This help screen' 41 | }) 42 | .alias({ 43 | 'server_address': 'a', 44 | 'bind_address': 'b', 45 | 'cache_dir': 'c', 46 | 'http_port': ['port','p'], 47 | 'config': 'C', 48 | 'version': 'v', 49 | 'help': 'h' 50 | }); 51 | 52 | var Argv = Optimist.argv, 53 | config = {}; 54 | 55 | /* CLI specific */ 56 | if (Argv.help) { 57 | console.log(Optimist.help()); 58 | process.exit(); 59 | 60 | } else if (Argv.version) { 61 | console.log('npm-lazy-mirror: ' + PackageJSON.version); 62 | process.exit(); 63 | 64 | } else if (Argv.config) { 65 | Config = JSON.parse(FS.readFileSync(Argv.config)); 66 | } 67 | 68 | /* Local server options */ 69 | config.httpPort = Argv.http_port || Config.http_port || 2000; 70 | config.serverAddress = Argv.server_address || Config.server_address || 'localhost'; 71 | config.bindAddress = Argv.bind_address || Config.bind_address || '127.0.0.1'; 72 | config.httpEnabled = Argv.http_enabled || Config.http_enabled; 73 | 74 | /* Local server HTTPS options */ 75 | config.httpsPort = Argv.https_port || Config.https_port || 443; 76 | config.httpsEnabled = Argv.https_enabled || Config.https_enabled; 77 | 78 | if (!config.httpEnabled && !config.httpsEnabled) { 79 | config.httpEnabled = true; 80 | } 81 | 82 | config.realExternalPort = Argv.real_external_port || (config.httpsEnabled ? config.httpsPort : config.httpPort); 83 | 84 | /* Logging */ 85 | config.logLevel = Argv.log_level || Config.log_level || 'info'; 86 | 87 | config.httpsKey = Argv.https_key || Config.https_key || null; 88 | config.httpsCert = Argv.https_cert || Config.https_cert || null; 89 | 90 | /* Upstream Registry */ 91 | config.upstreamHost = Argv.upstream_host || Config.upstream_host || 'registry.npmjs.org'; 92 | config.upstreamPort = Argv.upstream_port || Config.upstream_port || 80; 93 | config.upstreamUseHttps = Argv.upstream_use_https || Config.upstream_use_https || false; 94 | config.upstreamVerifySSL = Argv.upstream_verify_ssl || Config.upstream_verify_ssl || false; 95 | config.upstreamDBPath = Argv.upstream_db_path || Config.upstream_db_path || ''; 96 | 97 | /* Caching options */ 98 | config.cache = {}; 99 | config.cacheDir = Argv.cache_dir || Config.cache_dir || '/tmp/files'; 100 | config.cacheExpiry = Argv.cache_expiry || Config.cache_expiry || 24 * 60 * 60 * 1000; // 24 Hours 101 | config.cacheMem = Argv.cache_mem || Config.cache_mem || 200; // MB 102 | config.cacheOptions = { 103 | cache: { 104 | expire: 60 * 60, 105 | max_keys: 500, 106 | max_size: config.cache_mem 107 | } 108 | }; 109 | config.cachePermitStale = Argv.permit_stale_resources || Config.permit_stale_resources || false; 110 | 111 | /* HTTP/S proxy */ 112 | config.getProxyConfig = function(cb) { 113 | Npm.load(function(err, npm) { 114 | if (err) { return (err); } 115 | return cb(null, { 116 | httpProxy: Argv.http_proxy || Config.http_proxy || Npm.config.get('proxy') || process.env.HTTP_PROXY, 117 | httpsProxy: Argv.https_proxy || Config.https_proxy || Npm.config.get('https-proxy') || process.env.HTTPS_PROXY 118 | }); 119 | }); 120 | }; 121 | 122 | /* Package Blacklist */ 123 | config.packageBlacklist = (function() { 124 | if (Argv.package_blacklist || Config.package_blacklist) { 125 | return require(Argv.package_blacklist || Config.package_blacklist); 126 | } 127 | })(); 128 | 129 | module.exports = config; 130 | -------------------------------------------------------------------------------- /lib/constants.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | module.exports = { 8 | CACHE_NOT_EXIST: 0, 9 | CACHE_EXPIRED: 2, 10 | CACHE_VALID: 1, 11 | TYPE_META: 'meta', 12 | TYPE_TGZ: 'tgz' 13 | }; 14 | -------------------------------------------------------------------------------- /lib/handlers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2014 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var _ = require('lodash'); 8 | var Log = require('log'); 9 | var Package = require('./package'); 10 | 11 | var log = new Log(); 12 | 13 | var Handler = {}; 14 | 15 | /** 16 | * Serves the entire metadata for a package 17 | * 18 | * @param {Object} req - Http.req 19 | * @param {Object} res - Http.res 20 | */ 21 | Handler.servePackageMeta = function(req, res, config) { 22 | 23 | new Package({ 24 | name: req.url.replace(/^\/|\/$/g, ''), 25 | config: config 26 | }) 27 | .getMeta(function(err, meta) { 28 | if (err && err.code === 404) { return Handler.notFound(res, res); } 29 | if (err) { return Handler.internalErr(req, res, err); } 30 | meta = meta.replace(config.cacheDir, ''); 31 | config.cacheServer.serve(meta, req, res); 32 | }); 33 | }; 34 | 35 | /** 36 | * Serves the version subset metadata for a package 37 | * 38 | * @param {Object} req - Http.req 39 | * @param {Object} res - Http.res 40 | */ 41 | Handler.servePackageVersionMeta = function(req, res, config) { 42 | 43 | var pkg = _.compact(req.url.split('/')); 44 | new Package({ 45 | name: pkg[0], 46 | version: pkg[1], 47 | config: config 48 | }) 49 | .getVersionMeta(function(err, meta) { 50 | if (err || !meta) { return Handler.internalErr(req, res, err); } 51 | meta = meta.replace(config.cacheDir, ''); 52 | config.cacheServer.serve(meta, req, res); 53 | }); 54 | }; 55 | 56 | /** 57 | * Serves the latest version subset metadata for a package 58 | * 59 | * @param {Object} req - Http.req 60 | * @param {Object} res - Http.res 61 | */ 62 | Handler.serveLatestPackageMeta = function(req, res, config) { 63 | 64 | new Package({ 65 | name: req.url.replace(/^\/|\/$/g, '').replace(/\/latest\/?$/, ''), 66 | config: config 67 | }) 68 | .getLatestVersionMeta(function(err, meta) { 69 | if (err || !meta) { 70 | if (err.code === 404) { return Handler.notFound(res, res); } 71 | return Handler.internalErr(req, res, err); 72 | } 73 | meta = meta.replace(config.cacheDir, ''); 74 | config.cacheServer.serve(meta, req, res); 75 | }); 76 | 77 | }; 78 | 79 | /** 80 | * Serves the binary .tgz for a package, as requisitioned from the 'dist' key 81 | * in the package metadata. 82 | * 83 | * @param {Object} req - Http.req 84 | * @param {Object} res - Http.res 85 | */ 86 | Handler.servePackageTarball = function(req, res, config) { 87 | 88 | var name, version; 89 | var nameSplit = req.url.split('/-/'); 90 | name = nameSplit[0].replace('/', ''); 91 | if (nameSplit.length > 1) { 92 | version = nameSplit[1].replace('.tgz', '').replace(name + '-', ''); 93 | } 94 | 95 | new Package({ 96 | name: name, 97 | version: version, 98 | config: config 99 | }) 100 | .getTarball(function(err, tarball) { 101 | 102 | if (err && err.code === 404) { 103 | return Handler.notFound(req, res); 104 | } else if (err && !tarball) { 105 | return Handler.internalErr(req, res, err); 106 | } 107 | 108 | tarball = tarball.replace(config.cacheDir, ''); 109 | config.cacheServer.serve(tarball, req, res); 110 | }); 111 | }; 112 | 113 | /** 114 | * Handle generic 404s 115 | * 116 | * @param {Object} req - Http.req 117 | * @param {Object} res - Http.res 118 | * @param {Object} err - err object 119 | */ 120 | Handler.notFound = function(req, res) { 121 | var msg = 'Resource not found: ' + req.url; 122 | log.info(msg); 123 | res.statusCode = 404; 124 | res.end(msg); 125 | }; 126 | 127 | /** 128 | * Handle generic errors 129 | * 130 | * @param {Object} req - Http.req 131 | * @param {Object} res - Http.res 132 | * @param {Object} err - err object 133 | */ 134 | Handler.internalErr = function(req, res, err) { 135 | log.error('Internal request handler error: ', err); 136 | res.statusCode = 500; 137 | res.end('Internal server error ' + err); 138 | }; 139 | 140 | 141 | module.exports = Handler; 142 | 143 | -------------------------------------------------------------------------------- /lib/package.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2014 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Async = require('async'); 8 | var Cache = require('./cache'); 9 | var Constants = require('./constants'); 10 | var Fs = require('fs'); 11 | var Registry = require('./registry'); 12 | var Semver = require('semver'); 13 | 14 | /** 15 | * Provides a package class for working with package metadata and binary blobs. 16 | * 17 | * @constructor 18 | * @param {Object} opts - Options 19 | * @returns {Package} 20 | */ 21 | var Package = function (opts) { 22 | 23 | if (!this instanceof Package) { 24 | return new Package(opts); 25 | } 26 | 27 | opts = opts || {}; 28 | 29 | if (!opts.name) { return new Error('Package name not found'); } 30 | if (!opts.config && (!opts.config.upstreamHost || !opts.config.upstreamPort)) { 31 | return new Error('Registry is invalid: ' + opts.config); 32 | } 33 | 34 | this.name = opts.name; 35 | this.version = opts.version; 36 | 37 | this.config = opts.config; 38 | this.cache = new Cache(this.config); 39 | this.registry = new Registry(this.config); 40 | return this; 41 | 42 | }; 43 | 44 | /** 45 | * Gets the entire metadata for a package and stores it on disk and returns 46 | * the path to the stored file (JSON). 47 | * 48 | * @callback done - {err, path} 49 | */ 50 | Package.prototype.getMeta = function(cb) { 51 | 52 | var pkg = this; 53 | 54 | if (pkg.version) { 55 | var blacklisted = pkg.validateAgainstBlacklist(); 56 | if (blacklisted) { return cb(blacklisted); } 57 | } 58 | 59 | pkg.cache.validate({ 60 | type: Constants.TYPE_META, 61 | name: pkg.name 62 | }, 63 | function (err, validity) { 64 | if (err) { return cb(err); } 65 | 66 | if (validity === Constants.CACHE_EXPIRED && pkg.config.cachePermitStale) { 67 | Async.waterfall([ 68 | function (done) { 69 | pkg.registry.getMeta(pkg.name, function (err, meta) { 70 | done(null, meta); 71 | }); 72 | }, 73 | function (meta, done) { 74 | if (!meta) { 75 | pkg.cache.getMeta(pkg.name, done); 76 | } else { 77 | done(null, meta); 78 | } 79 | } 80 | ], cb); 81 | } else if (validity === Constants.CACHE_VALID) { 82 | pkg.cache.getMeta(pkg.name, cb); 83 | } else { 84 | Async.waterfall([ 85 | function (done) { 86 | pkg.registry.getMeta(pkg.name, done); 87 | }, 88 | function (meta, done) { 89 | pkg.cache.setMeta(pkg.name, meta, done); 90 | }, 91 | function (meta, done) { 92 | pkg.cache.getMeta(pkg.name, done); 93 | } 94 | ], function (err, result) { 95 | if (err) { return cb(err); } 96 | pkg.config.cache.fsStats.set(result, true); 97 | cb(null, result); 98 | }); 99 | } 100 | }); 101 | }; 102 | 103 | /** 104 | * Extracts the latest version metadata from the master metadata file 105 | * 106 | * @callback done - {err, path} 107 | */ 108 | Package.prototype.getLatestVersionMeta = function(done) { 109 | 110 | var pkg = this; 111 | 112 | pkg.getMeta(function (err, metaPath) { 113 | if (err) { return done(err); } 114 | 115 | Fs.readFile(metaPath, function (err, meta){ 116 | if (err) { return done(err); } 117 | meta = meta.toString(); 118 | var data; 119 | 120 | try { 121 | data = JSON.parse(meta); 122 | } catch (e) { 123 | return done(e); 124 | } 125 | 126 | if (data['dist-tags'].latest) { 127 | pkg.version = data['dist-tags'].latest; 128 | var blacklisted = pkg.validateAgainstBlacklist(); 129 | if (blacklisted) { return done(blacklisted); } 130 | pkg.getVersionMeta(done); 131 | } else { 132 | done(new Error('Cannot determine the latest version from the metadata')); 133 | } 134 | }); 135 | }); 136 | }; 137 | 138 | /** 139 | * Extracts the version metadata from the master metadata file, and creates a 140 | * separate file if necessary, for serving with a cache server. 141 | * 142 | * @callback done - {err, path} 143 | */ 144 | Package.prototype.getVersionMeta = function(done) { 145 | 146 | var pkg = this; 147 | 148 | if (!pkg.version) { return done(new Error('This package has no version')); } 149 | 150 | var blacklisted = pkg.validateAgainstBlacklist(); 151 | if (blacklisted) { return done(blacklisted); } 152 | 153 | pkg.getMeta(function (err, metaPath) { 154 | if (err) { return done(err); } 155 | 156 | Fs.readFile(metaPath, function (err, meta){ 157 | if (err) { return done(err); } 158 | meta = meta.toString(); 159 | var data; 160 | 161 | try { 162 | data = JSON.parse(meta); 163 | } catch (e) { 164 | return done(e); 165 | } 166 | 167 | if(data.versions[pkg.version]) { 168 | pkg.cache.setMetaVersion(pkg, data.versions[pkg.version], function (err){ 169 | if (err) { return done(err); } 170 | pkg.cache.getMetaVersion(pkg.name, pkg.version, done); 171 | }); 172 | } else { 173 | var error = new Error('Version not found'); 174 | error.code = 404; 175 | done(error); 176 | } 177 | }); 178 | }); 179 | }; 180 | 181 | /** 182 | * Assembles the package, checks the cache validity and reponds with the 183 | * path to the tarball 184 | * 185 | * @callback cb - {err, path} 186 | */ 187 | Package.prototype.getTarball = function(cb) { 188 | 189 | var pkg = this; 190 | 191 | var blacklisted = pkg.validateAgainstBlacklist(); 192 | if (blacklisted) { return cb(blacklisted); } 193 | 194 | 195 | Async.waterfall([ 196 | function(done) { 197 | pkg.cache.validate({ 198 | type: Constants.TYPE_TGZ, 199 | name: pkg.name, 200 | version: pkg.version 201 | }, done); 202 | }, 203 | function(validity, done){ 204 | if (validity === Constants.CACHE_VALID || (validity === Constants.CACHE_EXPIRED && pkg.config.cachePermitStale)) { 205 | pkg.getTarballFromMeta(done); 206 | } else { 207 | pkg.registry.getTarball(pkg, done); 208 | } 209 | } 210 | ], 211 | cb); 212 | }; 213 | 214 | /** 215 | * Extracts the tarball from the original JSON, and stores it in the package 216 | * directory, then returns the local path to it. 217 | * @callback cb = {err, path} 218 | */ 219 | Package.prototype.getTarballFromMeta = function (cb) { 220 | var pkg = this; 221 | 222 | pkg.getMeta(function(err, metaPath) { 223 | if (err) { return cb(err); } 224 | Fs.readFile(metaPath, function(err, meta){ 225 | if (err) { return cb(err); } 226 | meta = meta.toString(); 227 | var data; 228 | 229 | try { 230 | data = JSON.parse(meta); 231 | } catch (e) { 232 | return cb(e); 233 | } 234 | 235 | if (pkg.version) { 236 | if(data.versions[pkg.version]) { 237 | pkg.cache.getTarball(pkg.name, pkg.version, cb); 238 | } else { 239 | var error = new Error('Version not found'); 240 | error.code = 404; 241 | return cb(error); 242 | } 243 | } else { 244 | return cb(new Error('Version not supplied')); 245 | } 246 | }); 247 | }); 248 | }; 249 | 250 | /** 251 | * Validates the package name and semver against a supplied blacklist file. 252 | * Will return an error if the particular package is blacklisted, otherwise 253 | * undefined. 254 | * 255 | * @return {Error/undefined} 256 | */ 257 | Package.prototype.validateAgainstBlacklist = function () { 258 | 259 | var pkg = this; 260 | if (!pkg.version) { 261 | return new Error('Cannot validate a package with no version'); 262 | } 263 | 264 | if (pkg.config.packageBlacklist && pkg.config.packageBlacklist[pkg.name]) { 265 | if (Semver.satisfies(pkg.version, pkg.config.packageBlacklist[pkg.name])) { 266 | return new Error('Package ' + pkg.name + ' version: ' + pkg.config.packageBlacklist[pkg.name] + ' is blacklisted'); 267 | } 268 | } 269 | return; 270 | }; 271 | 272 | 273 | module.exports = Package; 274 | -------------------------------------------------------------------------------- /lib/registry.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2014 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Async = require('async'); 8 | var Constants = require('./constants'); 9 | var Fs = require('fs'); 10 | var Mkdirp = require('mkdirp'); 11 | var Path = require('path'); 12 | var Request = require('request'); 13 | var Url = require('url'); 14 | 15 | /** 16 | * Provides a class for interacting with the remote registry. 17 | * 18 | * @constructor 19 | * @param {Object} opts - Options 20 | */ 21 | var Registry = function (opts) { 22 | 23 | this.config = opts || {}; 24 | this.log = opts.log; 25 | 26 | var registry = this; 27 | 28 | var requestDefaults = { 29 | strictSSL: this.config.upstreamVerifySSL 30 | }; 31 | 32 | this.config.getProxyConfig(function (err, c){ 33 | if (!err) { 34 | requestDefaults.proxy = c.httpsProxy || c.httpProxy; 35 | registry.httpProxy = c.httpProxy; 36 | registry.httpsProxy = c.httpsProxy || c.httpProxy; 37 | } 38 | Request.defaults(requestDefaults); 39 | }); 40 | 41 | return this; 42 | }; 43 | 44 | /** 45 | * Pass thru proxy function to the upstream registry 46 | * 47 | * @param {Object} req - Http.Request 48 | * @param {Object} res - Http.Response 49 | */ 50 | Registry.prototype.proxyUpstream = function (req, res) { 51 | 52 | var registry = this; 53 | var reqMethod = req.method; 54 | var proto = this.config.upstreamUseHttps ? 'https:' : 'http:'; 55 | var port = this.config.upstreamPort; 56 | var url = Url.parse(req.url); 57 | var headers = req.headers; 58 | 59 | this.log.info('proxying request to registry ' + reqMethod + ' ' + req.url + ' ' + headers.host); 60 | 61 | url.hostname = registry.config.upstreamHost; 62 | url.protocol = proto; 63 | url.port = port; 64 | url.pathname = registry.config.upstreamDBPath + url.pathname; 65 | 66 | var options = { 67 | url: Url.format(url), 68 | headers: headers, 69 | strictSSL: this.config.upstreamVerifySSL 70 | }; 71 | 72 | if (registry.httpProxy) { options.proxy = registry.httpProxy; } 73 | if (registry.httpsProxy) { options.proxy = registry.httpsProxy; } 74 | 75 | new Request(options) 76 | .on('error', function (err) { 77 | registry.log.error('Error proxying to upstream registry: ' + err); 78 | }) 79 | .on('response', function (resp) { 80 | registry.log.debug('UPSTREAM REGISTRY RES:: CODE: ' + req.statusCode + ' HEADERS: ' + resp.headers); 81 | }) 82 | .pipe(res) 83 | .on('finish', function () { 84 | //TODO log request to separate access log 85 | }); 86 | }; 87 | 88 | 89 | /** 90 | * Gets the JSON metadata for all versions of a package. 91 | * 92 | * @param {String} pkg - the package name 93 | * @callback cb - (error, meta) 94 | */ 95 | Registry.prototype.getMeta = function (pkg, cb) { 96 | 97 | var registry = this; 98 | 99 | this.log.debug('proxy req GET METADATA: ' + pkg); 100 | 101 | var proto = this.config.upstreamUseHttps ? 'https' : 'http'; 102 | 103 | var requestOpts = { 104 | url: proto + '://' + this.config.upstreamHost + ':' + this.config.upstreamPort + this.config.upstreamDBPath + '/' + pkg, 105 | strictSSL: this.config.upstreamVerifySSL 106 | }; 107 | 108 | if (registry.httpProxy) { requestOpts.proxy = registry.httpProxy; } 109 | if (registry.httpsProxy) { requestOpts.proxy = registry.httpsProxy; } 110 | 111 | new Request(requestOpts, function (err, response, body) { 112 | if (err) { return cb(err); } 113 | 114 | if (response.statusCode !== 200) { 115 | var error = new Error('Invalid response code fetching metadata: ', response.statusCode); 116 | error.code = response.statusCode; 117 | return cb(error); 118 | } 119 | 120 | var meta; 121 | try { 122 | meta = JSON.parse(body); 123 | } catch (e) { 124 | return cb(e); 125 | } 126 | 127 | return cb(null, meta); 128 | }); 129 | }; 130 | 131 | /** 132 | * Retrieves the tarball for a specific version of a package. 133 | * 134 | * @param {Object} pkg - the package {name, version} 135 | * @callback cb - (error, meta) 136 | */ 137 | Registry.prototype.getTarball = function (pkg, cb) { 138 | 139 | var registry = this; 140 | var proto = registry.config.upstreamUseHttps ? 'https' : 'http'; 141 | var targetFile = pkg.cache.getTarballPath(pkg.name, pkg.version); 142 | var targetDir = Path.dirname(targetFile); 143 | 144 | Async.series([ 145 | function (done) { 146 | new Mkdirp(targetDir, done); 147 | }, 148 | function (done) { 149 | var requestOpts = { 150 | url: proto + '://' + pkg.config.upstreamHost + ':' + pkg.config.upstreamPort + pkg.config.upstreamDBPath + '/' + pkg.name + '/-/' + pkg.name + '-' + pkg.version + '.tgz', 151 | strictSSL: registry.config.upstreamVerifySSL 152 | }; 153 | 154 | if (registry.httpProxy) { requestOpts.proxy = registry.httpProxy; } 155 | if (registry.httpsProxy) { requestOpts.proxy = registry.httpsProxy; } 156 | 157 | var r = new Request(requestOpts) 158 | .on('response', function (res) { 159 | if (res.statusCode !== 200) { 160 | var error = new Error('Invalid response fetching tarball from remote repository: ' + res.statusCode); 161 | error.code = res.statusCode; 162 | return done(error); 163 | } 164 | r.pipe(Fs.createWriteStream(targetFile)) 165 | .on('finish', done) 166 | .on('error', done); 167 | }) 168 | .on('error', done); 169 | } 170 | ], function (err) { 171 | if (!err) { pkg.config.cache.fsStats.set(targetFile, Constants.CACHE_VALID); } 172 | cb(err, targetFile); 173 | }); 174 | }; 175 | 176 | module.exports = Registry; 177 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "npm-lazy-mirror", 3 | "version": "0.3.2", 4 | "description": "Lazy Mirroring for npmjs.org", 5 | "main": "index.js", 6 | "bin": { 7 | "npm-lazy-mirror": "./bin/npm-lazy-mirror" 8 | }, 9 | "bugs": { 10 | "url": "https://github.com/ActiveState/npm-lazy-mirror/issues" 11 | }, 12 | "homepage": "https://github.com/ActiveState/npm-lazy-mirror", 13 | "repository": { 14 | "type": "git", 15 | "url": "https://github.com/ActiveState/npm-lazy-mirror" 16 | }, 17 | "scripts": { 18 | "start": "./bin/npm-lazy-mirror", 19 | "test": "mocha --reporter list test/runner.js test/npm.js" 20 | }, 21 | "dependencies": { 22 | "async": "0.7.0", 23 | "async-cache": "0.1.5", 24 | "lactate": "0.13.12", 25 | "lodash": "2.4.1", 26 | "log": "1.4.0", 27 | "mkdirp": "0.3.5", 28 | "npm": "1.4.7", 29 | "optimist": "0.6.1", 30 | "request": "2.34.0", 31 | "semver": "2.2.1" 32 | }, 33 | "devDependencies": { 34 | "mocha": "", 35 | "supertest": "" 36 | }, 37 | "engines": { 38 | "node": ">= 0.10" 39 | }, 40 | "keywords": [ 41 | "npm", 42 | "mirror", 43 | "lazy", 44 | "cache" 45 | ], 46 | "author": "Jamie Paton ", 47 | "license": "MIT" 48 | } 49 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2014 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Async = require('async'); 8 | var AsyncCache = require('async-cache'); 9 | var Cache = require('./lib/cache'); 10 | var Config = require('./lib/config'); 11 | var Constants = require('./lib/constants'); 12 | var Fs = require('fs'); 13 | var Handlers = require('./lib/handlers'); 14 | var Http = require('http'); 15 | var Https = require('https'); 16 | var Lactate = require('lactate'); 17 | var Log = require('log'); 18 | var Registry = require('./lib/registry'); 19 | 20 | /* Generic logger */ 21 | var log = new Log(Config.logLevel); 22 | 23 | Config.log = log; 24 | 25 | /* Upstream Proxy */ 26 | var registry = new Registry(Config); 27 | 28 | /* HTTP asset caching server */ 29 | Config.cacheServer = Lactate.dir(Config.cacheDir, Config.cacheOptions); 30 | 31 | /** 32 | * If the file cannot be found locally, proxy to the upstream registry as 33 | * a last resort 34 | */ 35 | Config.cacheServer.notFound(function (req, res) { 36 | registry.proxyUpstream(req, res); 37 | }); 38 | 39 | /* Global async disk cache */ 40 | Config.cache.fsStats = new AsyncCache({ 41 | max: 5000, 42 | maxAge: Config.cacheExpiry, 43 | load: function(path, cb) { 44 | 45 | log.info('Cache MISS: ', path); 46 | 47 | var cache = new Cache(Config); 48 | Async.waterfall([ 49 | function (done) { 50 | cache.existsDisk(path, done); 51 | }, 52 | function (exists, done) { 53 | if (exists) { 54 | cache.diskExpired(path, done); 55 | } else { 56 | done(null, null, null); 57 | } 58 | }, 59 | function (expired, stats, done) { 60 | if (!expired && stats) { 61 | done(null, Constants.CACHE_VALID); 62 | } else { 63 | done(null, Constants.CACHE_NOT_EXIST); 64 | } 65 | } 66 | ], 67 | function (err, valid) { 68 | cb(err, valid); 69 | }); 70 | } 71 | }); 72 | 73 | /** 74 | * Primary HTTP/HTTPS request dispatcher 75 | * 76 | * @param {Object} req - HTTP.req 77 | * @param {Object} res - HTTP.res 78 | */ 79 | var serveRequest = function (req, res) { 80 | req.headers.host = Config.upstreamHost; 81 | 82 | log.info(req.method + ' ' + req.url); 83 | 84 | /* /package/latest */ 85 | if (req.url.match(/^\/[_-\w.]+?\/latest\/?$/)) { 86 | Handlers.serveLatestPackageMeta(req, res, Config); 87 | 88 | /* /package/ */ 89 | } else if (req.url.match(/^\/[_-\w.]+?\/[0-9]+\.[0-9]+\.[0-9]+/)) { 90 | Handlers.servePackageVersionMeta(req, res, Config); 91 | 92 | /* /-/all/ */ 93 | } else if (req.url.match(/\/-\/all\/.*/)) { 94 | registry.proxyUpstream(req, res); 95 | 96 | /* /-/- */ 97 | } else if (req.url.match(/^\/[_-\w.]+?\/-\/.*\.tgz/)) { 98 | Handlers.servePackageTarball(req, res, Config); 99 | 100 | /* /package */ 101 | } else if (req.url.match(/^\/[_-\w.]+?$/)) { 102 | Handlers.servePackageMeta(req, res, Config); 103 | 104 | /* void */ 105 | } else { 106 | registry.proxyUpstream(req, res); 107 | } 108 | }; 109 | 110 | /** 111 | * The main HTTP server 112 | */ 113 | if (Config.httpEnabled) { 114 | var httpServer = Http.createServer(serveRequest); 115 | 116 | httpServer.listen(Config.httpPort, Config.bindAddress, function () { 117 | log.info('Lazy mirror (HTTP) is listening @ ' + Config.bindAddress + ':' + Config.httpPort + ' External host: ' + Config.serverAddress); 118 | }); 119 | } 120 | 121 | /** 122 | * The main HTTPS server 123 | */ 124 | if (Config.httpsEnabled) { 125 | 126 | if (!Config.httpsKey || !Config.httpsCert) { 127 | throw new Error('Missing https_cert or http_key option'); 128 | } 129 | 130 | var httpsOptions = { 131 | key: Fs.readFileSync(Config.httpsKey), 132 | cert: Fs.readFileSync(Config.httpsCert), 133 | }; 134 | 135 | var httpsServer = Https.createServer(httpsOptions, serveRequest); 136 | 137 | httpsServer.listen(Config.httpsPort, Config.bindAddress, function () { 138 | log.info('Lazy mirror (HTTPS) is listening @ ' + Config.bindAddress + ':' + Config.httpsPort + ' External host: ' + Config.serverAddress); 139 | }); 140 | } 141 | 142 | 143 | -------------------------------------------------------------------------------- /test/helpers.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Config = require('../lib/config'); 8 | var Helpers = {}; 9 | 10 | Helpers.randomModules = [ 'express', 'supertest', 'mocha', 'hapi', 11 | 'coffee-script', 'lodash' ]; 12 | 13 | Helpers.getRandom = function(min, max) { 14 | return Math.floor(Math.random() * (max - min) + min); 15 | }; 16 | 17 | Helpers.getRandomModule = function() { 18 | return Helpers.randomModules[Helpers.getRandom(0, Helpers.randomModules.length)]; 19 | }; 20 | 21 | Helpers.handleRes = function(err) { 22 | if (err) { 23 | throw err; 24 | } 25 | }; 26 | 27 | Helpers.tarballPointsToMirror = function(res) { 28 | if (!res.body.dist.tarball) { return 'tarball field missing'; } 29 | if (res.body.dist.tarball.indexOf(Config.serverAddress + ':' + Config.realExternalPort) === -1) { 30 | return 'tarball URI does not map to the server external address:port'; 31 | } 32 | }; 33 | 34 | module.exports = Helpers; 35 | 36 | -------------------------------------------------------------------------------- /test/npm.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Config = require('../lib/config'); 8 | var Helpers = require('./helpers'); 9 | var Mkdirp = require('mkdirp'); 10 | var Npm = require('npm'); 11 | 12 | var npmLogLevel = 'warn'; 13 | var nodeModulesDir = '/tmp/npm-lazy-test-modules'; 14 | 15 | Npm.load({loglevel: npmLogLevel}, function (err, npm) { 16 | npm.config.set('registry', 'http://' + Config.serverAddress + ':' + Config.realExternalPort); 17 | npm.config.set('prefix', nodeModulesDir); 18 | npm.config.set('global', true); 19 | 20 | describe('It should make the test install folder', function () { 21 | it('should create ' + nodeModulesDir, function (done) { 22 | new Mkdirp(nodeModulesDir, done); 23 | }); 24 | }); 25 | 26 | describe('It should install packages using the npm module', function (){ 27 | this.timeout(20000); 28 | 29 | it('should install npm', function (done) { 30 | npm.commands.install(['npm'], done); 31 | }); 32 | 33 | var rand1 = Helpers.getRandomModule(); 34 | it('should install a random module: ' + rand1, function (done) { 35 | this.timeout(1000 * 60 * 3); 36 | npm.commands.install([rand1], done); 37 | }); 38 | 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /test/runner.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) ActiveState 2013 - ALL RIGHTS RESERVED. 3 | */ 4 | 5 | 'use strict'; 6 | 7 | var Helpers = require('./helpers'); 8 | 9 | var HOST = process.env.TEST_HOST || 'localhost'; 10 | var PORT = process.env.TEST_PORT || 2000; 11 | 12 | var request = require('supertest'); 13 | 14 | request = request('http://' + HOST + ':' + PORT); 15 | 16 | describe('GET /', function(){ 17 | it('respond with json', function(done) { 18 | request.get('/') 19 | .set('Accept', 'application/json') 20 | .expect(200) 21 | .expect(/registry/, done); 22 | }); 23 | }); 24 | 25 | describe('GET /express', function(){ 26 | it('responds 200', function(done) { 27 | request.get('/express') 28 | .set('Accept', 'application/json') 29 | .expect(200, done); 30 | }); 31 | }); 32 | 33 | describe('GET /express/latest', function(){ 34 | it('respond 200 with Cache-Control header', function(done) { 35 | request.get('/express/latest') 36 | .set('Accept', 'application/json') 37 | .expect(200) 38 | .expect('Cache-Control', /max-age/, done); 39 | }); 40 | }); 41 | 42 | describe('GET /express/3.4.4', function(){ 43 | it('gets the express 3.4.4 metadata', function(done) { 44 | request.get('/express/3.4.4') 45 | .expect(200) 46 | .expect('Cache-Control', /max-age/, done); 47 | }); 48 | }); 49 | 50 | describe('GET /multiparty/2.2.0', function(){ 51 | it('responds 200 for the multiparty 2.2.0 metadata', function(done) { 52 | request.get('/multiparty/2.2.0') 53 | .expect(200) 54 | .expect('Cache-Control', /max-age/, done); 55 | }); 56 | }); 57 | 58 | describe('GET /multiparty/-/multiparty-2.2.0.tgz', function(){ 59 | it('gets the multiparty 2.2.0 tarball', function(done) { 60 | 61 | this.timeout(10000); 62 | 63 | request.get('/multiparty/-/multiparty-2.2.0.tgz') 64 | .expect(200) 65 | .expect('Cache-Control', /max-age/) 66 | .expect('Content-Type', /application\/octet-stream/, done); 67 | }); 68 | }); 69 | 70 | describe('GET /multiparty/-/multiparty-2.222.0.tgz (fetch tarball)', function(){ 71 | it('404 response on non-existing tarball', function(done) { 72 | request.get('/multiparty/-/multiparty-2.222.0.tgz') 73 | .expect(404, done); 74 | }); 75 | }); 76 | 77 | describe('GET //does/not/exist', function(){ 78 | it('responds 404 on non-existing package', function(done) { 79 | request.get('//does/not/exist') 80 | .expect(404, done); 81 | }); 82 | }); 83 | 84 | describe('GET /doesnotexist', function(){ 85 | it('responds 404 on non-existing package', function(done) { 86 | request.get('/doesnotexists') 87 | .expect(404, done); 88 | }); 89 | }); 90 | var randomModule = Helpers.getRandomModule(); 91 | 92 | describe('GET /' + randomModule, function(){ 93 | it('returns a random module metadata', function(done) { 94 | request.get('/' + randomModule) 95 | .expect(200, done); 96 | }); 97 | }); 98 | 99 | describe('GET /raw-body/-/raw-body-1.1.2.tgz', function(){ 100 | it('returns a tarball for package with multiple hyphens', function(done) { 101 | request.get('/raw-body/-/raw-body-1.1.2.tgz') 102 | .expect(200, done); 103 | }); 104 | }); 105 | 106 | describe('GET /dateformat/1.0.7-1.2.3', function(){ 107 | it('returns metadata for a package with multiple versions', function(done) { 108 | request.get('/dateformat/1.0.7-1.2.3') 109 | .set('Accept', 'application/json') 110 | .expect('Content-Type', /json/) 111 | .expect(Helpers.tarballPointsToMirror) 112 | .expect(200, done); 113 | }); 114 | }); 115 | 116 | --------------------------------------------------------------------------------