├── .eslintignore ├── .travis.yml ├── .npmignore ├── src ├── index.js ├── constants.js ├── util │ ├── managed-child.js │ ├── format.js │ ├── file.js │ ├── log.js │ ├── managed.js │ └── managed-master.js ├── run-env-defaults.js ├── link-filter.js ├── scan.js ├── cli.compare-foo.mocha.man.js ├── link-filter.mocha.js ├── find-packages.js ├── cli-options.js ├── cli.js ├── link.js └── pack-ref.js ├── .gitignore ├── prettier.config.js ├── .eslintrc.js ├── fixtures ├── projects │ └── foo1 │ │ ├── package.json │ │ └── npm-shrinkwrap.json ├── save-for-bar1 │ ├── package.json │ └── npm-shrinkwrap.json └── cli-test-basic.bash ├── .editorconfig ├── babel.config.js ├── LICENSE ├── usage.txt ├── bin └── pkglink.js ├── package.json └── README.md /.eslintignore: -------------------------------------------------------------------------------- 1 | **/build-lib/** 2 | **/node_modules/** 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '10' 4 | - '12' 5 | - 'node' 6 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | /src/ 2 | /build-lib/**/*.mocha.js 3 | /build-lib/**/*.mocha.man.js 4 | /fixtures/ -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | export { prune } from './pack-ref'; 2 | export { default as scanAndLink } from './scan'; 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | *.log 3 | .nyc_output 4 | node_modules 5 | dist 6 | build-lib 7 | build-es 8 | coverage 9 | _book 10 | -------------------------------------------------------------------------------- /prettier.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | printWidth: 120, 3 | trailingComma: 'none', 4 | semi: true, 5 | singleQuote: true, 6 | tabWidth: 2 7 | }; 8 | -------------------------------------------------------------------------------- /src/constants.js: -------------------------------------------------------------------------------- 1 | export const DEFAULT_CONFIG_FILE = '.pkglink'; // in home directory 2 | export const DEFAULT_REFS_FILE = '.pkglink_refs'; // in home directory 3 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | mocha: true, 4 | node: true 5 | }, 6 | extends: ['standard', 'prettier', 'prettier/standard'], 7 | rules: { 8 | camelcase: 'off', 9 | 'handle-callback-err': 'off' 10 | }, 11 | parser: 'babel-eslint' 12 | }; 13 | -------------------------------------------------------------------------------- /fixtures/projects/foo1/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "foo1", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "devDependencies": { 12 | "expect": "1.20.2" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /fixtures/save-for-bar1/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "foo1", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "devDependencies": { 12 | "expect": "1.20.1" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig helps developers define and maintain 2 | # consistent coding styles between different editors and IDEs. 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | indent_style = space 12 | indent_size = 2 13 | 14 | [*.md] 15 | trim_trailing_whitespace = false 16 | 17 | -------------------------------------------------------------------------------- /src/util/managed-child.js: -------------------------------------------------------------------------------- 1 | import R from 'ramda'; 2 | 3 | export default function runAsChild(INTERRUPT_TYPE) { 4 | const onInterrupt = (fn) => { 5 | // set the onInterrupt fn to call 6 | const interruptOnce = R.once(fn); 7 | process 8 | .once('SIGINT', interruptOnce) 9 | .once('SIGTERM', interruptOnce) 10 | .on('message', (msg) => { 11 | if (msg && msg.type === INTERRUPT_TYPE) { 12 | interruptOnce(); 13 | } 14 | }); 15 | }; 16 | 17 | const shutdown = R.once(() => { 18 | process.disconnect(); 19 | }); 20 | 21 | return { 22 | onInterrupt, 23 | shutdown 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /src/util/format.js: -------------------------------------------------------------------------------- 1 | import numeral from 'numeral'; 2 | import truncate from 'cli-truncate'; 3 | 4 | export function formatBytes(bytes) { 5 | return numeral(bytes).format('0.[00]b'); 6 | } 7 | 8 | export function formatDevNameVersion(dev, name, version) { 9 | // use name-version first since device is usually constant 10 | return `${name}-${version}:${dev}`; 11 | } 12 | 13 | export function sortObjKeys(obj) { 14 | return Object.keys(obj) 15 | .sort() 16 | .reduce((acc, k) => { 17 | acc[k] = obj[k]; 18 | return acc; 19 | }, {}); 20 | } 21 | 22 | export function trunc(size, str) { 23 | return truncate(str, size, { position: 'middle' }); 24 | } 25 | -------------------------------------------------------------------------------- /src/util/file.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra-promise'; 2 | 3 | export function safeJsonReadSync(file) { 4 | // returns obj, error, or undefined when not found 5 | try { 6 | const stat = fs.statSync(file); 7 | if (stat && stat.size) { 8 | try { 9 | return fs.readJsonSync(file); 10 | } catch (err) { 11 | return err; 12 | } 13 | } 14 | return undefined; 15 | } catch (err) { 16 | if (err.code !== 'ENOENT') { 17 | console.error(err); 18 | return err; 19 | } 20 | return undefined; 21 | } 22 | } 23 | 24 | export function outputFileStderrSync(file) { 25 | const content = fs.readFileSync(file); 26 | process.stderr.write(content); 27 | } 28 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | const BABEL_ENV = process.env.BABEL_ENV; 2 | const CI = process.env.CI && process.env.CI === 'true'; 3 | 4 | const presets = [ 5 | [ 6 | '@babel/env', 7 | { 8 | // targets are specified in .browserslist 9 | // run `npx browserslist` will show resultant targets or see debug output from build 10 | useBuiltIns: 'usage', 11 | corejs: '3.6.4', 12 | modules: BABEL_ENV === 'es' ? false : 'auto', // not transforming modules for es 13 | debug: CI // show the browser target and plugins used when in CI mode 14 | } 15 | ] 16 | ]; 17 | 18 | const plugins = []; 19 | 20 | // these are merged with others 21 | const env = {}; 22 | 23 | module.exports = { 24 | env, 25 | plugins, 26 | presets 27 | }; 28 | -------------------------------------------------------------------------------- /src/run-env-defaults.js: -------------------------------------------------------------------------------- 1 | import { Observable } from 'rxjs'; 2 | 3 | /* 4 | The defaults for the runtime environment, used by CLI 5 | and main code for providing current status. Default log, 6 | log.clear, and out are no-ops, will likely be overridden 7 | */ 8 | 9 | function noop() {} 10 | 11 | // placeholder, can be overridden 12 | function noopLog(str) {} 13 | noopLog.clear = noop; 14 | 15 | const rtenv = { 16 | cancelled: false, 17 | cancelled$: Observable.never(), 18 | completedPackages: 0, 19 | currentPackageDir: '', 20 | existingPackRefs: {}, 21 | linkFn: null, // alternate linking promise fn 22 | log: noopLog, 23 | updatedPackRefs: {}, 24 | out: noop, 25 | packageCount: 0, 26 | savedByteCount: 0 27 | }; 28 | 29 | export default rtenv; 30 | -------------------------------------------------------------------------------- /src/link-filter.js: -------------------------------------------------------------------------------- 1 | /* 2 | Filter applied to files being considered for linking 3 | @returns true for files to perform a hard link on 4 | */ 5 | export default function linkFilter(config, dstPackInode, x) { 6 | // filter out missing targets 7 | return ( 8 | x.dstEI && 9 | // take only non-package.json files 10 | x.dstEI.stat.ino !== dstPackInode && 11 | // make sure not same inode as master 12 | x.srcEI.stat.ino !== x.dstEI.stat.ino && 13 | // same device 14 | x.srcEI.stat.dev === x.dstEI.stat.dev && 15 | // same size 16 | x.srcEI.stat.size === x.dstEI.stat.size && 17 | // ignoreModTime or is same modified datetime 18 | (config.ignoreModTime || 19 | x.srcEI.stat.mtime.getTime() === x.dstEI.stat.mtime.getTime()) && 20 | // big enough to care about 21 | x.dstEI.stat.size >= config.minFileSize 22 | ); 23 | } 24 | -------------------------------------------------------------------------------- /src/util/log.js: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import numeral from 'numeral'; 3 | import { Subject } from 'rxjs'; 4 | import { formatBytes, trunc } from './format'; 5 | 6 | // eslint-disable-next-line import/prefer-default-export 7 | export function createLogUpdate(config, rtenv) { 8 | // throttle logging of scan updates 9 | const logUpdate$ = new Subject(); 10 | const linkSaved = config.dryrun ? 'saves:' : 'saved:'; 11 | logUpdate$ 12 | .throttleTime(100) // throttle scan updates 100ms each 13 | .subscribe(() => { 14 | rtenv.log( 15 | `${chalk.blue('pkgs:')} ${numeral(rtenv.completedPackages).format('0,0')}/${numeral(rtenv.packageCount).format( 16 | '0,0' 17 | )} ${chalk.green(linkSaved)} ${chalk.bold(formatBytes(rtenv.savedByteCount))} ${chalk.dim( 18 | trunc(config.extraCols, rtenv.currentPackageDir) 19 | )}` 20 | ); 21 | }); 22 | 23 | return function logUpdate() { 24 | logUpdate$.next(); 25 | }; 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Jeff Barczewski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /src/util/managed.js: -------------------------------------------------------------------------------- 1 | import runAsManagedChild from './managed-child'; 2 | import runAsMaster from './managed-master'; 3 | 4 | const INTERRUPT_TYPE = 'INTERRUPT_SHUTDOWN'; 5 | 6 | /* 7 | Handles setting up signals and if we are running 8 | a child process then it hooks signals up so we can 9 | do a graceful shutdown on Control-c including from 10 | Windows. 11 | */ 12 | 13 | let launchWorkerMain = () => {}; // defined later 14 | 15 | function launchWorker(script, opts) { 16 | return launchWorkerMain(script, opts); 17 | } 18 | 19 | if (process.disconnect) { 20 | // running as a child 21 | const childMethods = runAsManagedChild(INTERRUPT_TYPE); 22 | launchWorker.onInterrupt = childMethods.onInterrupt; 23 | launchWorker.shutdown = childMethods.shutdown; 24 | } else { 25 | // otherwise running as master 26 | const masterMethods = runAsMaster(INTERRUPT_TYPE); 27 | launchWorkerMain = masterMethods.launchChildWorker; 28 | launchWorker.onInterrupt = masterMethods.onInterrupt; 29 | launchWorker.shutdown = masterMethods.shutdown; 30 | } 31 | 32 | export default launchWorker; 33 | -------------------------------------------------------------------------------- /src/scan.js: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import { createLogUpdate } from './util/log'; 3 | import { genModuleLinks, handleModuleLinking, determineLinks } from './link'; 4 | import { determinePackLinkSrcDst } from './pack-ref'; 5 | import findPackages from './find-packages'; 6 | 7 | export default function scanAndLink(config, rtenv, rootDirs) { 8 | const logUpdate = createLogUpdate(config, rtenv); 9 | 10 | const outputSrcDstForDryrun = config.dryrun 11 | ? (lnkSrcDst) => { 12 | const { devNameVer: dnv, src, dst } = lnkSrcDst; 13 | rtenv.log.clear(); 14 | rtenv.out(chalk.bold(dnv.split(':')[0])); // nameVersion 15 | rtenv.out(` ${src}`); 16 | rtenv.out(` ${dst}`); 17 | rtenv.out(''); 18 | } 19 | : () => {}; 20 | 21 | return findPackages(config, rtenv, rootDirs, logUpdate) 22 | .takeWhile(() => !rtenv.cancelled) 23 | .mergeMap((eiDN) => determinePackLinkSrcDst(config, rtenv, eiDN), config.concurrentOps) 24 | .takeWhile(() => !rtenv.cancelled) 25 | .do((lnkSrcDst) => outputSrcDstForDryrun(lnkSrcDst)) 26 | .do((lnkSrcDst) => { 27 | rtenv.currentPackageDir = lnkSrcDst.dst; 28 | logUpdate(); 29 | }) 30 | .mergeMap((lnkSrcDst) => { 31 | if (config.dryrun) { 32 | return determineLinks(config, rtenv, lnkSrcDst, false); 33 | } else if (config.genLnCmds) { 34 | return genModuleLinks(config, rtenv, lnkSrcDst); 35 | } 36 | return handleModuleLinking(config, rtenv, lnkSrcDst); 37 | }, config.concurrentOps) 38 | 39 | .scan((acc, [src, dst, size]) => { 40 | acc += size; 41 | return acc; 42 | }, 0) 43 | .do((savedBytes) => { 44 | rtenv.savedByteCount = savedBytes; 45 | }) 46 | .do((savedBytes) => logUpdate()); 47 | } 48 | -------------------------------------------------------------------------------- /src/util/managed-master.js: -------------------------------------------------------------------------------- 1 | import cluster from 'cluster'; 2 | import Readline from 'readline'; 3 | import R from 'ramda'; 4 | 5 | // child process is killed if doesn't shutdown in this time 6 | const STOP_TIMEOUT = 10000; // 10s 7 | 8 | export default function runAsMaster(INTERRUPT_TYPE) { 9 | const win32 = process.platform === 'win32'; 10 | let readline; 11 | if (win32) { 12 | readline = Readline.createInterface({ 13 | input: process.stdin, 14 | output: process.stdout 15 | }); 16 | 17 | readline.on('SIGINT', () => { 18 | process.emit('SIGINT'); 19 | }); 20 | } 21 | 22 | const shutdown = R.once(() => { 23 | if (readline) { 24 | readline.close(); 25 | } 26 | }); 27 | 28 | function launchChildWorker(script, opts) { 29 | const options = R.merge( 30 | { 31 | exec: script, 32 | stopTimeout: STOP_TIMEOUT 33 | }, 34 | opts 35 | ); 36 | cluster.setupMaster(options); 37 | const worker = cluster.fork(); 38 | 39 | let killTimeout = null; 40 | 41 | const cancel = R.once(() => { 42 | // for windows compatibility 43 | worker.send({ type: INTERRUPT_TYPE }); 44 | 45 | // failsafe timer, kills child if doesn't shutdown 46 | killTimeout = setTimeout(() => { 47 | console.log('killing child'); 48 | worker.kill('SIGTERM'); 49 | killTimeout = null; 50 | }, options.stopTimeout); 51 | }); 52 | 53 | process.once('SIGINT', cancel).once('SIGTERM', cancel); 54 | 55 | worker.on('exit', (code) => { 56 | process.exitCode = code; 57 | if (killTimeout) { 58 | try { 59 | clearTimeout(killTimeout); 60 | killTimeout = null; 61 | } catch (err) { 62 | console.error(err); 63 | } 64 | } 65 | shutdown(); 66 | }); 67 | 68 | return worker; 69 | } 70 | 71 | function onInterrupt(fn) { 72 | process.once('SIGINT', fn); 73 | } 74 | 75 | return { 76 | launchChildWorker, 77 | onInterrupt, 78 | shutdown 79 | }; 80 | } 81 | -------------------------------------------------------------------------------- /usage.txt: -------------------------------------------------------------------------------- 1 | Usage: pkglink {OPTIONS} [dir] [dirN] 2 | 3 | Description: 4 | 5 | pkglink - Space saving Node.js package hard linker 6 | 7 | pkglink recursively searches directories for Node.js packages 8 | installed in node_modules directories. It uses the package name 9 | and version to match up possible packages to share. Once it finds 10 | similar packages, pkglink walks through the package directory tree 11 | checking for files that can be linked. If each file's modified 12 | datetime and size match, it will create a hard link for that file 13 | to save disk space. (On win32, mtimes are inconsistent and ignored) 14 | 15 | It keeps track of modules linked in ~/.pkglink_refs to quickly 16 | locate similar modules on future runs. The refs are always 17 | double checked before being considered for linking. This makes 18 | it convenient to perform future pkglink runs on new directories 19 | without having to reprocess the old. 20 | 21 | Standard Options: 22 | 23 | -c, --config CONFIG_PATH 24 | 25 | This option overrides the config file path, default ~/.pkglink 26 | 27 | -d, --dryrun 28 | 29 | Instead of performing the linking, just display the modules that 30 | would be linked and the amount of disk space that would be saved. 31 | 32 | -g, --gen-ln-cmds 33 | 34 | Instead of performing the linking, just generate link commands 35 | that the system would perform and output 36 | 37 | -h, --help 38 | 39 | Show this message 40 | 41 | -m, --memory MEMORY_MB 42 | 43 | Run with increased or decreased memory specified in MB, overrides 44 | environment variable PKGLINK_NODE_OPTIONS and config.memory 45 | The default memory used is 2560. 46 | 47 | -p, --prune 48 | 49 | Prune the refs file by checking all of the refs clearing out any 50 | that have changed 51 | 52 | -r, --refs-file REFS_FILE_PATH 53 | 54 | Specify where to load and store the link refs file which is used to 55 | quickly locate previously linked modules. Default ~/pkglink_refs.json 56 | 57 | -t, --tree-depth N 58 | 59 | Maximum depth to search the directories specified for packages 60 | Default depth: 0 (unlimited) 61 | 62 | -v, --verbose 63 | 64 | Output additional information helpful for debugging 65 | -------------------------------------------------------------------------------- /src/cli.compare-foo.mocha.man.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-extraneous-dependencies */ 2 | import expect from 'expect'; 3 | import Path from 'path'; 4 | import fs from 'fs-extra-promise'; 5 | 6 | const projectsPath = Path.join(__dirname, '../fixtures/projects'); 7 | const masterDir = 'foo1'; 8 | 9 | const rootDirs = ['foo2', 'foo3', 'bar1', 'cat1']; 10 | 11 | const filesLinked = [ 12 | 'node_modules/define-properties/index.js', 13 | 'node_modules/tmatch/LICENSE', 14 | 'node_modules/tmatch/index.js' 15 | ]; 16 | 17 | const filesNotLinked = [ 18 | 'node_modules/define-properties/package.json', 19 | 'node_modules/define-properties/.editorconfig' 20 | ]; 21 | 22 | describe('cli.compare-foo', () => { 23 | describe('should link master files', () => { 24 | filesLinked.forEach((f) => { 25 | it(`${masterDir}: ${f}`, () => { 26 | const p = Path.join(projectsPath, masterDir, f); 27 | return fs 28 | .statAsync(p) 29 | .then((stat) => stat.nlink) 30 | .then((nlink) => expect(nlink).toBeGreaterThan(0)); 31 | }); 32 | }); 33 | }); 34 | 35 | describe('should link matching files', () => { 36 | rootDirs.forEach((root) => { 37 | filesLinked.forEach((f) => { 38 | it(`${root}: ${f}`, () => { 39 | const p1 = Path.join(projectsPath, masterDir, f); 40 | const p2 = Path.join(projectsPath, root, f); 41 | let p1Stat; 42 | return fs 43 | .statAsync(p1) 44 | .then((stat1) => { 45 | p1Stat = stat1; 46 | }) 47 | .then(() => fs.statAsync(p2)) 48 | .then((stat2) => { 49 | expect(stat2.ino).toBe(p1Stat.ino); 50 | }); 51 | }); 52 | }); 53 | }); 54 | }); 55 | 56 | describe('should not link non-matching files', () => { 57 | rootDirs.forEach((root) => { 58 | filesNotLinked.forEach((f) => { 59 | it(`${root}: ${f}`, () => { 60 | const p1 = Path.join(projectsPath, masterDir, f); 61 | const p2 = Path.join(projectsPath, root, f); 62 | let p1Stat; 63 | return fs 64 | .statAsync(p1) 65 | .then((stat1) => { 66 | p1Stat = stat1; 67 | }) 68 | .then(() => fs.statAsync(p2)) 69 | .then((stat2) => { 70 | expect(stat2.ino).toNotBe(p1Stat.ino); 71 | }); 72 | }); 73 | }); 74 | }); 75 | }); 76 | }); 77 | -------------------------------------------------------------------------------- /bin/pkglink.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var OS = require('os'); 4 | var Path = require('path'); 5 | var minimist = require('minimist'); 6 | // managed sets up process listeners and win32 SIGINT hook 7 | var managed = require('../build-lib/util/managed').default; // require in either case 8 | var Constants = require('../build-lib/constants'); 9 | var FSUtils = require('../build-lib/util/file'); 10 | 11 | /* 12 | We determine whether we need to run a child process so we can 13 | enable a larger memory footprint. util/managed handles all of 14 | the details and sets up the signal handlers. 15 | 16 | The main CLI is in cli.js 17 | */ 18 | 19 | var script = Path.join(__dirname, '..', 'build-lib', 'cli.js'); 20 | var freeMemoryMB = Math.floor(OS.freemem() / (1024 * 1024)); 21 | var minimistOpts = { 22 | boolean: ['v'], 23 | string: ['c', 'm'], 24 | alias: { 25 | c: 'config', 26 | m: 'memory', 27 | v: 'verbose' 28 | } 29 | }; 30 | var argv = minimist(process.argv.slice(2), minimistOpts); 31 | var envNodeOptions = process.env.PKGLINK_NODE_OPTIONS; 32 | var CONFIG_PATH = 33 | argv.config || Path.resolve(OS.homedir(), Constants.DEFAULT_CONFIG_FILE); 34 | var parsedConfigJson = FSUtils.safeJsonReadSync(CONFIG_PATH); 35 | var configMemory = parsedConfigJson && parsedConfigJson.memory; 36 | var DESIRED_MEM = configMemory || 2560; // MB 37 | var hasExtraMemory = DESIRED_MEM < freeMemoryMB; 38 | 39 | // check in order argv.memory, env, config/default for node options 40 | var nodeOptions = argv.memory 41 | ? ['--max-old-space-size=' + argv.memory] 42 | : envNodeOptions 43 | ? envNodeOptions.split(' ') 44 | : ['--max-old-space-size=' + DESIRED_MEM]; 45 | 46 | var options = { 47 | execArgv: process.execArgv.concat(nodeOptions) 48 | }; 49 | 50 | if (argv.verbose) { 51 | console.log('argv.memory', argv.memory); 52 | console.log('process.env.PKGLINK_NODE_OPTIONS', envNodeOptions); 53 | console.log('config', parsedConfigJson); 54 | console.log('freeMemoryMB', freeMemoryMB); 55 | } 56 | 57 | var noOverrideNotEnoughMemory = 58 | !argv.memory && !envNodeOptions && !configMemory && !hasExtraMemory; 59 | 60 | var alreadyHasOptions = nodeOptions.every( 61 | (o) => process.execArgv.indexOf(o) !== -1 62 | ); 63 | 64 | // no overrides and not enough extra memory or already has proper options 65 | if (noOverrideNotEnoughMemory || alreadyHasOptions) { 66 | if (!alreadyHasOptions) { 67 | // indicate that we are running as is 68 | console.log( 69 | 'running with reduced memory, free:%sMB desired:%sMB', 70 | freeMemoryMB, 71 | DESIRED_MEM 72 | ); 73 | } 74 | require(script); // already has options invoke directly 75 | } else { 76 | // need to use child to get right options 77 | if (argv.verbose) { 78 | console.log('using child process to adjust working memory'); 79 | console.log('execArgv:', options.execArgv); 80 | } 81 | managed(script, options); 82 | } 83 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pkglink", 3 | "version": "1.1.1", 4 | "description": "Space saving Node.js package hard linker. pkglink locates common JavaScript/Node.js packages from your node_modules directories and hard links the package files so they share disk space.", 5 | "main": "build-lib/index.js", 6 | "bin": { 7 | "pkglink": "bin/pkglink.js", 8 | "pkglink_low": "build-lib/cli.js" 9 | }, 10 | "scripts": { 11 | "clean": "mkdirp build-lib && rimraf build-lib/*", 12 | "build": "cross-env BABEL_ENV=commonjs babel src --out-dir build-lib", 13 | "lint": "eslint --ext .js --ext .jsx src", 14 | "prebuild": "npm run clean", 15 | "postbuild": "echo \"Finished\"", 16 | "prepublish": "run-s prod-build lint test", 17 | "prewatch": "npm run clean", 18 | "pretest": "cd \"fixtures/projects/foo1\" && npm ci && cd -", 19 | "prod-build": "npm run build --production", 20 | "start": "npm run watch", 21 | "test": "run-s test:*", 22 | "test:cli-basic-direct": "cd fixtures && bash cli-test-basic.bash \"../bin/pkglink.js\"", 23 | "test:cli-basic-low": "cd fixtures && bash cli-test-basic.bash \"../build-lib/cli.js\"", 24 | "test:mocha": "cross-env BABEL_ENV=test mocha --require @babel/register \"src/**/*.mocha.js*\"", 25 | "watch": "run-p -c watch:*", 26 | "watch:babel": "npm run build -- -w", 27 | "watch:mocha": "npm run test:mocha -- -w" 28 | }, 29 | "author": "Jeff Barczewski", 30 | "repository": { 31 | "type": "git", 32 | "url": "http://github.com/jeffbski/pkglink.git" 33 | }, 34 | "bugs": { 35 | "url": "http://github.com/jeffbski/pkglink/issues" 36 | }, 37 | "engines": { 38 | "node": ">=4" 39 | }, 40 | "license": "MIT", 41 | "dependencies": { 42 | "@hapi/joi": "^17.1.1", 43 | "babel-eslint": "^10.1.0", 44 | "bluebird": "^3.7.2", 45 | "chalk": "^1.1.3", 46 | "cli-truncate": "^0.2.1", 47 | "core-js": "^3.9.1", 48 | "fs-extra-promise": "^0.4.1", 49 | "minimist": "^1.2.5", 50 | "numeral": "^1.5.3", 51 | "ramda": "^0.27.0", 52 | "readdirp": "^2.1.0", 53 | "rxjs": "^5.0.0-rc.1", 54 | "single-line-log": "^1.1.2", 55 | "strip-ansi": "^3.0.1" 56 | }, 57 | "devDependencies": { 58 | "@babel/cli": "^7.13.10", 59 | "@babel/core": "^7.13.10", 60 | "@babel/preset-env": "^7.13.10", 61 | "@babel/register": "^7.13.8", 62 | "cross-env": "^3.1.3", 63 | "eslint": "^6.8.0", 64 | "eslint-config-prettier": "^6.10.1", 65 | "eslint-config-standard": "^14.1.1", 66 | "eslint-plugin-import": "^2.22.1", 67 | "eslint-plugin-node": "^11.1.0", 68 | "eslint-plugin-prettier": "^3.3.1", 69 | "eslint-plugin-promise": "^4.3.1", 70 | "eslint-plugin-standard": "^4.0.1", 71 | "expect": "^1.20.2", 72 | "mkdirp": "^1.0.4", 73 | "mocha": "^7.1.1", 74 | "npm-run-all": "^4.1.5", 75 | "prettier": "^2.2.1", 76 | "rimraf": "^3.0.2" 77 | }, 78 | "keywords": [ 79 | "package", 80 | "link", 81 | "linking", 82 | "hard link", 83 | "npm", 84 | "modules", 85 | "sharing", 86 | "consolidating", 87 | "space" 88 | ] 89 | } 90 | -------------------------------------------------------------------------------- /fixtures/cli-test-basic.bash: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PKGLINK_EXEC="$1" 4 | 5 | echo "NOTE: if this script fails, just see what the last command run before failing" 6 | echo "It will likely be a difference in package size or something" 7 | 8 | set -e 9 | set -x 10 | 11 | unamestr=$(uname) 12 | 13 | # cwd should be here in fixtures 14 | rimraf REFS.json 15 | rimraf projects/foo2 16 | rimraf projects/foo3 17 | rimraf projects/bar1 18 | rimraf projects/cat1 19 | 20 | # single dir runs 21 | rimraf projects/foo2 22 | cp -a projects/foo1 projects/foo2 23 | ${PKGLINK_EXEC} -vr REFS.json projects/foo1 | tee output.log 24 | grep "pkgs: 37 saved: 0" output.log 25 | ${PKGLINK_EXEC} -vr REFS.json projects/foo2 | tee output.log 26 | grep "pkgs: 37 saved: 1.62MB" output.log 27 | grep "define-properties" REFS.json 28 | 29 | # combined multi-dir runs 30 | rimraf projects/foo3 31 | cp -a projects/foo1 projects/foo3 32 | ${PKGLINK_EXEC} -vr REFS.json -d projects/foo1 projects/foo3 | tee output.log 33 | grep "# pkgs: 74 would save: 1.62MB" output.log 34 | ${PKGLINK_EXEC} -vr REFS.json -g projects/foo1 projects/foo3 | tee output.log 35 | grep "# pkgs: 74 would save: 1.62MB" output.log 36 | ${PKGLINK_EXEC} -vr REFS.json projects/foo1 projects/foo3 | tee output.log 37 | grep "pkgs: 74 saved: 1.62MB" output.log 38 | 39 | # combined projects run picks up projects/bar1 (expect ver different) 40 | rimraf projects/bar1 41 | cp -a projects/foo1 projects/bar1 42 | cd projects/bar1 43 | cp ../../save-for-bar1/* ./ # override package.json and npm-shrinkwrap 44 | npm ci 45 | cd - 46 | ${PKGLINK_EXEC} -vr REFS.json -d projects | tee output.log 47 | grep -e "# pkgs: 148 would save: 1.42MB" output.log 48 | ${PKGLINK_EXEC} -vr REFS.json -g projects | tee output.log 49 | grep -e "# pkgs: 148 would save: 1.42MB" output.log 50 | ${PKGLINK_EXEC} -vr REFS.json projects | tee output.log 51 | grep -e "pkgs: 148 saved: 1.42MB" output.log 52 | 53 | # different modified time excluded 54 | rimraf projects/cat1 55 | cp -a projects/foo1 projects/cat1 56 | if [[ "$unamestr" =~ _NT ]] ; then # windows can't do modtime 57 | ${PKGLINK_EXEC} -vr REFS.json projects | tee output.log 58 | grep "pkgs: 185 saved: 1.61MB" output.log 59 | else # non-windows, test modtime excluded 60 | touch projects/cat1/node_modules/expect/lib/Expectation.js 61 | ${PKGLINK_EXEC} -vr REFS.json -d projects | tee output.log 62 | grep "# pkgs: 185 would save: 1.61MB" output.log 63 | ${PKGLINK_EXEC} -vr REFS.json -g projects | tee output.log 64 | grep "# pkgs: 185 would save: 1.61MB" output.log 65 | ${PKGLINK_EXEC} -vr REFS.json projects | tee output.log 66 | grep "pkgs: 185 saved: 1.61MB" output.log 67 | fi 68 | 69 | cd .. 70 | npx cross-env BABEL_ENV=test mocha --require @babel/register src/cli.compare-foo.mocha.man.js 71 | cd - 72 | 73 | # REFS should contain foo2, delete foo2, prune, REFS no foo2 74 | grep "foo2" REFS.json 75 | rimraf projects/foo2/node_modules 76 | ${PKGLINK_EXEC} -vpr REFS.json -p | tee output.log 77 | grep "updated REFS.json" output.log 78 | grep -L "foo2" REFS.json | grep REFS.json 79 | 80 | rimraf projects/foo2 81 | rimraf projects/foo3 82 | rimraf projects/bar1 83 | rimraf projects/cat1 84 | rimraf REFS.json 85 | rimraf output.log 86 | -------------------------------------------------------------------------------- /src/link-filter.mocha.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-extraneous-dependencies */ 2 | import expect from 'expect'; 3 | import R from 'ramda'; 4 | import linkFilter from './link-filter'; 5 | 6 | const config = { 7 | minFileSize: 0 8 | }; 9 | 10 | const dstPackInode = 1; // package.json inode 11 | 12 | const masterEI = { 13 | stat: { 14 | ino: 100, 15 | dev: 'abc', 16 | size: 123, 17 | mtime: new Date() 18 | } 19 | }; 20 | 21 | const match1 = { 22 | stat: { 23 | ino: 101, 24 | dev: 'abc', 25 | size: 123, 26 | mtime: masterEI.stat.mtime 27 | } 28 | }; 29 | 30 | const linked1 = { 31 | stat: { 32 | ...masterEI.stat 33 | } 34 | }; 35 | 36 | describe('link-filter', () => { 37 | describe('missing targets', () => { 38 | it('should exclude', () => { 39 | const x = { srcEI: masterEI }; 40 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 41 | }); 42 | }); 43 | 44 | describe('non-package.json', () => { 45 | it('should exclude', () => { 46 | const x = R.set(R.lensPath(['dstEI', 'stat', 'ino']), dstPackInode, {}); 47 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 48 | }); 49 | }); 50 | 51 | describe('dstInode same as master', () => { 52 | it('should exclude', () => { 53 | const x = R.compose( 54 | R.assoc('srcEI', masterEI), 55 | R.assoc('dstEI', linked1) 56 | )({}); 57 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 58 | }); 59 | }); 60 | 61 | describe('different device, same inode', () => { 62 | it('should exclude', () => { 63 | const x = R.compose( 64 | R.assoc('srcEI', masterEI), 65 | R.assoc('dstEI', R.assocPath(['stat', 'dev'], 'def', linked1)) 66 | )({}); 67 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 68 | }); 69 | }); 70 | 71 | describe('different size', () => { 72 | it('should exclude', () => { 73 | const x = R.compose( 74 | R.assoc('srcEI', masterEI), 75 | R.assoc('dstEI', R.assocPath(['stat', 'size'], 999, match1)) 76 | )({}); 77 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 78 | }); 79 | }); 80 | 81 | describe('different mtime', () => { 82 | it('should exclude', () => { 83 | const x = R.compose( 84 | R.assoc('srcEI', masterEI), 85 | R.assoc('dstEI', R.assocPath(['stat', 'mtime'], new Date(100), match1)) 86 | )({}); 87 | expect(linkFilter(config, dstPackInode, x)).toNotExist(); 88 | }); 89 | }); 90 | 91 | describe('different size below config.minFileSize', () => { 92 | it('should exclude', () => { 93 | const config2 = { minFileSize: 10000 }; 94 | const x = R.compose( 95 | R.assoc('srcEI', masterEI), 96 | R.assoc('dstEI', match1) 97 | )({}); 98 | expect(linkFilter(config2, dstPackInode, x)).toNotExist(); 99 | }); 100 | }); 101 | 102 | describe('matching files not already linked', () => { 103 | it('should include', () => { 104 | const x = R.compose( 105 | R.assoc('srcEI', masterEI), 106 | R.assoc('dstEI', match1) 107 | )({}); 108 | expect(linkFilter(config, dstPackInode, x)).toExist(); 109 | }); 110 | }); 111 | }); 112 | -------------------------------------------------------------------------------- /src/find-packages.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra-promise'; 2 | import Path from 'path'; 3 | import R from 'ramda'; 4 | import readdirp from 'readdirp'; 5 | import { Observable } from 'rxjs'; 6 | import { formatDevNameVersion } from './util/format'; 7 | 8 | // unix or windows paths 9 | const ENDS_NODE_MOD_RE = /[\\/]node_modules$/; 10 | 11 | /* 12 | Special directory tree filter for finding node_module/X packages 13 | - no dirs starting with '.' 14 | - accept node_modules 15 | - if under ancestor of node_modules 16 | - allow if parent is node_modules (keep in node_modules/X tree) 17 | - otherwise allow (not yet found node_modules tree) 18 | */ 19 | function filterDirsNodeModPacks(ei) { 20 | const eiName = ei.name; 21 | if (eiName.charAt(0) === '.') { 22 | return false; 23 | } // no dot dirs 24 | if (eiName === 'node_modules') { 25 | return true; 26 | } // node_modules 27 | const eiFullParentDir = ei.fullParentDir; 28 | if (eiFullParentDir.indexOf('node_modules') !== -1) { 29 | // under node_modules 30 | // only if grand parent is node_modules will we continue down 31 | return Path.basename(eiFullParentDir) === 'node_modules'; 32 | } 33 | return true; // not in node_modules yet, so keep walking 34 | } 35 | 36 | export default function findPackages(config, rtenv, rootDirs, logUpdate) { 37 | // ret obs of eiDN 38 | return ( 39 | Observable.from(rootDirs) 40 | // find all package.json files 41 | .mergeMap((startDir) => { 42 | const readdirpOptions = { 43 | root: startDir, 44 | entryType: 'files', 45 | lstat: true, // want actual files not symlinked 46 | fileFilter: ['package.json'], 47 | directoryFilter: filterDirsNodeModPacks 48 | }; 49 | if (config.treeDepth) { 50 | readdirpOptions.depth = config.treeDepth; 51 | } 52 | const fstream = readdirp(readdirpOptions); 53 | rtenv.cancelled$.subscribe(() => fstream.destroy()); // stop reading 54 | return Observable.fromEvent(fstream, 'data') 55 | .takeWhile(() => !rtenv.cancelled) 56 | .takeUntil(Observable.fromEvent(fstream, 'close')) 57 | .takeUntil(Observable.fromEvent(fstream, 'end')); 58 | }, config.concurrentOps) 59 | // only parents ending in node_modules 60 | .filter((ei) => ENDS_NODE_MOD_RE.test(Path.dirname(ei.fullParentDir))) 61 | // get name and version from package.json 62 | .mergeMap( 63 | (ei) => 64 | Observable.from(fs.readJsonAsync(ei.fullPath, { throws: false })), 65 | (ei, pack) => ({ 66 | // returns eiDN 67 | entryInfo: truncEI(ei), 68 | devNameVer: 69 | pack && pack.name && pack.version 70 | ? formatDevNameVersion(ei.stat.dev, pack.name, pack.version) 71 | : null 72 | }), 73 | config.concurrentOps 74 | ) 75 | .filter((obj) => obj.devNameVer) // has name and version, not null 76 | .do((obj) => { 77 | rtenv.packageCount += 1; 78 | rtenv.currentPackageDir = obj.entryInfo.fullParentDir; 79 | }) 80 | .do((obj) => { 81 | logUpdate(); 82 | }) 83 | ); 84 | } 85 | 86 | /* 87 | Truncate entryInfo to just fullParentDir and stat to save memory 88 | */ 89 | function truncEI(ei) { 90 | return R.pick(['fullParentDir', 'stat'], ei); 91 | } 92 | -------------------------------------------------------------------------------- /src/cli-options.js: -------------------------------------------------------------------------------- 1 | import chalk from 'chalk'; 2 | import Joi from '@hapi/joi'; 3 | import minimist from 'minimist'; 4 | import OS from 'os'; 5 | import Path from 'path'; 6 | import R from 'ramda'; 7 | import { safeJsonReadSync } from './util/file'; 8 | import { DEFAULT_CONFIG_FILE, DEFAULT_REFS_FILE } from './constants'; 9 | 10 | const minimistOpts = { 11 | boolean: ['d', 'g', 'h', 'p', 'v'], 12 | string: ['c', 'm', 'r'], 13 | alias: { 14 | c: 'config', 15 | d: 'dryrun', 16 | g: 'gen-ln-cmds', 17 | h: 'help', 18 | m: 'memory', 19 | p: 'prune', 20 | r: 'refs-file', 21 | t: 'tree-depth', 22 | v: 'verbose' 23 | } 24 | }; 25 | 26 | export const argvSchema = Joi.object({ 27 | config: Joi.string(), 28 | 'refs-file': Joi.string(), 29 | 'tree-depth': Joi.number().integer().min(0) 30 | }).unknown(true); 31 | 32 | export const configSchema = Joi.object({ 33 | refsFile: Joi.string().default(Path.resolve(OS.homedir(), DEFAULT_REFS_FILE)), 34 | concurrentOps: Joi.number().integer().min(1).default(4), 35 | // windows does not maintain original modtimes for installs 36 | // so ignoreModTime is defaulted to true for win32 37 | ignoreModTime: Joi.boolean().default(OS.platform() === 'win32'), 38 | memory: Joi.number().integer().min(100).default(2560), // MB 39 | minFileSize: Joi.number().integer().min(0).default(0), // bytes 40 | treeDepth: Joi.number().integer().min(0).default(0), 41 | refSize: Joi.number().integer().min(1).default(5), 42 | consoleWidth: Joi.number().integer().min(30).default(70) 43 | }); 44 | 45 | export function gatherOptions(processArgv, displayHelp) { 46 | // processArgv is already sliced, process.argv.slice(2) 47 | const unvalidArgv = minimist(processArgv, minimistOpts); 48 | const argvVResult = argvSchema.validate(unvalidArgv); 49 | if (argvVResult.error) { 50 | if (displayHelp) { 51 | displayHelp(); 52 | } 53 | console.error(''); 54 | console.error(chalk.red('error: invalid argument specified')); 55 | argvVResult.error.details.forEach((err) => { 56 | console.error(err.message); 57 | }); 58 | process.exit(20); 59 | } 60 | const argv = argvVResult.value; // possibly updated by schema 61 | return argv; 62 | } 63 | 64 | export function gatherConfig(argv, unvalidatedConfig, configPath) { 65 | const configResult = configSchema.validate(unvalidatedConfig, { 66 | abortEarly: false 67 | }); 68 | if (configResult.error) { 69 | console.error(chalk.red('error: invalid JSON configuration')); 70 | console.error(`${chalk.bold('config file:')} ${configPath}`); 71 | configResult.error.details.forEach((err) => { 72 | console.error(err.message); 73 | }); 74 | process.exit(22); 75 | } 76 | const config = configResult.value; // with defaults applied 77 | R.toPairs({ 78 | // for these defined argv values override config 79 | dryrun: argv.dryrun, 80 | genLnCmds: argv['gen-ln-cmds'], 81 | memory: argv.memory, 82 | refsFile: argv['refs-file'], 83 | treeDepth: argv['tree-depth'] 84 | }).forEach((p) => { 85 | const k = p[0]; 86 | const v = p[1]; 87 | if (!R.isNil(v)) { 88 | // if defined, use it 89 | config[k] = v; 90 | } 91 | }); 92 | // define how much room is left for displaying paths 93 | config.extraCols = config.consoleWidth - 30; 94 | return config; 95 | } 96 | 97 | export function gatherOptionsConfig(processArgv, displayHelp) { 98 | const argv = gatherOptions(processArgv, displayHelp); 99 | 100 | const CONFIG_PATH = 101 | argv.config || Path.resolve(OS.homedir(), DEFAULT_CONFIG_FILE); 102 | 103 | const parsedConfigJson = safeJsonReadSync(CONFIG_PATH); 104 | if (parsedConfigJson instanceof Error) { 105 | console.error(chalk.red('error: invalid JSON configuration')); 106 | console.error(`${chalk.bold('config file:')} ${CONFIG_PATH}`); 107 | console.error(parsedConfigJson); // error 108 | process.exit(21); 109 | } 110 | const unvalidatedConfig = parsedConfigJson || {}; 111 | 112 | const config = gatherConfig(argv, unvalidatedConfig, CONFIG_PATH); 113 | 114 | return { 115 | argv, 116 | config 117 | }; 118 | } 119 | -------------------------------------------------------------------------------- /src/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import chalk from 'chalk'; 3 | import fs from 'fs-extra-promise'; 4 | import numeral from 'numeral'; 5 | import OS from 'os'; 6 | import Path from 'path'; 7 | import R from 'ramda'; 8 | import { Observable, ReplaySubject, Subject } from 'rxjs'; 9 | import SingleLineLog from 'single-line-log'; 10 | import stripAnsi from 'strip-ansi'; 11 | import { formatBytes, sortObjKeys } from './util/format'; 12 | import { outputFileStderrSync } from './util/file'; 13 | import defaultRTEnv from './run-env-defaults'; 14 | import { prune, scanAndLink } from './index'; 15 | import managed from './util/managed'; 16 | import { gatherOptionsConfig } from './cli-options'; 17 | 18 | const isTTY = process.stdout.isTTY; // truthy if in terminal 19 | const singleLineLog = SingleLineLog.stderr; 20 | 21 | const rtenv = { 22 | // create our copy 23 | ...defaultRTEnv 24 | }; 25 | 26 | const { argv, config } = gatherOptionsConfig( 27 | process.argv.slice(2), 28 | displayHelp 29 | ); 30 | 31 | // should we be using terminal output 32 | const isTermOut = isTTY && !argv['gen-ln-cmds']; 33 | 34 | if (argv.help || (!argv._.length && !argv.prune)) { 35 | // display help 36 | displayHelp(); 37 | process.exit(23); 38 | } 39 | 40 | function displayHelp() { 41 | outputFileStderrSync(Path.join(__dirname, '..', 'usage.txt')); 42 | } 43 | 44 | fs.ensureFileSync(config.refsFile); 45 | 46 | const startingDirs = argv._.map((x) => Path.resolve(x)); 47 | 48 | // key=nameVersion value: array of ref tuples [modPath, packJsonInode, packJsonMTimeEpoch] 49 | rtenv.existingPackRefs = 50 | fs.readJsonSync(config.refsFile, { throws: false }) || {}; 51 | 52 | rtenv.cancelled$ = new ReplaySubject(1); 53 | 54 | const singleLineLog$ = new Subject(); 55 | singleLineLog$ 56 | .filter((x) => isTermOut) // only if in terminal 57 | .distinctUntilChanged() 58 | .throttleTime(100) 59 | .takeUntil(rtenv.cancelled$) 60 | .subscribe({ 61 | next: (x) => singleLineLog(x), 62 | complete: () => { 63 | singleLineLog(''); 64 | singleLineLog.clear(); 65 | } 66 | }); 67 | const log = singleLineLog$.next.bind(singleLineLog$); 68 | log.clear = () => { 69 | if (isTermOut) { 70 | singleLineLog(''); 71 | singleLineLog.clear(); 72 | } 73 | }; 74 | rtenv.log = log; // share this logger in the rtenv 75 | 76 | function out(str) { 77 | const s = isTermOut ? str : stripAnsi(str); 78 | process.stdout.write(s); 79 | process.stdout.write(OS.EOL); 80 | } 81 | rtenv.out = out; // share this output fn in the rtenv 82 | 83 | const cancel = R.once(() => { 84 | rtenv.cancelled = true; 85 | rtenv.cancelled$.next(true); 86 | console.error('cancelling...'); 87 | }); 88 | const finalTasks = R.once(() => { 89 | singleLineLog$.complete(); 90 | if (argv.dryrun || argv['gen-ln-cmds']) { 91 | out( 92 | `# ${chalk.blue('pkgs:')} ${numeral(rtenv.packageCount).format( 93 | '0,0' 94 | )} ${chalk.yellow('would save:')} ${chalk.bold( 95 | formatBytes(rtenv.savedByteCount) 96 | )}` 97 | ); 98 | managed.shutdown(); 99 | return; 100 | } 101 | if (argv.prune || Object.keys(rtenv.updatedPackRefs).length) { 102 | const sortedExistingPackRefs = sortObjKeys( 103 | R.merge(rtenv.existingPackRefs, rtenv.updatedPackRefs) 104 | ); 105 | fs.outputJsonSync(config.refsFile, sortedExistingPackRefs); 106 | // if pruned or if no savings, at least let them know refs updated 107 | if (argv.prune || !rtenv.savedByteCount) { 108 | out(`updated ${config.refsFile}`); 109 | } 110 | } 111 | out( 112 | `${chalk.blue('pkgs:')} ${numeral(rtenv.packageCount).format( 113 | '0,0' 114 | )} ${chalk.green('saved:')} ${chalk.bold( 115 | formatBytes(rtenv.savedByteCount) 116 | )}` 117 | ); 118 | managed.shutdown(); 119 | }); 120 | 121 | managed.onInterrupt(cancel); // fires on SIGINT 122 | process.once('SIGTERM', cancel).once('EXIT', finalTasks); 123 | 124 | if (argv.verbose) { 125 | console.log('argv', argv); 126 | console.log('config', config); 127 | } 128 | 129 | out(''); // advance to full line 130 | 131 | // Main program start, create task$ and run 132 | const arrTaskObs = []; 133 | if (argv.prune) { 134 | arrTaskObs.push( 135 | Observable.defer(() => { 136 | log(`${chalk.bold('pruning...')}`); 137 | return prune(config, rtenv.existingPackRefs); 138 | }).do((newShares) => { 139 | rtenv.existingPackRefs = newShares; 140 | }) 141 | ); 142 | } 143 | if (startingDirs.length) { 144 | arrTaskObs.push( 145 | Observable.defer(() => scanAndLink(config, rtenv, startingDirs)) 146 | ); 147 | } 148 | 149 | // run all the task observables serially 150 | if (arrTaskObs.length) { 151 | Observable.concat(...arrTaskObs).subscribe({ 152 | error: (err) => console.error(err), 153 | complete: () => finalTasks() 154 | }); 155 | } 156 | -------------------------------------------------------------------------------- /src/link.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra-promise'; 2 | import Path from 'path'; 3 | import readdirp from 'readdirp'; 4 | import { Observable } from 'rxjs'; 5 | import { buildPackRef } from './pack-ref'; 6 | import linkFilter from './link-filter'; 7 | import { createLogUpdate } from './util/log'; 8 | 9 | /* 10 | Default hard link function which unlinks orig dst then creates link 11 | If failed to link (maybe fs doesn't support), recopy from src 12 | @return promise that resolves on success or rejects on failure 13 | */ 14 | function hardLink(src, dst) { 15 | return fs 16 | .unlinkAsync(dst) 17 | .then(() => fs.linkAsync(src, dst)) 18 | .catch((err) => { 19 | fs.copyAsync(src, dst, { 20 | clobber: false, 21 | preserveTimestamps: true 22 | }) 23 | .then(() => { 24 | console.error('INFO: recopied %s to %s to cleanup from link error which follows', src, dst); 25 | }) 26 | .catch((/* err */) => { 27 | console.error('ERROR: was not able to restore %s after link error that follows, reinstall package', dst); 28 | }); 29 | throw err; // rethrow original err 30 | }); 31 | } 32 | 33 | export function genModuleLinks(config, rtenv, lnkModSrcDst) { 34 | // returns observable 35 | return ( 36 | determineLinks(config, rtenv, lnkModSrcDst, true) 37 | // just output the ln commands 38 | .do(([src, dst, size]) => { 39 | rtenv.out(`ln -f "${src}" "${dst}"`); 40 | }) 41 | ); 42 | } 43 | 44 | export function handleModuleLinking(config, rtenv, lnkModSrcDst) { 45 | // returns observable 46 | return determineLinks(config, rtenv, lnkModSrcDst, true).mergeMap( 47 | (s_d_sz) => performLink(config, rtenv, s_d_sz), 48 | (s_d_sz, ops) => s_d_sz, 49 | config.concurrentOps 50 | ); 51 | } 52 | 53 | export function determineLinks(config, rtenv, lnkModSrcDst, updatePackRefs = false) { 54 | // returns observable of s_d_sz [srcFullPath, dstFullPath, size] 55 | 56 | const logUpdate = createLogUpdate(config, rtenv); 57 | 58 | // src is the master we link from, dst is the dst link 59 | const devNameVer = lnkModSrcDst.devNameVer; // device:nameVersion 60 | const srcRoot = lnkModSrcDst.src; 61 | const srcPackInode = lnkModSrcDst.srcPackInode; 62 | const srcPackMTimeEpoch = lnkModSrcDst.srcPackMTimeEpoch; 63 | const dstRoot = lnkModSrcDst.dst; 64 | const dstPackInode = lnkModSrcDst.dstPackInode; 65 | const dstPackMTimeEpoch = lnkModSrcDst.dstPackMTimeEpoch; 66 | 67 | if (updatePackRefs) { 68 | let packRefs = rtenv.updatedPackRefs[devNameVer] || []; 69 | if (!packRefs.length) { 70 | packRefs.push(buildPackRef(srcRoot, srcPackInode, srcPackMTimeEpoch)); 71 | } 72 | packRefs = packRefs.filter((packRef) => packRef[0] !== dstRoot); 73 | if (packRefs.length < config.refSize) { 74 | packRefs.push(buildPackRef(dstRoot, dstPackInode, dstPackMTimeEpoch)); 75 | } 76 | rtenv.updatedPackRefs[devNameVer] = packRefs; 77 | } 78 | 79 | const fstream = readdirp({ 80 | root: lnkModSrcDst.src, 81 | entryType: 'files', 82 | lstat: true, // want actual files not symlinked 83 | fileFilter: ['!.*'], 84 | directoryFilter: ['!.*', '!node_modules'] 85 | }); 86 | fstream.once('end', () => { 87 | rtenv.completedPackages += 1; 88 | logUpdate(); 89 | }); 90 | rtenv.cancelled$.subscribe(() => fstream.destroy()); // stop reading 91 | 92 | return ( 93 | Observable.fromEvent(fstream, 'data') 94 | .takeWhile(() => !rtenv.cancelled) 95 | .takeUntil(Observable.fromEvent(fstream, 'close')) 96 | .takeUntil(Observable.fromEvent(fstream, 'end')) 97 | // combine with stat for dst 98 | .mergeMap( 99 | (srcEI) => { 100 | const dstPath = Path.resolve(dstRoot, srcEI.path); 101 | return Observable.from( 102 | fs 103 | .statAsync(dstPath) 104 | .then((stat) => ({ 105 | fullPath: dstPath, 106 | stat 107 | })) 108 | .catch((err) => { 109 | if (err.code !== 'ENOENT') { 110 | console.error(err); 111 | } 112 | return null; 113 | }) 114 | ); 115 | }, 116 | (srcEI, dstEI) => ({ 117 | srcEI, 118 | dstEI 119 | }), 120 | config.concurrentOps 121 | ) 122 | .filter((x) => linkFilter(config, dstPackInode, x)) 123 | .map((x) => [ 124 | // s_d_sz 125 | x.srcEI.fullPath, 126 | x.dstEI.fullPath, 127 | x.srcEI.stat.size 128 | ]) 129 | ); 130 | } 131 | 132 | function performLink(config, rtenv, [src, dst, size]) { 133 | // returns observable 134 | const link = rtenv.linkFn || hardLink; // use custom link if provided 135 | return Observable.fromPromise( 136 | link(src, dst).catch((err) => { 137 | console.error(`ERROR: failed to unlink/link src:${src} dst:${dst}`, err); 138 | throw err; 139 | }) 140 | ); 141 | } 142 | -------------------------------------------------------------------------------- /src/pack-ref.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs-extra-promise'; 2 | import Path from 'path'; 3 | import Prom from 'bluebird'; 4 | import R from 'ramda'; 5 | import { Observable } from 'rxjs'; 6 | import { formatDevNameVersion } from './util/format'; 7 | 8 | export function prune(config, dnvPR) { 9 | // return obs of new dnvPR object 10 | return Observable.from( 11 | R.toPairs(dnvPR) // [dnv, arrPackRef] 12 | ) 13 | .mergeMap((dnv_PR) => verifyDMP(dnv_PR, config)) 14 | .reduce((acc, dnv_PR) => R.append(dnv_PR, acc), []) 15 | .map((flatDMR) => R.fromPairs(flatDMR)); 16 | } 17 | 18 | export function verifyDMP([dnv, arrPackRef], config) { 19 | // return obs of valid dnv_PR 20 | const { concurrentOps } = config; 21 | return ( 22 | Observable.from(arrPackRef) // obs of packRefs 23 | // returns obs of valid packRef 24 | .mergeMap((packRef) => verifyPackRef(dnv, packRef, false), concurrentOps) 25 | .reduce((acc, packRef) => R.append(packRef, acc), []) 26 | .filter((arrRefEI) => arrRefEI.length) 27 | .map((arrRefEI) => [dnv, arrRefEI]) 28 | ); // dnv_PR 29 | } 30 | 31 | export function buildPackRef(modFullPath, packageJsonInode, packageJsonMTimeEpoch) { 32 | return [modFullPath, packageJsonInode, packageJsonMTimeEpoch]; 33 | } 34 | 35 | export function verifyPackRef(dnv, packRef, returnEI = false) { 36 | // return obs of valid packRef 37 | const modDir = packRef[0]; 38 | const packInode = packRef[1]; 39 | const packMTimeEpoch = packRef[2]; 40 | const packPath = Path.join(modDir, 'package.json'); 41 | let packStat; 42 | return Observable.from( 43 | fs 44 | .statAsync(packPath) 45 | .then((stat) => { 46 | // eslint-disable-line consistent-return 47 | if (stat && stat.ino === packInode && stat.mtime.getTime() === packMTimeEpoch) { 48 | packStat = stat; // save for later use 49 | return fs.readJsonAsync(packPath, { throws: false }); 50 | } 51 | return undefined; 52 | }) 53 | // if json and matches, return packRef or EI 54 | .then((json) => { 55 | // eslint-disable-line consistent-return 56 | if (json) { 57 | const devNameVer = formatDevNameVersion(packStat.dev, json.name, json.version); 58 | if (devNameVer === dnv) { 59 | return returnEI 60 | ? { 61 | // masterEI 62 | stat: packStat, 63 | fullParentDir: modDir 64 | } 65 | : packRef; 66 | } 67 | } 68 | return undefined; 69 | }) 70 | .catch((err) => { 71 | if (err.code !== 'ENOENT') { 72 | console.error(err); 73 | } 74 | }) 75 | ).filter((x) => x); // filter any undefineds, those were invalid 76 | } 77 | 78 | const masterEICache = {}; 79 | 80 | function checkMasterCache(config, rtenv, dnv, packEI) { 81 | // ret obs of masterEI 82 | const masterEI = masterEICache[dnv]; 83 | if (masterEI) { 84 | if (!masterEI.then) { 85 | // it is not a promise 86 | return Observable.of(masterEI); 87 | } 88 | // otherwise it was a promise 89 | return Observable.fromPromise(masterEI); 90 | } 91 | // otherwise not found 92 | const masterEIProm = findExistingMaster(config, rtenv, dnv, packEI); 93 | masterEICache[dnv] = masterEIProm; 94 | // optimize future requests so they don't need to hit promise 95 | masterEIProm.then((masterEI) => { 96 | masterEICache[dnv] = masterEI; // eliminate promise overhead 97 | }); 98 | return Observable.fromPromise(masterEIProm); 99 | } 100 | 101 | export function determinePackLinkSrcDst(config, rtenv, destEIdn) { 102 | // ret obs of srcDstObj 103 | if (rtenv.cancelled) { 104 | return Observable.empty(); 105 | } 106 | const { entryInfo: dstEI, devNameVer: dnv } = destEIdn; 107 | 108 | return checkMasterCache(config, rtenv, dnv, dstEI) 109 | .takeWhile(() => !rtenv.cancelled) 110 | .filter((masterEI) => !isEISameInode(masterEI, dstEI)) 111 | .map((masterEI) => ({ 112 | devNameVer: dnv, // device:nameVersion 113 | src: masterEI.fullParentDir, 114 | srcPackInode: masterEI.stat.ino, 115 | srcPackMTimeEpoch: masterEI.stat.mtime.getTime(), 116 | dst: dstEI.fullParentDir, 117 | dstPackInode: dstEI.stat.ino, 118 | dstPackMTimeEpoch: dstEI.stat.mtime.getTime() 119 | })); 120 | } 121 | 122 | function isEISameInode(firstEI, secondEI) { 123 | return firstEI.stat.dev === secondEI.stat.dev && firstEI.stat.ino === secondEI.stat.ino; 124 | } 125 | 126 | // prepare for this to be async 127 | function getExistingPackRefs(config, rtenv, dnv) { 128 | // returns observable to arrPackRefs 129 | // check rtenv.existingPackRefs[dnv] for ref tuples 130 | const masterPackRefs = R.pathOr([], [dnv], rtenv.existingPackRefs); // array of [modDir, packInode, packMTimeEpoch] packRef tuples 131 | return Observable.of(masterPackRefs); 132 | } 133 | 134 | function findExistingMaster(config, rtenv, dnv, ei) { 135 | // returns promise resolving to masterEI 136 | /* 137 | we will be checking through the rtenv.existingPackRefs[dnv] packRefs 138 | to see if any are still valid. Resolve with the first one that is 139 | still valid, also returning the remaining packRefs. Not all of the 140 | packRefs will have been checked, just enough to find one valid one. 141 | A new array of refs will be updated in rtenv.updatedPackRefs 142 | which will omit any found to be invalid. 143 | Resolves with masterEI or uses ei provided 144 | */ 145 | return getExistingPackRefs(config, rtenv, dnv) 146 | .mergeMap((masterPackRefs) => { 147 | if (!masterPackRefs.length) { 148 | // no valid found, set to empty [] 149 | rtenv.updatedPackRefs[dnv] = [buildPackRef(ei.fullParentDir, ei.stat.ino, ei.stat.mtime.getTime())]; 150 | return Observable.of(ei); 151 | } 152 | // otherwise we have packrefs check them 153 | return Observable.from(masterPackRefs) 154 | .mergeMap( 155 | (packRef) => verifyPackRef(dnv, packRef, true), 156 | 1 // one at a time since only need first 157 | ) 158 | .first( 159 | (masterEI) => masterEI, // exists 160 | (masterEI, idx) => [masterEI, idx], 161 | false 162 | ) 163 | .map((masterEI_idx) => { 164 | if (!masterEI_idx) { 165 | // no valid found, set to empty [] 166 | rtenv.updatedPackRefs[dnv] = [buildPackRef(ei.fullParentDir, ei.stat.ino, ei.stat.mtime.getTime())]; 167 | return ei; 168 | } 169 | const idx = masterEI_idx[1]; 170 | // wasn't first one so needs slicing 171 | rtenv.updatedPackRefs[dnv] = masterPackRefs.slice(idx); 172 | const masterEI = masterEI_idx[0]; 173 | return masterEI; 174 | }); 175 | }) 176 | .toPromise(Prom); 177 | } 178 | -------------------------------------------------------------------------------- /fixtures/projects/foo1/npm-shrinkwrap.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "foo1", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "define-properties": { 8 | "version": "1.1.3", 9 | "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", 10 | "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", 11 | "dev": true, 12 | "requires": { 13 | "object-keys": "^1.0.12" 14 | } 15 | }, 16 | "es-abstract": { 17 | "version": "1.17.5", 18 | "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", 19 | "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", 20 | "dev": true, 21 | "requires": { 22 | "es-to-primitive": "^1.2.1", 23 | "function-bind": "^1.1.1", 24 | "has": "^1.0.3", 25 | "has-symbols": "^1.0.1", 26 | "is-callable": "^1.1.5", 27 | "is-regex": "^1.0.5", 28 | "object-inspect": "^1.7.0", 29 | "object-keys": "^1.1.1", 30 | "object.assign": "^4.1.0", 31 | "string.prototype.trimleft": "^2.1.1", 32 | "string.prototype.trimright": "^2.1.1" 33 | } 34 | }, 35 | "es-get-iterator": { 36 | "version": "1.1.0", 37 | "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.0.tgz", 38 | "integrity": "sha512-UfrmHuWQlNMTs35e1ypnvikg6jCz3SK8v8ImvmDsh36fCVUR1MqoFDiyn0/k52C8NqO3YsO8Oe0azeesNuqSsQ==", 39 | "dev": true, 40 | "requires": { 41 | "es-abstract": "^1.17.4", 42 | "has-symbols": "^1.0.1", 43 | "is-arguments": "^1.0.4", 44 | "is-map": "^2.0.1", 45 | "is-set": "^2.0.1", 46 | "is-string": "^1.0.5", 47 | "isarray": "^2.0.5" 48 | } 49 | }, 50 | "es-to-primitive": { 51 | "version": "1.2.1", 52 | "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", 53 | "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", 54 | "dev": true, 55 | "requires": { 56 | "is-callable": "^1.1.4", 57 | "is-date-object": "^1.0.1", 58 | "is-symbol": "^1.0.2" 59 | } 60 | }, 61 | "expect": { 62 | "version": "1.20.2", 63 | "resolved": "https://registry.npmjs.org/expect/-/expect-1.20.2.tgz", 64 | "integrity": "sha1-1Fj+TFYAQDa64yMkFqP2Nh8E+WU=", 65 | "dev": true, 66 | "requires": { 67 | "define-properties": "~1.1.2", 68 | "has": "^1.0.1", 69 | "is-equal": "^1.5.1", 70 | "is-regex": "^1.0.3", 71 | "object-inspect": "^1.1.0", 72 | "object-keys": "^1.0.9", 73 | "tmatch": "^2.0.1" 74 | } 75 | }, 76 | "function-bind": { 77 | "version": "1.1.1", 78 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", 79 | "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", 80 | "dev": true 81 | }, 82 | "functions-have-names": { 83 | "version": "1.2.1", 84 | "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.1.tgz", 85 | "integrity": "sha512-j48B/ZI7VKs3sgeI2cZp7WXWmZXu7Iq5pl5/vptV5N2mq+DGFuS/ulaDjtaoLpYzuD6u8UgrUKHfgo7fDTSiBA==", 86 | "dev": true 87 | }, 88 | "has": { 89 | "version": "1.0.3", 90 | "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", 91 | "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", 92 | "dev": true, 93 | "requires": { 94 | "function-bind": "^1.1.1" 95 | } 96 | }, 97 | "has-symbols": { 98 | "version": "1.0.1", 99 | "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", 100 | "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", 101 | "dev": true 102 | }, 103 | "is-arguments": { 104 | "version": "1.0.4", 105 | "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", 106 | "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", 107 | "dev": true 108 | }, 109 | "is-arrow-function": { 110 | "version": "2.0.3", 111 | "resolved": "https://registry.npmjs.org/is-arrow-function/-/is-arrow-function-2.0.3.tgz", 112 | "integrity": "sha1-Kb4sLY2UUIUri7r7Y1unuNjofsI=", 113 | "dev": true, 114 | "requires": { 115 | "is-callable": "^1.0.4" 116 | } 117 | }, 118 | "is-bigint": { 119 | "version": "1.0.0", 120 | "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.0.tgz", 121 | "integrity": "sha512-t5mGUXC/xRheCK431ylNiSkGGpBp8bHENBcENTkDT6ppwPzEVxNGZRvgvmOEfbWkFhA7D2GEuE2mmQTr78sl2g==", 122 | "dev": true 123 | }, 124 | "is-boolean-object": { 125 | "version": "1.0.1", 126 | "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.0.1.tgz", 127 | "integrity": "sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ==", 128 | "dev": true 129 | }, 130 | "is-callable": { 131 | "version": "1.1.5", 132 | "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", 133 | "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", 134 | "dev": true 135 | }, 136 | "is-date-object": { 137 | "version": "1.0.2", 138 | "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", 139 | "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", 140 | "dev": true 141 | }, 142 | "is-equal": { 143 | "version": "1.6.1", 144 | "resolved": "https://registry.npmjs.org/is-equal/-/is-equal-1.6.1.tgz", 145 | "integrity": "sha512-3/79QTolnfNFrxQAvqH8M+O01uGWsVq54BUPG2mXQH7zi4BE/0TY+fmA444t8xSBvIwyNMvsTmCZ5ViVDlqPJg==", 146 | "dev": true, 147 | "requires": { 148 | "es-get-iterator": "^1.0.1", 149 | "functions-have-names": "^1.2.0", 150 | "has": "^1.0.3", 151 | "is-arrow-function": "^2.0.3", 152 | "is-bigint": "^1.0.0", 153 | "is-boolean-object": "^1.0.0", 154 | "is-callable": "^1.1.4", 155 | "is-date-object": "^1.0.1", 156 | "is-generator-function": "^1.0.7", 157 | "is-number-object": "^1.0.3", 158 | "is-regex": "^1.0.4", 159 | "is-string": "^1.0.4", 160 | "is-symbol": "^1.0.3", 161 | "isarray": "^2.0.5", 162 | "object-inspect": "^1.7.0", 163 | "object.entries": "^1.1.0", 164 | "which-boxed-primitive": "^1.0.1", 165 | "which-collection": "^1.0.0" 166 | } 167 | }, 168 | "is-generator-function": { 169 | "version": "1.0.7", 170 | "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", 171 | "integrity": "sha512-YZc5EwyO4f2kWCax7oegfuSr9mFz1ZvieNYBEjmukLxgXfBUbxAWGVF7GZf0zidYtoBl3WvC07YK0wT76a+Rtw==", 172 | "dev": true 173 | }, 174 | "is-map": { 175 | "version": "2.0.1", 176 | "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.1.tgz", 177 | "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==", 178 | "dev": true 179 | }, 180 | "is-number-object": { 181 | "version": "1.0.4", 182 | "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", 183 | "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==", 184 | "dev": true 185 | }, 186 | "is-regex": { 187 | "version": "1.0.5", 188 | "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", 189 | "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", 190 | "dev": true, 191 | "requires": { 192 | "has": "^1.0.3" 193 | } 194 | }, 195 | "is-set": { 196 | "version": "2.0.1", 197 | "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.1.tgz", 198 | "integrity": "sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA==", 199 | "dev": true 200 | }, 201 | "is-string": { 202 | "version": "1.0.5", 203 | "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", 204 | "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", 205 | "dev": true 206 | }, 207 | "is-symbol": { 208 | "version": "1.0.3", 209 | "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", 210 | "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", 211 | "dev": true, 212 | "requires": { 213 | "has-symbols": "^1.0.1" 214 | } 215 | }, 216 | "is-weakmap": { 217 | "version": "2.0.1", 218 | "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", 219 | "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", 220 | "dev": true 221 | }, 222 | "is-weakset": { 223 | "version": "2.0.1", 224 | "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.1.tgz", 225 | "integrity": "sha512-pi4vhbhVHGLxohUw7PhGsueT4vRGFoXhP7+RGN0jKIv9+8PWYCQTqtADngrxOm2g46hoH0+g8uZZBzMrvVGDmw==", 226 | "dev": true 227 | }, 228 | "isarray": { 229 | "version": "2.0.5", 230 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", 231 | "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", 232 | "dev": true 233 | }, 234 | "object-inspect": { 235 | "version": "1.7.0", 236 | "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", 237 | "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", 238 | "dev": true 239 | }, 240 | "object-keys": { 241 | "version": "1.1.1", 242 | "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", 243 | "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", 244 | "dev": true 245 | }, 246 | "object.assign": { 247 | "version": "4.1.0", 248 | "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", 249 | "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", 250 | "dev": true, 251 | "requires": { 252 | "define-properties": "^1.1.2", 253 | "function-bind": "^1.1.1", 254 | "has-symbols": "^1.0.0", 255 | "object-keys": "^1.0.11" 256 | } 257 | }, 258 | "object.entries": { 259 | "version": "1.1.1", 260 | "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.1.tgz", 261 | "integrity": "sha512-ilqR7BgdyZetJutmDPfXCDffGa0/Yzl2ivVNpbx/g4UeWrCdRnFDUBrKJGLhGieRHDATnyZXWBeCb29k9CJysQ==", 262 | "dev": true, 263 | "requires": { 264 | "define-properties": "^1.1.3", 265 | "es-abstract": "^1.17.0-next.1", 266 | "function-bind": "^1.1.1", 267 | "has": "^1.0.3" 268 | } 269 | }, 270 | "string.prototype.trimend": { 271 | "version": "1.0.1", 272 | "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", 273 | "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", 274 | "dev": true, 275 | "requires": { 276 | "define-properties": "^1.1.3", 277 | "es-abstract": "^1.17.5" 278 | } 279 | }, 280 | "string.prototype.trimleft": { 281 | "version": "2.1.2", 282 | "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", 283 | "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", 284 | "dev": true, 285 | "requires": { 286 | "define-properties": "^1.1.3", 287 | "es-abstract": "^1.17.5", 288 | "string.prototype.trimstart": "^1.0.0" 289 | } 290 | }, 291 | "string.prototype.trimright": { 292 | "version": "2.1.2", 293 | "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", 294 | "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", 295 | "dev": true, 296 | "requires": { 297 | "define-properties": "^1.1.3", 298 | "es-abstract": "^1.17.5", 299 | "string.prototype.trimend": "^1.0.0" 300 | } 301 | }, 302 | "string.prototype.trimstart": { 303 | "version": "1.0.1", 304 | "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", 305 | "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", 306 | "dev": true, 307 | "requires": { 308 | "define-properties": "^1.1.3", 309 | "es-abstract": "^1.17.5" 310 | } 311 | }, 312 | "tmatch": { 313 | "version": "2.0.1", 314 | "resolved": "https://registry.npmjs.org/tmatch/-/tmatch-2.0.1.tgz", 315 | "integrity": "sha1-DFYkbzPzDaG409colauvFmYPOM8=", 316 | "dev": true 317 | }, 318 | "which-boxed-primitive": { 319 | "version": "1.0.1", 320 | "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.1.tgz", 321 | "integrity": "sha512-7BT4TwISdDGBgaemWU0N0OU7FeAEJ9Oo2P1PHRm/FCWoEi2VLWC9b6xvxAA3C/NMpxg3HXVgi0sMmGbNUbNepQ==", 322 | "dev": true, 323 | "requires": { 324 | "is-bigint": "^1.0.0", 325 | "is-boolean-object": "^1.0.0", 326 | "is-number-object": "^1.0.3", 327 | "is-string": "^1.0.4", 328 | "is-symbol": "^1.0.2" 329 | } 330 | }, 331 | "which-collection": { 332 | "version": "1.0.1", 333 | "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", 334 | "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", 335 | "dev": true, 336 | "requires": { 337 | "is-map": "^2.0.1", 338 | "is-set": "^2.0.1", 339 | "is-weakmap": "^2.0.1", 340 | "is-weakset": "^2.0.1" 341 | } 342 | } 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /fixtures/save-for-bar1/npm-shrinkwrap.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "foo1", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "define-properties": { 8 | "version": "1.1.3", 9 | "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz", 10 | "integrity": "sha512-3MqfYKj2lLzdMSf8ZIZE/V+Zuy+BgD6f164e8K2w7dgnpKArBDerGYpM46IYYcjnkdPNMjPk9A6VFB8+3SKlXQ==", 11 | "dev": true, 12 | "requires": { 13 | "object-keys": "^1.0.12" 14 | } 15 | }, 16 | "es-abstract": { 17 | "version": "1.17.5", 18 | "resolved": "https://registry.npmjs.org/es-abstract/-/es-abstract-1.17.5.tgz", 19 | "integrity": "sha512-BR9auzDbySxOcfog0tLECW8l28eRGpDpU3Dm3Hp4q/N+VtLTmyj4EUN088XZWQDW/hzj6sYRDXeOFsaAODKvpg==", 20 | "dev": true, 21 | "requires": { 22 | "es-to-primitive": "^1.2.1", 23 | "function-bind": "^1.1.1", 24 | "has": "^1.0.3", 25 | "has-symbols": "^1.0.1", 26 | "is-callable": "^1.1.5", 27 | "is-regex": "^1.0.5", 28 | "object-inspect": "^1.7.0", 29 | "object-keys": "^1.1.1", 30 | "object.assign": "^4.1.0", 31 | "string.prototype.trimleft": "^2.1.1", 32 | "string.prototype.trimright": "^2.1.1" 33 | } 34 | }, 35 | "es-get-iterator": { 36 | "version": "1.1.0", 37 | "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.0.tgz", 38 | "integrity": "sha512-UfrmHuWQlNMTs35e1ypnvikg6jCz3SK8v8ImvmDsh36fCVUR1MqoFDiyn0/k52C8NqO3YsO8Oe0azeesNuqSsQ==", 39 | "dev": true, 40 | "requires": { 41 | "es-abstract": "^1.17.4", 42 | "has-symbols": "^1.0.1", 43 | "is-arguments": "^1.0.4", 44 | "is-map": "^2.0.1", 45 | "is-set": "^2.0.1", 46 | "is-string": "^1.0.5", 47 | "isarray": "^2.0.5" 48 | } 49 | }, 50 | "es-to-primitive": { 51 | "version": "1.2.1", 52 | "resolved": "https://registry.npmjs.org/es-to-primitive/-/es-to-primitive-1.2.1.tgz", 53 | "integrity": "sha512-QCOllgZJtaUo9miYBcLChTUaHNjJF3PYs1VidD7AwiEj1kYxKeQTctLAezAOH5ZKRH0g2IgPn6KwB4IT8iRpvA==", 54 | "dev": true, 55 | "requires": { 56 | "is-callable": "^1.1.4", 57 | "is-date-object": "^1.0.1", 58 | "is-symbol": "^1.0.2" 59 | } 60 | }, 61 | "expect": { 62 | "version": "1.20.1", 63 | "resolved": "https://registry.npmjs.org/expect/-/expect-1.20.1.tgz", 64 | "integrity": "sha1-0/GEI75tBPgp2qvCQIedeGft9fM=", 65 | "dev": true, 66 | "requires": { 67 | "define-properties": "~1.1.2", 68 | "has": "^1.0.1", 69 | "is-equal": "^1.5.1", 70 | "is-regex": "^1.0.3", 71 | "object-inspect": "^1.1.0", 72 | "object-keys": "^1.0.9", 73 | "tmatch": "^2.0.1" 74 | } 75 | }, 76 | "function-bind": { 77 | "version": "1.1.1", 78 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", 79 | "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==", 80 | "dev": true 81 | }, 82 | "functions-have-names": { 83 | "version": "1.2.1", 84 | "resolved": "https://registry.npmjs.org/functions-have-names/-/functions-have-names-1.2.1.tgz", 85 | "integrity": "sha512-j48B/ZI7VKs3sgeI2cZp7WXWmZXu7Iq5pl5/vptV5N2mq+DGFuS/ulaDjtaoLpYzuD6u8UgrUKHfgo7fDTSiBA==", 86 | "dev": true 87 | }, 88 | "has": { 89 | "version": "1.0.3", 90 | "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", 91 | "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", 92 | "dev": true, 93 | "requires": { 94 | "function-bind": "^1.1.1" 95 | } 96 | }, 97 | "has-symbols": { 98 | "version": "1.0.1", 99 | "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.1.tgz", 100 | "integrity": "sha512-PLcsoqu++dmEIZB+6totNFKq/7Do+Z0u4oT0zKOJNl3lYK6vGwwu2hjHs+68OEZbTjiUE9bgOABXbP/GvrS0Kg==", 101 | "dev": true 102 | }, 103 | "is-arguments": { 104 | "version": "1.0.4", 105 | "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.0.4.tgz", 106 | "integrity": "sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==", 107 | "dev": true 108 | }, 109 | "is-arrow-function": { 110 | "version": "2.0.3", 111 | "resolved": "https://registry.npmjs.org/is-arrow-function/-/is-arrow-function-2.0.3.tgz", 112 | "integrity": "sha1-Kb4sLY2UUIUri7r7Y1unuNjofsI=", 113 | "dev": true, 114 | "requires": { 115 | "is-callable": "^1.0.4" 116 | } 117 | }, 118 | "is-bigint": { 119 | "version": "1.0.0", 120 | "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.0.tgz", 121 | "integrity": "sha512-t5mGUXC/xRheCK431ylNiSkGGpBp8bHENBcENTkDT6ppwPzEVxNGZRvgvmOEfbWkFhA7D2GEuE2mmQTr78sl2g==", 122 | "dev": true 123 | }, 124 | "is-boolean-object": { 125 | "version": "1.0.1", 126 | "resolved": "https://registry.npmjs.org/is-boolean-object/-/is-boolean-object-1.0.1.tgz", 127 | "integrity": "sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ==", 128 | "dev": true 129 | }, 130 | "is-callable": { 131 | "version": "1.1.5", 132 | "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.1.5.tgz", 133 | "integrity": "sha512-ESKv5sMCJB2jnHTWZ3O5itG+O128Hsus4K4Qh1h2/cgn2vbgnLSVqfV46AeJA9D5EeeLa9w81KUXMtn34zhX+Q==", 134 | "dev": true 135 | }, 136 | "is-date-object": { 137 | "version": "1.0.2", 138 | "resolved": "https://registry.npmjs.org/is-date-object/-/is-date-object-1.0.2.tgz", 139 | "integrity": "sha512-USlDT524woQ08aoZFzh3/Z6ch9Y/EWXEHQ/AaRN0SkKq4t2Jw2R2339tSXmwuVoY7LLlBCbOIlx2myP/L5zk0g==", 140 | "dev": true 141 | }, 142 | "is-equal": { 143 | "version": "1.6.1", 144 | "resolved": "https://registry.npmjs.org/is-equal/-/is-equal-1.6.1.tgz", 145 | "integrity": "sha512-3/79QTolnfNFrxQAvqH8M+O01uGWsVq54BUPG2mXQH7zi4BE/0TY+fmA444t8xSBvIwyNMvsTmCZ5ViVDlqPJg==", 146 | "dev": true, 147 | "requires": { 148 | "es-get-iterator": "^1.0.1", 149 | "functions-have-names": "^1.2.0", 150 | "has": "^1.0.3", 151 | "is-arrow-function": "^2.0.3", 152 | "is-bigint": "^1.0.0", 153 | "is-boolean-object": "^1.0.0", 154 | "is-callable": "^1.1.4", 155 | "is-date-object": "^1.0.1", 156 | "is-generator-function": "^1.0.7", 157 | "is-number-object": "^1.0.3", 158 | "is-regex": "^1.0.4", 159 | "is-string": "^1.0.4", 160 | "is-symbol": "^1.0.3", 161 | "isarray": "^2.0.5", 162 | "object-inspect": "^1.7.0", 163 | "object.entries": "^1.1.0", 164 | "which-boxed-primitive": "^1.0.1", 165 | "which-collection": "^1.0.0" 166 | } 167 | }, 168 | "is-generator-function": { 169 | "version": "1.0.7", 170 | "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.7.tgz", 171 | "integrity": "sha512-YZc5EwyO4f2kWCax7oegfuSr9mFz1ZvieNYBEjmukLxgXfBUbxAWGVF7GZf0zidYtoBl3WvC07YK0wT76a+Rtw==", 172 | "dev": true 173 | }, 174 | "is-map": { 175 | "version": "2.0.1", 176 | "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.1.tgz", 177 | "integrity": "sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==", 178 | "dev": true 179 | }, 180 | "is-number-object": { 181 | "version": "1.0.4", 182 | "resolved": "https://registry.npmjs.org/is-number-object/-/is-number-object-1.0.4.tgz", 183 | "integrity": "sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==", 184 | "dev": true 185 | }, 186 | "is-regex": { 187 | "version": "1.0.5", 188 | "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.0.5.tgz", 189 | "integrity": "sha512-vlKW17SNq44owv5AQR3Cq0bQPEb8+kF3UKZ2fiZNOWtztYE5i0CzCZxFDwO58qAOWtxdBRVO/V5Qin1wjCqFYQ==", 190 | "dev": true, 191 | "requires": { 192 | "has": "^1.0.3" 193 | } 194 | }, 195 | "is-set": { 196 | "version": "2.0.1", 197 | "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.1.tgz", 198 | "integrity": "sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA==", 199 | "dev": true 200 | }, 201 | "is-string": { 202 | "version": "1.0.5", 203 | "resolved": "https://registry.npmjs.org/is-string/-/is-string-1.0.5.tgz", 204 | "integrity": "sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==", 205 | "dev": true 206 | }, 207 | "is-symbol": { 208 | "version": "1.0.3", 209 | "resolved": "https://registry.npmjs.org/is-symbol/-/is-symbol-1.0.3.tgz", 210 | "integrity": "sha512-OwijhaRSgqvhm/0ZdAcXNZt9lYdKFpcRDT5ULUuYXPoT794UNOdU+gpT6Rzo7b4V2HUl/op6GqY894AZwv9faQ==", 211 | "dev": true, 212 | "requires": { 213 | "has-symbols": "^1.0.1" 214 | } 215 | }, 216 | "is-weakmap": { 217 | "version": "2.0.1", 218 | "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", 219 | "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", 220 | "dev": true 221 | }, 222 | "is-weakset": { 223 | "version": "2.0.1", 224 | "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.1.tgz", 225 | "integrity": "sha512-pi4vhbhVHGLxohUw7PhGsueT4vRGFoXhP7+RGN0jKIv9+8PWYCQTqtADngrxOm2g46hoH0+g8uZZBzMrvVGDmw==", 226 | "dev": true 227 | }, 228 | "isarray": { 229 | "version": "2.0.5", 230 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", 231 | "integrity": "sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==", 232 | "dev": true 233 | }, 234 | "object-inspect": { 235 | "version": "1.7.0", 236 | "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.7.0.tgz", 237 | "integrity": "sha512-a7pEHdh1xKIAgTySUGgLMx/xwDZskN1Ud6egYYN3EdRW4ZMPNEDUTF+hwy2LUC+Bl+SyLXANnwz/jyh/qutKUw==", 238 | "dev": true 239 | }, 240 | "object-keys": { 241 | "version": "1.1.1", 242 | "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", 243 | "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", 244 | "dev": true 245 | }, 246 | "object.assign": { 247 | "version": "4.1.0", 248 | "resolved": "https://registry.npmjs.org/object.assign/-/object.assign-4.1.0.tgz", 249 | "integrity": "sha512-exHJeq6kBKj58mqGyTQ9DFvrZC/eR6OwxzoM9YRoGBqrXYonaFyGiFMuc9VZrXf7DarreEwMpurG3dd+CNyW5w==", 250 | "dev": true, 251 | "requires": { 252 | "define-properties": "^1.1.2", 253 | "function-bind": "^1.1.1", 254 | "has-symbols": "^1.0.0", 255 | "object-keys": "^1.0.11" 256 | } 257 | }, 258 | "object.entries": { 259 | "version": "1.1.1", 260 | "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.1.tgz", 261 | "integrity": "sha512-ilqR7BgdyZetJutmDPfXCDffGa0/Yzl2ivVNpbx/g4UeWrCdRnFDUBrKJGLhGieRHDATnyZXWBeCb29k9CJysQ==", 262 | "dev": true, 263 | "requires": { 264 | "define-properties": "^1.1.3", 265 | "es-abstract": "^1.17.0-next.1", 266 | "function-bind": "^1.1.1", 267 | "has": "^1.0.3" 268 | } 269 | }, 270 | "string.prototype.trimend": { 271 | "version": "1.0.1", 272 | "resolved": "https://registry.npmjs.org/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz", 273 | "integrity": "sha512-LRPxFUaTtpqYsTeNKaFOw3R4bxIzWOnbQ837QfBylo8jIxtcbK/A/sMV7Q+OAV/vWo+7s25pOE10KYSjaSO06g==", 274 | "dev": true, 275 | "requires": { 276 | "define-properties": "^1.1.3", 277 | "es-abstract": "^1.17.5" 278 | } 279 | }, 280 | "string.prototype.trimleft": { 281 | "version": "2.1.2", 282 | "resolved": "https://registry.npmjs.org/string.prototype.trimleft/-/string.prototype.trimleft-2.1.2.tgz", 283 | "integrity": "sha512-gCA0tza1JBvqr3bfAIFJGqfdRTyPae82+KTnm3coDXkZN9wnuW3HjGgN386D7hfv5CHQYCI022/rJPVlqXyHSw==", 284 | "dev": true, 285 | "requires": { 286 | "define-properties": "^1.1.3", 287 | "es-abstract": "^1.17.5", 288 | "string.prototype.trimstart": "^1.0.0" 289 | } 290 | }, 291 | "string.prototype.trimright": { 292 | "version": "2.1.2", 293 | "resolved": "https://registry.npmjs.org/string.prototype.trimright/-/string.prototype.trimright-2.1.2.tgz", 294 | "integrity": "sha512-ZNRQ7sY3KroTaYjRS6EbNiiHrOkjihL9aQE/8gfQ4DtAC/aEBRHFJa44OmoWxGGqXuJlfKkZW4WcXErGr+9ZFg==", 295 | "dev": true, 296 | "requires": { 297 | "define-properties": "^1.1.3", 298 | "es-abstract": "^1.17.5", 299 | "string.prototype.trimend": "^1.0.0" 300 | } 301 | }, 302 | "string.prototype.trimstart": { 303 | "version": "1.0.1", 304 | "resolved": "https://registry.npmjs.org/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz", 305 | "integrity": "sha512-XxZn+QpvrBI1FOcg6dIpxUPgWCPuNXvMD72aaRaUQv1eD4e/Qy8i/hFTe0BUmD60p/QA6bh1avmuPTfNjqVWRw==", 306 | "dev": true, 307 | "requires": { 308 | "define-properties": "^1.1.3", 309 | "es-abstract": "^1.17.5" 310 | } 311 | }, 312 | "tmatch": { 313 | "version": "2.0.1", 314 | "resolved": "https://registry.npmjs.org/tmatch/-/tmatch-2.0.1.tgz", 315 | "integrity": "sha1-DFYkbzPzDaG409colauvFmYPOM8=", 316 | "dev": true 317 | }, 318 | "which-boxed-primitive": { 319 | "version": "1.0.1", 320 | "resolved": "https://registry.npmjs.org/which-boxed-primitive/-/which-boxed-primitive-1.0.1.tgz", 321 | "integrity": "sha512-7BT4TwISdDGBgaemWU0N0OU7FeAEJ9Oo2P1PHRm/FCWoEi2VLWC9b6xvxAA3C/NMpxg3HXVgi0sMmGbNUbNepQ==", 322 | "dev": true, 323 | "requires": { 324 | "is-bigint": "^1.0.0", 325 | "is-boolean-object": "^1.0.0", 326 | "is-number-object": "^1.0.3", 327 | "is-string": "^1.0.4", 328 | "is-symbol": "^1.0.2" 329 | } 330 | }, 331 | "which-collection": { 332 | "version": "1.0.1", 333 | "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", 334 | "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", 335 | "dev": true, 336 | "requires": { 337 | "is-map": "^2.0.1", 338 | "is-set": "^2.0.1", 339 | "is-weakmap": "^2.0.1", 340 | "is-weakset": "^2.0.1" 341 | } 342 | } 343 | } 344 | } 345 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # pkglink 2 | 3 | Space saving Node.js package hard linker. 4 | 5 | pkglink locates common JavaScript/Node.js packages from your node_modules directories and hard links the package files so they share disk space. 6 | 7 | [![Build Status](https://secure.travis-ci.org/jeffbski/pkglink.png?branch=master)](http://travis-ci.org/jeffbski/pkglink) [![Known Vulnerabilities](https://snyk.io/test/github/jeffbski/pkglink/cb67b52c10073cbd5a7e6cc6798931db779adb97/badge.svg)](https://snyk.io/test/github/jeffbski/pkglink/cb67b52c10073cbd5a7e6cc6798931db779adb97) 8 | 9 | demo 10 | 11 | ## Why? 12 | 13 | As an instructor, I create lots of JavaScript and Node.js projects and many of them use the same packages. **However due to the way packages are installed they all take up their own disk space.** It would be nice to have a way for the installations of the same package to **share disk space**. 14 | 15 | Modern operating systems and disk formats support the concept of **hard links** which is a way to have one copy of a file on disk that can be used from multiple paths. Since packages are generally read-only once they are installed, it would save much disk space if we could hard link their files. 16 | 17 | pkglink is a command line tool that searches directory tree that you specify for packages in your node_modules directories. When it finds matching packages of the same name and version that could share space, it hard links the files. As a safety precaution it checks many file attributes before considering them for linking ([see full details later in this doc](#what-files-will-it-link-in-the-packages)). 18 | 19 | pkglink keeps track of packages it has seen on previous scans so when you run on new directories in the future, it can quickly know where to look for previous package matches. It double checks the previous packages are still the proper version, inode, and modified time before linking, but this prevents performing full tree scans any time you add a new project. Simply run pkglink once on your project tree and then again on new projects as you create them. 20 | 21 | pkglink has been tested on Ubuntu, Mac OS X, and Windows. Hard links are supported on most modern disk formats with the exception of FAT and ReFS. 22 | 23 | ## How much savings? 24 | 25 | It all depends on how many matching packages you have on your system, but you will probably be surprised. 26 | 27 | After running pkglink on my project directories, **it found 128K packages and saved over 20GB of disk space**. 28 | 29 | ## Assumptions for use 30 | 31 | The main assumption that enables hard linking is that you are not manually modifying your packages after install from the registry. This means that installed packages of the same name and version should generally be the same. Additional checks at the file level are used to verify matches ([see filter criteria later in this doc](#what-files-will-it-link-in-the-packages)) before selecting them for linking. 32 | 33 | Before running any tool that can modify your file system it is always a good idea to have a current backup and sync code with your repositories. 34 | 35 | Hard linking will not work on FAT and ReFS file systems. Hard links can only be made between files on the same device (drive). pkglink has been tested on Mac OS X (hpfs), Ubuntu (ext4), and Windows (NTFS). 36 | 37 | If you had to recover from an unforeseen defect in pkglink, the recovery process is to simply delete your project's node_modules directory and perform npm install again. 38 | 39 | ## Installation 40 | 41 | ```bash 42 | npm install -g pkglink 43 | ``` 44 | 45 | ## Quick start 46 | 47 | ### To find and hard link matching packages 48 | 49 | To hard link packages just run pkglink with one or more directory trees that you wish it to scan and link. 50 | 51 | ```bash 52 | pkglink DIR1 DIR2 ... 53 | ``` 54 | 55 | You will get output similar to this: 56 | 57 | ``` 58 | jeffbski-laptop:~$ pkglink ~/projects ~/working 59 | 60 | pkgs: 128,383 saved: 5.11GB 61 | ``` 62 | 63 | The run above indicated that pkglink found 128K packages and after linking it saved over 5GB of disk space. (Actual savings was higher since I had run pkglink on a portion of the tree previously) 64 | 65 | ### Dryrun - just output a list of matching packages 66 | 67 | If you wish to see what packages pkglink would link you can use the `--dryrun` or `-d` option. pkglink will output matching packages that it would normally link but it will NOT perform any linking. 68 | 69 | ```bash 70 | pkglink -d DIR1 DIR2 ... 71 | ``` 72 | 73 | The `--dryrun` output looks like: 74 | 75 | ``` 76 | jeffbski-laptop:~$ pkglink -d ~/working/expect-test 77 | 78 | tmatch-2.0.1 79 | /Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/tmatch 80 | /Users/jeff/working/expect-test/node_modules/tmatch 81 | 82 | object.entries-1.0.3 83 | /Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/object.entries 84 | /Users/jeff/working/expect-test/node_modules/object.entries 85 | 86 | object-keys-1.0.11 87 | /Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/object-keys 88 | /Users/jeff/working/expect-test/node_modules/object-keys 89 | 90 | # pkgs: 21 would save: 3.88MB 91 | ``` 92 | 93 | ### Generate link commands only 94 | 95 | If you want to see exactly what it would be linking down to the file level, you can use the `--gen-ln-cmds` or `-g` option and it will output the equivalent bash commands for the hard links that it would normally create. It will not peform the linking. You can view this for correctness or even save it to a file and excute it with bash besides just running pkglink again wihout the `-g` option. 96 | 97 | ```bash 98 | pkglink -g DIR1 DIR2 ... 99 | ``` 100 | 101 | The `--gen-ln-cmds` output looks like 102 | 103 | ``` 104 | jeffbski-laptop:~$ pkglink -g ~/working/expect-test 105 | 106 | ln -f "/Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/define-properties/index.js" "/Users/jeff/working/expect-test/node_modules/define-properties/index.js" 107 | ln -f "/Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/expect/CHANGES.md" "/Users/jeff/working/expect-test/node_modules/expect/CHANGES.md" 108 | ln -f "/Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/expect/LICENSE.md" "/Users/jeff/working/expect-test/node_modules/expect/LICENSE.md" 109 | ln -f "/Users/jeff/projects/pkglink/fixtures/projects/foo1/node_modules/es-abstract/Makefile" "/Users/jeff/working/expect-test/node_modules/es-abstract/Makefile" 110 | # pkgs: 21 would save: 3.88MB 111 | ``` 112 | 113 | ## Full Usage 114 | 115 | ``` 116 | Usage: pkglink {OPTIONS} [dir] [dirN] 117 | 118 | Description: 119 | 120 | pkglink - Space saving Node.js package hard linker 121 | 122 | pkglink recursively searches directories for Node.js packages 123 | installed in node_modules directories. It uses the package name 124 | and version to match up possible packages to share. Once it finds 125 | similar packages, pkglink walks through the package directory tree 126 | checking for files that can be linked. If each file's modified 127 | datetime and size match, it will create a hard link for that file 128 | to save disk space. (On win32, mtimes are inconsistent and ignored) 129 | 130 | It keeps track of modules linked in ~/.pkglink_refs to quickly 131 | locate similar modules on future runs. The refs are always 132 | double checked before being considered for linking. This makes 133 | it convenient to perform future pkglink runs on new directories 134 | without having to reprocess the old. 135 | 136 | Standard Options: 137 | 138 | -c, --config CONFIG_PATH 139 | 140 | This option overrides the config file path, default ~/.pkglink 141 | 142 | -d, --dryrun 143 | 144 | Instead of performing the linking, just display the modules that 145 | would be linked and the amount of disk space that would be saved. 146 | 147 | -g, --gen-ln-cmds 148 | 149 | Instead of performing the linking, just generate link commands 150 | that the system would perform and output 151 | 152 | -h, --help 153 | 154 | Show this message 155 | 156 | -m, --memory MEMORY_MB 157 | 158 | Run with increased or decreased memory specified in MB, overrides 159 | environment variable PKGLINK_NODE_OPTIONS and config.memory 160 | The default memory used is 2560. 161 | 162 | -p, --prune 163 | 164 | Prune the refs file by checking all of the refs clearing out any 165 | that have changed 166 | 167 | -r, --refs-file REFS_FILE_PATH 168 | 169 | Specify where to load and store the link refs file which is used to 170 | quickly locate previously linked modules. Default ~/pkglink_refs.json 171 | 172 | -t, --tree-depth N 173 | 174 | Maximum depth to search the directories specified for packages 175 | Default depth: 0 (unlimited) 176 | 177 | -v, --verbose 178 | 179 | Output additional information helpful for debugging 180 | ``` 181 | 182 | If your machine has less than 2.5GB of memory you can use `pkglink_low` instead of `pkglink` and it will run with the normal 1.5GB memory default. 183 | 184 | ## Config 185 | 186 | The default config file path is `~/.pkglink` unless you override it with the `--config` command line option. If this file exists it should be a JSON file with an object having any of the following properties. 187 | 188 | - `refsFile` - location of the JSON file used to track the last 5 references to each package it finds, default: `~/.pkglink_refs`. This can also be overridden with the `--refs-file` command line argument. 189 | 190 | - `concurrentOps` - the number of concurrent operations allowed for IO operations, default: 4 191 | - `consoleWidth` - the number of columns in your console, default: 70 192 | - `ignoreModTime` - ignore the modification time of the files, default is true on Windows, otherwise false 193 | - `memory` - adjust the memory used in MB, default: 2560 (2.5GB). Can also be overridden by setting environment variable PKGLINK_NODE_OPTIONS=--max-old-space-size=1234 or by using the command line argument `--memory`. 194 | - `minFileSize` - the minimum size file to consider for linking in bytes, default: 0 195 | - `refSize` - number of package refs to keep in the refsFile which is used to find matching packages on successive runs, default: 5 196 | - `tree-depth` - the maximum depth to search the directories for packages, default: 0 (unlimited). Can also be overridden with `--tree-depth` command line option. 197 | 198 | ## How do I know it is working? 199 | 200 | Well if you check your disk space before and after a run it should be at least as much savings as pkglink indicates during a run. pkglink indicates the file size saved, but the actual savings can be greater due to the block size of the disk. 201 | 202 | On systems with bash, you can also use `ls -ali node_modules/XYZ` to see the number of hard links a particular file has (which is the number of times it is shared) and the actual inode values. 203 | 204 | When using the `-i` option with `ls` the first column is the inode of the file, so you can verify one directories' files with another. Also the 3rd column is the number of hard links, so you can see that CHANGELOG.md, LICENSE, README.md, and index.js all have 17 hard links. 205 | 206 | ```bash 207 | jeffbski-laptop:~/working/expect-test$ ls -ali node_modules/define-properties/ 208 | total 80 209 | 89543426 drwxr-xr-x 13 jeff staff 442 Oct 22 04:02 . 210 | 89543425 drwxr-xr-x 24 jeff staff 816 Oct 22 03:58 .. 211 | 89543473 -rw-r--r-- 1 jeff staff 276 Oct 14 2015 .editorconfig 212 | 89543474 -rw-r--r-- 1 jeff staff 156 Oct 14 2015 .eslintrc 213 | 89543475 -rw-r--r-- 1 jeff staff 3062 Oct 14 2015 .jscs.json 214 | 89543476 -rw-r--r-- 1 jeff staff 8 Oct 14 2015 .npmignore 215 | 89543477 -rw-r--r-- 1 jeff staff 1182 Oct 14 2015 .travis.yml 216 | 89212049 -rw-r--r-- 17 jeff staff 972 Oct 14 2015 CHANGELOG.md 217 | 89212004 -rw-r--r-- 17 jeff staff 1080 Oct 14 2015 LICENSE 218 | 89211984 -rw-r--r-- 17 jeff staff 2725 Oct 14 2015 README.md 219 | 89212027 -rw-r--r-- 17 jeff staff 1560 Oct 14 2015 index.js 220 | 89543482 -rw-r--r-- 1 jeff staff 1593 Oct 14 2015 package.json 221 | 89543447 drwxr-xr-x 3 jeff staff 102 Oct 22 04:02 test 222 | ``` 223 | 224 | ## What files will it link in the packages 225 | 226 | pkglink looks for packages in the node_modules directories of the directory trees that you specify as args on the command line. 227 | 228 | To be considered for linking the following criteria are checked: 229 | 230 | - package name and version from package.json must match 231 | - package.json is excluded from linking since npm often modifies it on install 232 | - files are on the same device (drive) - hard links only work on same device 233 | - files are not already the same inode (not already hard linked) 234 | - file size is the same 235 | - file modified time is the same (except on Windows which doesn't maintain the original modified times during npm installs) 236 | - file size is >= to config.minFileSize (defaults to 0 to include all) 237 | - directories starting with a `.` and all their descendents are ignored 238 | 239 | ## FAQ 240 | 241 | ### Q. Can I run this for a single project? 242 | 243 | Yes, pkglink is designed so that you can run it for individual projects or for a whole directory tree. It keeps track of packages it has already seen on previous runs (in its refs file) so it can perform links with those as well as any duplication in your project. 244 | 245 | ### Q. Once I use this do I need to do anything special when deleting or updating projects? 246 | 247 | No, since pkglink works by using hard links, your operating system will handle things appropriately under the covers. The OS updates the link count when packages are deleted from a particular path. If you update or reinstall then your packages will simply replace those that were there. You could run pkglink on the project again to hard link the new files. 248 | 249 | Also while pkglink keeps a list of packages it has found in its refs file (~/.pkglink_refs), it always double checks packages before using them for linking (and it updates the refs file). You may also run pkglink with the `--prune` option to check all the refs. 250 | 251 | ### Q. Can I interrupt pkglink during its run? 252 | 253 | Yes, type Control-c once and pkglink will cancel its processing and shutdown. Please allow time for it to gracefully shutdown. 254 | 255 | ### Q. What does the output mean? 256 | 257 | ``` 258 | jeffbski-laptop:~$ pkglink ~/projects ~/working 259 | 260 | pkgs: 128,383 saved: 5.11GB 261 | ``` 262 | 263 | For this pkglink found 128K packages and after performing linking it saved over 5GB of space. pkglink reports the total of the file size saved, but the actual savings on disk is likely larger due to drive block sizes. Using `df -H` before and after the run, the actual size saved was around 11GB. 264 | 265 | Since I had already run pkglink on portions of this tree, this was only the additional savings gained. I had already linked another 8GB previously so my total link savings was closer to 20GB. 266 | 267 | If you were to run pkglink again immediately after this previous run it will come back with the same pkg count but the savings reported this time would be 0 since everything had been linked previously. 268 | 269 | ### Q. What do I do if I get an out of memory error? 270 | 271 | If you run pkglink on a really large directory tree, you might get an out of memory error during the run. 272 | 273 | The error might look something like: 274 | 275 | ``` 276 | FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - JavaScript heap out of memory 277 | ``` 278 | 279 | You can either run pkglink on smaller portions of the tree at a time or you can allow pkglink to use more memory for its run. You can do this by using the `--memory` or `-m` option or changing the `memory` config option in the ~/.pgklink JSON file. 280 | 281 | By default pkglink runs with 2.5GB of memory, so to increase it to 4GB, you could use the following command: 282 | 283 | ```bash 284 | pkglink -m 4096 DIR1 DIR2 ... 285 | ``` 286 | 287 | If you don't even have 2.5GB of memory, you can use the low memory version of pkglink, `pkglink_low DIR1 DIR2 ...` and it will just run with the node.js defaults. Note that you may need to run pkglink_low on smaller portions of the directory tree at a time. 288 | 289 | ## Recovering from an unforeseen problem 290 | 291 | If you need to recover from a problem the standard way is to simply delete your project's `node_modules` directory and run `npm install` again. 292 | 293 | If pkglink exits early, failing to give you the summary output or if you get an out of memory error, see the FAQ above about [handling out of memory errors](#q-what-do-i-do-if-i-get-an-out-of-memory-error). You can run pkglink on smaller directory trees at a time or increase the memory available to it. 294 | 295 | ## License 296 | 297 | MIT license 298 | 299 | ## Credits 300 | 301 | This project was born out of discussions between @kevinold and @jeffbski at Strange Loop 2016. 302 | 303 | [CodeWinds Training](https://codewinds.com) sponsored the development of this project. 304 | --------------------------------------------------------------------------------