├── .gitignore ├── README.md ├── bundle-dependencies.js ├── index.js ├── package.json └── perf-tests ├── create-cache-data.js ├── raw-require.js └── require-with-cached-data.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | perf-tests/.cached-* 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cached-module-loader 2 | 3 | Caching bundler and loader of Node.js modules which uses the `cachedData` 4 | feature in the `vm` module (available since Node.js 6). 5 | 6 | **Don't use this.** 7 | 8 | ## Installation 9 | 10 | ``` 11 | $ npm install --save cached-module-loader 12 | ``` 13 | 14 | ## Usage 15 | 16 | Generate a cached bundle: 17 | 18 | ```js 19 | const { join } = require('path') 20 | const { writeFileSync } = require('fs') 21 | const cachedModuleLoader = require('cached-module-loader') 22 | 23 | cachedModuleLoader.bundleDependencies('babel-core').then(bundle => { 24 | writeFileSync(join(__dirname, '.cached-data.bin'), cachedData) 25 | writeFileSync(join(__dirname, '.cached-code.js'), code) 26 | }) 27 | ``` 28 | 29 | Read a bundle and load the module: 30 | 31 | ```js 32 | const { join } = require('path') 33 | const { readFileSync } = require('fs') 34 | const cachedModuleLoader = require('cached-module-loader') 35 | 36 | const cachedData = readFileSync(join(__dirname, '.cached-data.bin')) 37 | const code = readFileSync(join(__dirname, '.cached-code.js')) 38 | 39 | const babel = cachedModuleLoader.loadInThisContext(require.resolve('babel-core'), module, { 40 | moduleId: 'babel-core', 41 | cachedData, 42 | code 43 | }) 44 | ``` 45 | 46 | ## How it works 47 | 48 | First `cached-module-loader` records every `.js` dependency of the requested 49 | module. These dependencies (and the module itself) are concatenated into one 50 | bundle. `require()` calls are rewritten to use absolute paths. The dependencies 51 | are wrapped in Node.js' standard module wrapper, and when loading modules, a 52 | modified `require()` function is passed. This function can load modules from the 53 | bundle rather than from disk. 54 | 55 | A new 56 | [vm.Script](https://nodejs.org/dist/latest-v6.x/docs/api/vm.html#vm_class_vm_script) 57 | is created with the bundle as the script source, and the `produceCachedData` 58 | option enabled. This generates `cachedData`, which should be saved on the file 59 | system. Using both the source and `cachedData`, new 60 | [vm.Script](https://nodejs.org/dist/latest-v6.x/docs/api/vm.html#vm_class_vm_script) 61 | instances can be quickly created, making it much faster to load cached modules 62 | from the bundle. 63 | 64 | ## Performance tests 65 | 66 | ```console 67 | $ node perf-tests/create-cache-data.js 68 | 69 | $ time node perf-tests/require-with-cached-data.js 70 | $ time node perf-tests/raw-require.js 71 | ``` 72 | -------------------------------------------------------------------------------- /bundle-dependencies.js: -------------------------------------------------------------------------------- 1 | const bundling = new Map() 2 | let recording = true 3 | 4 | // Note that append-transform and its dependencies cannot be bundled. 5 | const appendTransform = require('append-transform') // eslint-disable-line 6 | appendTransform((code, filename) => { 7 | if (recording) { 8 | bundling.set(filename, code) 9 | } 10 | 11 | return code 12 | }) 13 | 14 | require(process.argv[2]) 15 | recording = false 16 | 17 | // Load further dependencies after recording the files that are to be bundled. 18 | const Module = require('module') 19 | const { dirname } = require('path') 20 | const { transform } = require('babel-core') 21 | const wrapListener = require('babel-plugin-detective/wrap-listener') 22 | const resolveFrom = require('resolve-from') 23 | 24 | const rewriteRequire = wrapListener((path, file) => { 25 | if (!path.isLiteral()) return 26 | 27 | const fromDir = dirname(file.opts.filename) 28 | const filename = resolveFrom(fromDir, path.node.value) 29 | if (bundling.has(filename)) { 30 | path.node.value = filename 31 | } 32 | }, 'rewrite-require-path', { 33 | require: true 34 | }) 35 | 36 | // Write a block so the script can be run in an existing context without the 37 | // variables leaking. 38 | process.stdout.write(`{ 39 | const files = new Map() 40 | 41 | function load (filename, parent) { 42 | if (!files.has(filename)) { 43 | throw new Error(\`\${filename} has not been flattened\`) 44 | } 45 | if (!parent) { 46 | throw new TypeError('Missing parent') 47 | } 48 | 49 | const Module = parent.constructor 50 | 51 | const cachedModule = Module._cache[filename] 52 | if (cachedModule) { 53 | return cachedModule.exports 54 | } 55 | 56 | const module = new Module(filename, parent) 57 | module.filename = filename 58 | // FIXME: Set module.paths? 59 | 60 | // Add to cache early to support circular references. 61 | Module._cache[filename] = module 62 | 63 | let threw = true 64 | try { 65 | const { dirname, compiledWrapper } = files.get(filename) 66 | const require = makeRequireFunction(module, dirname) 67 | const args = [module.exports, require, module, filename, dirname] 68 | compiledWrapper.apply(module.exports, args) 69 | threw = false 70 | } finally { 71 | if (threw) { 72 | delete Module._cache[filename] 73 | } 74 | } 75 | 76 | return module.exports 77 | } 78 | 79 | let resolveFrom; 80 | function makeRequireFunction(module, dirname) { 81 | const Module = module.constructor 82 | 83 | function require (path) { 84 | if (files.has(path)) { 85 | return load(path, module) 86 | } 87 | 88 | return module.require(resolve(path)) 89 | } 90 | 91 | function resolve (request) { 92 | if (!resolveFrom) { 93 | resolveFrom = module.require(${JSON.stringify(require.resolve('resolve-from'))}); 94 | } 95 | return resolveFrom(dirname, request) 96 | } 97 | require.resolve = resolve 98 | 99 | require.main = process.mainModule 100 | require.extensions = Module._extensions 101 | require.cache = Module._cache 102 | 103 | return require 104 | }\n\n`) 105 | 106 | for (const [filename, originalCode] of bundling) { 107 | const { code } = transform(originalCode, { 108 | filename, 109 | code: true, 110 | ast: false, 111 | babelrc: false, 112 | compact: false, 113 | plugins: [rewriteRequire] 114 | }) 115 | 116 | // Add the module wrapper, but ignore the trailing semicolon. 117 | const wrapped = Module.wrap(code).slice(0, -1) 118 | 119 | process.stdout.write(`files.set(${JSON.stringify(filename)}, { 120 | dirname: ${JSON.stringify(dirname(filename))}, 121 | compiledWrapper: ${wrapped} 122 | })\n\n`) 123 | } 124 | 125 | process.stdout.write('load\n}\n') 126 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { join } = require('path') 4 | const { Script } = require('vm') 5 | 6 | exports.bundleDependencies = function (moduleId) { 7 | const execa = require('execa') 8 | return execa.stdout( 9 | process.execPath, 10 | [join(__dirname, 'bundle-dependencies.js'), moduleId], 11 | { encoding: 'buffer', maxBuffer: Infinity, stripEof: false } 12 | ).then(code => { 13 | const { cachedData } = new Script(code.toString('utf8'), { 14 | filename: `${moduleId}.bundle`, 15 | produceCachedData: true 16 | }) 17 | 18 | return { cachedData, code, moduleId } 19 | }) 20 | } 21 | 22 | exports.loadInThisContext = function (filename, parent, bundle) { 23 | const { cachedData, code, moduleId } = bundle 24 | const script = new Script(code.toString('utf8'), { 25 | cachedData, 26 | filename: `${moduleId}.bundle` 27 | }) 28 | 29 | const load = script.runInThisContext() 30 | return load(filename, parent) 31 | } 32 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cached-module-loader", 3 | "version": "0.0.2", 4 | "description": "Highly experimental bundler and loader of Node.js modules", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "as-i-preach" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/novemberborn/cached-module-loader.git" 12 | }, 13 | "author": "Mark Wubben (https://novemberborn.net/)", 14 | "license": "ISC", 15 | "bugs": { 16 | "url": "https://github.com/novemberborn/cached-module-loader/issues" 17 | }, 18 | "homepage": "https://github.com/novemberborn/cached-module-loader#readme", 19 | "engines": { 20 | "node": ">=6" 21 | }, 22 | "dependencies": { 23 | "append-transform": "^0.4.0", 24 | "babel-core": "^6.14.0", 25 | "babel-plugin-detective": "^2.0.0", 26 | "execa": "^0.4.0", 27 | "resolve-from": "^2.0.0" 28 | }, 29 | "devDependencies": { 30 | "@novemberborn/as-i-preach": "^5.0.0" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /perf-tests/create-cache-data.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { writeFileSync } = require('fs') 4 | const { join } = require('path') 5 | 6 | const { bundleDependencies } = require('../') 7 | 8 | bundleDependencies('babel-core').then(({ cachedData, code }) => { 9 | writeFileSync(join(__dirname, '.cached-data.bin'), cachedData) 10 | writeFileSync(join(__dirname, '.cached-code.js'), code) 11 | return 12 | }).catch(err => { 13 | console.error(err && err.stack || err) 14 | process.exit(1) 15 | }) 16 | -------------------------------------------------------------------------------- /perf-tests/raw-require.js: -------------------------------------------------------------------------------- 1 | require('babel-core') 2 | -------------------------------------------------------------------------------- /perf-tests/require-with-cached-data.js: -------------------------------------------------------------------------------- 1 | const { strictEqual } = require('assert') 2 | const { readFile } = require('fs') 3 | const { join } = require('path') 4 | 5 | const { loadInThisContext } = require('../') 6 | 7 | function read (file) { 8 | return new Promise((resolve, reject) => { 9 | return readFile(file, (err, contents) => { 10 | if (err) reject(err) 11 | else resolve(contents) 12 | }) 13 | }) 14 | } 15 | 16 | Promise.all([ 17 | read(join(__dirname, '.cached-data.bin')), 18 | read(join(__dirname, '.cached-code.js')) 19 | ]).then(([cachedData, code]) => { 20 | const babel = loadInThisContext(require.resolve('babel-core'), module, { 21 | moduleId: 'babel-core', 22 | cachedData, 23 | code 24 | }) 25 | 26 | strictEqual(require('babel-core'), babel) 27 | return 28 | }).catch(err => { 29 | console.error(err && err.stack || err) 30 | process.exit(1) 31 | }) 32 | --------------------------------------------------------------------------------