├── .prettierrc.js ├── .editorconfig ├── package.json ├── FORMAT.md ├── LICENSE ├── cli.js ├── .gitignore ├── README.md └── index.js /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | printWidth: 80, 3 | useTabs: true, 4 | tabWidth: 2, 5 | overrides: [ 6 | { 7 | files: "*.json", 8 | options: { 9 | useTabs: false 10 | } 11 | } 12 | ] 13 | }; 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = tab 5 | indent_size = 2 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | insert_final_newline = true 9 | max_line_length = 80 10 | 11 | [*.{yml,yaml,json}] 12 | indent_style = space 13 | indent_size = 2 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-voo", 3 | "version": "0.4.0", 4 | "description": "Source Caching for Node.js", 5 | "main": "index.js", 6 | "bin": "cli.js", 7 | "repository": "https://github.com/sokra/node-voo", 8 | "author": "Tobias Koppers ", 9 | "license": "MIT", 10 | "files": [ 11 | "index.js", 12 | "cli.js" 13 | ], 14 | "engines": { 15 | "node": ">=12.0.0" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /FORMAT.md: -------------------------------------------------------------------------------- 1 | ``` 2 | struct { 3 | int32 version = 3; 4 | double created; 5 | int32 lifetime; 6 | int32 nameLength; 7 | int32 modulesCount; 8 | int32 scriptSourceLength; 9 | int32 cachedDataLength; 10 | int32 resolveCount; 11 | char name[nameLength]; 12 | char nodeModulesIntegrity[13]; 13 | struct ModuleInfo { 14 | int32 filenameLength; 15 | int32 sourceLength; 16 | } moduleInfo[modulesCount]; 17 | struct ModuleData { 18 | char filename[moduleInfo[i].filenameLength]; 19 | byte source[moduleInfo[i].sourceLength]; 20 | } moduleData[modulesCount]; 21 | char scriptSource[scriptSourceLength]; 22 | byte cachedData[cachedDataLength]; 23 | struct ResolveInfo { 24 | int32 keyLength; 25 | int32 resultLength; 26 | } resolveInfo[resolveCount]; 27 | struct ResolveData { 28 | char key[resolveInfo[i].keyLength]; 29 | char result[resolveInfo[i].resultLength]; 30 | } resolveData[resolveCount]; 31 | } 32 | ``` 33 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Tobias Koppers 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | let i = 2; 4 | w: while (i < process.argv.length) { 5 | switch (process.argv[i++]) { 6 | case "--yarn": 7 | process.env.NODE_VOO_YARN = "true"; 8 | break; 9 | case "--npm": 10 | process.env.NODE_VOO_NPM = "true"; 11 | break; 12 | case "--cache-only": 13 | process.env.NODE_VOO_CACHE_ONLY = "true"; 14 | break; 15 | case "--no-persist": 16 | process.env.NODE_VOO_NO_PERSIST = "true"; 17 | break; 18 | case "--warning": 19 | process.env.NODE_VOO_LOGLEVEL = "warning"; 20 | break; 21 | case "--info": 22 | process.env.NODE_VOO_LOGLEVEL = "info"; 23 | break; 24 | case "--verbose": 25 | process.env.NODE_VOO_LOGLEVEL = "verbose"; 26 | break; 27 | default: 28 | break w; 29 | } 30 | } 31 | process.argv.splice(1, Math.max(1, i - 2)); 32 | const index = require.resolve("./index"); 33 | const old = process.env.NODE_OPTIONS || ""; 34 | process.env.NODE_OPTIONS = `${old} -r ${JSON.stringify(index)}`; 35 | require(index); 36 | if (process.argv.length > 1) { 37 | require(require("path").resolve(process.cwd(), process.argv[1])); 38 | } else { 39 | require("repl").start(); 40 | } 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-voo 2 | 3 | Source Caching for Node.js 4 | 5 | - Creates v8 cache data for executed javascript files. 6 | - Groups javascript files that are used together. (These groups are called "Voo"s.) 7 | - Stores a cache file into the systems temp directory. 8 | - Puts lazy required files into separate cache files. 9 | - Learns which modules are required conditionally and puts them into separate cache files. 10 | - Improves startup speed of node.js applications after a few runs 11 | 12 | ## How it works 13 | 14 | The first run captures used modules and source code. It groups modules together. 15 | The cache file contains a list of modules with source code. 16 | 17 | The second run loads the modules from cache and creates a single source code for all modules of the group. 18 | This source code is executed and modules are served from this groups source code. 19 | The single source code and v8 cached data is added to the cache file. 20 | 21 | The 3rd run loads the single source code and v8 cached data from the cache and restores compiled code from it. 22 | This run uses optimized code which was generated on the second run. 23 | 24 | Note that code might get reoptimized when captured type info changes. When this happens cache files are updated. 25 | 26 | When the process exits, Voos are persisted by a probablilty process until the time limit has reached (default 100ms). 27 | The probablilty process ensures that all Voos get eventually persisted without increasing the exit delay too much. 28 | As the lifetime of the Voo increases, the probablility of persisting decreases. 29 | 30 | Voos are also persisted everytime their lifetime has doubled (minimum 10s, maximum 1h). 31 | 32 | ## Usage 33 | 34 | ```js 35 | require("node-voo"); 36 | require(""); 37 | ``` 38 | 39 | ## Command Line 40 | 41 | ```sh 42 | yarn add node-voo 43 | yarn node-voo --yarn 44 | ``` 45 | 46 | -or- 47 | 48 | ```sh 49 | npm install -g node-voo 50 | node-voo --npm 51 | ``` 52 | 53 | -or- 54 | 55 | ```sh 56 | yarn global add node-voo 57 | node-voo 58 | ``` 59 | 60 | -or- 61 | 62 | ```sh 63 | NODE_OPTIONS="-r node-voo" node 64 | ``` 65 | 66 | (\*nix only) 67 | 68 | -or- 69 | 70 | ```sh 71 | set NODE_OPTIONS=-r node-voo 72 | node 73 | ``` 74 | 75 | (windows only, stays active for all futures `node` calls too) 76 | 77 | -or- 78 | 79 | ```sh 80 | export NODE_OPTIONS="-r node-voo" 81 | node 82 | ``` 83 | 84 | (\*nix only, , stays active for all futures `node` calls too) 85 | 86 | -or- 87 | 88 | ```sh 89 | node -r node-voo 90 | ``` 91 | 92 | (doesn't capture child processes) 93 | 94 | -or- 95 | 96 | ```sh 97 | npx node-voo 98 | ``` 99 | 100 | (npx has a performance overhead) 101 | 102 | ## Options 103 | 104 | It's possible to pass options via environment variables: 105 | 106 | - `NODE_VOO_YARN=true`: Trust `node_modules/.yarn-integrity` to agressively cache resolving and modules in node_modules. Requirements: 107 | - Only use Yarn. 108 | - Do not modify files in node_modules manually. 109 | - `NODE_VOO_NPM=true`: Trust `package-lock.json` to agressively cache resolving and modules in node_modules. Requirements: 110 | - Only use Npm. 111 | - Do not modify files in node_modules manually. 112 | - Do not run node-voo between changes to `package-lock.json` and `npm i`. i. e. when switching branches, etc. 113 | - `NODE_VOO_LOGLEVEL=warning`: Display warnings on console when 114 | - cached data was rejected by v8 115 | - not all Voos can be persisted due to time limit 116 | - cache can't be restore due to an error 117 | - cache can't be persisted due to an error 118 | - `NODE_VOO_LOGLEVEL=info`: Display warnings and info on console when 119 | - any warning from above occurs 120 | - cache can't be used because source file changed 121 | - cache file was not found and will probably be created 122 | - a unoptimized Voo is restored (including count info) 123 | - a Voo is reorganized due to detected conditional requires 124 | - `NODE_VOO_LOGLEVEL=verbose`: Display warnings and info on console when 125 | - any warning or info from above occurs 126 | - a Voo could be reorganized but it's not worth it due to minor difference 127 | - a Voo is restored (including count and size info) 128 | - a Voo is persisted (including count and size info) 129 | - the process exit and all Voos are persisted (including timing info) 130 | - `NODE_VOO_CACHE_ONLY=true`: Always use the cache and never check if real files have changed. Also allows to cache resolving. 131 | - `NODE_VOO_NO_PERSIST=true`: Never persist Voos (Use only when cache files already reached the optimum) 132 | - `NODE_VOO_PERSIST_LIMIT=`: Time limit in milliseconds, how long node-voo persists Voos on process exit 133 | - `NODE_VOO_CACHE_DIRECTORY`: Directory to store cache files. Defaults to `NODE_VOO_TEMP_DIRECTORY`. 134 | - `NODE_VOO_TEMP_DIRECTORY`: Directory to store temporary files. Defaults to os temp directory. 135 | 136 | When using the CLI it's also possible to pass some options with argument: 137 | 138 | - `node-voo --yarn` = `NODE_VOO_YARN` 139 | - `node-voo --npm` = `NODE_VOO_NPM` 140 | - `node-voo --cache-only` = `NODE_VOO_CACHE_ONLY` 141 | - `node-voo --no-persist` = `NODE_VOO_NO_PERSIST` 142 | - `node-voo --warning` = `NODE_VOO_LOGLEVEL=warning` 143 | - `node-voo --info` = `NODE_VOO_LOGLEVEL=info` 144 | - `node-voo --verbose` = `NODE_VOO_LOGLEVEL=verbose` 145 | 146 | Performance hints: 147 | 148 | - Use `yarn` resp. `npm` optimization via `NODE_VOO_YARN` resp. `NODE_VOO_NPM` to enable caching for resolving. 149 | - Code has to run a few times to reach performance optimum. 150 | 151 | ## Examples 152 | 153 | All examples were run with `NODE_VOO_YARN=true NODE_VOO_LOGLEVEL=verbose NODE_OPTIONS=-r .../node-voo/index.js`: 154 | 155 | ### eslint 156 | 157 | ``` 158 | > node -e "console.time(); require('eslint'); console.timeEnd()" 159 | [node-voo] enabled (cache directory: ...) 160 | [node-voo] .../node_modules/eslint/lib/api.js no cache file 161 | default: 459.409ms 162 | [node-voo] .../node_modules/eslint/lib/api.js persisted [unoptimized] 170 modules 163 | [node-voo] 1 Voos persisted in 33ms 164 | 165 | > node -e "console.time(); require('eslint'); console.timeEnd()" 166 | [node-voo] enabled (cache directory: ...) 167 | [node-voo] .../node_modules/eslint/lib/api.js restored [unoptimized] 170 modules 168 | default: 268.427ms 169 | [node-voo] .../node_modules/eslint/lib/api.js persisted [optimized for 133ms] 170 modules 1.8 MiB Source Code 651 kiB Cached Data 220 Resolve Entries 170 | [node-voo] 1 Voos persisted in 83ms 171 | 172 | > node -e "console.time(); require('eslint'); console.timeEnd()" 173 | [node-voo] enabled (cache directory: ...) 174 | [node-voo] .../node_modules/eslint/lib/api.js restored [optimized for 133ms] 170 modules 1.8 MiB Source Code 651 kiB Cached Data 220 Resolve Entries 175 | default: 122.383ms 176 | [node-voo] .../node_modules/eslint/lib/api.js persisted [optimized for 220ms] 170 modules 1.8 MiB Source Code 651 kiB Cached Data 220 Resolve Entries 177 | [node-voo] 1 Voos persisted in 50ms 178 | 179 | > node -e "console.time(); require('eslint'); console.timeEnd()" 180 | [node-voo] enabled (cache directory: ...) 181 | [node-voo] .../node_modules/eslint/lib/api.js restored [optimized for 220ms] 170 modules 1.8 MiB Source Code 651 kiB Cached Data 220 Resolve Entries 182 | default: 123.616ms 183 | 184 | > node -e "console.time(); require('eslint'); console.timeEnd()" 185 | [node-voo] enabled (cache directory: ...) 186 | [node-voo] .../node_modules/eslint/lib/api.js restored [optimized for 220ms] 170 modules 1.8 MiB Source Code 651 kiB Cached Data 220 Resolve Entries 187 | default: 131.611ms 188 | ``` 189 | 190 | ### babel-core 191 | 192 | ``` 193 | > node -e "console.time(); require('babel-core'); console.timeEnd()" 194 | [node-voo] enabled (cache directory: ...) 195 | [node-voo] .../node_modules/babel-core/index.js no cache file 196 | default: 901.891ms 197 | [node-voo] .../node_modules/babel-core/index.js persisted [unoptimized] 487 modules 198 | [node-voo] 1 Voos persisted in 53ms 199 | 200 | > node -e "console.time(); require('babel-core'); console.timeEnd()" 201 | [node-voo] enabled (cache directory: ...) 202 | [node-voo] .../node_modules/babel-core/index.js restored [unoptimized] 487 modules 203 | default: 243.541ms 204 | [node-voo] .../node_modules/babel-core/index.js persisted [optimized for 122ms] 487 modules 1.1 MiB Source Code 750 kiB Cached Data 633 Resolve Entries 205 | [node-voo] 1 Voos persisted in 88ms 206 | 207 | > node -e "console.time(); require('babel-core'); console.timeEnd()" 208 | [node-voo] enabled (cache directory: ...) 209 | [node-voo] .../node_modules/babel-core/index.js restored [optimized for 122ms] 487 modules 1.1 MiB Source Code 750 kiB Cached Data 633 Resolve Entries 210 | default: 140.512ms 211 | [node-voo] .../node_modules/babel-core/index.js persisted [optimized for 223ms] 487 modules 1.1 MiB Source Code 750 kiB Cached Data 633 Resolve Entries 212 | [node-voo] 1 Voos persisted in 78ms 213 | 214 | > node -e "console.time(); require('babel-core'); console.timeEnd()" 215 | [node-voo] enabled (cache directory: ...) 216 | [node-voo] .../node_modules/babel-core/index.js restored [optimized for 223ms] 487 modules 1.1 MiB Source Code 750 kiB Cached Data 633 Resolve Entries 217 | default: 140.497ms 218 | [node-voo] .../node_modules/babel-core/index.js persisted [optimized for 325ms] 487 modules 1.1 MiB Source Code 750 kiB Cached Data 633 Resolve Entries 219 | [node-voo] 1 Voos persisted in 88ms 220 | ``` 221 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const path = require("path"); 3 | const vm = require("vm"); 4 | const url = require("url"); 5 | const Module = require("module"); 6 | 7 | const HEADER_SIZE = 32; 8 | const FORMAT_VERSION = 3; 9 | 10 | const loglevel = process.env.NODE_VOO_LOGLEVEL; 11 | const log = { $warning: 1, $info: 2, $verbose: 3 }["$" + loglevel] | 0; 12 | 13 | const trustYarn = !!process.env.NODE_VOO_YARN; 14 | const trustNpm = !!process.env.NODE_VOO_NPM; 15 | const cacheOnly = !!process.env.NODE_VOO_CACHE_ONLY; 16 | const noPersist = !!process.env.NODE_VOO_NO_PERSIST; 17 | const persistLimit = +process.env.NODE_VOO_PERSIST_LIMIT || 100; 18 | const tempDir = process.env.NODE_VOO_TEMP_DIRECTORY 19 | ? path.resolve(process.env.NODE_VOO_TEMP_DIRECTORY) 20 | : path.join(require("os").tmpdir(), "node-voo"); 21 | const cacheDir = process.env.NODE_VOO_CACHE_DIRECTORY 22 | ? path.resolve(process.env.NODE_VOO_CACHE_DIRECTORY) 23 | : tempDir; 24 | 25 | if (log >= 3) { 26 | console.log(`[node-voo] enabled (cache directory: ${cacheDir})`); 27 | } 28 | 29 | try { 30 | fs.mkdirSync(tempDir, { recursive: true }); 31 | } catch (e) {} 32 | 33 | try { 34 | fs.mkdirSync(cacheDir, { recursive: true }); 35 | } catch (e) {} 36 | 37 | const HASH_LENGTH = 13; 38 | const hashBuf = Buffer.allocUnsafe(HASH_LENGTH); 39 | const getHash = (str) => { 40 | hashBuf.fill(0); 41 | let x = 0; 42 | for (let i = 0; i < str.length; i++) { 43 | const c = str.charCodeAt(i); 44 | hashBuf[x] += c; 45 | x = (x + i + c) % HASH_LENGTH; 46 | } 47 | return hashBuf; 48 | }; 49 | 50 | // Find root node_modules 51 | let myBase, myNodeModules; 52 | const dirnameMatch = 53 | /((?:\/\.config\/yarn\/|\\Yarn\\Data\\)(?:link|global)|\/usr\/local\/lib|\\nodejs)?[/\\]node_modules[/\\]/.exec( 54 | __dirname 55 | ); 56 | if (dirnameMatch && !dirnameMatch[1]) { 57 | myBase = __dirname.slice(0, dirnameMatch.index); 58 | if (fs.existsSync(path.join(myBase, "package.json"))) { 59 | myNodeModules = path.join(myBase, "node_modules"); 60 | } 61 | } 62 | if (!myNodeModules) { 63 | let last; 64 | myBase = process.cwd(); 65 | while (myBase !== last) { 66 | if (fs.existsSync(path.join(myBase, "package.json"))) break; 67 | last = myBase; 68 | myBase = path.dirname(myBase); 69 | } 70 | myNodeModules = path.join(myBase, "node_modules"); 71 | } 72 | 73 | // Read integrity file 74 | let nodeModulesIntegrity; 75 | if (trustNpm && myNodeModules) { 76 | try { 77 | nodeModulesIntegrity = Buffer.from( 78 | getHash(fs.readFileSync(path.join(myBase, "package-lock.json"), "utf-8")) 79 | ); 80 | } catch (e) {} 81 | } 82 | if (trustYarn && myNodeModules) { 83 | try { 84 | nodeModulesIntegrity = Buffer.from( 85 | getHash( 86 | fs.readFileSync(path.join(myNodeModules, ".yarn-integrity"), "utf-8") 87 | ) 88 | ); 89 | } catch (e) {} 90 | } 91 | 92 | const stripBOM = (content) => { 93 | if (content.charCodeAt(0) === 0xfeff) { 94 | content = content.slice(1); 95 | } 96 | return content; 97 | }; 98 | 99 | const stripShebang = (content) => { 100 | // Remove shebang 101 | var contLen = content.length; 102 | if (contLen >= 2) { 103 | if ( 104 | content.charCodeAt(0) === 35 && 105 | content.charCodeAt(1) === 33 // /^#!/ 106 | ) { 107 | if (contLen === 2) { 108 | // Exact match 109 | content = ""; 110 | } else { 111 | // Find end of shebang line and slice it off 112 | var i = 2; 113 | for (; i < contLen; ++i) { 114 | var code = content.charCodeAt(i); 115 | if (code === 13 || code === 10) break; // /\r|\n/ 116 | } 117 | if (i === contLen) content = ""; 118 | else { 119 | // Note that this actually includes the newline character(s) in the 120 | // new output. This duplicates the behavior of the regular expression 121 | // that was previously used to replace the shebang line 122 | content = content.slice(i); 123 | } 124 | } 125 | } 126 | } 127 | return content; 128 | }; 129 | 130 | function validateString(value) { 131 | if (typeof value !== "string") { 132 | const err = new TypeError( 133 | `The "request" argument must be of type string. Received type ${typeof value}` 134 | ); 135 | err.code = "ERR_INVALID_ARG_TYPE"; 136 | throw err; 137 | } 138 | } 139 | 140 | const makeRequireFunction = (module) => { 141 | const Module = module.constructor; 142 | 143 | function require(path) { 144 | return module.require(path); 145 | } 146 | 147 | function resolve(request, options) { 148 | validateString(request); 149 | return Module._resolveFilename(request, module, false, options); 150 | } 151 | 152 | require.resolve = resolve; 153 | 154 | function paths(request) { 155 | validateString(request); 156 | return Module._resolveLookupPaths(request, module, true); 157 | } 158 | 159 | resolve.paths = paths; 160 | 161 | require.main = process.mainModule; 162 | 163 | // Enable support to add extra extension types. 164 | require.extensions = Module._extensions; 165 | 166 | require.cache = Module._cache; 167 | 168 | return require; 169 | }; 170 | 171 | const writeSync = (fd, buffer) => { 172 | const length = buffer.length; 173 | let offset = 0; 174 | do { 175 | const written = fs.writeSync(fd, buffer, offset, length); 176 | if (written === length) return; 177 | offset += written; 178 | length -= written; 179 | } while (true); 180 | }; 181 | 182 | const readInfoAndData = (file, start, count, valueFn, targetMap) => { 183 | let pos = start + count * 8; 184 | for (let i = 0; i < count; i++) { 185 | const keyLength = file.readInt32LE(start + i * 8, true); 186 | const valueLength = file.readInt32LE(start + 4 + i * 8, true); 187 | const key = file.slice(pos, pos + keyLength).toString("utf-8"); 188 | pos += keyLength; 189 | const value = valueFn(file.slice(pos, pos + valueLength)); 190 | pos += valueLength; 191 | targetMap.set(key, value); 192 | } 193 | return pos; 194 | }; 195 | 196 | const writeInfoAndData = (fd, map, valueFn) => { 197 | const info = Buffer.allocUnsafe(map.size * 8); 198 | const buffers = [info]; 199 | let pos = 0; 200 | for (const [key, value] of map) { 201 | const keyBuffer = Buffer.from(key, "utf-8"); 202 | const valueBuffer = valueFn(value); 203 | info.writeInt32LE(keyBuffer.length, pos); 204 | pos += 4; 205 | info.writeInt32LE(valueBuffer.length, pos); 206 | pos += 4; 207 | buffers.push(keyBuffer, valueBuffer); 208 | } 209 | for (const buffer of buffers) { 210 | writeSync(fd, buffer); 211 | } 212 | }; 213 | 214 | const resolveCache = new Map(); 215 | const moduleToVoo = new Map(); 216 | const allVoos = []; 217 | let uniqueId = process.pid + ""; 218 | try { 219 | uniqueId += "-" + require("worker_threads").threadId; 220 | } catch (e) {} 221 | 222 | class Voo { 223 | constructor(name) { 224 | this.name = name; 225 | this.hash = getHash(name).toString("hex"); 226 | this.filename = path.join(cacheDir, this.hash); 227 | this.created = Date.now() / 1000; 228 | this.started = 0; 229 | this.lifetime = 0; 230 | this.modules = new Map(); 231 | this.resolve = new Map(); 232 | this.timeout = undefined; 233 | this.currentModules = new Set(); 234 | this.scriptSource = undefined; 235 | this.scriptSourceBuffer = undefined; 236 | this.script = undefined; 237 | this.restored = false; 238 | this.integrityMatches = false; 239 | this.cache = new Map(); 240 | } 241 | 242 | persist() { 243 | const tempFile = path.join(tempDir, this.hash + "~" + uniqueId); 244 | try { 245 | this.mayRestructure(); 246 | let cachedData; 247 | let scriptSource; 248 | if (this.scriptSource !== undefined) { 249 | if (this.started) { 250 | const now = Date.now(); 251 | this.lifetime += now - this.started; 252 | this.started = now; 253 | } 254 | this.scriptSourceBuffer = 255 | this.scriptSourceBuffer || Buffer.from(this.scriptSource, "utf-8"); 256 | scriptSource = this.scriptSourceBuffer; 257 | if (this.script) { 258 | cachedData = this.script.createCachedData(); 259 | } 260 | } 261 | const fd = fs.openSync(tempFile, "w"); 262 | const header = Buffer.allocUnsafe(HEADER_SIZE); 263 | const nameBuffer = Buffer.from(this.name, "utf-8"); 264 | header.writeInt32LE(FORMAT_VERSION, 0, true); 265 | header.writeDoubleLE(this.created, 4, true); 266 | header.writeInt32LE(this.lifetime, 8, true); 267 | header.writeInt32LE(nameBuffer.length, 12, true); 268 | header.writeInt32LE(this.modules.size, 16, true); 269 | header.writeInt32LE(scriptSource ? scriptSource.length : 0, 20, true); 270 | header.writeInt32LE(cachedData ? cachedData.length : 0, 24, true); 271 | header.writeInt32LE(this.resolve.size, 28, true); 272 | writeSync(fd, header); 273 | writeSync(fd, nameBuffer); 274 | writeSync( 275 | fd, 276 | nodeModulesIntegrity || Buffer.allocUnsafe(HASH_LENGTH).fill(0) 277 | ); 278 | writeInfoAndData(fd, this.modules, (v) => v); 279 | if (scriptSource) { 280 | writeSync(fd, scriptSource); 281 | } 282 | if (cachedData) { 283 | writeSync(fd, cachedData); 284 | } 285 | writeInfoAndData(fd, this.resolve, (str) => Buffer.from(str, "utf-8")); 286 | fs.closeSync(fd); 287 | try { 288 | fs.unlinkSync(this.filename); 289 | } catch (e) {} 290 | try { 291 | fs.renameSync(tempFile, this.filename); 292 | } catch (e) {} 293 | if (log >= 3) { 294 | console.log( 295 | `[node-voo] ${this.name} persisted ${this.getInfo(cachedData)}` 296 | ); 297 | } 298 | } catch (e) { 299 | try { 300 | fs.unlinkSync(tempFile); 301 | } catch (e) {} 302 | if (log >= 1) { 303 | console.log(`[node-voo] ${this.name} failed to persist: ${e.stack}`); 304 | } 305 | } 306 | } 307 | 308 | tryRestore(Module) { 309 | try { 310 | // Read cache file 311 | const file = fs.readFileSync(this.filename); 312 | if (file.length < HEADER_SIZE) 313 | throw new Error("Incorrect cache file size"); 314 | if (file.readInt32LE(0, true) !== FORMAT_VERSION) 315 | throw new Error("Incorrect cache file version"); 316 | this.created = file.readInt32LE(4, true); 317 | this.lifetime = file.readInt32LE(8, true); 318 | const nameSize = file.readInt32LE(12, true); 319 | const numberOfModules = file.readInt32LE(16, true); 320 | const scriptSourceSize = file.readInt32LE(20, true); 321 | const cachedDataSize = file.readInt32LE(24, true); 322 | const numberOfResolveEntries = file.readInt32LE(28, true); 323 | let pos = HEADER_SIZE; 324 | const name = file.slice(pos, pos + nameSize).toString("utf-8"); 325 | pos += nameSize; 326 | if (name !== this.name) { 327 | throw new Error("Hash conflict"); 328 | } 329 | let integrityMatches = cacheOnly; 330 | if (!integrityMatches && nodeModulesIntegrity) { 331 | const hash = file.slice(pos, pos + HASH_LENGTH); 332 | integrityMatches = Buffer.compare(hash, nodeModulesIntegrity) === 0; 333 | } 334 | pos += HASH_LENGTH; 335 | pos = readInfoAndData(file, pos, numberOfModules, (v) => v, this.modules); 336 | let scriptSourceBuffer; 337 | if (scriptSourceSize > 0) { 338 | scriptSourceBuffer = file.slice(pos, pos + scriptSourceSize); 339 | pos += scriptSourceSize; 340 | this.scriptSourceBuffer = scriptSourceBuffer; 341 | this.scriptSource = scriptSourceBuffer.toString("utf-8"); 342 | } else { 343 | this.createScriptSource(Module); 344 | } 345 | let cachedData = undefined; 346 | if (cachedDataSize > 0) { 347 | cachedData = file.slice(pos, pos + cachedDataSize); 348 | pos += cachedDataSize; 349 | } 350 | if (cacheOnly || integrityMatches) { 351 | readInfoAndData( 352 | file, 353 | pos, 354 | numberOfResolveEntries, 355 | (buf) => buf.toString("utf-8"), 356 | this.resolve 357 | ); 358 | this.integrityMatches = true; 359 | } else if (numberOfResolveEntries > 0) { 360 | this.lifetime = 0; 361 | } 362 | 363 | this.script = new vm.Script(this.scriptSource, { 364 | cachedData, 365 | filename: this.filename + ".js", 366 | lineOffset: 0, 367 | displayErrors: true, 368 | importModuleDynamically: undefined, 369 | }); 370 | if (log >= 1 && this.script.cachedDataRejected) { 371 | console.warn(`[node-voo] ${this.name} cached data was rejected by v8`); 372 | this.lifetime = 0; 373 | } 374 | const result = this.script.runInThisContext(); 375 | 376 | // File cache with data 377 | if (this.modules.size === 1) { 378 | const filename = this.modules.keys().next().value; 379 | this.cache.set(filename, result); 380 | } else { 381 | for (const filename of this.modules.keys()) { 382 | const fn = result["$" + filename]; 383 | this.cache.set(filename, fn); 384 | } 385 | } 386 | 387 | for (const [key, result] of this.resolve) { 388 | resolveCache.set(key, result); 389 | } 390 | 391 | this.restored = true; 392 | this.started = Date.now(); 393 | 394 | if (log >= 2) { 395 | if (cachedData === undefined || log >= 3) { 396 | console.log( 397 | `[node-voo] ${this.name} restored ${this.getInfo(cachedData)}` 398 | ); 399 | } 400 | } 401 | } catch (e) { 402 | if (e.code !== "ENOENT") { 403 | if (log >= 1) { 404 | console.log( 405 | `[node-voo] ${this.name} (${this.filename}) failed to restore: ${e.stack}` 406 | ); 407 | } 408 | } else { 409 | if (log >= 2) { 410 | console.log(`[node-voo] ${this.name} no cache file`); 411 | } 412 | } 413 | } 414 | } 415 | 416 | createScriptSource(Module) { 417 | // Create optimizes source with cached data 418 | if (this.modules.size === 1) { 419 | const source = this.modules.values().next().value; 420 | this.scriptSource = Module.wrap( 421 | stripShebang(stripBOM(source.toString("utf-8"))) 422 | ); 423 | } else { 424 | this.scriptSource = `(function() {\nvar __node_voo_result = {};\n${Array.from( 425 | this.modules 426 | ) 427 | .map(([filename, source]) => { 428 | return `__node_voo_result[${JSON.stringify( 429 | "$" + filename 430 | )}] = ${Module.wrap( 431 | stripShebang(stripBOM(source.toString("utf-8"))) 432 | )}`; 433 | }) 434 | .join("\n")}\nreturn __node_voo_result;\n})();`; 435 | } 436 | this.scriptSourceBuffer = undefined; 437 | } 438 | 439 | mayRestructure() { 440 | if (this.currentModules !== undefined) { 441 | const removableModules = new Set(); 442 | let removableSize = 0; 443 | for (const [filename, source] of this.modules) { 444 | if (!this.currentModules.has(filename)) { 445 | removableModules.add(filename); 446 | removableSize += source.length; 447 | } 448 | } 449 | if (removableSize > 10240 || removableModules.size > 100) { 450 | if (log >= 2) { 451 | console.log( 452 | `[node-voo] ${this.name} restructured Voo ${ 453 | removableModules.size 454 | } modules (${Math.ceil(removableSize / 1024)} kiB) removed` 455 | ); 456 | } 457 | for (const filename of removableModules) { 458 | this.modules.delete(filename); 459 | } 460 | this.scriptSource = undefined; 461 | this.scriptSourceBuffer = undefined; 462 | this.lifetime = 0; 463 | this.currentModules = undefined; 464 | } else if (log >= 3 && removableModules.size > 0) { 465 | console.log( 466 | `[node-voo] ${this.name} restructuring not worth it: ${ 467 | removableModules.size 468 | } modules (${ 469 | Math.ceil(removableSize / 102.4) / 10 470 | } kiB) could be removed` 471 | ); 472 | } 473 | } 474 | } 475 | 476 | flipCoin() { 477 | if (this.lifetime === 0 || this.started === 0) return true; 478 | this.mayRestructure(); 479 | const runtime = Date.now() - this.started; 480 | const p = runtime / this.lifetime; 481 | return Math.random() < p; 482 | } 483 | 484 | start() { 485 | if (this.started === 0) { 486 | this.started = Date.now(); 487 | } 488 | if (!noPersist) { 489 | allVoos.push(this); 490 | if (this.scriptSource !== undefined) { 491 | this.updateTimeout(); 492 | } 493 | } 494 | } 495 | 496 | updateTimeout() { 497 | if (this.timeout) { 498 | clearTimeout(this.timeout); 499 | } 500 | const persistIn = Math.min(Math.max(1000, this.lifetime), 60 * 60 * 1000); 501 | this.timeout = setTimeout(() => { 502 | this.persist(); 503 | if (this.scriptSource !== undefined) { 504 | this.updateTimeout(); 505 | } 506 | }, persistIn); 507 | this.timeout.unref(); 508 | } 509 | 510 | has(filename) { 511 | return this.modules.has(filename); 512 | } 513 | 514 | isValid(filename) { 515 | if (cacheOnly) return true; 516 | if (this.integrityMatches && filename.startsWith(myNodeModules)) 517 | return true; 518 | try { 519 | return ( 520 | Buffer.compare( 521 | this.modules.get(filename), 522 | fs.readFileSync(filename) 523 | ) === 0 524 | ); 525 | } catch (e) { 526 | return false; 527 | } 528 | } 529 | 530 | track(filename) { 531 | this.currentModules.add(filename); 532 | } 533 | 534 | addModule(filename, source) { 535 | this.modules.set(filename, source); 536 | this.scriptSource = undefined; 537 | this.scriptSourceBuffer = undefined; 538 | this.lifetime = 0; 539 | } 540 | 541 | addResolve(key, result) { 542 | this.resolve.set(key, result); 543 | this.lifetime = 0; 544 | } 545 | 546 | getInfo(cachedData) { 547 | const formatTime = (t) => { 548 | if (t > 2000) { 549 | return `${Math.floor(t / 1000)}s`; 550 | } else if (t > 500) { 551 | return `${Math.floor(t / 100) / 10}s`; 552 | } else { 553 | return `${t}ms`; 554 | } 555 | }; 556 | const formatSize = (s) => { 557 | if (s > 1024 * 1024) { 558 | return `${Math.floor(s / 1024 / 102.4) / 10} MiB`; 559 | } else if (s > 10240) { 560 | return `${Math.floor(s / 1024)} kiB`; 561 | } else { 562 | return `${Math.floor(s / 102.4) / 10} kiB`; 563 | } 564 | }; 565 | if (cachedData === undefined) { 566 | return `[unoptimized] ${this.modules.size} modules`; 567 | } else { 568 | return `[optimized for ${formatTime(this.lifetime)}] ${ 569 | this.modules.size 570 | } modules ${formatSize( 571 | this.scriptSourceBuffer.length 572 | )} Source Code ${formatSize(cachedData.length)} Cached Data ${ 573 | this.resolve.size 574 | } Resolve Entries`; 575 | } 576 | } 577 | } 578 | 579 | if (!noPersist) { 580 | process.on("exit", () => { 581 | for (const voo of currentVoos) { 582 | allVoos.push(voo); 583 | } 584 | currentVoos.length = 0; 585 | 586 | let n = 0; 587 | const voos = allVoos.filter((voo) => voo.flipCoin()); 588 | const start = Date.now(); 589 | while (voos.length > 0) { 590 | const random = Math.floor(Math.random() * voos.length); 591 | const voo = voos[random]; 592 | voo.persist(); 593 | n++; 594 | voos.splice(random, 1); 595 | if (Date.now() - start >= persistLimit) break; 596 | } 597 | if (log >= 1) { 598 | if (voos.length === 0) { 599 | if (log >= 3 && n > 0) { 600 | console.log( 601 | `[node-voo] ${n} Voos persisted in ${Date.now() - start}ms` 602 | ); 603 | } 604 | } else { 605 | console.warn( 606 | `[node-voo] ${ 607 | voos.length 608 | } Voos not persisted because time limit reached (took ${ 609 | Date.now() - start 610 | }ms)` 611 | ); 612 | } 613 | } 614 | }); 615 | } 616 | 617 | let currentVoos = []; 618 | 619 | require.extensions[".js"] = (module, filename) => { 620 | let newVoo = false; 621 | let currentVoo; 622 | let content; 623 | let contentString; 624 | if (currentVoos.length === 0) { 625 | if ( 626 | /\bimport\b/.test( 627 | (contentString = (content = fs.readFileSync(filename)).toString( 628 | "utf-8" 629 | )) 630 | ) 631 | ) { 632 | // This can't be cached 633 | module._compile(stripBOM(contentString), filename); 634 | return; 635 | } 636 | currentVoo = new Voo(filename); 637 | currentVoo.tryRestore(module.constructor); 638 | currentVoos.push(currentVoo); 639 | newVoo = true; 640 | } else { 641 | for (const voo of currentVoos) { 642 | if (voo.has(filename)) { 643 | currentVoo = voo; 644 | break; 645 | } 646 | } 647 | if (currentVoo === undefined) { 648 | if ( 649 | (contentString = (content = fs.readFileSync(filename)).toString( 650 | "utf-8" 651 | )).includes("import") 652 | ) { 653 | // This can't be cached 654 | module._compile(stripBOM(contentString), filename); 655 | return; 656 | } 657 | const lastVoo = currentVoos[currentVoos.length - 1]; 658 | if (!lastVoo.restored) { 659 | currentVoo = lastVoo; 660 | } else { 661 | currentVoo = new Voo(lastVoo.hash + "|" + filename); 662 | currentVoo.tryRestore(module.constructor); 663 | currentVoos.push(currentVoo); 664 | } 665 | } 666 | } 667 | try { 668 | moduleToVoo.set(module, currentVoo); 669 | currentVoo.track(filename); 670 | const cacheEntry = currentVoo.cache.get(filename); 671 | if (cacheEntry !== undefined && currentVoo.isValid(filename)) { 672 | const dirname = path.dirname(filename); 673 | const require = makeRequireFunction(module); 674 | const exports = module.exports; 675 | cacheEntry.call(exports, exports, require, module, filename, dirname); 676 | } else { 677 | if (log >= 2 && cacheEntry !== undefined) { 678 | console.warn(`[node-voo] ${filename} has changed`); 679 | } 680 | if (content === undefined) { 681 | content = fs.readFileSync(filename); 682 | contentString = content.toString("utf-8"); 683 | } 684 | currentVoo.addModule(filename, content); 685 | module._compile(stripBOM(contentString), filename); 686 | } 687 | if (newVoo) { 688 | for (const voo of currentVoos) { 689 | voo.start(); 690 | } 691 | } 692 | } finally { 693 | if (newVoo) { 694 | currentVoos.length = 0; 695 | } 696 | } 697 | }; 698 | 699 | if (nodeModulesIntegrity || cacheOnly) { 700 | const cacheableModules = new WeakMap(); 701 | 702 | const originalResolveFilename = Module._resolveFilename; 703 | Module._resolveFilename = (request, parent, isMain, options) => { 704 | if (isMain || !parent || !parent.filename) { 705 | return originalResolveFilename(request, parent, isMain, options); 706 | } 707 | if (!cacheOnly) { 708 | let cacheable = cacheableModules.get(parent); 709 | if (cacheable === undefined) { 710 | cacheable = parent.filename.startsWith(myNodeModules); 711 | cacheableModules.set(parent, cacheable); 712 | } 713 | if (!cacheable) { 714 | return originalResolveFilename(request, parent, isMain, options); 715 | } 716 | } 717 | const key = request + path.dirname(parent.filename); 718 | const cacheEntry = resolveCache.get(key); 719 | if (cacheEntry !== undefined) { 720 | return cacheEntry; 721 | } 722 | const result = originalResolveFilename(request, parent, isMain, options); 723 | if (!cacheOnly) { 724 | const resultCacheable = result.startsWith(myNodeModules); 725 | if (!resultCacheable) { 726 | return result; 727 | } 728 | } 729 | resolveCache.set(key, result); 730 | const voo = moduleToVoo.get(parent); 731 | if (voo !== undefined) { 732 | voo.addResolve(key, result); 733 | } 734 | 735 | return result; 736 | }; 737 | } 738 | --------------------------------------------------------------------------------