├── test ├── basic-loader-test │ ├── dep.js │ ├── dep1.js │ ├── build-dep.js │ ├── directory │ │ └── file.js │ ├── file.js │ ├── style.less │ ├── mod.js │ ├── mod1.js │ ├── mod2.js │ ├── index.js │ ├── webpack.config.js │ └── test-loader.js ├── ts-loader-example │ ├── package.json │ ├── assets │ │ └── color_palette.scss │ ├── index.ts │ └── webpack.config.js ├── css-loader-example │ ├── style.css │ ├── style.modules.css │ ├── index.js │ └── webpack.config.js ├── sass-loader-example │ ├── assets │ │ └── color_palette.scss │ ├── _shared.scss │ ├── postcss.config.js │ ├── style.scss │ ├── package.json │ ├── index.js │ ├── webpack.config.js │ └── package-lock.json ├── less-loader-example │ ├── style.less │ ├── index.js │ └── webpack.config.js ├── __snapshots__ │ ├── pitch.test.js.snap │ ├── workerPool.test.js.snap │ └── webpack.test.js.snap ├── serializer.test.js ├── readBuffer.test.js ├── workerPool.test.js ├── webpack.test.js └── pitch.test.js ├── src ├── cjs.js ├── serializer.js ├── WorkerError.js ├── readBuffer.js ├── workerPools.js ├── index.js ├── template.js ├── WorkerPool.js └── worker.js ├── .prettierrc.js ├── example ├── _shared.scss ├── style.scss ├── .babelrc ├── package.json ├── index.js ├── webpack.config.js └── package-lock.json ├── .prettierignore ├── .gitattributes ├── husky.config.js ├── .eslintignore ├── .eslintrc.js ├── lint-staged.config.js ├── commitlint.config.js ├── .editorconfig ├── .gitignore ├── babel.config.js ├── .github └── workflows │ ├── dependency-review.yml │ └── nodejs.yml ├── .cspell.json ├── LICENSE ├── package.json ├── README.md └── CHANGELOG.md /test/basic-loader-test/dep.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/basic-loader-test/dep1.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/basic-loader-test/build-dep.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/basic-loader-test/directory/file.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/ts-loader-example/package.json: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /src/cjs.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./index'); 2 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { singleQuote: true }; 2 | -------------------------------------------------------------------------------- /test/basic-loader-test/file.js: -------------------------------------------------------------------------------- 1 | console.log('test'); 2 | -------------------------------------------------------------------------------- /example/_shared.scss: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /test/basic-loader-test/style.less: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /test/css-loader-example/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /test/sass-loader-example/assets/color_palette.scss: -------------------------------------------------------------------------------- 1 | $white: #ffffff; 2 | -------------------------------------------------------------------------------- /test/ts-loader-example/assets/color_palette.scss: -------------------------------------------------------------------------------- 1 | $white: #ffffff; 2 | -------------------------------------------------------------------------------- /test/css-loader-example/style.modules.css: -------------------------------------------------------------------------------- 1 | .class { 2 | color: red; 3 | } 4 | -------------------------------------------------------------------------------- /test/less-loader-example/style.less: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /test/sass-loader-example/_shared.scss: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | /coverage 2 | /dist 3 | /node_modules 4 | /test/fixtures 5 | CHANGELOG.md -------------------------------------------------------------------------------- /example/style.scss: -------------------------------------------------------------------------------- 1 | @import '_shared'; 2 | 3 | body { 4 | background: red; 5 | } 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | yarn.lock -diff 2 | * text=auto 3 | bin/* eol=lf 4 | package-lock.json -diff -------------------------------------------------------------------------------- /test/basic-loader-test/mod.js: -------------------------------------------------------------------------------- 1 | export default new URL('./style.less', import.meta.url); 2 | -------------------------------------------------------------------------------- /test/basic-loader-test/mod1.js: -------------------------------------------------------------------------------- 1 | export default new URL('./style.less', import.meta.url); 2 | -------------------------------------------------------------------------------- /test/ts-loader-example/index.ts: -------------------------------------------------------------------------------- 1 | const foo: string = 'foo'; 2 | 3 | console.log(foo); 4 | -------------------------------------------------------------------------------- /test/sass-loader-example/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: ['postcss-font-magician'], 3 | }; 4 | -------------------------------------------------------------------------------- /test/sass-loader-example/style.scss: -------------------------------------------------------------------------------- 1 | @import '_shared'; 2 | @import 'color_palette'; 3 | 4 | body { 5 | background: red; 6 | } 7 | -------------------------------------------------------------------------------- /test/basic-loader-test/mod2.js: -------------------------------------------------------------------------------- 1 | const test = require('./mod1'); 2 | 3 | module.exports = new URL('./style.less', import.meta.url); 4 | -------------------------------------------------------------------------------- /test/sass-loader-example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "lodash-es": "^4.17.21", 4 | "react": "^16.6.3" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/css-loader-example/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | import './style.css'; 4 | import './style.modules.css'; 5 | -------------------------------------------------------------------------------- /husky.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | hooks: { 3 | 'pre-commit': 'lint-staged', 4 | 'commit-msg': 'commitlint -E HUSKY_GIT_PARAMS', 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /test/basic-loader-test/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | // eslint-disable-next-line import/extensions 4 | import './file.js?q=1#hash'; 5 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | /coverage 2 | /dist 3 | /node_modules 4 | /test/fixtures 5 | /test/basic-loader-test/mod.js 6 | /test/basic-loader-test/mod1.js 7 | /test/basic-loader-test/mod2.js 8 | 9 | -------------------------------------------------------------------------------- /example/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": "> 0.25%, not dead" 7 | } 8 | ] 9 | ] 10 | } 11 | -------------------------------------------------------------------------------- /test/__snapshots__/pitch.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`runs pitch unsuccessfully when workPool throw an error 1`] = `"Unexpected Error"`; 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | root: true, 3 | extends: ['@webpack-contrib/eslint-config-webpack', 'prettier'], 4 | parserOptions: { 5 | ecmaVersion: 2020, 6 | }, 7 | }; 8 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "lodash-es": "^4.17.21", 4 | "react": "^16.6.3" 5 | }, 6 | "devDependencies": { 7 | "babel-loader": "^8.1.0" 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /lint-staged.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | '*': [ 3 | 'prettier --cache --write --ignore-unknown', 4 | 'cspell --cache --no-must-find-files', 5 | ], 6 | '*.js': ['eslint --cache --fix'], 7 | }; 8 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: ['@commitlint/config-conventional'], 3 | rules: { 4 | 'header-max-length': [0], 5 | 'body-max-line-length': [0], 6 | 'footer-max-line-length': [0], 7 | }, 8 | }; 9 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false -------------------------------------------------------------------------------- /test/__snapshots__/workerPool.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`workerPool should throw an error when worker.stdio is undefined 1`] = `"Failed to create the worker pool with workerId: 1 and configuration: {}. Please verify if you hit the OS open files limit."`; 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | *.log 3 | npm-debug.log* 4 | .eslintcache 5 | .cspellcache 6 | 7 | /coverage 8 | /dist 9 | /local 10 | /reports 11 | /node_modules 12 | 13 | .DS_Store 14 | Thumbs.db 15 | .idea 16 | *.iml 17 | .vscode 18 | *.sublime-project 19 | *.sublime-workspace 20 | .nyc_output 21 | test/outputs 22 | -------------------------------------------------------------------------------- /babel.config.js: -------------------------------------------------------------------------------- 1 | const MIN_BABEL_VERSION = 7; 2 | 3 | module.exports = (api) => { 4 | api.assertVersion(MIN_BABEL_VERSION); 5 | api.cache(true); 6 | 7 | return { 8 | presets: [ 9 | [ 10 | '@babel/preset-env', 11 | { 12 | targets: { 13 | node: '16.10.0', 14 | }, 15 | }, 16 | ], 17 | ], 18 | }; 19 | }; 20 | -------------------------------------------------------------------------------- /.github/workflows/dependency-review.yml: -------------------------------------------------------------------------------- 1 | name: 'Dependency Review' 2 | on: [pull_request] 3 | 4 | permissions: 5 | contents: read 6 | 7 | jobs: 8 | dependency-review: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: 'Checkout Repository' 12 | uses: actions/checkout@v5 13 | - name: 'Dependency Review' 14 | uses: actions/dependency-review-action@v4 15 | -------------------------------------------------------------------------------- /src/serializer.js: -------------------------------------------------------------------------------- 1 | export function replacer(_key, value) { 2 | if (value instanceof RegExp) { 3 | return { 4 | __serialized_type: 'RegExp', 5 | source: value.source, 6 | flags: value.flags, 7 | }; 8 | } 9 | return value; 10 | } 11 | 12 | export function reviver(_key, value) { 13 | if (typeof value === 'object' && value !== null) { 14 | // eslint-disable-next-line no-underscore-dangle 15 | if (value.__serialized_type === 'RegExp') { 16 | return new RegExp(value.source, value.flags); 17 | } 18 | } 19 | 20 | return value; 21 | } 22 | -------------------------------------------------------------------------------- /test/serializer.test.js: -------------------------------------------------------------------------------- 1 | const { replacer, reviver } = require('../src/serializer'); 2 | 3 | test('round-trips plain objects', () => { 4 | const json = JSON.stringify( 5 | { 6 | a: 1, 7 | b: 'foo', 8 | c: [null, false], 9 | }, 10 | replacer, 11 | ); 12 | expect(JSON.parse(json, reviver)).toEqual({ 13 | a: 1, 14 | b: 'foo', 15 | c: [null, false], 16 | }); 17 | }); 18 | 19 | test('round-trips regular expressions', () => { 20 | const json = JSON.stringify( 21 | { 22 | r: /hoge/g, 23 | s: /^(\w\s)+$/m, 24 | }, 25 | replacer, 26 | ); 27 | expect(JSON.parse(json, reviver)).toEqual({ 28 | r: /hoge/g, 29 | s: /^(\w\s)+$/m, 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /.cspell.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2", 3 | "language": "en,en-gb", 4 | "words": [ 5 | "hoge", 6 | "pathinfo", 7 | "iife", 8 | "fullhash", 9 | "cacheable", 10 | "elsewise", 11 | "Respawn", 12 | "respawning", 13 | "cpus", 14 | "vspace", 15 | "hspace", 16 | "memfs", 17 | "commitlint", 18 | "opencollective", 19 | "Koppers", 20 | "sokra", 21 | "lifecycles", 22 | "absolutify", 23 | "filebase", 24 | "chunkhash", 25 | "moduleid", 26 | "modulehash" 27 | ], 28 | "ignorePaths": [ 29 | "CHANGELOG.md", 30 | "package.json", 31 | "dist/**", 32 | "**/__snapshots__/**", 33 | "package-lock.json", 34 | "**/*.css", 35 | "**/fonts/**", 36 | "node_modules", 37 | "coverage", 38 | "*.log" 39 | ] 40 | } 41 | -------------------------------------------------------------------------------- /src/WorkerError.js: -------------------------------------------------------------------------------- 1 | const stack = (err, worker, workerId) => { 2 | const originError = (err.stack || '') 3 | .split('\n') 4 | .filter((line) => line.trim().startsWith('at')); 5 | 6 | const workerError = worker 7 | .split('\n') 8 | .filter((line) => line.trim().startsWith('at')); 9 | 10 | const diff = workerError 11 | .slice(0, workerError.length - originError.length) 12 | .join('\n'); 13 | 14 | originError.unshift(diff); 15 | originError.unshift(err.message); 16 | originError.unshift(`Thread Loader (Worker ${workerId})`); 17 | 18 | return originError.join('\n'); 19 | }; 20 | 21 | class WorkerError extends Error { 22 | constructor(err, workerId) { 23 | super(err); 24 | this.name = err.name; 25 | this.message = err.message; 26 | 27 | Error.captureStackTrace(this, this.constructor); 28 | 29 | this.stack = stack(err, this.stack, workerId); 30 | } 31 | } 32 | 33 | export default WorkerError; 34 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | import './style.scss?00'; 4 | import './style.scss?01'; 5 | import './style.scss?02'; 6 | import './style.scss?03'; 7 | import './style.scss?04'; 8 | import './style.scss?05'; 9 | import './style.scss?06'; 10 | import './style.scss?07'; 11 | import './style.scss?08'; 12 | import './style.scss?09'; 13 | 14 | import './style.scss?10'; 15 | import './style.scss?11'; 16 | import './style.scss?12'; 17 | import './style.scss?13'; 18 | import './style.scss?14'; 19 | import './style.scss?15'; 20 | import './style.scss?16'; 21 | import './style.scss?17'; 22 | import './style.scss?18'; 23 | import './style.scss?19'; 24 | 25 | import './style.scss?20'; 26 | import './style.scss?21'; 27 | import './style.scss?22'; 28 | import './style.scss?23'; 29 | import './style.scss?24'; 30 | import './style.scss?25'; 31 | import './style.scss?26'; 32 | import './style.scss?27'; 33 | import './style.scss?28'; 34 | import './style.scss?29'; 35 | -------------------------------------------------------------------------------- /test/less-loader-example/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | import './style.less?00'; 4 | import './style.less?01'; 5 | import './style.less?02'; 6 | import './style.less?03'; 7 | import './style.less?04'; 8 | import './style.less?05'; 9 | import './style.less?06'; 10 | import './style.less?07'; 11 | import './style.less?08'; 12 | import './style.less?09'; 13 | 14 | import './style.less?10'; 15 | import './style.less?11'; 16 | import './style.less?12'; 17 | import './style.less?13'; 18 | import './style.less?14'; 19 | import './style.less?15'; 20 | import './style.less?16'; 21 | import './style.less?17'; 22 | import './style.less?18'; 23 | import './style.less?19'; 24 | 25 | import './style.less?20'; 26 | import './style.less?21'; 27 | import './style.less?22'; 28 | import './style.less?23'; 29 | import './style.less?24'; 30 | import './style.less?25'; 31 | import './style.less?26'; 32 | import './style.less?27'; 33 | import './style.less?28'; 34 | import './style.less?29'; 35 | -------------------------------------------------------------------------------- /test/sass-loader-example/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | import './style.scss?00'; 4 | import './style.scss?01'; 5 | import './style.scss?02'; 6 | import './style.scss?03'; 7 | import './style.scss?04'; 8 | import './style.scss?05'; 9 | import './style.scss?06'; 10 | import './style.scss?07'; 11 | import './style.scss?08'; 12 | import './style.scss?09'; 13 | 14 | import './style.scss?10'; 15 | import './style.scss?11'; 16 | import './style.scss?12'; 17 | import './style.scss?13'; 18 | import './style.scss?14'; 19 | import './style.scss?15'; 20 | import './style.scss?16'; 21 | import './style.scss?17'; 22 | import './style.scss?18'; 23 | import './style.scss?19'; 24 | 25 | import './style.scss?20'; 26 | import './style.scss?21'; 27 | import './style.scss?22'; 28 | import './style.scss?23'; 29 | import './style.scss?24'; 30 | import './style.scss?25'; 31 | import './style.scss?26'; 32 | import './style.scss?27'; 33 | import './style.scss?28'; 34 | import './style.scss?29'; 35 | -------------------------------------------------------------------------------- /src/readBuffer.js: -------------------------------------------------------------------------------- 1 | export default function readBuffer(pipe, length, callback) { 2 | if (length === 0) { 3 | callback(null, Buffer.alloc(0)); 4 | return; 5 | } 6 | 7 | let remainingLength = length; 8 | 9 | const buffers = []; 10 | 11 | const readChunk = () => { 12 | const onChunk = (arg) => { 13 | let chunk = arg; 14 | let overflow; 15 | 16 | if (chunk.length > remainingLength) { 17 | overflow = chunk.slice(remainingLength); 18 | chunk = chunk.slice(0, remainingLength); 19 | remainingLength = 0; 20 | } else { 21 | remainingLength -= chunk.length; 22 | } 23 | 24 | buffers.push(chunk); 25 | 26 | if (remainingLength === 0) { 27 | pipe.removeListener('data', onChunk); 28 | pipe.pause(); 29 | 30 | if (overflow) { 31 | pipe.unshift(overflow); 32 | } 33 | 34 | callback(null, Buffer.concat(buffers, length)); 35 | } 36 | }; 37 | 38 | pipe.on('data', onChunk); 39 | pipe.resume(); 40 | }; 41 | readChunk(); 42 | } 43 | -------------------------------------------------------------------------------- /test/css-loader-example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const threadLoader = require('../../dist'); // eslint-disable-line import/no-extraneous-dependencies 4 | 5 | module.exports = (env) => { 6 | const workerPool = { 7 | workers: +env.threads, 8 | workerParallelJobs: 1, 9 | poolTimeout: env.watch ? Infinity : 2000, 10 | }; 11 | if (+env.threads > 0) { 12 | threadLoader.warmup(workerPool, ['css-loader']); 13 | } 14 | return { 15 | mode: 'none', 16 | context: __dirname, 17 | devtool: false, 18 | entry: ['./index.js'], 19 | output: { 20 | path: path.resolve('dist'), 21 | filename: 'bundle.js', 22 | }, 23 | module: { 24 | rules: [ 25 | { 26 | test: /\.css$/, 27 | use: [ 28 | env.threads !== 0 && { 29 | loader: path.resolve(__dirname, '../../dist/index.js'), 30 | options: workerPool, 31 | }, 32 | 'css-loader', 33 | ].filter(Boolean), 34 | }, 35 | ], 36 | }, 37 | stats: { 38 | children: false, 39 | }, 40 | }; 41 | }; 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright JS Foundation and other contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | 'Software'), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /test/ts-loader-example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const threadLoader = require('../../dist'); // eslint-disable-line import/no-extraneous-dependencies 4 | 5 | module.exports = (env) => { 6 | const workerPool = { 7 | workers: +env.threads, 8 | poolTimeout: env.watch ? Infinity : 2000, 9 | }; 10 | if (+env.threads > 0) { 11 | threadLoader.warmup(workerPool, ['ts-loader']); 12 | } 13 | return { 14 | mode: 'none', 15 | context: __dirname, 16 | devtool: false, 17 | entry: ['./index.ts'], 18 | output: { 19 | path: path.resolve('dist'), 20 | filename: 'bundle.js', 21 | }, 22 | module: { 23 | rules: [ 24 | { 25 | test: /\.ts$/, 26 | use: [ 27 | env.threads !== 0 && { 28 | loader: path.resolve(__dirname, '../../dist/index.js'), 29 | options: workerPool, 30 | }, 31 | { loader: 'ts-loader', options: { happyPackMode: true } }, 32 | ].filter(Boolean), 33 | }, 34 | ], 35 | }, 36 | stats: { 37 | children: false, 38 | }, 39 | }; 40 | }; 41 | -------------------------------------------------------------------------------- /src/workerPools.js: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | 3 | import WorkerPool from './WorkerPool'; 4 | 5 | const workerPools = Object.create(null); 6 | 7 | function calculateNumberOfWorkers() { 8 | // There are situations when this call will return undefined so 9 | // we are fallback here to 1. 10 | // More info on: https://github.com/nodejs/node/issues/19022 11 | const cpus = os.cpus() || { length: 1 }; 12 | 13 | return Math.max(1, cpus.length - 1); 14 | } 15 | 16 | function getPool(options) { 17 | const workerPoolOptions = { 18 | name: options.name || '', 19 | numberOfWorkers: options.workers || calculateNumberOfWorkers(), 20 | workerNodeArgs: options.workerNodeArgs, 21 | workerParallelJobs: options.workerParallelJobs || 20, 22 | poolTimeout: options.poolTimeout || 500, 23 | poolParallelJobs: options.poolParallelJobs || 200, 24 | poolRespawn: options.poolRespawn || false, 25 | }; 26 | const tpKey = JSON.stringify(workerPoolOptions); 27 | 28 | workerPools[tpKey] = workerPools[tpKey] || new WorkerPool(workerPoolOptions); 29 | 30 | return workerPools[tpKey]; 31 | } 32 | 33 | export { getPool }; // eslint-disable-line import/prefer-default-export 34 | -------------------------------------------------------------------------------- /test/readBuffer.test.js: -------------------------------------------------------------------------------- 1 | const stream = require('stream'); 2 | 3 | const readBuffer = require('../src/readBuffer'); 4 | 5 | test('data is read', (done) => { 6 | expect.assertions(3); 7 | let eventCount = 0; 8 | function read() { 9 | eventCount += 1; 10 | if (eventCount <= 8) { 11 | return this.push(Buffer.from(eventCount.toString())); 12 | } 13 | return this.push(null); 14 | } 15 | const mockEventStream = new stream.Readable({ 16 | objectMode: true, 17 | read, 18 | }); 19 | function cb(err, data) { 20 | expect(err).toBe(null); 21 | expect(data.length).toBe(8); 22 | expect(String.fromCharCode(data[0])).toBe('1'); 23 | done(); 24 | } 25 | readBuffer.default(mockEventStream, 8, cb); 26 | }); 27 | 28 | test('no data is read when early quit but no error is thrown', (done) => { 29 | expect.assertions(1); 30 | let eventCount = 0; 31 | function read() { 32 | eventCount += 1; 33 | if (eventCount <= 5) { 34 | return this.push(Buffer.from(eventCount.toString())); 35 | } 36 | return this.push(null); 37 | } 38 | const mockEventStream = new stream.Readable({ 39 | objectMode: true, 40 | read, 41 | }); 42 | 43 | const cb = jest.fn(); 44 | readBuffer.default(mockEventStream, 8, cb); 45 | 46 | expect(cb).not.toHaveBeenCalled(); 47 | done(); 48 | }); 49 | -------------------------------------------------------------------------------- /test/basic-loader-test/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const webpack = require('webpack'); 4 | 5 | const threadLoader = require('../../dist'); // eslint-disable-line import/no-extraneous-dependencies 6 | 7 | module.exports = (env) => { 8 | const workerPool = { 9 | workers: +env.threads, 10 | workerParallelJobs: 2, 11 | poolTimeout: env.watch ? Infinity : 2000, 12 | }; 13 | if (+env.threads > 0) { 14 | threadLoader.warmup(workerPool, [require.resolve('./test-loader.js')]); 15 | } 16 | return { 17 | mode: 'none', 18 | context: __dirname, 19 | devtool: false, 20 | entry: ['./index.js'], 21 | output: { 22 | path: path.resolve('dist'), 23 | filename: 'bundle.js', 24 | }, 25 | module: { 26 | rules: [ 27 | { 28 | test: /file\.js$/, 29 | use: [ 30 | env.threads !== 0 && { 31 | loader: path.resolve(__dirname, '../../dist/index.js'), 32 | options: workerPool, 33 | }, 34 | { 35 | loader: require.resolve('./test-loader'), 36 | options: { 37 | test: /test/i, 38 | }, 39 | }, 40 | ], 41 | }, 42 | ], 43 | }, 44 | plugins: [new webpack.HotModuleReplacementPlugin()], 45 | }; 46 | }; 47 | -------------------------------------------------------------------------------- /test/less-loader-example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); 4 | 5 | const threadLoader = require('../../dist'); // eslint-disable-line import/no-extraneous-dependencies 6 | 7 | module.exports = (env) => { 8 | const workerPool = { 9 | workers: +env.threads, 10 | workerParallelJobs: 2, 11 | poolTimeout: env.watch ? Infinity : 2000, 12 | }; 13 | if (+env.threads > 0) { 14 | threadLoader.warmup(workerPool, ['less-loader', 'css-loader']); 15 | } 16 | return { 17 | mode: 'none', 18 | context: __dirname, 19 | devtool: false, 20 | entry: ['./index.js'], 21 | output: { 22 | path: path.resolve('dist'), 23 | filename: 'bundle.js', 24 | }, 25 | module: { 26 | rules: [ 27 | { 28 | test: /\.less$/, 29 | use: [ 30 | MiniCssExtractPlugin.loader, 31 | env.threads !== 0 && { 32 | loader: path.resolve(__dirname, '../../dist/index.js'), 33 | options: workerPool, 34 | }, 35 | 'css-loader', 36 | 'less-loader', 37 | ].filter(Boolean), 38 | }, 39 | ], 40 | }, 41 | plugins: [ 42 | new MiniCssExtractPlugin({ 43 | filename: 'style.css', 44 | }), 45 | ], 46 | stats: { 47 | children: false, 48 | }, 49 | }; 50 | }; 51 | -------------------------------------------------------------------------------- /example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | // eslint-disable-next-line import/no-extraneous-dependencies 4 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); 5 | 6 | // eslint-disable-line import/no-extraneous-dependencies 7 | const threadLoader = require('../dist'); 8 | 9 | module.exports = (env = {}) => { 10 | const workerPool = { 11 | workers: +env.threads, 12 | poolTimeout: env.watch ? Infinity : 2000, 13 | }; 14 | const workerPoolSass = { 15 | workers: +env.threads, 16 | workerParallelJobs: 2, 17 | poolTimeout: env.watch ? Infinity : 2000, 18 | }; 19 | 20 | if (+env.threads > 0) { 21 | threadLoader.warmup(workerPool, ['babel-loader', 'babel-preset-env']); 22 | threadLoader.warmup(workerPoolSass, ['sass-loader', 'css-loader']); 23 | } 24 | 25 | return { 26 | mode: 'development', 27 | context: __dirname, 28 | entry: ['react', 'lodash-es', './index.js'], 29 | output: { 30 | path: path.resolve('dist'), 31 | filename: 'bundle.js', 32 | }, 33 | module: { 34 | rules: [ 35 | { 36 | test: /\.js$/, 37 | use: [ 38 | env.threads !== 0 && { 39 | loader: path.resolve(__dirname, '../dist/index.js'), 40 | options: workerPool, 41 | }, 42 | 'babel-loader', 43 | ].filter(Boolean), 44 | }, 45 | { 46 | test: /\.scss$/, 47 | use: [ 48 | MiniCssExtractPlugin.loader, 49 | env.threads !== 0 && { 50 | loader: path.resolve(__dirname, '../dist/index.js'), 51 | options: workerPoolSass, 52 | }, 53 | 'css-loader', 54 | 'sass-loader', 55 | ].filter(Boolean), 56 | }, 57 | ], 58 | }, 59 | plugins: [ 60 | new MiniCssExtractPlugin({ 61 | filename: 'style.css', 62 | }), 63 | ], 64 | stats: { 65 | children: false, 66 | }, 67 | }; 68 | }; 69 | -------------------------------------------------------------------------------- /test/workerPool.test.js: -------------------------------------------------------------------------------- 1 | import childProcess from 'child_process'; 2 | import stream from 'stream'; 3 | 4 | import WorkerPool from '../src/WorkerPool'; 5 | 6 | jest.mock('child_process', () => { 7 | return { 8 | spawn: jest.fn(() => { 9 | return { 10 | unref: jest.fn(), 11 | }; 12 | }), 13 | }; 14 | }); 15 | 16 | describe('workerPool', () => { 17 | it('should throw an error when worker.stdio is undefined', () => { 18 | const workerPool = new WorkerPool({}); 19 | expect(() => workerPool.createWorker()).toThrowErrorMatchingSnapshot(); 20 | expect(() => workerPool.createWorker()).toThrowError( 21 | 'Please verify if you hit the OS open files limit', 22 | ); 23 | }); 24 | 25 | it('should not throw an error when worker.stdio is defined', () => { 26 | childProcess.spawn.mockImplementationOnce(() => { 27 | return { 28 | stdio: new Array(5).fill(new stream.PassThrough()), 29 | unref: jest.fn(), 30 | }; 31 | }); 32 | 33 | const workerPool = new WorkerPool({}); 34 | expect(() => workerPool.createWorker()).not.toThrow(); 35 | }); 36 | 37 | it('should be able to run if the worker pool was not terminated', () => { 38 | const workerPool = new WorkerPool({}); 39 | expect(workerPool.isAbleToRun()).toBe(true); 40 | }); 41 | 42 | it('should not be able to run if the worker pool was terminated', () => { 43 | const workerPool = new WorkerPool({}); 44 | workerPool.terminate(); 45 | expect(workerPool.isAbleToRun()).toBe(false); 46 | }); 47 | 48 | it('should sanitize nodeArgs when spawn a child process', () => { 49 | childProcess.spawn.mockClear(); 50 | childProcess.spawn.mockImplementationOnce(() => { 51 | return { 52 | stdio: new Array(5).fill(new stream.PassThrough()), 53 | unref: jest.fn(), 54 | }; 55 | }); 56 | 57 | const workerPool = new WorkerPool({ 58 | workerNodeArgs: ['--max-old-space-size=1024', '', null], 59 | workerParallelJobs: 20, 60 | }); 61 | 62 | expect(() => workerPool.createWorker()).not.toThrow(); 63 | 64 | const nonSanitizedNodeArgs = childProcess.spawn.mock.calls[0][1].filter( 65 | (opt) => !opt, 66 | ); 67 | expect(nonSanitizedNodeArgs.length).toEqual(0); 68 | }); 69 | }); 70 | -------------------------------------------------------------------------------- /test/sass-loader-example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); // eslint-disable-line import/no-extraneous-dependencies 4 | 5 | const threadLoader = require('../../dist'); // eslint-disable-line import/no-extraneous-dependencies 6 | 7 | module.exports = (env) => { 8 | const workerPool = { 9 | workers: +env.threads, 10 | poolTimeout: env.watch ? Infinity : 2000, 11 | }; 12 | const workerPoolSass = { 13 | workers: +env.threads, 14 | workerParallelJobs: 2, 15 | poolTimeout: env.watch ? Infinity : 2000, 16 | }; 17 | const sassLoaderOptions = { 18 | sourceMap: true, 19 | sassOptions: { 20 | includePaths: [path.resolve(__dirname, 'assets')], 21 | }, 22 | }; 23 | if (+env.threads > 0) { 24 | threadLoader.warmup(workerPool, ['babel-loader', '@babel/preset-env']); 25 | threadLoader.warmup(workerPoolSass, [ 26 | 'sass-loader', 27 | 'postcss-loader', 28 | 'css-loader', 29 | ]); 30 | } 31 | return { 32 | mode: 'none', 33 | context: __dirname, 34 | devtool: false, 35 | entry: ['./index.js'], 36 | output: { 37 | path: path.resolve('dist'), 38 | filename: 'bundle.js', 39 | }, 40 | module: { 41 | rules: [ 42 | { 43 | test: /\.js$/, 44 | use: [ 45 | env.threads !== 0 && { 46 | loader: path.resolve(__dirname, '../../dist/index.js'), 47 | options: workerPool, 48 | }, 49 | 'babel-loader', 50 | ].filter(Boolean), 51 | }, 52 | { 53 | test: /\.scss$/, 54 | use: [ 55 | MiniCssExtractPlugin.loader, 56 | env.threads !== 0 && { 57 | loader: path.resolve(__dirname, '../../dist/index.js'), 58 | options: workerPoolSass, 59 | }, 60 | 'css-loader', 61 | { 62 | loader: 'postcss-loader', 63 | options: { 64 | postcssOptions: { 65 | config: path.resolve(__dirname, './postcss.config.js'), 66 | }, 67 | }, 68 | }, 69 | { loader: 'sass-loader', options: sassLoaderOptions }, 70 | ].filter(Boolean), 71 | }, 72 | ], 73 | }, 74 | plugins: [ 75 | new MiniCssExtractPlugin({ 76 | filename: 'style.css', 77 | }), 78 | ], 79 | stats: { 80 | children: false, 81 | }, 82 | }; 83 | }; 84 | -------------------------------------------------------------------------------- /test/webpack.test.js: -------------------------------------------------------------------------------- 1 | import webpack from 'webpack'; 2 | 3 | import basicLoaderConfig from './basic-loader-test/webpack.config'; 4 | import sassLoaderConfig from './sass-loader-example/webpack.config'; 5 | import tsLoaderConfig from './ts-loader-example/webpack.config'; 6 | import lessLoaderConfig from './less-loader-example/webpack.config'; 7 | import cssLoaderConfig from './css-loader-example/webpack.config'; 8 | 9 | test("Processes sass-loader's @import correctly", (done) => { 10 | const config = sassLoaderConfig({ threads: 1 }); 11 | 12 | webpack(config, (err, stats) => { 13 | if (err) { 14 | done(err); 15 | return; 16 | } 17 | 18 | expect(err).toBe(null); 19 | expect(stats.hasErrors()).toBe(false); 20 | done(); 21 | }); 22 | }, 30000); 23 | 24 | test('Processes ts-loader correctly', (done) => { 25 | const config = tsLoaderConfig({ threads: 1 }); 26 | 27 | webpack(config, (err, stats) => { 28 | if (err) { 29 | done(err); 30 | return; 31 | } 32 | 33 | expect(err).toBe(null); 34 | expect(stats.hasErrors()).toBe(false); 35 | done(); 36 | }); 37 | }, 30000); 38 | 39 | test('Works with less-loader', (done) => { 40 | const config = lessLoaderConfig({ threads: 1 }); 41 | 42 | webpack(config, (err, stats) => { 43 | if (err) { 44 | done(err); 45 | return; 46 | } 47 | 48 | expect(err).toBe(null); 49 | expect(stats.hasErrors()).toBe(false); 50 | done(); 51 | }); 52 | }, 30000); 53 | 54 | test('Works with css-loader', (done) => { 55 | const config = cssLoaderConfig({}); 56 | 57 | webpack(config, (err, stats) => { 58 | if (err) { 59 | done(err); 60 | return; 61 | } 62 | 63 | expect(err).toBe(null); 64 | expect(stats.hasErrors()).toBe(false); 65 | done(); 66 | }); 67 | }, 30000); 68 | 69 | test('Works with test-loader', (done) => { 70 | const config = basicLoaderConfig({ threads: 1 }); 71 | 72 | webpack(config, (err, stats) => { 73 | if (err) { 74 | done(err); 75 | return; 76 | } 77 | 78 | expect(stats.compilation.errors).toMatchSnapshot('errors'); 79 | expect(stats.compilation.warnings).toMatchSnapshot('warnings'); 80 | 81 | const logs = Array.from(stats.compilation.logging.entries()) 82 | .filter((item) => /file\.js\?q=1#hash/.test(item[0])) 83 | .map((item) => item[1].map(({ time, ...rest }) => rest)); 84 | 85 | expect(logs).toMatchSnapshot('logs'); 86 | 87 | const [testMod] = [...stats.compilation.modules].filter( 88 | (i) => i.rawRequest === './file.js?q=1#hash', 89 | ); 90 | 91 | expect(testMod.buildInfo.cacheable).toBe(false); 92 | // eslint-disable-next-line no-eval, no-underscore-dangle 93 | expect(eval(testMod._source.source())).toMatchSnapshot('result'); 94 | 95 | done(); 96 | }); 97 | }, 30000); 98 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import { getPool } from './workerPools'; 2 | 3 | function pitch() { 4 | const options = this.getOptions(); 5 | const workerPool = getPool(options); 6 | 7 | if (!workerPool.isAbleToRun()) { 8 | return; 9 | } 10 | 11 | const callback = this.async(); 12 | 13 | workerPool.run( 14 | { 15 | loaders: this.loaders.slice(this.loaderIndex + 1).map((l) => { 16 | return { 17 | loader: l.path, 18 | options: l.options, 19 | ident: l.ident, 20 | }; 21 | }), 22 | _compiler: { 23 | fsStartTime: this._compiler.fsStartTime, 24 | options: { plugins: [] }, 25 | }, 26 | _compilation: { 27 | outputOptions: { 28 | hashSalt: this._compilation.outputOptions.hashSalt, 29 | hashFunction: this._compilation.outputOptions.hashFunction, 30 | hashDigest: this._compilation.outputOptions.hashDigest, 31 | hashDigestLength: this._compilation.outputOptions.hashDigestLength, 32 | }, 33 | options: { 34 | devtool: 35 | this._compilation && 36 | this._compilation.options && 37 | this._compilation.options.devtool, 38 | }, 39 | }, 40 | resourcePath: this.resourcePath, 41 | resourceQuery: this.resourceQuery, 42 | resourceFragment: this.resourceFragment, 43 | environment: this.environment, 44 | resource: 45 | this.resourcePath + 46 | (this.resourceQuery || '') + 47 | (this.resourceFragment || ''), 48 | sourceMap: this.sourceMap, 49 | emitError: this.emitError, 50 | emitWarning: this.emitWarning, 51 | getLogger: this.getLogger, 52 | loggers: {}, 53 | loadModule: this.loadModule, 54 | importModule: this.importModule, 55 | resolve: this.resolve, 56 | getResolve: this.getResolve, 57 | target: this.target, 58 | mode: this.mode, 59 | minimize: this.minimize, 60 | optionsContext: this.rootContext || this.options.context, 61 | rootContext: this.rootContext, 62 | hot: this.hot, 63 | }, 64 | (err, r) => { 65 | if (r) { 66 | this.cacheable(r.cacheable); 67 | 68 | r.fileDependencies.forEach((d) => this.addDependency(d)); 69 | r.contextDependencies.forEach((d) => this.addContextDependency(d)); 70 | r.missingDependencies.forEach((d) => this.addMissingDependency(d)); 71 | r.buildDependencies.forEach((d) => 72 | // Compatibility with webpack v4 73 | this.addBuildDependency 74 | ? this.addBuildDependency(d) 75 | : this.addDependency(d), 76 | ); 77 | } 78 | 79 | if (err) { 80 | callback(err); 81 | return; 82 | } 83 | 84 | callback(null, ...r.result); 85 | }, 86 | ); 87 | } 88 | 89 | function warmup(options, requires) { 90 | const workerPool = getPool(options); 91 | 92 | workerPool.warmup(requires); 93 | } 94 | 95 | export { pitch, warmup }; // eslint-disable-line import/prefer-default-export 96 | -------------------------------------------------------------------------------- /.github/workflows/nodejs.yml: -------------------------------------------------------------------------------- 1 | name: thread-loader 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | - next 8 | pull_request: 9 | branches: 10 | - main 11 | - next 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | lint: 18 | name: Lint - ${{ matrix.os }} - Node v${{ matrix.node-version }} 19 | 20 | env: 21 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 22 | 23 | strategy: 24 | matrix: 25 | os: [ubuntu-latest] 26 | node-version: [lts/*] 27 | 28 | runs-on: ${{ matrix.os }} 29 | 30 | concurrency: 31 | group: lint-${{ matrix.os }}-v${{ matrix.node-version }}-${{ github.ref }} 32 | cancel-in-progress: true 33 | 34 | steps: 35 | - uses: actions/checkout@v5 36 | with: 37 | fetch-depth: 0 38 | 39 | - name: Use Node.js ${{ matrix.node-version }} 40 | uses: actions/setup-node@v4 41 | with: 42 | node-version: ${{ matrix.node-version }} 43 | cache: 'npm' 44 | 45 | - name: Install dependencies 46 | run: npm ci 47 | 48 | - name: Lint 49 | run: npm run lint 50 | 51 | - name: Security audit 52 | run: npm run security -- --only=prod 53 | 54 | - name: Validate PR commits with commitlint 55 | if: github.event_name == 'pull_request' 56 | run: npx commitlint --from ${{ github.event.pull_request.head.sha }}~${{ github.event.pull_request.commits }} --to ${{ github.event.pull_request.head.sha }} --verbose 57 | 58 | test: 59 | name: Test - ${{ matrix.os }} - Node v${{ matrix.node-version }}, Webpack ${{ matrix.webpack-version }} 60 | 61 | strategy: 62 | matrix: 63 | os: [ubuntu-latest, windows-latest, macos-latest] 64 | node-version: [16.x, 18.x, 20.x, 22.x, 24.x] 65 | webpack-version: [latest] 66 | 67 | runs-on: ${{ matrix.os }} 68 | 69 | concurrency: 70 | group: test-${{ matrix.os }}-v${{ matrix.node-version }}-${{ matrix.webpack-version }}-${{ github.ref }} 71 | cancel-in-progress: true 72 | 73 | steps: 74 | - name: Setup Git 75 | if: matrix.os == 'windows-latest' 76 | run: git config --global core.autocrlf input 77 | 78 | - uses: actions/checkout@v5 79 | 80 | - name: Use Node.js ${{ matrix.node-version }} 81 | uses: actions/setup-node@v4 82 | with: 83 | node-version: ${{ matrix.node-version }} 84 | cache: 'npm' 85 | 86 | - name: Install dependencies 87 | run: npm ci 88 | 89 | - name: Install webpack ${{ matrix.webpack-version }} 90 | if: matrix.webpack-version != 'latest' 91 | run: npm i webpack@${{ matrix.webpack-version }} 92 | 93 | - name: Run tests for webpack version ${{ matrix.webpack-version }} 94 | run: npm run test:coverage -- --ci 95 | 96 | - name: Submit coverage data to codecov 97 | uses: codecov/codecov-action@v5 98 | with: 99 | token: ${{ secrets.CODECOV_TOKEN }} 100 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "thread-loader", 3 | "version": "4.0.4", 4 | "description": "Runs the following loaders in a worker pool", 5 | "license": "MIT", 6 | "repository": "webpack/thread-loader", 7 | "author": "Tobias Koppers @sokra", 8 | "homepage": "https://github.com/webpack/thread-loader", 9 | "bugs": "https://github.com/webpack/thread-loader/issues", 10 | "funding": { 11 | "type": "opencollective", 12 | "url": "https://opencollective.com/webpack" 13 | }, 14 | "main": "dist/cjs.js", 15 | "engines": { 16 | "node": ">= 16.10.0" 17 | }, 18 | "scripts": { 19 | "start": "npm run build -- -w", 20 | "clean": "del-cli dist", 21 | "prebuild": "npm run clean", 22 | "build": "cross-env NODE_ENV=production babel src -d dist --copy-files", 23 | "commitlint": "commitlint --from=main", 24 | "security": "npm audit --production", 25 | "lint:prettier": "prettier --cache --list-different .", 26 | "lint:js": "eslint --cache .", 27 | "lint:spelling": "cspell --cache --no-must-find-files --quiet \"**/*.*\"", 28 | "lint": "npm-run-all -l -p \"lint:**\"", 29 | "test:only": "cross-env NODE_ENV=test jest", 30 | "fix:js": "npm run lint:js -- --fix", 31 | "fix:prettier": "npm run lint:prettier -- --write", 32 | "fix": "npm-run-all -l fix:js fix:prettier", 33 | "test:watch": "npm run test:only -- --watch", 34 | "test:manual": "npm run build && webpack-dev-server test/manual/src/index.js --open --config test/manual/webpack.config.js", 35 | "test:coverage": "npm run test:only -- --collectCoverageFrom=\"src/**/*.js\" --coverage", 36 | "pretest": "npm run lint", 37 | "test": "npm run test:coverage", 38 | "prepare": "husky install && npm run build", 39 | "release": "standard-version" 40 | }, 41 | "files": [ 42 | "dist" 43 | ], 44 | "peerDependencies": { 45 | "webpack": "^5.0.0" 46 | }, 47 | "dependencies": { 48 | "json-parse-better-errors": "^1.0.2", 49 | "loader-runner": "^4.1.0", 50 | "neo-async": "^2.6.2", 51 | "schema-utils": "^4.2.0" 52 | }, 53 | "devDependencies": { 54 | "@babel/cli": "^7.24.6", 55 | "@babel/core": "^7.24.6", 56 | "@babel/preset-env": "^7.24.6", 57 | "@commitlint/cli": "^17.8.1", 58 | "@commitlint/config-conventional": "^17.8.1", 59 | "@webpack-contrib/eslint-config-webpack": "^3.0.0", 60 | "babel-jest": "^29.7.0", 61 | "babel-loader": "^9.2.0", 62 | "cross-env": "^7.0.2", 63 | "cspell": "^7.3.9", 64 | "css-loader": "^6.11.0", 65 | "del": "^7.1.0", 66 | "del-cli": "^5.1.0", 67 | "eslint": "^8.57.0", 68 | "eslint-config-prettier": "^9.1.0", 69 | "eslint-plugin-import": "^2.29.1", 70 | "husky": "^8.0.3", 71 | "jest": "^29.7.0", 72 | "less-loader": "^11.1.4", 73 | "lint-staged": "^14.0.1", 74 | "lodash": "^4.17.20", 75 | "memfs": "^4.11.2", 76 | "mini-css-extract-plugin": "^2.9.0", 77 | "npm-run-all": "^4.1.5", 78 | "postcss": "^8.4.38", 79 | "postcss-font-magician": "^3.0.0", 80 | "postcss-loader": "^7.3.4", 81 | "prettier": "^3.3.3", 82 | "sass": "^1.77.4", 83 | "sass-loader": "^14.2.1", 84 | "standard-version": "^9.0.0", 85 | "ts-loader": "^9.5.1", 86 | "webpack": "^5.91.0" 87 | }, 88 | "keywords": [ 89 | "webpack" 90 | ] 91 | } 92 | -------------------------------------------------------------------------------- /test/pitch.test.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | 3 | import { pitch } from '../src/cjs'; 4 | import { getPool } from '../src/workerPools'; 5 | 6 | jest.mock('../src/workerPools', () => { 7 | return { 8 | getPool: jest.fn(), 9 | }; 10 | }); 11 | 12 | const runGetPoolMock = (error) => { 13 | getPool.mockImplementationOnce(() => { 14 | return { 15 | isAbleToRun: () => true, 16 | run: jest.fn((opts, cb) => { 17 | cb(error, { 18 | buildDependencies: [], 19 | missingDependencies: [], 20 | fileDependencies: [], 21 | contextDependencies: [], 22 | result: {}, 23 | }); 24 | }), 25 | }; 26 | }); 27 | }; 28 | 29 | const runPitch = (options) => 30 | pitch.call( 31 | Object.assign( 32 | {}, 33 | { 34 | query: options, 35 | loaders: [], 36 | _compiler: { fsStartTime: Date.now() }, 37 | _compilation: { 38 | outputOptions: { 39 | assetModuleFilename: '[hash][ext][query]', 40 | asyncChunks: true, 41 | charset: true, 42 | chunkFilename: '[id].bundle.js', 43 | chunkFormat: 'array-push', 44 | chunkLoading: 'jsonp', 45 | chunkLoadingGlobal: 'webpackChunk', 46 | chunkLoadTimeout: 120000, 47 | cssFilename: 'bundle.css', 48 | cssChunkFilename: '[id].bundle.css', 49 | clean: undefined, 50 | compareBeforeEmit: true, 51 | crossOriginLoading: false, 52 | devtoolFallbackModuleFilenameTemplate: undefined, 53 | devtoolModuleFilenameTemplate: undefined, 54 | devtoolNamespace: '', 55 | environment: { 56 | arrowFunction: true, 57 | const: true, 58 | destructuring: true, 59 | forOf: true, 60 | bigIntLiteral: undefined, 61 | dynamicImport: undefined, 62 | module: undefined, 63 | }, 64 | enabledChunkLoadingTypes: ['jsonp', 'import-scripts'], 65 | enabledLibraryTypes: [], 66 | enabledWasmLoadingTypes: ['fetch'], 67 | filename: 'bundle.js', 68 | globalObject: 'self', 69 | hashDigest: 'hex', 70 | hashDigestLength: 20, 71 | hashFunction: 'md4', 72 | hashSalt: undefined, 73 | hotUpdateChunkFilename: '[id].[fullhash].hot-update.js', 74 | hotUpdateGlobal: 'webpackHotUpdate', 75 | hotUpdateMainFilename: '[runtime].[fullhash].hot-update.json', 76 | iife: true, 77 | importFunctionName: 'import', 78 | importMetaName: 'import.meta', 79 | scriptType: false, 80 | library: undefined, 81 | module: false, 82 | path: '/Applications/SAPDevelop/forks/thread-loader/dist', 83 | pathinfo: false, 84 | publicPath: 'auto', 85 | sourceMapFilename: '[file].map[query]', 86 | sourcePrefix: undefined, 87 | strictModuleExceptionHandling: false, 88 | trustedTypes: undefined, 89 | uniqueName: '', 90 | wasmLoading: 'fetch', 91 | webassemblyModuleFilename: '[hash].module.wasm', 92 | workerPublicPath: '', 93 | workerChunkLoading: 'import-scripts', 94 | workerWasmLoading: 'fetch', 95 | }, 96 | }, 97 | rootContext: path.resolve('../'), 98 | getOptions: () => { 99 | return { workers: NaN, poolTimeout: 2000 }; 100 | }, 101 | cacheable: () => {}, 102 | async: () => (error) => { 103 | if (error) { 104 | throw error; 105 | } 106 | }, 107 | }, 108 | ), 109 | ); 110 | 111 | // it('runs pitch successfully when workPool not throw an error', () => { 112 | // runGetPoolMock(null); 113 | // expect(() => runPitch({})).not.toThrow(); 114 | // }); 115 | 116 | it('runs pitch unsuccessfully when workPool throw an error', () => { 117 | runGetPoolMock(new Error('Unexpected Error')); 118 | expect(() => runPitch({})).toThrowErrorMatchingSnapshot(); 119 | }); 120 | -------------------------------------------------------------------------------- /test/__snapshots__/webpack.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`Works with test-loader: errors 1`] = ` 4 | [ 5 | [ModuleError: Module Error (from ../../dist/index.js): 6 | Test Message Error], 7 | ] 8 | `; 9 | 10 | exports[`Works with test-loader: logs 1`] = ` 11 | [ 12 | [ 13 | { 14 | "args": [ 15 | "test message", 16 | ], 17 | "trace": undefined, 18 | "type": "info", 19 | }, 20 | ], 21 | [ 22 | { 23 | "args": [ 24 | "test message", 25 | ], 26 | "trace": undefined, 27 | "type": "log", 28 | }, 29 | ], 30 | ] 31 | `; 32 | 33 | exports[`Works with test-loader: result 1`] = ` 34 | { 35 | "addBuildDependency": "function", 36 | "addContextDependency": "function", 37 | "addDependency": "function", 38 | "addMissingDependency": "function", 39 | "async": "function", 40 | "cacheable": "function", 41 | "callback": "function", 42 | "clearDependencies": "function", 43 | "context": "/test/basic-loader-test", 44 | "currentRequest": "/test/basic-loader-test/test-loader.js??ruleSet[1].rules[0].use[1]!/test/basic-loader-test/file.js?q=1#hash", 45 | "data": null, 46 | "dependency": "function", 47 | "emitError": "function", 48 | "emitFile": "undefined", 49 | "emitWarning": "function", 50 | "environment": { 51 | "arrowFunction": true, 52 | "asyncFunction": true, 53 | "bigIntLiteral": true, 54 | "const": true, 55 | "destructuring": true, 56 | "document": true, 57 | "forOf": true, 58 | "nodePrefixForCoreModules": true, 59 | "optionalChaining": true, 60 | "templateLiteral": true, 61 | }, 62 | "getContextDependencies": "function", 63 | "getContextDependenciesResult": [ 64 | "/test/basic-loader-test/directory", 65 | ], 66 | "getDependencies": "function", 67 | "getDependenciesResult": [ 68 | "/test/basic-loader-test/file.js", 69 | "/test/basic-loader-test/dep1.js", 70 | "/test/basic-loader-test/dep.js", 71 | ], 72 | "getLogger": "function", 73 | "getMissingDependencies": "function", 74 | "getMissingDependenciesResult": [], 75 | "getOptions": "function", 76 | "getResolve": "function", 77 | "hot": true, 78 | "importModule": "function", 79 | "importModuleResult1": { 80 | "default": "http://test.com/first/777312cffc01c1457868.less", 81 | }, 82 | "importModuleResult2": { 83 | "default": "http://test.com/first/777312cffc01c1457868.less", 84 | }, 85 | "loadModule": "function", 86 | "loadModuleResult": { 87 | "map": null, 88 | "source": "const test = require('./mod1'); 89 | 90 | module.exports = new URL('./style.less', import.meta.url); 91 | ", 92 | }, 93 | "loaderIndex": 0, 94 | "loaders": [ 95 | { 96 | "data": null, 97 | "fragment": "", 98 | "ident": "ruleSet[1].rules[0].use[1]", 99 | "normalExecuted": true, 100 | "options": { 101 | "test": {}, 102 | }, 103 | "path": "/test/basic-loader-test/test-loader.js", 104 | "pitchExecuted": true, 105 | "query": "??ruleSet[1].rules[0].use[1]", 106 | "request": "/test/basic-loader-test/test-loader.js??ruleSet[1].rules[0].use[1]", 107 | }, 108 | ], 109 | "mode": "none", 110 | "options": { 111 | "test": {}, 112 | }, 113 | "previousRequest": "", 114 | "query": { 115 | "test": {}, 116 | }, 117 | "remainingRequest": "/test/basic-loader-test/file.js?q=1#hash", 118 | "request": "/test/basic-loader-test/test-loader.js??ruleSet[1].rules[0].use[1]!/test/basic-loader-test/file.js?q=1#hash", 119 | "resolve": "function", 120 | "resource": "/test/basic-loader-test/file.js?q=1#hash", 121 | "resourceFragment": "#hash", 122 | "resourcePath": "/test/basic-loader-test/file.js", 123 | "resourceQuery": "?q=1", 124 | "rootContext": "/test/basic-loader-test", 125 | "sourceMap": false, 126 | "target": "web", 127 | "utils": { 128 | "absolutify": "undefined", 129 | "contextify": "undefined", 130 | "createHash": "function", 131 | "createHashResult": "db346d691d7acc4dc2625db19f9e3f52", 132 | "createHashResult1": "4fdcca5ddb678139", 133 | }, 134 | "version": 2, 135 | "webpack": true, 136 | } 137 | `; 138 | 139 | exports[`Works with test-loader: warnings 1`] = ` 140 | [ 141 | [ModuleWarning: Module Warning (from ../../dist/index.js): 142 | Test Message Warning], 143 | ] 144 | `; 145 | -------------------------------------------------------------------------------- /test/basic-loader-test/test-loader.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const normalize = (str) => 4 | str.split(process.cwd()).join('').replace(/\\/g, '/'); 5 | 6 | module.exports = async function testLoader() { 7 | this.cacheable(false); 8 | 9 | const options = this.getOptions(); 10 | const callback = this.async(); 11 | 12 | this.emitWarning(new Error('Test Message Warning')); 13 | this.emitError(new Error('Test Message Error')); 14 | this.dependency(require.resolve('./dep1.js')); 15 | this.addDependency(require.resolve('./dep.js')); 16 | this.addBuildDependency(require.resolve('./build-dep.js')); 17 | this.addContextDependency(path.resolve(__dirname, './directory')); 18 | 19 | const logger = this.getLogger('name'); 20 | 21 | logger.info('test message'); 22 | 23 | const logger1 = this.getLogger(); 24 | 25 | logger1.log('test message'); 26 | 27 | // Todo fix me 28 | // this.addMissingDependency(require.resolve("./missing-dep.js")); 29 | 30 | callback( 31 | null, 32 | `module.exports = ${JSON.stringify({ 33 | options, 34 | getOptions: typeof this.getOptions, 35 | async: typeof this.async, 36 | version: this.version, 37 | mode: this.mode, 38 | webpack: this.webpack, 39 | sourceMap: this.sourceMap, 40 | target: this.target, 41 | rootContext: normalize(this.rootContext), 42 | context: normalize(this.context), 43 | environment: this.environment, 44 | loaderIndex: this.loaderIndex, 45 | loaders: this.loaders.map((item) => { 46 | return { 47 | ...item, 48 | path: normalize(item.path), 49 | request: normalize(item.request), 50 | }; 51 | }), 52 | resourcePath: normalize(this.resourcePath), 53 | resourceQuery: this.resourceQuery, 54 | resourceFragment: this.resourceFragment, 55 | resource: normalize(this.resource), 56 | request: normalize(this.request), 57 | remainingRequest: normalize(this.remainingRequest), 58 | currentRequest: normalize(this.currentRequest), 59 | previousRequest: this.previousRequest, 60 | query: this.query, 61 | // Todo fix me 62 | data: this.data, 63 | hot: this.hot, 64 | cacheable: typeof this.cacheable, 65 | emitWarning: typeof this.emitWarning, 66 | emitError: typeof this.emitError, 67 | resolve: typeof this.resolve, 68 | getResolve: typeof this.getResolve, 69 | getLogger: typeof this.getLogger, 70 | // Todo fix me 71 | emitFile: typeof this.emitFile, 72 | addBuildDependency: typeof this.addBuildDependency, 73 | utils: { 74 | absolutify: typeof this.utils.absolutify, 75 | contextify: typeof this.utils.contextify, 76 | createHash: typeof this.utils.createHash, 77 | createHashResult: this.utils.createHash().update('test').digest('hex'), 78 | createHashResult1: this.utils 79 | .createHash('xxhash64') 80 | .update('test') 81 | .digest('hex'), 82 | }, 83 | loadModule: typeof this.loadModule, 84 | loadModuleResult: await new Promise((resolve, reject) => 85 | this.loadModule('./mod2.js', (err, source, map, mod) => { 86 | if (err) { 87 | reject(err); 88 | return; 89 | } 90 | 91 | resolve({ source, map, mod }); 92 | }), 93 | ), 94 | importModule: typeof this.importModule, 95 | importModuleResult1: await this.importModule('./mod.js', { 96 | publicPath: 'http://test.com/first/', 97 | }), 98 | importModuleResult2: await new Promise((resolve, reject) => 99 | this.importModule( 100 | './mod1.js', 101 | { 102 | publicPath: 'http://test.com/first/', 103 | }, 104 | (err, result) => { 105 | if (err) { 106 | reject(err); 107 | return; 108 | } 109 | 110 | resolve(result); 111 | }, 112 | ), 113 | ), 114 | callback: typeof this.callback, 115 | addDependency: typeof this.addDependency, 116 | dependency: typeof this.addDependency, 117 | addContextDependency: typeof this.addContextDependency, 118 | addMissingDependency: typeof this.addMissingDependency, 119 | getDependencies: typeof this.getDependencies, 120 | getDependenciesResult: this.getDependencies().map(normalize), 121 | getContextDependencies: typeof this.getContextDependencies, 122 | getContextDependenciesResult: 123 | this.getContextDependencies().map(normalize), 124 | getMissingDependencies: typeof this.getMissingDependencies, 125 | getMissingDependenciesResult: this.getMissingDependencies(), 126 | clearDependencies: typeof this.clearDependencies, 127 | })};`, 128 | ); 129 | }; 130 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 | 4 | 5 |
6 | 7 | [![npm][npm]][npm-url] 8 | [![node][node]][node-url] 9 | [![tests][tests]][tests-url] 10 | [![coverage][cover]][cover-url] 11 | [![discussion][discussion]][discussion-url] 12 | [![size][size]][size-url] 13 | 14 | # thread-loader 15 | 16 | Runs the specified loaders in a worker pool. 17 | 18 | ## Getting Started 19 | 20 | ```bash 21 | npm install --save-dev thread-loader 22 | ``` 23 | 24 | or 25 | 26 | ```bash 27 | yarn add -D thread-loader 28 | ``` 29 | 30 | or 31 | 32 | ```bash 33 | pnpm add -D thread-loader 34 | ``` 35 | 36 | Put this loader in front of other loaders. 37 | The following loaders run in a worker pool. 38 | 39 | Loaders running in a worker pool have limitations. Examples: 40 | 41 | - Loaders cannot emit files. 42 | - Loaders cannot use custom loader APIs (i.e. by plugins). 43 | - Loaders cannot access webpack options. 44 | 45 | Each worker is a separate Node.js process, which has an overhead of ~600ms. There is also additional overhead from inter-process communication. 46 | 47 | > Use this loader only for expensive operations! 48 | 49 | ### Examples 50 | 51 | **webpack.config.js** 52 | 53 | ```js 54 | module.exports = { 55 | module: { 56 | rules: [ 57 | { 58 | test: /\.js$/, 59 | include: path.resolve('src'), 60 | use: [ 61 | 'thread-loader', 62 | // your expensive loader (e.g babel-loader) 63 | ], 64 | }, 65 | ], 66 | }, 67 | }; 68 | ``` 69 | 70 | **with options** 71 | 72 | ```js 73 | use: [ 74 | { 75 | loader: 'thread-loader', 76 | // loaders with equal options will share worker pools 77 | options: { 78 | // the number of spawned workers, defaults to (number of cpus - 1) or 79 | // fallback to 1 when require('os').cpus() is undefined 80 | workers: 2, 81 | 82 | // number of jobs a worker processes in parallel 83 | // defaults to 20 84 | workerParallelJobs: 50, 85 | 86 | // additional node.js arguments 87 | workerNodeArgs: ['--max-old-space-size=1024'], 88 | 89 | // Allow to respawn a dead worker pool 90 | // respawning slows down the entire compilation 91 | // and should be set to false for development 92 | poolRespawn: false, 93 | 94 | // timeout for killing the worker processes when idle 95 | // defaults to 500 (ms) 96 | // can be set to Infinity for watching builds to keep workers alive 97 | poolTimeout: 2000, 98 | 99 | // number of jobs the pool distributes to the workers 100 | // defaults to 200 101 | // decrease for less efficient but more fair distribution 102 | poolParallelJobs: 50, 103 | 104 | // name of the pool 105 | // can be used to create different pools with otherwise identical options 106 | name: 'my-pool', 107 | }, 108 | }, 109 | // your expensive loader (e.g babel-loader) 110 | ]; 111 | ``` 112 | 113 | **prewarming** 114 | 115 | To prevent the high delays when booting workers, it is possible to warm up the worker pool. 116 | 117 | This boots the max number of workers in the pool and loads the specified modules into the Node.js module cache. 118 | 119 | ```js 120 | const threadLoader = require('thread-loader'); 121 | 122 | threadLoader.warmup( 123 | { 124 | // pool options, like passed to loader options 125 | // must match loader options to boot the correct pool 126 | }, 127 | [ 128 | // modules to load 129 | // can be any module, i.e. 130 | 'babel-loader', 131 | '@babel/preset-env', 132 | 'sass-loader', 133 | ], 134 | ); 135 | ``` 136 | 137 | ## Contributing 138 | 139 | We welcome all contributions! 140 | If you're new here, please take a moment to review our contributing guidelines before submitting issues or pull requests. 141 | 142 | [CONTRIBUTING](https://github.com/webpack/thread-loader?tab=contributing-ov-file#contributing) 143 | 144 | ## License 145 | 146 | [MIT](./LICENSE) 147 | 148 | [npm]: https://img.shields.io/npm/v/thread-loader.svg 149 | [npm-url]: https://npmjs.com/package/thread-loader 150 | [node]: https://img.shields.io/node/v/thread-loader.svg 151 | [node-url]: https://nodejs.org 152 | [tests]: https://github.com/webpack/thread-loader/workflows/thread-loader/badge.svg 153 | [tests-url]: https://github.com/webpack/thread-loader/actions 154 | [cover]: https://codecov.io/gh/webpack/thread-loader/branch/main/graph/badge.svg 155 | [cover-url]: https://codecov.io/gh/webpack/thread-loader 156 | [discussion]: https://img.shields.io/github/discussions/webpack/webpack 157 | [discussion-url]: https://github.com/webpack/webpack/discussions 158 | [size]: https://packagephobia.now.sh/badge?p=thread-loader 159 | [size-url]: https://packagephobia.now.sh/result?p=thread-loader 160 | -------------------------------------------------------------------------------- /test/sass-loader-example/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sass-loader-example", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "dependencies": { 8 | "lodash-es": "^4.17.21", 9 | "react": "^16.6.3" 10 | } 11 | }, 12 | "node_modules/js-tokens": { 13 | "version": "4.0.0", 14 | "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", 15 | "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" 16 | }, 17 | "node_modules/lodash-es": { 18 | "version": "4.17.21", 19 | "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", 20 | "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" 21 | }, 22 | "node_modules/loose-envify": { 23 | "version": "1.4.0", 24 | "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", 25 | "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", 26 | "dependencies": { 27 | "js-tokens": "^3.0.0 || ^4.0.0" 28 | }, 29 | "bin": { 30 | "loose-envify": "cli.js" 31 | } 32 | }, 33 | "node_modules/object-assign": { 34 | "version": "4.1.1", 35 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 36 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=", 37 | "engines": { 38 | "node": ">=0.10.0" 39 | } 40 | }, 41 | "node_modules/react": { 42 | "version": "16.6.3", 43 | "resolved": "https://registry.npmjs.org/react/-/react-16.6.3.tgz", 44 | "integrity": "sha512-zCvmH2vbEolgKxtqXL2wmGCUxUyNheYn/C+PD1YAjfxHC54+MhdruyhO7QieQrYsYeTxrn93PM2y0jRH1zEExw==", 45 | "dependencies": { 46 | "loose-envify": "^1.1.0", 47 | "object-assign": "^4.1.1", 48 | "prop-types": "^15.6.2", 49 | "scheduler": "^0.11.2" 50 | }, 51 | "engines": { 52 | "node": ">=0.10.0" 53 | } 54 | }, 55 | "node_modules/react/node_modules/prop-types": { 56 | "version": "15.6.2", 57 | "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.2.tgz", 58 | "integrity": "sha512-3pboPvLiWD7dkI3qf3KbUe6hKFKa52w+AE0VCqECtf+QHAKgOL37tTaNCnuX1nAAQ4ZhyP+kYVKf8rLmJ/feDQ==", 59 | "dependencies": { 60 | "loose-envify": "^1.3.1", 61 | "object-assign": "^4.1.1" 62 | } 63 | }, 64 | "node_modules/react/node_modules/scheduler": { 65 | "version": "0.11.3", 66 | "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.11.3.tgz", 67 | "integrity": "sha512-i9X9VRRVZDd3xZw10NY5Z2cVMbdYg6gqFecfj79USv1CFN+YrJ3gIPRKf1qlY+Sxly4djoKdfx1T+m9dnRB8kQ==", 68 | "dependencies": { 69 | "loose-envify": "^1.1.0", 70 | "object-assign": "^4.1.1" 71 | } 72 | } 73 | }, 74 | "dependencies": { 75 | "js-tokens": { 76 | "version": "4.0.0", 77 | "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", 78 | "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" 79 | }, 80 | "lodash-es": { 81 | "version": "4.17.21", 82 | "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", 83 | "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" 84 | }, 85 | "loose-envify": { 86 | "version": "1.4.0", 87 | "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", 88 | "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", 89 | "requires": { 90 | "js-tokens": "^3.0.0 || ^4.0.0" 91 | } 92 | }, 93 | "object-assign": { 94 | "version": "4.1.1", 95 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 96 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" 97 | }, 98 | "react": { 99 | "version": "16.6.3", 100 | "resolved": "https://registry.npmjs.org/react/-/react-16.6.3.tgz", 101 | "integrity": "sha512-zCvmH2vbEolgKxtqXL2wmGCUxUyNheYn/C+PD1YAjfxHC54+MhdruyhO7QieQrYsYeTxrn93PM2y0jRH1zEExw==", 102 | "requires": { 103 | "loose-envify": "^1.1.0", 104 | "object-assign": "^4.1.1", 105 | "prop-types": "^15.6.2", 106 | "scheduler": "^0.11.2" 107 | }, 108 | "dependencies": { 109 | "prop-types": { 110 | "version": "15.6.2", 111 | "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.2.tgz", 112 | "integrity": "sha512-3pboPvLiWD7dkI3qf3KbUe6hKFKa52w+AE0VCqECtf+QHAKgOL37tTaNCnuX1nAAQ4ZhyP+kYVKf8rLmJ/feDQ==", 113 | "requires": { 114 | "loose-envify": "^1.3.1", 115 | "object-assign": "^4.1.1" 116 | } 117 | }, 118 | "scheduler": { 119 | "version": "0.11.3", 120 | "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.11.3.tgz", 121 | "integrity": "sha512-i9X9VRRVZDd3xZw10NY5Z2cVMbdYg6gqFecfj79USv1CFN+YrJ3gIPRKf1qlY+Sxly4djoKdfx1T+m9dnRB8kQ==", 122 | "requires": { 123 | "loose-envify": "^1.1.0", 124 | "object-assign": "^4.1.1" 125 | } 126 | } 127 | } 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/template.js: -------------------------------------------------------------------------------- 1 | // TODO export it from webpack 2 | 3 | const { basename, extname } = require('path'); 4 | const util = require('util'); 5 | 6 | const { Chunk } = require('webpack'); 7 | const { Module } = require('webpack'); 8 | const { parseResource } = require('webpack/lib/util/identifier'); 9 | 10 | const REGEXP = /\[\\*([\w:]+)\\*\]/gi; 11 | 12 | /** 13 | * @param {string | number} id id 14 | * @returns {string | number} result 15 | */ 16 | const prepareId = (id) => { 17 | if (typeof id !== 'string') return id; 18 | 19 | if (/^"\s\+*.*\+\s*"$/.test(id)) { 20 | const match = /^"\s\+*\s*(.*)\s*\+\s*"$/.exec(id); 21 | 22 | return `" + (${ 23 | /** @type {string[]} */ (match)[1] 24 | } + "").replace(/(^[.-]|[^a-zA-Z0-9_-])+/g, "_") + "`; 25 | } 26 | 27 | return id.replace(/(^[.-]|[^a-zA-Z0-9_-])+/g, '_'); 28 | }; 29 | 30 | /** 31 | * @callback ReplacerFunction 32 | * @param {string} match 33 | * @param {string | undefined} arg 34 | * @param {string} input 35 | */ 36 | 37 | /** 38 | * @param {ReplacerFunction} replacer replacer 39 | * @param {((arg0: number) => string) | undefined} handler handler 40 | * @param {AssetInfo | undefined} assetInfo asset info 41 | * @param {string} hashName hash name 42 | * @returns {ReplacerFunction} hash replacer function 43 | */ 44 | const hashLength = (replacer, handler, assetInfo, hashName) => { 45 | /** @type {ReplacerFunction} */ 46 | const fn = (match, arg, input) => { 47 | let result; 48 | const length = arg && Number.parseInt(arg, 10); 49 | 50 | if (length && handler) { 51 | result = handler(length); 52 | } else { 53 | const hash = replacer(match, arg, input); 54 | 55 | result = length ? hash.slice(0, length) : hash; 56 | } 57 | if (assetInfo) { 58 | // eslint-disable-next-line no-param-reassign 59 | assetInfo.immutable = true; 60 | if (Array.isArray(assetInfo[hashName])) { 61 | // eslint-disable-next-line no-param-reassign 62 | assetInfo[hashName] = [...assetInfo[hashName], result]; 63 | } else if (assetInfo[hashName]) { 64 | // eslint-disable-next-line no-param-reassign 65 | assetInfo[hashName] = [assetInfo[hashName], result]; 66 | } else { 67 | // eslint-disable-next-line no-param-reassign 68 | assetInfo[hashName] = result; 69 | } 70 | } 71 | return result; 72 | }; 73 | 74 | return fn; 75 | }; 76 | 77 | /** @typedef {(match: string, arg?: string, input?: string) => string} Replacer */ 78 | 79 | /** 80 | * @param {string | number | null | undefined | (() => string | number | null | undefined)} value value 81 | * @param {boolean=} allowEmpty allow empty 82 | * @returns {Replacer} replacer 83 | */ 84 | const replacer = (value, allowEmpty) => { 85 | /** @type {Replacer} */ 86 | const fn = (match, arg, input) => { 87 | if (typeof value === 'function') { 88 | // eslint-disable-next-line no-param-reassign 89 | value = value(); 90 | } 91 | // eslint-disable-next-line no-undefined 92 | if (value === null || value === undefined) { 93 | if (!allowEmpty) { 94 | throw new Error( 95 | `Path variable ${match} not implemented in this context: ${input}`, 96 | ); 97 | } 98 | 99 | return ''; 100 | } 101 | 102 | return `${value}`; 103 | }; 104 | 105 | return fn; 106 | }; 107 | 108 | const deprecationCache = new Map(); 109 | const deprecatedFunction = (() => () => {})(); 110 | /** 111 | * @param {Function} fn function 112 | * @param {string} message message 113 | * @param {string} code code 114 | * @returns {function(...any[]): void} function with deprecation output 115 | */ 116 | const deprecated = (fn, message, code) => { 117 | let d = deprecationCache.get(message); 118 | // eslint-disable-next-line no-undefined 119 | if (d === undefined) { 120 | d = util.deprecate(deprecatedFunction, message, code); 121 | deprecationCache.set(message, d); 122 | } 123 | return (...args) => { 124 | d(); 125 | return fn(...args); 126 | }; 127 | }; 128 | 129 | /** @typedef {string | function(PathData, AssetInfo=): string} TemplatePath */ 130 | 131 | /** 132 | * @param {TemplatePath} path the raw path 133 | * @param {PathData} data context data 134 | * @param {AssetInfo | undefined} assetInfo extra info about the asset (will be written to) 135 | * @returns {string} the interpolated path 136 | */ 137 | const replacePathVariables = (path, data, assetInfo) => { 138 | const { chunkGraph } = data; 139 | 140 | /** @type {Map} */ 141 | const replacements = new Map(); 142 | 143 | // Filename context 144 | // 145 | // Placeholders 146 | // 147 | // for /some/path/file.js?query#fragment: 148 | // [file] - /some/path/file.js 149 | // [query] - ?query 150 | // [fragment] - #fragment 151 | // [base] - file.js 152 | // [path] - /some/path/ 153 | // [name] - file 154 | // [ext] - .js 155 | if (typeof data.filename === 'string') { 156 | const { path: file, query, fragment } = parseResource(data.filename); 157 | 158 | const ext = extname(file); 159 | const base = basename(file); 160 | const name = base.slice(0, base.length - ext.length); 161 | // eslint-disable-next-line no-shadow 162 | const path = file.slice(0, file.length - base.length); 163 | 164 | replacements.set('file', replacer(file)); 165 | replacements.set('query', replacer(query, true)); 166 | replacements.set('fragment', replacer(fragment, true)); 167 | replacements.set('path', replacer(path, true)); 168 | replacements.set('base', replacer(base)); 169 | replacements.set('name', replacer(name)); 170 | replacements.set('ext', replacer(ext, true)); 171 | // Legacy 172 | replacements.set( 173 | 'filebase', 174 | deprecated( 175 | replacer(base), 176 | '[filebase] is now [base]', 177 | 'DEP_WEBPACK_TEMPLATE_PATH_PLUGIN_REPLACE_PATH_VARIABLES_FILENAME', 178 | ), 179 | ); 180 | } 181 | 182 | // Compilation context 183 | // 184 | // Placeholders 185 | // 186 | // [fullhash] - data.hash (3a4b5c6e7f) 187 | // 188 | // Legacy Placeholders 189 | // 190 | // [hash] - data.hash (3a4b5c6e7f) 191 | if (data.hash) { 192 | const hashReplacer = hashLength( 193 | replacer(data.hash), 194 | data.hashWithLength, 195 | assetInfo, 196 | 'fullhash', 197 | ); 198 | 199 | replacements.set('fullhash', hashReplacer); 200 | 201 | // Legacy 202 | replacements.set( 203 | 'hash', 204 | deprecated( 205 | hashReplacer, 206 | '[hash] is now [fullhash] (also consider using [chunkhash] or [contenthash], see documentation for details)', 207 | 'DEP_WEBPACK_TEMPLATE_PATH_PLUGIN_REPLACE_PATH_VARIABLES_HASH', 208 | ), 209 | ); 210 | } 211 | 212 | // Chunk Context 213 | // 214 | // Placeholders 215 | // 216 | // [id] - chunk.id (0.js) 217 | // [name] - chunk.name (app.js) 218 | // [chunkhash] - chunk.hash (7823t4t4.js) 219 | // [contenthash] - chunk.contentHash[type] (3256u3zg.js) 220 | if (data.chunk) { 221 | const { chunk } = data; 222 | 223 | const { contentHashType } = data; 224 | 225 | const idReplacer = replacer(chunk.id); 226 | const nameReplacer = replacer(chunk.name || chunk.id); 227 | const chunkhashReplacer = hashLength( 228 | replacer(chunk instanceof Chunk ? chunk.renderedHash : chunk.hash), 229 | // eslint-disable-next-line no-undefined 230 | 'hashWithLength' in chunk ? chunk.hashWithLength : undefined, 231 | assetInfo, 232 | 'chunkhash', 233 | ); 234 | const contenthashReplacer = hashLength( 235 | replacer( 236 | data.contentHash || 237 | (contentHashType && 238 | chunk.contentHash && 239 | chunk.contentHash[contentHashType]), 240 | ), 241 | data.contentHashWithLength || 242 | ('contentHashWithLength' in chunk && chunk.contentHashWithLength 243 | ? chunk.contentHashWithLength[/** @type {string} */ (contentHashType)] 244 | : // eslint-disable-next-line no-undefined 245 | undefined), 246 | assetInfo, 247 | 'contenthash', 248 | ); 249 | 250 | replacements.set('id', idReplacer); 251 | replacements.set('name', nameReplacer); 252 | replacements.set('chunkhash', chunkhashReplacer); 253 | replacements.set('contenthash', contenthashReplacer); 254 | } 255 | 256 | // Module Context 257 | // 258 | // Placeholders 259 | // 260 | // [id] - module.id (2.png) 261 | // [hash] - module.hash (6237543873.png) 262 | // 263 | // Legacy Placeholders 264 | // 265 | // [moduleid] - module.id (2.png) 266 | // [modulehash] - module.hash (6237543873.png) 267 | if (data.module) { 268 | const { module } = data; 269 | 270 | const idReplacer = replacer(() => 271 | prepareId( 272 | module instanceof Module 273 | ? /** @type {ModuleId} */ 274 | (/** @type {ChunkGraph} */ (chunkGraph).getModuleId(module)) 275 | : module.id, 276 | ), 277 | ); 278 | const moduleHashReplacer = hashLength( 279 | replacer(() => 280 | module instanceof Module 281 | ? /** @type {ChunkGraph} */ 282 | (chunkGraph).getRenderedModuleHash(module, data.runtime) 283 | : module.hash, 284 | ), 285 | // eslint-disable-next-line no-undefined 286 | 'hashWithLength' in module ? module.hashWithLength : undefined, 287 | assetInfo, 288 | 'modulehash', 289 | ); 290 | const contentHashReplacer = hashLength( 291 | replacer(/** @type {string} */ (data.contentHash)), 292 | // eslint-disable-next-line no-undefined 293 | undefined, 294 | assetInfo, 295 | 'contenthash', 296 | ); 297 | 298 | replacements.set('id', idReplacer); 299 | replacements.set('modulehash', moduleHashReplacer); 300 | replacements.set('contenthash', contentHashReplacer); 301 | replacements.set( 302 | 'hash', 303 | data.contentHash ? contentHashReplacer : moduleHashReplacer, 304 | ); 305 | // Legacy 306 | replacements.set( 307 | 'moduleid', 308 | deprecated( 309 | idReplacer, 310 | '[moduleid] is now [id]', 311 | 'DEP_WEBPACK_TEMPLATE_PATH_PLUGIN_REPLACE_PATH_VARIABLES_MODULE_ID', 312 | ), 313 | ); 314 | } 315 | 316 | // Other things 317 | if (data.url) { 318 | replacements.set('url', replacer(data.url)); 319 | } 320 | if (typeof data.runtime === 'string') { 321 | replacements.set( 322 | 'runtime', 323 | replacer(() => prepareId(/** @type {string} */ (data.runtime))), 324 | ); 325 | } else { 326 | replacements.set('runtime', replacer('_')); 327 | } 328 | 329 | if (typeof path === 'function') { 330 | // eslint-disable-next-line no-param-reassign 331 | path = path(data, assetInfo); 332 | } 333 | 334 | // eslint-disable-next-line no-param-reassign 335 | path = path.replace(REGEXP, (match, content) => { 336 | if (content.length + 2 === match.length) { 337 | const contentMatch = /^(\w+)(?::(\w+))?$/.exec(content); 338 | if (!contentMatch) return match; 339 | const [, kind, arg] = contentMatch; 340 | // eslint-disable-next-line no-shadow 341 | const replacer = replacements.get(kind); 342 | // eslint-disable-next-line no-undefined 343 | if (replacer !== undefined) { 344 | return replacer(match, arg, path); 345 | } 346 | } else if (match.startsWith('[\\') && match.endsWith('\\]')) { 347 | return `[${match.slice(2, -2)}]`; 348 | } 349 | return match; 350 | }); 351 | 352 | return path; 353 | }; 354 | 355 | module.exports = replacePathVariables; 356 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | ### [4.0.4](https://github.com/webpack-contrib/thread-loader/compare/v4.0.3...v4.0.4) (2024-09-18) 6 | 7 | 8 | ### Bug Fixes 9 | 10 | * respect cacheable ([d3bf829](https://github.com/webpack-contrib/thread-loader/commit/d3bf8294f4ff863a58091450534d6ce24fb06fe4)) 11 | * support `hot` ([#219](https://github.com/webpack-contrib/thread-loader/issues/219)) ([2de3b64](https://github.com/webpack-contrib/thread-loader/commit/2de3b640cd5d8be674654942c97c8f8de7ed3f3e)) 12 | * support `utils.createHash` ([#222](https://github.com/webpack-contrib/thread-loader/issues/222)) ([6bfc93c](https://github.com/webpack-contrib/thread-loader/commit/6bfc93cab75a8f44d35a7ac0b26745cda5168dfe)) 13 | 14 | ### [4.0.3](https://github.com/webpack-contrib/thread-loader/compare/v4.0.2...v4.0.3) (2024-09-17) 15 | 16 | 17 | ### Bug Fixes 18 | 19 | * getLogger is now defined in child loader ([158d52b](https://github.com/webpack-contrib/thread-loader/commit/158d52b4a8e97d2489b305be262f1994f9eeee76)) 20 | * pass `mode` ([#217](https://github.com/webpack-contrib/thread-loader/issues/217)) ([57c66b8](https://github.com/webpack-contrib/thread-loader/commit/57c66b8d93ac3ba547052345523b5dacb8ea255d)) 21 | * work with `importModule` ([#218](https://github.com/webpack-contrib/thread-loader/issues/218)) ([75b9b7a](https://github.com/webpack-contrib/thread-loader/commit/75b9b7aaba709e8282e27fa8196379d1712c886f)) 22 | 23 | ### [4.0.2](https://github.com/webpack-contrib/thread-loader/compare/v4.0.1...v4.0.2) (2023-05-22) 24 | 25 | 26 | ### Bug Fixes 27 | 28 | * compatibility with ts-loader ([#183](https://github.com/webpack-contrib/thread-loader/issues/183)) ([cbc9722](https://github.com/webpack-contrib/thread-loader/commit/cbc97221ca9f625b0b7d53d2570fb2c34d663c3b)) 29 | 30 | ### [4.0.1](https://github.com/webpack-contrib/thread-loader/compare/v4.0.0...v4.0.1) (2023-04-19) 31 | 32 | 33 | ### Bug Fixes 34 | 35 | * parsing circular json ([#163](https://github.com/webpack-contrib/thread-loader/issues/163)) ([71af48d](https://github.com/webpack-contrib/thread-loader/commit/71af48d18da286f7105a25a4adc9e48ae73e6918)) 36 | 37 | ## [4.0.0](https://github.com/webpack-contrib/thread-loader/compare/v3.0.4...v4.0.0) (2023-04-18) 38 | 39 | 40 | ### ⚠ BREAKING CHANGES 41 | 42 | * minimum supported `Node.js` version is `16.10.0` 43 | * minimum supported `webpack` version is `5` 44 | 45 | ### [3.0.4](https://github.com/webpack-contrib/thread-loader/compare/v3.0.3...v3.0.4) (2021-05-10) 46 | 47 | 48 | ### Bug Fixes 49 | 50 | * do not crash on `this.addBuildDependency` usage ([#117](https://github.com/webpack-contrib/thread-loader/issues/117)) ([1c7a8a2](https://github.com/webpack-contrib/thread-loader/commit/1c7a8a2454c7540a226b2a7fa6e0cbfef6ebf2c6)) 51 | * `this.addMissingDependency` works fine ([#119](https://github.com/webpack-contrib/thread-loader/issues/119)) ([5a0ea0c](https://github.com/webpack-contrib/thread-loader/commit/5a0ea0c4239e69cffd68e79a01f9615250c66755)) 52 | 53 | ### [3.0.3](https://github.com/webpack-contrib/thread-loader/compare/v3.0.2...v3.0.3) (2021-04-13) 54 | 55 | 56 | ### Bug Fixes 57 | 58 | * `getOptions` usage ([#113](https://github.com/webpack-contrib/thread-loader/issues/113)) ([d7531ef](https://github.com/webpack-contrib/thread-loader/commit/d7531efd39b90eff3e6cdd5e6917997f5b392bff)) 59 | 60 | ### [3.0.2](https://github.com/webpack-contrib/thread-loader/compare/v3.0.1...v3.0.2) (2021-04-12) 61 | 62 | 63 | ### Bug Fixes 64 | 65 | * support serialization of RegExp ([#102](https://github.com/webpack-contrib/thread-loader/issues/102)) ([3766560](https://github.com/webpack-contrib/thread-loader/commit/37665608bea01c4072fa974b038de1352a82961c)) 66 | 67 | ### [3.0.1](https://github.com/webpack-contrib/thread-loader/compare/v3.0.0...v3.0.1) (2020-10-27) 68 | 69 | 70 | ### Bug Fixes 71 | 72 | * pass rootContext to loaders ([#104](https://github.com/webpack-contrib/thread-loader/issues/104)) ([8e56785](https://github.com/webpack-contrib/thread-loader/commit/8e567853efa3a0d6b95423d3598a68ad77598bc4)) 73 | 74 | ## [3.0.0](https://github.com/webpack-contrib/thread-loader/compare/v2.1.3...v3.0.0) (2020-09-12) 75 | 76 | 77 | ### ⚠ BREAKING CHANGES 78 | 79 | * minimum supported `Node.js` version is `10.13` 80 | 81 | ### Bug Fixes 82 | 83 | * `loadModule` and `fs` are now available in a loader context ([#88](https://github.com/webpack-contrib/thread-loader/issues/88)) ([ea5c9ad](https://github.com/webpack-contrib/thread-loader/commit/ea5c9ad8ffd3898e1fe136cc3cf371b3d15e3f97)) 84 | * `getResolve` is now available in a loader context ([#99](https://github.com/webpack-contrib/thread-loader/issues/99)) ([16bbc23](https://github.com/webpack-contrib/thread-loader/commit/16bbc236dfdc26c857c97c8c005bbad6883c49ed)) 85 | 86 | 87 | ## [2.1.3](https://github.com/webpack-contrib/thread-loader/compare/v2.1.2...v2.1.3) (2019-08-08) 88 | 89 | 90 | ### Bug Fixes 91 | 92 | * correct default for workerParallelJobs option ([#74](https://github.com/webpack-contrib/thread-loader/issues/74)) ([79758d0](https://github.com/webpack-contrib/thread-loader/commit/79758d0)) 93 | * do not allow empty or invalid node args when spin up child process ([#73](https://github.com/webpack-contrib/thread-loader/issues/73)) ([b02d503](https://github.com/webpack-contrib/thread-loader/commit/b02d503)) 94 | 95 | 96 | 97 | 98 | ## [2.1.2](https://github.com/webpack-contrib/thread-loader/compare/v2.1.1...v2.1.2) (2019-01-25) 99 | 100 | 101 | ### Bug Fixes 102 | 103 | * lifecycle handling for signals 104 | 105 | 106 | 107 | 108 | ## [2.1.1](https://github.com/webpack-contrib/thread-loader/compare/v2.1.0...v2.1.1) (2018-12-21) 109 | 110 | 111 | ### Performance Improvements 112 | 113 | * use `neo-async` instead of `async` ([#54](https://github.com/webpack-contrib/thread-loader/issues/54)) ([d3a6664](https://github.com/webpack-contrib/thread-loader/commit/d3a6664)) 114 | 115 | 116 | 117 | 118 | # [2.1.0](https://github.com/webpack-contrib/thread-loader/compare/v2.0.2...v2.1.0) (2018-12-21) 119 | 120 | 121 | ### Features 122 | 123 | * add poolRespawn flag to speed up incremental builds ([#52](https://github.com/webpack-contrib/thread-loader/issues/52)) ([76535bf](https://github.com/webpack-contrib/thread-loader/commit/76535bf)) 124 | 125 | 126 | 127 | 128 | ## [2.0.2](https://github.com/webpack-contrib/thread-loader/compare/v2.0.1...v2.0.2) (2018-12-20) 129 | 130 | 131 | ### Bug Fixes 132 | 133 | * build hang ([#53](https://github.com/webpack-contrib/thread-loader/issues/53)) ([fa02b60](https://github.com/webpack-contrib/thread-loader/commit/fa02b60)) 134 | 135 | 136 | 137 | 138 | ## [2.0.1](https://github.com/webpack-contrib/thread-loader/compare/v2.0.0...v2.0.1) (2018-12-19) 139 | 140 | 141 | ### Bug Fixes 142 | 143 | * memory leaks, worker and main process lifecycles ([#51](https://github.com/webpack-contrib/thread-loader/issues/51)) ([f10fe55](https://github.com/webpack-contrib/thread-loader/commit/f10fe55)) 144 | 145 | 146 | 147 | 148 | ## [2.0.0](https://github.com/webpack-contrib/thread-loader/compare/v1.2.0...v2.0.0) (2018-12-18) 149 | 150 | 151 | ### Bug Fixes 152 | 153 | * calculate number of workers correctly ([#49](https://github.com/webpack-contrib/thread-loader/issues/49)) ([fcbd813](https://github.com/webpack-contrib/thread-loader/commit/fcbd813)) 154 | * check on `undefined` for `worker.stdio` ([#45](https://github.com/webpack-contrib/thread-loader/issues/45)) ([c891a9c](https://github.com/webpack-contrib/thread-loader/commit/c891a9c)) 155 | * listen `end` events ([#42](https://github.com/webpack-contrib/thread-loader/issues/42)) ([0f87683](https://github.com/webpack-contrib/thread-loader/commit/0f87683)) 156 | 157 | 158 | ### BREAKING CHANGE 159 | 160 | * drop support for node < 6.9 161 | 162 | 163 | 164 | 165 | # [1.2.0](https://github.com/webpack-contrib/thread-loader/compare/v1.1.5...v1.2.0) (2018-07-27) 166 | 167 | 168 | ### Features 169 | 170 | * add target, minimize and resourceQuery into context ([#25](https://github.com/webpack-contrib/thread-loader/issues/25)) ([f3c7a2c](https://github.com/webpack-contrib/thread-loader/commit/f3c7a2c)) 171 | 172 | 173 | 174 | 175 | ## [1.1.5](https://github.com/webpack-contrib/thread-loader/compare/v1.1.4...v1.1.5) (2018-02-26) 176 | 177 | 178 | ### Bug Fixes 179 | 180 | * **package:** add `webpack >= 4` (`peerDependencies`) ([#22](https://github.com/webpack-contrib/thread-loader/issues/22)) ([9345756](https://github.com/webpack-contrib/thread-loader/commit/9345756)) 181 | * **WorkerError:** handle undefined `error` stacks ([#20](https://github.com/webpack-contrib/thread-loader/issues/20)) ([6fb5daf](https://github.com/webpack-contrib/thread-loader/commit/6fb5daf)) 182 | 183 | 184 | 185 | 186 | ## [1.1.4](https://github.com/webpack-contrib/thread-loader/compare/v1.1.3...v1.1.4) (2018-02-21) 187 | 188 | 189 | ### Bug Fixes 190 | 191 | * **index:** add `webpack >= v4.0.0` support ([#16](https://github.com/webpack-contrib/thread-loader/issues/16)) ([5d33937](https://github.com/webpack-contrib/thread-loader/commit/5d33937)) 192 | 193 | 194 | 195 | 196 | ## [1.1.3](https://github.com/webpack-contrib/thread-loader/compare/v1.1.2...v1.1.3) (2018-02-07) 197 | 198 | 199 | ### Bug Fixes 200 | 201 | * **WorkerPool:** trace stacks to avoid duplicated `err.messages` from workers ([#13](https://github.com/webpack-contrib/thread-loader/issues/13)) ([80dda4f](https://github.com/webpack-contrib/thread-loader/commit/80dda4f)) 202 | 203 | 204 | 205 | 206 | ## [1.1.2](https://github.com/webpack-contrib/thread-loader/compare/v1.1.1...v1.1.2) (2017-10-09) 207 | 208 | 209 | ### Bug Fixes 210 | 211 | * **readBuffer:** handle 0-byte reads ([c7ca960](https://github.com/webpack-contrib/thread-loader/commit/c7ca960)) 212 | 213 | 214 | 215 | 216 | ## [1.1.1](https://github.com/webpack-contrib/thread-loader/compare/v1.1.0...v1.1.1) (2017-08-28) 217 | 218 | 219 | ### Bug Fixes 220 | 221 | * **context:** Pass context to loader ([29ced70](https://github.com/webpack-contrib/thread-loader/commit/29ced70)) 222 | * **deps:** pass along result for dependencies ([19832ec](https://github.com/webpack-contrib/thread-loader/commit/19832ec)) 223 | * **example:** fix for broken sass and add watch ([47ba43e](https://github.com/webpack-contrib/thread-loader/commit/47ba43e)) 224 | 225 | 226 | 227 | 228 | # [1.1.0](https://github.com/webpack-contrib/thread-loader/compare/v1.0.3...v1.1.0) (2017-07-14) 229 | 230 | 231 | ### Features 232 | 233 | * **pool:** add warmup method ([a0ce440](https://github.com/webpack-contrib/thread-loader/commit/a0ce440)) 234 | 235 | 236 | 237 | 238 | ## [1.0.3](https://github.com/webpack-contrib/thread-loader/compare/v1.0.2...v1.0.3) (2017-05-27) 239 | 240 | 241 | ### Bug Fixes 242 | 243 | * **resolve:** fix passing error to worker ([6561f57](https://github.com/webpack-contrib/thread-loader/commit/6561f57)) 244 | 245 | 246 | 247 | 248 | ## [1.0.2](https://github.com/webpack-contrib/thread-loader/compare/v1.0.1...v1.0.2) (2017-05-27) 249 | 250 | 251 | ### Bug Fixes 252 | 253 | * **resolve:** fix incorrect method for sending message ([bb92a28](https://github.com/webpack-contrib/thread-loader/commit/bb92a28)) 254 | 255 | 256 | 257 | 258 | ## 1.0.1 (2017-04-28) 259 | 260 | 261 | 262 | # Change Log 263 | 264 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 265 | 266 | x.x.x / -- 267 | ================== 268 | 269 | * Bug fix - 270 | * Feature - 271 | * Chore - 272 | * Docs - 273 | -------------------------------------------------------------------------------- /example/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "@types/json-schema": { 6 | "version": "7.0.6", 7 | "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.6.tgz", 8 | "integrity": "sha512-3c+yGKvVP5Y9TYBEibGNR+kLtijnj7mYrXRg+WpFb2X9xm04g/DXYkfg4hmzJQosc9snFNUPkbYIhu+KAm6jJw==", 9 | "dev": true 10 | }, 11 | "ajv": { 12 | "version": "6.12.4", 13 | "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.4.tgz", 14 | "integrity": "sha512-eienB2c9qVQs2KWexhkrdMLVDoIQCz5KSeLxwg9Lzk4DOfBtIK9PQwwufcsn1jjGuf9WZmqPMbGxOzfcuphJCQ==", 15 | "dev": true, 16 | "requires": { 17 | "fast-deep-equal": "^3.1.1", 18 | "fast-json-stable-stringify": "^2.0.0", 19 | "json-schema-traverse": "^0.4.1", 20 | "uri-js": "^4.2.2" 21 | } 22 | }, 23 | "ajv-keywords": { 24 | "version": "3.5.2", 25 | "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", 26 | "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==", 27 | "dev": true 28 | }, 29 | "babel-loader": { 30 | "version": "8.1.0", 31 | "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", 32 | "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", 33 | "dev": true, 34 | "requires": { 35 | "find-cache-dir": "^2.1.0", 36 | "loader-utils": "^1.4.0", 37 | "mkdirp": "^0.5.3", 38 | "pify": "^4.0.1", 39 | "schema-utils": "^2.6.5" 40 | } 41 | }, 42 | "big.js": { 43 | "version": "5.2.2", 44 | "resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz", 45 | "integrity": "sha512-vyL2OymJxmarO8gxMr0mhChsO9QGwhynfuu4+MHTAW6czfq9humCB7rKpUjDd9YUiDPU4mzpyupFSvOClAwbmQ==", 46 | "dev": true 47 | }, 48 | "commondir": { 49 | "version": "1.0.1", 50 | "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", 51 | "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=", 52 | "dev": true 53 | }, 54 | "emojis-list": { 55 | "version": "3.0.0", 56 | "resolved": "https://registry.npmjs.org/emojis-list/-/emojis-list-3.0.0.tgz", 57 | "integrity": "sha512-/kyM18EfinwXZbno9FyUGeFh87KC8HRQBQGildHZbEuRyWFOmv1U10o9BBp8XVZDVNNuQKyIGIu5ZYAAXJ0V2Q==", 58 | "dev": true 59 | }, 60 | "fast-deep-equal": { 61 | "version": "3.1.3", 62 | "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", 63 | "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", 64 | "dev": true 65 | }, 66 | "fast-json-stable-stringify": { 67 | "version": "2.1.0", 68 | "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", 69 | "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", 70 | "dev": true 71 | }, 72 | "find-cache-dir": { 73 | "version": "2.1.0", 74 | "resolved": "https://registry.npmjs.org/find-cache-dir/-/find-cache-dir-2.1.0.tgz", 75 | "integrity": "sha512-Tq6PixE0w/VMFfCgbONnkiQIVol/JJL7nRMi20fqzA4NRs9AfeqMGeRdPi3wIhYkxjeBaWh2rxwapn5Tu3IqOQ==", 76 | "dev": true, 77 | "requires": { 78 | "commondir": "^1.0.1", 79 | "make-dir": "^2.0.0", 80 | "pkg-dir": "^3.0.0" 81 | } 82 | }, 83 | "find-up": { 84 | "version": "3.0.0", 85 | "resolved": "https://registry.npmjs.org/find-up/-/find-up-3.0.0.tgz", 86 | "integrity": "sha512-1yD6RmLI1XBfxugvORwlck6f75tYL+iR0jqwsOrOxMZyGYqUuDhJ0l4AXdO1iX/FTs9cBAMEk1gWSEx1kSbylg==", 87 | "dev": true, 88 | "requires": { 89 | "locate-path": "^3.0.0" 90 | } 91 | }, 92 | "js-tokens": { 93 | "version": "4.0.0", 94 | "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", 95 | "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" 96 | }, 97 | "json-schema-traverse": { 98 | "version": "0.4.1", 99 | "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", 100 | "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", 101 | "dev": true 102 | }, 103 | "json5": { 104 | "version": "1.0.1", 105 | "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", 106 | "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", 107 | "dev": true, 108 | "requires": { 109 | "minimist": "^1.2.0" 110 | } 111 | }, 112 | "loader-utils": { 113 | "version": "1.4.0", 114 | "resolved": "https://registry.npmjs.org/loader-utils/-/loader-utils-1.4.0.tgz", 115 | "integrity": "sha512-qH0WSMBtn/oHuwjy/NucEgbx5dbxxnxup9s4PVXJUDHZBQY+s0NWA9rJf53RBnQZxfch7euUui7hpoAPvALZdA==", 116 | "dev": true, 117 | "requires": { 118 | "big.js": "^5.2.2", 119 | "emojis-list": "^3.0.0", 120 | "json5": "^1.0.1" 121 | } 122 | }, 123 | "locate-path": { 124 | "version": "3.0.0", 125 | "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-3.0.0.tgz", 126 | "integrity": "sha512-7AO748wWnIhNqAuaty2ZWHkQHRSNfPVIsPIfwEOWO22AmaoVrWavlOcMR5nzTLNYvp36X220/maaRsrec1G65A==", 127 | "dev": true, 128 | "requires": { 129 | "p-locate": "^3.0.0", 130 | "path-exists": "^3.0.0" 131 | } 132 | }, 133 | "lodash-es": { 134 | "version": "4.17.21", 135 | "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.21.tgz", 136 | "integrity": "sha512-mKnC+QJ9pWVzv+C4/U3rRsHapFfHvQFoFB92e52xeyGMcX6/OlIl78je1u8vePzYZSkkogMPJ2yjxxsb89cxyw==" 137 | }, 138 | "loose-envify": { 139 | "version": "1.4.0", 140 | "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", 141 | "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", 142 | "requires": { 143 | "js-tokens": "^3.0.0 || ^4.0.0" 144 | } 145 | }, 146 | "make-dir": { 147 | "version": "2.1.0", 148 | "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-2.1.0.tgz", 149 | "integrity": "sha512-LS9X+dc8KLxXCb8dni79fLIIUA5VyZoyjSMCwTluaXA0o27cCK0bhXkpgw+sTXVpPy/lSO57ilRixqk0vDmtRA==", 150 | "dev": true, 151 | "requires": { 152 | "pify": "^4.0.1", 153 | "semver": "^5.6.0" 154 | } 155 | }, 156 | "minimist": { 157 | "version": "1.2.8", 158 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", 159 | "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", 160 | "dev": true 161 | }, 162 | "mkdirp": { 163 | "version": "0.5.5", 164 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.5.tgz", 165 | "integrity": "sha512-NKmAlESf6jMGym1++R0Ra7wvhV+wFW63FaSOFPwRahvea0gMUcGUhVeAg/0BC0wiv9ih5NYPB1Wn1UEI1/L+xQ==", 166 | "dev": true, 167 | "requires": { 168 | "minimist": "^1.2.5" 169 | } 170 | }, 171 | "object-assign": { 172 | "version": "4.1.1", 173 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 174 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" 175 | }, 176 | "p-limit": { 177 | "version": "2.3.0", 178 | "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", 179 | "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", 180 | "dev": true, 181 | "requires": { 182 | "p-try": "^2.0.0" 183 | } 184 | }, 185 | "p-locate": { 186 | "version": "3.0.0", 187 | "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-3.0.0.tgz", 188 | "integrity": "sha512-x+12w/To+4GFfgJhBEpiDcLozRJGegY+Ei7/z0tSLkMmxGZNybVMSfWj9aJn8Z5Fc7dBUNJOOVgPv2H7IwulSQ==", 189 | "dev": true, 190 | "requires": { 191 | "p-limit": "^2.0.0" 192 | } 193 | }, 194 | "p-try": { 195 | "version": "2.2.0", 196 | "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", 197 | "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", 198 | "dev": true 199 | }, 200 | "path-exists": { 201 | "version": "3.0.0", 202 | "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-3.0.0.tgz", 203 | "integrity": "sha1-zg6+ql94yxiSXqfYENe1mwEP1RU=", 204 | "dev": true 205 | }, 206 | "pify": { 207 | "version": "4.0.1", 208 | "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", 209 | "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==", 210 | "dev": true 211 | }, 212 | "pkg-dir": { 213 | "version": "3.0.0", 214 | "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", 215 | "integrity": "sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==", 216 | "dev": true, 217 | "requires": { 218 | "find-up": "^3.0.0" 219 | } 220 | }, 221 | "punycode": { 222 | "version": "2.1.1", 223 | "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", 224 | "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==", 225 | "dev": true 226 | }, 227 | "react": { 228 | "version": "16.6.3", 229 | "resolved": "https://registry.npmjs.org/react/-/react-16.6.3.tgz", 230 | "integrity": "sha512-zCvmH2vbEolgKxtqXL2wmGCUxUyNheYn/C+PD1YAjfxHC54+MhdruyhO7QieQrYsYeTxrn93PM2y0jRH1zEExw==", 231 | "requires": { 232 | "loose-envify": "^1.1.0", 233 | "object-assign": "^4.1.1", 234 | "prop-types": "^15.6.2", 235 | "scheduler": "^0.11.2" 236 | }, 237 | "dependencies": { 238 | "prop-types": { 239 | "version": "15.6.2", 240 | "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.2.tgz", 241 | "integrity": "sha512-3pboPvLiWD7dkI3qf3KbUe6hKFKa52w+AE0VCqECtf+QHAKgOL37tTaNCnuX1nAAQ4ZhyP+kYVKf8rLmJ/feDQ==", 242 | "requires": { 243 | "loose-envify": "^1.3.1", 244 | "object-assign": "^4.1.1" 245 | } 246 | }, 247 | "scheduler": { 248 | "version": "0.11.3", 249 | "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.11.3.tgz", 250 | "integrity": "sha512-i9X9VRRVZDd3xZw10NY5Z2cVMbdYg6gqFecfj79USv1CFN+YrJ3gIPRKf1qlY+Sxly4djoKdfx1T+m9dnRB8kQ==", 251 | "requires": { 252 | "loose-envify": "^1.1.0", 253 | "object-assign": "^4.1.1" 254 | } 255 | } 256 | } 257 | }, 258 | "schema-utils": { 259 | "version": "2.7.1", 260 | "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", 261 | "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", 262 | "dev": true, 263 | "requires": { 264 | "@types/json-schema": "^7.0.5", 265 | "ajv": "^6.12.4", 266 | "ajv-keywords": "^3.5.2" 267 | } 268 | }, 269 | "semver": { 270 | "version": "5.7.1", 271 | "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", 272 | "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", 273 | "dev": true 274 | }, 275 | "uri-js": { 276 | "version": "4.4.0", 277 | "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.0.tgz", 278 | "integrity": "sha512-B0yRTzYdUCCn9n+F4+Gh4yIDtMQcaJsmYBDsTSG8g/OejKBodLQ2IHfN3bM7jUsRXndopT7OIXWdYqc1fjmV6g==", 279 | "dev": true, 280 | "requires": { 281 | "punycode": "^2.1.0" 282 | } 283 | } 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /src/WorkerPool.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | import childProcess from 'child_process'; 4 | 5 | import asyncQueue from 'neo-async/queue'; 6 | import asyncMapSeries from 'neo-async/mapSeries'; 7 | 8 | import readBuffer from './readBuffer'; 9 | import WorkerError from './WorkerError'; 10 | import { replacer, reviver } from './serializer'; 11 | 12 | const workerPath = require.resolve('./worker'); 13 | 14 | let workerId = 0; 15 | 16 | class PoolWorker { 17 | constructor(options, onJobDone) { 18 | this.disposed = false; 19 | this.nextJobId = 0; 20 | this.jobs = Object.create(null); 21 | this.activeJobs = 0; 22 | this.onJobDone = onJobDone; 23 | this.id = workerId; 24 | 25 | workerId += 1; 26 | // Empty or invalid node args would break the child process 27 | const sanitizedNodeArgs = (options.nodeArgs || []).filter((opt) => !!opt); 28 | 29 | this.worker = childProcess.spawn( 30 | process.execPath, 31 | [].concat(sanitizedNodeArgs).concat(workerPath, options.parallelJobs), 32 | { 33 | detached: true, 34 | stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'], 35 | }, 36 | ); 37 | 38 | this.worker.unref(); 39 | 40 | // This prevents a problem where the worker stdio can be undefined 41 | // when the kernel hits the limit of open files. 42 | // More info can be found on: https://github.com/webpack/thread-loader/issues/2 43 | if (!this.worker.stdio) { 44 | throw new Error( 45 | `Failed to create the worker pool with workerId: ${workerId} and ${''}configuration: ${JSON.stringify( 46 | options, 47 | )}. Please verify if you hit the OS open files limit.`, 48 | ); 49 | } 50 | 51 | const [, , , readPipe, writePipe] = this.worker.stdio; 52 | this.readPipe = readPipe; 53 | this.writePipe = writePipe; 54 | this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr); 55 | this.readNextMessage(); 56 | } 57 | 58 | listenStdOutAndErrFromWorker(workerStdout, workerStderr) { 59 | if (workerStdout) { 60 | workerStdout.on('data', this.writeToStdout); 61 | } 62 | 63 | if (workerStderr) { 64 | workerStderr.on('data', this.writeToStderr); 65 | } 66 | } 67 | 68 | ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) { 69 | if (workerStdout) { 70 | workerStdout.removeListener('data', this.writeToStdout); 71 | } 72 | 73 | if (workerStderr) { 74 | workerStderr.removeListener('data', this.writeToStderr); 75 | } 76 | } 77 | 78 | writeToStdout(data) { 79 | if (!this.disposed) { 80 | process.stdout.write(data); 81 | } 82 | } 83 | 84 | writeToStderr(data) { 85 | if (!this.disposed) { 86 | process.stderr.write(data); 87 | } 88 | } 89 | 90 | run(data, callback) { 91 | const jobId = this.nextJobId; 92 | this.nextJobId += 1; 93 | this.jobs[jobId] = { data, callback }; 94 | this.activeJobs += 1; 95 | this.writeJson({ 96 | type: 'job', 97 | id: jobId, 98 | data, 99 | }); 100 | } 101 | 102 | warmup(requires) { 103 | this.writeJson({ 104 | type: 'warmup', 105 | requires, 106 | }); 107 | } 108 | 109 | writeJson(data) { 110 | const lengthBuffer = Buffer.alloc(4); 111 | const messageBuffer = Buffer.from(JSON.stringify(data, replacer), 'utf-8'); 112 | lengthBuffer.writeInt32BE(messageBuffer.length, 0); 113 | this.writePipe.write(lengthBuffer); 114 | this.writePipe.write(messageBuffer); 115 | } 116 | 117 | writeEnd() { 118 | const lengthBuffer = Buffer.alloc(4); 119 | lengthBuffer.writeInt32BE(0, 0); 120 | this.writePipe.write(lengthBuffer); 121 | } 122 | 123 | readNextMessage() { 124 | this.state = 'read length'; 125 | this.readBuffer(4, (lengthReadError, lengthBuffer) => { 126 | if (lengthReadError) { 127 | console.error( 128 | `Failed to communicate with worker (read length) ${lengthReadError}`, 129 | ); 130 | return; 131 | } 132 | this.state = 'length read'; 133 | const length = lengthBuffer.readInt32BE(0); 134 | 135 | this.state = 'read message'; 136 | this.readBuffer(length, (messageError, messageBuffer) => { 137 | if (messageError) { 138 | console.error( 139 | `Failed to communicate with worker (read message) ${messageError}`, 140 | ); 141 | return; 142 | } 143 | this.state = 'message read'; 144 | const messageString = messageBuffer.toString('utf-8'); 145 | const message = JSON.parse(messageString, reviver); 146 | this.state = 'process message'; 147 | this.onWorkerMessage(message, (err) => { 148 | if (err) { 149 | console.error( 150 | `Failed to communicate with worker (process message) ${err}`, 151 | ); 152 | return; 153 | } 154 | this.state = 'soon next'; 155 | setImmediate(() => this.readNextMessage()); 156 | }); 157 | }); 158 | }); 159 | } 160 | 161 | onWorkerMessage(message, finalCallback) { 162 | const { type, id } = message; 163 | switch (type) { 164 | case 'job': { 165 | const { data, error, result } = message; 166 | asyncMapSeries( 167 | data, 168 | (length, callback) => this.readBuffer(length, callback), 169 | (eachErr, buffers) => { 170 | const { callback: jobCallback } = this.jobs[id]; 171 | const callback = (err, arg) => { 172 | if (jobCallback) { 173 | delete this.jobs[id]; 174 | this.activeJobs -= 1; 175 | this.onJobDone(); 176 | if (err) { 177 | jobCallback(err instanceof Error ? err : new Error(err), arg); 178 | } else { 179 | jobCallback(null, arg); 180 | } 181 | } 182 | finalCallback(); 183 | }; 184 | if (eachErr) { 185 | callback(eachErr); 186 | return; 187 | } 188 | let bufferPosition = 0; 189 | if (result.result) { 190 | result.result = result.result.map((r) => { 191 | if (r.buffer) { 192 | const buffer = buffers[bufferPosition]; 193 | bufferPosition += 1; 194 | if (r.string) { 195 | return buffer.toString('utf-8'); 196 | } 197 | return buffer; 198 | } 199 | return r.data; 200 | }); 201 | } 202 | if (error) { 203 | callback(this.fromErrorObj(error), result); 204 | return; 205 | } 206 | callback(null, result); 207 | }, 208 | ); 209 | break; 210 | } 211 | case 'loadModule': { 212 | const { request, questionId } = message; 213 | const { data } = this.jobs[id]; 214 | // eslint-disable-next-line no-unused-vars 215 | data.loadModule(request, (error, source, sourceMap, module) => { 216 | this.writeJson({ 217 | type: 'result', 218 | id: questionId, 219 | error: error 220 | ? { 221 | message: error.message, 222 | details: error.details, 223 | missing: error.missing, 224 | } 225 | : null, 226 | result: [ 227 | source, 228 | sourceMap, 229 | // TODO: Serialize module? 230 | // module, 231 | ], 232 | }); 233 | }); 234 | finalCallback(); 235 | break; 236 | } 237 | case 'importModule': { 238 | const { request, options, questionId } = message; 239 | const { data } = this.jobs[id]; 240 | 241 | data 242 | .importModule(request, options) 243 | .then((result) => { 244 | this.writeJson({ 245 | type: 'result', 246 | id: questionId, 247 | error: null, 248 | result, 249 | }); 250 | }) 251 | .catch((error) => { 252 | this.writeJson({ 253 | type: 'result', 254 | id: questionId, 255 | error: error 256 | ? { 257 | message: error.message, 258 | details: error.details, 259 | missing: error.missing, 260 | } 261 | : null, 262 | }); 263 | }); 264 | 265 | finalCallback(); 266 | break; 267 | } 268 | case 'resolve': { 269 | const { context, request, options, questionId } = message; 270 | const { data } = this.jobs[id]; 271 | if (options) { 272 | data.getResolve(options)(context, request, (error, result) => { 273 | this.writeJson({ 274 | type: 'result', 275 | id: questionId, 276 | error: error 277 | ? { 278 | message: error.message, 279 | details: error.details, 280 | missing: error.missing, 281 | } 282 | : null, 283 | result, 284 | }); 285 | }); 286 | } else { 287 | data.resolve(context, request, (error, result) => { 288 | this.writeJson({ 289 | type: 'result', 290 | id: questionId, 291 | error: error 292 | ? { 293 | message: error.message, 294 | details: error.details, 295 | missing: error.missing, 296 | } 297 | : null, 298 | result, 299 | }); 300 | }); 301 | } 302 | finalCallback(); 303 | break; 304 | } 305 | case 'emitWarning': { 306 | const { data } = message; 307 | const { data: jobData } = this.jobs[id]; 308 | jobData.emitWarning(this.fromErrorObj(data)); 309 | finalCallback(); 310 | break; 311 | } 312 | case 'emitError': { 313 | const { data } = message; 314 | const { data: jobData } = this.jobs[id]; 315 | jobData.emitError(this.fromErrorObj(data)); 316 | finalCallback(); 317 | break; 318 | } 319 | case 'getLogger': { 320 | // initialise logger by name in jobData 321 | const { data } = message; 322 | const { data: jobData } = this.jobs[id]; 323 | const internalName = data.name || '__internal__'; 324 | if (!Object.hasOwnProperty.call(jobData.loggers, internalName)) { 325 | jobData.loggers[internalName] = jobData.getLogger(data.name); 326 | } 327 | finalCallback(); 328 | break; 329 | } 330 | case 'logger': { 331 | const { data } = message; 332 | const { data: jobData } = this.jobs[id]; 333 | const internalName = data.name || '__internal__'; 334 | const logger = jobData.loggers[internalName]; 335 | logger[data.method](...data.args); 336 | finalCallback(); 337 | break; 338 | } 339 | default: { 340 | console.error(`Unexpected worker message ${type} in WorkerPool.`); 341 | finalCallback(); 342 | break; 343 | } 344 | } 345 | } 346 | 347 | fromErrorObj(arg) { 348 | let obj; 349 | if (typeof arg === 'string') { 350 | obj = { message: arg }; 351 | } else { 352 | obj = arg; 353 | } 354 | return new WorkerError(obj, this.id); 355 | } 356 | 357 | readBuffer(length, callback) { 358 | readBuffer(this.readPipe, length, callback); 359 | } 360 | 361 | dispose() { 362 | if (!this.disposed) { 363 | this.disposed = true; 364 | this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr); 365 | this.writeEnd(); 366 | } 367 | } 368 | } 369 | 370 | export default class WorkerPool { 371 | constructor(options) { 372 | this.options = options || {}; 373 | this.numberOfWorkers = options.numberOfWorkers; 374 | this.poolTimeout = options.poolTimeout; 375 | this.workerNodeArgs = options.workerNodeArgs; 376 | this.workerParallelJobs = options.workerParallelJobs; 377 | this.workers = new Set(); 378 | this.activeJobs = 0; 379 | this.timeout = null; 380 | this.poolQueue = asyncQueue( 381 | this.distributeJob.bind(this), 382 | options.poolParallelJobs, 383 | ); 384 | this.terminated = false; 385 | 386 | this.setupLifeCycle(); 387 | } 388 | 389 | isAbleToRun() { 390 | return !this.terminated; 391 | } 392 | 393 | terminate() { 394 | if (this.terminated) { 395 | return; 396 | } 397 | 398 | this.terminated = true; 399 | this.poolQueue.kill(); 400 | this.disposeWorkers(true); 401 | } 402 | 403 | setupLifeCycle() { 404 | process.on('exit', () => { 405 | this.terminate(); 406 | }); 407 | } 408 | 409 | run(data, callback) { 410 | if (this.timeout) { 411 | clearTimeout(this.timeout); 412 | this.timeout = null; 413 | } 414 | this.activeJobs += 1; 415 | this.poolQueue.push(data, callback); 416 | } 417 | 418 | distributeJob(data, callback) { 419 | // use worker with the fewest jobs 420 | let bestWorker; 421 | for (const worker of this.workers) { 422 | if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) { 423 | bestWorker = worker; 424 | } 425 | } 426 | if ( 427 | bestWorker && 428 | (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers) 429 | ) { 430 | bestWorker.run(data, callback); 431 | return; 432 | } 433 | const newWorker = this.createWorker(); 434 | newWorker.run(data, callback); 435 | } 436 | 437 | createWorker() { 438 | // spin up a new worker 439 | const newWorker = new PoolWorker( 440 | { 441 | nodeArgs: this.workerNodeArgs, 442 | parallelJobs: this.workerParallelJobs, 443 | }, 444 | () => this.onJobDone(), 445 | ); 446 | this.workers.add(newWorker); 447 | return newWorker; 448 | } 449 | 450 | warmup(requires) { 451 | while (this.workers.size < this.numberOfWorkers) { 452 | this.createWorker().warmup(requires); 453 | } 454 | } 455 | 456 | onJobDone() { 457 | this.activeJobs -= 1; 458 | if (this.activeJobs === 0 && isFinite(this.poolTimeout)) { 459 | this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout); 460 | } 461 | } 462 | 463 | disposeWorkers(fromTerminate) { 464 | if (!this.options.poolRespawn && !fromTerminate) { 465 | this.terminate(); 466 | return; 467 | } 468 | 469 | if (this.activeJobs === 0 || fromTerminate) { 470 | for (const worker of this.workers) { 471 | worker.dispose(); 472 | } 473 | this.workers.clear(); 474 | } 475 | } 476 | } 477 | -------------------------------------------------------------------------------- /src/worker.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | import fs from 'fs'; 3 | import NativeModule from 'module'; 4 | 5 | import querystring from 'querystring'; 6 | 7 | import loaderRunner from 'loader-runner'; 8 | import asyncQueue from 'neo-async/queue'; 9 | import parseJson from 'json-parse-better-errors'; 10 | import { validate } from 'schema-utils'; 11 | 12 | import readBuffer from './readBuffer'; 13 | import { replacer, reviver } from './serializer'; 14 | 15 | const writePipe = fs.createWriteStream(null, { fd: 3 }); 16 | const readPipe = fs.createReadStream(null, { fd: 4 }); 17 | 18 | writePipe.on('finish', onTerminateWrite); 19 | readPipe.on('end', onTerminateRead); 20 | writePipe.on('close', onTerminateWrite); 21 | readPipe.on('close', onTerminateRead); 22 | 23 | readPipe.on('error', onError); 24 | writePipe.on('error', onError); 25 | 26 | const PARALLEL_JOBS = +process.argv[2] || 20; 27 | 28 | let terminated = false; 29 | let nextQuestionId = 0; 30 | const callbackMap = Object.create(null); 31 | 32 | function onError(error) { 33 | console.error(error); 34 | } 35 | 36 | function onTerminateRead() { 37 | terminateRead(); 38 | } 39 | 40 | function onTerminateWrite() { 41 | terminateWrite(); 42 | } 43 | 44 | function writePipeWrite(...args) { 45 | if (!terminated) { 46 | writePipe.write(...args); 47 | } 48 | } 49 | 50 | function writePipeCork() { 51 | if (!terminated) { 52 | writePipe.cork(); 53 | } 54 | } 55 | 56 | function writePipeUncork() { 57 | if (!terminated) { 58 | writePipe.uncork(); 59 | } 60 | } 61 | 62 | function terminateRead() { 63 | terminated = true; 64 | readPipe.removeAllListeners(); 65 | } 66 | 67 | function terminateWrite() { 68 | terminated = true; 69 | writePipe.removeAllListeners(); 70 | } 71 | 72 | function terminate() { 73 | terminateRead(); 74 | terminateWrite(); 75 | } 76 | 77 | function toErrorObj(err) { 78 | return { 79 | message: err.message, 80 | details: err.details, 81 | stack: err.stack, 82 | hideStack: err.hideStack, 83 | }; 84 | } 85 | 86 | function toNativeError(obj) { 87 | if (!obj) return null; 88 | const err = new Error(obj.message); 89 | err.details = obj.details; 90 | err.missing = obj.missing; 91 | return err; 92 | } 93 | 94 | function writeJson(data) { 95 | writePipeCork(); 96 | process.nextTick(() => { 97 | writePipeUncork(); 98 | }); 99 | 100 | const lengthBuffer = Buffer.alloc(4); 101 | const messageBuffer = Buffer.from(JSON.stringify(data, replacer), 'utf-8'); 102 | lengthBuffer.writeInt32BE(messageBuffer.length, 0); 103 | 104 | writePipeWrite(lengthBuffer); 105 | writePipeWrite(messageBuffer); 106 | } 107 | 108 | const queue = asyncQueue(({ id, data }, taskCallback) => { 109 | try { 110 | const resolveWithOptions = (context, request, callback, options) => { 111 | callbackMap[nextQuestionId] = callback; 112 | writeJson({ 113 | type: 'resolve', 114 | id, 115 | questionId: nextQuestionId, 116 | context, 117 | request, 118 | options, 119 | }); 120 | nextQuestionId += 1; 121 | }; 122 | const importModule = (request, options, callback) => { 123 | callbackMap[nextQuestionId] = callback; 124 | writeJson({ 125 | type: 'importModule', 126 | id, 127 | questionId: nextQuestionId, 128 | request, 129 | options, 130 | }); 131 | nextQuestionId += 1; 132 | }; 133 | 134 | const buildDependencies = []; 135 | 136 | // eslint-disable-next-line no-underscore-dangle, no-param-reassign 137 | data._compilation.getPath = function getPath(filename, extraData = {}) { 138 | if (!extraData.hash) { 139 | // eslint-disable-next-line no-param-reassign 140 | extraData = { 141 | // eslint-disable-next-line no-underscore-dangle 142 | hash: data._compilation.hash, 143 | ...extraData, 144 | }; 145 | } 146 | 147 | // eslint-disable-next-line global-require 148 | const template = require('./template'); 149 | 150 | return template(filename, extraData); 151 | }; 152 | 153 | loaderRunner.runLoaders( 154 | { 155 | loaders: data.loaders, 156 | resource: data.resource, 157 | readResource: fs.readFile.bind(fs), 158 | context: { 159 | version: 2, 160 | fs, 161 | loadModule: (request, callback) => { 162 | callbackMap[nextQuestionId] = (error, result) => 163 | callback(error, ...result); 164 | writeJson({ 165 | type: 'loadModule', 166 | id, 167 | questionId: nextQuestionId, 168 | request, 169 | }); 170 | nextQuestionId += 1; 171 | }, 172 | // eslint-disable-next-line consistent-return 173 | importModule: (request, options, callback) => { 174 | if (callback) { 175 | importModule(request, options, callback); 176 | } else { 177 | return new Promise((resolve, reject) => { 178 | importModule(request, options, (err, result) => { 179 | if (err) { 180 | reject(err); 181 | } else { 182 | resolve(result); 183 | } 184 | }); 185 | }); 186 | } 187 | }, 188 | resolve: (context, request, callback) => { 189 | resolveWithOptions(context, request, callback); 190 | }, 191 | // eslint-disable-next-line consistent-return 192 | getResolve: (options) => (context, request, callback) => { 193 | if (callback) { 194 | resolveWithOptions(context, request, callback, options); 195 | } else { 196 | return new Promise((resolve, reject) => { 197 | resolveWithOptions( 198 | context, 199 | request, 200 | (err, result) => { 201 | if (err) { 202 | reject(err); 203 | } else { 204 | resolve(result); 205 | } 206 | }, 207 | options, 208 | ); 209 | }); 210 | } 211 | }, 212 | // Not an arrow function because it uses this 213 | getOptions(schema) { 214 | // loaders, loaderIndex will be defined by runLoaders 215 | const loader = this.loaders[this.loaderIndex]; 216 | 217 | // Verbatim copy from 218 | // https://github.com/webpack/webpack/blob/v5.31.2/lib/NormalModule.js#L471-L508 219 | // except eslint/prettier differences 220 | // -- unfortunate result of getOptions being synchronous functions. 221 | 222 | let { options } = loader; 223 | 224 | if (typeof options === 'string') { 225 | if (options.startsWith('{') && options.endsWith('}')) { 226 | try { 227 | options = parseJson(options); 228 | } catch (e) { 229 | throw new Error(`Cannot parse string options: ${e.message}`); 230 | } 231 | } else { 232 | options = querystring.parse(options, '&', '=', { 233 | maxKeys: 0, 234 | }); 235 | } 236 | } 237 | 238 | // eslint-disable-next-line no-undefined 239 | if (options === null || options === undefined) { 240 | options = {}; 241 | } 242 | 243 | if (schema) { 244 | let name = 'Loader'; 245 | let baseDataPath = 'options'; 246 | let match; 247 | // eslint-disable-next-line no-cond-assign 248 | if (schema.title && (match = /^(.+) (.+)$/.exec(schema.title))) { 249 | [, name, baseDataPath] = match; 250 | } 251 | validate(schema, options, { 252 | name, 253 | baseDataPath, 254 | }); 255 | } 256 | 257 | return options; 258 | }, 259 | getLogger: (name) => { 260 | function writeLoggerJson(method, args) { 261 | writeJson({ 262 | type: 'logger', 263 | id, 264 | data: { name, method, args }, 265 | }); 266 | } 267 | writeJson({ 268 | type: 'getLogger', 269 | id, 270 | data: { name }, 271 | }); 272 | // The logger interface should be aligned with the WebpackLogger class 273 | // https://github.com/webpack/webpack/blob/v5.94.0/lib/logging/Logger.js 274 | return { 275 | error(...args) { 276 | writeLoggerJson('error', args); 277 | }, 278 | 279 | warn(...args) { 280 | writeLoggerJson('warn', args); 281 | }, 282 | 283 | info(...args) { 284 | writeLoggerJson('info', args); 285 | }, 286 | 287 | log(...args) { 288 | writeLoggerJson('log', args); 289 | }, 290 | 291 | debug(...args) { 292 | writeLoggerJson('debug', args); 293 | }, 294 | 295 | assert(...args) { 296 | writeLoggerJson('assert', args); 297 | }, 298 | 299 | trace(...args) { 300 | writeLoggerJson('trace', args); 301 | }, 302 | 303 | clear(...args) { 304 | writeLoggerJson('clear', args); 305 | }, 306 | 307 | status(...args) { 308 | writeLoggerJson('status', args); 309 | }, 310 | 311 | group(...args) { 312 | writeLoggerJson('group', args); 313 | }, 314 | 315 | groupCollapsed(...args) { 316 | writeLoggerJson('groupCollapsed', args); 317 | }, 318 | 319 | groupEnd(...args) { 320 | writeLoggerJson('groupEnd', args); 321 | }, 322 | 323 | profile(...args) { 324 | writeLoggerJson('profile', args); 325 | }, 326 | 327 | profileEnd(...args) { 328 | writeLoggerJson('profileEnd', args); 329 | }, 330 | 331 | time(...args) { 332 | writeLoggerJson('time', args); 333 | }, 334 | 335 | timeLog(...args) { 336 | writeLoggerJson('timeLog', args); 337 | }, 338 | 339 | timeEnd(...args) { 340 | writeLoggerJson('timeEnd', args); 341 | }, 342 | 343 | timeAggregate(...args) { 344 | writeLoggerJson('timeAggregate', args); 345 | }, 346 | 347 | timeAggregateEnd(...args) { 348 | writeLoggerJson('timeAggregateEnd', args); 349 | }, 350 | }; 351 | }, 352 | emitWarning: (warning) => { 353 | writeJson({ 354 | type: 'emitWarning', 355 | id, 356 | data: toErrorObj(warning), 357 | }); 358 | }, 359 | emitError: (error) => { 360 | writeJson({ 361 | type: 'emitError', 362 | id, 363 | data: toErrorObj(error), 364 | }); 365 | }, 366 | exec: (code, filename) => { 367 | const module = new NativeModule(filename, this); 368 | module.paths = NativeModule._nodeModulePaths(this.context); // eslint-disable-line no-underscore-dangle 369 | module.filename = filename; 370 | module._compile(code, filename); // eslint-disable-line no-underscore-dangle 371 | return module.exports; 372 | }, 373 | addBuildDependency: (filename) => { 374 | buildDependencies.push(filename); 375 | }, 376 | options: { 377 | context: data.optionsContext, 378 | }, 379 | utils: { 380 | createHash: (type) => { 381 | // eslint-disable-next-line global-require 382 | const { createHash } = require('webpack').util; 383 | 384 | return createHash( 385 | // eslint-disable-next-line no-underscore-dangle 386 | type || data._compilation.outputOptions.hashFunction, 387 | ); 388 | }, 389 | }, 390 | webpack: true, 391 | 'thread-loader': true, 392 | mode: data.mode, 393 | sourceMap: data.sourceMap, 394 | target: data.target, 395 | minimize: data.minimize, 396 | resourceQuery: data.resourceQuery, 397 | resourceFragment: data.resourceFragment, 398 | environment: data.environment, 399 | rootContext: data.rootContext, 400 | hot: data.hot, 401 | // eslint-disable-next-line no-underscore-dangle 402 | _compilation: data._compilation, 403 | // eslint-disable-next-line no-underscore-dangle 404 | _compiler: data._compiler, 405 | resourcePath: data.resourcePath, 406 | }, 407 | }, 408 | (err, lrResult) => { 409 | const { 410 | result, 411 | cacheable, 412 | fileDependencies, 413 | contextDependencies, 414 | missingDependencies, 415 | } = lrResult; 416 | const buffersToSend = []; 417 | const convertedResult = 418 | Array.isArray(result) && 419 | result.map((item) => { 420 | const isBuffer = Buffer.isBuffer(item); 421 | if (isBuffer) { 422 | buffersToSend.push(item); 423 | return { 424 | buffer: true, 425 | }; 426 | } 427 | if (typeof item === 'string') { 428 | const stringBuffer = Buffer.from(item, 'utf-8'); 429 | buffersToSend.push(stringBuffer); 430 | return { 431 | buffer: true, 432 | string: true, 433 | }; 434 | } 435 | return { 436 | data: item, 437 | }; 438 | }); 439 | writeJson({ 440 | type: 'job', 441 | id, 442 | error: err && toErrorObj(err), 443 | result: { 444 | result: convertedResult, 445 | cacheable, 446 | fileDependencies, 447 | contextDependencies, 448 | missingDependencies, 449 | buildDependencies, 450 | }, 451 | data: buffersToSend.map((buffer) => buffer.length), 452 | }); 453 | buffersToSend.forEach((buffer) => { 454 | writePipeWrite(buffer); 455 | }); 456 | setImmediate(taskCallback); 457 | }, 458 | ); 459 | } catch (e) { 460 | writeJson({ 461 | type: 'job', 462 | id, 463 | error: toErrorObj(e), 464 | }); 465 | taskCallback(); 466 | } 467 | }, PARALLEL_JOBS); 468 | 469 | function dispose() { 470 | terminate(); 471 | 472 | queue.kill(); 473 | process.exit(0); 474 | } 475 | 476 | function onMessage(message) { 477 | try { 478 | const { type, id } = message; 479 | switch (type) { 480 | case 'job': { 481 | queue.push(message); 482 | break; 483 | } 484 | case 'result': { 485 | const { error, result } = message; 486 | const callback = callbackMap[id]; 487 | if (callback) { 488 | const nativeError = toNativeError(error); 489 | callback(nativeError, result); 490 | } else { 491 | console.error(`Worker got unexpected result id ${id}`); 492 | } 493 | delete callbackMap[id]; 494 | break; 495 | } 496 | case 'warmup': { 497 | const { requires } = message; 498 | // load modules into process 499 | requires.forEach((r) => require(r)); // eslint-disable-line import/no-dynamic-require, global-require 500 | break; 501 | } 502 | default: { 503 | console.error(`Worker got unexpected job type ${type}`); 504 | break; 505 | } 506 | } 507 | } catch (e) { 508 | console.error(`Error in worker ${e}`); 509 | } 510 | } 511 | 512 | function readNextMessage() { 513 | readBuffer(readPipe, 4, (lengthReadError, lengthBuffer) => { 514 | if (lengthReadError) { 515 | console.error( 516 | `Failed to communicate with main process (read length) ${lengthReadError}`, 517 | ); 518 | return; 519 | } 520 | 521 | const length = lengthBuffer.length && lengthBuffer.readInt32BE(0); 522 | 523 | if (length === 0) { 524 | // worker should dispose and exit 525 | dispose(); 526 | return; 527 | } 528 | readBuffer(readPipe, length, (messageError, messageBuffer) => { 529 | if (terminated) { 530 | return; 531 | } 532 | 533 | if (messageError) { 534 | console.error( 535 | `Failed to communicate with main process (read message) ${messageError}`, 536 | ); 537 | return; 538 | } 539 | const messageString = messageBuffer.toString('utf-8'); 540 | const message = JSON.parse(messageString, reviver); 541 | 542 | onMessage(message); 543 | setImmediate(() => readNextMessage()); 544 | }); 545 | }); 546 | } 547 | 548 | // start reading messages from main process 549 | readNextMessage(); 550 | --------------------------------------------------------------------------------