├── .eslintignore ├── .eslintrc ├── src ├── cjs.js ├── WorkerError.js ├── readBuffer.js ├── workerPools.js ├── index.js ├── worker.js └── WorkerPool.js ├── example ├── _shared.scss ├── style.scss ├── package.json ├── .babelrc ├── index.js ├── webpack.config.js └── package-lock.json ├── .gitattributes ├── test ├── __snapshots__ │ ├── pitch.test.js.snap │ └── workerPool.test.js.snap ├── pitch.test.js ├── readBuffer.test.js └── workerPool.test.js ├── .editorconfig ├── .gitignore ├── .github ├── PULL_REQUEST_TEMPLATE.md └── ISSUE_TEMPLATE.md ├── appveyor.yml ├── .babelrc ├── LICENSE ├── .travis.yml ├── package.json ├── README.md └── CHANGELOG.md /.eslintignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | /dist -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "webpack" 3 | } -------------------------------------------------------------------------------- /src/cjs.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./index'); 2 | -------------------------------------------------------------------------------- /example/_shared.scss: -------------------------------------------------------------------------------- 1 | body { 2 | background: red; 3 | } 4 | -------------------------------------------------------------------------------- /example/style.scss: -------------------------------------------------------------------------------- 1 | @import '_shared'; 2 | 3 | body { 4 | background: red; 5 | } 6 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | yarn.lock -diff 2 | * text=auto 3 | bin/* eol=lf 4 | package-lock.json -diff -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "lodash-es": "^4.17.4", 4 | "react": "^16.6.3" 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/__snapshots__/pitch.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`runs pitch unsuccessfully when workPool throw an error 1`] = `"Unexpected Error"`; 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | 3 | [*] 4 | charset = utf-8 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | 11 | [.md] 12 | insert_final_newline = false 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /test/__snapshots__/workerPool.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`workerPool should throw an error when worker.stdio is undefined 1`] = `"Failed to create the worker pool with workerId: 1 and configuration: {}. Please verify if you hit the OS open files limit."`; 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | logs 2 | *.log 3 | npm-debug.log* 4 | yarn-debug.log* 5 | .eslintcache 6 | /coverage 7 | /dist 8 | /example/dist 9 | /example/node_modules 10 | /local 11 | /reports 12 | /node_modules 13 | .DS_Store 14 | Thumbs.db 15 | .idea 16 | .vscode 17 | *.sublime-project 18 | *.sublime-workspace 19 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 8 | -------------------------------------------------------------------------------- /appveyor.yml: -------------------------------------------------------------------------------- 1 | branches: 2 | only: 3 | - master 4 | init: 5 | - git config --global core.autocrlf input 6 | environment: 7 | matrix: 8 | - nodejs_version: '8' 9 | webpack_version: latest 10 | job_part: test 11 | - nodejs_version: '6' 12 | webpack_version: latest 13 | job_part: test 14 | build: 'off' 15 | matrix: 16 | fast_finish: true 17 | install: 18 | - ps: Install-Product node $env:nodejs_version x64 19 | - npm i -g npm@latest 20 | - npm install 21 | before_test: 22 | - cmd: npm install webpack@%webpack_version% 23 | test_script: 24 | - node --version 25 | - npm --version 26 | - cmd: npm run appveyor:%job_part% 27 | -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", 5 | { 6 | "useBuiltIns": true, 7 | "targets": { 8 | "node": "6.9.0" 9 | }, 10 | "exclude": [ 11 | "transform-async-to-generator", 12 | "transform-regenerator" 13 | ] 14 | } 15 | ] 16 | ], 17 | "plugins": [ 18 | [ 19 | "transform-object-rest-spread", 20 | { 21 | "useBuiltIns": true 22 | } 23 | ] 24 | ], 25 | "env": { 26 | "test": { 27 | "presets": [ 28 | "env" 29 | ], 30 | "plugins": [ 31 | "transform-object-rest-spread" 32 | ] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /example/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", 5 | { 6 | "useBuiltIns": true, 7 | "targets": { 8 | "node": "6.9.0" 9 | }, 10 | "exclude": [ 11 | "transform-async-to-generator", 12 | "transform-regenerator" 13 | ] 14 | } 15 | ] 16 | ], 17 | "plugins": [ 18 | [ 19 | "transform-object-rest-spread", 20 | { 21 | "useBuiltIns": true 22 | } 23 | ] 24 | ], 25 | "env": { 26 | "test": { 27 | "presets": [ 28 | "env" 29 | ], 30 | "plugins": [ 31 | "transform-object-rest-spread" 32 | ] 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/WorkerError.js: -------------------------------------------------------------------------------- 1 | const stack = (err, worker, workerId) => { 2 | const originError = (err.stack || '') 3 | .split('\n') 4 | .filter(line => line.trim().startsWith('at')); 5 | 6 | const workerError = worker 7 | .split('\n') 8 | .filter(line => line.trim().startsWith('at')); 9 | 10 | const diff = workerError.slice(0, workerError.length - originError.length).join('\n'); 11 | 12 | originError.unshift(diff); 13 | originError.unshift(err.message); 14 | originError.unshift(`Thread Loader (Worker ${workerId})`); 15 | 16 | return originError.join('\n'); 17 | }; 18 | 19 | class WorkerError extends Error { 20 | constructor(err, workerId) { 21 | super(err); 22 | this.name = err.name; 23 | this.message = err.message; 24 | 25 | Error.captureStackTrace(this, this.constructor); 26 | 27 | this.stack = stack(err, this.stack, workerId); 28 | } 29 | } 30 | 31 | export default WorkerError; 32 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable import/no-unresolved */ 2 | // some file 3 | import './style.scss?00'; 4 | import './style.scss?01'; 5 | import './style.scss?02'; 6 | import './style.scss?03'; 7 | import './style.scss?04'; 8 | import './style.scss?05'; 9 | import './style.scss?06'; 10 | import './style.scss?07'; 11 | import './style.scss?08'; 12 | import './style.scss?09'; 13 | 14 | import './style.scss?10'; 15 | import './style.scss?11'; 16 | import './style.scss?12'; 17 | import './style.scss?13'; 18 | import './style.scss?14'; 19 | import './style.scss?15'; 20 | import './style.scss?16'; 21 | import './style.scss?17'; 22 | import './style.scss?18'; 23 | import './style.scss?19'; 24 | 25 | import './style.scss?20'; 26 | import './style.scss?21'; 27 | import './style.scss?22'; 28 | import './style.scss?23'; 29 | import './style.scss?24'; 30 | import './style.scss?25'; 31 | import './style.scss?26'; 32 | import './style.scss?27'; 33 | import './style.scss?28'; 34 | import './style.scss?29'; 35 | -------------------------------------------------------------------------------- /src/readBuffer.js: -------------------------------------------------------------------------------- 1 | export default function readBuffer(pipe, length, callback) { 2 | if (length === 0) { 3 | callback(null, Buffer.alloc(0)); 4 | return; 5 | } 6 | 7 | let remainingLength = length; 8 | const buffers = []; 9 | 10 | const readChunk = () => { 11 | const onChunk = (arg) => { 12 | let chunk = arg; 13 | let overflow; 14 | if (chunk.length > remainingLength) { 15 | overflow = chunk.slice(remainingLength); 16 | chunk = chunk.slice(0, remainingLength); 17 | remainingLength = 0; 18 | } else { 19 | remainingLength -= chunk.length; 20 | } 21 | buffers.push(chunk); 22 | if (remainingLength === 0) { 23 | pipe.removeListener('data', onChunk); 24 | pipe.pause(); 25 | 26 | if (overflow) { 27 | pipe.unshift(overflow); 28 | } 29 | 30 | callback(null, Buffer.concat(buffers, length)); 31 | } 32 | }; 33 | 34 | pipe.on('data', onChunk); 35 | pipe.resume(); 36 | }; 37 | readChunk(); 38 | } 39 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright JS Foundation and other contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | 'Software'), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 17 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 18 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 19 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 20 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /src/workerPools.js: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | import WorkerPool from './WorkerPool'; 3 | 4 | const workerPools = Object.create(null); 5 | 6 | function calculateNumberOfWorkers() { 7 | // There are situations when this call will return undefined so 8 | // we are fallback here to 1. 9 | // More info on: https://github.com/nodejs/node/issues/19022 10 | const cpus = os.cpus() || { length: 1 }; 11 | 12 | return Math.max(1, cpus.length - 1); 13 | } 14 | 15 | function getPool(options) { 16 | const workerPoolOptions = { 17 | name: options.name || '', 18 | numberOfWorkers: options.workers || calculateNumberOfWorkers(), 19 | workerNodeArgs: options.workerNodeArgs, 20 | workerParallelJobs: options.workerParallelJobs || 20, 21 | poolTimeout: options.poolTimeout || 500, 22 | poolParallelJobs: options.poolParallelJobs || 200, 23 | poolRespawn: options.poolRespawn || false, 24 | }; 25 | const tpKey = JSON.stringify(workerPoolOptions); 26 | workerPools[tpKey] = workerPools[tpKey] || new WorkerPool(workerPoolOptions); 27 | const workerPool = workerPools[tpKey]; 28 | return workerPool; 29 | } 30 | 31 | export { getPool }; // eslint-disable-line import/prefer-default-export 32 | -------------------------------------------------------------------------------- /test/pitch.test.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import { pitch } from '../src/cjs'; 3 | import { getPool } from '../src/workerPools'; 4 | 5 | jest.mock('../src/workerPools', () => { 6 | return { 7 | getPool: jest.fn(), 8 | }; 9 | }); 10 | 11 | const runGetPoolMock = (error) => { 12 | getPool.mockImplementationOnce(() => { 13 | return { 14 | isAbleToRun: () => true, 15 | run: jest.fn((opts, cb) => { 16 | cb(error, { 17 | fileDependencies: [], 18 | contextDependencies: [], 19 | result: {}, 20 | }); 21 | }), 22 | }; 23 | }); 24 | }; 25 | 26 | const runPitch = options => pitch.call( 27 | Object.assign( 28 | {}, 29 | { 30 | query: options, 31 | loaders: [], 32 | rootContext: path.resolve('../'), 33 | async: () => (error) => { 34 | if (error) { 35 | throw error; 36 | } 37 | }, 38 | }, 39 | ), 40 | ); 41 | 42 | it('runs pitch successfully when workPool not throw an error', () => { 43 | runGetPoolMock(null); 44 | expect(() => runPitch({})).not.toThrow(); 45 | }); 46 | 47 | it('runs pitch unsuccessfully when workPool throw an error', () => { 48 | runGetPoolMock(new Error('Unexpected Error')); 49 | expect(() => runPitch({})).toThrowErrorMatchingSnapshot(); 50 | }); 51 | -------------------------------------------------------------------------------- /test/readBuffer.test.js: -------------------------------------------------------------------------------- 1 | const stream = require('stream'); 2 | const readBuffer = require('../dist/readBuffer'); 3 | 4 | test('data is read', (done) => { 5 | expect.assertions(3); 6 | let eventCount = 0; 7 | function read() { 8 | eventCount += 1; 9 | if (eventCount <= 8) { 10 | return this.push(Buffer.from(eventCount.toString())); 11 | } 12 | return this.push(null); 13 | } 14 | const mockEventStream = new stream.Readable({ 15 | objectMode: true, 16 | read, 17 | }); 18 | function cb(err, data) { 19 | expect(err).toBe(null); 20 | expect(data.length).toBe(8); 21 | expect(String.fromCharCode(data[0])).toBe('1'); 22 | done(); 23 | } 24 | readBuffer.default(mockEventStream, 8, cb); 25 | }); 26 | 27 | test('no data is read when early quit but no error is thrown', (done) => { 28 | expect.assertions(1); 29 | let eventCount = 0; 30 | function read() { 31 | eventCount += 1; 32 | if (eventCount <= 5) { 33 | return this.push(Buffer.from(eventCount.toString())); 34 | } 35 | return this.push(null); 36 | } 37 | const mockEventStream = new stream.Readable({ 38 | objectMode: true, 39 | read, 40 | }); 41 | 42 | const cb = jest.fn(); 43 | readBuffer.default(mockEventStream, 8, cb); 44 | 45 | expect(cb).not.toHaveBeenCalled(); 46 | done(); 47 | }); 48 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import loaderUtils from 'loader-utils'; 2 | import { getPool } from './workerPools'; 3 | 4 | function pitch() { 5 | const options = loaderUtils.getOptions(this) || {}; 6 | const workerPool = getPool(options); 7 | if (!workerPool.isAbleToRun()) { 8 | return; 9 | } 10 | const callback = this.async(); 11 | workerPool.run({ 12 | loaders: this.loaders.slice(this.loaderIndex + 1).map((l) => { 13 | return { 14 | loader: l.path, 15 | options: l.options, 16 | ident: l.ident, 17 | }; 18 | }), 19 | resource: this.resourcePath + (this.resourceQuery || ''), 20 | sourceMap: this.sourceMap, 21 | emitError: this.emitError, 22 | emitWarning: this.emitWarning, 23 | resolve: this.resolve, 24 | target: this.target, 25 | minimize: this.minimize, 26 | resourceQuery: this.resourceQuery, 27 | optionsContext: this.rootContext || this.options.context, 28 | }, (err, r) => { 29 | if (r) { 30 | r.fileDependencies.forEach(d => this.addDependency(d)); 31 | r.contextDependencies.forEach(d => this.addContextDependency(d)); 32 | } 33 | if (err) { 34 | callback(err); 35 | return; 36 | } 37 | callback(null, ...r.result); 38 | }); 39 | } 40 | 41 | function warmup(options, requires) { 42 | const workerPool = getPool(options); 43 | workerPool.warmup(requires); 44 | } 45 | 46 | export { pitch, warmup }; // eslint-disable-line import/prefer-default-export 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | dist: trusty 3 | language: node_js 4 | branches: 5 | only: 6 | - master 7 | jobs: 8 | fast_finish: true 9 | allow_failures: 10 | - env: WEBPACK_VERSION=canary 11 | include: 12 | - &test-latest 13 | stage: Webpack latest 14 | node_js: 6 15 | env: WEBPACK_VERSION=latest JOB_PART=test 16 | script: npm run travis:$JOB_PART 17 | - <<: *test-latest 18 | node_js: 8 19 | env: WEBPACK_VERSION=latest JOB_PART=lint 20 | script: npm run travis:$JOB_PART 21 | - <<: *test-latest 22 | node_js: 10 23 | env: WEBPACK_VERSION=latest JOB_PART=coverage 24 | script: npm run travis:$JOB_PART 25 | after_success: 'bash <(curl -s https://codecov.io/bash)' 26 | - <<: *test-latest 27 | node_js: 10 28 | env: WEBPACK_VERSION=latest JOB_PART=test 29 | script: npm run travis:$JOB_PART 30 | - stage: Webpack canary 31 | before_script: npm i --no-save git://github.com/webpack/webpack.git#master 32 | script: npm run travis:$JOB_PART 33 | node_js: 6 34 | env: WEBPACK_VERSION=canary JOB_PART=test 35 | before_install: 36 | - 'if [[ `npm -v` != 5* ]]; then npm i -g npm@^5.0.0; fi' 37 | - nvm --version 38 | - node --version 39 | - npm --version 40 | before_script: 41 | - |- 42 | if [ "$WEBPACK_VERSION" ]; then 43 | npm i --no-save webpack@$WEBPACK_VERSION 44 | fi 45 | script: 46 | - 'npm run travis:$JOB_PART' 47 | after_success: 48 | - 'bash <(curl -s https://codecov.io/bash)' 49 | -------------------------------------------------------------------------------- /test/workerPool.test.js: -------------------------------------------------------------------------------- 1 | import childProcess from 'child_process'; 2 | import stream from 'stream'; 3 | import WorkerPool from '../src/WorkerPool'; 4 | 5 | jest.mock('child_process', () => { 6 | return { 7 | spawn: jest.fn(() => { 8 | return { 9 | unref: jest.fn(), 10 | }; 11 | }), 12 | }; 13 | }); 14 | 15 | describe('workerPool', () => { 16 | it('should throw an error when worker.stdio is undefined', () => { 17 | const workerPool = new WorkerPool({}); 18 | expect(() => workerPool.createWorker()).toThrowErrorMatchingSnapshot(); 19 | expect(() => workerPool.createWorker()).toThrowError('Please verify if you hit the OS open files limit'); 20 | }); 21 | 22 | it('should not throw an error when worker.stdio is defined', () => { 23 | childProcess.spawn.mockImplementationOnce(() => { 24 | return { 25 | stdio: new Array(5).fill(new stream.PassThrough()), 26 | unref: jest.fn(), 27 | }; 28 | }); 29 | 30 | const workerPool = new WorkerPool({}); 31 | expect(() => workerPool.createWorker()).not.toThrow(); 32 | }); 33 | 34 | it('should be able to run if the worker pool was not terminated', () => { 35 | const workerPool = new WorkerPool({}); 36 | expect(workerPool.isAbleToRun()).toBe(true); 37 | }); 38 | 39 | it('should not be able to run if the worker pool was terminated', () => { 40 | const workerPool = new WorkerPool({}); 41 | workerPool.terminate(); 42 | expect(workerPool.isAbleToRun()).toBe(false); 43 | }); 44 | }); 45 | -------------------------------------------------------------------------------- /example/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const MiniCssExtractPlugin = require('mini-css-extract-plugin'); // eslint-disable-line import/no-extraneous-dependencies 3 | const threadLoader = require('../src'); // eslint-disable-line import/no-extraneous-dependencies 4 | 5 | module.exports = (env) => { 6 | const workerPool = { 7 | workers: +env.threads, 8 | poolTimeout: env.watch ? Infinity : 2000, 9 | }; 10 | const workerPoolSass = { 11 | workers: +env.threads, 12 | workerParallelJobs: 2, 13 | poolTimeout: env.watch ? Infinity : 2000, 14 | }; 15 | if (+env.threads > 0) { 16 | threadLoader.warmup(workerPool, ['babel-loader', 'babel-preset-env']); 17 | threadLoader.warmup(workerPoolSass, ['sass-loader', 'css-loader']); 18 | } 19 | return { 20 | mode: 'none', 21 | context: __dirname, 22 | entry: ['react', 'lodash-es', './index.js'], 23 | output: { 24 | path: path.resolve('dist'), 25 | filename: 'bundle.js', 26 | }, 27 | module: { 28 | rules: [ 29 | { 30 | test: /\.js$/, 31 | use: [ 32 | env.threads !== 0 && { 33 | loader: path.resolve(__dirname, '../src/index.js'), 34 | options: workerPool, 35 | }, 36 | 'babel-loader', 37 | ].filter(Boolean), 38 | }, 39 | { 40 | test: /\.scss$/, 41 | use: [ 42 | MiniCssExtractPlugin.loader, 43 | env.threads !== 0 && { 44 | loader: path.resolve(__dirname, '../src/index.js'), 45 | options: workerPoolSass, 46 | }, 47 | 'css-loader', 48 | 'sass-loader', 49 | ].filter(Boolean), 50 | }, 51 | ], 52 | }, 53 | plugins: [ 54 | new MiniCssExtractPlugin({ 55 | filename: 'style.css', 56 | }), 57 | ], 58 | stats: { 59 | children: false, 60 | }, 61 | }; 62 | }; 63 | -------------------------------------------------------------------------------- /example/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "requires": true, 3 | "lockfileVersion": 1, 4 | "dependencies": { 5 | "js-tokens": { 6 | "version": "4.0.0", 7 | "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", 8 | "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" 9 | }, 10 | "lodash-es": { 11 | "version": "4.17.11", 12 | "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.11.tgz", 13 | "integrity": "sha512-DHb1ub+rMjjrxqlB3H56/6MXtm1lSksDp2rA2cNWjG8mlDUYFhUj3Di2Zn5IwSU87xLv8tNIQ7sSwE/YOX/D/Q==" 14 | }, 15 | "loose-envify": { 16 | "version": "1.4.0", 17 | "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", 18 | "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", 19 | "requires": { 20 | "js-tokens": "^3.0.0 || ^4.0.0" 21 | } 22 | }, 23 | "object-assign": { 24 | "version": "4.1.1", 25 | "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", 26 | "integrity": "sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM=" 27 | }, 28 | "react": { 29 | "version": "16.6.3", 30 | "resolved": "https://registry.npmjs.org/react/-/react-16.6.3.tgz", 31 | "integrity": "sha512-zCvmH2vbEolgKxtqXL2wmGCUxUyNheYn/C+PD1YAjfxHC54+MhdruyhO7QieQrYsYeTxrn93PM2y0jRH1zEExw==", 32 | "requires": { 33 | "loose-envify": "^1.1.0", 34 | "object-assign": "^4.1.1", 35 | "prop-types": "^15.6.2", 36 | "scheduler": "^0.11.2" 37 | }, 38 | "dependencies": { 39 | "prop-types": { 40 | "version": "15.6.2", 41 | "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.6.2.tgz", 42 | "integrity": "sha512-3pboPvLiWD7dkI3qf3KbUe6hKFKa52w+AE0VCqECtf+QHAKgOL37tTaNCnuX1nAAQ4ZhyP+kYVKf8rLmJ/feDQ==", 43 | "requires": { 44 | "loose-envify": "^1.3.1", 45 | "object-assign": "^4.1.1" 46 | } 47 | }, 48 | "scheduler": { 49 | "version": "0.11.3", 50 | "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.11.3.tgz", 51 | "integrity": "sha512-i9X9VRRVZDd3xZw10NY5Z2cVMbdYg6gqFecfj79USv1CFN+YrJ3gIPRKf1qlY+Sxly4djoKdfx1T+m9dnRB8kQ==", 52 | "requires": { 53 | "loose-envify": "^1.1.0", 54 | "object-assign": "^4.1.1" 55 | } 56 | } 57 | } 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "thread-loader", 3 | "version": "2.1.2", 4 | "description": "Runs the following loaders in a worker pool", 5 | "author": "Tobias Koppers @sokra", 6 | "license": "MIT", 7 | "main": "dist/cjs.js", 8 | "files": [ 9 | "dist" 10 | ], 11 | "scripts": { 12 | "start": "npm run build -- -w", 13 | "clean": "del-cli dist", 14 | "build": "cross-env NODE_ENV=production babel src -d dist --ignore 'src/**/*.test.js'", 15 | "lint": "eslint --cache src test", 16 | "lint-staged": "lint-staged", 17 | "prebuild": "npm run clean", 18 | "prepare": "npm run build", 19 | "release": "standard-version", 20 | "security": "npm audit", 21 | "test": "jest", 22 | "test:watch": "jest --watch", 23 | "test:coverage": "jest --collectCoverageFrom='src/**/*.js' --coverage", 24 | "travis:coverage": "npm run test:coverage -- --runInBand", 25 | "travis:lint": "npm run lint && npm run security", 26 | "travis:test": "npm run test -- --runInBand", 27 | "appveyor:test": "npm run test", 28 | "webpack-defaults": "webpack-defaults" 29 | }, 30 | "dependencies": { 31 | "neo-async": "^2.6.0", 32 | "loader-runner": "^2.3.1", 33 | "loader-utils": "^1.1.0" 34 | }, 35 | "devDependencies": { 36 | "babel-cli": "^6.26.0", 37 | "babel-core": "^6.26.3", 38 | "babel-jest": "^23.6.0", 39 | "babel-loader": "^7.1.5", 40 | "babel-plugin-transform-object-rest-spread": "^6.26.0", 41 | "babel-polyfill": "^6.26.0", 42 | "babel-preset-env": "^1.7.0", 43 | "cross-env": "^5.2.0", 44 | "css-loader": "^1.0.1", 45 | "del-cli": "^1.1.0", 46 | "eslint": "^5.10.0", 47 | "eslint-config-webpack": "^1.2.5", 48 | "eslint-plugin-import": "^2.14.0", 49 | "jest": "^23.6.0", 50 | "lint-staged": "^8.1.0", 51 | "lodash": "^4.17.11", 52 | "mini-css-extract-plugin": "^0.5.0", 53 | "nodemon": "^1.18.8", 54 | "node-sass": "^4.11.0", 55 | "pre-commit": "^1.2.2", 56 | "sass-loader": "^7.1.0", 57 | "standard-version": "^4.4.0", 58 | "webpack": "^4.27.1", 59 | "webpack-cli": "^3.1.2", 60 | "webpack-defaults": "^1.6.0" 61 | }, 62 | "engines": { 63 | "node": ">= 6.9.0 <7.0.0 || >= 8.9.0" 64 | }, 65 | "peerDependencies": { 66 | "webpack": "^2.0.0 || ^3.0.0 || ^4.0.0" 67 | }, 68 | "repository": "https://github.com/webpack-contrib/thread-loader.git", 69 | "bugs": "https://github.com/webpack-contrib/thread-loader/issues", 70 | "homepage": "https://github.com/webpack-contrib/thread-loader", 71 | "pre-commit": "lint-staged", 72 | "lint-staged": { 73 | "*.js": [ 74 | "eslint --fix", 75 | "git add" 76 | ] 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![npm][npm]][npm-url] 2 | [![deps][deps]][deps-url] 3 | [![test][test]][test-url] 4 | [![coverage][cover]][cover-url] 5 | [![chat][chat]][chat-url] 6 | 7 |
8 | 9 | 10 | 11 |

thread-loader

12 |

Runs the following loaders in a worker pool.

13 |
14 | 15 |

Install

16 | 17 | ```bash 18 | npm install --save-dev thread-loader 19 | ``` 20 | 21 |

Usage

22 | 23 | Put this loader in front of other loaders. The following loaders run in a worker pool. 24 | 25 | Loaders running in a worker pool are limited. Examples: 26 | 27 | * Loaders cannot emit files. 28 | * Loaders cannot use custom loader API (i. e. by plugins). 29 | * Loaders cannot access the webpack options. 30 | 31 | Each worker is a separate node.js process, which has an overhead of ~600ms. There is also an overhead of inter-process communication. 32 | 33 | Use this loader only for expensive operations! 34 | 35 |

Examples

36 | 37 | **webpack.config.js** 38 | 39 | ```js 40 | module.exports = { 41 | module: { 42 | rules: [ 43 | { 44 | test: /\.js$/, 45 | include: path.resolve("src"), 46 | use: [ 47 | "thread-loader", 48 | // your expensive loader (e.g babel-loader) 49 | ] 50 | } 51 | ] 52 | } 53 | } 54 | ``` 55 | 56 | **with options** 57 | 58 | ```js 59 | use: [ 60 | { 61 | loader: "thread-loader", 62 | // loaders with equal options will share worker pools 63 | options: { 64 | // the number of spawned workers, defaults to (number of cpus - 1) or 65 | // fallback to 1 when require('os').cpus() is undefined 66 | workers: 2, 67 | 68 | // number of jobs a worker processes in parallel 69 | // defaults to 20 70 | workerParallelJobs: 50, 71 | 72 | // additional node.js arguments 73 | workerNodeArgs: ['--max-old-space-size=1024'], 74 | 75 | // Allow to respawn a dead worker pool 76 | // respawning slows down the entire compilation 77 | // and should be set to false for development 78 | poolRespawn: false, 79 | 80 | // timeout for killing the worker processes when idle 81 | // defaults to 500 (ms) 82 | // can be set to Infinity for watching builds to keep workers alive 83 | poolTimeout: 2000, 84 | 85 | // number of jobs the poll distributes to the workers 86 | // defaults to 200 87 | // decrease of less efficient but more fair distribution 88 | poolParallelJobs: 50, 89 | 90 | // name of the pool 91 | // can be used to create different pools with elsewise identical options 92 | name: "my-pool" 93 | } 94 | }, 95 | // your expensive loader (e.g babel-loader) 96 | ] 97 | ``` 98 | 99 | **prewarming** 100 | 101 | To prevent the high delay when booting workers it possible to warmup the worker pool. 102 | 103 | This boots the max number of workers in the pool and loads specified modules into the node.js module cache. 104 | 105 | ``` js 106 | const threadLoader = require('thread-loader'); 107 | 108 | threadLoader.warmup({ 109 | // pool options, like passed to loader options 110 | // must match loader options to boot the correct pool 111 | }, [ 112 | // modules to load 113 | // can be any module, i. e. 114 | 'babel-loader', 115 | 'babel-preset-es2015', 116 | 'sass-loader', 117 | ]); 118 | ``` 119 | 120 | 121 |

Maintainers

122 | 123 | 124 | 125 | 126 | 133 | 134 | 135 |
127 | 128 | 129 |
130 | sokra 131 |
132 |
136 | 137 | 138 | [npm]: https://img.shields.io/npm/v/thread-loader.svg 139 | [npm-url]: https://npmjs.com/package/thread-loader 140 | 141 | [deps]: https://david-dm.org/webpack-contrib/thread-loader.svg 142 | [deps-url]: https://david-dm.org/webpack-contrib/thread-loader 143 | 144 | [chat]: https://img.shields.io/badge/gitter-webpack%2Fwebpack-brightgreen.svg 145 | [chat-url]: https://gitter.im/webpack/webpack 146 | 147 | [test]: http://img.shields.io/travis/webpack-contrib/thread-loader.svg 148 | [test-url]: https://travis-ci.org/webpack-contrib/thread-loader 149 | 150 | [cover]: https://codecov.io/gh/webpack-contrib/thread-loader/branch/master/graph/badge.svg 151 | [cover-url]: https://codecov.io/gh/webpack-contrib/thread-loader 152 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 4 | 5 | 6 | ## [2.1.2](https://github.com/webpack-contrib/thread-loader/compare/v2.1.1...v2.1.2) (2019-01-25) 7 | 8 | 9 | ### Bug Fixes 10 | 11 | * lifecycle handling for signals 12 | 13 | 14 | 15 | 16 | ## [2.1.1](https://github.com/webpack-contrib/thread-loader/compare/v2.1.0...v2.1.1) (2018-12-21) 17 | 18 | 19 | ### Performance Improvements 20 | 21 | * use `neo-async` instead of `async` ([#54](https://github.com/webpack-contrib/thread-loader/issues/54)) ([d3a6664](https://github.com/webpack-contrib/thread-loader/commit/d3a6664)) 22 | 23 | 24 | 25 | 26 | # [2.1.0](https://github.com/webpack-contrib/thread-loader/compare/v2.0.2...v2.1.0) (2018-12-21) 27 | 28 | 29 | ### Features 30 | 31 | * add poolRespawn flag to speed up incremental builds ([#52](https://github.com/webpack-contrib/thread-loader/issues/52)) ([76535bf](https://github.com/webpack-contrib/thread-loader/commit/76535bf)) 32 | 33 | 34 | 35 | 36 | ## [2.0.2](https://github.com/webpack-contrib/thread-loader/compare/v2.0.1...v2.0.2) (2018-12-20) 37 | 38 | 39 | ### Bug Fixes 40 | 41 | * build hang ([#53](https://github.com/webpack-contrib/thread-loader/issues/53)) ([fa02b60](https://github.com/webpack-contrib/thread-loader/commit/fa02b60)) 42 | 43 | 44 | 45 | 46 | ## [2.0.1](https://github.com/webpack-contrib/thread-loader/compare/v2.0.0...v2.0.1) (2018-12-19) 47 | 48 | 49 | ### Bug Fixes 50 | 51 | * memory leaks, worker and main process lifecycles ([#51](https://github.com/webpack-contrib/thread-loader/issues/51)) ([f10fe55](https://github.com/webpack-contrib/thread-loader/commit/f10fe55)) 52 | 53 | 54 | 55 | 56 | ## [2.0.0](https://github.com/webpack-contrib/thread-loader/compare/v1.2.0...v2.0.0) (2018-12-18) 57 | 58 | 59 | ### Bug Fixes 60 | 61 | * calculate number of workers correctly ([#49](https://github.com/webpack-contrib/thread-loader/issues/49)) ([fcbd813](https://github.com/webpack-contrib/thread-loader/commit/fcbd813)) 62 | * check on `undefined` for `worker.stdio` ([#45](https://github.com/webpack-contrib/thread-loader/issues/45)) ([c891a9c](https://github.com/webpack-contrib/thread-loader/commit/c891a9c)) 63 | * listen `end` events ([#42](https://github.com/webpack-contrib/thread-loader/issues/42)) ([0f87683](https://github.com/webpack-contrib/thread-loader/commit/0f87683)) 64 | 65 | 66 | ### BREAKING CHANGE 67 | 68 | * drop support for node < 6.9 69 | 70 | 71 | 72 | 73 | # [1.2.0](https://github.com/webpack-contrib/thread-loader/compare/v1.1.5...v1.2.0) (2018-07-27) 74 | 75 | 76 | ### Features 77 | 78 | * add target, minimize and resourceQuery into context ([#25](https://github.com/webpack-contrib/thread-loader/issues/25)) ([f3c7a2c](https://github.com/webpack-contrib/thread-loader/commit/f3c7a2c)) 79 | 80 | 81 | 82 | 83 | ## [1.1.5](https://github.com/webpack-contrib/thread-loader/compare/v1.1.4...v1.1.5) (2018-02-26) 84 | 85 | 86 | ### Bug Fixes 87 | 88 | * **package:** add `webpack >= 4` (`peerDependencies`) ([#22](https://github.com/webpack-contrib/thread-loader/issues/22)) ([9345756](https://github.com/webpack-contrib/thread-loader/commit/9345756)) 89 | * **WorkerError:** handle undefined `error` stacks ([#20](https://github.com/webpack-contrib/thread-loader/issues/20)) ([6fb5daf](https://github.com/webpack-contrib/thread-loader/commit/6fb5daf)) 90 | 91 | 92 | 93 | 94 | ## [1.1.4](https://github.com/webpack-contrib/thread-loader/compare/v1.1.3...v1.1.4) (2018-02-21) 95 | 96 | 97 | ### Bug Fixes 98 | 99 | * **index:** add `webpack >= v4.0.0` support ([#16](https://github.com/webpack-contrib/thread-loader/issues/16)) ([5d33937](https://github.com/webpack-contrib/thread-loader/commit/5d33937)) 100 | 101 | 102 | 103 | 104 | ## [1.1.3](https://github.com/webpack-contrib/thread-loader/compare/v1.1.2...v1.1.3) (2018-02-07) 105 | 106 | 107 | ### Bug Fixes 108 | 109 | * **WorkerPool:** trace stacks to avoid duplicated `err.messages` from workers ([#13](https://github.com/webpack-contrib/thread-loader/issues/13)) ([80dda4f](https://github.com/webpack-contrib/thread-loader/commit/80dda4f)) 110 | 111 | 112 | 113 | 114 | ## [1.1.2](https://github.com/webpack-contrib/thread-loader/compare/v1.1.1...v1.1.2) (2017-10-09) 115 | 116 | 117 | ### Bug Fixes 118 | 119 | * **readBuffer:** handle 0-byte reads ([c7ca960](https://github.com/webpack-contrib/thread-loader/commit/c7ca960)) 120 | 121 | 122 | 123 | 124 | ## [1.1.1](https://github.com/webpack-contrib/thread-loader/compare/v1.1.0...v1.1.1) (2017-08-28) 125 | 126 | 127 | ### Bug Fixes 128 | 129 | * **context:** Pass context to loader ([29ced70](https://github.com/webpack-contrib/thread-loader/commit/29ced70)) 130 | * **deps:** pass along result for dependencies ([19832ec](https://github.com/webpack-contrib/thread-loader/commit/19832ec)) 131 | * **example:** fix for broken sass and add watch ([47ba43e](https://github.com/webpack-contrib/thread-loader/commit/47ba43e)) 132 | 133 | 134 | 135 | 136 | # [1.1.0](https://github.com/webpack-contrib/thread-loader/compare/v1.0.3...v1.1.0) (2017-07-14) 137 | 138 | 139 | ### Features 140 | 141 | * **pool:** add warmup method ([a0ce440](https://github.com/webpack-contrib/thread-loader/commit/a0ce440)) 142 | 143 | 144 | 145 | 146 | ## [1.0.3](https://github.com/webpack-contrib/thread-loader/compare/v1.0.2...v1.0.3) (2017-05-27) 147 | 148 | 149 | ### Bug Fixes 150 | 151 | * **resolve:** fix passing error to worker ([6561f57](https://github.com/webpack-contrib/thread-loader/commit/6561f57)) 152 | 153 | 154 | 155 | 156 | ## [1.0.2](https://github.com/webpack-contrib/thread-loader/compare/v1.0.1...v1.0.2) (2017-05-27) 157 | 158 | 159 | ### Bug Fixes 160 | 161 | * **resolve:** fix incorrect method for sending message ([bb92a28](https://github.com/webpack-contrib/thread-loader/commit/bb92a28)) 162 | 163 | 164 | 165 | 166 | ## 1.0.1 (2017-04-28) 167 | 168 | 169 | 170 | # Change Log 171 | 172 | All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. 173 | 174 | x.x.x / -- 175 | ================== 176 | 177 | * Bug fix - 178 | * Feature - 179 | * Chore - 180 | * Docs - 181 | -------------------------------------------------------------------------------- /src/worker.js: -------------------------------------------------------------------------------- 1 | /* global require */ 2 | /* eslint-disable no-console */ 3 | import fs from 'fs'; 4 | import NativeModule from 'module'; 5 | import loaderRunner from 'loader-runner'; 6 | import asyncQueue from 'neo-async/queue'; 7 | import readBuffer from './readBuffer'; 8 | 9 | const writePipe = fs.createWriteStream(null, { fd: 3 }); 10 | const readPipe = fs.createReadStream(null, { fd: 4 }); 11 | 12 | writePipe.on('finish', onTerminateWrite); 13 | readPipe.on('end', onTerminateRead); 14 | writePipe.on('close', onTerminateWrite); 15 | readPipe.on('close', onTerminateRead); 16 | 17 | readPipe.on('error', onError); 18 | writePipe.on('error', onError); 19 | 20 | const PARALLEL_JOBS = +process.argv[2]; 21 | 22 | let terminated = false; 23 | let nextQuestionId = 0; 24 | const callbackMap = Object.create(null); 25 | 26 | function onError(error) { 27 | console.error(error); 28 | } 29 | 30 | function onTerminateRead() { 31 | terminateRead(); 32 | } 33 | 34 | function onTerminateWrite() { 35 | terminateWrite(); 36 | } 37 | 38 | function writePipeWrite(...args) { 39 | if (!terminated) { 40 | writePipe.write(...args); 41 | } 42 | } 43 | 44 | function writePipeCork() { 45 | if (!terminated) { 46 | writePipe.cork(); 47 | } 48 | } 49 | 50 | function writePipeUncork() { 51 | if (!terminated) { 52 | writePipe.uncork(); 53 | } 54 | } 55 | 56 | function terminateRead() { 57 | terminated = true; 58 | readPipe.removeAllListeners(); 59 | } 60 | 61 | function terminateWrite() { 62 | terminated = true; 63 | writePipe.removeAllListeners(); 64 | } 65 | 66 | function terminate() { 67 | terminateRead(); 68 | terminateWrite(); 69 | } 70 | 71 | function toErrorObj(err) { 72 | return { 73 | message: err.message, 74 | details: err.details, 75 | stack: err.stack, 76 | hideStack: err.hideStack, 77 | }; 78 | } 79 | 80 | function toNativeError(obj) { 81 | if (!obj) return null; 82 | const err = new Error(obj.message); 83 | err.details = obj.details; 84 | err.missing = obj.missing; 85 | return err; 86 | } 87 | 88 | function writeJson(data) { 89 | writePipeCork(); 90 | process.nextTick(() => { 91 | writePipeUncork(); 92 | }); 93 | 94 | const lengthBuffer = Buffer.alloc(4); 95 | const messageBuffer = Buffer.from(JSON.stringify(data), 'utf-8'); 96 | lengthBuffer.writeInt32BE(messageBuffer.length, 0); 97 | 98 | writePipeWrite(lengthBuffer); 99 | writePipeWrite(messageBuffer); 100 | } 101 | 102 | const queue = asyncQueue(({ id, data }, taskCallback) => { 103 | try { 104 | loaderRunner.runLoaders({ 105 | loaders: data.loaders, 106 | resource: data.resource, 107 | readResource: fs.readFile.bind(fs), 108 | context: { 109 | version: 2, 110 | resolve: (context, request, callback) => { 111 | callbackMap[nextQuestionId] = callback; 112 | writeJson({ 113 | type: 'resolve', 114 | id, 115 | questionId: nextQuestionId, 116 | context, 117 | request, 118 | }); 119 | nextQuestionId += 1; 120 | }, 121 | emitWarning: (warning) => { 122 | writeJson({ 123 | type: 'emitWarning', 124 | id, 125 | data: toErrorObj(warning), 126 | }); 127 | }, 128 | emitError: (error) => { 129 | writeJson({ 130 | type: 'emitError', 131 | id, 132 | data: toErrorObj(error), 133 | }); 134 | }, 135 | exec: (code, filename) => { 136 | const module = new NativeModule(filename, this); 137 | module.paths = NativeModule._nodeModulePaths(this.context); // eslint-disable-line no-underscore-dangle 138 | module.filename = filename; 139 | module._compile(code, filename); // eslint-disable-line no-underscore-dangle 140 | return module.exports; 141 | }, 142 | options: { 143 | context: data.optionsContext, 144 | }, 145 | webpack: true, 146 | 'thread-loader': true, 147 | sourceMap: data.sourceMap, 148 | target: data.target, 149 | minimize: data.minimize, 150 | resourceQuery: data.resourceQuery, 151 | }, 152 | }, (err, lrResult) => { 153 | const { 154 | result, 155 | cacheable, 156 | fileDependencies, 157 | contextDependencies, 158 | } = lrResult; 159 | const buffersToSend = []; 160 | const convertedResult = Array.isArray(result) && result.map((item) => { 161 | const isBuffer = Buffer.isBuffer(item); 162 | if (isBuffer) { 163 | buffersToSend.push(item); 164 | return { 165 | buffer: true, 166 | }; 167 | } 168 | if (typeof item === 'string') { 169 | const stringBuffer = Buffer.from(item, 'utf-8'); 170 | buffersToSend.push(stringBuffer); 171 | return { 172 | buffer: true, 173 | string: true, 174 | }; 175 | } 176 | return { 177 | data: item, 178 | }; 179 | }); 180 | writeJson({ 181 | type: 'job', 182 | id, 183 | error: err && toErrorObj(err), 184 | result: { 185 | result: convertedResult, 186 | cacheable, 187 | fileDependencies, 188 | contextDependencies, 189 | }, 190 | data: buffersToSend.map(buffer => buffer.length), 191 | }); 192 | buffersToSend.forEach((buffer) => { 193 | writePipeWrite(buffer); 194 | }); 195 | setImmediate(taskCallback); 196 | }); 197 | } catch (e) { 198 | writeJson({ 199 | type: 'job', 200 | id, 201 | error: toErrorObj(e), 202 | }); 203 | taskCallback(); 204 | } 205 | }, PARALLEL_JOBS); 206 | 207 | function dispose() { 208 | terminate(); 209 | 210 | queue.kill(); 211 | process.exit(0); 212 | } 213 | 214 | function onMessage(message) { 215 | try { 216 | const { type, id } = message; 217 | switch (type) { 218 | case 'job': { 219 | queue.push(message); 220 | break; 221 | } 222 | case 'result': { 223 | const { error, result } = message; 224 | const callback = callbackMap[id]; 225 | if (callback) { 226 | const nativeError = toNativeError(error); 227 | callback(nativeError, result); 228 | } else { 229 | console.error(`Worker got unexpected result id ${id}`); 230 | } 231 | delete callbackMap[id]; 232 | break; 233 | } 234 | case 'warmup': { 235 | const { requires } = message; 236 | // load modules into process 237 | requires.forEach(r => require(r)); // eslint-disable-line import/no-dynamic-require, global-require 238 | break; 239 | } 240 | default: { 241 | console.error(`Worker got unexpected job type ${type}`); 242 | break; 243 | } 244 | } 245 | } catch (e) { 246 | console.error(`Error in worker ${e}`); 247 | } 248 | } 249 | 250 | function readNextMessage() { 251 | readBuffer(readPipe, 4, (lengthReadError, lengthBuffer) => { 252 | if (lengthReadError) { 253 | console.error(`Failed to communicate with main process (read length) ${lengthReadError}`); 254 | return; 255 | } 256 | 257 | const length = lengthBuffer.length && lengthBuffer.readInt32BE(0); 258 | 259 | if (length === 0) { 260 | // worker should dispose and exit 261 | dispose(); 262 | return; 263 | } 264 | readBuffer(readPipe, length, (messageError, messageBuffer) => { 265 | if (terminated) { 266 | return; 267 | } 268 | 269 | if (messageError) { 270 | console.error(`Failed to communicate with main process (read message) ${messageError}`); 271 | return; 272 | } 273 | const messageString = messageBuffer.toString('utf-8'); 274 | const message = JSON.parse(messageString); 275 | 276 | onMessage(message); 277 | setImmediate(() => readNextMessage()); 278 | }); 279 | }); 280 | } 281 | 282 | // start reading messages from main process 283 | readNextMessage(); 284 | -------------------------------------------------------------------------------- /src/WorkerPool.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-console */ 2 | 3 | import childProcess from 'child_process'; 4 | import asyncQueue from 'neo-async/queue'; 5 | import asyncMapSeries from 'neo-async/mapSeries'; 6 | import readBuffer from './readBuffer'; 7 | import WorkerError from './WorkerError'; 8 | 9 | const workerPath = require.resolve('./worker'); 10 | 11 | let workerId = 0; 12 | 13 | class PoolWorker { 14 | constructor(options, onJobDone) { 15 | this.disposed = false; 16 | this.nextJobId = 0; 17 | this.jobs = Object.create(null); 18 | this.activeJobs = 0; 19 | this.onJobDone = onJobDone; 20 | this.id = workerId; 21 | workerId += 1; 22 | this.worker = childProcess.spawn(process.execPath, [].concat(options.nodeArgs || []).concat(workerPath, options.parallelJobs), { 23 | detached: true, 24 | stdio: ['ignore', 'pipe', 'pipe', 'pipe', 'pipe'], 25 | }); 26 | 27 | this.worker.unref(); 28 | 29 | // This prevents a problem where the worker stdio can be undefined 30 | // when the kernel hits the limit of open files. 31 | // More info can be found on: https://github.com/webpack-contrib/thread-loader/issues/2 32 | if (!this.worker.stdio) { 33 | throw new Error(`Failed to create the worker pool with workerId: ${workerId} and ${'' 34 | }configuration: ${JSON.stringify(options)}. Please verify if you hit the OS open files limit.`); 35 | } 36 | 37 | const [, , , readPipe, writePipe] = this.worker.stdio; 38 | this.readPipe = readPipe; 39 | this.writePipe = writePipe; 40 | this.listenStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr); 41 | this.readNextMessage(); 42 | } 43 | 44 | listenStdOutAndErrFromWorker(workerStdout, workerStderr) { 45 | if (workerStdout) { 46 | workerStdout.on('data', this.writeToStdout); 47 | } 48 | 49 | if (workerStderr) { 50 | workerStderr.on('data', this.writeToStderr); 51 | } 52 | } 53 | 54 | ignoreStdOutAndErrFromWorker(workerStdout, workerStderr) { 55 | if (workerStdout) { 56 | workerStdout.removeListener('data', this.writeToStdout); 57 | } 58 | 59 | if (workerStderr) { 60 | workerStderr.removeListener('data', this.writeToStderr); 61 | } 62 | } 63 | 64 | writeToStdout(data) { 65 | if (!this.disposed) { 66 | process.stdout.write(data); 67 | } 68 | } 69 | 70 | writeToStderr(data) { 71 | if (!this.disposed) { 72 | process.stderr.write(data); 73 | } 74 | } 75 | 76 | run(data, callback) { 77 | const jobId = this.nextJobId; 78 | this.nextJobId += 1; 79 | this.jobs[jobId] = { data, callback }; 80 | this.activeJobs += 1; 81 | this.writeJson({ 82 | type: 'job', 83 | id: jobId, 84 | data, 85 | }); 86 | } 87 | 88 | warmup(requires) { 89 | this.writeJson({ 90 | type: 'warmup', 91 | requires, 92 | }); 93 | } 94 | 95 | writeJson(data) { 96 | const lengthBuffer = Buffer.alloc(4); 97 | const messageBuffer = Buffer.from(JSON.stringify(data), 'utf-8'); 98 | lengthBuffer.writeInt32BE(messageBuffer.length, 0); 99 | this.writePipe.write(lengthBuffer); 100 | this.writePipe.write(messageBuffer); 101 | } 102 | 103 | writeEnd() { 104 | const lengthBuffer = Buffer.alloc(4); 105 | lengthBuffer.writeInt32BE(0, 0); 106 | this.writePipe.write(lengthBuffer); 107 | } 108 | 109 | readNextMessage() { 110 | this.state = 'read length'; 111 | this.readBuffer(4, (lengthReadError, lengthBuffer) => { 112 | if (lengthReadError) { 113 | console.error(`Failed to communicate with worker (read length) ${lengthReadError}`); 114 | return; 115 | } 116 | this.state = 'length read'; 117 | const length = lengthBuffer.readInt32BE(0); 118 | 119 | this.state = 'read message'; 120 | this.readBuffer(length, (messageError, messageBuffer) => { 121 | if (messageError) { 122 | console.error(`Failed to communicate with worker (read message) ${messageError}`); 123 | return; 124 | } 125 | this.state = 'message read'; 126 | const messageString = messageBuffer.toString('utf-8'); 127 | const message = JSON.parse(messageString); 128 | this.state = 'process message'; 129 | this.onWorkerMessage(message, (err) => { 130 | if (err) { 131 | console.error(`Failed to communicate with worker (process message) ${err}`); 132 | return; 133 | } 134 | this.state = 'soon next'; 135 | setImmediate(() => this.readNextMessage()); 136 | }); 137 | }); 138 | }); 139 | } 140 | 141 | onWorkerMessage(message, finalCallback) { 142 | const { type, id } = message; 143 | switch (type) { 144 | case 'job': { 145 | const { data, error, result } = message; 146 | asyncMapSeries(data, (length, callback) => this.readBuffer(length, callback), (eachErr, buffers) => { 147 | const { callback: jobCallback } = this.jobs[id]; 148 | const callback = (err, arg) => { 149 | if (jobCallback) { 150 | delete this.jobs[id]; 151 | this.activeJobs -= 1; 152 | this.onJobDone(); 153 | if (err) { 154 | jobCallback(err instanceof Error ? err : new Error(err), arg); 155 | } else { 156 | jobCallback(null, arg); 157 | } 158 | } 159 | finalCallback(); 160 | }; 161 | if (eachErr) { 162 | callback(eachErr); 163 | return; 164 | } 165 | let bufferPosition = 0; 166 | if (result.result) { 167 | result.result = result.result.map((r) => { 168 | if (r.buffer) { 169 | const buffer = buffers[bufferPosition]; 170 | bufferPosition += 1; 171 | if (r.string) { 172 | return buffer.toString('utf-8'); 173 | } 174 | return buffer; 175 | } 176 | return r.data; 177 | }); 178 | } 179 | if (error) { 180 | callback(this.fromErrorObj(error), result); 181 | return; 182 | } 183 | callback(null, result); 184 | }); 185 | break; 186 | } 187 | case 'resolve': { 188 | const { context, request, questionId } = message; 189 | const { data } = this.jobs[id]; 190 | data.resolve(context, request, (error, result) => { 191 | this.writeJson({ 192 | type: 'result', 193 | id: questionId, 194 | error: error ? { 195 | message: error.message, 196 | details: error.details, 197 | missing: error.missing, 198 | } : null, 199 | result, 200 | }); 201 | }); 202 | finalCallback(); 203 | break; 204 | } 205 | case 'emitWarning': { 206 | const { data } = message; 207 | const { data: jobData } = this.jobs[id]; 208 | jobData.emitWarning(this.fromErrorObj(data)); 209 | finalCallback(); 210 | break; 211 | } 212 | case 'emitError': { 213 | const { data } = message; 214 | const { data: jobData } = this.jobs[id]; 215 | jobData.emitError(this.fromErrorObj(data)); 216 | finalCallback(); 217 | break; 218 | } 219 | default: { 220 | console.error(`Unexpected worker message ${type} in WorkerPool.`); 221 | finalCallback(); 222 | break; 223 | } 224 | } 225 | } 226 | 227 | fromErrorObj(arg) { 228 | let obj; 229 | if (typeof arg === 'string') { 230 | obj = { message: arg }; 231 | } else { 232 | obj = arg; 233 | } 234 | return new WorkerError(obj, this.id); 235 | } 236 | 237 | readBuffer(length, callback) { 238 | readBuffer(this.readPipe, length, callback); 239 | } 240 | 241 | dispose() { 242 | if (!this.disposed) { 243 | this.disposed = true; 244 | this.ignoreStdOutAndErrFromWorker(this.worker.stdout, this.worker.stderr); 245 | this.writeEnd(); 246 | } 247 | } 248 | } 249 | 250 | export default class WorkerPool { 251 | constructor(options) { 252 | this.options = options || {}; 253 | this.numberOfWorkers = options.numberOfWorkers; 254 | this.poolTimeout = options.poolTimeout; 255 | this.workerNodeArgs = options.workerNodeArgs; 256 | this.workerParallelJobs = options.workerParallelJobs; 257 | this.workers = new Set(); 258 | this.activeJobs = 0; 259 | this.timeout = null; 260 | this.poolQueue = asyncQueue(this.distributeJob.bind(this), options.poolParallelJobs); 261 | this.terminated = false; 262 | 263 | this.setupLifeCycle(); 264 | } 265 | 266 | isAbleToRun() { 267 | return !this.terminated; 268 | } 269 | 270 | terminate() { 271 | if (this.terminated) { 272 | return; 273 | } 274 | 275 | this.terminated = true; 276 | this.poolQueue.kill(); 277 | this.disposeWorkers(true); 278 | } 279 | 280 | setupLifeCycle() { 281 | process.on('exit', () => { 282 | this.terminate(); 283 | }); 284 | } 285 | 286 | run(data, callback) { 287 | if (this.timeout) { 288 | clearTimeout(this.timeout); 289 | this.timeout = null; 290 | } 291 | this.activeJobs += 1; 292 | this.poolQueue.push(data, callback); 293 | } 294 | 295 | distributeJob(data, callback) { 296 | // use worker with the fewest jobs 297 | let bestWorker; 298 | for (const worker of this.workers) { 299 | if (!bestWorker || worker.activeJobs < bestWorker.activeJobs) { 300 | bestWorker = worker; 301 | } 302 | } 303 | if (bestWorker && (bestWorker.activeJobs === 0 || this.workers.size >= this.numberOfWorkers)) { 304 | bestWorker.run(data, callback); 305 | return; 306 | } 307 | const newWorker = this.createWorker(); 308 | newWorker.run(data, callback); 309 | } 310 | 311 | createWorker() { 312 | // spin up a new worker 313 | const newWorker = new PoolWorker({ 314 | nodeArgs: this.workerNodeArgs, 315 | parallelJobs: this.workerParallelJobs, 316 | }, () => this.onJobDone()); 317 | this.workers.add(newWorker); 318 | return newWorker; 319 | } 320 | 321 | warmup(requires) { 322 | while (this.workers.size < this.numberOfWorkers) { 323 | this.createWorker().warmup(requires); 324 | } 325 | } 326 | 327 | onJobDone() { 328 | this.activeJobs -= 1; 329 | if (this.activeJobs === 0 && isFinite(this.poolTimeout)) { 330 | this.timeout = setTimeout(() => this.disposeWorkers(), this.poolTimeout); 331 | } 332 | } 333 | 334 | disposeWorkers(fromTerminate) { 335 | if (!this.options.poolRespawn && !fromTerminate) { 336 | this.terminate(); 337 | return; 338 | } 339 | 340 | if (this.activeJobs === 0 || fromTerminate) { 341 | for (const worker of this.workers) { 342 | worker.dispose(); 343 | } 344 | this.workers.clear(); 345 | } 346 | } 347 | } 348 | --------------------------------------------------------------------------------