├── .gitignore
├── .travis.yml
├── LICENSE
├── README.md
├── debug
├── index.html
└── src
│ ├── main.js
│ └── worker.js
├── index.js
├── package.json
└── test
├── basic
├── dep-shared.js
├── dep.js
├── index.js
├── worker.js
└── worker2.js
├── multiple-same-type
├── index.js
└── worker.js
└── send-in-constructor
├── index.js
└── worker.js
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
2 | yarn.lock
3 | debug/main.js
4 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | language: node_js
2 | node_js: node
3 | addons:
4 | apt:
5 | packages:
6 | - xvfb
7 | install:
8 | - export DISPLAY=':99.0'
9 | - Xvfb :99 -screen 0 1024x768x24 > /dev/null 2>&1 &
10 | - npm install
11 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | ISC License
2 |
3 | Copyright (c) 2016, Mapbox
4 |
5 | Permission to use, copy, modify, and/or distribute this software for any purpose
6 | with or without fee is hereby granted, provided that the above copyright notice
7 | and this permission notice appear in all copies.
8 |
9 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
10 | REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND
11 | FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
12 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
13 | OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
14 | TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF
15 | THIS SOFTWARE.
16 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | ## workerpoolify [](https://travis-ci.org/mapbox/workerpoolify)
2 |
3 | An experimental worker pool for Browserify-bundled projects.
4 | Unlike [webworkify](https://github.com/substack/webworkify),
5 | it allows you to create many lightweight "workers"
6 | with an N:M ratio
7 | to a pool of native web workers.
8 |
9 | When you create a new pooled worker,
10 | its module dependencies are lazily loaded on the worker side with some clever tricks.
11 |
12 | ### Example
13 |
14 | #### main.js
15 |
16 | ```js
17 | var workerpoolify = require('workerpoolify');
18 | var PooledWorker = workerpoolify(4);
19 |
20 | var worker = new PooledWorker(require('./worker'));
21 | worker.onmessage = function (type, data) {
22 | if (type === 'foo') {
23 | console.log('got message foo');
24 | }
25 | };
26 | worker.send('bar');
27 | ```
28 |
29 | #### worker.js
30 |
31 | ```js
32 | module.exports = function TestWorker() {
33 | console.log('worker created');
34 | this.onmessage = function (type, data) {
35 | if (type === 'bar') {
36 | console.log('worker: got message bar');
37 | this.send('foo');
38 | }
39 | };
40 | };
41 |
--------------------------------------------------------------------------------
/debug/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | Test page
5 |
6 |
7 |
8 |
9 |
10 |
--------------------------------------------------------------------------------
/debug/src/main.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var createWorkerPool = require('../../');
4 | var TestWorker = require('./worker');
5 |
6 | var PooledWorker = createWorkerPool(4);
7 |
8 | console.log('main: creating worker');
9 | var worker = new PooledWorker(TestWorker);
10 |
11 | worker.onmessage = function (type) {
12 | if (type === 'bar') {
13 | console.log('main: got bar from worker');
14 | console.log('main: sending baz to worker');
15 | this.send('baz');
16 | }
17 | };
18 |
19 | console.log('main: sending foo to worker');
20 | worker.send('foo');
21 |
--------------------------------------------------------------------------------
/debug/src/worker.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports = TestWorker;
4 |
5 | function TestWorker() {
6 | console.log('worker: created');
7 | }
8 |
9 | TestWorker.prototype = {
10 | onmessage: function (type) {
11 | if (type === 'foo') {
12 | console.log('worker: got foo');
13 | console.log('worker: sending bar');
14 | this.send('bar');
15 | }
16 | if (type === 'baz') {
17 | console.log('worker: got baz');
18 | }
19 | },
20 | onterminate: function () {
21 | console.log('worker terminated');
22 | }
23 | };
24 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var browserifyBundleFn = arguments[3];
4 | var browserifySources = arguments[4];
5 | var browserifyCache = arguments[5];
6 |
7 | module.exports = createWorkerPool;
8 |
9 | function nativeWorkerFn(self) {
10 | var workersidePooledWorkers = {};
11 |
12 | function send(type, data, transferList) {
13 | self.postMessage({
14 | type: type,
15 | data: data,
16 | workerId: this.workerId
17 | }, transferList);
18 | }
19 |
20 | function createWorkersidePooledWorker(moduleId, workerId) {
21 | var WorkerClass = self.require(moduleId);
22 |
23 | function Worker() {
24 | WorkerClass.call(this);
25 | }
26 | Worker.prototype = Object.create(WorkerClass.prototype);
27 | if (Worker.prototype.send) {
28 | throw new Error('Pooled worker class should not have a send property.');
29 | }
30 | if (Worker.prototype.workerId) {
31 | throw new Error('Pooled worker class should not have a workerId property.');
32 | }
33 | Worker.prototype.send = send;
34 | Worker.prototype.workerId = workerId;
35 |
36 | workersidePooledWorkers[workerId] = new Worker();
37 | }
38 |
39 | self.onmessage = function (e) {
40 | var data = e.data;
41 | var worker;
42 |
43 | if (data.bundle) { // add missing dependencies
44 | self.importScripts(data.bundle);
45 | self.postMessage({revoke: data.bundle});
46 | }
47 | if (data.moduleId) { // create workerside pooled worker
48 | createWorkersidePooledWorker(data.moduleId, data.workerId);
49 | }
50 | if (data.type) { // process message to the worker
51 | worker = workersidePooledWorkers[data.workerId];
52 | if (worker.onmessage) {
53 | worker.onmessage(data.type, data.data);
54 | }
55 | }
56 | if (data.terminate) { // terminate the worker
57 | worker = workersidePooledWorkers[data.workerId];
58 | delete workersidePooledWorkers[data.workerId];
59 | if (worker.onterminate) {
60 | worker.onterminate();
61 | }
62 | }
63 | };
64 | }
65 |
66 | function createWorkerPool(workerCount) {
67 | var pooledWorkers = {}; // a pool-wide set of PooledWorker instances
68 | var nativeWorkers = []; // a pool of native web workers
69 | var lastWorkerId = 0;
70 |
71 | workerCount = workerCount || 4;
72 |
73 | function PooledWorker(moduleFn) {
74 | if (nativeWorkers.length === 0) {
75 | createNativeWorkers(workerCount);
76 | }
77 |
78 | this.id = lastWorkerId++;
79 | pooledWorkers[this.id] = this;
80 |
81 | // pick one of the native workers
82 | this.worker = nativeWorkers[this.id % workerCount];
83 |
84 | // propagate any bundle changes to the native worker
85 | var moduleId = findModuleId(moduleFn);
86 | updateBundle(moduleId, this.worker);
87 |
88 | // create workerside pooled worker
89 | this.worker.postMessage({
90 | workerId: this.id,
91 | moduleId: moduleId
92 | });
93 | }
94 |
95 | PooledWorker.prototype = {
96 |
97 | send: function (type, data, transferList) {
98 | this.worker.postMessage({
99 | workerId: this.id,
100 | type: type,
101 | data: data
102 | }, transferList);
103 | },
104 |
105 | terminate: function () {
106 | this.worker.postMessage({
107 | workerId: this.id,
108 | terminate: true
109 | });
110 | delete pooledWorkers[this.id];
111 | }
112 | };
113 |
114 | function createNativeWorkers() {
115 | var nativeWorkerUrl = createURL('(' + nativeWorkerFn + ')(self)');
116 |
117 | for (var i = 0; i < workerCount; i++) {
118 | var nativeWorker = new Worker(nativeWorkerUrl);
119 | nativeWorker.onmessage = handleWorkerMessage;
120 | nativeWorker.bundleSources = {};
121 | nativeWorkers.push(nativeWorker);
122 | }
123 | }
124 |
125 | function handleWorkerMessage(e) {
126 | if (e.data.revoke) {
127 | URL.revokeObjectURL(e.data.revoke); // the url won't be needed after importing
128 | } else {
129 | var worker = pooledWorkers[e.data.workerId];
130 | if (worker) {
131 | worker.onmessage(e.data.type, e.data.data);
132 | }
133 | }
134 | }
135 |
136 | return PooledWorker;
137 | }
138 |
139 | // make a blob URL out of any worker bundle additions and propagate it to the native worker
140 | function updateBundle(moduleId, nativeWorker) {
141 | var addedSources = {};
142 | resolveSources(nativeWorker.bundleSources, addedSources, moduleId);
143 |
144 | var deps = Object.keys(addedSources);
145 | if (!deps.length) return;
146 |
147 | var src = generateWorkerBundle(deps);
148 | var url = createURL(src);
149 | nativeWorker.postMessage({bundle: url});
150 | }
151 |
152 | // find the Browserify id of the required module
153 | function findModuleId(moduleFn) {
154 | for (var id in browserifyCache) {
155 | if (browserifyCache[id].exports === moduleFn) {
156 | return id;
157 | }
158 | }
159 | throw new Error('Module not found in Browserify bundle.');
160 | }
161 |
162 | // generate a bundle from a set of Browserify deps
163 | function generateWorkerBundle(deps) {
164 | return 'self.require=(' + browserifyBundleFn + ')({' + deps.map(function (key) {
165 | var source = browserifySources[key];
166 | return JSON.stringify(key) + ':[' + source[0] + ',' + JSON.stringify(source[1]) + ']';
167 | }).join(',') + '},{},[])';
168 | }
169 |
170 | // resolve Browserify deps and find all modules that are not yet on the worker side
171 | function resolveSources(workerSources, addedSources, key) {
172 | if (workerSources[key]) return;
173 |
174 | workerSources[key] = true;
175 | addedSources[key] = true;
176 |
177 | var deps = browserifySources[key][1];
178 | for (var depPath in deps) {
179 | resolveSources(workerSources, addedSources, deps[depPath]);
180 | }
181 | }
182 |
183 | // create an Blob object URL from code
184 | function createURL(src) {
185 | var URL = window.URL || window.webkitURL;
186 | var blob = new Blob([src], {type: 'text/javascript'});
187 | return URL.createObjectURL(blob);
188 | }
189 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "workerpoolify",
3 | "version": "0.0.0",
4 | "description": "A worker pool for Browserify-bundled projects.",
5 | "main": "index.js",
6 | "scripts": {
7 | "pretest": "eslint index.js test/**/*.js debug/src/*.js",
8 | "test": "browserify test/*/index.js | tape-run",
9 | "build-debug": "browserify debug/src/main.js > debug/main.js"
10 | },
11 | "keywords": [],
12 | "author": "Vladimir Agafonkin",
13 | "license": "ISC",
14 | "dependencies": {
15 | "browserify": "^13.1.0"
16 | },
17 | "devDependencies": {
18 | "eslint": "^3.8.1",
19 | "eslint-config-mourner": "^2.0.1",
20 | "tape": "^4.6.2",
21 | "tape-run": "^2.1.4"
22 | },
23 | "eslintConfig": {
24 | "extends": "mourner"
25 | }
26 | }
27 |
--------------------------------------------------------------------------------
/test/basic/dep-shared.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports = 'Hello';
4 |
--------------------------------------------------------------------------------
/test/basic/dep.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports = 42;
4 |
--------------------------------------------------------------------------------
/test/basic/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var test = require('tape');
4 | var createWorkerPool = require('../../');
5 | var TestWorker = require('./worker');
6 | var TestWorker2 = require('./worker2');
7 |
8 | var PooledWorker = createWorkerPool(2);
9 |
10 | test('roundtrip messages to one worker', {timeout: 200}, function (t) {
11 | var worker = new PooledWorker(TestWorker);
12 | t.pass('main: create worker');
13 |
14 | worker.onmessage = function (type, data) {
15 | if (type === 'bar') {
16 | t.pass('main: got bar');
17 |
18 | this.send('baz');
19 | t.pass('main: send baz');
20 |
21 | } else if (type === 'end') {
22 | t.equal(data, 'Hello', 'main: got end');
23 | worker.terminate();
24 | t.pass('worker terminated');
25 | t.end();
26 |
27 | } else {
28 | t.fail('main: unexpected message ' + type);
29 | }
30 | };
31 |
32 | worker.send('foo');
33 | t.pass('main: send foo');
34 | });
35 |
36 | test('delayed worker2 creation and more messages', {timeout: 300}, function (t) {
37 | setTimeout(function () {
38 | var worker2 = new PooledWorker(TestWorker2);
39 | t.pass('main: create worker2');
40 |
41 | worker2.onmessage = function (type, data) {
42 | if (type === 'answer') {
43 | t.equal(data, '100 Hello 42', 'main: got answer');
44 | worker2.terminate();
45 | t.pass('worker2 terminated');
46 | t.end();
47 |
48 | } else {
49 | t.fail('main: unexpected message ' + type);
50 | }
51 | };
52 |
53 | worker2.send('ask', 100);
54 | t.pass('main: send ask');
55 | }, 100);
56 | });
57 |
--------------------------------------------------------------------------------
/test/basic/worker.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var hello = require('./dep-shared.js');
4 |
5 | module.exports = TestWorker;
6 |
7 | function TestWorker() {}
8 |
9 | TestWorker.prototype = {
10 | onmessage: function (type) {
11 | if (type === 'foo') {
12 | this.send('bar');
13 | } else if (type === 'baz') {
14 | this.send('end', hello);
15 | } else {
16 | this.send('error');
17 | }
18 | }
19 | };
20 |
--------------------------------------------------------------------------------
/test/basic/worker2.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var hello = require('./dep-shared.js');
4 | var answer = require('./dep');
5 |
6 | module.exports = TestWorker2;
7 |
8 | function TestWorker2() {}
9 |
10 | TestWorker2.prototype = {
11 | onmessage: function (type, data) {
12 | if (type === 'ask') {
13 | this.send('answer', data + ' ' + hello + ' ' + answer);
14 | } else {
15 | this.send('error');
16 | }
17 | }
18 | };
19 |
--------------------------------------------------------------------------------
/test/multiple-same-type/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var test = require('tape');
4 | var createWorkerPool = require('../../');
5 | var TestWorker = require('./worker');
6 |
7 | var PooledWorker = createWorkerPool(1);
8 |
9 | test('multiple workers of the same type', {timeout: 200}, function (t) {
10 | var worker1 = new PooledWorker(TestWorker);
11 | var worker2 = new PooledWorker(TestWorker);
12 | t.pass('main: create two workers');
13 |
14 | worker1.onmessage = function gotId(type, data) {
15 | t.equal(data, 0, 'main: got id from worker1');
16 | worker2.send('id');
17 | t.pass('request worker2 id');
18 | };
19 | worker2.onmessage = function gotId(type, data) {
20 | t.equal(data, 1, 'main: got id from worker2');
21 | t.end();
22 | };
23 |
24 | worker1.send('id');
25 | t.pass('request worker1 id');
26 | });
27 |
--------------------------------------------------------------------------------
/test/multiple-same-type/worker.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports = function () {
4 | this.onmessage = function () {
5 | this.send('id', this.workerId);
6 | };
7 | };
8 |
--------------------------------------------------------------------------------
/test/send-in-constructor/index.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | var test = require('tape');
4 | var createWorkerPool = require('../../');
5 | var TestWorker = require('./worker');
6 |
7 | var PooledWorker = createWorkerPool(1);
8 |
9 | test('send in worker constructor', {timeout: 200}, function (t) {
10 | var worker = new PooledWorker(TestWorker);
11 | t.pass('main: create worker');
12 |
13 | worker.onmessage = function (type) {
14 | if (type === 'foo') {
15 | t.pass('main: got foo');
16 | t.end();
17 |
18 | } else {
19 | t.fail('main: unexpected message ' + type);
20 | }
21 | };
22 | });
23 |
--------------------------------------------------------------------------------
/test/send-in-constructor/worker.js:
--------------------------------------------------------------------------------
1 | 'use strict';
2 |
3 | module.exports = function () {
4 | this.send('foo');
5 | };
6 |
--------------------------------------------------------------------------------