├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── docs └── design.md ├── examples └── prefork │ ├── README.md │ ├── master.js │ └── worker.js ├── lib ├── webworker-child.js ├── webworker-util.js ├── webworker.js ├── ws.js └── ws │ ├── connection.js │ └── manager.js ├── package.json └── test ├── test-error.js ├── test-fd.js ├── test-simple.js └── workers ├── error.js ├── fd.js └── simple.js /.gitignore: -------------------------------------------------------------------------------- 1 | README.html 2 | docs/design.html 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2010, Peter Griess 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, 8 | this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | * Neither the name of node-webworkers nor the names of its contributors 15 | may be used to endorse or promote products derived from this software 16 | without specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 19 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 20 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 21 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 22 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 24 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 25 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Simple makefile is simple. See README.md. 2 | 3 | INSTALL_DIR ?= /opt/local/share/node 4 | 5 | .PHONY: test install 6 | 7 | test: 8 | for f in `ls ./test/test-*.js` ; do \ 9 | node $$f ; \ 10 | done 11 | 12 | install: 13 | install -m 755 -d $(INSTALL_DIR) 14 | install -m 444 lib/webworker.js lib/webworker-util.js \ 15 | lib/webworker-child.js $(INSTALL_DIR) 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | `node-webworkers` is an implementation of the [Web Workers 2 | API](http://www.whatwg.org/specs/web-workers/current-work/) for 3 | [node.js](http://nodejs.org). 4 | 5 | See the design document 6 | [here](http://blog.std.in/2010/07/08/nodejs-webworker-design/). 7 | 8 | ### Example 9 | 10 | #### Master source 11 | 12 | var sys = require('sys'); 13 | var Worker = require('webworker'); 14 | 15 | var w = new Worker('foo.js'); 16 | 17 | w.onmessage = function(e) { 18 | sys.debug('Received mesage: ' + sys.inspect(e)); 19 | w.terminate(); 20 | }; 21 | 22 | w.postMessage({ foo : 'bar' }); 23 | 24 | #### Worker source 25 | 26 | onmessage = function(e) { 27 | postMessage({ test : 'this is a test' }); 28 | }; 29 | 30 | onclose = function() { 31 | sys.debug('Worker shuttting down.'); 32 | }; 33 | 34 | ### API 35 | 36 | Supported API methods are 37 | 38 | * `postMessage(e)` in both workers and the parent; messages are in the 39 | parent if this is invoked before the child is fully initialized 40 | * `onmessage(e)` in both workers and the parent 41 | * `onerror(e)`in both workers and the parent 42 | * `terminate()` in the parent 43 | 44 | In addition, some nonstandard APIs are provided 45 | 46 | * `onclose()` in the worker (allows for graceful shutdown) 47 | * The `postMessage()` method takes an additional optional file descriptor parameter, which 48 | will be sent with the message. This descriptor will be passed to 49 | `onmessage` handlers as an optional `fd` field. Handlers receiving 50 | messages posted without file descriptors will not see an `fd` field. Both 51 | the parent and child can send file descriptors using this mechanism. 52 | * `Worker.onexit(code, signal)` in the master, which is invoked on the 53 | master `Worker` object when the worker process exits. 54 | * The `Worker` constructor takes an additional optional object argument, 55 | `opts`, which is used as a dictionary of options with the following keys 56 | * `args` : A string or array of strings to pass to the executable before the filename to invoke. This can be used to request that the worker start up in debug mode (e.g. `{ 'args' : '--debug-brk' }`). By default this is empty. 57 | * `path` : A string naming the executable to invoke for workers. By default this is the value of `process.execPath` (e.g. `node` or similar). 58 | 59 | ### Installation 60 | 61 | This package can be installed via [npm](http://npmjs.org/) as follows 62 | 63 | % npm install webworker 64 | 65 | Note that this requires 66 | [node-websocket-client](http://github.com/pgriess/node-websocket-client) v0.9.3 67 | or later. This dependency will be handled automatically by `npm`, but must be 68 | dealt with manually if installing using another procedure. 69 | 70 | ### Credits 71 | 72 | This package contains a static snapshot of Micheil Smith's excellent 73 | [node-websocket-server](http://github.com/miksago/node-websocket-server) with 74 | some fixes applied to handle UNIX sockets. 75 | -------------------------------------------------------------------------------- /docs/design.md: -------------------------------------------------------------------------------- 1 | ## Overview and Motivation 2 | 3 | `node-webworkers` aims to implement as much of the [HTML5 Web 4 | Workers](http://www.whatwg.org/specs/web-workers/current-work/) API as is 5 | practical and useful in the context of NodeJS. Extensions to the HTML5 API 6 | are provided where it makes sense (e.g. to allow file descriptor passing). 7 | 8 | The motivation for providing this API to NodeJS is as follows 9 | 10 | * A set of standard (well, emerging standard anyway) platform-independency 11 | concurrency APIs is a useful abstraction. Particularly as HTML5 gains 12 | wider adoption and JavaScript developers are likely to familiar with Web 13 | Workers from doing browser development. The set of Node.JS primitives 14 | for managing processes, `child_process` provides a lot of utility, but 15 | is easily misunderstood by developers who have not developed for a UNIX 16 | platofrm before (e.g. "why does `kill()` not kill my process?"). 17 | 18 | In addition, the error reporting APIs in the Web Workers spec are more 19 | full-featured and JavaScript-specific than that provided natively by 20 | `child_process` (e.g. one can get a stack trace, etc). 21 | 22 | * Existing communicaiton mechanisms with child processes involve 23 | communicating over `stdin`/`stdout`. These are opaque byte streams and 24 | require the application to implement their own framing logic to discern 25 | message boundaries. Further, use of these built-in streams prevents 26 | `sys.puts()` and friends from working. 27 | 28 | * [Shared Workers](http://dev.w3.org/html5/workers/#shared-workers-and-the-sharedworker-interface) 29 | provide a useful naming service for communicating with other workers by 30 | name. Without this, the application must maintain its own routing 31 | structure. 32 | 33 | Note that shared workers are *not* implemented yet. 34 | 35 | ## Design 36 | 37 | The design that follows for Web Workers is motivated by a handful of 38 | underlying assumptions / philosophies: 39 | 40 | * Worker instances should be relatively long-lived. That is, it is not 41 | considered an important workload to be able to create and destroy 42 | thousands of workers as quickly as possible. Passing messages to 43 | existing workers to dispatch work items is favored over creating a new 44 | worker for each work item. 45 | 46 | * In the future, it will be desirable to run workers off-box, and to 47 | implement workers in other application frameworks / languages. This is 48 | particularly relevant in the choice of communication medium. 49 | 50 | In addition, there is a general preference to embracing standards and 51 | existing building blocks, particularly those that are geared towards 52 | JavaScript and/or HTTP. 53 | 54 | ### Worker Processes 55 | 56 | Each worker executes in its own self-contained `node` process rather than as 57 | a separate thread and V8 context within the master process. 58 | 59 | Benefits of this approach include fault isolation (any worker running out of 60 | memory or triggering some buggy C++ code will not take down other workers); 61 | avoiding the complexity of managing multiple event loops in a single 62 | process; and typical OSes are more likely to schedule different processes on 63 | different CPUs (this may not always happen for multiple threads within the 64 | same process), allowing the application to utilize multiple CPUs. 65 | 66 | Drawbkacks include the cost of context switching between workers being more 67 | expensive when using a process-per-worker model than it would be in a 68 | thread-per-worker model; passing messages between processes typically 69 | requires a data copy and always requires serializing data; and overhead of 70 | spawning a new process. 71 | 72 | ### Worker Context 73 | 74 | Each worker is launched by `webworker-child.js`. 75 | 76 | This script is passed to `node` as the entry point for the process and is 77 | responsible for constructing a V8 script context populated with bits 78 | relevant to the Web Worker API (e.g. the `postMessage()`, `close()` and 79 | `location` primitives, etc). 80 | 81 | ### Inter-worker Communication 82 | 83 | The Web Workers spec provides a simple message passing API. 84 | 85 | Under the covers, this is implemented by connecting each dedicated worker to 86 | its parent process with a UNIX domain socket. This is lower overhead than 87 | TCP, and allows for UNIX goodies like file descriptor passing. 88 | 89 | Message passing is done over this UNIX socket by negotiating an HTML5 Web 90 | Socket connection over this transport. This is done to provide a 91 | reasonably-performant standards-based message framing implementation and to 92 | lay the groundwork or communicating with off-box workers via HTTP over TCP, 93 | which may be implemented in another application stack entirely (e.g. Java, 94 | etc). The overhead of negotiating and maintaining the Web Socket connection 95 | is 1 round trip for handshaking and the overhead of maintaining HTTP state 96 | objects (`http_parser` and such). The handshaking overhead is not considered 97 | an undue burden given that workers are expected to be relatively long-lived 98 | and the HTTP state overhead considered small. 99 | 100 | The format of the messages themselves is JSON, serialized using 101 | `JSON.stringify()` and de-serialized using `JSON.parse()`. Significantly, 102 | the use of a framing protocol allows the Web Workers implementation to wait 103 | for an entire, complete JSON blob to arrive before invoking `JSON.parse()`. 104 | -------------------------------------------------------------------------------- /examples/prefork/README.md: -------------------------------------------------------------------------------- 1 | An example of using Web Workers to implement a preforking HTTP server. 2 | -------------------------------------------------------------------------------- /examples/prefork/master.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | var netBindings = process.binding('net'); 3 | var Worker = require('../lib/webworker'); 4 | 5 | var fd = netBindings.socket('tcp4'); 6 | netBindings.bind(fd, 80); 7 | netBindings.listen(fd, 128); 8 | 9 | for (var i = 0; i < 8; i++) { 10 | var w = new Worker(path.join(__dirname, 'worker.js')); 11 | w.postMessage({ 'banner' : 'Hello, world!' }, fd); 12 | } 13 | -------------------------------------------------------------------------------- /examples/prefork/worker.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert'); 2 | var http = require('http'); 3 | 4 | process.setuid('nobody'); 5 | 6 | var banner = undefined; 7 | 8 | var srv = http.createServer(function(req, resp) { 9 | resp.writeHead(200, {'Content-Type' : 'text/plain'}); 10 | resp.write(banner + ' (pid ' + process.pid + ')\n'); 11 | resp.end(); 12 | }); 13 | 14 | onmessage = function(msg) { 15 | assert.ok(msg.fd && msg.fd > 0); 16 | 17 | banner = msg.data.banner; 18 | 19 | srv.listenFD(msg.fd); 20 | }; 21 | -------------------------------------------------------------------------------- /lib/webworker-child.js: -------------------------------------------------------------------------------- 1 | // Launcher script for WebWorkers. 2 | // 3 | // Sets up context and runs a worker script. This is not intended to be 4 | // invoked directly. Rather, it is invoked automatically when constructing a 5 | // new Worker() object. 6 | // 7 | // usage: node worker.js