├── .gitignore ├── History.md ├── Makefile ├── README.md ├── lib └── conflation.js ├── main.js ├── package.json └── test └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | lib-cov 3 | *.seed 4 | *.log 5 | *.csv 6 | *.dat 7 | *.out 8 | *.pid 9 | *.gz 10 | 11 | pids 12 | logs 13 | results 14 | 15 | node_modules 16 | npm-debug.log -------------------------------------------------------------------------------- /History.md: -------------------------------------------------------------------------------- 1 | 0.0.1 / 2012-08-13 2 | ================== 3 | 4 | * Initial release 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | TESTS = test/*.js 2 | REPORTER = dot 3 | 4 | test: 5 | @./node_modules/.bin/mocha \ 6 | --reporter $(REPORTER) \ 7 | --slow 500ms \ 8 | --bail \ 9 | $(TESTS) 10 | 11 | .PHONY: test 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | engine.io-conflation 2 | ==================== 3 | 4 | `engine.io-conflation` is an [engine.io](https://github.com/LearnBoost/engine.io) (>= 0.2.0) plugin that makes **[conflation](http://magmasystems.blogspot.jp/2006/08/conflation.html)**, **aggregation**, **alteration** and **filtering** of messages straightforward, especially when it has to based on the client's performance consuming messages from the server. 5 | 6 | This is useful to **reduce the size of the payload for slow consumers** that cannot keep up with the frequency of messages, because of a low bandwidth connection, or low processing power. But it is generic enough to allow for not only conflation, i.e. deletion of messages, but also additions and modifications, for whatever purpose that might be useful. 7 | 8 | To use it, you have to create a function, lets call it `myConflater`, and register it by means of `server.on('flush', createConflater(myConflater)`. 9 | 10 | Your function will then be called every time the buffered array of messages is about to be flushed to the client, which happens only when the client has consumed the previous batch of messages, i.e. the client is ready to receive again. The array of messages that `myConflater` returns is what will then actually be sent to the client. 11 | 12 | # Usage # 13 | --------- 14 | 15 | ## A simple example that changes all messages to upper case ## 16 | 17 | ```js 18 | var engine = require('engine.io') 19 | , server = engine.listen(80) 20 | , createConflater = require('engine.io-conflation').createConflater; 21 | 22 | server.on('connection', function (socket) { 23 | socket.send('utf 8 string'); 24 | }); 25 | 26 | var myConflater = function (messages) { 27 | return messages.map(function (msg) { return msg.toUpperCase(); }); 28 | }; 29 | 30 | server.on('flush', createConflater(myConflater)); 31 | 32 | ``` 33 | 34 | In the example above, `myConflater` would make the message *'UTF 8 STRING'* (upper case) be sent to the client. 35 | 36 | ## A more realistic example that does conflation ## 37 | ```js 38 | var engine = require('engine.io') 39 | , server = engine.listen(80) 40 | , createConflater = require('engine.io-conflation').createConflater 41 | , symbols = [ 'Wheat', 'Corn', 'Soybeans' ] 42 | , randomInt = function(maxInt) { return Math.floor(Math.random() * maxInt); } 43 | 44 | function PriceQuote(symbol, price) { 45 | this.symbol = symbol; 46 | this.price = price; 47 | } 48 | // toString() is used by engine.io to serialize the data before sending it 49 | PriceQuote.prototype.toString = function() { return JSON.stringify(this); } 50 | 51 | var keepLastQuoteOnlyConflater = function(quotes) { 52 | var lastQuotes = {}; 53 | for (var i = 0, l = quotes.length; i < l; i++) { 54 | lastQuotes[quotes[i].symbol] = quotes[i]; 55 | } 56 | var lastQuotesArray = []; 57 | for (var grain in lastQuotes) 58 | lastQuotesArray.push(lastQuotes[grain]); 59 | return lastQuotesArray; 60 | } 61 | 62 | server.on('connection', function (socket) { 63 | while (true) { // generate random price quotes in rapid succession 64 | socket.send(new PriceQuote(symbols[randomInt(symbols.length)], 500 + randomInt(50))); 65 | } 66 | }); 67 | 68 | server.on('flush', createConflater(keepLastQuoteOnlyConflater)); 69 | 70 | ``` 71 | 72 | With the above conflater, the client will only get the most recent price quotes when it is ready. Quotes that *come in* while the client is still processing the current buffer will silently overwrite previous quotes with the same `symbol` key, and the client will not receive those overwritten values. 73 | 74 | # How it works internally # 75 | --------------------------- 76 | 77 | `createConflater` returns a function that takes the internal buffer of messages as provided by `engine.io`'s `flush` event. This buffer may contain heartbeats and other low-level messages that the actual conflater is neither interested in nor should mess with. `createConflater` then passes only the high-level messages (i.e. those sent via `socket.send(msg)`) to `myConflater`, so that the latter can work on the level of abstraction that users of `engine.io` normally work on. 78 | 79 | 80 | # Hacking engine.io-conflation # 81 | -------------------------------- 82 | 83 | In case you want to start hacking `engine.io-conflation`, start by making sure that the tests run: 84 | 85 | ``` 86 | npm install 87 | make test 88 | ``` 89 | 90 | Then after any code change, make sure the tests still run. -------------------------------------------------------------------------------- /lib/conflation.js: -------------------------------------------------------------------------------- 1 | exports.createConflater = function createConflater(conflater) { 2 | return function conflateOnFlush(socket, writeBuffer) { 3 | 4 | var nonMessages = []; 5 | var messages = []; 6 | 7 | for (var i = 0, l = writeBuffer.length; i < l; i++) { 8 | var current = writeBuffer[i]; 9 | if (current.type === 'message') { 10 | messages.push(current.data); 11 | } else { 12 | nonMessages.push(current); 13 | } 14 | } 15 | 16 | if (messages.length > 0) { 17 | messages = conflater(messages); 18 | 19 | messages = Array.prototype.map.call(messages, function(msg) { return { type: 'message', data: msg }; }, []); 20 | writeBuffer.length = 0; 21 | Array.prototype.push.apply(writeBuffer, nonMessages); 22 | Array.prototype.push.apply(writeBuffer, messages); 23 | } 24 | }; 25 | } 26 | -------------------------------------------------------------------------------- /main.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/conflation'); 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "engine.io-conflation", 3 | "version": "0.0.0", 4 | "description": "engine.io plugin that makes conflation, aggregation, alteration and filtering of messages, based on the client's performance consuming messages from the server, straightforward.", 5 | "main": "main.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "repository": "", 10 | "keywords": [ 11 | "engine.io", 12 | "plugin", 13 | "conflation", 14 | "aggregation", 15 | "filtering" 16 | ], 17 | "devDependencies": { 18 | "expect.js": "*", 19 | "engine.io": "0.2.0", 20 | "engine.io-client": "*", 21 | "mocha": "*" 22 | }, 23 | "author": "Eugen Dueck", 24 | "license": "BSD" 25 | } 26 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var eio = require('engine.io') 2 | , eioc = require('engine.io-client') 3 | , expect = require('expect.js') 4 | , createConflater = require('../main').createConflater; 5 | 6 | var listen = function listen(opts, fn) { 7 | if ('function' == typeof opts) { 8 | fn = opts; 9 | opts = {}; 10 | } 11 | 12 | var e = eio.listen(null, opts, function () { 13 | fn(e.httpServer.address().port); 14 | }); 15 | 16 | return e; 17 | } 18 | 19 | var asIsConflater = createConflater(function asIsConflater(messages) { 20 | return messages; }); 21 | 22 | var droppingConflater = createConflater(function droppingConflater(messages) { 23 | return []; }); 24 | 25 | var dropEveryOtherMessageConflater = createConflater(function localScope() { 26 | var other = false; 27 | return function droppingConflater(messages) { 28 | return messages.filter(function (msg) { other = !other; return other; }); 29 | }}()); 30 | 31 | var upCasingConflater = createConflater(function upCasingConflater(messages) { 32 | return messages.map(function (msg) { return msg.toUpperCase(); }); }); 33 | 34 | var duplicatingConflater = createConflater(function upCasingConflater(messages) { 35 | return messages.reduce(function (acc, msg) { acc.push(msg); acc.push(msg); return acc; }, []); }); 36 | 37 | describe('conflation', function(){ 38 | 39 | describe('pass-through conflater', function(){ 40 | 41 | it('should let a single message pass through as is', function(done) { 42 | var engine = listen(function (port) { 43 | var socket = new eioc.Socket('ws://localhost:' + port) 44 | , expected = ['a'] 45 | , i = 0; 46 | 47 | engine.on('flush', asIsConflater); 48 | 49 | engine.on('connection', function (conn) { 50 | conn.send('a'); 51 | 52 | conn.on('close', function () { 53 | // since close fires right after the buffer is drained 54 | setTimeout(function () { 55 | expect(i).to.be(expected.length); 56 | done(); 57 | }, 50); 58 | }); 59 | 60 | conn.close(); 61 | }); 62 | 63 | socket.on('open', function () { 64 | socket.on('message', function (msg) { 65 | expect(msg).to.be(expected[i++]); 66 | }); 67 | }); 68 | }); 69 | }); 70 | 71 | it('should let multiple messages pass through as is', function(done) { 72 | var engine = listen(function (port) { 73 | var socket = new eioc.Socket('ws://localhost:' + port) 74 | , expected = ['a', 'b', 'c', 'd'] 75 | , i = 0; 76 | 77 | engine.on('flush', asIsConflater); 78 | 79 | engine.on('connection', function (conn) { 80 | 81 | conn.send('a'); 82 | conn.send('b'); 83 | 84 | // also send some of the messages in a later batch 85 | setTimeout(function () { 86 | conn.send('c'); 87 | conn.send('d'); 88 | 89 | setTimeout(function () { 90 | conn.close(); 91 | }, 50); 92 | }, 50); 93 | 94 | conn.on('close', function () { 95 | // since close fires right after the buffer is drained 96 | setTimeout(function () { 97 | expect(i).to.be(expected.length); 98 | done(); 99 | }, 50); 100 | }); 101 | 102 | }); 103 | 104 | socket.on('open', function () { 105 | socket.on('message', function (msg) { 106 | expect(msg).to.be(expected[i++]); 107 | }); 108 | }); 109 | }); 110 | }); 111 | 112 | }); 113 | 114 | describe('duplicating conflater', function(){ 115 | 116 | it('should duplicate a single message', function(done) { 117 | var engine = listen(function (port) { 118 | var socket = new eioc.Socket('ws://localhost:' + port) 119 | , expected = ['a', 'a'] 120 | , i = 0; 121 | 122 | engine.on('flush', duplicatingConflater); 123 | 124 | engine.on('connection', function (conn) { 125 | conn.send('a'); 126 | 127 | conn.on('close', function () { 128 | // since close fires right after the buffer is drained 129 | setTimeout(function () { 130 | expect(i).to.be(expected.length); 131 | done(); 132 | }, 50); 133 | }); 134 | 135 | conn.close(); 136 | }); 137 | 138 | socket.on('open', function () { 139 | socket.on('message', function (msg) { 140 | expect(msg).to.be(expected[i++]); 141 | }); 142 | }); 143 | }); 144 | }); 145 | 146 | it('should duplicate multiple messages', function(done) { 147 | var engine = listen(function (port) { 148 | var socket = new eioc.Socket('ws://localhost:' + port) 149 | , expected = ['a', 'a', 'b', 'b', 'c', 'c', 'd', 'd'] 150 | , i = 0; 151 | 152 | engine.on('flush', duplicatingConflater); 153 | 154 | engine.on('connection', function (conn) { 155 | 156 | conn.send('a'); 157 | conn.send('b'); 158 | 159 | // also send some of the messages in a later batch 160 | setTimeout(function () { 161 | conn.send('c'); 162 | conn.send('d'); 163 | 164 | setTimeout(function () { 165 | conn.close(); 166 | }, 50); 167 | }, 50); 168 | 169 | conn.on('close', function () { 170 | // since close fires right after the buffer is drained 171 | setTimeout(function () { 172 | expect(i).to.be(expected.length); 173 | done(); 174 | }, 50); 175 | }); 176 | 177 | }); 178 | 179 | socket.on('open', function () { 180 | socket.on('message', function (msg) { 181 | expect(msg).to.be(expected[i++]); 182 | }); 183 | }); 184 | }); 185 | }); 186 | 187 | }); 188 | 189 | describe('message dropping conflater', function(){ 190 | 191 | it('should swallow a single message', function(done) { 192 | var engine = listen(function (port) { 193 | var socket = new eioc.Socket('ws://localhost:' + port) 194 | , expected = [] 195 | , i = 0; 196 | 197 | engine.on('flush', droppingConflater); 198 | 199 | engine.on('connection', function (conn) { 200 | conn.send('a'); 201 | 202 | conn.on('close', function () { 203 | // since close fires right after the buffer is drained 204 | setTimeout(function () { 205 | expect(i).to.be(expected.length); 206 | done(); 207 | }, 50); 208 | }); 209 | conn.close(); 210 | }); 211 | 212 | socket.on('open', function () { 213 | socket.on('message', function (msg) { 214 | expect(msg).to.be(expected[i++]); 215 | }); 216 | }); 217 | }); 218 | }); 219 | 220 | it('should swallow multiple messages', function(done) { 221 | var engine = listen(function (port) { 222 | var socket = new eioc.Socket('ws://localhost:' + port) 223 | , expected = [] 224 | , i = 0; 225 | 226 | engine.on('flush', droppingConflater); 227 | 228 | engine.on('connection', function (conn) { 229 | conn.send('a'); 230 | conn.send('b'); 231 | 232 | // also send some of the messages in a later batch 233 | setTimeout(function () { 234 | conn.send('c'); 235 | conn.send('d'); 236 | conn.on('close', function () { 237 | // since close fires right after the buffer is drained 238 | setTimeout(function () { 239 | expect(i).to.be(expected.length); 240 | done(); 241 | }, 50); 242 | }); 243 | setTimeout(function () { 244 | conn.close(); 245 | }, 50); 246 | }, 50); 247 | }); 248 | 249 | socket.on('open', function () { 250 | socket.on('message', function (msg) { 251 | expect(msg).to.be(expected[i++]); 252 | }); 253 | }); 254 | }); 255 | }); 256 | 257 | it('should drop every other message', function(done) { 258 | var engine = listen(function (port) { 259 | var socket = new eioc.Socket('ws://localhost:' + port) 260 | , expected = ['a', 'c'] 261 | , i = 0; 262 | 263 | engine.on('flush', dropEveryOtherMessageConflater); 264 | 265 | engine.on('connection', function (conn) { 266 | 267 | conn.send('a'); 268 | conn.send('b'); 269 | 270 | // also send some of the messages in a later batch 271 | setTimeout(function () { 272 | conn.send('c'); 273 | conn.send('d'); 274 | 275 | setTimeout(function () { 276 | conn.close(); 277 | }, 50); 278 | }, 50); 279 | 280 | conn.on('close', function () { 281 | // since close fires right after the buffer is drained 282 | setTimeout(function () { 283 | expect(i).to.be(expected.length); 284 | done(); 285 | }, 50); 286 | }); 287 | 288 | }); 289 | 290 | socket.on('open', function () { 291 | socket.on('message', function (msg) { 292 | expect(msg).to.be(expected[i++]); 293 | }); 294 | }); 295 | }); 296 | }); 297 | 298 | }); 299 | 300 | describe('message modifying conflater', function(){ 301 | 302 | it('should upcase a single message', function(done) { 303 | var engine = listen(function (port) { 304 | var socket = new eioc.Socket('ws://localhost:' + port) 305 | , expected = ['A'] 306 | , i = 0; 307 | 308 | engine.on('flush', upCasingConflater); 309 | 310 | engine.on('connection', function (conn) { 311 | conn.send('a'); 312 | 313 | conn.on('close', function () { 314 | // since close fires right after the buffer is drained 315 | setTimeout(function () { 316 | expect(i).to.be(expected.length); 317 | done(); 318 | }, 50); 319 | }); 320 | 321 | conn.close(); 322 | }); 323 | 324 | socket.on('open', function () { 325 | socket.on('message', function (msg) { 326 | expect(msg).to.be(expected[i++]); 327 | }); 328 | }); 329 | }); 330 | }); 331 | 332 | it('should upcase multiple messages', function(done) { 333 | var engine = listen(function (port) { 334 | var socket = new eioc.Socket('ws://localhost:' + port) 335 | , expected = ['A', 'B', 'C', 'D'] 336 | , i = 0; 337 | 338 | engine.on('flush', upCasingConflater); 339 | 340 | engine.on('connection', function (conn) { 341 | 342 | conn.send('a'); 343 | conn.send('b'); 344 | 345 | // also send some of the messages in a later batch 346 | setTimeout(function () { 347 | conn.send('c'); 348 | conn.send('d'); 349 | 350 | setTimeout(function () { 351 | conn.close(); 352 | }, 50); 353 | }, 50); 354 | 355 | conn.on('close', function () { 356 | // since close fires right after the buffer is drained 357 | setTimeout(function () { 358 | expect(i).to.be(expected.length); 359 | done(); 360 | }, 50); 361 | }); 362 | 363 | }); 364 | 365 | socket.on('open', function () { 366 | socket.on('message', function (msg) { 367 | expect(msg).to.be(expected[i++]); 368 | }); 369 | }); 370 | }); 371 | }); 372 | 373 | }); 374 | 375 | }); --------------------------------------------------------------------------------