├── README ├── conf-dispatcher ├── couch-client.js └── server.js ├── flow ├── couch-client.js ├── flow2couchdb.config └── flow2couchdb.js ├── queue.js ├── sashimi.js ├── test_packet_source.js └── tun.c /README: -------------------------------------------------------------------------------- 1 | It's just good enough to run a conference 2 | 3 | but a far away from a good piece of software 4 | 5 | to start the server 6 | 7 | sudo tun server ip:port 8 | 9 | to start a client 10 | 11 | sudo tun client srcip:destip:port srcip:destip:port srcip:destip:port 12 | currently not implemented with the srcip 13 | 14 | where is the bind call for net.createConnection 15 | 16 | you need 17 | ip route 18 | ip rule 19 | 20 | 21 | -------------------------------------------------------------------------------- /conf-dispatcher/couch-client.js: -------------------------------------------------------------------------------- 1 | /*global Buffer */ 2 | 3 | var http = require('http'), 4 | https = require('https'), 5 | Url = require('url'), 6 | EventEmitter = require('events').EventEmitter, 7 | querystring = require('querystring'); 8 | 9 | 10 | // Handles changes made in node v0.3.0 11 | var NOT_FOUND_ERR_NO = process.ENOENT ? process.ENOENT : require('constants').ENOENT; 12 | var MAX_DOCS = 1000; // The maximum number of docs to send in a single batch 13 | var MAX_DOCS = 1; // The maximum number of docs to send in a single batch 14 | 15 | function noOp(err) { if (err) { throw err; } } 16 | 17 | var CONNECTION_DEFAULTS = { 18 | host: '127.0.0.1:5984', 19 | port: 5984, 20 | hostname: '127.0.0.1', 21 | pathname: "/" 22 | }; 23 | 24 | function CouchClient(url) { 25 | var uri = Url.parse(url); 26 | uri.secure = uri.protocol == 'https:'; 27 | uri.protocolHandler = uri.secure ? https : http; 28 | uri.__proto__ = CONNECTION_DEFAULTS; 29 | var revCache = {}; 30 | 31 | // A simple wrapper around node's http(s) request. 32 | function request(method, path, body, callback) { 33 | var stream; 34 | // Body is optional 35 | if (typeof body === 'function' && typeof callback === 'undefined') { 36 | callback = body; 37 | body = undefined; 38 | } 39 | // Return a stream if no callback is specified 40 | if (!callback) { 41 | stream = new EventEmitter(); 42 | stream.setEncoding = function () { 43 | throw new Error("This stream is always utf8"); 44 | }; 45 | } 46 | 47 | function errorHandler(err) { 48 | if (callback) { callback(err); } 49 | if (stream) { stream.emit('error', err); } 50 | } 51 | 52 | var headers = { 53 | "Host": uri.hostname 54 | }; 55 | // add the authorization header if provided and using https 56 | if (uri.auth) { 57 | headers["Authorization"] = "Basic " + new Buffer(uri.auth, "ascii").toString("base64"); 58 | } 59 | 60 | if (body) { 61 | body = JSON.stringify(body); 62 | headers["Content-Length"] = Buffer.byteLength(body); 63 | headers["Content-Type"] = "application/json"; 64 | } 65 | 66 | var options = { 67 | host: uri.hostname, 68 | method: method, 69 | path: path, 70 | port: uri.port, 71 | headers: headers 72 | }; 73 | //console.log('-4-'+require('util').inspect(options)+"="+(uri.protocolHandler == http)); 74 | var request = uri.protocolHandler.request(options, function (response) { 75 | response.setEncoding('utf8'); 76 | var body = ""; 77 | response.on('data', function (chunk) { 78 | if (callback) { body += chunk; } 79 | if (stream) { stream.emit('data', chunk); } 80 | }); 81 | response.on('end', function () { 82 | if (callback) { 83 | try { 84 | var parsedBody = JSON.parse(body); 85 | callback(null, parsedBody); 86 | } catch(err) { 87 | callback(err); 88 | } 89 | } 90 | if (stream) { stream.emit('end'); } 91 | }); 92 | response.on('error', errorHandler); 93 | }); 94 | request.on('error', errorHandler); 95 | 96 | if (body) { request.write(body, 'utf8'); } 97 | request.end(); 98 | 99 | return stream; 100 | } 101 | 102 | // Requests UUIDs from the couch server in tick batches 103 | var uuidQueue = []; 104 | function getUUID(callback) { 105 | uuidQueue.push(callback); 106 | if (uuidQueue.length > 1) { return; } 107 | function consumeQueue() { 108 | var pending = uuidQueue.splice(0, MAX_DOCS); 109 | if (uuidQueue.length) { process.nextTick(consumeQueue); } 110 | // console.log("Bulk getting UUIDs %s", pending.length); 111 | request("GET", "/_uuids?count=" + pending.length, function (err, result) { 112 | if (err) { 113 | pending.forEach(function (callback) { 114 | callback(err); 115 | }); 116 | return; 117 | } 118 | if (result.uuids.length !== pending.length) { 119 | throw new Error("Wrong number of UUIDs generated " + result.uuids.length + " != " + pending.length); 120 | } 121 | result.uuids.forEach(function (uuid, i) { 122 | pending[i](null, uuid); 123 | }); 124 | }); 125 | } 126 | process.nextTick(consumeQueue); 127 | } 128 | 129 | // Saves documents in batches 130 | var saveValues = []; 131 | var saveQueue = []; 132 | function realSave(doc, callback) { 133 | // Put key and rev on the value without changing the original 134 | saveValues.push(doc); 135 | saveQueue.push(callback); 136 | if (saveQueue.length > 1) { return; } 137 | function consumeQueue() { 138 | var pending = saveQueue.splice(0, MAX_DOCS); 139 | var body = saveValues.splice(0, MAX_DOCS); 140 | if (saveQueue.length) { process.nextTick(consumeQueue); } 141 | // console.log("Bulk saving %s", body.length); 142 | request("POST", uri.pathname + "/_bulk_docs", {docs: body}, function (err, results) { 143 | if (results.error) { 144 | err = new Error("CouchDB Error: " + JSON.stringify(results)); 145 | if (results.error === 'not_found') { err.errno = NOT_FOUND_ERR_NO; } 146 | } 147 | if (err) { 148 | pending.forEach(function (callback) { 149 | callback(err); 150 | }); 151 | return; 152 | } 153 | console.log("*****"+require('util').inspect(results)); 154 | results.forEach(function (result, i) { 155 | var doc = body[i]; 156 | doc._id = result.id; 157 | doc._rev = result.rev; 158 | revCache[result.id] = result.rev; 159 | pending[i](null, doc); 160 | }); 161 | }); 162 | } 163 | process.nextTick(consumeQueue); 164 | } 165 | 166 | var getQueue = []; 167 | var getKeys = []; 168 | function realGet(key, includeDoc, callback) { 169 | getKeys.push(key); 170 | getQueue.push(callback); 171 | if (getQueue.length > 1) { return; } 172 | function consumeQueue() { 173 | var pending = getQueue.splice(0, MAX_DOCS); 174 | var keys = getKeys.splice(0, MAX_DOCS); 175 | if (getQueue.length) { process.nextTick(consumeQueue); } 176 | var path = uri.pathname + "/_all_docs"; 177 | if (includeDoc) { path += "?include_docs=true"; } 178 | console.log("Bulk Getting %s documents", require('util').inspect(keys)); 179 | 180 | request("POST", path, {keys: keys}, function (err, results) { 181 | if (err) { 182 | debugger; 183 | pending.forEach(function (callback) { 184 | callback(err); 185 | }); 186 | return; 187 | } 188 | if (results && !results.rows) { 189 | err = new Error("CouchDB Error: " + JSON.stringify(results)); 190 | } 191 | results.rows.forEach(function (result, i) { 192 | var err; 193 | if (includeDoc) { 194 | if (result.error) { 195 | err = new Error("CouchDB Error: " + JSON.stringify(result)); 196 | if (result.error === 'not_found') { err.errno = NOT_FOUND_ERR_NO; } 197 | pending[i](err); 198 | return; 199 | } 200 | if (!result.doc) { 201 | err = new Error("Document not found for " + JSON.stringify(result.key)); 202 | err.errno = NOT_FOUND_ERR_NO; 203 | pending[i](err); 204 | return; 205 | } 206 | pending[i](null, result.doc); 207 | return; 208 | } 209 | pending[i](null, result.value); 210 | }); 211 | }); 212 | } 213 | process.nextTick(consumeQueue); 214 | } 215 | 216 | 217 | function save(doc, callback) { 218 | if (!callback) { callback = noOp; } 219 | if (doc._id) { 220 | if (!doc._rev) { 221 | if (!revCache.hasOwnProperty(doc._id)) { 222 | realGet(doc._id, false, function (err, result) { 223 | if (err) { return callback(err); } 224 | if (result) { 225 | revCache[doc._id] = result.rev; 226 | doc._rev = result.rev; 227 | } 228 | realSave(doc, callback); 229 | }); 230 | return; 231 | } 232 | doc._rev = revCache[doc._id]; 233 | } 234 | } 235 | realSave(doc, callback); 236 | } 237 | 238 | function get(key, callback) { 239 | realGet(key, true, callback); 240 | } 241 | 242 | function remove(doc, callback) { 243 | if (typeof doc === 'string') { 244 | doc = {_id: doc}; 245 | } 246 | doc._deleted = true; 247 | save(doc, callback); 248 | } 249 | 250 | function changes(since, callback) { 251 | var stream = request("GET", uri.pathname + "/_changes?feed=continuous&heartbeat=1000&since=" + since); 252 | var data = ""; 253 | function checkData() { 254 | var p = data.indexOf("\n"); 255 | if (p >= 0) { 256 | var line = data.substr(0, p).trim(); 257 | data = data.substr(p + 1); 258 | if (line.length) { 259 | callback(null, JSON.parse(line)); 260 | } 261 | checkData(); 262 | } 263 | } 264 | stream.on('error', callback); 265 | stream.on('data', function (chunk) { 266 | data += chunk; 267 | checkData(); 268 | }); 269 | stream.on('end', function () { 270 | throw new Error("Changes feed got broken!"); 271 | }); 272 | } 273 | 274 | function view(viewName, obj, callback) { 275 | var method = "GET"; 276 | var body = null; 277 | if (typeof obj === 'function') { 278 | callback = obj; 279 | obj = null; 280 | } 281 | 282 | if ( viewName.substr(0,1) != '/' ) { 283 | // assume viewname is designdocname/viewname 284 | var parts = viewName.split("/",2); 285 | if ( parts.length == 2 ) { 286 | viewName = uri.pathname+"/_design/"+parts[0]+"/_view/"+parts[1]; 287 | } 288 | } 289 | 290 | if (obj && typeof obj === 'object') { 291 | Object.keys(obj).forEach(function(key){ 292 | if ( key === 'keys' ) { 293 | body = { keys: obj[key] }; // body is json stringified in request fn 294 | method='POST'; 295 | } else { 296 | obj[key] = JSON.stringify(obj[key]); 297 | } 298 | }); 299 | var getParams = querystring.stringify(obj); 300 | if (getParams){ 301 | viewName = viewName + '?' + getParams; 302 | } 303 | } 304 | request(method, viewName, body, callback); 305 | } 306 | 307 | // Expose the public API 308 | return { 309 | get: get, 310 | save: save, 311 | remove: remove, 312 | changes: changes, 313 | request: request, 314 | uri: uri, 315 | view: view 316 | }; 317 | } 318 | 319 | module.exports = CouchClient; 320 | -------------------------------------------------------------------------------- /conf-dispatcher/server.js: -------------------------------------------------------------------------------- 1 | var http = require('http'); 2 | var url = require('url'); 3 | var util = require('util'); 4 | var fs = require('fs'); 5 | 6 | var CouchClient = require('./couch-client'); 7 | 8 | var debug = process.argv[2] == 'debug'; 9 | var production = !(process.argv[3] == 'test'); 10 | 11 | var listen = { host: "127.0.0.1", port: 8124 }; 12 | 13 | var callStreamie = function(token, fn) { 14 | var streamie = http.request({ 15 | host: 'streamie.org', 16 | port: 80, 17 | path: '/user_info', 18 | method: 'GET', 19 | headers: { 20 | 'Host': 'streamie.org', 21 | 'Cookie': 'token='+token 22 | } 23 | }, function(res) { 24 | res.setEncoding('utf8') 25 | var data = []; 26 | res.on('data', function(doc) { 27 | data.push(data.toString("utf-8")); 28 | }) 29 | res.on('end', function(doc) { 30 | try { 31 | doc = JSON.parse(data.join("")); 32 | doc.statusCode = res.statusCode; 33 | fn(doc); 34 | } catch(e) { 35 | console.error('callStreamie:exception:'+e) 36 | } 37 | }) 38 | }).end(); 39 | } 40 | 41 | var getMacAddress = function(address, fn) { 42 | fs.readFile('/proc/net/arp', 'utf8', function(err, data) { 43 | if (err) { 44 | console.error('can not read /proc/net/arp:'+err); 45 | return; 46 | } 47 | var lines = data.toString().split("\n") 48 | for(var i = lines.length-1; i >= 0; --i) { 49 | var line = lines[i].split(/\s+/) 50 | if (line[0] == address) { 51 | return fn(line[3]); 52 | } 53 | } 54 | fn(); 55 | }); 56 | } 57 | 58 | var updateStreamie = function(mac, req, ret, retryCnt) { 59 | retryCnt = retryCnt || 0; 60 | streamie.get(ret.user_id, function(err, doc) { 61 | if (err) { 62 | console.error('updateStreamie:couchdb:get:failure:'+err); 63 | } 64 | var client = { 65 | ipv4: req.headers['x-real-ip'] || req.socket.remoteAddress, 66 | hwaddr: mac, 67 | useragent: req.headers['user-agent'], 68 | created_at: new Date() 69 | }; 70 | if (!doc) { 71 | console.log('NEW-DOC:'+ret.user_id); 72 | doc = { 73 | _id: ret.user_id, 74 | twitter: ret, 75 | clients: [client] 76 | }; 77 | } else { 78 | doc.twitter = ret; 79 | delete doc.completed 80 | var found = false; 81 | for(var i = doc.clients.length-1; i >= 0; --i) { 82 | var r_ip = req.headers['x-real-ip'] || req.socket.remoteAddress; 83 | if (doc.clients[i].ipv4 == r_ip) { 84 | console.log('UPD-IPUPDATE:'+ret.user_id); 85 | doc.clients[i] = client; 86 | found = true; 87 | break; 88 | } 89 | } 90 | if (!found) { 91 | console.log('UPD-ADDCLIENT:'+ret.user_id); 92 | doc.clients.push(client); 93 | } 94 | } 95 | /* 96 | streamie.save(doc, function(err, doc) { 97 | if (err) { 98 | // retry raise condition 99 | if (retryCnt < 5) { 100 | console.error('updateStreamie:couchdb:save:failure:'+err+":"+retryCnt); 101 | setTimeout(function() { updateStreamie(mac, req, ret, retryCnt + 1); }, 500); 102 | } else { 103 | console.error('updateStreamie:couchdb:save:failure:'+err+":MAX-RETRIED"); 104 | } 105 | } 106 | }); 107 | */ 108 | }); 109 | } 110 | 111 | var streamie = CouchClient('http://127.0.0.1:5984/streamie'); 112 | streamie.request('PUT', '/streamie', function(err, result) { 113 | http.createServer(function (req, res) { 114 | var dispatch = url.parse(req.url, true); 115 | if (dispatch.pathname == '/authorize') { 116 | callStreamie(dispatch.query['token'], function(ret) { 117 | ret.oauth = dispatch.query['token'] 118 | res.writeHead(ret.statusCode+'', {'Content-Type': 'application/javascript'}); 119 | callback = dispatch.query['callback'] || 'callback'; 120 | res.end(callback + '(' + JSON.stringify(ret) + ')') 121 | if (ret.error) { return; } 122 | getMacAddress(req.headers['x-real-ip'] || req.socket.remoteAddress, function(mac) { 123 | if (mac) { updateStreamie(mac, req, ret); } 124 | }); 125 | }); 126 | return; 127 | } 128 | res.writeHead(404, {'Content-Type': 'text/plain'}); 129 | res.end("Weg hier\n"); 130 | }).listen(listen.port, listen.host); 131 | 132 | var iptables = function(para, fn) { 133 | if (debug) { 134 | console.log('iptables:'+para.join(' ')); 135 | fn(0); 136 | return; 137 | } 138 | if (production) { 139 | var iptables = require('child_process').spawn('sudo', ['/sbin/iptables'].concat(para)) 140 | iptables.on('exit', function(code) { 141 | ~~code && console.log('iptables:'+para.join(' ')+"=>"+code); 142 | fn(code); 143 | }); 144 | } else { 145 | console.log('iptables:'+para.join(' ')+"=>"+code); 146 | fn(code); 147 | } 148 | }; 149 | 150 | var writeIPTables = function(para, fn, cmds, codes) { 151 | cmds = cmds || ['-D', '-I']; 152 | codes = codes || []; 153 | var cmd = cmds.shift(); 154 | if (!cmd) { return true; } 155 | iptables([cmd].concat(para), function(code) { 156 | codes.push(code); 157 | writeIPTables(para, fn, cmds, codes) && fn(codes); 158 | }) 159 | return false; 160 | } 161 | 162 | var updateIPTables = function(doc, cmds, retryCnt) { 163 | retryCnt = retryCnt || 0; 164 | cmds = ['-D', '-I']; 165 | //console.log('DOC:'+util.inspect(doc)); 166 | // if (doc._rev != rev) { return; } 167 | if (doc.completed && doc.completed.pid == process.pid) { return; } 168 | var called = 0; 169 | for(var i = doc.clients.length-1; i >= 0; --i) { 170 | var client = doc.clients[i]; 171 | // $IPTABLES -t mangle -I FREE_MACS -i $CONF_IF -p all -m mac 172 | // --mac-source c8:bc:c8:4f:d4:66 -s 10.205.0.100 -j MARK --set-mark 0x1205 173 | var iptable = []; 174 | iptable.push('FREE_MACS'); 175 | iptable.push('-t', 'mangle'); 176 | iptable.push('-p', 'all'); 177 | iptable.push('-m', 'mac'); 178 | iptable.push('--mac-source', client.hwaddr); 179 | iptable.push('-s', client.ipv4); 180 | iptable.push('-j', 'MARK'); 181 | iptable.push('--set-mark', '0x1205'); 182 | writeIPTables(iptable, function(codes) { 183 | client.iptabled = { date: new Date(), exitcodes: codes, rev: doc._rev }; 184 | if (called++ == doc.clients.length) { 185 | doc.completed = { date: new Date(), pid: process.pid, rev: doc._rev }; 186 | streamie.save(doc, function(err, doc) { 187 | if (err) { 188 | if (retryCnt < 5) { 189 | console.error('updateIPTables:couchdb:save:failure:'+err+":"+retryCnt); 190 | setTimeout(function() { updateIPTables(id, rev, retryCnt + 1); }, 500); 191 | } else { 192 | console.error('updateIPTables:couchdb:save:failure:'+err+":MAX-RETRIED"); 193 | } 194 | } 195 | }) 196 | } 197 | }, cmds); 198 | } 199 | } 200 | iptables(['-t', 'mangle', '-F', 'FREE_MACS'], function(code) { 201 | iptables(['-t', 'mangle', '-A', 'FREE_MACS', '-j', 'RETURN'], function(code) { 202 | var docrevs = {}; 203 | streamie.changes(0, function(err, changes) { 204 | if (err) { 205 | console.error('ERROR:couchdb:changes:'+err); 206 | return; 207 | } 208 | if (changes.deleted) { 209 | return; 210 | streamie.request('GET', '/streamie/'+changes.id+'?rev='+changes.changes[0].rev, function(err, doc) { 211 | if (err) { 212 | console.log('ERROR:couchdb:get:'+err); 213 | return; 214 | } 215 | if (doc.blocked) { 216 | console.log('BLOCKED:'+doc._id); 217 | return; 218 | } 219 | console.log('DELETE:'+util.inspect(doc)); 220 | updateIPTables(doc, ['-D']); 221 | }) 222 | } 223 | for(var i in changes.changes) { 224 | if (docrevs[changes.id] != changes.changes[i].rev) { 225 | console.log('CHANGES:'+i+":"+util.inspect(changes.id)); 226 | streamie.get(changes.id, function(err, doc) { 227 | if (err) { 228 | console.log('ERROR:couchdb:get:'+changes.id+":"+err); 229 | return; 230 | } 231 | if (doc.blocked) { 232 | console.log('BLOCKED:'+doc.id); 233 | return; 234 | } 235 | docrevs[doc._id] = doc._rev; 236 | //console.log('CHANGES:'+util.inspect(doc)); 237 | updateIPTables(doc); 238 | }) 239 | } 240 | } 241 | }) 242 | }) 243 | }) 244 | }) 245 | 246 | console.log('Server running at http://'+listen.host+":"+listen.port+'/'); 247 | -------------------------------------------------------------------------------- /flow/couch-client.js: -------------------------------------------------------------------------------- 1 | /*global Buffer */ 2 | 3 | var http = require('http'), 4 | https = require('https'), 5 | Url = require('url'), 6 | EventEmitter = require('events').EventEmitter, 7 | querystring = require('querystring'); 8 | 9 | 10 | // Handles changes made in node v0.3.0 11 | var NOT_FOUND_ERR_NO = process.ENOENT ? process.ENOENT : require('constants').ENOENT; 12 | var MAX_DOCS = 1000; // The maximum number of docs to send in a single batch 13 | 14 | function noOp(err) { if (err) { throw err; } } 15 | 16 | var CONNECTION_DEFAULTS = { 17 | host: '127.0.0.1:5984', 18 | port: 5984, 19 | hostname: '127.0.0.1', 20 | pathname: "/" 21 | }; 22 | 23 | function CouchClient(url) { 24 | var uri = Url.parse(url); 25 | uri.secure = uri.protocol == 'https:'; 26 | uri.protocolHandler = uri.secure ? https : http; 27 | uri.__proto__ = CONNECTION_DEFAULTS; 28 | var revCache = {}; 29 | 30 | // A simple wrapper around node's http(s) request. 31 | function request(method, path, body, callback) { 32 | var stream; 33 | // Body is optional 34 | if (typeof body === 'function' && typeof callback === 'undefined') { 35 | callback = body; 36 | body = undefined; 37 | } 38 | // Return a stream if no callback is specified 39 | if (!callback) { 40 | stream = new EventEmitter(); 41 | stream.setEncoding = function () { 42 | throw new Error("This stream is always utf8"); 43 | }; 44 | } 45 | 46 | function errorHandler(err) { 47 | if (callback) { callback(err); } 48 | if (stream) { stream.emit('error', err); } 49 | } 50 | 51 | var headers = { 52 | "Host": uri.hostname 53 | }; 54 | // add the authorization header if provided and using https 55 | if (uri.auth) { 56 | headers["Authorization"] = "Basic " + new Buffer(uri.auth, "ascii").toString("base64"); 57 | } 58 | 59 | if (body) { 60 | body = JSON.stringify(body); 61 | headers["Content-Length"] = Buffer.byteLength(body); 62 | headers["Content-Type"] = "application/json"; 63 | } 64 | 65 | var options = { 66 | host: uri.hostname, 67 | method: method, 68 | path: path, 69 | port: uri.port, 70 | headers: headers 71 | }; 72 | var request = uri.protocolHandler.request(options, function (response) { 73 | response.setEncoding('utf8'); 74 | var body = ""; 75 | response.on('data', function (chunk) { 76 | if (callback) { body += chunk; } 77 | if (stream) { stream.emit('data', chunk); } 78 | }); 79 | response.on('end', function () { 80 | if (callback) { 81 | try { 82 | var parsedBody = JSON.parse(body); 83 | callback(null, parsedBody); 84 | } catch(err) { 85 | callback(err); 86 | } 87 | } 88 | if (stream) { stream.emit('end'); } 89 | }); 90 | response.on('error', errorHandler); 91 | }); 92 | request.on('error', errorHandler); 93 | 94 | if (body) { request.write(body, 'utf8'); } 95 | request.end(); 96 | 97 | return stream; 98 | } 99 | 100 | // Requests UUIDs from the couch server in tick batches 101 | var uuidQueue = []; 102 | function getUUID(callback) { 103 | uuidQueue.push(callback); 104 | if (uuidQueue.length > 1) { return; } 105 | function consumeQueue() { 106 | var pending = uuidQueue.splice(0, MAX_DOCS); 107 | if (uuidQueue.length) { process.nextTick(consumeQueue); } 108 | // console.log("Bulk getting UUIDs %s", pending.length); 109 | request("GET", "/_uuids?count=" + pending.length, function (err, result) { 110 | if (err) { 111 | pending.forEach(function (callback) { 112 | callback(err); 113 | }); 114 | return; 115 | } 116 | if (result.uuids.length !== pending.length) { 117 | throw new Error("Wrong number of UUIDs generated " + result.uuids.length + " != " + pending.length); 118 | } 119 | result.uuids.forEach(function (uuid, i) { 120 | pending[i](null, uuid); 121 | }); 122 | }); 123 | } 124 | process.nextTick(consumeQueue); 125 | } 126 | 127 | // Saves documents in batches 128 | var saveValues = []; 129 | var saveQueue = []; 130 | function realSave(doc, callback) { 131 | // Put key and rev on the value without changing the original 132 | saveValues.push(doc); 133 | saveQueue.push(callback); 134 | if (saveQueue.length > 1) { return; } 135 | function consumeQueue() { 136 | var pending = saveQueue.splice(0, MAX_DOCS); 137 | var body = saveValues.splice(0, MAX_DOCS); 138 | if (saveQueue.length) { process.nextTick(consumeQueue); } 139 | // console.log("Bulk saving %s", body.length); 140 | request("POST", uri.pathname + "/_bulk_docs", {docs: body}, function (err, results) { 141 | if (results.error) { 142 | err = new Error("CouchDB Error: " + JSON.stringify(results)); 143 | if (results.error === 'not_found') { err.errno = NOT_FOUND_ERR_NO; } 144 | } 145 | if (err) { 146 | pending.forEach(function (callback) { 147 | callback(err); 148 | }); 149 | return; 150 | } 151 | results.forEach(function (result, i) { 152 | var doc = body[i]; 153 | doc._id = result.id; 154 | doc._rev = result.rev; 155 | revCache[result.id] = result.rev; 156 | pending[i](null, doc); 157 | }); 158 | }); 159 | } 160 | process.nextTick(consumeQueue); 161 | } 162 | 163 | var getQueue = []; 164 | var getKeys = []; 165 | function realGet(key, includeDoc, callback) { 166 | getKeys.push(key); 167 | getQueue.push(callback); 168 | if (getQueue.length > 1) { return; } 169 | function consumeQueue() { 170 | var pending = getQueue.splice(0, MAX_DOCS); 171 | var keys = getKeys.splice(0, MAX_DOCS); 172 | if (getQueue.length) { process.nextTick(consumeQueue); } 173 | var path = uri.pathname + "/_all_docs"; 174 | if (includeDoc) { path += "?include_docs=true"; } 175 | // console.log("Bulk Getting %s documents", keys.length); 176 | request("POST", path, {keys: keys}, function (err, results) { 177 | if (!results.rows) { 178 | err = new Error("CouchDB Error: " + JSON.stringify(results)); 179 | } 180 | if (err) { 181 | pending.forEach(function (callback) { 182 | callback(err); 183 | }); 184 | return; 185 | } 186 | results.rows.forEach(function (result, i) { 187 | var err; 188 | if (includeDoc) { 189 | if (result.error) { 190 | err = new Error("CouchDB Error: " + JSON.stringify(result)); 191 | if (result.error === 'not_found') { err.errno = NOT_FOUND_ERR_NO; } 192 | pending[i](err); 193 | return; 194 | } 195 | if (!result.doc) { 196 | err = new Error("Document not found for " + JSON.stringify(result.key)); 197 | err.errno = NOT_FOUND_ERR_NO; 198 | pending[i](err); 199 | return; 200 | } 201 | pending[i](null, result.doc); 202 | return; 203 | } 204 | pending[i](null, result.value); 205 | }); 206 | }); 207 | } 208 | process.nextTick(consumeQueue); 209 | } 210 | 211 | 212 | function save(doc, callback) { 213 | if (!callback) { callback = noOp; } 214 | if (doc._id) { 215 | if (!doc._rev) { 216 | if (!revCache.hasOwnProperty(doc._id)) { 217 | realGet(doc._id, false, function (err, result) { 218 | if (err) { return callback(err); } 219 | if (result) { 220 | revCache[doc._id] = result.rev; 221 | doc._rev = result.rev; 222 | } 223 | realSave(doc, callback); 224 | }); 225 | return; 226 | } 227 | doc._rev = revCache[doc._id]; 228 | } 229 | } 230 | realSave(doc, callback); 231 | } 232 | 233 | function get(key, callback) { 234 | realGet(key, true, callback); 235 | } 236 | 237 | function remove(doc, callback) { 238 | if (typeof doc === 'string') { 239 | doc = {_id: doc}; 240 | } 241 | doc._deleted = true; 242 | save(doc, callback); 243 | } 244 | 245 | function changes(since, callback) { 246 | var stream = request("GET", uri.pathname + "/_changes?feed=continuous&heartbeat=1000&since=" + since); 247 | var data = ""; 248 | function checkData() { 249 | var p = data.indexOf("\n"); 250 | if (p >= 0) { 251 | var line = data.substr(0, p).trim(); 252 | data = data.substr(p + 1); 253 | if (line.length) { 254 | callback(null, JSON.parse(line)); 255 | } 256 | checkData(); 257 | } 258 | } 259 | stream.on('error', callback); 260 | stream.on('data', function (chunk) { 261 | data += chunk; 262 | checkData(); 263 | }); 264 | stream.on('end', function () { 265 | throw new Error("Changes feed got broken!"); 266 | }); 267 | } 268 | 269 | function view(viewName, obj, callback) { 270 | var method = "GET"; 271 | var body = null; 272 | if (typeof obj === 'function') { 273 | callback = obj; 274 | obj = null; 275 | } 276 | 277 | if ( viewName.substr(0,1) != '/' ) { 278 | // assume viewname is designdocname/viewname 279 | var parts = viewName.split("/",2); 280 | if ( parts.length == 2 ) { 281 | viewName = uri.pathname+"/_design/"+parts[0]+"/_view/"+parts[1]; 282 | } 283 | } 284 | 285 | if (obj && typeof obj === 'object') { 286 | Object.keys(obj).forEach(function(key){ 287 | if ( key === 'keys' ) { 288 | body = { keys: obj[key] }; // body is json stringified in request fn 289 | method='POST'; 290 | } else { 291 | obj[key] = JSON.stringify(obj[key]); 292 | } 293 | }); 294 | var getParams = querystring.stringify(obj); 295 | if (getParams){ 296 | viewName = viewName + '?' + getParams; 297 | } 298 | } 299 | request(method, viewName, body, callback); 300 | } 301 | 302 | // Expose the public API 303 | return { 304 | get: get, 305 | save: save, 306 | remove: remove, 307 | changes: changes, 308 | request: request, 309 | uri: uri, 310 | view: view 311 | }; 312 | } 313 | 314 | module.exports = CouchClient; 315 | -------------------------------------------------------------------------------- /flow/flow2couchdb.config: -------------------------------------------------------------------------------- 1 | {"startDate":"2011-04-28T10:44:00.000Z"} -------------------------------------------------------------------------------- /flow/flow2couchdb.js: -------------------------------------------------------------------------------- 1 | 2 | var fs = require('fs'); 3 | var util = require('util'); 4 | var http = require('http'); 5 | var cp = require('child_process'); 6 | var CouchClient = require('./couch-client'); 7 | require('datejs'); 8 | 9 | var Server = { host: '127.0.0.1', port: 5984, path: '/traffic', headers: { 'content-type': 'application/json' } }; 10 | 11 | var gid = 1; 12 | var running = 0; 13 | var storeCouch = function(cmds, complete, id, cmd, i, found, count) { 14 | console.log('ENTER-storeCouch'+util.inspect(cmds)); 15 | found = false; 16 | for (i = running; i < 12; ++i) { 17 | running = 12; 18 | found = true; 19 | storeCouch(cmds, complete, id, cmd); 20 | } 21 | if (found) { return } 22 | 23 | cmd = cmds.shift(); 24 | if (!cmd) { return; } 25 | 26 | console.log('START-storeCouch'+util.inspect(cmd)); 27 | id = ++gid; 28 | count = 0; 29 | var run = function() { 30 | cp.exec(cmd.cmd.join(' '), { maxBuffer: 10*1024*1024 }, function(error, stdout, stderr, req, lines, out) { 31 | if (error) { 32 | console.log('ERROR:'+id+":"+error+":"+cmd.cmd.join(' ')); 33 | if (count++ < 3) { return run(); } 34 | running--; 35 | storeCouch(cmds, complete); 36 | return; 37 | } 38 | lines = stdout.split("\n"); 39 | delete stdout; 40 | delete stderr; 41 | lines.shift(); 42 | out = []; 43 | var out = { 44 | _id: cmd.key, 45 | tuples: [], 46 | created_at: Date() 47 | }; 48 | for(var line in lines) { 49 | line = lines[line]; 50 | var cols = line.split(/[ ]+/); 51 | //srcIP dstIP prot srcPort dstPort octets packets 52 | if (cols.length >= 7) { 53 | out.tuples.push({ 54 | srcIP: ip2twitter[cols[0]] || cols[0], 55 | dstIP: ip2twitter[cols[1]] || cols[1], 56 | prot: cols[2], 57 | srcPort: cols[3], 58 | dstPort: cols[4], 59 | octets: cols[5], 60 | packets: cols[6] 61 | }); 62 | } 63 | } 64 | var completed = function() { 65 | delete lines; 66 | delete cols; 67 | delete data; 68 | delete req; 69 | running--; 70 | storeCouch(cmds, complete) 71 | if (!cmds.length) { console.log('storeCouch-COMPLETE'); complete(); } 72 | } 73 | if (!out.tuples.length) { 74 | completed(); 75 | } else { 76 | traffic.save(out, function(err, doc) { 77 | if (err) { 78 | 79 | } 80 | completed(); 81 | }) 82 | } 83 | }) 84 | } 85 | run(); 86 | } 87 | 88 | var ip2twitter = {}; 89 | var lastStreamieChange = null; 90 | var streamie = new CouchClient('http://localhost:5984/streamie'); 91 | streamie.changes(0, function(err, changes) { 92 | if (err) { 93 | console.error('couchdb:changes:failure:'+err); 94 | return; 95 | } 96 | if (changes.deleted) { return; } 97 | streamie.get(changes.id, function(err, doc) { 98 | if (err) { 99 | console.error('couchdb:changes:failure:'+err); 100 | return; 101 | } 102 | for (var i in doc.clients) { 103 | var client = doc.clients[i]; 104 | console.log('ADD IP2Twitter:'+client.ipv4+"=>"+doc.twitter.screen_name); 105 | ip2twitter[client.ipv4] = doc.twitter.screen_name; 106 | } 107 | lastStreamieChange = new Date(); 108 | }) 109 | }) 110 | 111 | crawler = function(base, completed, data) { 112 | data = data || { dirs: [], files: [], calls: 0 }; 113 | data.calls++; 114 | console.log('IN-CRAWLER:'+base+":"+data.calls); 115 | fs.readdir(base, function(err, in_files, dirs, cnt, i) { 116 | if (err) { 117 | console.log('fs.readdir:err:'+err); 118 | setTimeout(function() { 119 | crawler(data, completed, data); 120 | }, 1000) 121 | return; 122 | } 123 | dirs = []; 124 | cnt = 0; 125 | for(i in in_files) { 126 | (function(fname, dir) { 127 | fs.stat(base+'/'+fname, function(err, stat) { 128 | if (err) { 129 | console.log('fs.readdir:err:'+err); 130 | return; 131 | } 132 | if (stat.isFile()) { data.files.push(base+'/'+fname); } 133 | else if (stat.isDirectory()) { dirs.push(base+'/'+fname); } 134 | if (++cnt == in_files.length) { 135 | data.dirs.push.apply(data.dirs, dirs); 136 | for(dir in dirs) { 137 | crawler(dirs[dir], completed, data); 138 | } 139 | data.calls--; 140 | console.log('OUT-CRAWLER-A:'+base+":"+data.calls); 141 | if (!data.calls) { 142 | completed(data); 143 | } 144 | } 145 | }) 146 | })(in_files[i]); 147 | } 148 | if (!in_files.length) { 149 | data.calls--; 150 | console.log('OUT-CRAWLER-B:'+base+":"+data.calls); 151 | if (!data.calls) { 152 | completed(data); 153 | } 154 | } 155 | }) 156 | } 157 | 158 | var traffic = new CouchClient('http://localhost:5984/traffic'); 159 | var observedDirs = {}; 160 | setTimeout(function CheckChanges() { 161 | var now = new Date(); 162 | if (lastStreamieChange && (now.getTime()-lastStreamieChange.getTime()) > 1000) { 163 | console.log('START-CRAWLER'+util.inspect(ip2twitter)); 164 | traffic.request('PUT', '/traffic', function(err, result) { 165 | console.log('AAAA'); 166 | if (err) { 167 | console.error('couchdb:traffic:failure:'+err) 168 | return 169 | } 170 | var running = false; 171 | var crawlerQueue = []; 172 | crawler('/flows', function AddFlows(data) { 173 | for(var i in data.dirs) { 174 | (function CheckDir(dir) { 175 | if (observedDirs[dir]) { return; }; 176 | observedDirs[dir] = true; 177 | console.log('OBSERVER:'+dir); 178 | 179 | fs.watchFile(dir, function() { 180 | console.log('dir='+dir); 181 | if (running) { 182 | crawlerQueue.push(dir); 183 | return; 184 | } 185 | running = true; 186 | crawler(dir, AddFlows); 187 | }) 188 | 189 | })(data.dirs[i]); 190 | } 191 | //var re = new RegExp("^.*\/ft-v05.\(\d+\-\d+\-\d+.\d+\)\+0200"); 192 | var re = new RegExp("^.*\/ft-v05\.\(\\d+-\\d+-\\d+\.\\d+\)\\+\\d+$"); 193 | var cmds = []; 194 | for(var i in data.files) { 195 | var file = data.files[i]; 196 | var res = re.exec(file); 197 | if (res) { 198 | console.log('ADD-Flow:'+file); 199 | cmds.push({key: res[1], cmd: ['flow-print', '<', file, '&&', 'mv', file, file+'-done']}) 200 | //cmds.push({key: res[1], cmd: ['flow-print', '<', file]}); 201 | } 202 | } 203 | if (!cmds.length) { 204 | running = false; 205 | var dir = crawlerQueue.shift(); 206 | if (dir) { crawler(dir, AddFlows); } 207 | } 208 | storeCouch(cmds, function() { 209 | running = false; 210 | var dir = crawlerQueue.shift(); 211 | if (dir) { crawler(dir, AddFlows); } 212 | }); 213 | }); 214 | }) 215 | } else { 216 | setTimeout(CheckChanges, 1000); 217 | } 218 | }, 1000); 219 | 220 | -------------------------------------------------------------------------------- /queue.js: -------------------------------------------------------------------------------- 1 | var Queue = function () { 2 | this.emptyBuffer = new Buffer(0); 3 | this.queue = []; 4 | } 5 | Queue.prototype.add = function(obj, wait_len, found_fn, test) { 6 | if (wait_len == 0) { 7 | found_fn(this.emptyBuffer, ""); 8 | return; 9 | } 10 | if (obj) { 11 | if (obj.length == 0) { return; } 12 | this.queue.push({ data: obj, ofs: 0, test: test}); 13 | } 14 | var segs = 0; 15 | var need = wait_len; 16 | for (var i = 0, ql = this.queue.length; i < ql; ++i) { 17 | var qe = this.queue[i]; 18 | var diff = qe.data.length - qe.ofs; 19 | ++segs; 20 | // console.log('PROC qlen='+qe.data.length+" qofs="+qe.ofs+" diff="+diff+" need="+need+":"+util.inspect(qe)) 21 | if (diff >= need) { 22 | var buffer = new Buffer(wait_len); 23 | var buffer_ofs = 0; 24 | for (var i = 0; i < segs-1; ++i) { 25 | /* all but last */ 26 | qe = this.queue.shift(); 27 | qe.data.copy(buffer, buffer_ofs, qe.ofs); 28 | buffer_ofs += qe.data.length - qe.ofs; 29 | //console.log('TOTAL'); 30 | delete qe; 31 | } 32 | qe = this.queue[0]; 33 | qe.data.copy(buffer, buffer_ofs, qe.ofs, qe.ofs+need); 34 | //console.log('PARTIAL', qe.data.length, qe.ofs, need, util.inspect(buffer), util.inspect(qe)); 35 | if (qe.ofs+need == qe.data.length) { 36 | //console.log('DELETE') 37 | delete this.queue.shift(); 38 | } 39 | else { this.queue[0].ofs += need; } 40 | found_fn(buffer, qe['test'] && qe.test); 41 | break; 42 | } 43 | need -= diff; 44 | } 45 | } 46 | module.exports = Queue; 47 | -------------------------------------------------------------------------------- /sashimi.js: -------------------------------------------------------------------------------- 1 | var sys = require('sys'); 2 | var utils = require('util'); 3 | var fs = require('fs'); 4 | var net = require('net'); 5 | var Queue = require('./queue'); 6 | 7 | var split_host_port = function(str) { 8 | var ret = str.split(':'); 9 | return { port: parseInt(ret[1], 10), host: ret[0] }; 10 | } 11 | 12 | var split_srv = function(str) { 13 | var val = { 14 | my: { port: 0, host: '0.0.0.0' }, 15 | peer: { port: 0, host: '0.0.0.0' } 16 | }; 17 | var srv = str.split('-'); 18 | if (srv.length == 2) { 19 | val.my = split_host_port(srv[0]); 20 | val.peer = split_host_port(srv[1]); 21 | } else { 22 | val.peer = split_host_port(str); 23 | } 24 | return val; 25 | } 26 | 27 | /* MAIN */ 28 | console.log(JSON.stringify(process.argv)); 29 | 30 | arg = 2; // skip node and sashimi.js 31 | var tun_dev = split_host_port(process.argv[arg]).host; // not nice but working 32 | var tun_fd = split_host_port(process.argv[arg++]).port; 33 | 34 | var mode = process.argv[arg++]; 35 | var name = process.argv[arg++]; 36 | 37 | var key = { 38 | my: process.argv[arg++], 39 | peer: process.argv[arg++] 40 | }; 41 | 42 | var servers = []; 43 | for(var i = arg; i < process.argv.length; ++i) { 44 | var arg = process.argv[i]; 45 | var no_output = arg.slice(0,1); 46 | if (no_output == '!') { arg = arg.slice(1); } 47 | var srv = split_srv(arg); 48 | if (no_output == '!') { srv.no_output = true; } 49 | else { srv.no_output = false; } 50 | servers.push(srv); 51 | } 52 | console.log('node_version:'+process.version+' tun_dev='+tun_dev+' tun_fd='+tun_fd+" servers="+JSON.stringify(servers)); 53 | 54 | var output_streams = []; 55 | var status = { in: 0, 56 | out: 0, 57 | ping: { id: 0, pings: 0, pongs: 0, errors: 0 }, 58 | connections: { 59 | open: 0, 60 | client: { 61 | connects: 0 62 | }, 63 | server: { 64 | connects: 0, 65 | close: 0 66 | } 67 | }, 68 | tun: { 69 | input: { 70 | err: 0 71 | } 72 | }, 73 | failure: { write: 0 } }; 74 | 75 | var packet_input = function() { 76 | var packet = new Buffer(1600); 77 | /* tun has a 4byte header i currently not know what this means */ 78 | fs.read(tun_fd, packet, 4, packet.length-4, null, function(err, len) { 79 | if (err) { 80 | status.tun.input.err++; 81 | console.log('packet_input err:'+err); 82 | packet_input(); 83 | return; 84 | } 85 | var plen = (((len) + 10000)+'').slice(1); // leading zero's 86 | packet.write(plen, 0, 'ascii'); 87 | if (output_streams.length > 0) { 88 | ++status.in; 89 | try { 90 | output_streams[status.in%output_streams.length].write(packet.slice(0, len+4)); 91 | } catch(e) { 92 | //console.log("packet_input:DESTROY") 93 | output_streams[status.in%output_streams.length].destroy(); 94 | } 95 | } 96 | packet_input(); 97 | }) 98 | } 99 | 100 | var streamer = function(stream, fns, opts) { 101 | //stream.setEncoding() 102 | var connected = false; 103 | var wait_key_peer = true; 104 | var queue = new Queue(); 105 | var header = { 106 | active: true, 107 | len: 4, 108 | completed: function(data, test) { 109 | header.active = false; 110 | packet.active = true; 111 | packet.len = ~~data; 112 | queue.add(null, packet.len, packet.completed); 113 | return null 114 | } 115 | } 116 | var ping_packet = new Buffer(20); 117 | var plen = (((16) + 10000)+'').slice(1); // leading zero's 118 | ping_packet.write(plen, 0, 'ascii'); 119 | 120 | var ping_pong = { 121 | PING: function(id) { 122 | status.ping.pongs++ 123 | ping_packet.write("PONG"+id, 4, 'ascii') 124 | if (!stream.recvPing) { stream.recvPing = { id: null, recv: null } } 125 | stream.recvPing.id = id 126 | stream.recvPing.recv = (new Date()).getTime() 127 | //console.log("SEND=>"+ping_packet.toString('ascii')) 128 | stream.write(ping_packet) 129 | }, 130 | PONG: function(id) { 131 | if (stream.sendPing.id == id) { 132 | status.ping.pongs++ 133 | stream.sendPing.id = null 134 | stream.sendPing.send = null 135 | } else { 136 | status.ping.errors++ 137 | console.log("PING("+stream.sendPing.id+")<>PONG("+id+")") 138 | } 139 | } 140 | }; 141 | var packet = { 142 | active: false, 143 | len: 0, 144 | completed: function(data, test) { 145 | ++status.out; 146 | if (data.length == 16) { 147 | var str = data.toString("ascii"); 148 | //console.log("RECV8:"+str); 149 | ping_pong[str.slice(0,4)](str.slice(4,16)); 150 | } else { 151 | fs.write(tun_fd, data, 0, data.length); 152 | } 153 | header.active = true; 154 | packet.active = false; 155 | packet.len = 0; 156 | queue.add(null, header.len, header.completed); 157 | } 158 | } 159 | stream.on('connect', function() { 160 | stream.setNoDelay(true); 161 | console.log('client-connect:'+stream.remoteAddress+":"+stream.remotePort+":"+key.my+":"+opts['no_output']); 162 | stream.write(key.my, 'utf-8'); 163 | connected = true; 164 | }) 165 | var clear_output_streams = function() { 166 | connected && console.log('client-close:'+stream.remoteAddress+":"+stream.remotePort); 167 | for(var i = output_streams.length - 1, l = i; i >= 0; --i) { 168 | if (output_streams[i] === stream) { 169 | output_streams[i] = output_streams[l] 170 | output_streams.pop() 171 | //console.log("clear_output_streams:DELETE:"+i+":"+l+":"+output_streams.length); 172 | break; 173 | } 174 | } 175 | status.connections.server.close++; 176 | console.log("clear_output_streams:destroy") 177 | connected && stream.destroy(); 178 | connected = false; 179 | fns && fns.closed && fns.closed(stream); 180 | fns && (fns.closed = false); 181 | } 182 | stream.on('close', function() { 183 | console.log('streamer:close'); 184 | clear_output_streams(); 185 | }); 186 | stream.on('end', function() { 187 | console.log('streamer:end'); 188 | clear_output_streams(); 189 | }); 190 | stream.on('error', function() { 191 | console.log('streamer:error'+utils.inspect(arguments)); 192 | clear_output_streams(); 193 | }); 194 | stream.on('data', function(data) { 195 | var obj = data; 196 | if (wait_key_peer) { 197 | queue.add(obj, key.peer.length, function(in_key) { 198 | if (in_key == key.peer) { 199 | console.log('verified key='+in_key); 200 | //stream.setEncoding('binary'); 201 | wait_key_peer = false; 202 | if (!(opts && opts.no_output)) { output_streams.push(stream); } 203 | fns && fns.opened && fns.opened(stream); 204 | } else { 205 | console.log('not verified key='+in_key); 206 | stream.destroy(); 207 | } 208 | }) 209 | return; 210 | } 211 | if (header.active) { 212 | queue.add(obj, header.len, header.completed); 213 | } else if (packet.active) { 214 | queue.add(obj, packet.len, function(data, test ) { 215 | packet.completed(data, test); 216 | }) 217 | } 218 | }); 219 | } 220 | var ping = function(output_streams, mode) { 221 | return function() { 222 | var packet = new Buffer(20); 223 | var plen = (((16) + 10000)+'').slice(1); // leading zero's 224 | packet.write(plen, 0, 'ascii'); 225 | var pinger = function() { 226 | //console.log("PINGER:"+output_streams.length); 227 | for(var i = output_streams.length-1; i >= 0; --i) { 228 | var stream = output_streams[i] 229 | if (!stream.sendPing) { stream.sendPing = { id: null, send: null } } 230 | if (stream.sendPing.send) { 231 | var now = (new Date()).getTime() 232 | if ((now-stream.sendPing.send) >= 5000 && !stream.destroyed) { 233 | // CLOSE Stream 234 | console.log("PING DESTROY"); 235 | stream.destroyed = true 236 | stream.destroy(); 237 | } 238 | } else { 239 | var id = ((((status.ping.id++)%10000)+10000)+'').slice(1) 240 | var sid = ((((i)%10000)+10000)+'').slice(1) 241 | stream.sendPing.id = mode+sid+id 242 | stream.sendPing.send = (new Date()).getTime() 243 | var pingId = "PING"+stream.sendPing.id 244 | packet.write(pingId, 4, 'ascii'); 245 | //console.log("SEND=>"+packet.toString()) 246 | status.ping.pings++ 247 | stream.write(packet); 248 | } 249 | } 250 | setTimeout(pinger, 1000); 251 | } 252 | pinger(); 253 | } 254 | } 255 | 256 | if (mode == 'server') { 257 | console.log('SERVER-MODE'); 258 | servers.forEach(function(server) { 259 | console.log('LISTEN:'+server.peer.port+":"+server.peer.host); 260 | net.createServer(function(stream) { 261 | streamer(stream, null, server); 262 | }).listen(server.peer.port, server.peer.host); 263 | }) 264 | packet_input(); 265 | setTimeout(ping(output_streams, "SERV"), 1000); 266 | } else if (mode == 'client') { 267 | console.log('CLIENT-MODE'); 268 | var connections = [] 269 | var client_connect = function(server, stream) { 270 | console.log('Connect peer='+server.peer.host+":"+ server.peer.port+" my="+server.my.host+":"+ server.my.port+":"+server['no_output']) 271 | stream = net.createConnection(server.peer.port, server.peer.host, { bind: server.my }); 272 | status.connections.client.connects++; 273 | streamer(stream, { 274 | closed: function() { 275 | setTimeout(function() { 276 | client_connect(server, stream); 277 | }, 1000); 278 | } 279 | }, server) 280 | } 281 | servers.forEach(function(server) { 282 | client_connect(server); 283 | }) 284 | //setTimeout(ping(output_streams), 1000); 285 | packet_input(); 286 | setTimeout(function recvPing() { 287 | //console.log("recvPing:"+ output_streams.length) 288 | for(var i = output_streams.length - 1; i >= 0 ; --i) { 289 | var stream = output_streams[i] 290 | if (!stream.recvPing) { stream.recvPing = { id: null, recv: (new Date()).getTime() } } 291 | var now = (new Date()).getTime() 292 | if ((now - stream.recvPing.recv) > 5000 && !stream.destroyed) { 293 | console.log("LINK-DOWN-DETECTED:") 294 | stream.destroyed = true 295 | stream.destroy() 296 | } 297 | } 298 | setTimeout(recvPing, 1000); 299 | }, 1000); 300 | } 301 | 302 | var http = require('http'); 303 | http.createServer(function (req, res) { 304 | res.writeHead(200, {'Content-Type': 'application/json'}); 305 | status.connections.open = output_streams.length; 306 | res.end(JSON.stringify(status)) 307 | }).listen(1706, "0.0.0.0"); 308 | 309 | setInterval(function() { 310 | status.connections.open = output_streams.length; 311 | sys.print("Status:"+JSON.stringify(status)+"\n"); 312 | }, 10000); 313 | -------------------------------------------------------------------------------- /test_packet_source.js: -------------------------------------------------------------------------------- 1 | var Queue = require('./queue'); 2 | 3 | var test_packet_source = function(fn, runs) { 4 | var data = ""; 5 | var maxlen = 1420; 6 | var base = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 7 | for(var i = 0; i < maxlen; ++i) { data += base[i%base.length]; } 8 | runs = runs || 1; 9 | for(var loops = 0; loops < runs; ++loops) { 10 | console.log('LOOP:'+loops); 11 | for(var outlen = 0; outlen < 1400; ++outlen) { 12 | var segsize = ~~(outlen/3); 13 | if (segsize == 0) { segsize = 1; } 14 | var blen = outlen + 4; 15 | var buffer = new Buffer(outlen + blen); 16 | buffer.write(((10000+outlen)+"").slice(1), "ascii"); // leading zero's 17 | buffer.write(data.slice(0,outlen), "binary", 4); 18 | for(var ofs = 0; ofs < blen; ofs += segsize) { 19 | var rest = (blen - ofs); 20 | if (rest > segsize) { rest = segsize; } 21 | fn(buffer.slice(ofs, ofs + rest), rest, data.slice(0,outlen)); 22 | } 23 | delete buffer; 24 | } 25 | } 26 | console.log('yeah done:'+loops*(outlen*(outlen+1))/2); 27 | } 28 | 29 | var queue = new Queue(); 30 | var header = { 31 | len: 4, 32 | completed: function(data, test) { 33 | packet.len = ~~data; 34 | state = packet; 35 | queue.add(null, packet.len, packet.completed); 36 | } 37 | }; 38 | var packet = { 39 | len: 0, 40 | completed: function(data,test) { 41 | if (data != test) { 42 | throw('BUFFER-FAILURE:'+data+":"+test); 43 | } 44 | state = header; 45 | } 46 | } 47 | var state = header; 48 | test_packet_source(function(buf, rest, cmp) { 49 | queue.add(buf, state.len, state.completed, cmp); 50 | }, 100) 51 | -------------------------------------------------------------------------------- /tun.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | 14 | int tun_alloc(char *dev) 15 | { 16 | struct ifreq ifr; 17 | int fd, err; 18 | 19 | if( (fd = open("/dev/net/tun", O_RDWR)) < 0 ) { 20 | return -1; 21 | } 22 | memset(&ifr, 0, sizeof(ifr)); 23 | /* Flags: IFF_TUN - TUN device (no Ethernet headers) 24 | * IFF_TAP - TAP device 25 | * 26 | * IFF_NO_PI - Do not provide packet information 27 | */ 28 | ifr.ifr_flags = IFF_TUN; 29 | if (*dev) { 30 | strncpy(ifr.ifr_name, dev, IFNAMSIZ); 31 | } 32 | err = ioctl(fd, TUNSETIFF, &ifr); 33 | if (err < 0) { 34 | close(fd); 35 | return err; 36 | } 37 | strcpy(dev, ifr.ifr_name); 38 | return fd; 39 | } 40 | 41 | int main(int argc, char **argv) { 42 | char dev[100]; 43 | int tun_id = 0; 44 | int fd = -1; 45 | for(tun_id = 0; fd < 0 && tun_id < 0xff; ++tun_id) { 46 | snprintf(dev, sizeof(dev), "tun%d", tun_id); 47 | fd = tun_alloc(dev); 48 | } 49 | printf("allocated:%d:%s\n", fd, dev); 50 | char setup[1024]; 51 | snprintf(setup, sizeof(setup), "./setup.%s \"%s\" \"%s\" \"%s\"", argv[3], argv[2], argv[3], dev); 52 | system(setup); 53 | 54 | seteuid(1000); // UGLY BUT USEFUL The node has to by run as root!! 55 | char *cp_argv[argc+3]; 56 | cp_argv[0] = argv[1]; 57 | cp_argv[1] = "sashimi.js"; 58 | char tun_fd[16]; 59 | snprintf(tun_fd, sizeof(tun_fd), "%s:%d", dev, fd); 60 | cp_argv[2] = tun_fd; 61 | int i; 62 | for(i = 2; i <= argc;++i) { 63 | cp_argv[i+1] = argv[i]; 64 | } 65 | execvp(argv[1], cp_argv); 66 | } 67 | --------------------------------------------------------------------------------