├── examples ├── http_basicauth_jsonstore │ ├── users.json │ └── server.js ├── http_noauth_zeromq │ ├── zmq.js │ └── server.js ├── http_noauth_file │ ├── file.js │ └── server.js ├── http_noauth │ └── server.js ├── websocket_noauth │ └── server.js ├── http_noauth_mqtt │ ├── server.js │ └── mqtt.js └── http_noauth_customrouter │ └── server.js ├── .gitignore ├── lib ├── router │ ├── allow_all.js │ └── base.js ├── authentication │ ├── base.js │ ├── allow_all.js │ └── basic.js ├── store │ ├── base.js │ └── json_file.js ├── source │ ├── base.js │ ├── fake.js │ ├── file.js │ ├── mqtt.js │ └── zeromq.js ├── user │ └── base.js ├── transport │ ├── base.js │ ├── websocket.js │ └── http.js └── client │ └── base.js ├── package.json ├── README.md └── index.js /examples/http_basicauth_jsonstore/users.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": "1", 4 | "username": "test1", 5 | "password": "test1" 6 | } 7 | ] -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | npm-debug.log 15 | node_modules/ 16 | -------------------------------------------------------------------------------- /examples/http_noauth_zeromq/zmq.js: -------------------------------------------------------------------------------- 1 | //Test script that will send a ZMQ message a few times a second 2 | var zmq = require("zeromq"); 3 | var socket = zmq.createSocket("pub"); 4 | var endpoint = "tcp://127.0.0.1:1234"; 5 | socket.bind(endpoint, function(){ 6 | console.log("Bound to " + endpoint); 7 | setInterval(function(){ 8 | socket.send(["firehose", '{"some_data":' + (new Date().getTime()) + '}']); 9 | }, 500); 10 | }); -------------------------------------------------------------------------------- /examples/http_noauth_file/file.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | 3 | //Open file for writing (publicly and in tmp) 4 | var filePath = "/tmp/firehose.log"; 5 | fs.open(filePath, 'a', 777, function(error, fd) { 6 | if (error) { 7 | console.log("Error opening file " + filePath + " error: " + error.message); 8 | } else { 9 | setInterval(function(){ 10 | fs.writeSync(fd, "This is a log entry " + new Date().toString() + "\n"); 11 | }, 500); 12 | } 13 | }); -------------------------------------------------------------------------------- /lib/router/allow_all.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Router = require("./base").Router; 3 | 4 | /** 5 | * Allow all router 6 | * 7 | */ 8 | function AllRouter(config) { 9 | Router.call(this, config); 10 | } 11 | util.inherits(AllRouter, Router); 12 | 13 | /** 14 | * Check a transport 15 | */ 16 | AllRouter.prototype.check = function(transport) { 17 | return true; 18 | }; 19 | 20 | /** 21 | * Get topics 22 | */ 23 | AllRouter.prototype.getTopics = function(transport) { 24 | return ['firehose']; 25 | }; 26 | 27 | module.exports = { 28 | Router: AllRouter 29 | } 30 | -------------------------------------------------------------------------------- /lib/authentication/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | 3 | /** 4 | * Base auth 5 | * 6 | */ 7 | function Authentication(config) { 8 | 9 | //Config 10 | this.config = config; 11 | } 12 | 13 | /** 14 | * Get the config 15 | */ 16 | Authentication.prototype.getConfig = function() { 17 | return this.config; 18 | }; 19 | 20 | /** 21 | * Check a request object to see if it has passed authentication 22 | */ 23 | Authentication.prototype.getCredentials = function(transport) { 24 | return {}; 25 | }; 26 | 27 | module.exports = { 28 | Authentication: Authentication 29 | }; 30 | 31 | -------------------------------------------------------------------------------- /lib/authentication/allow_all.js: -------------------------------------------------------------------------------- 1 | var util = require("util"); 2 | var Authentication = require("./base").Authentication; 3 | 4 | /** 5 | * Allow all auth 6 | * 7 | */ 8 | function AllowAllAuthentication(config) { 9 | Authentication.call(this, config); 10 | } 11 | util.inherits(AllowAllAuthentication, Authentication); 12 | 13 | /** 14 | * Check a request object to see if it has passed authentication 15 | */ 16 | AllowAllAuthentication.prototype.getCredentials = function(transport) { 17 | return {}; 18 | }; 19 | 20 | module.exports = { 21 | Authentication: AllowAllAuthentication 22 | }; 23 | 24 | -------------------------------------------------------------------------------- /lib/router/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | 3 | /** 4 | * Base router 5 | * 6 | */ 7 | function Router(config) { 8 | //Config 9 | this.config = config; 10 | } 11 | 12 | /** 13 | * Get config 14 | */ 15 | Router.prototype.getConfig = function() { 16 | return this.config; 17 | }; 18 | 19 | /** 20 | * Check a transport 21 | */ 22 | Router.prototype.check = function(transport) { 23 | return true; 24 | }; 25 | 26 | /** 27 | * Get topics 28 | */ 29 | Router.prototype.getTopics = function(transport) { 30 | return []; 31 | }; 32 | 33 | module.exports = { 34 | Router: Router 35 | } 36 | -------------------------------------------------------------------------------- /lib/store/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var User = require("../user/base").User; 3 | 4 | /** 5 | * Base store 6 | * 7 | */ 8 | function Store(config) { 9 | //Config 10 | this.config = config; 11 | } 12 | 13 | /** 14 | * Get the config 15 | */ 16 | Store.prototype.getConfig = function() { 17 | return this.config; 18 | }; 19 | 20 | /** 21 | * Get a user using credentials 22 | */ 23 | Store.prototype.getUserFromCredentials = function(credentials, callback) { 24 | //Create a user 25 | var user = new User(); 26 | user.setId("1"); 27 | user.setCredentials(credentials); 28 | 29 | //Send the user back 30 | callback(undefined, user); 31 | }; 32 | 33 | module.exports = { 34 | Store: Store 35 | }; 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "firehose", 3 | "version": "0.1.0", 4 | "description": "Create an HTTP and WebSocket firefose from anything!", 5 | "author": "Ollie Parsley ", 6 | "contributors": [ 7 | { 8 | "name": "Ollie Parsley", 9 | "email": "ollie@ollieparsley.com" 10 | } 11 | ], 12 | "keywords": [ 13 | "ollieparsley", 14 | "firehose", 15 | "websockets", 16 | "streaming", 17 | "http", 18 | "chunked", 19 | "realtime", 20 | "real-time", 21 | "mqtt", 22 | "zeromq", 23 | "zmq" 24 | ], 25 | "repository": { 26 | "type": "git", 27 | "url": "http://github.com/ollieparsley/node-firehose.git" 28 | }, 29 | "main": "index.js", 30 | "dependencies": { 31 | "mqtt": "^3.0.0", 32 | "tail": "0.4.0", 33 | "ws": "7.2.0", 34 | "zeromq": "5.1.1" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /lib/source/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var events = require('events'); 3 | 4 | /** 5 | * Base source 6 | * 7 | */ 8 | function Source(config) { 9 | events.EventEmitter.call(this); 10 | 11 | //Config 12 | this.config = config; 13 | } 14 | util.inherits(Source, events.EventEmitter); 15 | 16 | /** 17 | * Get the config 18 | */ 19 | Source.prototype.getConfig = function() { 20 | return this.config; 21 | }; 22 | 23 | /** 24 | * Start the source 25 | */ 26 | Source.prototype.start = function() { 27 | //Do nothing 28 | }; 29 | 30 | /** 31 | * Stop the source 32 | */ 33 | Source.prototype.stop = function() { 34 | //Do nothing 35 | }; 36 | 37 | /** 38 | * Add a topic 39 | */ 40 | Source.prototype.addTopic = function(topic) { 41 | //Do nothing 42 | }; 43 | 44 | /** 45 | * Remove a topic 46 | */ 47 | Source.prototype.removeTopic = function(topic) { 48 | //Do nothing 49 | }; 50 | 51 | /** 52 | * Broadcast data on a topic 53 | */ 54 | Source.prototype.broadcast = function(topic, data) { 55 | this.emit("topic_" + topic, data); 56 | }; 57 | 58 | module.exports = { 59 | Source: Source 60 | }; 61 | 62 | -------------------------------------------------------------------------------- /examples/http_noauth/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | 3 | //Firehose server options 4 | var options = { 5 | //Config 6 | config: { 7 | max_buffer_size: 1048576, 8 | max_concurrent_connections: 1024, 9 | transport: { 10 | port: 1337, 11 | host: "0.0.0.0" 12 | } 13 | }, 14 | 15 | //The base source which sends fake data every 2 seconds 16 | source: new firehose.source.Fake(), 17 | 18 | //An HTTP transport 19 | transport: firehose.transport.Http 20 | }; 21 | 22 | //Create server, send the options and wait for a client to successfully connect 23 | var server = firehose.createServer(options, function(client) { 24 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 25 | 26 | //Listen for client disconnects 27 | client.on("close", function() { 28 | console.log("Client (" + client.getId() + ") has disconnected") 29 | }); 30 | 31 | }); 32 | 33 | //Start the server listening 34 | server.listen(function(){ 35 | console.log ("Firehose server is now listening"); 36 | }); 37 | 38 | -------------------------------------------------------------------------------- /examples/websocket_noauth/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | 3 | //Firehose server options 4 | var options = { 5 | //Config 6 | config: { 7 | max_buffer_size: 1048576, 8 | max_concurrent_connections: 1024, 9 | transport: { 10 | port: 1337, 11 | host: "0.0.0.0" 12 | } 13 | }, 14 | 15 | //The base source which sends fake data every 2 seconds 16 | source: new firehose.source.Fake(), 17 | 18 | //An HTTP transport 19 | transport: firehose.transport.Websocket 20 | }; 21 | 22 | //Create server, send the options and wait for a client to successfully connect 23 | var server = firehose.createServer(options, function(client) { 24 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 25 | 26 | //Listen for client disconnects 27 | client.on("close", function() { 28 | console.log("Client (" + client.getId() + ") has disconnected") 29 | }); 30 | 31 | }); 32 | 33 | //Start the server listening 34 | server.listen(function(){ 35 | console.log ("Firehose server is now listening"); 36 | }); 37 | 38 | -------------------------------------------------------------------------------- /lib/user/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | 3 | /** 4 | * Base user 5 | * 6 | */ 7 | function User(config) { 8 | 9 | //Config 10 | this.config = config; 11 | 12 | //ID 13 | this.id = null; 14 | 15 | //Credentials 16 | this.credentials = {}; 17 | 18 | //Data 19 | this.data = {}; 20 | } 21 | 22 | /** 23 | * Get the config 24 | */ 25 | User.prototype.getConfig = function() { 26 | return this.config; 27 | }; 28 | 29 | /** 30 | * Set the ID 31 | */ 32 | User.prototype.setId = function(value) { 33 | this.id = value; 34 | }; 35 | 36 | /** 37 | * Get the id 38 | */ 39 | User.prototype.getId = function() { 40 | return this.id; 41 | }; 42 | 43 | /** 44 | * Set the credentials 45 | */ 46 | User.prototype.setCredentials = function(value) { 47 | this.credentials = value; 48 | }; 49 | 50 | /** 51 | * Get the credentials 52 | */ 53 | User.prototype.getCredentuals = function() { 54 | return this.credentials; 55 | }; 56 | 57 | /** 58 | * Set a data item 59 | */ 60 | User.prototype.set = function(key, value) { 61 | this.data[key] = value; 62 | }; 63 | 64 | /** 65 | * Get the data item 66 | */ 67 | User.prototype.get = function(key) { 68 | return this.data[key]; 69 | }; 70 | 71 | module.exports = { 72 | User: User 73 | }; 74 | 75 | -------------------------------------------------------------------------------- /examples/http_noauth_file/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | 3 | //Firehose server options 4 | var options = { 5 | //Config 6 | config: { 7 | tick_interval: 5, 8 | max_buffer_size: 1048576, 9 | max_concurrent_connections: 1024, 10 | transport: { 11 | port: 1337, 12 | host: "0.0.0.0" 13 | } 14 | }, 15 | 16 | //The source will tail a file that has the same topic name 17 | source: new firehose.source.File({ 18 | base_path: "/tmp/", 19 | file_extension: "log" 20 | }), 21 | 22 | //An HTTP transport 23 | transport: firehose.transport.Http 24 | }; 25 | 26 | //Create server, send the options and wait for a client to successfully connect 27 | var server = firehose.createServer(options, function(client) { 28 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 29 | 30 | //Listen for client disconnects 31 | client.on("close", function() { 32 | console.log("Client (" + client.getId() + ") has disconnected") 33 | }); 34 | 35 | }); 36 | 37 | //Start the server listening 38 | server.listen(function(){ 39 | console.log ("Firehose server is now listening"); 40 | }); 41 | 42 | -------------------------------------------------------------------------------- /examples/http_noauth_mqtt/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | 3 | //Firehose server options 4 | var options = { 5 | //Config 6 | config: { 7 | max_buffer_size: 1048576, 8 | max_concurrent_connections: 1024, 9 | transport: { 10 | port: 1337, 11 | host: "0.0.0.0" 12 | } 13 | }, 14 | 15 | //The source creates a sub socket and connects to all endpoints in the array 16 | source: new firehose.source.MQTT({ 17 | qos: 1, 18 | broker: { 19 | "host": "127.0.0.1", 20 | "port": 1245 21 | } 22 | }), 23 | 24 | //An HTTP transport 25 | transport: firehose.transport.Http 26 | }; 27 | 28 | //Create server, send the options and wait for a client to successfully connect 29 | var server = firehose.createServer(options, function(client) { 30 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 31 | 32 | //Listen for client disconnects 33 | client.on("close", function() { 34 | console.log("Client (" + client.getId() + ") has disconnected") 35 | }); 36 | 37 | }); 38 | 39 | //Start the server listening 40 | server.listen(function(){ 41 | console.log ("Firehose server is now listening"); 42 | }); 43 | 44 | -------------------------------------------------------------------------------- /examples/http_basicauth_jsonstore/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); 2 | var path = require('path'); 3 | 4 | //Firehose server options 5 | var options = { 6 | config: { 7 | max_buffer_size: 1048576, 8 | max_concurrent_connections: 1024, 9 | transport: { 10 | port: 1337, 11 | host: "0.0.0.0" 12 | } 13 | }, 14 | 15 | //The base source which sends fake data every 2 seconds 16 | source: new firehose.source.Fake(), 17 | 18 | //An HTTP transport 19 | transport: firehose.transport.Http, 20 | 21 | //Basic authentication 22 | authentication: new firehose.authentication.Basic(), 23 | 24 | //JSON file store 25 | store: new firehose.store.JsonFile({ 26 | path: path.resolve(__dirname, 'users.json') 27 | }) 28 | }; 29 | 30 | //Create server, send the options and wait for a client to successfully connect 31 | var server = firehose.createServer(options, function(client) { 32 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 33 | 34 | //Listen for client disconnects 35 | client.on("close", function() { 36 | console.log("Client (" + client.getId() + ") has disconnected") 37 | }); 38 | 39 | }); 40 | 41 | //Start the server listening 42 | server.listen(); 43 | 44 | -------------------------------------------------------------------------------- /examples/http_noauth_zeromq/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | 3 | //Firehose server options 4 | var options = { 5 | //Config 6 | config: { 7 | tick_interval: 5, 8 | max_buffer_size: 1048576, 9 | max_concurrent_connections: 1024, 10 | transport: { 11 | port: 1337, 12 | host: "0.0.0.0" 13 | } 14 | }, 15 | 16 | //The source creates a sub socket and connects to all endpoints in the array 17 | source: new firehose.source.ZeroMQ({ 18 | type: "sub", 19 | endpoints: [ 20 | "tcp://127.0.0.1:1234", 21 | "tcp://127.0.0.1:1235", 22 | "tcp://127.0.0.1:1236", 23 | ] 24 | }), 25 | 26 | //An HTTP transport 27 | transport: firehose.transport.Http 28 | }; 29 | 30 | //Create server, send the options and wait for a client to successfully connect 31 | var server = firehose.createServer(options, function(client) { 32 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 33 | 34 | //Listen for client disconnects 35 | client.on("close", function() { 36 | console.log("Client (" + client.getId() + ") has disconnected") 37 | }); 38 | 39 | }); 40 | 41 | //Start the server listening 42 | server.listen(function(){ 43 | console.log ("Firehose server is now listening"); 44 | }); 45 | 46 | -------------------------------------------------------------------------------- /lib/authentication/basic.js: -------------------------------------------------------------------------------- 1 | var util = require("util"); 2 | var Authentication = require("./base").Authentication; 3 | 4 | /** 5 | * Basic auth auth 6 | * 7 | */ 8 | function BasicAuthentication(config) { 9 | Authentication.call(this, config); 10 | } 11 | util.inherits(BasicAuthentication, Authentication); 12 | 13 | /** 14 | * Check a request object to see if it has passed authentication 15 | */ 16 | BasicAuthentication.prototype.getCredentials = function(transport) { 17 | //Extract the headers from the request 18 | var headers = transport.getHeaders(); 19 | var headerValue = headers["authorization"]; 20 | if (!headerValue) { 21 | return new Error("No header Authorization header was set"); 22 | } 23 | 24 | //Now check the format 25 | if (headerValue.substr(0, 6) !== "Basic ") { 26 | return new Error("Authorization header was not the correct format"); 27 | } 28 | 29 | //Now get the username and password 30 | var encodedCredentials = headerValue.substr(6); 31 | var credentials = new Buffer(encodedCredentials, 'base64').toString('utf8'); 32 | 33 | //Now check to see if the credentials contains a colon 34 | if (credentials.indexOf(":") < 0) { 35 | return new Error("Credentials were not encoded properly"); 36 | } 37 | 38 | //Now extract the username and password 39 | var credentialArray = credentials.split(":"); 40 | return {username:credentialArray[0], password:credentialArray[1]}; 41 | 42 | }; 43 | 44 | module.exports = { 45 | Authentication: BasicAuthentication 46 | }; 47 | 48 | -------------------------------------------------------------------------------- /lib/source/fake.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Source = require("./base").Source; 3 | 4 | /** 5 | * Fake source source 6 | * 7 | */ 8 | function FakeSource(config) { 9 | Source.call(this, config); 10 | 11 | //Config 12 | this.config = config; 13 | 14 | //Topic intervals 15 | this.topicIntervals = {}; 16 | 17 | } 18 | util.inherits(FakeSource, Source); 19 | 20 | /** 21 | * Start the source 22 | */ 23 | FakeSource.prototype.start = function() { 24 | //Do nothing. Would be used for setting up connections etc 25 | }; 26 | 27 | /** 28 | * Stop the source 29 | */ 30 | FakeSource.prototype.stop = function() { 31 | //Make sure all topics are remove 32 | this.topicIntervals.forEach(function(topic){ 33 | this.removeTopic(topic); 34 | }.bind(this)); 35 | }; 36 | 37 | /** 38 | * Add a topic 39 | */ 40 | FakeSource.prototype.addTopic = function(topic) { 41 | if (this.topicIntervals[topic] === undefined) { 42 | this.topicIntervals[topic] = setInterval(function(){ 43 | this.broadcast(topic, JSON.stringify({ 44 | "data": {"foo": "bar"}, 45 | "time": parseInt(new Date().getTime()/ 1000, 10) 46 | })); 47 | }.bind(this), 2 * 1000); 48 | } 49 | }; 50 | 51 | /** 52 | * Remove a topic 53 | */ 54 | FakeSource.prototype.removeTopic = function(topic) { 55 | if (this.topicIntervals[topic] !== undefined) { 56 | clearInterval(this.topicIntervals[topic]); 57 | this.topicIntervals[topic] = null 58 | delete this.topicIntervals[topic]; 59 | } 60 | }; 61 | 62 | module.exports = { 63 | Source: FakeSource 64 | }; 65 | 66 | -------------------------------------------------------------------------------- /lib/store/json_file.js: -------------------------------------------------------------------------------- 1 | var util = require("util"); 2 | var Store = require("./base").Store; 3 | var User = require("../user/base").User; 4 | 5 | /** 6 | * File based user storage 7 | * 8 | */ 9 | function JsonFileStore(config) { 10 | Store.call(this, config); 11 | 12 | //Users 13 | this.users = require(config.path); 14 | } 15 | util.inherits(JsonFileStore, Store); 16 | 17 | /** 18 | * Check a credentials object to see if it has passed authentication 19 | */ 20 | JsonFileStore.prototype.getUserFromCredentials = function(credentials, callback) { 21 | //Check the credentials exist 22 | if (!credentials.username || !credentials.password) { 23 | return callback(new Error("Username and password are required")); 24 | } 25 | 26 | //Get user based on user id 27 | var item = null; 28 | this.users.forEach(function(userItem){ 29 | if (userItem.username.toLowerCase() === credentials.username.toLowerCase()) { 30 | item = userItem; 31 | } 32 | }); 33 | 34 | //Check for null user 35 | if (item === null) { 36 | return callback(new Error("Username not found")); 37 | } 38 | 39 | //Check the password is correct 40 | if (item.password !== credentials.password) { 41 | return callback(new Error("Password is incorrect")); 42 | } 43 | 44 | //Create a user 45 | var user = new User(); 46 | user.setId(new Date().getTime().toString()); 47 | user.setCredentials(credentials); 48 | 49 | //Return the credentials as that is all we know about the user 50 | callback(undefined, user); 51 | }; 52 | 53 | module.exports = { 54 | Store: JsonFileStore 55 | }; 56 | -------------------------------------------------------------------------------- /lib/source/file.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Source = require("./base").Source; 3 | var Tail = require("tail").Tail; 4 | 5 | /** 6 | * File source 7 | * Config {"base_path":"/tmp/firehose_", "file_extension":"log"} 8 | * 9 | */ 10 | function FileSource(config) { 11 | Source.call(this, config); 12 | 13 | //Config 14 | this.config = config; 15 | 16 | //The files 17 | this.files = {}; 18 | 19 | } 20 | util.inherits(FileSource, Source); 21 | 22 | /** 23 | * Start the source 24 | */ 25 | FileSource.prototype.start = function() { 26 | //Do nothing 27 | }; 28 | 29 | /** 30 | * Stop the source 31 | */ 32 | FileSource.prototype.stop = function() { 33 | //Close each file 34 | Object.keys(this.files).forEach(function(fileName) { 35 | var tail = this.files[fileName]; 36 | tail.unwatch(); 37 | this.files[fileName] = null; 38 | delete this.files[fileName]; 39 | }.bind(this)); 40 | }; 41 | 42 | /** 43 | * Add a topic 44 | */ 45 | FileSource.prototype.addTopic = function(topic) { 46 | //Subscribe to a topic if is not subscribe to already 47 | if (this.files[topic] === undefined) { 48 | var config = this.getConfig(); 49 | var tail = new Tail(config.base_path + topic + (config.file_extension ? "." + config.file_extension : "")); 50 | tail.on("line", function(data) { 51 | this.emit("topic_" + topic, data.toString("utf8")); 52 | }.bind(this)); 53 | this.files[topic] = tail; 54 | } 55 | }; 56 | 57 | /** 58 | * Remove a topic 59 | */ 60 | FileSource.prototype.removeTopic = function(topic) { 61 | //Unsubscribe from topic 62 | if (this.files[topic] !== undefined) { 63 | this.files[topic].removeAllListeners(); 64 | this.files[topic] = null; 65 | delete this.files[topic]; 66 | } 67 | }; 68 | 69 | module.exports = { 70 | Source: FileSource 71 | }; 72 | 73 | -------------------------------------------------------------------------------- /lib/source/mqtt.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Source = require("./base").Source; 3 | var mqtt = require("mqtt"); 4 | 5 | /** 6 | * MQTT source 7 | * Config {"broker":{"port":1245,"host":"127.0.0.1"}} 8 | * 9 | */ 10 | function MQTTSource(config) { 11 | Source.call(this, config); 12 | 13 | //Config 14 | this.config = config; 15 | 16 | //The client 17 | this.client = null; 18 | 19 | //The topics 20 | this.topics = {}; 21 | 22 | } 23 | util.inherits(MQTTSource, Source); 24 | 25 | /** 26 | * Start the source 27 | */ 28 | MQTTSource.prototype.start = function() { 29 | //Create socket 30 | var config = this.getConfig(); 31 | 32 | 33 | var mqtt = require('mqtt') 34 | var client = mqtt.connect('mqtt://' + config.broker.host, config.options) 35 | 36 | client.on('connect', function () { 37 | console.log("MQTT client connected"); 38 | }) 39 | 40 | client.on('message', function(mqttTopic, mqttMessage) { 41 | //Convert from buffers to string ready for comparison 42 | mqttTopic = mqttTopic.toString("utf8"); 43 | if (mqttMessage) { 44 | mqttMessage = mqttMessage.toString("utf8"); 45 | } else { 46 | mqttMessage = "{}"; 47 | } 48 | 49 | //Emit the message 50 | this.emit("topic_" + mqttTopic, mqttMessage); 51 | 52 | }.bind(this)) 53 | 54 | }; 55 | 56 | /** 57 | * Stop the source 58 | */ 59 | MQTTSource.prototype.stop = function() { 60 | //Close the client 61 | this.client.end(); 62 | }; 63 | 64 | /** 65 | * Add a topic 66 | */ 67 | MQTTSource.prototype.addTopic = function(topic) { 68 | //Subscribe to a topic 69 | this.client.subscribe(topic); 70 | }; 71 | 72 | /** 73 | * Remove a topic 74 | */ 75 | MQTTSource.prototype.removeTopic = function(topic) { 76 | //Unsubscribe from topic 77 | this.client.unsubscribe(topic); 78 | }; 79 | 80 | module.exports = { 81 | Source: MQTTSource 82 | }; 83 | 84 | -------------------------------------------------------------------------------- /examples/http_noauth_customrouter/server.js: -------------------------------------------------------------------------------- 1 | var firehose = require("../../index"); //This would be require("firehose"); 2 | var util = require('util'); 3 | 4 | 5 | /** 6 | * Create our own router 7 | */ 8 | function CustomRouter(config) { 9 | firehose.router.Base.call(this, config); 10 | this.routes = [ 11 | '/1/firehose.json' 12 | ]; 13 | } 14 | util.inherits(CustomRouter, firehose.router.Base); 15 | 16 | /** 17 | * Check the path is good 18 | */ 19 | CustomRouter.prototype.check = function(transport) { 20 | var found = false; 21 | this.routes.forEach(function(path){ 22 | if (path === transport.getPath()) { 23 | found = true; 24 | } 25 | }) 26 | return found; 27 | }; 28 | 29 | /** 30 | * Get topics 31 | */ 32 | CustomRouter.prototype.getTopics = function(transport) { 33 | return ['']; 34 | }; 35 | 36 | 37 | 38 | //Firehose server options 39 | var options = { 40 | //Config 41 | config: { 42 | max_buffer_size: 1048576, 43 | max_concurrent_connections: 1024, 44 | transport: { 45 | port: 1337, 46 | host: "0.0.0.0" 47 | } 48 | }, 49 | 50 | //The base source which sends fake data every 2 seconds 51 | source: new firehose.source.Fake(), 52 | 53 | //An HTTP transport 54 | transport: firehose.transport.Http, 55 | 56 | //Use our custom router defined below 57 | router: new CustomRouter() 58 | }; 59 | 60 | //Create server, send the options and wait for a client to successfully connect 61 | var server = firehose.createServer(options, function(client) { 62 | console.log("New client (" + client.getId() + ") has connected. Now there are " + Object.keys(server.clients).length + " clients"); 63 | 64 | //Listen for client disconnects 65 | client.on("close", function() { 66 | console.log("Client (" + client.getId() + ") has disconnected") 67 | }); 68 | 69 | }); 70 | 71 | //Start the server listening 72 | server.listen(function(){ 73 | console.log ("Firehose server is now listening"); 74 | }); 75 | 76 | -------------------------------------------------------------------------------- /lib/transport/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var events = require('events'); 3 | 4 | /** 5 | * Base transport 6 | * 7 | */ 8 | function Transport(config) { 9 | events.EventEmitter.call(this); 10 | 11 | //Config 12 | this.config = config; 13 | 14 | //We need to get the user now that we have a connection 15 | this.user = null; 16 | } 17 | util.inherits(Transport, events.EventEmitter); 18 | 19 | /** 20 | * Get the name for the transport 21 | */ 22 | Transport.prototype.getName = function() { 23 | return "Base"; 24 | }; 25 | 26 | /** 27 | * Get user 28 | */ 29 | Transport.prototype.getUser = function() { 30 | return this.user; 31 | }; 32 | 33 | /** 34 | * Get config 35 | */ 36 | Transport.prototype.getConfig = function() { 37 | return this.config; 38 | }; 39 | 40 | /** 41 | * Start 42 | */ 43 | Transport.prototype.start = function(success, message, statusCode) { 44 | throw new Error("Not implemented"); 45 | }; 46 | 47 | /** 48 | * Get headers 49 | */ 50 | Transport.prototype.getHeaders = function() { 51 | throw new Error("Not implemented"); 52 | }; 53 | 54 | /** 55 | * Write item 56 | */ 57 | Transport.prototype.sendItem = function(topic, item) { 58 | throw new Error("Not implemented"); 59 | }; 60 | 61 | /** 62 | * Send error 63 | */ 64 | Transport.prototype.sendError = function(message, statusCode) { 65 | throw new Error("Not implemented"); 66 | }; 67 | 68 | /** 69 | * Close item 70 | */ 71 | Transport.prototype.close = function(item) { 72 | this.emit("close"); 73 | }; 74 | 75 | /** 76 | * Send tick 77 | */ 78 | Transport.prototype.sendTick = function() { 79 | throw new Error("Not implemented"); 80 | }; 81 | 82 | /** 83 | * Get socket 84 | */ 85 | Transport.prototype.getSocket = function() { 86 | throw new Error("Not implemented"); 87 | }; 88 | 89 | /** 90 | * Get the path used in the request 91 | */ 92 | Transport.prototype.getPath = function() { 93 | throw new Error("Not implemented"); 94 | }; 95 | 96 | /** 97 | * Create server 98 | */ 99 | function createServer(config, callback) { 100 | throw new Error("Server not implemented"); 101 | } 102 | 103 | module.exports = { 104 | createServer: createServer, 105 | Transport: Transport 106 | } 107 | -------------------------------------------------------------------------------- /examples/http_noauth_mqtt/mqtt.js: -------------------------------------------------------------------------------- 1 | var mqtt = require('mqtt'); 2 | var util = require('util'); 3 | 4 | mqtt.createServer(function(client) { 5 | var self = this; 6 | 7 | if (!self.clients) self.clients = {}; 8 | 9 | client.on('connect', function(packet) { 10 | self.clients[packet.clientId] = client; 11 | client.id = packet.clientId; 12 | console.log("CONNECT: client id: " + client.id); 13 | client.subscriptions = []; 14 | client.connack({returnCode: 0}); 15 | }); 16 | 17 | client.on('subscribe', function(packet) { 18 | var granted = []; 19 | 20 | console.log("SUBSCRIBE(%s): %j", client.id, packet); 21 | 22 | for (var i = 0; i < packet.subscriptions.length; i++) { 23 | var qos = packet.subscriptions[i].qos 24 | , topic = packet.subscriptions[i].topic 25 | , reg = new RegExp(topic.replace('+', '[^\/]+').replace('#', '.+') + '$'); 26 | 27 | granted.push(qos); 28 | client.subscriptions.push(reg); 29 | } 30 | 31 | client.suback({messageId: packet.messageId, granted: granted}); 32 | }); 33 | 34 | client.on('publish', function(packet) { 35 | console.log("PUBLISH(%s): %j", client.id, packet); 36 | for (var k in self.clients) { 37 | var c = self.clients[k] 38 | , publish = false; 39 | 40 | for (var i = 0; i < c.subscriptions.length; i++) { 41 | var s = c.subscriptions[i]; 42 | 43 | if (s.test(packet.topic)) { 44 | publish = true; 45 | } 46 | } 47 | 48 | if (publish) { 49 | c.publish({topic: packet.topic, payload: packet.payload}); 50 | } 51 | } 52 | }); 53 | 54 | client.on('pingreq', function(packet) { 55 | console.log('PINGREQ(%s)', client.id); 56 | client.pingresp(); 57 | }); 58 | 59 | client.on('disconnect', function(packet) { 60 | client.stream.end(); 61 | }); 62 | 63 | client.on('close', function(packet) { 64 | delete self.clients[client.id]; 65 | }); 66 | 67 | client.on('error', function(e) { 68 | client.stream.end(); 69 | console.log(e); 70 | }); 71 | }).listen(1245, "127.0.0.1"); 72 | 73 | //Create a client to send data on the firehose topic 74 | var client = mqtt.createClient(1245, "127.0.0.1"); 75 | setInterval(function(){ 76 | client.publish("firehose", '{"some_data":' + (new Date().getTime()) + '}'); 77 | }, 500); 78 | 79 | -------------------------------------------------------------------------------- /lib/source/zeromq.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Source = require("./base").Source; 3 | var zmq = require("zeromq"); 4 | 5 | /** 6 | * ZeroMQ source 7 | * Config {"type":"sub","endpoints":["tcp://127.0.0.1:1234","tcp://127.0.0.1:1235"]} 8 | * 9 | */ 10 | function ZeroMQSource(config) { 11 | Source.call(this, config); 12 | 13 | //Config 14 | this.config = config; 15 | 16 | //The socket 17 | this.socket = null; 18 | 19 | //The topics 20 | this.topics = {}; 21 | 22 | } 23 | util.inherits(ZeroMQSource, Source); 24 | 25 | /** 26 | * Start the source 27 | */ 28 | ZeroMQSource.prototype.start = function() { 29 | //Create socket 30 | var config = this.getConfig(); 31 | this.socket = zmq.socket(config.type ? config.type : "sub"); 32 | this.socket.identity = 'node_firehose'; 33 | 34 | //Connect to all endpoints 35 | config.endpoints.forEach(function(endpoint) { 36 | try { 37 | this.socket.connect(endpoint); 38 | } catch (e) { 39 | throw new Error("Could not connect to ZeroMQ endpoint: " + endpoint + " Error: " + e.message); 40 | } 41 | }.bind(this)); 42 | 43 | //Add event listener and emit events on topics 44 | this.socket.on("message", function(zmqTopic, zmqData) { 45 | //Convert from buffers to string ready for comparison 46 | zmqTopic = zmqTopic.toString("utf8"); 47 | if (zmqData) { 48 | zmqData = zmqData.toString("utf8"); 49 | } else { 50 | zmqData = "{}"; 51 | } 52 | 53 | //Check this topic against the list we have so we emit correctly 54 | //This is due to zmqs substring matches for topics 55 | Object.keys(this.topics).forEach(function(topic) { 56 | //Check the topic appears at the beginning of the zmq received topic 57 | if (zmqTopic.substring(0, topic.length-1) === topic) { 58 | this.emit("topic_" + topic, zmqData); 59 | } 60 | }.bind(this)); 61 | }.bind(this)); 62 | 63 | }; 64 | 65 | /** 66 | * Stop the source 67 | */ 68 | ZeroMQSource.prototype.stop = function() { 69 | //Close the socket 70 | this.socket.close(); 71 | 72 | }; 73 | 74 | /** 75 | * Add a topic 76 | */ 77 | ZeroMQSource.prototype.addTopic = function(topic) { 78 | //Subscribe to a topic 79 | this.topics[topic] = true; 80 | this.socket.subscribe(topic); 81 | }; 82 | 83 | /** 84 | * Remove a topic 85 | */ 86 | ZeroMQSource.prototype.removeTopic = function(topic) { 87 | //Unsubscribe from topic 88 | this.topics[topic] = null; 89 | delete this.topics[topic]; 90 | this.socket.unsubscribe(topic); 91 | }; 92 | 93 | module.exports = { 94 | Source: ZeroMQSource 95 | }; 96 | 97 | -------------------------------------------------------------------------------- /lib/client/base.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var events = require('events'); 3 | 4 | /** 5 | * Base client 6 | * 7 | */ 8 | function Client(transport, source, user) { 9 | events.EventEmitter.call(this); 10 | 11 | //Transport 12 | this.transport = transport; 13 | 14 | //User 15 | this.user = user; 16 | 17 | //Source 18 | this.source = source; 19 | 20 | //Subscriptions 21 | this.subscriptions = {}; 22 | 23 | //Id 24 | this.id = new Date().getTime().toString(); 25 | 26 | //Last item time 27 | this.lastItem = 0; 28 | 29 | //Start listening for transport events 30 | this.transport.on("close", function(){ 31 | this.close(); 32 | }.bind(this)); 33 | 34 | // 35 | // Add listeners for errors etc (needs base transport to extend event emitter) 36 | // Then emit standard events for close etc 37 | // 38 | // 39 | 40 | } 41 | util.inherits(Client, events.EventEmitter); 42 | 43 | /** 44 | * Get the id 45 | */ 46 | Client.prototype.getId = function() { 47 | return this.id; 48 | }; 49 | 50 | /** 51 | * Get the transport 52 | */ 53 | Client.prototype.getTransport = function() { 54 | return this.transport; 55 | }; 56 | 57 | /** 58 | * Get the source 59 | */ 60 | Client.prototype.getSource = function() { 61 | return this.source; 62 | }; 63 | 64 | /** 65 | * Get the user 66 | */ 67 | Client.prototype.getUser = function() { 68 | return this.user; 69 | }; 70 | 71 | /** 72 | * Get the topics 73 | */ 74 | Client.prototype.getTopics = function() { 75 | return Object.keys(this.subscriptions); 76 | }; 77 | 78 | /** 79 | * Add a topic 80 | */ 81 | Client.prototype.addTopic = function(topic) { 82 | //Subscribe for updates 83 | if (this.subscriptions[topic] === undefined) { 84 | this.subscriptions[topic] = function(data){ 85 | //Write to the transport 86 | this.getTransport().sendItem(topic, data); 87 | this.lastItem = parseInt(new Date().getTime() / 1000); 88 | }.bind(this); 89 | 90 | //Subscribe for events on this topic 91 | this.getSource().on("topic_" + topic, this.subscriptions[topic]); 92 | 93 | //Tell anything listening that this is a new topic. Used by source to start sending data on a topic 94 | this.emit("add_topic", topic); 95 | 96 | } 97 | 98 | }; 99 | 100 | /** 101 | * Remove a topic 102 | */ 103 | Client.prototype.removeTopic = function(topic) { 104 | var subscription = this.subscriptions[topic]; 105 | if (subscription !== undefined) { 106 | 107 | //Remove listener 108 | this.getSource().removeListener(topic, subscription); 109 | 110 | //Tell anything listening that the client is no longer listening to the topic 111 | this.emit("remove_topic", topic); 112 | } 113 | }; 114 | 115 | /** 116 | * Remove all topics 117 | */ 118 | Client.prototype.removeAllTopics = function() { 119 | Object.keys(this.subscriptions).forEach(function(topic){ 120 | this.removeTopic(topic); 121 | }.bind(this)); 122 | }; 123 | 124 | /** 125 | * Get the user 126 | */ 127 | Client.prototype.close = function() { 128 | //Unsubscribe from topics 129 | this.removeAllTopics(); 130 | 131 | //Emit close 132 | this.emit("close"); 133 | }; 134 | 135 | 136 | module.exports = { 137 | Client: Client 138 | }; 139 | 140 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | node-firehose 2 | ============= 3 | 4 | ## This repo os archived. For a similar project checkput https://benthos.dev 5 | 6 | 7 | Create a firefose from anything! 8 | 9 | ## Background 10 | 11 | Creating a firehose for your data can be a daunting propect. You can create one fairly simply, but there are alot of other factors to take into accont. 12 | 13 | * What happens if a user is on a slow connection? Memory usage will shoot up as the socket is buffered 14 | * Different authentication methods 15 | * Making it stable. You don't want to kick off happy customers, just because one customer has an error/exception 16 | * Support different transports, user storage, authentication, routes and sources 17 | 18 | These services, by their very nature, are long lived processes. Any errors will affect all customers connected. I ([OllieParsley](http://ollieparsley.com/)) have experience handling the Twitter firehose and also developing the [DataSift](http://datasift.com/) HTTP Streaming API, which behaves in a similar way to the Twitter Streaming API 19 | 20 | ## Installation 21 | 22 | ### NPM 23 | 24 | ```bash 25 | npm install firehose 26 | ``` 27 | 28 | ### Source 29 | 30 | ```bash 31 | git clone git@github.com:ollieparsley/node-firehose.git 32 | ``` 33 | 34 | ## Components 35 | 36 | The firehose server is made up of a few big components. These can mostly be combined in anyway to customise the ways a user connects and authenticates with the server. 37 | 38 | ### Transport 39 | 40 | Transports are the different communication methods you have with the user. In the examples directory you will find a ready made HTTP Transport component 41 | 42 | Included in this module are: 43 | 44 | * HTTP 45 | * WebSockets 46 | 47 | ### Authentication 48 | 49 | It is really easy to customise how users are authenticated. This takes care of extracting credentials from a request. The default authentication doesn't check any credentials. In the examples directory you will find a Basic Authentication component. 50 | 51 | #### Included transports 52 | 53 | * No authentication 54 | * Basic Authentication 55 | 56 | ### Store 57 | 58 | Once the firehose server has grabbed credentials (if any) they are passed along to a store and expects a User object in response. The default Store returns a randomly generated user. In the examples directory you can find a JSON Store which has all users stored in a JSON file. 59 | 60 | #### Included stores 61 | 62 | * JSON file storage 63 | 64 | ### Router 65 | 66 | Routers are used to make sure the path specified in a request is correct. The default router allows all requests through. 67 | 68 | #### Included routers 69 | 70 | * Allow all paths 71 | * There is an example custom router (in the style of Twitter Streaming API) in the examples 72 | 73 | ### Source 74 | 75 | Arguably the most important component. This is the element that receives your own data and passes it to users. The fake source sends a bit of JSON every 2 seconds. 76 | 77 | #### Included sources: 78 | 79 | * Fake (outputs fake JSON date every 2 seconds) 80 | * MQTT (listen to topics on an MQTT broker) 81 | * ZeroMQ (SUB or PULL data from ZMQ sockets) 82 | * File (tail any file) 83 | 84 | ## Creating your own components 85 | 86 | You can create your components by extending our base classes. Check out the examples directory for how to do this. Essentially you include our base class for a component and extend any of the methods using prototype inheritance. 87 | -------------------------------------------------------------------------------- /lib/transport/websocket.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Transport = require("./base").Transport; 3 | var WebSocket = require('ws'); 4 | 5 | /** 6 | * Websocket transport 7 | * 8 | */ 9 | function WebsocketTransport(config, client) { 10 | Transport.call(this, config); 11 | 12 | //Client 13 | this.client = client; 14 | 15 | //Add listener for messages 16 | this.client.on("message", function(){ 17 | //Read message 18 | }.bind(this)); 19 | 20 | //Add listener for close 21 | this.client.on("close", function(){ 22 | this.close(); 23 | }.bind(this)); 24 | 25 | //Add listener for error 26 | this.client.on("error", function(e){ 27 | this.close(); 28 | }.bind(this)); 29 | 30 | //Closed 31 | this.closed = false; 32 | 33 | } 34 | util.inherits(WebsocketTransport, Transport); 35 | 36 | /** 37 | * Get the name for the transport 38 | */ 39 | WebsocketTransport.prototype.getName = function() { 40 | return "WebSocket"; 41 | }; 42 | 43 | /** 44 | * Start 45 | */ 46 | WebsocketTransport.prototype.start = function(success, message, statusCode) { 47 | if (!success) { 48 | this.sendError(message, statusCode); 49 | } 50 | }; 51 | 52 | /** 53 | * Get headers 54 | */ 55 | WebsocketTransport.prototype.getHeaders = function() { 56 | return this.client && this.client.request && this.client.request.headers ? this.client.request.headers : {}; 57 | }; 58 | 59 | /** 60 | * Send item 61 | */ 62 | WebsocketTransport.prototype.sendItem = function(topic, item) { 63 | this.write(item); 64 | }; 65 | 66 | /** 67 | * Send error 68 | */ 69 | WebsocketTransport.prototype.sendError = function(message, statusCode) { 70 | this.write(JSON.stringify({"message": message})); 71 | this.close(); 72 | }; 73 | 74 | /** 75 | * Write item 76 | */ 77 | WebsocketTransport.prototype.write = function(item) { 78 | if (this.client.readyState === WebSocket.OPEN) { 79 | this.client.send(data); 80 | } 81 | }; 82 | 83 | /** 84 | * Close item 85 | */ 86 | WebsocketTransport.prototype.close = function() { 87 | if (!this.closed) { 88 | this.closed = true; 89 | this.client.close(); 90 | } 91 | this.emit("close"); 92 | }; 93 | 94 | /** 95 | * Send tick 96 | */ 97 | WebsocketTransport.prototype.sendTick = function() { 98 | var topic = ''; 99 | this.sendItem(topic, JSON.stringify({"tick": parseInt(new Date().getTime() / 1000, 10)})); 100 | }; 101 | 102 | /** 103 | * Get socket 104 | */ 105 | WebsocketTransport.prototype.getSocket = function() { 106 | return this.client && this.client.socket ? this.client.socket : false; 107 | }; 108 | 109 | /** 110 | * Get the path used in the request 111 | */ 112 | Transport.prototype.getPath = function() { 113 | return this.request && this.request.url ? this.request.url : false; 114 | }; 115 | 116 | /** 117 | * Create server 118 | */ 119 | function createServer(config, callback) { 120 | //Check config 121 | if (!config.port) { 122 | throw new Error("No port specified in transport config"); 123 | } 124 | 125 | var server = new WebSocket.Server({ 126 | port: config.port 127 | }); 128 | 129 | //Listen for websocket connections 130 | server.on('connection', function (client) { 131 | //Create an instance of the transport 132 | callback(new WebsocketTransport(config, client)); 133 | }); 134 | 135 | //Return an object that will allow the user to start listening when they want 136 | return { 137 | listen: function(callback) { 138 | // N/A 139 | }, 140 | close: function(callback) { 141 | server.close(callback); 142 | } 143 | } 144 | 145 | }; 146 | 147 | module.exports = { 148 | createServer: createServer, 149 | Transport: WebsocketTransport 150 | } 151 | -------------------------------------------------------------------------------- /lib/transport/http.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var Transport = require("./base").Transport; 3 | 4 | /** 5 | * HTTP transport 6 | * 7 | */ 8 | function HttpTransport(config, request, response) { 9 | Transport.call(this, config); 10 | 11 | //Request 12 | this.request = request; 13 | 14 | //Response 15 | this.response = response; 16 | 17 | //Add listener 18 | this.response.on("close", function(){ 19 | this.close(); 20 | }.bind(this)); 21 | this.response.on("end", function(){ 22 | this.close(); 23 | }.bind(this)); 24 | this.response.on("error", function(){ 25 | this.close(); 26 | }.bind(this)); 27 | 28 | //Headers 29 | this.headers = { 30 | "Content-Type": "application/json", 31 | "X-Served-By": "node-firehose" 32 | }; 33 | 34 | //Header sent 35 | this.headersSent = false; 36 | 37 | //Closed 38 | this.closed = false; 39 | 40 | } 41 | util.inherits(HttpTransport, Transport); 42 | 43 | /** 44 | * Get the name for the transport 45 | */ 46 | HttpTransport.prototype.getName = function() { 47 | return "HTTP"; 48 | }; 49 | 50 | /** 51 | * Start 52 | */ 53 | HttpTransport.prototype.start = function(success, message, statusCode) { 54 | if (success) { 55 | this.response.writeHead(200, this.headers); 56 | } else { 57 | this.response.sendError(message, statusCode); 58 | this.response.close(); 59 | } 60 | }; 61 | 62 | /** 63 | * Get headers 64 | */ 65 | HttpTransport.prototype.getHeaders = function() { 66 | return this.request && this.request.headers ? this.request.headers : {}; 67 | }; 68 | 69 | /** 70 | * Send item 71 | */ 72 | HttpTransport.prototype.sendItem = function(topic, item) { 73 | this.write(item); 74 | }; 75 | 76 | /** 77 | * Send error 78 | */ 79 | HttpTransport.prototype.sendError = function(message, statusCode) { 80 | this.sendHeaders(statusCode ? statusCode : 500, this.headers); 81 | this.write(JSON.stringify({"message": message})); 82 | this.close(); 83 | }; 84 | 85 | /** 86 | * Send headers 87 | */ 88 | HttpTransport.prototype.sendHeaders = function(statusCode, headers) { 89 | if (!this.headersSent) { 90 | this.headersSent = true; 91 | this.response.writeHead(statusCode, headers); 92 | } 93 | }; 94 | 95 | /** 96 | * Write item 97 | */ 98 | HttpTransport.prototype.write = function(item) { 99 | this.sendHeaders(200, this.headers); 100 | this.response.write(item + "\r\n"); 101 | }; 102 | 103 | /** 104 | * Close item 105 | */ 106 | HttpTransport.prototype.close = function() { 107 | if (!this.closed) { 108 | this.closed = true; 109 | this.response.end(); 110 | } 111 | this.emit("close"); 112 | }; 113 | 114 | /** 115 | * Send tick 116 | */ 117 | HttpTransport.prototype.sendTick = function() { 118 | var topic = ''; 119 | this.sendItem(topic, JSON.stringify({"tick": parseInt(new Date().getTime() / 1000, 10)})); 120 | }; 121 | 122 | /** 123 | * Get socket 124 | */ 125 | HttpTransport.prototype.getSocket = function() { 126 | return this.response && this.response.socket ? this.response.socket : false; 127 | }; 128 | 129 | /** 130 | * Get the path used in the request 131 | */ 132 | Transport.prototype.getPath = function() { 133 | return this.request && this.request.url ? this.request.url : false; 134 | }; 135 | 136 | /** 137 | * Create server 138 | */ 139 | function createServer(config, callback) { 140 | 141 | //Check config 142 | if (!config.host) { 143 | throw new Error("No host specified in transport config"); 144 | } else if (!config.port) { 145 | throw new Error("No port specified in transport config"); 146 | } 147 | 148 | //Create the http server 149 | var server = require("http").createServer(function(request, response){ 150 | //Create an instance of the transport 151 | callback(new HttpTransport(config, request, response)); 152 | }); 153 | 154 | //Return an object that will allow the user to start listening when they want 155 | return { 156 | listen: function(callback) { 157 | server.listen( 158 | config.port ? config.port : 8080, //Specify the port 159 | config.host ? config.host : callback, //Specify the host or just the callback if the host is not specified 160 | config.host ? callback : undefined //Specify the callback if the host is specified 161 | ); 162 | }, 163 | close: function(callback) { 164 | server.close(callback); 165 | } 166 | } 167 | 168 | }; 169 | 170 | module.exports = { 171 | createServer: createServer, 172 | Transport: HttpTransport 173 | } 174 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var firehose = { 2 | authentication: { 3 | Base: require("./lib/authentication/base").Authentication, 4 | Basic: require("./lib/authentication/basic").Authentication, 5 | AllowAll: require("./lib/authentication/allow_all").Authentication 6 | }, 7 | router: { 8 | Base: require("./lib/router/base").Router, 9 | AllowAll: require("./lib/router/allow_all").Router 10 | }, 11 | source: { 12 | Base: require("./lib/source/base").Source, 13 | Fake: require("./lib/source/fake").Source, 14 | ZeroMQ: require("./lib/source/zeromq").Source, 15 | MQTT: require("./lib/source/mqtt").Source, 16 | File: require("./lib/source/file").Source 17 | }, 18 | store: { 19 | Base: require("./lib/store/base").Store, 20 | JsonFile: require("./lib/store/json_file").Store 21 | }, 22 | transport: { 23 | Base: require("./lib/transport/base"), 24 | Http: require("./lib/transport/http"), 25 | Websocket: require("./lib/transport/websocket") 26 | }, 27 | user: { 28 | Base: require("./lib/user/base").User 29 | }, 30 | client: { 31 | Base: require("./lib/client/base").Client 32 | }, 33 | createServer: createServer 34 | }; 35 | 36 | function createServer(options, callback) { 37 | 38 | //Check the source 39 | if (!options.source) { 40 | throw new Error("A source needs to be set"); 41 | } else if (!(options.source instanceof firehose.source.Base)) { 42 | throw new Error("The source must be an instance of Source"); 43 | } 44 | 45 | //Check the transport 46 | //console.log("\n\n", options.transport.Transport, firehose.transport.Base.Transport, "\n\n"); 47 | //process.exit(); 48 | if (!options.transport) { 49 | throw new Error("A transport needs to be set"); 50 | } else if (!options.transport.createServer) { 51 | throw new Error("The transport must have a createServer method"); 52 | } 53 | 54 | //Check the router 55 | if (!options.router) { 56 | //Choose a router that allows all through 57 | options.router = new firehose.router.AllowAll(); 58 | } else if (!(options.router instanceof firehose.router.Base)) { 59 | throw new Error("The router must be an instance of Router"); 60 | } 61 | 62 | //Check the authentication 63 | if (!options.authentication) { 64 | //Choose the default authentication 65 | options.authentication = new firehose.authentication.Base(); 66 | } else if (!(options.authentication instanceof firehose.authentication.Base)) { 67 | throw new Error("The authentication must be an instance of Authentication"); 68 | } 69 | 70 | //Check the store 71 | if (!options.store) { 72 | options.store = new firehose.store.Base(); 73 | } else if (!(options.store instanceof firehose.store.Base)) { 74 | throw new Error("The store must be an instance of Store"); 75 | } 76 | 77 | //Start the source 78 | options.source.start(); 79 | 80 | //Clients 81 | var clients = {}; 82 | 83 | //Topics - ref counts 84 | var topics = {}; 85 | 86 | //Ticks 87 | var tickIntervalSeconds = options.config && options.config.tick_interval ? options.config.tick_interval : 30. 88 | var tickInterval = setInterval(function(){ 89 | var now = parseInt(new Date().getTime() / 1000); 90 | Object.keys(clients).forEach(function(clientId){ 91 | var client = clients[clientId]; 92 | if (now - client.lastItem > tickIntervalSeconds) { 93 | //Client hasn't received data for over x seconds so send a tick 94 | client.getTransport().sendTick(); 95 | } 96 | }); 97 | }, tickIntervalSeconds * 1000); 98 | 99 | //If we have a max buffer size confit then check it every 10 seconds 100 | var bufferCheckInterval = null; 101 | if (options.config && options.config.max_buffer_size) { 102 | bufferCheckInterval = setInterval(function(){ 103 | Object.keys(clients).forEach(function(clientId){ 104 | var client = clients[clientId]; 105 | var transport = client.getTransport(); 106 | if (transport) { 107 | var socket = transport.getSocket(); 108 | if (socket && socket.bufferSize && socket.bufferSize > options.config.max_buffer_size) { 109 | client.sendError("You are consuming too slowy and have been disconnected", 400) 110 | } 111 | } 112 | }); 113 | }, 10 * 1000); 114 | } 115 | 116 | //Now create the transport server 117 | var transportServer = options.transport.createServer(options.config.transport, function(transport) { 118 | //Check to see if we are the max connections 119 | if (options.config && options.config.max_concurrent_connections <= Object.keys(clients).length) { 120 | //We are at capacity 121 | transport.sendError("The service has reached capacity", 503); 122 | transport.close(); 123 | return; 124 | } 125 | 126 | //Check we have a route match 127 | if (!options.router.check(transport)) { 128 | transport.sendError("Not found", 404); 129 | transport.close(); 130 | return; 131 | } 132 | 133 | //Now check the authentication 134 | var credentials = options.authentication.getCredentials(transport); 135 | if (credentials instanceof Error) { 136 | transport.sendError("Not authorized", 401); 137 | transport.close(); 138 | return; 139 | } 140 | 141 | //Get the user using the credentials 142 | options.store.getUserFromCredentials(credentials, function(error, user){ 143 | 144 | //Check if we have an error 145 | if (error) { 146 | transport.sendError(error.message, 401); 147 | transport.close(); 148 | return; 149 | } 150 | 151 | //Create a new client 152 | var client = new firehose.client.Base(transport, options.source, user); 153 | 154 | //Add client to client list 155 | clients[client.getId()] = client; 156 | 157 | //Remove client from list 158 | client.on("close", function(){ 159 | clients[client.getId()] = null; 160 | delete clients[client.getId()]; 161 | }); 162 | 163 | //Client added a topic 164 | client.on("add_topic", function(topic){ 165 | console.log("Client added topic: " + topic); 166 | if (topics[topic] === undefined) { 167 | //Add the topic to the list 168 | topics[topic] = {count:1}; 169 | 170 | //New topic so tell the source to add it 171 | options.source.addTopic(topic); 172 | } else { 173 | //Increment the ref count 174 | topics[topic].count++; 175 | } 176 | }); 177 | 178 | //Client removed a topic 179 | client.on("remove_topic", function(topic){ 180 | console.log("Client removed topic: " + topic); 181 | //Check if we have the topic at all 182 | if (topics[topic] === undefined) { 183 | return; 184 | } 185 | 186 | //Decrement 187 | topics[topic].count--; 188 | 189 | //Check if none left 190 | if (topics[topic].count <= 0) { 191 | //Remove the topic as there is nothing listening 192 | options.source.removeTopic(topic); 193 | } 194 | }); 195 | 196 | //Get topics from initial connection 197 | var topics = options.router.getTopics(transport); 198 | topics.forEach(function(topic){ 199 | //Add subscriptions for client 200 | client.addTopic(topic); 201 | }); 202 | 203 | //From the transport get a user 204 | callback(client) 205 | }); 206 | 207 | }); 208 | 209 | //Return an object so they can call the listen method 210 | return { 211 | listen: function(listenCallback) { 212 | if (transportServer && transportServer.listen) { 213 | transportServer.listen(listenCallback); 214 | } else { 215 | listenCallback(); 216 | } 217 | }, 218 | close: function(){ 219 | //Stop the source 220 | options.source.stop(); 221 | 222 | //Close the server itself 223 | transportServer.close(); 224 | 225 | //Stop the buffer check 226 | if (bufferCheckInterval !== null) { 227 | clearInterval(bufferCheckInterval); 228 | } 229 | 230 | //Clear tick interval 231 | if (tickInterval !== null) { 232 | clearInterval(tickInterval); 233 | } 234 | }, 235 | clients: clients 236 | }; 237 | } 238 | 239 | exports = module.exports = firehose; --------------------------------------------------------------------------------