├── .gitignore ├── .travis.yaml ├── Makefile ├── README.md ├── index.js ├── lib ├── client.js ├── connection.js └── message.js ├── package.json ├── test.sh └── test ├── client.js ├── connecting.js ├── helper.js ├── identify-client.js ├── index.js ├── message.js ├── stress.js ├── subscribe-callback.js └── tls.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | nsq.tar.gz 3 | nsq/ 4 | -------------------------------------------------------------------------------- /.travis.yaml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY : test-linux test-osx download-linux download-osx test 2 | 3 | test-osx: download-osx test 4 | 5 | download-linux: 6 | @echo "downloading linux nsqd" 7 | curl -L https://github.com/bitly/nsq/releases/download/v0.2.27/nsq-0.2.27.linux-amd64.go1.2.tar.gz > nsq.tar.gz 8 | 9 | download-osx: 10 | @echo "downloading osx nsqd" 11 | curl -L https://github.com/bitly/nsq/releases/download/v0.2.27/nsq-0.2.27.darwin-amd64.go1.1.2.tar.gz > nsq.tar.gz 12 | 13 | test: 14 | ./test.sh 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nsqueue 2 | 3 | Simple, well-tested node client for [nsqd](https://github.com/bitly/nsq). nslookupd smarts not included (yet). 4 | 5 | ## install 6 | 7 | ```sh 8 | $ npm install nsqueue 9 | ``` 10 | ## quick start 11 | 12 | - Download nsq and run it 13 | - If you're using mac run `make download-osx` 14 | - If you're using linux run `make download-linux` 15 | - After download you can test it by running `make test` 16 | 17 | ## example 18 | 19 | ```js 20 | var nsqueue = require('nsqueue') 21 | 22 | var client = new nsqueue.Client({ 23 | host: 'localhost', 24 | port: 4150 25 | }) 26 | 27 | //connect to the nsqd server process over TCP 28 | client.connect() 29 | 30 | //publish buffers 31 | client.publish('your-topic', Buffer('Hi Mom', 'ascii')) 32 | 33 | //publish strings (utf8 assumed) 34 | client.publish('your-topic', 'Hi again, Mom!') 35 | 36 | //publish json 37 | client.publish('your-topic', {say: 'hi', to: 'mom'}, function(err) { 38 | //OPTIONAL callback 39 | //called when your message has been received by nsqd 40 | }) 41 | 42 | //publish multiple messages in a batch 43 | var messages = [ 44 | Buffer('Hi Dad', 'ascii'), 45 | 'Hi again, Dad!', 46 | {say: 'hi', to: 'dad'} 47 | ] 48 | 49 | client.publishAll('your-topic', messages, function(err) { 50 | //OPTIONAL callback 51 | //called when all your messages 52 | //in this batch have been received by nsqd 53 | }) 54 | 55 | client.subscribe('your-topic', 'your-channel', function(err) { 56 | //OPTIONAL callback 57 | //called when you have successfully subscribed 58 | //to the given topic/channel pair 59 | }) 60 | 61 | client.on('message', function(msg) { 62 | console.log(msg.data) //Buffer <...> 63 | console.log(msg.data.toString()) // 'hi mom' 64 | 65 | msg.finish() //finish the message - nsqd does not support 66 | //success notification so there is no callback 67 | }) 68 | 69 | client.on('error', function(err) { 70 | //something bad happened. :( 71 | }) 72 | 73 | client.end(function(err) { 74 | console.log('client closed') 75 | }) 76 | 77 | ``` 78 | 79 | ## api 80 | 81 | ### new Client(options) 82 | 83 | __options: object__ `options` should be an object with `port` & `host` properties. No default value is assumed if they are not supplied. 84 | 85 | #### client.connect([callback]) 86 | 87 | Connects the client to the `host` & `port` pair supplied in the constructor. 88 | 89 | _optional_ __callback(error: Error)__ callback called with error event if a connection error was encountered. If no callback is supplied and an error is encountered during connection, the client will emit the error as an `error` event. 90 | 91 | 92 | #### client.publish(topic, data, [callback]) 93 | 94 | __topic: string__ must be a string of valid nsqd topic characters 95 | 96 | __data: buffer, string, or object__ the payload of the message 97 | - if string, assumed to be utf8 encoded 98 | - if object, passed to `JSON.stringify` before publishing 99 | 100 | _optional_ __callback(error: Error)__ called when the message has been received by the nsqd server. Passed an `Error` object if there was a problem publishing the message. 101 | 102 | If no callback is supplied any publish error will be emitted as an `error` event. 103 | 104 | 105 | #### client.publishAll(topic, datums, [callback]) 106 | 107 | The same as `client.publish` but takes an array of data payloads as the second argument and publishes them as a batch. Batch publishing is part of the nsqd protocol and is generally more efficient when publishing many messages at once, though error handling is harder because you only know `m of n` messages failed in the event of an error, not which ones. Each of the items in the `datam` array will be considered its own message by nsqd. 108 | 109 | __topic: string__ must be a string of valid nsqd topic characters 110 | 111 | __datam: Array of buffers, strings, and/or objects__ the payloads of the messages 112 | 113 | _optional_ __callback(error: Error)__ called when the messages have all been received by the nsqd server. Passed an `Error` object if there was a problem publishing any of the messages. 114 | 115 | If no callback is supplied any publish error will be emitted as an `error` event. 116 | 117 | 118 | #### client.subscribe(topic, channel, [callback]) 119 | 120 | __topic: string__ the topic this client should subscribe to 121 | __channel: string__ the channel this client should subscribe to 122 | _optional_ __callback(error: Error)__ called when the client has successfully subscribed to the topic/channel pair. Passed an `Error` object i there was a problem subscribing to the channel/topic pair. 123 | 124 | If no callback is supplied any error during subscribing will be emitted as an `error` event. 125 | 126 | #### client.on('message', callback(message)) 127 | 128 | Adds an event listener which is called _every time_ a message is received on this client. Messages will only be received on the channel/topic pair the client is subscribed to. Note: the client does no internal buffering of incomming messages. Once the client is subscribed to a topic/channel pair events will start 'flowing' in immediately after the subscribe callback is called. You can add a `message` event listener before even calling subscribe. If no callback is supplied, any er 129 | 130 | If no callback is supplied any publish error will be emitted as an `error` event. 131 | 132 | #### client.end([callback]) 133 | 134 | Disconnects the client. 135 | 136 | _optional_ __callback(error: Error)__ called when the client has disconnected cleanly from the server nsqd process and the socket is closed. 137 | 138 | #### client.concurrency: int 139 | 140 | Default value: `1` 141 | 142 | The maximum number of in-flight messages the nsqd server will deliver to this client at one time. Setting this to `5` for example will allow 5 in-flight messages sent to this client. As each message is finished or requeued the server will send more messages down to the client until it again has 5 in-flight at a time. This value can be changed at any time and will take affect as soon as the next in-flight message is either requeued or finished. 143 | ### message 144 | 145 | Message objects are not created by you directly. They are emitted from clients with an active subscription on a topic/channel pair through the `message` event. 146 | 147 | #### message.data: Buffer 148 | 149 | The raw binary data of the message. Call `message.data.toString('utf8')` for a string representation. 150 | 151 | _note:_ because the nsqd protocol does not allow empty messages, this will never be null. 152 | 153 | #### message.json(): Object 154 | 155 | A helper which calls `JSON.parse(message.data.toString('utf8'))` and returns the results because it is so common to send & receive JSON messages. 156 | 157 | _note:_ calling this on a message which has non-valid JSON contents will throw a json parsing exception. 158 | 159 | #### message.finish() : bool 160 | 161 | Call this when you're done processing the message. Tells the nsqd server process you have successfully finished processing this message. The server will remove this message from the queue and not send it out to any more clients. 162 | 163 | Returns `true` if the response was sent to the nsqd server. If the message has already been responded to -- it is no longer `.inFlight == true` -- then this is a no-op and returns `false`. 164 | 165 | If there is an error finishing this message, the client will emit an `error` event. 166 | 167 | _note:_ currently the binary protocol does not communicate anything back in the event of a successful `FIN` message. There's no way to have a callback for `message.finish()` at this time -- it's fire and forget. 168 | 169 | #### message.requeue(timeoutInMilliseconds: int) 170 | 171 | __timeoutInMilliseconds: int__ millisecond timeout the nsqd server will wait before attempting to deliver the message again. 172 | 173 | Signals the nsqd server to requeue the message and deliver it again. You usually call this if the message consumer has failed to process the message appropriately. 174 | 175 | Returns `true` if the response was sent to the nsqd server. If the message has already been responded to -- it is no longer `.inFlight == true` -- then this is a no-op and returns `false`. 176 | 177 | If there is a problem requeuing this message, the client will emit an `error` event. 178 | 179 | _note:_ currently the binary protocol does not communicate anything back in the event of a successful `REQ` message. There's no way to have a callback for `message.requeue(1000)` at this time -- it's fire and forget. 180 | 181 | #### message.touch() 182 | 183 | Signal the nsqd server you want more time to process this message. 184 | 185 | _note:_ currently the binary protocol does not communicate anything back in the event of a successful `TOUCH` message. There's no way to have a callback for `message.touch()` at this time -- it's fire and forget. 186 | 187 | #### message.inFlight: bool 188 | 189 | Initially set to `true`. This will be set to `false` after calling `message.finish()` or `message.requeu()`. 190 | 191 | _note:_ it is currently considered an error for a client to respond to a message more than once, so calling `message.finish()` or `message.requeue()` more than once on a message will only send the `FIN` or `REQ` packet to the nsqd server __once__ for each message. If you absolutely must send `FIN` or `REQ` twice (which will usually cause the nsqueue client to emit an error) then you have to manually toggle `message.inFlight = true` before calling `message.finish()` or `message.requeue()` 192 | again. 193 | 194 | ```js 195 | client.on('message', function(msg) { 196 | console.log(msg.inFlight) //true 197 | var actuallyResponded = msg.finish() 198 | console.log(actuallyResponded) //true 199 | console.log(msg.inFlight) //false 200 | 201 | //lets try sending a `REQ` packet for this message 202 | var actuallyRespondedAgain = msg.requeue(1000) 203 | console.log(actuallyRespondedAgain) //false 204 | }) 205 | ``` 206 | 207 | ## testing 208 | 209 | Most of the test are functional style tests. They assume a running instance of nsqd on `localhost:4150`. Once you have nsqd reachable at `localhost:4150` run the tests by typing `mocha` at the project root. 210 | 211 | ## contributions 212 | 213 | I love contributions! Fork & send pull requests please! After a few pull requests I can add you as a contributor with push & pull acess if you're interested. If you find any problems or want to undertake more advanced/crazy refactorings please feel free to open an issue and we can discuss. 214 | 215 | ## LICENSE 216 | 217 | Copyright (c) 2014 Brian Carlson (brian.m.carlson@gmail.com) 218 | 219 | Permission is hereby granted, free of charge, to any person obtaining a copy 220 | of this software and associated documentation files (the "Software"), to deal 221 | in the Software without restriction, including without limitation the rights 222 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 223 | copies of the Software, and to permit persons to whom the Software is 224 | furnished to do so, subject to the following conditions: 225 | 226 | The above copyright notice and this permission notice shall be included in 227 | all copies or substantial portions of the Software. 228 | 229 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 230 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 231 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 232 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 233 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 234 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 235 | THE SOFTWARE. 236 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var connect = module.exports = function(options) { 2 | return new module.exports.Client(options) 3 | } 4 | 5 | //export direct Client & Connection constructors 6 | module.exports.Connection = require('./lib/connection') 7 | module.exports.Client = require('./lib/client') 8 | -------------------------------------------------------------------------------- /lib/client.js: -------------------------------------------------------------------------------- 1 | var EventEmitter = require('events').EventEmitter 2 | var util = require('util') 3 | var Connection = require('./connection') 4 | 5 | var Client = module.exports = function(options) { 6 | if(!(this instanceof Client)) return new Client(options); 7 | EventEmitter.call(this) 8 | this.con = new Connection(options) 9 | var self = this 10 | this.concurrency = 1 11 | this.con.on('message', this._handleIncommingMessage.bind(this)) 12 | 13 | //only emit an error if we are the only person 14 | //listening for connection errors - not waiting 15 | //for a possible error from a callback 16 | this.con.on('error', function(err) { 17 | if(self.con.listeners('error').length == 1) { 18 | self.emit('error', err) 19 | } 20 | }) 21 | } 22 | 23 | util.inherits(Client, EventEmitter) 24 | 25 | Client.prototype._handleIncommingMessage = function(msg) { 26 | msg.setClient(this) 27 | this.emit('message', msg) 28 | } 29 | 30 | Client.prototype.connect = function(cb) { 31 | this.con.connect(cb) 32 | } 33 | 34 | var bufferize = function(message) { 35 | if(typeof message == 'object') { 36 | if(message instanceof Buffer) { 37 | return message 38 | } 39 | return Buffer(JSON.stringify(message), 'utf8') 40 | } 41 | return Buffer(message, 'utf8') 42 | } 43 | 44 | var maybeCallback = function(con, cb) { 45 | if(cb) { 46 | //absorb any error response 47 | con.once('error', cb) 48 | con.once('response', function(res) { 49 | con.removeListener('error', cb) 50 | cb(null, res) 51 | }) 52 | } 53 | } 54 | 55 | Client.prototype.publish = function(topic, message, cb) { 56 | this.con.PUB(topic, bufferize(message)) 57 | maybeCallback(this.con, cb) 58 | } 59 | 60 | Client.prototype.publishAll = function(topic, messages, cb) { 61 | this.con.MPUB(topic, messages.map(bufferize)) 62 | maybeCallback(this.con, cb) 63 | } 64 | 65 | Client.prototype.subscribe = function(topic, channel, cb) { 66 | //subscribe to a topic+channel & set the initial 67 | //READY counter to the concurrency level 68 | this.con.SUB(topic, channel, cb) 69 | this.con.RDY(this.concurrency) 70 | maybeCallback(this.con, cb) 71 | } 72 | 73 | Client.prototype.finish = function(message) { 74 | this.con.FIN(message.id) 75 | this.con.RDY(this.concurrency) 76 | } 77 | 78 | Client.prototype.requeue = function(message, delay) { 79 | this.con.REQ(message.id, delay) 80 | this.con.RDY(this.concurrency) 81 | } 82 | 83 | Client.prototype.touch = function(message) { 84 | this.con.TOUCH(message.id) 85 | } 86 | 87 | Client.prototype.identify = function(options, cb) { 88 | if(options.tls) { 89 | var tls = options.tls 90 | delete options.tls 91 | } 92 | this.con.IDENTIFY(options, cb) 93 | var con = this.con 94 | return maybeCallback(con, function(err, res) { 95 | if(err) return cb(err); 96 | try { 97 | var data = JSON.parse(res) 98 | } catch(e) {} 99 | if(!tls) return cb(null, data); 100 | //there are tls options. initiate upgrade 101 | con.upgradeToTLS(tls, function(err) { 102 | if(err) return cb(err); 103 | 104 | //nsqd responds with OK 105 | maybeCallback(con, function(err, res) { 106 | if(err) return cb(err); 107 | cb(null, data) 108 | }) 109 | }) 110 | }) 111 | } 112 | 113 | Client.prototype.end = function(cb) { 114 | this.con.end(cb) 115 | } 116 | -------------------------------------------------------------------------------- /lib/connection.js: -------------------------------------------------------------------------------- 1 | var net = require('net') 2 | var EventEmitter = require('events').EventEmitter 3 | var util = require('util') 4 | 5 | var Reader = require('packet-reader') 6 | 7 | var Message = require('./message') 8 | 9 | // low level, direct binary protocol 10 | var Connection = module.exports = function(options, cb) { 11 | EventEmitter.call(this) 12 | this.reader = new Reader() 13 | this.lastCommad = null 14 | this.stream = null 15 | this.options = options 16 | this.secure = options.secure 17 | } 18 | 19 | util.inherits(Connection, EventEmitter) 20 | 21 | //connect to the given host/port 22 | //and handle any connection errors 23 | //calls callback with error argument if there 24 | //was a connection error, or null if connection 25 | //was a success 26 | Connection.prototype.connect = function(cb) { 27 | var self = this 28 | var connectError = function(e) { 29 | if(cb) { 30 | return cb(e) 31 | } 32 | self.emit('error', e) 33 | } 34 | this.stream = net.connect(this.options, function() { 35 | self.stream.on('error', function(err) { 36 | self.emit('error', err) 37 | }) 38 | self.stream.removeListener('error', connectError) 39 | self._writeText(' V2') 40 | if(cb) { 41 | cb(null, self) 42 | } 43 | }) 44 | this.stream.on('error', connectError) 45 | this.stream.on('data', this.parse.bind(this)) 46 | } 47 | 48 | //parse raw bytes off the stream 49 | Connection.prototype.parse = function(buffer) { 50 | this.reader.addChunk(buffer) 51 | var packet; 52 | while(packet = this.reader.read()) { 53 | var frameId = packet.readInt32BE(0) 54 | this['_handleTypeResponse' + frameId](packet.slice(4)) 55 | } 56 | } 57 | 58 | Connection.prototype._handleTypeResponse0 = function(message) { 59 | if(message == '_heartbeat_') { 60 | return this.NOP() 61 | } 62 | return this.emit('response', message) 63 | } 64 | 65 | Connection.prototype._handleTypeResponse1 = function(packet) { 66 | this.emit('error', new Error(packet.toString('ascii'))) 67 | } 68 | 69 | Connection.prototype._handleTypeResponse2 = function(packet) { 70 | this.emit('message', new Message(packet)) 71 | } 72 | 73 | Connection.prototype._writeText = function(text) { 74 | this.stream.write(text, 'ascii') 75 | } 76 | 77 | //sends the SUB (subscribe) command 78 | Connection.prototype.SUB = function(topic, channel) { 79 | var cmdText = 'SUB ' + topic + ' ' + channel + '\n' 80 | this._writeText(cmdText) 81 | } 82 | 83 | //sends the RDY (ready) command 84 | Connection.prototype.RDY = function(count) { 85 | this._writeText('RDY ' + count + '\n') 86 | } 87 | 88 | //sends the NOP command 89 | Connection.prototype.NOP = function() { 90 | this._writeText('NOP\n') 91 | } 92 | 93 | //sends the FIN (finish) command 94 | Connection.prototype.FIN = function(messageId) { 95 | this._writeText('FIN ' + messageId + '\n') 96 | } 97 | 98 | var sizeBuffer = function(size) { 99 | var buff = new Buffer(4) 100 | buff.writeInt32BE(size, 0) 101 | return buff 102 | } 103 | 104 | //sends the PUB (publish) command 105 | Connection.prototype.PUB = function(topic, buffer) { 106 | this._writeText('PUB ' + topic + '\n') 107 | this.stream.write(sizeBuffer(buffer.length)) 108 | this.stream.write(buffer) 109 | } 110 | 111 | //sends the MPUB (publish multiple) command 112 | Connection.prototype.MPUB = function(topic, buffers) { 113 | this._writeText('MPUB ' + topic + '\n') 114 | var bodySize = buffers.reduce(function(val, buff) { 115 | return val + buff.length + 4 116 | }, 4 + 4) 117 | this.stream.write(sizeBuffer(bodySize)) 118 | this.stream.write(sizeBuffer(buffers.length)) 119 | var self = this 120 | buffers.forEach(function(buff) { 121 | self.stream.write(sizeBuffer(buff.length)) 122 | self.stream.write(buff) 123 | }) 124 | } 125 | 126 | //sends the REQ (requeue) command 127 | Connection.prototype.REQ = function(messageId, timeout) { 128 | this._writeText('REQ ' + messageId + ' ' + timeout + '\n') 129 | } 130 | 131 | //sends the TOUCH command 132 | Connection.prototype.TOUCH = function(messageId) { 133 | this._writeText('TOUCH ' + messageId + '\n') 134 | } 135 | 136 | //sends the IDENTIFY length prefixed json 137 | Connection.prototype.IDENTIFY = function(options, cb) { 138 | this._writeText('IDENTIFY\n') 139 | var body = Buffer(JSON.stringify(options), 'utf8') 140 | this.stream.write(sizeBuffer(body.length)) 141 | this.stream.write(body) 142 | } 143 | 144 | Connection.prototype.upgradeToTLS = function(options, callback) { 145 | var tls = require('tls') 146 | console.log('upgrading') 147 | 148 | this.stream.removeAllListeners('data') 149 | 150 | options.socket = this.stream 151 | var self = this 152 | var tlsstream = tls.connect(options, function(err) { 153 | if(err) return callback(error) 154 | console.log('upgraded') 155 | //self.stream.removeAllListeners() 156 | self.stream = tlsstream 157 | tlsstream.on('data', self.parse.bind(self)) 158 | callback() 159 | }) 160 | tlsstream.on('error', function(e) { 161 | self.emit(e) 162 | }) 163 | } 164 | 165 | //sends the CLS (close) command 166 | Connection.prototype.CLS = function() { 167 | this._writeText('CLS\n') 168 | } 169 | 170 | //handles the close logic 171 | //calls the callback when 172 | //an all-clear message is received 173 | Connection.prototype.end = function(cb) { 174 | this.CLS() 175 | if(cb) { 176 | var done = function(msg) { 177 | if(msg == 'CLOSE_WAIT') { 178 | this.removeListener('response', done) 179 | cb() 180 | } 181 | } 182 | this.on('response', done) 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /lib/message.js: -------------------------------------------------------------------------------- 1 | var Message = module.exports = function(buffer) { 2 | this.timestamp = buffer.readInt32BE(0) + '' + buffer.readInt32BE(4) 3 | this.attempts = buffer.readInt16BE(8) 4 | this.id = buffer.toString('ascii', 10, 26) 5 | this.data = buffer.slice(26) 6 | this.client = null 7 | this.inFlight = true 8 | } 9 | 10 | //sets the client to use for communcation 11 | Message.prototype.setClient = function(client) { 12 | this.client = client 13 | } 14 | 15 | Message.prototype.finish = function() { 16 | if(!this.inFlight) return false; 17 | this.inFlight = false 18 | this.client.finish(this) 19 | return true 20 | } 21 | 22 | Message.prototype.requeue = function(delay) { 23 | if(!this.inFlight) return false; 24 | this.inFlight = false 25 | this.client.requeue(this, delay) 26 | return true 27 | } 28 | 29 | Message.prototype.touch = function() { 30 | this.client.touch(this) 31 | } 32 | 33 | Message.prototype.json = function() { 34 | return JSON.parse(this.data.toString('utf8')) 35 | } 36 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nsqueue", 3 | "version": "0.6.0", 4 | "description": "node.js client for nsq", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "make test-linux" 8 | }, 9 | "author": "Brian M. Carlson", 10 | "license": "MIT", 11 | "devDependencies": { 12 | "mocha": "~1.17.0", 13 | "okay": "~0.3.0", 14 | "request": "~2.33.0" 15 | }, 16 | "dependencies": { 17 | "packet-reader": "0.3.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | set -e 4 | 5 | rm -r nsq || true 6 | mkdir nsq 7 | echo "extracting nsqd" 8 | tar -xzvf nsq.tar.gz -C nsq --strip-components=1 **/bin/nsqd 9 | echo "generating key" 10 | openssl req -x509 -newkey rsa:2048 -keyout nsq/key.pem -out nsq/cert.pem -days 365 -nodes -subj "/C=US/ST=Denial/L=Springfield/O=Dis/CN=localhost" 11 | echo "starting nsq" 12 | cd nsq 13 | bin/nsqd -tls-cert=cert.pem -tls-key=key.pem & 14 | cd .. 15 | node_modules/.bin/mocha --bail 16 | kill $$! || killall nsqd 17 | rm -r nsq 18 | rm nsq.tar.gz 19 | -------------------------------------------------------------------------------- /test/client.js: -------------------------------------------------------------------------------- 1 | //test high level client 2 | var Client = require('../').Client 3 | var helper = require('./helper') 4 | var assert = require('assert') 5 | var ok = require('okay') 6 | var request = require('request') 7 | 8 | describe('client', function() { 9 | var topic = helper.client() 10 | 11 | it('can publish buffer', function(done) { 12 | this.client.publish(topic, Buffer('hi', 'ascii'), done) 13 | }) 14 | 15 | it('can publish text', function(done) { 16 | this.client.publish(topic, 'briân', done) 17 | }) 18 | 19 | it('can publish json', function(done) { 20 | this.client.publish(topic, {name: 'Brian'}, done) 21 | }) 22 | 23 | it('can consume first message', function(done) { 24 | this.client.subscribe(topic, 'test') 25 | var self = this 26 | this.client.once('message', function(msg) { 27 | assert.equal(msg.data.toString('ascii'), 'hi') 28 | self.lastMessage = msg 29 | done() 30 | }) 31 | }) 32 | 33 | it('can consume utf8 text message', function(done) { 34 | this.lastMessage.finish() 35 | var self = this 36 | this.client.once('message', function(msg) { 37 | assert.equal(msg.data.toString('utf8'), 'briân') 38 | self.lastMessage = msg 39 | done() 40 | }) 41 | }) 42 | 43 | it('can consume JSON message', function(done) { 44 | this.lastMessage.finish() 45 | this.client.once('message', function(msg) { 46 | assert.equal(JSON.parse(msg.data.toString('utf8')).name, 'Brian') 47 | assert.equal(msg.data.toString('utf8'), JSON.stringify(msg.json())) 48 | assert.equal(msg.json().name, 'Brian') 49 | done() 50 | }) 51 | }) 52 | 53 | describe('topic error tests', function() { 54 | var topic = helper.client() 55 | it('will handle publish error due to queue name in callback', function(done) { 56 | this.client.publish('', {test: true}, function(err) { 57 | assert(err) 58 | done() 59 | }) 60 | }) 61 | }) 62 | 63 | describe('message error tests', function() { 64 | var topic = helper.client() 65 | it('will handle publish error due to null message in callback', function(done) { 66 | this.client.publish(topic, Buffer(0), function(err) { 67 | assert(err) 68 | done() 69 | }) 70 | }) 71 | }) 72 | }) 73 | 74 | describe('client publish multiple', function() { 75 | var topic = helper.client() 76 | 77 | it('publishes', function(done) { 78 | var messages = [ 79 | Buffer('one', 'ascii'), 80 | 'twœ', 81 | {number: 3} 82 | ] 83 | this.client.publishAll(topic, messages, done) 84 | }) 85 | 86 | it('publishes all the messages', function(done) { 87 | helper.stats(topic, ok(done, function(stat) { 88 | assert.equal(stat.depth, 3) 89 | done() 90 | })) 91 | }) 92 | 93 | it('receives multiple', function(done) { 94 | var count = 0 95 | this.client.concurrency = 2 96 | this.client.subscribe(topic, 'test') 97 | this.client.on('message', function(msg) { 98 | msg.finish() 99 | if(++count >= 3) done() 100 | }) 101 | }) 102 | }) 103 | 104 | describe('connection callback', function() { 105 | var topic = 'test-topic-' + Date.now() 106 | 107 | after(function(done) { 108 | request.get('http://localhost:4151/delete_topic?topic=' + topic, done) 109 | }) 110 | 111 | it('is called once only', function(done) { 112 | var client = this.client = new Client(helper.options()) 113 | var callCount = 0 114 | client.connect(function() { 115 | assert.equal(callCount++, 0) 116 | client.publishAll(topic, ['test', 'test', 'test'], function() { 117 | client.publishAll(topic, ['test', 'test', 'test'], function() { 118 | setTimeout(done, 1000) 119 | }) 120 | }) 121 | }) 122 | }) 123 | }) 124 | -------------------------------------------------------------------------------- /test/connecting.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var nsqueue = require('../') 3 | var helper = require('./helper') 4 | var ok = require('okay') 5 | 6 | describe('connection', function() { 7 | it('calls back on success', function(done) { 8 | nsqueue(helper.options()).connect(done) 9 | }) 10 | 11 | it('connects from helper', function(done) { 12 | helper.connect(done) 13 | }) 14 | }) 15 | 16 | describe('connection without callback', function() { 17 | //need to split out end() and close() 18 | //end() - end the socket hard 19 | //close() - send CLS and keep socket open 20 | //also use --no-exit flag 21 | it('works', false, function(done) { 22 | var client = nsqueue(helper.options()) 23 | client.connect() 24 | setTimeout(function() { 25 | client.end() 26 | }, 100) 27 | }) 28 | 29 | it('emits error', function(done) { 30 | var client = nsqueue({host: 'asdfalsdkfjsdf', port: 1}) 31 | client.connect() 32 | client.on('error', function(e) { 33 | assert(e, 'should have received an error argument on connection failure') 34 | done() 35 | }) 36 | }) 37 | }) 38 | 39 | describe('disconnection', function() { 40 | var topic = helper.connection() 41 | 42 | it('disconnects', function(done) { 43 | var connection = this.connection 44 | connection.SUB(topic, 'test') 45 | connection.once('response', function() { 46 | connection.end(done) 47 | }) 48 | }) 49 | }) 50 | 51 | describe('ending client', function() { 52 | var topic = helper.client() 53 | it('can disconnect', function(done) { 54 | var client = this.client 55 | client.subscribe(topic, 'test', function() { 56 | client.end(done) 57 | }) 58 | }) 59 | }) 60 | -------------------------------------------------------------------------------- /test/helper.js: -------------------------------------------------------------------------------- 1 | var nsqueue = require('../') 2 | var request = require('request') 3 | var ok = require('okay') 4 | var Client = require('../lib/client') 5 | 6 | var helper = module.exports = { 7 | options: function() { 8 | return { 9 | host: 'localhost', 10 | port: 4150 11 | } 12 | }, 13 | connect: function(cb) { 14 | var con = new nsqueue.Connection(module.exports.options()) 15 | con.connect(function(err) { 16 | cb(err, con) 17 | }) 18 | return con 19 | }, 20 | publish: function(topic, message, cb) { 21 | var options = { 22 | url: 'http://localhost:4151/pub?topic=' + topic, 23 | body: message 24 | } 25 | request.post(options, cb) 26 | }, 27 | client: function(config) { 28 | var topic = 'test-topic-' + Date.now() 29 | before(function(done) { 30 | var options = helper.options() 31 | if(config) { 32 | for(var key in config) { 33 | options[key] = config[key] 34 | } 35 | } 36 | var client = this.client = new Client(options) 37 | client.connect(done) 38 | }) 39 | 40 | after(function(done) { 41 | request.get('http://localhost:4151/delete_topic?topic=' + topic, done) 42 | }) 43 | 44 | return topic 45 | }, 46 | connection: function() { 47 | before(function(done) { 48 | this.connection = helper.connect(done) 49 | }) 50 | 51 | var topic = 'test-topic-' + Date.now() 52 | 53 | after(function(done) { 54 | request.get('http://localhost:4151/delete_topic?topic=' + topic, done) 55 | }) 56 | 57 | return topic 58 | }, 59 | stats: function(topicName, cb) { 60 | var options = { 61 | url: 'http://localhost:4151/stats?format=json', 62 | json: true 63 | } 64 | request.get(options, ok(cb, function(res, body) { 65 | for(var i = 0; i < body.data.topics.length; i++) { 66 | var topic = body.data.topics[i] 67 | if(topic.topic_name == topicName) { 68 | return cb(null, topic) 69 | } 70 | } 71 | cb(null, null) 72 | })) 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /test/identify-client.js: -------------------------------------------------------------------------------- 1 | var helper = require('./helper') 2 | var ok = require('okay') 3 | var assert = require('assert') 4 | 5 | describe('client.identify', function() { 6 | var topic = helper.client() 7 | 8 | it('works', function(done) { 9 | var options = { 10 | short_id: 'hi', 11 | long_id: 'hello!', 12 | feature_negotiation: true, 13 | heartbeat_interval: -1 14 | } 15 | this.client.identify(options, function(err, res) { 16 | assert.ifError(err) 17 | assert(res) 18 | assert.equal(res.msg_timeout, 60000) 19 | assert.equal(res.deflate, false) 20 | assert.equal(res.snappy, false) 21 | done() 22 | }) 23 | }) 24 | 25 | it('responds with nothing when no feature negotation is given', function(done) { 26 | this.client.identify({short_id: 'hi'}, done) 27 | }) 28 | 29 | it('responds with error', function(done) { 30 | var options = { 31 | feature_negotiation: true, 32 | snappy: true, 33 | deflate: true 34 | } 35 | this.client.identify(options, function(err) { 36 | assert(err) 37 | done() 38 | }) 39 | }) 40 | }) 41 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | var assert = require('assert') 2 | var nsqueue = require('../') 3 | var helper = require('./helper') 4 | var ok = require('okay') 5 | 6 | 7 | describe('stream errors', function() { 8 | describe('during connection', function() { 9 | 10 | it('calls back with error on problem', function(done) { 11 | var options = { 12 | host: 'laksjdflkajsfd', 13 | port: 88381 14 | } 15 | nsqueue(options).connect(function(err) { 16 | assert(err) 17 | done() 18 | }) 19 | }) 20 | 21 | }) 22 | 23 | describe('after connection', function() { 24 | var options = { 25 | host: 'localhost', 26 | port: 4150 27 | } 28 | it('is emitted by connection', function(done) { 29 | helper.connect(ok(done, function(connection) { 30 | connection.on('error', function(err) { 31 | assert.equal(err.message, 'test') 32 | done() 33 | }) 34 | connection.stream.emit('error', new Error('test')) 35 | })) 36 | }) 37 | }) 38 | }) 39 | 40 | describe('subscribe', function() { 41 | var topic = helper.connection() 42 | 43 | it('emits response', function(done) { 44 | this.connection.SUB(topic, 'channel') 45 | this.connection.on('response', function(res) { 46 | assert.equal(res, 'OK') 47 | done() 48 | }) 49 | }) 50 | 51 | }) 52 | 53 | describe('subscribe error', function() { 54 | var topic = helper.connection() 55 | it('returns error if topic is invalid', function(done) { 56 | this.connection.SUB('Hello!!!!', 'brooklyn') 57 | this.connection.on('error', function(err) { 58 | assert(err.message.indexOf('E_BAD_TOPIC') > -1) 59 | done() 60 | }) 61 | }) 62 | }) 63 | 64 | describe('getting a message', function() { 65 | var topic = helper.connection() 66 | 67 | it('works', function(done) { 68 | var connection = this.connection 69 | helper.publish(topic, 'hi', ok(done, function() { 70 | connection.SUB(topic, 'test') 71 | connection.RDY(10) 72 | connection.on('message', function(msg) { 73 | assert(msg.timestamp) 74 | assert.equal(msg.attempts, 1) 75 | assert.equal(msg.data.toString('utf8'), 'hi') 76 | done() 77 | }) 78 | })) 79 | }) 80 | }) 81 | 82 | describe('publishing a message', function() { 83 | var topic = helper.connection() 84 | 85 | it('works', function(done) { 86 | var connection = this.connection 87 | connection.PUB(topic, Buffer('yo', 'ascii')) 88 | connection.on('response', function(res) { 89 | assert.equal(res, 'OK') 90 | done() 91 | }) 92 | }) 93 | 94 | it('emits error on invalid body', function(done) { 95 | var connection = this.connection 96 | connection.PUB(topic, Buffer(0)) 97 | connection.on('error', function(err) { 98 | assert(err.message.indexOf('BAD_MESSAGE') >= 0) 99 | done() 100 | }) 101 | }) 102 | }) 103 | 104 | describe('publishing multiple messages', function() { 105 | var topic = helper.connection() 106 | it('works', function(done) { 107 | var buffers = [ 108 | Buffer('one', 'ascii'), 109 | Buffer('two', 'ascii') 110 | ] 111 | this.connection.MPUB(topic, buffers) 112 | this.connection.on('response', function(res) { 113 | assert.equal(res, 'OK') 114 | done() 115 | }) 116 | }) 117 | 118 | it('adds two messages', function(done) { 119 | helper.stats(topic, ok(done, function(topic) { 120 | assert.equal(topic.message_count, 2) 121 | done() 122 | })) 123 | }) 124 | }) 125 | 126 | describe('message life-cycle', function() { 127 | var topic = helper.connection() 128 | it('enqueues and dequeues', function(done) { 129 | var connection = this.connection 130 | connection.PUB(topic, Buffer('1', 'ascii')) 131 | connection.PUB(topic, Buffer('2', 'ascii')) 132 | setTimeout(function() { 133 | connection.PUB(topic, Buffer('3', 'ascii')) 134 | }, 50) 135 | connection.SUB(topic, 'test') 136 | connection.RDY(100) 137 | var count = 0 138 | var sum = 0 139 | connection.on('message', function(msg) { 140 | connection.FIN(msg.id) 141 | sum += parseInt(msg.data.toString('ascii')) 142 | if(++count === 3) { 143 | helper.stats(topic, function(err, topic) { 144 | assert.equal(topic.channels[0].clients[0].finish_count, 3) 145 | assert.equal(topic.message_count, 3) 146 | assert.equal(topic.depth, 0) 147 | done() 148 | }) 149 | } 150 | }) 151 | }) 152 | }) 153 | -------------------------------------------------------------------------------- /test/message.js: -------------------------------------------------------------------------------- 1 | //test message behaviors 2 | var Client = require('../').Client 3 | var helper = require('./helper') 4 | var assert = require('assert') 5 | var ok = require('okay') 6 | 7 | describe('message', function() { 8 | var topic = helper.client() 9 | 10 | it('can be requeued', function(done) { 11 | var client = this.client 12 | client.publish(topic, Buffer('hi', 'ascii'), ok(done, function(){})) 13 | client.subscribe(topic, 'test', ok(done, function() { 14 | client.once('message', function(msg) { 15 | assert.strictEqual(msg.inFlight, true) 16 | msg.requeue(100) 17 | assert.strictEqual(msg.inFlight, false) 18 | client.once('message', function(msg) { 19 | assert.strictEqual(msg.inFlight, true) 20 | assert.equal(msg.data.toString(), 'hi') 21 | msg.requeue(100) 22 | done() 23 | }) 24 | }) 25 | })) 26 | }) 27 | 28 | describe('responding', function() { 29 | var topic = helper.client() 30 | it('will indicate it has already been responeded to', function(done) { 31 | var client = this.client 32 | client.subscribe(topic, 'test') 33 | client.on('message', function(msg) { 34 | assert.equal(msg.data.toString(), 'test') 35 | msg.finish() 36 | assert.strictEqual(msg.inFlight, false, 'msg should have inFlight === false') 37 | msg.inFlight = true 38 | msg.finish() 39 | client.once('error', function(err) { 40 | assert(err.message.indexOf('E_FIN_FAILED') > -1, 'Error should contain E_FIN_FAILED message') 41 | done() 42 | }) 43 | }) 44 | client.publish(topic, 'test') 45 | }) 46 | }) 47 | 48 | 49 | //this test depends on the test above it 50 | //to have already created a message 51 | //and subscribed to the topic 52 | it('can be touched', function(done) { 53 | var client = this.client 54 | client.once('message', function(msg) { 55 | msg.touch() 56 | assert.strictEqual(msg.inFlight, true) 57 | done() 58 | }) 59 | }) 60 | 61 | }) 62 | 63 | describe('message', function() { 64 | var topic = helper.client() 65 | 66 | before(function(done) { 67 | var client = this.client 68 | client.subscribe(topic, 'test') 69 | client.publish(topic, 'yo', done) 70 | }) 71 | 72 | it('can be requeued multiple times without error', function(done) { 73 | this.timeout(10000) 74 | this.client.once('message', function(msg) { 75 | assert.equal(msg.inFlight, true) 76 | assert.equal(msg.requeue(100), true) 77 | assert.equal(msg.inFlight, false) 78 | assert.equal(msg.requeue(100), false) 79 | assert.equal(msg.inFlight, false) 80 | assert.equal(msg.requeue(100), false) 81 | assert.equal(msg.inFlight, false) 82 | done() 83 | }) 84 | }) 85 | 86 | it('can be finished multiple times', function(done) { 87 | this.client.once('message', function(msg) { 88 | assert.equal(msg.inFlight, true) 89 | assert.equal(msg.finish(), true) 90 | assert.equal(msg.inFlight, false) 91 | assert.equal(msg.finish(), false) 92 | assert.equal(msg.inFlight, false) 93 | assert.equal(msg.finish(), false) 94 | assert.equal(msg.inFlight, false) 95 | done() 96 | }) 97 | }) 98 | }) 99 | 100 | -------------------------------------------------------------------------------- /test/stress.js: -------------------------------------------------------------------------------- 1 | var helper = require('./helper') 2 | var ok = require('okay') 3 | describe('one thousand messages', function() { 4 | 5 | var testConcurrency = function(total, level) { 6 | describe(total + ' messages with concurrency level ' + level, function() { 7 | var topic = helper.client() 8 | it('queues all messages in one batch', function(done) { 9 | var messages = [] 10 | for(var j = 0; j < total; j++) { 11 | messages.push({number: 1}) 12 | } 13 | this.client.publishAll(topic, messages, done) 14 | }) 15 | 16 | it('reads ' + total + ' messages in chunks of ' + level, function(done) { 17 | this.client.concurrency = level 18 | this.client.subscribe(topic, 'test') 19 | var count = 0 20 | this.client.on('message', function(msg) { 21 | msg.finish() 22 | count += msg.json().number 23 | if(count >= total) { 24 | done() 25 | } 26 | }) 27 | }) 28 | }) 29 | } 30 | 31 | testConcurrency(1000, 100) 32 | testConcurrency(1000, 1000) 33 | testConcurrency(1000, 3) 34 | testConcurrency(100, 1) 35 | testConcurrency(44, 3) 36 | testConcurrency(10, 100) 37 | }) 38 | -------------------------------------------------------------------------------- /test/subscribe-callback.js: -------------------------------------------------------------------------------- 1 | var helper = require('./helper') 2 | describe('subscribe callback', function() { 3 | var topic = helper.client() 4 | it('works', function(done) { 5 | this.client.subscribe(topic, 'test', done) 6 | }) 7 | }) 8 | 9 | describe('subscribe and publish', function() { 10 | var topic = helper.client() 11 | 12 | it('works after publish callback', function(done) { 13 | var client = this.client 14 | client.publish(topic, 'test', function(err) { 15 | if(err) return done(err); 16 | client.subscribe(topic, 'test', done) 17 | }) 18 | }) 19 | }) 20 | -------------------------------------------------------------------------------- /test/tls.js: -------------------------------------------------------------------------------- 1 | //test high level client 2 | var Client = require('../').Client 3 | var helper = require('./helper') 4 | var assert = require('assert') 5 | var ok = require('okay') 6 | var request = require('request') 7 | var fs = require('fs') 8 | 9 | describe('client', function() { 10 | var topic = helper.client() 11 | 12 | it('works', function(done) { 13 | this.timeout(5000) 14 | var options = { 15 | feature_negotiation: true, 16 | tls_v1: true, 17 | tls: { 18 | secureProtocol: 'TLSv1_method', 19 | //rejectUnauthorized: false, 20 | //cert: fs.readFileSync(__dirname + '/../nsq/cert.pem'), 21 | //key: fs.readFileSync(__dirname + '/../nsq/key.pem'), 22 | ca: [fs.readFileSync(__dirname + '/../nsq/cert.pem')] 23 | } 24 | } 25 | var client = this.client 26 | this.client.identify(options, ok(done, function(res) { 27 | assert.equal(res.tls_v1, true) 28 | client.publish('test', 'ok', done) 29 | })) 30 | }) 31 | }) 32 | --------------------------------------------------------------------------------