├── CNAME ├── _config.yml ├── .npmignore ├── .gitignore ├── lib ├── version.js ├── debug.js ├── nsq.js ├── roundrobinlist.js ├── backofftimer.js ├── framebuffer.js ├── lookupd.js ├── message.js ├── writer.js ├── reader.js ├── wire.js ├── config.js ├── readerrdy.js └── nsqdconnection.js ├── .husky └── pre-commit ├── .github ├── dependabot.yml └── workflows │ ├── release-please.yml │ ├── main.yaml │ └── codeql-analysis.yml ├── .prettierrc ├── test ├── rawmessage.js ├── key.pem ├── cert.pem ├── backofftimer_test.js ├── message_test.js ├── roundrobinlist_test.js ├── reader_test.js ├── framebuffer_test.js ├── wire_test.js ├── writer_test.js ├── lookupd_test.js ├── config_test.js ├── nsqdconnection_test.js ├── z_integration_test.js └── readerrdy_test.js ├── .eslintrc.js ├── LICENSE-MIT ├── package.json └── README.md /CNAME: -------------------------------------------------------------------------------- 1 | nsqjs.com -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-tactile -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .Python 2 | .template 3 | bin 4 | node_modules/ 5 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | .envrc 4 | .idea -------------------------------------------------------------------------------- /lib/version.js: -------------------------------------------------------------------------------- 1 | module.exports = require('../package.json').version 2 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npm run lint 5 | npm test 6 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "npm" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "trailingComma": "es5", 3 | "tabWidth": 2, 4 | "semi": false, 5 | "singleQuote": true, 6 | "bracketSpacing": false 7 | } 8 | -------------------------------------------------------------------------------- /lib/debug.js: -------------------------------------------------------------------------------- 1 | try { 2 | const debug = require('debug') 3 | module.exports = debug 4 | } catch (e) { 5 | module.exports = () => { 6 | return () => {} 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /lib/nsq.js: -------------------------------------------------------------------------------- 1 | const {NSQDConnection, WriterNSQDConnection} = require('./nsqdconnection') 2 | const Message = require('./message') 3 | const Reader = require('./reader') 4 | const Writer = require('./writer') 5 | 6 | module.exports = { 7 | Message, 8 | Reader, 9 | Writer, 10 | NSQDConnection, 11 | WriterNSQDConnection, 12 | } 13 | -------------------------------------------------------------------------------- /.github/workflows/release-please.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | name: release-please 6 | jobs: 7 | release-please: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: google-github-actions/release-please-action@v3 11 | with: 12 | release-type: node 13 | package-name: release-please-action 14 | -------------------------------------------------------------------------------- /test/rawmessage.js: -------------------------------------------------------------------------------- 1 | function rawMessage(id, timestamp, attempts, body) { 2 | // Create the raw NSQ message 3 | id = Buffer.from(id) 4 | body = Buffer.from(body) 5 | 6 | const b = Buffer.alloc(8 + 2 + 16 + body.length) 7 | b.writeBigInt64BE(BigInt(timestamp), 0) 8 | b.writeInt16BE(attempts, 8) 9 | b.copy(id, 10, 0, 16) 10 | b.copy(body, 26, 0, body.length) 11 | 12 | return b 13 | } 14 | 15 | module.exports = rawMessage 16 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | env: { 3 | es2020: true, 4 | node: true, 5 | }, 6 | extends: ['eslint:recommended'], 7 | parserOptions: { 8 | ecmaVersion: 10, 9 | sourceType: 'module', 10 | }, 11 | plugins: [], 12 | rules: {}, 13 | settings: {}, 14 | globals: { 15 | after: false, 16 | afterEach: false, 17 | before: false, 18 | browser: false, 19 | describe: false, 20 | beforeEach: false, 21 | it: false, 22 | }, 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | jobs: 11 | test: 12 | runs-on: ubuntu-latest 13 | timeout-minutes: 10 14 | strategy: 15 | matrix: 16 | node: [18,20,21] 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - uses: actions/setup-node@v3 21 | with: 22 | node-version: ${{ matrix.node }} 23 | 24 | - name: "System dependencies" 25 | run: | 26 | sudo apt-get -y update 27 | sudo apt-get -y install libsnappy-dev python3 build-essential 28 | - name: "Install nsq" 29 | run: | 30 | NSQ_DIST="nsq-1.3.0.linux-amd64.go1.21.5" 31 | curl -sSL "https://bitly-downloads.s3.amazonaws.com/nsq/${NSQ_DIST}.tar.gz" \ 32 | | tar -xzv --strip-components=1 33 | - run: npm install 34 | - run: | 35 | export PATH=bin:$PATH 36 | npm test 37 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Dudley Carr 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nsqjs", 3 | "description": "NodeJS client for NSQ", 4 | "version": "0.13.0", 5 | "homepage": "https://github.com/dudleycarr/nsqjs", 6 | "author": { 7 | "name": "Dudley Carr", 8 | "email": "dudley.carr@gmail.com" 9 | }, 10 | "keywords": [ 11 | "nsq", 12 | "nsq client", 13 | "nsq client official", 14 | "nsqjs", 15 | "distributed messaging", 16 | "messaging", 17 | "task", 18 | "task management" 19 | ], 20 | "repository": { 21 | "type": "git", 22 | "url": "https://github.com/dudleycarr/nsqjs.git" 23 | }, 24 | "bugs": { 25 | "url": "https://github.com/dudleycarr/nsqjs/issues" 26 | }, 27 | "license": "MIT", 28 | "scripts": { 29 | "format": "prettier --write \"{lib,test}/**/*.js\"", 30 | "lint": "eslint lib test", 31 | "test": "mocha -R spec -b", 32 | "prepare": "husky install" 33 | }, 34 | "main": "lib/nsq", 35 | "engines": { 36 | "node": ">= 16.20.0" 37 | }, 38 | "devDependencies": { 39 | "async-retry": "^1.3.3", 40 | "debug": "^4.3.4", 41 | "eslint": "^8.46.0", 42 | "husky": "^9.0.11", 43 | "mocha": "^10.2.0", 44 | "nock": "^13.3.2", 45 | "p-event": "^4.2.0", 46 | "prettier": "^3.0.1", 47 | "should": "^13.2.3", 48 | "sinon": "^17.0.1", 49 | "temp": "^0.9.4" 50 | }, 51 | "dependencies": { 52 | "lodash": "^4.17.21", 53 | "node-fetch": "^2.6.12", 54 | "node-state": "~1.4.4" 55 | }, 56 | "optionalDependencies": { 57 | "snappystream": "^2.0.0" 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /lib/roundrobinlist.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Takes a list and cycles through the elements in the list repeatedly and 3 | * in-order. Adding and removing to the list does not perturb the order. 4 | * 5 | * Usage: 6 | * const list = RoundRobinList([1, 2, 3]); 7 | * list.next() ==> [1] 8 | * list.next(2) ==> [2, 3] 9 | * list.next(2) ==> [1, 2] 10 | * list.add(5) ==> 5 11 | * list.next(2) ==> [3, 5] 12 | */ 13 | class RoundRobinList { 14 | /** 15 | * Instantiate a new RoundRobinList. 16 | * 17 | * @param {Array} list 18 | */ 19 | constructor(list) { 20 | this.list = list.slice() 21 | this.index = 0 22 | } 23 | 24 | /** 25 | * Returns the length of the list. 26 | * 27 | * @return {Number} 28 | */ 29 | length() { 30 | return this.list.length 31 | } 32 | 33 | /** 34 | * Add an item to the list. 35 | * 36 | * @param {*} item 37 | * @return {*} The item added. 38 | */ 39 | add(item) { 40 | return this.list.push(item) 41 | } 42 | 43 | /** 44 | * Remove an item from the list. 45 | * 46 | * @param {*} item 47 | * @return {Array|undefined} 48 | */ 49 | remove(item) { 50 | const itemIndex = this.list.indexOf(item) 51 | if (itemIndex === -1) return 52 | 53 | if (this.index > itemIndex) { 54 | this.index -= 1 55 | } 56 | 57 | return this.list.splice(itemIndex, 1) 58 | } 59 | 60 | /** 61 | * Get the next items in the list, round robin style. 62 | * 63 | * @param {Number} [count=1] 64 | * @return {Array} 65 | */ 66 | next(count = 1) { 67 | const {index} = this 68 | this.index = (this.index + count) % this.list.length 69 | return this.list.slice(index, index + count) 70 | } 71 | } 72 | 73 | module.exports = RoundRobinList 74 | -------------------------------------------------------------------------------- /test/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIEpAIBAAKCAQEAy2b+JM1r0RNEACvzI61YFPUuN605WDYCn8CTJZygA5qQEdti 3 | 6SmT4oTP9MYx8Q3vslIPGUwg9XOpvJceFz0tqf0zgDUjxBSI4EWm840uiQonrOH/ 4 | TI1uK31SBCc4OAYqRUluGbwimGrGDlWHon05RQ/hTvBJO7IehkDNVqRuEuCPG29Z 5 | N7qbNHGEFpd03J7eTpU/oblJrehEQJuv1/pKY+1nTCnhB8RmQqu5l9jOKh2FWn0e 6 | zmSTvFWC7oEqKI+ZYDJ/kQnzsbVv7nWtmm1BXtd9TH7rRQbJsdUBlzjk4yXjCgds 7 | 2bG/XswMrQmnlLPR4rw67zscaohNEbZiP3RYRQIDAQABAoIBAQDH2tjYPGc1tWJZ 8 | cNWkNoyXexkAZ9oyjE8jvMpYaH7pS5NHmHKles7uAWV7ssobeoAMjIh9aMnxosYi 9 | obFVUC1wG3PhA1WzMiITixyxrgUNbcbyHdUF2OlpHefaYNbiZVxfW/ksnCi57h/Q 10 | scVlqPj+nM3bDEpIt4k6jK219jaZn/6Gxm3Cwx+h0kTUiDwl40EK1G+3YNO7Btjr 11 | hOBEE4miXnL5NMStYLLdIoxfXlu/D+kW1SkcdCUoum228EUA5xKa9yvQPKNSYSPt 12 | 5EjFzCc18AsQ2A65mFGv4o6/kzjA1qoL3Akffz+8ZVoxXxA18wSmsCSs6Zes/70K 13 | xjOOxGHBAoGBAPIVLVZvF7CKop4v8vbAy4LsSKYpibrS4FJ83hZzTXSqtxHTa0w3 14 | xnBKNpboYB7m/QBWD8CocMo38XMm8vDo7WTKjCYhK6E6F94/wJNFi5V2ehXFAefD 15 | Qxg7u28IRTnOGKfxjv5oH336qxtByNlzly00zoM4NwhQDO+0oqVjkQs9AoGBANcY 16 | ip4kfcqoFU1RLKjcaxldtqoSbxs7YzTvAtRxIp1sJntqWqUelzVbYwp1e0OnrWlg 17 | YVBvKRvEbiZW01u2Mwsw9WVHSNQGjLo+nuCE5vaBCz4BXZOzWdoG2xMgWlKwrg+q 18 | NHZA4Knu1qFTPGjws3TY2F8gx53MInKrnNYdlnGpAoGBAKWt8rKGI6typmsnG9zf 19 | BCmddvcp1JZlPpuV1BV/YlJniBWOIZAvWAN/9y5+6VIc+qihPfS4E0GC2h/aV0ij 20 | 0d06doKeDxMwBCfab/0bCrYHOPTvOSeaTrAmKViLS32NXPiL7TaWon8A8NwdwM16 21 | O8v3qSLLdXad+syPTwVsSFdtAoGABvpG1OZ+JirZtg5iEoBEASinGlFKxWlhyH/Q 22 | aa/Z3Z16ihpjImQhp5t1VQuGmiVAXODBh2hzvvDaWegLJzh742sNKuHrHOWxfwE1 23 | CjeFfo2lHbfRn078JTR/utkb1P7iRqPQ290y7JBQ6h7XINheGfZG3p8jmpzaqTmj 24 | 9tYy4TECgYBm+JdbfKNDh6W6XeUzHvsFzhxBFw2ysg8R8ZTArocH21XgwlWB7pg0 25 | UTedR89GCcHEABuB2RdeeqRTsZspybxDMPPbIRZfc44/oBiZ20pfaJHqmivIa8Km 26 | lUrC6yPneGq9iD78LKpAWagPAFPECXvheTHq2ayAcyQiZ9FPI/R4Qg== 27 | -----END RSA PRIVATE KEY----- 28 | -------------------------------------------------------------------------------- /test/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIEwTCCA6mgAwIBAgIJALU656z85FcbMA0GCSqGSIb3DQEBBQUAMIGbMQswCQYD 3 | VQQGEwJVUzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxITAfBgNVBAoT 4 | GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEkMCIGA1UEAxMbZ2l0aHViLmNvbS9k 5 | dWRsZXljYXJyL25zcWpzMSQwIgYJKoZIhvcNAQkBFhVkdWRsZXkuY2FyckBnbWFp 6 | bC5jb20wHhcNMTQwNzI4MTQ0NTIyWhcNMTUwNzI4MTQ0NTIyWjCBmzELMAkGA1UE 7 | BhMCVVMxCzAJBgNVBAgTAldBMRAwDgYDVQQHEwdTZWF0dGxlMSEwHwYDVQQKExhJ 8 | bnRlcm5ldCBXaWRnaXRzIFB0eSBMdGQxJDAiBgNVBAMTG2dpdGh1Yi5jb20vZHVk 9 | bGV5Y2Fyci9uc3FqczEkMCIGCSqGSIb3DQEJARYVZHVkbGV5LmNhcnJAZ21haWwu 10 | Y29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAy2b+JM1r0RNEACvz 11 | I61YFPUuN605WDYCn8CTJZygA5qQEdti6SmT4oTP9MYx8Q3vslIPGUwg9XOpvJce 12 | Fz0tqf0zgDUjxBSI4EWm840uiQonrOH/TI1uK31SBCc4OAYqRUluGbwimGrGDlWH 13 | on05RQ/hTvBJO7IehkDNVqRuEuCPG29ZN7qbNHGEFpd03J7eTpU/oblJrehEQJuv 14 | 1/pKY+1nTCnhB8RmQqu5l9jOKh2FWn0ezmSTvFWC7oEqKI+ZYDJ/kQnzsbVv7nWt 15 | mm1BXtd9TH7rRQbJsdUBlzjk4yXjCgds2bG/XswMrQmnlLPR4rw67zscaohNEbZi 16 | P3RYRQIDAQABo4IBBDCCAQAwHQYDVR0OBBYEFImAhHh2pitn+mOtAyqaAooFZZlv 17 | MIHQBgNVHSMEgcgwgcWAFImAhHh2pitn+mOtAyqaAooFZZlvoYGhpIGeMIGbMQsw 18 | CQYDVQQGEwJVUzELMAkGA1UECBMCV0ExEDAOBgNVBAcTB1NlYXR0bGUxITAfBgNV 19 | BAoTGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEkMCIGA1UEAxMbZ2l0aHViLmNv 20 | bS9kdWRsZXljYXJyL25zcWpzMSQwIgYJKoZIhvcNAQkBFhVkdWRsZXkuY2FyckBn 21 | bWFpbC5jb22CCQC1Oues/ORXGzAMBgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUA 22 | A4IBAQBaYRHDQpG0VpT5xoC0awzF9QyJjvpgM5lVKgQMwrw/pPGIt2WFeUw9CbrI 23 | hPmBKSWRoubDi2S22MmnVx/B1qsF1+lSqdq+WLfgSsijbKFn5sfXWxF7+1pkaNJa 24 | BFNPf21CYkiSfUGX1ZzFL5T4ooDXHFsQ/95uvZRgTnQqoovjgTvbWfjk9ntNUsmB 25 | WW+PaHXXAVgDAtY62K8E5u46j6aax+5/p9BEitYWCdIS9KGrTJMjr+4veOSKPtqm 26 | 4owyGFe8nJozFiC7o+7kjEBl+8bXFLCM4YyX2CsUaxmloTAeAI/Oxm3FGw6B60cc 27 | clQzl72CTKkdSWWbauWTA9IXgG5z 28 | -----END CERTIFICATE----- 29 | -------------------------------------------------------------------------------- /lib/backofftimer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * This is a timer that is smart about backing off exponentially 3 | * when there are problems. 4 | * 5 | * Ported from pynsq: 6 | * https://github.com/bitly/pynsq/blob/master/nsq/BackoffTimer.py 7 | */ 8 | class BackoffTimer { 9 | /** 10 | * Instantiates a new instance of BackoffTimer. 11 | * 12 | * @constructor 13 | * @param {Number} minInterval 14 | * @param {Number} maxInterval 15 | * @param {Number} [ratio=0.25] 16 | * @param {Number} [shortLength=10] 17 | * @param {Number} [longLength=250] 18 | */ 19 | constructor( 20 | minInterval, 21 | maxInterval, 22 | ratio = 0.25, 23 | shortLength = 10, 24 | longLength = 250 25 | ) { 26 | this.minInterval = minInterval 27 | this.maxInterval = maxInterval 28 | 29 | this.maxShortTimer = (maxInterval - minInterval) * ratio 30 | this.maxLongTimer = (maxInterval - minInterval) * (1 - ratio) 31 | 32 | this.shortUnit = this.maxShortTimer / shortLength 33 | this.longUnit = this.maxLongTimer / longLength 34 | 35 | this.shortInterval = 0 36 | this.longInterval = 0 37 | 38 | this.interval = 0.0 39 | } 40 | 41 | /** 42 | * On success updates the backoff timers. 43 | */ 44 | success() { 45 | if (this.interval === 0.0) return 46 | 47 | this.shortInterval = Math.max(this.shortInterval - this.shortUnit, 0) 48 | this.longInterval = Math.max(this.longInterval - this.longUnit, 0) 49 | 50 | this.updateInterval() 51 | } 52 | 53 | /** 54 | * On failure updates the backoff timers. 55 | */ 56 | failure() { 57 | this.shortInterval = Math.min( 58 | this.shortInterval + this.shortUnit, 59 | this.maxShortTimer 60 | ) 61 | this.longInterval = Math.min( 62 | this.longInterval + this.longUnit, 63 | this.maxLongTimer 64 | ) 65 | 66 | this.updateInterval() 67 | } 68 | 69 | updateInterval() { 70 | this.interval = this.minInterval + this.shortInterval + this.longInterval 71 | } 72 | 73 | /** 74 | * Get the next backoff interval in seconds. 75 | * 76 | * @return {Number} 77 | */ 78 | getInterval() { 79 | return this.interval 80 | } 81 | } 82 | 83 | module.exports = BackoffTimer 84 | -------------------------------------------------------------------------------- /lib/framebuffer.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | 3 | /** 4 | * From the NSQ protocol documentation: 5 | * http://bitly.github.io/nsq/clients/tcp_protocol_spec.html 6 | * 7 | * The Frame format: 8 | * 9 | * [x][x][x][x][x][x][x][x][x][x][x][x]... 10 | * | (int32) || (int32) || (binary) 11 | * | 4-byte || 4-byte || N-byte 12 | * ------------------------------------... 13 | * size frame ID data 14 | */ 15 | class FrameBuffer { 16 | /** 17 | * Consume a raw message into the buffer. 18 | * 19 | * @param {String} raw 20 | */ 21 | consume(raw) { 22 | this.buffer = Buffer.concat(_.compact([this.buffer, raw])) 23 | } 24 | 25 | /** 26 | * Advance the buffer and return the current slice. 27 | * 28 | * @return {String} 29 | */ 30 | nextFrame() { 31 | if (!this.buffer) return 32 | 33 | if (!this.frameSize(0) || !(this.frameSize(0) <= this.buffer.length)) { 34 | return 35 | } 36 | 37 | const frame = this.pluckFrame() 38 | const nextOffset = this.nextOffset() 39 | this.buffer = this.buffer.slice(nextOffset) 40 | 41 | if (!this.buffer.length) { 42 | delete this.buffer 43 | } 44 | 45 | return frame 46 | } 47 | 48 | /** 49 | * Given an offset into a buffer, get the frame ID and data tuple. 50 | * 51 | * @param {Number} [offset=0] 52 | * @return {Array} 53 | */ 54 | pluckFrame(offset = 0) { 55 | const frame = this.buffer.slice(offset, offset + this.frameSize(offset)) 56 | const frameId = frame.readInt32BE(4) 57 | return [frameId, frame.slice(8)] 58 | } 59 | 60 | /** 61 | * Given the offset of the current frame in the buffer, find the offset 62 | * of the next buffer. 63 | * 64 | * @param {Number} [offset=0] 65 | * @return {Number} 66 | */ 67 | nextOffset(offset = 0) { 68 | const size = this.frameSize(offset) 69 | if (size) { 70 | return offset + size 71 | } 72 | } 73 | 74 | /** 75 | * Given the frame offset, return the frame size. 76 | * 77 | * @param {Number} offset 78 | * @return {Number} 79 | */ 80 | frameSize(offset) { 81 | if (!this.buffer || !(this.buffer.length > 4)) return 82 | 83 | if (offset + 4 <= this.buffer.length) { 84 | return 4 + this.buffer.readInt32BE(offset) 85 | } 86 | } 87 | } 88 | 89 | module.exports = FrameBuffer 90 | -------------------------------------------------------------------------------- /test/backofftimer_test.js: -------------------------------------------------------------------------------- 1 | const BackoffTimer = require('../lib/backofftimer') 2 | 3 | describe('backofftimer', () => { 4 | let timer = null 5 | beforeEach(() => { 6 | timer = new BackoffTimer(0, 128) 7 | }) 8 | 9 | describe('constructor', () => { 10 | it('should have @maxShortTimer eq 1', () => { 11 | timer.maxShortTimer.toString().should.eql('32') 12 | }) 13 | 14 | it('should have a @maxLongTimer eq 3', () => { 15 | timer.maxLongTimer.toString().should.eql('96') 16 | }) 17 | 18 | it('should have a @shortUnit equal to 0.1', () => { 19 | timer.shortUnit.toString().should.eql('3.2') 20 | }) 21 | 22 | it('should have a @longUnit equal to 0.3', () => { 23 | timer.longUnit.toString().should.eql('0.384') 24 | }) 25 | }) 26 | 27 | describe('success', () => { 28 | it('should adjust @shortInterval to 0', () => { 29 | timer.success() 30 | timer.shortInterval.toString().should.eql('0') 31 | }) 32 | 33 | it('should adjust @longInterval to 0', () => { 34 | timer.success() 35 | timer.longInterval.toString().should.eql('0') 36 | }) 37 | }) 38 | 39 | describe('failure', () => { 40 | it('should adjust @shortInterval to 3.2 after 1 failure', () => { 41 | timer.failure() 42 | timer.shortInterval.toString().should.eql('3.2') 43 | }) 44 | 45 | it('should adjust @longInterval to .384 after 1 failure', () => { 46 | timer.failure() 47 | timer.longInterval.toString().should.eql('0.384') 48 | }) 49 | }) 50 | 51 | describe('getInterval', () => { 52 | it('should initially be 0', () => { 53 | timer.getInterval().toString().should.eql('0') 54 | }) 55 | 56 | it('should be 0 after 1 success', () => { 57 | timer.success() 58 | timer.getInterval().toString().should.eql('0') 59 | }) 60 | 61 | it('should be 0 after 2 successes', () => { 62 | timer.success() 63 | timer.success() 64 | timer.getInterval().toString().should.eql('0') 65 | }) 66 | 67 | it('should be 3.584 after 1 failure', () => { 68 | timer.failure() 69 | timer.getInterval().toString().should.eql('3.584') 70 | }) 71 | 72 | it('should be 7.168 after 2 failure', () => { 73 | timer.failure() 74 | timer.failure() 75 | timer.getInterval().toString().should.eql('7.168') 76 | }) 77 | }) 78 | }) 79 | -------------------------------------------------------------------------------- /test/message_test.js: -------------------------------------------------------------------------------- 1 | const should = require('should') 2 | const sinon = require('sinon') 3 | const Message = require('../lib/message') 4 | const rawMessage = require('./rawmessage') 5 | 6 | const createMessage = (body, requeueDelay, timeout, maxTimeout) => { 7 | return new Message( 8 | rawMessage('1', Date.now(), 0, body), 9 | requeueDelay, 10 | timeout, 11 | maxTimeout 12 | ) 13 | } 14 | 15 | describe('Message', () => 16 | describe('timeout', () => { 17 | it('should not allow finishing a message twice', (done) => { 18 | const msg = createMessage('body', 90, 50, 100) 19 | 20 | const firstFinish = () => msg.finish() 21 | const secondFinish = () => { 22 | msg.hasResponded.should.eql(true) 23 | done() 24 | } 25 | 26 | setTimeout(firstFinish, 10) 27 | setTimeout(secondFinish, 20) 28 | }) 29 | 30 | it('should not allow requeue after finish', (done) => { 31 | const msg = createMessage('body', 90, 50, 100) 32 | 33 | const responseSpy = sinon.spy() 34 | msg.on(Message.RESPOND, responseSpy) 35 | 36 | const firstFinish = () => msg.finish() 37 | const secondRequeue = () => msg.requeue() 38 | 39 | const check = () => { 40 | responseSpy.calledOnce.should.be.true() 41 | done() 42 | } 43 | 44 | setTimeout(firstFinish, 10) 45 | setTimeout(secondRequeue, 20) 46 | setTimeout(check, 20) 47 | }) 48 | 49 | it('should allow touch and then finish post first timeout', (done) => { 50 | const touchIn = 15 51 | const timeoutIn = 20 52 | const finishIn = 25 53 | const checkIn = 30 54 | 55 | const msg = createMessage('body', 90, timeoutIn, 100) 56 | const responseSpy = sinon.spy() 57 | msg.on(Message.RESPOND, responseSpy) 58 | 59 | const touch = () => msg.touch() 60 | 61 | const finish = () => { 62 | msg.timedOut.should.be.eql(false) 63 | msg.finish() 64 | } 65 | 66 | const check = () => { 67 | responseSpy.calledTwice.should.be.true() 68 | done() 69 | } 70 | 71 | setTimeout(touch, touchIn) 72 | setTimeout(finish, finishIn) 73 | setTimeout(check, checkIn) 74 | }) 75 | 76 | return it('should clear timeout on finish', (done) => { 77 | const msg = createMessage('body', 10, 60, 120) 78 | msg.finish() 79 | 80 | return process.nextTick(() => { 81 | should.not.exist(msg.trackTimeoutId) 82 | return done() 83 | }) 84 | }) 85 | })) 86 | -------------------------------------------------------------------------------- /test/roundrobinlist_test.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert') 2 | 3 | const _ = require('lodash') 4 | const should = require('should') 5 | 6 | const RoundRobinList = require('../lib/roundrobinlist') 7 | 8 | describe('roundrobinlist', () => { 9 | let list = null 10 | let rrl = null 11 | 12 | beforeEach(() => { 13 | list = [1, 2, 3] 14 | rrl = new RoundRobinList(list) 15 | }) 16 | 17 | describe('constructor', () => { 18 | it('should have @list eq to passed in list', () => 19 | assert(_.isEqual(rrl.list, list))) 20 | 21 | it('should have made a copy of the list argument', () => 22 | assert(rrl.list !== list)) 23 | 24 | it('should have @index eq to 0', () => rrl.index.should.eql(0)) 25 | }) 26 | 27 | describe('add', () => 28 | it('@list should include the item', () => { 29 | rrl.add(10) 30 | should.ok(Array.from(rrl.list).includes(10)) 31 | })) 32 | 33 | describe('next', () => { 34 | it('should return a list of 1 item by default', () => { 35 | assert(_.isEqual(rrl.next(), list.slice(0, 1))) 36 | rrl.index.should.eql(1) 37 | }) 38 | 39 | it('should return a list as large as the count provided', () => { 40 | assert(_.isEqual(rrl.next(2), list.slice(0, 2))) 41 | rrl.index.should.eql(2) 42 | }) 43 | 44 | it('should return all items and and then start over', () => { 45 | assert(_.isEqual(rrl.next(), [1])) 46 | assert(_.isEqual(rrl.next(), [2])) 47 | assert(_.isEqual(rrl.next(), [3])) 48 | assert(_.isEqual(rrl.next(), [1])) 49 | }) 50 | }) 51 | 52 | describe('remove', () => { 53 | it('should remove the item if it exists in the list', () => { 54 | rrl.remove(3) 55 | should.ok(!Array.from(rrl.list).includes(3)) 56 | }) 57 | 58 | it('should not affect the order of items returned', () => { 59 | rrl.remove(1) 60 | assert(_.isEqual(rrl.next(), [2])) 61 | assert(_.isEqual(rrl.next(), [3])) 62 | assert(_.isEqual(rrl.next(), [2])) 63 | }) 64 | 65 | it('should not affect the order of items returned with items consumed', () => { 66 | assert(_.isEqual(rrl.next(), [1])) 67 | assert(_.isEqual(rrl.next(), [2])) 68 | rrl.remove(2) 69 | assert(_.isEqual(rrl.next(), [3])) 70 | assert(_.isEqual(rrl.next(), [1])) 71 | }) 72 | 73 | it('should silently fail when it does not have the item', () => { 74 | rrl.remove(10) 75 | assert(_.isEqual(rrl.list, [1, 2, 3])) 76 | rrl.index.should.eql(0) 77 | }) 78 | }) 79 | }) 80 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ main ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ main ] 20 | schedule: 21 | - cron: '25 20 * * 0' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'javascript' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] 37 | # Learn more: 38 | # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed 39 | 40 | steps: 41 | - name: Checkout repository 42 | uses: actions/checkout@v2 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v2 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v2 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v2 72 | -------------------------------------------------------------------------------- /test/reader_test.js: -------------------------------------------------------------------------------- 1 | const should = require('should') 2 | const sinon = require('sinon') 3 | 4 | const nsq = require('../lib/nsq') 5 | 6 | describe('reader', () => { 7 | const readerWithAttempts = (attempts) => 8 | new nsq.Reader('topic', 'default', { 9 | nsqdTCPAddresses: ['127.0.0.1:4150'], 10 | maxAttempts: attempts, 11 | }) 12 | 13 | describe('max attempts', () => 14 | describe('exceeded', () => { 15 | it('should process msg while attempts do not exceed max', (done) => { 16 | const maxAttempts = 1 17 | const reader = readerWithAttempts(maxAttempts) 18 | 19 | reader.on(nsq.Reader.DISCARD, () => { 20 | done(new Error('should not be discarded')) 21 | }) 22 | reader.on(nsq.Reader.MESSAGE, () => done()) 23 | reader.handleMessage({attempts: 1, finish: () => {}}) 24 | }) 25 | 26 | it('should finish after exceeding specified max attempts', (done) => { 27 | const maxAttempts = 2 28 | const reader = readerWithAttempts(maxAttempts) 29 | 30 | // Message that has exceed the maximum number of attempts 31 | const message = { 32 | attempts: maxAttempts + 1, 33 | finish: sinon.spy(), 34 | } 35 | 36 | reader.handleMessage(message) 37 | 38 | process.nextTick(() => { 39 | should.equal(message.finish.called, true) 40 | done() 41 | }) 42 | }) 43 | 44 | it('should call the DISCARD message hanlder if registered', (done) => { 45 | const maxAttempts = 2 46 | const reader = readerWithAttempts(maxAttempts) 47 | 48 | const message = { 49 | attempts: maxAttempts + 1, 50 | finish() {}, 51 | } 52 | 53 | reader.on(nsq.Reader.DISCARD, () => done()) 54 | reader.handleMessage(message) 55 | }) 56 | 57 | it('should call the MESSAGE handler by default', (done) => { 58 | const maxAttempts = 2 59 | const reader = readerWithAttempts(maxAttempts) 60 | 61 | const message = { 62 | attempts: maxAttempts + 1, 63 | finish() {}, 64 | } 65 | 66 | reader.on(nsq.Reader.MESSAGE, () => done()) 67 | reader.handleMessage(message) 68 | }) 69 | })) 70 | 71 | describe('off by default', () => 72 | it('should not finish the message', (done) => { 73 | const reader = readerWithAttempts(0) 74 | 75 | const message = { 76 | attempts: 100, 77 | finish: sinon.spy(), 78 | } 79 | 80 | // Registering this to make sure that even if the listener is available, 81 | // it should not be getting called. 82 | reader.on(nsq.Reader.DISCARD, () => { 83 | done(new Error('Unexpected discard message')) 84 | }) 85 | 86 | const messageHandlerSpy = sinon.spy() 87 | reader.on(nsq.Reader.MESSAGE, messageHandlerSpy) 88 | reader.handleMessage(message) 89 | 90 | process.nextTick(() => { 91 | should.equal(messageHandlerSpy.called, true) 92 | should.equal(message.finish.called, false) 93 | done() 94 | }) 95 | })) 96 | }) 97 | -------------------------------------------------------------------------------- /lib/lookupd.js: -------------------------------------------------------------------------------- 1 | const debug = require('./debug') 2 | const fetch = require('node-fetch') 3 | const url = require('url') 4 | const {joinHostPort} = require('./config') 5 | 6 | const log = debug('nsqjs:lookup') 7 | /** 8 | * lookupdRequest returns the list of producers from a lookupd given a 9 | * URL to query. 10 | * 11 | * The callback will not return an error since it's assumed that there might 12 | * be transient issues with lookupds. 13 | * 14 | * @param {String} url 15 | * @param {Function} callback 16 | */ 17 | async function lookupdRequest(url) { 18 | try { 19 | log(`Query: ${url}`) 20 | const response = await fetch(url) 21 | if (!response.ok) { 22 | log(`Request to nsqlookupd failed. Response code = ${response.status}`) 23 | return [] 24 | } 25 | 26 | // Pre nsq 1.x contained producers within data 27 | const {producers, data} = await response.json() 28 | return producers || data.producers 29 | } catch (e) { 30 | log(`Request to nslookupd failed without a response`) 31 | return [] 32 | } 33 | } 34 | 35 | /** 36 | * Takes a list of responses from lookupds and dedupes the nsqd 37 | * hosts based on host / port pair. 38 | * 39 | * @param {Array} results - list of lists of nsqd node objects. 40 | * @return {Array} 41 | */ 42 | function dedupeOnHostPort(results) { 43 | const uniqueNodes = {} 44 | for (const lookupdResult of results) { 45 | for (const item of lookupdResult) { 46 | const key = item.broadcast_address 47 | ? joinHostPort(item.broadcast_address, item.tcp_port) 48 | : joinHostPort(item.hostname, item.tcp_port) 49 | uniqueNodes[key] = item 50 | } 51 | } 52 | 53 | return Object.values(uniqueNodes) 54 | } 55 | 56 | /** 57 | * Construct a lookupd URL to query for a particular topic. 58 | * 59 | * @param {String} endpoint - host/port pair or a URL 60 | * @param {String} topic - nsq topic 61 | * @returns {String} lookupd URL 62 | */ 63 | function lookupdURL(endpoint, topic) { 64 | endpoint = endpoint.indexOf('://') !== -1 ? endpoint : `http://${endpoint}` 65 | 66 | const parsedUrl = new url.URL(endpoint) 67 | const pathname = parsedUrl.pathname 68 | parsedUrl.pathname = pathname && pathname !== '/' ? pathname : '/lookup' 69 | parsedUrl.searchParams.set('topic', topic) 70 | return parsedUrl.toString() 71 | } 72 | 73 | /** 74 | * Queries lookupds for known nsqd nodes given a topic and returns 75 | * a deduped list. 76 | * 77 | * @param {String} lookupdEndpoints - a string or a list of strings of 78 | * lookupd HTTP endpoints. eg. ['127.0.0.1:4161'] 79 | * @param {String} topic - a string of the topic name 80 | */ 81 | async function lookup(lookupdEndpoints, topic) { 82 | // Ensure we have a list of endpoints for lookupds. 83 | if (!Array.isArray(lookupdEndpoints)) { 84 | lookupdEndpoints = [lookupdEndpoints] 85 | } 86 | 87 | // URLs for querying `nodes` on each of the lookupds. 88 | const urls = Array.from(lookupdEndpoints).map((e) => lookupdURL(e, topic)) 89 | 90 | const results = await Promise.all(urls.map(lookupdRequest)) 91 | return dedupeOnHostPort(results) 92 | } 93 | 94 | module.exports = lookup 95 | -------------------------------------------------------------------------------- /test/framebuffer_test.js: -------------------------------------------------------------------------------- 1 | const should = require('should') 2 | 3 | const wire = require('../lib/wire') 4 | const FrameBuffer = require('../lib/framebuffer') 5 | 6 | const createFrame = (frameId, payload) => { 7 | const frame = Buffer.alloc(4 + 4 + payload.length) 8 | frame.writeInt32BE(payload.length + 4, 0) 9 | frame.writeInt32BE(frameId, 4) 10 | frame.write(payload, 8) 11 | return frame 12 | } 13 | 14 | describe('FrameBuffer', () => { 15 | it('should parse a single, full frame', () => { 16 | const frameBuffer = new FrameBuffer() 17 | const data = createFrame(wire.FRAME_TYPE_RESPONSE, 'OK') 18 | frameBuffer.consume(data) 19 | 20 | const [frameId, payload] = Array.from(frameBuffer.nextFrame()) 21 | frameId.should.eql(wire.FRAME_TYPE_RESPONSE) 22 | payload.toString().should.eql('OK') 23 | }) 24 | 25 | it('should parse two full frames', () => { 26 | const frameBuffer = new FrameBuffer() 27 | 28 | const firstFrame = createFrame(wire.FRAME_TYPE_RESPONSE, 'OK') 29 | const secondFrame = createFrame( 30 | wire.FRAME_TYPE_ERROR, 31 | JSON.stringify({shortname: 'localhost'}) 32 | ) 33 | 34 | frameBuffer.consume(Buffer.concat([firstFrame, secondFrame])) 35 | const frames = [frameBuffer.nextFrame(), frameBuffer.nextFrame()] 36 | frames.length.should.eql(2) 37 | 38 | let [frameId, data] = Array.from(frames.shift()) 39 | frameId.should.eql(wire.FRAME_TYPE_RESPONSE) 40 | data.toString().should.eql('OK') 41 | ;[frameId, data] = Array.from(frames.shift()) 42 | frameId.should.eql(wire.FRAME_TYPE_ERROR) 43 | data.toString().should.eql(JSON.stringify({shortname: 'localhost'})) 44 | }) 45 | 46 | it('should parse frame delivered in partials', () => { 47 | const frameBuffer = new FrameBuffer() 48 | const data = createFrame(wire.FRAME_TYPE_RESPONSE, 'OK') 49 | 50 | // First frame is 10 bytes long. Don't expect to get anything back. 51 | frameBuffer.consume(data.slice(0, 3)) 52 | should.not.exist(frameBuffer.nextFrame()) 53 | 54 | // Yup, still haven't received the whole frame. 55 | frameBuffer.consume(data.slice(3, 8)) 56 | should.not.exist(frameBuffer.nextFrame()) 57 | 58 | // Got the whole first frame. 59 | frameBuffer.consume(data.slice(8)) 60 | should.exist(frameBuffer.nextFrame()) 61 | }) 62 | 63 | it('should parse multiple frames delivered in partials', () => { 64 | const frameBuffer = new FrameBuffer() 65 | const first = createFrame(wire.FRAME_TYPE_RESPONSE, 'OK') 66 | const second = createFrame(wire.FRAME_TYPE_RESPONSE, '{}') 67 | const data = Buffer.concat([first, second]) 68 | 69 | // First frame is 10 bytes long. Don't expect to get anything back. 70 | frameBuffer.consume(data.slice(0, 3)) 71 | should.not.exist(frameBuffer.nextFrame()) 72 | 73 | // Yup, still haven't received the whole frame. 74 | frameBuffer.consume(data.slice(3, 8)) 75 | should.not.exist(frameBuffer.nextFrame()) 76 | 77 | // Got the whole first frame and part of the 2nd frame. 78 | frameBuffer.consume(data.slice(8, 12)) 79 | should.exist(frameBuffer.nextFrame()) 80 | 81 | // Got the 2nd frame. 82 | frameBuffer.consume(data.slice(12)) 83 | should.exist(frameBuffer.nextFrame()) 84 | }) 85 | 86 | return it('empty internal buffer when all frames are consumed', () => { 87 | const frameBuffer = new FrameBuffer() 88 | const data = createFrame(wire.FRAME_TYPE_RESPONSE, 'OK') 89 | 90 | frameBuffer.consume(data) 91 | while (frameBuffer.nextFrame()); 92 | 93 | should.not.exist(frameBuffer.buffer) 94 | }) 95 | }) 96 | -------------------------------------------------------------------------------- /test/wire_test.js: -------------------------------------------------------------------------------- 1 | const should = require('should') 2 | const wire = require('../lib/wire') 3 | 4 | const matchCommand = (commandFn, args, expected) => { 5 | const commandOut = commandFn(...args) 6 | should.equal(commandOut.toString(), expected) 7 | } 8 | 9 | describe('nsq wire', () => { 10 | it('should construct an identity message', () => { 11 | matchCommand( 12 | wire.identify, 13 | [{short_id: 1, long_id: 2}], 14 | 'IDENTIFY\n\u0000\u0000\u0000\u001a{"short_id":1,"long_id":2}' 15 | ) 16 | }) 17 | 18 | it('should construct an identity message with unicode', () => 19 | matchCommand( 20 | wire.identify, 21 | [{long_id: 'w\u00c3\u00a5\u00e2\u0080\u00a0'}], 22 | 'IDENTIFY\n\u0000\u0000\u0000-{"long_id":"w\\u00c3\\u00a5\\u00e2' + 23 | '\\u0080\\u00a0"}' 24 | )) 25 | 26 | it('should subscribe to a topic and channel', () => 27 | matchCommand( 28 | wire.subscribe, 29 | ['test_topic', 'test_channel'], 30 | 'SUB test_topic test_channel\n' 31 | )) 32 | 33 | it('should finish a message', () => 34 | matchCommand(wire.finish, ['test'], 'FIN test\n')) 35 | 36 | it('should finish a message with a unicode id', () => 37 | matchCommand( 38 | wire.finish, 39 | ['\u00fcn\u00ee\u00e7\u00f8\u2202\u00e9'], 40 | 'FIN \u00fcn\u00ee\u00e7\u00f8\u2202\u00e9\n' 41 | )) 42 | 43 | it('should requeue a message', () => 44 | matchCommand(wire.requeue, ['test'], 'REQ test 0\n')) 45 | 46 | it('should requeue a message with timeout', () => 47 | matchCommand(wire.requeue, ['test', 60], 'REQ test 60\n')) 48 | 49 | it('should touch a message', () => 50 | matchCommand(wire.touch, ['test'], 'TOUCH test\n')) 51 | 52 | it('should construct a ready message', () => 53 | matchCommand(wire.ready, [100], 'RDY 100\n')) 54 | 55 | it('should construct a no-op message', () => 56 | matchCommand(wire.nop, [], 'NOP\n')) 57 | 58 | it('should publish a message', () => 59 | matchCommand( 60 | wire.pub, 61 | ['test_topic', 'abcd'], 62 | 'PUB test_topic\n\u0000\u0000\u0000\u0004abcd' 63 | )) 64 | 65 | it('should publish a multi-byte string message', () => 66 | matchCommand( 67 | wire.pub, 68 | ['test_topic', 'こんにちは'], 69 | 'PUB test_topic\n\u0000\u0000\u0000\u000fこんにちは' 70 | )) 71 | 72 | it('should publish multiple string messages', () => 73 | matchCommand( 74 | wire.mpub, 75 | ['test_topic', ['abcd', 'efgh', 'ijkl']], 76 | [ 77 | 'MPUB test_topic\n\u0000\u0000\u0000\u001c\u0000\u0000\u0000\u0003', 78 | '\u0000\u0000\u0000\u0004abcd', 79 | '\u0000\u0000\u0000\u0004efgh', 80 | '\u0000\u0000\u0000\u0004ijkl', 81 | ].join('') 82 | )) 83 | 84 | it('should publish multiple buffer messages', () => 85 | matchCommand( 86 | wire.mpub, 87 | ['test_topic', [Buffer.from('abcd'), Buffer.from('efgh')]], 88 | [ 89 | 'MPUB test_topic\n\u0000\u0000\u0000\u0014\u0000\u0000\u0000\u0002', 90 | '\u0000\u0000\u0000\u0004abcd', 91 | '\u0000\u0000\u0000\u0004efgh', 92 | ].join('') 93 | )) 94 | 95 | return it('should unpack a received message', () => { 96 | const msgPayload = [ 97 | '132cb60626e9fd7a00013035356335626531636534333330323769747265616c6c7974', 98 | '696564746865726f6f6d746f676574686572', 99 | ] 100 | const msgParts = wire.unpackMessage(Buffer.from(msgPayload.join(''), 'hex')) 101 | 102 | const [id, timestamp, attempts] = Array.from(msgParts) 103 | timestamp.toString(10).should.eql('1381679323234827642') 104 | id.should.eql('055c5be1ce433027') 105 | return attempts.should.eql(1) 106 | }) 107 | }) 108 | -------------------------------------------------------------------------------- /test/writer_test.js: -------------------------------------------------------------------------------- 1 | const should = require('should') 2 | const sinon = require('sinon') 3 | 4 | const nsq = require('../lib/nsq') 5 | 6 | describe('writer', () => { 7 | let writer = null 8 | 9 | beforeEach(() => { 10 | writer = new nsq.Writer('127.0.0.1', '4150') 11 | writer.conn = {produceMessages: sinon.stub()} 12 | }) 13 | 14 | afterEach(() => { 15 | writer = null 16 | }) 17 | 18 | describe('publish', () => { 19 | it('should publish a string', () => { 20 | const topic = 'test_topic' 21 | const msg = 'hello world!' 22 | 23 | writer.publish(topic, msg, () => { 24 | should.equal(writer.conn.produceMessages.calledOnce, true) 25 | should.equal(writer.conn.produceMessages.calledWith(topic, [msg]), true) 26 | }) 27 | }) 28 | 29 | it('should defer publish a string', () => { 30 | const topic = 'test_topic' 31 | const msg = 'hello world!' 32 | 33 | writer.publish(topic, msg, 300, () => { 34 | should.equal(writer.conn.produceMessages.calledOnce, true) 35 | should.equal(writer.conn.produceMessages.calledWith(topic, [msg]), true) 36 | }) 37 | }) 38 | 39 | // Add test where it is not ready yet 40 | 41 | it('should publish a list of strings', () => { 42 | const topic = 'test_topic' 43 | const msgs = ['hello world!', 'another message'] 44 | 45 | writer.publish(topic, msgs, () => { 46 | should.equal(writer.conn.produceMessages.calledOnce, true) 47 | should.equal(writer.conn.produceMessages.calledWith(topic, msgs), true) 48 | }) 49 | }) 50 | 51 | it('should publish a buffer', () => { 52 | const topic = 'test_topic' 53 | const msg = Buffer.from('a buffer message') 54 | 55 | writer.publish(topic, msg, () => { 56 | should.equal(writer.conn.produceMessages.calledOnce, true) 57 | should.equal(writer.conn.produceMessages.calledWith(topic, [msg]), true) 58 | }) 59 | }) 60 | 61 | it('should publish an object as JSON', () => { 62 | const topic = 'test_topic' 63 | const msg = {a: 1} 64 | 65 | writer.publish(topic, msg, () => { 66 | should.equal(writer.conn.produceMessages.calledOnce, true) 67 | should.equal( 68 | writer.conn.produceMessages.calledWith(topic, [JSON.stringify(msg)]), 69 | true 70 | ) 71 | }) 72 | }) 73 | 74 | it('should publish a list of buffers', () => { 75 | const topic = 'test_topic' 76 | const msgs = [Buffer.from('a buffer message'), Buffer.from('another msg')] 77 | 78 | writer.publish(topic, msgs, () => { 79 | should.equal(writer.conn.produceMessages.calledOnce, true) 80 | should.equal(writer.conn.produceMessages.calledWith(topic, msgs), true) 81 | }) 82 | }) 83 | 84 | it('should publish a list of objects as JSON', () => { 85 | const topic = 'test_topic' 86 | const msgs = [{a: 1}, {b: 2}] 87 | const encodedMsgs = Array.from(msgs).map((i) => JSON.stringify(i)) 88 | 89 | writer.publish(topic, msgs, () => { 90 | should.equal(writer.conn.produceMessages.calledOnce, true) 91 | should.equal( 92 | writer.conn.produceMessages.calledWith(topic, encodedMsgs), 93 | true 94 | ) 95 | }) 96 | }) 97 | 98 | it('should fail when publishing Null', (done) => { 99 | const topic = 'test_topic' 100 | const msg = null 101 | 102 | writer.publish(topic, msg, (err) => { 103 | should.exist(err) 104 | done() 105 | }) 106 | }) 107 | 108 | it('should fail when publishing Undefined', (done) => { 109 | const topic = 'test_topic' 110 | const msg = undefined 111 | 112 | writer.publish(topic, msg, (err) => { 113 | should.exist(err) 114 | done() 115 | }) 116 | }) 117 | 118 | it('should fail when publishing an empty string', (done) => { 119 | const topic = 'test_topic' 120 | const msg = '' 121 | 122 | writer.publish(topic, msg, (err) => { 123 | should.exist(err) 124 | done() 125 | }) 126 | }) 127 | 128 | it('should fail when publishing an empty list', (done) => { 129 | const topic = 'test_topic' 130 | const msg = [] 131 | 132 | writer.publish(topic, msg, (err) => { 133 | should.exist(err) 134 | done() 135 | }) 136 | }) 137 | 138 | it('should fail when the Writer is not connected', (done) => { 139 | writer = new nsq.Writer('127.0.0.1', '4150') 140 | writer.publish('test_topic', 'a briliant message', (err) => { 141 | should.exist(err) 142 | done() 143 | }) 144 | }) 145 | }) 146 | }) 147 | -------------------------------------------------------------------------------- /test/lookupd_test.js: -------------------------------------------------------------------------------- 1 | const nock = require('nock') 2 | const should = require('should') 3 | 4 | const lookup = require('../lib/lookupd') 5 | 6 | const NSQD_1 = { 7 | address: 'localhost', 8 | broadcast_address: 'localhost', 9 | hostname: 'localhost', 10 | http_port: 4151, 11 | remote_address: 'localhost:12345', 12 | tcp_port: 4150, 13 | topics: ['sample_topic'], 14 | version: '0.2.23', 15 | } 16 | const NSQD_2 = { 17 | address: 'localhost', 18 | broadcast_address: 'localhost', 19 | hostname: 'localhost', 20 | http_port: 5151, 21 | remote_address: 'localhost:56789', 22 | tcp_port: 5150, 23 | topics: ['sample_topic'], 24 | version: '0.2.23', 25 | } 26 | const NSQD_3 = { 27 | address: 'localhost', 28 | broadcast_address: 'localhost', 29 | hostname: 'localhost', 30 | http_port: 6151, 31 | remote_address: 'localhost:23456', 32 | tcp_port: 6150, 33 | topics: ['sample_topic'], 34 | version: '0.2.23', 35 | } 36 | const NSQD_4 = { 37 | address: 'localhost', 38 | broadcast_address: 'localhost', 39 | hostname: 'localhost', 40 | http_port: 7151, 41 | remote_address: 'localhost:34567', 42 | tcp_port: 7150, 43 | topics: ['sample_topic'], 44 | version: '0.2.23', 45 | } 46 | 47 | const LOOKUPD_1 = '127.0.0.1:4161' 48 | const LOOKUPD_2 = '127.0.0.1:5161' 49 | const LOOKUPD_3 = 'http://127.0.0.1:6161/' 50 | const LOOKUPD_4 = 'http://127.0.0.1:7161/path/lookup' 51 | 52 | const nockUrlSplit = (url) => { 53 | const match = url.match(/^(https?:\/\/[^/]+)(\/.*$)/i) 54 | return { 55 | baseUrl: match[1], 56 | path: match[2], 57 | } 58 | } 59 | 60 | const registerWithLookupd = (lookupdAddress, nsqd) => { 61 | const producers = nsqd != null ? [nsqd] : [] 62 | 63 | if (nsqd != null) { 64 | nsqd.topics.forEach((topic) => { 65 | if (lookupdAddress.indexOf('://') === -1) { 66 | nock(`http://${lookupdAddress}`) 67 | .get(`/lookup?topic=${topic}`) 68 | .reply(200, { 69 | status_code: 200, 70 | status_txt: 'OK', 71 | producers, 72 | }) 73 | } else { 74 | const params = nockUrlSplit(lookupdAddress) 75 | const {baseUrl} = params 76 | let {path} = params 77 | if (!path || path === '/') { 78 | path = '/lookup' 79 | } 80 | 81 | nock(baseUrl).get(`${path}?topic=${topic}`).reply(200, { 82 | status_code: 200, 83 | status_txt: 'OK', 84 | producers, 85 | }) 86 | } 87 | }) 88 | } 89 | } 90 | 91 | const setFailedTopicReply = (lookupdAddress, topic) => 92 | nock(`http://${lookupdAddress}`).get(`/lookup?topic=${topic}`).reply(200, { 93 | status_code: 404, 94 | status_txt: 'TOPIC_NOT_FOUND', 95 | }) 96 | 97 | describe('lookupd.lookup', () => { 98 | afterEach(() => nock.cleanAll()) 99 | 100 | describe('querying a single lookupd for a topic', () => { 101 | it('should return an empty list if no nsqd nodes', async () => { 102 | setFailedTopicReply(LOOKUPD_1, 'sample_topic') 103 | 104 | const nodes = await lookup(LOOKUPD_1, 'sample_topic') 105 | nodes.should.be.empty() 106 | }) 107 | 108 | it('should return a list of nsqd nodes for a success reply', async () => { 109 | registerWithLookupd(LOOKUPD_1, NSQD_1) 110 | 111 | const nodes = await lookup(LOOKUPD_1, 'sample_topic') 112 | nodes.should.have.length(1) 113 | 114 | const props = ['address', 'broadcast_address', 'tcp_port', 'http_port'] 115 | for (const prop of props) { 116 | should.ok(Object.keys(nodes[0]).includes(prop)) 117 | } 118 | }) 119 | }) 120 | 121 | describe('querying a multiple lookupd', () => { 122 | it('should combine results from multiple lookupds', async () => { 123 | registerWithLookupd(LOOKUPD_1, NSQD_1) 124 | registerWithLookupd(LOOKUPD_2, NSQD_2) 125 | registerWithLookupd(LOOKUPD_3, NSQD_3) 126 | registerWithLookupd(LOOKUPD_4, NSQD_4) 127 | 128 | const lookupdAddresses = [LOOKUPD_1, LOOKUPD_2, LOOKUPD_3, LOOKUPD_4] 129 | const nodes = await lookup(lookupdAddresses, 'sample_topic') 130 | nodes.should.have.length(4) 131 | 132 | const ports = nodes.map((n) => n['tcp_port']) 133 | ports.sort() 134 | ports.should.eql([4150, 5150, 6150, 7150]) 135 | }) 136 | 137 | it('should dedupe combined results', async () => { 138 | registerWithLookupd(LOOKUPD_1, NSQD_1) 139 | registerWithLookupd(LOOKUPD_2, NSQD_1) 140 | registerWithLookupd(LOOKUPD_3, NSQD_1) 141 | registerWithLookupd(LOOKUPD_4, NSQD_1) 142 | 143 | const lookupdAddresses = [LOOKUPD_1, LOOKUPD_2, LOOKUPD_3, LOOKUPD_4] 144 | const nodes = await lookup(lookupdAddresses, 'sample_topic') 145 | nodes.should.have.length(1) 146 | }) 147 | 148 | return it('should succeed inspite of failures to query a lookupd', async () => { 149 | registerWithLookupd(LOOKUPD_1, NSQD_1) 150 | nock(`http://${LOOKUPD_2}`).get('/lookup?topic=sample_topic').reply(500) 151 | 152 | const lookupdAddresses = [LOOKUPD_1, LOOKUPD_2] 153 | const nodes = await lookup(lookupdAddresses, 'sample_topic') 154 | nodes.should.have.length(1) 155 | }) 156 | }) 157 | }) 158 | -------------------------------------------------------------------------------- /lib/message.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events') 2 | const wire = require('./wire') 3 | 4 | /** 5 | * Message - a high-level message object, which exposes stateful methods 6 | * for responding to nsqd (FIN, REQ, TOUCH, etc.) as well as metadata 7 | * such as attempts and timestamp. 8 | * @type {Message} 9 | */ 10 | class Message extends EventEmitter { 11 | // Event types 12 | static get BACKOFF() { 13 | return 'backoff' 14 | } 15 | static get RESPOND() { 16 | return 'respond' 17 | } 18 | 19 | // Response types 20 | static get FINISH() { 21 | return 0 22 | } 23 | static get REQUEUE() { 24 | return 1 25 | } 26 | static get TOUCH() { 27 | return 2 28 | } 29 | 30 | /** 31 | * Instantiates a new instance of a Message. 32 | * @constructor 33 | * @param {String} id 34 | * @param {String|Number} timestamp 35 | * @param {Number} attempts 36 | * @param {String} body 37 | * @param {Number} requeueDelay 38 | * @param {Number} msgTimeout 39 | * @param {Number} maxMsgTimeout 40 | */ 41 | constructor(rawMessage, requeueDelay, msgTimeout, maxMsgTimeout) { 42 | super(...arguments) // eslint-disable-line prefer-rest-params 43 | this.rawMessage = rawMessage 44 | this.requeueDelay = requeueDelay 45 | this.msgTimeout = msgTimeout 46 | this.maxMsgTimeout = maxMsgTimeout 47 | this.hasResponded = false 48 | this.receivedOn = Date.now() 49 | this.lastTouched = this.receivedOn 50 | this.touchCount = 0 51 | this.trackTimeoutId = null 52 | 53 | // Keep track of when this message actually times out. 54 | this.timedOut = false 55 | this.trackTimeout() 56 | } 57 | 58 | get id() { 59 | return wire.unpackMessageId(this.rawMessage) 60 | } 61 | 62 | get timestamp() { 63 | return wire.unpackMessageTimestamp(this.rawMessage) 64 | } 65 | 66 | get attempts() { 67 | return wire.unpackMessageAttempts(this.rawMessage) 68 | } 69 | 70 | get body() { 71 | return wire.unpackMessageBody(this.rawMessage) 72 | } 73 | 74 | /** 75 | * track whether or not a message has timed out. 76 | */ 77 | trackTimeout() { 78 | if (this.hasResponded) return 79 | 80 | const soft = this.timeUntilTimeout() 81 | const hard = this.timeUntilTimeout(true) 82 | 83 | // Both values have to be not null otherwise we've timedout. 84 | this.timedOut = !soft || !hard 85 | if (!this.timedOut) { 86 | clearTimeout(this.trackTimeoutId) 87 | this.trackTimeoutId = setTimeout( 88 | () => this.trackTimeout(), 89 | Math.min(soft, hard) 90 | ).unref() 91 | } 92 | } 93 | 94 | /** 95 | * Safely parse the body into JSON. 96 | * 97 | * @return {Object} 98 | */ 99 | json() { 100 | if (this.parsed == null) { 101 | try { 102 | this.parsed = JSON.parse(this.body) 103 | } catch (err) { 104 | throw new Error('Invalid JSON in Message') 105 | } 106 | } 107 | 108 | return this.parsed 109 | } 110 | 111 | /** 112 | * Returns in milliseconds the time until this message expires. Returns 113 | * null if that time has already ellapsed. There are two different timeouts 114 | * for a message. There are the soft timeouts that can be extended by touching 115 | * the message. There is the hard timeout that cannot be exceeded without 116 | * reconfiguring the nsqd. 117 | * 118 | * @param {Boolean} [hard=false] 119 | * @return {Number|null} 120 | */ 121 | timeUntilTimeout(hard = false) { 122 | if (this.hasResponded) return null 123 | 124 | let delta 125 | if (hard) { 126 | delta = this.receivedOn + this.maxMsgTimeout - Date.now() 127 | } else { 128 | delta = this.lastTouched + this.msgTimeout - Date.now() 129 | } 130 | 131 | if (delta > 0) { 132 | return delta 133 | } 134 | 135 | return null 136 | } 137 | 138 | /** 139 | * Respond with a `FINISH` event. 140 | */ 141 | finish() { 142 | this.respond(Message.FINISH, wire.finish(this.id)) 143 | } 144 | 145 | /** 146 | * Requeue the message with the specified amount of delay. If backoff is 147 | * specifed, then the subscribed Readers will backoff. 148 | * 149 | * @param {Number} [delay=this.requeueDelay] 150 | * @param {Boolean} [backoff=true] [description] 151 | */ 152 | requeue(delay = this.requeueDelay, backoff = true) { 153 | this.respond(Message.REQUEUE, wire.requeue(this.id, delay)) 154 | if (backoff) { 155 | this.emit(Message.BACKOFF) 156 | } 157 | } 158 | 159 | /** 160 | * Emit a `TOUCH` command. `TOUCH` command can be used to reset the timer 161 | * on the nsqd side. This can be done repeatedly until the message 162 | * is either FIN or REQ, up to the sending nsqd’s configured max_msg_timeout. 163 | */ 164 | touch() { 165 | this.touchCount += 1 166 | this.lastTouched = Date.now() 167 | this.respond(Message.TOUCH, wire.touch(this.id)) 168 | } 169 | 170 | /** 171 | * Emit a `RESPOND` event. 172 | * 173 | * @param {Number} responseType 174 | * @param {Buffer} wireData 175 | * @return {undefined} 176 | */ 177 | respond(responseType, wireData) { 178 | // TODO: Add a debug/warn when we moved to debug.js 179 | if (this.hasResponded) return 180 | 181 | process.nextTick(() => { 182 | if (responseType !== Message.TOUCH) { 183 | this.hasResponded = true 184 | clearTimeout(this.trackTimeoutId) 185 | this.trackTimeoutId = null 186 | } else { 187 | this.lastTouched = Date.now() 188 | } 189 | 190 | this.emit(Message.RESPOND, responseType, wireData) 191 | }) 192 | } 193 | } 194 | 195 | module.exports = Message 196 | -------------------------------------------------------------------------------- /lib/writer.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const debug = require('./debug') 3 | const {EventEmitter} = require('events') 4 | 5 | const {ConnectionConfig, joinHostPort} = require('./config') 6 | const {WriterNSQDConnection} = require('./nsqdconnection') 7 | 8 | /** 9 | * Publish messages to nsqds. 10 | * 11 | * Usage: 12 | * const writer = new Writer('127.0.0.1', 4150); 13 | * writer.connect(); 14 | * 15 | * writer.on(Writer.READY, () => { 16 | * // Send a single message 17 | * writer.publish('sample_topic', 'one'); 18 | * // Send multiple messages 19 | * writer.publish('sample_topic', ['two', 'three']); 20 | * }); 21 | * 22 | * writer.on(Writer.CLOSED, () => { 23 | * console.log('Writer closed'); 24 | * }); 25 | */ 26 | class Writer extends EventEmitter { 27 | // Writer events 28 | static get READY() { 29 | return 'ready' 30 | } 31 | static get CLOSED() { 32 | return 'closed' 33 | } 34 | static get ERROR() { 35 | return 'error' 36 | } 37 | 38 | /** 39 | * Instantiates a new Writer. 40 | * 41 | * @constructor 42 | * @param {String} nsqdHost 43 | * @param {String} nsqdPort 44 | * @param {Object} options 45 | */ 46 | constructor(nsqdHost, nsqdPort, options) { 47 | super() 48 | 49 | this.nsqdHost = nsqdHost 50 | this.nsqdPort = nsqdPort 51 | 52 | // Handy in the event that there are tons of publish calls 53 | // while the Writer is connecting. 54 | this.setMaxListeners(10000) 55 | 56 | this.debug = debug(`nsqjs:writer:${joinHostPort(this.nsqdHost, this.nsqdPort)}`) 57 | this.config = new ConnectionConfig(options) 58 | this.config.validate() 59 | this.ready = false 60 | 61 | this.debug('Configuration') 62 | this.debug(this.config) 63 | } 64 | 65 | /** 66 | * Connect establishes a new nsqd writer connection. 67 | */ 68 | connect() { 69 | this.conn = new WriterNSQDConnection( 70 | this.nsqdHost, 71 | this.nsqdPort, 72 | this.config 73 | ) 74 | 75 | this.debug('connect') 76 | this.conn.connect() 77 | 78 | this.conn.on(WriterNSQDConnection.READY, () => { 79 | this.debug('ready') 80 | this.ready = true 81 | this.emit(Writer.READY) 82 | }) 83 | 84 | this.conn.on(WriterNSQDConnection.CLOSED, () => { 85 | this.debug('closed') 86 | this.ready = false 87 | this.emit(Writer.CLOSED) 88 | }) 89 | 90 | this.conn.on(WriterNSQDConnection.ERROR, (err) => { 91 | this.debug('error', err) 92 | this.ready = false 93 | this.emit(Writer.ERROR, err) 94 | }) 95 | 96 | this.conn.on(WriterNSQDConnection.CONNECTION_ERROR, (err) => { 97 | this.debug('error', err) 98 | this.ready = false 99 | this.emit(Writer.ERROR, err) 100 | }) 101 | } 102 | 103 | /** 104 | * Publish a message or a list of messages to the connected nsqd. The contents 105 | * of the messages should either be strings or buffers with the payload encoded. 106 | 107 | * @param {String} topic 108 | * @param {String|Buffer|Object|Array} msgs - A string, a buffer, a 109 | * JSON serializable object, or a list of string / buffers / 110 | * JSON serializable objects. 111 | * @param {Function} callback 112 | * @return {undefined} 113 | */ 114 | publish(topic, msgs, callback) { 115 | let err = this._checkStateValidity() 116 | err = err || this._checkMsgsValidity(msgs) 117 | 118 | if (err) { 119 | return this._throwOrCallback(err, callback) 120 | } 121 | 122 | // Call publish again once the Writer is ready. 123 | if (!this.ready) { 124 | const onReady = (err) => { 125 | if (err) return callback(err) 126 | this.publish(topic, msgs, callback) 127 | } 128 | this._callwhenReady(onReady) 129 | } 130 | 131 | if (!_.isArray(msgs)) { 132 | msgs = [msgs] 133 | } 134 | 135 | // Automatically serialize as JSON if the message isn't a String or a Buffer 136 | msgs = msgs.map(this._serializeMsg) 137 | 138 | return this.conn.produceMessages(topic, msgs, undefined, callback) 139 | } 140 | 141 | /** 142 | * Publish a message to the connected nsqd with delay. The contents 143 | * of the messages should either be strings or buffers with the payload encoded. 144 | 145 | * @param {String} topic 146 | * @param {String|Buffer|Object} msg - A string, a buffer, a 147 | * JSON serializable object, or a list of string / buffers / 148 | * JSON serializable objects. 149 | * @param {Number} timeMs - defer time 150 | * @param {Function} callback 151 | * @return {undefined} 152 | */ 153 | deferPublish(topic, msg, timeMs, callback) { 154 | let err = this._checkStateValidity() 155 | err = err || this._checkMsgsValidity(msg) 156 | err = err || this._checkTimeMsValidity(timeMs) 157 | 158 | if (err) { 159 | return this._throwOrCallback(err, callback) 160 | } 161 | 162 | // Call publish again once the Writer is ready. 163 | if (!this.ready) { 164 | const onReady = (err) => { 165 | if (err) return callback(err) 166 | this.deferPublish(topic, msg, timeMs, callback) 167 | } 168 | this._callwhenReady(onReady) 169 | } 170 | 171 | return this.conn.produceMessages(topic, msg, timeMs, callback) 172 | } 173 | 174 | /** 175 | * Close the writer connection. 176 | * @return {undefined} 177 | */ 178 | close() { 179 | return this.conn.close() 180 | } 181 | 182 | _serializeMsg(msg) { 183 | if (_.isString(msg) || Buffer.isBuffer(msg)) { 184 | return msg 185 | } 186 | return JSON.stringify(msg) 187 | } 188 | 189 | _checkStateValidity() { 190 | let connState = '' 191 | 192 | if (this.conn && this.conn.statemachine) { 193 | connState = this.conn.statemachine.current_state_name 194 | } 195 | 196 | if (!this.conn || ['CLOSED', 'ERROR'].includes(connState)) { 197 | return new Error('No active Writer connection to send messages') 198 | } 199 | } 200 | 201 | _checkMsgsValidity(msgs) { 202 | // maybe when an array check every message to not be empty 203 | if (!msgs || _.isEmpty(msgs)) { 204 | return new Error('Attempting to publish an empty message') 205 | } 206 | } 207 | 208 | _checkTimeMsValidity(timeMs) { 209 | return _.isNumber(timeMs) && timeMs > 0 210 | ? undefined 211 | : new Error('The Delay must be a (positiv) number') 212 | } 213 | 214 | _throwOrCallback(err, callback) { 215 | if (callback) { 216 | return callback(err) 217 | } 218 | throw err 219 | } 220 | 221 | _callwhenReady(fn) { 222 | const ready = () => { 223 | remove() 224 | fn() 225 | } 226 | 227 | const failed = (err) => { 228 | if (!err) { 229 | err = new Error('Connection closed!') 230 | } 231 | remove() 232 | fn(err) 233 | } 234 | 235 | const remove = () => { 236 | this.removeListener(Writer.READY, ready) 237 | this.removeListener(Writer.ERROR, failed) 238 | this.removeListener(Writer.CLOSED, failed) 239 | } 240 | 241 | this.on(Writer.READY, ready) 242 | this.on(Writer.ERROR, failed) 243 | this.on(Writer.CLOSED, failed) 244 | } 245 | } 246 | 247 | module.exports = Writer 248 | -------------------------------------------------------------------------------- /lib/reader.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events') 2 | 3 | const debug = require('./debug') 4 | 5 | const RoundRobinList = require('./roundrobinlist') 6 | const lookup = require('./lookupd') 7 | const {NSQDConnection} = require('./nsqdconnection') 8 | const {ReaderConfig, splitHostPort, joinHostPort} = require('./config') 9 | const {ReaderRdy} = require('./readerrdy') 10 | 11 | /** 12 | * Reader provides high-level functionality for building robust NSQ 13 | * consumers. Reader is built upon the EventEmitter and thus supports various 14 | * hooks when different events occur. 15 | * @type {Reader} 16 | */ 17 | class Reader extends EventEmitter { 18 | static get ERROR() { 19 | return 'error' 20 | } 21 | static get MESSAGE() { 22 | return 'message' 23 | } 24 | static get READY() { 25 | return 'ready' 26 | } 27 | static get NOT_READY() { 28 | return 'not_ready' 29 | } 30 | static get DISCARD() { 31 | return 'discard' 32 | } 33 | static get NSQD_CONNECTED() { 34 | return 'nsqd_connected' 35 | } 36 | static get NSQD_CLOSED() { 37 | return 'nsqd_closed' 38 | } 39 | 40 | /** 41 | * @constructor 42 | * @param {String} topic 43 | * @param {String} channel 44 | * @param {Object} options 45 | */ 46 | constructor(topic, channel, options, ...args) { 47 | super(topic, channel, options, ...args) 48 | this.topic = topic 49 | this.channel = channel 50 | this.debug = debug(`nsqjs:reader:${this.topic}/${this.channel}`) 51 | this.config = new ReaderConfig(options) 52 | this.config.validate() 53 | 54 | this.debug('Configuration') 55 | this.debug(this.config) 56 | 57 | this.roundrobinLookupd = new RoundRobinList( 58 | this.config.lookupdHTTPAddresses 59 | ) 60 | 61 | this.readerRdy = new ReaderRdy( 62 | this.config.maxInFlight, 63 | this.config.maxBackoffDuration, 64 | `${this.topic}/${this.channel}`, 65 | this.config.lowRdyTimeout 66 | ) 67 | 68 | this.lookupdIntervalId = null 69 | this.directIntervalId = null 70 | this.connectionIds = [] 71 | this.isClosed = false 72 | } 73 | 74 | /** 75 | * Adds a connection to nsqd at the configured address. 76 | * 77 | * @return {undefined} 78 | */ 79 | connect() { 80 | this._connectTCPAddresses() 81 | this._connectLookupd() 82 | } 83 | 84 | _connectInterval() { 85 | return this.config.lookupdPollInterval * 1000 86 | } 87 | 88 | _connectTCPAddresses() { 89 | const directConnect = () => { 90 | // Don't establish new connections while the Reader is paused. 91 | if (this.isPaused()) return 92 | 93 | if (this.connectionIds.length < this.config.nsqdTCPAddresses.length) { 94 | return this.config.nsqdTCPAddresses.forEach((addr) => { 95 | const [address, port] = splitHostPort(addr) 96 | this.connectToNSQD(address, Number(port)) 97 | }) 98 | } 99 | } 100 | 101 | this.lookupdIntervalId = setInterval(() => { 102 | directConnect() 103 | }, this._connectInterval()) 104 | 105 | // Connect immediately. 106 | directConnect() 107 | } 108 | 109 | _connectLookupd() { 110 | this.directIntervalId = setInterval(() => { 111 | this.queryLookupd() 112 | }, this._connectInterval()) 113 | 114 | // Connect immediately. 115 | this.queryLookupd() 116 | } 117 | 118 | /** 119 | * Close all connections and prevent any periodic callbacks. 120 | * @return {Array} The closed connections. 121 | */ 122 | close() { 123 | this.isClosed = true 124 | clearInterval(this.directIntervalId) 125 | clearInterval(this.lookupdIntervalId) 126 | return this.readerRdy.close() 127 | } 128 | 129 | /** 130 | * Pause all connections 131 | * @return {Array} The paused connections. 132 | */ 133 | pause() { 134 | this.debug('pause') 135 | return this.readerRdy.pause() 136 | } 137 | 138 | /** 139 | * Unpause all connections 140 | * @return {Array} The unpaused connections. 141 | */ 142 | unpause() { 143 | this.debug('unpause') 144 | return this.readerRdy.unpause() 145 | } 146 | 147 | /** 148 | * @return {Boolean} 149 | */ 150 | isPaused() { 151 | return this.readerRdy.isPaused() 152 | } 153 | 154 | /** 155 | * Trigger a query of the configured nsq_lookupd_http_addresses. 156 | * @return {undefined} 157 | */ 158 | async queryLookupd() { 159 | // Don't establish new connections while the Reader is paused. 160 | if (this.isPaused()) return 161 | 162 | // Trigger a query of the configured `lookupdHTTPAddresses`. 163 | const endpoint = this.roundrobinLookupd.next() 164 | const nodes = await lookup(endpoint, this.topic) 165 | 166 | for (const n of nodes) { 167 | this.connectToNSQD(n.broadcast_address || n.hostname, n.tcp_port) 168 | } 169 | } 170 | 171 | /** 172 | * Adds a connection to nsqd at the specified address. 173 | * 174 | * @param {String} host 175 | * @param {Number|String} port 176 | * @return {Object|undefined} The newly created nsqd connection. 177 | */ 178 | connectToNSQD(host, port) { 179 | if (this.isClosed) { 180 | return 181 | } 182 | 183 | this.debug(`discovered ${joinHostPort(host, port)} for ${this.topic} topic`) 184 | const conn = new NSQDConnection( 185 | host, 186 | port, 187 | this.topic, 188 | this.channel, 189 | this.config 190 | ) 191 | 192 | // Ensure a connection doesn't already exist to this nsqd instance. 193 | if (this.connectionIds.indexOf(conn.id()) !== -1) { 194 | return 195 | } 196 | 197 | this.debug(`connecting to ${joinHostPort(host, port)}`) 198 | this.connectionIds.push(conn.id()) 199 | 200 | this.registerConnectionListeners(conn) 201 | this.readerRdy.addConnection(conn) 202 | 203 | return conn.connect() 204 | } 205 | 206 | /** 207 | * Registers event handlers for the nsqd connection. 208 | * @param {Object} conn 209 | */ 210 | registerConnectionListeners(conn) { 211 | conn.on(NSQDConnection.CONNECTED, () => { 212 | this.debug(Reader.NSQD_CONNECTED) 213 | this.emit(Reader.NSQD_CONNECTED, conn.nsqdHost, conn.nsqdPort) 214 | }) 215 | 216 | conn.on(NSQDConnection.READY, () => { 217 | // Emit only if this is the first connection for this Reader. 218 | if (this.connectionIds.length === 1) { 219 | this.debug(Reader.READY) 220 | this.emit(Reader.READY) 221 | } 222 | }) 223 | 224 | conn.on(NSQDConnection.ERROR, (err) => { 225 | this.debug(Reader.ERROR) 226 | this.debug(err) 227 | this.emit(Reader.ERROR, err) 228 | }) 229 | 230 | conn.on(NSQDConnection.CONNECTION_ERROR, (err) => { 231 | this.debug(Reader.ERROR) 232 | this.debug(err) 233 | this.emit(Reader.ERROR, err) 234 | }) 235 | 236 | // On close, remove the connection id from this reader. 237 | conn.on(NSQDConnection.CLOSED, () => { 238 | this.debug(Reader.NSQD_CLOSED) 239 | 240 | const index = this.connectionIds.indexOf(conn.id()) 241 | if (index === -1) { 242 | return 243 | } 244 | this.connectionIds.splice(index, 1) 245 | 246 | this.emit(Reader.NSQD_CLOSED, conn.nsqdHost, conn.nsqdPort) 247 | 248 | if (this.connectionIds.length === 0) { 249 | this.debug(Reader.NOT_READY) 250 | this.emit(Reader.NOT_READY) 251 | } 252 | }) 253 | 254 | /** 255 | * On message, send either a message or discard event depending on the 256 | * number of attempts. 257 | */ 258 | conn.on(NSQDConnection.MESSAGE, (message) => { 259 | this.handleMessage(message) 260 | }) 261 | } 262 | 263 | /** 264 | * Asynchronously handles an nsqd message. 265 | * 266 | * @param {Object} message 267 | */ 268 | handleMessage(message) { 269 | /** 270 | * Give the internal event listeners a chance at the events 271 | * before clients of the Reader. 272 | */ 273 | process.nextTick(() => { 274 | const autoFinishMessage = 275 | this.config.maxAttempts > 0 && 276 | this.config.maxAttempts < message.attempts 277 | const numDiscardListeners = this.listeners(Reader.DISCARD).length 278 | 279 | if (autoFinishMessage && numDiscardListeners > 0) { 280 | this.emit(Reader.DISCARD, message) 281 | } else { 282 | this.emit(Reader.MESSAGE, message) 283 | } 284 | 285 | if (autoFinishMessage) { 286 | message.finish() 287 | } 288 | }) 289 | } 290 | } 291 | 292 | module.exports = Reader 293 | -------------------------------------------------------------------------------- /lib/wire.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | 3 | const MAGIC_V2 = ' V2' 4 | const FRAME_TYPE_RESPONSE = 0 5 | const FRAME_TYPE_ERROR = 1 6 | const FRAME_TYPE_MESSAGE = 2 7 | 8 | /** 9 | * Stringifies an object. Supports unicode. 10 | * 11 | * @param {Object} obj 12 | * @param {Boolean} emitUnicode 13 | * @return {String} 14 | */ 15 | function jsonStringify(obj, emitUnicode) { 16 | const json = JSON.stringify(obj) 17 | if (emitUnicode) return json 18 | 19 | return json.replace( 20 | /[\u007f-\uffff]/g, 21 | (c) => `\\u${`0000${c.charCodeAt(0).toString(16)}`.slice(-4)}` 22 | ) 23 | } 24 | 25 | /** 26 | * Compute the byte length of an nsq message. 27 | * 28 | * @param {String|Array} msg 29 | * @return {Number} 30 | */ 31 | function byteLength(msg) { 32 | if (_.isString(msg)) return Buffer.byteLength(msg.toString()) 33 | 34 | return msg.length 35 | } 36 | 37 | /** 38 | * Unpack a message payload. The message is returned as an array in the format 39 | * [id, timestamp, attempts, body]. 40 | * 41 | * @param {Object} data 42 | * @return {Array} 43 | */ 44 | function unpackMessage(data) { 45 | let timestamp = data.readBigInt64BE(0) 46 | 47 | const attempts = data.readInt16BE(8) 48 | const id = data.slice(10, 26).toString() 49 | const body = data.slice(26) 50 | return [id, timestamp, attempts, body] 51 | } 52 | 53 | function unpackMessageId(rawMessage) { 54 | return rawMessage.slice(10, 26).toString() 55 | } 56 | 57 | function unpackMessageTimestamp(rawMessage) { 58 | return rawMessage.readBigInt64BE(0) 59 | } 60 | 61 | function unpackMessageAttempts(rawMessage) { 62 | return rawMessage.readInt16BE(8) 63 | } 64 | 65 | function unpackMessageBody(rawMessage) { 66 | return rawMessage.slice(26) 67 | } 68 | 69 | /** 70 | * Performs the requested command on the buffer. 71 | * 72 | * @param {String} cmd 73 | * @param {String} body 74 | * @return {Buffer} 75 | */ 76 | function command(cmd, body, ...parameters) { 77 | const buffers = [] 78 | 79 | if (parameters.length > 0) { 80 | parameters.unshift('') 81 | } 82 | 83 | const parametersStr = parameters.join(' ') 84 | const header = `${cmd + parametersStr}\n` 85 | 86 | buffers.push(Buffer.from(header)) 87 | 88 | // Body into output buffer it is not empty 89 | if (body != null) { 90 | // Write the size of the payload 91 | const lengthBuffer = Buffer.alloc(4) 92 | lengthBuffer.writeInt32BE(byteLength(body), 0) 93 | buffers.push(lengthBuffer) 94 | 95 | if (_.isString(body)) { 96 | buffers.push(Buffer.from(body)) 97 | } else { 98 | buffers.push(body) 99 | } 100 | } 101 | 102 | return Buffer.concat(buffers) 103 | } 104 | 105 | /** 106 | * Emit a `SUB` command. 107 | * 108 | * @param {String} topic 109 | * @param {String} channel 110 | * @return {Buffer} 111 | */ 112 | function subscribe(topic, channel) { 113 | if (!validTopicName(topic)) { 114 | throw new Error(`Invalid topic: ${topic}`) 115 | } 116 | 117 | if (!validChannelName(channel)) { 118 | throw new Error(`Invalid channel: ${channel}`) 119 | } 120 | 121 | return command('SUB', null, topic, channel) 122 | } 123 | 124 | /** 125 | * Emit an `INDENTIFY` command. 126 | * 127 | * @param {Object} data 128 | * @return {Buffer} 129 | */ 130 | function identify(data) { 131 | const validIdentifyKeys = [ 132 | 'client_id', 133 | 'deflate', 134 | 'deflate_level', 135 | 'feature_negotiation', 136 | 'hostname', 137 | 'heartbeat_interval', 138 | 'long_id', 139 | 'msg_timeout', 140 | 'output_buffer_size', 141 | 'output_buffer_timeout', 142 | 'sample_rate', 143 | 'short_id', 144 | 'snappy', 145 | 'tls_v1', 146 | 'user_agent', 147 | ] 148 | 149 | // Make sure there are no unexpected keys 150 | const unexpectedKeys = _.filter( 151 | _.keys(data), 152 | (k) => !Array.from(validIdentifyKeys).includes(k) 153 | ) 154 | 155 | if (unexpectedKeys.length) { 156 | throw new Error(`Unexpected IDENTIFY keys: ${unexpectedKeys}`) 157 | } 158 | 159 | return command('IDENTIFY', jsonStringify(data)) 160 | } 161 | 162 | /** 163 | * Emit a `RDY` command. 164 | * 165 | * @param {Number} count 166 | * @return {Buffer} 167 | */ 168 | function ready(count) { 169 | if (!_.isNumber(count)) { 170 | throw new Error(`RDY count (${count}) is not a number`) 171 | } 172 | 173 | if (!(count >= 0)) { 174 | throw new Error(`RDY count (${count}) is not positive`) 175 | } 176 | 177 | return command('RDY', null, count.toString()) 178 | } 179 | 180 | /** 181 | * Emit a `FIN` command. 182 | * 183 | * @param {String} id 184 | * @return {Buffer} 185 | */ 186 | function finish(id) { 187 | if (!(Buffer.byteLength(id) <= 16)) { 188 | throw new Error(`FINISH invalid id (${id})`) 189 | } 190 | return command('FIN', null, id) 191 | } 192 | 193 | function close() { 194 | return command('CLS', null) 195 | } 196 | 197 | /** 198 | * Emit a requeue command. 199 | * 200 | * @param {String} id 201 | * @param {Number} timeMs 202 | * @return {Buffer} 203 | */ 204 | function requeue(id, timeMs = 0) { 205 | if (!(Buffer.byteLength(id) <= 16)) { 206 | throw new Error(`REQUEUE invalid id (${id})`) 207 | } 208 | 209 | if (!_.isNumber(timeMs)) { 210 | throw new Error(`REQUEUE delay time is invalid (${timeMs})`) 211 | } 212 | 213 | const parameters = ['REQ', null, id, timeMs] 214 | return command(...parameters) 215 | } 216 | 217 | /** 218 | * Emit a `TOUCH` command. 219 | * 220 | * @param {String} id 221 | * @return {Buffer} 222 | */ 223 | function touch(id) { 224 | return command('TOUCH', null, id) 225 | } 226 | 227 | /** 228 | * Emit a `NOP` command. 229 | * 230 | * @return {Buffer} 231 | */ 232 | function nop() { 233 | return command('NOP', null) 234 | } 235 | 236 | /** 237 | * Emit a `PUB` command. 238 | * 239 | * @param {String} topic 240 | * @param {Object} data 241 | * @return {Buffer} 242 | */ 243 | function pub(topic, data) { 244 | return command('PUB', data, topic) 245 | } 246 | 247 | /** 248 | * Emit an `MPUB` command. 249 | * 250 | * @param {String} topic 251 | * @param {Object} data 252 | * @return {Buffer} 253 | */ 254 | function mpub(topic, data) { 255 | if (!_.isArray(data)) { 256 | throw new Error('MPUB requires an array of message') 257 | } 258 | const messages = _.map(data, (message) => { 259 | const buffer = Buffer.alloc(4 + byteLength(message)) 260 | buffer.writeInt32BE(byteLength(message), 0) 261 | 262 | if (_.isString(message)) { 263 | buffer.write(message, 4) 264 | } else { 265 | message.copy(buffer, 4, 0, buffer.length) 266 | } 267 | 268 | return buffer 269 | }) 270 | 271 | const numMessagesBuffer = Buffer.alloc(4) 272 | numMessagesBuffer.writeInt32BE(messages.length, 0) 273 | messages.unshift(numMessagesBuffer) 274 | 275 | return command('MPUB', Buffer.concat(messages), topic) 276 | } 277 | 278 | /** 279 | * Emit a `DPUB` command. 280 | * 281 | * @param {String} topic 282 | * @param {Object} data 283 | * @param {Number} timeMs 284 | * @return {Buffer} 285 | */ 286 | function dpub(topic, data, timeMs = 0) { 287 | return command('DPUB', data, topic, timeMs) 288 | } 289 | 290 | /** 291 | * Emit an `AUTH` command. 292 | * 293 | * @param {String} token 294 | * @return {Buffer} 295 | */ 296 | function auth(token) { 297 | return command('AUTH', token) 298 | } 299 | 300 | /** 301 | * Validate topic names. Topic names must be no longer than 302 | * 65 characters. 303 | * 304 | * @param {String} topic 305 | * @return {Boolean} 306 | */ 307 | function validTopicName(topic) { 308 | return ( 309 | topic && 310 | topic.length > 0 && 311 | topic.length < 65 && 312 | topic.match(/^[\w._-]+(?:#ephemeral)?$/) != null 313 | ) 314 | } 315 | 316 | /** 317 | * Validate channel names. Follows the same restriction as 318 | * topic names. 319 | * 320 | * @param {String} channel 321 | * @return {Boolean} 322 | */ 323 | function validChannelName(channel) { 324 | return ( 325 | channel && 326 | channel.length > 0 && 327 | channel.length < 65 && 328 | channel.match(/^[\w._-]+(?:#ephemeral)?$/) != null 329 | ) 330 | } 331 | 332 | module.exports = { 333 | FRAME_TYPE_ERROR, 334 | FRAME_TYPE_MESSAGE, 335 | FRAME_TYPE_RESPONSE, 336 | MAGIC_V2, 337 | auth, 338 | close, 339 | dpub, 340 | finish, 341 | identify, 342 | mpub, 343 | nop, 344 | pub, 345 | ready, 346 | requeue, 347 | subscribe, 348 | touch, 349 | unpackMessage, 350 | unpackMessageId, 351 | unpackMessageTimestamp, 352 | unpackMessageAttempts, 353 | unpackMessageBody, 354 | } 355 | -------------------------------------------------------------------------------- /test/config_test.js: -------------------------------------------------------------------------------- 1 | const {ConnectionConfig, ReaderConfig} = require('../lib/config') 2 | 3 | describe('ConnectionConfig', () => { 4 | let config = null 5 | 6 | beforeEach(() => { 7 | config = new ConnectionConfig() 8 | }) 9 | 10 | it('should use all defaults if nothing is provided', () => { 11 | config.maxInFlight.should.eql(1) 12 | }) 13 | 14 | it('should validate with defaults', () => { 15 | const check = () => config.validate() 16 | check.should.not.throw() 17 | }) 18 | 19 | it('should remove an unrecognized option', () => { 20 | config = new ConnectionConfig({unknownOption: 20}) 21 | config.should.not.have.property('unknownOption') 22 | }) 23 | 24 | describe('isNonEmptyString', () => { 25 | it('should correctly validate a non-empty string', () => { 26 | const check = () => config.isNonEmptyString('name', 'worker') 27 | check.should.not.throw() 28 | }) 29 | 30 | it('should throw on an empty string', () => { 31 | const check = () => config.isNonEmptyString('name', '') 32 | check.should.throw() 33 | }) 34 | 35 | it('should throw on a non-string', () => { 36 | const check = () => config.isNonEmptyString('name', {}) 37 | check.should.throw() 38 | }) 39 | }) 40 | 41 | describe('isNumber', () => { 42 | it('should validate with a value equal to the lower bound', () => { 43 | const check = () => config.isNumber('maxInFlight', 1, 1) 44 | check.should.not.throw() 45 | }) 46 | 47 | it('should validate with a value between the lower and upper bound', () => { 48 | const check = () => config.isNumber('maxInFlight', 5, 1, 10) 49 | check.should.not.throw() 50 | }) 51 | 52 | it('should validate with a value equal to the upper bound', () => { 53 | const check = () => config.isNumber('maxInFlight', 10, 1, 10) 54 | check.should.not.throw() 55 | }) 56 | 57 | it('should not validate with a value less than the lower bound', () => { 58 | const check = () => config.isNumber('maxInFlight', -1, 1) 59 | check.should.throw() 60 | }) 61 | 62 | it('should not validate with a value greater than the upper bound', () => { 63 | const check = () => config.isNumber('maxInFlight', 11, 1, 10) 64 | check.should.throw() 65 | }) 66 | 67 | it('should not validate against a non-number', () => { 68 | const check = () => config.isNumber('maxInFlight', null, 0) 69 | check.should.throw() 70 | }) 71 | }) 72 | 73 | describe('isNumberExclusive', () => { 74 | it('should not validate with a value equal to the lower bound', () => { 75 | const check = () => config.isNumberExclusive('maxInFlight', 1, 1) 76 | check.should.throw() 77 | }) 78 | 79 | it('should validate with a value between the lower and upper bound', () => { 80 | const check = () => config.isNumberExclusive('maxInFlight', 5, 1, 10) 81 | check.should.not.throw() 82 | }) 83 | 84 | it('should not validate with a value equal to the upper bound', () => { 85 | const check = () => config.isNumberExclusive('maxInFlight', 10, 1, 10) 86 | check.should.throw() 87 | }) 88 | 89 | it('should not validate with a value less than the lower bound', () => { 90 | const check = () => config.isNumberExclusive('maxInFlight', -1, 1) 91 | check.should.throw() 92 | }) 93 | 94 | it('should not validate with a value greater than the upper bound', () => { 95 | const check = () => config.isNumberExclusive('maxInFlight', 11, 1, 10) 96 | check.should.throw() 97 | }) 98 | 99 | it('should not validate against a non-number', () => { 100 | const check = () => config.isNumberExclusive('maxInFlight', null, 0) 101 | check.should.throw() 102 | }) 103 | }) 104 | 105 | describe('isBoolean', () => { 106 | it('should validate against true', () => { 107 | const check = () => config.isBoolean('tls', true) 108 | check.should.not.throw() 109 | }) 110 | 111 | it('should validate against false', () => { 112 | const check = () => config.isBoolean('tls', false) 113 | check.should.not.throw() 114 | }) 115 | 116 | it('should not validate against null', () => { 117 | const check = () => config.isBoolean('tls', null) 118 | check.should.throw() 119 | }) 120 | 121 | it('should not validate against a non-boolean value', () => { 122 | const check = () => config.isBoolean('tls', 'hi') 123 | check.should.throw() 124 | }) 125 | }) 126 | 127 | describe('isBuffer', () => { 128 | it('should require tls keys to be buffers', () => { 129 | const check = () => config.isBuffer('key', Buffer.from('a buffer')) 130 | check.should.not.throw() 131 | }) 132 | 133 | it('should require tls keys to be buffers', () => { 134 | const check = () => config.isBuffer('key', 'not a buffer') 135 | check.should.throw() 136 | }) 137 | 138 | it('should require tls certs to be buffers', () => { 139 | const check = () => 140 | config.isBuffer('cert', Buffer.from('definitely a buffer')) 141 | check.should.not.throw() 142 | }) 143 | 144 | it('should throw when a tls cert is not a buffer', () => { 145 | const check = () => config.isBuffer('cert', 'still not a buffer') 146 | check.should.throw() 147 | }) 148 | }) 149 | 150 | describe('isArray', () => { 151 | it('should require cert authority chains to be arrays', () => { 152 | const check = () => config.isArray('ca', ['cat', 'dog']) 153 | check.should.not.throw() 154 | }) 155 | 156 | it('should require cert authority chains to be arrays', () => { 157 | const check = () => config.isArray('ca', 'not an array') 158 | check.should.throw() 159 | }) 160 | }) 161 | 162 | describe('isBareAddresses', () => { 163 | it('should validate against a validate address list of 1', () => { 164 | const check = () => 165 | config.isBareAddresses('nsqdTCPAddresses', ['127.0.0.1:4150']) 166 | check.should.not.throw() 167 | }) 168 | 169 | it('should validate against a validate ipv6 address list of 1', () => { 170 | const check = () => 171 | config.isBareAddresses('nsqdTCPAddresses', ['[::1]:4150']) 172 | check.should.not.throw() 173 | }) 174 | 175 | it('should validate against a validate address list of 2', () => { 176 | const check = () => { 177 | const addrs = ['127.0.0.1:4150', 'localhost:4150'] 178 | config.isBareAddresses('nsqdTCPAddresses', addrs) 179 | } 180 | check.should.not.throw() 181 | }) 182 | 183 | it('should validate against a validate ipv6 address list of 2', () => { 184 | const check = () => 185 | config.isBareAddresses('nsqdTCPAddresses', ['[::1]:4150', '[::]:4150']) 186 | check.should.not.throw() 187 | }) 188 | 189 | it('should not validate non-numeric port', () => { 190 | const check = () => 191 | config.isBareAddresses('nsqdTCPAddresses', ['localhost']) 192 | check.should.throw() 193 | }) 194 | 195 | it('should invalidate ipv6 address port', () => { 196 | const check = () => 197 | config.isBareAddresses('nsqdTCPAddresses', ['[::1]']) 198 | check.should.throw() 199 | }) 200 | }) 201 | 202 | describe('isLookupdHTTPAddresses', () => { 203 | it('should validate against a validate address list of 1', () => { 204 | const check = () => 205 | config.isLookupdHTTPAddresses('lookupdHTTPAddresses', [ 206 | '127.0.0.1:4150', 207 | ]) 208 | check.should.not.throw() 209 | }) 210 | 211 | it('should validate against a validate address list of 2', () => { 212 | const check = () => { 213 | const addrs = [ 214 | '127.0.0.1:4150', 215 | 'localhost:4150', 216 | '[::1]:4150', 217 | '[::]:4150', 218 | 'http://localhost/nsq/lookup', 219 | 'https://localhost/nsq/lookup', 220 | ] 221 | config.isLookupdHTTPAddresses('lookupdHTTPAddresses', addrs) 222 | } 223 | check.should.not.throw() 224 | }) 225 | 226 | it('should not validate non-numeric port', () => { 227 | const check = () => 228 | config.isLookupdHTTPAddresses('lookupdHTTPAddresses', ['localhost']) 229 | check.should.throw() 230 | }) 231 | 232 | it('should not validate non-HTTP/HTTPs address', () => { 233 | const check = () => 234 | config.isLookupdHTTPAddresses('lookupdHTTPAddresses', ['localhost']) 235 | check.should.throw() 236 | }) 237 | }) 238 | }) 239 | 240 | describe('ReaderConfig', () => { 241 | let config = null 242 | 243 | beforeEach(() => { 244 | config = new ReaderConfig() 245 | }) 246 | 247 | it('should use all defaults if nothing is provided', () => { 248 | config.maxInFlight.should.eql(1) 249 | }) 250 | 251 | it('should validate with defaults', () => { 252 | const check = () => { 253 | config = new ReaderConfig({nsqdTCPAddresses: ['127.0.0.1:4150']}) 254 | config.validate() 255 | } 256 | check.should.not.throw() 257 | }) 258 | 259 | it('should convert a string address to an array', () => { 260 | config = new ReaderConfig({lookupdHTTPAddresses: '127.0.0.1:4161'}) 261 | config.validate() 262 | config.lookupdHTTPAddresses.length.should.equal(1) 263 | }) 264 | }) 265 | -------------------------------------------------------------------------------- /test/nsqdconnection_test.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const should = require('should') 3 | const sinon = require('sinon') 4 | const rawMessage = require('./rawmessage') 5 | 6 | const wire = require('../lib/wire') 7 | const { 8 | ConnectionState, 9 | NSQDConnection, 10 | WriterNSQDConnection, 11 | WriterConnectionState, 12 | } = require('../lib/nsqdconnection') 13 | 14 | describe('Reader ConnectionState', () => { 15 | const state = { 16 | sent: [], 17 | connection: null, 18 | statemachine: null, 19 | } 20 | 21 | beforeEach(() => { 22 | const sent = [] 23 | 24 | const connection = new NSQDConnection( 25 | '127.0.0.1', 26 | 4150, 27 | 'topic_test', 28 | 'channel_test' 29 | ) 30 | sinon 31 | .stub(connection, 'write') 32 | .callsFake((data) => sent.push(data.toString())) 33 | sinon.stub(connection, 'close').callsFake(() => {}) 34 | sinon.stub(connection, 'destroy').callsFake(() => {}) 35 | 36 | const statemachine = new ConnectionState(connection) 37 | 38 | return _.extend(state, { 39 | sent, 40 | connection, 41 | statemachine, 42 | }) 43 | }) 44 | 45 | it('handle initial handshake', () => { 46 | const {statemachine, sent} = state 47 | statemachine.raise('connecting') 48 | statemachine.raise('connected') 49 | sent[0].should.match(/^ {2}V2$/) 50 | sent[1].should.match(/^IDENTIFY/) 51 | }) 52 | 53 | it('handle OK identify response', () => { 54 | const {statemachine, connection} = state 55 | statemachine.raise('connecting') 56 | statemachine.raise('connected') 57 | statemachine.raise('response', Buffer.from('OK')) 58 | 59 | should.equal(connection.maxRdyCount, 2500) 60 | should.equal(connection.maxMsgTimeout, 900000) 61 | should.equal(connection.msgTimeout, 60000) 62 | }) 63 | 64 | it('handle identify response', () => { 65 | const {statemachine, connection} = state 66 | statemachine.raise('connecting') 67 | statemachine.raise('connected') 68 | 69 | statemachine.raise( 70 | 'response', 71 | JSON.stringify({ 72 | max_rdy_count: 1000, 73 | max_msg_timeout: 10 * 60 * 1000, 74 | msg_timeout: 2 * 60 * 1000, 75 | }) 76 | ) 77 | 78 | should.equal(connection.maxRdyCount, 1000) 79 | should.equal(connection.maxMsgTimeout, 600000) 80 | should.equal(connection.msgTimeout, 120000) 81 | }) 82 | 83 | it('create a subscription', (done) => { 84 | const {sent, statemachine, connection} = state 85 | 86 | // Subscribe notification 87 | connection.on(NSQDConnection.READY, () => done()) 88 | 89 | statemachine.raise('connecting') 90 | statemachine.raise('connected') 91 | statemachine.raise('response', 'OK') // Identify response 92 | 93 | sent[2].should.match(/^SUB topic_test channel_test\n$/) 94 | statemachine.raise('response', 'OK') 95 | }) 96 | 97 | it('handle a message', (done) => { 98 | const {statemachine, connection} = state 99 | connection.on(NSQDConnection.MESSAGE, () => done()) 100 | 101 | statemachine.raise('connecting') 102 | statemachine.raise('connected') 103 | statemachine.raise('response', 'OK') // Identify response 104 | statemachine.raise('response', 'OK') // Subscribe response 105 | 106 | should.equal(statemachine.current_state_name, 'READY_RECV') 107 | 108 | statemachine.raise('consumeMessage', {}) 109 | should.equal(statemachine.current_state_name, 'READY_RECV') 110 | }) 111 | 112 | it('handle a message finish after a disconnect', (done) => { 113 | const {statemachine, connection} = state 114 | sinon 115 | .stub(wire, 'unpackMessage') 116 | .callsFake(() => ['1', 0, 0, Buffer.from(''), 60, 60, 120]) 117 | 118 | connection.on(NSQDConnection.MESSAGE, (msg) => { 119 | const fin = () => { 120 | msg.finish() 121 | done() 122 | } 123 | setTimeout(fin, 10) 124 | }) 125 | 126 | // Advance the connection to the READY state. 127 | statemachine.raise('connecting') 128 | statemachine.raise('connected') 129 | statemachine.raise('response', 'OK') // Identify response 130 | statemachine.raise('response', 'OK') // Subscribe response 131 | 132 | // Receive message 133 | const msg = connection.createMessage(rawMessage('1', Date.now(), 0, 'msg')) 134 | statemachine.raise('consumeMessage', msg) 135 | 136 | // Close the connection before the message has been processed. 137 | connection.destroy() 138 | statemachine.goto('CLOSED') 139 | 140 | // Undo stub 141 | wire.unpackMessage.restore() 142 | }) 143 | 144 | it('handles non-fatal errors', (done) => { 145 | const {connection, statemachine} = state 146 | 147 | // Note: we still want an error event raised, just not a closed connection 148 | connection.on(NSQDConnection.ERROR, () => done()) 149 | 150 | // Yields an error if the connection actually closes 151 | connection.on(NSQDConnection.CLOSED, () => { 152 | done(new Error('Should not have closed!')) 153 | }) 154 | 155 | statemachine.goto('ERROR', new Error('E_REQ_FAILED')) 156 | }) 157 | }) 158 | 159 | describe('WriterConnectionState', () => { 160 | const state = { 161 | sent: [], 162 | connection: null, 163 | statemachine: null, 164 | } 165 | 166 | beforeEach(() => { 167 | const sent = [] 168 | const connection = new WriterNSQDConnection('127.0.0.1', 4150) 169 | sinon.stub(connection, 'destroy') 170 | 171 | sinon.stub(connection, 'write').callsFake((data) => { 172 | sent.push(data.toString()) 173 | }) 174 | 175 | const statemachine = new WriterConnectionState(connection) 176 | connection.statemachine = statemachine 177 | 178 | _.extend(state, { 179 | sent, 180 | connection, 181 | statemachine, 182 | }) 183 | }) 184 | 185 | it('should generate a READY event after IDENTIFY', (done) => { 186 | const {statemachine, connection} = state 187 | 188 | connection.on(WriterNSQDConnection.READY, () => { 189 | should.equal(statemachine.current_state_name, 'READY_SEND') 190 | done() 191 | }) 192 | 193 | statemachine.raise('connecting') 194 | statemachine.raise('connected') 195 | statemachine.raise('response', 'OK') 196 | }) 197 | 198 | it('should use PUB when sending a single message', (done) => { 199 | const {statemachine, connection, sent} = state 200 | 201 | connection.on(WriterNSQDConnection.READY, () => { 202 | connection.produceMessages('test', ['one']) 203 | sent[sent.length - 1].should.match(/^PUB/) 204 | done() 205 | }) 206 | 207 | statemachine.raise('connecting') 208 | statemachine.raise('connected') 209 | statemachine.raise('response', 'OK') 210 | }) 211 | 212 | it('should use MPUB when sending multiplie messages', (done) => { 213 | const {statemachine, connection, sent} = state 214 | 215 | connection.on(WriterNSQDConnection.READY, () => { 216 | connection.produceMessages('test', ['one', 'two']) 217 | sent[sent.length - 1].should.match(/^MPUB/) 218 | done() 219 | }) 220 | 221 | statemachine.raise('connecting') 222 | statemachine.raise('connected') 223 | statemachine.raise('response', 'OK') 224 | }) 225 | 226 | it('should call the callback when supplied on publishing a message', (done) => { 227 | const {statemachine, connection} = state 228 | 229 | connection.on(WriterNSQDConnection.READY, () => { 230 | connection.produceMessages('test', ['one'], undefined, () => done()) 231 | statemachine.raise('response', 'OK') 232 | }) 233 | 234 | statemachine.raise('connecting') 235 | statemachine.raise('connected') 236 | statemachine.raise('response', 'OK') 237 | }) 238 | 239 | it('should call the the right callback on several messages', (done) => { 240 | const {statemachine, connection} = state 241 | 242 | connection.on(WriterNSQDConnection.READY, () => { 243 | connection.produceMessages('test', ['one'], undefined) 244 | connection.produceMessages('test', ['two'], undefined, () => { 245 | // There should be no more callbacks 246 | should.equal(connection.messageCallbacks.length, 0) 247 | done() 248 | }) 249 | 250 | statemachine.raise('response', 'OK') 251 | statemachine.raise('response', 'OK') 252 | }) 253 | 254 | statemachine.raise('connecting') 255 | statemachine.raise('connected') 256 | statemachine.raise('response', 'OK') 257 | }) 258 | 259 | it('should call all callbacks on nsqd disconnect', (done) => { 260 | const {statemachine, connection} = state 261 | 262 | const firstCb = sinon.spy() 263 | const secondCb = sinon.spy() 264 | 265 | connection.on(WriterNSQDConnection.ERROR, () => {}) 266 | 267 | connection.on(WriterNSQDConnection.READY, () => { 268 | connection.produceMessages('test', ['one'], undefined, firstCb) 269 | connection.produceMessages('test', ['two'], undefined, secondCb) 270 | statemachine.goto('ERROR', 'lost connection') 271 | }) 272 | 273 | connection.on(WriterNSQDConnection.CLOSED, () => { 274 | firstCb.calledOnce.should.be.ok() 275 | secondCb.calledOnce.should.be.ok() 276 | done() 277 | }) 278 | 279 | statemachine.raise('connecting') 280 | statemachine.raise('connected') 281 | statemachine.raise('response', 'OK') 282 | }) 283 | }) 284 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const url = require('url') 3 | const net = require('net') 4 | 5 | function joinHostPort(host, port) { 6 | if (net.isIPv6(host)) { 7 | return `[${host}]:${port}` 8 | } 9 | 10 | return `${host}:${port}` 11 | } 12 | 13 | function splitHostPort(addr) { 14 | const parts = addr.split(':') 15 | const port = parts[parts.length - 1] 16 | let host = parts.slice(0, parts.length - 1).join() 17 | 18 | if (host[0] === '[' && host[host.length - 1] === ']') { 19 | host = host.slice(1, host.length - 1) 20 | } 21 | 22 | return [ 23 | host, 24 | port, 25 | ] 26 | } 27 | 28 | /** 29 | * Responsible for configuring the official defaults for nsqd connections. 30 | * @type {ConnectionConfig} 31 | */ 32 | class ConnectionConfig { 33 | static get DEFAULTS() { 34 | return { 35 | authSecret: null, 36 | clientId: null, 37 | deflate: false, 38 | deflateLevel: 6, 39 | heartbeatInterval: 30, 40 | maxInFlight: 1, 41 | messageTimeout: null, 42 | outputBufferSize: null, 43 | outputBufferTimeout: null, 44 | requeueDelay: 90000, 45 | sampleRate: null, 46 | snappy: false, 47 | tls: false, 48 | tlsVerification: true, 49 | key: null, 50 | cert: null, 51 | ca: null, 52 | idleTimeout: 0, 53 | } 54 | } 55 | 56 | /** 57 | * Indicates if an address has the host pair combo. 58 | * 59 | * @param {String} addr 60 | * @return {Boolean} 61 | */ 62 | static isBareAddress(addr) { 63 | const [host, port] = splitHostPort(addr) 64 | return host.length > 0 && port > 0 65 | } 66 | 67 | /** 68 | * Instantiates a new ConnectionConfig. 69 | * 70 | * @constructor 71 | * @param {Object} [options={}] 72 | */ 73 | constructor(options = {}) { 74 | Object.assign(this, this.constructor.DEFAULTS) 75 | 76 | // Pick only supported options. 77 | Object.keys(this.constructor.DEFAULTS).forEach((key) => { 78 | if (Object.prototype.hasOwnProperty.call(options, key)) { 79 | this[key] = options[key] 80 | } 81 | }) 82 | } 83 | 84 | /** 85 | * Throws an error if the value is not a non empty string. 86 | * 87 | * @param {String} option 88 | * @param {*} value 89 | */ 90 | isNonEmptyString(option, value) { 91 | if (!_.isString(value) || !(value.length > 0)) { 92 | throw new Error(`${option} must be a non-empty string`) 93 | } 94 | } 95 | 96 | /** 97 | * Throws an error if the value is not a number. 98 | * 99 | * @param {String} option 100 | * @param {*} value 101 | * @param {*} lower 102 | * @param {*} upper 103 | */ 104 | isNumber(option, value, lower, upper) { 105 | if (_.isNaN(value) || !_.isNumber(value)) { 106 | throw new Error(`${option}(${value}) is not a number`) 107 | } 108 | 109 | if (upper) { 110 | if (!(lower <= value && value <= upper)) { 111 | throw new Error(`${lower} <= ${option}(${value}) <= ${upper}`) 112 | } 113 | } else if (!(lower <= value)) { 114 | throw new Error(`${lower} <= ${option}(${value})`) 115 | } 116 | } 117 | 118 | /** 119 | * Throws an error if the value is not exclusive. 120 | * 121 | * @param {String} option 122 | * @param {*} value 123 | * @param {*} lower 124 | * @param {*} upper 125 | */ 126 | isNumberExclusive(option, value, lower, upper) { 127 | if (_.isNaN(value) || !_.isNumber(value)) { 128 | throw new Error(`${option}(${value}) is not a number`) 129 | } 130 | 131 | if (upper) { 132 | if (!(lower < value && value < upper)) { 133 | throw new Error(`${lower} < ${option}(${value}) < ${upper}`) 134 | } 135 | } else if (!(lower < value)) { 136 | throw new Error(`${lower} < ${option}(${value})`) 137 | } 138 | } 139 | 140 | /** 141 | * Throws an error if the option is not a Boolean. 142 | * 143 | * @param {String} option 144 | * @param {*} value 145 | */ 146 | isBoolean(option, value) { 147 | if (!_.isBoolean(value)) { 148 | throw new Error(`${option} must be either true or false`) 149 | } 150 | } 151 | 152 | /** 153 | * Throws an error if the option is not a bare address. 154 | * 155 | * @param {String} option 156 | * @param {*} value 157 | */ 158 | isBareAddresses(option, value) { 159 | if (!_.isArray(value) || !_.every(value, ConnectionConfig.isBareAddress)) { 160 | throw new Error(`${option} must be a list of addresses 'host:port'`) 161 | } 162 | } 163 | 164 | /** 165 | * Throws an error if the option is not a valid lookupd http address. 166 | * 167 | * @param {String} option 168 | * @param {*} value 169 | */ 170 | isLookupdHTTPAddresses(option, value) { 171 | const isAddr = (addr) => { 172 | if (addr.indexOf('://') === -1) { 173 | return ConnectionConfig.isBareAddress(addr) 174 | } 175 | 176 | const parsedUrl = url.parse(addr) 177 | return ( 178 | ['http:', 'https:'].includes(parsedUrl.protocol) && !!parsedUrl.host 179 | ) 180 | } 181 | 182 | if (!_.isArray(value) || !_.every(value, isAddr)) { 183 | throw new Error( 184 | `${option} must be a list of addresses 'host:port' or \ 185 | HTTP/HTTPS URI` 186 | ) 187 | } 188 | } 189 | 190 | /** 191 | * Throws an error if the option is not a buffer. 192 | * 193 | * @param {String} option 194 | * @param {*} value 195 | */ 196 | isBuffer(option, value) { 197 | if (!Buffer.isBuffer(value)) { 198 | throw new Error(`${option} must be a buffer`) 199 | } 200 | } 201 | 202 | /** 203 | * Throws an error if the option is not an array. 204 | * 205 | * @param {String} option 206 | * @param {*} value 207 | */ 208 | isArray(option, value) { 209 | if (!_.isArray(value)) { 210 | throw new Error(`${option} must be an array`) 211 | } 212 | } 213 | 214 | /** 215 | * Returns the validated client config. Throws an error if any values are 216 | * not correct. 217 | * 218 | * @return {Object} 219 | */ 220 | conditions() { 221 | return { 222 | authSecret: [this.isNonEmptyString], 223 | clientId: [this.isNonEmptyString], 224 | deflate: [this.isBoolean], 225 | deflateLevel: [this.isNumber, 0, 9], 226 | heartbeatInterval: [this.isNumber, 1], 227 | maxInFlight: [this.isNumber, 1], 228 | messageTimeout: [this.isNumber, 1], 229 | outputBufferSize: [this.isNumber, 64], 230 | outputBufferTimeout: [this.isNumber, 1], 231 | requeueDelay: [this.isNumber, 0], 232 | sampleRate: [this.isNumber, 1, 99], 233 | snappy: [this.isBoolean], 234 | tls: [this.isBoolean], 235 | tlsVerification: [this.isBoolean], 236 | key: [this.isBuffer], 237 | cert: [this.isBuffer], 238 | ca: [this.isBuffer], 239 | idleTimeout: [this.isNumber, 0], 240 | } 241 | } 242 | 243 | /** 244 | * Helper function that will validate a condition with the given args. 245 | * 246 | * @param {String} option 247 | * @param {String} value 248 | * @return {Boolean} 249 | */ 250 | validateOption(option, value) { 251 | const [fn, ...args] = this.conditions()[option] 252 | return fn(option, value, ...args) 253 | } 254 | 255 | /** 256 | * Validate the connection options. 257 | */ 258 | validate() { 259 | const options = Object.keys(this) 260 | for (const option of options) { 261 | // dont validate our methods 262 | const value = this[option] 263 | 264 | if (_.isFunction(value)) { 265 | continue 266 | } 267 | 268 | // Skip options that default to null 269 | if (_.isNull(value) && this.constructor.DEFAULTS[option] === null) { 270 | continue 271 | } 272 | 273 | // Disabled via -1 274 | const keys = ['outputBufferSize', 'outputBufferTimeout'] 275 | if (keys.includes(option) && value === -1) { 276 | continue 277 | } 278 | 279 | this.validateOption(option, value) 280 | } 281 | 282 | // Mutually exclusive options 283 | if (this.snappy && this.deflate) { 284 | throw new Error('Cannot use both deflate and snappy') 285 | } 286 | 287 | if (this.snappy) { 288 | try { 289 | require('snappystream') 290 | } catch (err) { 291 | throw new Error( 292 | 'Cannot use snappy since it did not successfully install via npm.' 293 | ) 294 | } 295 | } 296 | } 297 | } 298 | 299 | /** 300 | * Responsible for configuring the official defaults for Reader connections. 301 | * @type {[type]} 302 | */ 303 | class ReaderConfig extends ConnectionConfig { 304 | static get DEFAULTS() { 305 | return _.extend({}, ConnectionConfig.DEFAULTS, { 306 | lookupdHTTPAddresses: [], 307 | lookupdPollInterval: 60, 308 | lookupdPollJitter: 0.3, 309 | lowRdyTimeout: 50, 310 | name: null, 311 | nsqdTCPAddresses: [], 312 | maxAttempts: 0, 313 | maxBackoffDuration: 128, 314 | }) 315 | } 316 | 317 | /** 318 | * Returns the validated reader client config. Throws an error if any 319 | * values are not correct. 320 | * 321 | * @return {Object} 322 | */ 323 | conditions() { 324 | return _.extend({}, super.conditions(), { 325 | lookupdHTTPAddresses: [this.isLookupdHTTPAddresses], 326 | lookupdPollInterval: [this.isNumber, 1], 327 | lookupdPollJitter: [this.isNumberExclusive, 0, 1], 328 | lowRdyTimeout: [this.isNumber, 1], 329 | name: [this.isNonEmptyString], 330 | nsqdTCPAddresses: [this.isBareAddresses], 331 | maxAttempts: [this.isNumber, 0], 332 | maxBackoffDuration: [this.isNumber, 0], 333 | }) 334 | } 335 | 336 | /** 337 | * Validate the connection options. 338 | */ 339 | validate(...args) { 340 | const addresses = ['nsqdTCPAddresses', 'lookupdHTTPAddresses'] 341 | 342 | /** 343 | * Either a string or list of strings can be provided. Ensure list of 344 | * strings going forward. 345 | */ 346 | for (const key of Array.from(addresses)) { 347 | if (_.isString(this[key])) { 348 | this[key] = [this[key]] 349 | } 350 | } 351 | 352 | super.validate(...args) 353 | 354 | const pass = _.chain(addresses) 355 | .map((key) => this[key].length) 356 | .some(_.identity) 357 | .value() 358 | 359 | if (!pass) { 360 | throw new Error(`Need to provide either ${addresses.join(' or ')}`) 361 | } 362 | } 363 | } 364 | 365 | module.exports = {ConnectionConfig, ReaderConfig, joinHostPort, splitHostPort} 366 | -------------------------------------------------------------------------------- /test/z_integration_test.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash') 2 | const child_process = require('child_process') // eslint-disable-line camelcase 3 | const fetch = require('node-fetch') 4 | const pEvent = require('p-event') 5 | const retry = require('async-retry') 6 | const should = require('should') 7 | const temp = require('temp').track() 8 | const url = require('url') 9 | const util = require('util') 10 | 11 | const nsq = require('../lib/nsq') 12 | const EventEmitter = require('events') 13 | 14 | let TCP_PORT = 4150 15 | let HTTP_PORT = 4151 16 | 17 | const startNSQD = async (dataPath, additionalOptions = {}) => { 18 | let options = { 19 | 'http-address': `127.0.0.1:${HTTP_PORT}`, 20 | 'tcp-address': `127.0.0.1:${TCP_PORT}`, 21 | 'broadcast-address': '127.0.0.1', 22 | 'data-path': dataPath, 23 | 'tls-cert': './test/cert.pem', 24 | 'tls-key': './test/key.pem', 25 | } 26 | 27 | _.extend(options, additionalOptions) 28 | 29 | // Convert to array for child_process. 30 | options = Object.keys(options).map((option) => [ 31 | `-${option}`, 32 | options[option], 33 | ]) 34 | 35 | const process = child_process.spawn('nsqd', _.flatten(options), { 36 | stdio: ['ignore', 'ignore', 'ignore'], 37 | }) 38 | 39 | process.on('error', (err) => { 40 | throw err 41 | }) 42 | 43 | await retry( 44 | async () => { 45 | const response = await fetch(`http://127.0.0.1:${HTTP_PORT}/ping`) 46 | if (!response.ok) { 47 | throw new Error('not ready') 48 | } 49 | }, 50 | {retries: 10, minTimeout: 50} 51 | ) 52 | 53 | return process 54 | } 55 | 56 | const topicOp = async (op, topic) => { 57 | const u = new url.URL(`http://127.0.0.1:${HTTP_PORT}/${op}`) 58 | u.searchParams.set('topic', topic) 59 | 60 | await fetch(u.toString(), {method: 'POST'}) 61 | } 62 | 63 | const createTopic = async (topic) => topicOp('topic/create', topic) 64 | const deleteTopic = async (topic) => topicOp('topic/delete', topic) 65 | 66 | // Publish a single message via HTTP 67 | const publish = async (topic, message) => { 68 | const u = new url.URL(`http://127.0.0.1:${HTTP_PORT}/pub`) 69 | u.searchParams.set('topic', topic) 70 | 71 | await fetch(u.toString(), { 72 | method: 'POST', 73 | body: message, 74 | }) 75 | } 76 | 77 | describe('integration', () => { 78 | let nsqdProcess = null 79 | let reader = null 80 | 81 | beforeEach(async () => { 82 | nsqdProcess = await startNSQD(await temp.mkdir('/nsq')) 83 | await createTopic('test') 84 | }) 85 | 86 | afterEach(async () => { 87 | const closeEvent = pEvent(reader, 'nsqd_closed') 88 | const exitEvent = pEvent(nsqdProcess, 'exit') 89 | 90 | reader.close() 91 | await closeEvent 92 | 93 | await deleteTopic('test') 94 | 95 | nsqdProcess.kill('SIGKILL') 96 | await exitEvent 97 | 98 | // After each start, increment the ports to prevent possible conflict the 99 | // next time an NSQD instance is started. Sometimes NSQD instances do not 100 | // exit cleanly causing odd behavior for tests and the test suite. 101 | TCP_PORT = TCP_PORT + 50 102 | HTTP_PORT = HTTP_PORT + 50 103 | 104 | reader = null 105 | }) 106 | 107 | describe('stream compression and encryption', () => { 108 | const optionPermutations = [ 109 | {deflate: true}, 110 | {snappy: true}, 111 | {tls: true, tlsVerification: false}, 112 | {tls: true, tlsVerification: false, snappy: true}, 113 | {tls: true, tlsVerification: false, deflate: true}, 114 | ] 115 | 116 | optionPermutations.forEach((options) => { 117 | const compression = ['deflate', 'snappy'] 118 | .filter((key) => key in options) 119 | .map((key) => key) 120 | 121 | compression.push('none') 122 | 123 | // Figure out what compression is enabled 124 | const description = `reader with compression (${ 125 | compression[0] 126 | }) and tls (${options.tls != null})` 127 | 128 | describe(description, () => { 129 | it('should send and receive a message', async () => { 130 | const topic = 'test' 131 | const channel = 'default' 132 | const message = 'a message for our reader' 133 | 134 | await publish(topic, message) 135 | 136 | reader = new nsq.Reader( 137 | topic, 138 | channel, 139 | Object.assign( 140 | {nsqdTCPAddresses: [`127.0.0.1:${TCP_PORT}`]}, 141 | options 142 | ) 143 | ) 144 | reader.on('error', () => {}) 145 | 146 | const messageEvent = pEvent(reader, 'message') 147 | reader.connect() 148 | 149 | const msg = await messageEvent 150 | should.equal(msg.body.toString(), message) 151 | msg.finish() 152 | }) 153 | 154 | it('should send and receive a large message', async () => { 155 | const topic = 'test' 156 | const channel = 'default' 157 | const message = _.range(0, 100000) 158 | .map(() => 'a') 159 | .join('') 160 | 161 | await publish(topic, message) 162 | 163 | reader = new nsq.Reader( 164 | topic, 165 | channel, 166 | Object.assign( 167 | {nsqdTCPAddresses: [`127.0.0.1:${TCP_PORT}`]}, 168 | options 169 | ) 170 | ) 171 | reader.on('error', () => {}) 172 | const messageEvent = pEvent(reader, 'message') 173 | reader.connect() 174 | 175 | const msg = await messageEvent 176 | should.equal(msg.body.toString(), message) 177 | msg.finish() 178 | }) 179 | }) 180 | }) 181 | }) 182 | 183 | describe('end to end', () => { 184 | const topic = 'test' 185 | const channel = 'default' 186 | let writer = null 187 | reader = null 188 | 189 | beforeEach((done) => { 190 | writer = new nsq.Writer('127.0.0.1', TCP_PORT) 191 | writer.on('ready', () => { 192 | reader = new nsq.Reader(topic, channel, { 193 | nsqdTCPAddresses: [`127.0.0.1:${TCP_PORT}`], 194 | }) 195 | reader.on('nsqd_connected', () => done()) 196 | reader.connect() 197 | }) 198 | 199 | writer.on('error', () => {}) 200 | writer.connect() 201 | }) 202 | 203 | afterEach(() => { 204 | writer.close() 205 | }) 206 | 207 | it('should send and receive a string', (done) => { 208 | const message = 'hello world' 209 | writer.publish(topic, message, (err) => { 210 | if (err) done(err) 211 | }) 212 | 213 | reader.on('error', (err) => { 214 | console.log(err) 215 | }) 216 | 217 | reader.on('message', (msg) => { 218 | msg.body.toString().should.eql(message) 219 | msg.finish() 220 | done() 221 | }) 222 | }) 223 | 224 | it('should send and receive a String object', (done) => { 225 | const message = new String('hello world') 226 | writer.publish(topic, message, (err) => { 227 | if (err) done(err) 228 | }) 229 | 230 | reader.on('error', (err) => { 231 | console.log(err) 232 | }) 233 | 234 | reader.on('message', (msg) => { 235 | msg.body.toString().should.eql(message.toString()) 236 | msg.finish() 237 | done() 238 | }) 239 | }) 240 | 241 | it('should send and receive a Buffer', (done) => { 242 | const message = Buffer.from([0x11, 0x22, 0x33]) 243 | writer.publish(topic, message) 244 | 245 | reader.on('error', () => {}) 246 | 247 | reader.on('message', (readMsg) => { 248 | for (let i = 0; i < readMsg.body.length; i++) { 249 | should.equal(readMsg.body[i], message[i]) 250 | } 251 | readMsg.finish() 252 | done() 253 | }) 254 | }) 255 | 256 | it('should not receive messages when immediately paused', (done) => { 257 | setTimeout(done, 50) 258 | 259 | // Note: because NSQDConnection.connect() does most of it's work in 260 | // process.nextTick(), we're really pausing before the reader is 261 | // connected. 262 | // 263 | reader.pause() 264 | reader.on('message', (msg) => { 265 | msg.finish() 266 | done(new Error('Should not have received a message while paused')) 267 | }) 268 | 269 | writer.publish(topic, 'pause test') 270 | }) 271 | 272 | it('should not receive any new messages when paused', (done) => { 273 | writer.publish(topic, {messageShouldArrive: true}) 274 | 275 | reader.on('error', (err) => { 276 | console.log(err) 277 | }) 278 | 279 | reader.on('message', (msg) => { 280 | // check the message 281 | msg.json().messageShouldArrive.should.be.true() 282 | msg.finish() 283 | 284 | if (reader.isPaused()) return done() 285 | 286 | reader.pause() 287 | 288 | process.nextTick(() => { 289 | // send it again, shouldn't get this one 290 | writer.publish(topic, {messageShouldArrive: false}) 291 | setTimeout(done, 50) 292 | }) 293 | }) 294 | }) 295 | 296 | it('should not receive any requeued messages when paused', (done) => { 297 | writer.publish(topic, 'requeue me') 298 | let id = '' 299 | 300 | reader.on('message', (msg) => { 301 | // this will fail if the msg comes through again 302 | id.should.equal('') 303 | id = msg.id 304 | 305 | reader.pause() 306 | 307 | // send it again, shouldn't get this one 308 | msg.requeue(0, false) 309 | setTimeout(done, 50) 310 | }) 311 | 312 | reader.on('error', () => {}) 313 | }) 314 | 315 | it('should start receiving messages again after unpause async', async () => { 316 | const wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) 317 | 318 | const publish = util.promisify((topic, msg, cb) => { 319 | writer.publish(topic, msg, cb) 320 | }) 321 | 322 | let paused = false 323 | const messageEvents = new EventEmitter() 324 | const firstEvent = pEvent(messageEvents, 'first') 325 | const secondEvent = pEvent(messageEvents, 'second') 326 | 327 | reader.on('message', (msg) => { 328 | should.equal(paused, false) 329 | messageEvents.emit(msg.body.toString(), msg) 330 | }) 331 | 332 | // Pubish message 333 | publish(topic, 'first') 334 | 335 | // Handle first message 336 | let msg = await firstEvent 337 | paused.should.be.false() 338 | msg.finish() 339 | 340 | // Pause reader 341 | reader.pause() 342 | paused = true 343 | 344 | // Publish second message 345 | await publish(topic, 'second') 346 | 347 | // Unpause after delay 348 | await wait(50) 349 | reader.unpause() 350 | paused = false 351 | 352 | // Handle second message 353 | msg = await secondEvent 354 | msg.finish() 355 | }) 356 | 357 | it('should successfully publish a message before fully connected', (done) => { 358 | writer = new nsq.Writer('127.0.0.1', TCP_PORT) 359 | writer.connect() 360 | 361 | // The writer is connecting, but it shouldn't be ready to publish. 362 | should.equal(writer.ready, false) 363 | 364 | writer.on('error', () => {}) 365 | 366 | // Publish the message. It should succeed since the writer will queue up 367 | // the message while connecting. 368 | writer.publish('a_topic', 'a message', (err) => { 369 | should.not.exist(err) 370 | done() 371 | }) 372 | }) 373 | }) 374 | }) 375 | 376 | describe('failures', () => { 377 | let nsqdProcess = null 378 | 379 | before(async () => { 380 | nsqdProcess = await startNSQD(await temp.mkdir('/nsq')) 381 | }) 382 | 383 | describe('Writer', () => { 384 | describe('nsqd disconnect before publish', () => { 385 | it('should fail to publish a message', async () => { 386 | const writer = new nsq.Writer('127.0.0.1', TCP_PORT) 387 | writer.on('error', () => {}) 388 | 389 | const readyEvent = pEvent(writer, 'ready') 390 | const exitEvent = pEvent(nsqdProcess, 'exit') 391 | 392 | writer.connect() 393 | await readyEvent 394 | 395 | nsqdProcess.kill('SIGKILL') 396 | await exitEvent 397 | 398 | const publish = util.promisify((topic, msg, cb) => 399 | writer.publish(topic, msg, cb) 400 | ) 401 | try { 402 | await publish('test_topic', 'a failing message') 403 | should.fail() 404 | } catch (e) { 405 | should.exist(e) 406 | } 407 | }) 408 | }) 409 | }) 410 | }) 411 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nsqjs 2 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fdudleycarr%2Fnsqjs.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fdudleycarr%2Fnsqjs?ref=badge_shield) 3 | 4 | 5 | The official NodeJS client for the [nsq](http://nsq.io/) client protocol. This implementation attempts to be 6 | fully compliant and maintain feature parity with the official Go ([go-nsq](https://github.com/nsqio/go-nsq)) and Python ([pynsq](https://github.com/nsqio/pynsq)) clients. 7 | 8 | 9 | ## Usage 10 | 11 | ### new Reader(topic, channel, options) 12 | 13 | The topic and channel arguments are strings and must be specified. The options 14 | argument is optional. Below are the parameters that can be specified in the 15 | options object. 16 | 17 | * ```maxInFlight: 1```
18 | The maximum number of messages to process at once. This value is shared between nsqd connections. It's highly recommended that this value is greater than the number of nsqd connections. 19 | * ```heartbeatInterval: 30```
20 | The frequency in seconds at which the nsqd will send heartbeats to this Reader. 21 | * ```maxBackoffDuration: 128```
22 | The maximum amount of time (seconds) the Reader will backoff for any single backoff 23 | event. 24 | * ```maxAttempts: 0```
25 | The number of times a given message will be attempted (given to MESSAGE handler) before it will be handed to the DISCARD handler and then automatically finished. 0 means that there is **no limit.** If no DISCARD handler is specified and `maxAttempts > 0`, then the message will be finished automatically when the number of attempts has been exhausted. 26 | * ```requeueDelay: 90,000 (90secs)```
27 | The default amount of time (milliseconds) a message requeued should be delayed by before being dispatched by nsqd. 28 | * ```nsqdTCPAddresses```
29 | A string or an array of strings representing the host/port pair for nsqd instances. 30 |
For example: `['localhost:4150']` 31 | * ```lookupdHTTPAddresses```
32 | A string or an array of strings representing the host/port pair of nsqlookupd instaces or the full HTTP/HTTPS URIs of the nsqlookupd instances. 33 |
For example: `['localhost:4161']`, `['http://localhost/lookup']`, `['http://localhost/path/lookup?extra_param=true']` 34 | * ```lookupdPollInterval: 60```
35 | The frequency in seconds for querying lookupd instances. 36 | * ```lookupdPollJitter: 0.3```
37 | The jitter applied to the start of querying lookupd instances periodically. 38 | * ```lowRdyTimeout: 50```
39 | The timeout in milliseconds for switching between connections when the Reader 40 | maxInFlight is less than the number of connected NSQDs. 41 | * ```tls: false```
42 | Use TLS if nsqd has TLS support enabled. 43 | * ```tlsVerification: true```
44 | Require verification of the TLS cert. This needs to be false if you're using 45 | a self signed cert. 46 | * ```deflate: false```
47 | Use zlib Deflate compression. 48 | * ```deflateLevel: 6```
49 | Use zlib Deflate compression level. 50 | * ```snappy: false```
51 | Use Snappy compression. 52 | * ```authSecret: null```
53 | Authenticate using the provided auth secret. 54 | * ```outputBufferSize: null```
55 | The size in bytes of the buffer nsqd will use when writing to this client. -1 56 | disables buffering. ```outputBufferSize >= 64``` 57 | * ```outputBufferTimeout: null```
58 | The timeout after which any data that nsqd has buffered will be flushed to this client. Value is in milliseconds. ```outputBufferTimeout >= 1```. A value of ```-1``` disables timeouts. 59 | * ```messageTimeout: null```
60 | Sets the server-side message timeout in milliseconds for messages delivered to this client. 61 | * ```sampleRate: null```
62 | Deliver a percentage of all messages received to this connection. ```1 <= 63 | sampleRate <= 99``` 64 | * ```clientId: null```
65 | An identifier used to disambiguate this client. 66 | * ```idleTimeout: 0```
67 | Socket timeout after idling for the duration in second (default to 0 means disabled). 68 | 69 | Reader events are: 70 | 71 | * `Reader.READY` or `ready` 72 | * `Reader.NOT_READY` or `not_ready` 73 | * `Reader.MESSAGE` or `message` 74 | * `Reader.DISCARD` or `discard` 75 | * `Reader.ERROR` or `error` 76 | * `Reader.NSQD_CONNECTED` or `nsqd_connected` 77 | * `Reader.NSQD_CLOSED` or `nsqd_closed` 78 | 79 | `Reader.MESSAGE` and `Reader.DISCARD` both produce `Message` objects. 80 | `Reader.NSQD_CONNECTED` and `Reader.NSQD_CLOSED` events both provide the host 81 | and port of the nsqd to which the event pertains. 82 | 83 | These methods are available on a Reader object: 84 | * `connect()`
85 | Connect to the nsqds specified or connect to nsqds discovered via 86 | lookupd. 87 | * `close()`
88 | Disconnect from all nsqds. Does not wait for in-flight messages to complete. 89 | * `pause()`
90 | Pause the Reader by stopping message flow. Does not affect in-flight 91 | messages. 92 | * `unpause()`
93 | Unpauses the Reader by resuming normal message flow. 94 | * `isPaused()`
95 | `true` if paused, `false` otherwise. 96 | 97 | ### Message 98 | 99 | The following properties and methods are available on Message objects produced by a Reader 100 | instance. 101 | 102 | * `timestamp`
103 | Numeric timestamp for the Message provided by nsqd. 104 | * `attempts`
105 | The number of attempts that have been made to process this message. 106 | * `id`
107 | The opaque string id for the Message provided by nsqd. 108 | * `hasResponded`
109 | Boolean for whether or not a response has been sent. 110 | * `body`
111 | The message payload as a Buffer object. 112 | * `json()`
113 | Parses message payload as JSON and caches the result. 114 | * `timeUntilTimeout(hard=false)`:
115 | Returns the amount of time until the message times out. If the hard argument 116 | is provided, then it calculates the time until the hard timeout when nsqd 117 | will requeue inspite of touch events. 118 | * `finish()`
119 | Finish the message as successful. 120 | * `requeue(delay=null, backoff=true)` 121 | The delay is in milliseconds. This is how long nsqd will hold on the message 122 | before attempting it again. The backoff parameter indicates that we should 123 | treat this as an error within this process and we need to backoff to recover. 124 | * `touch()`
125 | Tell nsqd that you want extra time to process the message. It extends the 126 | soft timeout by the normal timeout amount. 127 | 128 | ### new Writer(nsqdHost, nsqdPort, options) 129 | 130 | Allows messages to be sent to an nsqd. 131 | 132 | Available Writer options: 133 | 134 | * ```tls: false```
135 | Use TLS if nsqd has TLS support enabled. 136 | * ```tlsVerification: true```
137 | Require verification of the TLS cert. This needs to be false if you're using 138 | a self signed cert. 139 | * ```deflate: false```
140 | Use zlib Deflate compression. 141 | * ```deflateLevel: 6```
142 | Use zlib Deflate compression level. 143 | * ```snappy: false```
144 | Use Snappy compression. 145 | * ```clientId: null```
146 | An identifier used to disambiguate this client. 147 | 148 | Writer events are: 149 | 150 | * `Writer.READY` or `ready` 151 | * `Writer.CLOSED` or `closed` 152 | * `Writer.ERROR` or `error` 153 | 154 | These methods are available on a Writer object: 155 | 156 | * `connect()`
157 | Connect to the nsqd specified. 158 | * `close()`
159 | Disconnect from nsqd. 160 | * `publish(topic, msgs, [callback])`
161 | `topic` is a string. `msgs` is either a string, a `Buffer`, JSON serializable 162 | object, a list of strings / `Buffers` / JSON serializable objects. `callback` takes a single `error` argument. 163 | * `deferPublish(topic, msg, timeMs, [callback])`
164 | `topic` is a string. `msg` is either a string, a `Buffer`, JSON serializable object. `timeMs` is the delay by which the message should be delivered. `callback` takes a single `error` argument. 165 | 166 | ### Simple example 167 | 168 | Start [nsqd](http://nsq.io/components/nsqd.html) and 169 | [nsqdlookupd](http://nsq.io/components/nsqlookupd.html) 170 | 171 | ```bash 172 | # nsqdLookupd Listens on 4161 for HTTP requests and 4160 for TCP requests 173 | $ nsqlookupd & 174 | $ nsqd -lookupd-tcp-address=127.0.0.1:4160 -broadcast-address=127.0.0.1 & 175 | ``` 176 | 177 | ```js 178 | const nsq = require('nsqjs') 179 | 180 | const reader = new nsq.Reader('sample_topic', 'test_channel', { 181 | lookupdHTTPAddresses: '127.0.0.1:4161' 182 | }) 183 | 184 | reader.connect() 185 | 186 | reader.on('message', msg => { 187 | console.log('Received message [%s]: %s', msg.id, msg.body.toString()) 188 | msg.finish() 189 | }) 190 | ``` 191 | 192 | Publish a message to nsqd to be consumed by the sample client: 193 | 194 | ```bash 195 | $ curl -d "it really tied the room together" http://localhost:4151/pub?topic=sample_topic 196 | ``` 197 | 198 | ### Example with message timeouts 199 | 200 | This script simulates a message that takes a long time to process or at least 201 | longer than the default message timeout. To ensure that the message doesn't 202 | timeout while being processed, touch events are sent to keep it alive. 203 | 204 | ```js 205 | const nsq = require('nsqjs') 206 | 207 | const reader = new nsq.Reader('sample_topic', 'test_channel', { 208 | lookupdHTTPAddresses: '127.0.0.1:4161' 209 | }) 210 | 211 | reader.connect() 212 | 213 | reader.on('message', msg => { 214 | console.log('Received message [%s]', msg.id) 215 | 216 | const touch = () => { 217 | if (!msg.hasResponded) { 218 | console.log('Touch [%s]', msg.id) 219 | msg.touch() 220 | 221 | // Touch the message again a second before the next timeout. 222 | setTimeout(touch, msg.timeUntilTimeout() - 1000) 223 | } 224 | } 225 | 226 | const finish = () => { 227 | console.log('Finished message [%s]: %s', msg.id, msg.body.toString()) 228 | msg.finish() 229 | } 230 | 231 | console.log('Message timeout is %f secs.', msg.timeUntilTimeout() / 1000) 232 | setTimeout(touch, msg.timeUntilTimeout() - 1000) 233 | 234 | // Finish the message after 2 timeout periods and 1 second. 235 | setTimeout(finish, msg.timeUntilTimeout() * 2 + 1000) 236 | }) 237 | ``` 238 | 239 | ### Enable nsqjs debugging 240 | 241 | nsqjs uses [debug](https://github.com/visionmedia/debug) to log debug output. 242 | 243 | To see all nsqjs events: 244 | ``` 245 | $ DEBUG=nsqjs:* node my_nsqjs_script.js 246 | ``` 247 | 248 | To see all reader events: 249 | ``` 250 | $ DEBUG=nsqjs:reader:* node my_nsqjs_script.js 251 | ``` 252 | 253 | To see a specific reader's events: 254 | ``` 255 | $ DEBUG=nsqjs:reader:/:* node my_nsqjs_script.js 256 | ``` 257 | > Replace `` and `` 258 | 259 | To see all writer events: 260 | ``` 261 | $ DEBUG=nsqjs:writer:* node my_nsqjs_script.js 262 | ``` 263 | 264 | ### A Writer Example 265 | 266 | The writer sends a single message and then a list of messages. 267 | 268 | ```js 269 | const nsq = require('nsqjs') 270 | 271 | const w = new nsq.Writer('127.0.0.1', 4150) 272 | 273 | w.connect() 274 | 275 | w.on('ready', () => { 276 | w.publish('sample_topic', 'it really tied the room together') 277 | w.deferPublish('sample_topic', ['This message gonna arrive 1 sec later.'], 1000) 278 | w.publish('sample_topic', [ 279 | 'Uh, excuse me. Mark it zero. Next frame.', 280 | 'Smokey, this is not \'Nam. This is bowling. There are rules.' 281 | ]) 282 | w.publish('sample_topic', 'Wu?', err => { 283 | if (err) { return console.error(err.message) } 284 | console.log('Message sent successfully') 285 | w.close() 286 | }) 287 | }) 288 | 289 | w.on('closed', () => { 290 | console.log('Writer closed') 291 | }) 292 | ``` 293 | 294 | ## Changes 295 | 296 | * **0.13.0** 297 | * Fix: SnappyStream initialization race condition. (#353) 298 | * Fix: IPv6 address support (#352) 299 | * Fix: Support JavaScript String objects as messages (#341) 300 | * Deprecated: Node versions < 12 (#346) 301 | * Change: Debug is now a dev dependency (#349) 302 | * Change: Use SnappyStream 2.0 (#354) 303 | * Change: Replaced BigNumber.js with built-in BigInt (#337) 304 | * Change: Replaced request.js with node-fetch (#347) 305 | * Change: Removed dependency on async.js (#347) 306 | * **0.12.0** 307 | * Expose `lowRdyTimeout` parameter for Readers. 308 | * Change the default value for `lowRdyTimeout` from 1.5s to 50ms. 309 | * **0.11.0** 310 | * Support NodeJS 10 311 | * Fix Snappy issues with NSQD 1.0 and 1.1 312 | * Fix `close` behavior for Readers 313 | * Added `"ready"` and `"not_ready"` events for Reader. 314 | * Fix short timeout for connection IDENTIFY requests. (Thanks @emaincourt) 315 | * Lazy deserialization of messages 316 | * Cached Backoff Timer calculations 317 | * **0.10.1** 318 | * Fix debug.js memory leak when destroying NSQDConnection objects. 319 | * **0.10.0** 320 | * Fix off by one error for Message maxAttempts. (Thanks @tomc974) 321 | * Fix requeueDelay default to be 90secs instead of 90msec. Updated docs. 322 | (Thanks @tomc974) 323 | * **0.9.2** 324 | * Fix `Reader.close` to cleanup all intervals to allow node to exit cleanly. 325 | * Upraded Sinon 326 | * Removed .eslintrc 327 | * **0.9.1** 328 | * Fixed Reader close exceptions. (Thanks @ekristen) 329 | * Bump Sinon version 330 | * Bump bignumber.js version 331 | * **0.9** 332 | * **Breaking change:** Node versions 6 and greater supported from now on. 333 | * Support for deferred message publishing! (thanks @spruce) 334 | * Added idleTimeout for Reader and Writer (thanks @abbshr) 335 | * Snappy support is now optional. (thanks @bcoe) 336 | * Snappy support is now fixed and working with 1.0.0-compat! 337 | * Fixed backoff behavior if mulitiple messages fail at the same time. Should recover much faster. 338 | * Use TCP NO_DELAY for nsqd connections. 339 | * Chores: 340 | * Replaced underscore with lodash 341 | * Move `src` to `lib` 342 | * Dropped Babel support 343 | * Use Standard JS style 344 | * Less flakey tests 345 | * **0.8.4** 346 | * Move to ES6 using Babel. 347 | * **0.7.12** 348 | * Bug: Fix issue introduced by not sending RDY count to main max-in-flight. 349 | Readers connected to mulitple nsqds do not set RDY count for connections 350 | made after the first nsqd connection. 351 | * **0.7.11** 352 | * Improvement: Keep track of touch count in Message instances. 353 | * Improvement: Stop sending RDY count to main max-in-flight for newer 354 | versions of nsqd. 355 | * Improvement: Make the connect debug message more accurate in Reader. 356 | Previously lookupd poll results suggested new connections were being made 357 | when they were not. 358 | * Bug: Non-fatal nsqd errors would cause RDY count to decrease and never 359 | return to normal. This will happen for example when finishing messages 360 | that have exceeded their amount of time to process a message. 361 | * 362 | * **0.7.10** 363 | * Properly handles non-string errors 364 | * **0.7.9** 365 | * Treat non-fatal errors appropriately 366 | * **0.7.7** 367 | * Build with Node v4 368 | * **0.7.6** 369 | * Fix npm install by adding .npmignore. 370 | * **0.7.3** 371 | * Slightly better invalid topic and channel error messages. 372 | * Handle more conditions for failing to publish a message. 373 | * **0.7.2** 374 | * Fix build for iojs and node v0.12 375 | * Bumped snappystream version. 376 | * **0.7.1** 377 | * Fix connection returning to max connection RDY after backoff 378 | * Fix backoff ignored when `message.finish` is called after backoff event. 379 | * **0.7.0** 380 | * Fixes for configuration breakages 381 | * Fix for AUTH 382 | * Fix for pause / unpause 383 | * Automatically finish messages when maxAttempts have been exceeded. 384 | * `maxAttempts` is now by default 0. [ Breaking Change! ] 385 | * discarded messages will now be sent to the `MESSAGE` listener if there's no 386 | `DISCARD` listener. 387 | * Support for emphemeral topics. 388 | * Support for 64 char topic / channel names. 389 | * Support for Lookupd URLs 390 | * Deprecate StateChangeLogger infavor of `debug` [ Breaking Change! ] 391 | * **0.6.0** 392 | * Added support for authentication 393 | * Added support for sample rate 394 | * Added support for specifying outputBufferSize and outputBufferTimeout 395 | * Added support for specifying msg_timeout 396 | * Refactored configuration checks 397 | * Breaking change for NSQDConnection constructor. NSQDConnection takes an 398 | options argument instead of each option as a parameter. 399 | * **0.5.1** 400 | * Fix for not failing when the nsqd isn't available on start. 401 | * **0.5.0** 402 | * Reworked FrameBuffer 403 | * Added TLS support for Reader and Writer 404 | * Added Deflate support 405 | * Added Snappy support 406 | * **0.4.1** 407 | * Fixed a logging issue on NSQConnection disconnected 408 | * **0.4.0** 409 | * Added `close`, `pause`, and `unpause` to Reader 410 | * Added callback for Writer publish 411 | * Expose error events on Reader/Writer 412 | * Expose nsqd connect / disconnect events 413 | * Fix crash when Message `finish`, `requeue`, etc after nsqd disconnect 414 | * Fix lookupd only queried on startup. 415 | * **0.3.1** 416 | * Fixed sending an array of Buffers 417 | * Fixed sending a message with multi-byte characters 418 | * **0.3.0** 419 | * Added Writer implementation 420 | * **0.2.1** 421 | * Added prepublish compilation to JavaScript. 422 | * **0.2.0** 423 | * `ReaderRdy`, `ConnectionRdy` implementation 424 | * `Reader` implementation 425 | * Initial documentation 426 | * `NSQDConnection` 427 | * Moved defaults to `Reader` 428 | * Support protocol / state logging 429 | * `connect()` now happens on next tick so that it can be called before event 430 | handlers are registered. 431 | * `Message` 432 | * Correctly support `TOUCH` events 433 | * Support soft and hard timeout timings 434 | * JSON parsing of message body 435 | * **0.1.0** 436 | * `NSQDConnection` implementation 437 | * `wire` implementation 438 | * `Message` implementation 439 | 440 | 441 | ## License 442 | [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fdudleycarr%2Fnsqjs.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fdudleycarr%2Fnsqjs?ref=badge_large) -------------------------------------------------------------------------------- /lib/readerrdy.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events') 2 | 3 | const NodeState = require('node-state') 4 | const _ = require('lodash') 5 | const debug = require('./debug') 6 | 7 | const BackoffTimer = require('./backofftimer') 8 | const RoundRobinList = require('./roundrobinlist') 9 | const {NSQDConnection} = require('./nsqdconnection') 10 | 11 | /** 12 | * Maintains the RDY and in-flight counts for a nsqd connection. ConnectionRdy 13 | * ensures that the RDY count will not exceed the max set for this connection. 14 | * The max for the connection can be adjusted at any time. 15 | * 16 | * Usage: 17 | * const connRdy = ConnectionRdy(conn); 18 | * const connRdy.setConnectionRdyMax(10); 19 | * 20 | * // On a successful message, bump up the RDY count for this connection. 21 | * conn.on('message', () => connRdy.raise('bump')); 22 | * 23 | * // We're backing off when we encounter a requeue. Wait 5 seconds to try 24 | * // again. 25 | * conn.on('requeue', () => connRdy.raise('backoff')); 26 | * setTimeout(() => connRdy.raise (bump'), 5000); 27 | */ 28 | class ConnectionRdy extends EventEmitter { 29 | // Events emitted by ConnectionRdy 30 | static get READY() { 31 | return 'ready' 32 | } 33 | static get STATE_CHANGE() { 34 | return 'statechange' 35 | } 36 | 37 | /** 38 | * Instantiates a new ConnectionRdy event emitter. 39 | * 40 | * @param {Object} conn 41 | * @constructor 42 | */ 43 | constructor(conn, ...args) { 44 | super(conn, ...args) 45 | this.conn = conn 46 | const readerId = `${this.conn.topic}/${this.conn.channel}` 47 | const connId = `${this.conn.id().replace(':', '/')}` 48 | this.debug = debug(`nsqjs:reader:${readerId}:rdy:conn:${connId}`) 49 | 50 | this.maxConnRdy = 0 // The absolutely maximum the RDY count can be per conn. 51 | this.inFlight = 0 // The num. messages currently in-flight for this conn. 52 | this.lastRdySent = 0 // The RDY value last sent to the server. 53 | this.availableRdy = 0 // The RDY count remaining on the server for this conn. 54 | this.statemachine = new ConnectionRdyState(this) 55 | 56 | this.conn.on(NSQDConnection.ERROR, (err) => this.log(err)) 57 | this.conn.on(NSQDConnection.MESSAGE, () => { 58 | if (this.idleId != null) { 59 | clearTimeout(this.idleId) 60 | } 61 | this.idleId = null 62 | this.inFlight += 1 63 | this.availableRdy -= 1 64 | }) 65 | this.conn.on(NSQDConnection.FINISHED, () => this.inFlight--) 66 | this.conn.on(NSQDConnection.REQUEUED, () => this.inFlight--) 67 | this.conn.on(NSQDConnection.READY, () => this.start()) 68 | } 69 | 70 | /** 71 | * Close the reader ready connection. 72 | */ 73 | close() { 74 | this.conn.close() 75 | } 76 | 77 | /** 78 | * Return the name of the local port connection. 79 | * 80 | * @return {String} 81 | */ 82 | name() { 83 | return String(this.conn.conn.localPort) 84 | } 85 | 86 | /** 87 | * Emit that the connection is ready. 88 | * 89 | * @return {Boolean} Returns true if the event had listeners, false otherwise. 90 | */ 91 | start() { 92 | this.statemachine.start() 93 | return this.emit(ConnectionRdy.READY) 94 | } 95 | 96 | /** 97 | * Initialize the max number of connections ready. 98 | * 99 | * @param {Number} maxConnRdy 100 | */ 101 | setConnectionRdyMax(maxConnRdy) { 102 | this.log(`setConnectionRdyMax ${maxConnRdy}`) 103 | // The RDY count for this connection should not exceed the max RDY count 104 | // configured for this nsqd connection. 105 | this.maxConnRdy = Math.min(maxConnRdy, this.conn.maxRdyCount) 106 | this.statemachine.raise('adjustMax') 107 | } 108 | 109 | /** 110 | * Raises a `BUMP` event. 111 | */ 112 | bump() { 113 | this.statemachine.raise('bump') 114 | } 115 | 116 | /** 117 | * Raises a `BACKOFF` event. 118 | */ 119 | backoff() { 120 | this.statemachine.raise('backoff') 121 | } 122 | 123 | /** 124 | * Used to identify when buffered messages should be processed 125 | * and responded to. 126 | * 127 | * @return {Boolean} [description] 128 | */ 129 | isStarved() { 130 | if (!(this.inFlight <= this.maxConnRdy)) { 131 | throw new Error('isStarved check is failing') 132 | } 133 | return this.inFlight === this.lastRdySent 134 | } 135 | 136 | /** 137 | * Assign the number of readers available. 138 | * 139 | * @param {Number} rdyCount 140 | */ 141 | setRdy(rdyCount) { 142 | this.log(`RDY ${rdyCount}`) 143 | if (rdyCount < 0 || rdyCount > this.maxConnRdy) return 144 | 145 | this.conn.setRdy(rdyCount) 146 | this.availableRdy = this.lastRdySent = rdyCount 147 | } 148 | 149 | /** 150 | * @param {String} message 151 | * @return {String} 152 | */ 153 | log(message) { 154 | if (message) return this.debug(message) 155 | } 156 | } 157 | 158 | /** 159 | * Internal statemachine used handle the various reader ready states. 160 | * @type {NodeState} 161 | */ 162 | class ConnectionRdyState extends NodeState { 163 | /** 164 | * Instantiates a new ConnectionRdyState. 165 | * 166 | * @param {Object} connRdy reader connection 167 | * @constructor 168 | */ 169 | constructor(connRdy) { 170 | super({ 171 | autostart: false, 172 | initial_state: 'INIT', 173 | sync_goto: true, 174 | }) 175 | 176 | this.connRdy = connRdy 177 | } 178 | 179 | /** 180 | * Utility function to log a message through debug. 181 | * 182 | * @param {Message} message 183 | * @return {String} 184 | */ 185 | log(message) { 186 | this.connRdy.debug(this.current_state_name) 187 | if (message) { 188 | return this.connRdy.debug(message) 189 | } 190 | } 191 | } 192 | 193 | ConnectionRdyState.prototype.states = { 194 | INIT: { 195 | // RDY is implicitly zero 196 | bump() { 197 | if (this.connRdy.maxConnRdy > 0) { 198 | return this.goto('MAX') 199 | } 200 | }, 201 | backoff() {}, // No-op 202 | adjustMax() {}, 203 | }, // No-op 204 | 205 | BACKOFF: { 206 | Enter() { 207 | return this.connRdy.setRdy(0) 208 | }, 209 | bump() { 210 | if (this.connRdy.maxConnRdy > 0) return this.goto('ONE') 211 | }, 212 | backoff() {}, // No-op 213 | adjustMax() {}, 214 | }, // No-op 215 | 216 | ONE: { 217 | Enter() { 218 | return this.connRdy.setRdy(1) 219 | }, 220 | bump() { 221 | return this.goto('MAX') 222 | }, 223 | backoff() { 224 | return this.goto('BACKOFF') 225 | }, 226 | adjustMax() {}, 227 | }, // No-op 228 | 229 | MAX: { 230 | Enter() { 231 | return this.connRdy.setRdy(this.connRdy.maxConnRdy) 232 | }, 233 | bump() { 234 | // No need to keep setting the RDY count for versions of NSQD >= 0.3.0. 235 | const version = 236 | this.connRdy.conn != null ? this.connRdy.conn.nsqdVersion : undefined 237 | if (!version || version.split('.') < [0, 3, 0]) { 238 | if (this.connRdy.availableRdy <= this.connRdy.lastRdySent * 0.25) { 239 | return this.connRdy.setRdy(this.connRdy.maxConnRdy) 240 | } 241 | } 242 | }, 243 | backoff() { 244 | return this.goto('BACKOFF') 245 | }, 246 | adjustMax() { 247 | this.log(`adjustMax RDY ${this.connRdy.maxConnRdy}`) 248 | return this.connRdy.setRdy(this.connRdy.maxConnRdy) 249 | }, 250 | }, 251 | } 252 | 253 | ConnectionRdyState.prototype.transitions = { 254 | '*': { 255 | '*': function (data, callback) { 256 | this.log() 257 | callback(data) 258 | return this.connRdy.emit(ConnectionRdy.STATE_CHANGE) 259 | }, 260 | }, 261 | } 262 | 263 | /** 264 | * Usage: 265 | * const backoffTime = 90; 266 | * const heartbeat = 30; 267 | * 268 | * const [topic, channel] = ['sample', 'default']; 269 | * const [host1, port1] = ['127.0.0.1', '4150']; 270 | * const c1 = new NSQDConnection(host1, port1, topic, channel, 271 | * backoffTime, heartbeat); 272 | * 273 | * const readerRdy = new ReaderRdy(1, 128); 274 | * readerRdy.addConnection(c1); 275 | * 276 | * const message = (msg) => { 277 | * console.log(`Callback [message]: ${msg.attempts}, ${msg.body.toString()}1); 278 | * if (msg.attempts >= 5) { 279 | * msg.finish(); 280 | * return; 281 | * } 282 | * 283 | * if (msg.body.toString() === 'requeue') 284 | * msg.requeue(); 285 | * else 286 | * msg.finish(); 287 | * } 288 | * 289 | * const discard = (msg) => { 290 | * console.log(`Giving up on this message: ${msg.id}`); 291 | * msg.finish(); 292 | * } 293 | * 294 | * c1.on(NSQDConnection.MESSAGE, message); 295 | * c1.connect(); 296 | */ 297 | let READER_COUNT = 0 298 | 299 | /** 300 | * ReaderRdy statemachine. 301 | * @type {[type]} 302 | */ 303 | class ReaderRdy extends NodeState { 304 | /** 305 | * Generates a new ID for a reader connection. 306 | * 307 | * @return {Number} 308 | */ 309 | static getId() { 310 | return READER_COUNT++ 311 | } 312 | 313 | /** 314 | * @constructor 315 | * @param {Number} maxInFlight Maximum number of messages in-flight 316 | * across all connections. 317 | * @param {Number} maxBackoffDuration The longest amount of time (secs) 318 | * for a backoff event. 319 | * @param {Number} readerId The descriptive id for the Reader 320 | * @param {Number} [lowRdyTimeout=1.5] Time (milliseconds) to rebalance RDY 321 | * count among connections 322 | */ 323 | constructor(maxInFlight, maxBackoffDuration, readerId, lowRdyTimeout = 50) { 324 | super({ 325 | autostart: true, 326 | initial_state: 'ZERO', 327 | sync_goto: true, 328 | }) 329 | 330 | this.maxInFlight = maxInFlight 331 | this.maxBackoffDuration = maxBackoffDuration 332 | this.readerId = readerId 333 | this.lowRdyTimeout = lowRdyTimeout 334 | this.debug = debug(`nsqjs:reader:${this.readerId}:rdy`) 335 | 336 | this.id = ReaderRdy.getId() 337 | this.backoffTimer = new BackoffTimer(0, this.maxBackoffDuration) 338 | this.backoffId = null 339 | this.balanceId = null 340 | this.connections = [] 341 | this.roundRobinConnections = new RoundRobinList([]) 342 | this.isClosed = false 343 | } 344 | 345 | /** 346 | * Close all reader connections. 347 | * 348 | * @return {Array} The closed connections. 349 | */ 350 | close() { 351 | this.isClosed = true 352 | clearTimeout(this.backoffId) 353 | clearTimeout(this.balanceId) 354 | return _.clone(this.connections).map((conn) => conn.close()) 355 | } 356 | 357 | /** 358 | * Raise a `PAUSE` event. 359 | */ 360 | pause() { 361 | this.raise('pause') 362 | } 363 | 364 | /** 365 | * Raise a `UNPAUSE` event. 366 | */ 367 | unpause() { 368 | this.raise('unpause') 369 | } 370 | 371 | /** 372 | * Indicates if a the reader ready connection has been paused. 373 | * 374 | * @return {Boolean} 375 | */ 376 | isPaused() { 377 | return this.current_state_name === 'PAUSE' 378 | } 379 | 380 | /** 381 | * @param {String} message 382 | * @return {String} 383 | */ 384 | log(message) { 385 | if (this.debug) { 386 | this.debug(this.current_state_name) 387 | 388 | if (message) return this.debug(message) 389 | } 390 | } 391 | 392 | /** 393 | * Used to identify when buffered messages should be processed 394 | * and responded to. 395 | * 396 | * @return {Boolean} [description] 397 | */ 398 | isStarved() { 399 | if (_.isEmpty(this.connections)) return false 400 | 401 | return this.connections.filter((conn) => conn.isStarved()).length > 0 402 | } 403 | 404 | /** 405 | * Creates a new ConnectionRdy statemachine. 406 | * @param {Object} conn 407 | * @return {ConnectionRdy} 408 | */ 409 | createConnectionRdy(conn) { 410 | return new ConnectionRdy(conn) 411 | } 412 | 413 | /** 414 | * Indicates if a producer is in a state where RDY counts are re-distributed. 415 | * @return {Boolean} 416 | */ 417 | isLowRdy() { 418 | return this.maxInFlight < this.connections.length 419 | } 420 | 421 | /** 422 | * Message success handler. 423 | * 424 | * @param {ConnectionRdy} connectionRdy 425 | */ 426 | onMessageSuccess(connectionRdy) { 427 | if (!this.isPaused()) { 428 | if (this.isLowRdy()) { 429 | // Balance the RDY count amoung existing connections given the 430 | // low RDY condition. 431 | this.balance() 432 | } else { 433 | // Restore RDY count for connection to the connection max. 434 | connectionRdy.bump() 435 | } 436 | } 437 | } 438 | 439 | /** 440 | * Add a new connection to the pool. 441 | * 442 | * @param {Object} conn 443 | */ 444 | addConnection(conn) { 445 | const connectionRdy = this.createConnectionRdy(conn) 446 | 447 | conn.on(NSQDConnection.CLOSED, () => { 448 | this.removeConnection(connectionRdy) 449 | this.balance() 450 | }) 451 | 452 | conn.on(NSQDConnection.FINISHED, () => this.raise('success', connectionRdy)) 453 | 454 | conn.on(NSQDConnection.REQUEUED, () => { 455 | // Since there isn't a guaranteed order for the REQUEUED and BACKOFF 456 | // events, handle the case when we handle BACKOFF and then REQUEUED. 457 | if (this.current_state_name !== 'BACKOFF' && !this.isPaused()) { 458 | connectionRdy.bump() 459 | } 460 | }) 461 | 462 | conn.on(NSQDConnection.BACKOFF, () => this.raise('backoff')) 463 | 464 | connectionRdy.on(ConnectionRdy.READY, () => { 465 | // Aborting the connection. ReaderRdy received a close while the 466 | // nsqdConnection was still being established. 467 | if (this.isClosed) { 468 | conn.close() 469 | return 470 | } 471 | 472 | this.connections.push(connectionRdy) 473 | this.roundRobinConnections.add(connectionRdy) 474 | 475 | this.balance() 476 | if (this.current_state_name === 'ZERO') { 477 | this.goto('MAX') 478 | } else if (['TRY_ONE', 'MAX'].includes(this.current_state_name)) { 479 | connectionRdy.bump() 480 | } 481 | }) 482 | } 483 | 484 | /** 485 | * Remove a connection from the pool. 486 | * 487 | * @param {Object} conn 488 | */ 489 | removeConnection(conn) { 490 | this.connections.splice(this.connections.indexOf(conn), 1) 491 | this.roundRobinConnections.remove(conn) 492 | 493 | if (this.connections.length === 0) { 494 | this.goto('ZERO') 495 | } 496 | } 497 | 498 | /** 499 | * Raise a `BUMP` event for each connection in the pool. 500 | * 501 | * @return {Array} The bumped connections 502 | */ 503 | bump() { 504 | return this.connections.map((conn) => conn.bump()) 505 | } 506 | 507 | /** 508 | * Try to balance the connection pool. 509 | */ 510 | try() { 511 | this.balance() 512 | } 513 | 514 | /** 515 | * Raise a `BACKOFF` event for each connection in the pool. 516 | */ 517 | backoff() { 518 | this.connections.forEach((conn) => conn.backoff()) 519 | 520 | if (this.backoffId) { 521 | clearTimeout(this.backoffId) 522 | } 523 | 524 | const onTimeout = () => { 525 | this.log('Backoff done') 526 | this.raise('try') 527 | } 528 | 529 | const delay = this.backoffTimer.getInterval() * 1000 530 | this.backoffId = setTimeout(onTimeout, delay) 531 | this.log(`Backoff for ${delay}`) 532 | } 533 | 534 | /** 535 | * Return the number of connections inflight. 536 | * 537 | * @return {Number} 538 | */ 539 | inFlight() { 540 | const add = (previous, conn) => previous + conn.inFlight 541 | return this.connections.reduce(add, 0) 542 | } 543 | 544 | /** 545 | * The max connections readily available. 546 | * 547 | * @return {Number} 548 | */ 549 | maxConnectionsRdy() { 550 | switch (this.current_state_name) { 551 | case 'TRY_ONE': 552 | return 1 553 | case 'PAUSE': 554 | return 0 555 | default: 556 | return this.maxInFlight 557 | } 558 | } 559 | 560 | /** 561 | * Evenly or fairly distributes RDY count based on the maxInFlight across 562 | * all nsqd connections. 563 | * 564 | * In the perverse situation where there are more connections than max in 565 | * flight, we do the following: 566 | * 567 | * There is a sliding window where each of the connections gets a RDY count 568 | * of 1. When the connection has processed it's single message, then 569 | * the RDY count is distributed to the next waiting connection. If 570 | * the connection does nothing with it's RDY count, then it should 571 | * timeout and give it's RDY count to another connection. 572 | */ 573 | balance() { 574 | this.log('balance') 575 | 576 | if (this.balanceId != null) { 577 | clearTimeout(this.balanceId) 578 | this.balanceId = null 579 | } 580 | 581 | const max = this.maxConnectionsRdy() 582 | const perConnectionMax = Math.floor(max / this.connections.length) 583 | 584 | // Low RDY and try conditions 585 | if (perConnectionMax === 0) { 586 | /** 587 | * Backoff on all connections. In-flight messages from 588 | * connections will still be processed. 589 | */ 590 | this.connections.forEach((conn) => conn.backoff()) 591 | 592 | // Distribute available RDY count to the connections next in line. 593 | this.roundRobinConnections.next(max - this.inFlight()).forEach((conn) => { 594 | conn.setConnectionRdyMax(1) 595 | conn.bump() 596 | }) 597 | 598 | // Rebalance periodically. Needed when no messages are received. 599 | this.balanceId = setTimeout(() => { 600 | this.balance() 601 | }, this.lowRdyTimeout) 602 | } else { 603 | let rdyRemainder = this.maxInFlight % this.connections.length 604 | this.connections.forEach((c) => { 605 | let connMax = perConnectionMax 606 | 607 | /** 608 | * Distribute the remainder RDY count evenly between the first 609 | * n connections. 610 | */ 611 | if (rdyRemainder > 0) { 612 | connMax += 1 613 | rdyRemainder -= 1 614 | } 615 | 616 | c.setConnectionRdyMax(connMax) 617 | c.bump() 618 | }) 619 | } 620 | } 621 | } 622 | 623 | /** 624 | * The following events results in transitions in the ReaderRdy state machine: 625 | * 1. Adding the first connection 626 | * 2. Remove the last connections 627 | * 3. Finish event from message handling 628 | * 4. Backoff event from message handling 629 | * 5. Backoff timeout 630 | */ 631 | ReaderRdy.prototype.states = { 632 | ZERO: { 633 | Enter() { 634 | if (this.backoffId) { 635 | return clearTimeout(this.backoffId) 636 | } 637 | }, 638 | backoff() {}, // No-op 639 | success() {}, // No-op 640 | try() {}, // No-op 641 | pause() { 642 | // No-op 643 | return this.goto('PAUSE') 644 | }, 645 | unpause() {}, 646 | }, // No-op 647 | 648 | PAUSE: { 649 | Enter() { 650 | return this.connections.map((conn) => conn.backoff()) 651 | }, 652 | backoff() {}, // No-op 653 | success() {}, // No-op 654 | try() {}, // No-op 655 | pause() {}, // No-op 656 | unpause() { 657 | return this.goto('TRY_ONE') 658 | }, 659 | }, 660 | 661 | TRY_ONE: { 662 | Enter() { 663 | return this.try() 664 | }, 665 | backoff() { 666 | return this.goto('BACKOFF') 667 | }, 668 | success(connectionRdy) { 669 | this.backoffTimer.success() 670 | this.onMessageSuccess(connectionRdy) 671 | return this.goto('MAX') 672 | }, 673 | try() {}, // No-op 674 | pause() { 675 | return this.goto('PAUSE') 676 | }, 677 | unpause() {}, 678 | }, // No-op 679 | 680 | MAX: { 681 | Enter() { 682 | this.balance() 683 | return this.bump() 684 | }, 685 | backoff() { 686 | return this.goto('BACKOFF') 687 | }, 688 | success(connectionRdy) { 689 | this.backoffTimer.success() 690 | return this.onMessageSuccess(connectionRdy) 691 | }, 692 | try() {}, // No-op 693 | pause() { 694 | return this.goto('PAUSE') 695 | }, 696 | unpause() {}, 697 | }, // No-op 698 | 699 | BACKOFF: { 700 | Enter() { 701 | this.backoffTimer.failure() 702 | return this.backoff() 703 | }, 704 | backoff() {}, // No-op 705 | success() {}, // No-op 706 | try() { 707 | return this.goto('TRY_ONE') 708 | }, 709 | pause() { 710 | return this.goto('PAUSE') 711 | }, 712 | unpause() {}, 713 | }, // No-op 714 | } 715 | 716 | ReaderRdy.prototype.transitions = { 717 | '*': { 718 | '*': function (data, callback) { 719 | this.log() 720 | return callback(data) 721 | }, 722 | }, 723 | } 724 | 725 | module.exports = {ReaderRdy, ConnectionRdy} 726 | -------------------------------------------------------------------------------- /test/readerrdy_test.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events') 2 | const _ = require('lodash') 3 | const should = require('should') 4 | const sinon = require('sinon') 5 | const rawMessage = require('./rawmessage') 6 | 7 | const Message = require('../lib/message') 8 | const {NSQDConnection} = require('../lib/nsqdconnection') 9 | const {ReaderRdy, ConnectionRdy} = require('../lib/readerrdy') 10 | 11 | class StubNSQDConnection extends EventEmitter { 12 | constructor( 13 | nsqdHost, 14 | nsqdPort, 15 | topic, 16 | channel, 17 | requeueDelay, 18 | heartbeatInterval 19 | ) { 20 | super() 21 | this.nsqdHost = nsqdHost 22 | this.nsqdPort = nsqdPort 23 | this.topic = topic 24 | this.channel = channel 25 | this.requeueDelay = requeueDelay 26 | this.heartbeatInterval = heartbeatInterval 27 | this.conn = {localPort: 1} 28 | this.maxRdyCount = 2500 29 | this.msgTimeout = 60 * 1000 30 | this.maxMsgTimeout = 15 * 60 * 1000 31 | this.rdyCounts = [] 32 | } 33 | 34 | id() { 35 | return `${this.nsqdHost}:${this.nsqdPort}` 36 | } 37 | 38 | // Empty 39 | connect() {} 40 | 41 | // Empty 42 | close() {} 43 | 44 | // Empty 45 | destroy() {} 46 | 47 | // Empty 48 | setRdy(rdyCount) { 49 | this.rdyCounts.push(rdyCount) 50 | } 51 | 52 | createMessage(msgId, msgTimestamp, attempts, msgBody) { 53 | const msgArgs = [ 54 | rawMessage(msgId, msgTimestamp, attempts, msgBody), 55 | this.requeueDelay, 56 | this.msgTimeout, 57 | this.maxMsgTimeout, 58 | ] 59 | const msg = new Message(...msgArgs) 60 | 61 | msg.on(Message.RESPOND, (responseType) => { 62 | if (responseType === Message.FINISH) { 63 | this.emit(NSQDConnection.FINISHED) 64 | } else if (responseType === Message.REQUEUE) { 65 | this.emit(NSQDConnection.REQUEUED) 66 | } 67 | }) 68 | 69 | msg.on(Message.BACKOFF, () => this.emit(NSQDConnection.BACKOFF)) 70 | 71 | this.emit(NSQDConnection.MESSAGE, msg) 72 | return msg 73 | } 74 | } 75 | 76 | const createNSQDConnection = (id) => { 77 | const conn = new StubNSQDConnection( 78 | `host${id}`, 79 | '4150', 80 | 'test', 81 | 'default', 82 | 60, 83 | 30 84 | ) 85 | conn.conn.localPort = id 86 | return conn 87 | } 88 | 89 | describe('ConnectionRdy', () => { 90 | let [conn, spy, cRdy] = Array.from([null, null, null]) 91 | 92 | beforeEach(() => { 93 | conn = createNSQDConnection(1) 94 | spy = sinon.spy(conn, 'setRdy') 95 | cRdy = new ConnectionRdy(conn) 96 | cRdy.start() 97 | }) 98 | 99 | it('should register listeners on a connection', () => { 100 | conn = new NSQDConnection('localhost', 1234, 'test', 'test') 101 | const mock = sinon.mock(conn) 102 | mock.expects('on').withArgs(NSQDConnection.ERROR) 103 | mock.expects('on').withArgs(NSQDConnection.FINISHED) 104 | mock.expects('on').withArgs(NSQDConnection.MESSAGE) 105 | mock.expects('on').withArgs(NSQDConnection.REQUEUED) 106 | mock.expects('on').withArgs(NSQDConnection.READY) 107 | cRdy = new ConnectionRdy(conn) 108 | mock.verify() 109 | }) 110 | 111 | it('should have a connection RDY max of zero', () => { 112 | should.equal(cRdy.maxConnRdy, 0) 113 | }) 114 | 115 | it('should not increase RDY when connection RDY max has not been set', () => { 116 | // This bump should be a no-op 117 | cRdy.bump() 118 | should.equal(cRdy.maxConnRdy, 0) 119 | should.equal(spy.called, false) 120 | }) 121 | 122 | it('should not allow RDY counts to be negative', () => { 123 | cRdy.setConnectionRdyMax(10) 124 | cRdy.setRdy(-1) 125 | should.equal(spy.notCalled, true) 126 | }) 127 | 128 | it('should not allow RDY counts to exceed the connection max', () => { 129 | cRdy.setConnectionRdyMax(10) 130 | cRdy.setRdy(9) 131 | cRdy.setRdy(10) 132 | cRdy.setRdy(20) 133 | should.equal(spy.calledTwice, true) 134 | should.equal(spy.firstCall.args[0], 9) 135 | should.equal(spy.secondCall.args[0], 10) 136 | }) 137 | 138 | it('should set RDY to max after initial bump', () => { 139 | cRdy.setConnectionRdyMax(3) 140 | cRdy.bump() 141 | should.equal(spy.firstCall.args[0], 3) 142 | }) 143 | 144 | it('should keep RDY at max after 1+ bumps', () => { 145 | cRdy.setConnectionRdyMax(3) 146 | for (let i = 1; i <= 3; i++) { 147 | cRdy.bump() 148 | } 149 | 150 | cRdy.maxConnRdy.should.eql(3) 151 | for (let i = 0; i < spy.callCount; i++) { 152 | should.ok(spy.getCall(i).args[0] <= 3) 153 | } 154 | }) 155 | 156 | it('should set RDY to zero from after first bump and then backoff', () => { 157 | cRdy.setConnectionRdyMax(3) 158 | cRdy.bump() 159 | cRdy.backoff() 160 | should.equal(spy.lastCall.args[0], 0) 161 | }) 162 | 163 | it('should set RDY to zero after 1+ bumps and then a backoff', () => { 164 | cRdy.setConnectionRdyMax(3) 165 | cRdy.bump() 166 | cRdy.backoff() 167 | should.equal(spy.lastCall.args[0], 0) 168 | }) 169 | 170 | it('should raise RDY when new connection RDY max is lower', () => { 171 | cRdy.setConnectionRdyMax(3) 172 | cRdy.bump() 173 | cRdy.setConnectionRdyMax(5) 174 | should.equal(cRdy.maxConnRdy, 5) 175 | should.equal(spy.lastCall.args[0], 5) 176 | }) 177 | 178 | it('should reduce RDY when new connection RDY max is higher', () => { 179 | cRdy.setConnectionRdyMax(3) 180 | cRdy.bump() 181 | cRdy.setConnectionRdyMax(2) 182 | should.equal(cRdy.maxConnRdy, 2) 183 | should.equal(spy.lastCall.args[0], 2) 184 | }) 185 | 186 | it('should update RDY when 75% of previous RDY is consumed', () => { 187 | let msg 188 | cRdy.setConnectionRdyMax(10) 189 | cRdy.bump() 190 | 191 | should.equal(spy.firstCall.args[0], 10) 192 | 193 | for (let i = 1; i <= 7; i++) { 194 | msg = conn.createMessage(`${i}`, Date.now(), 0, `Message ${i}`) 195 | msg.finish() 196 | cRdy.bump() 197 | } 198 | 199 | should.equal(spy.callCount, 1) 200 | 201 | msg = conn.createMessage('8', Date.now(), 0, 'Message 8') 202 | msg.finish() 203 | cRdy.bump() 204 | 205 | should.equal(spy.callCount, 2) 206 | should.equal(spy.lastCall.args[0], 10) 207 | }) 208 | }) 209 | 210 | describe('ReaderRdy', () => { 211 | let readerRdy = null 212 | 213 | beforeEach(() => { 214 | readerRdy = new ReaderRdy(1, 128, 'topic/channel') 215 | }) 216 | 217 | afterEach(() => readerRdy.close()) 218 | 219 | it('should register listeners on a connection', () => { 220 | // Stub out creation of ConnectionRdy to ignore the events registered by 221 | // ConnectionRdy. 222 | sinon.stub(readerRdy, 'createConnectionRdy').callsFake(() => ({on() {}})) 223 | // Empty 224 | 225 | const conn = createNSQDConnection(1) 226 | const mock = sinon.mock(conn) 227 | mock.expects('on').withArgs(NSQDConnection.CLOSED) 228 | mock.expects('on').withArgs(NSQDConnection.FINISHED) 229 | mock.expects('on').withArgs(NSQDConnection.REQUEUED) 230 | mock.expects('on').withArgs(NSQDConnection.BACKOFF) 231 | 232 | readerRdy.addConnection(conn) 233 | mock.verify() 234 | }) 235 | 236 | it('should be in the zero state until a new connection is READY', () => { 237 | const conn = createNSQDConnection(1) 238 | readerRdy.current_state_name.should.eql('ZERO') 239 | readerRdy.addConnection(conn) 240 | readerRdy.current_state_name.should.eql('ZERO') 241 | conn.emit(NSQDConnection.READY) 242 | readerRdy.current_state_name.should.eql('MAX') 243 | }) 244 | 245 | it('should be in the zero state if it loses all connections', () => { 246 | const conn = createNSQDConnection(1) 247 | readerRdy.addConnection(conn) 248 | conn.emit(NSQDConnection.READY) 249 | conn.emit(NSQDConnection.CLOSED) 250 | readerRdy.current_state_name.should.eql('ZERO') 251 | }) 252 | 253 | it('should evenly distribute RDY count across connections', () => { 254 | readerRdy = new ReaderRdy(100, 128, 'topic/channel') 255 | 256 | const conn1 = createNSQDConnection(1) 257 | const conn2 = createNSQDConnection(2) 258 | 259 | const setRdyStub1 = sinon.spy(conn1, 'setRdy') 260 | const setRdyStub2 = sinon.spy(conn2, 'setRdy') 261 | 262 | readerRdy.addConnection(conn1) 263 | conn1.emit(NSQDConnection.READY) 264 | 265 | setRdyStub1.lastCall.args[0].should.eql(100) 266 | 267 | readerRdy.addConnection(conn2) 268 | conn2.emit(NSQDConnection.READY) 269 | 270 | setRdyStub1.lastCall.args[0].should.eql(50) 271 | setRdyStub2.lastCall.args[0].should.eql(50) 272 | }) 273 | 274 | describe('low RDY conditions', () => { 275 | const assertAlternatingRdyCounts = (conn1, conn2) => { 276 | const minSize = Math.min(conn1.rdyCounts.length, conn2.rdyCounts.length) 277 | 278 | const zippedCounts = _.zip( 279 | conn1.rdyCounts.slice(-minSize), 280 | conn2.rdyCounts.slice(-minSize) 281 | ) 282 | 283 | // We expect the connection RDY counts to look like this: 284 | // conn 0: [1, 0, 1, 0] 285 | // conn 1: [0, 1, 0, 1] 286 | zippedCounts.forEach(([firstRdy, secondRdy]) => { 287 | should.ok(firstRdy + secondRdy === 1) 288 | }) 289 | } 290 | 291 | it('should periodically redistribute RDY', (done) => { 292 | // Shortening the periodically `balance` calls to every 10ms. 293 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 294 | 295 | const connections = [1, 2].map((i) => createNSQDConnection(i)) 296 | 297 | // Add the connections and trigger the NSQDConnection event that tells 298 | // listeners that the connections are connected and ready for message flow. 299 | connections.forEach((conn) => { 300 | readerRdy.addConnection(conn) 301 | conn.emit(NSQDConnection.READY) 302 | }) 303 | 304 | // Given the number of connections and the maxInFlight, we should be in low 305 | // RDY conditions. 306 | should.equal(readerRdy.isLowRdy(), true) 307 | 308 | const checkRdyCounts = () => { 309 | assertAlternatingRdyCounts(...connections) 310 | done() 311 | } 312 | 313 | // We have to wait a small period of time for log events to occur since the 314 | // `balance` call is invoked perdiocally. 315 | setTimeout(checkRdyCounts, 50) 316 | }) 317 | 318 | it('should handle the transition from normal', (done) => { 319 | // Shortening the periodica `balance` calls to every 10ms. 320 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 321 | 322 | const conn1 = createNSQDConnection(1) 323 | const conn2 = createNSQDConnection(2) 324 | 325 | // Add the connections and trigger the NSQDConnection event that tells 326 | // listeners that the connections are connected and ready for message flow. 327 | readerRdy.addConnection(conn1) 328 | conn1.emit(NSQDConnection.READY) 329 | 330 | should.equal(readerRdy.isLowRdy(), false) 331 | 332 | const addConnection = () => { 333 | readerRdy.addConnection(conn2) 334 | conn2.emit(NSQDConnection.READY) 335 | 336 | // Given the number of connections and the maxInFlight, we should be in 337 | // low RDY conditions. 338 | should.equal(readerRdy.isLowRdy(), true) 339 | } 340 | 341 | // Add the 2nd connections after some duration to simulate a new nsqd being 342 | // discovered and connected. 343 | setTimeout(addConnection, 20) 344 | 345 | const checkRdyCounts = () => { 346 | assertAlternatingRdyCounts(conn1, conn2) 347 | done() 348 | } 349 | 350 | // We have to wait a small period of time for log events to occur since the 351 | // `balance` call is invoked perdiocally. 352 | setTimeout(checkRdyCounts, 40) 353 | }) 354 | 355 | it('should handle the transition to normal conditions', (done) => { 356 | // Shortening the periodica `balance` calls to every 10ms. 357 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 358 | 359 | const connections = [1, 2].map((i) => createNSQDConnection(i)) 360 | 361 | // Add the connections and trigger the NSQDConnection event that tells 362 | // listeners that the connections are connected and ready for message flow. 363 | connections.forEach((conn) => { 364 | readerRdy.addConnection(conn) 365 | conn.emit(NSQDConnection.READY) 366 | }) 367 | 368 | should.equal(readerRdy.isLowRdy(), true) 369 | readerRdy.isLowRdy().should.eql(true) 370 | 371 | const checkNormal = () => { 372 | should.equal(readerRdy.isLowRdy(), false) 373 | should.equal(readerRdy.balanceId, null) 374 | should.equal(readerRdy.connections[0].lastRdySent, 1) 375 | done() 376 | } 377 | 378 | const removeConnection = () => { 379 | connections[1].emit(NSQDConnection.CLOSED) 380 | setTimeout(checkNormal, 20) 381 | } 382 | 383 | // Remove a connection after some period of time to get back to normal 384 | // conditions. 385 | setTimeout(removeConnection, 20) 386 | }) 387 | 388 | it('should move to normal conditions with connections in backoff', (done) => { 389 | /* 390 | 1. Create two nsqd connections 391 | 2. Close the 2nd connection when the first connection is in the BACKOFF 392 | state. 393 | 3. Check to see if the 1st connection does get it's RDY count. 394 | */ 395 | 396 | // Shortening the periodica `balance` calls to every 10ms. 397 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 398 | 399 | const connections = [1, 2].map((i) => createNSQDConnection(i)) 400 | 401 | connections.forEach((conn) => { 402 | readerRdy.addConnection(conn) 403 | conn.emit(NSQDConnection.READY) 404 | }) 405 | 406 | should.equal(readerRdy.isLowRdy(), true) 407 | 408 | const checkNormal = () => { 409 | should.equal(readerRdy.isLowRdy(), false) 410 | should.equal(readerRdy.balanceId, null) 411 | should.equal(readerRdy.connections[0].lastRdySent, 1) 412 | done() 413 | } 414 | 415 | const removeConnection = _.once(() => { 416 | connections[1].emit(NSQDConnection.CLOSED) 417 | setTimeout(checkNormal, 30) 418 | }) 419 | 420 | const removeOnBackoff = () => { 421 | const connRdy1 = readerRdy.connections[0] 422 | connRdy1.on(ConnectionRdy.STATE_CHANGE, () => { 423 | if (connRdy1.statemachine.current_state_name === 'BACKOFF') { 424 | // If we don't do the connection CLOSED in the next tick, we remove 425 | // the connection immediately which leaves `@connections` within 426 | // `balance` in an inconsistent state which isn't possible normally. 427 | setTimeout(removeConnection, 0) 428 | } 429 | }) 430 | } 431 | 432 | // Remove a connection after some period of time to get back to normal 433 | // conditions. 434 | setTimeout(removeOnBackoff, 20) 435 | }) 436 | 437 | it('should not exceed maxInFlight for long running message.', (done) => { 438 | // Shortening the periodica `balance` calls to every 10ms. 439 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 440 | 441 | const connections = [1, 2].map((i) => createNSQDConnection(i)) 442 | 443 | connections.forEach((conn) => { 444 | readerRdy.addConnection(conn) 445 | conn.emit(NSQDConnection.READY) 446 | }) 447 | 448 | // Handle the message but delay finishing the message so that several 449 | // balance calls happen and the check to ensure that RDY count is zero for 450 | // all connections. 451 | const handleMessage = (msg) => { 452 | const finish = () => { 453 | msg.finish() 454 | done() 455 | } 456 | 457 | setTimeout(finish, 40) 458 | } 459 | 460 | connections.forEach((conn) => { 461 | conn.on(NSQDConnection.MESSAGE, handleMessage) 462 | }) 463 | 464 | // When the message is in-flight, balance cannot give a RDY count out to 465 | // any of the connections. 466 | const checkRdyCount = () => { 467 | should.equal(readerRdy.isLowRdy(), true) 468 | should.equal(readerRdy.connections[0].lastRdySent, 0) 469 | should.equal(readerRdy.connections[1].lastRdySent, 0) 470 | } 471 | 472 | const sendMessageOnce = _.once(() => { 473 | connections[1].createMessage('1', Date.now(), 0, Buffer.from('test')) 474 | setTimeout(checkRdyCount, 20) 475 | }) 476 | 477 | // Send a message on the 2nd connection when we can. Only send the message 478 | // once so that we don't violate the maxInFlight count. 479 | const sendOnRdy = () => { 480 | const connRdy2 = readerRdy.connections[1] 481 | connRdy2.on(ConnectionRdy.STATE_CHANGE, () => { 482 | if ( 483 | ['ONE', 'MAX'].includes(connRdy2.statemachine.current_state_name) 484 | ) { 485 | sendMessageOnce() 486 | } 487 | }) 488 | } 489 | 490 | // We have to wait a small period of time for log events to occur since the 491 | // `balance` call is invoked perdiocally. 492 | setTimeout(sendOnRdy, 20) 493 | }) 494 | 495 | it('should recover losing a connection with a message in-flight', (done) => { 496 | /* 497 | Detailed description: 498 | 1. Connect to 5 nsqds and add them to the ReaderRdy 499 | 2. When the 1st connection has the shared RDY count, it receives a 500 | message. 501 | 3. On receipt of a message, the 1st connection will process the message 502 | for a long period of time. 503 | 4. While the message is being processed, the 1st connection will close. 504 | 5. Finally, check that the other connections are indeed now getting the 505 | RDY count. 506 | */ 507 | 508 | // Shortening the periodica `balance` calls to every 10ms. 509 | readerRdy = new ReaderRdy(1, 128, 'topic/channel', 0.01) 510 | 511 | const connections = [1, 2, 3, 4, 5].map((i) => createNSQDConnection(i)) 512 | 513 | // Add the connections and trigger the NSQDConnection event that tells 514 | // listeners that the connections are connected and ready for message flow. 515 | connections.forEach((conn) => { 516 | readerRdy.addConnection(conn) 517 | conn.emit(NSQDConnection.READY) 518 | }) 519 | 520 | const closeConnection = _.once(() => { 521 | connections[0].emit(NSQDConnection.CLOSED) 522 | }) 523 | 524 | // When the message is in-flight, balance cannot give a RDY count out to 525 | // any of the connections. 526 | const checkRdyCount = () => { 527 | should.equal(readerRdy.isLowRdy(), true) 528 | 529 | const rdyCounts = Array.from(readerRdy.connections).map( 530 | (connRdy) => connRdy.lastRdySent 531 | ) 532 | 533 | should.equal(readerRdy.connections.length, 4) 534 | should.ok(Array.from(rdyCounts).includes(1)) 535 | } 536 | 537 | const handleMessage = (msg) => { 538 | const delayFinish = () => { 539 | msg.finish() 540 | done() 541 | } 542 | 543 | setTimeout(closeConnection, 10) 544 | setTimeout(checkRdyCount, 30) 545 | setTimeout(delayFinish, 50) 546 | } 547 | 548 | connections.forEach((conn) => { 549 | conn.on(NSQDConnection.MESSAGE, handleMessage) 550 | }) 551 | 552 | const sendMessageOnce = _.once(() => { 553 | connections[0].createMessage('1', Date.now(), 0, Buffer.from('test')) 554 | }) 555 | 556 | // Send a message on the 2nd connection when we can. Only send the message 557 | // once so that we don't violate the maxInFlight count. 558 | const sendOnRdy = () => { 559 | const connRdy = readerRdy.connections[0] 560 | connRdy.on(ConnectionRdy.STATE_CHANGE, () => { 561 | if ( 562 | ['ONE', 'MAX'].includes(connRdy.statemachine.current_state_name) 563 | ) { 564 | sendMessageOnce() 565 | } 566 | }) 567 | } 568 | 569 | // We have to wait a small period of time for log events to occur since the 570 | // `balance` call is invoked perdiocally. 571 | setTimeout(sendOnRdy, 10) 572 | }) 573 | }) 574 | 575 | describe('try', () => { 576 | it('should on completion of backoff attempt a single connection', (done) => { 577 | /* 578 | Detailed description: 579 | 1. Create ReaderRdy with connections to 5 nsqds. 580 | 2. Generate a message from an nsqd that causes a backoff. 581 | 3. Verify that all the nsqds are in backoff mode. 582 | 4. At the end of the backoff period, verify that only one ConnectionRdy 583 | is in the try one state and the others are still in backoff. 584 | */ 585 | 586 | // Shortening the periodic `balance` calls to every 10ms. Changing the 587 | // max backoff duration to 10 sec. 588 | readerRdy = new ReaderRdy(100, 10, 'topic/channel', 0.01) 589 | 590 | const connections = [1, 2, 3, 4, 5].map((i) => createNSQDConnection(i)) 591 | 592 | connections.forEach((conn) => { 593 | readerRdy.addConnection(conn) 594 | conn.emit(NSQDConnection.READY) 595 | }) 596 | 597 | connections[0] 598 | .createMessage('1', Date.now(), 0, 'Message causing a backoff') 599 | .requeue() 600 | 601 | const checkInBackoff = () => { 602 | readerRdy.connections.forEach((connRdy) => { 603 | connRdy.statemachine.current_state_name.should.eql('BACKOFF') 604 | }) 605 | } 606 | 607 | checkInBackoff() 608 | 609 | const afterBackoff = () => { 610 | const states = readerRdy.connections.map( 611 | (connRdy) => connRdy.statemachine.current_state_name 612 | ) 613 | 614 | const ones = states.filter((state) => state === 'ONE') 615 | const backoffs = states.filter((state) => state === 'BACKOFF') 616 | 617 | should.equal(ones.length, 1) 618 | should.equal(backoffs.length, 4) 619 | done() 620 | } 621 | 622 | // Add 50ms to the delay so that we're confident that the event fired. 623 | const delay = readerRdy.backoffTimer.getInterval() + 0.05 624 | 625 | setTimeout(afterBackoff, delay.valueOf() * 1000) 626 | }) 627 | 628 | it('should after backoff with a successful message go to MAX', (done) => { 629 | /* 630 | Detailed description: 631 | 1. Create ReaderRdy with connections to 5 nsqds. 632 | 2. Generate a message from an nsqd that causes a backoff. 633 | 3. At the end of backoff, generate a message that will succeed. 634 | 4. Verify that ReaderRdy is in MAX and ConnectionRdy instances are in 635 | either ONE or MAX. At least on ConnectionRdy should be in MAX as well. 636 | */ 637 | 638 | // Shortening the periodica `balance` calls to every 10ms. Changing the 639 | // max backoff duration to 1 sec. 640 | readerRdy = new ReaderRdy(100, 1, 'topic/channel', 0.01) 641 | 642 | const connections = [1, 2, 3, 4, 5].map((i) => createNSQDConnection(i)) 643 | 644 | connections.forEach((conn) => { 645 | readerRdy.addConnection(conn) 646 | conn.emit(NSQDConnection.READY) 647 | }) 648 | 649 | let msg = connections[0].createMessage( 650 | '1', 651 | Date.now(), 652 | 0, 653 | 'Message causing a backoff' 654 | ) 655 | 656 | msg.requeue() 657 | 658 | const afterBackoff = () => { 659 | const [connRdy] = readerRdy.connections.filter( 660 | (conn) => conn.statemachine.current_state_name === 'ONE' 661 | ) 662 | 663 | msg = connRdy.conn.createMessage('1', Date.now(), 0, 'Success') 664 | msg.finish() 665 | 666 | const verifyMax = () => { 667 | const states = readerRdy.connections.map( 668 | (conn) => conn.statemachine.current_state_name 669 | ) 670 | 671 | const max = states.filter((s) => ['ONE', 'MAX'].includes(s)) 672 | 673 | max.length.should.eql(5) 674 | should.equal(max.length, 5) 675 | should.ok(states.includes('MAX')) 676 | done() 677 | } 678 | 679 | setTimeout(verifyMax, 0) 680 | } 681 | 682 | const delay = readerRdy.backoffTimer.getInterval() + 100 683 | setTimeout(afterBackoff, delay) 684 | }) 685 | }) 686 | 687 | describe('backoff', () => { 688 | it('should not increase interval with more failures during backoff', () => { 689 | readerRdy = new ReaderRdy(100, 1, 'topic/channel', 0.01) 690 | 691 | // Create a connection and make it ready. 692 | const c = createNSQDConnection(0) 693 | readerRdy.addConnection(c) 694 | c.emit(NSQDConnection.READY) 695 | 696 | readerRdy.raise('backoff') 697 | const interval = readerRdy.backoffTimer.getInterval() 698 | 699 | readerRdy.raise('backoff') 700 | readerRdy.backoffTimer.getInterval().should.eql(interval) 701 | }) 702 | }) 703 | 704 | describe('pause / unpause', () => { 705 | beforeEach(() => { 706 | // Shortening the periodic `balance` calls to every 10ms. Changing the 707 | // max backoff duration to 1 sec. 708 | readerRdy = new ReaderRdy(100, 1, 'topic/channel', 0.01) 709 | 710 | const connections = [1, 2, 3, 4, 5].map((i) => createNSQDConnection(i)) 711 | 712 | connections.forEach((conn) => { 713 | readerRdy.addConnection(conn) 714 | conn.emit(NSQDConnection.READY) 715 | }) 716 | }) 717 | 718 | it('should drop ready count to zero on all connections when paused', () => { 719 | readerRdy.pause() 720 | should.equal(readerRdy.current_state_name, 'PAUSE') 721 | readerRdy.connections.forEach((conn) => should.equal(conn.lastRdySent, 0)) 722 | }) 723 | 724 | it('should unpause by trying one', () => { 725 | readerRdy.pause() 726 | readerRdy.unpause() 727 | should.equal(readerRdy.current_state_name, 'TRY_ONE') 728 | }) 729 | 730 | it('should update the value of @isPaused when paused', () => { 731 | readerRdy.pause() 732 | should.equal(readerRdy.isPaused(), true) 733 | readerRdy.unpause() 734 | should.equal(readerRdy.isPaused(), false) 735 | }) 736 | }) 737 | }) 738 | -------------------------------------------------------------------------------- /lib/nsqdconnection.js: -------------------------------------------------------------------------------- 1 | const {EventEmitter} = require('events') 2 | const net = require('net') 3 | const os = require('os') 4 | const tls = require('tls') 5 | const zlib = require('zlib') 6 | 7 | const NodeState = require('node-state') 8 | const _ = require('lodash') 9 | const debug = require('./debug') 10 | 11 | const wire = require('./wire') 12 | const FrameBuffer = require('./framebuffer') 13 | const Message = require('./message') 14 | const version = require('./version') 15 | const {ConnectionConfig, joinHostPort} = require('./config') 16 | 17 | /** 18 | * NSQDConnection is a reader connection to a nsqd instance. It manages all 19 | * aspects of the nsqd connection with the exception of the RDY count which 20 | * needs to be managed across all nsqd connections for a given topic / channel 21 | * pair. 22 | * 23 | * This shouldn't be used directly. Use a Reader instead. 24 | * 25 | * Usage: 26 | * const c = new NSQDConnection('127.0.0.1', 4150, 'test', 'default', 60, 30) 27 | * 28 | * c.on(NSQDConnection.MESSAGE, (msg) => { 29 | * console.log(`[message]: ${msg.attempts}, ${msg.body.toString()}`) 30 | * console.log(`Timeout of message is ${msg.timeUntilTimeout()}`) 31 | * setTimeout(() => console.log(`${msg.timeUntilTimeout()}`), 5000) 32 | * msg.finish() 33 | * }) 34 | * 35 | * c.on(NSQDConnection.FINISHED, () => c.setRdy(1)) 36 | * 37 | * c.on(NSQDConnection.READY, () => { 38 | * console.log('Callback [ready]: Set RDY to 100') 39 | * c.setRdy(10) 40 | * }) 41 | * 42 | * c.on(NSQDConnection.CLOSED, () => { 43 | * console.log('Callback [closed]: Lost connection to nsqd') 44 | * }) 45 | * 46 | * c.on(NSQDConnection.ERROR, (err) => { 47 | * console.log(`Callback [error]: ${err}`) 48 | * }) 49 | * 50 | * c.on(NSQDConnection.BACKOFF, () => { 51 | * console.log('Callback [backoff]: RDY 0') 52 | * c.setRdy(0) 53 | * setTimeout(() => { 54 | * c.setRdy 100; 55 | * console.log('RDY 100') 56 | * }, 10 * 1000) 57 | * }) 58 | * 59 | * c.connect() 60 | */ 61 | class NSQDConnection extends EventEmitter { 62 | // Events emitted by NSQDConnection 63 | static get BACKOFF() { 64 | return 'backoff' 65 | } 66 | static get CONNECTED() { 67 | return 'connected' 68 | } 69 | static get CLOSED() { 70 | return 'closed' 71 | } 72 | static get CONNECTION_ERROR() { 73 | return 'connection_error' 74 | } 75 | static get ERROR() { 76 | return 'error' 77 | } 78 | static get FINISHED() { 79 | return 'finished' 80 | } 81 | static get MESSAGE() { 82 | return 'message' 83 | } 84 | static get REQUEUED() { 85 | return 'requeued' 86 | } 87 | static get READY() { 88 | return 'ready' 89 | } 90 | 91 | /** 92 | * Instantiates a new NSQDConnection. 93 | * 94 | * @constructor 95 | * @param {String} nsqdHost 96 | * @param {String|Number} nsqdPort 97 | * @param {String} topic 98 | * @param {String} channel 99 | * @param {Object} [options={}] 100 | */ 101 | constructor(nsqdHost, nsqdPort, topic, channel, options = {}) { 102 | super(nsqdHost, nsqdPort, topic, channel, options) 103 | 104 | this.nsqdHost = nsqdHost 105 | this.nsqdPort = nsqdPort 106 | this.topic = topic 107 | this.channel = channel 108 | const connId = this.id().replace(':', '/') 109 | this.debug = debug( 110 | `nsqjs:reader:${this.topic}/${this.channel}:conn:${connId}` 111 | ) 112 | 113 | this.config = new ConnectionConfig(options) 114 | this.config.validate() 115 | 116 | this.frameBuffer = new FrameBuffer() 117 | this.statemachine = this.connectionState() 118 | 119 | this.maxRdyCount = 0 // Max RDY value for a conn to this NSQD 120 | this.msgTimeout = 0 // Timeout time in milliseconds for a Message 121 | this.maxMsgTimeout = 0 // Max time to process a Message in millisecs 122 | this.nsqdVersion = null // Version returned by nsqd 123 | this.lastMessageTimestamp = null // Timestamp of last message received 124 | this.lastReceivedTimestamp = null // Timestamp of last data received 125 | this.conn = null // Socket connection to NSQD 126 | this.identifyTimeoutId = null // Timeout ID for triggering identifyFail 127 | this.messageCallbacks = [] // Callbacks on message sent responses 128 | 129 | this.writeQueue = [] 130 | this.onDataFn = null 131 | this.outWriter = null 132 | } 133 | 134 | /** 135 | * The nsqd host:port pair. 136 | * 137 | * @return {[type]} [description] 138 | */ 139 | id() { 140 | return joinHostPort(this.nsqdHost, this.nsqdPort) 141 | } 142 | 143 | /** 144 | * Instantiates or returns a new ConnectionState. 145 | * 146 | * @return {ConnectionState} 147 | */ 148 | connectionState() { 149 | return this.statemachine || new ConnectionState(this) 150 | } 151 | 152 | /** 153 | * Creates a new nsqd connection. 154 | */ 155 | connect() { 156 | this.statemachine.raise('connecting') 157 | 158 | // Using nextTick so that clients of Reader can register event listeners 159 | // right after calling connect. 160 | process.nextTick(() => { 161 | this.conn = net.connect( 162 | {port: this.nsqdPort, host: this.nsqdHost}, 163 | () => { 164 | this.statemachine.raise('connected') 165 | this.emit(NSQDConnection.CONNECTED) 166 | 167 | // Once there's a socket connection, give it 5 seconds to receive an 168 | // identify response. 169 | this.identifyTimeoutId = setTimeout(() => { 170 | this.identifyTimeout() 171 | }, 5000) 172 | 173 | this.identifyTimeoutId 174 | } 175 | ) 176 | this.conn.setNoDelay(true) 177 | this.outWriter = this.conn 178 | 179 | this.registerStreamListeners(this.conn) 180 | }) 181 | } 182 | 183 | /** 184 | * Register event handlers for the nsqd connection. 185 | * 186 | * @param {Object} conn 187 | */ 188 | registerStreamListeners(conn) { 189 | this.onDataFn = (data) => this.receiveData(data) 190 | conn.on('data', this.onDataFn) 191 | conn.on('end', () => { 192 | this.statemachine.goto('CLOSED') 193 | }) 194 | conn.on('error', (err) => { 195 | this.statemachine.goto('ERROR', err) 196 | this.emit('connection_error', err) 197 | }) 198 | conn.on('close', () => this.statemachine.raise('close')) 199 | conn.setTimeout(this.config.idleTimeout * 1000, () => 200 | this.statemachine.raise('close') 201 | ) 202 | } 203 | 204 | /** 205 | * Connect via tls. 206 | * 207 | * @param {Function} callback 208 | */ 209 | startTLS(callback) { 210 | for (const event of ['data', 'error', 'close']) { 211 | this.conn.removeAllListeners(event) 212 | } 213 | 214 | const options = { 215 | socket: this.conn, 216 | rejectUnauthorized: this.config.tlsVerification, 217 | ca: this.config.ca, 218 | key: this.config.key, 219 | cert: this.config.cert, 220 | } 221 | 222 | let tlsConn = tls.connect(options, () => { 223 | this.conn = tlsConn 224 | typeof callback === 'function' ? callback() : undefined 225 | }) 226 | 227 | this.outWriter = tlsConn 228 | this.registerStreamListeners(tlsConn) 229 | } 230 | 231 | /** 232 | * startCompression wraps the TCP connection stream. 233 | * 234 | * @param {Stream} inflater - Decompression stream 235 | * @param {Stream} deflater - Compression stream 236 | */ 237 | startCompression(inflater, deflater) { 238 | this.inflater = inflater 239 | this.deflater = deflater 240 | 241 | this.conn.removeListener('data', this.onDataFn) 242 | this.conn.pipe(this.inflater) 243 | this.inflater.on('data', this.onDataFn) 244 | 245 | this.outWriter = this.deflater 246 | this.outWriter.pipe(this.conn) 247 | 248 | if (this.frameBuffer.buffer) { 249 | const b = this.frameBuffer.buffer 250 | this.frameBuffer.buffer = null 251 | setImmediate(() => this.inflater.write(b)) 252 | } 253 | } 254 | 255 | /** 256 | * Begin deflating the frame buffer. Actualy deflating is handled by 257 | * zlib. 258 | * 259 | * @param {Number} level 260 | */ 261 | startDeflate(level) { 262 | this.startCompression( 263 | zlib.createInflateRaw({flush: zlib.constants.Z_SYNC_FLUSH}), 264 | zlib.createDeflateRaw({ 265 | level, 266 | flush: zlib.constants.Z_SYNC_FLUSH, 267 | }) 268 | ) 269 | } 270 | 271 | /** 272 | * Create a snappy stream. 273 | */ 274 | startSnappy() { 275 | const {SnappyStream, UnsnappyStream} = require('snappystream') 276 | this.startCompression(new UnsnappyStream(), new SnappyStream()) 277 | } 278 | 279 | /** 280 | * Raise a `READY` event with the specified count. 281 | * 282 | * @param {Number} rdyCount 283 | */ 284 | setRdy(rdyCount) { 285 | this.statemachine.raise('ready', rdyCount) 286 | } 287 | 288 | /** 289 | * Handle receiveing the message payload frame by frame. 290 | * 291 | * @param {Object} data 292 | */ 293 | receiveData(data) { 294 | this.lastReceivedTimestamp = Date.now() 295 | this.frameBuffer.consume(data) 296 | 297 | let frame = this.frameBuffer.nextFrame() 298 | 299 | while (frame) { 300 | const [frameId, payload] = Array.from(frame) 301 | switch (frameId) { 302 | case wire.FRAME_TYPE_RESPONSE: 303 | this.statemachine.raise('response', payload) 304 | break 305 | case wire.FRAME_TYPE_ERROR: 306 | this.statemachine.goto('ERROR', new Error(payload.toString())) 307 | break 308 | case wire.FRAME_TYPE_MESSAGE: 309 | this.lastMessageTimestamp = this.lastReceivedTimestamp 310 | this.statemachine.raise('consumeMessage', this.createMessage(payload)) 311 | break 312 | } 313 | 314 | frame = this.frameBuffer.nextFrame() 315 | } 316 | } 317 | 318 | /** 319 | * Generates client metadata so that nsqd can identify connections. 320 | * 321 | * @return {Object} The connection metadata. 322 | */ 323 | identify() { 324 | const longName = os.hostname() 325 | const shortName = longName.split('.')[0] 326 | 327 | const identify = { 328 | client_id: this.config.clientId || shortName, 329 | deflate: this.config.deflate, 330 | deflate_level: this.config.deflateLevel, 331 | feature_negotiation: true, 332 | heartbeat_interval: this.config.heartbeatInterval * 1000, 333 | hostname: longName, 334 | long_id: longName, // Remove when deprecating pre 1.0 335 | msg_timeout: this.config.messageTimeout, 336 | output_buffer_size: this.config.outputBufferSize, 337 | output_buffer_timeout: this.config.outputBufferTimeout, 338 | sample_rate: this.config.sampleRate, 339 | short_id: shortName, // Remove when deprecating pre 1.0 340 | snappy: this.config.snappy, 341 | tls_v1: this.config.tls, 342 | user_agent: `nsqjs/${version}`, 343 | } 344 | 345 | // Remove some keys when they're effectively not provided. 346 | const removableKeys = [ 347 | 'msg_timeout', 348 | 'output_buffer_size', 349 | 'output_buffer_timeout', 350 | 'sample_rate', 351 | ] 352 | 353 | removableKeys.forEach((key) => { 354 | if (identify[key] === null) { 355 | delete identify[key] 356 | } 357 | }) 358 | 359 | return identify 360 | } 361 | 362 | /** 363 | * Throws an error if the connection timed out while identifying the nsqd. 364 | */ 365 | identifyTimeout() { 366 | this.statemachine.goto( 367 | 'ERROR', 368 | new Error('Timed out identifying with nsqd') 369 | ) 370 | } 371 | 372 | /** 373 | * Clears an identify timeout. Useful for retries. 374 | */ 375 | clearIdentifyTimeout() { 376 | clearTimeout(this.identifyTimeoutId) 377 | this.identifyTimeoutId = null 378 | } 379 | 380 | /** 381 | * Create a new message from the payload. 382 | * 383 | * @param {Buffer} msgPayload 384 | * @return {Message} 385 | */ 386 | createMessage(msgPayload) { 387 | const msg = new Message( 388 | msgPayload, 389 | this.config.requeueDelay, 390 | this.msgTimeout, 391 | this.maxMsgTimeout 392 | ) 393 | 394 | this.debug(`Received message [${msg.id}] [attempts: ${msg.attempts}]`) 395 | 396 | msg.on(Message.RESPOND, (responseType, wireData) => { 397 | this.write(wireData) 398 | 399 | if (responseType === Message.FINISH) { 400 | this.debug( 401 | `Finished message [${msg.id}] [timedout=${msg.timedout === true}, \ 402 | elapsed=${Date.now() - msg.receivedOn}ms, \ 403 | touch_count=${msg.touchCount}]` 404 | ) 405 | this.emit(NSQDConnection.FINISHED) 406 | } else if (responseType === Message.REQUEUE) { 407 | this.debug(`Requeued message [${msg.id}]`) 408 | this.emit(NSQDConnection.REQUEUED) 409 | } 410 | }) 411 | 412 | msg.on(Message.BACKOFF, () => this.emit(NSQDConnection.BACKOFF)) 413 | 414 | return msg 415 | } 416 | 417 | /** 418 | * Write a message to the connection. Deflate it if necessary. 419 | * @param {Object} data 420 | */ 421 | write(data) { 422 | if (Buffer.isBuffer(data)) { 423 | this.outWriter.write(data) 424 | } else { 425 | this.outWriter.write(Buffer.from(data)) 426 | } 427 | } 428 | 429 | _flush() { 430 | if (this.writeQueue.length > 0) { 431 | const data = Buffer.concat(this.writeQueue) 432 | 433 | if (this.deflater) { 434 | this.deflater.write(data, () => this.conn.write(this.deflater.read())) 435 | } else { 436 | this.conn.write(data) 437 | } 438 | } 439 | 440 | this.writeQueue = [] 441 | } 442 | 443 | /** 444 | * Close the nsqd connection. 445 | */ 446 | close() { 447 | if ( 448 | !this.conn.destroyed && 449 | this.statemachine.current_state !== 'CLOSED' && 450 | this.statemachine.current_state !== 'ERROR' 451 | ) { 452 | try { 453 | this.conn.end(wire.close()) 454 | } catch (e) { 455 | // Continue regardless of error. 456 | } 457 | } 458 | this.statemachine.goto('CLOSED') 459 | } 460 | 461 | /** 462 | * Destroy the nsqd connection. 463 | */ 464 | destroy() { 465 | if (!this.conn.destroyed) { 466 | this.conn.destroy() 467 | } 468 | } 469 | } 470 | 471 | /** 472 | * A statemachine modeling the connection state of an nsqd connection. 473 | * @type {ConnectionState} 474 | */ 475 | class ConnectionState extends NodeState { 476 | /** 477 | * Instantiates a new instance of ConnectionState. 478 | * 479 | * @constructor 480 | * @param {Object} conn 481 | */ 482 | constructor(conn) { 483 | super({ 484 | autostart: true, 485 | initial_state: 'INIT', 486 | sync_goto: true, 487 | }) 488 | 489 | this.conn = conn 490 | this.identifyResponse = null 491 | } 492 | 493 | /** 494 | * @param {*} message 495 | */ 496 | log(message) { 497 | if (this.current_state_name !== 'INIT') { 498 | this.conn.debug(`${this.current_state_name}`) 499 | } 500 | if (message) { 501 | this.conn.debug(message) 502 | } 503 | } 504 | 505 | /** 506 | * @return {String} 507 | */ 508 | afterIdentify() { 509 | return 'SUBSCRIBE' 510 | } 511 | } 512 | 513 | ConnectionState.prototype.states = { 514 | INIT: { 515 | connecting() { 516 | return this.goto('CONNECTING') 517 | }, 518 | 519 | close() { 520 | return this.goto('CLOSED') 521 | }, 522 | }, 523 | 524 | CONNECTING: { 525 | connected() { 526 | return this.goto('CONNECTED') 527 | }, 528 | 529 | close() { 530 | return this.goto('CLOSED') 531 | }, 532 | }, 533 | 534 | CONNECTED: { 535 | Enter() { 536 | return this.goto('SEND_MAGIC_IDENTIFIER') 537 | }, 538 | }, 539 | 540 | SEND_MAGIC_IDENTIFIER: { 541 | Enter() { 542 | // Send the magic protocol identifier to the connection 543 | this.conn.write(wire.MAGIC_V2) 544 | return this.goto('IDENTIFY') 545 | }, 546 | }, 547 | 548 | IDENTIFY: { 549 | Enter() { 550 | // Send configuration details 551 | const identify = this.conn.identify() 552 | this.conn.debug(identify) 553 | this.conn.write(wire.identify(identify)) 554 | return this.goto('IDENTIFY_RESPONSE') 555 | }, 556 | }, 557 | 558 | IDENTIFY_RESPONSE: { 559 | response(data) { 560 | if (data.toString() === 'OK') { 561 | data = JSON.stringify({ 562 | max_rdy_count: 2500, 563 | max_msg_timeout: 15 * 60 * 1000, // 15 minutes 564 | msg_timeout: 60 * 1000, 565 | }) // 1 minute 566 | } 567 | 568 | this.identifyResponse = JSON.parse(data) 569 | this.conn.debug(this.identifyResponse) 570 | this.conn.maxRdyCount = this.identifyResponse.max_rdy_count 571 | this.conn.maxMsgTimeout = this.identifyResponse.max_msg_timeout 572 | this.conn.msgTimeout = this.identifyResponse.msg_timeout 573 | this.conn.nsqdVersion = this.identifyResponse.version 574 | this.conn.clearIdentifyTimeout() 575 | 576 | if (this.identifyResponse.tls_v1) { 577 | return this.goto('TLS_START') 578 | } 579 | return this.goto('IDENTIFY_COMPRESSION_CHECK') 580 | }, 581 | 582 | close() { 583 | return this.goto('CLOSED') 584 | }, 585 | }, 586 | 587 | IDENTIFY_COMPRESSION_CHECK: { 588 | Enter() { 589 | const {deflate, snappy} = this.identifyResponse 590 | 591 | if (deflate) { 592 | return this.goto('DEFLATE_START', this.identifyResponse.deflate_level) 593 | } 594 | if (snappy) { 595 | return this.goto('SNAPPY_START') 596 | } 597 | return this.goto('AUTH') 598 | }, 599 | }, 600 | 601 | TLS_START: { 602 | Enter() { 603 | this.conn.startTLS() 604 | return this.goto('TLS_RESPONSE') 605 | }, 606 | }, 607 | 608 | TLS_RESPONSE: { 609 | response(data) { 610 | if (data.toString() === 'OK') { 611 | return this.goto('IDENTIFY_COMPRESSION_CHECK') 612 | } 613 | return this.goto('ERROR', new Error('TLS negotiate error with nsqd')) 614 | }, 615 | 616 | close() { 617 | return this.goto('CLOSED') 618 | }, 619 | }, 620 | 621 | DEFLATE_START: { 622 | Enter(level) { 623 | this.conn.startDeflate(level) 624 | return this.goto('COMPRESSION_RESPONSE') 625 | }, 626 | }, 627 | 628 | SNAPPY_START: { 629 | Enter() { 630 | this.conn.startSnappy() 631 | return this.goto('COMPRESSION_RESPONSE') 632 | }, 633 | }, 634 | 635 | COMPRESSION_RESPONSE: { 636 | response(data) { 637 | if (data.toString() === 'OK') { 638 | return this.goto('AUTH') 639 | } 640 | return this.goto( 641 | 'ERROR', 642 | new Error('Bad response when enabling compression') 643 | ) 644 | }, 645 | 646 | close() { 647 | return this.goto('CLOSED') 648 | }, 649 | }, 650 | 651 | AUTH: { 652 | Enter() { 653 | if (!this.conn.config.authSecret) { 654 | return this.goto(this.afterIdentify()) 655 | } 656 | this.conn.write(wire.auth(this.conn.config.authSecret)) 657 | return this.goto('AUTH_RESPONSE') 658 | }, 659 | }, 660 | 661 | AUTH_RESPONSE: { 662 | response(data) { 663 | this.conn.auth = JSON.parse(data) 664 | return this.goto(this.afterIdentify()) 665 | }, 666 | 667 | close() { 668 | return this.goto('CLOSED') 669 | }, 670 | }, 671 | 672 | SUBSCRIBE: { 673 | Enter() { 674 | this.conn.write(wire.subscribe(this.conn.topic, this.conn.channel)) 675 | return this.goto('SUBSCRIBE_RESPONSE') 676 | }, 677 | }, 678 | 679 | SUBSCRIBE_RESPONSE: { 680 | response(data) { 681 | if (data.toString() === 'OK') { 682 | this.goto('READY_RECV') 683 | // Notify listener that this nsqd connection has passed the subscribe 684 | // phase. Do this only once for a connection. 685 | return this.conn.emit(NSQDConnection.READY) 686 | } 687 | }, 688 | 689 | close() { 690 | return this.goto('CLOSED') 691 | }, 692 | }, 693 | 694 | READY_RECV: { 695 | consumeMessage(msg) { 696 | return this.conn.emit(NSQDConnection.MESSAGE, msg) 697 | }, 698 | 699 | response(data) { 700 | if (data.toString() === '_heartbeat_') { 701 | return this.conn.write(wire.nop()) 702 | } 703 | }, 704 | 705 | ready(rdyCount) { 706 | // RDY count for this nsqd cannot exceed the nsqd configured 707 | // max rdy count. 708 | if (rdyCount > this.conn.maxRdyCount) { 709 | rdyCount = this.conn.maxRdyCount 710 | } 711 | return this.conn.write(wire.ready(rdyCount)) 712 | }, 713 | 714 | close() { 715 | return this.goto('CLOSED') 716 | }, 717 | }, 718 | 719 | READY_SEND: { 720 | Enter() { 721 | // Notify listener that this nsqd connection is ready to send. 722 | return this.conn.emit(NSQDConnection.READY) 723 | }, 724 | 725 | produceMessages(data) { 726 | const [topic, msgs, timeMs, callback] = Array.from(data) 727 | this.conn.messageCallbacks.push(callback) 728 | 729 | if (!_.isArray(msgs)) { 730 | throw new Error('Expect an array of messages to produceMessages') 731 | } 732 | 733 | if (msgs.length === 1) { 734 | if (!timeMs) { 735 | return this.conn.write(wire.pub(topic, msgs[0])) 736 | } else { 737 | return this.conn.write(wire.dpub(topic, msgs[0], timeMs)) 738 | } 739 | } 740 | if (!timeMs) { 741 | return this.conn.write(wire.mpub(topic, msgs)) 742 | } else { 743 | throw new Error('DPUB can only defer one message at a time') 744 | } 745 | }, 746 | 747 | response(data) { 748 | let cb 749 | switch (data.toString()) { 750 | case 'OK': 751 | cb = this.conn.messageCallbacks.shift() 752 | return typeof cb === 'function' ? cb(null) : undefined 753 | case '_heartbeat_': 754 | return this.conn.write(wire.nop()) 755 | } 756 | }, 757 | 758 | close() { 759 | return this.goto('CLOSED') 760 | }, 761 | }, 762 | 763 | ERROR: { 764 | Enter(err) { 765 | // If there's a callback, pass it the error. 766 | const cb = this.conn.messageCallbacks.shift() 767 | if (typeof cb === 'function') { 768 | cb(err) 769 | } 770 | 771 | this.conn.emit(NSQDConnection.ERROR, err) 772 | 773 | // According to NSQ docs, the following errors are non-fatal and should 774 | // not close the connection. See here for more info: 775 | // http://nsq.io/clients/building_client_libraries.html 776 | if (!_.isString(err)) { 777 | err = err.toString() 778 | } 779 | const errorCode = err.split(/\s+/)[1] 780 | 781 | if ( 782 | ['E_REQ_FAILED', 'E_FIN_FAILED', 'E_TOUCH_FAILED'].includes(errorCode) 783 | ) { 784 | return this.goto('READY_RECV') 785 | } 786 | return this.goto('CLOSED') 787 | }, 788 | 789 | close() { 790 | return this.goto('CLOSED') 791 | }, 792 | }, 793 | 794 | CLOSED: { 795 | Enter() { 796 | if (!this.conn) { 797 | return 798 | } 799 | 800 | // If there are callbacks, then let them error on the closed connection. 801 | const err = new Error('nsqd connection closed') 802 | for (const cb of this.conn.messageCallbacks) { 803 | if (typeof cb === 'function') { 804 | cb(err) 805 | } 806 | } 807 | 808 | this.conn.messageCallbacks = [] 809 | this.disable() 810 | this.conn.destroy() 811 | this.conn.emit(NSQDConnection.CLOSED) 812 | return delete this.conn 813 | }, 814 | 815 | // No-op. Once closed, subsequent calls should do nothing. 816 | close() {}, 817 | }, 818 | } 819 | 820 | ConnectionState.prototype.transitions = { 821 | '*': { 822 | '*': function (data, callback) { 823 | this.log() 824 | return callback(data) 825 | }, 826 | 827 | CONNECTED(data, callback) { 828 | this.log() 829 | return callback(data) 830 | }, 831 | 832 | ERROR(err, callback) { 833 | this.log(`${err}`) 834 | return callback(err) 835 | }, 836 | }, 837 | } 838 | 839 | /** 840 | * WriterConnectionState 841 | * 842 | * Usage: 843 | * c = new NSQDConnectionWriter '127.0.0.1', 4150, 30 844 | * c.connect() 845 | * 846 | * c.on NSQDConnectionWriter.CLOSED, -> 847 | * console.log "Callback [closed]: Lost connection to nsqd" 848 | * 849 | * c.on NSQDConnectionWriter.ERROR, (err) -> 850 | * console.log "Callback [error]: #{err}" 851 | * 852 | * c.on NSQDConnectionWriter.READY, -> 853 | * c.produceMessages 'sample_topic', ['first message'] 854 | * c.produceMessages 'sample_topic', ['second message', 'third message'] 855 | * c.destroy() 856 | */ 857 | class WriterNSQDConnection extends NSQDConnection { 858 | /** 859 | * @constructor 860 | * @param {String} nsqdHost 861 | * @param {String|Number} nsqdPort 862 | * @param {Object} [options={}] 863 | */ 864 | constructor(nsqdHost, nsqdPort, options = {}) { 865 | super(nsqdHost, nsqdPort, null, null, options) 866 | this.debug = debug(`nsqjs:writer:conn:${nsqdHost}/${nsqdPort}`) 867 | } 868 | 869 | /** 870 | * Instantiates a new instance of WriterConnectionState or returns an 871 | * existing one. 872 | * 873 | * @return {WriterConnectionState} 874 | */ 875 | connectionState() { 876 | return this.statemachine || new WriterConnectionState(this) 877 | } 878 | 879 | /** 880 | * Emits a `produceMessages` event with the specified topic, msgs, timeMs and a 881 | * callback. 882 | * 883 | * @param {String} topic 884 | * @param {Array} msgs 885 | * @param {Number} timeMs 886 | * @param {Function} callback 887 | */ 888 | produceMessages(topic, msgs, timeMs, callback) { 889 | this.statemachine.raise('produceMessages', [topic, msgs, timeMs, callback]) 890 | } 891 | } 892 | 893 | /** 894 | * A statemachine modeling the various states a writer connection can be in. 895 | */ 896 | class WriterConnectionState extends ConnectionState { 897 | /** 898 | * Returned when the connection is ready to send messages. 899 | * 900 | * @return {String} 901 | */ 902 | afterIdentify() { 903 | return 'READY_SEND' 904 | } 905 | } 906 | 907 | module.exports = { 908 | NSQDConnection, 909 | ConnectionState, 910 | WriterNSQDConnection, 911 | WriterConnectionState, 912 | } 913 | --------------------------------------------------------------------------------