├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── README.md ├── bench.js ├── eslint.config.js ├── examples ├── consumer.js └── producer.js ├── mqemitter-mongodb.js ├── package.json └── test.js /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: npm 4 | directory: "/" 5 | schedule: 6 | interval: daily 7 | open-pull-requests-limit: 10 8 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: ci 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | runs-on: ubuntu-latest 12 | strategy: 13 | matrix: 14 | node-version: [20,22] 15 | 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Use Node.js 20 | uses: actions/setup-node@v4 21 | with: 22 | node-version: ${{ matrix.node-version }} 23 | 24 | - name: Install mongo 25 | run: | 26 | npm run mongodb 27 | - name: Install 28 | run: | 29 | npm install 30 | - name: Run tests 31 | run: | 32 | npm run test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # Compiled binary addons (http://nodejs.org/api/addons.html) 20 | build/Release 21 | 22 | # Dependency directory 23 | # Commenting this out is preferred by some people, see 24 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git- 25 | node_modules 26 | 27 | # Users Environment Variables 28 | .lock-wscript 29 | db 30 | 31 | package-lock.json 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014-2020 Matteo Collina 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | mqemitter-mongodb  ![ci](https://github.com/mcollina/mqemitter-mongodb/workflows/ci/badge.svg) 2 | ================= 3 | 4 | MongoDB powered [MQEmitter](http://github.com/mcollina/mqemitter). 5 | 6 | See [MQEmitter](http://github.com/mcollina/mqemitter) for the actual 7 | API. 8 | 9 | [![js-standard-style](https://raw.githubusercontent.com/feross/standard/master/badge.png)](https://github.com/feross/standard) 10 | 11 | Install 12 | ------- 13 | 14 | ```bash 15 | $ npm install mqemitter-mongodb 16 | ``` 17 | 18 | Example 19 | ------- 20 | 21 | ```js 22 | const mongodb = require('mqemitter-mongodb') 23 | const mq = mongodb({ 24 | url: 'mongodb://127.0.0.1/mqemitter?auto_reconnect' 25 | }) 26 | const msg = { 27 | topic: 'hello world', 28 | payload: 'or any other fields' 29 | } 30 | 31 | mq.on('hello world', function (message, cb) { 32 | // call callback when you are done 33 | // do not pass any errors, the emitter cannot handle it. 34 | cb() 35 | }) 36 | 37 | // topic is mandatory 38 | mq.emit(msg, function () { 39 | // emitter will never return an error 40 | }) 41 | ``` 42 | 43 | API 44 | ----- 45 | ### MQEmitterMongoDB([opts]) 46 | 47 | Create a new instance of mqemitter-mongodb. 48 | 49 | Options: 50 | 51 | * `url`: a mongodb endpoint url 52 | * `database`: a mongodb database name, by default it comes from the uri 53 | * `mongo`: options for mongodb client 54 | * `db`: a db instance of mongodb (instead of url) 55 | 56 | 57 | Acknowledgements 58 | ---------------- 59 | 60 | Code ported from [Ascoltatori](http://github.com/mcollina/ascoltatori). 61 | 62 | License 63 | ------- 64 | 65 | MIT 66 | -------------------------------------------------------------------------------- /bench.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2014-2015, Matteo Collina 3 | * 4 | * Permission to use, copy, modify, and/or distribute this software for any 5 | * purpose with or without fee is hereby granted, provided that the above 6 | * copyright notice and this permission notice appear in all copies. 7 | * 8 | * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 9 | * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 10 | * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 11 | * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 12 | * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 13 | * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR 14 | * IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 15 | */ 16 | 'use strict' 17 | 18 | const mqemitter = require('./') 19 | const emitter = mqemitter({ concurrency: 10 }) 20 | const total = 1000000 21 | let written = 0 22 | let received = 0 23 | const timerKey = 'time for sending ' + total + ' messages' 24 | 25 | function write () { 26 | if (written === total) { 27 | return 28 | } 29 | 30 | written++ 31 | 32 | emitter.emit({ topic: 'hello', payload: 'world' }, write) 33 | } 34 | 35 | emitter.on('hello', function (msg, cb) { 36 | received++ 37 | if (received === total) { 38 | console.timeEnd(timerKey) 39 | } 40 | setImmediate(cb) 41 | }) 42 | 43 | console.time(timerKey) 44 | write() 45 | write() 46 | write() 47 | write() 48 | write() 49 | write() 50 | write() 51 | write() 52 | write() 53 | write() 54 | write() 55 | write() 56 | -------------------------------------------------------------------------------- /eslint.config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('neostandard')({}) 4 | -------------------------------------------------------------------------------- /examples/consumer.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const mqemitter = require('../') 4 | const instance = mqemitter({ 5 | url: 'mongodb://localhost/aaa' 6 | }) 7 | 8 | instance.on('hello', function (data, cb) { 9 | console.log(data) 10 | cb() 11 | }) 12 | -------------------------------------------------------------------------------- /examples/producer.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const mqemitter = require('../') 4 | const instance = mqemitter({ 5 | url: 'mongodb://localhost/aaa' 6 | }) 7 | 8 | setInterval(function () { 9 | instance.emit({ topic: 'hello', payload: 'world' }) 10 | }, 1000) 11 | -------------------------------------------------------------------------------- /mqemitter-mongodb.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const urlModule = require('url') 4 | const mongodb = require('mongodb') 5 | const MongoClient = mongodb.MongoClient 6 | const ObjectId = mongodb.ObjectId 7 | const { inherits } = require('util') 8 | const MQEmitter = require('mqemitter') 9 | const { pipeline, Transform } = require('stream') 10 | const EE = require('events').EventEmitter 11 | 12 | function connectClient (url, opts, cb) { 13 | MongoClient.connect(url, opts) 14 | .then(client => { 15 | process.nextTick(cb, null, client) 16 | }) 17 | .catch(err => { 18 | process.nextTick(cb, err) 19 | }) 20 | } 21 | 22 | // check if the collection exists and is capped 23 | // if not create it 24 | async function checkCollection (ctx, next) { 25 | // ping to see if database is connected 26 | try { 27 | await ctx._db.command({ ping: 1 }) 28 | } catch (err) { 29 | ctx.status.emit('error', err) 30 | return 31 | } 32 | const collectionName = ctx._opts.collection 33 | const collections = await ctx._db.listCollections({ name: collectionName }).toArray() 34 | if (collections.length > 0) { 35 | ctx._collection = ctx._db.collection(collectionName) 36 | if (!await ctx._collection.isCapped()) { 37 | // the collection is not capped, make it so 38 | await ctx._db.command({ 39 | convertToCapped: collectionName, 40 | size: ctx._opts.size, 41 | max: ctx._opts.max 42 | }) 43 | } 44 | } else { 45 | // collection does not exist yet create it 46 | await ctx._db.createCollection(collectionName, { 47 | capped: true, 48 | size: ctx._opts.size, 49 | max: ctx._opts.max 50 | }) 51 | ctx._collection = ctx._db.collection(collectionName) 52 | } 53 | process.nextTick(next) 54 | } 55 | 56 | // Create a Transform stream to process each object 57 | function buildTransform (ctx, failures, oldEmit) { 58 | return new Transform({ 59 | objectMode: true, 60 | transform (obj, enc, cb) { 61 | if (ctx.closed) { 62 | return cb() // Stop processing if closed 63 | } 64 | 65 | // convert mongo binary to buffer 66 | if (obj.payload && obj.payload._bsontype) { 67 | obj.payload = obj.payload.read(0, obj.payload.length()) 68 | } 69 | 70 | ctx._started = true 71 | failures = 0 72 | ctx._lastObj = obj 73 | 74 | oldEmit.call(ctx, obj, cb) 75 | 76 | if (ctx._waiting.has(obj._stringId)) { 77 | process.nextTick(ctx._waiting.get(obj._stringId)) 78 | ctx._waiting.delete(obj._stringId) 79 | } 80 | } 81 | }) 82 | } 83 | 84 | function MQEmitterMongoDB (opts) { 85 | if (!(this instanceof MQEmitterMongoDB)) { 86 | return new MQEmitterMongoDB(opts) 87 | } 88 | 89 | opts = opts || {} 90 | opts.size = opts.size || 10 * 1024 * 1024 // 10 MB 91 | opts.max = opts.max || 10000 // documents 92 | opts.collection = opts.collection || 'pubsub' 93 | 94 | const url = opts.url || 'mongodb://127.0.0.1/mqemitter' 95 | this.status = new EE() 96 | this.status.setMaxListeners(0) 97 | 98 | this._opts = opts 99 | 100 | const that = this 101 | 102 | this._db = null 103 | 104 | if (opts.db) { 105 | that._db = opts.db 106 | setImmediate(waitStartup) 107 | } else { 108 | const defaultOpts = { } 109 | const mongoOpts = that._opts.mongo ? Object.assign(defaultOpts, that._opts.mongo) : defaultOpts 110 | connectClient(url, mongoOpts, function (err, client) { 111 | if (err) { 112 | return that.status.emit('error', err) 113 | } 114 | const urlParsed = new urlModule.URL(that._opts.url) 115 | let databaseName = that._opts.database || (urlParsed.pathname ? urlParsed.pathname.substr(1) : undefined) 116 | databaseName = databaseName.substr(databaseName.lastIndexOf('/') + 1) 117 | 118 | that._client = client 119 | that._db = client.db(databaseName) 120 | 121 | waitStartup() 122 | }) 123 | } 124 | 125 | this._hasStream = false 126 | this._started = false 127 | 128 | function waitStartup () { 129 | checkCollection(that, setLast) 130 | } 131 | 132 | const oldEmit = MQEmitter.prototype.emit 133 | 134 | this._waiting = new Map() 135 | this._queue = [] 136 | this._executingBulk = false 137 | let failures = 0 138 | 139 | async function setLast () { 140 | try { 141 | const results = await that._collection 142 | .find({}, { timeout: false }) 143 | .sort({ $natural: -1 }) 144 | .limit(1) 145 | .toArray() 146 | const doc = results[0] 147 | that._lastObj = doc || { _id: new ObjectId() } 148 | 149 | if (!that._lastObj._stringId) { 150 | that._lastObj._stringId = that._lastObj._id.toString() 151 | } 152 | 153 | await start() 154 | } catch (error) { 155 | that.status.emit('error', error) 156 | } 157 | } 158 | 159 | async function start () { 160 | if (that.closed) { return } 161 | 162 | try { 163 | const cursor = await that._collection.find({ _id: { $gt: that._lastObj._id } }, { 164 | tailable: true, 165 | timeout: false, 166 | awaitData: true 167 | }) 168 | that._stream = cursor.stream() 169 | 170 | const processStream = buildTransform(that, failures, oldEmit) 171 | that._stream = pipeline(that._stream, processStream, function () { 172 | if (that.closed) { 173 | return 174 | } 175 | 176 | if (that._started && ++failures === 10) { 177 | that.status.emit('error', new Error('Lost connection to MongoDB')) 178 | } 179 | setTimeout(start, 100) 180 | }) 181 | 182 | that._hasStream = true 183 | that.status.emit('stream') 184 | that._bulkInsert() 185 | } catch (error) { 186 | that._hasStream = false 187 | that.status.emit('error', error) 188 | } 189 | } 190 | 191 | MQEmitter.call(this, opts) 192 | } 193 | 194 | inherits(MQEmitterMongoDB, MQEmitter) 195 | 196 | MQEmitterMongoDB.prototype._bulkInsert = async function () { 197 | if (!this._executingBulk && this._queue.length > 0) { 198 | this._executingBulk = true 199 | const operations = [] 200 | 201 | while (this._queue.length) { 202 | const p = this._queue.shift() 203 | operations.push({ insertOne: p.obj }) 204 | } 205 | 206 | await this._collection.bulkWrite(operations) 207 | this._executingBulk = false 208 | this._bulkInsert() 209 | } 210 | } 211 | 212 | MQEmitterMongoDB.prototype._insertDoc = function (obj, cb) { 213 | if (cb) { 214 | this._waiting.set(obj._stringId, cb) 215 | } 216 | this._queue.push({ obj }) 217 | 218 | if (this._hasStream) { 219 | this._bulkInsert() 220 | } 221 | } 222 | 223 | MQEmitterMongoDB.prototype.emit = function (obj, cb) { 224 | if (!this.closed && !this._stream) { 225 | // actively poll if stream is available 226 | this.status.once('stream', this.emit.bind(this, obj, cb)) 227 | return this 228 | } else if (this.closed) { 229 | const err = new Error('MQEmitterMongoDB is closed') 230 | if (cb) { 231 | cb(err) 232 | } 233 | } else { 234 | this._insertDoc(obj, cb) 235 | } 236 | 237 | return this 238 | } 239 | 240 | MQEmitterMongoDB.prototype.close = function (cb) { 241 | cb = cb || noop 242 | 243 | if (this.closed) { 244 | return cb() 245 | } 246 | 247 | if (!this._stream) { 248 | this.status.once('stream', this.close.bind(this, cb)) 249 | return 250 | } 251 | 252 | this._stream.destroy() 253 | this._stream.on('error', function () { }) 254 | this._stream = null 255 | 256 | this.closed = true 257 | 258 | const that = this 259 | MQEmitter.prototype.close.call(this, async function () { 260 | if (that._opts.db) { 261 | cb() 262 | } else { 263 | that._client.close().then(() => { 264 | process.nextTick(cb) 265 | }).catch(err => { 266 | that.status.emit('error', err) 267 | process.nextTick(cb, err) 268 | }) 269 | } 270 | }) 271 | 272 | return this 273 | } 274 | 275 | function noop () { } 276 | 277 | module.exports = MQEmitterMongoDB 278 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mqemitter-mongodb", 3 | "version": "9.0.1", 4 | "description": "MongoDB based MQEmitter", 5 | "main": "mqemitter-mongodb.js", 6 | "scripts": { 7 | "unit": "node --test --test-timeout=180000 test.js", 8 | "lint": "eslint", 9 | "lint:fix": "eslint --fix", 10 | "test": "npm run lint && npm run unit", 11 | "mongodb": "docker run -d --rm --name mongodb -p 27017:27017 mongo:8" 12 | }, 13 | "pre-commit": "test", 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/mcollina/mqemitter-mongodb.git" 17 | }, 18 | "author": "Matteo Collina ", 19 | "license": "MIT", 20 | "engines": { 21 | "node": ">=20.0.0" 22 | }, 23 | "devDependencies": { 24 | "@fastify/pre-commit": "^2.2.0", 25 | "eslint": "^9.26.0", 26 | "neostandard": "^0.12.1" 27 | }, 28 | "dependencies": { 29 | "mongodb": "^6.16.0", 30 | "mqemitter": "^7.0.0" 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const mongodb = require('mongodb') 4 | const MongoClient = mongodb.MongoClient 5 | const ObjectId = mongodb.ObjectId 6 | 7 | const MongoEmitter = require('./') 8 | const { test } = require('node:test') 9 | const abstractTests = require('mqemitter/abstractTest.js') 10 | const dbname = 'mqemitter-test' 11 | const collectionName = 'pubsub' 12 | const url = 'mongodb://127.0.0.1/' + dbname 13 | 14 | async function clean (db, cb) { 15 | const collections = await db.listCollections({ name: collectionName }).toArray() 16 | if (collections.length > 0) { 17 | await db.collection(collectionName).drop() 18 | } 19 | if (cb) { 20 | process.nextTick(cb) 21 | } 22 | } 23 | 24 | function connectClient (url, opts, cb) { 25 | MongoClient.connect(url, opts) 26 | .then(client => { 27 | process.nextTick(cb, null, client) 28 | }) 29 | .catch(err => { 30 | process.nextTick(cb, err) 31 | }) 32 | } 33 | 34 | connectClient(url, { w: 1 }, function (err, client) { 35 | const db = client.db(dbname) 36 | if (err) { 37 | throw (err) 38 | } 39 | 40 | clean(db, function () { 41 | client.close() 42 | 43 | abstractTests({ 44 | builder: function (opts) { 45 | opts = opts || {} 46 | opts.url = url 47 | 48 | return MongoEmitter(opts) 49 | }, 50 | test 51 | }) 52 | 53 | test('with default database name', async function (t) { 54 | t.plan(2) 55 | 56 | await new Promise((resolve) => { 57 | const mqEmitterMongoDB = MongoEmitter({ 58 | url 59 | }) 60 | 61 | mqEmitterMongoDB.status.once('stream', async function () { 62 | t.assert.equal(mqEmitterMongoDB._db.databaseName, dbname) 63 | t.assert.ok(true, 'database name is default db name') 64 | mqEmitterMongoDB.close() 65 | resolve() 66 | }) 67 | }) 68 | }) 69 | 70 | test('should fetch last packet id', async function (t) { 71 | t.plan(3) 72 | 73 | let started = 0 74 | const lastId = new ObjectId() 75 | 76 | function startInstance (cb) { 77 | const mqEmitterMongoDB = MongoEmitter({ 78 | url 79 | }) 80 | 81 | mqEmitterMongoDB.status.once('stream', function () { 82 | t.assert.equal(mqEmitterMongoDB._db.databaseName, dbname, 'database name is default db name') 83 | if (++started === 1) { 84 | mqEmitterMongoDB.emit({ topic: 'last/packet', payload: 'I\'m the last' }, () => { 85 | mqEmitterMongoDB.close(cb) 86 | }) 87 | } else { 88 | t.assert.ok(mqEmitterMongoDB._lastObj._stringId > lastId.toString(), 'Should fetch last Id') 89 | mqEmitterMongoDB.close(cb) 90 | } 91 | }) 92 | } 93 | 94 | await new Promise((resolve) => { 95 | startInstance(function () { 96 | startInstance(resolve) 97 | }) 98 | }) 99 | }) 100 | 101 | test('with database option', async function (t) { 102 | t.plan(2) 103 | 104 | await new Promise((resolve) => { 105 | const mqEmitterMongoDB = MongoEmitter({ 106 | url, 107 | database: 'test-custom-db-name' 108 | }) 109 | 110 | mqEmitterMongoDB.status.once('stream', function () { 111 | t.assert.equal(mqEmitterMongoDB._db.databaseName, 'test-custom-db-name') 112 | t.assert.ok(true, 'database name is custom db name') 113 | mqEmitterMongoDB.close() 114 | resolve() 115 | }) 116 | }) 117 | }) 118 | 119 | test('with mongodb options', async function (t) { 120 | t.plan(2) 121 | 122 | await new Promise((resolve) => { 123 | const mqEmitterMongoDB = MongoEmitter({ 124 | url, 125 | mongo: { 126 | appName: 'mqemitter-mongodb-test' 127 | } 128 | }) 129 | 130 | mqEmitterMongoDB.status.once('stream', function () { 131 | t.assert.equal(mqEmitterMongoDB._db.client.options.appName, 'mqemitter-mongodb-test') 132 | t.assert.ok(true, 'database name is default db name') 133 | mqEmitterMongoDB.close() 134 | resolve() 135 | }) 136 | }) 137 | }) 138 | 139 | // keep this test as last 140 | test('doesn\'t throw db errors', async function (t) { 141 | t.plan(1) 142 | 143 | await new Promise((resolve) => { 144 | client.close(true) 145 | 146 | const mqEmitterMongoDB = MongoEmitter({ 147 | url, 148 | db 149 | }) 150 | 151 | mqEmitterMongoDB.status.on('error', function (err) { 152 | t.assert.ok(true, 'error event emitted') 153 | if (err.message !== 'Client must be connected before running operations') { 154 | t.assert.fail('throws error') 155 | } 156 | mqEmitterMongoDB.close() 157 | resolve() 158 | }) 159 | }) 160 | }) 161 | }) 162 | }) 163 | --------------------------------------------------------------------------------