├── .gitignore
├── CONTRIBUTORS.md
├── README.md
├── index.js
├── package.json
└── test
└── index_spec.js
/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules
--------------------------------------------------------------------------------
/CONTRIBUTORS.md:
--------------------------------------------------------------------------------
1 | [Guillaume Flandre](https://github.com/gflandre)
2 | - Author of the initial project
3 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Mongo Aggregation Debugger
2 |
3 | [](https://nodei.co/npm/mongo-aggregation-debugger/)
4 |
5 | Mongo Aggregation Debugger helps debug MongoDb aggregation queries
6 | by being able to visualize each stage of the pipeline
7 |
8 | ## Why use it
9 | It is pretty hard to understand why a specific aggregation query fails or doesn't output the
10 | right results since it can be pretty complex and go through a lot of stages before returning values.
11 |
12 | The Mongo Aggregation Debugger helps you understand what is going on by either:
13 | - outputting in the console the results of each stage of the aggregation pipeline
14 | - returning an array of results of each stage of teh aggregation pipeline for programmatic use
15 | - running the query in a temporary database and outputting the results,
16 | very useful for automated testing
17 |
18 | ## How it works
19 | You give the debugger access to your instance of mongodb, and it creates a temporary collection
20 | in which it will run each stage of the aggregation query in series.
21 | The temporary database is dropped after each debug.
22 |
23 | ## Install
24 | ```
25 | npm install mongo-aggregation-debugger
26 | ```
27 |
28 | ## Instantiation
29 | ```
30 | var mad = require('mongo-aggregation-debugger')();
31 | ```
32 |
33 | You can provide an optional object as an argument to specify the mongodb connection information:
34 |
35 | key | default value | description
36 | ------------ | ------------- | -------------
37 | host | `localhost` | mongodb host name
38 | port | `27017` | mongodb port number
39 | username | `null` | (optional) username of the mongodb instance
40 | password | `null` | (optional) password of the mongodb instance
41 | options | `{}` | (optional) additional mongodb [options](http://mongodb.github.io/node-mongodb-native/2.0/api/MongoClient.html)
42 |
43 | ## API
44 | ### `log`
45 | This method outputs in the console the result of each stage of the aggregation pipeline.
46 |
47 | #### Use
48 | `log(data, query[, options][, callback])`
49 |
50 | argument | type | values | description
51 | ------------ | ------------- | ------------- | -------------
52 | data | `array` | | The data to run the query against
53 | query | `array` | | The aggregation query
54 | options | `object` | `showQuery`: `boolean` | Whether to show the query of the stage being run or not
55 | callback | `function(err)` | | The callback returned when all stages were executed
56 |
57 | #### Example:
58 | ```javascript
59 | var mad = require('mongo-aggregation-debugger')();
60 |
61 | var data = [{
62 | foo: 'bar',
63 | test: true,
64 | array: [ 1, 2, 3 ]
65 | }, {
66 | foo: 'bar2',
67 | test: false,
68 | array: [ 10, 20 ]
69 | }];
70 |
71 | var query = [{
72 | '$match': {
73 | test: true
74 | }
75 | }, {
76 | '$project': {
77 | foo: 1,
78 | array: 1
79 | }
80 | }, {
81 | '$unwind': "$array"
82 | }, {
83 | '$group': {
84 | _id: "$foo",
85 | foo: { $first: "$foo" },
86 | sum: { $sum: "$array" }
87 | }
88 | }];
89 |
90 | mad.log(data, query, function (err) {
91 | if (err) {
92 | // do something
93 | }
94 |
95 | console.log('All done!');
96 | });
97 | ```
98 |
99 | Running the code above would output this in your console:
100 |
101 |
102 | Example with the `showQuery` option:
103 | ```
104 | mad.log(data, query, { showQuery: true }, function (err) {
105 | if (err) {
106 | // do something
107 | }
108 |
109 | console.log('All done!');
110 | });
111 | ```
112 |
113 |
114 | ### `stages`
115 | This method returns the result of each stage of the aggregation pipeline for programmatic use.
116 |
117 | #### Use
118 | `stages(data, query[, callback])`
119 |
120 | argument | type | description
121 | ------------ | ------------- | ------------- | -------------
122 | data | `array` | The data to run the query against
123 | query | `array` | The aggregation query
124 | callback | `function(err, results)` | `results` is an array composed of as many objects as there are stages in the aggregation pipeline. Each object has a `query` attribute which is the query of the stage and a `result` attribute with the results of that query
125 |
126 | #### Example:
127 | ```javascript
128 | var util = require('util');
129 | var mad = require('mongo-aggregation-debugger')();
130 |
131 | var data = [{
132 | foo: 'bar',
133 | test: true,
134 | array: [ 1, 2, 3 ]
135 | }, {
136 | foo: 'bar2',
137 | test: false,
138 | array: [ 10, 20 ]
139 | }];
140 |
141 | var query = [{
142 | '$match': {
143 | test: true
144 | }
145 | }, {
146 | '$project': {
147 | foo: 1,
148 | array: 1
149 | }
150 | }, {
151 | '$unwind': "$array"
152 | }, {
153 | '$group': {
154 | _id: "$foo",
155 | foo: { $first: "$foo" },
156 | sum: { $sum: "$array" }
157 | }
158 | }];
159 |
160 | mad.stages(data, query, function (err, results) {
161 | if (err) {
162 | // do something
163 | }
164 |
165 | console.log(util.inspect(results, { depth: null }));
166 | });
167 | ```
168 |
169 | The output is:
170 | ```javascript
171 | [ { query: [ { '$match': { test: true } } ],
172 | results:
173 | [ { _id: 5599279a731b5aba47df6d97,
174 | foo: 'bar',
175 | test: true,
176 | array: [ 1, 2, 3 ] } ] },
177 | { query:
178 | [ { '$match': { test: true } },
179 | { '$project': { foo: 1, array: 1 } } ],
180 | results: [ { _id: 5599279a731b5aba47df6d97, foo: 'bar', array: [ 1, 2, 3 ] } ] },
181 | { query:
182 | [ { '$match': { test: true } },
183 | { '$project': { foo: 1, array: 1 } },
184 | { '$unwind': '$array' } ],
185 | results:
186 | [ { _id: 5599279a731b5aba47df6d97, foo: 'bar', array: 1 },
187 | { _id: 5599279a731b5aba47df6d97, foo: 'bar', array: 2 },
188 | { _id: 5599279a731b5aba47df6d97, foo: 'bar', array: 3 } ] },
189 | { query:
190 | [ { '$match': { test: true } },
191 | { '$project': { foo: 1, array: 1 } },
192 | { '$unwind': '$array' },
193 | { '$group':
194 | { _id: '$foo',
195 | foo: { '$first': '$foo' },
196 | sum: { '$sum': '$array' } } } ],
197 | results: [ { _id: 'bar', foo: 'bar', sum: 6 } ] } ]
198 | ```
199 |
200 | ### `exec`
201 | This method only runs the entire query passed, not all the stages seperately. It is useful for automated tests since it creates and drops a temporary database.
202 |
203 | #### Use
204 | `exec(data, query[, callback])`
205 |
206 | argument | type | description
207 | ------------ | ------------- | ------------- | -------------
208 | data | `array` | The data to run the query against
209 | query | `array` | The aggregation query
210 | callback | `function(err, results)` | `results` is the results of the query being run
211 |
212 | #### Example:
213 | ```javascript
214 | var util = require('util');
215 | var mad = require('mongo-aggregation-debugger')();
216 |
217 | var data = [{
218 | foo: 'bar',
219 | test: true,
220 | array: [ 1, 2, 3 ]
221 | }, {
222 | foo: 'bar2',
223 | test: false,
224 | array: [ 10, 20 ]
225 | }];
226 |
227 | var query = [{
228 | '$match': {
229 | test: true
230 | }
231 | }, {
232 | '$project': {
233 | foo: 1,
234 | array: 1
235 | }
236 | }, {
237 | '$unwind': "$array"
238 | }, {
239 | '$group': {
240 | _id: "$foo",
241 | foo: { $first: "$foo" },
242 | sum: { $sum: "$array" }
243 | }
244 | }];
245 |
246 | mad.exec(data, query, function (err, results) {
247 | if (err) {
248 | // do something
249 | }
250 |
251 | console.log(util.inspect(results, { depth: null }));
252 | });
253 | ```
254 |
255 | The output is:
256 | ```javascript
257 | [ { _id: 'bar', foo: 'bar', sum: 6 } ]
258 | ```
259 |
260 | ## Unit tests
261 | In order to test this lib you'll need to install mocha: `npm install -g mocha`.
262 | Then just run the `mocha` command at the root of the project.
263 |
264 | ## More info
265 | - [MongoDb](https://www.mongodb.org/)
266 | - [MongoDb Aggregation Framework](http://docs.mongodb.org/manual/core/aggregation-introduction/)
267 | - [MongoDb Native NodeJS Driver](https://github.com/mongodb/node-mongodb-native)
268 |
269 | ## Contribute
270 | If you think it would make sense to add some features/methods don't hesitate to fork and
271 | make pull requests.
272 |
273 | You can contact the main contributor on [Twitter](http://twitter.com/gflandre)
274 |
275 | ## Licence
276 | Distributed under the MIT License.
277 |
--------------------------------------------------------------------------------
/index.js:
--------------------------------------------------------------------------------
1 | /**
2 | * index.js
3 | */
4 | var util = require('util');
5 | var _ = require('underscore');
6 | var async = require('async');
7 | var clor = require("clor");
8 | var MongoClient = require('mongodb').MongoClient;
9 |
10 | /**
11 | * Mongo Aggregation Debugger module
12 | *
13 | * Provides ways to debug mongo's aggregation framework
14 | *
15 | * Initialize by providing MongoDb connection credentials
16 | * - host {string} (default: 'localhost') the host info
17 | * - port {number} (default: 27017) the port info
18 | * - username {string} (optional) the mongodb username
19 | * - password {string} (optional) the mongodb password
20 | * - options {object} (opional) standard mongodb options
21 | */
22 | var mongoAggregationDebugger = function (mongoParams) {
23 | my = {};
24 |
25 | my.defaultMongoParams = {
26 | host: 'localhost',
27 | port: 27017,
28 | username: null,
29 | password: null,
30 | options: {}
31 | };
32 |
33 | my.mongoParams = _.extend(my.defaultMongoParams, mongoParams || {});
34 |
35 | my.collectionName = 'documents';
36 |
37 | /**
38 | * Private
39 | */
40 | var generateDatabaseName;
41 | var debug;
42 | var buildConnectionUrl;
43 | var getMongoConnection;
44 |
45 | /**
46 | * Public
47 | */
48 | var log;
49 | var stages;
50 | var exec;
51 |
52 | /**
53 | * that
54 | */
55 | var that = {};
56 |
57 | /*************************************************************************************************
58 | * PRIVATE METHODS
59 | ************************************************************************************************/
60 | /**
61 | * Generates a rather unique database name
62 | * @return {string} the database name
63 | */
64 | generateDatabaseName = function () {
65 | return 'mongo_aggregation_debugger_' + process.pid + '_' + Date.now();
66 | };
67 |
68 | /**
69 | * Builds the mongodb connection url from the params passed
70 | * @return {string} the mongodb connection url
71 | */
72 | buildConnectionUrl = function () {
73 | if (!my.mongoParams.host) {
74 | my.mongoParams.host = my.defaultMongoParams.host;
75 | }
76 |
77 | var url = 'mongodb://';
78 |
79 | if (my.mongoParams.username && my.mongoParams.password) {
80 | url += my.mongoParams.username + ':' + my.mongoParams.password + '@';
81 | }
82 |
83 | url += my.mongoParams.host;
84 |
85 | if (my.mongoParams.port) {
86 | url += ':' + my.mongoParams.port;
87 | }
88 |
89 | my.debugDatabaseName = my.debugDatabaseName || generateDatabaseName();
90 |
91 | url += '/' + my.debugDatabaseName;
92 |
93 | return url;
94 | };
95 |
96 | /**
97 | * Get the mongodb's Db instance
98 | * @param {function} cb(err, db)
99 | */
100 | getMongoConnection = function (cb) {
101 | MongoClient.connect(buildConnectionUrl(), my.mongoParams.options, cb);
102 | };
103 |
104 | /**
105 | * Partition an aggregation query into several subsets
106 | * @param {array} query The aggregation query
107 | * @param {boolean} skipStages Whether to directly run the full query or not
108 | * @return {array} Query parts
109 | */
110 | partitionQuery = function (query, skipStages) {
111 | var queryParts = [];
112 |
113 | if (skipStages) {
114 | queryParts.push(query);
115 | } else {
116 | query.forEach(function (queryPart, index) {
117 | if (index === 0) {
118 | queryParts.push([ queryPart ]);
119 | } else {
120 | var part = _.clone(queryParts[index - 1]);
121 | part.push(queryPart);
122 | queryParts.push(part);
123 | }
124 | });
125 | }
126 |
127 | return queryParts;
128 | };
129 |
130 | /**
131 | * Actually runs the debugging part
132 | * @param {mixed} data An array of objects that will be the data
133 | * the aggregation will be performed on.
134 | * Also accepts a single object.
135 | * @param {array} query The aggregation query
136 | * @param {function} beforeEach Callback called before each stage of the aggregation.
137 | * Arguments passed:
138 | * - queryPart {array} The query run for this stage
139 | * of the aggregation
140 | * - index {number} The current aggregation stage number
141 | * @param {function} afterEach Callback called after each stage of the aggregation.
142 | * Arguments passed:
143 | * - results {array} The results of that aggregation stage
144 | * - index {number} The current aggregation stage number
145 | * @param {boolean} skipStages (optional, default: false) Whether to directly run the
146 | * full query or not
147 | * @param {function} cb(err)
148 | */
149 | debug = function (data, query, beforeEach, afterEach, skipStages, cb) {
150 | var doSkipStegaes = false;
151 |
152 | if (typeof skipStages === 'function' && typeof cb === 'undefined') {
153 | cb = skipStages;
154 | doSkipStages = false;
155 | } else if (typeof skipStages === 'boolean') {
156 | doSkipStages = skipStages;
157 | } else {
158 | return cb(new Error('Invalid `skipStages` value'));
159 | }
160 |
161 | if (typeof cb !== 'function') {
162 | throw new Error('Invalid callback');
163 | }
164 |
165 | if (!Array.isArray(data)) {
166 | if (data && typeof data === 'object') {
167 | data = [ data ];
168 | } else {
169 | return cb(new Error('Invalid `data`'));
170 | }
171 | }
172 |
173 | if (!Array.isArray(query)) {
174 | return cb(new Error('Invalid `query`'));
175 | }
176 |
177 | if (typeof beforeEach !== 'function') {
178 | beforeEach = function () {};
179 | }
180 |
181 | if (typeof afterEach !== 'function') {
182 | afterEach = function () {};
183 | }
184 |
185 | getMongoConnection(function (err, db) {
186 | if (err) {
187 | return cb(err);
188 | }
189 |
190 | var collection = db.collection(my.collectionName);
191 | var queryParts = partitionQuery(query, doSkipStages);
192 |
193 | var series = [
194 | function insert (cb) {
195 | collection.insert(data, cb);
196 | }
197 | ];
198 |
199 | queryParts.forEach(function (queryPart, index) {
200 | series.push(
201 | function (cb) {
202 | beforeEach(queryPart, index);
203 |
204 | (function (index) {
205 | collection.aggregate(queryPart, function (err, results) {
206 | if (err) {
207 | return cb(err);
208 | }
209 |
210 | afterEach(results, index);
211 | return cb();
212 | });
213 | })(index);
214 | }
215 | );
216 | });
217 |
218 | series.push(
219 | function dropDatabase (cb) {
220 | db.dropDatabase(cb);
221 | }
222 | );
223 |
224 | async.series(series, function (err) {
225 | if (err) {
226 | return db.dropDatabase(function (anotherErr) {
227 | db.close();
228 | cb(anotherErr || err);
229 | });
230 | }
231 |
232 | db.close();
233 | return cb();
234 | });
235 | });
236 | };
237 |
238 | /*************************************************************************************************
239 | * PUBLIC METHODS
240 | ************************************************************************************************/
241 | /**
242 | * Logging mode, displays results in console
243 | * @param {mixed} data An array of objects that will be the data
244 | * the aggregation will be performed on.
245 | * Also accepts a single object.
246 | * @param {array} query The aggregation query
247 | * @param {object} options (optional) Display options:
248 | * - showQuery {boolean} (default: false) Outputs each stage's query
249 | * @param {function} cb(err)
250 | */
251 | log = function (data, query, options, cb) {
252 | if (typeof options === 'function' && typeof cb === 'undefined') {
253 | cb = options;
254 | }
255 |
256 | if (typeof cb !== 'function') {
257 | cb = function () {};
258 | }
259 |
260 | options = options || {};
261 |
262 | util.debug(clor.cyan('Mongo aggregation debugger [Start]\n'));
263 |
264 | var beforeEach = function (queryPart, index) {
265 | var operationType = _.keys(queryPart[queryPart.length - 1])[0];
266 | console.log(clor.bgWhite.black(' Stage ' + (index + 1) + ' ') + ' ' +
267 | clor.bgBlue(' ' + operationType + ' '));
268 | if (options.showQuery) {
269 | console.log(util.inspect(queryPart, {
270 | depth: null,
271 | colors: true
272 | }));
273 | console.log('\n' + clor.bgGreen.black(' Results '));
274 | }
275 | };
276 |
277 | var afterEach = function (results) {
278 | console.log(util.inspect(results, {
279 | depth: null,
280 | colors: true
281 | }));
282 | console.log('\n');
283 | };
284 |
285 | debug(data, query, beforeEach, afterEach, function (err) {
286 | if (err) {
287 | return cb(err);
288 | } else {
289 | util.debug(clor.cyan('Mongo aggregation debugger [End]'));
290 | return cb();
291 | }
292 | });
293 | };
294 |
295 | /**
296 | * Programmatic mode, returns an array of each aggregation stage's data
297 | * @param {mixed} data An array of objects that will be the data
298 | * the aggregation will be performed on.
299 | * Also accepts a single object.
300 | * @param {array} query The aggregation query
301 | * @param {function} cb(err, output)
302 | */
303 | stages = function (data, query, cb) {
304 | if (typeof cb !== 'function') {
305 | cb = function () {};
306 | }
307 |
308 | var output = [];
309 |
310 | var beforeEach = function (queryPart, index) {
311 | output.push({
312 | query: queryPart
313 | });
314 | };
315 |
316 | var afterEach = function (results, index) {
317 | output[index] = output[index] || {};
318 | output[index].results = results;
319 | };
320 |
321 | debug(data, query, beforeEach, afterEach, function (err) {
322 | if (err) {
323 | return cb(err);
324 | } else {
325 | return cb(null, output);
326 | }
327 | });
328 | };
329 |
330 | /**
331 | * Exec mode, returns only the last result, without running intermediate aggregation stages
332 | * @param {mixed} data An array of objects that will be the data
333 | * the aggregation will be performed on.
334 | * Also accepts a single object.
335 | * @param {array} query The aggregation query
336 | * @param {function} cb(err, output)
337 | */
338 | exec = function (data, query, cb) {
339 | if (typeof cb !== 'function') {
340 | cb = function () {};
341 | }
342 |
343 | var output;
344 |
345 | var afterEach = function (results) {
346 | output = results;
347 | };
348 |
349 | debug(data, query, null, afterEach, true, function (err) {
350 | if (err) {
351 | return cb(err);
352 | } else {
353 | return cb(null, output);
354 | }
355 | });
356 | };
357 |
358 | that.log = log;
359 | that.stages = stages;
360 | that.exec = exec;
361 |
362 | return that;
363 | };
364 |
365 | module.exports = mongoAggregationDebugger;
366 |
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "mongo-aggregation-debugger",
3 | "version": "1.0.4",
4 | "description": "Node.js MongoDB aggregation framework debugger methods",
5 | "keywords": ["nodejs", "mongo", "mongodb", "aggregation", "aggregate", "debug", "debugger", "unit tests", "tests", "unit"],
6 | "homepage": "https://github.com/gflandre/mongo-aggregation-debugger",
7 | "author": { "name": "Guillaume Flandre",
8 | "email": "guiome.flandre@gmail.com",
9 | "url": "http://twitter.com/gflandre" },
10 | "repository" : { "type" : "git",
11 | "url" : "https://github.com/gflandre/mongo-aggregation-debugger.git" },
12 | "dependencies": {
13 | "async": "1.3.x",
14 | "clor": "0.2.x",
15 | "mocha": "2.2.x",
16 | "mongodb": "2.0.x",
17 | "proxyquire": "1.6.x",
18 | "sinon": "1.15.x",
19 | "underscore": "1.8.x"
20 | },
21 | "main" : "./index.js",
22 | "engines" : {
23 | "node" : ">=v0.10.0"
24 | },
25 | "scripts": {
26 | "test": "./node_modules/mocha/bin/mocha"
27 | }
28 | }
--------------------------------------------------------------------------------
/test/index_spec.js:
--------------------------------------------------------------------------------
1 | /**
2 | * @file index_spec.js
3 | */
4 | var assert = require('assert');
5 | var sinon = require('sinon');
6 | var proxyquire = require('proxyquire');
7 |
8 | describe('mongo-aggregation-debugger', function () {
9 | var data = {};
10 | var query = [];
11 | var mad;
12 |
13 | before(function () {
14 | data = [{
15 | foo: 'bar',
16 | test: true,
17 | array: [ 1, 2, 3 ]
18 | }, {
19 | foo: 'bar2',
20 | test: false,
21 | array: [ 10, 20 ]
22 | }];
23 |
24 | query = [{
25 | '$match': {
26 | test: true
27 | }
28 | }, {
29 | '$project': {
30 | foo: 1,
31 | array: 1
32 | }
33 | }, {
34 | '$unwind': "$array"
35 | }, {
36 | '$group': {
37 | _id: "$foo",
38 | foo: { $first: "$foo" },
39 | sum: { $sum: "$array" }
40 | }
41 | }];
42 |
43 | sinon.spy(console, 'log');
44 |
45 | mad = require(__dirname + '/../index')();
46 | });
47 |
48 | describe('#log()', function () {
49 | beforeEach(function () {
50 | console.log.reset();
51 | });
52 |
53 | it('should output results if no query is shown', function (done) {
54 | mad.log(data, query, function (err) {
55 | assert.equal(console.log.callCount, 12);
56 | done();
57 | });
58 | });
59 |
60 | it('should output more results if query is shown', function (done) {
61 | mad.log(data, query, { showQuery: true }, function (err) {
62 | assert.equal(console.log.callCount, 20);
63 | done();
64 | });
65 | });
66 | });
67 |
68 | describe('#stages()', function () {
69 | it('should return an error if data is invalid', function (done) {
70 | mad.stages(null, query, function (err) {
71 | assert.notEqual(typeof err, undefined);
72 | done();
73 | });
74 | });
75 |
76 | it('should return an error if query is invalid', function (done) {
77 | mad.stages(data, 'test', function (err) {
78 | assert.notEqual(typeof err, undefined);
79 | done();
80 | });
81 | });
82 |
83 | it('should return valid data stages', function (done) {
84 | mad.stages(data, query, function (err, results) {
85 | assert.equal(!!err, false);
86 | assert.equal(results.length, 4);
87 |
88 | assert.equal(results[0].query.length, 1);
89 | assert.equal(results[0].results.length, 1);
90 | assert.equal(results[0].results[0].test, true);
91 |
92 | assert.equal(results[1].query.length, 2);
93 | assert.equal(results[1].results.length, 1);
94 | assert.equal(typeof results[1].results[0].test, 'undefined');
95 |
96 | assert.equal(results[2].query.length, 3);
97 | assert.equal(results[2].results.length, 3);
98 |
99 | assert.equal(results[3].query.length, 4);
100 | assert.equal(results[3].results.length, 1);
101 | assert.equal(results[3].results[0]._id, 'bar');
102 | assert.equal(results[3].results[0].sum, 6);
103 |
104 | done();
105 | });
106 | });
107 | });
108 |
109 | describe('#exec()', function () {
110 | it('should return valid data', function (done) {
111 | mad.exec(data, query, function (err, results) {
112 | assert.equal(!!err, false);
113 | assert.equal(results.length, 1);
114 |
115 | assert.equal(results[0]._id, 'bar');
116 | assert.equal(results[0].sum, 6);
117 |
118 | done();
119 | });
120 | });
121 | });
122 | });
123 |
--------------------------------------------------------------------------------