├── test
    ├── TestJoinFunction.js
    └── TestJoinUtilityFunctions.js
├── package.json
├── LICENSE.TXT
├── README.md
└── Join.js


/test/TestJoinFunction.js:
--------------------------------------------------------------------------------
 1 | var assert = require("assert"),
 2 |     Join = require("../Join");
 3 | 
 4 | describe("Join", function () {
 5 |     var join = new Join();
 6 |     //going to need the keyhashbin (ffffffffff!)
 7 | 
 8 |     describe("#_performJoining()", function () {
 9 |         it("should add a new subdocument with the given key to the correct item in the", function () {
10 | 
11 |         });
12 |     });
13 | });
14 | 


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "mongo-fast-join",
 3 |     "version": "0.0.11",
 4 |     "description": "Join subdocuments into other mongo documents from other collections",
 5 |     "main": "Join.js",
 6 |     "scripts": {
 7 |         "test": "echo \"Error: no test specified\" && exit 1"
 8 |     },
 9 |     "repository": {
10 |         "type": "git",
11 |         "url": "https://github.com/Bill4Time/mongo-fast-join.git"
12 |     },
13 |     "keywords": [
14 |         "mongodb",
15 |         "mongo",
16 |         "nodejs",
17 |         "node",
18 |         "javascript",
19 |         "join",
20 |         "mongo-join"
21 |     ],
22 |     "author": "Bill4Time",
23 |     "contributors": [{
24 |         "name": "Kyle Housley",
25 |         "email": "khousley@bill4time.com"
26 |     }, {
27 |         "name": "Jeremy Diviney",
28 |         "email": "jeremy@bill4time.com"
29 |     }],
30 |     "license": "The MIT License (MIT)",
31 |     "bugs": {
32 |         "url": "https://github.com/Bill4Time/mongo-fast-join/issues"
33 |     }
34 | }
35 | 


--------------------------------------------------------------------------------
/LICENSE.TXT:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | Copyright (c) <year> <copyright holders>
 3 | 
 4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 5 | documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
 6 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
 7 | persons to whom the Software is furnished to do so, subject to the following conditions:
 8 | 
 9 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
10 | Software.
11 | 
12 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
13 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
15 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ##mongo-fast-join
 2 | 
 3 | 
 4 | Join sub documents from other collections into the original query documents and do it as fast as mongo can.
 5 | 
 6 | ###Intro
 7 | 
 8 | This is our stab at the mongo join problem. As you know, joins are not supported by MongoDB natively, and this can be a pain.
 9 | We sought to create a project which performs a join query on mongodb and does it quickly. After a few attempts with less 
10 | than stellar results, we arrived at the current implementation.
11 | 
12 | ####How did we do it?
13 | 
14 | mongo-fast-join is fast because we paginate our join queries. Each document that is going to be joined to another document 
15 | represents a unique query. This is accomplished with an $or clause. When dealing with only 10,000 original documents, this 
16 | was miserably slow, taking up to a minute to return results on the local network. What a useless tool this would be if it 
17 | took that long to join a measly 10,000 records. Turns out that splitting query into small queries with only 5 conditions 
18 | in each $or clause sped the performance up by many orders of magnitude, joining 10,000 documents in less than 200ms.
19 | 
20 | We think that the reason the single query performed so poorly is because the $or clause was not intended to handle 10,000
21 | conditions. It also seems that the query is executed in a single thread (just a guess). 
22 | 
23 | 
24 | ####Shut up and take my query!
25 | 
26 | This is the syntax we arrived at:
27 | 
28 | ```
29 | 
30 | var MJ = require("mongo-fast-join"),
31 |     mongoJoin = new MJ();
32 | 
33 | /*
34 |     Say we have a collection of sales where each document holds a manual reference to the product sold. We can join the
35 |     full product document into each sale document. Lets also assume that each product has a reference to some
36 |     manufacturer info.
37 | */
38 |         
39 | mongoJoin
40 |     .query(
41 |       //say we have sales records and we store all the products for sale in a different collection
42 |       db.collection("sales"),
43 |         {}, //query statement
44 |         {}, //fields
45 |         {
46 |             limit: 10000//options
47 |         }
48 |     )
49 |     .join({
50 |         joinCollection: db.collection("products"),
51 |         //respects the dot notation, multiple keys can be specified in this array
52 |         leftKeys: ["product_id"],
53 |         //This is the key of the document in the right hand document
54 |         rightKeys: ["_id"],
55 |         //This is the new subdocument that will be added to the result document
56 |         newKey: "product"
57 |     })
58 |     .join({
59 |         //say that we want to get the users that commented too
60 |         joinCollection: db.collection("manufacturers"),
61 |         //This is cool, you can join on the new documents you add to the source document
62 |         leftKeys: ["product.manufacturer_id"],//This field accepts many keys, which amounts to a composite key
63 |         rightKeys: ["_id"],
64 |         //unfortunately, as of now, you can only add subdocuments at the root level, not to arrays of subdocuments
65 |         newKey: "manufacturer"//The upside is that this serve the majority of cases
66 |     })
67 |     //Call exec to run the compiled query and catch any errors and results, in the callback
68 |     .exec(function (err, items) {
69 |         console.log(items);
70 |     });
71 | 
72 | ```
73 | 
74 | The resulting document should have 10000 sales records with the product sold and manufaturer info attached to each sale
75 | in a structure like this:
76 | 
77 | ```
78 | 
79 | {
80 |     _id: 1,
81 |     product_id: 2,
82 |     product: {
83 |         _id: 2,
84 |         manufacturer_id: 3
85 |         name: "Wooden Spoon"
86 |     },
87 |     manufacturer: {
88 |         _id: 3,
89 |         name: "Betty Crocker"
90 |     }
91 | }
92 | 
93 | ```
94 | 
95 | 
96 | 


--------------------------------------------------------------------------------
/test/TestJoinUtilityFunctions.js:
--------------------------------------------------------------------------------
  1 | var assert = require("assert"),
  2 |     Join = require("../Join");
  3 | 
  4 | describe("Join", function () {
  5 |     var join = (new Join());
  6 |     describe("#callIfFunction()", function () {
  7 |         it("should not throw an exception when a non function is passed as a argument", function () {
  8 |             assert.doesNotThrow(function () {
  9 |                 join._callIfFunction("not a function");
 10 |                 join._callIfFunction(false);
 11 |                 join._callIfFunction([]);
 12 |             });
 13 |         });
 14 |         it("should call the function passed in as an argument", function () {
 15 |             var itRan = false;
 16 |             join._callIfFunction(function () {
 17 |                 itRan = true;
 18 |             });
 19 |             assert(itRan);
 20 |         });
 21 |     });
 22 | 
 23 |     describe("#_removeNonMatchesLeft()", function () {
 24 |         var testArrayWithOneElement = function () { return [{
 25 |                 k1: {},
 26 |                 k2: {}
 27 |             }];
 28 |         };
 29 | 
 30 |         it("should remove items where there is no matching key in the object", function () {
 31 |             var testAr = testArrayWithOneElement();
 32 |             join._removeNonMatchesLeft(testAr, "non existent key");
 33 |             assert(testAr.length === 0);
 34 |         });
 35 | 
 36 |         it("should not remove elements when there is a matching key", function () {
 37 |             var testAr = testArrayWithOneElement();
 38 | 
 39 |             join._removeNonMatchesLeft(testAr, "k1");
 40 |             assert(testAr.length === 1);
 41 | 
 42 |             join._removeNonMatchesLeft(testAr, "k2");
 43 |             assert(testAr.length === 1);
 44 |         });
 45 |     });
 46 | 
 47 |     describe("#_safeObjectAccess()", function () {
 48 |         it("should return a value when given a correct path to a property in a given object", function () {
 49 |             var testObj = { here: { is: { the: { path: { to: { test: true } } } } } };
 50 | 
 51 |             assert(join._safeObjectAccess(testObj, "here", "is", "the", "path", "to", "test") === true);
 52 |         });
 53 | 
 54 |         it("should return undefined when given and incorrect path to a property in a given object", function () {
 55 |             var testObj = {
 56 |                 path: {}
 57 |             };
 58 | 
 59 |             assert(typeof join._safeObjectAccess(testObj, "path", "is", "not", "good") === "undefined");
 60 |             assert(typeof join._safeObjectAccess(undefined, "bogus", "path") === "undefined");
 61 |         });
 62 | 
 63 |         it("should return undefined when given an incorrect path in an undefined value", function () {
 64 |             assert(typeof join._safeObjectAccess(undefined, ""));
 65 |         });
 66 | 
 67 |         it("should return a list of value when an array is encountered in the lookup path", function () {
 68 |             var testObj = {
 69 |                     k1: [{k2: true}, {k2: true}, {k2: true}]
 70 |                 },
 71 |                 result = join._safeObjectAccess(testObj, "k1", "k2"),
 72 |                 testObjWithArrayOfSubObjects = {
 73 |                     k1: [{
 74 |                         k2: {
 75 |                             val: true
 76 |                         }
 77 |                     }, {
 78 |                         k2: {
 79 |                             val: true
 80 |                         }
 81 |                     }, {
 82 |                         k2: {
 83 |                             val: true
 84 |                         }
 85 |                     }]
 86 |                 },
 87 |                 result2 = join._safeObjectAccess(testObjWithArrayOfSubObjects, "k1", "k2");
 88 | 
 89 |             assert(Array.isArray(result));
 90 |             assert(result.length === 3);
 91 |             result.forEach(function (item) {
 92 |                 assert(item === true);//Each item returned should be the true boolean
 93 |             });
 94 | 
 95 |             assert(Array.isArray(result2));
 96 |             assert(result2.length === 3);
 97 |             result2.forEach(function (item) {
 98 |                 assert(item.val === true);//Each item returned should be the true boolean
 99 |             });
100 |         });
101 |     });
102 | 
103 |     describe("#_isNullOrUndefined()", function () {
104 |         var isNullOrUndefined = join._isNullOrUndefined;
105 | 
106 |         it("should return true when a null value is passed as the argument", function () {
107 |             assert(isNullOrUndefined(null));
108 |         });
109 | 
110 |         it("should return true when undefined is passed as the argument", function () {
111 |             var un,
112 |                 obj = {};
113 |             assert(isNullOrUndefined(undefined));
114 |             assert(isNullOrUndefined(un));
115 |             assert(isNullOrUndefined(obj.un));
116 |         });
117 | 
118 |         it("should return false when a non-null, defined value is passed as an argument", function () {
119 |             assert(!isNullOrUndefined({}));
120 |             assert(!isNullOrUndefined([]));
121 |             assert(!isNullOrUndefined(""));
122 |             assert(!isNullOrUndefined(1));
123 |             assert(!isNullOrUndefined(0));
124 |             assert(!isNullOrUndefined(/1/));
125 |             assert(!isNullOrUndefined(true));
126 |             assert(!isNullOrUndefined(false));
127 |             assert(!isNullOrUndefined(function () {}));
128 |         });
129 |     });
130 | });


--------------------------------------------------------------------------------
/Join.js:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * This module will perform an inner join on mongo native collections and any other collection which supports the
  3 |  * mongo-native collection API. We realize that mongodb is not meant to be used this way, but there are certainly instances
  4 |  * where a fk relationship is an appropriate way to NRDB and a necessary way to retrieve data. Honed in the fires of
  5 |  * artificial perf testing, this is mongo-fast-join.
  6 |  *
  7 |  * The novel aspect of this project is that we paginate requests for the join documents, which leverages the parallelism
  8 |  * of mongodb resulting in much quicker queries.
  9 |  *
 10 |  * Pass the query function, the queried collection, and the regular query arguments after that. At this stage you have
 11 |  * the join object. Call join on this object to begin compiling a join command. The main idea in the join arguments is
 12 |  * to specify the collection to join on, the key of the document in that collection to join on, and the key in the source
 13 |  * document to join on. It's an ad hoc foreign key relationship.
 14 |  */
 15 | module.exports = function () {
 16 |     var _this = this;
 17 |     /**
 18 |      * The initial query which will define the set of documents being joined on. If there is only an object specified
 19 |      * for querycollection, we are assuming that you are passing in the docs you want to join on, the left side.
 20 |      * @param queryCollection The collection to query from. Should be an object which implements the mongo native
 21 |      * collection API. _OR_ an array of documents that you want to join on.
 22 |      * @param query The query by example object to use to retrieve documents
 23 |      * @param [fields] The fields to request in the initial query
 24 |      * @param [options] The query options
 25 |      * @returns {exports}
 26 |      */
 27 |     this.query = function (queryCollection, query, fields, options) {
 28 |         //joinStack holds all the join args to be used, is used like a callstack when exec is called
 29 |         fields = fields || {};
 30 |         options = options || {};
 31 | 
 32 |         var joinStack = [],
 33 |             finalCallback,//This is the final function specified as the callback to call
 34 |             cursor,
 35 |             that = this,
 36 |             noInitialQuery = false,
 37 |             joinDocs;
 38 | 
 39 |         if (arguments.length === 1) {//The documents to join on are given
 40 |             noInitialQuery = true;
 41 |             joinDocs = queryCollection;
 42 |         } else {
 43 |             cursor = queryCollection.find(query, fields, options);
 44 |         }
 45 | 
 46 |         /**
 47 |          * Begin setting up the join operation
 48 |          *
 49 |          * @param args.joinCollection MongoDB.Collection the collection to join on
 50 |          * @param args.leftKeys Array(String) The foreign key(s) in the left hand document
 51 |          * @param {String} args.leftKey Same as leftKeys, better syntax for when there is no composite key
 52 |          * @param {Array} args.rightKeys The primary key(s) in the right hand document which will uniquely
 53 |          * identify that document
 54 |          * @param {Object} args.joinQuery A filtering query to perform at this level of the join corresponding to
 55 |          * mongodb's query by example objects.
 56 |          * @param {String} [args.joinType] Either 'inner' or 'left' is supported. Inner excludes records for which
 57 |          * there is no match in the right hand table at each level of the join. THIS MAY CAUSE YOU PROBLEMS WITH RECORDS
 58 |          * DISAPPEARING, RECOMMEND NOT SPECIFYING JOIN TYPE!
 59 |          * @param {String} args.rightKey The right hand key, same as right keys just allows for no array
 60 |          * @param args.newKey String The name of the property to map the joined document into
 61 |          * @param args.callback Function The function that will be called with the error message and results set at each
 62 |          * level of the join operation
 63 |          * @param [args.pageSize] Number The number of documents matched per request. The default is 25 which was a good
 64 |          * performer in our case
 65 |          */
 66 |         this.join = function (args) {
 67 |             joinStack.push(args);
 68 |             return that;
 69 |         };
 70 | 
 71 |         /**
 72 |          * Start the join operations, don't stop til we get there
 73 |          * @param {Function} [callback] Optional callback function which will be called with the final result set
 74 |          * Same as supplying a callback on the final joint argument.
 75 |          */
 76 |         this.exec = function (callback) {
 77 |             finalCallback = callback;
 78 |             if (noInitialQuery) {
 79 |                 arrayJoin(joinDocs, joinStack.shift());
 80 |             } else {
 81 |                 cursor.toArray(function (err, results) {
 82 |                     arrayJoin(results, joinStack.shift());
 83 |                 });
 84 |             }
 85 |         };
 86 | 
 87 |         /**
 88 |          * Put a new key in the hash map/bin and return the new location.
 89 |          * @param bin The object in which the key should be placed
 90 |          * @param value The key value
 91 |          * @returns {*}
 92 |          */
 93 |         function putKeyInBin (bin, value) {
 94 |             if (isNullOrUndefined(bin[value])) {
 95 |                 bin = bin[value] = {$val: value};
 96 |             } else {
 97 |                 bin = bin[value];
 98 |             }
 99 | 
100 |             return bin;//where we are in the hash tree
101 |         }
102 | 
103 |         this._putKeyInBin = putKeyInBin;
104 | 
105 |         /**
106 |          * Put a new index value into the array at the given bin, making a new array if there is none.
107 |          * @param currentBin The location in which to put the array
108 |          * @param index the index value to put in the array
109 |          * @returns the newly created array if there was one created. Must be assigned into the correct spot
110 |          */
111 |         function pushOrPut (currentBin, index) {
112 |             var temp = currentBin;
113 |             if (Array.isArray(currentBin)) {
114 |                 if (currentBin.indexOf(index) === -1) {
115 |                     currentBin.push(index);//only add if this is a unique index to prevent dup'ing indexes, and associations
116 |                 }
117 |             } else {
118 |                 temp = [index];
119 |             }
120 |             return temp;
121 |         }
122 | 
123 |         this._pushOrPut = pushOrPut;
124 | 
125 |         /**
126 |          * Create the hash table which maps unique left key combination to an array of numbers representing the indexes of the
127 |          * source data where those unique left key combinations were found. Use this map in the joining process to add
128 |          * join subdocuments to the source data.
129 |          * @param bin The bin object into which keys and indexes will be put
130 |          * @param leftValue the left document from which to retrieve key values
131 |          * @param index the index in the source data that the leftvalue lives at
132 |          * @param accessors The accessor functions which retrieve the value for each join key from the leftValue.
133 |          * @param rightKeys The keys in the right hand set which are being joined on
134 |          */
135 |         function buildHashTableIndices (bin, leftValue, index, accessors, rightKeys) {
136 |             var i,
137 |                 length = accessors.length,
138 |                 lastBin,
139 |                 val;
140 | 
141 |             for (i = 0; i < length; i += 1) {
142 |                 val = accessors[i](leftValue);
143 |                 if (typeof val === "undefined") {
144 |                     return;
145 |                 }
146 | 
147 |                 if (i + 1 === length) {
148 |                     if (Array.isArray(val)) {
149 |                         //for each value in val, put that key in
150 |                         val.forEach(function (subValue) {
151 |                             var boot = putKeyInBin(bin, subValue);
152 |                             bin[subValue] = pushOrPut(boot, index);
153 |                             bin[subValue].$val = subValue;
154 |                         });
155 | 
156 |                     } else {
157 |                         bin[val] = pushOrPut(bin[val], index);
158 |                         bin[val].$val = val;
159 |                     }
160 |                 } else if (Array.isArray(val)) {
161 |                     lastBin = bin;
162 |                     val.forEach(function (subDocumentValue) {//sub vals are for basically supposed to be payment ids
163 |                         var rightKeySubset = rightKeys.slice(i + 1);
164 | 
165 |                         if (isNullOrUndefined(bin[subDocumentValue])) {
166 |                             bin[subDocumentValue] = {$val: subDocumentValue};//Store $val to maintain the data type
167 |                         }
168 | 
169 |                         buildHashTableIndices(bin[subDocumentValue], leftValue, index, accessors.slice(i + 1), rightKeySubset);
170 |                     });
171 |                     return;//don't go through the rest of the accessors, this recursion will take care of those
172 |                 } else {
173 |                     bin = putKeyInBin(bin, val);
174 |                 }
175 |             }
176 |         }
177 | 
178 |         this._buildHashTableIndices = buildHashTableIndices;
179 | 
180 |         /**
181 |          * Build the in and or queries that will be used to query for the join documents.
182 |          * @param keyHashBin The bin to use to retrieve the query vals from
183 |          * @param rightKeys The keys which will exists in the join documents
184 |          * @param level The current level of recursion which will correspond to the accessor and the index of the right key
185 |          * @param valuePath The values that have been gathered so far. Ordinally corresponding to the right keys
186 |          * @param orQueries The total list of $or queries generated
187 |          * @param inQueries The total list of $in queries generated
188 |          */
189 |         function buildQueriesFromHashBin (keyHashBin, rightKeys, level, valuePath, orQueries, inQueries) {
190 |             var keys = Object.getOwnPropertyNames(keyHashBin),
191 |                 or;
192 | 
193 |             valuePath = valuePath || [];
194 | 
195 |             if (level === rightKeys.length) {
196 |                 or = {};
197 |                 rightKeys.forEach(function (key, i) {
198 |                     inQueries[i].push(valuePath[i]);
199 |                     or[key] = valuePath[i];
200 |                 });
201 | 
202 |                 orQueries.push(or);
203 |                 //start returning
204 |                 //take the value path and begin making objects out of it
205 |             } else {
206 |                 keys.forEach(function (key) {
207 |                     if (key !== "$val") {
208 |                         var newPath = valuePath.slice(),
209 |                             value = keyHashBin[key].$val;
210 | 
211 |                         newPath.push(value);//now have a copied array
212 |                         buildQueriesFromHashBin(keyHashBin[key], rightKeys, level + 1, newPath, orQueries, inQueries);
213 |                     }
214 |                 });
215 |             }
216 |         }
217 | 
218 |         this._buildQueriesFromHashBin = buildQueriesFromHashBin;
219 | 
220 |         /**
221 |          * Begin the joining process by compiling some data and performing a query for the objects to be joined.
222 |          * @param results The results of the previous join or query
223 |          * @param args The user supplied arguments which will configure this join
224 |          */
225 |         function arrayJoin (results, args) {
226 |             var srcDataArray = results,//use these results as the source of the join
227 |                 joinCollection = args.joinCollection,//This is the mongoDB.Collection to use to join on
228 |                 joinQuery = args.joinQuery,
229 |                 joinType = args.joinType || 'left',
230 |                 rightKeys = args.rightKeys || [args.rightKey],//Get the value of the key being joined upon
231 |                 newKey = args.newKey,//The new field onto which the joined document will be mapped
232 |                 callback = args.callback,//The callback to call at this level of the join
233 | 
234 |                 length,
235 |                 i,
236 | 
237 |                 subqueries,
238 |                 keyHashBin = {},
239 |                 accessors = [],
240 |                 joinLookups = [],
241 |                 inQueries = [],
242 |                 leftKeys = args.leftKeys || [args.leftKey];//place to put incoming join results
243 | 
244 |             rightKeys.forEach(function () {
245 |                 inQueries.push([]);
246 |             });
247 | 
248 |             leftKeys.forEach(function (key) {//generate the accessors for each entry in the composite key
249 |                 accessors.push(getKeyValueAccessorFromKey(key));
250 |             });
251 | 
252 |             length = results.length;
253 | 
254 |             //get the path first
255 |             for (i = 0; i < length; i += 1) {
256 |                 buildHashTableIndices(keyHashBin, results[i], i, accessors, rightKeys, inQueries, joinLookups, {});
257 |             }//create the path
258 | 
259 |             buildQueriesFromHashBin(keyHashBin, rightKeys, 0, [], joinLookups, inQueries);
260 | 
261 |             if (!Array.isArray(srcDataArray)) {
262 |                 srcDataArray = [srcDataArray];
263 |             }
264 | 
265 |             subqueries = getSubqueries(inQueries, joinLookups, joinQuery, args.pageSize || 25, rightKeys);//example
266 |             runSubqueries(subqueries, function (items) {
267 |                 var un;
268 |                 performJoining(srcDataArray, items, {
269 |                     rightKeyPropertyPaths: rightKeys,
270 |                     newKey: newKey,
271 |                     keyHashBin: keyHashBin
272 |                 });
273 | 
274 |                 if (joinType === "inner") {
275 |                     removeNonMatchesLeft(srcDataArray, newKey);
276 |                 }
277 | 
278 |                 if (joinStack.length > 0) {
279 |                     arrayJoin(srcDataArray, joinStack.shift());
280 |                 } else {
281 |                     callIfFunction(finalCallback, [un, srcDataArray]);
282 |                 }
283 |                 callIfFunction(callback, [un, srcDataArray]);
284 |             }, joinCollection);
285 |         }
286 | 
287 |         this._arrayJoin = arrayJoin;
288 | 
289 |         return this;
290 |     };
291 | 
292 |     /**
293 |      * Get the paged subqueries
294 |      */
295 |     function getSubqueries (inQueries, orQueries, otherQuery, pageSize, rightKeys) {
296 |         var subqueries = [],
297 |             numberOfChunks,
298 |             i,
299 |             inQuery,
300 |             orQuery,
301 |             queryArray,
302 |             from,
303 |             to;
304 |         //                                                   this is a stupid way to turn numbers into 1
305 |         numberOfChunks = (orQueries.length / pageSize) + (!!(orQueries.length % pageSize));
306 | 
307 |         for (i = 0; i < numberOfChunks; i += 1) {
308 |             inQuery = {};
309 |             from = i * pageSize;
310 |             to = from + pageSize;
311 | 
312 |             rightKeys.forEach(function (key, index) {
313 |                 inQuery[rightKeys[index]] = {$in: inQueries[index].slice(from, to)};
314 |             });
315 | 
316 |             orQuery = { $or: orQueries.slice(from, to)};
317 | 
318 |             queryArray = [ { $match: inQuery }, { $match: orQuery } ];
319 | 
320 |             if(otherQuery) {
321 |                 queryArray.push({ $match: otherQuery });
322 |                 //Push this to the end on the assumption that the join properties will be indexed, and the arbitrary 
323 |                 //filter properties won't be indexed.
324 |             }
325 |             subqueries.push(queryArray);
326 |         }
327 |         return subqueries;
328 |     }
329 | 
330 |     this._getSubqueries = getSubqueries;
331 | 
332 |     /**
333 |      * Run the sub queries individually, leveraging concurrency on the server for better performance.
334 |      */
335 |     function runSubqueries (subQueries, callback, collection) {
336 |         var i,
337 |             responsesReceived = 0,
338 |             length = subQueries.length,
339 |             joinedSet = [];//The array where the results are going to get stuffed
340 | 
341 |         if (subQueries.length > 0) {
342 |             for (i = 0; i < subQueries.length; i += 1) {
343 | 
344 |                 collection.aggregate(subQueries[i], function (err, results) {
345 |                     joinedSet = joinedSet.concat(results);
346 |                     responsesReceived += 1;
347 | 
348 |                     if (responsesReceived === length) {
349 |                         callback(joinedSet);
350 |                     }
351 |                 });
352 |             }
353 |         } else {
354 |             callback([]);
355 |         }
356 | 
357 |         return joinedSet;
358 |     }
359 | 
360 |     this._runSubqueries = runSubqueries;
361 | 
362 |     /**
363 |      * Use the lookup value type to build an accessor function for each join key. The lookup algorithm respects dot
364 |      * notation. Currently supports strings and functions.
365 |      * @param lookupValue The key being used to lookup the value.
366 |      * @returns {Function} used to lookup value from an object
367 |      */
368 |     function getKeyValueAccessorFromKey (lookupValue) {
369 |         var accessorFunction;
370 |         if (typeof lookupValue === "string") {
371 |             accessorFunction = function (resultValue) {
372 |                 var args = [resultValue];
373 | 
374 |                 args = args.concat(lookupValue.split("."));
375 | 
376 |                 return safeObjectAccess.apply(this, args);
377 |             };
378 |         } else if (typeof lookupValue === "function") {
379 |             accessorFunction = lookupValue;
380 |         }
381 | 
382 |         return accessorFunction;
383 |     }
384 | 
385 |     this._getKeyValueAccessorFromKey = getKeyValueAccessorFromKey;
386 | 
387 |     /**
388 |      * Join the join set with the original query results at the new key.
389 |      * @param sourceData The original result set
390 |      * @param joinSet The results returned from the join query
391 |      * @param joinArgs The arguments used to join the source to the join set
392 |      */
393 |     function performJoining (sourceData, joinSet, joinArgs) {
394 |         var length = joinSet.length,
395 |             i,
396 |             rightKeyAccessors = [];
397 | 
398 |         joinArgs.rightKeyPropertyPaths.forEach(function (keyValue) {
399 |             rightKeyAccessors.push(getKeyValueAccessorFromKey(keyValue));
400 |         });
401 | 
402 |         for (i = 0; i < length; i += 1) {
403 |             var rightRecord = joinSet[i],
404 |                 currentBin = joinArgs.keyHashBin;
405 | 
406 |             if (isNullOrUndefined(rightRecord)) {
407 |                 continue;//move onto the next, can't join on records that don't exist
408 |             }
409 | 
410 |             //for each entry in the join set add it to the source document at the correct index
411 |             rightKeyAccessors.forEach(function (accessor) {
412 |                 currentBin = currentBin[accessor(rightRecord)];
413 |             });
414 | 
415 |             currentBin.forEach(function (sourceDataIndex) {
416 |                 var theObject = sourceData[sourceDataIndex][joinArgs.newKey];
417 | 
418 |                 if (isNullOrUndefined(theObject)) {//Handle adding multiple matches to the same sub document
419 |                     sourceData[sourceDataIndex][joinArgs.newKey] = rightRecord;
420 |                 } else if (Array.isArray(theObject)) {
421 |                     theObject.push(rightRecord);
422 |                 } else {
423 |                     sourceData[sourceDataIndex][joinArgs.newKey] = [theObject, rightRecord];
424 |                 }
425 |             });
426 |         }
427 |     }
428 | 
429 |     this._performJoining = performJoining;
430 | 
431 |     function isNullOrUndefined (val) {
432 |         return typeof val === "undefined" || val === null;
433 |     }
434 | 
435 |     this._isNullOrUndefined = isNullOrUndefined;
436 | 
437 |     /**
438 |      * Access an object without having to worry about "cannot access property '' of undefined" errors.
439 |      * Some extra, necessary and ugly convenience built in is that, if we encounter an array on the lookup
440 |      * path, we recursively drill down into each array value, returning the values discovered in each of those
441 |      * paths. It's kind of a headache, but necessary.
442 |      * @returns The value you were looking for or undefined
443 |      */
444 |     function safeObjectAccess () {
445 |         var object = arguments[0],
446 |             length = arguments.length,
447 |             args = arguments,
448 |             i,
449 |             results,
450 |             temp;
451 | 
452 |         if (!isNullOrUndefined(object)) {
453 |             for (i = 1; i < length; i += 1) {
454 |                 if (Array.isArray(object)) {//if it's an array find the values from those results
455 |                     results = [];
456 |                     object.forEach(function (subDocument) {
457 |                         temp = safeObjectAccess.apply(
458 |                             safeObjectAccess,
459 |                             [subDocument].concat(Array.prototype.slice.apply(args, [i, length]))
460 |                         );
461 | 
462 |                         if (Array.isArray(temp)) {
463 |                             results = results.concat(temp);
464 |                         } else {
465 |                             results.push(temp);
466 |                         }
467 |                     });
468 |                     break;
469 |                 }
470 |                 if (typeof object !== "undefined") {
471 |                     object = object[arguments[i]];
472 |                 } else {
473 |                     break;
474 |                 }
475 |             }
476 |         }
477 | 
478 |         return results || object
479 |     }
480 | 
481 |     this._safeObjectAccess = safeObjectAccess;
482 | 
483 |     /**
484 |      * Simply call the first argument if it is the typeof a function
485 |      * @param fn The argument to call if it is a function
486 |      * @param args the arguments to call the function with
487 |      */
488 |     function callIfFunction (fn, args) {
489 |         if (typeof fn === "function") {
490 |             fn.apply(fn, args);
491 |         }
492 |     }
493 | 
494 |     this._callIfFunction = callIfFunction;
495 | 
496 |     /**
497 |      * Remove element from array if key doesn't exist for that element.
498 |      * @param array The array to be modified
499 |      * @param key The key that should exist if the element will not be removed
500 |      */
501 |     function removeNonMatchesLeft (array, key) {
502 |         var i;
503 |         for (i = 0; i < array.length; i += 1) {
504 |             if(!array[i][key]) {//remember, you're inserting sub docs, there is no valid falsy here
505 |                 array.splice(i, 1);
506 |                 i -= 1;//Account for the removed element
507 |             }
508 |         }
509 |     }
510 | 
511 |     this._removeNonMatchesLeft = removeNonMatchesLeft;
512 | };
513 | 


--------------------------------------------------------------------------------