├── .gitignore ├── README.md ├── index.js ├── lib ├── denormalize.js └── filter.js ├── package.json └── test └── TODO /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | lib-cov 3 | *.seed 4 | *.log 5 | *.csv 6 | *.dat 7 | *.out 8 | *.pid 9 | *.gz 10 | 11 | pids 12 | logs 13 | results 14 | 15 | npm-debug.log 16 | node_modules 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mongoose-Filter-Denormalize 2 | 3 | Simple filtering and denormalization for Mongoose. Useful for REST APIs where you might not want to 4 | send entire objects down the pipe. Allows you to store sensitive data directly on objects without worrying 5 | about it being sent to end users. 6 | 7 | ## Installation 8 | 9 | ``` 10 | npm install mongoose-filter-denormalize 11 | ``` 12 | 13 | ## Compatibility 14 | 15 | `mongoose <= v3.4` 16 | 17 | ## Filter Usage 18 | 19 | Filtering functionality is provided via a schema plugin. 20 | 21 | ### Schema 22 | 23 | ```javascript 24 | var filter = require('mongoose-filter-denormalize').filter; 25 | var ObjectId = mongoose.Schema.ObjectId; 26 | var UserSchema = new Mongoose.schema({ 27 | name : String, 28 | address : String, 29 | fb : { 30 | id : Number, 31 | accessToken : String 32 | }, 33 | writeOnlyField : String, 34 | readOnlyField : String 35 | }); 36 | UserSchema.plugin(filter, { 37 | readFilter: { 38 | "owner" : ['name', 'address', 'fb.id', 'fb.name', 'readOnlyField'], 39 | "public": ['name', 'fb.name'] 40 | }, 41 | writeFilter: { 42 | "owner" : ['name', 'address', 'fb.id', 'writeOnlyField'] 43 | }, 44 | // 'nofilter' is a built-in filter that does no processing, be careful with this 45 | defaultFilterRole: 'nofilter', 46 | sanitize: true, // Escape HTML in strings 47 | compat: true // Enable compatibility for Mongoose versions prior to 3.6 (default false) 48 | }); 49 | ``` 50 | 51 | ### Example Read 52 | 53 | ```javascript 54 | User.findOne({name: 'Foo Bar'}, User.getReadFilterKeys('public')), function(err, user){ 55 | if(err) return next(err); 56 | res.send({success: true, users: [user]}); 57 | }); 58 | ``` 59 | 60 | ### Example Write 61 | 62 | ```javascript 63 | User.findById(req.params.id, function(err, user){ 64 | if(err) next(err); 65 | if(user.id !== req.user.id) next(403); 66 | user.extendWithWriteFilter(inputRecord, 'owner'); // Similar to jQuery.extend() 67 | user.save(function(err, user){ 68 | if(err) return next(err); 69 | user.applyReadFilter('owner'); // Make sure the doc you return does not contain forbidden fields 70 | res.send({success: true, users: [user]}); 71 | }); 72 | }); 73 | ``` 74 | 75 | ### Options 76 | 77 | - `readFilter` (Object): Object mapping filtering profiles to string arrays of allowed fields. Used when reading 78 | a doc - useful for GET queries that must return only selected fields. 79 | - `writeFilter` (Object): As above, but used when when applied during a PUT or POST. This filters fields out of a given 80 | object so they will not be written even when specified. 81 | Useful for protected attributes like fb.accessToken. 82 | - `defaultFilterRole` (String)(default: 'nofilter'): Profile to use when one is not given, or the given profile does not exist. 83 | - `sanitize` (Boolean)(default: false): True to automatically escape HTML in strings. 84 | - `compat` (Boolean)(default: false): True to enable compatibility with Mongoose versions prior to 3.6 85 | 86 | ### Statics 87 | 88 | This plugin adds the following statics to your schema: 89 | 90 | - `getReadFilterKeys(filterRole)` 91 | - `getWriteFilterKeys(filterRole)` 92 | - `applyReadFilter(input, filterRole)` 93 | - `applyWriteFilter(input, filterRole` 94 | - `_applyFilter(input, filterKeys) // private helper` 95 | - `_getFilterKeys(type, filterRole) // private helper` 96 | 97 | ### Methods 98 | 99 | This plugin adds the following methods to your schema: 100 | 101 | - `extendWithWriteFilter(input, filterRole)` 102 | - `applyReadFilter(filterRole) // convenience method, calls statics.applyReadFilter` 103 | - `applyWriteFilter(filterRole) // convenience method, calls statics.applyWriteFilter` 104 | 105 | ## Denormalize Usage 106 | 107 | Denormalization functionality is provided via a schema plugin. 108 | This plugin has support for, but does not require, the filter.js plugin in the same package. 109 | 110 | ### Schema 111 | 112 | ```javascript 113 | var denormalize = require('mongoose-filter-denormalize').denormalize; 114 | var ObjectId = mongoose.Schema.ObjectId; 115 | var UserSchema = new Mongoose.schema({ 116 | name : String, 117 | transactions : [{type:ObjectId, ref:'Transaction'}], 118 | address : {type:ObjectId, ref:'Address'}, 119 | tickets : [{type:ObjectId, ref:'Ticket'}], 120 | bankaccount : {type:ObjectId, ref:'BankAccount'} 121 | }); 122 | 123 | // Running .denormalize() during a query will by default denormalize the selected defaults. 124 | // Excluded collections are never denormalized, even when asked for. 125 | // This is useful if passing query params directly to your methods. 126 | UserSchema.plugin(denormalize, {defaults: ['address', 'transactions', 'tickets'], 127 | exclude: 'bankaccount'}); 128 | ``` 129 | 130 | ### Querying 131 | 132 | ```javascript 133 | // Create a query. 134 | // The 'conditions' object allows you to query on denormalized objects! 135 | var opts = { 136 | refs: ["transactions", "address"], // Denormalize these refs. If blank, will use defaults 137 | filter: "public"; // Filter requires use of filter.js and profiles 138 | conditions: { 139 | address: {city : {$eq: "Seattle"}} // Only return the user if he is in Seattle 140 | } 141 | }; 142 | User.findOne({name: 'Foo Bar'}).denormalize(opts).run(function(err, user){ 143 | if(err) next(err); 144 | res.send({success: true, users: [user]}); 145 | }); 146 | ``` 147 | 148 | ### Options 149 | 150 | * `exclude` (String[] or String): References to never denormalize, even when explicitly asked 151 | Use this when generating refs programmatically, to prevent unintended leakage. 152 | * `defaults` (String[] or String): References to denormalize when called without options. 153 | Defaults to all refs (except those in 'exclude'). Useful to define 154 | this if you have hasMany references that can easily get large. 155 | * `suffix` (String): A suffix to add to all denormalized objects. This is not yet supported in Mongoose 156 | but hopefully will be soon. E.g. a suffix of '_obj' would denormalize the story.comment 157 | object to story.comment_obj, leaving the id in story.comment. This is necessary 158 | for compatibility with ExtJS. 159 | 160 | ### Notes 161 | 162 | If you are building your array of refs to denormalize programmatically, make sure it returns 163 | an empty array if you do not want it to denormalize - falsy values will cause this plugin 164 | to use defaults. 165 | 166 | ## Credits 167 | 168 | [Mongoose](https://github.com/LearnBoost/mongoose) 169 | 170 | 171 | ## License 172 | 173 | MIT 174 | 175 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports.filter = require('./lib/filter'); 2 | module.exports.denormalize = require('./lib/denormalize'); -------------------------------------------------------------------------------- /lib/denormalize.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Adds denormalize functionality for a doc. 3 | * 4 | * This plugin has support for, but does not require, the mongoose-filter-plugin in the same package. 5 | * 6 | * Options: 7 | * exclude (String[] or String) - References to never denormalize, even when explicitly asked 8 | * defaults (String[] or String) - References to denormalize when called without options. 9 | * Defaults to all refs (except those in 'exclude'). Useful to define 10 | * this if you have hasMany references that can easily get large. 11 | * 12 | * Example Usage: 13 | * 14 | * var denormalize = require('mongoose-filter-denormalize').denormalize; 15 | * var ObjectId = mongoose.Schema.ObjectId; 16 | * var UserSchema = new Mongoose.schema({ 17 | * name : String, 18 | * transactions : [{type:ObjectId, ref:'Transaction'}], 19 | * address : {type:ObjectId, ref:'Address'}, 20 | * tickets : [{type:ObjectId, ref:'Ticket'}], 21 | * bankaccount : {type:ObjectId, ref:'BankAccount'} 22 | * }); 23 | * 24 | * UserSchema.plugin(denormalize, {defaults: ['address', 'transactions', 'tickets'], exclude: 'bankaccount'}); 25 | * 26 | * var opts = {refs: ["transactions", "address"], filter: "public"}; // Filter requires use of mongoose-filter-plugin 27 | * opts.conditions = {address: {city : {$eq: "Seattle"}}}; // Only return the user if he is in Seattle 28 | * User.findOne({name: 'Foo Bar'}).denormalize(opts).run(function(err, user){ 29 | * if(err) return next(err); 30 | * res.send({success: true, users: [user]}); 31 | * }); 32 | * 33 | * If you are building your array of refs to denormalize programmatically, make sure it returns 34 | * an empty array if you do not want it to denormalize - falsy values will cause this plugin 35 | * to use defaults. 36 | * 37 | * 38 | */ 39 | 40 | var mongoose = require('mongoose'), 41 | _ = require('lodash'); 42 | 43 | module.exports = function denormalize(schema, options) { 44 | if(!options) options = {}; 45 | var defaultRefs = _parseRefs(options.defaults), 46 | excludedRefs = _.flatten([options.exclude]); 47 | 48 | /** 49 | * Chains populate() calls together. Requires model.getDenormalizationRefs(refs) to parse excludes 50 | * and defaults. 51 | */ 52 | if(!mongoose.Query.prototype.denormalize){ 53 | mongoose.Query.prototype.denormalize = function denormalize(opts) { 54 | var query = this, 55 | filter; 56 | 57 | if(!opts) opts = {options: {}}; 58 | if(!opts.options) opts.options = {}; 59 | if(!opts.conditions) opts.conditions = {}; 60 | opts.options.suffix = options.suffix || ""; 61 | 62 | if(_.isFunction(query.model.getDenormalizationRefs)){ 63 | var refs = query.model.getDenormalizationRefs(opts.refs); 64 | _.each(refs, function(ref){ 65 | query.populate(ref, _getReadFilterKeys(ref, opts), opts.conditions[ref] || {}, opts.options); 66 | }); 67 | } 68 | return this; 69 | 70 | function _getReadFilterKeys(ref, opts){ 71 | var modelName = query.model.schema.path(ref).options.ref; 72 | var model = mongoose.model(modelName); 73 | if(_.isFunction(model.getReadFilterKeys)){ 74 | return model.getReadFilterKeys(opts.filter); 75 | } 76 | } 77 | }; 78 | } 79 | 80 | /** 81 | * Used by query.denormalize 82 | */ 83 | schema.statics.getDenormalizationRefs = function(refs){ 84 | if(refs === 'false') return []; 85 | if(!refs || refs === 'true'){ 86 | refs = defaultRefs; 87 | } else { 88 | refs = _parseRefs(refs); 89 | } 90 | return _filterExcludes(refs); 91 | }; 92 | 93 | // -------- Private Functions ------------- 94 | 95 | // Given refs, match to existing schema paths. 96 | function _parseRefs(refs){ 97 | if(refs) refs = _.flatten([refs]); 98 | if(_.isEmpty(refs)) return _findAllRefs(); 99 | 100 | var ret = []; 101 | 102 | // Make sure that each input given is a real ref 103 | _.each(refs, function(ref){ 104 | if(_.isString(ref) && ref.length > 0){ 105 | if(schema.paths[ref].instance === "ObjectID"){ 106 | ret.push(ref); 107 | } 108 | } 109 | }); 110 | return ret; 111 | } 112 | 113 | function _filterExcludes(refsToDenormalize){ 114 | var ret = _.filter(refsToDenormalize, function(ref){ 115 | if(excludedRefs.indexOf(ref) === -1 && excludedRefs.indexOf(ref.replace(/_id$/, "")) === -1) 116 | return ref; 117 | }); 118 | return ret; 119 | } 120 | 121 | // If no defaults are given, grab every ObjectId in this schema (except _id) 122 | function _findAllRefs(){ 123 | var ret = []; 124 | _.each(schema.paths, function(path){ 125 | if (path.instance === "ObjectID" && path.path !== "_id") { 126 | ret.push(path.path); 127 | } 128 | }); 129 | return ret; 130 | } 131 | }; 132 | -------------------------------------------------------------------------------- /lib/filter.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Filter.js adds data security for a doc. You may apply named profiles that determine which fields will be 3 | * retrieved, to ensure that JSON sent to a user does not include inappropriate fields. 4 | * Optionally, you may automatically sanitize HTML by adding sanitize: true to plugin initialization. 5 | * 6 | * Options: 7 | * readFilter - Object mapping filtering profiles to string arrays of allowed fields. Used when reading 8 | * a doc - useful for GET queries that must return only selected fields. 9 | * writeFilter - As above, but used when when applied during a PUT or POST. This filters fields out of a given 10 | * object so they will not be written even when specified. 11 | * Useful for protected attributes like fb.accessToken. 12 | * defaultFilterRole - Profile to use when one is not given, or the given profile does not exist. 13 | * sanitize - True to automatically escape HTML in strings. 14 | * 15 | * 16 | * 17 | * This plugin adds the following statics to your schema: 18 | * getReadFilterKeys(filterRole) 19 | * getWriteFilterKeys(filterRole) 20 | * applyReadFilter(input, filterRole) 21 | * applyWriteFilter(input, filterRole) 22 | * _applyFilter(input, filterKeys) // private helper 23 | * _getFilterKeys(type, filterRole) // private helper 24 | * 25 | * and the following methods: 26 | * extendWithWriteFilter(input, filterRole) 27 | * applyReadFilter(filterRole) // convenience method, calls statics.applyReadFilter 28 | * applyWriteFilter(filterRole) // convenience method, calls statics.applyWriteFilter 29 | * 30 | * ---------- ---------- 31 | * ---------- Getting Started ---------- 32 | * ---------- ---------- 33 | * 34 | * ----- ----- 35 | * ----- Environment: ----- 36 | * ----- ----- 37 | * 38 | * var filter = require('mongoose-filter-denormalize').filter; 39 | * var ObjectId = mongoose.Schema.ObjectId; 40 | * var UserSchema = new Mongoose.schema({ 41 | * name : String, 42 | * address : String, 43 | * fb : { 44 | * id : Number, 45 | * accessToken : String 46 | * }, 47 | * writeOnlyField : String, 48 | * readOnlyField : String 49 | * }); 50 | * 51 | * UserSchema.plugin(filter, { 52 | * readFilter: { 53 | * "owner" : ['name', 'address', 'fb.id', 'fb.name', 'readOnlyField'], 54 | * "public": ['name', 'fb.name'] 55 | * }, 56 | * writeFilter: { 57 | * "owner" : ['name', 'address', 'fb.id', 'writeOnlyField'] 58 | * }, 59 | * defaultFilterRole: 'nofilter', // 'nofilter' is a built-in filter that does no processing, be careful with this 60 | * sanitize: true // Escape HTML in strings 61 | * }); 62 | * 63 | * ----- ----- 64 | * ----- Example read: ----- 65 | * ----- ----- 66 | * 67 | * User.findOne({name: 'Foo Bar'}, User.getReadFilterKeys('public')), function(err, user){ 68 | * if(err) next(err); 69 | * res.send({success: true, users: [user]}); 70 | * }); 71 | * 72 | * 73 | * ----- ----- 74 | * ----- Example write: ----- 75 | * ----- ----- 76 | * 77 | * User.findById(req.params.id, function(err, user){ 78 | * if(err) next(err); 79 | * if(user.id !== req.user.id) next(403); 80 | * user.extendWithWriteFilter(inputRecord, 'owner'); // Function added by plugin, similar to jQuery.extend()/Ext.extend() 81 | * user.save(function(err, user){ 82 | * if(err) return next(err); 83 | * user.applyReadFilter('owner'); // Make sure the doc you return does not contain forbidden fields 84 | * res.send({success: true, users: [user]}); 85 | * }); 86 | * }); 87 | * 88 | * 89 | */ 90 | 91 | var mongoose = require('mongoose'), 92 | _ = require('lodash'), 93 | sanitizer = require('sanitizer'); 94 | 95 | module.exports = function filter(schema, options) { 96 | 97 | var defaults = { 98 | sanitize: false, // escapes HTML 99 | defaultFilterRole : 'nofilter', 100 | compat: false 101 | }; 102 | 103 | options = _.extend(defaults, options); 104 | 105 | schema.statics.getReadFilterKeys = function(filterRole){ 106 | var filters = this._getFilterKeys("readFilter", filterRole); 107 | filters = filters ? filters.concat('_id') : filters; // Always send _id property 108 | if (options.compat) return filters; 109 | else return filters.join(' '); 110 | }; 111 | 112 | schema.statics.getWriteFilterKeys = function(filterRole){ 113 | return this._getFilterKeys("writeFilter", filterRole); 114 | }; 115 | 116 | schema.statics._getFilterKeys = function(type, filterRole){ 117 | var filter = {nofilter: null}; 118 | _.extend(filter, options[type]); 119 | return filterRole in filter ? filter[filterRole] : filter[options.defaultFilterRole]; 120 | }; 121 | 122 | /** 123 | * When executing a 'put', this is called on the retrieved doc with given inputs. 124 | * It is the controller's job to define a filterRole, which determines 125 | * which properties of the doc may be overwritten. 126 | */ 127 | schema.methods.extendWithWriteFilter = function(input, filterRole){ 128 | input = this.constructor.applyWriteFilter(input, filterRole); 129 | _.extend(this, input); 130 | }; 131 | 132 | /** 133 | * Helper for quickly applying a read filter. 134 | */ 135 | schema.methods.applyReadFilter = function(filterRole){ 136 | this._doc = this.constructor.applyReadFilter(this._doc, filterRole); 137 | }; 138 | 139 | /** 140 | * Helper for quickly applying a write filter. 141 | */ 142 | schema.methods.applyWriteFilter = function(filterRole){ 143 | this._doc = this.constructor.applyWriteFilter(this._doc, filterRole); 144 | }; 145 | 146 | /** 147 | * This will apply the read filter onto a given input or array of inputs. Useful if you need to take out 148 | * properties the user is not supposed to see after an update or other complicated logic. 149 | * Use just before calling res.send. 150 | */ 151 | schema.statics.applyReadFilter = function applyReadFilter(input, filterRole){ 152 | if(_.isArray(input)){ 153 | var ret = []; 154 | _.each(input, function(doc){ 155 | ret.push(applyReadFilter(doc, filterRole)); 156 | }); 157 | return ret; 158 | } 159 | 160 | var filters = this.getReadFilterKeys(filterRole); 161 | return this._applyFilter(input, filters); 162 | }; 163 | 164 | /** 165 | * Use this to manually apply a write filter on an input or array of inputs. 166 | * Note that this WILL DELETE ALL PROPERTIES NOT IN THE FILTER! 167 | * Do not use this on a doc before using doc.save(), as you will save it with filtered 168 | * properties missing - they will be overwritten with undefined. 169 | * Instead use methods.extendWithWriteFilter(input, filterRole), which only modifies allowed 170 | * properties but does not delete the rest. 171 | */ 172 | schema.statics.applyWriteFilter = function applyWriteFilter(input, filterRole){ 173 | if(_.isArray(input)){ 174 | var ret = []; 175 | _.each(input, function(doc){ 176 | ret.push(applyWriteFilter(doc, filterRole)); 177 | }); 178 | return ret; 179 | } 180 | 181 | // Sanitize strings 182 | if(options.sanitize){ 183 | input = sanitizeObject(input); 184 | } 185 | var filterKeys = this._getFilterKeys('writeFilter', filterRole); 186 | return this._applyFilter(input, filterKeys); 187 | }; 188 | 189 | 190 | /** 191 | * Applies a filter to a given object. 192 | */ 193 | schema.statics._applyFilter = function _applyFilter(input, filters){ 194 | if(filters === null) // no filter applied 195 | return input; 196 | 197 | var fieldKeys = _.keys(input), 198 | keysToRemove = [], 199 | schema = this; 200 | 201 | input = _.clone(input); // don't modify original object 202 | 203 | _.each(fieldKeys, function(key){ // search recursively for keys not matching the filter 204 | // We have an object, look through subobjects 205 | if(_.isObject(input[key]) && !_.isEmpty(input[key]) && !_.isArray(input[key]) && input[key]._bsontype !== "ObjectID"){ 206 | var reducedFilters = []; 207 | if(_.include(filters, key)) // whole object is allowed in filter, don't bother diving in 208 | return; 209 | 210 | // Find filters that may apply to this object's subfields 211 | _.each(filters, function(filter){ 212 | if(filter.indexOf(key) === 0) // Look for filters of the format "object.field" 213 | reducedFilters.push(filter.substring(key.length + 1)); 214 | }); 215 | 216 | // If some filters match, recusively call _applyFilter on this object 217 | if(reducedFilters.length > 0){ 218 | input[key] = _applyFilter(input[key], reducedFilters); 219 | } else { 220 | keysToRemove.push(key); // No matches, delete the whole object 221 | } 222 | } 223 | 224 | else if(filters.indexOf(key) === -1) keysToRemove.push(key); 225 | 226 | // Remove empty string objectIDs, this causes problems on writes 227 | else if(input[key] === "" && _.isObject(schema.prototype.schema.paths[key]) && 228 | schema.prototype.schema.paths[key].instance === "ObjectID"){ 229 | keysToRemove.push(key); 230 | } 231 | 232 | }); 233 | 234 | _.each(keysToRemove, function(key){ // remove keys 235 | delete input[key]; 236 | }); 237 | 238 | return input; 239 | }; 240 | 241 | /** Helpers **/ 242 | function sanitizeObject(obj){ 243 | _.each(obj, function(child, key){ 244 | if(_.isObject(child)) 245 | obj[key] = sanitizeObject(child); 246 | else if(_.isString(child)) 247 | obj[key] = sanitizer.escape(child); 248 | }); 249 | return obj; 250 | } 251 | 252 | }; 253 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongoose-filter-denormalize", 3 | "version": "0.2.1", 4 | "description": "Simple collection filtering and denormalization.", 5 | "author": "Samuel Reed