├── LICENSE ├── README.md ├── lib ├── client.js └── server.js └── package.js /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 MeteorHacks Pvt Ltd (Sri Lanka). 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | search-source 2 | ============= 3 | 4 | #### Reactive Data Source for building search solutions with Meteor 5 | 6 | If you are new to search source, it's a good idea to look at this introductory [article](https://meteorhacks.com/implementing-an-instant-search-solution-with-meteor.html) on MeteorHacks. 7 | 8 | ## Installation 9 | 10 | ``` 11 | meteor add meteorhacks:search-source 12 | ``` 13 | 14 | ### Creating a source in client 15 | 16 | ```js 17 | var options = { 18 | keepHistory: 1000 * 60 * 5, 19 | localSearch: true 20 | }; 21 | var fields = ['packageName', 'description']; 22 | 23 | PackageSearch = new SearchSource('packages', fields, options); 24 | ``` 25 | 26 | * First parameter for the source is the name of the source itself. You need to use it for defining the data source on the server. 27 | * second arguments is the number of fields to search on the client (used for client side search and text transformation) 28 | * set of options. Here are they 29 | * `keepHistory` - cache the search data locally. You need to give an expire time(in millis) to cache it on the client. Caching is done based on the search term. Then if you search again for that term, it search source won't ask the server to get data again. 30 | * `localSearch` - allow to search locally with the data it has. 31 | 32 | ### Define the data source on the server 33 | 34 | In the server, get data from any backend and send those data to the client as shown below. You need to return an array of documents where each of those object consists of `_id` field. 35 | 36 | > Just like inside a method, you can use `Meteor.userId()` and `Meteor.user()` inside a source definition. 37 | 38 | ```js 39 | SearchSource.defineSource('packages', function(searchText, options) { 40 | var options = {sort: {isoScore: -1}, limit: 20}; 41 | 42 | if(searchText) { 43 | var regExp = buildRegExp(searchText); 44 | var selector = {packageName: regExp, description: regExp}; 45 | return Packages.find(selector, options).fetch(); 46 | } else { 47 | return Packages.find({}, options).fetch(); 48 | } 49 | }); 50 | 51 | function buildRegExp(searchText) { 52 | var words = searchText.trim().split(/[ \-\:]+/); 53 | var exps = _.map(words, function(word) { 54 | return "(?=.*" + word + ")"; 55 | }); 56 | var fullExp = exps.join('') + ".+"; 57 | return new RegExp(fullExp, "i"); 58 | } 59 | ``` 60 | 61 | ### Get the reactive data source 62 | 63 | You can get the reactive data source with the `PackageSearch.getData` api. This is an example usage of that: 64 | 65 | ```js 66 | Template.searchResult.helpers({ 67 | getPackages: function() { 68 | return PackageSearch.getData({ 69 | transform: function(matchText, regExp) { 70 | return matchText.replace(regExp, "$&") 71 | }, 72 | sort: {isoScore: -1} 73 | }); 74 | } 75 | }); 76 | ``` 77 | 78 | `.getData()` api accepts an object with options (and an optional argument to ask for a cursor instead of a fetched array; see example below). These are the options you can pass: 79 | 80 | * `transform` - a transform function to alter the selected search texts. See above for an example usage. 81 | * `sort` - an object with MongoDB sort specifiers 82 | * `limit` - no of objects to limit 83 | * `docTransform` - a transform function to transform the documents in the search result. Use this for computed values or model helpers. (see example below) 84 | 85 | 86 | ```js 87 | Template.searchResult.helpers({ 88 | getPackages: function() { 89 | return PackageSearch.getData({ 90 | docTransform: function(doc) { 91 | return _.extend(doc, { 92 | owner: function() { 93 | return Meteor.users.find({_id: this.ownerId}) 94 | } 95 | }) 96 | }, 97 | sort: {isoScore: -1} 98 | }, true); 99 | } 100 | }); 101 | ``` 102 | 103 | ### Searching 104 | 105 | Finally we can invoke search queries by invoking following API. 106 | 107 | ```js 108 | PackageSearch.search("the text to search"); 109 | ``` 110 | 111 | ### Status 112 | 113 | You can get the status of the search source by invoking following API. It's reactive too. 114 | 115 | ``` 116 | var status = PackageSearch.getStatus(); 117 | ``` 118 | 119 | Status has following fields depending on the status. 120 | 121 | * loading - indicator when loading 122 | * loaded - indicator after loaded 123 | * error - the error object, mostly if backend data source throws an error 124 | 125 | ### Metadata 126 | 127 | With metadata, you get some useful information about search along with the search results. These metadata can be time it takes to process the search or the number of results for this search term. 128 | 129 | You can get the metadata with following API. It's reactive too. 130 | 131 | ```js 132 | var metadata = PackageSearch.getMetadata(); 133 | ``` 134 | 135 | Now we need a way to send metadata to the client. This is how we can do it. You need to change the server side search source as follows 136 | 137 | ```js 138 | SearchSource.defineSource('packages', function(searchText, options) { 139 | var data = getSearchResult(searchText); 140 | var metadata = getMetadata(); 141 | 142 | return { 143 | data: data, 144 | metadata: metadata 145 | } 146 | }); 147 | ``` 148 | 149 | ### Passing Options with Search 150 | 151 | We can also pass some options while searching. This is the way we can implement pagination and other extra functionality. 152 | 153 | Let's pass some options to the server: 154 | 155 | ```js 156 | // In the client 157 | var options = {page: 10}; 158 | PackageSearch.search("the text to search", options); 159 | ``` 160 | 161 | Now you can get the options object from the server. See: 162 | 163 | ```js 164 | // In the server 165 | SearchSource.defineSource('packages', function(searchText, options) { 166 | // do anything with options 167 | console.log(options); // {"page": 10} 168 | }); 169 | ``` 170 | 171 | ### Get Current Search Query 172 | 173 | You can get the current search query with following API. It's reactive too. 174 | 175 | ```js 176 | var searchText = PackageSearch.getCurrentQuery(); 177 | ``` 178 | 179 | ### Clean History 180 | 181 | You can clear the stored history (if enabled the `keepHistory` option) via the following API. 182 | 183 | ```js 184 | PackageSearch.cleanHistory(); 185 | ``` 186 | 187 | ### Defining Data Source in the Client 188 | 189 | Sometime, we don't need to fetch data from the server. We need to get it from a data source aleady available on the client. So, this is how we do it: 190 | 191 | ```js 192 | PackageSearch.fetchData = function(searchText, options, success) { 193 | SomeOtherDDPConnection.call('getPackages', searchText, options, function(err, data) { 194 | success(err, data); 195 | }); 196 | }; 197 | ``` 198 | -------------------------------------------------------------------------------- /lib/client.js: -------------------------------------------------------------------------------- 1 | SearchSource = function SearchSource(source, fields, options) { 2 | this.source = source; 3 | this.searchFields = fields; 4 | this.currentQuery = null; 5 | this.options = options || {}; 6 | 7 | this.status = new ReactiveVar({loaded: true}); 8 | this.metaData = new ReactiveVar({}); 9 | this.history = {}; 10 | this.store = new Mongo.Collection(null); 11 | 12 | this._storeDep = new Tracker.Dependency(); 13 | this._currentQueryDep = new Tracker.Dependency(); 14 | this._currentVersion = 0; 15 | this._loadedVersion = 0; 16 | } 17 | 18 | SearchSource.prototype._loadData = function(query, options) { 19 | var self = this; 20 | var version = 0; 21 | var historyKey = query + EJSON.stringify(options); 22 | if(this._canUseHistory(historyKey)) { 23 | this._updateStore(this.history[historyKey].data); 24 | this.metaData.set(this.history[historyKey].metadata); 25 | self._storeDep.changed(); 26 | } else { 27 | this.status.set({loading: true}); 28 | version = ++this._currentVersion; 29 | this._fetch(this.source, query, options, handleData); 30 | } 31 | 32 | function handleData(err, payload) { 33 | if(err) { 34 | self.status.set({error: err}); 35 | throw err; 36 | } else { 37 | if(payload instanceof Array) { 38 | var data = payload; 39 | var metadata = {}; 40 | } else { 41 | var data = payload.data; 42 | var metadata = payload.metadata; 43 | self.metaData.set(payload.metadata || {}); 44 | } 45 | 46 | if(self.options.keepHistory) { 47 | self.history[historyKey] = {data: data, loaded: new Date(), metadata: metadata}; 48 | } 49 | 50 | if(version > self._loadedVersion) { 51 | self._updateStore(data); 52 | self._loadedVersion = version; 53 | } 54 | 55 | if(version == self._currentVersion) { 56 | self.status.set({loaded: true}); 57 | } 58 | 59 | self._storeDep.changed(); 60 | } 61 | } 62 | }; 63 | 64 | SearchSource.prototype._canUseHistory = function(historyKey) { 65 | var historyItem = this.history[historyKey]; 66 | if(this.options.keepHistory && historyItem) { 67 | var diff = Date.now() - historyItem.loaded.getTime(); 68 | return diff < this.options.keepHistory; 69 | } 70 | 71 | return false; 72 | }; 73 | 74 | SearchSource.prototype._updateStore = function(data) { 75 | var self = this; 76 | var storeIds = _.pluck(this.store.find().fetch(), "_id"); 77 | var currentIds = []; 78 | data.forEach(function(item) { 79 | currentIds.push(item._id); 80 | self.store.update(item._id, item, {upsert: true}); 81 | }); 82 | 83 | // Remove items in client DB that we no longer need 84 | var currentIdMappings = {}; 85 | _.each(currentIds, function(currentId) { 86 | // to support Object Ids 87 | var str = (currentId._str)? currentId._str : currentId; 88 | currentIdMappings[str] = true; 89 | }); 90 | 91 | _.each(storeIds, function(storeId) { 92 | // to support Object Ids 93 | var str = (storeId._str)? storeId._str : storeId; 94 | if(!currentIdMappings[str]) { 95 | self.store.remove(storeId); 96 | } 97 | }); 98 | }; 99 | 100 | SearchSource.prototype.search = function(query, options) { 101 | this.currentQuery = query; 102 | this._currentQueryDep.changed(); 103 | 104 | this._loadData(query, options); 105 | 106 | if(this.options.localSearch) { 107 | this._storeDep.changed(); 108 | } 109 | }; 110 | 111 | SearchSource.prototype.getData = function(options, getCursor) { 112 | options = options || {}; 113 | var self = this; 114 | this._storeDep.depend(); 115 | var selector = {$or: []}; 116 | 117 | var regExp = this._buildRegExp(self.currentQuery); 118 | 119 | // only do client side searching if we are on the loading state 120 | // once loaded, we need to send all of them 121 | if(this.getStatus().loading) { 122 | self.searchFields.forEach(function(field) { 123 | var singleQuery = {}; 124 | singleQuery[field] = regExp; 125 | selector['$or'].push(singleQuery); 126 | }); 127 | } else { 128 | selector = {}; 129 | } 130 | 131 | function transform(doc) { 132 | if(options.transform) { 133 | self.searchFields.forEach(function(field) { 134 | if(self.currentQuery && doc[field]) { 135 | doc[field] = options.transform(doc[field], regExp, field, self.currentQuery); 136 | } 137 | }); 138 | } 139 | if(options.docTransform) { 140 | return options.docTransform(doc); 141 | } 142 | 143 | return doc; 144 | } 145 | 146 | var cursor = this.store.find(selector, { 147 | sort: options.sort, 148 | limit: options.limit, 149 | transform: transform 150 | }); 151 | 152 | if(getCursor) { 153 | return cursor; 154 | } 155 | 156 | return cursor.fetch(); 157 | }; 158 | 159 | SearchSource.prototype._fetch = function(source, query, options, callback) { 160 | if(typeof this.fetchData == 'function') { 161 | this.fetchData(query, options, callback); 162 | } else if(Meteor.status().connected) { 163 | this._fetchDDP.apply(this, arguments); 164 | } else { 165 | this._fetchHttp.apply(this, arguments); 166 | } 167 | }; 168 | 169 | SearchSource.prototype._fetchDDP = function(source, query, options, callback) { 170 | Meteor.call("search.source", this.source, query, options, callback); 171 | }; 172 | 173 | SearchSource.prototype._fetchHttp = function(source, query, options, callback) { 174 | var payload = { 175 | source: source, 176 | query: query, 177 | options: options 178 | }; 179 | 180 | var headers = { 181 | "Content-Type": "text/ejson" 182 | }; 183 | 184 | HTTP.post('/_search-source', { 185 | content: EJSON.stringify(payload), 186 | headers: headers 187 | }, function(err, res) { 188 | if(err) { 189 | callback(err); 190 | } else { 191 | var response = EJSON.parse(res.content); 192 | if(response.error) { 193 | callback(response.error); 194 | } else { 195 | callback(null, response.data); 196 | } 197 | } 198 | }); 199 | }; 200 | 201 | SearchSource.prototype.getMetadata = function() { 202 | return this.metaData.get(); 203 | }; 204 | 205 | SearchSource.prototype.getCurrentQuery = function() { 206 | this._currentQueryDep.depend(); 207 | return this.currentQuery; 208 | } 209 | 210 | SearchSource.prototype.getStatus = function() { 211 | return this.status.get(); 212 | }; 213 | 214 | SearchSource.prototype.cleanHistory = function() { 215 | this.history = {}; 216 | }; 217 | 218 | SearchSource.prototype._buildRegExp = function(query) { 219 | query = query || ""; 220 | 221 | var afterFilteredRegExpChars = query.replace(this._getRegExpFilterRegExp(), "\\$&"); 222 | var parts = afterFilteredRegExpChars.trim().split(' '); 223 | 224 | return new RegExp("(" + parts.join('|') + ")", "ig"); 225 | }; 226 | 227 | SearchSource.prototype._getRegExpFilterRegExp = _.once(function() { 228 | var regExpChars = [ 229 | "\\", "^", "$", "*", "+", "?", ".", 230 | "(", ")", ":", "|", "{", "}", "[", "]", 231 | "=", "!", "," 232 | ]; 233 | var regExpCharsReplace = _.map(regExpChars, function(c) { 234 | return "\\" + c; 235 | }).join("|"); 236 | return new RegExp("(" + regExpCharsReplace + ")", "g"); 237 | }); -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | SearchSource = {}; 2 | SearchSource._sources = {}; 3 | var bodyParser = Npm.require('body-parser'); 4 | 5 | SearchSource.defineSource = function(name, callback) { 6 | SearchSource._sources[name] = callback; 7 | }; 8 | 9 | Meteor.methods({ 10 | "search.source": function(name, query, options) { 11 | check(name, String); 12 | check(query, Match.OneOf(String, null, undefined)); 13 | check(options, Match.OneOf(Object, null, undefined)); 14 | this.unblock(); 15 | 16 | // we need to send the context of the method 17 | // that's why we use .call instead just invoking the function 18 | return getSourceData.call(this, name, query, options); 19 | } 20 | }); 21 | 22 | var postRoutes = Picker.filter(function(req, res) { 23 | return req.method == "POST"; 24 | }); 25 | 26 | postRoutes.middleware(bodyParser.text({ 27 | type: "text/ejson" 28 | })); 29 | 30 | postRoutes.route('/_search-source', function(params, req, res, next) { 31 | if(req.body) { 32 | var payload = EJSON.parse(req.body); 33 | try { 34 | // supporting the use of Meteor.userId() 35 | var data = DDP._CurrentInvocation.withValue({userId: null}, function() { 36 | return getSourceData(payload.source, payload.query, payload.options); 37 | }); 38 | sendData(res, null, data); 39 | } catch(ex) { 40 | if(ex instanceof Meteor.Error) { 41 | var error = { code: ex.error, message: ex.reason }; 42 | } else { 43 | var error = { message: ex.message }; 44 | } 45 | sendData(res, error); 46 | } 47 | } else { 48 | next(); 49 | } 50 | }); 51 | 52 | 53 | function sendData(res, err, data) { 54 | var payload = { 55 | error: err, 56 | data: data 57 | }; 58 | 59 | res.end(EJSON.stringify(payload)); 60 | } 61 | 62 | function getSourceData(name, query, options) { 63 | var source = SearchSource._sources[name]; 64 | if(source) { 65 | return source.call(this, query, options); 66 | } else { 67 | throw new Meteor.Error(404, "No such search source: " + name); 68 | } 69 | } -------------------------------------------------------------------------------- /package.js: -------------------------------------------------------------------------------- 1 | Package.describe({ 2 | "summary": "Reactive Data Source for Search", 3 | "version": "1.4.3 4 | "git": "https://github.com/meteorhacks/search-source.git", 5 | "name": "meteorhacks:search-source" 6 | }); 7 | 8 | Npm.depends({ 9 | "body-parser": "1.10.1" 10 | }); 11 | 12 | Package.onUse(function(api) { 13 | configurePackage(api); 14 | api.export(['SearchSource']); 15 | }); 16 | 17 | Package.onTest(function(api) { 18 | configurePackage(api); 19 | 20 | api.use(['tinytest', 'mongo-livedata'], ['client', 'server']); 21 | }); 22 | 23 | function configurePackage(api) { 24 | api.versionsFrom('METEOR@0.9.2'); 25 | api.use([ 26 | 'tracker', 'underscore', 'mongo', 'reactive-var', 27 | 'http', 'ejson', 'check', 'ddp' 28 | ], ['client']); 29 | 30 | api.use(['ejson', 'check', 'ddp'], ['server']); 31 | 32 | api.use('meteorhacks:picker@1.0.1', 'server'); 33 | 34 | api.add_files([ 35 | 'lib/server.js', 36 | ], ['server']); 37 | 38 | api.add_files([ 39 | 'lib/client.js', 40 | ], ['client']); 41 | } 42 | --------------------------------------------------------------------------------