├── .gitignore ├── .versions ├── README.md ├── minimongo-bulk.js ├── package.js ├── richsilv:dumb-collections.js ├── richsilv:dumb-collections_tests.js └── versions.json /.gitignore: -------------------------------------------------------------------------------- 1 | .build* 2 | -------------------------------------------------------------------------------- /.versions: -------------------------------------------------------------------------------- 1 | amplify@1.0.0 2 | application-configuration@1.0.4 3 | base64@1.0.2 4 | binary-heap@1.0.2 5 | callback-hook@1.0.2 6 | check@1.0.4 7 | ddp@1.0.14 8 | ejson@1.0.5 9 | follower-livedata@1.0.3 10 | geojson-utils@1.0.2 11 | id-map@1.0.2 12 | jquery@1.11.3 13 | json@1.0.2 14 | local-test:richsilv:dumb-collections@1.1.4 15 | logging@1.0.6 16 | meteor@1.1.4 17 | minimongo@1.0.6 18 | mongo@1.0.11 19 | ordered-dict@1.0.2 20 | random@1.0.2 21 | reactive-var@1.0.4 22 | retry@1.0.2 23 | richsilv:dumb-collections@1.1.4 24 | tinytest@1.0.4 25 | tracker@1.0.5 26 | underscore@1.0.2 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | richsilv:dumb-collections 2 | ======================= 3 | 4 | Dumb Collections for Meteor, in which the client syncs with server only on demand. 5 | 6 | ## Why? 7 | 8 | To avoid the livedata overhead associated with the usual pub/sub model, whilst retaining the benefits of minimongo and the Collections API. Also utilises localStorage to improve load times. 9 | 10 | ## NEW IN VERSION 1.1.0 11 | 12 | * Now compatible with Collection2. 13 | * Ability to specify `options` as well as `query` when syncing. 14 | * Insertion and removal is now far, far more efficient. 15 | * `reactive` parameter to allow client-side collection to be cleared non-reactively. 16 | 17 | ## Usage 18 | 19 | ```javascript``` 20 | MyCollection = new DumbCollection('mycollection'); 21 | ``` 22 | 23 | ## Extra Methods *(Client only)* 24 | 25 | #### DumbCollection.ready() 26 | 27 | Reactive variable: `true` once the initial load from localStorage is complete (whether it has yielded any documents or not), `false` beforehand. 28 | 29 | #### DumbCollection.synced() 30 | 31 | Reactive variable:`true` once the first server `sync` has been completed. On further `sync` calls, this will revert to `false` until the new sync is completed. 32 | 33 | #### DumbCollection.sync(options) 34 | 35 | Synchronise data with the server. Synchronisation is *always from server to client*. If you want to write any data from the client to the server, you need to write Meteor.methods and call insert/update*/remove on the server. This would work in exactly the same way as for any normal collection (although see below for update). 36 | 37 | ##### *Options* 38 | 39 | __*query*__ - the Mongo query to apply on the server, which will dictate what data is returned. If absent, the entire collection will be sent. Note that documents which fall outside the query will be removed from the client collection unless `retain` is set to true. 40 | 41 | __*options*__ - query options, as per [the Collections API](http://docs.meteor.com/#/full/find). `reactive` will be ignored, `transform` is untested. 42 | 43 | __*retain*__ - if set to `true`, the client collection will retain any documents which are not present in the server result set. Default is `false`. 44 | 45 | __*reject*__ - if set to `true`, the client collection will reject any new documents sent by the server. Default is `false`. 46 | 47 | __*removalCallback(removedDocs)*__ - callback to run once any unrecognised documents are removed from the client collection, with the removed documents passed as an argument. 48 | 49 | __*insertionCallback(insertedDocs)*__ - callback to run once any new documents sent by the server have been inserted, with the inserted documents passed as an argument. 50 | 51 | __*syncCallback(results)*__ - callback to run once the whole synchronisation is complete, with the results object containing the keys `removed` and `inserted`, which contain the same details as the previous callbacks respectively. 52 | 53 | __*failCallback(error)*__ - callback to run on the failure to store the client collection in localStorage once synchronisation is complete. This is passed the error object, which is almost always the result of the storage limit being exceeded. 54 | 55 | #### DumbCollection.clear(reactive) 56 | 57 | Clear the contents of the client-side collection, and associated local storage. 58 | 59 | ##### reactive [BOOLEAN] 60 | 61 | This method will set the reactive variable returned by `synced()` to `false`. However, by passing this flag, the value will be changed in a way which does not invalidate dependent computations. Note that computations dependent on the documents in the collection itself will still be invalidated. 62 | 63 | #### DumbCollection.ironRouterReady() 64 | 65 | A convenience method for allowing the use of Iron Router's wait and waitOn methods with Dumb Collections. Returns a handle with a single, reactive "ready" method, which indicates whether the collection has been synchronised. This allows one to do the following: 66 | 67 | ```javascript 68 | waitOn: function() { 69 | return MyDumbCollection.ironRouterReady(); 70 | } 71 | ``` 72 | 73 | **Note 1** - This relates to the 1.0.0 release of Iron Router - it is untested with previous versions. 74 | 75 | **Note 2** - If you only want to wait for the collection to be loaded from localStorage rather than being synchronised, just return the collection itself (i.e. `return MyDumbCollection;`). 76 | 77 | **Note 3** - If you want to run the synchronisation from Iron Router hooks, you *must* do this from the `onRun` hook, rather than `onBeforeAction`, `data`, or any of the other reactive hooks. If you don't then the route will continually reload reactively and the Dumb Collection will try to resynchronise each time. See the demo for an example. 78 | 79 | ## Limitations 80 | 81 | * Synchronisation is always from the server to the client by design, so the user will need to write Meteor.methods with appropriate security to perform CUD in the opposite direction. 82 | * The `update` method on the server collection has been renamed `_update` in an attempt to discourage its use - the synchronisation is based on `_id`s, and so any `update`s made on the server side cannot be synchronised. Given that one cannot update the `_id` field in MongoDB, it is necessary to remove and then insert, and rather than writing a method for this, it has been left to the user to tailor to their use case. 83 | * localStorage is a limited size, and if this is exceeded then *no* documents will be stored locally, with synchronisation required to populate the collection on the server. The `failCallback` will be fired in this case. 84 | 85 | ## Demo 86 | 87 | There is a demo deployed at [dumb-collections-demo.meteor.com](http://dumb-collections-demo.meteor.com), with the code available at [github.com/richsilv/dumb-collections-demo](https://github.com/richsilv/dumb-collections-demo). 88 | -------------------------------------------------------------------------------- /minimongo-bulk.js: -------------------------------------------------------------------------------- 1 | // ADAPTED FROM FROZEMAN: https://gist.github.com/frozeman/88a3e47679dd74242cab 2 | /* 3 | These two functions provide a simple extra minimongo functionality to add and remove documents in bulk, 4 | without unnecessary re-renders. 5 | 6 | The process is simple. It will add the documents to the collections "_map" and only the last item will be inserted properly 7 | to cause reactive dependencies to re-run. 8 | */ 9 | 10 | 11 | Models = {}; 12 | 13 | /* 14 | Inserts documents in bulk. 15 | */ 16 | Models.insertBulk = function(collection, documents){ 17 | if(collection) { 18 | var last = _.last(documents); 19 | _.each(documents, function(item){ 20 | if(_.isObject(item)) { 21 | if (item._id === last._id) 22 | collection.insert(item); 23 | else 24 | collection._collection._docs._map[item._id] = item; 25 | } 26 | }); 27 | } 28 | }, 29 | 30 | /* 31 | Removes documents in bulk. 32 | */ 33 | Models.removeBulk = function(collection, ids){ 34 | var _this = this; 35 | 36 | if (collection) { 37 | var lastId = _.last(ids); 38 | _.each(ids, function(id){ 39 | if (id === lastId) 40 | collection.remove({_id: id}); 41 | else 42 | delete collection._collection._docs._map[id]; 43 | }); 44 | } 45 | }; 46 | 47 | /* 48 | Removes all documents in a collection. 49 | */ 50 | Models.removeAll = function(collection){ 51 | var _this = this; 52 | 53 | if (collection) { 54 | var exampleId = Object.keys(collection._collection._docs._map)[0]; 55 | var newObj = {}; 56 | if (exampleId) newObj[exampleId] = collection._collection._docs._map[exampleId]; 57 | collection._collection._docs._map = newObj; 58 | if (exampleId) collection.remove({_id: exampleId}); 59 | } 60 | }; -------------------------------------------------------------------------------- /package.js: -------------------------------------------------------------------------------- 1 | Package.describe({ 2 | name: 'richsilv:dumb-collections', 3 | version: '1.1.5', 4 | summary: 'Meteor Collections which only sync with the server on request and are saved in local storage.', 5 | git: 'https://github.com/richsilv/meteor-dumb-collections.git' 6 | }); 7 | 8 | Package.on_use(function (api) { 9 | /* Use or imply other packages. 10 | 11 | * Example: 12 | * api.use('ui', 'client'); 13 | * api.use('iron-router', ['client', 'server']); 14 | */ 15 | 16 | /* 17 | * Add files that should be used with this 18 | * package. 19 | */ 20 | api.use('amplify@1.0.0', 'client'); 21 | api.use('mongo@1.0.7', ['client', 'server']); 22 | api.use('underscore@1.0.1', ['client', 'server']); 23 | api.use('tracker@1.0.3', ['client']); 24 | api.use('reactive-var@1.0.3', ['client']); 25 | 26 | api.add_files('minimongo-bulk.js', ['client']); 27 | api.add_files('richsilv:dumb-collections.js', ['client', 'server']); 28 | 29 | /* 30 | * Export global symbols. 31 | * 32 | * Example: 33 | * api.export('GlobalSymbol'); 34 | */ 35 | api.export('DumbCollection'); 36 | }); 37 | 38 | Package.on_test(function (api) { 39 | api.use('richsilv:dumb-collections'); 40 | api.use('tinytest'); 41 | 42 | api.add_files('richsilv:dumb-collections_tests.js'); 43 | }); 44 | -------------------------------------------------------------------------------- /richsilv:dumb-collections.js: -------------------------------------------------------------------------------- 1 | if (Meteor.isServer) { 2 | 3 | var collections = {}; 4 | 5 | DumbCollection = function(name, options) { 6 | 7 | var newCollection = new Mongo.Collection(name, options), 8 | _this = this; 9 | 10 | newCollection._update = newCollection.update; 11 | newCollection.update = function() { 12 | throw new Meteor.Error(500, "Please do not use update on a Dumb Collection on the server - remove and reinsert docs instead."); 13 | } 14 | 15 | collections[name] = newCollection; 16 | 17 | return newCollection; 18 | 19 | } 20 | 21 | Meteor.methods({ 22 | 23 | dumbCollectionGetNew: function(existing, name, query, options) { 24 | 25 | return collections[name].find(_.extend(query || {}, { 26 | _id: { 27 | $nin: existing 28 | } 29 | }), options || {}).fetch(); 30 | 31 | }, 32 | 33 | dumbCollectionGetRemoved: function(existing, name, query) { 34 | 35 | var currentIds = {}; 36 | 37 | collections[name].find(query || {}, { 38 | fields: { 39 | _id: true 40 | } 41 | }).forEach(function(doc) { 42 | currentIds[doc._id] = true; 43 | }); 44 | 45 | this.unblock(); 46 | 47 | var missingIds = existing.filter(function(docId){ 48 | return !(docId in currentIds); 49 | }); 50 | 51 | return missingIds || []; 52 | 53 | } 54 | 55 | }); 56 | 57 | } else if (Meteor.isClient) { 58 | 59 | DumbCollection = function(name, options) { 60 | 61 | var coll = new Mongo.Collection(null, options); 62 | 63 | var existingDocs = amplify.store('dumbCollection_' + name) || []; 64 | 65 | coll.name = name; 66 | coll.syncing = false; 67 | coll._readyFlag = new ReactiveVar(false); 68 | coll._syncFlag = new ReactiveVar(false); 69 | 70 | Models.insertBulk(coll, existingDocs); 71 | coll._readyFlag.set(true); 72 | console.log("Dumb Collection " + name + " seeded with " + existingDocs.length.toString() + " docs from local storage."); 73 | 74 | 75 | coll.sync = function(options) { 76 | 77 | options = options || {}; 78 | 79 | if (coll.syncing) throw new Meteor.Error('already_syncing', 'Cannot sync whilst already syncing'); 80 | 81 | var jobsComplete = { 82 | remove: options.retain, 83 | insert: options.reject 84 | }, 85 | completionDep = new Deps.Dependency(), 86 | results = {}, 87 | currentIds = []; 88 | 89 | coll._syncFlag.set(false); 90 | 91 | Tracker.autorun(function(outerComp) { 92 | 93 | if (coll.ready() && !coll.syncing) { 94 | 95 | coll.syncing = true; 96 | 97 | currentIds = _.pluck(coll.find({}, { 98 | reactive: false, 99 | fields: { 100 | _id: 1 101 | } 102 | }).fetch(), '_id'); 103 | 104 | if (!options.retain) { 105 | Meteor.call('dumbCollectionGetRemoved', currentIds, coll.name, options.query, function(err, res) { 106 | Models.removeBulk(coll, res); 107 | results.removed = res; 108 | jobsComplete.remove = true; 109 | completionDep.changed(); 110 | options.removalCallback && options.removalCallback.call(coll, removed); 111 | }); 112 | } else jobsComplete.remove = true; 113 | 114 | if (!options.reject) { 115 | Meteor.call('dumbCollectionGetNew', currentIds, coll.name, options.query, options.options, function(err, res) { 116 | results.inserted = res; 117 | Models.insertBulk(coll, res); 118 | jobsComplete.insert = true; 119 | completionDep.changed(); 120 | options.insertionCallback && options.insertionCallback.call(coll, res); 121 | }); 122 | } else jobsComplete.insert = true; 123 | 124 | Tracker.autorun(function(innerComp) { 125 | 126 | completionDep.depend(); 127 | 128 | if (jobsComplete.remove && jobsComplete.insert) { 129 | 130 | innerComp.stop() 131 | outerComp.stop(); 132 | coll._syncFlag.set(true); 133 | coll.syncing = false; 134 | 135 | var syncedCollection = coll.find().fetch(); 136 | try { 137 | amplify.store('dumbCollection_' + coll.name, syncedCollection); 138 | } 139 | catch (e) { 140 | console.log("Collection cannot be stored in Local Storage."); 141 | options.failCallback && options.failCallback.call(coll, e); 142 | } 143 | finally { 144 | console.log("Dumb Collection " + coll.name + " now has " + syncedCollection.length + " documents stored locally."); 145 | options.syncCallback && options.syncCallback.call(coll, results); 146 | } 147 | } 148 | 149 | }); 150 | 151 | } 152 | 153 | }); 154 | 155 | }; 156 | 157 | coll.clear = function(reactive) { 158 | 159 | Models.removeAll(coll); 160 | amplify.store('dumbCollection_' + coll.name, []); 161 | if (reactive) { 162 | coll._syncFlag.set(false); 163 | } else { 164 | coll._syncFlag.curValue = false; 165 | } 166 | }; 167 | 168 | coll.ready = function() { 169 | 170 | return coll._readyFlag.get(); 171 | 172 | }; 173 | 174 | coll.synced = function() { 175 | 176 | return coll._syncFlag.get(); 177 | 178 | }; 179 | 180 | coll.ironRouterReady = function() { 181 | 182 | return { 183 | ready: function() { 184 | return coll._syncFlag.get(); 185 | } 186 | } 187 | 188 | }; 189 | 190 | return coll; 191 | 192 | } 193 | 194 | } 195 | -------------------------------------------------------------------------------- /richsilv:dumb-collections_tests.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Test this package by running this command from you app 3 | * folder: 4 | * 5 | * > meteor test-packages richsilv:polledcollection 6 | * 7 | */ 8 | 9 | Tinytest.add('richsilv:dumb-collections - main test', function (test) { 10 | test.ok(); 11 | }); 12 | -------------------------------------------------------------------------------- /versions.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | [ 4 | "amplify", 5 | "1.0.0" 6 | ], 7 | [ 8 | "application-configuration", 9 | "1.0.3" 10 | ], 11 | [ 12 | "base64", 13 | "1.0.1" 14 | ], 15 | [ 16 | "binary-heap", 17 | "1.0.1" 18 | ], 19 | [ 20 | "callback-hook", 21 | "1.0.1" 22 | ], 23 | [ 24 | "check", 25 | "1.0.2" 26 | ], 27 | [ 28 | "ddp", 29 | "1.0.12" 30 | ], 31 | [ 32 | "ejson", 33 | "1.0.4" 34 | ], 35 | [ 36 | "follower-livedata", 37 | "1.0.2" 38 | ], 39 | [ 40 | "geojson-utils", 41 | "1.0.1" 42 | ], 43 | [ 44 | "id-map", 45 | "1.0.1" 46 | ], 47 | [ 48 | "jquery", 49 | "1.0.1" 50 | ], 51 | [ 52 | "json", 53 | "1.0.1" 54 | ], 55 | [ 56 | "logging", 57 | "1.0.5" 58 | ], 59 | [ 60 | "meteor", 61 | "1.1.3" 62 | ], 63 | [ 64 | "minimongo", 65 | "1.0.5" 66 | ], 67 | [ 68 | "mongo", 69 | "1.0.9" 70 | ], 71 | [ 72 | "ordered-dict", 73 | "1.0.1" 74 | ], 75 | [ 76 | "random", 77 | "1.0.1" 78 | ], 79 | [ 80 | "reactive-var", 81 | "1.0.3" 82 | ], 83 | [ 84 | "retry", 85 | "1.0.1" 86 | ], 87 | [ 88 | "tracker", 89 | "1.0.3" 90 | ], 91 | [ 92 | "underscore", 93 | "1.0.1" 94 | ] 95 | ], 96 | "pluginDependencies": [], 97 | "toolVersion": "meteor-tool@1.0.36", 98 | "format": "1.0" 99 | } --------------------------------------------------------------------------------