├── lib ├── export.js ├── init.coffee └── mongo2es.coffee ├── README.md ├── package.js └── tests └── test.coffee /lib/export.js: -------------------------------------------------------------------------------- 1 | Mongo2ES = this.Mongo2ES; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mongo2ES meteor package 2 | 3 | This is standalone package which you can use inside your meteor app, to sync MongoDB to ElasticSearch. 4 | 5 | Install this package with ```meteor add alino:mongo2es``` 6 | 7 | For more information about Mongo2ES visit https://github.com/Alino/Mongo2ES 8 | -------------------------------------------------------------------------------- /lib/init.coffee: -------------------------------------------------------------------------------- 1 | if Meteor.isServer 2 | Meteor.startup -> 3 | if not process.env.elasticsearchHost? and not Meteor.settings.elasticsearchHost? 4 | error = "elasticsearch host was not set, you must set it up!!!" 5 | log.error(error) 6 | throw new Meteor.Error(error) 7 | else 8 | Meteor.settings.elasticsearchHost = process.env.elasticsearchHost or Meteor.settings.elasticsearchHost 9 | 10 | if not process.env.logitHost? and not Meteor.settings.logit? 11 | log.warn "warning: logit host wasn't set. Logging into console only" 12 | else 13 | Meteor.settings.logit = 14 | host: process.env.logitHost or Meteor.settings.logit.host 15 | port: process.env.logitPort or Meteor.settings.logit.port 16 | 17 | Meteor.startup -> 18 | options = { host: Meteor.settings.elasticsearchHost } 19 | options.auth = Mongo2ES::getESAuth() 20 | ESresponse = Mongo2ES::getStatusForES(options) 21 | if ESresponse.statusCode isnt 200 22 | error = "bad ES response, exiting..." 23 | throw new Meteor.Error(error, null, ESresponse) 24 | log.error(error) 25 | else 26 | log.info 'connection with ElasticSearch successful' 27 | -------------------------------------------------------------------------------- /package.js: -------------------------------------------------------------------------------- 1 | Package.describe({ 2 | name: 'poetic:mongo2es', 3 | summary: 'create hooks for MongoDB collection to sync with ElasticSearch', 4 | version: '0.0.7', 5 | git: 'https://github.com/poetic/alino-mongo2es.git' 6 | }); 7 | 8 | Package.onUse(function (api) { 9 | api.versionsFrom(['METEOR@1.2']); 10 | 11 | var packages = [ 12 | "mongo@1.1.2", 13 | "http@1.1.1", 14 | "coffeescript", 15 | "underscore", 16 | "alino:logit@0.0.3" 17 | ]; 18 | 19 | api.use(packages); 20 | 21 | api.addFiles([ 22 | "lib/mongo2es.coffee", 23 | "lib/init.coffee", 24 | "lib/export.js" 25 | ], 'server'); 26 | 27 | api.export('Mongo2ES', 'server'); 28 | 29 | }); 30 | 31 | Package.onTest(function(api) { 32 | api.versionsFrom("METEOR@1.2"); 33 | api.use([ 34 | "mongo@1.1.2", 35 | "http@1.1.1", 36 | 'coffeescript', 37 | 'underscore', 38 | "alino:logit@0.0.3", 39 | 'sanjo:jasmine@0.20.2' 40 | ]); 41 | api.addFiles([ 42 | "lib/mongo2es.coffee", 43 | "lib/init.coffee", 44 | 'tests/test.coffee' 45 | ], 'server'); 46 | }); 47 | -------------------------------------------------------------------------------- /lib/mongo2es.coffee: -------------------------------------------------------------------------------- 1 | # TODO - consider using collection hooks instead of observe 2 | # https://atmospherejs.com/matb33/collection-hooks 3 | 4 | class @Mongo2ES 5 | constructor: (@options, @transform, @copyAlreadyExistingData = false) -> 6 | self = @ 7 | @verbose = self.options.verbose or process.env.MONGO2ES_VERBOSE? 8 | self.options.collectionName = self.getCollection(self.options.collectionName) 9 | self.watcher = self.options.collectionName.find().observe( 10 | added: (newDocument) -> 11 | if self.copyAlreadyExistingData 12 | if self.transform? and _.isFunction(self.transform) 13 | if self.transform(newDocument) is false then return 14 | else newDocument = self.transform(newDocument) 15 | self.options.ES.auth = getESAuth() 16 | self.addToES(self.options.collectionName, self.options.ES, newDocument) 17 | 18 | changed: (newDocument, oldDocument) -> 19 | if self.transform? and _.isFunction(self.transform) 20 | if self.transform(newDocument) is false then return 21 | else newDocument = self.transform(newDocument) 22 | self.options.ES.auth = getESAuth() 23 | self.updateToES(self.options.collectionName, self.options.ES, newDocument, oldDocument) 24 | 25 | removed: (oldDocument) -> 26 | if self.transform? and _.isFunction(self.transform) 27 | if self.transform() is false then return 28 | self.options.ES.auth = getESAuth() 29 | self.removeESdocument(self.options.ES, oldDocument._id) 30 | ) 31 | self.copyAlreadyExistingData = true 32 | 33 | getESAuth: -> 34 | return if Meteor.settings.elasticsearchAuth then Meteor.settings.elasticsearchAuth else process.env.elasticsearchAuth 35 | 36 | getCollection: (collectionName) -> 37 | if _.isString(collectionName) then return new Mongo.Collection collectionName 38 | else return collectionName 39 | 40 | stopWatch: -> 41 | self = @ 42 | self.watcher.stop() 43 | self.watcher 44 | 45 | getStatusForES: (ES) -> 46 | console.log "checking connectivity with ElasticSearch on #{ES.host}" 47 | options = { data: '/' } 48 | if ES.auth then options.auth = ES.auth 49 | try 50 | response = HTTP.get(ES.host, options) 51 | catch e 52 | log.error(e) 53 | return e 54 | return response 55 | 56 | addToES: (collectionName, ES, newDocument) -> 57 | log.info("adding doc #{newDocument._id} to ES") 58 | url = "#{ES.host}/#{ES.index}/#{ES.type}/#{newDocument._id}" 59 | query = _.omit(newDocument, '_id') 60 | options = { data: query } 61 | if ES.auth then options.auth = ES.auth 62 | if @verbose 63 | console.log url 64 | console.log query 65 | try 66 | response = HTTP.post(url, options) 67 | catch e 68 | log.error(e) 69 | return e 70 | return response 71 | 72 | updateToES: (collectionName, ES, newDocument, oldDocument) -> 73 | if newDocument._id isnt oldDocument._id 74 | log.info "document ID #{oldDocument._id} was changed to #{newDocument._id}" 75 | @removeESdocument(ES, oldDocument._id) 76 | log.info "updating doc #{newDocument._id} to ES" 77 | url = "#{ES.host}/#{ES.index}/#{ES.type}/#{newDocument._id}" 78 | query = _.omit(newDocument, '_id') 79 | options = { data: query } 80 | if ES.auth then options.auth = ES.auth 81 | if @verbose 82 | console.log url 83 | console.log query 84 | try 85 | response = HTTP.post(url, options) 86 | catch e 87 | log.error(e) 88 | return e 89 | return response 90 | 91 | removeESdocument: (ES, documentID) -> 92 | log.info "removing doc #{documentID} from ES" 93 | url = "#{ES.host}/#{ES.index}/#{ES.type}/#{documentID}" 94 | options = {} 95 | if ES.auth then options.auth = ES.auth 96 | if @verbose 97 | console.log url 98 | try 99 | response = HTTP.del(url, options) 100 | catch e 101 | log.error(e) 102 | return e 103 | return response 104 | -------------------------------------------------------------------------------- /tests/test.coffee: -------------------------------------------------------------------------------- 1 | describe 'Mongo2ES', -> 2 | testCollection1 = new Mongo.Collection('testCollection1') 3 | testCollection2 = new Mongo.Collection('testCollection2') 4 | testCollection3 = new Mongo.Collection('testCollection3') 5 | testCollection4 = new Mongo.Collection('testCollection4') 6 | testCollection5 = new Mongo.Collection('testCollection5') 7 | testCollection6 = new Mongo.Collection('testCollection6') 8 | testCollection7 = new Mongo.Collection('testCollection7') 9 | testCollection9 = new Mongo.Collection('testCollection9') 10 | testCollection11 = new Mongo.Collection('testCollection11') 11 | testCollection12 = new Mongo.Collection('testCollection12') 12 | testCollection13 = new Mongo.Collection('testCollection13') 13 | testCollection14 = new Mongo.Collection('testCollection14') 14 | testCollection15 = new Mongo.Collection('testCollection15') 15 | 16 | ESdefault = host: Meteor.settings.elasticsearchHost 17 | optionsDefault = 18 | collectionName: testCollection1 19 | ES: 20 | host: ESdefault.host 21 | index: 'admin' 22 | type: 'test' 23 | 24 | 25 | clearDB = -> 26 | testCollection1.remove({}) 27 | testCollection2.remove({}) 28 | testCollection3.remove({}) 29 | testCollection4.remove({}) 30 | testCollection5.remove({}) 31 | testCollection6.remove({}) 32 | testCollection7.remove({}) 33 | testCollection9.remove({}) 34 | testCollection11.remove({}) 35 | testCollection12.remove({}) 36 | testCollection13.remove({}) 37 | testCollection14.remove({}) 38 | testCollection15.remove({}) 39 | 40 | beforeEach -> 41 | clearDB() 42 | 43 | 44 | describe 'getCollection', -> 45 | it 'should return meteor collection if it receives string', -> 46 | options = optionsDefault 47 | options.collectionName = 'testCollection10' 48 | x = new Mongo2ES(optionsDefault) 49 | collection = x.getCollection('testCollection8') 50 | expect(collection._collection).toBeDefined() 51 | expect(collection._name).toBe 'testCollection8' 52 | 53 | it 'should return meteor collection if it receives meteor collection', -> 54 | options = optionsDefault 55 | options.collectionName = testCollection9 56 | x = new Mongo2ES(optionsDefault) 57 | collection = x.getCollection(testCollection9) 58 | expect(collection._collection).toBeDefined() 59 | expect(collection._name).toBe 'testCollection9' 60 | 61 | describe 'getStatusForES', -> 62 | it 'should return ES statusCode 200', -> 63 | x = new Mongo2ES(optionsDefault) 64 | response = x.getStatusForES(optionsDefault.ES) 65 | expect(response.statusCode).toBeDefined 66 | expect(response.statusCode).toBe 200 67 | 68 | it 'should return an error if ES host unreachable', -> 69 | options = 70 | collectionName: testCollection2 71 | ES: 72 | host: "http://192.168.666.666:9200" 73 | index: 'admin' 74 | type: 'test' 75 | x = new Mongo2ES(options) 76 | response = x.getStatusForES(options.ES) 77 | expect(response.error).toBeDefined 78 | 79 | describe 'addToES', -> 80 | it 'should save document to ES', (done) -> 81 | options = optionsDefault 82 | options.collectionName = testCollection3 83 | x = new Mongo2ES(options) 84 | testCollection3.find().observe( 85 | added: (newDocument) -> 86 | expect(newDocument._id).toBe 'kvak' 87 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('kvak')}" 88 | try 89 | result = HTTP.get(url) 90 | expect(result).toBeDefined() 91 | expect(result.data.found).toBe(true) 92 | expect(result.data._source).toEqual _.omit(newDocument, '_id') 93 | catch e 94 | console.error e 95 | expect(e).toBeUndefined() 96 | finally 97 | done() 98 | ) 99 | testCollection3.insert({ _id: 'kvak', query: 'mnau' }) 100 | 101 | 102 | it 'should save TRANSFORMED document to ES', (done) -> 103 | options = optionsDefault 104 | options.collectionName = testCollection7 105 | transform = (doc) -> 106 | doc.trans_query = "#{doc.query}_TRANSFORMED" 107 | return doc 108 | x = new Mongo2ES(options, transform) 109 | testCollection7.find().observe( 110 | added: (newDocument) -> 111 | expect(x.transform).toBeDefined() 112 | expect(newDocument._id).toBe 'kvak' 113 | expect(newDocument.query).toBe 'mnau' 114 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('kvak')}" 115 | try 116 | result = HTTP.get(url) 117 | expect(result).toBeDefined() 118 | expect(result.data.found).toBe(true) 119 | transformedDocument = newDocument 120 | transformedDocument.trans_query = 'mnau_TRANSFORMED' 121 | expect(result.data._source).toEqual _.omit(transformedDocument, '_id') 122 | catch e 123 | console.error e 124 | expect(e).toBeUndefined() 125 | finally 126 | done() 127 | ) 128 | testCollection7.insert({ _id: 'kvak', query: 'mnau' }) 129 | 130 | it 'should skip creating the document in ES if transform function returns false', (done) -> 131 | options = optionsDefault 132 | options.collectionName = testCollection13 133 | transform = (doc) -> return false 134 | x = new Mongo2ES(options, transform) 135 | testCollection13.find().observe( 136 | added: (newDocument) -> 137 | expect(x.transform).toBeDefined() 138 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('neprujdesdal')}" 139 | try 140 | result = HTTP.get(url) 141 | expect(result).toBeUndefined() 142 | catch e 143 | expect(e).toBeDefined() 144 | finally 145 | done() 146 | ) 147 | testCollection13.insert({ _id: 'neprujdesdal', query: 'gandalf huli travu' }) 148 | 149 | it 'should copy already existing mongo data to ES if third parameter is true', (done) -> 150 | testCollection11.insert({ _id: 'toto tu uz bolo', query: 'mnau' }) 151 | options = optionsDefault 152 | options.collectionName = testCollection11 153 | x = new Mongo2ES(options, undefined, true) 154 | testCollection11.find().observe( 155 | added: (newDocument) -> 156 | expect(x.transform).toBeUndefined() 157 | expect(x.copyAlreadyExistingData).toBe true 158 | expect(newDocument._id).toBe 'toto tu uz bolo' 159 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('toto tu uz bolo')}" 160 | try 161 | result = HTTP.get(url) 162 | expect(result).toBeDefined() 163 | expect(result.data.found).toBe(true) 164 | expect(result.data._source).toEqual _.omit(newDocument, '_id') 165 | catch e 166 | console.error e 167 | expect(e).toBeUndefined() 168 | finally 169 | done() 170 | ) 171 | 172 | it 'should not copy already existing mongo data to ES if third parameter is not defined', (done) -> 173 | testCollection12.insert({ _id: 'toto by tam nemalo byt', query: 'kvak' }) 174 | options = optionsDefault 175 | options.collectionName = testCollection12 176 | x = new Mongo2ES(options) 177 | testCollection12.find().observe( 178 | added: (newDocument) -> 179 | expect(x.transform).toBeUndefined() 180 | expect().toBeUndefined() 181 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('toto by tam nemalo byt')}" 182 | try 183 | result = HTTP.get(url) 184 | expect(result).toBeUndefined() 185 | catch e 186 | expect(e).toBeDefined() 187 | finally 188 | done() 189 | ) 190 | 191 | describe 'updateToES', -> 192 | it 'should update document in ES', (done) -> 193 | options = optionsDefault 194 | options.collectionName = testCollection6 195 | x = new Mongo2ES(options) 196 | testCollection6.insert({ _id: "42", query: 'kvak' }) 197 | testCollection6.find().observe( 198 | changed: (newDocument, oldDocument) -> 199 | expect(newDocument._id).toBe '42' 200 | expect(newDocument.query).toBe 'mnau' 201 | expect(oldDocument._id).toBe '42' 202 | expect(oldDocument.query).toBe 'kvak' 203 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('42')}" 204 | try 205 | result = HTTP.get(url) 206 | expect(result).toBeDefined() 207 | expect(result.data._source).toEqual _.omit(newDocument, '_id') 208 | catch e 209 | expect(e).toBeUndefined() 210 | finally 211 | done() 212 | ) 213 | testCollection6.update({ _id: "42" }, { $set: { query: 'mnau' } }) 214 | 215 | it 'should skip updating the document in ES if transform function returns false', (done) -> 216 | options = optionsDefault 217 | options.collectionName = testCollection14 218 | testCollection14.insert({ _id: 'neprujdesdal_update', query: 'gandalf huli kravu' }) 219 | doc = testCollection14.findOne({ _id: 'neprujdesdal_update' }) 220 | Mongo2ES::addToES(testCollection14, options.ES, doc) 221 | transform = (doc) -> return false 222 | x = new Mongo2ES(options, transform) 223 | testCollection14.find().observe( 224 | changed: (newDocument, oldDocument) -> 225 | expect(x.transform).toBeDefined() 226 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('neprujdesdal_update')}" 227 | try 228 | result = HTTP.get(url) 229 | expect(oldDocument.query).toBe('gandalf huli kravu') 230 | expect(newDocument.query).toBe('gandalf huli travu') 231 | expect(result).toBeDefined() 232 | expect(result.data._source.query).toBe('gandalf huli kravu') 233 | catch e 234 | expect(e).toBeUndefined() 235 | finally 236 | done() 237 | ) 238 | testCollection14.update({ _id: 'neprujdesdal_update' }, { $set: { query: 'gandalf huli travu' } }) 239 | 240 | 241 | describe 'removeESdocument', -> 242 | it 'should remove document from ES', (done) -> 243 | options = optionsDefault 244 | options.collectionName = testCollection4 245 | x = new Mongo2ES(options) 246 | testCollection4.insert({ _id: 'kvak', query: 'mnau' }) 247 | testCollection4.find().observe( 248 | removed: (oldDocument) -> 249 | expect(oldDocument._id).toBe 'kvak' 250 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('kvak')}" 251 | try 252 | result = HTTP.get(url) 253 | expect(result).toBeUndefined() 254 | catch e 255 | expect(e).toBeDefined() 256 | finally 257 | done() 258 | ) 259 | testCollection4.remove({ _id: 'kvak', query: 'mnau' }) 260 | 261 | it 'should not remove document from ES if transform function returns false', (done) -> 262 | options = optionsDefault 263 | options.collectionName = testCollection15 264 | transform = (doc) -> return false 265 | x = new Mongo2ES(options, transform) 266 | testCollection15.insert({ _id: 'kvak', query: 'mnau' }) 267 | doc = testCollection15.findOne({ _id: 'kvak' }) 268 | Mongo2ES::addToES(testCollection15, options.ES, doc) 269 | testCollection15.find().observe( 270 | removed: (oldDocument) -> 271 | expect(oldDocument._id).toBe 'kvak' 272 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('kvak')}" 273 | try 274 | result = HTTP.get(url) 275 | expect(result).toBeDefined() 276 | expect(result.data._id).toBe('kvak') 277 | catch e 278 | expect(e).toBeUndefined() 279 | finally 280 | done() 281 | ) 282 | testCollection15.remove({ _id: 'kvak', query: 'mnau' }) 283 | 284 | describe 'stopWatch', -> 285 | it 'should stop watching the collection', (done) -> 286 | options = optionsDefault 287 | options.collectionName = testCollection5 288 | x = new Mongo2ES(options) 289 | watcher = x.stopWatch() 290 | expect(watcher._stopped).toBe true 291 | testCollection5.find().observe( 292 | added: (newDocument) -> 293 | expect(newDocument._id).toBe 'kvak' 294 | url = "#{x.options.ES.host}/#{x.options.ES.index}/#{x.options.ES.type}/#{encodeURI('kvak')}" 295 | try 296 | result = HTTP.get(url) 297 | expect(result).toBeUndefined() 298 | catch e 299 | expect(e).toBeDefined() 300 | finally 301 | done() 302 | ) 303 | testCollection5.insert({ _id: 'kvak', query: 'mnau' }) 304 | --------------------------------------------------------------------------------