├── .npm └── package │ ├── .gitignore │ ├── README │ └── npm-shrinkwrap.json ├── .versions ├── README.md ├── mysql-shadow.js ├── package.js ├── smart.json └── versions.json /.npm/package/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /.npm/package/README: -------------------------------------------------------------------------------- 1 | This directory and the files immediately inside it are automatically generated 2 | when you change this package's NPM dependencies. Commit the files in this 3 | directory (npm-shrinkwrap.json, .gitignore, and this README) to source control 4 | so that others run the same versions of sub-dependencies. 5 | 6 | You should NOT check in the node_modules directory that Meteor automatically 7 | creates; if you are using git, the .gitignore file tells git to ignore it. 8 | -------------------------------------------------------------------------------- /.npm/package/npm-shrinkwrap.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": { 3 | "mysql": { 4 | "version": "http://registry.npmjs.org/mysql/-/mysql-2.4.3.tgz", 5 | "dependencies": { 6 | "bignumber.js": { 7 | "version": "http://registry.npmjs.org/bignumber.js/-/bignumber.js-1.4.0.tgz" 8 | }, 9 | "readable-stream": { 10 | "version": "1.1.13", 11 | "dependencies": { 12 | "core-util-is": { 13 | "version": "1.0.1" 14 | }, 15 | "isarray": { 16 | "version": "http://registry.npmjs.org/isarray/-/isarray-0.0.1.tgz" 17 | }, 18 | "string_decoder": { 19 | "version": "0.10.31" 20 | }, 21 | "inherits": { 22 | "version": "2.0.1" 23 | } 24 | } 25 | }, 26 | "require-all": { 27 | "version": "http://registry.npmjs.org/require-all/-/require-all-0.0.8.tgz" 28 | } 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /.versions: -------------------------------------------------------------------------------- 1 | base64@1.0.3 2 | binary-heap@1.0.3 3 | callback-hook@1.0.3 4 | check@1.0.5 5 | ddp@1.1.0 6 | ejson@1.0.6 7 | geojson-utils@1.0.3 8 | id-map@1.0.3 9 | json@1.0.3 10 | logging@1.0.7 11 | matb33:collection-hooks@0.7.11 12 | meteor@1.1.6 13 | minimongo@1.0.8 14 | mongo@1.1.0 15 | ordered-dict@1.0.3 16 | perak:mysql-shadow@1.0.9 17 | random@1.0.3 18 | retry@1.0.3 19 | tracker@1.0.7 20 | underscore@1.0.3 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MySQL Shadow 2 | ============ 3 | 4 | This package allows you to clone MySQL database into your Mongo database. 5 | Then write your meteor application as you normally do - you are working with mongo database. 6 | Collection inserts/updates/deletes can be automaticaly catched server side, transformed into SQL statements and executed on MySQL server. 7 | 8 | 9 | Warning! 10 | -------- 11 | 12 | This is super early prototype so be careful with your MySQL databases: **Don't use this with production database!** 13 | 14 | 15 | Usage 16 | ===== 17 | 18 | Do something like this in your server startup code: 19 | 20 | ``` 21 | Meteor.startup(function() { 22 | 23 | var connectionName = "my_connection"; // Name your connection as you wish 24 | 25 | CreateMySQLConnection(connectionName, { 26 | host : "localhost", // MySQL host address or IP 27 | database : "database", // MySQL database name 28 | user : "root", // MySQL username 29 | password : "password" // MySQL password 30 | }); 31 | 32 | OpenMySQLConnection(connectionName, function(e) { 33 | if(e) { 34 | console.log("Error: " + e.code + " " + e.reason); 35 | return; 36 | } 37 | 38 | console.log("Connected. Initializing shadow..."); 39 | 40 | CreateMySQLShadow(connectionName, {}, function(e) { 41 | if(e) { 42 | console.log("Error: " + e.code + " " + e.reason); 43 | return; 44 | } 45 | 46 | console.log("Shadow initialized. Copying data to mongo..."); 47 | 48 | MySQLShadowSyncAll(connectionName, {}, function(e) { 49 | if(e) { 50 | console.log("Error: " + e.code + " " + e.reason); 51 | return; 52 | } 53 | 54 | // If you want changes to your collections to be automatically replicated back to MySQL do something like this: 55 | // MySQLShadowCollection(SomeCollection, connectionName, {}); 56 | 57 | console.log("Success."); 58 | }); 59 | }); 60 | }); 61 | 62 | ``` 63 | **That's it:** now your Mongo database contains collections with data from all tables found in MySQL database. For each MySQL table you have collection with the same name in your Mongo database. 64 | 65 | 66 | Declare collections as you normally do: 67 | 68 | ``` 69 | SomeCollection = new Meteor.Collection("customers"); 70 | ``` 71 | 72 | 73 | Replicate changes back to MySQL 74 | ------------------------------- 75 | 76 | All inserts/updates/deletes on imported collections can be automatically replicated back to MySQL database. 77 | 78 | To activate this, call `MySQLShadowCollection` for all collections that you want to replicate. 79 | You can do this only after shadow is already created, so best place is to put it into `CreateMySQLShadow` callback: 80 | 81 | ``` 82 | ... 83 | MySQLShadowCollection(SomeCollection, connectionName, {}); 84 | ... 85 | ``` 86 | 87 | Now, when you do `SomeCollection.insert(...)`, `SomeCollection.update(...)` or `SomeCollection.remove(...)`, server will execute SQL `INSERT`, `UPDATE` or `DELETE` statement in your MySQL database. 88 | 89 | 90 | TODO 91 | ==== 92 | 93 | - test it (!) 94 | 95 | - inserts, updates and deletes can affect other tables in MySQL database (via triggers). In that case affected records should be cloned back to mongo. 96 | 97 | - improve performance 98 | 99 | 100 | Version history 101 | =============== 102 | 103 | 1.0.10 104 | ------ 105 | 106 | - Fixed bug (crashing) 107 | 108 | 109 | 1.0.9 110 | ----- 111 | 112 | - Fixed bug (crashing) 113 | 114 | 115 | 1.0.8 116 | ----- 117 | 118 | - Fixed bug (crashing) 119 | 120 | 121 | 1.0.7 122 | ----- 123 | 124 | - Fixed bug with time part in dates (if date object's hour, minute or second was zero, time part was not written) 125 | 126 | 127 | 1.0.6 128 | ----- 129 | 130 | - If record exists in mongo but doesn't exists in mysql - it's not deleted anymore 131 | 132 | 133 | 1.0.5 134 | ----- 135 | 136 | - Fixed some minor bugs 137 | 138 | 139 | 1.0.4 140 | ----- 141 | 142 | - Fixed bug with call to MongoInternals.defaultRemoteCollectionDriver(). Thanks to xxronis. 143 | 144 | 145 | 1.0.3 146 | ----- 147 | 148 | - Improved SQL statement creation (removed squel, and now it works with dates) 149 | -------------------------------------------------------------------------------- /mysql-shadow.js: -------------------------------------------------------------------------------- 1 | var mysql = Npm.require("mysql"); 2 | var Fiber = Npm.require("fibers"); 3 | 4 | var MySQLConnections = {}; 5 | 6 | var db = null; 7 | 8 | CreateMySQLConnection = function(connectionName, options) { 9 | db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; 10 | 11 | var connection = mysql.createConnection(options); 12 | MySQLConnections[connectionName] = connection; 13 | return connection; 14 | }; 15 | 16 | OpenMySQLConnection = function(connectionName, callback) { 17 | var connection = MySQLConnections[connectionName]; 18 | if(!connection) { 19 | var err = new Meteor.Error(404, "Unknown connection \"" + connectionName + "\"."); 20 | if(callback) { 21 | callback(err); 22 | return; 23 | } else { 24 | throw err; 25 | } 26 | } 27 | 28 | connection.connect(function(err) { 29 | if(err) { 30 | if(callback) { 31 | callback(err); 32 | return; 33 | } else { 34 | throw err; 35 | } 36 | } 37 | 38 | if(callback) { 39 | callback(); 40 | } 41 | }); 42 | }; 43 | 44 | CloseMySQLConnection = function(connectionName, callback) { 45 | var connection = MySQLConnections[connectionName]; 46 | if(!connection) { 47 | var err = new Meteor.Error(404, "Unknown connection \"" + connectionName + "\"."); 48 | if(callback) { 49 | callback(err); 50 | return; 51 | } else { 52 | throw err; 53 | } 54 | } 55 | 56 | connection.end(); 57 | }; 58 | 59 | 60 | /* 61 | Shadow 62 | */ 63 | 64 | MySQLShadows = {}; 65 | 66 | CreateMySQLShadow = function(connectionName, options, callback) { 67 | var connection = MySQLConnections[connectionName]; 68 | if(!connection) { 69 | var err = new Meteor.Error(404, "Unknown connection \"" + connectionName + "\"."); 70 | if(callback) { 71 | callback(err); 72 | return; 73 | } else { 74 | throw err; 75 | } 76 | } 77 | 78 | var schemaName = connection.config.database; 79 | var showTablesQuery = "SELECT k.TABLE_NAME, k.COLUMN_NAME as PRIMARY_KEY FROM information_schema.table_constraints t LEFT JOIN information_schema.key_column_usage k USING(constraint_name,table_schema,table_name) WHERE t.constraint_type='PRIMARY KEY' AND t.table_schema='" + schemaName + "';"; 80 | 81 | var shadow = { 82 | tables: [] 83 | }; 84 | 85 | connection.query(showTablesQuery, function(e, rows) { 86 | if(e) { 87 | var err = new Meteor.Error(500, e.message); 88 | if(callback) { 89 | callback(err); 90 | return; 91 | } else { 92 | throw err; 93 | } 94 | } 95 | 96 | _.each(rows, function(row) { 97 | shadow.tables.push(row); 98 | }); 99 | 100 | MySQLShadows[connectionName] = shadow; 101 | 102 | if(callback) { 103 | callback(); 104 | } 105 | }); 106 | }; 107 | 108 | MySQLShadowSyncTable = function(connectionName, tableName, options, callback) { 109 | var connection = MySQLConnections[connectionName]; 110 | if(!connection) { 111 | var err = new Meteor.Error(404, "Unknown connection \"" + connectionName + "\"."); 112 | if(callback) { 113 | callback(err); 114 | return; 115 | } else { 116 | throw err; 117 | } 118 | } 119 | 120 | var shadow = MySQLShadows[connectionName]; 121 | if(!shadow) { 122 | var err = new Meteor.Error(404, "Unknown shadow \"" + connectionName + "\"."); 123 | if(callback) { 124 | callback(err); 125 | return; 126 | } else { 127 | throw err; 128 | } 129 | } 130 | 131 | var table = _.find(shadow.tables, function(table) { return table.TABLE_NAME == tableName; }); 132 | if(!table) { 133 | var err = new Meteor.Error(404, "Unknown table \"" + tableName + "\"."); 134 | if(callback) { 135 | callback(err); 136 | return; 137 | } else { 138 | throw err; 139 | } 140 | } 141 | 142 | var tableDataQuery = "SELECT * FROM " + table.TABLE_NAME + ";"; 143 | var tableData = []; 144 | connection.query(tableDataQuery, function(e, rows) { 145 | if(e) { 146 | var err = new Meteor.Error(500, e.message); 147 | if(callback) { 148 | callback(err); 149 | return; 150 | } else { 151 | throw err; 152 | } 153 | } 154 | 155 | var collection = db.collection(table.TABLE_NAME); 156 | 157 | // upsert complete table from mysql to mongo 158 | _.each(rows, function(mysqlRow) { 159 | var selector = {}; 160 | selector[table.PRIMARY_KEY] = mysqlRow[table.PRIMARY_KEY]; 161 | 162 | collection.findOne(selector, function(err, mongoRow) { 163 | if(err) { 164 | if(callback) { 165 | callback(err); 166 | return; 167 | } else { 168 | throw err; 169 | } 170 | } 171 | 172 | if(!mongoRow) { 173 | mysqlRow._id = Random.id(); 174 | collection.insert(mysqlRow, function(err, inserted) { 175 | if(err) { 176 | if(callback) { 177 | callback(err); 178 | return; 179 | } else { 180 | throw err; 181 | } 182 | } 183 | }); 184 | } else { 185 | collection.update(selector, mysqlRow, { upsert: false }, function(err) { 186 | if(err) { 187 | if(callback) { 188 | callback(err); 189 | return; 190 | } else { 191 | throw err; 192 | } 193 | } 194 | }); 195 | } 196 | }); 197 | }); 198 | 199 | // remove documents from mongo that not exists in mysql 200 | /* 201 | collection.find({}).each(function(e, doc) { 202 | if(doc) { 203 | var mysqlRow = _.find(rows, function(row) { return row[table.PRIMARY_KEY] == doc[table.PRIMARY_KEY]; }); 204 | if(!mysqlRow) { 205 | var selector = {}; 206 | selector[table.PRIMARY_KEY] = doc[table.PRIMARY_KEY]; 207 | collection.remove(selector, function(err) { 208 | if(err) { 209 | if(callback) { 210 | callback(err); 211 | return; 212 | } else { 213 | throw err; 214 | } 215 | } 216 | }); 217 | } 218 | } else { 219 | if(err) { 220 | 221 | } 222 | } 223 | }); 224 | */ 225 | if(callback) { 226 | callback(); 227 | } 228 | }); 229 | } 230 | 231 | MySQLShadowSyncAll = function(connectionName, options, callback) { 232 | var shadow = MySQLShadows[connectionName]; 233 | if(!shadow) { 234 | var err = new Meteor.Error(404, "Unknown shadow \"" + connectionName + "\"."); 235 | if(callback) { 236 | callback(err); 237 | return; 238 | } else { 239 | throw err; 240 | } 241 | } 242 | 243 | _.each(shadow.tables, function(table) { 244 | MySQLShadowSyncTable(connectionName, table.TABLE_NAME, options, function(err) { 245 | if(err) { 246 | if(callback) { 247 | callback(err); 248 | return; 249 | } else { 250 | throw err; 251 | } 252 | } 253 | }); 254 | }); 255 | 256 | if(callback) { 257 | callback(); 258 | } 259 | }; 260 | 261 | function dateToMySQLDateLiteral(date) { 262 | var res = date.getFullYear() + "-" + (date.getMonth() + 1) + "-" + date.getDate(); 263 | 264 | if(date.getHours() != 0 || date.getMinutes() != 0 || date.getSeconds() != 0) { 265 | res = res + " " + date.getHours() + ":" + date.getMinutes() + ":" + date.getSeconds(); 266 | } 267 | return res; 268 | } 269 | 270 | MySQLShadowCollection = function(collection, connectionName, options, callback) { 271 | var tableName = collection._name; 272 | 273 | var connection = MySQLConnections[connectionName]; 274 | if(!connection) { 275 | var err = new Meteor.Error(404, "Unknown connection \"" + connectionName + "\"."); 276 | if(callback) { 277 | callback(err); 278 | return; 279 | } else { 280 | throw err; 281 | } 282 | } 283 | 284 | var shadow = MySQLShadows[connectionName]; 285 | if(!shadow) { 286 | var err = new Meteor.Error(404, "Unknown shadow \"" + connectionName + "\"."); 287 | if(callback) { 288 | callback(err); 289 | return; 290 | } else { 291 | throw err; 292 | } 293 | } 294 | 295 | var table = _.find(shadow.tables, function(table) { return table.TABLE_NAME == tableName; }); 296 | if(!table) { 297 | var err = new Meteor.Error(404, "Unknown table \"" + tableName + "\"."); 298 | if(callback) { 299 | callback(err); 300 | return; 301 | } else { 302 | throw err; 303 | } 304 | } 305 | 306 | // after insert hook 307 | collection.after.insert(function(userId, doc) { 308 | var row = JSON.parse(JSON.stringify(doc)); 309 | 310 | if(row._id) delete row._id; 311 | 312 | var sql = "INSERT INTO " + table.TABLE_NAME + " ("; 313 | var i = 0; 314 | for(var field in row) { 315 | if(!options || !options.skipFields || options.skipFields.indexOf(field) < 0) { 316 | if(i > 0) { 317 | sql = sql + ", "; 318 | } 319 | 320 | sql = sql + field; 321 | 322 | i++; 323 | } 324 | } 325 | sql = sql + ") VALUES ("; 326 | 327 | i = 0; 328 | for(var field in row) { 329 | if(!options || !options.skipFields || options.skipFields.indexOf(field) < 0) { 330 | if(i > 0) { 331 | sql = sql + ", "; 332 | } 333 | 334 | var value = doc[field]; 335 | if(_.isDate(value)) { 336 | sql = sql + "\'" + dateToMySQLDateLiteral(value) + "\'"; 337 | } else { 338 | sql = sql + connection.escape(value); 339 | } 340 | 341 | i++; 342 | } 343 | } 344 | sql = sql + ");"; 345 | 346 | // insert 347 | connection.query(sql, function(err, res) { 348 | if(err) { 349 | throw new Meteor.Error(500, err.message); 350 | } 351 | 352 | // read new record (record is maybe changed by triggers) 353 | newRecordSql = "SELECT * FROM " + table.TABLE_NAME + " WHERE " + table.PRIMARY_KEY + " = " + connection.escape(res.insertId) + ";"; 354 | connection.query(newRecordSql, function(err, rows) { 355 | if(err) { 356 | throw new Meteor.Error(500, err.message); 357 | } 358 | 359 | if(rows.length) { 360 | var newRow = rows[0]; 361 | Fiber(function() { 362 | // update newly inserted mongo doc 363 | collection.update({ _id: doc._id }, { $set: newRow }); 364 | }).run(); 365 | } 366 | 367 | // !!! refresh affected tables (by triggers) here 368 | 369 | }); 370 | }); 371 | }); 372 | 373 | // before update - check $set operator 374 | collection.before.update(function(userId, doc, fieldNames, modifier, options) { 375 | if(!modifier.$set) { 376 | throw new Meteor.Error(500, "MySQL collection must be updated using $set operator!"); 377 | } 378 | }); 379 | 380 | // after update hook 381 | collection.after.update(function(userId, doc, fieldNames, modifier, options) { 382 | if(modifier.$set[table.PRIMARY_KEY]) { 383 | // after.insert and after.update hooks will update entire record (including primary key). In this case we don't want to update record in mysql. 384 | // NOTE: maybe this is not a best solution (you cannot modify primary key / if you modify primary key record will not be replicated to mysql) 385 | return; 386 | } 387 | 388 | var sql = "UPDATE " + table.TABLE_NAME + " SET "; 389 | i = 0; 390 | for(var field in modifier.$set) { 391 | if(!options.skipFields || options.skipFields.indexOf(field) < 0) { 392 | if(i > 0) { 393 | sql = sql + ", "; 394 | } 395 | 396 | sql = sql + field + "="; 397 | 398 | var value = modifier.$set[field]; 399 | if(_.isDate(value)) { 400 | sql = sql + "\'" + dateToMySQLDateLiteral(value) + "\'"; 401 | } else { 402 | sql = sql + connection.escape(value); 403 | } 404 | 405 | i++; 406 | } 407 | } 408 | sql = sql + " WHERE " + table.PRIMARY_KEY + " = " + connection.escape(doc[table.PRIMARY_KEY]) + ";"; 409 | 410 | connection.query(sql, function(err) { 411 | if(err) { 412 | throw new Meteor.Error(500, err.message); 413 | } 414 | 415 | // !!! refresh affected tables (by triggers) here 416 | }); 417 | }); 418 | 419 | // after remove hook 420 | collection.after.remove(function(userId, doc) { 421 | var sql = "DELETE FROM " + table.TABLE_NAME + " WHERE " + table.PRIMARY_KEY + " = " + connection.escape(doc[table.PRIMARY_KEY]) + ";"; 422 | connection.query(sql, function(err) { 423 | if(err) { 424 | throw new Meteor.Error(500, err.message); 425 | } 426 | 427 | // !!! refresh affected tables (by triggers) here 428 | }); 429 | }); 430 | 431 | }; 432 | -------------------------------------------------------------------------------- /package.js: -------------------------------------------------------------------------------- 1 | Package.describe({ 2 | summary: "Clone MySQL database into Mongo and auto replicate collection changes back to MySQL", 3 | version: "1.0.10", 4 | git: "https://github.com/perak/mysql-shadow.git" 5 | }); 6 | 7 | // Before Meteor 0.9? 8 | if(!Package.onUse) Package.onUse = Package.on_use; 9 | 10 | Npm.depends({ 11 | mysql: "2.4.3" 12 | }); 13 | 14 | Package.onUse(function(api) { 15 | // Meteor >= 0.9? 16 | if(api.versionsFrom) api.versionsFrom("METEOR@0.9.0"); 17 | 18 | api.use("underscore"); 19 | api.use("matb33:collection-hooks@0.7.6"); 20 | 21 | api.add_files("mysql-shadow.js", "server"); 22 | 23 | api.export("CreateMySQLConnection", "server"); 24 | api.export("OpenMySQLConnection", "server"); 25 | api.export("CloseMySQLConnection", "server"); 26 | 27 | api.export("CreateMySQLShadow", "server"); 28 | api.export("MySQLShadowSyncTable", "server"); 29 | api.export("MySQLShadowSyncAll", "server"); 30 | api.export("MySQLShadowCollection", "server"); 31 | }); 32 | -------------------------------------------------------------------------------- /smart.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mysql-shadow", 3 | "description": "Clone MySQL database into Mongo and auto replicate collection changes back to MySQL", 4 | "homepage": "https://github.com/perak/mysql-shadow", 5 | "author": "Petar Korponaic", 6 | "version": "1.0.10", 7 | "git": "https://github.com/perak/mysql-shadow.git", 8 | "packages": {} 9 | } -------------------------------------------------------------------------------- /versions.json: -------------------------------------------------------------------------------- 1 | { 2 | "dependencies": [ 3 | [ 4 | "application-configuration", 5 | "1.0.2" 6 | ], 7 | [ 8 | "base64", 9 | "1.0.0" 10 | ], 11 | [ 12 | "binary-heap", 13 | "1.0.0" 14 | ], 15 | [ 16 | "callback-hook", 17 | "1.0.0" 18 | ], 19 | [ 20 | "check", 21 | "1.0.1" 22 | ], 23 | [ 24 | "ddp", 25 | "1.0.9" 26 | ], 27 | [ 28 | "ejson", 29 | "1.0.3" 30 | ], 31 | [ 32 | "follower-livedata", 33 | "1.0.1" 34 | ], 35 | [ 36 | "geojson-utils", 37 | "1.0.0" 38 | ], 39 | [ 40 | "id-map", 41 | "1.0.0" 42 | ], 43 | [ 44 | "json", 45 | "1.0.0" 46 | ], 47 | [ 48 | "logging", 49 | "1.0.3" 50 | ], 51 | [ 52 | "matb33:collection-hooks", 53 | "0.7.6" 54 | ], 55 | [ 56 | "meteor", 57 | "1.1.1" 58 | ], 59 | [ 60 | "minimongo", 61 | "1.0.3" 62 | ], 63 | [ 64 | "mongo", 65 | "1.0.6" 66 | ], 67 | [ 68 | "ordered-dict", 69 | "1.0.0" 70 | ], 71 | [ 72 | "random", 73 | "1.0.0" 74 | ], 75 | [ 76 | "retry", 77 | "1.0.0" 78 | ], 79 | [ 80 | "tracker", 81 | "1.0.2" 82 | ], 83 | [ 84 | "underscore", 85 | "1.0.0" 86 | ] 87 | ], 88 | "pluginDependencies": [], 89 | "toolVersion": "meteor-tool@1.0.36", 90 | "format": "1.0" 91 | } --------------------------------------------------------------------------------