├── .gitignore ├── README.md ├── app.js ├── config.json ├── lib ├── mongo.js ├── mysql.js └── tailer.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | node_modules 14 | 15 | npm-debug.log 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ##MongoDB to MySQL Data Streaming 2 | 3 | Streaming data in MongoDB to MySQL database in realtime. Enable SQL query on 4 | data in NoSQL database. 5 | 6 | ## Configurations: 7 | 8 | 1. Update the mongodb configuration in `config.json` 9 | 10 | ```json 11 | { 12 | "service": "mycol001", 13 | "mongodb": { 14 | "host": "127.0.0.1", 15 | "port": 27017, 16 | "db": "test", 17 | "collection": "blog_posts" 18 | }, 19 | "mysql": { 20 | "host": "localhost", 21 | "user": "root", 22 | "password": "", 23 | "db": "test", 24 | "table": "blog_posts" 25 | }, 26 | "sync_fields": { // fields store in MySQL 27 | "_id": "int", // required field 28 | "field1": "int", 29 | "field2": "int", 30 | "field3": "string", 31 | "field4": "string" 32 | }, 33 | "transform" : { 34 | "field_name" : { // field name changes 35 | "order": "_order" 36 | }, 37 | "field_value" : { 38 | "cid": "transform_cid", // field value changes, need function 39 | } 40 | } 41 | } 42 | ``` 43 | 44 | 2. Add index manually, for example: 45 | 46 | ALTER TABLE blog_posts ADD PRIMARY KEY (_id); 47 | 48 | ALTER TABLE blog_posts ADD INDEX field1 (field1); 49 | 50 | ALTER TABLE blog_posts ADD INDEX _order_id (_order, _id); 51 | 52 | 3. Import the old data in MongoDB collection to MySQL table: 53 | 54 | node app.js import 55 | 56 | 4. Start the daemon to streaming data 57 | 58 | node start app.js or forever start app.js 59 | 60 | 5. A MySQL table mongo_to_mysql will be created to store required information. 61 | 62 | 6. Update the transform() to change field names or modify values during streaming. 63 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var Mongo = require('./lib/mongo.js'), 2 | MySQL = require('./lib/mysql.js'), 3 | Tailer = require('./lib/tailer.js'), 4 | fs = require('fs'), 5 | config = JSON.parse(fs.readFileSync(process.cwd() + '/config.json')); 6 | 7 | //var heapdump = require('heapdump'); 8 | var app = {}; 9 | app.last_timestamp = 0; 10 | app.refresh = (process.argv[2] && process.argv[2] === 'import'); // import all data 11 | app.config = config; 12 | app.mongo = new Mongo(app); 13 | app.mysql = new MySQL(app, function () { 14 | app.tailer = new Tailer(app); 15 | if (app.refresh === true) { 16 | app.tailer.import(function () { 17 | app.tailer.start(); 18 | }); 19 | } else { 20 | app.tailer.start(); 21 | } 22 | }); -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "service": "mycol001", 3 | "mongodb": { 4 | "host": "127.0.0.1", 5 | "port": 27017, 6 | "db": "test", 7 | "collection": "blog_posts" 8 | }, 9 | "mysql": { 10 | "host": "localhost", 11 | "user": "root", 12 | "password": "", 13 | "db": "test", 14 | "table": "blog_posts" 15 | }, 16 | "sync_fields": { 17 | "_id": "int", 18 | "field1": "int", 19 | "field2": "string", 20 | "order": "int" 21 | }, 22 | "transform" : { 23 | "field_name" : { 24 | "order": "_order" 25 | }, 26 | "field_value" : { 27 | "cid": "transform_cid" 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lib/mongo.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | Db = require('mongodb').Db, 3 | Server = require('mongodb').Server, 4 | async = require('async'), 5 | _ = require('underscore'); 6 | 7 | module.exports = Mongo; 8 | 9 | function Mongo(app) { 10 | util.log('Connect to MongoDB...'); 11 | this.config = app.config; 12 | var server = new Server(this.config.mongodb.host, this.config.mongodb.port, { 13 | auto_reconnect: true 14 | }); 15 | this.db = new Db('local', server, { 16 | safe: true 17 | }); 18 | this.db.open(function (err, db) { 19 | if (err) throw err; 20 | }); 21 | this.db.on("close", function (error) { 22 | util.log("Connection to the database was closed!"); 23 | }); 24 | 25 | var server2 = new Server(this.config.mongodb.host, this.config.mongodb.port, { 26 | auto_reconnect: true 27 | }); 28 | 29 | this.db2 = new Db(this.config.mongodb.db, server2, { 30 | safe: true 31 | }); 32 | this.db2.open(function (err, db) { 33 | if (err) throw err; 34 | }); 35 | this.db2.on("close", function (error) { 36 | util.log("Connection to the database was closed!"); 37 | }); 38 | } -------------------------------------------------------------------------------- /lib/mysql.js: -------------------------------------------------------------------------------- 1 | var mysql = require('mysql'), 2 | _ = require('underscore'), 3 | util = require('util'); 4 | 5 | module.exports = MySQL; 6 | 7 | function MySQL(app, callback) { 8 | util.log('Connect to MySQL...'); 9 | var self = this; 10 | this.config = app.config; 11 | if (app.refresh) { 12 | this.create_table(function () { 13 | callback(); 14 | }); 15 | } else { 16 | this.read_timestamp(function () { 17 | callback(); 18 | }); 19 | } 20 | this.app = app; 21 | } 22 | 23 | MySQL.prototype.getConnection = function () { 24 | 25 | if (this.client && this.client._socket && this.client._socket.readable && this.client._socket.writable) { 26 | return this.client; 27 | } 28 | this.client = mysql.createConnection({ 29 | host: this.config.mysql.host, 30 | user: this.config.mysql.user, 31 | password: this.config.mysql.password, 32 | multipleStatements: true 33 | }); 34 | 35 | this.client.connect(function (err) { 36 | if (err) { 37 | util.log("SQL CONNECT ERROR: " + err); 38 | } else { 39 | util.log("SQL CONNECT SUCCESSFUL."); 40 | } 41 | }); 42 | 43 | this.client.on("close", function (err) { 44 | util.log("SQL CONNECTION CLOSED."); 45 | }); 46 | this.client.on("error", function (err) { 47 | util.log("SQL CONNECTION ERROR: " + err); 48 | }); 49 | 50 | this.client.query('USE ' + this.config.mysql.db); 51 | return this.client; 52 | }; 53 | 54 | MySQL.prototype.insert = function (item, callback) { 55 | 56 | item = transform(item); 57 | 58 | var self = this; 59 | var keys = _.keys(this.config.sync_fields); 60 | var fields = [], 61 | values = []; 62 | _.each(item, function (val, key) { 63 | if (_.contains(keys, key)) { 64 | fields.push(key); 65 | if (self.config.sync_fields[key] === 'string') { 66 | val = val || ''; 67 | val = val.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); 68 | values.push('"' + val + '"'); 69 | } else if (self.config.sync_fields[key] === 'int') { 70 | values.push(val || 0); 71 | } 72 | } 73 | }); 74 | var fields_str = fields.join(', '); 75 | var values_str = values.join(', '); 76 | var sql = 'INSERT INTO ' + this.config.mysql.table + ' (' + fields_str + ') VALUES (' + values_str + ');'; 77 | var conn = self.getConnection(); 78 | conn.query(sql, function (err, results) { 79 | if (err) { 80 | util.log(sql); 81 | throw err; 82 | } 83 | return callback(); 84 | }); 85 | }; 86 | 87 | MySQL.prototype.update = function (id, item, unset_items, callback) { 88 | if (item) { 89 | item = transform(item); 90 | } 91 | if (unset_items) { 92 | unset_items = transform(unset_items); 93 | } 94 | var self = this; 95 | var keys = _.keys(this.config.sync_fields); 96 | var sets = []; 97 | _.each(item, function (val, key) { 98 | if (_.contains(keys, key)) { 99 | if (self.config.sync_fields[key] === 'string') { 100 | val = val || ''; 101 | val = val.replace(/\\/g, '\\\\').replace(/"/g, '\\"'); 102 | sets.push(key + ' = "' + val + '"'); 103 | } else if (self.config.sync_fields[key] === 'int') { 104 | sets.push(key + ' = ' + (val || 0)); 105 | } 106 | } 107 | }); 108 | 109 | _.each(unset_items, function (val, key) { 110 | if (_.contains(keys, key)) { 111 | if (self.config.sync_fields[key] === 'string') { 112 | sets.push(key + ' = ""'); 113 | } else if (self.config.sync_fields[key] === 'int') { 114 | sets.push(key + ' = 0'); 115 | } 116 | } 117 | }); 118 | 119 | if (sets.length === 0) return; 120 | 121 | var sets_str = sets.join(', '); 122 | var sql; 123 | if (this.config.sync_fields['_id'] === 'int') { 124 | sql = 'UPDATE ' + this.config.mysql.table + ' SET ' + sets_str + ' WHERE _id = ' + id + ';'; 125 | } else { 126 | sql = 'UPDATE ' + this.config.mysql.table + ' SET ' + sets_str + ' WHERE _id = "' + id + '";'; 127 | } 128 | var conn = self.getConnection(); 129 | conn.query(sql, function (err, results) { 130 | if (err) { 131 | util.log(sql); 132 | throw err; 133 | } 134 | return callback(); 135 | }); 136 | }; 137 | 138 | MySQL.prototype.remove = function (id, callback) { 139 | var sql; 140 | if (this.config.sync_fields['_id'] === 'int') { 141 | sql = 'DELETE FROM ' + this.config.mysql.table + ' WHERE _id = ' + id + ';'; 142 | } else { 143 | sql = 'DELETE FROM ' + this.config.mysql.table + ' WHERE _id = \'' + id + '\';'; 144 | } 145 | var conn = this.getConnection(); 146 | conn.query(sql, function (err, results) { 147 | if (err) { 148 | util.log(sql); 149 | throw err; 150 | } 151 | return callback(); 152 | }); 153 | }; 154 | 155 | MySQL.prototype.create_table = function (callback) { 156 | var fields = []; 157 | _.each(this.config.sync_fields, function (val, key) { 158 | if (val === 'string') { 159 | fields.push(key + ' VARCHAR(1000)'); 160 | } else if (val === 'int') { 161 | fields.push(key + ' BIGINT'); 162 | } 163 | }); 164 | var fields_str = fields.join(', '); 165 | var sql = 'DROP TABLE IF EXISTS ' + this.config.mysql.table + '; ' + 'CREATE TABLE ' + this.config.mysql.table + ' (' + fields_str + ') ENGINE INNODB;'; 166 | var sql2 = 'DROP TABLE IF EXISTS mongo_to_mysql; ' + 'CREATE TABLE mongo_to_mysql (service varchar(20), timestamp BIGINT) ENGINE INNODB;'; 167 | var sql3 = 'INSERT INTO mongo_to_mysql (service, timestamp) VALUES ("' + this.config.service + '", 0);'; 168 | 169 | var conn = this.getConnection(); 170 | conn.query(sql, function (err, results) { 171 | if (err) { 172 | util.log(err); 173 | } 174 | conn.query(sql2, function (err, results) { 175 | conn.query(sql3, function (err, results) { 176 | callback(); 177 | }); 178 | }); 179 | }); 180 | }; 181 | 182 | MySQL.prototype.read_timestamp = function (callback) { 183 | var self = this; 184 | var conn = this.getConnection(); 185 | conn.query('SELECT timestamp FROM mongo_to_mysql WHERE service = "' + this.config.service + '"', function (err, results) { 186 | if (results && results[0]) { 187 | self.app.last_timestamp = results[0].timestamp; 188 | callback(); 189 | } 190 | }); 191 | }; 192 | 193 | MySQL.prototype.update_timestamp = function (timestamp) { 194 | var conn = this.getConnection(); 195 | conn.query('UPDATE mongo_to_mysql SET timestamp = ' + timestamp + ' WHERE service = \'' + this.config.service + '\';', function (err, results) {}); 196 | }; 197 | 198 | // field name, value type, value 199 | 200 | function transform(item) { 201 | if (item.cid) { 202 | item.cid = parseInt(item.cid.replace('c', ''), 10); 203 | } 204 | if (item.vid) { 205 | item.vid = parseInt(item.vid, 10); 206 | } 207 | if (item.order) { 208 | item._order = item.order; 209 | delete item.order; 210 | } 211 | return item; 212 | } -------------------------------------------------------------------------------- /lib/tailer.js: -------------------------------------------------------------------------------- 1 | var util = require('util'), 2 | mongo = require('mongodb'); 3 | 4 | module.exports = Tailer; 5 | 6 | function Tailer(app) { 7 | util.log('Starting Tailer...'); 8 | this.config = app.config; 9 | this.mongo = app.mongo; 10 | this.mysql = app.mysql; 11 | this.app = app; 12 | } 13 | 14 | Tailer.prototype.start = function () { 15 | this.tail(); 16 | }; 17 | 18 | Tailer.prototype.import = function (callback) { 19 | util.log('Begin to import...'); 20 | var self = this; 21 | var stream = this.mongo.db2.collection(self.config.mongodb.collection).find().stream(); 22 | stream.on("data", function (item) { 23 | stream.pause(); 24 | self.mysql.insert(item, function () { 25 | stream.resume(); 26 | }); 27 | }); 28 | stream.on('end', function (item) { 29 | util.log('Import Done.'); 30 | self.app.tailer.read_timestamp(function () { 31 | callback(); 32 | }); 33 | }); 34 | }; 35 | 36 | Tailer.prototype.read_timestamp = function (callback) { 37 | var self = this; 38 | var last = this.mongo.db.collection('oplog.$main').find().sort({ 39 | $natural: -1 40 | }).limit(1); 41 | last.nextObject(function (err, item) { 42 | var timestamp = item.ts.toNumber(); 43 | self.app.mysql.update_timestamp(timestamp); 44 | self.app.last_timestamp = timestamp; 45 | callback(); 46 | }); 47 | }; 48 | 49 | Tailer.prototype.tail = function () { 50 | var self = this; 51 | util.log('Last timestamp: ' + this.app.last_timestamp); 52 | //console.log(new mongo.Timestamp.fromNumber(5856968703085642000)); 53 | var options = { 54 | 'ns': self.config.mongodb.db + '.' + self.config.mongodb.collection, 55 | 'ts': { 56 | '$gt': new mongo.Timestamp.fromNumber(this.app.last_timestamp) 57 | } 58 | }; 59 | 60 | var stream = this.mongo.db.collection('oplog.$main').find(options, { 61 | tailable: true, 62 | awaitdata: true, 63 | numberOfRetries: -1 64 | }).stream(); 65 | 66 | stream.on('data', function (item) { 67 | if (item.op !== 'n' && item.ts.toNumber() !== self.app.last_timestamp) { 68 | //util.log(JSON.stringify(item)+'\r\n'); 69 | self.process(item, function () {}); 70 | } 71 | }); 72 | 73 | stream.on('close', function () { 74 | util.log("No more...."); 75 | }); 76 | }; 77 | 78 | Tailer.prototype.process = function (log, callback) { 79 | this.mysql.update_timestamp(log.ts.toNumber()); 80 | switch (log.op) { 81 | case 'i': 82 | this.mysql.insert(log.o, callback); 83 | break; 84 | case 'u': 85 | this.mysql.update(log.o2._id, log.o['$set'], log.o['$unset'], callback); 86 | break; 87 | case 'd': 88 | this.mysql.remove(log.o._id, callback); 89 | break; 90 | } 91 | }; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mongo2mysql", 3 | "version": "0.0.0", 4 | "description": "Streaming data from MongoDB into MySQL", 5 | "main": "app.js", 6 | "scripts": { 7 | "test": "" 8 | }, 9 | "dependencies": { 10 | "underscore": "=1.4.3", 11 | "forever": "=0.10.0", 12 | "async": "~0.2.7", 13 | "colors": "~0.6.0-1", 14 | "mongodb": "~3.6.1", 15 | "mysql": "~2.0.0-alpha7" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git://github.com/doubaokun/MongoDB-to-MySQL" 20 | }, 21 | "author": "Bruce Dou", 22 | "license": "MIT" 23 | } 24 | --------------------------------------------------------------------------------