├── .gitignore ├── LICENSE ├── README.md ├── index.js ├── lib ├── commands │ ├── hash.js │ ├── index.js │ ├── keys.js │ ├── list.js │ ├── mapping.js │ ├── server.js │ └── string.js ├── dbsync.js ├── rewriter │ ├── filerewriter.js │ └── rewriter.js ├── timer │ └── synctimer.js └── utils │ ├── constant.js │ ├── queue.js │ └── utils.js ├── package.json └── test ├── bench.js ├── lib └── mysql.js ├── mapping ├── bag.js └── player.js └── sync.js /.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | */node-log.log 3 | logs/*.log 4 | !.gitignore 5 | node_modules/* 6 | .project 7 | .settings/ 8 | **/*.svn 9 | *.svn 10 | *.sublime-project 11 | *.sublime-workspace 12 | *.swp 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2012 NetEase, Inc. and other pomelo contributors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #data-sync 2 | data sync module is simple sync memory data into store engine like mysql,redis,file. 3 | 4 | As we known, updating data is very frequently in game application. Especial in MMORPG kind game. User game data,such as location,flood,equipment,etc. almost always change as time going. For the purpose of avoid such update action cost, we decide to keep a copy data in memory. And keep synchronized with a timer and log; 5 | 6 | Data sync can support both timer call and instance invoke for the different 7 | situation. Most of time the developer don't pay attention to it; 8 | 9 | Data sync also can support memory operation like NOSQL database such as 10 | redis,mongodb etc. most of time developer can seem as a memory database without 11 | transaction. 12 | 13 | Data sync features include timer sync,set,get,mset,mget,hset,hget,incr,decr,flush,merger,showdown,info,etc. and the developer can extend it very easily. 14 | 15 | ##Installation 16 | ``` 17 | npm install pomelo-sync 18 | ``` 19 | 20 | ##Usage 21 | ``` javascript 22 | 23 | var opt = opt || {}; 24 | 25 | var updateUser = function(dbclient,val) { 26 | console.log('mock save %j',val); 27 | } 28 | 29 | var dbclient = {};//db connection etc; 30 | var id = 10001; 31 | var optKey = 'updateUser'; 32 | var mapping = {}; //key function mapping 33 | mapping[optKey]=updateUser; 34 | opt.mapping = mapping; 35 | opt.client = dbclient; 36 | opt.interval = 2000; 37 | 38 | var Sync = require('pomelo-sync'); 39 | var sync = new Sync(opt) ; 40 | sync.exec(optKey,id,{name:'hello'}); 41 | 42 | ``` 43 | 44 | ##API 45 | ###sync.exec(key,id,val,cb) 46 | Add a object to sync for timer exec call back. 47 | ####Arguments 48 | + key - the key function mapping for wanted to call back,it must be unique. 49 | + id - object primary key for merger operation. 50 | + val - the object wanted to synchronized. 51 | + cb - the function call back when timer exec. 52 | 53 | ###sync.flush(key,id,val,cb) 54 | immediately synchronized the memory data with out waiting timer and will remove 55 | waiting queue data; 56 | ####Arguments 57 | + key - the key function mapping for wanted to call back,it must be unique. 58 | + id - object primary key for merger operation. 59 | + val - the object wanted to synchronized. 60 | + cb - the function call back when timer exec. 61 | 62 | ###sync.isDone 63 | get the db sync status when the queue is empty,it should return true;otherwise 64 | return false; 65 | 66 | 67 | 68 | ##Notice 69 | system default sync time is 1000 * 60, 70 | if you use mysql or redis sync,you should set options.client,the file sync is default but it doesn't load in current. 71 | Mysql OR mapping in this modules do not support,user should realize it self. 72 | 73 | ##ADD 74 | for more usage detail , reading source and benchmark and test case from 75 | source is recommended; 76 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('./lib/dbsync'); -------------------------------------------------------------------------------- /lib/commands/hash.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | 5 | var utils = require('../utils/utils'); 6 | /** 7 | * HLEN 8 | */ 9 | exports.hlen = function(key){ 10 | var obj = this.lookup(key); 11 | if (!!obj && 'hash' == obj.type) { 12 | return Object.keys(obj.val).length; 13 | } else { 14 | return -1; 15 | } 16 | }; 17 | 18 | /** 19 | * HVALS 20 | */ 21 | 22 | exports.hvals = function(key){ 23 | var obj = this.lookup(key); 24 | if (!!obj && 'hash' == obj.type) { 25 | return (obj.val); 26 | } else { 27 | return null; 28 | } 29 | }; 30 | 31 | /** 32 | * HKEYS 33 | */ 34 | 35 | exports.hkeys = function(key){ 36 | var obj = this.lookup(key); 37 | if (!!obj && 'hash' == obj.type) { 38 | return Object.keys(obj.val); 39 | } else { 40 | return null; 41 | } 42 | }; 43 | 44 | /** 45 | * HSET 46 | */ 47 | 48 | (exports.hset = function(key, field, val){ 49 | var obj = this.lookup(key); 50 | 51 | if (obj && 'hash' != obj.type) { return false;} 52 | obj = obj || (this.db.data[key] = { type: 'hash', val: {} }); 53 | 54 | obj.val[field] = val; 55 | 56 | return true; 57 | 58 | }).mutates = true; 59 | 60 | /** 61 | * HMSET ( )+ 62 | */ 63 | 64 | (exports.hmset = function(data){ 65 | var len = data.length , key = data[0] , obj = this.lookup(key) , field , val; 66 | if (obj && 'hash' != obj.type) { return false;} 67 | obj = obj || (this.db.data[key] = { type: 'hash', val: {} }); 68 | var i = 1; 69 | for (i = 1; i < len; ++i) { 70 | field = data[i++]; 71 | val = data[i]; 72 | obj.val[field] = val; 73 | } 74 | return true; 75 | 76 | }).mutates = true; 77 | 78 | exports.hmset.multiple = 2; 79 | exports.hmset.skip = 1; 80 | 81 | /** 82 | * HGET 83 | */ 84 | 85 | exports.hget = function(key, field){ 86 | var obj = this.lookup(key) , val; 87 | if (!obj) { 88 | return null; 89 | } else if ('hash' == obj.type) { 90 | return obj.val[field] || null; 91 | } else { 92 | return null; 93 | } 94 | }; 95 | 96 | /** 97 | * HGETALL 98 | */ 99 | 100 | exports.hgetall = function(key){ 101 | var obj = this.lookup(key); 102 | var list = []; 103 | var field; 104 | if (!!obj && 'hash' == obj.type) { 105 | for (field in obj.val) { 106 | list.push(field, obj.val[field]); 107 | } 108 | return list; 109 | } else { 110 | return null; 111 | } 112 | }; 113 | 114 | /** 115 | * HEXISTS 116 | */ 117 | 118 | exports.hexists = function(key, field){ 119 | var obj = this.lookup(key); 120 | if (!!obj && 'hash' == obj.type) { 121 | var result = (hfield in obj.val); 122 | return result; 123 | } else { 124 | return false; 125 | } 126 | }; 127 | -------------------------------------------------------------------------------- /lib/commands/index.js: -------------------------------------------------------------------------------- 1 | (function merge(type){ 2 | var cmds = require('./' + type); 3 | for (var cmd in cmds) { 4 | exports[cmd] = cmds[cmd]; 5 | } 6 | return merge; 7 | })('keys')('string')('list')('hash')('server')('mapping'); 8 | -------------------------------------------------------------------------------- /lib/commands/keys.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | var utils = require('../utils/utils'); 5 | 6 | /** 7 | * EXPIRE 8 | */ 9 | 10 | exports.expire = function(key, seconds){ 11 | var obj = this.lookup(key); 12 | if (!!obj) { 13 | obj.expires = Date.now() + (seconds * 1000); 14 | return true; 15 | } else { 16 | return false; 17 | } 18 | }; 19 | 20 | /** 21 | * EXPIREAT 22 | */ 23 | 24 | exports.expireat = function(key, seconds){ 25 | var obj = this.lookup(key); 26 | if (!!obj) { 27 | obj.expires = +seconds * 1000; 28 | return true; 29 | } else { 30 | return false; 31 | } 32 | }; 33 | 34 | 35 | (exports.del = function(key){ 36 | if (this.lookup(key)) { 37 | delete this.db.data[key]; 38 | return true; 39 | } else { 40 | return false; 41 | } 42 | }).mutates = true; 43 | 44 | /** 45 | * PERSIST 46 | */ 47 | 48 | exports.persist = function(key){ 49 | var obj = this.lookup(key); 50 | if (!!obj && 'number' == typeof obj.expires) { 51 | delete obj.expires; 52 | return true; 53 | } else { 54 | return false; 55 | } 56 | }; 57 | 58 | /** 59 | * TTL 60 | */ 61 | 62 | exports.ttl = function(key){ 63 | var obj = this.lookup(key); 64 | if (!!obj && 'number' == typeof obj.expires) { 65 | return (Math.round((obj.expires - Date.now()) / 1000)); 66 | } else { 67 | return 0; 68 | } 69 | }; 70 | 71 | /** 72 | * TYPE 73 | */ 74 | 75 | exports.type = function(key){ 76 | var obj = this.lookup(key); 77 | if (!!obj) { 78 | return (obj.type); 79 | } else { 80 | return undefined; 81 | } 82 | }; 83 | 84 | /** 85 | * EXISTS 86 | */ 87 | 88 | exports.exists = function(key){ 89 | return this.lookup(key); 90 | }; 91 | 92 | /** 93 | * RANDOMKEY 94 | */ 95 | 96 | exports.randomkey = function(){ 97 | var keys = Object.keys(this.db.data); 98 | var len = keys.length; 99 | if (len) { 100 | var key = keys[Math.random() * len | 0]; 101 | return (key); 102 | } else { 103 | return null; 104 | } 105 | }; 106 | 107 | 108 | 109 | /** 110 | * RENAME 111 | */ 112 | 113 | (exports.rename = function(from, to){ 114 | var data = this.db.data; 115 | if (from == to) { throw Error('source and destination objects are the same');} 116 | // Fail if attempting to rename a non-existant key 117 | if (!this.lookup(from)) {throw Error('no such key');} 118 | // Map key val / key type 119 | var type = data[from].type; 120 | var obj = data[to] = data[from]; 121 | obj.type = type; 122 | delete data[from]; 123 | 124 | return true; 125 | }).mutates = true; 126 | 127 | /** 128 | * KEYS 129 | */ 130 | 131 | exports.keys = function(pattern){ 132 | var keys = Object.keys(this.db.data); 133 | var matched = []; 134 | 135 | // Optimize for common "*" 136 | if ('*' == pattern) { return keys;} 137 | 138 | // Convert pattern to regexp 139 | pattern = utils.parsePattern(pattern); 140 | 141 | // Filter 142 | for (var i = 0, len = keys.length; i < len; ++i) { 143 | if (pattern.test(keys[i])) { 144 | matched.push(keys[i]); 145 | } 146 | } 147 | 148 | return (matched); 149 | }; 150 | -------------------------------------------------------------------------------- /lib/commands/list.js: -------------------------------------------------------------------------------- 1 | var utils = require('../utils/utils'); 2 | 3 | /** 4 | * add val to list 5 | */ 6 | exports.sadd = function(key,val){ 7 | this.writeToAOF('sadd', [key,val]); 8 | var obj = this.lookup(key); 9 | if (!!obj) { 10 | obj.val.push(val); 11 | return true; 12 | } else { 13 | var list = []; 14 | list.push(val); 15 | this.set(key,list); 16 | return true; 17 | } 18 | }; 19 | /** 20 | * del from list 21 | */ 22 | exports.sdel = function(key,val){ 23 | this.writeToAOF('sdel', [key,val]); 24 | var obj = this.lookup(key); 25 | if (!!obj) { 26 | var index = obj.val.indexOf(val); 27 | if (index==-1) { 28 | return false; 29 | } else { 30 | delete obj.val[index]; 31 | return true; 32 | } 33 | } else { 34 | return false; 35 | } 36 | }; 37 | 38 | -------------------------------------------------------------------------------- /lib/commands/mapping.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var fs = require('fs'); 3 | var path = require('path'); 4 | 5 | /** 6 | * Auto-load bundled components with getters. 7 | * 8 | * @param {String} mappingPath 9 | * @return {Object} mapping 10 | */ 11 | exports.loadMapping = function(mappingPath) { 12 | var mapping = {}; 13 | var logger = this.log; 14 | var self = this; 15 | mappingPath+='/'; 16 | if (!!self.debug) { 17 | logger.info('[data sync compoment] load mapping file ' + mappingPath); 18 | } 19 | fs.readdirSync(mappingPath).forEach(function(filename){ 20 | if (!/\.js$/.test(filename)) {return;} 21 | var name = path.basename(filename, '.js'),key,pro; 22 | var fullPath = mappingPath + name; 23 | if (!!self.debug) { 24 | logger.log('loading ' + fullPath); 25 | } 26 | pro = require(fullPath); 27 | for (key in pro){ 28 | var fullKey = name+'.'+key; 29 | if (mapping[fullKey]){ 30 | logger.error('[data sync component] exist duplicated key map function ' + key + ' ignore it now.'); 31 | } else { 32 | mapping[fullKey] = pro[key]; 33 | } 34 | } 35 | }); 36 | logger.info('[data sync component] load mapping file done.' ); 37 | return mapping; 38 | }; 39 | 40 | 41 | -------------------------------------------------------------------------------- /lib/commands/server.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | var utils = require('../utils/utils'); 5 | var invoke = utils.invoke; 6 | var clone = utils.clone; 7 | /** 8 | * 9 | * invoke tick instant 10 | * 11 | * @module 12 | * 13 | * @param {String} key 14 | * @param {Object} val 15 | * @param {Function} cb 16 | * 17 | */ 18 | exports.execSync = function(key,val,cb){ 19 | this.rewriter.tick(key,val,cb); 20 | }; 21 | 22 | /** 23 | * exec add be synced data to queue 24 | * invoke by timer 25 | * 26 | * @module 27 | */ 28 | exports.exec = function(){ 29 | var mergerKey; 30 | switch (arguments.length) { 31 | case 2: 32 | this.enqueue(arguments[0],arguments[1]); 33 | break; 34 | case 3: 35 | case 4: 36 | mergerKey = [arguments[0],arguments[1]].join(''); 37 | this.mergerMap[mergerKey] = {key: arguments[0], val: clone(arguments[2]), cb: arguments[3]}; 38 | this.writeToAOF(arguments[0], [arguments[2]]); 39 | break; 40 | default: 41 | break; 42 | } 43 | }; 44 | 45 | /** 46 | * 47 | * enqueue data 48 | * 49 | * @param {String} key 50 | * @param {Object} val 51 | * 52 | */ 53 | exports.enqueue = function(key, val){ 54 | var target = clone(val); 55 | if (!!target) { 56 | this.writeToAOF(key, [val]); 57 | this.flushQueue.push({key:key,val:val}); 58 | } 59 | }; 60 | 61 | /** 62 | * flush all data go head 63 | */ 64 | exports.sync = function(){ 65 | if (this.rewriter) { 66 | this.rewriter.sync(this); 67 | } 68 | }; 69 | /** 70 | * reutrn queue is empty or not when shutdown server 71 | * 72 | * @module 73 | * 74 | */ 75 | exports.isDone = function(){ 76 | var writerEmpty = true,queueEmpty = false,mapEmpty = false; 77 | if (!!this.rewriter) { 78 | writerEmpty = this.rewriter.isDone(); 79 | } 80 | queueEmpty = (this.flushQueue.getLength()===0); 81 | mapEmpty = (utils.getMapLength(this.mergerMap)===0); 82 | return writerEmpty && queueEmpty && mapEmpty; 83 | }; 84 | 85 | /* 86 | * 87 | * flush single data to db 88 | * first remove from cache map 89 | */ 90 | exports.flush = function(){ 91 | var mergerKey; 92 | if (arguments.length>=3) { 93 | mergerKey = [arguments[0],arguments[1]].join(''); 94 | var exists = this.mergerMap[mergerKey]; 95 | if (!!exists) { 96 | this.writeToAOF([arguments[0],['_remove']].join(''),[exists]); 97 | delete this.mergerMap[mergerKey]; 98 | } 99 | this.writeToAOF(arguments[0], [arguments[2]]); 100 | return this.rewriter.flush(arguments[0], arguments[2], arguments[3]); 101 | } else { 102 | this.log.error('invaild arguments,flush must have at least 3 arguments'); 103 | return false; 104 | } 105 | }; 106 | 107 | /** 108 | * get dbsync info INFO 109 | * 110 | * 111 | */ 112 | exports.info = function(){ 113 | var buf = '' 114 | , day = 86400000 115 | , uptime = new Date - this.server.start; 116 | 117 | this.dbs.forEach(function(db, i){ 118 | var keys = Object.keys(db) 119 | , len = keys.length; 120 | if (len) { 121 | buf += 'db' + i + ':keys=' + len + ',expires=0\r\n'; 122 | } 123 | }); 124 | 125 | return (buf); 126 | }; 127 | 128 | 129 | -------------------------------------------------------------------------------- /lib/commands/string.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | 5 | var utils = require('../utils/utils'); 6 | /** 7 | * GET set value 8 | * 9 | * @param {String} key 10 | * 11 | */ 12 | exports.get = function(key){ 13 | var obj = this.lookup(key); 14 | if (!!obj) { 15 | return obj.val; 16 | } else { 17 | return null; 18 | } 19 | }; 20 | 21 | /** 22 | * GETSET 23 | * 24 | * @param {String} key 25 | * @param {String} val 26 | * 27 | */ 28 | exports.getset = function(key, val){ 29 | this.writeToAOF('getset', [key,val]); 30 | this.db.data[key] = { val: val }; 31 | 32 | return this.get(key); 33 | }; 34 | 35 | /** 36 | * SET db value by key 37 | * 38 | * @param {String} key 39 | * @param {Object} val 40 | */ 41 | 42 | (exports.set = function(key, val){ 43 | this.writeToAOF('set', [key,val]); 44 | this.db.data[key] = { val: val}; 45 | return true; 46 | }).mutates = true; 47 | 48 | 49 | /** 50 | * INCR counter 51 | * 52 | * @param {String} key 53 | */ 54 | 55 | (exports.incr = function(key){ 56 | var obj = this.lookup(key); 57 | 58 | if (!obj) { 59 | this.db.data[key] = {val: 1 }; 60 | return 1; 61 | } else { 62 | return ++obj.val; 63 | } 64 | }).mutates = true; 65 | 66 | /** 67 | * INCRBY counter with step 68 | * 69 | * @param {String} key 70 | * @param {number} num 71 | * 72 | */ 73 | (exports.incrby = function(key, num){ 74 | var obj = this.lookup(key); 75 | if (isNaN(num)) { throw new Error("TypeError");} 76 | if (!obj) { 77 | obj = this.db.data[key] = {val: num }; 78 | return (obj.val); 79 | } else { 80 | return (obj.val += num); 81 | } 82 | }).mutates = true; 83 | 84 | /** 85 | * DECRBY 86 | */ 87 | 88 | (exports.decrby = function(key, num){ 89 | var obj = this.lookup(key); 90 | if (isNaN(num)) { throw new Error(" TypoeError");} 91 | if (!obj) { 92 | obj = this.db.data[key] = {val: -num }; 93 | return (obj.val); 94 | } else { 95 | obj.val = obj.val-num; 96 | return obj.val; 97 | } 98 | }).mutates = true; 99 | 100 | /** 101 | * DECR 102 | */ 103 | 104 | (exports.decr = function(key){ 105 | var obj = this.lookup(key); 106 | 107 | if(!obj) { 108 | this.db.data[key] = { val: -1 }; 109 | return -1; 110 | } else { 111 | return --obj.val; 112 | } 113 | }).mutates = true; 114 | 115 | /** 116 | * STRLEN 117 | */ 118 | 119 | exports.strlen = function(key){ 120 | var val = this.lookup(key); 121 | if (val) { 122 | return val.length; 123 | } else { 124 | return 0; 125 | } 126 | }; 127 | 128 | /** 129 | * MGET + 130 | */ 131 | 132 | (exports.mget = function(keys){ 133 | var len = keys.length; 134 | var list = []; 135 | for (var i = 0; i < len; ++i) { 136 | var obj = this.lookup(keys[i]); 137 | list.push(obj); 138 | } 139 | return list; 140 | }).multiple = 1; 141 | 142 | /** 143 | * MSET ( )+ 144 | */ 145 | 146 | exports.mset = function(strs){ 147 | var len = strs.length 148 | , key 149 | , val; 150 | 151 | for (var i = 0; i < len; ++i) { 152 | key = strs[i++]; 153 | this.db.data[key] = { val: strs[i] }; 154 | } 155 | return true; 156 | }; 157 | 158 | exports.mset.multiple = 2; 159 | exports.mset.mutates = true; 160 | 161 | -------------------------------------------------------------------------------- /lib/dbsync.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | var commands = require('./commands'); 5 | var utils = require('./utils/utils'); 6 | var Queue = require('./utils/queue'); 7 | var fs = require('fs'); 8 | var crypto = require('crypto'); 9 | var Rewriter = require('../lib/rewriter/rewriter'); 10 | var SyncTimer = require('../lib/timer/synctimer'); 11 | 12 | /** 13 | * 14 | * DataSync Components. 15 | * 16 | * Initialize a new `DataSync` with the given `options`. 17 | * 18 | * DataSync's prototype is based on `commands` under the same directory; 19 | * 20 | * @class DataSync 21 | * @constructor 22 | * @param {Object} options init params include aof,log,interval,mapping and mappingPath etc. 23 | * 24 | */ 25 | var DataSync = module.exports = function(options) { 26 | options = options || {}; 27 | this.dbs = []; 28 | this.selectDB(0); 29 | this.client = options.client; 30 | this.aof = options.aof || false; 31 | this.debug = options.debug || false; 32 | this.log = options.log || console; 33 | this.interval = options.interval || 1000 * 60; 34 | this.flushQueue = new Queue(); 35 | this.mergerMap = {}; 36 | if (!!this.aof) { 37 | if (!!options.filename) { 38 | this.filename = options.filename; 39 | } else { 40 | var path = process.cwd() + '/logs'; 41 | fs.mkdirSync(path); 42 | this.filename = path+'/dbsync.log'; 43 | } 44 | this.stream = fs.createWriteStream(this.filename, { flags: 'a' }); 45 | } 46 | if (!!options.mapping){ 47 | this.mapping = options.mapping; 48 | } else if (!!options.mappingPath) { 49 | this.mapping = this.loadMapping(options.mappingPath); 50 | } 51 | this.rewriter = options.rewriter || new Rewriter(this); 52 | this.timer = options.timer || new SyncTimer(); 53 | this.timer.start(this); 54 | }; 55 | 56 | /** 57 | * Expose commands to store. 58 | */ 59 | DataSync.prototype = commands; 60 | 61 | /** 62 | * Select database at the given `index`. 63 | * @api private 64 | * @param {Number} index 65 | */ 66 | 67 | DataSync.prototype.selectDB = function(index){ 68 | var db = this.dbs[index]; 69 | if (!db) { 70 | db = {}; 71 | db.data = {}; 72 | this.dbs[index] = db; 73 | } 74 | this.db = db; 75 | }; 76 | 77 | /** 78 | *return the first used db 79 | * 80 | * @api private 81 | */ 82 | DataSync.prototype.use = function() { 83 | this.selectDB(0); 84 | var db = this.dbs[0]; 85 | var keys = Object.keys(db); 86 | var dbkey = keys[0]; 87 | return db[dbkey]; 88 | }; 89 | 90 | /** 91 | * Lookup `key`, when volatile compare timestamps to 92 | * expire the key. 93 | * 94 | * @api private 95 | * @param {String} key 96 | * @return {Object} 97 | */ 98 | 99 | DataSync.prototype.lookup = function(key){ 100 | var obj = this.db.data[key]; 101 | if (obj && 'number' == typeof obj.expires && Date.now() > obj.expires) { 102 | delete this.db.data[key]; 103 | return; 104 | } 105 | return obj; 106 | }; 107 | 108 | /** 109 | * Write the given `cmd`, and `args` to the AOF. 110 | * 111 | * @api private 112 | * @param {String} cmd 113 | * @param {Array} args 114 | */ 115 | 116 | DataSync.prototype.writeToAOF = function(cmd, args){ 117 | var self = this; 118 | if (!self.aof) {return;} 119 | 120 | var argc = args.length; 121 | var op = '*' + (argc + 1) + '\r\n' + cmd + '\r\n'; 122 | 123 | // Write head length 124 | this.stream.write(op); 125 | var i = 0; 126 | // Write Args 127 | for (i = 0; i < argc; ++i) { 128 | var key = utils.string(args[i]); 129 | this.stream.write(key); 130 | this.stream.write('\r\n'); 131 | } 132 | }; 133 | -------------------------------------------------------------------------------- /lib/rewriter/filerewriter.js: -------------------------------------------------------------------------------- 1 | 2 | var utils = require('../utils/utils'); 3 | var fs = require('fs'); 4 | 5 | /** 6 | * Initialize a new AOF FileRewriter with the given `db`. 7 | * 8 | * @param {options} 9 | */ 10 | 11 | var FileRewriter = module.exports = function FileRewriter(server) { 12 | var self = this; 13 | this.server = server; 14 | this.filename = process.cwd() + '/logs/dump.db'; //+ (Math.random() * 0xfffffff | 0); 15 | this.streams = fs.createWriteStream(this.filename,{ flags: 'w' }); 16 | this.filter = options.filter || null ; 17 | }; 18 | 19 | /** 20 | * Initiate sync. 21 | */ 22 | 23 | FileRewriter.prototype.sync = function(){ 24 | var server = this.server; 25 | var db = server.use(); 26 | for(var key in db){ 27 | if (!!this.filter){ 28 | if (!!!this.filter(key)) { continue ; } 29 | } 30 | var val = db[key]; 31 | if (!!server.mapping) { 32 | server.mapping(key, val);} 33 | else { 34 | this.aof(key,val);} 35 | } 36 | //server.queue.shiftEach(function(key){}); 37 | //this.end(); 38 | }; 39 | 40 | /** 41 | * Close tmpfile streams, and replace AOF 42 | * will our tempfile, then callback `fn(err)`. 43 | */ 44 | 45 | FileRewriter.prototype.end = function(fn){ 46 | this.streams.end(); 47 | }; 48 | 49 | /** 50 | * Write key / val. 51 | */ 52 | 53 | FileRewriter.prototype.aof = function(key, val){ 54 | var type = val.type || 'string'; 55 | return this[type](key, val); 56 | }; 57 | 58 | /** 59 | * Write string to `streams`. 60 | */ 61 | 62 | FileRewriter.prototype.string = function(key, val) { 63 | this.streams.write('$' + key.length + '\r\n'); 64 | this.streams.write(key); 65 | this.streams.write('\r\n'); 66 | this.streams.write(JSON.stringify(val)); 67 | this.streams.write('\r\n'); 68 | }; 69 | 70 | FileRewriter.prototype.hash = function(key, val) { 71 | this.string(key,val); 72 | }; 73 | -------------------------------------------------------------------------------- /lib/rewriter/rewriter.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | var utils = require('../utils/utils'); 5 | var invoke = utils.invoke; 6 | /** 7 | * Initialize a new AOF Rewriter with the given `db`. 8 | * 9 | * @param {options} 10 | * 11 | */ 12 | var Rewriter = module.exports = function Rewriter(server) { 13 | this.server = server; 14 | this.count = 0; 15 | }; 16 | 17 | /** 18 | * Initiate sync. 19 | */ 20 | 21 | Rewriter.prototype.sync = function(){ 22 | var self = this,server = self.server; 23 | server.flushQueue.shiftEach(function(element){ 24 | self.tick(element.key,element.val); 25 | }); 26 | var mergerMap = server.mergerMap; 27 | for (var mergerKey in mergerMap){ 28 | var entry = mergerMap[mergerKey]; 29 | self.tick(entry.key, entry.val, entry.cb); 30 | delete mergerMap[mergerKey]; 31 | } 32 | return true; 33 | }; 34 | 35 | /* 36 | * 37 | * flush db 38 | * 39 | */ 40 | Rewriter.prototype.flush = function(key, val, cb){ 41 | this.tick(key, val, cb); 42 | }; 43 | /* 44 | * 45 | * judge task is done 46 | * 47 | */ 48 | Rewriter.prototype.tick = function(key,val,cb){ 49 | var self = this,server = self.server; 50 | if (!server.client){ 51 | server.log.error('db sync client is null'); 52 | return ; 53 | } 54 | var syncb = server.mapping[key]; 55 | if (!syncb) { 56 | server.log.error(key + ' callback function not exist '); 57 | return; 58 | } 59 | if (!cb) { 60 | self.count+=1; 61 | return invoke(syncb,server.client,val,function(){self.count-=1;}); 62 | } else { 63 | invoke(syncb,server.client,val,cb); 64 | } 65 | }; 66 | /* 67 | * 68 | * judge task is done 69 | * 70 | */ 71 | Rewriter.prototype.isDone = function() { 72 | return this.count===0; 73 | }; 74 | -------------------------------------------------------------------------------- /lib/timer/synctimer.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Module dependencies. 3 | */ 4 | 5 | var SyncTimer = module.exports = function SyncTimer() { 6 | var self = this; 7 | }; 8 | 9 | /** 10 | * start sync timer . 11 | */ 12 | 13 | SyncTimer.prototype.start = function(db){ 14 | setInterval(function(){ 15 | //console.log('Background append only file rewriting started'); 16 | db.sync(); 17 | },db.interval); 18 | }; 19 | 20 | -------------------------------------------------------------------------------- /lib/utils/constant.js: -------------------------------------------------------------------------------- 1 | /** 2 | * type for error message 3 | */ 4 | var constant = module.exports = {}; 5 | 6 | constant.TypeError = "type error"; 7 | 8 | constant.OutOfRange = "out of range"; 9 | 10 | -------------------------------------------------------------------------------- /lib/utils/queue.js: -------------------------------------------------------------------------------- 1 | /** 2 | * queue for sync deleted data 3 | */ 4 | var Queue = function(){ 5 | this.tail = []; 6 | this.head = []; 7 | this.offset = 0; 8 | }; 9 | 10 | Queue.prototype.shift = function () { 11 | if (this.offset === this.head.length) { 12 | var tmp = this.head; 13 | tmp.length = 0; 14 | this.head = this.tail; 15 | this.tail = tmp; 16 | this.offset = 0; 17 | if (this.head.length === 0) { 18 | return; 19 | } 20 | } 21 | return this.head[this.offset++]; 22 | }; 23 | 24 | Queue.prototype.push = function (item) { 25 | return this.tail.push(item); 26 | }; 27 | 28 | Queue.prototype.forEach = function (fn) { 29 | var array = this.head.slice(this.offset), i, il; 30 | for (i = 0, il = array.length; i < il; i += 1) { 31 | fn(array[i]); 32 | } 33 | return array; 34 | }; 35 | 36 | Queue.prototype.shiftEach = function (fn) { 37 | while (this.tail.length>0) { 38 | var element = this.tail.shift(); 39 | fn(element); 40 | } 41 | }; 42 | 43 | Queue.prototype.getLength = function () { 44 | return this.head.length - this.offset + this.tail.length; 45 | }; 46 | 47 | Object.defineProperty(Queue.prototype, 'length', { 48 | get: function () { 49 | return this.getLength(); 50 | } 51 | }); 52 | 53 | module.exports = Queue; 54 | -------------------------------------------------------------------------------- /lib/utils/utils.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | /** 3 | * Convert object to a string. 4 | * 5 | * @param {object} buf 6 | * @return {String} 7 | */ 8 | 9 | exports.string = function(o) { 10 | try { 11 | return JSON.stringify(o);} 12 | catch(ex){ 13 | return util.inspect(o,true,100,true); 14 | } 15 | return o; 16 | }; 17 | 18 | /** 19 | * Parse a `pattern` and return a RegExp. 20 | * 21 | * @param {String} pattern 22 | * @return {RegExp} 23 | */ 24 | 25 | exports.parsePattern = function(pattern){ 26 | pattern = pattern 27 | .replace(/\*/g, '.*') 28 | .replace(/\?/g, '.'); 29 | return new RegExp('^' + pattern + '$'); 30 | }; 31 | 32 | /** 33 | * invoke callback function 34 | * @param cb 35 | */ 36 | exports.invoke = function(cb) { 37 | if(!!cb && typeof cb == 'function') { 38 | cb.apply(null, Array.prototype.slice.call(arguments, 1)); 39 | } 40 | }; 41 | 42 | 43 | 44 | /*** 45 | * clone new object 46 | * 47 | * @param {Object} obj; 48 | * 49 | */ 50 | exports.clone = function(obj){ 51 | if (obj === Object(obj)){ 52 | if (Object.prototype.toString.call(obj) == '[object Array]'){ 53 | return obj.slice(); 54 | } else { 55 | var ret = {}; 56 | Object.keys(obj).forEach(function (val) { 57 | ret[val] = obj[val]; 58 | }); 59 | return ret; 60 | } 61 | } else { 62 | return null; 63 | } 64 | }; 65 | 66 | /** 67 | *return the merge length 68 | */ 69 | exports.getMapLength = function(map){ 70 | var length = 0; 71 | for (var key in map) { 72 | length+=1; 73 | } 74 | return length; 75 | } 76 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pomelo-sync", 3 | "version": "0.0.4", 4 | "dependencies": { 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /test/bench.js: -------------------------------------------------------------------------------- 1 | // bench for data-snyc 2 | 3 | var MemDatabase = require('../lib/dbsync'); 4 | var db = null; 5 | var assert = require('assert'); 6 | 7 | var best = {writes: 0, hwrites:0, jwrites:0, reads: 0, hreads: 0, jreads: 0 } 8 | , avg = {writes: 0, writesCnt: 0, hwrites: 0, hwritesCnt: 0, jwrites: 0, jwritesCnt: 0 9 | , reads: 0, readsCnt: 0, hreads: 0, hreadsCnt: 0, jreads:0, jreadsCnt: 0 10 | } 11 | 12 | var big = [] 13 | for (var i=1000; i--; ) { 14 | big.push(i) 15 | } 16 | 17 | var objects = [ 18 | JSON.stringify('tiny') 19 | 20 | , JSON.stringify('hello I am a medium sized string') 21 | 22 | , JSON.stringify({ 23 | there: 'is' 24 | , only: 'chaos' 25 | , butterfly: ['says', ['there', 'is', 'only', 'chaos']] 26 | , pi: Math.PI 27 | }) 28 | 29 | , JSON.stringify({ 30 | there: 'is' 31 | , only: 'chaos' 32 | , butterfly: ['says', ['there', 'is', 'only', 'chaos']] 33 | , pi: Math.PI 34 | , big: big 35 | }) 36 | ] 37 | 38 | function bench(obj, what, num, cb) { 39 | console.log(' obj length:', obj.length) 40 | console.log(' operations:', num) 41 | console.log('-------------------') 42 | switch (what) { 43 | case 'all': 44 | sets(obj, num, function() { 45 | gets(obj, num, function() { 46 | hsets(obj, num, function() { 47 | hgets(obj, num, function() { 48 | console.log(''); 49 | cb(); 50 | }); 51 | }); 52 | }) 53 | }) 54 | break 55 | case 'sets': 56 | sets(obj, num, function() { 57 | console.log(''); 58 | cb(); 59 | }) 60 | break 61 | case 'gets': 62 | gets(obj, num, function() { 63 | console.log(''); 64 | cb(); 65 | }) 66 | break 67 | case 'hsets': 68 | hsets(obj, num, function() { 69 | console.log(''); 70 | cb(); 71 | }) 72 | break 73 | case 'hgets': 74 | hgets(obj, num, function() { 75 | console.log(''); 76 | cb(); 77 | }) 78 | break 79 | default: 80 | cb(); 81 | break 82 | } 83 | } 84 | 85 | function sets(obj, num, cb) { 86 | var done = 0 87 | , clients = 0 88 | , timer = new Date() 89 | for (var i=num; i--; ) { 90 | var res = db.set(i, obj); 91 | if (res) { 92 | done++ 93 | } 94 | if (done === num) { 95 | var result = ( (num) / ((new Date() - timer) / 1000)) 96 | if (result > best.writes) best.writes = result 97 | avg.writes += result 98 | avg.writesCnt += 1 99 | console.log('sets writes:', result.toFixed(2) + '/s') 100 | cb(); 101 | } 102 | } 103 | } 104 | 105 | function gets(obj, num, cb) { 106 | var done = 0 107 | , clients = 0 108 | , timer = new Date() 109 | 110 | for (var i=num; i--; ) { 111 | var data = db.get(i); 112 | if (!!data) { 113 | done++ 114 | } 115 | if (done === num) { 116 | var result = ( (num) / ((new Date() - timer) / 1000)) 117 | if (result > best.reads) best.reads = result 118 | avg.reads += result 119 | avg.readsCnt += 1 120 | console.log('gets reads:', result.toFixed(2) + '/s') 121 | cb(); 122 | } 123 | 124 | } 125 | } 126 | 127 | function hsets(obj, num, cb) { 128 | var done = 0 129 | , clients = 0 130 | , timer = new Date() 131 | 132 | for (var i=num; i--; ) { 133 | var res = db.hset('hkey','num',num) 134 | if (!!res) { 135 | done++ 136 | } 137 | if (done === num) { 138 | var result = ( (num) / ((new Date() - timer) / 1000)) 139 | if (result > best.hwrites) best.hwrites = result 140 | avg.hwrites += result 141 | avg.hwritesCnt += 1 142 | console.log('hkey writes:', result.toFixed(2) + '/s') 143 | cb() 144 | } 145 | } 146 | } 147 | 148 | function hgets(obj, num, cb) { 149 | var done = 0 150 | , clients = 0 151 | , timer = new Date() 152 | for (var i=num; i--; ) { 153 | var res = db.hget('hkey','num'); 154 | if (!!res) { 155 | done++ 156 | } 157 | if (done === num) { 158 | var result = ( (num) / ((new Date() - timer) / 1000)) 159 | if (result > best.hreads) best.hreads = result 160 | avg.hreads += result 161 | avg.hreadsCnt += 1 162 | console.log('hkey reads:', result.toFixed(2) + '/s') 163 | cb() 164 | } 165 | } 166 | } 167 | 168 | 169 | var scenario = [ ['all', 1000] , ['all', 2000] , ['sets', 5000] , ['sets', 10000] , ['gets', 5000] , ['gets', 10000] , ['hsets', 5000] , ['hgets', 10000] ]; 170 | var scenarioLen = scenario.length; 171 | 172 | var next = function(i, o) { 173 | if (i < scenarioLen) { 174 | bench(objects[o], scenario[i][0], scenario[i][1], function() { 175 | setTimeout(function() { 176 | next(++i, o) 177 | }, scenario[i][1] / 3) // give some time for the hd to breath 178 | }) 179 | } else { 180 | o++ 181 | if (o===objects.length) { 182 | console.log('---------------------') 183 | console.log('') 184 | console.log('best writes:', best.writes.toFixed(2) + '/s') 185 | console.log('best hkey writes:', best.hwrites.toFixed(2) + '/s') 186 | console.log('best reads:', best.reads.toFixed(2) + '/s') 187 | console.log('best hkey reads:', best.hreads.toFixed(2) + '/s') 188 | console.log('avg writes:', (avg.writes / avg.writesCnt).toFixed(2) + '/s') 189 | console.log('avg hwrites:', (avg.hwrites / avg.hwritesCnt).toFixed(2) + '/s') 190 | console.log('avg reads:', (avg.reads / avg.readsCnt).toFixed(2) + '/s') 191 | console.log('avg hreads:', (avg.hreads / avg.hreadsCnt).toFixed(2) + '/s') 192 | console.log('---------------------') 193 | console.log('') 194 | console.log('all done!') 195 | } else { 196 | next(0, o) 197 | } 198 | } 199 | } 200 | 201 | var consistency = function(cb) { 202 | var done = 0 203 | , num = 100 204 | console.log('writes...') 205 | for (var i=num; i--; ) { 206 | if (db.set(i, '1234567890')) { 207 | done++; 208 | } 209 | } 210 | if (done===num) { 211 | done = 0 212 | console.log('reads...') 213 | for (var i=num; i--; ) { 214 | var res = db.get(i); 215 | if (!!res) { 216 | done++; 217 | var data = res; 218 | assert.equal(data, '1234567890', 'Consistency error!') 219 | if (done===num) { 220 | cb(); 221 | } 222 | } 223 | } 224 | } 225 | } 226 | 227 | var doesitwork = function(cb) { 228 | var cnt = 0 229 | , max = 6 230 | 231 | var cntincr = 0 232 | for (var i=50; i--; ) { 233 | var number = db.incr('incr'); 234 | cntincr++ 235 | if (cntincr == 50) { 236 | console.log('incr test:', number) 237 | cnt++ 238 | if (cnt === max) cb() 239 | } 240 | } 241 | var cntdecr = 0 242 | for (var i=50; i--; ) { 243 | if (db.decr('decr')) 244 | cntdecr++ 245 | } 246 | if (cntdecr == 50) { 247 | console.log('decr test:', number) 248 | cnt++ 249 | if (cnt === max) cb() 250 | } 251 | var cntmass = 0 252 | for (var i=50; i--;) { 253 | if (db.set('mass', 'writes')) 254 | cntmass++ 255 | } 256 | if (cntmass===50) { 257 | var data = db.get('mass'); 258 | console.log('mass:', data) 259 | cnt++ 260 | if (cnt === max) cb() 261 | } 262 | } 263 | var start = function(db) { 264 | console.log('checking consistency...') 265 | consistency(function() { 266 | console.log('done.') 267 | console.log('=====================') 268 | console.log('benchmark starting...') 269 | console.log('') 270 | next(0, 0) 271 | }); 272 | } 273 | 274 | var db = new MemDatabase(); 275 | start(db); 276 | -------------------------------------------------------------------------------- /test/lib/mysql.js: -------------------------------------------------------------------------------- 1 | var Client = require('mysql').Client; 2 | var client = new Client(); 3 | 4 | client.host = 'pomelo.163.com'; 5 | client.user = 'xy'; 6 | client.password = 'dev'; 7 | client.database = 'Pomelo'; 8 | 9 | exports.client = client; 10 | -------------------------------------------------------------------------------- /test/mapping/bag.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | 3 | selectUser:function(client,val,cb){ 4 | console.error(' selectUser ' + JSON.stringify(val)); 5 | var sql = 'select * from Hero where id = ?'; 6 | var args = [val]; 7 | client.query(sql, args, function(err, res){ 8 | if(err !== null){ 9 | console.error('selectUser mysql failed!　' + sql + ' ' + JSON.stringify(val)); 10 | cb(null,'-1'); 11 | } else { 12 | console.info('selectUser mysql success! flash dbok ' + sql + ' ' + JSON.stringify(val)); 13 | cb(null,res[0]['name']); 14 | } 15 | }); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /test/mapping/player.js: -------------------------------------------------------------------------------- 1 | module.exports = player = {}; 2 | 3 | player.updateUser=function(client,val,cb){ 4 | console.error(' updatewrite ' + JSON.stringify(val)+ ' ' + val.x+ ' ' +val.y + ' ' +val.uid); 5 | var sql = 'update Hero set x = ? ,y = ? ,sceneId = ? where id = ?'; 6 | var args = [val.x, val.y, val.sceneId, val.uid]; 7 | client.query(sql, args, function(err, res){ 8 | if(err !== null){ 9 | console.error('write mysql failed!　' + sql + ' ' + JSON.stringify(val)); 10 | } else { 11 | console.info('write mysql success! flash dbok ' + sql + ' ' + JSON.stringify(val)); 12 | cb(); 13 | } 14 | }); 15 | } 16 | 17 | -------------------------------------------------------------------------------- /test/sync.js: -------------------------------------------------------------------------------- 1 | var DataSync = require('../'); 2 | 3 | var dbclient = require('./lib/mysql').client; 4 | 5 | 6 | var opt = {}; 7 | var mappingPath = __dirname+ '/mapping'; 8 | opt.client = dbclient; 9 | opt.interval = 1000 * 10; 10 | opt.aof = false; 11 | var sync = new DataSync(opt); 12 | console.log('before loading ') 13 | sync.mapping = sync.loadMapping(mappingPath); 14 | 15 | 16 | console.log(sync.mapping); 17 | 18 | var key = 'user_key'; 19 | var User = function User(name){ 20 | this.name = name; 21 | }; 22 | 23 | var user1 = new User('hello'); 24 | user1.x = user1.y = 999; 25 | user1.uid = 10003; 26 | user1.sceneId = 1; 27 | var resp = sync.set(key,user1); 28 | 29 | console.log('resp %j' , sync.get(key)); 30 | 31 | sync.execSync('bag.selectUser',10004,function(err,data){ 32 | console.log(err + ' select data ' + data); 33 | }); 34 | 35 | user1.x = 888; 36 | user1.y = 777; 37 | 38 | console.log(' count ' + sync.rewriter.count); 39 | 40 | sync.exec('player.updateUser',10003,user1); 41 | 42 | user1.x = 999; 43 | 44 | sync.flush('player.updateUser',10003,user1); 45 | 46 | setInterval(function(){ 47 | console.log(' count:' + sync.rewriter.count + ' isDone: ' + sync.isDone()); 48 | },1000); 49 | --------------------------------------------------------------------------------