├── .gitignore ├── app ├── daemon.js ├── master.js ├── worker.js ├── ui.js └── pages │ └── nodefox.html ├── lib ├── sqlCount.js ├── .pool.js.swp ├── rules │ ├── mirror.js │ ├── numsplit.js │ └── hashes.js ├── render.js ├── tool.js ├── request_queue.js ├── daemon │ ├── sqlNormalize.js │ └── fireWallClient.js ├── factory.js ├── parse.js ├── pool.js ├── env.js ├── log.js ├── cache │ ├── lcache.js │ └── mcache.js ├── routecalc.js ├── decare.js ├── dataloader.js ├── query.js ├── hash.js ├── column.js ├── mysqlloader.js ├── parser │ └── lexter.js ├── mysql.js ├── datamerge.js ├── quickeval.js └── reform.js ├── bin ├── restart.sh ├── start.sh └── stop.sh ├── DEVELOPERS ├── Makefile ├── package.json ├── Jakefile.js ├── install.sh ├── test └── unit │ ├── test.Pool.js │ ├── test.LCache.js │ ├── test.Parse.js │ ├── test.Decare.js │ ├── test.QuickEval.js │ ├── test.Column.js │ ├── test.DataMerge.js │ ├── test.Lexter.js │ └── test.Select.js ├── DIRECTIONS.md ├── README.md ├── src └── calculate.js └── init.sql /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /app/daemon.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincent-zhao/Myfox-query-module/HEAD/app/daemon.js -------------------------------------------------------------------------------- /lib/sqlCount.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincent-zhao/Myfox-query-module/HEAD/lib/sqlCount.js -------------------------------------------------------------------------------- /lib/.pool.js.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vincent-zhao/Myfox-query-module/HEAD/lib/.pool.js.swp -------------------------------------------------------------------------------- /bin/restart.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | # vim: set expandtab tabstop=4 shiftwidth=4 foldmethod=marker: # 3 | 4 | ./stop.sh && ./start.sh 5 | -------------------------------------------------------------------------------- /bin/start.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | # vim: set expandtab tabstop=4 shiftwidth=4 foldmethod=marker: # 3 | 4 | cd ../app 5 | 6 | nohup node master.js & 7 | -------------------------------------------------------------------------------- /DEVELOPERS: -------------------------------------------------------------------------------- 1 | myfox查询模块 开发人员： 2 | 3 | * Zhao Zhiqiang (yixuan) 4 | * Zhao Lei (xuyi) 5 | 6 | 如果你有任何问题，请和我们联系。 7 | 8 | -------------------------------------------------------------------------------- /bin/stop.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | # vim: set expandtab tabstop=4 shiftwidth=4 foldmethod=marker: # 3 | 4 | ps -ef|grep "nohup node master.js"|grep -v grep|cut -c 9-15|xargs kill -9 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ENV=test 2 | 3 | test: conf 4 | @npm install 5 | -./node_modules/mocha/bin/mocha --reporter tap --ignore-leaks -t 20000 test/unit/test.*.js 6 | 7 | conf: 8 | @npm install -g jake 9 | -mkdir conf 10 | @jake 'generateConfig[$(ENV)]' 11 | 12 | clean: 13 | @cd conf && touch useless && rm * 14 | @cd run && touch useless && rm * 15 | 16 | .PHONY: test conf 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodefox", 3 | "version": "3.0.0", 4 | "author": "xuyi.zl,yixuan.zzq", 5 | "contributors": [ 6 | ], 7 | "homepage": "https://github.com/vincent-zhao/Myfox-query-module", 8 | "description": "A mysql layer.", 9 | "keywords": [ "nodefox" ], 10 | "dependencies": { 11 | "mysql-libmysqlclient" : "=1.2.10", 12 | "node-cluster" : ">=0.1.10" 13 | }, 14 | "engines": { 15 | "node": ">=0.6.9" 16 | }, 17 | "devDependencies": { 18 | "should" : ">=0.4.2", 19 | "mocha" : ">=0.9.0", 20 | "jake" : ">=0.1.14" 21 | }, 22 | "scripts": { 23 | "test": "make test" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /lib/rules/mirror.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: mirror.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: mirror策略 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | /*{{{ route()*/ 15 | /** 16 | * mirror策略的路由计算组合 17 | * @param empty 18 | * @return {String=""} 19 | */ 20 | exports.route = function(){ 21 | return ""; 22 | } 23 | /*}}}*/ 24 | -------------------------------------------------------------------------------- /lib/rules/numsplit.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: numsplit.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: numsplit策略 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | var hashes = require(__dirname + '/hashes'); 15 | 16 | /*{{{ route()*/ 17 | /** 18 | * numsplit策略的路由值计算组合 19 | * @param empty 20 | * @return void 21 | */ 22 | exports.route = hashes.route; 23 | /*}}}*/ 24 | -------------------------------------------------------------------------------- /lib/rules/hashes.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: hash.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: hash路由策略 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | /*{{{ route()*/ 15 | /** 16 | * hash策略的路由值计算组合 17 | * @param {Object} fields 路由值 18 | * @return {String} 19 | */ 20 | exports.route = function(fields){ 21 | var ret = ""; 22 | for(var i in fields){ 23 | ret += (i + ":" + fields[i] + ","); 24 | } 25 | return ret.substr(0,ret.length - 1); 26 | } 27 | /*}}}*/ 28 | -------------------------------------------------------------------------------- /lib/render.js: -------------------------------------------------------------------------------- 1 | var sep = String.fromCharCode('\x01'); 2 | 3 | //code:506 means that obj lacks key elements 4 | function render(obj){ 5 | var res = ""; 6 | var version = (obj.version !== undefined) ? obj.version : "2.0"; 7 | var code = (obj.code !== undefined) ? obj.code : 506; 8 | var msg = (obj.msg !== undefined) ? obj.msg : ""; 9 | var data = (obj.data !== undefined) ? obj.data : []; 10 | res += (version + sep + code + sep + msg + sep + data.length + "\r\n"); 11 | if(data.length == 0){return res;} 12 | for(var i in data[0]){ 13 | res += (i + sep); 14 | } 15 | res = (res.substr(0,res.length-1) + "\r\n"); 16 | for(var i = 0; i < data.length; i++){ 17 | for(j in data[i]){ 18 | res += (data[i][j] + sep); 19 | } 20 | res = (res.substr(0,res.length-1) + "\r\n"); 21 | } 22 | return res; 23 | } 24 | 25 | module.exports = render; 26 | -------------------------------------------------------------------------------- /lib/tool.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: tool.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: 工具类 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | /*{{{ objectClone()*/ 15 | /** 16 | * 复制对象 17 | * @param {Object} obj 需要复制的对象 18 | * @param {String} preventName 需要屏蔽复制的字段名字 19 | * @return {Object} 返回的复制对象 20 | */ 21 | function objectClone(obj,preventName){ 22 | if((typeof obj) == 'object' && obj !== null){ 23 | var res = (!obj.sort)?{}:[]; 24 | for(var i in obj){ 25 | if(i != preventName) 26 | res[i] = objectClone(obj[i],preventName); 27 | } 28 | return res; 29 | }else if((typeof obj) == 'function'){ 30 | return (new obj()).constructor; 31 | } 32 | return obj; 33 | } 34 | exports.objectClone = objectClone; 35 | /*}}}*/ 36 | 37 | -------------------------------------------------------------------------------- /Jakefile.js: -------------------------------------------------------------------------------- 1 | var fs = require("fs"); 2 | var configPath = __dirname + "/conf"; 3 | 4 | desc('default jake'); 5 | task('default',function(params){ 6 | console.log("not tell to generate which env"); 7 | }); 8 | 9 | desc('run jake for config generation'); 10 | task("generateConfig",function(envName,path){ 11 | if(envName !== "test" && envName !== "rc1" && envName !== "rc2" && envName !== "release"){ 12 | console.log("no config for " + envName); 13 | return; 14 | } 15 | if(path !== undefined){ 16 | configPath = path; 17 | } 18 | console.log("start generate " + envName + " enviroment....."); 19 | var getConfig = require(__dirname + "/build/config/" + envName + ".js"); 20 | for(var i in getConfig){ 21 | var getTpl = fs.readFileSync(__dirname + "/build/tpl/" + i + "_tpl.js").toString(); 22 | for(var j in getConfig[i]){ 23 | var reg = new RegExp("##"+j+"##",""); 24 | getTpl = getTpl.replace(reg,getConfig[i][j].toString()); 25 | } 26 | fs.writeFileSync(configPath + "/" + i + ".js","var Log = require(\"../lib/log\");\nmodule.exports = " + getTpl); 27 | } 28 | console.log("generate " + envName + " enviroment succeed!"); 29 | }); 30 | -------------------------------------------------------------------------------- /lib/request_queue.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: request_queue.js 9 | Author: xuyi (xuyi.zl@taobao.com) 10 | Description: 11 | Last Modified: 2012-03-30 12 | */ 13 | 14 | /*{{{ request_queue()*/ 15 | var request_queue = function () { 16 | this.queue = {}; 17 | this.len = 0; 18 | }; 19 | /*}}}*/ 20 | 21 | /*{{{ request_queue.push()*/ 22 | request_queue.prototype.push = function (query) { 23 | var exist = false; 24 | if(!this.queue[query.key]) { 25 | this.queue[query.key] = [query]; 26 | } else { 27 | exist = true; 28 | this.queue[query.key].push(query); 29 | } 30 | this.len ++; 31 | return exist; 32 | }; 33 | /*}}}*/ 34 | 35 | /*{{{ request_queue.end()*/ 36 | request_queue.prototype.end = function(query, callback) { 37 | var _self = this; 38 | var ele = null; 39 | while(ele = _self.queue[query.key].pop()) { 40 | _self.len --; 41 | callback(ele); 42 | } 43 | delete _self.queue[query.key]; 44 | }; 45 | /*}}}*/ 46 | 47 | exports.queue = request_queue; 48 | 49 | 50 | -------------------------------------------------------------------------------- /install.sh: -------------------------------------------------------------------------------- 1 | # !/bin/bash 2 | # vim: set expandtab tabstop=4 shiftwidth=4 foldmethod=marker: # 3 | 4 | #declare -r MYSQL="mysql -h127.0.0.1 -uMysqlUsername -pMysqlPass -PMysqlport --local-infile=1 --default-character-set=utf8" 5 | #-h{your_mysqlServer_ip} 6 | #-u{your_mysqlServer_user} 7 | #-P{your_mysqlServer_port} 8 | #-p{your_mysqlServer_pass} 9 | 10 | #修改下面句子 11 | declare -r MYSQL="mysql -h127.0.0.1 -u用户名 -p密码 --local-infile=1 --default-character-set=utf8" 12 | #example:如果我的用户名为root,密码为mypassword,端口为3306,则将declare句子改为如下样子。 13 | #declare -r MYSQL="mysql5 -h127.0.0.1 -uroot -pmypassword -P3306 --local-infile=1 --default-character-set=utf8" 14 | 15 | # {{{ function usage() # 16 | function usage() { 17 | cat < 0 && me.stack.length > 0) { 83 | var id = me.stack.pop(); 84 | var cb = me.queue.shift(); 85 | cb(me.conn[id], id); 86 | } 87 | 88 | if (!me.stoped) { 89 | return; 90 | } 91 | 92 | for (var i = 0; i < me.conn.length; i++) { 93 | calls.close(me.conn[i]); 94 | delete me.conn[i]; 95 | } 96 | me.stack = []; 97 | 98 | if (me.onclose) { 99 | me.onclose(); 100 | me.onclose = null; 101 | } 102 | } 103 | 104 | check(); 105 | return me; 106 | } 107 | /*}}}*/ 108 | 109 | -------------------------------------------------------------------------------- /test/unit/test.Decare.js: -------------------------------------------------------------------------------- 1 | var should = require('should'); 2 | var Decare = require(__dirname + "/../../lib/decare"); 3 | 4 | describe("Decare test",function(){ 5 | 6 | /*{{{ normal decare test*/ 7 | it("normal decare test",function(done){ 8 | var decare = Decare.create(); 9 | decare.register("age",[1,2,3]); 10 | decare.register("sex",["F","M"]); 11 | decare.register("name",["a","b","c","d"]); 12 | decare.register("hair",["blue","red"]); 13 | decare.unregister("hair"); 14 | var res = decare.cal(); 15 | var expect = []; 16 | var t1 = [];t1["age"] = 1;t1["sex"] = "F";t1["name"] = "a";expect.push(t1); 17 | var t2 = [];t2["age"] = 1;t2["sex"] = "F";t2["name"] = "b";expect.push(t2); 18 | var t3 = [];t3["age"] = 1;t3["sex"] = "F";t3["name"] = "c";expect.push(t3); 19 | var t4 = [];t4["age"] = 1;t4["sex"] = "F";t4["name"] = "d";expect.push(t4); 20 | var t5 = [];t5["age"] = 1;t5["sex"] = "M";t5["name"] = "a";expect.push(t5); 21 | var t6 = [];t6["age"] = 1;t6["sex"] = "M";t6["name"] = "b";expect.push(t6); 22 | var t7 = [];t7["age"] = 1;t7["sex"] = "M";t7["name"] = "c";expect.push(t7); 23 | var t8 = [];t8["age"] = 1;t8["sex"] = "M";t8["name"] = "d";expect.push(t8); 24 | var t9 = [];t9["age"] = 2;t9["sex"] = "F";t9["name"] = "a";expect.push(t9); 25 | var t10 = [];t10["age"] = 2;t10["sex"] = "F";t10["name"] = "b";expect.push(t10); 26 | var t11 = [];t11["age"] = 2;t11["sex"] = "F";t11["name"] = "c";expect.push(t11); 27 | var t12 = [];t12["age"] = 2;t12["sex"] = "F";t12["name"] = "d";expect.push(t12); 28 | var t13 = [];t13["age"] = 2;t13["sex"] = "M";t13["name"] = "a";expect.push(t13); 29 | var t14 = [];t14["age"] = 2;t14["sex"] = "M";t14["name"] = "b";expect.push(t14); 30 | var t15 = [];t15["age"] = 2;t15["sex"] = "M";t15["name"] = "c";expect.push(t15); 31 | var t16 = [];t16["age"] = 2;t16["sex"] = "M";t16["name"] = "d";expect.push(t16); 32 | var t17 = [];t17["age"] = 3;t17["sex"] = "F";t17["name"] = "a";expect.push(t17); 33 | var t18 = [];t18["age"] = 3;t18["sex"] = "F";t18["name"] = "b";expect.push(t18); 34 | var t19 = [];t19["age"] = 3;t19["sex"] = "F";t19["name"] = "c";expect.push(t19); 35 | var t20 = [];t20["age"] = 3;t20["sex"] = "F";t20["name"] = "d";expect.push(t20); 36 | var t21 = [];t21["age"] = 3;t21["sex"] = "M";t21["name"] = "a";expect.push(t21); 37 | var t22 = [];t22["age"] = 3;t22["sex"] = "M";t22["name"] = "b";expect.push(t22); 38 | var t23 = [];t23["age"] = 3;t23["sex"] = "M";t23["name"] = "c";expect.push(t23); 39 | var t24 = [];t24["age"] = 3;t24["sex"] = "M";t24["name"] = "d";expect.push(t24); 40 | res.should.eql(expect); 41 | done(); 42 | }) 43 | /*}}}*/ 44 | 45 | }); 46 | -------------------------------------------------------------------------------- /test/unit/test.QuickEval.js: -------------------------------------------------------------------------------- 1 | var should = require('should'); 2 | var QuickEval = require(__dirname + "/../../lib/quickeval"); 3 | var Lexter = require(__dirname + "/../../lib/parser/lexter"); 4 | 5 | describe("QuickEval test",function(){ 6 | 7 | /*{{{ test normal quick eval*/ 8 | it("test normal quick eval",function(done){ 9 | var e = QuickEval.create("2"); 10 | e.execute().should.eql(2); 11 | e = QuickEval.create("((((((1%3)&(0|1))||0)&&1) << 2)>>1)"); 12 | e.execute().should.eql(2); 13 | e = QuickEval.create("(4>=2)&&(5>4)&&(1<2)&&(2<=3)"); 14 | e.execute().should.eql(1); 15 | e = QuickEval.create("1+2*3.6/(-2)"); 16 | e.execute().should.eql(-2.6); 17 | e = QuickEval.create("1+x"); 18 | e.execute({x:1.4}).should.eql(2.4); 19 | var e = QuickEval.create("x-2"); 20 | e.execute({x:3}).should.eql(1); 21 | e = QuickEval.create("1+ROUND(x,1)"); 22 | e.execute({x:1.4999}).should.eql(2.5); 23 | e = QuickEval.create("ABS(-1)"); 24 | e.execute().should.eql(1); 25 | e = QuickEval.create("CEIL(x+1.5)"); 26 | e.execute({x:1}).should.eql(3); 27 | e = QuickEval.create("FLOOR(1.5)"); 28 | e.execute().should.eql(1); 29 | e = QuickEval.create("EXP(1.5)"); 30 | e.execute().should.eql(4.4816890703380645); 31 | e = QuickEval.create("SQRT(4)"); 32 | e.execute().should.eql(2); 33 | e = QuickEval.create("SIN(3)"); 34 | e.execute().should.eql(0.1411200080598672); 35 | e = QuickEval.create("COS(3)"); 36 | e.execute().should.eql(-0.9899924966004454); 37 | e = QuickEval.create("MD5(\"abc\")"); 38 | e.execute().should.eql("900150983cd24fb0d6963f7d28e17f72"); 39 | e = QuickEval.create("INT(3)"); 40 | e.execute().should.eql(3); 41 | e = QuickEval.create("LN(10)"); 42 | e.execute().should.eql(2.302585092994046); 43 | e = QuickEval.create("LOG(100)"); 44 | e.execute().should.eql(2); 45 | e = QuickEval.create("LENGTH(\"abab\")"); 46 | e.execute().should.eql(4); 47 | 48 | e = QuickEval.create([ 49 | { 50 | type : Lexter.types.NUMBER, 51 | text : 1 52 | }, 53 | { 54 | type : Lexter.types.OPERATOR, 55 | text : "+" 56 | }, 57 | { 58 | type : Lexter.types.VARIABLE, 59 | text : "ABS(c)" 60 | } 61 | ]); 62 | e.execute({"ABS(c)":1}).should.eql(2); 63 | done(); 64 | }); 65 | /*}}}*/ 66 | 67 | /*{{{ test throw exception when undefined function */ 68 | it("test throw exception when undefined function",function(done){ 69 | try{ 70 | var e = QuickEval.create("I_AM_NOT_EXISTS(1)"); 71 | e.execute(); 72 | }catch(e){ 73 | "Undefined function named as \"I_AM_NOT_EXISTS\"".should.eql(e.message); 74 | done(); 75 | } 76 | }); 77 | /*}}}*/ 78 | 79 | /*{{{ test operator after function*/ 80 | it("test operator after function",function(done){ 81 | var e = QuickEval.create("POW(2, 4 - ( 100 + tfix ) / 100) - 1"); 82 | e.execute({tfix:100}).should.eql(3); 83 | var e2 = QuickEval.create("POW(2, 4-(100+tfix)/100)-1"); 84 | e2.execute({tfix:100}).should.eql(3); 85 | done(); 86 | }); 87 | /*}}}*/ 88 | 89 | /*{{{ test compare operate */ 90 | it("test compare operate",function(done){ 91 | var e = QuickEval.create("IF(version >= 2, 0, 1)"); 92 | e.execute({version:2}).should.eql(0); 93 | e.execute({version:1}).should.eql(1); 94 | done(); 95 | }); 96 | /*}}}*/ 97 | 98 | }); 99 | -------------------------------------------------------------------------------- /lib/env.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: env.js 9 | Author: xuyi (xuyi.zl@taobao.com) 10 | Description: 环境配置 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | 15 | global.Util = require('util'); 16 | global.Events = require("events"); 17 | global.tool = require(__dirname + '/tool.js'); 18 | global.Log = require(__dirname + '/log.js'); 19 | global.factory = require(__dirname + '/factory'); 20 | 21 | global.DEBUG = true; 22 | global.CLOSING = false; 23 | global.READCACHE = true; 24 | global.ANALYSIZE_UNIQUEKEY = true; 25 | global.USE_UNIQUEKEY = true; 26 | global.SPLIT_TIMES = 0; 27 | 28 | global.splitReqNum = 0; 29 | global.splitCachedNum = 0; 30 | 31 | global.masterConf = require(__dirname + '/../conf/master_config.js'); 32 | global.workerConf = require(__dirname + '/../conf/worker_config.js'); 33 | global.fireWallConf = require(__dirname + '/../conf/firewall_config.js'); 34 | global.memCacheConf = require(__dirname + '/../conf/memcache_config.js'); 35 | global.dataLoaderConf = require(__dirname + '/../conf/dataloader_config.js'); 36 | global.mysqlLoaderConf = require(__dirname + '/../conf/mysqlloader_config.js'); 37 | 38 | global.keyVersion = ""; 39 | 40 | 41 | global.workerLogger = Log.create(workerConf.logLevel, workerConf.logPath, "worker"); 42 | global.masterLogger = Log.create(workerConf.logLevel, workerConf.logPath, "master"); 43 | global.mysqlLogger = Log.create(workerConf.logLevel, workerConf.logPath, "mysql"); 44 | global.slowLogger = Log.create(workerConf.logLevel, workerConf.logPath, "slow"); 45 | global.memcacheLogger = Log.create(workerConf.logLevel, workerConf.logPath, "memcache"); 46 | global.fireWallLogger = Log.create(workerConf.logLevel, workerConf.logPath, "fireWall"); 47 | 48 | global.__STAT__ = { 49 | totalReqs : 0, 50 | cachedReqs : 0, 51 | totalSplits : 0, 52 | nowReqs : 0, 53 | cachedSplits : 0, 54 | allUseTime : 0, 55 | logLevel : workerLogger.getLevel(), 56 | }; 57 | 58 | /* {{{ function length()*/ 59 | /** 60 | * 全局方法 得到数组或对象的长度 61 | * @param {Array|Object} obj 62 | * @return {Integer} 63 | */ 64 | global.length = function(obj){ 65 | var num = 0, i; 66 | for (i in obj) { 67 | num++; 68 | } 69 | return num; 70 | } 71 | /*}}}*/ 72 | 73 | /* {{{ function empty()*/ 74 | /** 75 | * 全局方法 验证数组|对象是否为空 76 | * @param {Array|Object} obj 77 | * @return {Boolean} 78 | */ 79 | global.empty = function(obj){ 80 | var i; 81 | for (i in obj) { 82 | return false; 83 | } 84 | return true; 85 | } 86 | /*}}}*/ 87 | 88 | /* {{{ function debug()*/ 89 | /** 90 | * 全局方法 debug信息输出 91 | * @param {String} str debug字符串 92 | * @return {None} 93 | */ 94 | global.debug = function(str){ 95 | if(global.DEBUG){ 96 | console.log(str); 97 | } 98 | } 99 | /*}}}*/ 100 | 101 | var exec = require('child_process').exec; 102 | 103 | /*{{{ function getLocalIp()*/ 104 | /** 105 | * 获得本机ip 106 | * @return void 107 | */ 108 | global.getLocalIp = function(){ 109 | exec('hostname -i',function(err, stdout, stderr){ 110 | if(!err){ 111 | global.localIp = stdout; 112 | } 113 | }); 114 | } 115 | 116 | getLocalIp(); 117 | 118 | 119 | -------------------------------------------------------------------------------- /lib/log.js: -------------------------------------------------------------------------------- 1 | /* 2 | File: log.js 3 | Author: yixuan (yixuan.zzq@taobao.com) 4 | Description: 日志类 5 | Last Modified: 2012-02-07 6 | */ 7 | 8 | var fs = require('fs'); 9 | 10 | var cwd = process.cwd() + '/', 11 | INFO = 1, 12 | DEBUG = 2, 13 | NOTICE = 4, 14 | WARNING = 8, 15 | ERROR = 16, 16 | type = ['INFO','DEBUG','NOTICE','WARNING','ERROR'], 17 | bufferSize = 1; 18 | 19 | exports.INFO = INFO; 20 | exports.DEBUG = DEBUG; 21 | exports.NOTICE = NOTICE; 22 | exports.WARNING = WARNING; 23 | exports.ERROR = ERROR; 24 | 25 | function pad2(num) { 26 | return num > 9 ? num : '0' + num; 27 | } 28 | 29 | function getTime() { 30 | var t = new Date(); 31 | return [t.getFullYear(), '-', pad2(t.getMonth() + 1) , '-', pad2(t.getDate()), ' ', 32 | pad2(t.getHours()), ':', pad2(t.getMinutes()), ':', pad2(t.getSeconds())].join(''); 33 | } 34 | 35 | function getDate(){ 36 | var t = new Date(); 37 | return t.getFullYear()+"-"+pad2(t.getMonth()+1)+"-"+pad2(t.getDate()); 38 | } 39 | 40 | function formatLog(log) { 41 | var pos; 42 | if(log.type & 1){pos=0;} 43 | else if(log.type & 2){pos=1;} 44 | else if(log.type & 4){pos=2;} 45 | else if(log.type & 8){pos=3;} 46 | else{pos=4;} 47 | return [log.time, '\t[', type[pos], ']','\tTag:',log.tag,'\tMsg:', log.msg,"\n"].join(''); 48 | } 49 | 50 | exports.create = function(levelNum, dict , fileName) { 51 | try{ 52 | fs.mkdirSync(dict,0755); 53 | }catch(e){} 54 | var file = dict+"/"+fileName; 55 | var buffer = new Buffer(bufferSize); 56 | var pos = 0; 57 | var fd = fs.openSync(file, 'a'); 58 | var dat = fs.statSync(file).atime; 59 | process.on('exit', function(){ 60 | fs.writeSync(fd, buffer, 0, pos, null); 61 | }); 62 | function log(type, tag, msg) { 63 | try{ 64 | msg = msg.toString(); 65 | if(!(type & levelNum)){return;} 66 | var log = {type:type, tag:tag, msg:msg.replace("\n",";").replace("\t","_"), time:getTime()}; 67 | var logContent = formatLog(log); 68 | var bl = Buffer.byteLength(logContent); 69 | if( bl+pos > buffer.length){ 70 | try{ 71 | if(fs.statSync(file).atime > dat){ 72 | fs.closeSync(fd); 73 | fd = fs.openSync(file,'a'); 74 | dat = fs.statSync(file).atime; 75 | }; 76 | }catch(e){ 77 | if(e.errno == 34){ 78 | fs.closeSync(fd); 79 | fd = fs.openSync(file,'a'); 80 | dat = fs.statSync(file).atime; 81 | }else{ 82 | throw new Error(e); 83 | } 84 | } 85 | fs.writeSync(fd,buffer,0,pos,null); 86 | fs.writeSync(fd,new Buffer(logContent),0,bl,null); 87 | }else{ 88 | pos += buffer.write(logContent,pos); 89 | } 90 | }catch(e){} 91 | } 92 | function addLev(type){ 93 | if(!(levelNum & type)){ 94 | levelNum += type; 95 | } 96 | } 97 | function removeLev(type){ 98 | if(levelNum & type){ 99 | levelNum -= type; 100 | } 101 | } 102 | function getLev(){ 103 | return levelNum; 104 | } 105 | return { 106 | info : function(tag,msg) {log(INFO,tag, msg);}, 107 | debug : function(tag,msg) {log(DEBUG,tag, msg);}, 108 | warning : function(tag,msg) {log(WARNING,tag, msg);}, 109 | error : function(tag,msg) {log(ERROR,tag, msg);}, 110 | notice : function(tag,msg) {log(NOTICE,tag, msg);}, 111 | addLevel : function(type) {addLev(type);}, 112 | removeLevel : function(type) {removeLev(type);}, 113 | getLevel : function(){return getLev();} 114 | }; 115 | } 116 | -------------------------------------------------------------------------------- /lib/cache/lcache.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: lcache.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: 本地缓存类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | //缓存答应到日志里的间隔时间 15 | 16 | require(__dirname + '/../env.js'); 17 | var printLengthInterval = 5*60*1000; 18 | 19 | /*{{{ LCache constructor*/ 20 | /** 21 | * LCache构造函数 22 | * @param {int} size 缓存容量 23 | * @return void 24 | */ 25 | var LCache = function(size){ 26 | this.buffer = []; 27 | this.size = size; 28 | this.totalQuery = 0; 29 | this.cachedQuery = 0; 30 | this.totalResearchTimes = 0; 31 | var _self = this; 32 | setInterval(function(){ 33 | workerLogger.notice("LcacheLength","Lcache Length:" + _self.buffer.length + "|Total:"+ _self.totalQuery + "|Cached:" + _self.cachedQuery + "|totalResearchTimes:" + _self.totalResearchTimes + "|pid:" + process.pid); 34 | },printLengthInterval); 35 | } 36 | /*}}}*/ 37 | 38 | /*{{{ get()*/ 39 | /** 40 | * 读取缓存 41 | * @param {String} key 缓存键 42 | * @return {Object} 43 | */ 44 | LCache.prototype.get = function(key){ 45 | var buf = this.buffer; 46 | var len = buf.length; 47 | var res = false; 48 | var tmp; 49 | for(var i = 0; i < len; i++){ 50 | if(buf[i].key == key){ 51 | this.totalResearchTimes += i; 52 | res = objectClone(buf[i].value); 53 | if(i != 0){ 54 | tmp = buf[i-1]; 55 | buf[i-1] = buf[i]; 56 | buf[i] = tmp; 57 | } 58 | break; 59 | } 60 | } 61 | this.totalQuery++; 62 | if(res !== false){ 63 | this.cachedQuery++; 64 | } 65 | return res; 66 | } 67 | /*}}}*/ 68 | 69 | /*{{{ set()*/ 70 | /** 71 | * 设置缓存 72 | * @param {String} key 缓存键 73 | * @param {String} value 缓存值 74 | * @return void 75 | */ 76 | LCache.prototype.set = function(key,value){ 77 | if(this.buffer.length == this.size){ 78 | this.buffer.pop(); 79 | } 80 | this.buffer.push({"key":key,"value":value}); 81 | } 82 | /*}}}*/ 83 | 84 | /*{{{ clean()*/ 85 | /** 86 | * 清空缓存 87 | * @param empty 88 | * @return void 89 | */ 90 | LCache.prototype.clean = function(){ 91 | workerLogger.warning("before_clean_lcache","lcache length:" + this.buffer.length + "|pid:" + process.pid); 92 | while(this.buffer.length > 0){ 93 | this.buffer.pop(); 94 | } 95 | workerLogger.warning("after_clean_lcache","lcache length:" + this.buffer.length + "|pid:" + process.pid); 96 | this.totalQuery = 0; 97 | this.cachedQuery = 0; 98 | } 99 | /*}}}*/ 100 | 101 | /*{{{ create()*/ 102 | /** 103 | * 创建缓存对象 104 | * @param {int} size 缓存容量 105 | * @return {Object} 缓存对象 106 | */ 107 | exports.create = function(size){ 108 | return new LCache(size); 109 | } 110 | /*}}}*/ 111 | 112 | /*{{{ objectClone*/ 113 | /** 114 | * 对象复制 115 | * @param {Object} obj 需要复制的对象 116 | * @param {String} preventName屏蔽复制的元素 117 | * @return {Object} 118 | */ 119 | function objectClone(obj,preventName){ 120 | if((typeof obj)=='object' && obj !== null){ 121 | var res=(!obj.sort)?{}:[]; 122 | for(var i in obj){ 123 | if(i!=preventName) 124 | res[i]=objectClone(obj[i],preventName); 125 | } 126 | return res; 127 | }else if((typeof obj)=='function'){ 128 | return (new obj()).constructor; 129 | } 130 | return obj; 131 | } 132 | /*}}}*/ 133 | -------------------------------------------------------------------------------- /lib/routecalc.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: routecalc.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: 路由计算类 11 | Last Modified: 2012-04-11 12 | */ 13 | 14 | require(__dirname + '/env.js'); 15 | var Hash = require(__dirname + '/hash'); 16 | var hash = require(__dirname + '/rules/hashes') 17 | var mirror = require(__dirname + '/rules/mirror'); 18 | var numsplit = require(__dirname + '/rules/numsplit'); 19 | var LCache = require(__dirname + '/cache/lcache'); 20 | var dataloader = factory.getDataLoader(); 21 | 22 | if(!Conf){ 23 | var Conf = {lcacheLength:1000} 24 | } 25 | 26 | var lcache = LCache.create(Conf.lcacheLength); 27 | 28 | var ROUTE = { 29 | MIRROR: 0, 30 | HASH: 1, 31 | SPLIT: 2 32 | } 33 | 34 | /*{{{ routeValue() */ 35 | /** 36 | * 拼合路由值 37 | * @param {Object} table 表对象 38 | * @param {Object} field 路由值Map 39 | * @return {String} 路由值组合结果 40 | */ 41 | function routeValue(table,field){ 42 | switch(table.tableInfo.route_type){ 43 | case ROUTE.MIRROR: 44 | return mirror.route(); 45 | case ROUTE.HASH: 46 | return hash.route(field); 47 | case ROUTE.SPLIT: 48 | return numsplit.route(field); 49 | default: 50 | break; 51 | } 52 | } 53 | /*}}}*/ 54 | 55 | /*{{{ findNodes() */ 56 | /** 57 | * 查询路由节点 58 | * @param {Object} table 表对象 59 | * @param {Object} field 路由值Map 60 | * @param {Function} cb 回调函数 61 | * @return void 62 | */ 63 | function findNodes(reqObj,table,field,cb){ 64 | var route = routeValue(table,field); 65 | var key = table.tableInfo.table_name + "|" + route; 66 | var get = lcache.get(key); 67 | if(get !== false){ 68 | cb(null,get); 69 | }else{ 70 | var sql = "SELECT hosts_list,real_table,modtime,unique_key FROM $$v1 WHERE route_sign = $$v2 AND table_name = $$v3 AND route_text=$$v4 AND route_flag >= 300 AND route_flag < 400"; 71 | var r_sign = sign(route,table.tableInfo.table_name); 72 | sql = sql.replace('$$v1', dataLoaderConf.routeTable.dbname + "." + dataLoaderConf.routeTable.table_prefix + dataLoaderConf.routeTable.table_name); 73 | sql = sql.replace('$$v2', r_sign); 74 | sql = sql.replace('$$v3', "'" + table.tableInfo.table_name + "'"); 75 | sql = sql.replace('$$v4', "'" + route + "'"); 76 | 77 | dataloader.query(sql,reqObj,function(err,data){ 78 | if(err){ 79 | cb(err,data); 80 | return; 81 | } 82 | 83 | var fields = {}; 84 | for(var i in field){ 85 | fields[i] = field[i]; 86 | } 87 | for(var nodeidx in data){ 88 | data[nodeidx]["route_val"] = fields; 89 | } 90 | 91 | lcache.set(key,data); 92 | cb(err,data); 93 | }); 94 | } 95 | } 96 | /*}}}*/ 97 | 98 | /*{{{ sign() */ 99 | /** 100 | * route_sign签名计算 101 | * @param {String} str 拼合好的路由值字符串 102 | * @param {String} tbname 表名 103 | * @return {int} 签名结果 104 | */ 105 | function sign(str,tbname){ 106 | return Hash.crc32(trim(str) + "|" + tbname); 107 | } 108 | /*}}}*/ 109 | 110 | /*{{{ trim()*/ 111 | /** 112 | * 过滤字符 113 | * @param {String} str 需要过滤的字符串 114 | * @return {String} 过滤后的字符串 115 | */ 116 | function trim(str){ 117 | return str.replace(/(^[\\s]*)|([\\s]*$)/g, ""); 118 | } 119 | /*}}}*/ 120 | 121 | exports.findNodes = findNodes; 122 | exports.ROUTE = ROUTE; 123 | exports.cleanRouteInfo = function(){ 124 | lcache.clean(); 125 | } 126 | -------------------------------------------------------------------------------- /lib/decare.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: decare.js 9 | Author: yixuan.zzq (yixuan.zzq@taobao.com) 10 | Description: 笛卡尔积计算类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | /*{{{ Decare constructor*/ 15 | /** 16 | * Decare对象构造函数 17 | * @param empty 18 | * @return void 19 | */ 20 | var Decare = function(){ 21 | this.params = []; 22 | this.result = []; 23 | } 24 | /*}}}*/ 25 | 26 | /*{{{ register()*/ 27 | /** 28 | * decare计算项注册 29 | * @param {String} key 注册项的键 30 | * @param {Object} val 注册项的值 31 | * @return {Object} 32 | */ 33 | Decare.prototype.register = function(key,val){ 34 | this.params[key+""] = val; 35 | this.result = []; 36 | return this; 37 | } 38 | /*}}}*/ 39 | 40 | /*{{{ unregister()*/ 41 | /** 42 | * decare注册项取消 43 | * @param {String} key 注册项的键 44 | * @return {Object} 45 | */ 46 | Decare.prototype.unregister = function(key){ 47 | delete this.params[key+""]; 48 | this.result = []; 49 | return this; 50 | } 51 | /*}}}*/ 52 | 53 | /*{{{ cal()*/ 54 | /** 55 | * 做笛卡尔积 56 | * @param empty 57 | * @return {Object} 58 | */ 59 | Decare.prototype.cal = function(){ 60 | if(this.result.length == 0){ 61 | var chunk = []; 62 | var parts = arrayChunk(this.params,2,true); 63 | var len = parts.length; 64 | for(var i = 0;i 1){ 82 | var temp = []; 83 | var parts = arrayChunk(chunk,2,false); 84 | var len = parts.length; 85 | for(var i = 0;i 0){ 41 | master.push(Mysql.create(conf.mysql.master.pop())); 42 | } 43 | while(conf.mysql.slave.length > 0){ 44 | slave.push(Mysql.create(conf.mysql.slave.pop())); 45 | } 46 | _loadTable(); 47 | } 48 | /*}}}*/ 49 | 50 | /*{{{ function query()*/ 51 | /** 52 | * 执行sql语句 53 | * @param {String} sql 需要执行的sql 54 | * @param {Function} cb 回调函数 55 | * @return {None} 56 | */ 57 | var query = function(sql, reqObj, cb){ 58 | var req = slave[parseInt(Math.random() * 100) % slave.length] 59 | .query(sql,reqObj); 60 | 61 | req.on('err',function(err){ 62 | master[parseInt(Math.random() * 100) % master.length] 63 | .query(sql,reqObj,function(err, res){ 64 | 65 | cb(err, res); 66 | 67 | }); 68 | }); 69 | req.on('res',function(res){ 70 | cb('', res); 71 | }); 72 | } 73 | exports.query = query; 74 | /*}}}*/ 75 | 76 | /*{{{ function _loadTable()*/ 77 | /** 78 | * 加载表信息到内存，(此方法为阻塞式，供初始化用) 79 | * @return {None} 80 | */ 81 | function _loadTable(){ 82 | workerLogger.notice("loadTable","loadTable From Server"); 83 | 84 | // load hostlist 85 | var sql = Util.format('SELECT host_id,conn_host,conn_port,read_user,read_pass FROM %s.%shost_list WHERE host_stat = %s', config.routeTable.dbname, config.routeTable.table_prefix, SERVER.ONLINE); 86 | table['hostList'] = slave[0].querySync(sql); 87 | 88 | // load tableInfo 89 | var sql = Util.format("SELECT table_name,route_type FROM %s.%stable_list", config.routeTable.dbname, config.routeTable.table_prefix); 90 | table['tableInfo'] = slave[0].querySync(sql); 91 | 92 | // get routeValue for each table 93 | table['tableInfo'].forEach(function(t, index){ 94 | var sql = Util.format("SELECT column_name,tidy_return FROM %s.%stable_route WHERE table_name='%s'", config.routeTable.dbname, config.routeTable.table_prefix, t.table_name); 95 | table['tableInfo'][index]['routeFields'] = slave[0].querySync(sql); 96 | 97 | // sort routeFields from db according to key 'column_name' 98 | var routeFields = table['tableInfo'][index]["routeFields"]; 99 | for(var i = 0;i < routeFields.length; i++){ 100 | for(var j = routeFields.length - 1;j > i;j--){ 101 | if(routeFields[j].column_name < routeFields[j-1].column_name){ 102 | var tmp = routeFields[j]; 103 | routeFields[j] = routeFields[j-1]; 104 | routeFields[j-1] = tmp; 105 | } 106 | } 107 | } 108 | 109 | }); 110 | } 111 | exports.reLoadTable = _loadTable; 112 | /*}}}*/ 113 | 114 | /*{{{ function getRouteTable()*/ 115 | /** 116 | * 获取路由表名 117 | * @param {String} value 路由值 118 | * @return {String} 119 | */ 120 | var getRouteTable = function(value) { 121 | return config.routeTable.dbname + '.myfox_route_info_' + value.substr(0,1); 122 | } 123 | exports.getRouteTable = getRouteTable; 124 | /*}}}*/ 125 | 126 | /*{{{ function getHostList()*/ 127 | /** 128 | * 获取机器列表 129 | * @return {Array} 130 | */ 131 | var getHostList = function() { 132 | if(table['hostList']){ 133 | return table['hostList']; 134 | } 135 | return ''; 136 | } 137 | exports.getHostList = getHostList; 138 | /*}}}*/ 139 | 140 | /*{{{ function getTableInfo()*/ 141 | /** 142 | * 获取表信息 143 | * @param {String} tbname 表名 144 | * @return {Object} 145 | */ 146 | var getTableInfo = function(tbname) { 147 | return _getTableInfo(table['tableInfo'],tbname); 148 | } 149 | 150 | function _getTableInfo(tableInfo, tbname){ 151 | for(var i = 0; i < tableInfo.length; i++){ 152 | if(tableInfo[i].table_name === tbname){ 153 | return tableInfo[i]; 154 | break; 155 | } 156 | } 157 | return ''; 158 | } 159 | exports.getTableInfo = getTableInfo; 160 | /*}}}*/ 161 | 162 | process.on('exit',function(){ 163 | master.forEach(function(m){ 164 | m.close(); 165 | }); 166 | slave.forEach(function(s){ 167 | s.close(); 168 | }); 169 | }); 170 | 171 | -------------------------------------------------------------------------------- /lib/query.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | File: query.js 4 | Author: xuyi (xuyi.zl@taobao.com) 5 | Description: 6 | Last Modified: 2012-03-30 7 | */ 8 | 9 | require(__dirname + '/../lib/env'); 10 | 11 | var Hash = require(__dirname + '/hash'); 12 | var Mcache = require(__dirname + '/cache/mcache.js'); 13 | var requestDealer = require(__dirname + '/../src/requestDealer'); 14 | var Calc = require(__dirname + '/../src/calculate'); 15 | var fWallClient = require(__dirname + '/daemon/fireWallClient.js'); 16 | 17 | var cache = factory.getMcache(); 18 | var fireWall = fWallClient.create(fireWallConf.client); 19 | 20 | var CACHE_ERROR = { 21 | 'NO_USE' : 1, 22 | 'NO_DATA' : 2, 23 | 'TIMEOUT' : 4, 24 | 'PARSE_WRONG' : 8, 25 | }; 26 | var EXPIRE_TIME = { 27 | 'NORMAL' : 86400, 28 | 'GRAY' : 300, 29 | }; 30 | 31 | 32 | /*{{{ function cacheKey()*/ 33 | function cacheKey(str) { 34 | str = workerConf.mcachePrefix + keyVersion + ':data:' + encodeURI(str); 35 | if(str.length > 250){ 36 | str = 'Hash{' + Hash.md5(str) + Hash.fnv(str) + '}'; 37 | } 38 | return str; 39 | } 40 | /*}}}*/ 41 | 42 | /*{{{ function setCache()*/ 43 | function setCache(key,data,ttl){ 44 | cache.set(cacheKey(key), escape(JSON.stringify({t:Date.now(),d:data,e:ttl})), ttl); 45 | } 46 | /*}}} */ 47 | 48 | /*{{{ function checkSql()*/ 49 | function checkSql(route, ip) { 50 | var routes = route.res.route; 51 | var sqlFortest = routes[0].sql; 52 | if(!ip || !sqlFortest){ 53 | return false; 54 | } 55 | var res = fireWall.banSql(sqlFortest, ip); 56 | 57 | if( !!res ){ 58 | fireWallLogger.warning('SQL_BANNED', JSON.stringify(routes[0])); 59 | return res; 60 | } 61 | return false; 62 | } 63 | /*}}}*/ 64 | 65 | /*{{{ query()*/ 66 | var query = function(parseData) { 67 | this.key = parseData.sql.replace(/ /g, '') + JSON.stringify(parseData.params); 68 | this.token = process.pid + "&" + __STAT__.totalReqs; 69 | this.start = Date.now(); 70 | this.parseData = parseData; 71 | this.route = null; 72 | this.error = null; 73 | this.result = null; 74 | this.ip = null; 75 | this.expire = 86400; 76 | }; 77 | /*}}}*/ 78 | 79 | /*{{{ query.getData()*/ 80 | query.prototype.getData = function (callback) { 81 | var _self = this; 82 | var start = Date.now(); 83 | 84 | var cacheDone = function (err, res) { 85 | if(err) { 86 | requestDealer.dealRequest(_self.parseData, routeDone); 87 | }else { 88 | if(__STAT__) { 89 | __STAT__.cachedReqs ++; 90 | } 91 | _self.result = res.data; 92 | _self.explain = res.explainData; 93 | _self.debugInfo = res.debugInfo; 94 | callback(_self); 95 | } 96 | }; 97 | 98 | var routeDone = function (err, route) { 99 | if (err) { 100 | _self.error = err; 101 | callback(_self); 102 | return; 103 | } 104 | _self.route = route; 105 | _self.routeTime = (Date.now() - start) /1000; 106 | /*sql封禁检查*/ 107 | var checkRes = checkSql(route, _self.ip); 108 | if(checkRes) { 109 | _self.error = checkRes; 110 | callback(_self); 111 | return; 112 | } 113 | Calc.create(route, calcDone).exec(); 114 | }; 115 | 116 | var calcDone = function (err, res, debugInfo, explainData, expire) { 117 | if (err || !res || !res.length) { 118 | _self.error = err; 119 | expire = EXPIRE_TIME.GRAY; 120 | } 121 | if(parseInt(expire) >= 0) { 122 | _self.expire = expire; 123 | } 124 | if(_self.parseData.writeCache){ 125 | setCache(_self.key, {data : res, explain : explainData, debugInfo : debugInfo}, _self.expire); 126 | } 127 | _self.result = res; 128 | _self.explain = explainData; 129 | _self.debugInfo = debugInfo; 130 | callback(_self); 131 | }; 132 | 133 | _self.cacheGet(_self.key, cacheDone); 134 | }; 135 | /*}}}*/ 136 | 137 | /*{{{ query.cacheGet()*/ 138 | query.prototype.cacheGet = function (key, callback) { 139 | var _self = this; 140 | var error = null; 141 | var result = null; 142 | if(!READCACHE || !_self.parseData.readCache) { 143 | error = CACHE_ERROR['NO_USE']; 144 | callback(error, result); 145 | return; 146 | } 147 | cache.get(cacheKey(key), function (res) { 148 | if(!res) { 149 | error = CACHE_ERROR['NO_DATA']; 150 | callback(error, result); 151 | return; 152 | } 153 | try{ 154 | res = JSON.parse(unescape(res)); 155 | }catch(e){ 156 | error = CACHE_ERROR['PARSE_WRONG']; 157 | callback(error, result); 158 | return; 159 | } 160 | if(!res.d || !res.t ) { 161 | error = CACHE_ERROR['NO_DATA']; 162 | callback(error, result); 163 | return; 164 | } 165 | _self.expire = res.e - Math.round((Date.now() - res.t) / 1000); 166 | if(_self.expire < 0) { 167 | _self.expire = 0; 168 | } 169 | result = res.d; 170 | callback(error, result); 171 | }); 172 | } 173 | /*}}}*/ 174 | 175 | exports.create = function (parseData) { 176 | return new query(parseData); 177 | } 178 | -------------------------------------------------------------------------------- /lib/daemon/fireWallClient.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: fireWallClient.js 9 | Author: xuyi (xuyi.zl@taobao.com) 10 | Description: 防火墙客户端 11 | Last Modified: 2012-02-29 12 | */ 13 | var fs = require('fs'); 14 | var os = require('os'); 15 | var util = require('util'); 16 | var normal = require(__dirname + '/sqlNormalize.js').create(); 17 | 18 | normal.register(/(\w+)_\d+\.\w+/g,"$1"); 19 | var isArray = Array.isArray; 20 | var response = { 21 | SQL : JSON.stringify({data:[],code:500,expire:'%d'}), 22 | IP : JSON.stringify({data:[],code:501,expire:-1}), 23 | ALL : JSON.stringify({data:[],code:502,expire:-1}), 24 | }; 25 | var MIN_COUNT_USE_PRECENT = 100; 26 | 27 | /* {{{ function _parse()*/ 28 | function _parse(data){ 29 | var ret = {}; 30 | data = data.toString().split("\n"); 31 | if(isArray(data)) { 32 | for(var i = 0; i < data.length; i++){ 33 | if(data[i]){ 34 | var arr = data[i].split("\t"); 35 | ret[arr[0].trim()] = Date.parse(arr[1]); 36 | } 37 | } 38 | } 39 | return ret; 40 | } 41 | /* }}}*/ 42 | 43 | /* {{{ function _empty()*/ 44 | function _empty(obj){ 45 | for(var key in obj){ 46 | return false; 47 | } 48 | return true; 49 | } 50 | /* }}}*/ 51 | 52 | /* {{{ fireWall()*/ 53 | var fireWall = function(config){ 54 | this.bannedSql = {}; 55 | this.bannedIp = {}; 56 | this.recordIp = {}; 57 | this.banAll = false; 58 | this.config = config; 59 | var _self = this; 60 | _self._read(); 61 | fs.watchFile(_self.config.blackListFile, function(curr, prev) { 62 | if (curr.mtime.getTime() !== prev.mtime.getTime()) { 63 | _self._read(); 64 | } 65 | }); 66 | 67 | var times = 0; 68 | //周期执行 69 | setInterval(function() { 70 | times ++; 71 | _self.sysCheck(); 72 | if(times % 60 === 0){ 73 | _self.fresh(); 74 | } 75 | if(times === 60 * 30) { /* 30 minutes */ 76 | times = 0; 77 | _self.empty(); 78 | } 79 | }, 1000); 80 | 81 | } 82 | /* }}}*/ 83 | 84 | /*{{{ function _read()*/ 85 | fireWall.prototype._read = function () { 86 | var _self = this; 87 | fs.readFile(_self.config.blackListFile,function(err, data) { 88 | if(err){ 89 | console.log(err); 90 | } 91 | if (data) { 92 | _self.bannedSql = _parse(data); 93 | } 94 | }); 95 | } 96 | /*}}}*/ 97 | 98 | /* {{{ function sysCheck()*/ 99 | fireWall.prototype.sysCheck = function(){ 100 | var load = os.loadavg(); 101 | if(load[0] > this.config.sysLoadMax){ 102 | this.banAll = true; 103 | }else{ 104 | this.banAll = false; 105 | } 106 | } 107 | /* }}}*/ 108 | 109 | /* {{{ function banIp()*/ 110 | fireWall.prototype.banIp = function (ip) { 111 | if(this.recordIp[ip]){ 112 | this.recordIp[ip].total ++; 113 | var minPercentage = this.config.slowQueryPercentage / 2; 114 | if(this.bannedIp[ip] && ((this.recordIp[ip].banned / this.recordIp[ip].total) < minPercentage)){ 115 | delete this.bannedIp[ip]; 116 | } 117 | } 118 | if(this.banAll){ 119 | //return true; 120 | return response.ALL; 121 | } 122 | if(this.bannedIp[ip]){ 123 | //return true; 124 | return response.IP; 125 | } 126 | return false; 127 | } 128 | /* }}}*/ 129 | 130 | /* {{{ function banSql()*/ 131 | fireWall.prototype.banSql = function (sql, ip) { 132 | var _self = this; 133 | if(_empty(this.bannedSql)) { /*黑名单为空不进行检查*/ 134 | return false; 135 | } 136 | sql = normal.execute(sql); 137 | ip = ip.trim(); 138 | if(this.bannedSql[sql]){ 139 | /*开始记录Ip*/ 140 | if(this.recordIp[ip]){ 141 | this.recordIp[ip].banned ++; 142 | if((this.recordIp[ip].total > MIN_COUNT_USE_PRECENT) && 143 | ((this.recordIp[ip].banned / this.recordIp[ip].total) > this.config.slowQueryPercentage)){ 144 | this.bannedIp[ip] = true; 145 | } 146 | }else{ 147 | this.recordIp[ip] = {total : 1, banned : 1}; 148 | } 149 | //return true; 150 | var now = Date.now(); 151 | return util.format(response.SQL, Math.round((_self.bannedSql[sql] - now)/1000) ); 152 | } 153 | return false; 154 | } 155 | /* }}}*/ 156 | 157 | /* {{{ function getBannedSql()*/ 158 | fireWall.prototype.getBannedSql = function(){ 159 | return this.bannedSql; 160 | } 161 | /* }}}*/ 162 | 163 | /* {{{ function getBannedIp()*/ 164 | fireWall.prototype.getBannedIp = function(){ 165 | return this.bannedIp; 166 | } 167 | /* }}}*/ 168 | 169 | /* {{{ function fresh()*/ 170 | fireWall.prototype.fresh = function(){ 171 | //this.bannedIp = {}; 172 | //this.ips = {}; 173 | //更新sql黑名单 174 | for(var sql in this.bannedSql){ 175 | if(this.bannedSql[sql] < Date.now()){ 176 | delete this.bannedSql[sql]; 177 | } 178 | } 179 | } 180 | /* }}}*/ 181 | 182 | /* {{{ function empty()*/ 183 | fireWall.prototype.empty = function(){ 184 | this.bannedIp = {}; 185 | this.recordIp = {}; 186 | } 187 | /* }}}*/ 188 | 189 | exports.create = function(config){ 190 | return new fireWall(config); 191 | } 192 | 193 | -------------------------------------------------------------------------------- /src/calculate.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: calculate.js 9 | Author: xuyi,yixuan.zzq (xuyi.zl@taobao.com,yixuan.zzq@taobao.com) 10 | Description: 数据加载及计算 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | require(__dirname + '/../lib/env.js'); 15 | var DataMerge = require(__dirname + '/../lib/datamerge.js'); 16 | var MysqlLoader = factory.getMysqlLoader(); 17 | 18 | /*{{{ function calc()*/ 19 | /** 20 | * calc 构造函数 21 | * @param {Object} req 请求对象 22 | * @param {Function} cb 回调 23 | * @return {None} 24 | */ 25 | var calc = function(req, cb){ 26 | this.route = req.res; 27 | this.reqObj = req.reqObj; 28 | this.merge = _mergeInit(req.res); 29 | this.cb = cb; 30 | this.debugInfo = {}; 31 | } 32 | /*}}}*/ 33 | 34 | /* {{{ function calc.exec()*/ 35 | /** 36 | *数据加载&整合计算 主函数 37 | * @return {None} 38 | */ 39 | calc.prototype.exec = function(){ 40 | var _self = this; 41 | var res = ''; 42 | _self.getData(function(err,res,r){ 43 | if(err){ 44 | _self.cb(err, ''); 45 | return; 46 | } 47 | res = _getMerge(_self.merge, res); 48 | if(_self.reqObj.explain){ 49 | _self.getExplain(r,_self.format(res),_self.debugInfo); 50 | }else{ 51 | _self.cb('',_self.format(res),_self.debugInfo); 52 | } 53 | //_self.cb('', res); 54 | }); 55 | } 56 | /*}}}*/ 57 | 58 | /*{{{ function calc.format()*/ 59 | /** 60 | * 过滤结果集 61 | * @param {Object} data 结果集 62 | * @return {Object} 63 | */ 64 | calc.prototype.format = function(data){ 65 | var _self = this; 66 | var ret = []; 67 | var columns = _self.route.columns; 68 | if (columns['*']) { 69 | ret = data; 70 | } else { 71 | ret = data.map(function(d){ 72 | var n = {}; 73 | for (var key in columns) { 74 | if (!columns[key]["hide"]) { 75 | n[key] = d[key]; 76 | } 77 | } 78 | return n; 79 | }); 80 | } 81 | return ret; 82 | } 83 | /*}}}*/ 84 | 85 | /*{{{ function _getMerge()*/ 86 | /** 87 | * 获取Merge结果 88 | * @param {Object} merge merge对象 89 | * @param {Object} data 需要merge的结果集 90 | * @return {Object} 91 | */ 92 | function _getMerge(merge, data){ 93 | while (data.length !== 0) { 94 | merge.push(data.shift()); 95 | } 96 | return merge.getData(); 97 | } 98 | /*}}}*/ 99 | 100 | /*{{{ function _getData()*/ 101 | /** 102 | * 获取数据 103 | * @param {Function} cb 104 | * @return {None} 105 | */ 106 | calc.prototype.getData = function(cb){ 107 | var _self = this; 108 | var route = _self.route; 109 | var num = route.route.length; 110 | var ret = []; 111 | var isReturned = false; 112 | var expire = null; 113 | _self.debugInfo.splitData = []; 114 | var maxLenRoute = { 115 | len : 0, 116 | route : undefined 117 | } 118 | var error = ''; 119 | MysqlLoader.getData(route,function(err, res , debug, r){ 120 | num --; 121 | if(isReturned){ 122 | return; 123 | } 124 | if(err){ 125 | /* 126 | cb(err, ''); 127 | isReturned = true; 128 | return; 129 | */ 130 | error = err; 131 | var res = []; 132 | } 133 | if(res.length === 0){ 134 | expire = 5*60; //如果有分片为空，设置总缓存过期时间为5分钟 135 | } 136 | if(res.length > maxLenRoute.len || maxLenRoute.route === undefined){ 137 | maxLenRoute.len = res.length; 138 | maxLenRoute.route = r; 139 | } 140 | if(_self.reqObj.isDebug){ 141 | _self.debugInfo.splitData.push({info:debug,data:res}); 142 | } 143 | ret.push(res); 144 | if( !num ){ 145 | isReturned = true; 146 | cb(error, ret, maxLenRoute.route, expire); 147 | } 148 | },_self.reqObj); 149 | } 150 | /*}}}*/ 151 | 152 | /*{{{ function getExplain()*/ 153 | /** 154 | * 获得explain信息 155 | * @param {Object} route 查询explain信息的路由对象 156 | * @param {Object} data 原始sql的结果，用于查好explain信息一起传回 157 | * @param {Object} debugData 原始sqldebug信息 158 | * @param void 159 | */ 160 | calc.prototype.getExplain = function(route,data,debugData){ 161 | var _self = this; 162 | if(route === undefined){ 163 | _self.cb("no explain",""); 164 | return; 165 | } 166 | MysqlLoader.getExplain(route,function(err,res){ 167 | if(err){ 168 | _self.cb(err,''); 169 | return; 170 | } 171 | var explainData = { 172 | sql:route.sql, 173 | data:res 174 | } 175 | _self.cb("",data,debugData,explainData); 176 | }); 177 | } 178 | /*}}}*/ 179 | 180 | /*{{{ function _mergeInit()*/ 181 | /** 182 | * Merge 初始化 183 | * @param {Object} info 配置 184 | * @return {Object} 185 | */ 186 | function _mergeInit(info){ 187 | var merge = DataMerge.create(); 188 | if (info.distinct){ 189 | merge.setDistinct(info.distinct); 190 | } 191 | if (info.limits) { 192 | merge.setLimit(info.limits['offset'], info.limits['length']); 193 | } 194 | if (!empty(info.groups)) { 195 | merge.setGroupBy(info.groups); 196 | } 197 | if (!empty(info.orders)) { 198 | merge.setSortKey(info.orders); 199 | } 200 | var output = info.columns; 201 | var evals = []; 202 | var groups = []; 203 | var hidden = []; 204 | for (var key in output) { 205 | var opt = output[key]; 206 | opt.expr && (evals[key] = opt.expr); 207 | opt.merge && (groups[key] = opt.merge); 208 | opt.hide && (hidden[key] = opt.hide); 209 | } 210 | merge.setEvals(evals); 211 | merge.setMerge(groups); 212 | merge.setHidden(hidden); 213 | return merge; 214 | } 215 | /*}}}*/ 216 | 217 | exports.create = function(route, cb){ 218 | return new calc(route, cb); 219 | } 220 | -------------------------------------------------------------------------------- /lib/hash.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: hash.js 9 | Author: xuyi (xuyi.zl@taobao.com) 10 | Description: 常用字符串HASH函数 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | var crypto = require('crypto'); 15 | 16 | /* {{{ function additive() */ 17 | exports.additive = function(key) { 18 | key = key instanceof Buffer ? key: new Buffer(key); 19 | for (var hash = key.length, i = 0; i < key.length; i++) { 20 | hash += key[i]; 21 | } 22 | 23 | return hash; 24 | } 25 | /* }}} */ 26 | 27 | /* {{{ function rotating() */ 28 | exports.rotating = function(key) { 29 | key = key instanceof Buffer ? key: new Buffer(key); 30 | for (var hash = key.length, i = 0; i < key.length; ++i) { 31 | hash = (hash << 4) ^ (hash >> 28) ^ key[i]; 32 | } 33 | return hash; 34 | } 35 | /* }}} */ 36 | 37 | /* {{{ function bernstein() */ 38 | exports.bernstein = function(key) { 39 | key = key instanceof Buffer ? key: new Buffer(key); 40 | var hash = 5381; 41 | for (i = 0; i < key.length; ++i) { 42 | hash = (hash << 5 + hash) + key[i]; 43 | } 44 | 45 | return hash; 46 | } 47 | /* }}} */ 48 | 49 | /* {{{ function fnv() */ 50 | exports.fnv = function(key) { 51 | key = key instanceof Buffer ? key: new Buffer(key); 52 | var p = 16777619, 53 | hash = 0x811C9DC5; 54 | for (var i = 0; i < key.length; i++) { 55 | hash = (hash * p) ^ key[i]; 56 | } 57 | hash += hash << 13; 58 | hash ^= hash >> 7; 59 | hash += hash << 3; 60 | hash ^= hash >> 17; 61 | hash += hash << 5; 62 | 63 | return hash; 64 | } 65 | /* }}} */ 66 | 67 | /* {{{ function fnv1a() */ 68 | exports.fnv1a = function(key) { 69 | key = key instanceof Buffer ? key: new Buffer(key); 70 | var p = 16777619, 71 | hash = 0x811C9DC5; 72 | for (var i = 0; i < key.length; i++) { 73 | hash = (hash ^ key[i]) * p; 74 | } 75 | hash += hash << 13; 76 | hash ^= hash >> 7; 77 | hash += hash << 3; 78 | hash ^= hash >> 17; 79 | hash += hash << 5; 80 | 81 | return hash; 82 | } 83 | /* }}} */ 84 | 85 | /* {{{ md5() */ 86 | exports.md5 = function(key) { 87 | return crypto.createHash('md5').update(key).digest("hex"); 88 | } 89 | /*}}}*/ 90 | 91 | /* {{{ crc32() */ 92 | exports.crc32 = function(str, hex) { 93 | var crc = ~0, 94 | i, l; 95 | for (i = 0, l = str.length; i < l; i++) { 96 | crc = (crc >>> 8) ^ crc32tab[(crc ^ str.charCodeAt(i)) & 0xff]; 97 | } 98 | crc = crc ^ - 1; 99 | var ret = hex ? crc.toString(16) : crc; 100 | if(ret < 0){ 101 | ret = ret >>> 0; 102 | } 103 | return ret; 104 | }; 105 | 106 | var crc32tab = [ 107 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d]; 108 | /*}}}*/ 109 | 110 | -------------------------------------------------------------------------------- /app/worker.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | File: worker.js 4 | Author: xuyi (xuyi.zl@taobao.com) 5 | Description: 6 | Last Modified: 2012-03-29 7 | */ 8 | require(__dirname + '/../lib/env'); 9 | var Http = require('http'); 10 | var fs = require('fs'); 11 | var Worker = require('node-cluster').Worker; 12 | var Query = require(__dirname + '/../lib/query'); 13 | var parse = require(__dirname + '/../lib/parse'); 14 | var requestDealer = require(__dirname + '/../src/requestDealer'); 15 | var request_queue = require(__dirname + '/../lib/request_queue').queue; 16 | 17 | var fWallClient = require(__dirname + '/../lib/daemon/fireWallClient.js'); 18 | var fireWall = fWallClient.create(fireWallConf.client); 19 | 20 | var REQUEST_QUEUE = new request_queue(); 21 | 22 | var TYPE = { 23 | 'CONTROL' : 30, 24 | }; 25 | 26 | var ACTION = { 27 | 'GET_INFO' : 1, 28 | 'LOG_LEVEL' : 30, 29 | 'CLEAN_CACHE' : 20, 30 | }; 31 | 32 | var last_count = 0; /* 记录时间间隔内请求数 **/ 33 | var trend_length = 24; /* 内存中维护队列长度 **/ 34 | var trend = []; /* 固定时间内请求数记录队列 **/ 35 | 36 | setInterval(updateTrend, 30 * 60 * 1000); 37 | 38 | var admin = new Worker(); 39 | 40 | /*{{{ server()*/ 41 | var server = Http.createServer(function (req, res) { 42 | switch (req.method) { 43 | case 'GET' : 44 | get_handle(req, res); 45 | break; 46 | case 'POST' : 47 | __STAT__.totalReqs ++; 48 | __STAT__.nowReqs ++; 49 | post_handle(req, res); 50 | break; 51 | default : 52 | res.end(); 53 | } 54 | }); 55 | 56 | admin.ready(function (socket) { 57 | server.emit('connection', socket); 58 | }); 59 | /*}}}*/ 60 | 61 | /*{{{ function checkIp()*/ 62 | function checkIp(req, res){ 63 | var ip = req.connection.remoteAddress; 64 | var response = fireWall.banIp(req.connection.remoteAddress); 65 | if( !!response ){ 66 | fireWallLogger.warning('IP_BANNED', ip); 67 | res.end(response); 68 | return true; 69 | //return false; 70 | } 71 | return false; 72 | } 73 | /*}}}*/ 74 | 75 | /*{{{ msg_handle*/ 76 | var msg_handle = function (data) { 77 | if(TYPE.CONTROL !== data.type) { 78 | return; 79 | } 80 | var result = { 81 | id : data.id, 82 | type : TYPE.CONTROL, 83 | data : null 84 | }; 85 | if(ACTION.GET_INFO == data.action) { 86 | result.data = __STAT__; 87 | result.memUse = (process.memoryUsage().rss/1000000).toFixed(2); 88 | result.data.trend = trend; 89 | 90 | }else if (ACTION.LOG_LEVEL == data.action) { 91 | var cmd = data.command.split('|'); 92 | if(cmd[0] == "add"){ 93 | workerLogger.addLevel(parseInt(cmd[1])); 94 | }else{ 95 | workerLogger.removeLevel(parseInt(cmd[1])); 96 | } 97 | result.data = { res : 'ok' }; 98 | 99 | }else if (ACTION.CLEAN_CACHE == data.action) { 100 | switch (data.command) { 101 | case 'tableCache' : 102 | requestDealer.cleanTable(); 103 | break; 104 | case 'routeInfoCache' : 105 | requestDealer.cleanRouteInfo(); 106 | break; 107 | default: break; 108 | } 109 | result.data = { res : 'ok' }; 110 | } 111 | process.send(result); 112 | }; 113 | /*}}}*/ 114 | 115 | /*{{{ get_handle*/ 116 | var get_handle = function (req, res) { 117 | res.end('ok'); 118 | } 119 | /*}}}*/ 120 | 121 | /*{{{ post_handle*/ 122 | var post_handle = function (req, res) { 123 | if(checkIp(req, res)) { 124 | return; 125 | } 126 | var data = ''; 127 | req.on('data', function (trunk) { 128 | data += trunk; 129 | }); 130 | req.on('end', function () { 131 | admin.transact(); 132 | var parseData = parse(data); 133 | if(!parseData) { 134 | res.end(); 135 | admin.release(); 136 | return; 137 | } 138 | var query = Query.create(parseData); 139 | query.ip = req.connection.remoteAddress; 140 | query.appIp = req.headers['x-itier-myfox-appip']; 141 | query.req = req; 142 | query.res = res; 143 | workerLogger.notice( 'getQuery|token:' + query.token + '[' + query.ip + '(' + req.appIp + ')' + ']',JSON.stringify(parseData)); 144 | if(!REQUEST_QUEUE.push(query)) { /* 若queryQueue中不存在，插入queryQueue，并且执行后续请求动作;若存在则到此为止 **/ 145 | query.getData(function(query){ 146 | REQUEST_QUEUE.end(query, function(query){ 147 | if(__STAT__) { 148 | __STAT__.allUseTime += Date.now() - query.start; 149 | __STAT__.nowReqs --; 150 | } 151 | var result = formatResult(query); 152 | query.res.end(result); 153 | workerLogger.notice('RES|token:'+query.token, JSON.stringify({timeUse:Date.now() - query.start,length:result.length})); 154 | admin.release(); 155 | }); 156 | }); 157 | } 158 | }); 159 | } 160 | /*}}}*/ 161 | 162 | /*{{{ formatResult()*/ 163 | var formatResult = function (query) { 164 | if(query.error && query.error.toString) { 165 | query.error = query.error.toString(); 166 | } 167 | if(query.parseData.isDebug) { 168 | var ret = { 169 | data : query.result || [], 170 | msg : query.error, 171 | route : query.route ? query.route.res.route : null , 172 | columns : query.route ? query.route.res.columns : null , 173 | routeTime : query.routeTime, 174 | getResDebug : query.debugInfo, 175 | explain : query.explain, 176 | expire : query.expire 177 | }; 178 | workerLogger.debug('DebugRes', JSON.stringify(ret)); 179 | } else { 180 | var ret = { data : query.result || [], msg : query.error, expire : query.expire}; 181 | } 182 | return JSON.stringify(ret); 183 | }; 184 | /*}}}*/ 185 | 186 | /*{{{ updateTrend()*/ 187 | function updateTrend(){ 188 | if(trend.length >= trend_length) { 189 | trend.shift(); 190 | } 191 | trend.push({d:Date.now(), queryNum:__STAT__.totalReqs - last_count }); 192 | last_count = __STAT__.totalReqs; 193 | }; 194 | /*}}}*/ 195 | 196 | process.on('message', msg_handle); 197 | 198 | /*{{{ readStatesFile()*/ 199 | function readStatesFile (file) { 200 | var data = fs.readFileSync(file).toString(); 201 | try { 202 | var get = JSON.parse(data); 203 | keyVersion = get["sqlCacheVersion"]; 204 | }catch(e) {} 205 | } 206 | 207 | readStatesFile(masterConf.statesFile); 208 | 209 | fs.watchFile(masterConf.statesFile, function (curr,prev) { 210 | if(curr.mtime.getTime() !== prev.mtime.getTime()) { 211 | readStatesFile(masterConf.statesFile); 212 | } 213 | }); 214 | /*}}}*/ 215 | -------------------------------------------------------------------------------- /test/unit/test.DataMerge.js: -------------------------------------------------------------------------------- 1 | var should = require('should'); 2 | var DataMerge = require(__dirname + "/../../lib/datamerge"); 3 | var Util = require('util'); 4 | 5 | describe("DataMerge test",function(){ 6 | 7 | /*{{{ test push and sort*/ 8 | it("test push and sort",function(done){ 9 | var arr1 = [ 10 | {a:3,b:1,c:1}, 11 | {a:2,b:2,c:2}, 12 | {a:1,b:2,c:3}, 13 | ]; 14 | var arr2 = [ 15 | {a:423,b:0,c:4}, 16 | {a:3,b:1,c:5}, 17 | {a:1,b:223,c:6} 18 | ]; 19 | var arr3 = [ 20 | {a:423,b:0,c:7}, 21 | {a:32,b:14,c:8}, 22 | {a:2,b:200,c:9} 23 | ]; 24 | var arr4 = [ 25 | {a:4,b:6,c:10}, 26 | {a:4,b:9,c:14}, 27 | {a:2,b:200,c:12} 28 | ]; 29 | var arr5 = [ 30 | {a:6,b:6,c:13}, 31 | {a:4,b:9,c:11}, 32 | {a:3,b:15,c:15} 33 | ]; 34 | var sort = DataMerge.create(); 35 | sort.push(arr1); 36 | sort.push(arr2); 37 | sort.push(arr3); 38 | sort.push(arr4); 39 | sort.push(arr5); 40 | sort.setSortKey({a:"DESC",b:"DESC",c:"ASC"}); 41 | sort.setLimit(2,8); 42 | var expect = [ 43 | // {a:423,b:0,c:4}, 44 | // {a:423,b:0,c:7}, 45 | {a:32,b:14,c:8}, 46 | {a:6,b:6,c:13}, 47 | {a:4,b:9,c:11}, 48 | {a:4,b:9,c:14}, 49 | {a:4,b:6,c:10}, 50 | {a:3,b:15,c:15}, 51 | {a:3,b:1,c:1}, 52 | {a:3,b:1,c:5}, 53 | // {a:2,b:200,c:9}, 54 | // {a:2,b:200,c:12}, 55 | // {a:2,b:2,c:2}, 56 | // {a:1,b:223,c:6}, 57 | // {a:1,b:2,c:3}, 58 | ]; 59 | sort.getData(); 60 | 61 | sort.getData().should.eql(expect); 62 | done(); 63 | }); 64 | /*}}}*/ 65 | 66 | /*{{{ test max min sum with group by */ 67 | it("test max min sum with group by",function(done){ 68 | var data1 = [ 69 | {a:1,b:2,c:3,d:4,e:5,f:"a"}, 70 | {a:2,b:2,c:4,d:5,e:3,f:"b"}, 71 | {a:2,b:9,c:4,d:5,e:13,f:"c"}, 72 | {a:2,b:10,c:4,d:5,e:-6,f:"d"}, 73 | {a:2,b:11,c:4,d:5,e:5,f:"e"}, 74 | {a:2,b:12,c:4,d:5,e:1,f:"f"}, 75 | {a:2,b:13,c:4,d:5,e:6,f:"g"}, 76 | {a:2,b:14,c:4,d:5,e:-3,f:"h"}, 77 | ]; 78 | var data2 = [ 79 | {a:1,b:2,c:1,d:9,e:-1,f:"i"}, 80 | {a:2,b:3,c:4,d:5,e:3,f:"j"}, 81 | {a:2,b:4,c:4,d:5,e:0,f:"k"}, 82 | {a:2,b:5,c:4,d:5,e:5,f:"l"}, 83 | {a:2,b:6,c:4,d:5,e:2,f:"m"}, 84 | {a:2,b:7,c:4,d:5,e:8,f:"n"}, 85 | {a:2,b:8,c:4,d:5,e:5,f:"o"}, 86 | ]; 87 | var merge = DataMerge.create(); 88 | merge.push(data1); 89 | merge.push(data2); 90 | merge.setGroupBy(["a","b"]); 91 | merge.setMerge({ 92 | c:DataMerge.REFORM_MIN, 93 | d:DataMerge.REFORM_MAX, 94 | e:DataMerge.REFORM_SUM, 95 | f:DataMerge.REFORM_CONCAT, 96 | }); 97 | var expect = 98 | [ { a: 1, b: 2, c: 1, d: 9, e: 4 ,f:"a,i"}, 99 | { a: 2, b: 2, c: 4, d: 5, e: 3 ,f:"b"}, 100 | { a: 2, b: 9, c: 4, d: 5, e: 13 ,f:"c"}, 101 | { a: 2, b: 10, c: 4, d: 5, e: -6 ,f:"d"}, 102 | { a: 2, b: 11, c: 4, d: 5, e: 5 ,f:"e"}, 103 | { a: 2, b: 12, c: 4, d: 5, e: 1 ,f:"f"}, 104 | { a: 2, b: 13, c: 4, d: 5, e: 6 ,f:"g"}, 105 | { a: 2, b: 14, c: 4, d: 5, e: -3 ,f:"h"}, 106 | { a: 2, b: 3, c: 4, d: 5, e: 3 ,f:"j"}, 107 | { a: 2, b: 4, c: 4, d: 5, e: 0 ,f:"k"}, 108 | { a: 2, b: 5, c: 4, d: 5, e: 5 ,f:"l"}, 109 | { a: 2, b: 6, c: 4, d: 5, e: 2 ,f:"m"}, 110 | { a: 2, b: 7, c: 4, d: 5, e: 8 ,f:"n"}, 111 | { a: 2, b: 8, c: 4, d: 5, e: 5 ,f:"o"} ]; 112 | 113 | merge.getData().should.eql(expect); 114 | merge.setLimit(1,2); 115 | 116 | expect = 117 | [ { a: 2, b: 2, c: 4, d: 5, e: 3 , f: "b"}, 118 | { a: 2, b: 9, c: 4, d: 5, e: 13, f: "c"} ]; 119 | 120 | merge.getData().should.eql(expect); 121 | merge.setSortKey({e:"ASC",b:"DESC"}); 122 | merge.setHidden({d:true,f:false}); 123 | merge.setLimit(-1,2); 124 | 125 | expect = 126 | [ 127 | { a: 2, b: 10, c: 4, e: -6, f: "d" }, 128 | { a: 2, b: 14, c: 4, e: -3, f: "h" }, 129 | ]; 130 | 131 | merge.getData().should.eql(expect); 132 | 133 | done(); 134 | }); 135 | /*}}}*/ 136 | 137 | /*{{{ test max min sum without group by */ 138 | it("test max min sum without group by",function(done){ 139 | var data1 = [{c:1,d:4,e:5}]; 140 | var data2 = [{c:1,d:9,e:-1}]; 141 | var merge = DataMerge.create(); 142 | merge.push(data1); 143 | merge.push(data2); 144 | merge.setMerge({ 145 | c:DataMerge.REFORM_MIN, 146 | d:DataMerge.REFORM_MAX, 147 | e:DataMerge.REFORM_SUM 148 | }); 149 | merge.getData().should.eql([{c:1,d:9,e:4}]); 150 | done(); 151 | }); 152 | /*}}}*/ 153 | 154 | /*{{{ test_sort_with_expression */ 155 | it("test_sort_with_expression",function(done){ 156 | var data1 = [ 157 | {b:2,c:3,d:4}, 158 | {b:4,c:4,d:5} 159 | ]; 160 | var data2 = [ 161 | {b:2,c:1,d:4}, 162 | {b:3,c:4,d:2} 163 | ]; 164 | var merge = DataMerge.create(); 165 | merge.push(data1); 166 | merge.push(data2); 167 | merge.setMerge({ 168 | c:DataMerge.REFORM_SUM, 169 | d:DataMerge.REFORM_SUM 170 | }); 171 | merge.setGroupBy(["b"]); 172 | merge.setEvals({e:"c/d"}); 173 | merge.setSortKey({e:"DESC"}); 174 | var expect = [ 175 | {b:3,c:4,d:2,e:4/2}, 176 | {b:4,c:4,d:5,e:4/5}, 177 | {b:2,c:4,d:8,e:4/8} 178 | ]; 179 | merge.getData().should.eql(expect); 180 | done(); 181 | }); 182 | /*}}}*/ 183 | 184 | /*{{{ test distinct*/ 185 | it("test distinct",function(done){ 186 | var merge = DataMerge.create(); 187 | merge.setDistinct(true); 188 | var data1 = [{c:3,d:4,e:5}]; 189 | var data2 = [{c:3,d:4,e:5}]; 190 | merge.push(data1); 191 | merge.push(data2); 192 | merge.getData().should.eql([{c:3,d:4,e:5}]); 193 | done(); 194 | }); 195 | /*}}}*/ 196 | 197 | /*{{{ test push empty return empty*/ 198 | it("test push empty return empty",function(done){ 199 | var sort = DataMerge.create(); 200 | sort.setEvals({f3:"id"}); 201 | sort.setSortKey({id:"DESC"}); 202 | sort.getData().should.eql([]); 203 | done(); 204 | }); 205 | /*}}}*/ 206 | 207 | /*{{{ test number string distinct bug case*/ 208 | it("test number string distinct bug case",function(done){ 209 | var merge = DataMerge.create(); 210 | merge.setDistinct(true); 211 | merge.setSortKey({category_id:"ASC"}); 212 | var data1 = [ 213 | {category_id:16}, 214 | {category_id:1625}, 215 | {category_id:30}, 216 | {category_id:50006842}, 217 | {category_id:50006843}, 218 | ]; 219 | var data2 = [ 220 | {category_id:16}, 221 | {category_id:1625}, 222 | {category_id:30}, 223 | {category_id:50006842}, 224 | ]; 225 | merge.push(data1); 226 | merge.push(data2); 227 | var expect = [ 228 | {category_id:16}, 229 | {category_id:30}, 230 | {category_id:1625}, 231 | {category_id:50006842}, 232 | {category_id:50006843}, 233 | ]; 234 | merge.getData().should.eql(expect); 235 | done(); 236 | }); 237 | /*}}}*/ 238 | 239 | }); 240 | -------------------------------------------------------------------------------- /test/unit/test.Lexter.js: -------------------------------------------------------------------------------- 1 | var should = require('should'); 2 | var Lexter = require(__dirname + '/../../lib/parser/lexter.js'); 3 | 4 | describe("Lexter test",function(){ 5 | 6 | /*{{{ test simple select with comment parse*/ 7 | it("test simple select with comment parse",function(done){ 8 | var lexter = Lexter.create( 9 | "SELECT /** 我是注释 **/ 123, `password`, MD5(\"123456\") FROM mysql.user WHERE user=\"测试\\\"引号\" AND host!='%'" 10 | ); 11 | var eql = [ 12 | {'text' : 'SELECT', 'type' : Lexter.types.KEYWORD}, 13 | {'text' : '我是注释', 'type' : Lexter.types.COMMENT}, 14 | {'text' : 123, 'type' : Lexter.types.NUMBER}, 15 | {'text' : ',', 'type' : Lexter.types.COMMAS}, 16 | {'text' : 'password', 'type' : Lexter.types.VARIABLE}, 17 | {'text' : ',', 'type' : Lexter.types.COMMAS}, 18 | {'text' : 'MD5', 'type' : Lexter.types.FUNCTION}, 19 | {'text' : '(', 'type' : Lexter.types.COMMAS}, 20 | {'text' : '123456', 'type' : Lexter.types.STRING}, 21 | {'text' : ')', 'type' : Lexter.types.COMMAS}, 22 | {'text' : 'FROM', 'type' : Lexter.types.KEYWORD}, 23 | {'text' : 'mysql.user', 'type' : Lexter.types.KEYWORD}, 24 | {'text' : 'WHERE', 'type' : Lexter.types.KEYWORD}, 25 | {'text' : 'user', 'type' : Lexter.types.KEYWORD}, 26 | {'text' : '=', 'type' : Lexter.types.OPERATOR}, 27 | {'text' : '测试"引号', 'type' : Lexter.types.STRING}, 28 | {'text' : 'AND', 'type' : Lexter.types.KEYWORD}, 29 | {'text' : 'host', 'type' : Lexter.types.KEYWORD}, 30 | {'text' : '!=', 'type' : Lexter.types.OPERATOR}, 31 | {'text' : '%', 'type' : Lexter.types.STRING}, 32 | ]; 33 | lexter.getAll().should.eql(eql); 34 | done(); 35 | }); 36 | /*}}}*/ 37 | 38 | /*{{{ test negative number be parsed */ 39 | it("test negative number be parsed",function(done){ 40 | var lexter = Lexter.create('SELECT a, c-1 FROM table WHERE b=-2'); 41 | var eql = [ 42 | {text : 'SELECT', type : Lexter.types.KEYWORD}, 43 | {text : 'a', type : Lexter.types.KEYWORD}, 44 | {text : ',', type : Lexter.types.COMMAS}, 45 | {text : 'c', type : Lexter.types.KEYWORD}, 46 | {text : -1, type : Lexter.types.NUMBER}, 47 | {text : 'FROM', type : Lexter.types.KEYWORD}, 48 | {text : 'table', type : Lexter.types.KEYWORD}, 49 | {text : 'WHERE', type : Lexter.types.KEYWORD}, 50 | {text : 'b', type : Lexter.types.KEYWORD}, 51 | {text : '=', type : Lexter.types.OPERATOR}, 52 | {text : -2, type : Lexter.types.NUMBER}, 53 | ]; 54 | lexter.getAll().should.eql(eql); 55 | done(); 56 | }); 57 | /*}}}*/ 58 | 59 | /*{{{ test parse bind variable */ 60 | it("test parse bind variable",function(done){ 61 | var lexter = Lexter.create('c=:id AND t=:V_1'); 62 | var eql = [ 63 | {text : 'c', type : Lexter.types.KEYWORD}, 64 | {text : '=', type : Lexter.types.OPERATOR}, 65 | {text : ':id', type : Lexter.types.PARAMS}, 66 | {text : 'AND', type : Lexter.types.KEYWORD}, 67 | {text : 't', type : Lexter.types.KEYWORD}, 68 | {text : '=', type : Lexter.types.OPERATOR}, 69 | {text : ':V_1', type : Lexter.types.PARAMS}, 70 | ]; 71 | lexter.getAll().should.eql(eql); 72 | done(); 73 | }); 74 | /*}}}*/ 75 | 76 | /*{{{ test number be parsed*/ 77 | it("test number be parsed",function(done){ 78 | var lexter = Lexter.create(123402); 79 | var eql = [ 80 | {text : 123402, type : Lexter.types.NUMBER} 81 | ]; 82 | lexter.getAll().should.eql(eql); 83 | done(); 84 | }); 85 | /*}}}*/ 86 | 87 | /*{{{ test expression parsed ok */ 88 | it("test expression parsed ok",function(done){ 89 | var lexter = Lexter.create('1+2*3.783/ABS(-3.6)'); 90 | var eql = [ 91 | {text : 1, type : Lexter.types.NUMBER}, 92 | {text : '+', type : Lexter.types.OPERATOR}, 93 | {text : 2, type : Lexter.types.NUMBER}, 94 | {text : '*', type : Lexter.types.OPERATOR}, 95 | {text : 3.783, type : Lexter.types.NUMBER}, 96 | {text : '/', type : Lexter.types.OPERATOR}, 97 | {text : 'ABS', type : Lexter.types.FUNCTION}, 98 | {text : '(', type : Lexter.types.COMMAS}, 99 | {text : -3.6, type : Lexter.types.NUMBER}, 100 | {text : ')', type : Lexter.types.COMMAS}, 101 | ]; 102 | lexter.getAll().should.eql(eql); 103 | done(); 104 | }); 105 | /*}}}*/ 106 | 107 | /*{{{ test token index of */ 108 | it("test token index of",function(done){ 109 | var lexter = Lexter.create('1+2*3.783/ABS(x + 3.6) + y'); 110 | var commas = { 111 | type : Lexter.types.OPERATOR, 112 | text : '+' 113 | }; 114 | 115 | lexter.indexOf(commas).should.eql(1); 116 | lexter.indexOf(commas,1).should.eql(12); 117 | 118 | // xxx: x 后边的+号不应该被识别出 119 | 120 | lexter.indexOf(commas,8).should.eql(12); 121 | lexter.indexOf(commas,12).should.eql(-1); 122 | 123 | // xxx: 正则表达式匹配 124 | lexter.indexOf({ 125 | type : Lexter.types.FUNCTION, 126 | text : 'a', 127 | }).should.eql(6); 128 | 129 | lexter.indexOf({ 130 | type : Lexter.types.FUNCTION, 131 | text : '^a$', 132 | }).should.eql(-1); 133 | 134 | done(); 135 | }); 136 | /*}}}*/ 137 | 138 | /*{{{ test get operator vars */ 139 | it("test get operator vars",function(done){ 140 | var lexter = Lexter.create("-1+v+FUNCTION(a,FUNC2(b))"); 141 | var expect = [ 142 | {text:-1,type:Lexter.types.NUMBER}, 143 | {text:"v",type:Lexter.types.KEYWORD} 144 | ]; 145 | Lexter.vars(1,"-1+v+FUNCTION(a,FUNC2(b))",true).should.eql(expect); 146 | expect = [ 147 | [{text:"a",type:Lexter.types.KEYWORD}], 148 | [{text:"FUNC2",type:Lexter.types.FUNCTION}, 149 | {text:"(",type:Lexter.types.COMMAS}, 150 | {text:"b",type:Lexter.types.KEYWORD}, 151 | {text:")",type:Lexter.types.COMMAS}] 152 | ]; 153 | Lexter.vars(4,lexter.getAll(),false).should.eql(expect); 154 | expect = [ 155 | [{text:"b",type:Lexter.types.KEYWORD}] 156 | ]; 157 | Lexter.vars(8,lexter.getAll(),false).should.eql(expect); 158 | done(); 159 | }); 160 | /*}}}*/ 161 | 162 | /*{{{ test get text*/ 163 | it("test get text",function(done){ 164 | var t = [ 165 | {type:Lexter.types.FUNCTION,text:"ABS"}, 166 | {type:Lexter.types.COMMAS,text:"("}, 167 | {type:Lexter.types.NUMBER,text:-3.6}, 168 | {type:Lexter.types.STRING,text:")"}, 169 | {type:Lexter.types.STRING,text:true}, 170 | {type:Lexter.types.STRING,text:123}, 171 | {type:Lexter.types.STRING,text:"\nA\0B\rC\bD\tE"}, 172 | ]; 173 | Lexter.text(t," OR ").join("").should.eql("ABS OR ( OR -3.6 OR \')\' OR true OR 123 OR \'\\nA\\0B\\rC\\bD\\tE\'"); 174 | done(); 175 | }); 176 | /*}}}*/ 177 | 178 | }); 179 | -------------------------------------------------------------------------------- /lib/column.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: column.js 9 | Author: yixuan.zzq (yixuan.zzq@taobao.com) 10 | Description: sql列操作类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | var Lexter = require(__dirname + "/parser/lexter"); 15 | 16 | var R_MAX = 2; 17 | var R_MIN = 4; 18 | var R_SUM = 8; 19 | var R_COUNT = 16; 20 | var R_CONCAT = 32; 21 | var N_COLUMN_PREFIX = "i_am_virtual_"; 22 | 23 | exports.REFORM_MAX = R_MAX; 24 | exports.REFORM_MIN = R_MIN; 25 | exports.REFORM_SUM = R_SUM; 26 | exports.REFORM_COUNT = R_COUNT; 27 | exports.REFORM_CONCAT = R_CONCAT; 28 | exports.NEW_COLUMN_PREFIX = N_COLUMN_PREFIX; 29 | 30 | var groupFunctions = ["SUM","AVG","COUNT","MAX","MIN","GROUP_CONCAT"]; 31 | var l = groupFunctions.length; 32 | var mergeMaps = []; 33 | mergeMaps["SUM"] = R_SUM; 34 | mergeMaps["MAX"] = R_MAX; 35 | mergeMaps["MIN"] = R_MIN; 36 | mergeMaps["AVG"] = R_SUM; 37 | mergeMaps["COUNT"] = R_SUM; 38 | mergeMaps["GROUP_CONCAT"] = R_CONCAT; 39 | 40 | var columnCounter = 1; 41 | var columnAlias = []; 42 | var selectColumn = []; 43 | 44 | function init(){ 45 | columnCounter = 1; 46 | columnAlias = []; 47 | selectColumn = []; 48 | } 49 | 50 | /*{{{ public build()*/ 51 | /** 52 | * 重构sql列,保存在全局变量中 53 | * @param {Object} token 词法解析结果 54 | * @param {String} name 列名字 55 | * @param {String} dist distinct类型 56 | * @param {boolean} hide 是否隐藏该列 57 | * @return void 58 | */ 59 | function build(token,name,dist,hide){ 60 | dist = dist ? dist : ""; 61 | hide = hide ? hide : false; 62 | name = name.trim(); 63 | 64 | var chars = []; 65 | var myfox = []; 66 | var len = token.length; 67 | var ch; 68 | for(var i = 0;i < len;i++){ 69 | ch = token[i]; 70 | if(ch.type == Lexter.types.STRING){ 71 | chars[i] = "\'"+ch.text+"\'"; 72 | }else{ 73 | chars[i] = ch.text; 74 | } 75 | if(ch.type != Lexter.types.FUNCTION){ 76 | continue; 77 | } 78 | for(var j = 0;j < l;j++){ 79 | if(groupFunctions[j] == ch.text.toUpperCase()){ 80 | myfox.push(i); 81 | } 82 | } 83 | } 84 | if(myfox.length == 0){ 85 | var column = chars.join(""); 86 | var alias = getName(column,name); 87 | if(!selectColumn[alias]){ 88 | selectColumn[alias] = { 89 | expr : column, 90 | dist : dist, 91 | fetch : true, 92 | hide : hide, 93 | merge : null 94 | }; 95 | } 96 | if(alias != name){ 97 | selectColumn[name] = { 98 | expr : alias, 99 | fetch : false, 100 | hide : hide 101 | }; 102 | } 103 | return; 104 | } 105 | var offset = 0; 106 | var output = []; 107 | len = myfox.length; 108 | var text,getChild,clen,expr,elen,funame,funObj,alias; 109 | for(var i = 0;i define.MAX_VALUE_LEN){ /* value超长丢弃*/ 231 | !!callback && callback(false); 232 | return; 233 | } 234 | var cmd = "set " + key + " 0 " + ttl + " " + value.length + "\r\n" + value + "\r\n"; 235 | writeSocket(obj, key, cmd, function(data) { 236 | data = Parser(data, obj); 237 | if (callback) { 238 | callback(data); 239 | } 240 | }); 241 | } 242 | /* }}} */ 243 | 244 | /* {{{ public void add() */ 245 | Mcache.prototype.add = function(key, value, ttl, callback) { 246 | var obj = this; 247 | var cmd = "add " + key + " 0 " + ttl + " " + value.length + "\r\n" + value + "\r\n"; 248 | writeSocket(obj, key, cmd, function(data) { 249 | data = Parser(data, obj); 250 | if (callback) { 251 | callback(data); 252 | } 253 | }); 254 | } 255 | /* }}} */ 256 | 257 | /* {{{ public void delete() */ 258 | Mcache.prototype.delete = function(key, callback) { 259 | var obj = this; 260 | writeSocket(obj, key, "delete " + key + " 0\r\n", function(data) { 261 | data = Parser(data, obj); 262 | if (callback) { 263 | callback(data); 264 | } 265 | }); 266 | } 267 | /* }}} */ 268 | 269 | /* {{{ public void close() */ 270 | Mcache.prototype.close = function(callback) { 271 | var len = this.pools.length; 272 | for (var i = 0; i < len; i++) { 273 | this.pools[i].close(function() { 274 | if ((--len) <= 0 && callback) { 275 | this.pools = []; 276 | callback(); 277 | } 278 | }); 279 | } 280 | } 281 | /* }}} */ 282 | 283 | /* {{{ public integer getResultCode() */ 284 | Mcache.prototype.getResultCode = function() { 285 | if (undefined === errors[this.error]) { 286 | return errors.UNKNOWN; 287 | } 288 | 289 | return errors[this.error]; 290 | } 291 | /* }}} */ 292 | 293 | /* {{{ public string getResultMessage() */ 294 | Mcache.prototype.getResultMessage = function() { 295 | return this.error; 296 | } 297 | /* }}} */ 298 | 299 | exports.create = function(option) { 300 | return new Mcache(option); 301 | } 302 | 303 | -------------------------------------------------------------------------------- /lib/mysqlloader.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: mysqlloader.js 9 | Author: xuyi,yixuan (xuyi.zl@taobao.com,yixuan.zzq@taobao.com) 10 | Description: Mysql加载类 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | require(__dirname + '/env'); 15 | 16 | var Mysql = require(__dirname + '/mysql.js'); 17 | var Hash = require(__dirname + '/hash.js'); 18 | 19 | var cache = factory.getMcache(); 20 | var DataLoader = factory.getDataLoader(); 21 | 22 | var node = {}; 23 | var server = {}; 24 | 25 | /*{{{ function cacheKey()*/ 26 | /** 27 | * 制造memcache mey 28 | * @param {String} str 29 | * @return {String} 30 | */ 31 | function cacheKey(str) { 32 | str = mysqlLoaderConf.mcachePrefix + ':data:' + encodeURI(str.replace(/ /g, '')); 33 | if (str.length > 250) { 34 | str = 'Hash{' + Hash.md5(str) + Hash.fnv(str) + '}'; 35 | } 36 | return str; 37 | } 38 | /*}}}*/ 39 | 40 | /*{{{ function setCache()*/ 41 | /** 42 | * 设置缓存 43 | * @param {String} key 缓存Key 44 | * @param {Object} data 缓存Value 45 | * @param {Integer} ttl 有效时间 46 | */ 47 | function setCache(key, data, ttl) { 48 | cache.set(cacheKey(key), escape(JSON.stringify(data)), ttl); 49 | } 50 | /*}}}*/ 51 | 52 | /*{{{ function init()*/ 53 | /** 54 | * 初始化DataLoader 55 | * @return {None} 56 | */ 57 | function init() { 58 | DataLoader.getHostList().forEach(function(host) { 59 | server[host.host_id] = Mysql.create(mysqlConfWraper(host)); 60 | }); 61 | } 62 | 63 | init(); 64 | /*}}}*/ 65 | 66 | /*{{{ function mysqlConfWraper()*/ 67 | /** 68 | * 格式化Config Object 69 | * @param {Object} conf 配置文件 70 | * @return {Object} 71 | */ 72 | function mysqlConfWraper(conf) { 73 | return { 74 | poolSize: mysqlLoaderConf.poolSize, 75 | timeout: mysqlLoaderConf.timeout, 76 | slow: mysqlLoaderConf.slow, 77 | connInfo: { 78 | conn_host: conf.conn_host, 79 | conn_user: conf.read_user, 80 | conn_port: conf.conn_port, 81 | conn_pass: conf.read_pass.toString(), 82 | } 83 | }; 84 | } 85 | //}}} 86 | 87 | /*{{{ function getData()*/ 88 | /** 89 | * 获取数据 90 | * @param {Object} route 路由信息 91 | * @param {Function} cb 回调 92 | * @param {Boolean} debug debug开关 93 | * @param {Boolean} usecache Usecache开关 94 | * @return {None} 95 | */ 96 | var getData = function(route, cb, reqObj) { 97 | route = route.route; 98 | var debugInfo = {}; 99 | route.forEach(function(r) { 100 | if(__STAT__) { 101 | __STAT__.totalSplits ++; 102 | } 103 | if (READCACHE && reqObj.readCache) { 104 | cache.get(cacheKey(r.sql), function(res) { 105 | var wrong = false; 106 | try{ 107 | res = JSON.parse(unescape(res)); 108 | }catch(e){ 109 | wrong = true; 110 | } 111 | memcacheLogger.debug('CACHE_INFO|token:'+reqObj.token, JSON.stringify({'k':r.sql,'h':cacheKey(r.sql),'v':res})); 112 | if (!wrong && res && res.d && res.d.length > 0 && res.t && r.time <= res.t) { 113 | if(__STAT__) { 114 | __STAT__.cachedSplits ++; 115 | } 116 | cb('',res.d); 117 | } else { 118 | queryMysql(r, cb, 0, reqObj); 119 | } 120 | }); 121 | } else { 122 | queryMysql(r, cb, 0, reqObj); 123 | } 124 | }); 125 | } 126 | exports.getData = getData; 127 | //}}} 128 | 129 | /*{{{ function getExplain()*/ 130 | /** 131 | * 获取sql语句explain信息 132 | * @param {Object} r 用来查询explain的某个分片对象 133 | * @param {Function} cb 回调函数 134 | * @return void 135 | */ 136 | var getExplain = function(r,cb){ 137 | var explainRoute = tool.objectClone(r); 138 | explainRoute.sql = "explain (" + explainRoute.sql + ")"; 139 | //只是为了复用queryMysql伪造的reqObj 140 | var reqObj = { 141 | readCache : false, 142 | writeCache : false, 143 | isDebug : false, 144 | explain : false 145 | } 146 | queryMysql(explainRoute,cb,0,reqObj); 147 | } 148 | exports.getExplain = getExplain; 149 | /*}}}*/ 150 | 151 | /*{{{ function queryMysql()*/ 152 | /** 153 | * 请求Mysql 154 | * @param {Object} route 路由信息 155 | * @param {Function} cb 156 | * @param {Integer} index 标志位 157 | * @param {Boolean} debug debug开关 158 | * @return {None} 159 | */ 160 | function queryMysql(route, cb, index, reqObj) { 161 | SPLIT_TIMES ++; 162 | index = index || 0; 163 | //route.node like this "6,8" 164 | serverIds = route.host.split(","); 165 | var num = index; 166 | if(num === 0){ 167 | num = SPLIT_TIMES % serverIds.length; 168 | } 169 | if (reqObj.isDebug) { 170 | var debugInfo = {}; 171 | debugInfo.serverId = serverIds[index]; 172 | debugInfo.servers = serverIds; 173 | debugInfo.splitSql = route.sql; 174 | var begin = Date.now(); 175 | } 176 | var query = server[parseInt(serverIds[num],10)].query(route.sql,reqObj); 177 | query.on('err', function(err) { 178 | mysqlLogger.error('MysqlQueryError|token:'+reqObj.token,JSON.stringify({err : err.toString(), route : route,num : serverIds[num]})); 179 | if (reqObj.isDebug) { 180 | debugInfo.queryTime = Date.now() - begin; 181 | } 182 | if (index < serverIds.length - 1) { 183 | index++; 184 | queryMysql(route, cb, index, reqObj); 185 | } else { 186 | cb(err, '', debugInfo,route); 187 | } 188 | }); 189 | query.on('res', function(res) { 190 | if (reqObj.isDebug) { 191 | debugInfo.queryTime = Date.now() - begin; 192 | } 193 | res = format(res); 194 | if (reqObj.writeCache) { 195 | setCache(route.sql, { t: route.time, d: res }, 0); 196 | } 197 | cb('', res, debugInfo,route); 198 | }); 199 | } 200 | /*}}}*/ 201 | 202 | /*{{{ function format()*/ 203 | /** 204 | * 格式化mysql query 结果 205 | * @param {Array} row 结果集 206 | * @return {Array} 207 | */ 208 | function format(row) { 209 | for (var tk in row) { 210 | for (var tkk in row[tk]) { 211 | if ('object' === typeof(row[tk][tkk]) && row[tk][tkk] instanceof Buffer) { 212 | row[tk][tkk] = row[tk][tkk].toString(); 213 | } 214 | if ('string' === typeof(row[tk][tkk]) ) { 215 | if(/^(-)?(\d)*\.(\d)*$/.test(row[tk][tkk])){ 216 | if( row[tk-1] && /^(-)?(\d)*\.(\d)*$/.test(row[tk-1][tkk])){ /*匹配前一个或后一个，若都为同样格式则转换*/ 217 | row[tk][tkk] = parseFloat(row[tk][tkk]); 218 | }else if( row[tk+1] && /^(-)?(\d)*\.(\d)*$/.test(row[tk+1][tkk])){ 219 | row[tk][tkk] = parseFloat(row[tk][tkk]); 220 | } 221 | } 222 | } 223 | if ('object' === typeof(row[tk][tkk]) && row[tk][tkk] instanceof Date) { 224 | row[tk][tkk] = formatDate(row[tk][tkk]); 225 | } 226 | } 227 | } 228 | return row; 229 | } 230 | /*}}}*/ 231 | 232 | /*{{{ function formatDate()*/ 233 | /** 234 | * 日期格式化 235 | * @param {Date} date 236 | * @return {String} 237 | */ 238 | function formatDate(date) { 239 | var datestr = date.toLocaleDateString(); 240 | var res = []; 241 | var map = { 242 | 'January' : '01', 243 | 'February' : '02', 244 | 'March' : '03', 245 | 'April' : '04', 246 | 'May' : '05', 247 | 'June' : '06', 248 | 'July' : '07', 249 | 'August' : '08', 250 | 'September' : '09', 251 | 'October' : '10', 252 | 'November' : '11', 253 | 'December' : '12' 254 | }; 255 | datestr = datestr.split(','); 256 | (function() { 257 | for (var i = 1; i < datestr.length; i++) { 258 | if (i === 1) { 259 | var tmp = datestr[i].trim().split(' '); 260 | res.push(map[tmp[0].trim()]); 261 | res.push(tmp[1]); 262 | } else { 263 | res.unshift(datestr[i].trim()); 264 | } 265 | } 266 | })(); 267 | return res.join('-'); 268 | } 269 | /*}}}*/ 270 | 271 | /*{{{ process.on('exit');*/ 272 | process.on('exit', function() { 273 | for(var id in server){ 274 | server[id].close(); 275 | } 276 | }); 277 | /*}}}*/ 278 | 279 | -------------------------------------------------------------------------------- /test/unit/test.Select.js: -------------------------------------------------------------------------------- 1 | var should = require('should'); 2 | var Select = require(__dirname + '/../../lib/parser/select.js'); 3 | var Lexter = require(__dirname + '/../../lib/parser/lexter.js'); 4 | 5 | describe('Select test',function(){ 6 | 7 | /*{{{ throw exception when not begin with select */ 8 | it("throw exception when not begin with select",function(done){ 9 | try { 10 | var select = Select.create("`SELECT` 1 as b"); 11 | select.get(); 12 | true.should.not.be.ok; 13 | } catch(err) { 14 | true.should.be.ok; 15 | "SQL command should begin with keyword *SELECT*".should.eql(err.message); 16 | } 17 | done(); 18 | }); 19 | /*}}}*/ 20 | 21 | /*{{{ parse right select column*/ 22 | it("parse right select column",function(done){ 23 | var select = Select.create('sElECt distiNct a, 1 b, 1+MD5("123") AS `select`, MAX(d), DistInct user.username'); 24 | var eql = { 25 | 'a' : { 26 | dist : { text: 'distiNct', type: 1 }, 27 | expr: [{ 28 | text: 'a', 29 | type: 1 30 | }] 31 | }, 32 | 'b' : { 33 | dist: null, 34 | expr: [{ 35 | text: 1, 36 | type: 2 37 | }] 38 | }, 39 | 'select' : { 40 | dist: null, 41 | expr: [{ 42 | text: 1, 43 | type: 2 44 | }, 45 | { 46 | text: '+', 47 | type: 7 48 | }, 49 | { 50 | text: 'MD5', 51 | type: 4 52 | }, 53 | { 54 | text: '(', 55 | type: 8 56 | }, 57 | { 58 | text: '123', 59 | type: 3 60 | }, 61 | { 62 | text: ')', 63 | type: 8 64 | }] 65 | }, 66 | 'MAX(d)' : { 67 | dist: null, 68 | expr: [{ 69 | text: 'MAX', 70 | type: 4 71 | }, 72 | { 73 | text: '(', 74 | type: 8 75 | }, 76 | { 77 | text: 'd', 78 | type: 1 79 | }, 80 | { 81 | text: ')', 82 | type: 8 83 | }] 84 | }, 85 | 'username' : { 86 | dist: { 87 | text: 'DistInct', 88 | type: 1 89 | }, 90 | expr: [{ 91 | text: 'user.username', 92 | type: 1 93 | }] 94 | } 95 | } 96 | select.get('columns').should.eql(eql); 97 | done(); 98 | }); 99 | /*}}}*/ 100 | 101 | /*{{{ parse right table*/ 102 | it("parse right table",function(done){ 103 | var sql = Select.create('SELECT .. FROM mysql.tableA As a, tableB b, c WHERE ..... ORDER BY bbbb ...'); 104 | var eql = { 105 | 'a' : { 106 | db: 'mysql', 107 | table: 'tableA' 108 | }, 109 | 'b' : { 110 | db: '', 111 | table: 'tableB' 112 | }, 113 | 'c' : { 114 | db: '', 115 | table: 'c' 116 | } 117 | } 118 | sql.get("tables").should.eql(eql); 119 | done(); 120 | }); 121 | /*}}}*/ 122 | 123 | /*{{{ parse right join*/ 124 | it("parse right join",function(done){ 125 | var sql = Select.create('SELECT a.c1, b.c2 FROM a LEFT JOIN b ON a.c3=b.c3 AND a.c4=b.c4 JOIN db.tab_c as c ON c.id=a.c2'); 126 | sql.get('joinmap').b.should.eql({ 127 | table: 'b', 128 | method: 'LEFT JOIN', 129 | where: 'a.c3 = b.c3 AND a.c4 = b.c4' 130 | }); 131 | sql.get('joinmap').c.should.eql({ 132 | table: 'db.tab_c', 133 | method: 'JOIN', 134 | where: 'c.id = a.c2' 135 | }); 136 | done(); 137 | }); 138 | /*}}}*/ 139 | 140 | /*{{{ parse right where*/ 141 | it("parse right where",function(done){ 142 | var sql = Select.create('SelEcT * FROM table WHERE a=b AND c >= "id" AND thedate BETWEEN (100 AND 200) AND t IN (2,5,"6") and m LIKE "%abc%" AND 1 <> 2 AND d is not null AND p NOT LIKE "8" AND db.table.x NOT IN (2) AND z is null'); 143 | var eql = [ 144 | { 145 | relate: Select.WHERE.EQ, 146 | values: [{text: 'b',type: 1}], 147 | column: {text: 'a',type: 1} 148 | }, 149 | { 150 | relate: Select.WHERE.GE, 151 | values: [{text: 'id',type: 3}], 152 | column: {text: 'c',type: 1} 153 | }, 154 | { 155 | relate: Select.WHERE.BETWEEN, 156 | values: [{text: 100,type: 2},{text: 200,type: 2}], 157 | column: {text: 'thedate',type: 1} 158 | }, 159 | { 160 | relate: Select.WHERE.IN, 161 | values: [{text: 2,type: 2},{text: 5,type: 2},{text: '6',type: 3}], 162 | column: {text: 't',type: 1} 163 | }, 164 | { 165 | relate: Select.WHERE.LIKE, 166 | values: [{text: '%abc%',type: 3}], 167 | column: {text: 'm',type: 1} 168 | }, 169 | { 170 | relate: Select.WHERE.NE, 171 | values: [{text: 2,type: 2}], 172 | column: {text: 1,type: 2} 173 | }, 174 | { 175 | relate: Select.WHERE.NOTNULL, 176 | values: null, 177 | column: {text: 'd',type: 1} 178 | }, 179 | { 180 | relate: Select.WHERE.NOTLIKE, 181 | values: [{text: '8',type: 3}], 182 | column: {text: 'p',type: 1} 183 | }, 184 | { 185 | relate: Select.WHERE.NOTIN, 186 | values: [{text: 2,type: 2}], 187 | column: {text: 'db.table.x',type: 1} 188 | }, 189 | { 190 | relate: Select.WHERE.ISNULL, 191 | values: null, 192 | column: {text: 'z',type: 1} 193 | } 194 | ] 195 | sql.get('where').should.eql(eql); 196 | done(); 197 | }); 198 | /*}}}*/ 199 | 200 | /*{{{ parse right groupby*/ 201 | it("parse right groupby",function(done){ 202 | var sql = Select.create('SELECT * FROM table gRouP by c, CONCAT(`status`, "wo")'); 203 | var eql = [ 204 | [{text: 'c',type: 1}], 205 | [ 206 | {text: 'CONCAT',type: 4}, 207 | {text: '(',type: 8}, 208 | {text: 'status',type: 5}, 209 | {text: ',',type: 8}, 210 | {text: 'wo',type: 3}, 211 | {text: ')',type: 8}, 212 | ] 213 | ] 214 | sql.get('groupby').should.eql(eql); 215 | done(); 216 | }); 217 | /*}}}*/ 218 | 219 | /*{{{ parse right orderby*/ 220 | it("parse right orderby",function(done){ 221 | var sql = Select.create('SELECT * FROM tab ORDER BY a DESC, MD5(b), c ASC'); 222 | var eql = [ 223 | { 224 | type: 2, 225 | expr: [{text: 'a',type: 1}] 226 | }, 227 | { 228 | type: 1, 229 | expr: [{text: 'MD5',type: 4},{text: '(',type: 8},{text: 'b',type: 1},{text: ')',type: 8}] 230 | }, 231 | { 232 | type: 1, 233 | expr: [{text: 'c',type: 1}] 234 | } 235 | ] 236 | sql.get('orderby').should.eql(eql); 237 | done(); 238 | }); 239 | /*}}}*/ 240 | 241 | /*{{{ parse right limit*/ 242 | it("parse right limit",function(done){ 243 | var sql = Select.create('Select * from asldf LIMIT 10'); 244 | var eql = [ 245 | {text: 0,type: 2}, 246 | {text: 10,type: 2} 247 | ] 248 | sql.get("limits").should.eql(eql); 249 | done(); 250 | }); 251 | /*}}}*/ 252 | 253 | /*{{{ replace works fine*/ 254 | it("replace works fine",function(done){ 255 | var sql = Select.create('SELECT a.c1, b.c2 FROM a LEFT JOIN b ON a.c3=b.c3 AND a.c4=b.c4 WHERE a.c1 = 2 AND a.c1 > 1 AND b.c2 = 0'); 256 | 257 | sql.replaceWhere('a.c1', "a.c4 = 5 AND a.pid = '01'"); 258 | sql.replaceTable('a', 'new_b'); 259 | 260 | var eql = [ 261 | { 262 | relate: 1, 263 | values: [{text: 0,type: 2}], 264 | column: {text: 'b.c2',type: 1} 265 | }, 266 | { 267 | relate: 1, 268 | values: [{text: 5,type: 2}], 269 | column: {text: 'a.c4',type: 1} 270 | }, 271 | { 272 | relate: 1, 273 | values: [{text: '01',type: 3}], 274 | column: {text: 'a.pid',type: 1} 275 | } 276 | ] 277 | sql.get('where').should.eql(eql); 278 | sql.get('tables').a.table.should.eql("new_b"); 279 | done(); 280 | }); 281 | /*}}}*/ 282 | 283 | /*{{{parse right elements*/ 284 | it("parse right elements",function(done){ 285 | var sql = Select.create('SELECT brand_id,SUM(collector_num) from rpt_brand_buyer_area_d where thedate = "2010-10-10" and brand_id = "10122" and category_id = "110520" limit 6'); 286 | var res = ''; 287 | for(var tk in sql.get()){ 288 | res += tk + ','; 289 | } 290 | res = res.substr(0,res.length-2); 291 | res.should.eql('columns,tables,joinmap,where,groupby,orderby,limit'); 292 | done(); 293 | }); 294 | /*}}}*/ 295 | 296 | }) 297 | -------------------------------------------------------------------------------- /app/pages/nodefox.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | nodefox 6 | 31 | 32 | 196 | 197 | 198 |

NodeFox

199 |
200 | 201 |
202 | readCache 203 | writeCache 204 | explain 205 |

206 | 207 |

208 | 209 |

210 | 211 |
212 | 213 |
214 |

215 | 216 |
217 | 218 |
219 |

220 | 221 |
222 | 223 |
224 |

225 | 226 |
227 | 228 |
229 | 230 |
231 |

232 | 233 |
234 | 235 |
236 | 237 | 238 | 239 | -------------------------------------------------------------------------------- /lib/parser/lexter.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: lexter.js 9 | Author: pengchun,yixuan (pengchun@taobao.com,yixuan.zzq@taobao.com) 10 | Description: 词法分析类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | /*{{{ mysqlEscape()*/ 15 | var mysqlEscape = function(val){ 16 | if (val === undefined || val === null) { 17 | return 'NULL'; 18 | } 19 | 20 | switch (typeof val) { 21 | case 'boolean': return (val) ? 'true' : 'false'; 22 | case 'number': return val+''; 23 | } 24 | 25 | if (typeof val === 'object') { 26 | val = (typeof val.toISOString === 'function') 27 | ? val.toISOString() 28 | : val.toString(); 29 | } 30 | 31 | val = val.replace(/[\0\n\r\b\t\\\'\"\x1a]/g, function(s) { 32 | switch(s) { 33 | case "\0": return "\\0"; 34 | case "\n": return "\\n"; 35 | case "\r": return "\\r"; 36 | case "\b": return "\\b"; 37 | case "\t": return "\\t"; 38 | case "\x1a": return "\\Z"; 39 | default: return "\\"+s; 40 | } 41 | }); 42 | return "'"+val+"'"; 43 | } 44 | /*}}}*/ 45 | 46 | 47 | var Types = { 48 | UNKNOWN : 0,/**<未知的 */ 49 | KEYWORD : 1,/**<关键字 */ 50 | NUMBER : 2,/**<数 字 */ 51 | STRING : 3,/**<字符串 */ 52 | FUNCTION : 4,/**<函数名 */ 53 | VARIABLE : 5,/**<变 量 */ 54 | PARAMS : 6,/**<绑定值 */ 55 | OPERATOR : 7,/**<运算符 */ 56 | COMMAS : 8,/**<标 点 */ 57 | MEMORY : 9, 58 | 59 | COMMENT : 99,/**<注 释 */ 60 | } 61 | 62 | var Parser = function(query) { 63 | var tks = [], pre = Types.UNKNOWN; 64 | var tmp = [], cur = '', sub = '', nxt = ''; 65 | 66 | var len = query.length; 67 | for (var i = 0; i < len; i++) { 68 | cur = query.charAt(i); 69 | 70 | /* {{{ 注释 */ 71 | if ('/' == cur && '*' == query.charAt(i + 1)) { 72 | tmp = []; 73 | i++; 74 | while (++i < len) { 75 | sub = query.charAt(i); 76 | nxt = query.charAt(++i); 77 | if ('*' == sub && '/' == nxt) { 78 | break; 79 | } 80 | tmp.push(sub); 81 | tmp.push(nxt); 82 | } 83 | tmp = tmp.join(""); 84 | tks.push({ 85 | 'text' : tmp.replace(/^[\*\s]+/, '').replace(/[\s\*]+$/, ''), 86 | 'type' : Types.COMMENT, 87 | }); 88 | } 89 | /* }}} */ 90 | 91 | /* {{{ 字符串 */ 92 | else if ("'" == cur || '"' == cur || '`' == cur) { 93 | tmp = []; 94 | while (i < len && cur != (sub = query.charAt(++i))) { 95 | ("\\" == sub) ? tmp.push(query.charAt(++i)) : tmp.push(sub); 96 | } 97 | tmp = tmp.join(""); 98 | tks.push({ 99 | 'text' : tmp, 100 | 'type' : ('`' == cur) ? Types.VARIABLE : Types.STRING, 101 | }); 102 | } 103 | /* }}} */ 104 | 105 | /* {{{ 绑定变量 */ 106 | else if (':' == cur) { 107 | tmp = [cur]; 108 | while (i < len) { 109 | sub = query.charAt(++i); 110 | if (!(/^\w+$/i.test(sub))) { 111 | break; 112 | } 113 | tmp.push(sub); 114 | } 115 | tmp = tmp.join(""); 116 | tks.push({ 117 | 'text' : tmp, 118 | 'type' : Types.PARAMS, 119 | }); 120 | } 121 | /* }}} */ 122 | 123 | /* {{{ 函数名 */ 124 | else if (/^[a-z_]+$/i.test(cur)) { 125 | tmp = [cur]; 126 | while (i < len) { 127 | sub = query.charAt(++i); 128 | if (!(/^[\w\.]+$/i.test(sub))) { 129 | break; 130 | } 131 | tmp.push(sub); 132 | } 133 | tmp = tmp.join(""); 134 | i--; 135 | tks.push({ 136 | 'text' : tmp, 137 | 'type' : '(' == sub ? Types.FUNCTION : Types.KEYWORD, 138 | }); 139 | } 140 | /* }}} */ 141 | 142 | /* {{{ 数字 */ 143 | 144 | else if (('-' == cur && Types.VARIABLE != pre) || /\d+/.test(cur)) { 145 | tmp = [cur]; 146 | while (i < len) { 147 | sub = query.charAt(++i); 148 | if (!(/^[\d\.]+$/.test(sub))) { 149 | break; 150 | } 151 | tmp.push(sub); 152 | } 153 | tmp = tmp.join(""); 154 | i--; 155 | 156 | if("-" == tmp){ 157 | tks.push({ 158 | 'text' : '-', /**< 类型转换 */ 159 | 'type' : Types.OPERATOR 160 | }); 161 | }else{ 162 | tks.push({ 163 | 'text' : tmp - 0, /**< 类型转换 */ 164 | 'type' : Types.NUMBER 165 | }); 166 | } 167 | } 168 | /* }}} */ 169 | 170 | /* {{{ 标点 */ 171 | else if (/^[\,;\(\)]+$/.test(cur)) { 172 | tks.push({ 173 | 'text' : cur, 174 | 'type' : Types.COMMAS, 175 | }); 176 | } 177 | /* }}} */ 178 | 179 | /* {{{ 运算符 */ 180 | else if (/^(\+|\-|\*|\/|>|<|=|!|%|&|\||\^)$/.test(cur)) { 181 | tmp = [cur]; 182 | while (i < len) { 183 | sub = query.charAt(++i); 184 | if (!(/^(\+|\*|\/|>|<|=|!|%|&|\||\^)+$/.test(sub))) { 185 | break; 186 | } 187 | tmp.push(sub); 188 | } 189 | tmp = tmp.join(""); 190 | i--; 191 | 192 | tks.push({ 193 | 'text' : tmp, 194 | 'type' : Types.OPERATOR, 195 | }); 196 | } 197 | /* }}} */ 198 | 199 | pre = tks[tks.length - 1].type; 200 | } 201 | 202 | return tks; 203 | } 204 | 205 | /* {{{ public construct() */ 206 | var Lexter = function(query) { 207 | this.tokens = Parser(query.toString()); 208 | this.blocks = []; 209 | 210 | var express = 0; 211 | var calcmap = { 212 | "(" : 1, 213 | ")" : -1, 214 | }; 215 | 216 | for (var i = 0; i < this.tokens.length; ++i) { 217 | var tks = this.tokens[i]; 218 | if (tks.type == Types.COMMAS && undefined !== calcmap[tks.text]) { 219 | express += calcmap[tks.text]; 220 | } else if (!express) { 221 | this.blocks.push(i); 222 | } 223 | } 224 | } 225 | /* }}} */ 226 | 227 | /* {{{ public getAll() */ 228 | Lexter.prototype.getAll = function() { 229 | return this.tokens; 230 | } 231 | /* }}} */ 232 | 233 | /* {{{ public indexOf() */ 234 | Lexter.prototype.indexOf = function(who, off) { 235 | var pos = 0; 236 | var tks = null; 237 | 238 | try { 239 | var exp = new RegExp(who.text, 'i'); 240 | } catch (e) { 241 | var exp = who.text.toLowerCase(); 242 | } 243 | var off = (off === undefined || off < 0) ? 0 : off + 1; 244 | 245 | // xxx: 这里可以用二分法 246 | var bls = this.blocks; 247 | var len = bls.length; 248 | for (var i = 0; i < len; ++i) { 249 | pos = bls[i]; 250 | if (pos < off) { 251 | continue; 252 | } 253 | 254 | tks = this.tokens[pos]; 255 | if (who.type == tks.type && ((exp instanceof RegExp && exp.test(tks.text)) || (!(exp instanceof RegExp) && exp == tks.text.toLowerCase()))) { 256 | return bls[i]; 257 | } 258 | } 259 | 260 | return -1; 261 | } 262 | /* }}} */ 263 | 264 | /*{{{ static vars()*/ 265 | exports.vars = function(idx,tokens,isString){ 266 | var res; 267 | if(isString){ 268 | var lexter = new Lexter(tokens); 269 | tokens = lexter.getAll(); 270 | } 271 | if(!tokens[idx]){return null;} 272 | switch(tokens[idx]["type"]){ 273 | case Types.OPERATOR: 274 | res = [tokens[idx-1],tokens[idx+1]]; 275 | break; 276 | case Types.FUNCTION: 277 | res = []; 278 | var temp = []; 279 | var expr = 0; 280 | for(var i = idx+1, count=tokens.length;i0){ 289 | temp.push(tk); 290 | } 291 | expr++; 292 | break; 293 | case ")": 294 | if((--expr)==0){ 295 | res.push(temp); 296 | temp = []; 297 | i = count; 298 | break; 299 | }else{temp.push(tk);} 300 | break; 301 | case ",": 302 | if(expr == 1){ 303 | res.push(temp); 304 | temp = []; 305 | }else{temp.push(tk);} 306 | break; 307 | default: 308 | break; 309 | } 310 | } 311 | break; 312 | default : 313 | res = null; 314 | break; 315 | } 316 | return res; 317 | } 318 | /*}}}*/ 319 | 320 | /*{{{ static text()*/ 321 | exports.text = function(stack,comma){ 322 | var res = []; 323 | var len = stack.length; 324 | for(var i = 0;i < len; i++){ 325 | var token = stack[i]; 326 | var type = token.type; 327 | var text = token.text; 328 | if(!token || !type || text == null){ 329 | res.push(null); 330 | }else{ 331 | switch(type){ 332 | case Types.STRING : 333 | res.push(mysqlEscape(text)); 334 | break; 335 | case Types.VARIABLE : 336 | res.push(text); 337 | break; 338 | default: 339 | res.push(text); 340 | break; 341 | } 342 | } 343 | if(comma){ 344 | res.push(comma); 345 | } 346 | } 347 | if(comma){res.pop();} 348 | return res; 349 | } 350 | /*}}}*/ 351 | 352 | exports.types = Types; 353 | exports.create = function(query) { 354 | return new Lexter(query); 355 | } 356 | 357 | -------------------------------------------------------------------------------- /lib/mysql.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: mysql.js 9 | Author: xuyi (xuyi.zl@taobao.com) 10 | Description: mysql操作类 11 | Last Modified: 2012-02-20 12 | */ 13 | 14 | require(__dirname + '/env.js'); 15 | DEBUG = false; 16 | 17 | var Mysql = require('mysql-libmysqlclient'); 18 | var Pool = require(__dirname + '/pool.js'); 19 | var inQuery = []; 20 | 21 | var queryStatus = { 22 | 'QUERYING' : 1, 23 | 'KILLED' : -1 24 | }; 25 | 26 | /* {{{ _back()*/ 27 | /** 28 | * 处理返回抽象 29 | * @param {Error} err 错误信息对象 30 | * @param {Object} res 返回结果集 31 | * @param {this} self 运行环境this 32 | * @param {Function} callback 运行环境callback 33 | * @return {None} 34 | */ 35 | function _back(err, res, self, callback){ 36 | if(err){ 37 | if(callback){ 38 | callback(err, ''); 39 | return; 40 | } 41 | self.emit('err',err); 42 | return; 43 | } 44 | if(callback){ 45 | callback('', res); 46 | return; 47 | } 48 | self.emit('res',res); 49 | } 50 | /* }}}*/ 51 | 52 | /*{{{ Object Query()*/ 53 | /** 54 | * 封装事件式query实例 55 | * @param {Object} who Mysql实例 56 | * @param {Object} reqObj 请求对象 57 | * @param {String} sql 需执行的sql 58 | * @param {Function} cb 回调 59 | */ 60 | function Query(who, sql, reqObj, cb){ 61 | Events.EventEmitter.call(this); 62 | var _self = this; 63 | 64 | var key = sql.toLowerCase(); 65 | 66 | /* 若有相同请求在运行，则将此次请求合并*/ 67 | if(inQuery[key]){ 68 | //console.log('query Merged'); 69 | inQuery[key].scopes.push({'callback' : cb, 'self' : _self}); 70 | return; 71 | } 72 | inQuery[key] = { 73 | 'status' : queryStatus.QUERYING, 74 | 'scopes' : [{'callback' : cb, 'self' : _self}] 75 | }; 76 | 77 | var debugInfo = { 78 | sql : sql, 79 | }; 80 | var isTimeout = false; 81 | var start = Date.now(); 82 | /* 连接池利用率大于阈值，进行sql kill */ 83 | if(who.client.stack.length / who.client.conn.length < 0.3){ 84 | connRecover(who); 85 | } 86 | 87 | who.client.get(function(conn, pos){ 88 | /*{{{ debug-- */ 89 | debugInfo.connInfo = who.opt.connInfo; 90 | var getConnTm = Date.now(); 91 | debugInfo.getConnTm = getConnTm - start; 92 | /*-- }}}debug */ 93 | conn.query(sql,function(err,row){ 94 | /*{{{ debug-- */ 95 | debugInfo.poolPos = pos; 96 | var getRowTm = Date.now(); 97 | debugInfo.getRowTm = getRowTm - getConnTm; 98 | /*-- }}}debug */ 99 | who.client.release(pos); 100 | if(err){ 101 | var fetchAllTm = Date.now(); 102 | debugInfo.fetchAllTm = fetchAllTm - getRowTm; 103 | debugInfo.queryTm = fetchAllTm - start; 104 | mysqlLogger.debug('QueryInfo|token:'+reqObj.token,JSON.stringify(debugInfo)); 105 | if(debugInfo.queryTm > who.opt.slow){ 106 | var cloneInfo = tool.objectClone(debugInfo); 107 | var connectInfo = cloneInfo.connInfo; 108 | connectInfo.conn_user = connectInfo.conn_user.charAt(0) + "******" + connectInfo.conn_user.charAt(connectInfo.conn_user.length-1); 109 | connectInfo.conn_pass = connectInfo.conn_pass.charAt(0) + "******" + connectInfo.conn_pass.charAt(connectInfo.conn_pass.length-1); 110 | slowLogger.warning('SLOW|token:'+reqObj.token, JSON.stringify(cloneInfo)); 111 | } 112 | var sc = null; 113 | while(sc = inQuery[key].scopes.pop ? inQuery[key].scopes.pop() : null){ 114 | _back(err, '', sc.self, sc.callback); 115 | } 116 | delete inQuery[key]; 117 | return; 118 | 119 | }else if (inQuery[key].status === queryStatus.KILLED){ 120 | var sc = null; 121 | while(sc = inQuery[key].scopes.pop ? inQuery[key].scopes.pop() : null){ 122 | _back('SLOW QUERY KILLED', '', sc.self, sc.callback); 123 | } 124 | delete inQuery[key]; 125 | return; 126 | 127 | }else { 128 | row.fetchAll(function(err,res){ 129 | /*{{{ debug-- */ 130 | var fetchAllTm = Date.now(); 131 | debugInfo.fetchAllTm = fetchAllTm - getRowTm; 132 | debugInfo.queryTm = fetchAllTm - start; 133 | mysqlLogger.debug('QueryInfo|token:'+reqObj.token,JSON.stringify(debugInfo)); 134 | if(debugInfo.queryTm > who.opt.slow){ 135 | var cloneInfo = tool.objectClone(debugInfo); 136 | var connectInfo = cloneInfo.connInfo; 137 | connectInfo.conn_user = connectInfo.conn_user.charAt(0) + "******" + connectInfo.conn_user.charAt(connectInfo.conn_user.length-1); 138 | connectInfo.conn_pass = connectInfo.conn_pass.charAt(0) + "******" + connectInfo.conn_pass.charAt(connectInfo.conn_pass.length-1); 139 | slowLogger.warning('SLOW|token:'+reqObj.token, JSON.stringify(cloneInfo)); 140 | } 141 | /* --}}}debug */ 142 | 143 | var sc = null; 144 | while(sc = inQuery[key].scopes.pop ? inQuery[key].scopes.pop() : null){ 145 | _back(err, res, sc.self, sc.callback); 146 | } 147 | delete inQuery[key]; 148 | }); 149 | } 150 | }); 151 | }); 152 | } 153 | Util.inherits(Query, Events.EventEmitter); 154 | 155 | /*}}}*/ 156 | 157 | /* {{{ connRecover()*/ 158 | /** 159 | * 连接满时，杀掉数据库处慢连接 160 | * @param {Object} who 与具体某台mysql服务器对应的对象 161 | * @return void 162 | */ 163 | var pTimeout = 2; 164 | 165 | function connRecover(who){ 166 | var sql = "show full processlist"; 167 | 168 | who.client.get(function(conn, pos){ 169 | conn.query(sql,function(err,row){ 170 | if(row){ 171 | row.fetchAll(function(err,processList){ 172 | var forKill = []; 173 | for(var k in processList){ 174 | var pcs = processList[k]; 175 | if(pcs.User === who.opt.connInfo.conn_user && 176 | (pcs.Host.indexOf(localIp.toString().trim()) > -1) && 177 | pcs.Command === 'Query' && 178 | pcs.Time >= pTimeout && 179 | pcs.Info && 180 | inQuery[pcs.Info.toString().toLowerCase()] 181 | ){ 182 | var killsql = "kill query " + pcs.Id; 183 | forKill.push(killsql); 184 | mysqlLogger.warning('KILL_QUERY',JSON.stringify(pcs)); 185 | inQuery[pcs.Info.toLowerCase()].status = queryStatus.KILLED; 186 | } 187 | } 188 | var klen = forKill.length; 189 | if(!klen){ 190 | who.client.release(pos); 191 | return; 192 | } 193 | var counter = 0; 194 | for(var killid = 0; killid < klen; killid++ ){ 195 | conn.query(forKill[killid],function(err, res){ 196 | counter ++; 197 | if(counter === klen){ 198 | who.client.release(pos); 199 | } 200 | }); 201 | } 202 | }); 203 | } 204 | if(err){ 205 | who.client.release(pos); 206 | } 207 | }); 208 | }); 209 | } 210 | /* }}}*/ 211 | 212 | /*{{{ function mysql()*/ 213 | /** 214 | * Mysql 215 | * @param {Object} opt 配置对象 216 | * @return {None} 217 | */ 218 | var mysql = function(opt) { 219 | var _self = this; 220 | this.opt = opt; 221 | try{ 222 | this.client = Pool.create(opt.poolSize, { 223 | 'conn' : function(){ 224 | var conn = Mysql.createConnectionSync(); 225 | conn.initSync(); 226 | conn.setOptionSync(conn.MYSQL_OPT_CONNECT_TIMEOUT, 2); 227 | conn.setOptionSync(conn.MYSQL_OPT_READ_TIMEOUT, opt.timeout); // *3 retry 228 | conn.setOptionSync(conn.MYSQL_OPT_RECONNECT, 1); //自动重连 229 | conn.realConnectSync( 230 | opt.connInfo.conn_host, 231 | opt.connInfo.conn_user, 232 | opt.connInfo.conn_pass, 233 | opt.connInfo.conn_db, 234 | opt.connInfo.conn_port 235 | ); 236 | conn.setCharsetSync('utf8'); 237 | return conn; 238 | }, 239 | 'close' : function(conn){ 240 | try{ 241 | conn.closeSync(); 242 | }catch(e){ 243 | mysqlLogger.warning('CloserError',e); 244 | } 245 | } 246 | }); 247 | }catch(e){ 248 | console.log('mysql make connpool error'); 249 | console.log(opt); 250 | console.log(e); 251 | } 252 | } 253 | /*}}}*/ 254 | 255 | /*{{{ function query()*/ 256 | /** 257 | * 执行sql 258 | * @param {String} sql sql语句 259 | * @param {Object} reqObj 请求对象 260 | * @param {Function} cb 回调函数 261 | * @return {None} 262 | */ 263 | mysql.prototype.query = function(sql,reqObj,cb) { 264 | return new Query(this,sql,reqObj,cb); 265 | } 266 | /*}}}*/ 267 | 268 | /*{{{ function querySync()*/ 269 | /** 270 | * 阻塞式query 271 | * @param {String} sql 272 | * @return {Object} 273 | */ 274 | mysql.prototype.querySync = function(sql){ 275 | var conn = Mysql.createConnectionSync( 276 | this.opt.connInfo.conn_host, 277 | this.opt.connInfo.conn_user, 278 | this.opt.connInfo.conn_pass, 279 | this.opt.connInfo.conn_db, 280 | this.opt.connInfo.conn_port 281 | ); 282 | 283 | var row = conn.querySync(sql); 284 | if(!row.fetchAllSync ){ 285 | return row; 286 | } 287 | var res = row.fetchAllSync(); 288 | conn.closeSync(); 289 | return res; 290 | } 291 | /*}}}*/ 292 | 293 | /*{{{ function close()*/ 294 | /** 295 | * 关闭连接 296 | * @return {None} 297 | */ 298 | mysql.prototype.close = function(){ 299 | if(this.client === undefined){return;} 300 | this.client.conn.forEach(function(conn){ 301 | try{ 302 | conn.closeSync(); 303 | }catch(e){ 304 | console.log(e); 305 | } 306 | }); 307 | } 308 | /*}}}*/ 309 | 310 | /*{{{ create()*/ 311 | exports.create = function(opt) { 312 | return new mysql(opt); 313 | } 314 | /*}}}*/ 315 | -------------------------------------------------------------------------------- /init.sql: -------------------------------------------------------------------------------- 1 | 2 | DROP DATABASE IF EXISTS meta_myfox_config; 3 | CREATE DATABASE meta_myfox_config; 4 | 5 | USE meta_myfox_config; 6 | 7 | -- 系统状态表 8 | DROP TABLE IF EXISTS dev_settings; 9 | CREATE TABLE IF NOT EXISTS dev_settings ( 10 | autokid int(10) unsigned not null auto_increment, 11 | cfgname varchar(32) not null default '', 12 | ownname varchar(32) not null default '', 13 | cfgvalue varchar(255) not null default '', 14 | addtime datetime not null default '0000-00-00 00:00:00', 15 | modtime datetime not null default '0000-00-00 00:00:00', 16 | PRIMARY KEY pk_setting_id (autokid), 17 | UNIQUE KEY uk_setting_name (ownname,cfgname) 18 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 19 | 20 | -- 机器表 21 | DROP TABLE IF EXISTS dev_host_list; 22 | CREATE TABLE dev_host_list ( 23 | host_id int(10) unsigned not null auto_increment, 24 | host_type tinyint(2) unsigned not null default 0, 25 | host_stat tinyint(2) unsigned not null default 0, 26 | host_pos int(10) unsigned not null default 0, 27 | host_name char(16) not null default '', 28 | addtime datetime not null default '0000-00-00 00:00:00', 29 | modtime datetime not null default '0000-00-00 00:00:00', 30 | conn_host varchar(64) not null default '', 31 | conn_port smallint(5) unsigned not null default 0, 32 | read_user varchar(64) not null default '', 33 | read_pass varchar(64) not null default '', 34 | write_user varchar(64) not null default '', 35 | write_pass varchar(64) not null default '', 36 | PRIMARY KEY pk_host_id (host_id), 37 | UNIQUE KEY uk_host_name (host_name), 38 | KEY idx_host_stat (host_stat, host_type), 39 | KEY idx_host_pos (host_pos) 40 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 41 | 42 | INSERT INTO dev_host_list VALUES (1,1,0,0,'host1',NOW(),NOW(),'127.0.0.1',3306,'##用户名##','##密码##','',''); 43 | INSERT INTO dev_host_list VALUES (2,1,0,0,'host2',NOW(),NOW(),'127.0.0.1',3306,'##用户名##','##密码##','',''); 44 | 45 | UPDATE dev_host_list SET host_pos = INET_ATON(conn_host); 46 | 47 | -- 配置表 48 | DROP TABLE IF EXISTS dev_table_list; 49 | CREATE TABLE dev_table_list ( 50 | autokid int(10) unsigned not null auto_increment, 51 | addtime datetime not null default '0000-00-00 00:00:00', 52 | modtime datetime not null default '0000-00-00 00:00:00', 53 | backups tinyint(2) unsigned not null default 1, 54 | max_index_num tinyint(2) unsigned not null default 0, 55 | split_threshold int(10) unsigned not null default 0, 56 | split_drift decimal(5,2) unsigned not null default 0.00, 57 | load_type tinyint(2) unsigned not null default 0, 58 | route_type tinyint(2) unsigned not null default 0, 59 | table_name varchar(64) not null default '', 60 | table_desc varchar(128) not null default '', 61 | unique_key varchar(256) not null default '', 62 | table_sign varchar(32) not null default '', 63 | sql_import text not null default '', 64 | PRIMARY KEY pk_table_id (autokid), 65 | UNIQUE KEY uk_table_name (table_name) 66 | ) ENGINE = InnoDB DEFAULT CHARSET=UTF8; 67 | 68 | INSERT INTO dev_table_list VALUES (1,NOW(),NOW(),2,5,1000,'0.20',1,0,'mirror','测试镜像表','','',''); 69 | INSERT INTO dev_table_list VALUES (2,NOW(),NOW(),2,5,1000,'0.20',0,1,'numsplit','测试切分表','thedate,cid','',''); 70 | 71 | -- 路由字段表 72 | DROP TABLE IF EXISTS dev_table_route; 73 | CREATE TABLE dev_table_route ( 74 | autokid int(10) unsigned not null auto_increment, 75 | addtime datetime not null default '0000-00-00 00:00:00', 76 | modtime datetime not null default '0000-00-00 00:00:00', 77 | table_name varchar(64) not null default '', 78 | column_name varchar(64) not null default '', 79 | tidy_method varchar(64) not null default '', 80 | tidy_return varchar(20) not null default 'int', 81 | is_primary tinyint(2) unsigned not null default 0, 82 | PRIMARY KEY pk_auto_kid (autokid), 83 | UNIQUE KEY uk_table_column (table_name, column_name) 84 | ) ENGINE = InnoDB DEFAULT CHARSET=UTF8; 85 | 86 | INSERT INTO dev_table_route VALUES (1,NOW(),NOW(),'numsplit','thedate','','date',1); 87 | INSERT INTO dev_table_route VALUES (2,NOW(),NOW(),'numsplit','cid','','int',1); 88 | 89 | -- 表字段配置表 90 | DROP TABLE IF EXISTS dev_table_column; 91 | CREATE TABLE dev_table_column ( 92 | autokid int(10) unsigned not null auto_increment, 93 | column_order smallint(5) unsigned not null default 0, 94 | addtime int(10) unsigned not null default 0, 95 | modtime int(10) unsigned not null default 0, 96 | table_name varchar(64) not null default '', 97 | column_name varchar(64) not null default '', 98 | column_type varchar(64) not null default '', 99 | column_size varchar(64) not null default '', 100 | default_value varchar(64) not null default '', 101 | column_desc varchar(256) not null default '', 102 | PRIMARY KEY pk_column_id (autokid), 103 | UNIQUE KEY uk_column_name (table_name,column_name), 104 | KEY idx_column_order (table_name(10), column_order) 105 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 106 | INSERT INTO dev_table_column VALUES (1,1,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'mirror','cid','uint','10','0','类目ID'); 107 | INSERT INTO dev_table_column VALUES (2,2,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'mirror','cname','char','255','','类目名字'); 108 | INSERT INTO dev_table_column VALUES (3,100,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'mirror','autokid','autokid','10','0','自增键'); 109 | INSERT INTO dev_table_column VALUES (4,5,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','char1','char','32','',''); 110 | INSERT INTO dev_table_column VALUES (5,2,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','cid','uint','10','0',''); 111 | INSERT INTO dev_table_column VALUES (6,3,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','num1','uint','10','0',''); 112 | INSERT INTO dev_table_column VALUES (7,4,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','num2','float','20,14','0.00',''); 113 | INSERT INTO dev_table_column VALUES (8,1,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','thedate','date','','0000-00-00',''); 114 | INSERT INTO dev_table_column VALUES (9,100,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),'numsplit','autokid','autokid','10','0',''); 115 | 116 | DROP TABLE IF EXISTS dev_table_index; 117 | CREATE TABLE dev_table_index ( 118 | autokid int(10) unsigned not null auto_increment, 119 | addtime int(10) unsigned not null default 0, 120 | modtime int(10) unsigned not null default 0, 121 | create_type tinyint(2) unsigned not null default 0, 122 | table_name varchar(64) not null default '', 123 | index_name varchar(64) not null default '', 124 | index_text varchar(1024) not null default '', 125 | PRIMARY KEY pk_index_id (autokid), 126 | UNIQUE KEY uk_table_index (table_name, index_name) 127 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 128 | 129 | INSERT INTO dev_table_index VALUES (1,UNIX_TIMESTAMP(),UNIX_TIMESTAMP(),0,'numsplit','idx_cid','cid'); 130 | 131 | -- 任务队列 132 | DROP TABLE IF EXISTS dev_task_queque; 133 | CREATE TABLE IF NOT EXISTS dev_task_queque ( 134 | autokid bigint(20) unsigned not null auto_increment, 135 | agentpos smallint(5) unsigned not null default 0, 136 | priority smallint(5) unsigned not null default 0, 137 | trytimes smallint(5) unsigned not null default 0, 138 | addtime datetime NOT NULL DEFAULT '0000-00-00 00:00:00', 139 | begtime datetime NOT NULL DEFAULT '0000-00-00 00:00:00', 140 | endtime datetime NOT NULL DEFAULT '0000-00-00 00:00:00', 141 | task_flag smallint(5) unsigned not null default 0, 142 | task_type varchar(100) not null default '', 143 | adduser varchar(100) not null default '', 144 | last_error varchar(200) not null default '', 145 | tmp_status varchar(1000) not null default '', 146 | task_info text, 147 | PRIMARY KEY pk_queque_id (autokid), 148 | KEY idx_queque_flag (task_flag, trytimes, priority), 149 | KEY idx_queque_time (addtime, task_flag) 150 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 151 | 152 | -- 路由表 153 | DROP TABLE IF EXISTS dev_route_info; 154 | CREATE TABLE dev_route_info ( 155 | autokid int(10) unsigned not null auto_increment, 156 | addtime int(10) unsigned not null default 0, 157 | modtime int(10) unsigned not null default 0, 158 | hittime int(10) unsigned not null default 0, 159 | route_sign int(10) unsigned not null default 0, 160 | is_archive tinyint(2) unsigned not null default 0, 161 | route_flag smallint(5) unsigned not null default 0, 162 | table_name varchar(64) not null default '', 163 | real_table varchar(128) not null default '', 164 | hosts_list varchar(1024) not null default '', 165 | route_text varchar(1024) not null default '', 166 | unique_key varchar(1024) not null default '', 167 | PRIMARY KEY pk_route_id (autokid), 168 | KEY idx_route_sign (route_sign, route_flag), 169 | KEY idx_route_time (modtime, is_archive) 170 | ) ENGINE = MyISAM DEFAULT CHARSET=UTF8; 171 | 172 | INSERT INTO dev_route_info VALUES (NULL,'1329102431',0,0,'3264109376',0,100,'numsplit','numsplit_0.t_2_2','$','cid:1,thedate:20110610',''); 173 | INSERT INTO dev_route_info VALUES (NULL,'1329102431',0,0,'3264109376',0,300,'numsplit','numsplit_0.t_2_0','1,2$','cid:1,thedate:20110610',''); 174 | -- 归一sql统计表 175 | DROP TABLE IF EXISTS sql_format_stat_v2; 176 | CREATE TABLE `sql_format_stat_v2` ( 177 | `autokid` int(10) unsigned NOT NULL AUTO_INCREMENT, 178 | `total_score` int(10) unsigned NOT NULL DEFAULT '0', 179 | `query_nums` int(10) unsigned NOT NULL DEFAULT '0', 180 | `last_visit` int(10) unsigned NOT NULL DEFAULT '0', 181 | `is_trustful` tinyint(2) unsigned NOT NULL DEFAULT '0', 182 | `sql_sign` int(10) unsigned NOT NULL DEFAULT '0', 183 | `addtime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', 184 | `modtime` datetime NOT NULL DEFAULT '0000-00-00 00:00:00', 185 | `ref_tables` varchar(200) NOT NULL DEFAULT '', 186 | `sql_remark` varchar(200) NOT NULL DEFAULT '', 187 | `sql_format` text NOT NULL, 188 | `sql_sample` text NOT NULL, 189 | PRIMARY KEY (`autokid`), 190 | KEY `idx_sql_sign` (`sql_sign`), 191 | KEY `idx_sql_stat` (`total_score`), 192 | FULLTEXT KEY `full_sql_table` (`ref_tables`) 193 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 194 | 195 | DROP DATABASE IF EXISTS numsplit_0; 196 | CREATE DATABASE numsplit_0; 197 | CREATE TABLE numsplit_0.t_2_0 ( 198 | `thedate` date NOT NULL DEFAULT '0000-00-00', 199 | `cid` int(10) unsigned NOT NULL DEFAULT '0', 200 | `num1` int(10) unsigned NOT NULL DEFAULT '0', 201 | `num2` decimal(20,14) NOT NULL DEFAULT '0.00000000000000', 202 | `char1` varchar(32) NOT NULL DEFAULT '', 203 | `autokid` int(10) unsigned NOT NULL AUTO_INCREMENT, 204 | PRIMARY KEY (`autokid`), 205 | KEY `idx_split_cid` (`cid`) 206 | ) ENGINE=MyISAM DEFAULT CHARSET=utf8; 207 | 208 | INSERT INTO numsplit_0.t_2_0 VALUES ('20110610','1','1','20','for_test',NULL); 209 | 210 | -------------------------------------------------------------------------------- /lib/datamerge.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: datamerge.js 9 | Author: yixuan,xuyi (yixuan.zzq@taobao.com,xuyi.zl@taobao.com) 10 | Description: 分片数据整合类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | require(__dirname + '/env'); 15 | var QuickEval = require(__dirname + "/quickeval"); 16 | 17 | var ORDER_DESC = -1; 18 | var ORDER_ASC = 1; 19 | 20 | var REFORM_MAX = 2; 21 | var REFORM_MIN = 4; 22 | var REFORM_SUM = 8; 23 | var REFORM_COUNT = 16; 24 | var REFORM_CONCAT = 32; 25 | 26 | var DataMerge = function(){ 27 | this.evals = []; 28 | this.hidden = []; 29 | this.sortArr = []; 30 | this.sortKey = []; 31 | this.groupBy = []; 32 | this.groupUse = []; 33 | this.types = []; 34 | this.distinct = false; 35 | this.limitStart = 0; 36 | this.length = -1; 37 | } 38 | 39 | /*{{{ function emptyAll()*/ 40 | /** 41 | * 清空排序结果 42 | * @return {None} 43 | */ 44 | DataMerge.prototype.emptyAll = function(){ 45 | var _self = this; 46 | _self.sortArr = []; 47 | } 48 | /**}}}/ 49 | 50 | /*{{{ function setDistinct()*/ 51 | /** 52 | * 设置distinct 53 | * @param {string} distinct 54 | */ 55 | DataMerge.prototype.setDistinct = function(distinct){ 56 | var _self = this; 57 | if(arguments.length == 0){ 58 | distinct = false; 59 | } 60 | _self.distinct = distinct; 61 | } 62 | /*}}}*/ 63 | 64 | /*{{{ function setEvals()*/ 65 | /** 66 | * 设置Evals 67 | * @param {String} evals 68 | */ 69 | DataMerge.prototype.setEvals = function(evals){ 70 | var _self = this; 71 | _self.evals = []; 72 | for(var key in evals){ 73 | var expr = evals[key]; 74 | _self.evals[key] = QuickEval.create(expr); 75 | } 76 | } 77 | /*}}}*/ 78 | 79 | /*{{{ function setSortKey()*/ 80 | DataMerge.prototype.setSortKey = function(sortKey){ 81 | for(var key in sortKey){ 82 | if('string' == typeof(sortKey[key]) && sortKey[key].trim() == 'DESC'){ 83 | sortKey[key] = -1; 84 | }else{ 85 | sortKey[key] = 1; 86 | } 87 | } 88 | this.sortKey = sortKey; 89 | } 90 | /*}}}*/ 91 | 92 | /*{{{ function setLimit()*/ 93 | DataMerge.prototype.setLimit = function(limitStart,length){ 94 | var _self = this; 95 | if(limitStart <= 0){ 96 | limitStart = 0; 97 | } 98 | _self.limitStart = limitStart; 99 | _self.length = length; 100 | } 101 | /*}}}*/ 102 | 103 | /*{{{ function setGroupBy()*/ 104 | DataMerge.prototype.setGroupBy = function(groupBy){ 105 | var _self = this; 106 | _self.groupBy = groupBy; 107 | } 108 | /*}}}*/ 109 | 110 | /*{{{ function setMerge()*/ 111 | DataMerge.prototype.setMerge = function(groupUse){ 112 | var _self = this; 113 | _self.groupUse = groupUse; 114 | } 115 | /*}}}*/ 116 | 117 | /*{{{ function setHidden()*/ 118 | DataMerge.prototype.setHidden = function(hidden){ 119 | var _self = this; 120 | _self.hidden = hidden; 121 | } 122 | /*}}}*/ 123 | 124 | /*{{{ function filterEvals()*/ 125 | /** 126 | * 表达式求值 127 | * @param {Object} data 128 | * @return {Object} 129 | */ 130 | DataMerge.prototype.filterEvals = function(data){ 131 | var _self = this; 132 | if( empty(_self.evals) ){ 133 | return data; 134 | } 135 | for(var i in data){ 136 | var row = data[i]; 137 | for(var key in _self.evals){ 138 | row[key] = _self.evals[key].execute(row); 139 | } 140 | } 141 | return data; 142 | } 143 | /*}}}*/ 144 | 145 | /*{{{ function filterHidden()*/ 146 | /** 147 | * 过滤结果集隐藏字段 148 | * @param {Object} data 149 | * @return {Object} 150 | */ 151 | DataMerge.prototype.filterHidden = function(data){ 152 | var _self = this; 153 | var count = 0; 154 | for(var i in _self.hidden){ 155 | count ++; 156 | break; 157 | } 158 | if(count === 0){return data;} 159 | for(var i in data){ 160 | var row = data[i]; 161 | for(var j in _self.hidden){ 162 | if(_self.hidden[j] === true){ 163 | delete row[j]; 164 | } 165 | } 166 | } 167 | return data; 168 | } 169 | /*}}}*/ 170 | 171 | /*{{{function filterLimit()*/ 172 | /** 173 | * 按Limit过滤结果集 174 | * @param {Object} data 175 | * @return {Object} 176 | */ 177 | DataMerge.prototype.filterLimit = function(data){ 178 | var _self = this; 179 | if(_self.length < 0){ 180 | return data; 181 | } 182 | return data.slice(_self.limitStart,_self.limitStart+_self.length); 183 | } 184 | /*}}}*/ 185 | 186 | /*{{{ function push()*/ 187 | DataMerge.prototype.push = function(arr){ 188 | var _self = this; 189 | _self.sortArr.push(arr); 190 | } 191 | /*}}}*/ 192 | 193 | /*{{{ function getData()*/ 194 | /** 195 | * 取得Merge结果 196 | * @return {Object} 197 | */ 198 | DataMerge.prototype.getData = function(){ 199 | var _self = this; 200 | if( !empty(_self.groupUse) ){ 201 | return _self.filterHidden(_self.polymerize()); 202 | } 203 | var data = _self.mergeSort(); 204 | if(data.length == 0){ 205 | return []; 206 | } 207 | return _self.filterHidden(_self.filterEvals(_self.filterLimit(data))); 208 | } 209 | /*}}}*/ 210 | 211 | /*{{{ function mergeSort()*/ 212 | /** 213 | * Merge结果集排序 214 | * @return {Object} 215 | */ 216 | DataMerge.prototype.mergeSort = function(){ 217 | var _self = this; 218 | var arr = format(_self.sortArr); 219 | var keys = _self.sortKey; 220 | for(var key in keys){ 221 | arr = kSort(arr,keys); 222 | } 223 | if(_self.distinct){ 224 | return distinct(arr); 225 | } 226 | return arr; 227 | } 228 | /*}}}*/ 229 | 230 | /*{{{ function polymerize()*/ 231 | /** 232 | * 进行merge计算 233 | * @return {Object} 234 | */ 235 | DataMerge.prototype.polymerize = function(){ 236 | var _self = this; 237 | var group = []; 238 | (function(){ 239 | for(var i in _self.sortArr){ 240 | var twoArr = _self.sortArr[i]; 241 | for(var j in twoArr){ 242 | var oneArr = twoArr[j]; 243 | var groupKey = _self.getKey(oneArr); 244 | if(!group[groupKey]){ 245 | group[groupKey] = {}; 246 | for(var key in oneArr){ 247 | group[groupKey][key] = oneArr[key]; 248 | } 249 | }else{ 250 | for(var key in _self.groupUse){ 251 | var type = _self.groupUse[key]; 252 | if(type == REFORM_MAX){ 253 | if(oneArr[key] !== null){ 254 | if(group[groupKey][key] === null){ 255 | group[groupKey][key] = oneArr[key]; 256 | }else{ 257 | group[groupKey][key] = oneArr[key] > group[groupKey][key] ? oneArr[key] : group[groupKey][key]; 258 | } 259 | } 260 | }else if(type == REFORM_MIN){ 261 | if(oneArr[key] !== null){ 262 | if(group[groupKey][key] === null){ 263 | group[groupKey][key] = oneArr[key]; 264 | }else{ 265 | group[groupKey][key] = oneArr[key] < group[groupKey][key] ? oneArr[key] : group[groupKey][key]; 266 | } 267 | } 268 | }else if(type == REFORM_SUM){ 269 | if(oneArr[key] !== null){ 270 | if(group[groupKey][key] === null){ 271 | group[groupKey][key] = oneArr[key]; 272 | }else{ 273 | group[groupKey][key] = Math.round((parseFloat(oneArr[key]) + parseFloat(group[groupKey][key])) * 100) / 100; 274 | } 275 | } 276 | }else if(type == REFORM_CONCAT){ 277 | group[groupKey][key] = (group[groupKey][key] === null ? "":group[groupKey][key]); 278 | if(oneArr[key] === null){ 279 | oneArr[key] = ""; 280 | } 281 | group[groupKey][key] = group[groupKey][key]+","+oneArr[key]; 282 | }else{} 283 | } 284 | } 285 | } 286 | } 287 | for(var i in group){ 288 | for(var j in group[i]){ 289 | group[i][j] = (group[i][j] === null ? 0:group[i][j]); 290 | } 291 | } 292 | })(); 293 | return _self.heapSort(group); 294 | } 295 | /*}}}*/ 296 | 297 | /*{{{ function kSort() */ 298 | /** 299 | * 排序函数，主要运用js内置sort 300 | * 301 | * @param {Array} arr 需要排序的结果集 302 | * @param {Array} keys 按照keys中key的顺序排序 303 | * @return {Array} 304 | */ 305 | function kSort(arr,keys){ 306 | arr = arr.sort(function(a,b){ 307 | for(var key in keys){ 308 | if(a[key] == b[key]){ 309 | continue ; 310 | } 311 | if(a[key] > b[key]){ 312 | return keys[key]; 313 | }else{ 314 | return keys[key] * -1; 315 | } 316 | break; 317 | } 318 | return 0; 319 | }); 320 | return arr; 321 | } 322 | /*}}}*/ 323 | 324 | /*{{{ function format() */ 325 | /** 326 | * 结果集格式化 327 | * @param {Array} arr 328 | * @return {Array} 329 | */ 330 | function format(arr){ 331 | var res = []; 332 | var alen = arr.length; 333 | for(var i = 0; i< alen; i++){ 334 | res = res.concat(arr[i]); 335 | } 336 | return res; 337 | } 338 | /*}}}*/ 339 | 340 | /*{{{ function distinct()*/ 341 | /** 342 | * 对结果集进行distinct处理 343 | * @param {Array} arr 344 | * @return {Array} 345 | */ 346 | function distinct(arr){ 347 | var tmp = []; 348 | var alen = arr.length; 349 | for (var i = 0 ; i< alen; i++){ 350 | var ele = arr[i]; 351 | if(tmp.length == 0){ 352 | tmp.push(ele); 353 | continue; 354 | } 355 | var push = true; 356 | var tlen = tmp.length; 357 | for(var a = 0 ; a < tlen ; a++){ 358 | if(objCompare(tmp[a],ele)){ 359 | push = false; 360 | break; 361 | }else{ 362 | continue; 363 | } 364 | } 365 | if(push){ 366 | tmp.push(ele); 367 | } 368 | } 369 | return tmp; 370 | } 371 | /*}}} */ 372 | 373 | /*{{{ function objCompare()*/ 374 | /** 375 | * 对象比较函数 376 | * @param {Object} obj1 377 | * @param {Object} obj2 378 | * @return {Boolean} 379 | */ 380 | function objCompare(obj1,obj2){ 381 | if(obj1 == null || obj2 == null){ 382 | return false; 383 | } 384 | for(var i in obj1){ 385 | if(obj2[i] != obj1[i]){ return false; } 386 | } 387 | for(var i in obj2){ 388 | if(obj2[i] != obj1[i]){return false;} 389 | } 390 | return true; 391 | } 392 | /*}}}*/ 393 | 394 | /*{{{ getKey()*/ 395 | /** 396 | * 生成用于merge的group key 397 | * @param {Object} row 398 | * @return {String} 399 | */ 400 | DataMerge.prototype.getKey = function(row){ 401 | var _self = this; 402 | var res = []; 403 | for(var i in _self.groupBy){ 404 | var column = _self.groupBy[i]; 405 | res.push(column+","+row[column]); 406 | } 407 | return res.join(","); 408 | } 409 | /*}}}*/ 410 | 411 | /*{{{ function heapSort() */ 412 | /** 413 | * 结果集排序函数，基于ksort 414 | * @param {Array} data 415 | * @return {Array} 416 | */ 417 | DataMerge.prototype.heapSort = function(data){ 418 | var _self = this; 419 | var arr = []; 420 | for(var key in data){ 421 | arr.push(data[key]); 422 | } 423 | arr = _self.filterEvals(arr); 424 | for(var key in _self.sortKey){ 425 | arr = kSort(arr,_self.sortKey); 426 | } 427 | return _self.filterLimit(arr); 428 | } 429 | /*}}}*/ 430 | 431 | /*{{{ exports*/ 432 | exports.ORDER_DESC = ORDER_DESC; 433 | exports.ORDER_ASC = ORDER_ASC; 434 | exports.REFORM_MAX = REFORM_MAX; 435 | exports.REFORM_MIN = REFORM_MIN; 436 | exports.REFORM_SUM = REFORM_SUM; 437 | exports.REFORM_COUNT = REFORM_COUNT; 438 | exports.REFORM_CONCAT = REFORM_CONCAT; 439 | /*}}}*/ 440 | 441 | /*{{{ function create()*/ 442 | exports.create = function(){ 443 | return new DataMerge(); 444 | } 445 | /*}}}*/ 446 | 447 | -------------------------------------------------------------------------------- /lib/quickeval.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: quickeval.js 9 | Author: yixuan (yixuan.zzq@taobao.com) 10 | Description: 表达式计算模块 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | var Lexter = require(__dirname + "/parser/lexter"); 15 | var Hash = require(__dirname + "/../lib/hash"); 16 | var util = require("util"); 17 | 18 | /*{{{ consts*/ 19 | var consts = { 20 | "PI" : Math.PI, 21 | "E" : Math.E, 22 | "KB" : 1024, 23 | "MB" : 1048576, 24 | "GB" : 1073741824, 25 | "TB" : 1099511627776, 26 | }; 27 | /*}}}*/ 28 | 29 | /*{{{ function consts*/ 30 | var functions = { 31 | "ABS" : {"call":function(arg){return arg[0]===null ? 0:Math.abs(parseFloat(arg[0]));},"args":1,"cache":true}, 32 | "CEIL" : {"call":function(arg){return arg[0]===null ? 0:Math.ceil(parseFloat(arg[0]));},"args":1,"cache":true}, 33 | "FLOOR" : {"call":function(arg){return arg[0]===null ? 0:Math.floor(parseFloat(arg[0]));},"args":1,"cache":true}, 34 | "POW" : {"call":function(arg){return arg[0]===null ? 0:Math.pow(parseFloat(arg[0]),parseFloat(arg[1]));},"args":2,"cache":true}, 35 | "EXP" : {"call":function(arg){return arg[0]===null ? 0:Math.exp(parseFloat(arg[0]));},"args":1,"cache":true}, 36 | "LOG" : {"call":function(arg){return arg[0]===null ? 0:Math.log(parseFloat(arg[0]))/parseFloat(Math.log(10));},"args":1,"cache":true}, 37 | "LN" : {"call":function(arg){return arg[0]===null ? 0:Math.log(parseFloat(arg[0]));},"args":1,"cache":true}, 38 | "SQRT" : {"call":function(arg){return arg[0]===null ? 0:Math.sqrt(parseFloat(arg[0]));},"args":1,"cache":true}, 39 | "SIN" : {"call":function(arg){return arg[0]===null ? 0:Math.sin(parseFloat(arg[0]));},"args":1,"cache":true}, 40 | "COS" : {"call":function(arg){return arg[0]===null ? 0:Math.cos(parseFloat(arg[0]));},"args":1,"cache":true}, 41 | "MD5" : {"call":function(arg){return Hash.md5(arg[0]);},"args":1,"cache":true}, 42 | "INT" : {"call":function(arg){return parseInt(arg[0]);},"args":1,"cache":true}, 43 | "LENGTH" : {"call":function(arg){return arg[0].length;},"args":1,"cache":true}, 44 | "ROUND" : {"call":round,"args":1,"cache":true}, 45 | "OPERATE": {"call":operate,"args":3,"cache":true}, 46 | "STATIC": {"call":function(arg){return arg[0];},"args":1,"cache":true}, 47 | "IF" : {"call":function(arg){return arg[0] ? arg[1]:arg[2]},"args":3,"cache":true} 48 | }; 49 | /*}}}*/ 50 | 51 | /*{{{ operator consts*/ 52 | var operators = { 53 | "*" : 2, 54 | "/" : 2, 55 | "%" : 2, 56 | "+" : 3, 57 | "-" : 3, 58 | ">>" : 10, 59 | "<<" : 10, 60 | ">" : 20, 61 | ">=" : 20, 62 | "<" : 20, 63 | "<=" : 20, 64 | "==" : 21, 65 | "!=" : 21, 66 | "&" : 30, 67 | "^" : 31, 68 | "|" : 32, 69 | "&&" : 40, 70 | "||" : 41 71 | } 72 | /*}}}*/ 73 | 74 | /*{{{ QuickEval constructor*/ 75 | /** 76 | * QuickEval对象的构造函数 77 | * @param {String} expression 表达式字符串 78 | * @param {String} charset 字符集 79 | * @return void 80 | */ 81 | var QuickEval = function(expression,charset){ 82 | var _self = this; 83 | _self.token = []; 84 | _self.stack = []; 85 | if(expression instanceof Array){ 86 | _self.token = expression; 87 | }else{ 88 | var lexter = Lexter.create("("+expression+")"); 89 | _self.token = lexter.getAll(); 90 | } 91 | _self.patch(); 92 | _self.prepare(); 93 | _self.optimize(); 94 | } 95 | /*}}}*/ 96 | 97 | /*{{{ execute()*/ 98 | /** 99 | * 执行计算 100 | * @param {Object} variable 表达式中相应变量的变量值 101 | * @return {int} 计算结果 102 | */ 103 | QuickEval.prototype.execute = function(variable){ 104 | var _self = this; 105 | var res = []; 106 | for(key in _self.stack){ 107 | var stack = _self.stack[key]; 108 | var call = stack["call"].toUpperCase(); 109 | if(call == null || !functions[call]){ 110 | throw new Error("Undefined function named as \""+stack["call"]+"\""); 111 | } 112 | call = functions[call]; 113 | args = []; 114 | for(argidx in stack["args"]){ 115 | var arg = stack["args"][argidx]; 116 | switch(arg["type"]){ 117 | case Lexter.types.MEMORY: 118 | args.push(res[arg["text"]]); 119 | break; 120 | case Lexter.types.NUMBER: 121 | case Lexter.types.STRING: 122 | case Lexter.types.OPERATOR: 123 | args.push(arg["text"]); 124 | break; 125 | 126 | default: 127 | var t = (variable[arg["text"]] === null || variable[arg["text"]] === undefined) ? null : variable[arg["text"]]; 128 | args.push(t); 129 | break; 130 | } 131 | } 132 | if(args.length < call["args"]){ 133 | throw new Error("Function "+stack["call"]+" needs "+call["args"]+" args at least,given "+args.length); 134 | } 135 | res[key] = call["call"](args); 136 | } 137 | return res.pop(); 138 | } 139 | /*}}}*/ 140 | 141 | /*{{{ patch()*/ 142 | QuickEval.prototype.patch = function(){ 143 | var _self = this; 144 | var tokens = []; 145 | var last = Lexter.types.OPERATOR; 146 | var text = ""; 147 | for(tokenidx in _self.token){ 148 | var token = _self.token[tokenidx]; 149 | if(token["type"] == Lexter.types.COMMENT){ 150 | continue; 151 | } 152 | if(token["type"] == Lexter.types.NUMBER && token["text"]<0){ 153 | if(text == "(" || text == ","){ 154 | }else{ 155 | tokens.push({type:Lexter.types.OPERATOR,text:"+"}); 156 | } 157 | } 158 | tokens.push(token); 159 | last = token["type"]; 160 | text = token["text"]; 161 | } 162 | for(tidx in tokens){ 163 | tokens[tidx]["mark"] = -1; 164 | } 165 | _self.token = tokens; 166 | } 167 | /*}}}*/ 168 | 169 | /*{{{ prepare() */ 170 | /** 171 | * 表达式分析 172 | * @param empty 173 | * @return void 174 | */ 175 | QuickEval.prototype.prepare = function(){ 176 | var _self = this; 177 | var count = _self.token.length; 178 | var left = count; 179 | var right = 0; 180 | var index = 0; 181 | while(left > 0){ 182 | 183 | /*{{{ find the innest parenthesses*/ 184 | var find = false; 185 | var pos = 0; 186 | for(var i = right + 1;i < count;i++){ 187 | var token = _self.token[i]; 188 | if(token["mark"] != -1){ 189 | continue; 190 | } 191 | if(token["type"] == Lexter.types.COMMAS && token["text"] == ")"){ 192 | pos = i; 193 | break; 194 | } 195 | } 196 | right = pos; 197 | left = (left == count) ? right : left; 198 | 199 | pos = 0; 200 | for(var i = right - 1;i >= 0;i--){ 201 | var token = _self.token[i]; 202 | if(token["mark"] != -1){ 203 | continue; 204 | } 205 | _self.token[i]["mark"] = 0; 206 | if(token["type"] == Lexter.types.COMMAS && token["text"] == "("){ 207 | pos = i; 208 | find = true; 209 | break; 210 | } 211 | } 212 | 213 | left = pos; 214 | if(right == 0){ 215 | left = 0; 216 | right = count-1; 217 | } 218 | /*}}}*/ 219 | 220 | /*{{{ find the calculation with the highest priority*/ 221 | while(true){ 222 | pos = -1; 223 | var max = 65535; 224 | 225 | for(var i = left;i<=right;i++){ 226 | var token = _self.token[i]; 227 | if((token["mark"] != -1 && token["mark"] != 0) || token["type"] == Lexter.types.COMMAS){ 228 | continue; 229 | } 230 | var level = _self.priority(token["text"]); 231 | if(level < max){ 232 | max = level; 233 | pos = i; 234 | } 235 | } 236 | if(pos < 0){ 237 | var begin = left; 238 | if(left > 0 && _self.token[left-1]["type"] == Lexter.types.FUNCTION){ 239 | begin = left - 1; 240 | _self.stack.push({ 241 | "call" : _self.token[begin]["text"], 242 | "args" : _self.get_token_args(begin) 243 | }); 244 | index++; 245 | }else if(_self.token[left]["mark"] === -1 || (right-left) === 2 ){ 246 | var a = [_self.get_token_var(find ? left+1 : left)]; 247 | _self.stack.push({ 248 | "call" : "STATIC", 249 | "args" : a 250 | }); 251 | index++; 252 | } 253 | for(var i = begin;i<=right;i++){ 254 | _self.token[i]["mark"] = index; 255 | } 256 | 257 | break; 258 | } 259 | 260 | _self.stack.push({ 261 | "call" : "OPERATE", 262 | "args" : _self.get_token_args(pos) 263 | }); 264 | index++; 265 | _self.token[pos]["mark"] = index; 266 | 267 | for(i in [-1,1]){ 268 | var offset = [-1,1][i]; 269 | if(_self.token[pos+offset]["mark"] == -1 || _self.token[pos+offset]["mark"] == 0){ 270 | _self.token[pos+offset]["mark"] = index; 271 | }else{ 272 | tmp = _self.token[pos+offset]["mark"]; 273 | for(var j = pos+offset;j>=0 && j>": 503 | return val1 >> parseInt(val2); 504 | 505 | case ">": 506 | return val1 > val2 ? 1 : 0; 507 | 508 | case ">=": 509 | return val1 >= val2 ? 1 : 0; 510 | 511 | case "<": 512 | return val1 < val2 ? 1 : 0; 513 | 514 | case "<=": 515 | return val1 <= val2 ? 1 : 0; 516 | 517 | case "==": 518 | return val1 == val2 ? 1 : 0; 519 | 520 | case "!=": 521 | return val1 != val2 ? 1 : 0; 522 | 523 | case "&&": 524 | return (val1 && val2) ? 1 : 0; 525 | 526 | case "||": 527 | return (val1 || val2) ? 1 : 0; 528 | 529 | default: 530 | break; 531 | } 532 | return null; 533 | } 534 | /*}}}*/ 535 | 536 | /*{{{ round()*/ 537 | /** 538 | * 精度控制 539 | * @param {Array} args 第一个元素是要控制精度的数字，二个是控制精度的位数 540 | * @return {int} 541 | */ 542 | function round(args){ 543 | var intpre = null; 544 | var num = args[0]; 545 | if(args[1] === undefined || args[1] === null){intpre = 2;} 546 | else{intpre = args[1];} 547 | intpre = parseInt(intpre); 548 | var round = Math.pow(10,intpre); 549 | return parseInt(num * round + 0.5)/round; 550 | } 551 | /*}}}*/ 552 | 553 | exports.create = function(expression,charset){ 554 | return new QuickEval(expression,charset); 555 | } 556 | -------------------------------------------------------------------------------- /lib/reform.js: -------------------------------------------------------------------------------- 1 | /* vim: set expandtab tabstop=2 shiftwidth=2 foldmethod=marker: */ 2 | /* 3 | (C) 2011-2012 Alibaba Group Holding Limited. 4 | This program is free software; you can redistribute it and/or 5 | modify it under the terms of the GNU General Public License 6 | version 2 as published by the Free Software Foundation. 7 | 8 | File: reform.js 9 | Author: yixuan.zzq (yixuan.zzq@taobao.com) 10 | Description: sql重组类 11 | Last Modified: 2012-02-07 12 | */ 13 | 14 | require(__dirname + '/env.js'); 15 | var Select = require(__dirname + "/parser/select"); 16 | var Log = require(__dirname + "/log"); 17 | var Decare = require(__dirname + "/decare"); 18 | var Column = require(__dirname + "/column"); 19 | var Lexter = require(__dirname + "/parser/lexter"); 20 | var util = require('util'); 21 | 22 | var nodeMap = []; 23 | 24 | exports.CACHE_PREFIX = "#myfox#reform#node#"; 25 | 26 | var relateMap = []; 27 | relateMap[Select.WHERE.EQ] = "="; 28 | relateMap[Select.WHERE.GT] = ">"; 29 | relateMap[Select.WHERE.GE] = ">="; 30 | relateMap[Select.WHERE.LT] = "<"; 31 | relateMap[Select.WHERE.LE] = "<="; 32 | relateMap[Select.WHERE.NE] = "!="; 33 | relateMap[Select.WHERE.BETWEEN] = "BETWEEN"; 34 | relateMap[Select.WHERE.IN] = "IN"; 35 | relateMap[Select.WHERE.NOTIN] = "NOT IN"; 36 | relateMap[Select.WHERE.LIKE] = "LIKE"; 37 | relateMap[Select.WHERE.NOTLIKE] = "NOT LIKE"; 38 | relateMap[Select.WHERE.ISNULL] = "IS NULL"; 39 | relateMap[Select.WHERE.NOTNULL] = "IS NOT NULL"; 40 | 41 | var Reform = function(query,tbnode,reqObj){ 42 | this.query = query; 43 | this.distinct = null; 44 | this.tbnode = tbnode; 45 | this.reqObj = reqObj; 46 | this.output = []; 47 | this.column = []; 48 | this.groups = []; 49 | this.orders = []; 50 | this.limits = null; 51 | this.colmap = []; 52 | this.wheres = []; 53 | } 54 | 55 | /*{{{ distinct()*/ 56 | /** 57 | * 是否带有distinct 58 | * @param empty 59 | * @return {boolean} 60 | */ 61 | Reform.prototype.dist = function(){ 62 | 63 | if(this.distinct && this.distinct.text.toUpperCase()=="DISTINCT"){ 64 | return true; 65 | } 66 | return false; 67 | } 68 | /*}}}*/ 69 | 70 | /*{{{ result()*/ 71 | /** 72 | * 生成分片路由信息主方法 73 | * @param empty 74 | * @return {Object} 路由结果对象 75 | */ 76 | Reform.prototype.result = function(){ 77 | var stack = this.query; 78 | var merged = []; 79 | var existJoin = false; 80 | 81 | for(var i in stack["tables"]){ 82 | merged[i] = stack["tables"][i]; 83 | } 84 | for(var i in stack["joinmap"]){ 85 | existJoin = true; 86 | merged[i] = stack["joinmap"][i]; 87 | } 88 | existJoin = existJoin | (!ANALYSIZE_UNIQUEKEY); 89 | 90 | var tables = this.reformTable(merged,existJoin); 91 | if(typeof(tables) == 'string'){ 92 | return tables; 93 | } 94 | 95 | var wheres = this.reformWhere(stack["where"]); 96 | var orders = this.buildColumn(stack["columns"],stack["orderby"]); 97 | var gb; 98 | if(this.groups.length == 0){ 99 | gb = ""; 100 | }else{ 101 | gb = " GROUP BY "+this.groups.join(","); 102 | } 103 | 104 | var normalPattern,uniquePattern; 105 | if(orders.length === 0){ 106 | normalPattern = "SELECT "+this.column.join(",")+" FROM {TABLES}{WHERES}"+gb+this.reformLimits(stack["limits"]); 107 | uniquePattern = "SELECT "+removeFunc(this.column.join(","))+" FROM {TABLES}{WHERES}"+gb+this.reformLimits(stack["limits"]); 108 | }else{ 109 | normalPattern = "SELECT "+this.column.join(",")+" FROM {TABLES}{WHERES}"+gb+" ORDER BY "+orders.join(",")+this.reformLimits(stack["limits"]); 110 | uniquePattern = "SELECT "+removeFunc(this.column.join(","))+" FROM {TABLES}{WHERES}"+gb+" ORDER BY "+removeFunc(orders.join(","))+this.reformLimits(stack["limits"]); 111 | } 112 | 113 | var tabColMap = []; 114 | if(!existJoin){ 115 | tabColMap = getColumnMap(); 116 | } 117 | 118 | var reform = []; 119 | var table,where,tw,getSrvs; 120 | for(var i = 0;i < tables.length; i++){ 121 | table = tables[i]; 122 | where = []; 123 | tw = table.where ? table.where : ""; 124 | if(tw!=""){where.push(tw);} 125 | if(wheres.length !== 0){where.push(wheres);} 126 | 127 | var uni = false; 128 | if(!existJoin){ 129 | uni = isUnique(tabColMap,table); 130 | } 131 | 132 | if(uni){ 133 | var originalSql = (normalPattern.replace(/{TABLES}/,table.table).replace(/{WHERES}/,where.length == 0 ? "" : " WHERE "+where.join(" AND "))).trim(); 134 | var changedSql = (uniquePattern.replace(/{TABLES}/,table.table).replace(/{WHERES}/,where.length == 0 ? "" : " WHERE "+where.join(" AND "))).trim(); 135 | workerLogger.warning("UNIQUE_KEY|token:"+this.reqObj.token,"original:"+originalSql+"|changed:"+changedSql); 136 | 137 | reform.push({ 138 | host : table.hosts.join(","), 139 | time : table.time, 140 | sql : USE_UNIQUEKEY ? changedSql : originalSql 141 | }); 142 | }else{ 143 | reform.push({ 144 | host : table.hosts.join(","), 145 | time : table.time, 146 | sql : (normalPattern.replace(/{TABLES}/,table.table).replace(/{WHERES}/,where.length == 0 ? "" : " WHERE "+where.join(" AND "))).trim() 147 | }); 148 | } 149 | } 150 | 151 | return { 152 | 'distinct' : this.dist(), 153 | 'columns' : this.output, 154 | 'groups' : this.groups, 155 | 'orders' : this.orders, 156 | 'limits' : this.limits, 157 | 'route' : reform 158 | } 159 | } 160 | /*}}}*/ 161 | 162 | /*{{{ buildColumn()*/ 163 | /** 164 | * 重组分片sql中的列，并且重组分片sql中的orderby字段 165 | * @param {Object} stack 原始sql语句中的column部分内容 166 | * @param {Object} orders 原始sql中的orderby字段内容 167 | * @return {String} 168 | */ 169 | Reform.prototype.buildColumn = function(stack,orders){ 170 | var dist = false; 171 | var s; 172 | for(var name in stack){ 173 | s = stack[name]; 174 | dist = dist ? dist : s.dist; 175 | Column.build(s.expr,name,s.dist); 176 | } 177 | this.output = Column.transform(); 178 | this.orders = {}; 179 | var colmap = Column.maps(); 180 | var res = []; 181 | 182 | var order,cl,al,value; 183 | var orderLen = orders.length; 184 | var op = this.output; 185 | for(var i = 0;i < orderLen;i++){ 186 | order = orders[i]; 187 | cl = Lexter.text(order.expr).join(""); 188 | al = cl; 189 | if(!op[cl] && !op["*"]){ 190 | if(!colmap[cl]){ 191 | Column.build(order.expr,cl,"",true); 192 | }else{ 193 | al = colmap[cl]; 194 | } 195 | } 196 | 197 | value = order.type == Select.ORDER.DESC ? "DESC" : "ASC"; 198 | this.orders[al] = value; 199 | res.push(cl+" "+value); 200 | } 201 | 202 | this.distinct = dist; 203 | this.column = Column.getAll(this.groups); 204 | this.output = Column.transform(); 205 | 206 | return res; 207 | } 208 | /*}}}*/ 209 | 210 | /*{{{ reformTable()*/ 211 | /** 212 | * 整合分片sql的表字段和部分where字段（部分是指只是where字段中的路由字段） 213 | * @param {Object} tables 查询牵涉到的各个表 214 | * @return {Array} 每个表的具体路由节点等信息 215 | */ 216 | Reform.prototype.reformTable = function(tables,existJoin){ 217 | var routes = {}; 218 | var tab,tbname,tabNodeInfo,len,tmp,tmpLen,table,run; 219 | for(var alias in tables){ 220 | routes[alias] = []; 221 | tab = tables[alias]; 222 | tbname = tab.table; 223 | if(!this.tbnode[tbname]){ 224 | return "Undefined table named as "+tbname; 225 | } 226 | 227 | tabNodeInfo = this.tbnode[tbname]; 228 | len = tabNodeInfo.length; 229 | table = routes[alias]; 230 | for(var i = 0;i < len; i++){ 231 | tmp = tabNodeInfo[i]; 232 | tmpLen = tmp.length; 233 | for(var j = 0;j < tmpLen; j++){ 234 | run = tmp[j]; 235 | table.push({ 236 | table : run.real_table, 237 | time : run.modtime ? run.modtime : 0, 238 | hosts : hostSplit(run.hosts_list), 239 | route : run.route_val ? run.route_val : null, 240 | uniqueKey : run.unique_key, 241 | joins : tab.method ? tab.method : null, 242 | where : tab.where ? tab.where : null 243 | }); 244 | } 245 | } 246 | } 247 | var decare = Decare.create(); 248 | for(var alias in routes){ 249 | decare.register(alias,routes[alias]); 250 | } 251 | var res = []; 252 | var result = decare.cal(); 253 | var len = result.length; 254 | var run,hosts,mtime,table,joins,where,configRoute; 255 | for(var i = 0;i < len; i++){ 256 | run = result[i]; 257 | if(!run){continue;} 258 | hosts = []; 259 | mtime = 0; 260 | table = []; 261 | joins = []; 262 | where = []; 263 | unique = []; 264 | /*{{{*/ 265 | var config,nodeLen; 266 | for(var alias in run){ 267 | config = run[alias]; 268 | nodeLen = hosts.length; 269 | var tmp; 270 | if(nodeLen != 0){ 271 | tmp = []; 272 | var cn = config.hosts; 273 | var cnLen = cn.length; 274 | for(var j = 0;j < nodeLen; j++){ 275 | for(var k = 0;k < cnLen;k++){ 276 | if(hosts[j] == cn[k]){ 277 | tmp.push(hosts[j]); 278 | break; 279 | } 280 | } 281 | } 282 | hosts = tmp; 283 | }else{ 284 | hosts = config.hosts ? config.hosts : []; 285 | } 286 | if(hosts.length == 0){console.log("Crossing nodes between joined tables");} 287 | if(config.time){ 288 | mtime = mtime > config.time ? mtime : config.time; 289 | } 290 | configRoute = config.route; 291 | if(configRoute){ 292 | var t,configRouteVal; 293 | for(var key in configRoute){ 294 | t = alias+"."+key; 295 | configRouteVal = configRoute[key]; 296 | where.push(t+" = "+configRouteVal); 297 | this.wheres[t] = configRouteVal; 298 | this.wheres[key] = configRouteVal; 299 | } 300 | } 301 | if(!config.joins){ 302 | table.push(config.table+" AS "+alias); 303 | if(!existJoin){ 304 | unique[alias] = config.uniqueKey.split(";"); 305 | if(unique[alias].length !== 0){ 306 | var last = unique[alias].pop(); 307 | if(last.charAt(last.length-1) === "$"){ 308 | unique[alias].push(last.substr(0,last.length-1)); 309 | } 310 | } 311 | } 312 | }else{ 313 | joins.push(config.joins+" "+config.table+" AS "+alias+" ON "+config.where); 314 | } 315 | } 316 | /*}}}*/ 317 | res.push({ 318 | hosts : hosts, 319 | time : mtime, 320 | uniqueKey : unique, 321 | table : (table.join(",")+" "+joins.join(" ")).trim(), 322 | where : where.join(" AND ") 323 | }); 324 | } 325 | 326 | return res; 327 | } 328 | /*}}}*/ 329 | 330 | /*{{{ reformWhere()*/ 331 | /** 332 | * 整合分片sql中非路由字段的where字段 333 | * @param {Array} stack 原始sql语句的where字段内容 334 | * @return {String} 分片sql的部分where 335 | */ 336 | Reform.prototype.reformWhere = function(stack){ 337 | var wheres = []; 338 | var op,t,col,values; 339 | var whereLen = stack.length; 340 | for(var i = 0;i < whereLen;i++){ 341 | op = stack[i]; 342 | t = []; 343 | if(op.dbname){t.push(op.dbname);} 344 | if(op.tbname){t.push(op.tbname);} 345 | if(op.column){t.push(op.column.text);} 346 | col = t.join("."); 347 | if(this.wheres[col]){continue;} 348 | 349 | switch(op.relate){ 350 | case Select.WHERE.BETWEEN: 351 | values = Lexter.text(op.values," AND "); 352 | break; 353 | case Select.WHERE.IN: 354 | case Select.WHERE.NOTIN: 355 | values = Lexter.text(op.values,","); 356 | values.unshift("("); 357 | values.push(")"); 358 | break; 359 | default: 360 | values = Lexter.text(op.values); 361 | break; 362 | } 363 | wheres.push(col+" "+relateMap[op.relate]+" "+values.join("")); 364 | } 365 | if(wheres.length == 0){ 366 | return ""; 367 | } 368 | return wheres.join(" AND "); 369 | } 370 | /*}}}*/ 371 | 372 | /*{{{ reformLimits()*/ 373 | /** 374 | * 整合分片sql中的limit字段 375 | * @param {Array} stack 原始sql中的limit字段内容 376 | * @param {String} 377 | */ 378 | Reform.prototype.reformLimits = function(stack){ 379 | if(!stack || stack.length == 0){ 380 | this.limits = null; 381 | return ""; 382 | } 383 | this.limits = { 384 | offset : stack[0].text, 385 | length : stack[1].text 386 | }; 387 | 388 | return " LIMIT 0, "+(stack[0].text+stack[1].text); 389 | } 390 | /*}}}*/ 391 | 392 | /*{{{ hostSplit()*/ 393 | /** 394 | * 过滤host列表是否溢出 395 | * @param {String} str 含有host节点的string 396 | * @return {Array} 过滤后的host节点数组 397 | */ 398 | function hostSplit(str){ 399 | var res = []; 400 | if(str.charAt(str.length-1) === "$"){ 401 | var get = str.substr(0,str.length-1).split(","); 402 | for(var i = 0;i < get.length; i++){ 403 | if(!isNaN(get[i])){res.push(get[i]);} 404 | } 405 | }else{ 406 | var get = str.substr(0,str.length).pop(); 407 | for(var i = 0;i < get.length; i++){ 408 | if(!isNaN(get[i])){res.push(get[i]);} 409 | } 410 | } 411 | return res; 412 | } 413 | /*}}}*/ 414 | 415 | /*{{{ removeFunc()*/ 416 | /** 417 | * 去除SUM等方法 418 | * @param {String} str 需要处理的字符串 419 | * @return {String} 过滤好后的字符串 420 | */ 421 | function removeFunc(str){ 422 | var exp = new RegExp("((SUM)|(AVG)|(MAX)|(MIN))\\(","i"); 423 | var res = ""; 424 | 425 | while(true){ 426 | var pos = str.search(exp); 427 | if(pos === -1){ 428 | return res+str; 429 | } 430 | if(str.charAt(pos-1) !== undefined && str.charAt(pos-1).match(/\w/i)){ 431 | continue; 432 | } 433 | var lev = 0; 434 | for(var i = pos;i < str.length;i++){ 435 | if(str.charAt(i) === "("){ 436 | lev++; 437 | continue; 438 | } 439 | if(str.charAt(i) === ")" && --lev === 0){ 440 | res += str.substr(0,pos) + str.substr(pos+4,i-pos-4); 441 | str = str.substr(i+1); 442 | break; 443 | } 444 | } 445 | } 446 | } 447 | /*}}}*/ 448 | 449 | /*{{{ getColumnMap()*/ 450 | /** 451 | * 生成一张表名对应字段名的map 452 | * @return Array 453 | */ 454 | function getColumnMap(){ 455 | var tabColMap = []; 456 | var selectColumn = Column.getSelectColumn(); 457 | 458 | for(var i in selectColumn){ 459 | if(selectColumn[i].dist !== undefined){ 460 | var split = selectColumn[i].expr.split("."); 461 | if(split.length === 2){ 462 | if(tabColMap[split[0]] === undefined){ 463 | tabColMap[split[0]] = []; 464 | } 465 | tabColMap[split[0]].push(split[1]); 466 | }else{ 467 | if(tabColMap[""] === undefined){ 468 | tabColMap[""] = []; 469 | } 470 | tabColMap[""].push(split[0]); 471 | } 472 | } 473 | } 474 | 475 | return tabColMap; 476 | } 477 | /*}}}*/ 478 | 479 | /*{{{ isUnique()*/ 480 | function isUnique(tabColMap,table){ 481 | if(tabColMap[''] !== undefined){ 482 | var count = 0; 483 | for(var i in tabColMap){ 484 | if(++count > 1){ 485 | return false; 486 | } 487 | } 488 | for(var j in table["uniqueKey"]){ 489 | var tmp = table["uniqueKey"][j]; 490 | for(var k in tabColMap[""]){ 491 | if(isInArray(tmp,tabColMap[""][k])){ 492 | return true; 493 | } 494 | } 495 | } 496 | }else{ 497 | for(var j in table["uniqueKey"]){ 498 | var tmp = table["uniqueKey"][j]; 499 | for(var k in tabColMap[j]){ 500 | if(isInArray(tmp,tabColMap[j][k])){ 501 | return true; 502 | } 503 | } 504 | } 505 | } 506 | return false; 507 | } 508 | /*}}}*/ 509 | 510 | /*{{{ isInArray()*/ 511 | /** 512 | * 判断元素是否在数组中 513 | * @param {Array} arr 数组 514 | * @param {string|int} element 所要判断元素 515 | * @return {boolean} 是否存在 516 | */ 517 | function isInArray(arr,element){ 518 | for(var i in arr){ 519 | if(arr[i] === element){return true;} 520 | } 521 | return false; 522 | } 523 | /*}}}*/ 524 | 525 | /*{{{ create()*/ 526 | /** 527 | * 创建Reform对象方法 528 | * @param {Array} query sql解析后的token 529 | * @param {Object} tbnode 每个表的路由信息 530 | * @return {Object} Reform类对象 531 | */ 532 | exports.create = function(query,tbnode,reqObj){ 533 | Column.init(); 534 | return new Reform(query,tbnode,reqObj); 535 | } 536 | /*}}}*/ 537 | 538 | exports.cleanNodeMap = function(){ 539 | nodeMap = []; 540 | } 541 | --------------------------------------------------------------------------------