├── README.md ├── mglory ├── .classpath ├── .project ├── .settings │ ├── org.eclipse.core.resources.prefs │ └── org.eclipse.jdt.core.prefs ├── bulider.xml ├── lib │ ├── commons-logging-1.1.1.jar │ ├── log4j-1.2.15.jar │ └── netty-all-4.0.18.Final.jar └── src │ ├── com │ ├── lingo │ │ └── mglory │ │ │ ├── job │ │ │ └── JobExecutor.java │ │ │ ├── listener │ │ │ ├── ExitSystemExecutor.java │ │ │ └── StartMglory.java │ │ │ ├── mq │ │ │ ├── MQReceiver.java │ │ │ ├── MQStore.java │ │ │ ├── MgloryMQ.java │ │ │ └── MsgExecutorTask.java │ │ │ ├── param │ │ │ ├── MgloryIn.java │ │ │ ├── MgloryOut.java │ │ │ └── WorkFlowDAG.java │ │ │ ├── resource │ │ │ ├── ElectManager.java │ │ │ ├── NodeManager.java │ │ │ ├── ResourceHelper.java │ │ │ ├── ResourceManager.java │ │ │ ├── ResourcePool.java │ │ │ ├── ResourceScheduler.java │ │ │ ├── ResourceUsage.java │ │ │ └── SystemResource.java │ │ │ ├── route │ │ │ ├── MgloryHash.java │ │ │ ├── MgloryHashHelper.java │ │ │ └── Node.java │ │ │ ├── server │ │ │ ├── MgloryGroupServer.java │ │ │ └── MgloryWorkerServer.java │ │ │ ├── store │ │ │ ├── MgloryMap.java │ │ │ ├── MgloryMapManager.java │ │ │ └── MgloryMapStore.java │ │ │ ├── task │ │ │ └── TaskState.java │ │ │ ├── transport │ │ │ ├── client │ │ │ │ └── Client.java │ │ │ ├── handler │ │ │ │ ├── ClientHandler.java │ │ │ │ └── ServerHandler.java │ │ │ └── server │ │ │ │ └── Server.java │ │ │ └── util │ │ │ ├── PropertiesUtil.java │ │ │ └── Utils.java │ └── test │ │ ├── MQTest.java │ │ ├── MgloryMapTest.java │ │ ├── ResourceUsageTest.java │ │ ├── Test.java │ │ ├── Test4.java │ │ ├── Test5.java │ │ ├── WordCount.java │ │ └── WordCount2.java │ ├── config.properties │ └── log4j.properties └── mglory简介.doc /README.md: -------------------------------------------------------------------------------- 1 | # mglory 2 | mglory是一个java分布式框架。具备以下功能：分布式计算、分布式内存存储(支持持久化)、分布式消息队列 3 | -------------------------------------------------------------------------------- /mglory/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /mglory/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | mglory 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | 15 | org.eclipse.jdt.core.javanature 16 | 17 | 18 | -------------------------------------------------------------------------------- /mglory/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding/=UTF-8 3 | -------------------------------------------------------------------------------- /mglory/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled 3 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 4 | org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve 5 | org.eclipse.jdt.core.compiler.compliance=1.6 6 | org.eclipse.jdt.core.compiler.debug.lineNumber=generate 7 | org.eclipse.jdt.core.compiler.debug.localVariable=generate 8 | org.eclipse.jdt.core.compiler.debug.sourceFile=generate 9 | org.eclipse.jdt.core.compiler.problem.assertIdentifier=error 10 | org.eclipse.jdt.core.compiler.problem.enumIdentifier=error 11 | org.eclipse.jdt.core.compiler.source=1.6 12 | -------------------------------------------------------------------------------- /mglory/bulider.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | uniTrade jar builder 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /mglory/lib/commons-logging-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lingo623/mglory/a89bf52fd91698de799736f21cb772c8c476abe9/mglory/lib/commons-logging-1.1.1.jar -------------------------------------------------------------------------------- /mglory/lib/log4j-1.2.15.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lingo623/mglory/a89bf52fd91698de799736f21cb772c8c476abe9/mglory/lib/log4j-1.2.15.jar -------------------------------------------------------------------------------- /mglory/lib/netty-all-4.0.18.Final.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lingo623/mglory/a89bf52fd91698de799736f21cb772c8c476abe9/mglory/lib/netty-all-4.0.18.Final.jar -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/job/JobExecutor.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.job; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.concurrent.Callable; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.FutureTask; 11 | 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | 15 | import com.lingo.mglory.param.WorkFlowDAG; 16 | import com.lingo.mglory.param.MgloryIn; 17 | import com.lingo.mglory.param.MgloryOut; 18 | import com.lingo.mglory.resource.ResourceHelper; 19 | import com.lingo.mglory.task.TaskState; 20 | import com.lingo.mglory.transport.client.Client; 21 | 22 | public class JobExecutor { 23 | private static Log log = LogFactory.getLog(JobExecutor.class); 24 | private String masterGroup;//中心服务（资源管理器）主机地址 如：114.215.178.115:1099 25 | private String slaveGroup;//中心服务（资源管理器）备机地址 26 | private WorkFlowDAG dag; 27 | private boolean isInterrupted=false; 28 | private static ExecutorService taskExecutor=null; 29 | static 30 | { 31 | taskExecutor=Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); 32 | } 33 | public JobExecutor(String masterGroup,String slaveGroup) 34 | { 35 | this.masterGroup=masterGroup; 36 | this.slaveGroup=slaveGroup; 37 | } 38 | /** 39 | * 任务执行 40 | * @param in 41 | * @return 42 | */ 43 | public MgloryOut execute(MgloryIn in) 44 | { 45 | return execute(in,""); 46 | } 47 | /** 48 | * 任务执行 49 | * @param in 50 | * @param execResource 51 | * @return 52 | */ 53 | public MgloryOut execute(MgloryIn in,String execResource) 54 | { 55 | String taskId=in.getTaskId(); 56 | MgloryOut out =new MgloryOut(); 57 | out.setTaskId(taskId); 58 | try 59 | { 60 | String resource=execResource; 61 | if (resource.equals("")) 62 | { 63 | resource=ResourceHelper.getInstance().getIdleResource(masterGroup, slaveGroup); 64 | } 65 | dag.setTaskExcutor(taskId,resource); 66 | String ip=resource.split(":")[0]; 67 | int port=Integer.parseInt(resource.split(":")[1]); 68 | out=Client.getInstance().call(ip, port,in); 69 | out.setTaskId(taskId); 70 | } 71 | catch(Exception e) 72 | { 73 | out.setTaskId(taskId); 74 | out.setStatus(MgloryOut.EXCEPTION); 75 | out.setErrorMsg(e.getMessage()); 76 | log.warn("忽略执行任务异常："+e); 77 | } 78 | return out; 79 | } 80 | /** 81 | * 任务执行 82 | * @param in 83 | * @param dag 84 | * @return 85 | * @throws Exception 86 | */ 87 | public MgloryOut execute(MgloryIn in,WorkFlowDAG dag) throws Exception 88 | { 89 | MgloryOut out =new MgloryOut(); 90 | if (dag==null || dag.getTaskMap().isEmpty()) 91 | { 92 | out.setErrorMsg("空DAG无须执行"); 93 | return out; 94 | } 95 | this.dag=dag; 96 | preDealForStartTaskNode(in); 97 | execute("start"); 98 | if (isInterrupted) 99 | { 100 | interrupt(); 101 | } 102 | out=dag.getTaskResult().get("end"); 103 | return out; 104 | } 105 | private void execute(String taskId) 106 | { 107 | Map needDealTaskMap=getNeedDealTask(taskId); 108 | if (!needDealTaskMap.isEmpty()) 109 | { 110 | //end任务节点不处理 111 | if(needDealTaskMap.containsKey("end")) 112 | { 113 | dag.setTaskState(taskId,TaskState.COMPLETE); 114 | //taskExecutor.shutdown(); 115 | return; 116 | } 117 | //ExecutorService taskExecutor=Executors.newFixedThreadPool(needDealTaskMap.size()); 118 | List> futureTasks = new ArrayList>(); 119 | for (final String waitExecuteTaskId:needDealTaskMap.keySet()) 120 | { 121 | final String taskClazzName=dag.getTaskMap().get(waitExecuteTaskId).get("taskClazzName"); 122 | final List preTaskList=dag.getPreTaskMap().get(waitExecuteTaskId); 123 | final String operMethod=dag.getTaskMap().get(waitExecuteTaskId).get("operMethod"); 124 | final long timeout=Long.parseLong(dag.getTaskMap().get(waitExecuteTaskId).get("timeout")); 125 | dag.setTaskState(waitExecuteTaskId,TaskState.RUNNING); 126 | FutureTask futureTask = new FutureTask( 127 | new Callable() { 128 | @Override 129 | public MgloryOut call() throws Exception { 130 | String execResource=""; 131 | String locationType=MgloryOut.RESULT; 132 | MgloryIn in=new MgloryIn(); 133 | in.setTaskId(waitExecuteTaskId); 134 | in.setInvokeMethod(taskClazzName+"."+operMethod); 135 | Map inMap=new HashMap(); 136 | for (String preTaskId:preTaskList) 137 | { 138 | MgloryOut out=dag.getTaskResult().get(preTaskId); 139 | inMap.put(preTaskId,out); 140 | if (out.getStoreLocation().equals(MgloryOut.LOCAL_MEM) && !locationType.equals(MgloryOut.LOCAL_FILE) 141 | && execResource.equals("")) 142 | { 143 | execResource=out.getMap().get("locationAddress")!=null?out.getMap().get("locationAddress").toString():""; 144 | } 145 | else if (out.getStoreLocation().equals(MgloryOut.LOCAL_FILE) && !locationType.equals(MgloryOut.LOCAL_FILE)) 146 | { 147 | if (out.getMap().get("locationAddress")!=null) 148 | { 149 | execResource=out.getMap().get("locationAddress").toString(); 150 | locationType=MgloryOut.LOCAL_FILE; 151 | } 152 | } 153 | } 154 | in.setMap(inMap); 155 | in.setTimeout(timeout); 156 | return execute(in,execResource); 157 | } 158 | }); 159 | futureTasks.add(futureTask); 160 | taskExecutor.submit(futureTask); 161 | } 162 | for (FutureTask futureTask : futureTasks) { 163 | try { 164 | if (!isInterrupted) 165 | { 166 | MgloryOut out=futureTask.get(); 167 | if (out.getStatus().equals(MgloryOut.NORMAL)) 168 | { 169 | dag.setTaskState(out.getTaskId(),TaskState.COMPLETE); 170 | dag.setTaskResult(out.getTaskId(),out); 171 | if (dag.getNextTaskMap().get(out.getTaskId())!=null && dag.getNextTaskMap().get(out.getTaskId()).get(0).equals("end")) 172 | { 173 | dag.setTaskResult("end",out); 174 | } 175 | execute(out.getTaskId()); 176 | } 177 | else 178 | { 179 | dag.setTaskState(out.getTaskId(),TaskState.ERROR); 180 | dag.setTaskResult(out.getTaskId(),out); 181 | dag.setTaskResult("end",out); 182 | //终止任务执行 183 | if(!futureTask.isCancelled()) 184 | { 185 | futureTask.cancel(true); 186 | } 187 | isInterrupted=true; 188 | } 189 | } 190 | else 191 | { 192 | //终止任务执行 193 | if(!futureTask.isCancelled()) 194 | { 195 | futureTask.cancel(true); 196 | } 197 | } 198 | 199 | } catch (Exception e) { 200 | if (!isInterrupted) 201 | { 202 | MgloryOut out=new MgloryOut(); 203 | out.setStatus(MgloryOut.EXCEPTION); 204 | out.setErrorMsg(e.getMessage()); 205 | dag.setTaskResult("end",out); 206 | isInterrupted=true; 207 | log.warn("忽略执行任务异常："+e); 208 | } 209 | 210 | } 211 | } 212 | //taskExecutor.shutdown(); 213 | } 214 | } 215 | private void preDealForStartTaskNode(MgloryIn in) 216 | { 217 | dag.setTaskState("start",TaskState.COMPLETE); 218 | MgloryOut startOut =new MgloryOut(); 219 | startOut.setTaskId("start"); 220 | startOut.setMap(in.getMap()); 221 | dag.setTaskResult("start", startOut); 222 | } 223 | private Map getNeedDealTask(String taskId) 224 | { 225 | Map needDealTaskMap=new HashMap (); 226 | //获取孩子节点 227 | List nextTaskList=dag.getNextTaskMap().get(taskId); 228 | if (nextTaskList!=null && nextTaskList.size()>0) 229 | { 230 | for (int i=0;i preTaskList=dag.getPreTaskMap().get(childTaskId); 234 | //判断是否可以执行 235 | boolean flag=true; 236 | for (int j=0;j program exit now."); 18 | System.exit(0); 19 | } 20 | String startType=args[0].trim(); 21 | String lockName=""; 22 | //集团服务 23 | if (startType.equals("group")) 24 | { 25 | try 26 | { 27 | MgloryGroupServer.getInstance().startGroupServer(); 28 | NodeManager.getInstance().startHeart(true); 29 | } 30 | catch (Exception e) 31 | { 32 | System.err.println("start mgloryGroupServer failed, program exit now."); 33 | System.err.println(e); 34 | System.exit(0); 35 | } 36 | if (MgloryGroupServer.getInstance().getMasterFlag()) 37 | { 38 | lockName="localhost_"+MgloryGroupServer.getInstance().getPort()+"_group_master"; 39 | } 40 | else 41 | { 42 | lockName="localhost__"+MgloryGroupServer.getInstance().getPort()+"_group_slave"; 43 | } 44 | } 45 | else 46 | { 47 | if (args.length < 4) { 48 | System.err.println("Usage:\tcom.lingo.mglory.server.StartMglory program exit now."); 49 | System.exit(0); 50 | } 51 | String ip=args[1].trim(); 52 | String port=args[2].trim(); 53 | String resourceType=args[3].trim(); 54 | lockName="localhost__"+port+"_worker"; 55 | try 56 | { 57 | MgloryWorkerServer.getInstance().startResource(ip, Integer.parseInt(port),resourceType); 58 | NodeManager.getInstance().startHeart(false); 59 | } 60 | catch (Exception e) 61 | { 62 | System.err.println("start mgloryWorkerServer failed, program exit now."); 63 | System.err.println(e); 64 | System.exit(0); 65 | } 66 | } 67 | // 根据当前输入参数的文件名生成lock锁文件 68 | File flock = new File("./lock", lockName + ".lock"); 69 | if (flock.exists()) { 70 | System.out.println("Lock file " + flock.getName() + " \t exists already,program continue."); 71 | System.out.println("flock :" + flock.getAbsolutePath()); 72 | } else { 73 | try { 74 | if (flock.createNewFile()) { 75 | System.out.println("Lock successfully created"); 76 | } else { 77 | System.out.println("Create Lock file failed,but this program continue"); 78 | } 79 | } catch (IOException e) { 80 | System.err.println("Create File Lock failed, program exit now."); 81 | System.err.println(e); 82 | System.exit(0); 83 | } 84 | } 85 | ScheduledExecutorService executorService = Executors.newScheduledThreadPool(1); 86 | ExitSystemExecutor exitSystemExecutor=new ExitSystemExecutor(flock); 87 | executorService.scheduleAtFixedRate(exitSystemExecutor, 1, 1, TimeUnit.SECONDS); 88 | System.out.println("mglory start complete......"); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/mq/MQReceiver.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.mq; 2 | 3 | import java.util.Map; 4 | 5 | import com.lingo.mglory.param.MgloryOut; 6 | 7 | public class MQReceiver{ 8 | public MgloryOut receive(String taskId,Map inMap)throws Exception 9 | { 10 | MgloryOut out =new MgloryOut(); 11 | MQStore.getInstance().msgQueue.add(inMap); 12 | return out; 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/mq/MQStore.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.mq; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Properties; 6 | import java.util.concurrent.Callable; 7 | import java.util.concurrent.ConcurrentHashMap; 8 | import java.util.concurrent.ConcurrentLinkedQueue; 9 | import java.util.concurrent.ConcurrentMap; 10 | import java.util.concurrent.ExecutorService; 11 | import java.util.concurrent.Executors; 12 | import java.util.concurrent.FutureTask; 13 | import java.util.concurrent.ScheduledExecutorService; 14 | import java.util.concurrent.TimeUnit; 15 | 16 | import org.apache.commons.logging.Log; 17 | import org.apache.commons.logging.LogFactory; 18 | 19 | import com.lingo.mglory.param.MgloryOut; 20 | import com.lingo.mglory.store.MgloryMap; 21 | import com.lingo.mglory.util.PropertiesUtil; 22 | 23 | 24 | public class MQStore { 25 | private static Log log = LogFactory.getLog(MQStore.class); 26 | private static MQStore mqStore; 27 | private Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 28 | public ConcurrentLinkedQueue> msgQueue=new ConcurrentLinkedQueue>(); 29 | //存储已经发送的消费者（消息整体没发送完） 30 | private ConcurrentMap> haveConsumedMap =new ConcurrentHashMap>(); 31 | private Map> topicMap =new HashMap>(); 32 | private MQStore() 33 | { 34 | init(); 35 | } 36 | public synchronized static MQStore getInstance() 37 | { 38 | if (mqStore==null) 39 | { 40 | mqStore=new MQStore(); 41 | } 42 | return mqStore; 43 | } 44 | private void init() 45 | { 46 | getTopic(); 47 | ScheduledExecutorService dealExecutorService = Executors.newScheduledThreadPool(Runtime.getRuntime().availableProcessors()); 48 | MsgExecutorTask msgExecutorTask=new MsgExecutorTask(); 49 | msgExecutorTask.setMsgQueue(msgQueue); 50 | msgExecutorTask.setHaveConsumedMap(haveConsumedMap); 51 | msgExecutorTask.setTopic(topicMap); 52 | msgExecutorTask.setScheduler(dealExecutorService); 53 | dealExecutorService.scheduleWithFixedDelay(msgExecutorTask, 1,1, TimeUnit.SECONDS); 54 | } 55 | private void getTopic() 56 | { 57 | String masterIp=config.getProperty("masterIp"); 58 | String masterPort=config.getProperty("masterPort"); 59 | String slaveIp=config.getProperty("slaveIp"); 60 | String slavePort=config.getProperty("slavePort"); 61 | final String masterGroup=masterIp+":"+masterPort; 62 | final String slaveGroup=slaveIp+":"+slavePort; 63 | ExecutorService executor=Executors.newFixedThreadPool(1); 64 | FutureTask> futureTask = new FutureTask>( 65 | new Callable>() { 66 | @Override 67 | public Map call() throws Exception{ 68 | while(true) 69 | { 70 | MgloryMap mgloryMap=new MgloryMap(masterGroup,slaveGroup); 71 | try { 72 | MgloryOut mgloryOut=mgloryMap.getAllByRootKey("topic"); 73 | Map outMap=mgloryOut.getMap(); 74 | if (!outMap.isEmpty()) 75 | { 76 | for (String key:outMap.keySet()) 77 | { 78 | String topic=outMap.get(key).toString(); 79 | try 80 | { 81 | MgloryOut topicOut=mgloryMap.getAllByRootKey(topic); 82 | Map topicOutMap=topicOut.getMap(); 83 | topicMap.put(topic,topicOutMap); 84 | } 85 | catch(Exception e) 86 | { 87 | log.warn("忽略异常:"+e); 88 | } 89 | 90 | } 91 | } 92 | else{ 93 | topicMap.clear(); 94 | } 95 | } catch (Exception e) { 96 | log.warn("忽略异常:"+e); 97 | } 98 | Thread.sleep(3000); 99 | } 100 | } 101 | }); 102 | executor.submit(futureTask); 103 | executor.shutdown(); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/mq/MgloryMQ.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.mq; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | 9 | import com.lingo.mglory.param.MgloryIn; 10 | import com.lingo.mglory.param.MgloryOut; 11 | import com.lingo.mglory.resource.ResourceHelper; 12 | import com.lingo.mglory.store.MgloryMap; 13 | import com.lingo.mglory.transport.client.Client; 14 | import com.lingo.mglory.util.Utils; 15 | 16 | public class MgloryMQ{ 17 | private static Log log = LogFactory.getLog(MgloryMQ.class); 18 | private String masterGroup;//中心服务（资源管理器）主机地址 如：114.215.178.115:1099 19 | private String slaveGroup;//中心服务（资源管理器）备机地址 20 | public MgloryMQ(String masterGroup,String slaveGroup){ 21 | this.masterGroup=masterGroup; 22 | this.slaveGroup=slaveGroup; 23 | } 24 | /** 25 | * 订阅(发布订阅模式) 26 | * @param topic 27 | * @param consumerId 28 | * @param consumerAddr 29 | * @param msgModel 30 | * @throws Exception 31 | */ 32 | public void subscribe(String topic,String consumerId,String consumerAddr,String consumerMethod) throws Exception 33 | { 34 | if (topic==null || topic.equals("") ||consumerId==null || consumerId.equals("") || consumerAddr==null || consumerAddr.equals("") 35 | || consumerMethod==null || consumerMethod.equals("")) 36 | { 37 | throw new Exception("输入参数不合法"); 38 | } 39 | Map valueMap=new HashMap(); 40 | valueMap.put("topic",topic); 41 | valueMap.put("consumerId", consumerId); 42 | valueMap.put("consumerAddr",consumerAddr); 43 | valueMap.put("consumerMethod",consumerMethod); 44 | String value=Utils.transMapToStr(valueMap); 45 | MgloryMap mgloryMap=new MgloryMap(masterGroup,slaveGroup); 46 | try { 47 | mgloryMap.put("topic"+"__"+topic,topic); 48 | mgloryMap.put(topic+"__"+consumerId,value); 49 | } catch (Exception e) { 50 | throw new Exception("订阅主题:"+topic+"失败:"+e); 51 | } 52 | } 53 | /** 54 | * 取消订阅(发布订阅模式) 55 | * @param topic 56 | * @param subscriber 57 | * @throws Exception 58 | */ 59 | public void unsubscribe(String topic)throws Exception 60 | { 61 | MgloryMap mgloryMap=new MgloryMap(masterGroup,slaveGroup); 62 | mgloryMap.remove("topic"+"__"+topic); 63 | MgloryOut mgloryOut=mgloryMap.getAllByRootKey(topic); 64 | Map outMap=mgloryOut.getMap(); 65 | if (!outMap.isEmpty()) 66 | { 67 | for (String key:outMap.keySet()) 68 | { 69 | try { 70 | mgloryMap.remove(key); 71 | } catch (Exception e) { 72 | throw new Exception("取消订阅主题:"+topic+"失败:"+e); 73 | } 74 | } 75 | } 76 | } 77 | /** 78 | * 发布消息 79 | * @param topic 80 | * @param produceId 81 | * @param msgId 82 | * @param msg 83 | * @throws Exception 84 | */ 85 | public int publish(String topic,String produceId,String msgId,String msg) throws Exception 86 | { 87 | log.info("msg====>"+msg); 88 | int result=-1; 89 | if (topic==null || topic.equals("") ||produceId==null || produceId.equals("") || msgId==null || msgId.equals("")) 90 | { 91 | throw new Exception("输入参数不合法"); 92 | } 93 | String resource=ResourceHelper.getInstance().getIdleResource(masterGroup, slaveGroup); 94 | String ip=resource.split(":")[0]; 95 | int port=Integer.parseInt(resource.split(":")[1]); 96 | Map inMap=new HashMap(); 97 | inMap.put("topic", topic); 98 | inMap.put("produceId",produceId); 99 | inMap.put("msgId",msgId); 100 | inMap.put("msg",msg); 101 | MgloryIn mgloryIn=new MgloryIn(); 102 | mgloryIn.setInvokeMethod("com.lingo.mglory.mq.MQReceiver.receive"); 103 | mgloryIn.setMap(inMap); 104 | MgloryOut out=Client.getInstance().call(ip, port, mgloryIn); 105 | if (out.getStatus().equals(MgloryOut.NORMAL)) 106 | { 107 | result=0; 108 | } 109 | return result; 110 | } 111 | } -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/mq/MsgExecutorTask.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.mq; 2 | 3 | import java.util.Map; 4 | import java.util.TimerTask; 5 | import java.util.concurrent.ConcurrentLinkedQueue; 6 | import java.util.concurrent.ConcurrentMap; 7 | import java.util.concurrent.ExecutorService; 8 | 9 | import org.apache.commons.logging.Log; 10 | import org.apache.commons.logging.LogFactory; 11 | 12 | import com.lingo.mglory.param.MgloryIn; 13 | import com.lingo.mglory.param.MgloryOut; 14 | import com.lingo.mglory.resource.ResourceHelper; 15 | import com.lingo.mglory.transport.client.Client; 16 | import com.lingo.mglory.util.Utils; 17 | 18 | public class MsgExecutorTask extends TimerTask{ 19 | private static Log log = LogFactory.getLog(MsgExecutorTask.class); 20 | private ConcurrentLinkedQueue> msgQueue; 21 | private ConcurrentMap> haveConsumedMap; 22 | private Map> topicMap; 23 | protected ExecutorService scheduler; 24 | public void setScheduler(ExecutorService scheduler) { 25 | this.scheduler = scheduler; 26 | } 27 | 28 | public void setMsgQueue(ConcurrentLinkedQueue> msgQueue) { 29 | this.msgQueue = msgQueue; 30 | } 31 | public void setTopic(Map> topicMap) 32 | { 33 | this.topicMap=topicMap; 34 | } 35 | public void setHaveConsumedMap(ConcurrentMap> haveConsumedMap) 36 | { 37 | this.haveConsumedMap=haveConsumedMap; 38 | } 39 | public void run() { 40 | Map msgMap=msgQueue.poll(); 41 | if (msgMap!=null) 42 | { 43 | try { 44 | String topic=msgMap.get("topic").toString(); 45 | Map outMap=topicMap.get(topic); 46 | if (outMap==null || outMap.isEmpty()) 47 | { 48 | msgQueue.add(msgMap); 49 | } 50 | else 51 | { 52 | String msgId=msgMap.get("msgId").toString(); 53 | for (String key:outMap.keySet()) 54 | { 55 | String value=outMap.get(key).toString(); 56 | Map valueMap=Utils.transStrToMap(value); 57 | String consumerAddr=valueMap.get("consumerAddr"); 58 | if (haveConsumedMap.get(msgId)!=null && haveConsumedMap.get(msgId).contains(consumerAddr)) 59 | { 60 | continue; 61 | } 62 | else 63 | { 64 | String consumerMethod=valueMap.get("consumerMethod"); 65 | try 66 | { 67 | String resource=ResourceHelper.getInstance().getIdleResource(consumerAddr, consumerAddr); 68 | 69 | String ip=resource.split(":")[0]; 70 | int port=Integer.parseInt(resource.split(":")[1]); 71 | MgloryIn in=new MgloryIn(); 72 | in.setMap(msgMap); 73 | in.setInvokeMethod(consumerMethod); 74 | MgloryOut out=Client.getInstance().call(ip, port,in); 75 | if (out.getStatus().equals(MgloryOut.NORMAL)) 76 | { 77 | if (haveConsumedMap.get(msgId)==null) 78 | { 79 | ConcurrentLinkedQueue queue=new ConcurrentLinkedQueue(); 80 | queue.add(consumerAddr); 81 | haveConsumedMap.put(msgId,queue); 82 | } 83 | else 84 | { 85 | haveConsumedMap.get(msgId).add(consumerAddr); 86 | } 87 | } 88 | } 89 | catch(Exception e) 90 | { 91 | log.warn("忽略异常:"+e); 92 | } 93 | } 94 | } 95 | if (haveConsumedMap.get(msgId).size()==outMap.size()) 96 | { 97 | haveConsumedMap.remove(msgId); 98 | } 99 | else 100 | { 101 | //加入队列继续处理 102 | msgQueue.add(msgMap); 103 | } 104 | } 105 | } catch (Exception e) { 106 | log.warn("发送消息失败："+e); 107 | } 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/param/MgloryIn.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.param; 2 | 3 | import java.io.Serializable; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | public class MgloryIn implements Serializable{ 8 | /** 9 | * 10 | */ 11 | private static final long serialVersionUID = 3111696841231705271L; 12 | private String taskId; 13 | private Map map; 14 | private String invokeMethod; 15 | private long timeout; 16 | public MgloryIn() 17 | { 18 | taskId=""; 19 | map=new HashMap(); 20 | timeout=0; 21 | } 22 | public void setMap(Map map) 23 | { 24 | this.map=map; 25 | } 26 | public Map getMap() 27 | { 28 | return map; 29 | } 30 | public void setTaskId(String taskId) 31 | { 32 | this.taskId=taskId; 33 | } 34 | public String getTaskId() 35 | { 36 | return taskId; 37 | } 38 | public void setInvokeMethod(String invokeMethod) 39 | { 40 | this.invokeMethod=invokeMethod; 41 | } 42 | public String getInvokeMethod() 43 | { 44 | return invokeMethod; 45 | } 46 | public void setTimeout(long timeout) 47 | { 48 | this.timeout=timeout; 49 | } 50 | public long getTimeout() 51 | { 52 | return timeout; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/param/MgloryOut.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.param; 2 | 3 | import java.io.Serializable; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | public class MgloryOut implements Serializable{ 8 | /** 9 | * 10 | */ 11 | private static final long serialVersionUID = 6822952442841004132L; 12 | private String taskId; 13 | private Map map; 14 | private String status; 15 | private String errorMsg; 16 | private String operMethod; 17 | private String storeLocation; 18 | private int storeNum; 19 | public static final String NORMAL="NORMAL";//正常 20 | public static final String EXCEPTION="EXCEPTION";//异常 21 | public static final String INTERRUPT="INTERRUPT";//中断 22 | public static final String RESULT="RESULT";//直接结果 23 | public static final String LOCAL_MEM="LOCAL_MEM";//本地内存 24 | public static final String LOCAL_FILE="LOCAL_FILE";//本地文件 25 | public static final String STORE_MEM="STORE_MEM";//内存存储系统 26 | public MgloryOut() 27 | { 28 | taskId=""; 29 | map=new HashMap(); 30 | status=NORMAL; 31 | errorMsg=""; 32 | operMethod=""; 33 | storeLocation=RESULT; 34 | storeNum=1; 35 | } 36 | public void setMap(Map map) 37 | { 38 | this.map=map; 39 | } 40 | public Map getMap() 41 | { 42 | return map; 43 | } 44 | public void setStatus(String status) 45 | { 46 | this.status=status; 47 | } 48 | public String getStatus() 49 | { 50 | return status; 51 | } 52 | public void setErrorMsg(String errorMsg) 53 | { 54 | this.errorMsg=errorMsg; 55 | } 56 | public String getErrorMsg() 57 | { 58 | return errorMsg; 59 | } 60 | public void setOperMethod(String operMethod) 61 | { 62 | this.operMethod=operMethod; 63 | } 64 | public String getOperMethod() 65 | { 66 | return operMethod; 67 | } 68 | public void setStoreLocation(String storeLocation) 69 | { 70 | this.storeLocation=storeLocation; 71 | } 72 | public String getStoreLocation() 73 | { 74 | return storeLocation; 75 | } 76 | public void setTaskId(String taskId) 77 | { 78 | this.taskId=taskId; 79 | } 80 | public String getTaskId() 81 | { 82 | return taskId; 83 | } 84 | public void setStoreNum(int storeNum) 85 | { 86 | this.storeNum=storeNum; 87 | } 88 | public int getStoreNum() 89 | { 90 | return storeNum; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/param/WorkFlowDAG.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.param; 2 | 3 | import java.io.Serializable; 4 | import java.util.HashMap; 5 | import java.util.LinkedList; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.concurrent.ConcurrentMap; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | import com.lingo.mglory.task.TaskState; 12 | 13 | public class WorkFlowDAG implements Serializable{ 14 | /** 15 | * 16 | */ 17 | private static final long serialVersionUID = -8470987561329561707L; 18 | private Map> taskMap=new HashMap>();//taskClazzName、operMethod、timeout 19 | private Map> preTaskMap=new HashMap>(); 20 | private Map> nextTaskMap=new HashMap>(); 21 | private ConcurrentMap taskStateMap=new ConcurrentHashMap(); 22 | private ConcurrentMap taskExcutorMap=new ConcurrentHashMap(); 23 | private ConcurrentMap taskResult=new ConcurrentHashMap(); 24 | public WorkFlowDAG() 25 | { 26 | 27 | } 28 | public void setTask(String taskId,String taskClazzName,String operMethod,String timeout) 29 | { 30 | Map taskInfoMap=new HashMap(); 31 | taskInfoMap.put("taskClazzName", taskClazzName);//执行类 32 | taskInfoMap.put("operMethod", operMethod);//执行方法 33 | taskInfoMap.put("timeout", timeout);//服务器执行超时时间 34 | taskMap.put(taskId, taskInfoMap); 35 | setTaskState(taskId,TaskState.WAIT_RUN); 36 | } 37 | public void setPreTask(String taskId,String preTaskId) 38 | { 39 | if (preTaskMap.get(taskId)==null) 40 | { 41 | List list=new LinkedList(); 42 | list.add(preTaskId); 43 | preTaskMap.put(taskId, list); 44 | } 45 | else 46 | { 47 | List list=preTaskMap.get(taskId); 48 | list.add(preTaskId); 49 | preTaskMap.put(taskId,list); 50 | } 51 | } 52 | public void setNextTask(String taskId,String nextTaskId) 53 | { 54 | if (nextTaskMap.get(taskId)==null) 55 | { 56 | List list=new LinkedList(); 57 | list.add(nextTaskId); 58 | nextTaskMap.put(taskId, list); 59 | } 60 | else 61 | { 62 | List list=nextTaskMap.get(taskId); 63 | list.add(nextTaskId); 64 | nextTaskMap.put(taskId,list); 65 | } 66 | } 67 | public void setTaskResult(String taskId,MgloryOut out) 68 | { 69 | taskResult.put(taskId,out); 70 | } 71 | public void setTaskState(String taskId,String state) 72 | { 73 | taskStateMap.put(taskId, state); 74 | } 75 | public void setTaskExcutor(String taskId,String resource) 76 | { 77 | taskExcutorMap.put(taskId, resource); 78 | } 79 | public Map> getTaskMap() 80 | { 81 | return taskMap; 82 | } 83 | public Map> getPreTaskMap() 84 | { 85 | return preTaskMap; 86 | } 87 | public Map> getNextTaskMap() 88 | { 89 | return nextTaskMap; 90 | } 91 | public ConcurrentMap getTaskResult() 92 | { 93 | return taskResult; 94 | } 95 | public ConcurrentMap getTaskStateMap() 96 | { 97 | return taskStateMap; 98 | } 99 | public ConcurrentMap getTaskExcutorMap() 100 | { 101 | return taskExcutorMap; 102 | } 103 | public void interruptTaskState() 104 | { 105 | for (String taskId:taskStateMap.keySet()) 106 | { 107 | if (taskStateMap.get(taskId).equals(TaskState.WAIT_RUN) || taskStateMap.get(taskId).equals(TaskState.RUNNING)) 108 | { 109 | taskStateMap.put(taskId, TaskState.INTERRUPT); 110 | } 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ElectManager.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.concurrent.ConcurrentMap; 6 | 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | 10 | import com.lingo.mglory.param.MgloryIn; 11 | import com.lingo.mglory.param.MgloryOut; 12 | import com.lingo.mglory.server.MgloryGroupServer; 13 | import com.lingo.mglory.transport.client.Client; 14 | 15 | public class ElectManager { 16 | private static Log log = LogFactory.getLog(ElectManager.class); 17 | private static ElectManager electManager; 18 | private String primary=""; 19 | private long primaryLease=0; 20 | private String elector=""; 21 | private int validateNum=0; 22 | private ElectManager() 23 | { 24 | 25 | } 26 | public synchronized static ElectManager getInstance() 27 | { 28 | if (electManager==null) 29 | { 30 | electManager=new ElectManager(); 31 | } 32 | return electManager; 33 | } 34 | public String getPrimary() 35 | { 36 | return primary; 37 | } 38 | public void setPrimary(String primary) 39 | { 40 | this.primary=primary; 41 | } 42 | public long getPrimaryLease() 43 | { 44 | return primaryLease; 45 | } 46 | public void setPrimaryLease(long primaryLease) 47 | { 48 | this.primaryLease=primaryLease; 49 | } 50 | public String getElector() 51 | { 52 | return elector; 53 | } 54 | public void setElector(String elector) 55 | { 56 | this.elector=elector; 57 | } 58 | /** 59 | * 选举primary 60 | * @return 61 | */ 62 | public void electPrimary() 63 | { 64 | String primaryWaitElect=getPrimary(); 65 | String masterGroup=ResourcePool.getInstance().getMasterGroup(); 66 | String slaveGroup=ResourcePool.getInstance().getSlaveGroup(); 67 | String localResource=MgloryGroupServer.getInstance().getIp()+":"+MgloryGroupServer.getInstance().getPort(); 68 | if (primaryWaitElect.equals("")) 69 | { 70 | primaryWaitElect=localResource; 71 | } 72 | if (getPrimary().equals(masterGroup) 73 | || getPrimary().equals(slaveGroup)) 74 | { 75 | ResourceManager resourceManager=new ResourceManager(); 76 | ConcurrentMap frontResourceMap=resourceManager.getFrontResource(); 77 | if (!frontResourceMap.isEmpty()) 78 | { 79 | primaryWaitElect=frontResourceMap.keySet().toArray()[0].toString(); 80 | } 81 | } 82 | if (validateNum>5) 83 | { 84 | //重新选举 85 | ResourceManager resourceManager=new ResourceManager(); 86 | ConcurrentMap frontResourceMap=resourceManager.getFrontResource(); 87 | if (!frontResourceMap.isEmpty()) 88 | { 89 | for (String key:frontResourceMap.keySet()) 90 | { 91 | if (key!=primaryWaitElect) 92 | { 93 | primaryWaitElect=key; 94 | break; 95 | } 96 | } 97 | } 98 | //没选到前端机时，选举本地资源为primary 99 | if(primaryWaitElect.equals(getPrimary())) 100 | { 101 | primaryWaitElect=localResource; 102 | } 103 | 104 | } 105 | if(!primaryWaitElect.equals(getPrimary())) 106 | { 107 | setPrimary(primaryWaitElect); 108 | setPrimaryLease(System.currentTimeMillis()); 109 | } 110 | if (!primaryWaitElect.equals(localResource)) 111 | { 112 | //校验 113 | MgloryIn mgloryIn=new MgloryIn(); 114 | Map inMap=new HashMap(); 115 | inMap.put("elector", localResource); 116 | inMap.put("primary", getPrimary()); 117 | inMap.put("primaryLease", getPrimaryLease()); 118 | mgloryIn.setMap(inMap); 119 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceScheduler.synPrimary"); 120 | try 121 | { 122 | String ip=primary.split(":")[0]; 123 | int port=Integer.parseInt(primary.split(":")[1]); 124 | MgloryOut out=Client.getInstance().call(ip,port,mgloryIn); 125 | String elector=out.getMap().get("elector").toString(); 126 | NodeManager.getInstance().setElectFlag(elector.equals(localResource)); 127 | validateNum=0; 128 | } 129 | catch(Exception e) 130 | { 131 | log.error("校验primary失败===>"+e); 132 | validateNum++; 133 | } 134 | } 135 | else 136 | { 137 | //资源为primary 138 | NodeManager.getInstance().setPrimaryFlag(true); 139 | } 140 | } 141 | 142 | } 143 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/NodeManager.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * @Description:节点管理器 3 | * @author sunxuhang 4 | * @revision 1.0 5 | * @date 2013-12-17 6 | *******************************************************************************/ 7 | package com.lingo.mglory.resource; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.Properties; 12 | import java.util.concurrent.Callable; 13 | import java.util.concurrent.ConcurrentMap; 14 | import java.util.concurrent.ExecutorService; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.FutureTask; 17 | 18 | import org.apache.commons.logging.Log; 19 | import org.apache.commons.logging.LogFactory; 20 | 21 | import com.lingo.mglory.param.MgloryIn; 22 | import com.lingo.mglory.param.MgloryOut; 23 | import com.lingo.mglory.server.MgloryGroupServer; 24 | import com.lingo.mglory.server.MgloryWorkerServer; 25 | import com.lingo.mglory.transport.client.Client; 26 | import com.lingo.mglory.transport.server.Server; 27 | import com.lingo.mglory.util.PropertiesUtil; 28 | 29 | public class NodeManager { 30 | private static Log log = LogFactory.getLog(NodeManager.class); 31 | private Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 32 | private static NodeManager nodeManager; 33 | private String masterGroup; 34 | private String slaveGroup; 35 | private boolean isGroup; 36 | private boolean synResourceFlag; 37 | private boolean cleanUpFlag; 38 | private boolean electFlag=false; 39 | private boolean primaryFlag=true; 40 | private NodeManager() 41 | { 42 | 43 | } 44 | public synchronized static NodeManager getInstance() 45 | { 46 | if (nodeManager==null) 47 | { 48 | nodeManager=new NodeManager(); 49 | } 50 | return nodeManager; 51 | } 52 | public void startHeart(boolean isGroup) 53 | { 54 | String masterIp=config.getProperty("masterIp"); 55 | String masterPort=config.getProperty("masterPort"); 56 | String slaveIp=config.getProperty("slaveIp"); 57 | String slavePort=config.getProperty("slavePort"); 58 | masterGroup=masterIp+":"+masterPort; 59 | slaveGroup=slaveIp+":"+slavePort; 60 | this.isGroup=isGroup; 61 | if (isGroup) 62 | { 63 | String localResource=MgloryGroupServer.getInstance().getIp()+":"+MgloryGroupServer.getInstance().getPort(); 64 | //启动时只有主资源管理器启动选举器 65 | if (localResource.equals(masterGroup)) 66 | { 67 | electFlag=true; 68 | } 69 | //选举primary 70 | electPrimary(); 71 | } 72 | else 73 | { 74 | //执行服务报告资源信息 75 | reportResource(); 76 | } 77 | String resource=MgloryWorkerServer.getInstance().getResource(); 78 | registerShutdownHook(resource); 79 | } 80 | //选举primary 81 | private void electPrimary() 82 | { 83 | ExecutorService executor=Executors.newFixedThreadPool(1); 84 | FutureTask> futureTask = new FutureTask>( 85 | new Callable>() { 86 | @Override 87 | public Map call() throws Exception { 88 | while(true) 89 | { 90 | if (electFlag) 91 | { 92 | try 93 | { 94 | ElectManager.getInstance().electPrimary(); 95 | } 96 | catch(Exception e) 97 | { 98 | log.error("elect primary error===>"+e); 99 | } 100 | } 101 | log.info("primary====>"+ElectManager.getInstance().getPrimary()); 102 | String localResource=MgloryGroupServer.getInstance().getIp()+":"+MgloryGroupServer.getInstance().getPort(); 103 | setElectFlag(localResource.equals(ElectManager.getInstance().getElector())); 104 | Thread.sleep(5000); 105 | } 106 | } 107 | }); 108 | executor.submit(futureTask); 109 | executor.shutdown(); 110 | } 111 | private void synResuorceToFront() 112 | { 113 | synResourceFlag=true; 114 | ExecutorService executor=Executors.newFixedThreadPool(1); 115 | FutureTask futureTask = new FutureTask( 116 | new Callable() { 117 | @Override 118 | public MgloryOut call() throws Exception { 119 | while(primaryFlag) 120 | { 121 | try 122 | { 123 | Map frontResourceMap=ResourcePool.getInstance().frontResourceMap; 124 | frontResourceMap.put(masterGroup, masterGroup); 125 | frontResourceMap.put(slaveGroup, slaveGroup); 126 | frontResourceMap.remove(ElectManager.getInstance().getPrimary()); 127 | for (String resource:frontResourceMap.keySet()) 128 | { 129 | String ip=resource.split(":")[0]; 130 | int port=Integer.parseInt(resource.split(":")[1]); 131 | Map map=new HashMap(); 132 | map.put("resourceMap",ResourcePool.getInstance().resourceMap); 133 | map.put("execResourceMap",ResourcePool.getInstance().execResourceMap); 134 | map.put("bakExecResourceMap",ResourcePool.getInstance().bakExecResourceMap); 135 | map.put("resourceMonitorMap",ResourcePool.getInstance().resourceMonitorMap); 136 | map.put("resourceMonitorTimeMap",ResourcePool.getInstance().resourceMonitorTimeMap); 137 | map.put("primary",ElectManager.getInstance().getPrimary()); 138 | map.put("primaryLease",ElectManager.getInstance().getPrimaryLease()); 139 | map.put("elector",ElectManager.getInstance().getElector()); 140 | MgloryIn mgloryIn=new MgloryIn(); 141 | mgloryIn.setMap(map); 142 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.synResuorceToFront"); 143 | Client.getInstance().call(ip, port, mgloryIn); 144 | } 145 | } 146 | catch(Exception e) 147 | { 148 | log.error("忽略异常====>"+e); 149 | } 150 | Thread.sleep(5000); 151 | } 152 | synResourceFlag=false; 153 | return null; 154 | } 155 | 156 | }); 157 | executor.submit(futureTask); 158 | executor.shutdown(); 159 | } 160 | //清除僵死资源 161 | private void cleanUpDeadResource() 162 | { 163 | cleanUpFlag=true; 164 | ExecutorService executor=Executors.newFixedThreadPool(1); 165 | FutureTask> futureTask = new FutureTask>( 166 | new Callable>() { 167 | @Override 168 | public Map call() throws Exception { 169 | while(primaryFlag) 170 | { 171 | try 172 | { 173 | log.info("清除僵死资源"); 174 | ResourceManager resourceManager=new ResourceManager(); 175 | ConcurrentMap resourceMonitorTimeMap=resourceManager.getResourceMonitorTime(); 176 | for (String key:resourceMonitorTimeMap.keySet()) 177 | { 178 | long t=System.currentTimeMillis(); 179 | long monitorTime=resourceMonitorTimeMap.get(key); 180 | if (monitorTime-t>10000) 181 | { 182 | resourceManager.removeResource(key); 183 | } 184 | } 185 | } 186 | catch(Exception e) 187 | { 188 | log.error("cleanup resource error===>"+e); 189 | } 190 | Thread.sleep(3000); 191 | } 192 | cleanUpFlag=false; 193 | return null; 194 | } 195 | }); 196 | executor.submit(futureTask); 197 | executor.shutdown(); 198 | } 199 | //心跳程序，同步资源给primary 200 | private void reportResource() 201 | { 202 | ExecutorService executor=Executors.newFixedThreadPool(1); 203 | FutureTask> futureTask = new FutureTask>( 204 | new Callable>() { 205 | @Override 206 | public Map call() throws Exception { 207 | while(true) 208 | { 209 | log.info("报告资源"); 210 | String primary=ElectManager.getInstance().getPrimary(); 211 | if (primary.equals("")) 212 | { 213 | primary=masterGroup; 214 | } 215 | String resource=MgloryWorkerServer.getInstance().getResource(); 216 | if (!primary.equals(resource)) 217 | { 218 | Map inMap=new HashMap(); 219 | inMap.put("resource",resource); 220 | inMap.put("resourceType", MgloryWorkerServer.getInstance().getResourceType()); 221 | inMap.put("systemResource",ResourceUsage.getResUsage()); 222 | MgloryIn mgloryIn=new MgloryIn(); 223 | mgloryIn.setMap(inMap); 224 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.synResuorce"); 225 | try 226 | { 227 | String ip=primary.split(":")[0]; 228 | int port=Integer.parseInt(primary.split(":")[1]); 229 | MgloryOut out=Client.getInstance().call(ip,port, mgloryIn); 230 | ElectManager.getInstance().setPrimary(out.getMap().get("primary").toString()); 231 | } 232 | catch(Exception e) 233 | { 234 | log.error("同步资源到primary服务失败===>"+e); 235 | MgloryIn in=new MgloryIn(); 236 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceScheduler.getPrimary"); 237 | try 238 | { 239 | String ip=masterGroup.split(":")[0]; 240 | int port=Integer.parseInt(masterGroup.split(":")[1]); 241 | MgloryOut out=Client.getInstance().call(ip,port,in); 242 | ElectManager.getInstance().setPrimary(out.getMap().get("primary").toString()); 243 | } 244 | catch(Exception em) 245 | { 246 | log.error("获取primary失败===>"+em); 247 | if (!slaveGroup.equals(masterGroup)) 248 | try 249 | { 250 | String ip=slaveGroup.split(":")[0]; 251 | int port=Integer.parseInt(slaveGroup.split(":")[1]); 252 | MgloryOut out=Client.getInstance().call(ip,port,in); 253 | ElectManager.getInstance().setPrimary(out.getMap().get("primary").toString()); 254 | } 255 | catch(Exception es) 256 | { 257 | log.error("获取primary失败===>"+es); 258 | } 259 | } 260 | } 261 | 262 | } 263 | Thread.sleep(3000); 264 | } 265 | } 266 | }); 267 | executor.submit(futureTask); 268 | executor.shutdown(); 269 | } 270 | public void setElectFlag(boolean electFlag) 271 | { 272 | this.electFlag=electFlag; 273 | } 274 | public void setPrimaryFlag(boolean primaryFlag) 275 | { 276 | if (primaryFlag && !synResourceFlag) 277 | { 278 | synResuorceToFront(); 279 | } 280 | if (primaryFlag && !cleanUpFlag) 281 | { 282 | cleanUpDeadResource(); 283 | } 284 | this.primaryFlag=primaryFlag; 285 | } 286 | /** 287 | * 钩子方法 288 | * @param resource 289 | */ 290 | private void registerShutdownHook(final String resource){ 291 | Runtime.getRuntime().addShutdownHook(new Thread(){ 292 | public void run(){ 293 | if (!isGroup) 294 | { 295 | String primary=ElectManager.getInstance().getPrimary(); 296 | if (primary.equals("")) 297 | { 298 | primary=masterGroup; 299 | } 300 | if (!primary.equals(resource)) 301 | { 302 | Map inMap=new HashMap(); 303 | inMap.put("resource", resource); 304 | MgloryIn mgloryIn=new MgloryIn(); 305 | mgloryIn.setMap(inMap); 306 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.removeResource"); 307 | try 308 | { 309 | String ip=primary.split(":")[0]; 310 | int port=Integer.parseInt(primary.split(":")[1]); 311 | Client.getInstance().call(ip, port, mgloryIn); 312 | } 313 | catch(Exception e) 314 | { 315 | log.error("清除资源失败===>"+e); 316 | } 317 | } 318 | } 319 | Server.shutdown(); 320 | } 321 | }); 322 | } 323 | 324 | } 325 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ResourceHelper.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.Random; 8 | import java.util.concurrent.Callable; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | import java.util.concurrent.ConcurrentMap; 11 | import java.util.concurrent.ExecutorService; 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.FutureTask; 14 | 15 | import org.apache.commons.logging.Log; 16 | import org.apache.commons.logging.LogFactory; 17 | 18 | import com.lingo.mglory.param.MgloryIn; 19 | import com.lingo.mglory.param.MgloryOut; 20 | import com.lingo.mglory.transport.client.Client; 21 | 22 | public class ResourceHelper { 23 | private static Log log = LogFactory.getLog(ResourceHelper.class); 24 | private static ResourceHelper helper; 25 | private ConcurrentMap groupMap=new ConcurrentHashMap(); 26 | private ConcurrentMap> groupFrontMap=new ConcurrentHashMap>(); 27 | private String primary=""; 28 | private ResourceHelper(){ 29 | init(); 30 | } 31 | public synchronized static ResourceHelper getInstance() 32 | { 33 | if (helper==null) 34 | { 35 | helper=new ResourceHelper(); 36 | } 37 | return helper; 38 | } 39 | /** 40 | * 获取空闲资源 41 | * @param masterGroup 42 | * @param slaveGroup 43 | * @param num 44 | * @return 45 | * @throws Exception 46 | */ 47 | @SuppressWarnings("unchecked") 48 | public List getIdleResource(String masterGroup,String slaveGroup,int num) throws Exception 49 | { 50 | ListresourceList=new ArrayList(); 51 | String managerResource=getFrontResource(masterGroup,slaveGroup); 52 | Map inMap=new HashMap(); 53 | inMap.put("num",num); 54 | MgloryIn mgloryIn=new MgloryIn(); 55 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceScheduler.getIdleResource"); 56 | mgloryIn.setMap(inMap); 57 | String ip=managerResource.split(":")[0]; 58 | int port=Integer.parseInt(managerResource.split(":")[1]); 59 | MgloryOut mgloryOut=Client.getInstance().call(ip, port, mgloryIn); 60 | resourceList=(List)mgloryOut.getMap().get("resourceList"); 61 | if (resourceList==null) 62 | { 63 | for (int i=0;i<5;i++) 64 | { 65 | try 66 | { 67 | String masterIp=masterGroup.split(":")[0]; 68 | int masterPort=Integer.parseInt(masterGroup.split(":")[1]); 69 | MgloryOut out=Client.getInstance().call(masterIp, masterPort, mgloryIn); 70 | resourceList=(List)out.getMap().get("resourceList"); 71 | break; 72 | } catch(Exception e) 73 | { 74 | if (i==4) 75 | { 76 | String slaveIp=slaveGroup.split(":")[0]; 77 | int slavePort=Integer.parseInt(slaveGroup.split(":")[1]); 78 | MgloryOut out=Client.getInstance().call(slaveIp, slavePort, mgloryIn); 79 | resourceList=(List)out.getMap().get("resourceList"); 80 | } 81 | } 82 | } 83 | } 84 | if (resourceList==null) 85 | { 86 | resourceList=new ArrayList(); 87 | } 88 | return resourceList; 89 | } 90 | /** 91 | * 获取空闲资源 92 | * @param masterGroup 93 | * @param slaveGroup 94 | * @return 95 | * @throws Exception 96 | */ 97 | public String getIdleResource(String masterGroup,String slaveGroup) throws Exception 98 | { 99 | List resourceList=new ArrayList(); 100 | resourceList=getIdleResource(masterGroup, slaveGroup,1); 101 | if (resourceList.isEmpty()) 102 | { 103 | throw new Exception("没有获取到空闲资源"); 104 | } 105 | String resource=resourceList.get(0); 106 | return resource; 107 | } 108 | private void init() 109 | { 110 | ExecutorService executor=Executors.newFixedThreadPool(1); 111 | FutureTask> futureTask = new FutureTask>( 112 | new Callable>() { 113 | @SuppressWarnings("unchecked") 114 | @Override 115 | public Map call() throws Exception { 116 | while(true) 117 | { 118 | for (String group:groupMap.keySet()) 119 | { 120 | try 121 | { 122 | String ip=group.split(":")[0]; 123 | int port=Integer.parseInt(group.split(":")[1]); 124 | MgloryIn mgloryIn=new MgloryIn(); 125 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.getFrontResource"); 126 | MgloryOut mgloryOut=Client.getInstance().call(ip, port, mgloryIn); 127 | primary= mgloryOut.getMap().get("primary").toString(); 128 | groupFrontMap.put(group,(Map)mgloryOut.getMap().get("front")); 129 | } 130 | catch(Exception e) 131 | { 132 | if (groupFrontMap.size()>1) 133 | { 134 | groupFrontMap.remove(group); 135 | } 136 | log.error("忽略异常："+e); 137 | } 138 | 139 | } 140 | Thread.sleep(60000); 141 | } 142 | } 143 | 144 | }); 145 | executor.submit(futureTask); 146 | executor.shutdown(); 147 | } 148 | public String getFrontResource(String masterGroup,String slaveGroup) 149 | { 150 | if (!groupMap.containsKey(masterGroup)) 151 | { 152 | groupMap.put(masterGroup, masterGroup); 153 | } 154 | if (!groupMap.containsKey(slaveGroup)) 155 | { 156 | groupMap.put(slaveGroup, slaveGroup); 157 | } 158 | String resource=""; 159 | Map masterFrontMap=groupFrontMap.get(masterGroup); 160 | Map slaveFrontMap=groupFrontMap.get(slaveGroup); 161 | if (masterFrontMap!=null) 162 | { 163 | masterFrontMap.remove(primary); 164 | if (!masterFrontMap.isEmpty()) 165 | { 166 | Random r = new Random(); 167 | int n=r.nextInt(masterFrontMap.size()); 168 | resource=masterFrontMap.keySet().toArray()[n].toString(); 169 | } 170 | } 171 | else if(slaveFrontMap!=null && !masterGroup.equals(slaveGroup)) 172 | { 173 | slaveFrontMap.remove(primary); 174 | if (!slaveFrontMap.isEmpty()) 175 | { 176 | Random r = new Random(); 177 | int n=r.nextInt(slaveFrontMap.size()); 178 | resource=slaveFrontMap.keySet().toArray()[n].toString(); 179 | } 180 | } 181 | if (resource.equals("")) 182 | { 183 | Random r = new Random(); 184 | int n=r.nextInt(groupMap.size()); 185 | resource = groupMap.keySet().toArray()[n].toString(); 186 | } 187 | return resource; 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ResourceManager.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * @Description:资源管理类 3 | * @author sunxuhang 4 | * @revision 1.0 5 | * @date 2013-12-17 6 | *******************************************************************************/ 7 | package com.lingo.mglory.resource; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentMap; 12 | 13 | import com.lingo.mglory.param.MgloryIn; 14 | import com.lingo.mglory.param.MgloryOut; 15 | import com.lingo.mglory.transport.client.Client; 16 | import com.lingo.mglory.util.Utils; 17 | public class ResourceManager{ 18 | /** 19 | * 获取注册资源 20 | * @return ConcurrentMap 21 | */ 22 | public ConcurrentMap getResource() 23 | { 24 | return ResourcePool.getInstance().resourceMap; 25 | } 26 | /** 27 | * 获取执行类资源 28 | * @return ConcurrentMap 29 | */ 30 | public ConcurrentMap getExecResource() 31 | { 32 | return ResourcePool.getInstance().execResourceMap; 33 | } 34 | /** 35 | * 获取前置类资源 36 | * @return ConcurrentMap 37 | */ 38 | public ConcurrentMap getFrontResource() 39 | { 40 | return ResourcePool.getInstance().frontResourceMap; 41 | } 42 | /** 43 | * 获取执行类备份资源 44 | * @return ConcurrentMap 45 | */ 46 | public ConcurrentMap getBakExecResource() 47 | { 48 | return ResourcePool.getInstance().bakExecResourceMap; 49 | } 50 | /** 51 | * 获取resourceMonitor 52 | * @return 53 | */ 54 | public ConcurrentMap getResourceMonitor() 55 | { 56 | return ResourcePool.getInstance().resourceMonitorMap; 57 | } 58 | /** 59 | * 注册资源 60 | * @param taskId 61 | * @param inMap resource：127.0.0.1:8888,resourceType:exec、front、bakExec 62 | * @return 63 | * @throws Exception 64 | */ 65 | public MgloryOut addResource(String taskId,Map inMap) throws Exception 66 | { 67 | MgloryOut out =new MgloryOut(); 68 | String resource=inMap.get("resource").toString(); 69 | String resourceType=inMap.get("resourceType").toString(); 70 | //执行类资源 71 | if (resourceType.equals(ResourcePool.EXEC_RESOURCE)) 72 | { 73 | if (!ResourcePool.getInstance().execResourceMap.containsKey(resource)) 74 | { 75 | ResourcePool.getInstance().execResourceMap.put(resource,resource);//value为在用资源 76 | } 77 | else if (!ResourcePool.getInstance().execResourceMap.get(resource).equals(resource)) 78 | { 79 | //数据迁移(备迁往主) 80 | String bakResource=ResourcePool.getInstance().execResourceMap.get(resource); 81 | String ip=bakResource.split(":")[0]; 82 | int port=Integer.parseInt(bakResource.split(":")[1]); 83 | MgloryIn in=new MgloryIn(); 84 | inMap=new HashMap(); 85 | inMap.put("resource",resource); 86 | in.setMap(inMap); 87 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.bakToPrimary"); 88 | Client.getInstance().call(ip, port, in); 89 | ResourcePool.getInstance().execResourceMap.put(resource,resource); 90 | } 91 | } 92 | //前置类资源 93 | else if (resourceType.equals(ResourcePool.FRONT_RESOURCE)) 94 | { 95 | if (!ResourcePool.getInstance().frontResourceMap.containsKey(resource)) 96 | { 97 | ResourcePool.getInstance().frontResourceMap.put(resource,resourceType); 98 | } 99 | } 100 | //执行类备份资源 101 | else 102 | { 103 | //resourceType即需要备份的资源 ,如127.0.0.1:1096 104 | if (!ResourcePool.getInstance().bakExecResourceMap.containsKey(resourceType)) 105 | { 106 | ResourcePool.getInstance().bakExecResourceMap.put(resourceType,resource); 107 | } 108 | } 109 | ResourcePool.getInstance().resourceMap.put(resource,resourceType); 110 | return out; 111 | } 112 | /** 113 | * 移除资源 114 | * @param taskId 115 | * @param inMap 116 | * @throws RemoteException 117 | */ 118 | public MgloryOut removeResource(String taskId,Map inMap) 119 | { 120 | MgloryOut out =new MgloryOut(); 121 | String resource=inMap.get("resource").toString(); 122 | removeResource(resource); 123 | return out; 124 | 125 | } 126 | /** 127 | * 移除资源 128 | * @param resource 129 | */ 130 | public void removeResource(String resource) 131 | { 132 | if (!ResourcePool.getInstance().resourceMap.containsKey(resource)) 133 | { 134 | return; 135 | } 136 | if (ResourcePool.getInstance().resourceMap.get(resource).equals(ResourcePool.EXEC_RESOURCE)) 137 | { 138 | //execResourceMap.remove(resource); 139 | if (ResourcePool.getInstance().bakExecResourceMap.containsKey(resource)) 140 | { 141 | //备用资源切换成在用资源 142 | ResourcePool.getInstance().execResourceMap.put(resource, ResourcePool.getInstance().bakExecResourceMap.get(resource)); 143 | } 144 | else 145 | { 146 | ResourcePool.getInstance().execResourceMap.remove(resource); 147 | //数据迁移,暂未实现 148 | } 149 | } 150 | else if (ResourcePool.getInstance().resourceMap.get(resource).equals(ResourcePool.FRONT_RESOURCE)) 151 | { 152 | ResourcePool.getInstance().frontResourceMap.remove(resource); 153 | } 154 | else 155 | { 156 | if (ResourcePool.getInstance().bakExecResourceMap.containsValue(resource)) 157 | { 158 | String bakResource=""; 159 | for (String key:ResourcePool.getInstance().bakExecResourceMap.keySet()) 160 | { 161 | if (ResourcePool.getInstance().bakExecResourceMap.get(key).toString().equals(resource)) 162 | { 163 | bakResource=key; 164 | break; 165 | } 166 | } 167 | if (!bakResource.equals("")) 168 | { 169 | ResourcePool.getInstance().bakExecResourceMap.remove(bakResource); 170 | if(ResourcePool.getInstance().execResourceMap.get(bakResource).toString().equals(resource)) 171 | { 172 | ResourcePool.getInstance().execResourceMap.remove(bakResource); 173 | //数据迁移,暂未实现 174 | } 175 | } 176 | } 177 | } 178 | ResourcePool.getInstance().resourceMap.remove(resource); 179 | ResourcePool.getInstance().resourceMonitorMap.remove(resource); 180 | ResourcePool.getInstance().resourceMonitorTimeMap.remove(resource); 181 | 182 | } 183 | /** 184 | * 获取primary资源 185 | * @param taskId 186 | * @param inMap 187 | * @return 188 | */ 189 | public MgloryOut getPrimary(String taskId,Map inMap) 190 | { 191 | MgloryOut out =new MgloryOut(); 192 | Map outMap=new HashMap(); 193 | outMap.put("primary",ElectManager.getInstance().getPrimary()); 194 | out.setMap(outMap); 195 | return out; 196 | } 197 | /** 198 | * 获取前置类资源 199 | * @param taskId 200 | * @param inMap 201 | * @return 202 | */ 203 | public MgloryOut getFrontResource(String taskId,Map inMap) 204 | { 205 | MgloryOut out =new MgloryOut(); 206 | Map outMap=new HashMap(); 207 | outMap.put("primary",ElectManager.getInstance().getPrimary()); 208 | outMap.put("front",Utils.transMap(ResourcePool.getInstance().frontResourceMap)); 209 | out.setMap(outMap); 210 | return out; 211 | } 212 | /** 213 | * 获取执行类资源 214 | * @param taskId 215 | * @param inMap 216 | * @return 217 | */ 218 | public MgloryOut getExecResource(String taskId,Map inMap) 219 | { 220 | MgloryOut out =new MgloryOut(); 221 | out.setMap(Utils.transMap(ResourcePool.getInstance().execResourceMap)); 222 | return out; 223 | } 224 | /** 225 | * 获取执行类备份资源 226 | * @param taskId 227 | * @param inMap 228 | * @return 229 | */ 230 | public MgloryOut getBakExecResource(String taskId,Map inMap) 231 | { 232 | MgloryOut out =new MgloryOut(); 233 | out.setMap(Utils.transMap(ResourcePool.getInstance().bakExecResourceMap)); 234 | return out; 235 | } 236 | /** 237 | * 获取resourceMonitorTime 238 | * @param taskId 239 | * @param inMap 240 | * @return 241 | */ 242 | public MgloryOut getResourceMonitorTime(String taskId,Map inMap) 243 | { 244 | MgloryOut out =new MgloryOut(); 245 | out.setMap(Utils.transMap(ResourcePool.getInstance().resourceMonitorTimeMap)); 246 | return out; 247 | } 248 | /** 249 | * 获取resourceMonitorTime 250 | * @return 251 | */ 252 | public ConcurrentMap getResourceMonitorTime() 253 | { 254 | return ResourcePool.getInstance().resourceMonitorTimeMap; 255 | } 256 | /** 257 | * @param taskId 258 | * @param inMap 259 | * @return 260 | * @throws Exception 261 | */ 262 | public MgloryOut synResuorce(String taskId,Map inMap)throws Exception 263 | { 264 | MgloryOut out =new MgloryOut(); 265 | String resource=inMap.get("resource").toString(); 266 | String resourceType=inMap.get("resourceType").toString(); 267 | SystemResource systemResourc=(SystemResource)inMap.get("systemResource"); 268 | Map map=new HashMap(); 269 | map.put("resource", resource); 270 | map.put("resourceType", resourceType); 271 | addResource(taskId,map); 272 | ResourcePool.getInstance().resourceMonitorMap.put(resource, systemResourc); 273 | ResourcePool.getInstance().resourceMonitorTimeMap.put(resource,System.currentTimeMillis()); 274 | Map outMap=new HashMap(); 275 | outMap.put("primary",ElectManager.getInstance().getPrimary()); 276 | out.setMap(outMap); 277 | return out; 278 | } 279 | /** 280 | * 同步资源到前端机 281 | * @param taskId 282 | * @param inMap 283 | * @return 284 | * @throws Exception 285 | */ 286 | @SuppressWarnings("unchecked") 287 | public MgloryOut synResuorceToFront(String taskId,Map inMap)throws Exception 288 | { 289 | MgloryOut out =new MgloryOut(); 290 | ConcurrentMap resourceMap=(ConcurrentMap)inMap.get("resourceMap"); 291 | ConcurrentMap execResourceMap=(ConcurrentMap)inMap.get("execResourceMap"); 292 | ConcurrentMap bakExecResourceMap=(ConcurrentMap)inMap.get("bakExecResourceMap"); 293 | ConcurrentMap resourceMonitorMap=(ConcurrentMap)inMap.get("resourceMonitorMap"); 294 | ConcurrentMap resourceMonitorTimeMap=(ConcurrentMap)inMap.get("resourceMonitorTimeMap"); 295 | String primary=inMap.get("primary").toString(); 296 | long primaryLease=Long.parseLong(inMap.get("primaryLease").toString()); 297 | String elector=inMap.get("elector").toString(); 298 | setResource(resourceMap,execResourceMap,bakExecResourceMap, resourceMonitorMap, resourceMonitorTimeMap); 299 | setPrimary(primary,primaryLease,elector); 300 | return out; 301 | } 302 | /** 303 | * 设置资源 304 | * @param resourceMap 305 | * @param execResourceMap 306 | * @param bakExecResourceMap 307 | * @param serviceMap 308 | * @param resourceMonitorMap 309 | * @param resourceMonitorTimeMap 310 | */ 311 | public void setResource(ConcurrentMap resourceMap,ConcurrentMap execResourceMap, 312 | ConcurrentMap bakExecResourceMap,ConcurrentMap resourceMonitorMap, 313 | ConcurrentMap resourceMonitorTimeMap) 314 | { 315 | ResourcePool.getInstance().resourceMap=resourceMap; 316 | ResourcePool.getInstance().execResourceMap=execResourceMap; 317 | ResourcePool.getInstance().bakExecResourceMap=bakExecResourceMap; 318 | ResourcePool.getInstance().resourceMonitorMap=resourceMonitorMap; 319 | ResourcePool.getInstance().resourceMonitorTimeMap=resourceMonitorTimeMap; 320 | } 321 | public void setPrimary(String primary,long primaryLease,String elector) 322 | { 323 | if (primaryLease>ElectManager.getInstance().getPrimaryLease()) 324 | { 325 | NodeManager.getInstance().setPrimaryFlag(false); 326 | ElectManager.getInstance().setPrimary(primary); 327 | ElectManager.getInstance().setPrimaryLease(primaryLease); 328 | ElectManager.getInstance().setElector(elector); 329 | } 330 | } 331 | } 332 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ResourcePool.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.util.concurrent.ConcurrentHashMap; 4 | import java.util.concurrent.ConcurrentMap; 5 | 6 | public class ResourcePool{ 7 | //前置类 8 | public static final String FRONT_RESOURCE="front"; 9 | //执行类 10 | public static final String EXEC_RESOURCE="exec"; 11 | //存储备份类 12 | public static final String BAK_EXEC_RESOURCE="bakExec"; 13 | private static ResourcePool resourcePool; 14 | //注册的资源 15 | public ConcurrentMap resourceMap=new ConcurrentHashMap(); 16 | //前置类资源 17 | public ConcurrentMap frontResourceMap=new ConcurrentHashMap(); 18 | //执行类资源 19 | public ConcurrentMap execResourceMap=new ConcurrentHashMap(); 20 | //执行类备份资源 21 | public ConcurrentMap bakExecResourceMap=new ConcurrentHashMap(); 22 | //注册资源上的服务信息 23 | public ConcurrentMap serviceMap=new ConcurrentHashMap(); 24 | //注册资源的系统信息 25 | public ConcurrentMap resourceMonitorMap=new ConcurrentHashMap(); 26 | //资源报告时间信息 27 | public ConcurrentMap resourceMonitorTimeMap=new ConcurrentHashMap(); 28 | private String masterGroup; 29 | private String slaveGroup; 30 | private ResourcePool(){ 31 | 32 | } 33 | public synchronized static ResourcePool getInstance() 34 | { 35 | if (resourcePool==null) 36 | { 37 | resourcePool=new ResourcePool(); 38 | } 39 | return resourcePool; 40 | } 41 | public String getMasterGroup() 42 | { 43 | return masterGroup; 44 | } 45 | public void setMasterGroup(String masterGroup) 46 | { 47 | this.masterGroup=masterGroup; 48 | } 49 | public String getSlaveGroup() 50 | { 51 | return slaveGroup; 52 | } 53 | public void setSlaveGroup(String slaveGroup) 54 | { 55 | this.slaveGroup=slaveGroup; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ResourceScheduler.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.rmi.RemoteException; 4 | import java.util.ArrayList; 5 | import java.util.HashMap; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.Random; 9 | import java.util.concurrent.Callable; 10 | import java.util.concurrent.ConcurrentMap; 11 | import java.util.concurrent.ExecutorService; 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.FutureTask; 14 | 15 | import org.apache.commons.logging.Log; 16 | import org.apache.commons.logging.LogFactory; 17 | 18 | import com.lingo.mglory.param.MgloryOut; 19 | 20 | public class ResourceScheduler{ 21 | private static Log log = LogFactory.getLog(ResourceScheduler.class); 22 | /** 23 | * 获取空闲的注册资源 24 | * @param taskId 25 | * @param inMap 26 | * @return 27 | * @throws Exception 28 | */ 29 | public MgloryOut getIdleResource(String taskId,Map inMap) throws Exception 30 | { 31 | MgloryOut out =new MgloryOut(); 32 | List resourceList=new ArrayList(); 33 | int num=Integer.parseInt(inMap.get("num").toString()); 34 | for (int i=0;i outMap=new HashMap(); 39 | outMap.put("resourceList", resourceList); 40 | out.setMap(outMap); 41 | return out; 42 | } 43 | /** 44 | * 获取空闲的注册资源 45 | * @return String 46 | * @throws RemoteException 47 | */ 48 | public String getIdleResource() throws Exception 49 | { 50 | ResourceManager resourceManager=new ResourceManager(); 51 | String resource=""; 52 | ConcurrentMap resourceMap=resourceManager.getResource(); 53 | final ConcurrentMap execResourceMap=resourceManager.getExecResource(); 54 | final ConcurrentMap resourceMonitorMap=resourceManager.getResourceMonitor(); 55 | List idleResourceList=new ArrayList(); 56 | if (resourceMap.isEmpty()) 57 | { 58 | throw new Exception("没有可用资源!"); 59 | } 60 | final long t=System.currentTimeMillis(); 61 | ExecutorService executor=Executors.newFixedThreadPool(1); 62 | FutureTask> futureTask = new FutureTask >( 63 | new Callable >() { 64 | @Override 65 | public List call() throws Exception { 66 | List resourceList=new ArrayList(); 67 | while(resourceList.isEmpty() && !resourceMonitorMap.isEmpty() 68 | && !execResourceMap.isEmpty() && (System.currentTimeMillis()-t)<12000) 69 | { 70 | log.info("获取空闲资源"); 71 | // for (String key:resourceMonitorMap.keySet()) 72 | // { 73 | // if (resourceMonitorMap.get(key).getFreeMemory()>1024*64 74 | // && resourceMonitorMap.get(key).getCpuRatio()<0.95 75 | // && resourceMonitorMap.get(key).getIoRatio()<0.70 76 | // && resourceMonitorMap.get(key).getNetRatio()<0.90 77 | // && execResourceMap.containsKey(key)) 78 | // { 79 | // resourceList.add(key); 80 | // } 81 | // } 82 | for (String key:execResourceMap.keySet()) 83 | { 84 | String value=execResourceMap.get(key).toString(); 85 | if (resourceMonitorMap.get(value).getFreeMemory()>1024*64 86 | && resourceMonitorMap.get(value).getCpuRatio()<0.95 87 | && resourceMonitorMap.get(value).getIoRatio()<0.70 88 | && resourceMonitorMap.get(value).getNetRatio()<0.90) 89 | { 90 | resourceList.add(value); 91 | } 92 | } 93 | if (!resourceList.isEmpty()) 94 | { 95 | break; 96 | } 97 | Thread.sleep(3000); 98 | } 99 | return resourceList; 100 | } 101 | }); 102 | executor.submit(futureTask); 103 | try { 104 | idleResourceList=futureTask.get(); 105 | } catch (Exception e) { 106 | log.error("忽略异常====>"+e); 107 | } 108 | executor.shutdown(); 109 | if (idleResourceList.isEmpty()) 110 | { 111 | Random r = new Random(); 112 | int n=r.nextInt(execResourceMap.size()); 113 | resource=execResourceMap.keySet().toArray()[n].toString(); 114 | } 115 | else 116 | { 117 | Random r = new Random(); 118 | int n=r.nextInt(idleResourceList.size()); 119 | resource=idleResourceList.get(n); 120 | } 121 | return resource; 122 | } 123 | /** 124 | * 获取primary 125 | * @param taskId 126 | * @param inMap 127 | * @return 128 | * @throws Exception 129 | */ 130 | public MgloryOut getPrimary(String taskId,Map inMap)throws Exception 131 | { 132 | MgloryOut out =new MgloryOut(); 133 | Map outMap=new HashMap(); 134 | outMap.put("primary",ElectManager.getInstance().getPrimary()); 135 | out.setMap(outMap); 136 | return out; 137 | } 138 | /** 139 | * 同步primary 140 | * @param taskId 141 | * @param inMap 142 | * @return 143 | * @throws Exception 144 | */ 145 | public MgloryOut synPrimary(String taskId,Map inMap)throws Exception 146 | { 147 | String primarySyn=inMap.get("primary").toString(); 148 | String elector=inMap.get("elector").toString(); 149 | long primaryLeaseSyn=Long.parseLong(inMap.get("primaryLease").toString()); 150 | long primaryLease=ElectManager.getInstance().getPrimaryLease(); 151 | if (primaryLeaseSyn>primaryLease) 152 | { 153 | ElectManager.getInstance().setPrimary(primarySyn); 154 | ElectManager.getInstance().setPrimaryLease(primaryLease); 155 | ElectManager.getInstance().setElector(elector); 156 | //资源为primary 157 | NodeManager.getInstance().setPrimaryFlag(true); 158 | } 159 | MgloryOut out =new MgloryOut(); 160 | Map outMap=new HashMap(); 161 | outMap.put("elector",ElectManager.getInstance().getElector()); 162 | out.setMap(outMap); 163 | return out; 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/ResourceUsage.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.io.PrintWriter; 7 | import java.io.StringWriter; 8 | 9 | import org.apache.commons.logging.Log; 10 | import org.apache.commons.logging.LogFactory; 11 | 12 | public class ResourceUsage { 13 | private static Log log = LogFactory.getLog(ResourceUsage.class); 14 | public static SystemResource getResUsage() throws Exception { 15 | SystemResource resUsage= new SystemResource(); 16 | resUsage.setCpuRatio(getCpuUsage()); 17 | resUsage.setFreeMemory(getFreeMemory()); 18 | resUsage.setIoRatio(getIoUsage()); 19 | resUsage.setNetRatio(getNetUsage()); 20 | return resUsage; 21 | } 22 | /** 23 | * Purpose:采集CPU使用率 24 | * @param args 25 | * @return float,CPU使用率,小于1 26 | */ 27 | public static float getCpuUsage() { 28 | //log.info("开始收集cpu使用率"); 29 | float cpuUsage = 0; 30 | Process pro1,pro2; 31 | Runtime r = Runtime.getRuntime(); 32 | try { 33 | String command = "cat /proc/stat"; 34 | pro1 = r.exec(command); 35 | BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream())); 36 | String line = null; 37 | long idleCpuTime1 = 0, totalCpuTime1 = 0; //分别为系统启动后空闲的CPU时间和总的CPU时间 38 | while((line=in1.readLine()) != null){ 39 | if(line.startsWith("cpu")){ 40 | line = line.trim(); 41 | //log.info(line); 42 | String[] temp = line.split("\\s+"); 43 | idleCpuTime1 = Long.parseLong(temp[4]); 44 | for(String s : temp){ 45 | if(!s.equals("cpu")){ 46 | totalCpuTime1 += Long.parseLong(s); 47 | } 48 | } 49 | //log.info("IdleCpuTime: " + idleCpuTime1 + ", " + "TotalCpuTime" + totalCpuTime1); 50 | break; 51 | } 52 | } 53 | in1.close(); 54 | pro1.destroy(); 55 | try { 56 | Thread.sleep(100); 57 | } catch (InterruptedException e) { 58 | StringWriter sw = new StringWriter(); 59 | e.printStackTrace(new PrintWriter(sw)); 60 | log.error("CpuUsage休眠时发生InterruptedException. " + e.getMessage()); 61 | log.error(sw.toString()); 62 | } 63 | pro2 = r.exec(command); 64 | BufferedReader in2 = new BufferedReader(new InputStreamReader(pro2.getInputStream())); 65 | long idleCpuTime2 = 0, totalCpuTime2 = 0; //分别为系统启动后空闲的CPU时间和总的CPU时间 66 | while((line=in2.readLine()) != null){ 67 | if(line.startsWith("cpu")){ 68 | line = line.trim(); 69 | //log.info(line); 70 | String[] temp = line.split("\\s+"); 71 | idleCpuTime2 = Long.parseLong(temp[4]); 72 | for(String s : temp){ 73 | if(!s.equals("cpu")){ 74 | totalCpuTime2 += Long.parseLong(s); 75 | } 76 | } 77 | //log.info("IdleCpuTime: " + idleCpuTime2 + ", " + "TotalCpuTime" + totalCpuTime2); 78 | break; 79 | } 80 | } 81 | if(idleCpuTime1 != 0 && totalCpuTime1 !=0 && idleCpuTime2 != 0 && totalCpuTime2 !=0){ 82 | cpuUsage = 1 - (float)(idleCpuTime2 - idleCpuTime1)/(float)(totalCpuTime2 - totalCpuTime1); 83 | //log.info("本节点CPU使用率为: " + cpuUsage); 84 | } 85 | in2.close(); 86 | pro2.destroy(); 87 | } catch (IOException e) { 88 | StringWriter sw = new StringWriter(); 89 | e.printStackTrace(new PrintWriter(sw)); 90 | log.error("CpuUsage发生InstantiationException. " + e.getMessage()); 91 | log.error(sw.toString()); 92 | } 93 | return cpuUsage; 94 | } 95 | /** 96 | * Purpose:采集空闲内存 97 | * @param args 98 | * @return 99 | */ 100 | public static long getFreeMemory() { 101 | //log.info("开始收集空闲内存"); 102 | //float memUsage = 0.0f; 103 | Process pro = null; 104 | Runtime r = Runtime.getRuntime(); 105 | //long totalMem = 0, freeMem = 0; 106 | long freeMem=0; 107 | try { 108 | String command = "cat /proc/meminfo"; 109 | pro = r.exec(command); 110 | BufferedReader in = new BufferedReader(new InputStreamReader(pro.getInputStream())); 111 | String line = null; 112 | int count = 0; 113 | while((line=in.readLine()) != null){ 114 | //log.info(line); 115 | String[] memInfo = line.split("\\s+"); 116 | // if(memInfo[0].startsWith("MemTotal")){ 117 | // totalMem = Long.parseLong(memInfo[1]); 118 | // } 119 | if(memInfo[0].startsWith("MemFree")){ 120 | freeMem = Long.parseLong(memInfo[1]); 121 | } 122 | // memUsage = 1- (float)freeMem/(float)totalMem; 123 | // log.info("本节点内存使用率为: " + memUsage); 124 | if(++count == 2){ 125 | break; 126 | } 127 | } 128 | in.close(); 129 | pro.destroy(); 130 | } catch (IOException e) { 131 | StringWriter sw = new StringWriter(); 132 | e.printStackTrace(new PrintWriter(sw)); 133 | log.error("MemUsage发生InstantiationException. " + e.getMessage()); 134 | log.error(sw.toString()); 135 | } 136 | //log.info("空闲内存:"+freeMem); 137 | return freeMem; 138 | } 139 | /** 140 | * @Purpose:采集磁盘IO使用率 141 | * @param args 142 | * @return float,磁盘IO使用率,小于1 143 | */ 144 | public static float getIoUsage() { 145 | //log.info("开始收集磁盘IO使用率"); 146 | float ioUsage = 0.0f; 147 | Process pro = null; 148 | Runtime r = Runtime.getRuntime(); 149 | try { 150 | String command = "iostat -d -x"; 151 | pro = r.exec(command); 152 | BufferedReader in = new BufferedReader(new InputStreamReader(pro.getInputStream())); 153 | String line = null; 154 | int count = 0; 155 | while((line=in.readLine()) != null){ 156 | if(++count >= 4){ 157 | // log.info(line); 158 | String[] temp = line.split("\\s+"); 159 | if(temp.length > 1){ 160 | float util = Float.parseFloat(temp[temp.length-1]); 161 | ioUsage = (ioUsage>util)?ioUsage:util; 162 | } 163 | } 164 | } 165 | if(ioUsage > 0){ 166 | //log.info("本节点磁盘IO使用率为: " + ioUsage); 167 | ioUsage /= 100; 168 | } 169 | in.close(); 170 | pro.destroy(); 171 | } catch (IOException e) { 172 | StringWriter sw = new StringWriter(); 173 | e.printStackTrace(new PrintWriter(sw)); 174 | log.error("IoUsage发生InstantiationException. " + e.getMessage()); 175 | log.error(sw.toString()); 176 | } 177 | return ioUsage; 178 | } 179 | /** 180 | * @Purpose:采集网络带宽使用率 181 | * @param args 182 | * @return float,网络带宽使用率,小于1 183 | */ 184 | public static float getNetUsage() { 185 | float TotalBandwidth = 1000; //网口带宽,Mbps 186 | //log.info("开始收集网络带宽使用率"); 187 | float netUsage = 0.0f; 188 | Process pro1,pro2; 189 | Runtime r = Runtime.getRuntime(); 190 | try { 191 | String command = "cat /proc/net/dev"; 192 | //第一次采集流量数据 193 | long startTime = System.currentTimeMillis(); 194 | pro1 = r.exec(command); 195 | BufferedReader in1 = new BufferedReader(new InputStreamReader(pro1.getInputStream())); 196 | String line = null; 197 | long inSize1 = 0, outSize1 = 0; 198 | while((line=in1.readLine()) != null){ 199 | line = line.trim(); 200 | if(line.startsWith("eth0")){ 201 | //log.info(line); 202 | String[] temp = line.split("\\s+"); 203 | inSize1 = Long.parseLong(temp[0].substring(5)); //Receive bytes,单位为Byte 204 | outSize1 = Long.parseLong(temp[8]); //Transmit bytes,单位为Byte 205 | break; 206 | } 207 | } 208 | in1.close(); 209 | pro1.destroy(); 210 | try { 211 | Thread.sleep(1000); 212 | } catch (InterruptedException e) { 213 | StringWriter sw = new StringWriter(); 214 | e.printStackTrace(new PrintWriter(sw)); 215 | log.error("NetUsage休眠时发生InterruptedException. " + e.getMessage()); 216 | log.error(sw.toString()); 217 | } 218 | //第二次采集流量数据 219 | long endTime = System.currentTimeMillis(); 220 | pro2 = r.exec(command); 221 | BufferedReader in2 = new BufferedReader(new InputStreamReader(pro2.getInputStream())); 222 | long inSize2 = 0 ,outSize2 = 0; 223 | while((line=in2.readLine()) != null){ 224 | line = line.trim(); 225 | if(line.startsWith("eth0")){ 226 | //log.info(line); 227 | String[] temp = line.split("\\s+"); 228 | inSize2 = Long.parseLong(temp[0].substring(5)); 229 | outSize2 = Long.parseLong(temp[8]); 230 | break; 231 | } 232 | } 233 | if(inSize1 != 0 && outSize1 !=0 && inSize2 != 0 && outSize2 !=0){ 234 | float interval = (float)(endTime - startTime)/1000; 235 | //网口传输速度,单位为bps 236 | float curRate = (float)(inSize2 - inSize1 + outSize2 - outSize1)*8/(1000000*interval); 237 | netUsage = curRate/TotalBandwidth; 238 | //log.info("本节点网口速度为: " + curRate + "Mbps"); 239 | //log.info("本节点网络带宽使用率为: " + netUsage); 240 | } 241 | in2.close(); 242 | pro2.destroy(); 243 | } catch (IOException e) { 244 | StringWriter sw = new StringWriter(); 245 | e.printStackTrace(new PrintWriter(sw)); 246 | log.error("NetUsage发生InstantiationException. " + e.getMessage()); 247 | log.error(sw.toString()); 248 | } 249 | return netUsage; 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/resource/SystemResource.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.resource; 2 | 3 | import java.io.Serializable; 4 | 5 | 6 | public class SystemResource implements Serializable{ 7 | 8 | private static final long serialVersionUID = -1553098501230376578L; 9 | 10 | 11 | /** 剩余内存 */ 12 | private long freeMemory; 13 | 14 | /** cpu使用率 */ 15 | private float cpuRatio; 16 | 17 | /** 磁盘IO使用率 */ 18 | private float ioRatio; 19 | 20 | /** 网络带宽使用率 */ 21 | private float netRatio; 22 | 23 | public long getFreeMemory() { 24 | return freeMemory; 25 | } 26 | 27 | public void setFreeMemory(long freeMemory) { 28 | this.freeMemory = freeMemory; 29 | } 30 | 31 | public float getCpuRatio() { 32 | return cpuRatio; 33 | } 34 | 35 | public void setCpuRatio(float cpuRatio) { 36 | this.cpuRatio = cpuRatio; 37 | } 38 | 39 | public float getIoRatio() { 40 | return ioRatio; 41 | } 42 | 43 | public void setIoRatio(float ioRatio) { 44 | this.ioRatio = ioRatio; 45 | } 46 | 47 | public float getNetRatio() { 48 | return netRatio; 49 | } 50 | 51 | public void setNetRatio(float netRatio) { 52 | this.netRatio = netRatio; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/route/MgloryHash.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.route; 2 | 3 | import java.nio.ByteBuffer; 4 | import java.nio.ByteOrder; 5 | import java.util.Collection; 6 | import java.util.SortedMap; 7 | import java.util.TreeMap; 8 | 9 | public class MgloryHash { 10 | 11 | private final int numberOfReplicas;// 每个机器节点关联的虚拟节点个数 12 | private final SortedMap circle = new TreeMap();// 环形虚拟节点 13 | 14 | /** 15 | * 16 | * @param numberOfReplicas 17 | * 每个机器节点关联的虚拟节点个数 18 | * @param nodes 19 | * 真实机器节点 20 | */ 21 | public MgloryHash(int numberOfReplicas, Collection nodes) { 22 | this.numberOfReplicas = numberOfReplicas; 23 | for (T node : nodes) { 24 | add(node); 25 | } 26 | } 27 | public Long hash(String key) { 28 | ByteBuffer buf = ByteBuffer.wrap(key.getBytes()); 29 | int seed = 0x1234ABCD; 30 | 31 | ByteOrder byteOrder = buf.order(); 32 | buf.order(ByteOrder.LITTLE_ENDIAN); 33 | 34 | long m = 0xc6a4a7935bd1e995L; 35 | int r = 47; 36 | 37 | long h = seed ^ (buf.remaining() * m); 38 | 39 | long k; 40 | while (buf.remaining() >= 8) { 41 | k = buf.getLong(); 42 | 43 | k *= m; 44 | k ^= k >>> r; 45 | k *= m; 46 | 47 | h ^= k; 48 | h *= m; 49 | } 50 | 51 | if (buf.remaining() > 0) { 52 | ByteBuffer finish = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); 53 | finish.put(buf).rewind(); 54 | h ^= finish.getLong(); 55 | h *= m; 56 | } 57 | 58 | h ^= h >>> r; 59 | h *= m; 60 | h ^= h >>> r; 61 | 62 | buf.order(byteOrder); 63 | return h; 64 | } 65 | /** 66 | * 增加真实机器节点 67 | * 68 | * @param node 69 | */ 70 | public void add(T node) { 71 | for (int i = 0; i < this.numberOfReplicas; i++) { 72 | circle.put(hash(node.toString() + i), node); 73 | } 74 | } 75 | 76 | /** 77 | * 删除真实机器节点 78 | * 79 | * @param node 80 | */ 81 | public void remove(T node) { 82 | for (int i = 0; i < this.numberOfReplicas; i++) { 83 | circle.remove(hash(node.toString() + i)); 84 | } 85 | } 86 | 87 | /** 88 | * 取得真实机器节点 89 | * 90 | * @param key 91 | * @return 92 | */ 93 | public T get(String key) { 94 | if (circle.isEmpty()) { 95 | return null; 96 | } 97 | 98 | long hash =hash(key); 99 | if (!circle.containsKey(hash)) { 100 | SortedMap tailMap = circle.tailMap(hash);// 沿环的顺时针找到一个虚拟节点 101 | hash = tailMap.isEmpty() ? circle.firstKey() : tailMap.firstKey(); 102 | } 103 | 104 | return circle.get(hash); // 返回该虚拟节点对应的真实机器节点的信息 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/route/MgloryHashHelper.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.route; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Map; 6 | 7 | import com.lingo.mglory.param.MgloryIn; 8 | import com.lingo.mglory.param.MgloryOut; 9 | import com.lingo.mglory.resource.ResourceHelper; 10 | import com.lingo.mglory.transport.client.Client; 11 | import com.lingo.mglory.util.Utils; 12 | 13 | public class MgloryHashHelper { 14 | public static String getHashResource(String masterGroup,String slaveGroup,String key) throws Exception 15 | { 16 | Map execResourceMap=getExecResourceMap(masterGroup,slaveGroup); 17 | return getResource(key, getMgloryHash(execResourceMap),execResourceMap,masterGroup,slaveGroup); 18 | } 19 | public static String getHashResource(String key,MgloryHash> mgloryHash,String masterGroup,String slaveGroup) throws Exception 20 | { 21 | Map execResourceMap=getExecResourceMap(masterGroup,slaveGroup); 22 | return getResource(key,mgloryHash,execResourceMap,masterGroup,slaveGroup); 23 | } 24 | public static MgloryHash> getMgloryHash(Map execResourceMap) 25 | { 26 | List> nodes=new ArrayList>(); 27 | for (String resource:execResourceMap.keySet()) 28 | { 29 | Node node=new Node(resource); 30 | nodes.add(node); 31 | } 32 | MgloryHash> mgloryHash=new MgloryHash>(1000,nodes); 33 | return mgloryHash; 34 | } 35 | public static String getResource(String key,MgloryHash> mgloryHash,Map execResourceMap, 36 | String masterGroup,String slaveGroup)throws Exception 37 | { 38 | if (key==null || key.equals("")) 39 | { 40 | throw new Exception("key不能为空"); 41 | } 42 | int point=key.lastIndexOf("__"); 43 | if (point!=-1) 44 | { 45 | key=key.substring(0,point); 46 | } 47 | String hashResource=mgloryHash.get(key).getResource(); 48 | String resource=execResourceMap.get(hashResource); 49 | if (resource==null) 50 | { 51 | Map bakExecResourceMap=getBakExecResource(masterGroup,slaveGroup); 52 | throw new Exception("资源："+hashResource+"和"+bakExecResourceMap.get(hashResource)+"都处于不可用状态，请检查！"); 53 | } 54 | return resource; 55 | } 56 | public static Map getExecResourceMap(String masterGroup,String slaveGroup) throws Exception 57 | { 58 | String resource=ResourceHelper.getInstance().getFrontResource(masterGroup, slaveGroup); 59 | String ip=resource.split(":")[0]; 60 | int port=Integer.parseInt(resource.split(":")[1]); 61 | MgloryIn in=new MgloryIn(); 62 | in.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.getExecResource"); 63 | MgloryOut out=Client.getInstance().call(ip, port, in); 64 | Map execResourceMap=out.getMap(); 65 | return Utils.transToStrMap(execResourceMap); 66 | } 67 | public static Map getBakExecResource(String masterGroup,String slaveGroup) throws Exception 68 | { 69 | String resource=ResourceHelper.getInstance().getFrontResource(masterGroup, slaveGroup); 70 | String ip=resource.split(":")[0]; 71 | int port=Integer.parseInt(resource.split(":")[1]); 72 | MgloryIn in=new MgloryIn(); 73 | in.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.getBakExecResource"); 74 | MgloryOut out=Client.getInstance().call(ip, port, in); 75 | Map bakExecResource=out.getMap(); 76 | return Utils.transToStrMap(bakExecResource); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/route/Node.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.route; 2 | 3 | public class Node { 4 | 5 | private String resource; 6 | 7 | public Node(String resource) { 8 | this.resource = resource; 9 | } 10 | 11 | public String getResource() { 12 | return resource; 13 | } 14 | 15 | public void setResource(String resource) { 16 | this.resource = resource; 17 | } 18 | /** 19 | * 复写toString方法，使用节点resource当做hash的KEY 20 | */ 21 | @Override 22 | public String toString() { 23 | return resource; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/server/MgloryGroupServer.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * @Description:中心服务类 (资源管理) 3 | * @author sunxuhang 4 | * @revision 1.0 5 | * @date 2013-12-17 6 | *******************************************************************************/ 7 | package com.lingo.mglory.server; 8 | import java.util.Properties; 9 | 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | 13 | import com.lingo.mglory.resource.ResourcePool; 14 | import com.lingo.mglory.transport.server.Server; 15 | import com.lingo.mglory.util.PropertiesUtil; 16 | public class MgloryGroupServer { 17 | private static Log log = LogFactory.getLog(MgloryGroupServer.class); 18 | private static MgloryGroupServer groupServer; 19 | private String ip; 20 | private int port; 21 | private boolean masterFlag=false; 22 | private Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 23 | private MgloryGroupServer() 24 | { 25 | init(); 26 | } 27 | private void init() 28 | { 29 | if (config.getProperty("masterFlag")!=null) 30 | { 31 | masterFlag=Boolean.parseBoolean(config.getProperty("masterFlag")); 32 | } 33 | if (masterFlag) 34 | { 35 | ip=config.getProperty("masterIp"); 36 | port=Integer.parseInt(config.getProperty("masterPort")); 37 | } 38 | else 39 | { 40 | if (config.getProperty("slavePort")!=null) 41 | { 42 | ip=config.getProperty("slaveIp"); 43 | port=Integer.parseInt(config.getProperty("slavePort")); 44 | } 45 | else 46 | { 47 | throw new RuntimeException("请配置slave服务器IP、端口"); 48 | } 49 | } 50 | String masterGroup=config.getProperty("masterIp")+":"+config.getProperty("masterPort"); 51 | String slaveGroup=config.getProperty("slaveIp")+":"+config.getProperty("slavePort"); 52 | ResourcePool.getInstance().setMasterGroup(masterGroup); 53 | ResourcePool.getInstance().setSlaveGroup(slaveGroup); 54 | } 55 | public synchronized static MgloryGroupServer getInstance() 56 | { 57 | if (groupServer==null) 58 | { 59 | groupServer=new MgloryGroupServer(); 60 | } 61 | return groupServer; 62 | } 63 | /** 64 | * 启动中心管理服务 65 | */ 66 | public void startGroupServer() throws Exception 67 | { 68 | startGroupServer(ip,port); 69 | } 70 | public void startGroupServer(String ip,int port) throws Exception 71 | { 72 | try 73 | { 74 | Server.start(ip, port); 75 | } 76 | catch(Exception e) 77 | { 78 | log.error("管理服务启动失败===>"+e); 79 | throw new Exception("管理服务启动失败:"+e); 80 | } 81 | } 82 | public boolean getMasterFlag() 83 | { 84 | return masterFlag; 85 | } 86 | public String getIp() 87 | { 88 | return ip; 89 | } 90 | public int getPort() 91 | { 92 | return port; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/server/MgloryWorkerServer.java: -------------------------------------------------------------------------------- 1 | /******************************************************************************* 2 | * @Description:执行服务类(负责具体事务) 3 | * @author sunxuhang 4 | * @revision 1.0 5 | * @date 2013-12-17 6 | *******************************************************************************/ 7 | package com.lingo.mglory.server; 8 | import java.util.Properties; 9 | 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | 13 | import com.lingo.mglory.transport.server.Server; 14 | import com.lingo.mglory.util.PropertiesUtil; 15 | public class MgloryWorkerServer { 16 | private static Log log = LogFactory.getLog(MgloryWorkerServer.class); 17 | private Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 18 | private static MgloryWorkerServer workerServer; 19 | private String masterGroup; 20 | private String slaveGroup; 21 | private String resource; 22 | private String resourceType; 23 | private MgloryWorkerServer() 24 | { 25 | init(); 26 | } 27 | private void init() 28 | { 29 | String masterIp=config.getProperty("masterIp"); 30 | String masterPort=config.getProperty("masterPort"); 31 | String slaveIp=config.getProperty("slaveIp"); 32 | String slavePort=config.getProperty("slavePort"); 33 | masterGroup=masterIp+":"+masterPort; 34 | slaveGroup=slaveIp+":"+slavePort; 35 | } 36 | public synchronized static MgloryWorkerServer getInstance() 37 | { 38 | if(workerServer==null) 39 | { 40 | workerServer=new MgloryWorkerServer(); 41 | } 42 | return workerServer; 43 | } 44 | /** 45 | * 注册资源 46 | * @param ip 47 | * @param port 48 | * @param resourceType 49 | */ 50 | public void startResource(String ip,int port,String resourceType)throws Exception 51 | { 52 | try 53 | { 54 | Server.start(ip, port); 55 | } 56 | catch(Exception e) 57 | { 58 | log.error("启动工作服务器失败===>"+e); 59 | throw new Exception("启动工作服务器失败===>"+e); 60 | } 61 | resource=ip+":"+port; 62 | this.resourceType=resourceType; 63 | } 64 | public String getMasterGroup() 65 | { 66 | return masterGroup; 67 | } 68 | public String getSlaveGroup() 69 | { 70 | return slaveGroup; 71 | } 72 | public String getResource() 73 | { 74 | return resource; 75 | } 76 | public String getResourceType() 77 | { 78 | return resourceType; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/store/MgloryMap.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.store; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.concurrent.Callable; 8 | import java.util.concurrent.ExecutorService; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.FutureTask; 11 | 12 | import org.apache.commons.logging.Log; 13 | import org.apache.commons.logging.LogFactory; 14 | 15 | import com.lingo.mglory.param.MgloryIn; 16 | import com.lingo.mglory.param.MgloryOut; 17 | import com.lingo.mglory.route.MgloryHashHelper; 18 | import com.lingo.mglory.transport.client.Client; 19 | 20 | public class MgloryMap{ 21 | private static Log log = LogFactory.getLog(MgloryMap.class); 22 | private String masterGroup;//中心服务（资源管理器）主机地址 如：114.215.178.115:1099 23 | private String slaveGroup;//中心服务（资源管理器）备机地址 24 | public MgloryMap(String masterGroup,String slaveGroup) 25 | { 26 | this.masterGroup=masterGroup; 27 | this.slaveGroup=slaveGroup; 28 | } 29 | /** 30 | * 写入 31 | * @param key 32 | * @param value 33 | * @throws Exception 34 | */ 35 | public void put(String key,Object value) throws Exception 36 | { 37 | String resource=MgloryHashHelper.getHashResource(masterGroup, slaveGroup,key); 38 | String ip=resource.split(":")[0]; 39 | int port=Integer.parseInt(resource.split(":")[1]); 40 | Map inMap=new HashMap(); 41 | inMap.put("key",key); 42 | inMap.put("value",value); 43 | MgloryIn in=new MgloryIn(); 44 | in.setMap(inMap); 45 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.put"); 46 | Client.getInstance().call(ip, port, in); 47 | } 48 | /** 49 | * 批量写入 50 | * @param list 51 | * @throws Exception 52 | */ 53 | public void batchPut(List> memorylist) throws Exception 54 | { 55 | Map>> resourceUpdateListMap=new HashMap>>(); 56 | for (Map map:memorylist) 57 | { 58 | if (!map.containsKey("key") || !map.containsKey("value")) 59 | { 60 | log.error("key缺失"); 61 | return; 62 | } 63 | else 64 | { 65 | String key=map.get("key").toString(); 66 | String resource=MgloryHashHelper.getHashResource(masterGroup, slaveGroup,key); 67 | if (resourceUpdateListMap.containsKey(resource)) 68 | { 69 | List> updateList=resourceUpdateListMap.get(resource); 70 | updateList.add(map); 71 | } 72 | else 73 | { 74 | List> updateList=new ArrayList>(); 75 | updateList.add(map); 76 | resourceUpdateListMap.put(resource,updateList); 77 | } 78 | } 79 | } 80 | if (!resourceUpdateListMap.isEmpty()) 81 | { 82 | ExecutorService taskExecutor=Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); 83 | List> futureTasks = new ArrayList>(); 84 | for (String key:resourceUpdateListMap.keySet()) { 85 | final String resource=key; 86 | final List> updateList=resourceUpdateListMap.get(resource); 87 | FutureTask futureTask = new FutureTask( 88 | new Callable() { 89 | @Override 90 | public MgloryOut call() throws Exception { 91 | Map inMap=new HashMap(); 92 | inMap.put("list",updateList); 93 | MgloryIn in=new MgloryIn(); 94 | in.setMap(inMap); 95 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.batchPut"); 96 | String ip=resource.split(":")[0]; 97 | int port=Integer.parseInt(resource.split(":")[1]); 98 | return Client.getInstance().call(ip, port, in); 99 | } 100 | }); 101 | 102 | futureTasks.add(futureTask); 103 | taskExecutor.submit(futureTask); 104 | } 105 | for (FutureTask futureTask : futureTasks) { 106 | try { 107 | futureTask.get(); 108 | } catch (Exception e) { 109 | //throw e; 110 | log.error("error===>"+e); 111 | } 112 | } 113 | taskExecutor.shutdown(); 114 | } 115 | } 116 | /** 117 | * 读 118 | * @param key 119 | * @return 120 | * @throws Exception 121 | */ 122 | public Object get(String key) throws Exception 123 | { 124 | String resource=MgloryHashHelper.getHashResource(masterGroup, slaveGroup,key); 125 | String ip=resource.split(":")[0]; 126 | int port=Integer.parseInt(resource.split(":")[1]); 127 | Map inMap=new HashMap(); 128 | inMap.put("key",key); 129 | MgloryIn in=new MgloryIn(); 130 | in.setMap(inMap); 131 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.get"); 132 | MgloryOut out=Client.getInstance().call(ip, port, in); 133 | Object result=out.getMap().get(key); 134 | return result; 135 | 136 | } 137 | /** 138 | * 读一级key下的所有信息 139 | * @param rootKey 140 | * @return 141 | * @throws Exception 142 | */ 143 | public MgloryOut getAllByRootKey(String rootKey) throws Exception 144 | { 145 | String resource=MgloryHashHelper.getHashResource(masterGroup, slaveGroup,rootKey); 146 | String ip=resource.split(":")[0]; 147 | int port=Integer.parseInt(resource.split(":")[1]); 148 | Map inMap=new HashMap(); 149 | inMap.put("rootKey",rootKey); 150 | MgloryIn in=new MgloryIn(); 151 | in.setMap(inMap); 152 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.getAllByRootKey"); 153 | MgloryOut out=Client.getInstance().call(ip, port, in); 154 | return out; 155 | } 156 | /** 157 | * 读所有数据 158 | * @return 159 | * @throws Exception 160 | */ 161 | public List getAll() throws Exception 162 | { 163 | List result=new ArrayList(); 164 | Map execResourceMap=MgloryHashHelper.getExecResourceMap(masterGroup,slaveGroup); 165 | for (String resource:execResourceMap.keySet()) 166 | { 167 | try 168 | { 169 | String ip=resource.split(":")[0]; 170 | int port=Integer.parseInt(resource.split(":")[1]); 171 | MgloryIn in=new MgloryIn(); 172 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.getAll"); 173 | MgloryOut out=Client.getInstance().call(ip, port, in); 174 | result.add(out); 175 | } 176 | catch(Exception e) 177 | { 178 | log.error("error===>"+e); 179 | throw new Exception("读是有数据失败:"+e); 180 | } 181 | } 182 | return result; 183 | } 184 | /** 185 | * 移除key 186 | * @param key 187 | */ 188 | public void remove(String key)throws Exception 189 | { 190 | String resource=MgloryHashHelper.getHashResource(masterGroup, slaveGroup,key); 191 | String ip=resource.split(":")[0]; 192 | int port=Integer.parseInt(resource.split(":")[1]); 193 | Map inMap=new HashMap(); 194 | inMap.put("key",key); 195 | MgloryIn in=new MgloryIn(); 196 | in.setMap(inMap); 197 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.remove"); 198 | Client.getInstance().call(ip, port, in); 199 | } 200 | /** 201 | * 清空 202 | * @throws Exception 203 | */ 204 | public void clear()throws Exception 205 | { 206 | Map execResourceMap=MgloryHashHelper.getExecResourceMap(masterGroup,slaveGroup); 207 | for (String resource:execResourceMap.keySet()) 208 | { 209 | try 210 | { 211 | String ip=resource.split(":")[0]; 212 | int port=Integer.parseInt(resource.split(":")[1]); 213 | MgloryIn in=new MgloryIn(); 214 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.clear"); 215 | Client.getInstance().call(ip, port, in); 216 | } 217 | catch(Exception e) 218 | { 219 | log.error("error===>"+e); 220 | throw new Exception("清空失败:"+e); 221 | } 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/store/MgloryMapManager.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.store; 2 | 3 | import java.lang.reflect.Method; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.concurrent.ConcurrentMap; 8 | 9 | import org.apache.commons.logging.Log; 10 | import org.apache.commons.logging.LogFactory; 11 | 12 | import com.lingo.mglory.param.MgloryIn; 13 | import com.lingo.mglory.param.MgloryOut; 14 | import com.lingo.mglory.transport.client.Client; 15 | 16 | public class MgloryMapManager { 17 | private static Log log = LogFactory.getLog(MgloryMapManager.class); 18 | public MgloryOut put(String taskId,Map inMap) 19 | { 20 | MgloryOut out=new MgloryOut(); 21 | String key=inMap.get("key").toString(); 22 | Object value=inMap.get("value"); 23 | MgloryMapStore.getInstance().mgloryMap.put(key,value); 24 | if (MgloryMapStore.getInstance().synFlag) 25 | { 26 | //加入待同步队列 27 | Map> operMap=new HashMap>(); 28 | operMap.put("put",inMap); 29 | MgloryMapStore.getInstance().operQueue.add(operMap); 30 | } 31 | return out; 32 | } 33 | @SuppressWarnings("unchecked") 34 | public MgloryOut batchPut(String taskId,Map inMap) 35 | { 36 | MgloryOut out=new MgloryOut(); 37 | List> updateList=(List>)inMap.get("list"); 38 | for (Map map:updateList) 39 | { 40 | String key=map.get("key").toString(); 41 | Object value=map.get("value"); 42 | MgloryMapStore.getInstance().mgloryMap.put(key,value); 43 | } 44 | if (MgloryMapStore.getInstance().synFlag) 45 | { 46 | //加入待同步队列 47 | Map> operMap=new HashMap>(); 48 | operMap.put("batchPut",inMap); 49 | MgloryMapStore.getInstance().operQueue.add(operMap); 50 | } 51 | return out; 52 | } 53 | public MgloryOut get(String taskId,Map inMap) 54 | { 55 | MgloryOut out=new MgloryOut(); 56 | String key=inMap.get("key").toString(); 57 | Map resultMap=new HashMap(); 58 | resultMap.put(key,MgloryMapStore.getInstance().mgloryMap.get(key)); 59 | out.setMap(resultMap); 60 | return out; 61 | } 62 | public MgloryOut getAllByRootKey(String taskId,Map inMap) 63 | { 64 | MgloryOut out=new MgloryOut(); 65 | String rootKey=inMap.get("rootKey").toString(); 66 | Map resultMap=new HashMap(); 67 | for (String key:MgloryMapStore.getInstance().mgloryMap.keySet()) 68 | { 69 | if (key.startsWith(rootKey+"__")) 70 | { 71 | resultMap.put(key,MgloryMapStore.getInstance().mgloryMap.get(key)); 72 | } 73 | } 74 | out.setMap(resultMap); 75 | return out; 76 | } 77 | public MgloryOut getAll(String taskId,Map inMap) 78 | { 79 | MgloryOut out=new MgloryOut(); 80 | out.setMap(MgloryMapStore.getInstance().mgloryMap); 81 | return out; 82 | } 83 | public MgloryOut remove(String taskId,Map inMap) 84 | { 85 | MgloryOut out=new MgloryOut(); 86 | String key=inMap.get("key").toString(); 87 | MgloryMapStore.getInstance().mgloryMap.remove(key); 88 | if (MgloryMapStore.getInstance().synFlag) 89 | { 90 | //加入待同步队列 91 | Map> operMap=new HashMap>(); 92 | operMap.put("remove", inMap); 93 | MgloryMapStore.getInstance().operQueue.add(operMap); 94 | } 95 | return out; 96 | } 97 | public MgloryOut clear(String taskId,Map inMap) 98 | { 99 | MgloryOut out=new MgloryOut(); 100 | MgloryMapStore.getInstance().mgloryMap.clear(); 101 | if (MgloryMapStore.getInstance().synFlag) 102 | { 103 | //加入待同步队列 104 | Map> operMap=new HashMap>(); 105 | operMap.put("clear", inMap); 106 | MgloryMapStore.getInstance().operQueue.add(operMap); 107 | } 108 | return out; 109 | } 110 | public MgloryOut getKeys(String taskId,Map inMap) 111 | { 112 | MgloryOut out=new MgloryOut(); 113 | Map keyMap=new HashMap(); 114 | for (String key:MgloryMapStore.getInstance().mgloryMap.keySet()) 115 | { 116 | keyMap.put(key,""); 117 | } 118 | out.setMap(keyMap); 119 | return out; 120 | } 121 | @SuppressWarnings("unchecked") 122 | public MgloryOut syn(String taskId,Map inMap) 123 | { 124 | MgloryOut out=new MgloryOut(); 125 | List>> list=(List>>)inMap.get("list"); 126 | for (Map> operMap:list) 127 | { 128 | String operType=operMap.keySet().toArray()[0].toString(); 129 | Method method = MgloryMapStore.getInstance().methodMap.get(operType); 130 | try { 131 | method.invoke(this,operMap.get(operType)); 132 | } catch (Exception e) { 133 | log.error("异常忽略不计:"+e); 134 | } 135 | } 136 | return out; 137 | } 138 | public MgloryOut bakToPrimary(String taskId,Map inMap) 139 | { 140 | MgloryOut out=new MgloryOut(); 141 | String resource=inMap.get("resource").toString(); 142 | String ip=resource.split(":")[0]; 143 | int port=Integer.parseInt(resource.split(":")[1]); 144 | MgloryIn in=new MgloryIn(); 145 | inMap=new HashMap(); 146 | inMap.put("mgloryMap", MgloryMapStore.getInstance().mgloryMap); 147 | in.setMap(inMap); 148 | in.setInvokeMethod("com.lingo.mglory.store.MgloryMapManager.receiveFromBak"); 149 | Client.getInstance().call(ip, port, in); 150 | return out; 151 | } 152 | @SuppressWarnings("unchecked") 153 | public MgloryOut receiveFromBak(String taskId,Map inMap) 154 | { 155 | MgloryOut out=new MgloryOut(); 156 | MgloryMapStore.getInstance().mgloryMap=(ConcurrentMap)inMap.get("mgloryMap"); 157 | return out; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/store/MgloryMapStore.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.store; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.BufferedWriter; 5 | import java.io.File; 6 | import java.io.FileInputStream; 7 | import java.io.FileWriter; 8 | import java.io.IOException; 9 | import java.io.InputStreamReader; 10 | import java.lang.reflect.Method; 11 | import java.util.ArrayList; 12 | import java.util.HashMap; 13 | import java.util.List; 14 | import java.util.Map; 15 | import java.util.concurrent.Callable; 16 | import java.util.concurrent.ConcurrentHashMap; 17 | import java.util.concurrent.ConcurrentLinkedQueue; 18 | import java.util.concurrent.ConcurrentMap; 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.Executors; 21 | import java.util.concurrent.FutureTask; 22 | 23 | import org.apache.commons.logging.Log; 24 | import org.apache.commons.logging.LogFactory; 25 | 26 | import com.lingo.mglory.param.MgloryIn; 27 | import com.lingo.mglory.param.MgloryOut; 28 | import com.lingo.mglory.resource.ResourceHelper; 29 | import com.lingo.mglory.server.MgloryWorkerServer; 30 | import com.lingo.mglory.transport.client.Client; 31 | import com.lingo.mglory.util.Utils; 32 | 33 | public class MgloryMapStore { 34 | private static Log log = LogFactory.getLog(MgloryMapStore.class); 35 | private static MgloryMapStore mgloryMapStore; 36 | private String masterGroup; 37 | public boolean synFlag=false; 38 | public ConcurrentMap mgloryMap=new ConcurrentHashMap(); 39 | public ConcurrentLinkedQueue>> operQueue=new ConcurrentLinkedQueue>>(); 40 | public final Map methodMap = new HashMap(); 41 | private MgloryMapStore(){ 42 | init(); 43 | if (synFlag) 44 | { 45 | synToBakTask(); 46 | } 47 | //首次需要进行加载 48 | loadMemory(); 49 | persistenceTask(); 50 | } 51 | public synchronized static MgloryMapStore getInstance() 52 | { 53 | if (mgloryMapStore==null) 54 | { 55 | mgloryMapStore=new MgloryMapStore(); 56 | } 57 | return mgloryMapStore; 58 | } 59 | private void init() 60 | { 61 | try 62 | { 63 | masterGroup=MgloryWorkerServer.getInstance().getMasterGroup(); 64 | MgloryIn mgloryIn=new MgloryIn(); 65 | mgloryIn.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.getBakExecResource"); 66 | String ip=masterGroup.split(":")[0]; 67 | int port=Integer.parseInt(masterGroup.split(":")[1]); 68 | MgloryOut mgloryOut=Client.getInstance().call(ip, port, mgloryIn); 69 | Map bakExecResourceMap=mgloryOut.getMap(); 70 | String resource=MgloryWorkerServer.getInstance().getResource(); 71 | if (bakExecResourceMap.containsKey(resource)) 72 | { 73 | synFlag=true; 74 | } 75 | if (methodMap.isEmpty()) 76 | { 77 | Method[] methods = new MgloryMapManager().getClass().getMethods(); 78 | for (int i=0; i"+e); 86 | } 87 | } 88 | private void synToBakTask() 89 | { 90 | ExecutorService executor=Executors.newFixedThreadPool(1); 91 | FutureTask> futureTask = new FutureTask>( 92 | new Callable>() { 93 | @Override 94 | public Map call() throws Exception { 95 | String resource=ResourceHelper.getInstance().getFrontResource(masterGroup,masterGroup); 96 | String ip=resource.split(":")[0]; 97 | int port=Integer.parseInt(resource.split(":")[1]); 98 | MgloryIn in=new MgloryIn(); 99 | in.setInvokeMethod("com.lingo.mglory.resource.ResourceManager.getBakExecResource"); 100 | while(true) 101 | { 102 | try 103 | { 104 | MgloryOut out=Client.getInstance().call(ip, port, in); 105 | Map bakExecResourceMap=out.getMap(); 106 | if (bakExecResourceMap.containsKey(resource)) 107 | { 108 | String bakResource=bakExecResourceMap.get(resource).toString(); 109 | List>> list=new ArrayList>>(); 110 | for (int i=0;i map=new HashMap(); 115 | map.put("list",list); 116 | MgloryIn mgloryIn=new MgloryIn(); 117 | mgloryIn.setInvokeMethod("com.lingo.mglory.application.MgloryMapApp.syn"); 118 | mgloryIn.setMap(map); 119 | String bakIp=bakResource.split(":")[0]; 120 | int bakPort=Integer.parseInt(bakResource.split(":")[1]); 121 | out=Client.getInstance().call(bakIp, bakPort, mgloryIn); 122 | } 123 | } 124 | catch(Exception e) 125 | { 126 | log.error("同步异常忽略不计:"+e); 127 | } 128 | Thread.sleep(600000); 129 | } 130 | } 131 | 132 | }); 133 | executor.submit(futureTask); 134 | executor.shutdown(); 135 | } 136 | private void persistenceTask() 137 | { 138 | ExecutorService executor=Executors.newFixedThreadPool(1); 139 | FutureTask> futureTask = new FutureTask>( 140 | new Callable>() { 141 | @Override 142 | public Map call() throws Exception { 143 | while(true) 144 | { 145 | try 146 | { 147 | persistence(); 148 | } 149 | catch(Exception e) 150 | { 151 | log.error("同步异常忽略不计:"+e); 152 | } 153 | Thread.sleep(60000); 154 | } 155 | } 156 | 157 | }); 158 | executor.submit(futureTask); 159 | executor.shutdown(); 160 | } 161 | public void loadMemory() 162 | { 163 | String resource=MgloryWorkerServer.getInstance().getResource(); 164 | String port=resource.split(":")[1]; 165 | String fileName="./db/mglory"+port+".db"; 166 | File file=new File(fileName); 167 | if(file.exists()) 168 | { 169 | InputStreamReader read=null; 170 | try 171 | { 172 | read = new InputStreamReader(new FileInputStream(file)); 173 | BufferedReader bufferedReader = new BufferedReader(read); 174 | String lineTxt = null; 175 | while((lineTxt = bufferedReader.readLine()) != null){ 176 | String key=lineTxt.split("<=>")[0]; 177 | String value=lineTxt.split("<=>").length==2?lineTxt.split("<=>")[1]:null; 178 | //ConcurrentMap map=transStrToMap(value); 179 | //mgloryMap.put(key,map); 180 | mgloryMap.put(key,value); 181 | } 182 | } 183 | catch(Exception e) 184 | { 185 | e.printStackTrace(); 186 | } 187 | finally 188 | { 189 | if (read!=null) 190 | { 191 | try { 192 | read.close(); 193 | } catch (IOException e) { 194 | // TODO Auto-generated catch block 195 | e.printStackTrace(); 196 | } 197 | } 198 | } 199 | } 200 | } 201 | //持久化 202 | public void persistence() 203 | { 204 | long t=System.currentTimeMillis(); 205 | String resource=MgloryWorkerServer.getInstance().getResource(); 206 | String port=resource.split(":")[1]; 207 | String fileName="./db/mglory"+port+".db"; 208 | String tempFileName="./db/mglory"+port+t+".db"; 209 | File file=new File(fileName); 210 | if(file.exists()) 211 | { 212 | file.renameTo(new File(tempFileName)); 213 | } 214 | else 215 | { 216 | try { 217 | file.createNewFile(); 218 | } catch (IOException e) { 219 | // TODO Auto-generated catch block 220 | e.printStackTrace(); 221 | } 222 | } 223 | BufferedWriter bufferedWriter = null; 224 | try { 225 | bufferedWriter = new BufferedWriter(new FileWriter(file)); 226 | for (String key:mgloryMap.keySet()) 227 | { 228 | Object obj=mgloryMap.get(key); 229 | String value=Utils.transMapToStr(obj); 230 | bufferedWriter.write(key+"<=>"+value); 231 | bufferedWriter.newLine(); 232 | } 233 | } catch (Exception ex) { 234 | ex.printStackTrace(); 235 | } finally { 236 | //Close the BufferedWriter 237 | try { 238 | if (bufferedWriter != null) { 239 | bufferedWriter.flush(); 240 | bufferedWriter.close(); 241 | } 242 | } catch (IOException ex) { 243 | ex.printStackTrace(); 244 | } 245 | } 246 | File tempFile=new File(tempFileName); 247 | if (tempFile.exists()) 248 | { 249 | tempFile.delete(); 250 | } 251 | //loadMemory(); 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/task/TaskState.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.task; 2 | 3 | public class TaskState { 4 | public static final String WAIT_RUN="WAIT_RUN";//待执行 5 | public static final String RUNNING="RUNNING";//执行中 6 | public static final String ERROR="ERROR";//错误 7 | public static final String COMPLETE="COMPLETE";//完成 8 | public static final String INTERRUPT="INTERRUPT";//中断 9 | } 10 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/transport/client/Client.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.transport.client; 2 | 3 | import java.util.Properties; 4 | 5 | import com.lingo.mglory.param.MgloryIn; 6 | import com.lingo.mglory.param.MgloryOut; 7 | import com.lingo.mglory.transport.handler.ClientHandler; 8 | import com.lingo.mglory.util.PropertiesUtil; 9 | 10 | import io.netty.bootstrap.Bootstrap; 11 | import io.netty.channel.ChannelFuture; 12 | import io.netty.channel.ChannelInitializer; 13 | import io.netty.channel.ChannelOption; 14 | import io.netty.channel.ChannelPipeline; 15 | import io.netty.channel.EventLoopGroup; 16 | import io.netty.channel.nio.NioEventLoopGroup; 17 | import io.netty.channel.socket.SocketChannel; 18 | import io.netty.channel.socket.nio.NioSocketChannel; 19 | import io.netty.handler.codec.LengthFieldBasedFrameDecoder; 20 | import io.netty.handler.codec.LengthFieldPrepender; 21 | import io.netty.handler.codec.serialization.ClassResolvers; 22 | import io.netty.handler.codec.serialization.ObjectDecoder; 23 | import io.netty.handler.codec.serialization.ObjectEncoder; 24 | public class Client 25 | { 26 | private Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 27 | private int CONNECT_TIMEOUT_MILLIS=Integer.parseInt(config.getProperty("clientTimeOut")); 28 | private static Client client; 29 | private Client() 30 | { 31 | 32 | } 33 | public synchronized static Client getInstance() 34 | { 35 | if (client==null) 36 | { 37 | client=new Client(); 38 | } 39 | return client; 40 | } 41 | public MgloryOut call(String ip,int port,MgloryIn mgloryIn) 42 | { 43 | EventLoopGroup group = new NioEventLoopGroup(); 44 | MgloryOut mgloryOut=new MgloryOut(); 45 | try 46 | { 47 | Bootstrap b = new Bootstrap(); 48 | b.group(group); 49 | final ClientHandler chl=new ClientHandler(); 50 | b.channel(NioSocketChannel.class).option(ChannelOption.TCP_NODELAY, true); 51 | b.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, CONNECT_TIMEOUT_MILLIS); 52 | b.handler(new ChannelInitializer() { 53 | @Override 54 | protected void initChannel(SocketChannel ch) throws Exception { 55 | ChannelPipeline pipeline = ch.pipeline(); 56 | pipeline.addLast("frameDecoder", new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4)); 57 | pipeline.addLast("frameEncoder", new LengthFieldPrepender(4)); 58 | pipeline.addLast("encode", new ObjectEncoder()); 59 | pipeline.addLast("decode", new ObjectDecoder(ClassResolvers.weakCachingConcurrentResolver(null))); 60 | pipeline.addLast("handler", chl); 61 | } 62 | }); 63 | ChannelFuture f = b.connect(ip, port).sync(); 64 | f.channel().writeAndFlush(mgloryIn); 65 | f.channel().closeFuture().sync(); 66 | mgloryOut=chl.getMessage(); 67 | } 68 | catch (Exception e) 69 | { 70 | e.printStackTrace(); 71 | } 72 | finally 73 | { 74 | group.shutdownGracefully(); 75 | } 76 | return mgloryOut; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/transport/handler/ClientHandler.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.transport.handler; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | import com.lingo.mglory.param.MgloryOut; 7 | 8 | import io.netty.channel.ChannelHandlerContext; 9 | import io.netty.channel.ChannelInboundHandlerAdapter; 10 | 11 | public class ClientHandler extends ChannelInboundHandlerAdapter 12 | { 13 | private static Log log = LogFactory.getLog(ClientHandler.class); 14 | private MgloryOut mgloryOut=new MgloryOut(); 15 | @Override 16 | public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception 17 | { 18 | //log.debug("client接收到服务器返回的消息"); 19 | mgloryOut=(MgloryOut)msg; 20 | } 21 | public MgloryOut getMessage() 22 | { 23 | return this.mgloryOut; 24 | } 25 | @Override 26 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception 27 | { 28 | log.warn("client exception is general:"+cause); 29 | } 30 | 31 | } -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/transport/handler/ServerHandler.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.transport.handler; 2 | 3 | 4 | import java.lang.reflect.Method; 5 | import java.util.Map; 6 | import java.util.concurrent.Callable; 7 | import java.util.concurrent.ExecutorService; 8 | import java.util.concurrent.Executors; 9 | import java.util.concurrent.FutureTask; 10 | 11 | import org.apache.commons.logging.Log; 12 | import org.apache.commons.logging.LogFactory; 13 | 14 | import com.lingo.mglory.param.MgloryIn; 15 | import com.lingo.mglory.param.MgloryOut; 16 | 17 | import io.netty.channel.ChannelHandlerContext; 18 | 19 | import io.netty.channel.ChannelInboundHandlerAdapter; 20 | 21 | 22 | public class ServerHandler extends ChannelInboundHandlerAdapter 23 | { 24 | private static Log log = LogFactory.getLog(ServerHandler.class); 25 | private boolean taskCompleteFlag; 26 | @Override 27 | public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception 28 | { 29 | MgloryIn mgloryIn=(MgloryIn)msg; 30 | //log.debug("SERVER接收方法调用==>"+mgloryIn.getInvokeMethod()); 31 | //log.debug("SERVER接收参数==>"+mgloryIn.getMap()); 32 | String taskId=mgloryIn.getTaskId(); 33 | String invokeMethod=mgloryIn.getInvokeMethod(); 34 | Map inMap=mgloryIn.getMap(); 35 | long timeout=mgloryIn.getTimeout(); 36 | MgloryOut mgloryOut=invoke(invokeMethod,taskId,inMap,timeout); 37 | ctx.channel().writeAndFlush(mgloryOut); 38 | ctx.close(); 39 | } 40 | @Override 41 | public void channelActive(ChannelHandlerContext ctx) throws Exception 42 | { 43 | //log.info(">>>>>>>>"); 44 | } 45 | @Override 46 | public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception 47 | { 48 | log.warn("server exception is general:"+cause); 49 | } 50 | public MgloryOut invoke(final String invokeMethod,final String taskId,final Map inMap,final long timeout) 51 | { 52 | final long t=System.currentTimeMillis(); 53 | MgloryOut out=new MgloryOut(); 54 | ExecutorService taskExecutor=Executors.newFixedThreadPool(1); 55 | final FutureTask futureTask = new FutureTask( 56 | new Callable() { 57 | @Override 58 | public MgloryOut call() throws Exception { 59 | return invoke(invokeMethod,taskId,inMap); 60 | } 61 | }); 62 | taskExecutor.submit(futureTask); 63 | ExecutorService interruptedExecutor=Executors.newFixedThreadPool(1); 64 | FutureTask> interruptedTask = new FutureTask>( 65 | new Callable>() { 66 | @Override 67 | public Map call() throws Exception { 68 | while(!taskCompleteFlag) 69 | { 70 | if (timeout!=0 && System.currentTimeMillis()-t>timeout) 71 | { 72 | //终止任务执行 73 | if(!futureTask.isCancelled()) 74 | { 75 | futureTask.cancel(true); 76 | } 77 | taskCompleteFlag=true; 78 | break; 79 | } 80 | else 81 | { 82 | if (timeout==0) 83 | { 84 | break; 85 | } 86 | else 87 | { 88 | Thread.sleep(1000); 89 | } 90 | } 91 | } 92 | return null; 93 | } 94 | }); 95 | interruptedExecutor.submit(interruptedTask); 96 | interruptedExecutor.shutdown(); 97 | try 98 | { 99 | out=futureTask.get(); 100 | } 101 | catch(Exception e) 102 | { 103 | //throw new RuntimeException(e); 104 | out.setStatus(MgloryOut.INTERRUPT); 105 | out.setErrorMsg(e.getMessage()); 106 | } 107 | finally 108 | { 109 | taskCompleteFlag=true; 110 | taskExecutor.shutdown(); 111 | } 112 | return out; 113 | } 114 | public MgloryOut invoke(String invokeMethod,final String taskId,Map inMap) 115 | { 116 | MgloryOut mgloryOut=new MgloryOut(); 117 | if (invokeMethod.equals("")) 118 | { 119 | mgloryOut.setStatus(MgloryOut.EXCEPTION); 120 | mgloryOut.setErrorMsg("类方法名不能为空"); 121 | } 122 | else 123 | { 124 | String className=""; 125 | String methodName=""; 126 | int lastPointIndex=invokeMethod.lastIndexOf("."); 127 | className=invokeMethod.substring(0,lastPointIndex); 128 | methodName=invokeMethod.substring(lastPointIndex+1,invokeMethod.length()); 129 | try{ 130 | Class cls = Class.forName(className); 131 | Object instance = cls.newInstance(); 132 | Method method = cls.getMethod(methodName,String.class, Map.class); 133 | mgloryOut=(MgloryOut)method.invoke(instance,taskId,inMap); 134 | } 135 | catch(Exception e) 136 | { 137 | mgloryOut.setStatus(MgloryOut.EXCEPTION); 138 | mgloryOut.setErrorMsg("执行异常："+e); 139 | } 140 | } 141 | return mgloryOut; 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/transport/server/Server.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.transport.server; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | 6 | import com.lingo.mglory.transport.handler.ServerHandler; 7 | 8 | import io.netty.bootstrap.ServerBootstrap; 9 | import io.netty.channel.ChannelInitializer; 10 | import io.netty.channel.ChannelOption; 11 | import io.netty.channel.ChannelPipeline; 12 | import io.netty.channel.EventLoopGroup; 13 | import io.netty.channel.nio.NioEventLoopGroup; 14 | import io.netty.channel.socket.SocketChannel; 15 | import io.netty.channel.socket.nio.NioServerSocketChannel; 16 | import io.netty.handler.codec.LengthFieldBasedFrameDecoder; 17 | import io.netty.handler.codec.LengthFieldPrepender; 18 | import io.netty.handler.codec.serialization.ClassResolvers; 19 | import io.netty.handler.codec.serialization.ObjectDecoder; 20 | import io.netty.handler.codec.serialization.ObjectEncoder; 21 | public class Server { 22 | private static Log log = LogFactory.getLog(Server.class); 23 | protected static final int BIZGROUPSIZE = Runtime.getRuntime().availableProcessors() * 2; 24 | //protected static final int BIZTHREADSIZE = 4; 25 | private static final EventLoopGroup bossGroup = new NioEventLoopGroup(BIZGROUPSIZE); 26 | //private static final EventLoopGroup workerGroup = new NioEventLoopGroup(BIZTHREADSIZE); 27 | private static final EventLoopGroup workerGroup = new NioEventLoopGroup(); 28 | private Server() 29 | { 30 | 31 | } 32 | public static void start(String ip,int port) throws Exception { 33 | ServerBootstrap b = new ServerBootstrap(); 34 | b.group(bossGroup, workerGroup); 35 | b.channel(NioServerSocketChannel.class); 36 | b.option(ChannelOption.SO_BACKLOG, 1000000); 37 | b.childHandler(new ChannelInitializer() { 38 | @Override 39 | public void initChannel(SocketChannel ch) throws Exception { 40 | ChannelPipeline pipeline = ch.pipeline(); 41 | pipeline.addLast(new LengthFieldBasedFrameDecoder(Integer.MAX_VALUE, 0, 4, 0, 4)); 42 | pipeline.addLast(new LengthFieldPrepender(4)); 43 | pipeline.addLast("encode", new ObjectEncoder()); 44 | pipeline.addLast("decode", new ObjectDecoder(ClassResolvers.weakCachingConcurrentResolver(null))); 45 | pipeline.addLast(new ServerHandler()); 46 | } 47 | }); 48 | b.bind(ip, port).sync(); 49 | log.info("服务器已启动"); 50 | } 51 | public static void shutdown() { 52 | workerGroup.shutdownGracefully(); 53 | bossGroup.shutdownGracefully(); 54 | } 55 | 56 | public static void main(String[] args) throws Exception { 57 | System.out.println("开始启动服务器..."); 58 | start("localhost",9990); 59 | System.out.println("11111111111111111111111"); 60 | //shutdown(); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/util/PropertiesUtil.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.util; 2 | import java.io.FileInputStream; 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.Properties; 6 | 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | 10 | import com.lingo.mglory.server.MgloryGroupServer; 11 | 12 | public class PropertiesUtil { 13 | private static Log log = LogFactory.getLog(MgloryGroupServer.class); 14 | private static PropertiesUtil propertiesUtil; 15 | private static Map configMap=new HashMap(); 16 | public synchronized static PropertiesUtil getInstance() 17 | { 18 | if (propertiesUtil==null) 19 | { 20 | propertiesUtil=new PropertiesUtil(); 21 | } 22 | return propertiesUtil; 23 | } 24 | public Properties getConfig(String fileName) 25 | { 26 | Properties config=new Properties(); 27 | if (configMap.get(fileName)!=null) 28 | { 29 | config= configMap.get(fileName); 30 | } 31 | else 32 | { 33 | String configFile=PropertiesUtil.class.getClassLoader().getResource(fileName).getPath().toString(); 34 | configFile=configFile.replaceAll("%20", " "); 35 | FileInputStream inputStream=null; 36 | try 37 | { 38 | inputStream=new FileInputStream(configFile); 39 | config.load(inputStream); 40 | configMap.put(fileName,config); 41 | } 42 | catch(Exception e) 43 | { 44 | log.error("load properties error id :"+e.getLocalizedMessage()); 45 | } 46 | finally 47 | { 48 | if(inputStream!=null) 49 | { 50 | try 51 | { 52 | inputStream.close(); 53 | } 54 | catch(Exception e) 55 | { 56 | log.error("load properties error id :"+e.getLocalizedMessage()); 57 | } 58 | } 59 | } 60 | } 61 | 62 | return config; 63 | } 64 | } -------------------------------------------------------------------------------- /mglory/src/com/lingo/mglory/util/Utils.java: -------------------------------------------------------------------------------- 1 | package com.lingo.mglory.util; 2 | 3 | import java.text.SimpleDateFormat; 4 | import java.util.Date; 5 | import java.util.HashMap; 6 | import java.util.Map; 7 | import java.util.Properties; 8 | 9 | public class Utils { 10 | public static void setRmiPara() 11 | { 12 | Properties config= PropertiesUtil.getInstance().getConfig("config.properties"); 13 | //设置调用超时时间 14 | if (config.getProperty("clientTimeOut")!=null && System.getProperty("sun.rmi.transport.proxy.connectTimeout")!=null) 15 | { 16 | String timeOut=config.getProperty("clientTimeOut"); 17 | System.setProperty("sun.rmi.transport.proxy.connectTimeout", timeOut); 18 | System.setProperty("sun.rmi.transport.tcp.responseTimeout", timeOut); 19 | } 20 | } 21 | public static T newInstance(Class clazz) { 22 | try { 23 | return clazz.newInstance(); 24 | } catch (Exception e) { 25 | throw new RuntimeException(e); 26 | } 27 | } 28 | @SuppressWarnings("unchecked") 29 | public static T newInstance(String className) { 30 | try { 31 | Class clazz=(Class) Class.forName(className); 32 | return clazz.newInstance(); 33 | } catch (Exception e) { 34 | throw new RuntimeException(e); 35 | } 36 | } 37 | public static Map transMap(Map map) 38 | { 39 | Map result=new HashMap(); 40 | for (String key:map.keySet()) 41 | { 42 | result.put(key, map.get(key)); 43 | } 44 | return result; 45 | } 46 | public static Map transToStrMap(Map map) 47 | { 48 | Map result=new HashMap(); 49 | for (String key:map.keySet()) 50 | { 51 | result.put(key, map.get(key)==null?"":map.get(key).toString()); 52 | } 53 | return result; 54 | } 55 | public static Map transStrToMap(String str) 56 | { 57 | 58 | Map map=new HashMap(); 59 | if (str!=null) 60 | { 61 | String[] kvs=str.split("!,!"); 62 | for (String kv:kvs) 63 | { 64 | String k=kv.split("!=!")[0]; 65 | String v=kv.split("!=!").length==2?kv.split("!=!")[1]:null; 66 | map.put(k, v); 67 | } 68 | } 69 | return map; 70 | 71 | } 72 | @SuppressWarnings("unchecked") 73 | public static String transMapToStr(Object obj){ 74 | if (obj instanceof Map) 75 | { 76 | Map map=(Map)obj; 77 | StringBuffer sb = new StringBuffer(); 78 | for(String key:map.keySet()) 79 | { 80 | sb.append(key+"!=!"+map.get(key)+"!,!"); 81 | } 82 | return sb.toString(); 83 | } 84 | else 85 | { 86 | return obj==null?null:obj.toString(); 87 | } 88 | 89 | } 90 | /** 91 | * 获取序列 92 | * @param rLength 生成随机数的长度 93 | * @return 时间数字(System.nanoTime()的后6位)+rLength位随机数 94 | */ 95 | public static String getSeqId() { 96 | int rLength=10; 97 | StringBuilder b = new StringBuilder(""); 98 | String t = String.valueOf(System.nanoTime()); 99 | if (t.length()>6) 100 | { 101 | t=t.substring(t.length()-6,t.length()); 102 | } 103 | String r=getFixLenthString(rLength-4); 104 | b.append(getSysdate("MMdd")).append(t).append(r); 105 | return b.toString(); 106 | } 107 | /** 108 | * 返回长度为【strLength】的随机数，在后面补0 109 | */ 110 | public static String getFixLenthString(int strLength) { 111 | double d=Double.parseDouble(String.valueOf(Math.pow(10, strLength))); 112 | long r = (long) Math.floor(Math.random() *d); 113 | String random=String.valueOf(r); 114 | for (int i=0;i"+i); 23 | System.out.println("test====>"+i); 24 | } 25 | System.exit(0); 26 | } 27 | private static void subscribe(String topic,String consumerId,String consumerAddr,String consumerMethod) throws Exception 28 | { 29 | MgloryMQ mq=new MgloryMQ("114.215.178.115:1099","114.215.178.115:1099"); 30 | mq.subscribe(topic, consumerId, consumerAddr, consumerMethod); 31 | } 32 | public static void send(String topic,String produceId,String msgId,String msg) throws Exception 33 | { 34 | MgloryMQ mq=new MgloryMQ("114.215.178.115:1099","114.215.178.115:1099"); 35 | mq.publish(topic, produceId, msgId, msg); 36 | } 37 | public MgloryOut consume(String taskId,Map inMap) 38 | { 39 | log.info("msg===>"+inMap.get("msg")); 40 | MgloryOut out=new MgloryOut(); 41 | return out; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /mglory/src/com/test/MgloryMapTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 分布式存储测试类 3 | */ 4 | package com.test; 5 | import java.rmi.RemoteException; 6 | import java.util.ArrayList; 7 | import java.util.HashMap; 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | import com.lingo.mglory.store.MgloryMap; 12 | 13 | public class MgloryMapTest{ 14 | /** 15 | * 16 | */ 17 | private static final long serialVersionUID = 6192616050186217549L; 18 | public MgloryMapTest() throws RemoteException { 19 | super(); 20 | } 21 | public static void main(String[] args)throws Exception 22 | { 23 | List> list=new ArrayList>(); 24 | long t3=System.currentTimeMillis(); 25 | for (int i=1;i<10;i++) 26 | { 27 | Map map=new HashMap(); 28 | map.put("key", "9121219283392192"+i); 29 | Map map1=new HashMap(); 30 | map1.put("SERIAL_NUMBER", "18602567623"); 31 | map1.put("TRADE_ID", "9718971897189718"); 32 | map1.put("USER_ID", "9718971897189717"); 33 | map1.put("CUST_NAME", "孙绪杭"); 34 | map1.put("NET_TYPE_CODE", "0017"); 35 | map1.put("TRADE_TYPE_CODE", "0010"); 36 | map1.put("CERT_ADDR", "江苏省南京市中山南路弓箭坊40号305"); 37 | map1.put("BRAND_CODE", "ADSL"); 38 | map.put("value",map1); 39 | list.add(map); 40 | } 41 | MgloryMap mgloryMap=new MgloryMap("114.215.178.115:1099","114.215.178.115:1099"); 42 | mgloryMap.batchPut(list); 43 | long t4=System.currentTimeMillis(); 44 | System.out.println("value==>"+mgloryMap.get( "9121219283392192"+5)); 45 | System.out.println("耗时==>"+(System.currentTimeMillis()-t4)); 46 | System.out.println("耗时==>"+(System.currentTimeMillis()-t3)); 47 | System.exit(0); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /mglory/src/com/test/ResourceUsageTest.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import java.rmi.RemoteException; 4 | 5 | import com.lingo.mglory.param.MgloryIn; 6 | import com.lingo.mglory.param.MgloryOut; 7 | import com.lingo.mglory.resource.ResourceUsage; 8 | 9 | public class ResourceUsageTest{ 10 | public ResourceUsageTest() throws RemoteException { 11 | super(); 12 | } 13 | /** 14 | * 15 | */ 16 | public MgloryOut task(MgloryIn in) throws RemoteException 17 | { 18 | MgloryOut out=new MgloryOut(); 19 | System.out.println("cpu:"+ResourceUsage.getCpuUsage()); 20 | System.out.println("io:"+ResourceUsage.getIoUsage()); 21 | System.out.println("memory:"+ResourceUsage.getFreeMemory()); 22 | System.out.println("net:"+ResourceUsage.getNetUsage()); 23 | 24 | out.setStatus(MgloryOut.NORMAL); 25 | return out; 26 | } 27 | public static void main(String[] args)throws Exception 28 | { 29 | System.out.println("cpu:"+ResourceUsage.getCpuUsage()); 30 | System.out.println("io:"+ResourceUsage.getIoUsage()); 31 | System.out.println("memory:"+ResourceUsage.getFreeMemory()); 32 | System.out.println("net:"+ResourceUsage.getNetUsage()); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /mglory/src/com/test/Test.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | public class Test { 7 | public static void main(String[] args) 8 | { 9 | Map map1=new HashMap(); 10 | Map map=new HashMap(); 11 | for (int i=0;i<2;i++) 12 | { 13 | map.put("hello"+i, "word"+i); 14 | } 15 | map1.put("test", map); 16 | map=new HashMap(); 17 | System.out.println("map1==>"+map1); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /mglory/src/com/test/Test4.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | 6 | import com.lingo.mglory.route.MgloryHash; 7 | import com.lingo.mglory.route.Node; 8 | 9 | public class Test4 { 10 | public static void main(String[] args) 11 | { 12 | System.out.println("name===>"+Test4.class.getName()); 13 | long t=System.currentTimeMillis(); 14 | List> nodes = new ArrayList>();// 真实机器节点 15 | for (int i=0;i<5;i++) 16 | { 17 | Node node=new Node("127.0.0."+i); 18 | nodes.add(node); 19 | } 20 | MgloryHash> mgloryHash=new MgloryHash>(1000,nodes); 21 | for (int i=0;i<10000;i++) 22 | { 23 | String key="hello"+i; 24 | mgloryHash.get(key); 25 | System.out.println(key+"===>"+mgloryHash.get(key)); 26 | } 27 | System.out.println("耗时====>"+(System.currentTimeMillis()-t)); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /mglory/src/com/test/Test5.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import com.lingo.mglory.resource.ResourceHelper; 4 | 5 | public class Test5 { 6 | public static void main(String[] args) throws Exception 7 | { 8 | for(int i=0;i<10;i++) 9 | { 10 | long t=System.currentTimeMillis(); 11 | String resource=ResourceHelper.getInstance().getIdleResource("114.215.178.115:1099","114.215.178.115:1099"); 12 | System.out.println("resource==>"+resource); 13 | System.out.println("耗时==>"+(System.currentTimeMillis()-t)); 14 | //Thread.sleep(3000); 15 | } 16 | 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /mglory/src/com/test/WordCount.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.StringTokenizer; 6 | 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | 10 | import com.lingo.mglory.job.JobExecutor; 11 | import com.lingo.mglory.param.WorkFlowDAG; 12 | import com.lingo.mglory.param.MgloryIn; 13 | import com.lingo.mglory.param.MgloryOut; 14 | import com.lingo.mglory.util.Utils; 15 | public class WordCount{ 16 | private static Log log = LogFactory.getLog(WordCount.class); 17 | public WordCount() { 18 | 19 | } 20 | public MgloryOut map(String taskId,Map inMap) throws Exception 21 | { 22 | MgloryOut out=new MgloryOut(); 23 | Map collector=new HashMap(); 24 | for (String preTaskId:inMap.keySet()) 25 | { 26 | Map paramMap=((MgloryOut)inMap.get(preTaskId)).getMap(); 27 | String text=paramMap.get(taskId).toString(); 28 | StringTokenizer tokenizer = new StringTokenizer(text); 29 | while (tokenizer.hasMoreTokens()) { 30 | String word = tokenizer.nextToken(); 31 | if (collector.containsKey(word)) 32 | { 33 | collector.put(word, Integer.parseInt(collector.get(word).toString())+1); 34 | } 35 | else 36 | { 37 | collector.put(word, 1); 38 | } 39 | } 40 | 41 | } 42 | log.info("collector===>"+collector); 43 | out.setMap(collector); 44 | return out; 45 | } 46 | public MgloryOut reduce(String taskId,Map inMap) throws Exception 47 | { 48 | MgloryOut out=new MgloryOut(); 49 | Map collector=new HashMap(); 50 | for (String taskId1:inMap.keySet()) 51 | { 52 | Map paramMap=((MgloryOut)inMap.get(taskId1)).getMap(); 53 | for (String word:paramMap.keySet()) 54 | { 55 | if (collector.containsKey(word)) 56 | { 57 | collector.put(word, Integer.parseInt(collector.get(word).toString())+Integer.parseInt(paramMap.get(word).toString())); 58 | } 59 | else 60 | { 61 | collector.put(word,paramMap.get(word)); 62 | } 63 | } 64 | } 65 | log.info("collector===>"+collector); 66 | out.setMap(collector); 67 | return out; 68 | } 69 | public static void main(String[] args)throws Exception 70 | { 71 | long t=System.currentTimeMillis(); 72 | Map paramMap=new HashMap(); 73 | String taskId1="map1"+Utils.getSeqId(); 74 | String taskId2="map2"+Utils.getSeqId(); 75 | paramMap.put(taskId1, "Hello World Bye World Hello Hadoop GoodBye Hadoop"); 76 | paramMap.put(taskId2, "Happy New Year Today Is Good Day Hello Through producing different" + 77 | " mature RNAs from one same gene alternative splicing (AS) helps improve the capacity " + 78 | "of transcriptome and proteome It has been proposed that human genome permits more AS " + 79 | "events which explains the fact that human are far more complex than lower animals with " + 80 | "similar gene numbers (such as C elegans) By taking advantage of high throughput transcriptome " + 81 | "sequencing (RNA-seq) technology researchers have reported that at least 95% of human multi-exon " + 82 | "genes are alternative spliced Such result seems supporting the former hypothesis However it’s " + 83 | "still unclear whether all these AS events are functional"); 84 | MgloryIn in=new MgloryIn(); 85 | in.setMap(paramMap); 86 | in.setInvokeMethod(WordCount.class.getName()+".map"); 87 | WorkFlowDAG dag=new WorkFlowDAG(); 88 | dag.setTask("start", "", "", ""); 89 | dag.setTask("end", "", "", ""); 90 | //String taskId1="map1"+Utils.getSeqId(); 91 | dag.setTask(taskId1, WordCount.class.getName(), "map", "0"); 92 | dag.setPreTask(taskId1, "start"); 93 | //dag.setPreTask("end", taskId1); 94 | dag.setNextTask("start", taskId1); 95 | //dag.setNextTask(taskId1, "end"); 96 | 97 | //String taskId2="map2"+Utils.getSeqId(); 98 | dag.setTask(taskId2, WordCount.class.getName(), "map", "0"); 99 | dag.setPreTask(taskId2, "start"); 100 | //dag.setPreTask("end", taskId2); 101 | dag.setNextTask("start", taskId2); 102 | //dag.setNextTask(taskId2, "end"); 103 | 104 | 105 | String taskId3="reduce"+Utils.getSeqId(); 106 | dag.setTask(taskId3, WordCount.class.getName(), "reduce", "0"); 107 | dag.setPreTask(taskId3, taskId1); 108 | dag.setPreTask(taskId3, taskId2); 109 | dag.setPreTask("end", taskId3); 110 | dag.setNextTask(taskId1, taskId3); 111 | dag.setNextTask(taskId2, taskId3); 112 | dag.setNextTask(taskId3, "end"); 113 | 114 | JobExecutor jobExecutor=new JobExecutor("114.215.178.115:1099","114.215.178.115:1099"); 115 | MgloryOut out =jobExecutor.execute(in, dag); 116 | System.out.println("map==>"+out.getMap()); 117 | System.out.println("status==>"+out.getStatus()); 118 | System.out.println("msg==>"+out.getErrorMsg()); 119 | System.out.println("耗时===>"+(System.currentTimeMillis()-t)); 120 | System.exit(0); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /mglory/src/com/test/WordCount2.java: -------------------------------------------------------------------------------- 1 | package com.test; 2 | 3 | import java.util.ArrayList; 4 | import java.util.HashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | import java.util.concurrent.Callable; 8 | import java.util.concurrent.ExecutionException; 9 | import java.util.concurrent.ExecutorService; 10 | import java.util.concurrent.Executors; 11 | import java.util.concurrent.FutureTask; 12 | 13 | //import org.apache.commons.logging.Log; 14 | //import org.apache.commons.logging.LogFactory; 15 | 16 | import com.lingo.mglory.job.JobExecutor; 17 | import com.lingo.mglory.param.WorkFlowDAG; 18 | import com.lingo.mglory.param.MgloryIn; 19 | import com.lingo.mglory.param.MgloryOut; 20 | import com.lingo.mglory.util.Utils; 21 | public class WordCount2{ 22 | //private static Log log = LogFactory.getLog(WordCount.class); 23 | public WordCount2() { 24 | 25 | } 26 | public static void main(String[] args)throws Exception 27 | { 28 | WordCount2 test=new WordCount2(); 29 | test.syn(); 30 | System.exit(0); 31 | } 32 | public void syn() 33 | { 34 | ExecutorService taskExecutor=Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); 35 | List> futureTasks = new ArrayList>(); 36 | for (int i=0;i futureTask = new FutureTask( 39 | new Callable() { 40 | @Override 41 | public MgloryOut call() throws Exception { 42 | return test(); 43 | } 44 | }); 45 | futureTasks.add(futureTask); 46 | taskExecutor.submit(futureTask); 47 | } 48 | for (FutureTask futureTask : futureTasks) { 49 | try { 50 | futureTask.get(); 51 | } catch (InterruptedException e) { 52 | // TODO Auto-generated catch block 53 | e.printStackTrace(); 54 | } catch (ExecutionException e) { 55 | // TODO Auto-generated catch block 56 | e.printStackTrace(); 57 | } 58 | } 59 | taskExecutor.shutdown(); 60 | } 61 | public MgloryOut test() throws Exception 62 | { 63 | long t=System.currentTimeMillis(); 64 | Map paramMap=new HashMap(); 65 | String taskId1="map1"+Utils.getSeqId(); 66 | String taskId2="map2"+Utils.getSeqId(); 67 | paramMap.put(taskId1, "Hello World Bye World Hello Hadoop GoodBye Hadoop"); 68 | paramMap.put(taskId2, "Happy New Year Today Is Good Day Hello Through producing different" + 69 | " mature RNAs from one same gene alternative splicing (AS) helps improve the capacity " + 70 | "of transcriptome and proteome It has been proposed that human genome permits more AS " + 71 | "events which explains the fact that human are far more complex than lower animals with " + 72 | "similar gene numbers (such as C elegans) By taking advantage of high throughput transcriptome " + 73 | "sequencing (RNA-seq) technology researchers have reported that at least 95% of human multi-exon " + 74 | "genes are alternative spliced Such result seems supporting the former hypothesis However it’s " + 75 | "still unclear whether all these AS events are functional"); 76 | MgloryIn in=new MgloryIn(); 77 | in.setMap(paramMap); 78 | in.setInvokeMethod(WordCount.class.getName()+".map"); 79 | WorkFlowDAG dag=new WorkFlowDAG(); 80 | dag.setTask("start", "", "", ""); 81 | dag.setTask("end", "", "", ""); 82 | //String taskId1="map1"+Utils.getSeqId(); 83 | dag.setTask(taskId1, WordCount.class.getName(), "map", "0"); 84 | dag.setPreTask(taskId1, "start"); 85 | //dag.setPreTask("end", taskId1); 86 | dag.setNextTask("start", taskId1); 87 | //dag.setNextTask(taskId1, "end"); 88 | 89 | //String taskId2="map2"+Utils.getSeqId(); 90 | dag.setTask(taskId2, WordCount.class.getName(), "map", "0"); 91 | dag.setPreTask(taskId2, "start"); 92 | //dag.setPreTask("end", taskId2); 93 | dag.setNextTask("start", taskId2); 94 | //dag.setNextTask(taskId2, "end"); 95 | 96 | 97 | String taskId3="reduce"+Utils.getSeqId(); 98 | dag.setTask(taskId3, WordCount.class.getName(), "reduce", "0"); 99 | dag.setPreTask(taskId3, taskId1); 100 | dag.setPreTask(taskId3, taskId2); 101 | dag.setPreTask("end", taskId3); 102 | dag.setNextTask(taskId1, taskId3); 103 | dag.setNextTask(taskId2, taskId3); 104 | dag.setNextTask(taskId3, "end"); 105 | 106 | JobExecutor jobExecutor=new JobExecutor("114.215.178.115:1099","114.215.178.115:1099"); 107 | MgloryOut out =jobExecutor.execute(in, dag); 108 | System.out.println("map==>"+out.getMap()); 109 | System.out.println("status==>"+out.getStatus()); 110 | System.out.println("msg==>"+out.getErrorMsg()); 111 | System.out.println("耗时===>"+(System.currentTimeMillis()-t)); 112 | return new MgloryOut(); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /mglory/src/config.properties: -------------------------------------------------------------------------------- 1 | masterIp=127.0.0.1 2 | masterPort=1099 3 | masterFlag=true 4 | slaveIp=127.0.0.1 5 | slavePort=1099 6 | clientTimeOut=30000 -------------------------------------------------------------------------------- /mglory/src/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=DEBUG,A1 2 | 3 | log4j.appender.A1=org.apache.log4j.RollingFileAppender 4 | log4j.appender.A1.layout=org.apache.log4j.PatternLayout 5 | 6 | #log file path 7 | log4j.appender.A1.File=./log/mglory.log 8 | log4j.appender.A1.encoding=UTF-8 9 | log4j.appender.A1.MaxFileSize=20MB 10 | log4j.appender.A1.MaxBackupIndex=30 11 | log4j.appender.A1.layout.ConversionPattern=%-d{yyyy-MM-dd HH:mm:ss,SSS} [%c{1}]-[%p]:%L - %m%n 12 | 13 | # Print only messages of level DEBUG or above in the package. 14 | 15 | log4j.logger.org.springframework=INFO 16 | log4j.logger.org.apache=INFO 17 | log4j.logger.com.lingo=DEBUG -------------------------------------------------------------------------------- /mglory简介.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lingo623/mglory/a89bf52fd91698de799736f21cb772c8c476abe9/mglory简介.doc --------------------------------------------------------------------------------