├── .gitignore ├── lu-raft-kv ├── src │ ├── main │ │ ├── java │ │ │ ├── cn │ │ │ │ └── think │ │ │ │ │ └── in │ │ │ │ │ └── java │ │ │ │ │ ├── LifeCycle.java │ │ │ │ │ ├── rpc │ │ │ │ │ ├── RpcClient.java │ │ │ │ │ ├── RpcServer.java │ │ │ │ │ ├── RaftUserProcessor.java │ │ │ │ │ ├── DefaultRpcClient.java │ │ │ │ │ ├── Response.java │ │ │ │ │ ├── Request.java │ │ │ │ │ └── DefaultRpcServer.java │ │ │ │ │ ├── membership │ │ │ │ │ └── changes │ │ │ │ │ │ ├── Server.java │ │ │ │ │ │ ├── ClusterMembershipChanges.java │ │ │ │ │ │ └── Result.java │ │ │ │ │ ├── util │ │ │ │ │ └── LongConvert.java │ │ │ │ │ ├── exception │ │ │ │ │ ├── RaftNotSupportException.java │ │ │ │ │ └── RaftRemotingException.java │ │ │ │ │ ├── LogModule.java │ │ │ │ │ ├── entity │ │ │ │ │ ├── BaseParam.java │ │ │ │ │ ├── RvoteResult.java │ │ │ │ │ ├── Command.java │ │ │ │ │ ├── AentryResult.java │ │ │ │ │ ├── RvoteParam.java │ │ │ │ │ ├── ReplicationFailModel.java │ │ │ │ │ ├── LogEntry.java │ │ │ │ │ └── AentryParam.java │ │ │ │ │ ├── StateMachine.java │ │ │ │ │ ├── common │ │ │ │ │ ├── NodeConfig.java │ │ │ │ │ ├── NodeStatus.java │ │ │ │ │ ├── Peer.java │ │ │ │ │ └── PeerSet.java │ │ │ │ │ ├── current │ │ │ │ │ ├── RaftThread.java │ │ │ │ │ ├── SleepHelper.java │ │ │ │ │ ├── RaftThreadPoolExecutor.java │ │ │ │ │ └── RaftThreadPool.java │ │ │ │ │ ├── constant │ │ │ │ │ └── StateMachineSaveType.java │ │ │ │ │ ├── Consensus.java │ │ │ │ │ ├── Node.java │ │ │ │ │ ├── RaftNodeBootStrap.java │ │ │ │ │ └── impl │ │ │ │ │ ├── ClusterMembershipChangesImpl.java │ │ │ │ │ ├── RedisStateMachine.java │ │ │ │ │ ├── DefaultStateMachine.java │ │ │ │ │ ├── DefaultLogModule.java │ │ │ │ │ ├── DefaultConsensus.java │ │ │ │ │ └── DefaultNode.java │ │ │ └── raft │ │ │ │ └── client │ │ │ │ ├── ClientKVAck.java │ │ │ │ ├── ClientKVReq.java │ │ │ │ ├── RaftClient2.java │ │ │ │ ├── RaftClient.java │ │ │ │ └── RaftClient3.java │ │ └── resources │ │ │ └── log4j.xml │ └── test │ │ └── java │ │ └── cn │ │ └── think │ │ └── in │ │ └── java │ │ └── impl │ │ ├── DefaultStateMachineTest.java │ │ ├── DefaultLogModuleTest.java │ │ └── RocksDBTest.java └── pom.xml ├── readme.md └── pom.xml /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | /target 3 | lu-raft/.idea 4 | *.iml 5 | /rocksDB-raft 6 | .idea 7 | target/ 8 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/LifeCycle.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | /** 4 | * 5 | * @author 莫那·鲁道 6 | */ 7 | public interface LifeCycle { 8 | 9 | void init() throws Throwable; 10 | 11 | void destroy() throws Throwable; 12 | } 13 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/RpcClient.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | /** 4 | * 5 | * @author 莫那·鲁道 6 | */ 7 | public interface RpcClient { 8 | 9 | Response send(Request request); 10 | 11 | Response send(Request request, int timeout); 12 | } 13 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/RpcServer.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | /** 4 | * @author 莫那·鲁道 5 | */ 6 | public interface RpcServer { 7 | 8 | void start(); 9 | 10 | void stop(); 11 | 12 | Response handlerRequest(Request request); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/membership/changes/Server.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.membership.changes; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * @author 莫那·鲁道 8 | */ 9 | @Getter 10 | @Setter 11 | public class Server { 12 | 13 | String address; 14 | 15 | } 16 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/util/LongConvert.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.util; 2 | 3 | /** 4 | * 5 | * @author 莫那·鲁道 6 | */ 7 | public class LongConvert { 8 | 9 | public static long convert(Long l) { 10 | if (l == null) { 11 | return 0; 12 | } 13 | return l; 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/exception/RaftNotSupportException.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.exception; 2 | 3 | /** 4 | * 5 | */ 6 | public class RaftNotSupportException extends RuntimeException { 7 | 8 | public RaftNotSupportException() { 9 | } 10 | 11 | public RaftNotSupportException(String message) { 12 | super(message); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/exception/RaftRemotingException.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.exception; 2 | 3 | /** 4 | * 5 | */ 6 | public class RaftRemotingException extends RuntimeException { 7 | 8 | public RaftRemotingException() { 9 | super(); 10 | } 11 | 12 | public RaftRemotingException(String message) { 13 | super(message); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/LogModule.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | import cn.think.in.java.entity.LogEntry; 4 | 5 | /** 6 | * 7 | * @see cn.think.in.java.entity.LogEntry 8 | * @author 莫那·鲁道 9 | */ 10 | public interface LogModule { 11 | 12 | void write(LogEntry logEntry); 13 | 14 | LogEntry read(Long index); 15 | 16 | void removeOnStartIndex(Long startIndex); 17 | 18 | LogEntry getLast(); 19 | 20 | Long getLastIndex(); 21 | } 22 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/BaseParam.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import lombok.ToString; 8 | 9 | /** 10 | * 11 | */ 12 | @Getter 13 | @Setter 14 | @ToString 15 | public class BaseParam implements Serializable { 16 | 17 | /** 候选人的任期号 */ 18 | public long term; 19 | 20 | /** 被请求者 ID(ip:selfPort) */ 21 | public String serverId; 22 | 23 | } 24 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/membership/changes/ClusterMembershipChanges.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.membership.changes; 2 | 3 | import cn.think.in.java.common.Peer; 4 | 5 | /** 6 | * 7 | * 集群配置变更接口. 8 | * 9 | * @author 莫那·鲁道 10 | */ 11 | public interface ClusterMembershipChanges { 12 | 13 | /** 14 | * 添加节点. 15 | * @param newPeer 16 | * @return 17 | */ 18 | Result addPeer(Peer newPeer); 19 | 20 | /** 21 | * 删除节点. 22 | * @param oldPeer 23 | * @return 24 | */ 25 | Result removePeer(Peer oldPeer); 26 | } 27 | 28 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/StateMachine.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | import cn.think.in.java.entity.LogEntry; 4 | 5 | /** 6 | * 状态机接口. 7 | * @author 莫那·鲁道 8 | */ 9 | public interface StateMachine { 10 | 11 | /** 12 | * 将数据应用到状态机. 13 | * 14 | * 原则上,只需这一个方法(apply). 其他的方法是为了更方便的使用状态机. 15 | * @param logEntry 日志中的数据. 16 | */ 17 | void apply(LogEntry logEntry); 18 | 19 | LogEntry get(String key); 20 | 21 | String getString(String key); 22 | 23 | void setString(String key, String value); 24 | 25 | void delString(String... key); 26 | 27 | } 28 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/common/NodeConfig.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.common; 2 | 3 | import java.util.List; 4 | 5 | import cn.think.in.java.constant.StateMachineSaveType; 6 | import lombok.Getter; 7 | import lombok.Setter; 8 | import lombok.ToString; 9 | 10 | /** 11 | * 12 | * 节点配置 13 | * 14 | * @author 莫那·鲁道 15 | */ 16 | @Getter 17 | @Setter 18 | @ToString 19 | public class NodeConfig { 20 | 21 | /** 自身 selfPort */ 22 | public int selfPort; 23 | 24 | /** 所有节点地址. */ 25 | public List peerAddrs; 26 | /** 27 | * 状态快照存储类型 28 | */ 29 | public StateMachineSaveType stateMachineSaveType; 30 | } 31 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/current/RaftThread.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.current; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | /** 7 | * 8 | * @author 莫那·鲁道 9 | */ 10 | public class RaftThread extends Thread { 11 | 12 | private static final Logger LOGGER = LoggerFactory.getLogger(RaftThread.class); 13 | private static final UncaughtExceptionHandler uncaughtExceptionHandler = (t, e) 14 | -> LOGGER.warn("Exception occurred from thread {}", t.getName(), e); 15 | 16 | public RaftThread(String threadName, Runnable r) { 17 | super(r, threadName); 18 | setUncaughtExceptionHandler(uncaughtExceptionHandler); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/common/NodeStatus.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.common; 2 | 3 | import lombok.Getter; 4 | 5 | /** 6 | * 7 | * @author 莫那·鲁道 8 | */ 9 | public interface NodeStatus { 10 | 11 | int FOLLOWER = 0; 12 | int CANDIDATE = 1; 13 | int LEADER = 2; 14 | 15 | @Getter 16 | enum Enum { 17 | FOLLOWER(0), CANDIDATE(1), LEADER(2); 18 | 19 | Enum(int code) { 20 | this.code = code; 21 | } 22 | 23 | int code; 24 | 25 | public static Enum value(int i) { 26 | for (Enum value : Enum.values()) { 27 | if (value.code == i) { 28 | return value; 29 | } 30 | } 31 | return null; 32 | } 33 | 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/RaftUserProcessor.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | import com.alipay.remoting.AsyncContext; 4 | import com.alipay.remoting.BizContext; 5 | import com.alipay.remoting.rpc.protocol.AbstractUserProcessor; 6 | 7 | import cn.think.in.java.exception.RaftNotSupportException; 8 | 9 | /** 10 | * 11 | * @author 莫那·鲁道 12 | */ 13 | public abstract class RaftUserProcessor extends AbstractUserProcessor { 14 | 15 | @Override 16 | public void handleRequest(BizContext bizCtx, AsyncContext asyncCtx, T request) { 17 | throw new RaftNotSupportException( 18 | "Raft Server not support handleRequest(BizContext bizCtx, AsyncContext asyncCtx, T request) "); 19 | } 20 | 21 | 22 | @Override 23 | public String interest() { 24 | return Request.class.getName(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/current/SleepHelper.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.current; 2 | 3 | import java.util.concurrent.TimeUnit; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | /** 9 | * 10 | * @author 莫那·鲁道 11 | */ 12 | public class SleepHelper { 13 | 14 | private static final Logger LOGGER = LoggerFactory.getLogger(SleepHelper.class); 15 | 16 | 17 | public static void sleep(int ms) { 18 | try { 19 | Thread.sleep(ms); 20 | } catch (InterruptedException e) { 21 | LOGGER.warn(e.getMessage()); 22 | } 23 | 24 | } 25 | 26 | public static void sleep2(int seconds) { 27 | try { 28 | TimeUnit.SECONDS.sleep(seconds); 29 | } catch (InterruptedException e) { 30 | LOGGER.warn(e.getMessage()); 31 | } 32 | 33 | } 34 | 35 | 36 | } 37 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/constant/StateMachineSaveType.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.constant; 2 | 3 | import cn.think.in.java.StateMachine; 4 | import cn.think.in.java.impl.DefaultStateMachine; 5 | import cn.think.in.java.impl.RedisStateMachine; 6 | 7 | /** 8 | * 9 | * 快照存储类型 10 | * 11 | * @author rensailong 12 | */ 13 | public enum StateMachineSaveType { 14 | REDIS("redis", "redis存储", RedisStateMachine.getInstance()), 15 | ROCKS_DB("RocksDB", "RocksDB本地存储", DefaultStateMachine.getInstance()) 16 | ; 17 | 18 | public StateMachine getStateMachine() { 19 | return this.stateMachine; 20 | } 21 | 22 | private String typeName; 23 | 24 | private String desc; 25 | 26 | private StateMachine stateMachine; 27 | 28 | StateMachineSaveType(String typeName, String desc, StateMachine stateMachine) { 29 | this.typeName = typeName; 30 | this.desc = desc; 31 | this.stateMachine = stateMachine; 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/common/Peer.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.common; 2 | 3 | import java.util.Objects; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | 8 | /** 9 | * 当前节点的 同伴. 10 | * 11 | * @author 莫那·鲁道 12 | */ 13 | @Getter 14 | @Setter 15 | public class Peer { 16 | 17 | /** ip:selfPort */ 18 | private final String addr; 19 | 20 | 21 | public Peer(String addr) { 22 | this.addr = addr; 23 | } 24 | 25 | @Override 26 | public boolean equals(Object o) { 27 | if (this == o) { 28 | return true; 29 | } 30 | if (o == null || getClass() != o.getClass()) { 31 | return false; 32 | } 33 | Peer peer = (Peer) o; 34 | return Objects.equals(addr, peer.addr); 35 | } 36 | 37 | @Override 38 | public int hashCode() { 39 | 40 | return Objects.hash(addr); 41 | } 42 | 43 | @Override 44 | public String toString() { 45 | return "Peer{" + 46 | "addr='" + addr + '\'' + 47 | '}'; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/Consensus.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | import cn.think.in.java.entity.AentryParam; 4 | import cn.think.in.java.entity.AentryResult; 5 | import cn.think.in.java.entity.RvoteParam; 6 | import cn.think.in.java.entity.RvoteResult; 7 | 8 | /** 9 | * 10 | * Raft 一致性模块. 11 | * @author 莫那·鲁道 12 | */ 13 | public interface Consensus { 14 | 15 | /** 16 | * 请求投票 RPC 17 | * 18 | * 接收者实现： 19 | * 20 | * 如果term < currentTerm返回 false （5.2 节） 21 | * 如果 votedFor 为空或者就是 candidateId，并且候选人的日志至少和自己一样新，那么就投票给他（5.2 节，5.4 节） 22 | * @return 23 | */ 24 | RvoteResult requestVote(RvoteParam param); 25 | 26 | /** 27 | * 附加日志(多个日志,为了提高效率) RPC 28 | * 29 | * 接收者实现： 30 | * 31 | * 如果 term < currentTerm 就返回 false （5.1 节） 32 | * 如果日志在 prevLogIndex 位置处的日志条目的任期号和 prevLogTerm 不匹配，则返回 false （5.3 节） 33 | * 如果已经存在的日志条目和新的产生冲突（索引值相同但是任期号不同），删除这一条和之后所有的 （5.3 节） 34 | * 附加任何在已有的日志中不存在的条目 35 | * 如果 leaderCommit > commitIndex，令 commitIndex 等于 leaderCommit 和 新日志条目索引值中较小的一个 36 | * @return 37 | */ 38 | AentryResult appendEntries(AentryParam param); 39 | 40 | 41 | } 42 | -------------------------------------------------------------------------------- /lu-raft-kv/src/test/java/cn/think/in/java/impl/DefaultStateMachineTest.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import org.junit.Before; 4 | import org.junit.Test; 5 | import org.rocksdb.RocksDBException; 6 | 7 | import cn.think.in.java.entity.Command; 8 | import cn.think.in.java.entity.LogEntry; 9 | 10 | /** 11 | * 12 | * @author 莫那·鲁道 13 | */ 14 | public class DefaultStateMachineTest { 15 | 16 | static { 17 | System.setProperty("serverPort", "8777"); 18 | DefaultStateMachine.dbDir = "/Users/cxs/code/lu-raft-revert/rocksDB-raft/" + System.getProperty("serverPort"); 19 | DefaultStateMachine.stateMachineDir = DefaultStateMachine.dbDir + "/stateMachine"; 20 | } 21 | DefaultStateMachine machine; 22 | 23 | @Before 24 | public void before() { 25 | machine = DefaultStateMachine.getInstance(); 26 | 27 | } 28 | 29 | @Test 30 | public void apply() { 31 | LogEntry logEntry = LogEntry.newBuilder().term(1).command(Command.newBuilder().key("hello").value("value1").build()).build(); 32 | machine.apply(logEntry); 33 | } 34 | 35 | 36 | @Test 37 | public void applyRead() throws RocksDBException { 38 | 39 | System.out.println(machine.get("hello:7")); 40 | } 41 | } -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/raft/client/ClientKVAck.java: -------------------------------------------------------------------------------- 1 | package raft.client; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import lombok.ToString; 8 | 9 | /** 10 | * 11 | * @author 莫那·鲁道 12 | */ 13 | @Getter 14 | @Setter 15 | @ToString 16 | public class ClientKVAck implements Serializable { 17 | 18 | Object result; 19 | 20 | public ClientKVAck(Object result) { 21 | this.result = result; 22 | } 23 | 24 | private ClientKVAck(Builder builder) { 25 | setResult(builder.result); 26 | } 27 | 28 | public static ClientKVAck ok() { 29 | return new ClientKVAck("ok"); 30 | } 31 | 32 | public static ClientKVAck fail() { 33 | return new ClientKVAck("fail"); 34 | } 35 | 36 | public static Builder newBuilder() { 37 | return new Builder(); 38 | } 39 | 40 | 41 | public static final class Builder { 42 | 43 | private Object result; 44 | 45 | private Builder() { 46 | } 47 | 48 | public Builder result(Object val) { 49 | result = val; 50 | return this; 51 | } 52 | 53 | public ClientKVAck build() { 54 | return new ClientKVAck(this); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/Node.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | import cn.think.in.java.common.NodeConfig; 4 | import cn.think.in.java.entity.AentryParam; 5 | import cn.think.in.java.entity.AentryResult; 6 | import cn.think.in.java.entity.RvoteParam; 7 | import cn.think.in.java.entity.RvoteResult; 8 | import raft.client.ClientKVAck; 9 | import raft.client.ClientKVReq; 10 | 11 | /** 12 | * 13 | * @author 莫那·鲁道 14 | */ 15 | public interface Node extends LifeCycle{ 16 | 17 | /** 18 | * 设置配置文件. 19 | * 20 | * @param config 21 | */ 22 | void setConfig(NodeConfig config); 23 | 24 | /** 25 | * 处理请求投票 RPC. 26 | * 27 | * @param param 28 | * @return 29 | */ 30 | RvoteResult handlerRequestVote(RvoteParam param); 31 | 32 | /** 33 | * 处理附加日志请求. 34 | * 35 | * @param param 36 | * @return 37 | */ 38 | AentryResult handlerAppendEntries(AentryParam param); 39 | 40 | /** 41 | * 处理客户端请求. 42 | * 43 | * @param request 44 | * @return 45 | */ 46 | ClientKVAck handlerClientRequest(ClientKVReq request); 47 | 48 | /** 49 | * 转发给 leader 节点. 50 | * @param request 51 | * @return 52 | */ 53 | ClientKVAck redirect(ClientKVReq request); 54 | 55 | } 56 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/DefaultRpcClient.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | import com.alipay.remoting.exception.RemotingException; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import cn.think.in.java.exception.RaftRemotingException; 9 | 10 | /** 11 | * 12 | * @author 莫那·鲁道 13 | */ 14 | public class DefaultRpcClient implements RpcClient { 15 | 16 | public static Logger logger = LoggerFactory.getLogger(DefaultRpcClient.class.getName()); 17 | 18 | private final static com.alipay.remoting.rpc.RpcClient CLIENT = new com.alipay.remoting.rpc.RpcClient(); 19 | static { 20 | CLIENT.init(); 21 | } 22 | 23 | 24 | @Override 25 | public Response send(Request request) { 26 | return send(request, 200000); 27 | } 28 | 29 | @Override 30 | public Response send(Request request, int timeout) { 31 | Response result = null; 32 | try { 33 | result = (Response) CLIENT.invokeSync(request.getUrl(), request, timeout); 34 | } catch (RemotingException e) { 35 | e.printStackTrace(); 36 | logger.info("rpc RaftRemotingException "); 37 | throw new RaftRemotingException(); 38 | } catch (InterruptedException e) { 39 | e.printStackTrace(); 40 | } 41 | return (result); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/Response.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | 8 | /** 9 | * 10 | * @author 莫那·鲁道 11 | */ 12 | @Getter 13 | @Setter 14 | public class Response implements Serializable { 15 | 16 | 17 | private T result; 18 | 19 | public Response(T result) { 20 | this.result = result; 21 | } 22 | 23 | private Response(Builder builder) { 24 | setResult((T) builder.result); 25 | } 26 | 27 | public static Response ok() { 28 | return new Response<>("ok"); 29 | } 30 | 31 | public static Response fail() { 32 | return new Response<>("fail"); 33 | } 34 | 35 | public static Builder newBuilder() { 36 | return new Builder(); 37 | } 38 | 39 | 40 | @Override 41 | public String toString() { 42 | return "Response{" + 43 | "result=" + result + 44 | '}'; 45 | } 46 | 47 | public static final class Builder { 48 | 49 | private Object result; 50 | 51 | private Builder() { 52 | } 53 | 54 | public Builder result(Object val) { 55 | result = val; 56 | return this; 57 | } 58 | 59 | public Response build() { 60 | return new Response(this); 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/RaftNodeBootStrap.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java; 2 | 3 | import java.util.Arrays; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | import cn.think.in.java.common.NodeConfig; 8 | import cn.think.in.java.constant.StateMachineSaveType; 9 | import cn.think.in.java.impl.DefaultNode; 10 | 11 | /** 12 | * -DserverPort=8775 13 | * -DserverPort=8776 14 | * -DserverPort=8777 15 | * -DserverPort=8778 16 | * -DserverPort=8779 17 | */ 18 | public class RaftNodeBootStrap { 19 | 20 | public static void main(String[] args) throws Throwable { 21 | main0(); 22 | } 23 | 24 | public static void main0() throws Throwable { 25 | String[] peerAddr = {"localhost:8775","localhost:8776","localhost:8777", "localhost:8778", "localhost:8779"}; 26 | 27 | NodeConfig config = new NodeConfig(); 28 | 29 | // 自身节点 30 | config.setSelfPort(Integer.valueOf(System.getProperty("serverPort"))); 31 | 32 | // 其他节点地址 33 | config.setPeerAddrs(Arrays.asList(peerAddr)); 34 | config.setStateMachineSaveType(StateMachineSaveType.ROCKS_DB); 35 | Node node = DefaultNode.getInstance(); 36 | node.setConfig(config); 37 | 38 | node.init(); 39 | 40 | Runtime.getRuntime().addShutdownHook(new Thread(() -> { 41 | try { 42 | node.destroy(); 43 | } catch (Throwable throwable) { 44 | throwable.printStackTrace(); 45 | } 46 | })); 47 | 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/RvoteResult.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | 8 | /** 9 | * 10 | * 请求投票 RPC 返回值. 11 | * 12 | */ 13 | @Getter 14 | @Setter 15 | public class RvoteResult implements Serializable { 16 | 17 | /** 当前任期号，以便于候选人去更新自己的任期 */ 18 | long term; 19 | 20 | /** 候选人赢得了此张选票时为真 */ 21 | boolean voteGranted; 22 | 23 | public RvoteResult(boolean voteGranted) { 24 | this.voteGranted = voteGranted; 25 | } 26 | 27 | private RvoteResult(Builder builder) { 28 | setTerm(builder.term); 29 | setVoteGranted(builder.voteGranted); 30 | } 31 | 32 | public static RvoteResult fail() { 33 | return new RvoteResult(false); 34 | } 35 | 36 | public static RvoteResult ok() { 37 | return new RvoteResult(true); 38 | } 39 | 40 | public static Builder newBuilder() { 41 | return new Builder(); 42 | } 43 | 44 | 45 | public static final class Builder { 46 | 47 | private long term; 48 | private boolean voteGranted; 49 | 50 | private Builder() { 51 | } 52 | 53 | public Builder term(long term) { 54 | this.term = term; 55 | return this; 56 | } 57 | 58 | public Builder voteGranted(boolean voteGranted) { 59 | this.voteGranted = voteGranted; 60 | return this; 61 | } 62 | 63 | public RvoteResult build() { 64 | return new RvoteResult(this); 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /lu-raft-kv/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 8 | cn.think.in.java 9 | lu-raft-parent 10 | 1.0-RELEASE 11 | 12 | 13 | lu-raft-kv 14 | 15 | 16 | 2.7.1 17 | 18 | 19 | 20 | 21 | junit 22 | junit 23 | 4.12 24 | test 25 | 26 | 27 | 28 | redis.clients 29 | jedis 30 | ${jedis-jar.version} 31 | 32 | 33 | 34 | com.google.guava 35 | guava 36 | 23.0 37 | 38 | 41 | 42 | org.rocksdb 43 | rocksdbjni 44 | 5.14.3 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/current/RaftThreadPoolExecutor.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.current; 2 | 3 | import java.util.concurrent.BlockingQueue; 4 | import java.util.concurrent.ThreadPoolExecutor; 5 | import java.util.concurrent.TimeUnit; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | /** 11 | * 12 | * @author 莫那·鲁道 13 | */ 14 | public class RaftThreadPoolExecutor extends ThreadPoolExecutor { 15 | 16 | private static final Logger LOGGER = LoggerFactory.getLogger(RaftThreadPoolExecutor.class); 17 | 18 | private static final ThreadLocal COST_TIME_WATCH = ThreadLocal.withInitial(System::currentTimeMillis); 19 | 20 | public RaftThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit, 21 | BlockingQueue workQueue, RaftThreadPool.NameThreadFactory nameThreadFactory) { 22 | super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, nameThreadFactory); 23 | } 24 | 25 | @Override 26 | protected void beforeExecute(Thread t, Runnable r) { 27 | COST_TIME_WATCH.get(); 28 | LOGGER.debug("raft thread pool before Execute"); 29 | } 30 | 31 | @Override 32 | protected void afterExecute(Runnable r, Throwable t) { 33 | LOGGER.debug("raft thread pool after Execute, cost time : {}", System.currentTimeMillis() - COST_TIME_WATCH.get()); 34 | COST_TIME_WATCH.remove(); 35 | } 36 | 37 | @Override 38 | protected void terminated() { 39 | LOGGER.info("active count : {}, queueSize : {}, poolSize : {}", getActiveCount(), getQueue().size(), getPoolSize()); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/common/PeerSet.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.common; 2 | 3 | import java.io.Serializable; 4 | import java.util.ArrayList; 5 | import java.util.List; 6 | 7 | /** 8 | * 9 | * 节点集合. 去重. 10 | * 11 | * @author 莫那·鲁道 12 | */ 13 | public class PeerSet implements Serializable { 14 | 15 | private List list = new ArrayList<>(); 16 | 17 | private volatile Peer leader; 18 | 19 | /** final */ 20 | private volatile Peer self; 21 | 22 | private PeerSet() { 23 | } 24 | 25 | public static PeerSet getInstance() { 26 | return PeerSetLazyHolder.INSTANCE; 27 | } 28 | 29 | private static class PeerSetLazyHolder { 30 | 31 | private static final PeerSet INSTANCE = new PeerSet(); 32 | } 33 | 34 | public void setSelf(Peer peer) { 35 | self = peer; 36 | } 37 | 38 | public Peer getSelf() { 39 | return self; 40 | } 41 | 42 | public void addPeer(Peer peer) { 43 | list.add(peer); 44 | } 45 | 46 | public void removePeer(Peer peer) { 47 | list.remove(peer); 48 | } 49 | 50 | public List getPeers() { 51 | return list; 52 | } 53 | 54 | public List getPeersWithOutSelf() { 55 | List list2 = new ArrayList<>(list); 56 | list2.remove(self); 57 | return list2; 58 | } 59 | 60 | 61 | public Peer getLeader() { 62 | return leader; 63 | } 64 | 65 | public void setLeader(Peer peer) { 66 | leader = peer; 67 | } 68 | 69 | @Override 70 | public String toString() { 71 | return "PeerSet{" + 72 | "list=" + list + 73 | ", leader=" + leader + 74 | ", self=" + self + 75 | '}'; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/Command.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.io.Serializable; 4 | import java.util.Objects; 5 | 6 | import lombok.Getter; 7 | import lombok.Setter; 8 | import lombok.ToString; 9 | 10 | /** 11 | * 12 | * @author 莫那·鲁道 13 | */ 14 | @Getter 15 | @Setter 16 | @ToString 17 | public class Command implements Serializable { 18 | 19 | String key; 20 | 21 | String value; 22 | 23 | public Command(String key, String value) { 24 | this.key = key; 25 | this.value = value; 26 | } 27 | 28 | private Command(Builder builder) { 29 | setKey(builder.key); 30 | setValue(builder.value); 31 | } 32 | 33 | public static Builder newBuilder() { 34 | return new Builder(); 35 | } 36 | 37 | 38 | @Override 39 | public boolean equals(Object o) { 40 | if (this == o) { 41 | return true; 42 | } 43 | if (o == null || getClass() != o.getClass()) { 44 | return false; 45 | } 46 | Command command = (Command) o; 47 | return Objects.equals(key, command.key) && 48 | Objects.equals(value, command.value); 49 | } 50 | 51 | @Override 52 | public int hashCode() { 53 | return Objects.hash(key, value); 54 | } 55 | 56 | public static final class Builder { 57 | 58 | private String key; 59 | private String value; 60 | 61 | private Builder() { 62 | } 63 | 64 | public Builder key(String val) { 65 | key = val; 66 | return this; 67 | } 68 | 69 | public Builder value(String val) { 70 | value = val; 71 | return this; 72 | } 73 | 74 | public Command build() { 75 | return new Command(this); 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/AentryResult.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import lombok.ToString; 8 | 9 | /** 10 | * 11 | * 附加 RPC 日志返回值. 12 | * 13 | * @author 莫那·鲁道 14 | */ 15 | @Setter 16 | @Getter 17 | @ToString 18 | public class AentryResult implements Serializable { 19 | 20 | /** 当前的任期号，用于领导人去更新自己 */ 21 | long term; 22 | 23 | /** 跟随者包含了匹配上 prevLogIndex 和 prevLogTerm 的日志时为真 */ 24 | boolean success; 25 | 26 | public AentryResult(long term) { 27 | this.term = term; 28 | } 29 | 30 | public AentryResult(boolean success) { 31 | this.success = success; 32 | } 33 | 34 | public AentryResult(long term, boolean success) { 35 | this.term = term; 36 | this.success = success; 37 | } 38 | 39 | private AentryResult(Builder builder) { 40 | setTerm(builder.term); 41 | setSuccess(builder.success); 42 | } 43 | 44 | public static AentryResult fail() { 45 | return new AentryResult(false); 46 | } 47 | 48 | public static AentryResult ok() { 49 | return new AentryResult(true); 50 | } 51 | 52 | public static Builder newBuilder() { 53 | return new Builder(); 54 | } 55 | 56 | 57 | public static final class Builder { 58 | 59 | private long term; 60 | private boolean success; 61 | 62 | private Builder() { 63 | } 64 | 65 | public Builder term(long val) { 66 | term = val; 67 | return this; 68 | } 69 | 70 | public Builder success(boolean val) { 71 | success = val; 72 | return this; 73 | } 74 | 75 | public AentryResult build() { 76 | return new AentryResult(this); 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/raft/client/ClientKVReq.java: -------------------------------------------------------------------------------- 1 | package raft.client; 2 | 3 | import java.io.Serializable; 4 | 5 | import lombok.Getter; 6 | import lombok.Setter; 7 | import lombok.ToString; 8 | 9 | /** 10 | * 11 | * @author 莫那·鲁道 12 | */ 13 | @Getter 14 | @Setter 15 | @ToString 16 | public class ClientKVReq implements Serializable { 17 | 18 | public static int PUT = 0; 19 | public static int GET = 1; 20 | 21 | int type; 22 | 23 | String key; 24 | 25 | String value; 26 | 27 | private ClientKVReq(Builder builder) { 28 | setType(builder.type); 29 | setKey(builder.key); 30 | setValue(builder.value); 31 | } 32 | 33 | public static Builder newBuilder() { 34 | return new Builder(); 35 | } 36 | 37 | public enum Type { 38 | PUT(0), GET(1); 39 | int code; 40 | 41 | Type(int code) { 42 | this.code = code; 43 | } 44 | 45 | public static Type value(int code ) { 46 | for (Type type : values()) { 47 | if (type.code == code) { 48 | return type; 49 | } 50 | } 51 | return null; 52 | } 53 | } 54 | 55 | 56 | public static final class Builder { 57 | 58 | private int type; 59 | private String key; 60 | private String value; 61 | 62 | private Builder() { 63 | } 64 | 65 | 66 | public Builder type(int val) { 67 | type = val; 68 | return this; 69 | } 70 | 71 | public Builder key(String val) { 72 | key = val; 73 | return this; 74 | } 75 | 76 | public Builder value(String val) { 77 | value = val; 78 | return this; 79 | } 80 | 81 | public ClientKVReq build() { 82 | return new ClientKVReq(this); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/membership/changes/Result.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.membership.changes; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * 8 | * @author 莫那·鲁道 9 | */ 10 | @Getter 11 | @Setter 12 | public class Result { 13 | 14 | public static final int FAIL = 0; 15 | public static final int SUCCESS = 1; 16 | 17 | int status; 18 | 19 | String leaderHint; 20 | 21 | public Result() { 22 | } 23 | 24 | public Result(Builder builder) { 25 | setStatus(builder.status); 26 | setLeaderHint(builder.leaderHint); 27 | } 28 | 29 | @Override 30 | public String toString() { 31 | return "Result{" + 32 | "status=" + status + 33 | ", leaderHint='" + leaderHint + '\'' + 34 | '}'; 35 | } 36 | 37 | public static Builder newBuilder() { 38 | return new Builder(); 39 | } 40 | 41 | @Getter 42 | public enum Status { 43 | FAIL(0), SUCCESS(1); 44 | 45 | int code; 46 | 47 | Status(int code) { 48 | this.code = code; 49 | } 50 | 51 | public static Status value(int v) { 52 | for (Status i : values()) { 53 | if (i.code == v) { 54 | return i; 55 | } 56 | } 57 | return null; 58 | } 59 | } 60 | 61 | public static final class Builder { 62 | 63 | private int status; 64 | private String leaderHint; 65 | 66 | private Builder() { 67 | } 68 | 69 | public Builder status(int val) { 70 | status = val; 71 | return this; 72 | } 73 | 74 | public Builder leaderHint(String val) { 75 | leaderHint = val; 76 | return this; 77 | } 78 | 79 | public Result build() { 80 | return new Result(this); 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /lu-raft-kv/src/test/java/cn/think/in/java/impl/DefaultLogModuleTest.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import org.junit.After; 4 | import org.junit.Assert; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | import cn.think.in.java.entity.Command; 9 | import cn.think.in.java.entity.LogEntry; 10 | 11 | 12 | /** 13 | * 14 | * @author 莫那·鲁道 15 | */ 16 | public class DefaultLogModuleTest { 17 | 18 | static { 19 | System.setProperty("serverPort", "8779"); 20 | DefaultLogModule.dbDir = "/Users/cxs/code/lu-raft-revert/rocksDB-raft/" + System.getProperty("serverPort"); 21 | DefaultLogModule.logsDir = DefaultLogModule.dbDir + "/logModule"; 22 | } 23 | 24 | DefaultLogModule defaultLogs; 25 | 26 | @Before 27 | public void setUp() throws Exception { 28 | System.setProperty("serverPort", "8777"); 29 | 30 | defaultLogs = DefaultLogModule.getInstance(); 31 | } 32 | 33 | @After 34 | public void tearDown() throws Exception { 35 | 36 | } 37 | 38 | @Test 39 | public void write() { 40 | LogEntry entry = LogEntry.newBuilder(). 41 | term(1). 42 | command(Command.newBuilder().key("hello").value("world").build()). 43 | build(); 44 | defaultLogs.write(entry); 45 | 46 | Assert.assertEquals(entry, defaultLogs.read(entry.getIndex())); 47 | } 48 | 49 | @Test 50 | public void read() { 51 | System.out.println(defaultLogs.getLastIndex()); 52 | } 53 | 54 | @Test 55 | public void remove() { 56 | defaultLogs.removeOnStartIndex(3L); 57 | } 58 | 59 | @Test 60 | public void getLast() { 61 | 62 | } 63 | 64 | @Test 65 | public void getLastIndex() { 66 | } 67 | 68 | @Test 69 | public void getDbDir() { 70 | } 71 | 72 | @Test 73 | public void getLogsDir() { 74 | } 75 | 76 | @Test 77 | public void setDbDir() { 78 | } 79 | } -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/raft/client/RaftClient2.java: -------------------------------------------------------------------------------- 1 | package raft.client; 2 | 3 | import java.util.List; 4 | 5 | import com.google.common.collect.Lists; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import cn.think.in.java.entity.LogEntry; 11 | import cn.think.in.java.rpc.DefaultRpcClient; 12 | import cn.think.in.java.rpc.Request; 13 | import cn.think.in.java.rpc.Response; 14 | import cn.think.in.java.rpc.RpcClient; 15 | 16 | /** 17 | * 18 | * @author 莫那·鲁道 19 | */ 20 | public class RaftClient2 { 21 | 22 | private static final Logger LOGGER = LoggerFactory.getLogger(RaftClient.class); 23 | 24 | 25 | private final static RpcClient client = new DefaultRpcClient(); 26 | 27 | static String addr = "localhost:8778"; 28 | static List list3 = Lists.newArrayList("localhost:8777", "localhost:8778", "localhost:8779"); 29 | static List list2 = Lists.newArrayList( "localhost:8777", "localhost:8779"); 30 | static List list1 = Lists.newArrayList( "localhost:8779"); 31 | 32 | public static void main(String[] args) throws InterruptedException { 33 | for (int i = 3; ; i++) { 34 | 35 | try { 36 | Request r = new Request<>(); 37 | 38 | int size = list2.size(); 39 | 40 | ClientKVReq obj = ClientKVReq.newBuilder().key("hello:" + i).type(ClientKVReq.GET).build(); 41 | int index = (i) % size; 42 | addr = list2.get(index); 43 | r.setUrl(addr); 44 | r.setObj(obj); 45 | r.setCmd(Request.CLIENT_REQ); 46 | 47 | Response response2 = client.send(r); 48 | 49 | LOGGER.info("request content : {}, url : {}, get response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response2.getResult()); 50 | } catch (Exception e) { 51 | e.printStackTrace(); 52 | } finally { 53 | Thread.sleep(1000); 54 | 55 | } 56 | 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## Lu-Raft-KV-Storage 2 | 3 | 这是一个 Java 版本的 Raft(CP) KV 分布式存储实现. 可用于 Raft 初学者深入学习 Raft 协议. 4 | 5 | 相关文章 http://thinkinjava.cn/2019/01/12/2019/2019-01-12-lu-raft-kv/ 6 | 7 | 为了尽可能的保证数据一致性，该实现的"性能"没有基于 AP 的实现好。 8 | 9 | 目前实现了 Raft 4 大核心功能的其中 2 个功能. 10 | 11 | 1. leader 选举 12 | 2. 日志复制 13 | 3. 成员变更(未测试) 14 | 4. 快照压缩(未实现) 15 | 16 | ## Design 17 | 18 | 完全是参照 RAFT 论文来写的. 没有任何妥协. 19 | 20 | ![image](https://user-images.githubusercontent.com/24973360/50371851-b13de880-05fd-11e9-958a-5813b3b6d761.png) 21 | 22 | 23 | 24 | ## quick start 25 | #### 验证 "leader 选举" 26 | 27 | 1. 在 idea 中配置 5 个 application 启动项,配置 main 类为 RaftNodeBootStrap 类, 加入 -DserverPort=8775 -DserverPort=8776 -DserverPort=8777 -DserverPort=8778 -DserverPort=8779 28 | 系统配置, 表示分布式环境下的 5 个机器节点. 29 | 2. 依次启动 5 个 RaftNodeBootStrap 节点, 端口分别是 8775，8776， 8777, 8778, 8779. 30 | 3. 观察控制台, 约 6 秒后, 会发生选举事件,此时,会产生一个 leader. 而 leader 会立刻发送心跳维持自己的地位. 31 | 4. 如果leader 的端口是 8775, 使用 idea 关闭 8775 端口，模拟节点挂掉, 大约 15 秒后, 会重新开始选举, 并且会在剩余的 4 个节点中,产生一个新的 leader. 并开始发送心跳日志。 32 | 33 | #### 验证"日志复制" 34 | 35 | ##### 正常状态下 36 | 37 | 1. 在 idea 中配置 5 个 application 启动项,配置 main 类为 RaftNodeBootStrap 类, 加入 -DserverPort=8775 -DserverPort=8776 -DserverPort=8777 -DserverPort=8778 -DserverPort=8779 38 | 2. 依次启动 5 个 RaftNodeBootStrap 节点, 端口分别是 8775，8776， 8777, 8778, 8779. 39 | 3. 使用客户端写入 kv 数据. 40 | 4. 杀掉所有节点, 使用 junit test 读取每个 rocksDB 的值, 验证每个节点的数据是否一致. 41 | 42 | ##### 非正常状态下 43 | 44 | 1. 在 idea 中配置 5 个 application 启动项,配置 main 类为 RaftNodeBootStrap 类, 加入 -DserverPort=8775 -DserverPort=8776 -DserverPort=8777 -DserverPort=8778 -DserverPort=8779 45 | 2. 依次启动 5 个 RaftNodeBootStrap 节点, 端口分别是 8775，8776， 8777, 8778, 8779. 46 | 3. 使用客户端写入 kv 数据. 47 | 4. 杀掉 leader （假设是 8775）. 48 | 5. 再次写入数据. 49 | 6. 重启 8775. 50 | 7. 关闭所有节点, 读取 RocksDB 验证数据一致性. 51 | 52 | 53 | ## And 54 | 55 | 欢迎提交 RP, issue. 加微信一起探讨 Raft。 56 | 57 | 本人微信： 58 | 59 | ![image](https://user-images.githubusercontent.com/24973360/50372024-5f975d00-0601-11e9-8247-139e145b1123.png) 60 | 61 | ## Acknowledgments 62 | 63 | 感谢 SOFA-Bolt 提供 RPC 网络框架 https://github.com/alipay/sofa-bolt 64 | 65 | 感谢 rocksDB 提供 KV 存储 https://github.com/facebook/rocksdb 66 | 67 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/Request.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | import java.io.Serializable; 4 | 5 | import cn.think.in.java.entity.AentryParam; 6 | import cn.think.in.java.entity.RvoteParam; 7 | import lombok.Getter; 8 | import lombok.Setter; 9 | import lombok.ToString; 10 | import raft.client.ClientKVReq; 11 | 12 | /** 13 | * @author 莫那·鲁道 14 | */ 15 | @Getter 16 | @Setter 17 | @ToString 18 | public class Request implements Serializable { 19 | 20 | /** 请求投票 */ 21 | public static final int R_VOTE = 0; 22 | /** 附加日志 */ 23 | public static final int A_ENTRIES = 1; 24 | /** 客户端 */ 25 | public static final int CLIENT_REQ = 2; 26 | /** 配置变更. add*/ 27 | public static final int CHANGE_CONFIG_ADD = 3; 28 | /** 配置变更. remove*/ 29 | public static final int CHANGE_CONFIG_REMOVE = 4; 30 | /** 请求类型 */ 31 | private int cmd = -1; 32 | 33 | /** param 34 | * @see AentryParam 35 | * @see RvoteParam 36 | * @see ClientKVReq 37 | * */ 38 | private T obj; 39 | 40 | String url; 41 | 42 | public Request() { 43 | } 44 | 45 | public Request(T obj) { 46 | this.obj = obj; 47 | } 48 | 49 | public Request(int cmd, T obj, String url) { 50 | this.cmd = cmd; 51 | this.obj = obj; 52 | this.url = url; 53 | } 54 | 55 | private Request(Builder builder) { 56 | setCmd(builder.cmd); 57 | setObj((T) builder.obj); 58 | setUrl(builder.url); 59 | } 60 | 61 | public static Builder newBuilder() { 62 | return new Builder<>(); 63 | } 64 | 65 | 66 | public final static class Builder { 67 | 68 | private int cmd; 69 | private Object obj; 70 | private String url; 71 | 72 | private Builder() { 73 | } 74 | 75 | public Builder cmd(int val) { 76 | cmd = val; 77 | return this; 78 | } 79 | 80 | public Builder obj(Object val) { 81 | obj = val; 82 | return this; 83 | } 84 | 85 | public Builder url(String val) { 86 | url = val; 87 | return this; 88 | } 89 | 90 | public Request build() { 91 | return new Request(this); 92 | } 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/RvoteParam.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import cn.think.in.java.Consensus; 4 | import lombok.Getter; 5 | import lombok.Setter; 6 | 7 | /** 8 | * 请求投票 RPC 参数. 9 | * 10 | * @author 莫那·鲁道 11 | * @see Consensus#requestVote(RvoteParam) 12 | */ 13 | @Getter 14 | @Setter 15 | public class RvoteParam extends BaseParam { 16 | 17 | /** 请求选票的候选人的 Id(ip:selfPort) */ 18 | String candidateId; 19 | 20 | /** 候选人的最后日志条目的索引值 */ 21 | long lastLogIndex; 22 | 23 | /** 候选人最后日志条目的任期号 */ 24 | long lastLogTerm; 25 | 26 | private RvoteParam(Builder builder) { 27 | setTerm(builder.term); 28 | setServerId(builder.serverId); 29 | setCandidateId(builder.candidateId); 30 | setLastLogIndex(builder.lastLogIndex); 31 | setLastLogTerm(builder.lastLogTerm); 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return "RvoteParam{" + 37 | "candidateId='" + candidateId + '\'' + 38 | ", lastLogIndex=" + lastLogIndex + 39 | ", lastLogTerm=" + lastLogTerm + 40 | ", term=" + term + 41 | ", serverId='" + serverId + '\'' + 42 | '}'; 43 | } 44 | 45 | public static Builder newBuilder() { 46 | return new Builder(); 47 | } 48 | 49 | 50 | public static final class Builder { 51 | 52 | private long term; 53 | private String serverId; 54 | private String candidateId; 55 | private long lastLogIndex; 56 | private long lastLogTerm; 57 | 58 | private Builder() { 59 | } 60 | 61 | public Builder term(long val) { 62 | term = val; 63 | return this; 64 | } 65 | 66 | public Builder serverId(String val) { 67 | serverId = val; 68 | return this; 69 | } 70 | 71 | public Builder candidateId(String val) { 72 | candidateId = val; 73 | return this; 74 | } 75 | 76 | public Builder lastLogIndex(long val) { 77 | lastLogIndex = val; 78 | return this; 79 | } 80 | 81 | public Builder lastLogTerm(long val) { 82 | lastLogTerm = val; 83 | return this; 84 | } 85 | 86 | public RvoteParam build() { 87 | return new RvoteParam(this); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/ReplicationFailModel.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.util.concurrent.Callable; 4 | 5 | import cn.think.in.java.common.Peer; 6 | 7 | /** 8 | * 9 | * @author 莫那·鲁道 10 | */ 11 | public class ReplicationFailModel { 12 | static String count = "_count"; 13 | static String success = "_success"; 14 | 15 | public String countKey; 16 | public String successKey; 17 | public Callable callable; 18 | public LogEntry logEntry; 19 | public Peer peer; 20 | public Long offerTime; 21 | 22 | public ReplicationFailModel(Callable callable, LogEntry logEntry, Peer peer, Long offerTime) { 23 | this.callable = callable; 24 | this.logEntry = logEntry; 25 | this.peer = peer; 26 | this.offerTime = offerTime; 27 | countKey = logEntry.getCommand().getKey() + count; 28 | successKey = logEntry.getCommand().getKey() + success; 29 | } 30 | 31 | private ReplicationFailModel(Builder builder) { 32 | countKey = builder.countKey; 33 | successKey = builder.successKey; 34 | callable = builder.callable; 35 | logEntry = builder.logEntry; 36 | peer = builder.peer; 37 | offerTime = builder.offerTime; 38 | } 39 | 40 | public static Builder newBuilder() { 41 | return new Builder(); 42 | } 43 | 44 | 45 | public static final class Builder { 46 | 47 | private String countKey; 48 | private String successKey; 49 | private Callable callable; 50 | private LogEntry logEntry; 51 | private Peer peer; 52 | private Long offerTime; 53 | 54 | private Builder() { 55 | } 56 | 57 | public Builder countKey(String val) { 58 | countKey = val; 59 | return this; 60 | } 61 | 62 | public Builder successKey(String val) { 63 | successKey = val; 64 | return this; 65 | } 66 | 67 | public Builder callable(Callable val) { 68 | callable = val; 69 | return this; 70 | } 71 | 72 | public Builder logEntry(LogEntry val) { 73 | logEntry = val; 74 | return this; 75 | } 76 | 77 | public Builder peer(Peer val) { 78 | peer = val; 79 | return this; 80 | } 81 | 82 | public Builder offerTime(Long val) { 83 | offerTime = val; 84 | return this; 85 | } 86 | 87 | public ReplicationFailModel build() { 88 | return new ReplicationFailModel(this); 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/current/RaftThreadPool.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.current; 2 | 3 | import java.util.concurrent.Callable; 4 | import java.util.concurrent.Future; 5 | import java.util.concurrent.LinkedBlockingQueue; 6 | import java.util.concurrent.ScheduledExecutorService; 7 | import java.util.concurrent.ScheduledThreadPoolExecutor; 8 | import java.util.concurrent.ThreadFactory; 9 | import java.util.concurrent.ThreadPoolExecutor; 10 | import java.util.concurrent.TimeUnit; 11 | 12 | /** 13 | * 14 | * @author 莫那·鲁道 15 | */ 16 | public class RaftThreadPool { 17 | 18 | private static int cup = Runtime.getRuntime().availableProcessors(); 19 | private static int maxPoolSize = cup * 2; 20 | private static final int queueSize = 1024; 21 | private static final long keepTime = 1000 * 60; 22 | private static TimeUnit keepTimeUnit = TimeUnit.MILLISECONDS; 23 | 24 | private static ScheduledExecutorService ss = getScheduled(); 25 | private static ThreadPoolExecutor te = getThreadPool(); 26 | 27 | private static ThreadPoolExecutor getThreadPool() { 28 | return new RaftThreadPoolExecutor( 29 | cup, 30 | maxPoolSize, 31 | keepTime, 32 | keepTimeUnit, 33 | new LinkedBlockingQueue<>(queueSize), 34 | new NameThreadFactory()); 35 | } 36 | 37 | private static ScheduledExecutorService getScheduled() { 38 | return new ScheduledThreadPoolExecutor(cup, new NameThreadFactory()); 39 | } 40 | 41 | 42 | public static void scheduleAtFixedRate(Runnable r, long initDelay, long delay) { 43 | ss.scheduleAtFixedRate(r, initDelay, delay, TimeUnit.MILLISECONDS); 44 | } 45 | 46 | 47 | public static void scheduleWithFixedDelay(Runnable r, long delay) { 48 | ss.scheduleWithFixedDelay(r, 0, delay, TimeUnit.MILLISECONDS); 49 | } 50 | 51 | @SuppressWarnings("unchecked") 52 | public static Future submit(Callable r) { 53 | return te.submit(r); 54 | } 55 | 56 | public static void execute(Runnable r) { 57 | te.execute(r); 58 | } 59 | 60 | public static void execute(Runnable r, boolean sync) { 61 | if (sync) { 62 | r.run(); 63 | } else { 64 | te.execute(r); 65 | } 66 | } 67 | 68 | static class NameThreadFactory implements ThreadFactory { 69 | 70 | @Override 71 | public Thread newThread(Runnable r) { 72 | Thread t = new RaftThread("Raft thread", r); 73 | t.setDaemon(true); 74 | t.setPriority(5); 75 | return t; 76 | } 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/rpc/DefaultRpcServer.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.rpc; 2 | 3 | import com.alipay.remoting.BizContext; 4 | 5 | import cn.think.in.java.common.Peer; 6 | import cn.think.in.java.entity.AentryParam; 7 | import cn.think.in.java.entity.RvoteParam; 8 | import cn.think.in.java.impl.DefaultNode; 9 | import cn.think.in.java.membership.changes.ClusterMembershipChanges; 10 | import raft.client.ClientKVReq; 11 | 12 | /** 13 | * 14 | * Raft Server 15 | * 16 | * @author 莫那·鲁道 17 | * 18 | */ 19 | @SuppressWarnings("unchecked") 20 | public class DefaultRpcServer implements RpcServer { 21 | 22 | private volatile boolean flag; 23 | 24 | private DefaultNode node; 25 | 26 | private com.alipay.remoting.rpc.RpcServer rpcServer; 27 | 28 | public DefaultRpcServer(int port, DefaultNode node) { 29 | 30 | if (flag) { 31 | return; 32 | } 33 | synchronized (this) { 34 | if (flag) { 35 | return; 36 | } 37 | 38 | rpcServer = new com.alipay.remoting.rpc.RpcServer(port, false, false); 39 | 40 | rpcServer.registerUserProcessor(new RaftUserProcessor() { 41 | 42 | @Override 43 | public Object handleRequest(BizContext bizCtx, Request request) throws Exception { 44 | return handlerRequest(request); 45 | } 46 | }); 47 | 48 | this.node = node; 49 | flag = true; 50 | } 51 | 52 | } 53 | 54 | @Override 55 | public void start() { 56 | rpcServer.start(); 57 | } 58 | 59 | @Override 60 | public void stop() { 61 | rpcServer.stop(); 62 | } 63 | 64 | @Override 65 | public Response handlerRequest(Request request) { 66 | if (request.getCmd() == Request.R_VOTE) { 67 | return new Response(node.handlerRequestVote((RvoteParam) request.getObj())); 68 | } else if (request.getCmd() == Request.A_ENTRIES) { 69 | return new Response(node.handlerAppendEntries((AentryParam) request.getObj())); 70 | } else if (request.getCmd() == Request.CLIENT_REQ) { 71 | return new Response(node.handlerClientRequest((ClientKVReq) request.getObj())); 72 | } else if (request.getCmd() == Request.CHANGE_CONFIG_REMOVE) { 73 | return new Response(((ClusterMembershipChanges) node).removePeer((Peer) request.getObj())); 74 | } else if (request.getCmd() == Request.CHANGE_CONFIG_ADD) { 75 | return new Response(((ClusterMembershipChanges) node).addPeer((Peer) request.getObj())); 76 | } 77 | return null; 78 | } 79 | 80 | 81 | } 82 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cn.think.in.java 8 | lu-raft-parent 9 | pom 10 | 1.0-RELEASE 11 | 12 | 13 | lu-raft-kv 14 | 15 | 16 | 17 | 18 | 19 | 20 | org.slf4j 21 | slf4j-api 22 | 1.7.25 23 | 24 | 25 | 26 | org.slf4j 27 | slf4j-log4j12 28 | 1.7.21 29 | 30 | 31 | 32 | 33 | 34 | com.alipay.sofa 35 | bolt 36 | 1.4.1 37 | 38 | 39 | slf4j-api 40 | org.slf4j 41 | 42 | 43 | 44 | 45 | 46 | 47 | redis.clients 48 | jedis 49 | 2.9.0 50 | 51 | 52 | 53 | 54 | com.alibaba 55 | fastjson 56 | 1.2.49 57 | 58 | 59 | 60 | 61 | org.projectlombok 62 | lombok 63 | 1.18.2 64 | 65 | 66 | 67 | 68 | com.google.guava 69 | guava 70 | 23.0 71 | 72 | 73 | 74 | com.alipay.sofa 75 | hessian 76 | 3.3.2 77 | 78 | 79 | 80 | 81 | lu-raft 82 | 83 | 84 | org.apache.maven.plugins 85 | maven-compiler-plugin 86 | 87 | 8 88 | 8 89 | 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/ClusterMembershipChangesImpl.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import cn.think.in.java.common.NodeStatus; 7 | import cn.think.in.java.common.Peer; 8 | import cn.think.in.java.entity.LogEntry; 9 | import cn.think.in.java.membership.changes.ClusterMembershipChanges; 10 | import cn.think.in.java.membership.changes.Result; 11 | import cn.think.in.java.rpc.Request; 12 | import cn.think.in.java.rpc.Response; 13 | 14 | /** 15 | * 16 | * 集群配置变更接口默认实现. 17 | * 18 | * @author 莫那·鲁道 19 | */ 20 | public class ClusterMembershipChangesImpl implements ClusterMembershipChanges { 21 | 22 | private static final Logger LOGGER = LoggerFactory.getLogger(ClusterMembershipChangesImpl.class); 23 | 24 | 25 | private final DefaultNode node; 26 | 27 | public ClusterMembershipChangesImpl(DefaultNode node) { 28 | this.node = node; 29 | } 30 | 31 | /** 必须是同步的,一次只能添加一个节点 32 | * @param newPeer*/ 33 | @Override 34 | public synchronized Result addPeer(Peer newPeer) { 35 | // 已经存在 36 | if (node.peerSet.getPeersWithOutSelf().contains(newPeer)) { 37 | return new Result(); 38 | } 39 | 40 | node.peerSet.getPeersWithOutSelf().add(newPeer); 41 | 42 | if (node.status == NodeStatus.LEADER) { 43 | node.nextIndexs.put(newPeer, 0L); 44 | node.matchIndexs.put(newPeer, 0L); 45 | 46 | for (long i = 0; i < node.logModule.getLastIndex(); i++) { 47 | LogEntry e = node.logModule.read(i); 48 | if (e != null) { 49 | node.replication(newPeer, e); 50 | } 51 | } 52 | 53 | for (Peer item : node.peerSet.getPeersWithOutSelf()) { 54 | // TODO 同步到其他节点. 55 | Request request = Request.newBuilder() 56 | .cmd(Request.CHANGE_CONFIG_ADD) 57 | .url(newPeer.getAddr()) 58 | .obj(newPeer) 59 | .build(); 60 | 61 | Response response = node.rpcClient.send(request); 62 | Result result = (Result) response.getResult(); 63 | if (result != null && result.getStatus() == Result.Status.SUCCESS.getCode()) { 64 | LOGGER.info("replication config success, peer : {}, newServer : {}", newPeer, newPeer); 65 | } else { 66 | LOGGER.warn("replication config fail, peer : {}, newServer : {}", newPeer, newPeer); 67 | } 68 | } 69 | 70 | } 71 | 72 | return new Result(); 73 | } 74 | 75 | 76 | /** 必须是同步的,一次只能删除一个节点 77 | * @param oldPeer*/ 78 | @Override 79 | public synchronized Result removePeer(Peer oldPeer) { 80 | node.peerSet.getPeersWithOutSelf().remove(oldPeer); 81 | node.nextIndexs.remove(oldPeer); 82 | node.matchIndexs.remove(oldPeer); 83 | 84 | return new Result(); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/LogEntry.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.io.Serializable; 4 | import java.util.Objects; 5 | 6 | import cn.think.in.java.LogModule; 7 | import lombok.Getter; 8 | import lombok.Setter; 9 | 10 | /** 11 | * 日志条目 12 | * 13 | * @author 莫那·鲁道 14 | * @see LogModule 15 | */ 16 | @Getter 17 | @Setter 18 | public class LogEntry implements Serializable, Comparable { 19 | 20 | private Long index; 21 | 22 | private long term; 23 | 24 | private Command command; 25 | 26 | public LogEntry() { 27 | } 28 | 29 | public LogEntry(long term, Command command) { 30 | this.term = term; 31 | this.command = command; 32 | } 33 | 34 | public LogEntry(Long index, long term, Command command) { 35 | this.index = index; 36 | this.term = term; 37 | this.command = command; 38 | } 39 | 40 | private LogEntry(Builder builder) { 41 | setIndex(builder.index); 42 | setTerm(builder.term); 43 | setCommand(builder.command); 44 | } 45 | 46 | public static Builder newBuilder() { 47 | return new Builder(); 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return "{" + 53 | "index=" + index + 54 | ", term=" + term + 55 | ", command=" + command + 56 | '}'; 57 | } 58 | 59 | @Override 60 | public int compareTo(Object o) { 61 | if (o == null) { 62 | return -1; 63 | } 64 | if (this.getIndex() > ((LogEntry) o).getIndex()) { 65 | return 1; 66 | } 67 | return -1; 68 | } 69 | 70 | @Override 71 | public boolean equals(Object o) { 72 | if (this == o) { 73 | return true; 74 | } 75 | if (o == null || getClass() != o.getClass()) { 76 | return false; 77 | } 78 | LogEntry logEntry = (LogEntry) o; 79 | return term == logEntry.term && 80 | Objects.equals(index, logEntry.index) && 81 | Objects.equals(command, logEntry.command); 82 | } 83 | 84 | @Override 85 | public int hashCode() { 86 | return Objects.hash(index, term, command); 87 | } 88 | 89 | public static final class Builder { 90 | 91 | private Long index; 92 | private long term; 93 | private Command command; 94 | 95 | private Builder() { 96 | } 97 | 98 | public Builder index(Long val) { 99 | index = val; 100 | return this; 101 | } 102 | 103 | public Builder term(long val) { 104 | term = val; 105 | return this; 106 | } 107 | 108 | public Builder command(Command val) { 109 | command = val; 110 | return this; 111 | } 112 | 113 | public LogEntry build() { 114 | return new LogEntry(this); 115 | } 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/raft/client/RaftClient.java: -------------------------------------------------------------------------------- 1 | package raft.client; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | import com.alipay.remoting.exception.RemotingException; 7 | import com.google.common.collect.Lists; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import cn.think.in.java.current.SleepHelper; 13 | import cn.think.in.java.entity.LogEntry; 14 | import cn.think.in.java.rpc.DefaultRpcClient; 15 | import cn.think.in.java.rpc.Request; 16 | import cn.think.in.java.rpc.Response; 17 | import cn.think.in.java.rpc.RpcClient; 18 | 19 | /** 20 | * 21 | * @author 莫那·鲁道 22 | */ 23 | public class RaftClient { 24 | 25 | private static final Logger LOGGER = LoggerFactory.getLogger(RaftClient.class); 26 | 27 | 28 | private final static RpcClient client = new DefaultRpcClient(); 29 | 30 | static String addr = "localhost:8778"; 31 | static List list = Lists.newArrayList("localhost:8777", "localhost:8778", "localhost:8779"); 32 | 33 | public static void main(String[] args) throws RemotingException, InterruptedException { 34 | 35 | AtomicLong count = new AtomicLong(3); 36 | 37 | for (int i = 3; ; i++) { 38 | try { 39 | int index = (int) (count.incrementAndGet() % list.size()); 40 | addr = list.get(index); 41 | 42 | ClientKVReq obj = ClientKVReq.newBuilder().key("hello:" + i).value("world:" + i).type(ClientKVReq.PUT).build(); 43 | 44 | Request r = new Request<>(); 45 | r.setObj(obj); 46 | r.setUrl(addr); 47 | r.setCmd(Request.CLIENT_REQ); 48 | Response response; 49 | try { 50 | response = client.send(r); 51 | } catch (Exception e) { 52 | r.setUrl(list.get((int) ((count.incrementAndGet()) % list.size()))); 53 | response = client.send(r); 54 | } 55 | 56 | LOGGER.info("request content : {}, url : {}, put response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response.getResult()); 57 | 58 | SleepHelper.sleep(1000); 59 | 60 | obj = ClientKVReq.newBuilder().key("hello:" + i).type(ClientKVReq.GET).build(); 61 | 62 | addr = list.get(index); 63 | addr = list.get(index); 64 | r.setUrl(addr); 65 | r.setObj(obj); 66 | 67 | Response response2; 68 | try { 69 | response2 = client.send(r); 70 | } catch (Exception e) { 71 | r.setUrl(list.get((int) ((count.incrementAndGet()) % list.size()))); 72 | response2 = client.send(r); 73 | } 74 | 75 | LOGGER.info("request content : {}, url : {}, get response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response2.getResult()); 76 | } catch (Exception e) { 77 | e.printStackTrace(); 78 | i = i - 1; 79 | } 80 | 81 | SleepHelper.sleep(5000); 82 | } 83 | 84 | 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/raft/client/RaftClient3.java: -------------------------------------------------------------------------------- 1 | package raft.client; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | import com.alipay.remoting.exception.RemotingException; 7 | import com.google.common.collect.Lists; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import cn.think.in.java.current.SleepHelper; 13 | import cn.think.in.java.entity.LogEntry; 14 | import cn.think.in.java.rpc.DefaultRpcClient; 15 | import cn.think.in.java.rpc.Request; 16 | import cn.think.in.java.rpc.Response; 17 | import cn.think.in.java.rpc.RpcClient; 18 | 19 | /** 20 | * 21 | * @author 莫那·鲁道 22 | */ 23 | public class RaftClient3 { 24 | 25 | private static final Logger LOGGER = LoggerFactory.getLogger(RaftClient3.class); 26 | 27 | 28 | private final static RpcClient client = new DefaultRpcClient(); 29 | 30 | static String addr = "localhost:8777"; 31 | static List list = Lists.newArrayList("localhost:8777", "localhost:8778", "localhost:8779"); 32 | 33 | public static void main(String[] args) throws RemotingException, InterruptedException { 34 | 35 | AtomicLong count = new AtomicLong(3); 36 | 37 | int keyNum = 4; 38 | try { 39 | int index = (int) (count.incrementAndGet() % list.size()); 40 | index = 1; 41 | addr = list.get(index); 42 | 43 | ClientKVReq obj = ClientKVReq.newBuilder().key("hello:" + keyNum).value("world:" + keyNum).type(ClientKVReq.PUT).build(); 44 | 45 | Request r = new Request<>(); 46 | r.setObj(obj); 47 | r.setUrl(addr); 48 | r.setCmd(Request.CLIENT_REQ); 49 | Response response = null; 50 | try { 51 | response = client.send(r); 52 | } catch (Exception e) { 53 | } 54 | 55 | LOGGER.info("request content : {}, url : {}, put response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response.getResult()); 56 | 57 | SleepHelper.sleep(1000); 58 | 59 | obj = ClientKVReq.newBuilder().key("hello:" + keyNum).type(ClientKVReq.GET).build(); 60 | 61 | addr = list.get(index); 62 | addr = list.get(index); 63 | r.setUrl(addr); 64 | r.setObj(obj); 65 | 66 | Response response2; 67 | try { 68 | response2 = client.send(r); 69 | } catch (Exception e) { 70 | r.setUrl(list.get((int) ((count.incrementAndGet()) % list.size()))); 71 | response2 = client.send(r); 72 | } 73 | 74 | if (response.getResult() == null) { 75 | LOGGER.error("request content : {}, url : {}, get response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response2.getResult()); 76 | System.exit(1); 77 | return; 78 | } 79 | LOGGER.info("request content : {}, url : {}, get response : {}", obj.key + "=" + obj.getValue(), r.getUrl(), response2.getResult()); 80 | } catch (Exception e) { 81 | e.printStackTrace(); 82 | } 83 | 84 | System.exit(1); 85 | 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/entity/AentryParam.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.entity; 2 | 3 | import java.util.Arrays; 4 | 5 | import cn.think.in.java.Consensus; 6 | import lombok.Getter; 7 | import lombok.Setter; 8 | import lombok.ToString; 9 | 10 | /** 11 | * 12 | * 附加日志 RPC 参数. handlerAppendEntries 13 | * 14 | * @author 莫那·鲁道 15 | * @see Consensus#appendEntries(AentryParam) 16 | */ 17 | @Getter 18 | @Setter 19 | @ToString 20 | public class AentryParam extends BaseParam { 21 | 22 | /** 领导人的 Id，以便于跟随者重定向请求 */ 23 | String leaderId; 24 | 25 | /**新的日志条目紧随之前的索引值 */ 26 | long prevLogIndex; 27 | 28 | /** prevLogIndex 条目的任期号 */ 29 | long preLogTerm; 30 | 31 | /** 准备存储的日志条目（表示心跳时为空；一次性发送多个是为了提高效率） */ 32 | LogEntry[] entries; 33 | 34 | /** 领导人已经提交的日志的索引值 */ 35 | long leaderCommit; 36 | 37 | public AentryParam() { 38 | } 39 | 40 | private AentryParam(Builder builder) { 41 | setTerm(builder.term); 42 | setServerId(builder.serverId); 43 | setLeaderId(builder.leaderId); 44 | setPrevLogIndex(builder.prevLogIndex); 45 | setPreLogTerm(builder.preLogTerm); 46 | setEntries(builder.entries); 47 | setLeaderCommit(builder.leaderCommit); 48 | } 49 | 50 | @Override 51 | public String toString() { 52 | return "AentryParam{" + 53 | "leaderId='" + leaderId + '\'' + 54 | ", prevLogIndex=" + prevLogIndex + 55 | ", preLogTerm=" + preLogTerm + 56 | ", entries=" + Arrays.toString(entries) + 57 | ", leaderCommit=" + leaderCommit + 58 | ", term=" + term + 59 | ", serverId='" + serverId + '\'' + 60 | '}'; 61 | } 62 | 63 | public static Builder newBuilder() { 64 | return new Builder(); 65 | } 66 | 67 | 68 | public static final class Builder { 69 | 70 | private long term; 71 | private String serverId; 72 | private String leaderId; 73 | private long prevLogIndex; 74 | private long preLogTerm; 75 | private LogEntry[] entries; 76 | private long leaderCommit; 77 | 78 | private Builder() { 79 | } 80 | 81 | public Builder term(long val) { 82 | term = val; 83 | return this; 84 | } 85 | 86 | public Builder serverId(String val) { 87 | serverId = val; 88 | return this; 89 | } 90 | 91 | public Builder leaderId(String val) { 92 | leaderId = val; 93 | return this; 94 | } 95 | 96 | public Builder prevLogIndex(long val) { 97 | prevLogIndex = val; 98 | return this; 99 | } 100 | 101 | public Builder preLogTerm(long val) { 102 | preLogTerm = val; 103 | return this; 104 | } 105 | 106 | public Builder entries(LogEntry[] val) { 107 | entries = val; 108 | return this; 109 | } 110 | 111 | public Builder leaderCommit(long val) { 112 | leaderCommit = val; 113 | return this; 114 | } 115 | 116 | public AentryParam build() { 117 | return new AentryParam(this); 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/RedisStateMachine.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import org.apache.commons.pool2.impl.GenericObjectPoolConfig; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import com.alibaba.fastjson.JSON; 8 | 9 | import cn.think.in.java.StateMachine; 10 | import cn.think.in.java.entity.Command; 11 | import cn.think.in.java.entity.LogEntry; 12 | import redis.clients.jedis.Jedis; 13 | import redis.clients.jedis.JedisPool; 14 | 15 | /** 16 | * redis实现状态机存储 17 | * 18 | * @author rensailong 19 | */ 20 | public class RedisStateMachine implements StateMachine { 21 | private static final Logger LOGGER = LoggerFactory.getLogger(RedisStateMachine.class); 22 | 23 | private static JedisPool jedisPool; 24 | 25 | static { 26 | GenericObjectPoolConfig redisConfig = new GenericObjectPoolConfig(); 27 | redisConfig.setMaxTotal(100); 28 | redisConfig.setMaxWaitMillis(10 * 1000); 29 | redisConfig.setMaxIdle(1000); 30 | redisConfig.setTestOnBorrow(true); 31 | jedisPool = new JedisPool(redisConfig, "127.0.0.1", 6379); 32 | } 33 | 34 | public static RedisStateMachine getInstance() { 35 | return RedisStateMachine.RedisStateMachineLazyHolder.INSTANCE; 36 | } 37 | 38 | private static class RedisStateMachineLazyHolder { 39 | 40 | private static final RedisStateMachine INSTANCE = new RedisStateMachine(); 41 | } 42 | 43 | @Override 44 | public void apply(LogEntry logEntry) { 45 | Jedis jedis = null; 46 | try { 47 | jedis = jedisPool.getResource(); 48 | Command command = logEntry.getCommand(); 49 | if (command == null) { 50 | throw new IllegalArgumentException("command can not be null, logEntry : " + logEntry.toString()); 51 | } 52 | String key = command.getKey(); 53 | jedis.set(key.getBytes(), JSON.toJSONBytes(logEntry)); 54 | } catch (Exception e) { 55 | LOGGER.info(e.getMessage()); 56 | } finally { 57 | if (jedis != null) { 58 | jedis.close(); 59 | } 60 | } 61 | } 62 | 63 | @Override 64 | public LogEntry get(String key) { 65 | LogEntry result = null; 66 | Jedis jedis = null; 67 | try { 68 | jedis = jedisPool.getResource(); 69 | result = JSON.parseObject(jedis.get(key), LogEntry.class); 70 | } catch (Exception e) { 71 | LOGGER.error("redis error ", e); 72 | } finally { 73 | if (jedis != null) { 74 | jedis.close(); 75 | } 76 | } 77 | return result; 78 | } 79 | 80 | @Override 81 | public String getString(String key) { 82 | String result = null; 83 | Jedis jedis = null; 84 | try { 85 | jedis = jedisPool.getResource(); 86 | result = jedis.get(key); 87 | } catch (Exception e) { 88 | LOGGER.error("redis error ", e); 89 | } finally { 90 | if (jedis != null) { 91 | jedis.close(); 92 | } 93 | } 94 | return result; 95 | } 96 | 97 | @Override 98 | public void setString(String key, String value) { 99 | Jedis jedis = null; 100 | try { 101 | jedis = jedisPool.getResource(); 102 | jedis.set(key, value); 103 | } catch (Exception e) { 104 | LOGGER.error("redis error ", e); 105 | } finally { 106 | if (jedis != null) { 107 | jedis.close(); 108 | } 109 | } 110 | } 111 | 112 | @Override 113 | public void delString(String... keys) { 114 | Jedis jedis = null; 115 | try { 116 | jedis.del(keys); 117 | } catch (Exception e) { 118 | LOGGER.error("redis error ", e); 119 | } finally { 120 | if (jedis != null) { 121 | jedis.close(); 122 | } 123 | } 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/DefaultStateMachine.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import java.io.File; 4 | 5 | import com.alibaba.fastjson.JSON; 6 | 7 | import org.rocksdb.Options; 8 | import org.rocksdb.RocksDB; 9 | import org.rocksdb.RocksDBException; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import cn.think.in.java.StateMachine; 14 | import cn.think.in.java.entity.Command; 15 | import cn.think.in.java.entity.LogEntry; 16 | 17 | /** 18 | * 19 | * 默认的状态机实现. 20 | * 21 | * @author 莫那·鲁道 22 | */ 23 | public class DefaultStateMachine implements StateMachine { 24 | 25 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultStateMachine.class); 26 | 27 | /** public just for test */ 28 | public static String dbDir; 29 | public static String stateMachineDir; 30 | 31 | public static RocksDB machineDb; 32 | 33 | static { 34 | if (dbDir == null) { 35 | dbDir = "./rocksDB-raft/" + System.getProperty("serverPort"); 36 | } 37 | if (stateMachineDir == null) { 38 | stateMachineDir =dbDir + "/stateMachine"; 39 | } 40 | RocksDB.loadLibrary(); 41 | } 42 | 43 | 44 | private DefaultStateMachine() { 45 | try { 46 | File file = new File(stateMachineDir); 47 | boolean success = false; 48 | if (!file.exists()) { 49 | success = file.mkdirs(); 50 | } 51 | if (success) { 52 | LOGGER.warn("make a new dir : " + stateMachineDir); 53 | } 54 | Options options = new Options(); 55 | options.setCreateIfMissing(true); 56 | machineDb = RocksDB.open(options, stateMachineDir); 57 | 58 | } catch (RocksDBException e) { 59 | LOGGER.info(e.getMessage()); 60 | } 61 | } 62 | 63 | public static DefaultStateMachine getInstance() { 64 | return DefaultStateMachineLazyHolder.INSTANCE; 65 | } 66 | 67 | private static class DefaultStateMachineLazyHolder { 68 | 69 | private static final DefaultStateMachine INSTANCE = new DefaultStateMachine(); 70 | } 71 | 72 | @Override 73 | public LogEntry get(String key) { 74 | try { 75 | byte[] result = machineDb.get(key.getBytes()); 76 | if (result == null) { 77 | return null; 78 | } 79 | return JSON.parseObject(result, LogEntry.class); 80 | } catch (RocksDBException e) { 81 | LOGGER.info(e.getMessage()); 82 | } 83 | return null; 84 | } 85 | 86 | @Override 87 | public String getString(String key) { 88 | try { 89 | byte[] bytes = machineDb.get(key.getBytes()); 90 | if (bytes != null) { 91 | return new String(bytes); 92 | } 93 | } catch (RocksDBException e) { 94 | LOGGER.info(e.getMessage()); 95 | } 96 | return ""; 97 | } 98 | 99 | @Override 100 | public void setString(String key, String value) { 101 | try { 102 | machineDb.put(key.getBytes(), value.getBytes()); 103 | } catch (RocksDBException e) { 104 | LOGGER.info(e.getMessage()); 105 | } 106 | } 107 | 108 | @Override 109 | public void delString(String... key) { 110 | try { 111 | for (String s : key) { 112 | machineDb.delete(s.getBytes()); 113 | } 114 | } catch (RocksDBException e) { 115 | LOGGER.info(e.getMessage()); 116 | } 117 | } 118 | 119 | @Override 120 | public synchronized void apply(LogEntry logEntry) { 121 | 122 | try { 123 | Command command = logEntry.getCommand(); 124 | 125 | if (command == null) { 126 | throw new IllegalArgumentException("command can not be null, logEntry : " + logEntry.toString()); 127 | } 128 | String key = command.getKey(); 129 | machineDb.put(key.getBytes(), JSON.toJSONBytes(logEntry)); 130 | } catch (RocksDBException e) { 131 | LOGGER.info(e.getMessage()); 132 | } 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /lu-raft-kv/src/test/java/cn/think/in/java/impl/RocksDBTest.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import java.io.File; 4 | 5 | import com.alibaba.fastjson.JSON; 6 | 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | import org.rocksdb.Options; 10 | import org.rocksdb.RocksDB; 11 | import org.rocksdb.RocksDBException; 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import lombok.Getter; 16 | import lombok.Setter; 17 | import lombok.ToString; 18 | 19 | /** 20 | * 21 | * @author 莫那·鲁道 22 | */ 23 | public class RocksDBTest { 24 | 25 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultStateMachine.class); 26 | 27 | private String dbDir = "./rocksDB-raft/" + System.getProperty("serverPort"); 28 | private String stateMachineDir = dbDir + "/test"; 29 | 30 | public RocksDB machineDb; 31 | 32 | static { 33 | RocksDB.loadLibrary(); 34 | } 35 | 36 | public byte[] lastIndexKey = "LAST_INDEX_KEY".getBytes(); 37 | 38 | public RocksDBTest() { 39 | try { 40 | System.setProperty("serverPort", "8078"); 41 | File file = new File(stateMachineDir); 42 | if (!file.exists()) { 43 | file.mkdirs(); 44 | } 45 | Options options = new Options(); 46 | options.setCreateIfMissing(true); 47 | machineDb = RocksDB.open(options, stateMachineDir); 48 | 49 | } catch (RocksDBException e) { 50 | e.printStackTrace(); 51 | } 52 | } 53 | 54 | 55 | public static RocksDBTest getInstance() { 56 | return RocksDBTestLazyHolder.INSTANCE; 57 | } 58 | 59 | private static class RocksDBTestLazyHolder { 60 | 61 | private static final RocksDBTest INSTANCE = new RocksDBTest(); 62 | } 63 | 64 | RocksDBTest instance; 65 | 66 | @Before 67 | public void before() { 68 | instance = getInstance(); 69 | } 70 | 71 | @Test 72 | public void test() throws RocksDBException { 73 | System.out.println(getLastIndex()); 74 | System.out.println(get(getLastIndex())); 75 | 76 | write(new Cmd("hello", "value")); 77 | 78 | System.out.println(getLastIndex()); 79 | 80 | System.out.println(get(getLastIndex())); 81 | 82 | deleteOnStartIndex(getLastIndex()); 83 | 84 | write(new Cmd("hello", "value")); 85 | 86 | deleteOnStartIndex(1L); 87 | 88 | System.out.println(getLastIndex()); 89 | 90 | System.out.println(get(getLastIndex())); 91 | 92 | 93 | } 94 | 95 | public synchronized void write(Cmd cmd) { 96 | try { 97 | cmd.setIndex(getLastIndex() + 1); 98 | machineDb.put(cmd.getIndex().toString().getBytes(), JSON.toJSONBytes(cmd)); 99 | } catch (RocksDBException e) { 100 | e.printStackTrace(); 101 | } finally { 102 | updateLastIndex(cmd.getIndex()); 103 | } 104 | } 105 | 106 | public synchronized void deleteOnStartIndex(Long index) { 107 | try { 108 | for (long i = index; i <= getLastIndex(); i++) { 109 | try { 110 | machineDb.delete((i + "").getBytes()); 111 | } catch (RocksDBException e) { 112 | e.printStackTrace(); 113 | } 114 | } 115 | 116 | } finally { 117 | updateLastIndex(index - 1); 118 | } 119 | } 120 | 121 | public Cmd get(Long index) { 122 | try { 123 | if (index == null) { 124 | throw new IllegalArgumentException(); 125 | } 126 | byte[] cmd = machineDb.get(index.toString().getBytes()); 127 | if (cmd != null) { 128 | return JSON.parseObject(machineDb.get(index.toString().getBytes()), Cmd.class); 129 | } 130 | } catch (RocksDBException e) { 131 | e.printStackTrace(); 132 | } 133 | return null; 134 | } 135 | 136 | 137 | public void updateLastIndex(Long index) { 138 | try { 139 | // overWrite 140 | machineDb.put(this.lastIndexKey, index.toString().getBytes()); 141 | } catch (RocksDBException e) { 142 | e.printStackTrace(); 143 | } 144 | } 145 | 146 | public Long getLastIndex() { 147 | byte[] lastIndex = new byte[0]; 148 | try { 149 | lastIndex = machineDb.get(this.lastIndexKey); 150 | if (lastIndex == null) { 151 | lastIndex = "0".getBytes(); 152 | } 153 | } catch (RocksDBException e) { 154 | e.printStackTrace(); 155 | } 156 | return Long.valueOf(new String(lastIndex)); 157 | } 158 | 159 | @Setter 160 | @Getter 161 | @ToString 162 | static class Cmd { 163 | 164 | Long index; 165 | String key; 166 | String value; 167 | 168 | public Cmd() { 169 | } 170 | 171 | public Cmd(String key, String value) { 172 | this.key = key; 173 | this.value = value; 174 | } 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/DefaultLogModule.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import java.io.File; 4 | import java.util.concurrent.locks.ReentrantLock; 5 | 6 | import com.alibaba.fastjson.JSON; 7 | 8 | import org.rocksdb.Options; 9 | import org.rocksdb.RocksDB; 10 | import org.rocksdb.RocksDBException; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import cn.think.in.java.LogModule; 15 | import cn.think.in.java.entity.LogEntry; 16 | import lombok.Getter; 17 | import lombok.Setter; 18 | 19 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 20 | 21 | /** 22 | * 23 | * 默认的日志实现. 日志模块不关心 key, 只关心 index. 24 | * 25 | * @author 莫那·鲁道 26 | * @see cn.think.in.java.entity.LogEntry 27 | */ 28 | @Setter 29 | @Getter 30 | public class DefaultLogModule implements LogModule { 31 | 32 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultLogModule.class); 33 | 34 | 35 | /** public just for test */ 36 | public static String dbDir; 37 | public static String logsDir; 38 | 39 | private static RocksDB logDb; 40 | 41 | public final static byte[] LAST_INDEX_KEY = "LAST_INDEX_KEY".getBytes(); 42 | 43 | ReentrantLock lock = new ReentrantLock(); 44 | 45 | static { 46 | if (dbDir == null) { 47 | dbDir = "./rocksDB-raft/" + System.getProperty("serverPort"); 48 | } 49 | if (logsDir == null) { 50 | logsDir = dbDir + "/logModule"; 51 | } 52 | RocksDB.loadLibrary(); 53 | } 54 | 55 | private DefaultLogModule() { 56 | Options options = new Options(); 57 | options.setCreateIfMissing(true); 58 | 59 | File file = new File(logsDir); 60 | boolean success = false; 61 | if (!file.exists()) { 62 | success = file.mkdirs(); 63 | } 64 | if (success) { 65 | LOGGER.warn("make a new dir : " + logsDir); 66 | } 67 | try { 68 | logDb = RocksDB.open(options, logsDir); 69 | } catch (RocksDBException e) { 70 | LOGGER.warn(e.getMessage()); 71 | } 72 | } 73 | 74 | public static DefaultLogModule getInstance() { 75 | return DefaultLogsLazyHolder.INSTANCE; 76 | } 77 | 78 | private static class DefaultLogsLazyHolder { 79 | 80 | private static final DefaultLogModule INSTANCE = new DefaultLogModule(); 81 | } 82 | 83 | /** 84 | * logEntry 的 index 就是 key. 严格保证递增. 85 | * 86 | * @param logEntry 87 | */ 88 | @Override 89 | public void write(LogEntry logEntry) { 90 | 91 | boolean success = false; 92 | try { 93 | lock.tryLock(3000, MILLISECONDS); 94 | logEntry.setIndex(getLastIndex() + 1); 95 | logDb.put(logEntry.getIndex().toString().getBytes(), JSON.toJSONBytes(logEntry)); 96 | success = true; 97 | LOGGER.info("DefaultLogModule write rocksDB success, logEntry info : [{}]", logEntry); 98 | } catch (RocksDBException | InterruptedException e) { 99 | LOGGER.warn(e.getMessage()); 100 | } finally { 101 | if (success) { 102 | updateLastIndex(logEntry.getIndex()); 103 | } 104 | lock.unlock(); 105 | } 106 | } 107 | 108 | 109 | @Override 110 | public LogEntry read(Long index) { 111 | try { 112 | byte[] result = logDb.get(convert(index)); 113 | if (result == null) { 114 | return null; 115 | } 116 | return JSON.parseObject(result, LogEntry.class); 117 | } catch (RocksDBException e) { 118 | LOGGER.warn(e.getMessage(), e); 119 | } 120 | return null; 121 | } 122 | 123 | @Override 124 | public void removeOnStartIndex(Long startIndex) { 125 | boolean success = false; 126 | int count = 0; 127 | try { 128 | lock.tryLock(3000, MILLISECONDS); 129 | for (long i = startIndex; i <= getLastIndex(); i++) { 130 | logDb.delete(String.valueOf(i).getBytes()); 131 | ++count; 132 | } 133 | success = true; 134 | LOGGER.warn("rocksDB removeOnStartIndex success, count={} startIndex={}, lastIndex={}", count, startIndex, getLastIndex()); 135 | } catch (InterruptedException | RocksDBException e) { 136 | LOGGER.warn(e.getMessage()); 137 | } finally { 138 | if (success) { 139 | updateLastIndex(getLastIndex() - count); 140 | } 141 | lock.unlock(); 142 | } 143 | } 144 | 145 | 146 | @Override 147 | public LogEntry getLast() { 148 | try { 149 | byte[] result = logDb.get(convert(getLastIndex())); 150 | if (result == null) { 151 | return null; 152 | } 153 | return JSON.parseObject(result, LogEntry.class); 154 | } catch (RocksDBException e) { 155 | e.printStackTrace(); 156 | } 157 | return null; 158 | } 159 | 160 | @Override 161 | public Long getLastIndex() { 162 | byte[] lastIndex = "-1".getBytes(); 163 | try { 164 | lastIndex = logDb.get(LAST_INDEX_KEY); 165 | if (lastIndex == null) { 166 | lastIndex = "-1".getBytes(); 167 | } 168 | } catch (RocksDBException e) { 169 | e.printStackTrace(); 170 | } 171 | return Long.valueOf(new String(lastIndex)); 172 | } 173 | 174 | private byte[] convert(Long key) { 175 | return key.toString().getBytes(); 176 | } 177 | 178 | // on lock 179 | private void updateLastIndex(Long index) { 180 | try { 181 | // overWrite 182 | logDb.put(LAST_INDEX_KEY, index.toString().getBytes()); 183 | } catch (RocksDBException e) { 184 | e.printStackTrace(); 185 | } 186 | } 187 | 188 | 189 | } 190 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/DefaultConsensus.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import java.util.concurrent.locks.ReentrantLock; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import cn.think.in.java.Consensus; 9 | import cn.think.in.java.common.NodeStatus; 10 | import cn.think.in.java.common.Peer; 11 | import cn.think.in.java.entity.AentryParam; 12 | import cn.think.in.java.entity.AentryResult; 13 | import cn.think.in.java.entity.LogEntry; 14 | import cn.think.in.java.entity.RvoteParam; 15 | import cn.think.in.java.entity.RvoteResult; 16 | import io.netty.util.internal.StringUtil; 17 | import lombok.Getter; 18 | import lombok.Setter; 19 | 20 | /** 21 | * 22 | * 默认的一致性模块实现. 23 | * 24 | * @author 莫那·鲁道 25 | */ 26 | @Setter 27 | @Getter 28 | public class DefaultConsensus implements Consensus { 29 | 30 | 31 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultConsensus.class); 32 | 33 | 34 | public final DefaultNode node; 35 | 36 | public final ReentrantLock voteLock = new ReentrantLock(); 37 | public final ReentrantLock appendLock = new ReentrantLock(); 38 | 39 | public DefaultConsensus(DefaultNode node) { 40 | this.node = node; 41 | } 42 | 43 | /** 44 | * 请求投票 RPC 45 | * 46 | * 接收者实现： 47 | * 如果term < currentTerm返回 false （5.2 节） 48 | * 如果 votedFor 为空或者就是 candidateId，并且候选人的日志至少和自己一样新，那么就投票给他（5.2 节，5.4 节） 49 | */ 50 | @Override 51 | public RvoteResult requestVote(RvoteParam param) { 52 | try { 53 | RvoteResult.Builder builder = RvoteResult.newBuilder(); 54 | if (!voteLock.tryLock()) { 55 | return builder.term(node.getCurrentTerm()).voteGranted(false).build(); 56 | } 57 | 58 | // 对方任期没有自己新 59 | if (param.getTerm() < node.getCurrentTerm()) { 60 | return builder.term(node.getCurrentTerm()).voteGranted(false).build(); 61 | } 62 | 63 | // (当前节点并没有投票 或者 已经投票过了且是对方节点) && 对方日志和自己一样新 64 | LOGGER.info("node {} current vote for [{}], param candidateId : {}", node.peerSet.getSelf(), node.getVotedFor(), param.getCandidateId()); 65 | LOGGER.info("node {} current term {}, peer term : {}", node.peerSet.getSelf(), node.getCurrentTerm(), param.getTerm()); 66 | 67 | if ((StringUtil.isNullOrEmpty(node.getVotedFor()) || node.getVotedFor().equals(param.getCandidateId()))) { 68 | 69 | if (node.getLogModule().getLast() != null) { 70 | // 对方没有自己新 71 | if (node.getLogModule().getLast().getTerm() > param.getLastLogTerm()) { 72 | return RvoteResult.fail(); 73 | } 74 | // 对方没有自己新 75 | if (node.getLogModule().getLastIndex() > param.getLastLogIndex()) { 76 | return RvoteResult.fail(); 77 | } 78 | } 79 | 80 | // 切换状态 81 | node.status = NodeStatus.FOLLOWER; 82 | // 更新 83 | node.peerSet.setLeader(new Peer(param.getCandidateId())); 84 | node.setCurrentTerm(param.getTerm()); 85 | node.setVotedFor(param.serverId); 86 | // 返回成功 87 | return builder.term(node.currentTerm).voteGranted(true).build(); 88 | } 89 | 90 | return builder.term(node.currentTerm).voteGranted(false).build(); 91 | 92 | } finally { 93 | voteLock.unlock(); 94 | } 95 | } 96 | 97 | 98 | /** 99 | * 附加日志(多个日志,为了提高效率) RPC 100 | * 101 | * 接收者实现： 102 | * 如果 term < currentTerm 就返回 false （5.1 节） 103 | * 如果日志在 prevLogIndex 位置处的日志条目的任期号和 prevLogTerm 不匹配，则返回 false （5.3 节） 104 | * 如果已经存在的日志条目和新的产生冲突（索引值相同但是任期号不同），删除这一条和之后所有的 （5.3 节） 105 | * 附加任何在已有的日志中不存在的条目 106 | * 如果 leaderCommit > commitIndex，令 commitIndex 等于 leaderCommit 和 新日志条目索引值中较小的一个 107 | */ 108 | @Override 109 | public AentryResult appendEntries(AentryParam param) { 110 | AentryResult result = AentryResult.fail(); 111 | try { 112 | if (!appendLock.tryLock()) { 113 | return result; 114 | } 115 | 116 | result.setTerm(node.getCurrentTerm()); 117 | // 不够格 118 | if (param.getTerm() < node.getCurrentTerm()) { 119 | return result; 120 | } 121 | 122 | node.preHeartBeatTime = System.currentTimeMillis(); 123 | node.preElectionTime = System.currentTimeMillis(); 124 | node.peerSet.setLeader(new Peer(param.getLeaderId())); 125 | 126 | // 够格 127 | if (param.getTerm() >= node.getCurrentTerm()) { 128 | LOGGER.debug("node {} become FOLLOWER, currentTerm : {}, param Term : {}, param serverId", 129 | node.peerSet.getSelf(), node.currentTerm, param.getTerm(), param.getServerId()); 130 | // 认怂 131 | node.status = NodeStatus.FOLLOWER; 132 | } 133 | // 使用对方的 term. 134 | node.setCurrentTerm(param.getTerm()); 135 | 136 | //心跳 137 | if (param.getEntries() == null || param.getEntries().length == 0) { 138 | LOGGER.info("node {} append heartbeat success , he's term : {}, my term : {}", 139 | param.getLeaderId(), param.getTerm(), node.getCurrentTerm()); 140 | return AentryResult.newBuilder().term(node.getCurrentTerm()).success(true).build(); 141 | } 142 | 143 | // 真实日志 144 | // 第一次 145 | if (node.getLogModule().getLastIndex() != 0 && param.getPrevLogIndex() != 0) { 146 | LogEntry logEntry; 147 | if ((logEntry = node.getLogModule().read(param.getPrevLogIndex())) != null) { 148 | // 如果日志在 prevLogIndex 位置处的日志条目的任期号和 prevLogTerm 不匹配，则返回 false 149 | // 需要减小 nextIndex 重试. 150 | if (logEntry.getTerm() != param.getPreLogTerm()) { 151 | return result; 152 | } 153 | } else { 154 | // index 不对, 需要递减 nextIndex 重试. 155 | return result; 156 | } 157 | 158 | } 159 | 160 | // 如果已经存在的日志条目和新的产生冲突（索引值相同但是任期号不同），删除这一条和之后所有的 161 | LogEntry existLog = node.getLogModule().read(((param.getPrevLogIndex() + 1))); 162 | if (existLog != null && existLog.getTerm() != param.getEntries()[0].getTerm()) { 163 | // 删除这一条和之后所有的, 然后写入日志和状态机. 164 | node.getLogModule().removeOnStartIndex(param.getPrevLogIndex() + 1); 165 | } else if (existLog != null) { 166 | // 已经有日志了, 不能重复写入. 167 | result.setSuccess(true); 168 | return result; 169 | } 170 | 171 | // 写进日志并且应用到状态机 172 | for (LogEntry entry : param.getEntries()) { 173 | node.getLogModule().write(entry); 174 | node.stateMachine.apply(entry); 175 | result.setSuccess(true); 176 | } 177 | 178 | //如果 leaderCommit > commitIndex，令 commitIndex 等于 leaderCommit 和 新日志条目索引值中较小的一个 179 | if (param.getLeaderCommit() > node.getCommitIndex()) { 180 | int commitIndex = (int) Math.min(param.getLeaderCommit(), node.getLogModule().getLastIndex()); 181 | node.setCommitIndex(commitIndex); 182 | node.setLastApplied(commitIndex); 183 | } 184 | 185 | result.setTerm(node.getCurrentTerm()); 186 | 187 | node.status = NodeStatus.FOLLOWER; 188 | // TODO, 是否应当在成功回复之后, 才正式提交? 防止 leader "等待回复"过程中 挂掉. 189 | return result; 190 | } finally { 191 | appendLock.unlock(); 192 | } 193 | } 194 | 195 | 196 | } 197 | -------------------------------------------------------------------------------- /lu-raft-kv/src/main/java/cn/think/in/java/impl/DefaultNode.java: -------------------------------------------------------------------------------- 1 | package cn.think.in.java.impl; 2 | 3 | import static cn.think.in.java.common.NodeStatus.LEADER; 4 | import static java.util.concurrent.TimeUnit.MILLISECONDS; 5 | 6 | import java.util.ArrayList; 7 | import java.util.Collections; 8 | import java.util.LinkedList; 9 | import java.util.List; 10 | import java.util.Map; 11 | import java.util.concurrent.Callable; 12 | import java.util.concurrent.CancellationException; 13 | import java.util.concurrent.ConcurrentHashMap; 14 | import java.util.concurrent.CopyOnWriteArrayList; 15 | import java.util.concurrent.CountDownLatch; 16 | import java.util.concurrent.ExecutionException; 17 | import java.util.concurrent.Future; 18 | import java.util.concurrent.LinkedBlockingQueue; 19 | import java.util.concurrent.ThreadLocalRandom; 20 | import java.util.concurrent.TimeoutException; 21 | import java.util.concurrent.atomic.AtomicInteger; 22 | 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import cn.think.in.java.Consensus; 27 | import cn.think.in.java.LifeCycle; 28 | import cn.think.in.java.LogModule; 29 | import cn.think.in.java.Node; 30 | import cn.think.in.java.StateMachine; 31 | import cn.think.in.java.common.NodeConfig; 32 | import cn.think.in.java.common.NodeStatus; 33 | import cn.think.in.java.common.Peer; 34 | import cn.think.in.java.common.PeerSet; 35 | import cn.think.in.java.current.RaftThreadPool; 36 | import cn.think.in.java.entity.AentryParam; 37 | import cn.think.in.java.entity.AentryResult; 38 | import cn.think.in.java.entity.Command; 39 | import cn.think.in.java.entity.LogEntry; 40 | import cn.think.in.java.entity.ReplicationFailModel; 41 | import cn.think.in.java.entity.RvoteParam; 42 | import cn.think.in.java.entity.RvoteResult; 43 | import cn.think.in.java.exception.RaftRemotingException; 44 | import cn.think.in.java.membership.changes.ClusterMembershipChanges; 45 | import cn.think.in.java.membership.changes.Result; 46 | import cn.think.in.java.rpc.DefaultRpcClient; 47 | import cn.think.in.java.rpc.DefaultRpcServer; 48 | import cn.think.in.java.rpc.Request; 49 | import cn.think.in.java.rpc.Response; 50 | import cn.think.in.java.rpc.RpcClient; 51 | import cn.think.in.java.rpc.RpcServer; 52 | import cn.think.in.java.util.LongConvert; 53 | import lombok.Getter; 54 | import lombok.Setter; 55 | import raft.client.ClientKVAck; 56 | import raft.client.ClientKVReq; 57 | 58 | /** 59 | * 抽象机器节点, 初始为 follower, 角色随时变化. 60 | * @author 莫那·鲁道 61 | */ 62 | @Getter 63 | @Setter 64 | public class DefaultNode implements Node, LifeCycle, ClusterMembershipChanges { 65 | 66 | private static final Logger LOGGER = LoggerFactory.getLogger(DefaultNode.class); 67 | 68 | /** 选举时间间隔基数 */ 69 | public volatile long electionTime = 15 * 1000; 70 | /** 上一次选举时间 */ 71 | public volatile long preElectionTime = 0; 72 | 73 | /** 上次一心跳时间戳 */ 74 | public volatile long preHeartBeatTime = 0; 75 | /** 心跳间隔基数 */ 76 | public final long heartBeatTick = 5 * 1000; 77 | 78 | 79 | private HeartBeatTask heartBeatTask = new HeartBeatTask(); 80 | private ElectionTask electionTask = new ElectionTask(); 81 | private ReplicationFailQueueConsumer replicationFailQueueConsumer = new ReplicationFailQueueConsumer(); 82 | 83 | private LinkedBlockingQueue replicationFailQueue = new LinkedBlockingQueue<>(2048); 84 | 85 | 86 | /** 87 | * 节点当前状态 88 | * @see NodeStatus 89 | */ 90 | public volatile int status = NodeStatus.FOLLOWER; 91 | 92 | public PeerSet peerSet; 93 | 94 | 95 | 96 | /* ============ 所有服务器上持久存在的 ============= */ 97 | 98 | /** 服务器最后一次知道的任期号（初始化为 0，持续递增） */ 99 | volatile long currentTerm = 0; 100 | /** 在当前获得选票的候选人的 Id */ 101 | volatile String votedFor; 102 | /** 日志条目集；每一个条目包含一个用户状态机执行的指令，和收到时的任期号 */ 103 | LogModule logModule; 104 | 105 | 106 | 107 | /* ============ 所有服务器上经常变的 ============= */ 108 | 109 | /** 已知的最大的已经被提交的日志条目的索引值 */ 110 | volatile long commitIndex; 111 | 112 | /** 最后被应用到状态机的日志条目索引值（初始化为 0，持续递增) */ 113 | volatile long lastApplied = 0; 114 | 115 | /* ========== 在领导人里经常改变的(选举后重新初始化) ================== */ 116 | 117 | /** 对于每一个服务器，需要发送给他的下一个日志条目的索引值（初始化为领导人最后索引值加一） */ 118 | Map nextIndexs; 119 | 120 | /** 对于每一个服务器，已经复制给他的日志的最高索引值 */ 121 | Map matchIndexs; 122 | 123 | 124 | 125 | /* ============================== */ 126 | 127 | public volatile boolean started; 128 | 129 | public NodeConfig config; 130 | 131 | public static RpcServer RPC_SERVER; 132 | 133 | public RpcClient rpcClient = new DefaultRpcClient(); 134 | 135 | public StateMachine stateMachine; 136 | 137 | /* ============================== */ 138 | 139 | /** 一致性模块实现 */ 140 | Consensus consensus; 141 | 142 | ClusterMembershipChanges delegate; 143 | 144 | 145 | /* ============================== */ 146 | 147 | private DefaultNode() { 148 | } 149 | 150 | public static DefaultNode getInstance() { 151 | return DefaultNodeLazyHolder.INSTANCE; 152 | } 153 | 154 | 155 | private static class DefaultNodeLazyHolder { 156 | 157 | private static final DefaultNode INSTANCE = new DefaultNode(); 158 | } 159 | 160 | @Override 161 | public void init() throws Throwable { 162 | if (started) { 163 | return; 164 | } 165 | synchronized (this) { 166 | if (started) { 167 | return; 168 | } 169 | RPC_SERVER.start(); 170 | 171 | consensus = new DefaultConsensus(this); 172 | delegate = new ClusterMembershipChangesImpl(this); 173 | 174 | RaftThreadPool.scheduleWithFixedDelay(heartBeatTask, 500); 175 | RaftThreadPool.scheduleAtFixedRate(electionTask, 6000, 500); 176 | RaftThreadPool.execute(replicationFailQueueConsumer); 177 | 178 | LogEntry logEntry = logModule.getLast(); 179 | if (logEntry != null) { 180 | currentTerm = logEntry.getTerm(); 181 | } 182 | 183 | started = true; 184 | 185 | LOGGER.info("start success, selfId : {} ", peerSet.getSelf()); 186 | } 187 | } 188 | 189 | @Override 190 | public void setConfig(NodeConfig config) { 191 | this.config = config; 192 | stateMachine = config.getStateMachineSaveType().getStateMachine(); 193 | logModule = DefaultLogModule.getInstance(); 194 | 195 | peerSet = PeerSet.getInstance(); 196 | for (String s : config.getPeerAddrs()) { 197 | Peer peer = new Peer(s); 198 | peerSet.addPeer(peer); 199 | if (s.equals("localhost:" + config.getSelfPort())) { 200 | peerSet.setSelf(peer); 201 | } 202 | } 203 | 204 | RPC_SERVER = new DefaultRpcServer(config.selfPort, this); 205 | } 206 | 207 | 208 | @Override 209 | public RvoteResult handlerRequestVote(RvoteParam param) { 210 | LOGGER.warn("handlerRequestVote will be invoke, param info : {}", param); 211 | return consensus.requestVote(param); 212 | } 213 | 214 | @Override 215 | public AentryResult handlerAppendEntries(AentryParam param) { 216 | if (param.getEntries() != null) { 217 | LOGGER.warn("node receive node {} append entry, entry content = {}", param.getLeaderId(), param.getEntries()); 218 | } 219 | 220 | return consensus.appendEntries(param); 221 | 222 | } 223 | 224 | 225 | @SuppressWarnings("unchecked") 226 | @Override 227 | public ClientKVAck redirect(ClientKVReq request) { 228 | Request r = Request.newBuilder(). 229 | obj(request).url(peerSet.getLeader().getAddr()).cmd(Request.CLIENT_REQ).build(); 230 | Response response = rpcClient.send(r); 231 | return (ClientKVAck) response.getResult(); 232 | } 233 | 234 | /** 235 | * 客户端的每一个请求都包含一条被复制状态机执行的指令。 236 | * 领导人把这条指令作为一条新的日志条目附加到日志中去，然后并行的发起附加条目 RPCs 给其他的服务器，让他们复制这条日志条目。 237 | * 当这条日志条目被安全的复制（下面会介绍），领导人会应用这条日志条目到它的状态机中然后把执行的结果返回给客户端。 238 | * 如果跟随者崩溃或者运行缓慢，再或者网络丢包， 239 | * 领导人会不断的重复尝试附加日志条目 RPCs （尽管已经回复了客户端）直到所有的跟随者都最终存储了所有的日志条目。 240 | * @param request 241 | * @return 242 | */ 243 | @Override 244 | public synchronized ClientKVAck handlerClientRequest(ClientKVReq request) { 245 | 246 | LOGGER.warn("handlerClientRequest handler {} operation, and key : [{}], value : [{}]", 247 | ClientKVReq.Type.value(request.getType()), request.getKey(), request.getValue()); 248 | 249 | if (status != LEADER) { 250 | LOGGER.warn("I not am leader , only invoke redirect method, leader addr : {}, my addr : {}", 251 | peerSet.getLeader(), peerSet.getSelf().getAddr()); 252 | return redirect(request); 253 | } 254 | 255 | if (request.getType() == ClientKVReq.GET) { 256 | LogEntry logEntry = stateMachine.get(request.getKey()); 257 | if (logEntry != null) { 258 | return new ClientKVAck(logEntry.getCommand()); 259 | } 260 | return new ClientKVAck(null); 261 | } 262 | 263 | LogEntry logEntry = LogEntry.newBuilder() 264 | .command(Command.newBuilder(). 265 | key(request.getKey()). 266 | value(request.getValue()). 267 | build()) 268 | .term(currentTerm) 269 | .build(); 270 | 271 | // 预提交到本地日志, TODO 预提交 272 | logModule.write(logEntry); 273 | LOGGER.info("write logModule success, logEntry info : {}, log index : {}", logEntry, logEntry.getIndex()); 274 | 275 | final AtomicInteger success = new AtomicInteger(0); 276 | 277 | List> futureList = new CopyOnWriteArrayList<>(); 278 | 279 | int count = 0; 280 | // 复制到其他机器 281 | for (Peer peer : peerSet.getPeersWithOutSelf()) { 282 | // TODO check self and RaftThreadPool 283 | count++; 284 | // 并行发起 RPC 复制. 285 | futureList.add(replication(peer, logEntry)); 286 | } 287 | 288 | CountDownLatch latch = new CountDownLatch(futureList.size()); 289 | List resultList = new CopyOnWriteArrayList<>(); 290 | 291 | getRPCAppendResult(futureList, latch, resultList); 292 | 293 | try { 294 | latch.await(4000, MILLISECONDS); 295 | } catch (InterruptedException e) { 296 | e.printStackTrace(); 297 | } 298 | 299 | for (Boolean aBoolean : resultList) { 300 | if (aBoolean) { 301 | success.incrementAndGet(); 302 | } 303 | } 304 | 305 | // 如果存在一个满足N > commitIndex的 N，并且大多数的matchIndex[i] ≥ N成立， 306 | // 并且log[N].term == currentTerm成立，那么令 commitIndex 等于这个 N （5.3 和 5.4 节） 307 | List matchIndexList = new ArrayList<>(matchIndexs.values()); 308 | // 小于 2, 没有意义 309 | int median = 0; 310 | if (matchIndexList.size() >= 2) { 311 | Collections.sort(matchIndexList); 312 | median = matchIndexList.size() / 2; 313 | } 314 | Long N = matchIndexList.get(median); 315 | if (N > commitIndex) { 316 | LogEntry entry = logModule.read(N); 317 | if (entry != null && entry.getTerm() == currentTerm) { 318 | commitIndex = N; 319 | } 320 | } 321 | 322 | // 响应客户端(成功一半) 323 | if (success.get() >= (count / 2)) { 324 | // 更新 325 | commitIndex = logEntry.getIndex(); 326 | // 应用到状态机 327 | getStateMachine().apply(logEntry); 328 | lastApplied = commitIndex; 329 | 330 | LOGGER.info("success apply local state machine, logEntry info : {}", logEntry); 331 | // 返回成功. 332 | return ClientKVAck.ok(); 333 | } else { 334 | // 回滚已经提交的日志. 335 | logModule.removeOnStartIndex(logEntry.getIndex()); 336 | LOGGER.warn("fail apply local state machine, logEntry info : {}", logEntry); 337 | // TODO 不应用到状态机,但已经记录到日志中.由定时任务从重试队列取出,然后重复尝试,当达到条件时,应用到状态机. 338 | // 这里应该返回错误, 因为没有成功复制过半机器. 339 | return ClientKVAck.fail(); 340 | } 341 | } 342 | 343 | private void getRPCAppendResult(List> futureList, CountDownLatch latch, List resultList) { 344 | for (Future future : futureList) { 345 | RaftThreadPool.execute(new Runnable() { 346 | @Override 347 | public void run() { 348 | try { 349 | resultList.add(future.get(3000, MILLISECONDS)); 350 | } catch (CancellationException | TimeoutException | ExecutionException | InterruptedException e) { 351 | e.printStackTrace(); 352 | resultList.add(false); 353 | } finally { 354 | latch.countDown(); 355 | } 356 | } 357 | }); 358 | } 359 | } 360 | 361 | 362 | /** 复制到其他机器 */ 363 | public Future replication(Peer peer, LogEntry entry) { 364 | 365 | return RaftThreadPool.submit(new Callable() { 366 | @Override 367 | public Boolean call() throws Exception { 368 | 369 | long start = System.currentTimeMillis(), end = start; 370 | 371 | // 20 秒重试时间 372 | while (end - start < 20 * 1000L) { 373 | 374 | AentryParam aentryParam = new AentryParam(); 375 | aentryParam.setTerm(currentTerm); 376 | aentryParam.setServerId(peer.getAddr()); 377 | aentryParam.setLeaderId(peerSet.getSelf().getAddr()); 378 | 379 | aentryParam.setLeaderCommit(commitIndex); 380 | 381 | // 以我这边为准, 这个行为通常是成为 leader 后,首次进行 RPC 才有意义. 382 | Long nextIndex = nextIndexs.get(peer); 383 | LinkedList logEntries = new LinkedList<>(); 384 | if (entry.getIndex() >= nextIndex) { 385 | for (long i = nextIndex; i <= entry.getIndex(); i++) { 386 | LogEntry l = logModule.read(i); 387 | if (l != null) { 388 | logEntries.add(l); 389 | } 390 | } 391 | } else { 392 | logEntries.add(entry); 393 | } 394 | // 最小的那个日志. 395 | LogEntry preLog = getPreLog(logEntries.getFirst()); 396 | aentryParam.setPreLogTerm(preLog.getTerm()); 397 | aentryParam.setPrevLogIndex(preLog.getIndex()); 398 | 399 | aentryParam.setEntries(logEntries.toArray(new LogEntry[0])); 400 | 401 | Request request = Request.newBuilder() 402 | .cmd(Request.A_ENTRIES) 403 | .obj(aentryParam) 404 | .url(peer.getAddr()) 405 | .build(); 406 | 407 | try { 408 | Response response = getRpcClient().send(request); 409 | if (response == null) { 410 | return false; 411 | } 412 | AentryResult result = (AentryResult) response.getResult(); 413 | if (result != null && result.isSuccess()) { 414 | LOGGER.info("append follower entry success , follower=[{}], entry=[{}]", peer, aentryParam.getEntries()); 415 | // update 这两个追踪值 416 | nextIndexs.put(peer, entry.getIndex() + 1); 417 | matchIndexs.put(peer, entry.getIndex()); 418 | return true; 419 | } else if (result != null) { 420 | // 对方比我大 421 | if (result.getTerm() > currentTerm) { 422 | LOGGER.warn("follower [{}] term [{}] than more self, and my term = [{}], so, I will become follower", 423 | peer, result.getTerm(), currentTerm); 424 | currentTerm = result.getTerm(); 425 | // 认怂, 变成跟随者 426 | status = NodeStatus.FOLLOWER; 427 | return false; 428 | } // 没我大, 却失败了,说明 index 不对.或者 term 不对. 429 | else { 430 | // 递减 431 | if (nextIndex == 0) { 432 | nextIndex = 1L; 433 | } 434 | nextIndexs.put(peer, nextIndex - 1); 435 | LOGGER.warn("follower {} nextIndex not match, will reduce nextIndex and retry RPC append, nextIndex : [{}]", peer.getAddr(), 436 | nextIndex); 437 | // 重来, 直到成功. 438 | } 439 | } 440 | 441 | end = System.currentTimeMillis(); 442 | 443 | } catch (Exception e) { 444 | LOGGER.warn(e.getMessage(), e); 445 | // TODO 到底要不要放队列重试? 446 | // ReplicationFailModel model = ReplicationFailModel.newBuilder() 447 | // .callable(this) 448 | // .logEntry(entry) 449 | // .peer(peer) 450 | // .offerTime(System.currentTimeMillis()) 451 | // .build(); 452 | // replicationFailQueue.offer(model); 453 | return false; 454 | } 455 | } 456 | // 超时了,没办法了 457 | return false; 458 | } 459 | }); 460 | 461 | } 462 | 463 | private LogEntry getPreLog(LogEntry logEntry) { 464 | LogEntry entry = logModule.read(logEntry.getIndex() - 1); 465 | 466 | if (entry == null) { 467 | LOGGER.warn("get perLog is null , parameter logEntry : {}", logEntry); 468 | entry = LogEntry.newBuilder().index(0L).term(0).command(null).build(); 469 | } 470 | return entry; 471 | } 472 | 473 | 474 | class ReplicationFailQueueConsumer implements Runnable { 475 | 476 | /** 一分钟 */ 477 | long intervalTime = 1000 * 60; 478 | 479 | @Override 480 | public void run() { 481 | for (; ; ) { 482 | 483 | try { 484 | ReplicationFailModel model = replicationFailQueue.take(); 485 | if (status != LEADER) { 486 | // 应该清空? 487 | replicationFailQueue.clear(); 488 | continue; 489 | } 490 | LOGGER.warn("replication Fail Queue Consumer take a task, will be retry replication, content detail : [{}]", model.logEntry); 491 | long offerTime = model.offerTime; 492 | if (System.currentTimeMillis() - offerTime > intervalTime) { 493 | LOGGER.warn("replication Fail event Queue maybe full or handler slow"); 494 | } 495 | 496 | Callable callable = model.callable; 497 | Future future = RaftThreadPool.submit(callable); 498 | Boolean r = future.get(3000, MILLISECONDS); 499 | // 重试成功. 500 | if (r) { 501 | // 可能有资格应用到状态机. 502 | tryApplyStateMachine(model); 503 | } 504 | 505 | } catch (InterruptedException e) { 506 | // ignore 507 | } catch (ExecutionException | TimeoutException e) { 508 | LOGGER.warn(e.getMessage()); 509 | } 510 | } 511 | } 512 | } 513 | 514 | private void tryApplyStateMachine(ReplicationFailModel model) { 515 | 516 | String success = stateMachine.getString(model.successKey); 517 | stateMachine.setString(model.successKey, String.valueOf(Integer.valueOf(success) + 1)); 518 | 519 | String count = stateMachine.getString(model.countKey); 520 | 521 | if (Integer.valueOf(success) >= Integer.valueOf(count) / 2) { 522 | stateMachine.apply(model.logEntry); 523 | stateMachine.delString(model.countKey, model.successKey); 524 | } 525 | } 526 | 527 | 528 | @Override 529 | public void destroy() throws Throwable { 530 | RPC_SERVER.stop(); 531 | } 532 | 533 | 534 | /** 535 | * 1. 在转变成候选人后就立即开始选举过程 536 | * 自增当前的任期号（currentTerm） 537 | * 给自己投票 538 | * 重置选举超时计时器 539 | * 发送请求投票的 RPC 给其他所有服务器 540 | * 2. 如果接收到大多数服务器的选票，那么就变成领导人 541 | * 3. 如果接收到来自新的领导人的附加日志 RPC，转变成跟随者 542 | * 4. 如果选举过程超时，再次发起一轮选举 543 | */ 544 | class ElectionTask implements Runnable { 545 | 546 | @Override 547 | public void run() { 548 | 549 | if (status == LEADER) { 550 | return; 551 | } 552 | 553 | long current = System.currentTimeMillis(); 554 | // 基于 RAFT 的随机时间,解决冲突. 555 | electionTime = electionTime + ThreadLocalRandom.current().nextInt(50); 556 | if (current - preElectionTime < electionTime) { 557 | return; 558 | } 559 | status = NodeStatus.CANDIDATE; 560 | LOGGER.error("node {} will become CANDIDATE and start election leader, current term : [{}], LastEntry : [{}]", 561 | peerSet.getSelf(), currentTerm, logModule.getLast()); 562 | 563 | preElectionTime = System.currentTimeMillis() + ThreadLocalRandom.current().nextInt(200) + 150; 564 | 565 | currentTerm = currentTerm + 1; 566 | // 推荐自己. 567 | votedFor = peerSet.getSelf().getAddr(); 568 | 569 | List peers = peerSet.getPeersWithOutSelf(); 570 | 571 | ArrayList futureArrayList = new ArrayList<>(); 572 | 573 | LOGGER.info("peerList size : {}, peer list content : {}", peers.size(), peers); 574 | 575 | // 发送请求 576 | for (Peer peer : peers) { 577 | 578 | futureArrayList.add(RaftThreadPool.submit(new Callable() { 579 | @Override 580 | public Object call() throws Exception { 581 | long lastTerm = 0L; 582 | LogEntry last = logModule.getLast(); 583 | if (last != null) { 584 | lastTerm = last.getTerm(); 585 | } 586 | 587 | RvoteParam param = RvoteParam.newBuilder(). 588 | term(currentTerm). 589 | candidateId(peerSet.getSelf().getAddr()). 590 | lastLogIndex(LongConvert.convert(logModule.getLastIndex())). 591 | lastLogTerm(lastTerm). 592 | build(); 593 | 594 | Request request = Request.newBuilder() 595 | .cmd(Request.R_VOTE) 596 | .obj(param) 597 | .url(peer.getAddr()) 598 | .build(); 599 | 600 | try { 601 | @SuppressWarnings("unchecked") 602 | Response response = getRpcClient().send(request); 603 | return response; 604 | 605 | } catch (RaftRemotingException e) { 606 | LOGGER.error("ElectionTask RPC Fail , URL : " + request.getUrl()); 607 | return null; 608 | } 609 | } 610 | })); 611 | } 612 | 613 | AtomicInteger success2 = new AtomicInteger(0); 614 | CountDownLatch latch = new CountDownLatch(futureArrayList.size()); 615 | 616 | LOGGER.info("futureArrayList.size() : {}", futureArrayList.size()); 617 | // 等待结果. 618 | for (Future future : futureArrayList) { 619 | RaftThreadPool.submit(new Callable() { 620 | @Override 621 | public Object call() throws Exception { 622 | try { 623 | 624 | @SuppressWarnings("unchecked") 625 | Response response = (Response) future.get(3000, MILLISECONDS); 626 | if (response == null) { 627 | return -1; 628 | } 629 | boolean isVoteGranted = response.getResult().isVoteGranted(); 630 | 631 | if (isVoteGranted) { 632 | success2.incrementAndGet(); 633 | } else { 634 | // 更新自己的任期. 635 | long resTerm = response.getResult().getTerm(); 636 | if (resTerm >= currentTerm) { 637 | currentTerm = resTerm; 638 | } 639 | } 640 | return 0; 641 | } catch (Exception e) { 642 | LOGGER.error("future.get exception , e : ", e); 643 | return -1; 644 | } finally { 645 | latch.countDown(); 646 | } 647 | } 648 | }); 649 | } 650 | 651 | try { 652 | // 稍等片刻 653 | latch.await(3500, MILLISECONDS); 654 | } catch (InterruptedException e) { 655 | LOGGER.warn("InterruptedException By Master election Task"); 656 | } 657 | 658 | int success = success2.get(); 659 | LOGGER.info("node {} maybe become leader , success count = {} , status : {}", peerSet.getSelf(), success, NodeStatus.Enum.value(status)); 660 | // 如果投票期间,有其他服务器发送 appendEntry , 就可能变成 follower ,这时,应该停止. 661 | if (status == NodeStatus.FOLLOWER) { 662 | return; 663 | } 664 | // 加上自身. 665 | if (success >= peers.size() / 2) { 666 | LOGGER.warn("node {} become leader ", peerSet.getSelf()); 667 | status = LEADER; 668 | peerSet.setLeader(peerSet.getSelf()); 669 | votedFor = ""; 670 | becomeLeaderToDoThing(); 671 | } else { 672 | // else 重新选举 673 | votedFor = ""; 674 | } 675 | 676 | } 677 | } 678 | 679 | /** 680 | * 初始化所有的 nextIndex 值为自己的最后一条日志的 index + 1. 如果下次 RPC 时, 跟随者和leader 不一致,就会失败. 681 | * 那么 leader 尝试递减 nextIndex 并进行重试.最终将达成一致. 682 | */ 683 | private void becomeLeaderToDoThing() { 684 | nextIndexs = new ConcurrentHashMap<>(); 685 | matchIndexs = new ConcurrentHashMap<>(); 686 | for (Peer peer : peerSet.getPeersWithOutSelf()) { 687 | nextIndexs.put(peer, logModule.getLastIndex() + 1); 688 | matchIndexs.put(peer, 0L); 689 | } 690 | } 691 | 692 | 693 | class HeartBeatTask implements Runnable { 694 | 695 | @Override 696 | public void run() { 697 | 698 | if (status != LEADER) { 699 | return; 700 | } 701 | 702 | long current = System.currentTimeMillis(); 703 | if (current - preHeartBeatTime < heartBeatTick) { 704 | return; 705 | } 706 | LOGGER.info("=========== NextIndex ============="); 707 | for (Peer peer : peerSet.getPeersWithOutSelf()) { 708 | LOGGER.info("Peer {} nextIndex={}", peer.getAddr(), nextIndexs.get(peer)); 709 | } 710 | 711 | preHeartBeatTime = System.currentTimeMillis(); 712 | 713 | // 心跳只关心 term 和 leaderID 714 | for (Peer peer : peerSet.getPeersWithOutSelf()) { 715 | 716 | AentryParam param = AentryParam.newBuilder() 717 | .entries(null)// 心跳,空日志. 718 | .leaderId(peerSet.getSelf().getAddr()) 719 | .serverId(peer.getAddr()) 720 | .term(currentTerm) 721 | .build(); 722 | 723 | Request request = new Request<>( 724 | Request.A_ENTRIES, 725 | param, 726 | peer.getAddr()); 727 | 728 | RaftThreadPool.execute(new Runnable() { 729 | @Override 730 | public void run() { 731 | try { 732 | Response response = getRpcClient().send(request); 733 | AentryResult aentryResult = (AentryResult) response.getResult(); 734 | long term = aentryResult.getTerm(); 735 | 736 | if (term > currentTerm) { 737 | LOGGER.error("self will become follower, he's term : {}, my term : {}", term, currentTerm); 738 | currentTerm = term; 739 | votedFor = ""; 740 | status = NodeStatus.FOLLOWER; 741 | } 742 | } catch (Exception e) { 743 | LOGGER.error("HeartBeatTask RPC Fail, request URL : {} ", request.getUrl()); 744 | } 745 | } 746 | }, false); 747 | } 748 | } 749 | } 750 | 751 | @Override 752 | public Result addPeer(Peer newPeer) { 753 | return delegate.addPeer(newPeer); 754 | } 755 | 756 | @Override 757 | public Result removePeer(Peer oldPeer) { 758 | return delegate.removePeer(oldPeer); 759 | } 760 | 761 | } 762 | --------------------------------------------------------------------------------