├── README.md ├── main ├── run.sh ├── config.json └── main.go ├── malicious.go ├── common.go ├── network.go ├── rpc.go ├── client.go ├── debug.go └── hotstuff.go /README.md: -------------------------------------------------------------------------------- 1 | # hotstuff-go 2 | Implement HotStuff consensus with go-lang 3 | -------------------------------------------------------------------------------- /main/run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $1 = "start" ]; then 3 | for (( i=0; i<4; i++ )) do 4 | ./main server $i --tag=hotstuff-server & 5 | done 6 | for (( i=0; i<3; i++ )) do 7 | ./main client $i --tag=hotstuff-client & 8 | done 9 | elif [ $1 = "kill" ]; then 10 | pgrep -f hotstuff-server | xargs kill 11 | pgrep -f hotstuff-client | xargs kill 12 | elif [ $1 = "build" ]; then 13 | go build 14 | fi -------------------------------------------------------------------------------- /main/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": [ 3 | { 4 | "id": 0, 5 | "address": "127.0.0.1:10010", 6 | "debug": "127.0.0.1:20010" 7 | }, 8 | { 9 | "id": 1, 10 | "address": "127.0.0.1:10011", 11 | "debug": "127.0.0.1:20011" 12 | }, 13 | { 14 | "id": 2, 15 | "address": "127.0.0.1:10012", 16 | "debug": "127.0.0.1:20012" 17 | }, 18 | { 19 | "id": 3, 20 | "address": "127.0.0.1:10013", 21 | "debug": "127.0.0.1:20013" 22 | } 23 | ], 24 | "clients": [ 25 | { 26 | "id": 0, 27 | "address": "127.0.0.1:30010", 28 | "debug": "127.0.0.1:30110" 29 | }, 30 | { 31 | "id": 1, 32 | "address": "127.0.0.1:30011", 33 | "debug": "127.0.0.1:30111" 34 | }, 35 | { 36 | "id": 2, 37 | "address": "127.0.0.1:30012", 38 | "debug": "127.0.0.1:30112" 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /malicious.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "errors" 5 | "strconv" 6 | ) 7 | 8 | func (hs *HotStuff) setMaliciousMode(maliciousMode int) error { 9 | hs.mu.Lock() 10 | defer hs.mu.Unlock() 11 | 12 | if maliciousMode < 0 || maliciousMode > PartialMaliciousMode { 13 | return errors.New("Invalid malicious mode") 14 | } 15 | 16 | hs.maliciousMode = MaliciousBehaviorMode(maliciousMode) 17 | return nil 18 | } 19 | 20 | func (hs *HotStuff) sendMaliciousMsg(id int, rpcname string, rpcacgs interface{}, isPartial bool) { 21 | args := rpcacgs.(*MsgArgs) 22 | if !args.ParSig { 23 | // From Leader 24 | if !isPartial { 25 | if args.Node.Parent != "" { 26 | if hs.lockedQC.NodeId != "" { 27 | node, ok := hs.nodeMap[hs.lockedQC.NodeId] 28 | if ok { 29 | fakeReq := &RequestArgs{} 30 | fakeReq.ClientId = 1 31 | fakeReq.Operation = "fakeop" 32 | newId := getLogNodeId(hs.viewId, fakeReq) 33 | newIdwithColor := "\033[0;31m" + newId + "_" + strconv.Itoa(hs.me) + "\033[0m" 34 | args.Node.Id = newIdwithColor 35 | args.Node.Parent = node.Parent 36 | args.Node.Justify = node.Justify 37 | } 38 | } 39 | } 40 | } 41 | 42 | hs.rawSendMsg(id, rpcname, args) 43 | } else { 44 | // To Leader 45 | // Generic message 46 | if args.Node.Id != "" { 47 | fakeReq := &RequestArgs{} 48 | fakeReq.ClientId = 1 49 | fakeReq.Operation = "fakeop" 50 | newId := getLogNodeId(hs.viewId, fakeReq) 51 | newIdwithColor := "\033[0;31m" + newId + "_" + strconv.Itoa(hs.me) + "\033[0m" 52 | args.Node.Id = newIdwithColor 53 | args.Node.Request = *fakeReq 54 | hs.rawSendMsg(id, rpcname, args) 55 | } else { 56 | hs.rawSendMsg(id, rpcname, args) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /main/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strconv" 8 | "sync" 9 | 10 | "github.com/myzWILLmake/hotstuff-go" 11 | "github.com/spf13/viper" 12 | ) 13 | 14 | type NodeInfo struct { 15 | Id int `json:"id"` 16 | Address string `json:"address"` 17 | Debug string `json:"debug"` 18 | } 19 | 20 | type X struct { 21 | Servers []NodeInfo `json:"servers"` 22 | Clients []NodeInfo `json:"clients"` 23 | } 24 | 25 | func main() { 26 | if len(os.Args) < 3 { 27 | log.Fatal("Invalid augments") 28 | return 29 | } 30 | 31 | nodeType := os.Args[1] 32 | if nodeType != "client" && nodeType != "server" { 33 | log.Fatal("Invalid node type") 34 | return 35 | } 36 | 37 | id, err := strconv.Atoi(os.Args[2]) 38 | if err != nil { 39 | log.Fatal("Invalid id") 40 | return 41 | } 42 | viper.SetConfigName("config.json") 43 | viper.AddConfigPath(".") 44 | viper.SetConfigType("json") 45 | err = viper.ReadInConfig() 46 | if err != nil { 47 | fmt.Printf("config file error: %s\n", err) 48 | os.Exit(1) 49 | } 50 | var x X 51 | viper.Unmarshal(&x) 52 | 53 | serverAddrs := make([]string, len(x.Servers)) 54 | for _, node := range x.Servers { 55 | serverAddrs[node.Id] = node.Address 56 | } 57 | clientAddrs := make([]string, len(x.Clients)) 58 | for _, node := range x.Clients { 59 | clientAddrs[node.Id] = node.Address 60 | } 61 | 62 | if nodeType == "server" { 63 | debugAddr := x.Servers[id].Debug 64 | wg := &sync.WaitGroup{} 65 | hotstuff.RunHotStuffServer(id, serverAddrs, clientAddrs, true, debugAddr, wg) 66 | wg.Wait() 67 | } else if nodeType == "client" { 68 | clientAddr := x.Clients[id].Address 69 | debugAddr := x.Clients[id].Debug 70 | wg := &sync.WaitGroup{} 71 | hotstuff.RunClient(id, clientAddr, serverAddrs, true, debugAddr, wg) 72 | wg.Wait() 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /common.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | ) 7 | 8 | type TimerWithCancel struct { 9 | d time.Duration 10 | t *time.Timer 11 | c chan interface{} 12 | f func() 13 | } 14 | 15 | func NewTimerWithCancel(d time.Duration) *TimerWithCancel { 16 | t := &TimerWithCancel{} 17 | t.d = d 18 | t.c = make(chan interface{}) 19 | return t 20 | } 21 | 22 | func (t *TimerWithCancel) Start() { 23 | t.t = time.NewTimer(t.d) 24 | go func() { 25 | select { 26 | case <-t.t.C: 27 | t.f() 28 | case <-t.c: 29 | } 30 | }() 31 | } 32 | 33 | func (t *TimerWithCancel) SetTimeout(f func()) { 34 | t.f = f 35 | } 36 | 37 | func (t *TimerWithCancel) Cancel() { 38 | t.c <- nil 39 | } 40 | 41 | type LogNode struct { 42 | Id string 43 | Parent string 44 | ViewId int 45 | Request RequestArgs 46 | Justify QC 47 | } 48 | 49 | func getLogNodeId(viewId int, request *RequestArgs) string { 50 | s := strconv.Itoa(viewId) + "_" + request.Operation.(string) 51 | // h := sha1.New() 52 | // h.Write([]byte(s)) 53 | // bs := h.Sum(nil) 54 | // ns := fmt.Sprintf("%x", bs) 55 | // return ns[:8] 56 | return s 57 | } 58 | 59 | type QC struct { 60 | ViewId int 61 | NodeId string 62 | //TODO: threshold signatures 63 | } 64 | 65 | type DefaultReply struct { 66 | Err string 67 | } 68 | 69 | type RequestArgs struct { 70 | Operation interface{} 71 | Timestamp int64 72 | ClientId int 73 | } 74 | 75 | type ReplyArgs struct { 76 | ViewId int 77 | Timestamp int64 78 | ReplicaId int 79 | Result interface{} 80 | } 81 | 82 | type MsgArgs struct { 83 | RepId int 84 | ViewId int 85 | Node LogNode 86 | QC QC 87 | // just a mark for partial signature 88 | // TODO: implement partial signature 89 | ParSig bool 90 | } 91 | 92 | type MaliciousBehaviorMode int 93 | 94 | const ( 95 | NormalMode = iota 96 | CrashedLike 97 | MaliciousMode 98 | PartialMaliciousMode 99 | ) 100 | -------------------------------------------------------------------------------- /network.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "net" 7 | "net/http" 8 | "net/rpc" 9 | "sync" 10 | ) 11 | 12 | type peerWrapper struct { 13 | client *rpc.Client 14 | address string 15 | } 16 | 17 | func (c *peerWrapper) Call(serviceMethod string, args interface{}, reply interface{}) error { 18 | var err error 19 | if c.client == nil { 20 | err = errors.New("") 21 | } else { 22 | err = c.client.Call(serviceMethod, args, reply) 23 | } 24 | if err != nil { 25 | var errdial error 26 | c.client, errdial = rpc.DialHTTP("tcp", c.address) 27 | if errdial != nil { 28 | return errdial 29 | } 30 | err = c.client.Call(serviceMethod, args, reply) 31 | } 32 | return err 33 | } 34 | 35 | func createPeers(addresses []string) []peerWrapper { 36 | peers := make([]peerWrapper, len(addresses)) 37 | for i := 0; i < len(addresses); i++ { 38 | peers[i].client = nil 39 | peers[i].address = addresses[i] 40 | } 41 | 42 | return peers 43 | } 44 | 45 | func RunHotStuffServer(id int, serverAddrs, clientAddrs []string, debug bool, debugAddr string, wg *sync.WaitGroup) *HotStuff { 46 | debugCh := make(chan interface{}, 1024) 47 | servers := createPeers(serverAddrs) 48 | clients := createPeers(clientAddrs) 49 | hotStuff := MakeHotStuff(id, servers, clients, debugCh) 50 | 51 | if debug { 52 | MakeHotStuffDebugServer(debugAddr, debugCh, hotStuff, wg) 53 | } 54 | 55 | rpc.Register(hotStuff) 56 | rpc.HandleHTTP() 57 | l, err := net.Listen("tcp", serverAddrs[id]) 58 | if err != nil { 59 | log.Fatal("listen error:", err) 60 | return nil 61 | } 62 | 63 | go http.Serve(l, nil) 64 | return hotStuff 65 | } 66 | 67 | func RunClient(id int, clientAddr string, hotStuffAddrs []string, debug bool, debugAddr string, wg *sync.WaitGroup) *Client { 68 | debugCh := make(chan interface{}, 1024) 69 | peers := createPeers(hotStuffAddrs) 70 | client := MakeClient(id, peers, debugCh) 71 | 72 | if debug { 73 | MakeClientDebugServer(debugAddr, debugCh, client, wg) 74 | } 75 | 76 | rpc.Register(client) 77 | rpc.HandleHTTP() 78 | l, err := net.Listen("tcp", clientAddr) 79 | if err != nil { 80 | log.Fatal("listen error:", err) 81 | return nil 82 | } 83 | 84 | go http.Serve(l, nil) 85 | return client 86 | } 87 | -------------------------------------------------------------------------------- /rpc.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import "fmt" 4 | 5 | func (hs *HotStuff) Request(args *RequestArgs, reply *DefaultReply) error { 6 | hs.mu.Lock() 7 | defer hs.mu.Unlock() 8 | 9 | // TODO: only one client request should be serviced 10 | msg := fmt.Sprintf("Recieve Request: id[%d] op[%s] time[%d]\n", args.ClientId, args.Operation.(string), args.Timestamp) 11 | hs.debugPrint(msg) 12 | if hs.isLeader() { 13 | hs.processClientRequest(args) 14 | if hs.noopTimer != nil { 15 | hs.noopTimer.Cancel() 16 | hs.noopTimer = nil 17 | } 18 | } 19 | 20 | return nil 21 | } 22 | 23 | func (hs *HotStuff) Msg(args *MsgArgs, reply *DefaultReply) error { 24 | hs.mu.Lock() 25 | defer hs.mu.Unlock() 26 | 27 | if !args.ParSig { 28 | // From Leader 29 | msg := fmt.Sprintf("\033[1;36mReceive Msg From Leader:\033[0m rid[%d] viewId[%d] nodeId[%s]\n", args.RepId, args.ViewId, args.Node.Id) 30 | hs.debugPrint(msg) 31 | if args.ViewId%hs.n != args.RepId { 32 | reply.Err = fmt.Sprintf("Generic msg from invalid leader[%d].\n", args.RepId) 33 | return nil 34 | } 35 | 36 | if args.ViewId > hs.viewId { 37 | // TODO: check threshold signature 38 | hs.newView(args.ViewId) 39 | } 40 | 41 | if args.ViewId != hs.viewId { 42 | reply.Err = fmt.Sprintf("Generic msg from invalid viewId[%d].\n", args.ViewId) 43 | return nil 44 | } 45 | 46 | hs.update(&args.Node) 47 | } else { 48 | // To Leader 49 | msg := fmt.Sprintf("\033[1;36mReceive Msg to Leader:\033[0m rid[%d] viewId[%d] nodeId[%s] qcId[%s]\n", args.RepId, args.ViewId, args.Node.Id, args.QC.NodeId) 50 | hs.debugPrint(msg) 51 | // if args.ViewId != hs.viewId { 52 | // reply.Err = fmt.Sprintf("Vote msg from invalid viewId[%d].\n", args.ViewId) 53 | // return nil 54 | // } 55 | 56 | if hs.isNextLeader() { 57 | hs.savedMsgs[args.RepId] = args 58 | hs.processSavedMsgs() 59 | } 60 | } 61 | 62 | return nil 63 | } 64 | 65 | func (c *Client) Reply(args *ReplyArgs, reply *DefaultReply) error { 66 | c.mu.Lock() 67 | defer c.mu.Unlock() 68 | c.debugPrint(fmt.Sprintf("Received Reply[%d, %s, %d] from ReplicaId[%d]\n", args.Timestamp, args.Result, args.ViewId, args.ReplicaId)) 69 | c.saveReply(args) 70 | c.processReplies(args.Timestamp) 71 | return nil 72 | } 73 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | // save operation as string 10 | type Client struct { 11 | mu *sync.Mutex 12 | me int 13 | n int 14 | f int 15 | peers []peerWrapper 16 | requests map[int64]string 17 | replies map[int64]map[int]string 18 | 19 | debugCh chan interface{} 20 | } 21 | 22 | func (c *Client) broadcast(rpcname string, rpcargs interface{}) { 23 | reply := &DefaultReply{} 24 | for _, peer := range c.peers { 25 | p := peer 26 | go p.Call("HotStuff."+rpcname, rpcargs, reply) 27 | } 28 | } 29 | 30 | func (c *Client) newRequest(command string) { 31 | requestArgs := &RequestArgs{} 32 | requestArgs.ClientId = c.me 33 | requestArgs.Operation = command 34 | requestArgs.Timestamp = time.Now().Unix() 35 | 36 | c.mu.Lock() 37 | defer c.mu.Unlock() 38 | 39 | c.replies[requestArgs.Timestamp] = make(map[int]string) 40 | c.requests[requestArgs.Timestamp] = command 41 | c.broadcast("Request", requestArgs) 42 | } 43 | 44 | func (c *Client) saveReply(replyArgs *ReplyArgs) { 45 | timestamp := replyArgs.Timestamp 46 | if c.replies[timestamp] == nil { 47 | return 48 | } 49 | 50 | c.replies[timestamp][replyArgs.ReplicaId] = replyArgs.Result.(string) 51 | } 52 | 53 | func (c *Client) processReplies(timestamp int64) { 54 | replies := c.replies[timestamp] 55 | if replies == nil || len(replies) <= c.f { 56 | return 57 | } 58 | 59 | resultMap := make(map[string]int) 60 | maxCnt := 0 61 | maxResult := "" 62 | for _, result := range replies { 63 | resultMap[result]++ 64 | if resultMap[result] > maxCnt { 65 | maxCnt = resultMap[result] 66 | maxResult = result 67 | } 68 | } 69 | 70 | if maxCnt > c.f { 71 | // accept Reply 72 | c.acceptReply(timestamp, maxResult) 73 | } 74 | } 75 | 76 | func (c *Client) acceptReply(timestamp int64, result string) { 77 | if c.requests[timestamp] == "" { 78 | return 79 | } 80 | 81 | // output the result 82 | command := c.requests[timestamp] 83 | msg := fmt.Sprintf("Client [%d]: Command[%s] got Result[%s]\n", c.me, command, result) 84 | c.debugPrint(msg) 85 | 86 | delete(c.requests, timestamp) 87 | delete(c.replies, timestamp) 88 | } 89 | 90 | func (c *Client) debugPrint(msg string) { 91 | c.debugCh <- msg 92 | } 93 | 94 | func MakeClient(id int, peers []peerWrapper, ch chan interface{}) *Client { 95 | c := &Client{} 96 | c.mu = &sync.Mutex{} 97 | c.me = id 98 | c.peers = peers 99 | c.requests = make(map[int64]string) 100 | c.replies = make(map[int64]map[int]string) 101 | c.debugCh = ch 102 | c.n = len(c.peers) 103 | c.f = (c.n - 1) / 3 104 | 105 | return c 106 | } 107 | -------------------------------------------------------------------------------- /debug.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "net" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | type IDebugServer interface { 13 | handleConnArgs(net.Conn, []string) 14 | } 15 | 16 | type DebugServerBase struct { 17 | addr string 18 | tcpl net.Listener 19 | clients map[string]net.Conn 20 | notifyCh chan interface{} 21 | } 22 | 23 | func (ds *DebugServerBase) getNotifyMsg() { 24 | for true { 25 | msg := <-ds.notifyCh 26 | for _, conn := range ds.clients { 27 | conn.Write([]byte(msg.(string))) 28 | } 29 | } 30 | } 31 | 32 | func (ds *DebugServerBase) readConnData(ids IDebugServer, conn net.Conn) { 33 | buf := make([]byte, 4096) 34 | for { 35 | cnt, err := conn.Read(buf) 36 | if err != nil { 37 | return 38 | } 39 | msg := string(buf[:cnt]) 40 | args := strings.Fields(msg) 41 | if len(args) > 0 { 42 | ids.handleConnArgs(conn, args) 43 | } 44 | } 45 | } 46 | 47 | func (ds *DebugServerBase) run(ids IDebugServer, wg *sync.WaitGroup) { 48 | go ds.getNotifyMsg() 49 | for true { 50 | tcpConn, err := ds.tcpl.Accept() 51 | if err != nil { 52 | break 53 | } 54 | ds.clients[tcpConn.RemoteAddr().String()] = tcpConn 55 | go ds.readConnData(ids, tcpConn) 56 | } 57 | wg.Done() 58 | } 59 | 60 | type HotStuffDebugServer struct { 61 | DebugServerBase 62 | hotStuffServer *HotStuff 63 | } 64 | 65 | func (hds *HotStuffDebugServer) handlePrint(conn net.Conn) { 66 | info := hds.hotStuffServer.getServerInfo() 67 | msg := fmt.Sprintf(` 68 | HotStuff Server State: 69 | id: %d 70 | n: %d 71 | viewId %d 72 | gQC %d 73 | %s 74 | lQC %d 75 | %s 76 | `, info["id"].(int), info["n"].(int), info["viewId"].(int), 77 | info["genericQCView"].(int), info["genericQCId"].(string), 78 | info["lockedQCView"].(int), info["lockedQCId"].(string)) 79 | conn.Write([]byte(msg)) 80 | } 81 | 82 | func (hds *HotStuffDebugServer) handleMaliciousBehavior(conn net.Conn, args []string) { 83 | if len(args) < 2 { 84 | conn.Write([]byte("Arguments not enough\n")) 85 | return 86 | } 87 | 88 | mbmode, err := strconv.Atoi(args[1]) 89 | if err != nil { 90 | conn.Write([]byte("Invalid malicious mode\n")) 91 | return 92 | } 93 | 94 | err = hds.hotStuffServer.setMaliciousMode(mbmode) 95 | if err != nil { 96 | conn.Write([]byte(err.Error() + "\n")) 97 | return 98 | } 99 | 100 | conn.Write([]byte(fmt.Sprintf("malicious behavior set. mode[%d]\n", mbmode))) 101 | } 102 | 103 | func (hds *HotStuffDebugServer) handleNodes(conn net.Conn) { 104 | msg := hds.hotStuffServer.getRecentNodesWithLock() 105 | conn.Write([]byte(msg)) 106 | } 107 | 108 | func (hds *HotStuffDebugServer) handleConnArgs(conn net.Conn, args []string) { 109 | switch args[0] { 110 | case "mb": 111 | hds.handleMaliciousBehavior(conn, args) 112 | case "kill": 113 | conn.Write([]byte("Kill Server...\n")) 114 | conn.Close() 115 | hds.tcpl.Close() 116 | case "print": 117 | hds.handlePrint(conn) 118 | case "nodes": 119 | hds.handleNodes(conn) 120 | case "quit": 121 | conn.Write([]byte("Bye!\n")) 122 | case "echo": 123 | msg := strings.Join(args[1:], " ") 124 | conn.Write([]byte(msg)) 125 | } 126 | } 127 | 128 | func MakeHotStuffDebugServer(addr string, ch chan interface{}, hotStuff *HotStuff, wg *sync.WaitGroup) *HotStuffDebugServer { 129 | tcpListener, err := net.Listen("tcp", addr) 130 | if err != nil { 131 | log.Fatal("debug server listen error:", addr, err) 132 | } 133 | 134 | hds := &HotStuffDebugServer{} 135 | hds.tcpl = tcpListener 136 | hds.addr = addr 137 | hds.clients = make(map[string]net.Conn) 138 | hds.notifyCh = ch 139 | hds.hotStuffServer = hotStuff 140 | wg.Add(1) 141 | go hds.run(hds, wg) 142 | return hds 143 | } 144 | 145 | type ClientDebugServer struct { 146 | DebugServerBase 147 | clientServer *Client 148 | } 149 | 150 | func (cds *ClientDebugServer) handleRequest(conn net.Conn, args []string) { 151 | request := strings.Join(args[1:], " ") 152 | cds.clientServer.newRequest(request) 153 | reply := fmt.Sprintf("Request[%s] sent.\n", request) 154 | conn.Write([]byte(reply)) 155 | } 156 | 157 | func (cds *ClientDebugServer) handleConnArgs(conn net.Conn, args []string) { 158 | switch args[0] { 159 | case "req": 160 | cds.handleRequest(conn, args) 161 | case "kill": 162 | conn.Write([]byte("Kill Server...\n")) 163 | conn.Close() 164 | cds.tcpl.Close() 165 | case "print": 166 | // handlePrint() 167 | case "quit": 168 | conn.Write([]byte("Bye!\n")) 169 | conn.Close() 170 | case "echo": 171 | msg := strings.Join(args[1:], " ") 172 | conn.Write([]byte(msg)) 173 | } 174 | } 175 | 176 | func MakeClientDebugServer(addr string, ch chan interface{}, client *Client, wg *sync.WaitGroup) *ClientDebugServer { 177 | tcpListener, err := net.Listen("tcp", addr) 178 | if err != nil { 179 | log.Fatal("clientSrv listen error:", addr, err) 180 | } 181 | cds := &ClientDebugServer{} 182 | cds.tcpl = tcpListener 183 | cds.clients = make(map[string]net.Conn) 184 | cds.addr = addr 185 | cds.notifyCh = ch 186 | cds.clientServer = client 187 | wg.Add(1) 188 | go cds.run(cds, wg) 189 | return cds 190 | } 191 | -------------------------------------------------------------------------------- /hotstuff.go: -------------------------------------------------------------------------------- 1 | package hotstuff 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | const ViewTimeOut = 15000 10 | const NoopTimeOut = 4000 11 | 12 | type HotStuff struct { 13 | mu *sync.Mutex 14 | servers []peerWrapper 15 | clients []peerWrapper 16 | request *RequestArgs 17 | n int 18 | f int 19 | me int 20 | viewId int 21 | nodeMap map[string]*LogNode 22 | lastNode *LogNode 23 | genericQC QC 24 | lockedQC QC 25 | savedMsgs map[int]*MsgArgs 26 | viewTimer *TimerWithCancel 27 | noopTimer *TimerWithCancel 28 | maliciousMode MaliciousBehaviorMode 29 | 30 | debugCh chan interface{} 31 | } 32 | 33 | func (hs *HotStuff) isLeader() bool { 34 | return hs.me == hs.viewId%hs.n 35 | } 36 | 37 | func (hs *HotStuff) isNextLeader() bool { 38 | return hs.me == (hs.viewId+1)%hs.n 39 | } 40 | 41 | func (hs *HotStuff) getLeader(viewId int) peerWrapper { 42 | id := viewId % hs.n 43 | return hs.servers[id] 44 | } 45 | 46 | func (hs *HotStuff) broadcast(rpcname string, rpcargs interface{}) { 47 | for id := range hs.servers { 48 | hs.sendMsg(id, rpcname, rpcargs) 49 | } 50 | } 51 | 52 | func (hs *HotStuff) sendMsg(id int, rpcname string, rpcacgs interface{}) { 53 | switch hs.maliciousMode { 54 | case NormalMode: 55 | hs.rawSendMsg(id, rpcname, rpcacgs) 56 | case CrashedLike: 57 | return 58 | case MaliciousMode: 59 | hs.sendMaliciousMsg(id, rpcname, rpcacgs, false) 60 | case PartialMaliciousMode: 61 | hs.sendMaliciousMsg(id, rpcname, rpcacgs, true) 62 | } 63 | } 64 | 65 | func (hs *HotStuff) rawSendMsg(id int, rpcname string, rpcacgs interface{}) { 66 | p := hs.servers[id] 67 | reply := &DefaultReply{} 68 | go p.Call("HotStuff."+rpcname, rpcacgs, reply) 69 | } 70 | 71 | func (hs *HotStuff) replyClient(clientId int, replyArgs *ReplyArgs) { 72 | client := hs.clients[clientId] 73 | defaultReply := &DefaultReply{} 74 | go client.Call("Client.Reply", replyArgs, defaultReply) 75 | } 76 | 77 | func (hs *HotStuff) debugPrint(msg string) { 78 | hs.debugCh <- msg 79 | } 80 | 81 | func (hs *HotStuff) processClientRequest(request *RequestArgs) { 82 | curProposal := hs.createLeaf(hs.genericQC.NodeId, request, hs.genericQC) 83 | genericMsg := &MsgArgs{} 84 | genericMsg.RepId = hs.me 85 | genericMsg.ViewId = hs.viewId 86 | genericMsg.Node = *curProposal 87 | hs.broadcast("Msg", genericMsg) 88 | } 89 | 90 | func (hs *HotStuff) createLeaf(parent string, request *RequestArgs, qc QC) *LogNode { 91 | if request == nil { 92 | return nil 93 | } 94 | 95 | parentNode, ok := hs.nodeMap[parent] 96 | if ok { 97 | tmpView := parentNode.ViewId + 1 98 | for tmpView < hs.viewId { 99 | dummyNode := &LogNode{} 100 | dummyNode.ViewId = tmpView 101 | dummyNode.Parent = parent 102 | dummyNode.Request = RequestArgs{} 103 | dummyNode.Request.Operation = "dummy" 104 | dummyNode.Id = getLogNodeId(dummyNode.ViewId, &dummyNode.Request) 105 | dummyNode.Justify = QC{} 106 | hs.nodeMap[dummyNode.Id] = dummyNode 107 | parent = dummyNode.Id 108 | tmpView++ 109 | } 110 | } 111 | 112 | node := &LogNode{} 113 | node.ViewId = hs.viewId 114 | node.Parent = parent 115 | node.Request = *request 116 | node.Id = getLogNodeId(hs.viewId, request) 117 | node.Justify = qc 118 | 119 | hs.saveNode(node) 120 | msg := fmt.Sprintf("\033[0;32mCreate Leaf:\033[0m id[%s] parent[%s] view[%d] op[%s]\n", node.Id, node.Parent, node.ViewId, node.Request.Operation.(string)) 121 | hs.debugPrint(msg) 122 | return node 123 | } 124 | 125 | func (hs *HotStuff) safeNode(n *LogNode, qc QC) bool { 126 | for n != nil { 127 | if n.Parent == hs.lockedQC.NodeId { 128 | return true 129 | } 130 | n = hs.nodeMap[n.Parent] 131 | } 132 | if qc.ViewId > hs.lockedQC.ViewId { 133 | return true 134 | } 135 | return false 136 | } 137 | 138 | func (hs *HotStuff) saveNode(n *LogNode) { 139 | hs.lastNode = n 140 | hs.nodeMap[n.Id] = n 141 | } 142 | 143 | func (hs *HotStuff) update(n *LogNode) { 144 | var prepare, precommit, commit, decide *LogNode 145 | var nodeId string 146 | prepare = n 147 | if prepare != nil { 148 | nodeId = prepare.Justify.NodeId 149 | precommit = hs.nodeMap[nodeId] 150 | } 151 | if precommit != nil { 152 | nodeId = precommit.Justify.NodeId 153 | commit = hs.nodeMap[nodeId] 154 | } 155 | if commit != nil { 156 | nodeId = commit.Justify.NodeId 157 | decide = hs.nodeMap[nodeId] 158 | } 159 | 160 | if hs.safeNode(prepare, prepare.Justify) { 161 | // node saved 162 | msg := fmt.Sprintf("\033[1;32mLogNode saved:\033[0m id[%s] qcId[%s] qcview[%d] \n", n.Id, n.Justify.NodeId, n.Justify.ViewId) 163 | hs.debugPrint(msg) 164 | 165 | hs.saveNode(n) 166 | voteMsg := &MsgArgs{} 167 | voteMsg.RepId = hs.me 168 | voteMsg.ViewId = hs.viewId 169 | voteMsg.Node = *prepare 170 | voteMsg.ParSig = true 171 | nextLeaderId := (hs.viewId + 1) % hs.n 172 | hs.sendMsg(nextLeaderId, "Msg", voteMsg) 173 | } else { 174 | return 175 | } 176 | 177 | if prepare != nil && precommit != nil && prepare.Parent == precommit.Id { 178 | hs.genericQC = prepare.Justify 179 | if precommit != nil && commit != nil && precommit.Parent == commit.Id { 180 | hs.lockedQC = precommit.Justify 181 | if commit != nil && decide != nil && commit.Parent == decide.Id { 182 | //execute decide 183 | msg := fmt.Sprintf("\033[1;34mExecute Request:\033[0m id[%s], op[%s]\n", decide.Id, decide.Request.Operation.(string)) 184 | hs.debugPrint(msg) 185 | request := decide.Request 186 | if request.Timestamp != 0 { 187 | reply := &ReplyArgs{} 188 | reply.ViewId = hs.viewId 189 | reply.Timestamp = request.Timestamp 190 | reply.ReplicaId = hs.me 191 | reply.Result = request.Operation 192 | hs.replyClient(request.ClientId, reply) 193 | } 194 | } 195 | } 196 | } 197 | } 198 | 199 | func (hs *HotStuff) processSavedMsgs() { 200 | cnt := 0 201 | for _, msg := range hs.savedMsgs { 202 | if msg.ViewId == hs.viewId { 203 | cnt++ 204 | } 205 | } 206 | 207 | if cnt >= hs.n-hs.f { 208 | checkVoteMap := make(map[string]int) 209 | // try to find genericQC (get consensus) 210 | for _, msg := range hs.savedMsgs { 211 | if msg.Node.Id != "" { 212 | node := msg.Node 213 | checkVoteMap[node.Id]++ 214 | if checkVoteMap[node.Id] > hs.f { 215 | // get valid consensus 216 | newQc := QC{} 217 | newQc.ViewId = node.ViewId 218 | newQc.NodeId = node.Id 219 | hs.genericQC = newQc 220 | } 221 | } 222 | } 223 | // if cannot find genericQC, it will generate a dummy node 224 | hs.newView(hs.viewId + 1) 225 | } 226 | } 227 | 228 | func (hs *HotStuff) newView(viewId int) { 229 | if hs.viewId >= viewId { 230 | return 231 | } 232 | 233 | msg := hs.getRecentNodes() 234 | hs.debugPrint(msg) 235 | 236 | msg = fmt.Sprintf("=== HotStuff: Change to NewView[%d] ===\n", viewId) 237 | hs.debugPrint(msg) 238 | if hs.viewTimer != nil { 239 | hs.viewTimer.Cancel() 240 | hs.viewTimer = nil 241 | } 242 | 243 | if hs.noopTimer != nil { 244 | hs.noopTimer.Cancel() 245 | hs.noopTimer = nil 246 | } 247 | 248 | hs.viewId = viewId 249 | if hs.isLeader() { 250 | var highNode *LogNode 251 | for _, msg := range hs.savedMsgs { 252 | if msg.QC.NodeId != "" { 253 | node, ok := hs.nodeMap[msg.QC.NodeId] 254 | if ok { 255 | if highNode == nil || node.ViewId > highNode.ViewId { 256 | highNode = node 257 | } 258 | } 259 | } 260 | } 261 | 262 | if highNode != nil { 263 | highQC := highNode.Justify 264 | if highQC.ViewId > hs.genericQC.ViewId { 265 | hs.genericQC = highQC 266 | } 267 | } 268 | 269 | hs.noopTimer = NewTimerWithCancel(time.Duration(NoopTimeOut * time.Millisecond)) 270 | hs.noopTimer.SetTimeout(func() { 271 | hs.mu.Lock() 272 | defer hs.mu.Unlock() 273 | hs.noopTimer = nil 274 | 275 | noopRequset := &RequestArgs{} 276 | noopRequset.Operation = "noop" 277 | hs.processClientRequest(noopRequset) 278 | }) 279 | hs.noopTimer.Start() 280 | } 281 | 282 | hs.viewTimer = NewTimerWithCancel(time.Duration(ViewTimeOut * time.Millisecond)) 283 | hs.viewTimer.SetTimeout(func() { 284 | hs.mu.Lock() 285 | defer hs.mu.Unlock() 286 | hs.viewTimer = nil 287 | msg := fmt.Sprintf("\033[1;33mNewView timeout:\033[0m rep[%d] oldview[%d]\n", hs.me, hs.viewId) 288 | hs.debugPrint(msg) 289 | 290 | newViewMsg := &MsgArgs{} 291 | newViewMsg.ViewId = hs.viewId 292 | newViewMsg.RepId = hs.me 293 | newViewMsg.QC = hs.genericQC 294 | newViewMsg.ParSig = true 295 | nextLeaderId := (hs.viewId + 1) % hs.n 296 | hs.sendMsg(nextLeaderId, "Msg", newViewMsg) 297 | hs.newView(hs.viewId + 1) 298 | }) 299 | hs.viewTimer.Start() 300 | } 301 | 302 | func (hs *HotStuff) getServerInfo() map[string]interface{} { 303 | hs.mu.Lock() 304 | defer hs.mu.Unlock() 305 | 306 | info := make(map[string]interface{}) 307 | info["id"] = hs.me 308 | info["viewId"] = hs.viewId 309 | info["n"] = hs.n 310 | info["f"] = hs.f 311 | info["genericQCId"] = hs.genericQC.NodeId 312 | info["genericQCView"] = hs.genericQC.ViewId 313 | info["lockedQCId"] = hs.lockedQC.NodeId 314 | info["lockedQCView"] = hs.lockedQC.ViewId 315 | return info 316 | } 317 | 318 | func (hs *HotStuff) getRecentNodesWithLock() string { 319 | hs.mu.Lock() 320 | defer hs.mu.Unlock() 321 | return hs.getRecentNodes() 322 | } 323 | 324 | func (hs *HotStuff) getRecentNodes() string { 325 | node := hs.nodeMap[hs.genericQC.NodeId] 326 | msg := "Recent valid nodes: \n" 327 | for i := 0; i < 5; i++ { 328 | if node == nil { 329 | return msg 330 | } 331 | 332 | msg += fmt.Sprintf(" nodeId[%s] view[%d] parent[%s] qc[%s]\n", node.Id, node.ViewId, node.Parent, node.Justify.NodeId) 333 | qcId := node.Justify.NodeId 334 | node = hs.nodeMap[qcId] 335 | } 336 | 337 | return msg 338 | } 339 | 340 | func MakeHotStuff(id int, serverPeers, clientPeers []peerWrapper, debugCh chan interface{}) *HotStuff { 341 | hs := &HotStuff{} 342 | hs.mu = &sync.Mutex{} 343 | hs.me = id 344 | hs.servers = serverPeers 345 | hs.clients = clientPeers 346 | hs.viewId = 0 347 | hs.nodeMap = make(map[string]*LogNode) 348 | hs.n = len(hs.servers) 349 | hs.f = (hs.n - 1) / 3 350 | hs.savedMsgs = make(map[int]*MsgArgs) 351 | hs.maliciousMode = NormalMode 352 | go hs.newView(1) 353 | 354 | hs.debugCh = debugCh 355 | return hs 356 | } 357 | --------------------------------------------------------------------------------