├── .gitignore ├── LICENSE ├── README.md ├── asset └── run.png └── src ├── labrpc ├── labrpc.go └── test_test.go └── raft ├── config.go ├── persister.go ├── raft.go ├── test_test.go └── util.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Xingkai Yu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NJU-DisSys 2 | 3 | 南京大学分布式系统Raft算法，本项目力求最简洁和最直观的实现，完整代码以及必要注释不超过350行，并能通过500小时循环测试。 4 | 5 | 代码将在实验截止后开源，README已经给出大致框架，只需在注释中补充[论文](https://raft.github.io/raft.pdf)第四页内容即可。 6 | 7 | ## 运行说明 8 | 9 | ![](asset/run.png) 10 | 11 | ## 数据结构 12 | 13 | `Raft`结构在论文基础上添加了如下成员： 14 | 15 | ```go 16 | type Raft struct { 17 | // fields in the paper 18 | role int 19 | votes int 20 | timer *time.Timer 21 | } 22 | ``` 23 | 24 | 其他结构与论文描述一致。 25 | 26 | 一些常量： 27 | 28 | ```go 29 | const ( 30 | FOLLOWER = 0 31 | CANDIDATE = 1 32 | LEADER = 2 33 | ) 34 | 35 | const ( 36 | electionTimeoutMin = 300 * time.Millisecond 37 | electionTimeoutMax = 600 * time.Millisecond 38 | heartbeatTimeout = 50 * time.Millisecond 39 | checkTimeout = 5 * time.Millisecond 40 | ) 41 | ``` 42 | 43 | 两个实用函数： 44 | ```go 45 | func randTime() time.Duration { 46 | diff := (electionTimeoutMax - electionTimeoutMin).Milliseconds() 47 | return electionTimeoutMin + time.Duration(rand.Intn(int(diff)))*time.Millisecond 48 | } 49 | 50 | func wait(n int, ch chan bool) { 51 | for i := 1; i < n; i++ { 52 | select { 53 | case <-ch: 54 | case <-time.After(checkTimeout): 55 | return 56 | } 57 | } 58 | } 59 | ``` 60 | 61 | ## 模块设计 62 | 63 | Raft算法包含两种RPC，本项目将两种处理流程尽可能统一。 64 | 65 | 定义`RPC`（具体是`RequestVote`或`AppendEntries`）相关结构： 66 | 67 | ```go 68 | type RPCArgs struct { 69 | // fields in the paper 70 | } 71 | 72 | type RPCReply struct { 73 | // fields in the paper 74 | } 75 | ``` 76 | 77 | 接收者响应逻辑： 78 | 79 | ```go 80 | func (rf *Raft) RPC(args RPCArgs, reply *RPCReply) { 81 | // receiver responds to the request 82 | // e.g. convert to follower if outdated 83 | // or vote for candidate 84 | // or update logs 85 | } 86 | ``` 87 | 88 | 发送者单次请求逻辑，包含可以**立即处理响应**的部分逻辑，通过`channel`发送响应成功： 89 | 90 | ```go 91 | func (rf *Raft) sendRPC(server int, args RPCArgs, reply *RPCReply, ch chan bool) { 92 | if !rf.peers[server].Call("Raft.RPC", args, reply) { 93 | return 94 | } 95 | // handle the response immediately 96 | // e.g. convert to follower and return if outdated 97 | // or increase the candidate's votes 98 | // or update the leader's nextIndex 99 | // finally execute ch <- true 100 | } 101 | ``` 102 | 103 | 发送者批量请求逻辑，等待所有发收结束或者超时，之后处理剩余事务（竞选成功或日志提交）： 104 | 105 | ```go 106 | ch := make(chan bool) 107 | for i := 0; i < n; i++ { 108 | if i != rf.me { 109 | // construct args and reply 110 | go rf.sendRPC(i, args, &reply, ch) 111 | } 112 | } 113 | 114 | // wait all goroutines go well or time out 115 | wait(n, ch) 116 | 117 | // handle the remaining transactions 118 | // e.g. decide whether to become a leader 119 | // or find the appropriate index to commit 120 | ``` 121 | 122 | 在`Make()`函数中两个无限循环的`goroutine`，一个定时检查日志应用情况，另一个计时器触发对于`FOLLOWER`只需变为`CANDIDATE`，对于`CANDIDATE`或`LEADER`执行上面代码逻辑： 123 | 124 | ```go 125 | go func() { 126 | for rf.me != -1 { 127 | time.Sleep(checkTimeout) 128 | // check whether to apply log to state machine 129 | } 130 | }() 131 | 132 | go func() { 133 | for rf.me != -1 { 134 | <-rf.timer.C 135 | switch rf.role { 136 | case FOLLOWER: 137 | // become candidate 138 | case CANDIDATE: 139 | // run for election 140 | case LEADER: 141 | // manage log replication 142 | } 143 | } 144 | }() 145 | ``` 146 | 147 | 在`Kill()`函数中执行`rf.me = -1`结束上面两个`goroutine`。 148 | 149 | ## 注意事项 150 | 151 | 1. 并发执行数据访问常加锁 152 | 153 | 2. 修改非易失成员立即持久化 154 | 155 | ## 鲁棒测试 156 | 157 | 成功通过一次测试并不是终点，多次运行仍然可能出错，可以在`shell`中定义如下函数： 158 | 159 | ```shell 160 | run() { 161 | go test 162 | while [[ $? -ne 1 ]] 163 | do 164 | go test 165 | done 166 | } 167 | ``` 168 | 169 | 执行`run`命令无限测试直至失败，本项目通过了500小时的循环测试。 170 | -------------------------------------------------------------------------------- /asset/run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GeeeekExplorer/NJU-DisSys/4ae79144e586d772f9801e324a24853dc2c8fd64/asset/run.png -------------------------------------------------------------------------------- /src/labrpc/labrpc.go: -------------------------------------------------------------------------------- 1 | package labrpc 2 | 3 | // 4 | // channel-based RPC, for 824 labs. 5 | // allows tests to disconnect RPC connections. 6 | // 7 | // we will use the original labrpc.go to test your code for grading. 8 | // so, while you can modify this code to help you debug, please 9 | // test against the original before submitting. 10 | // 11 | // adapted from Go net/rpc/server.go. 12 | // 13 | // sends gob-encoded values to ensure that RPCs 14 | // don't include references to program objects. 15 | // 16 | // net := MakeNetwork() -- holds network, clients, servers. 17 | // end := net.MakeEnd(endname) -- create a client end-point, to talk to one server. 18 | // net.AddServer(servername, server) -- adds a named server to network. 19 | // net.DeleteServer(servername) -- eliminate the named server. 20 | // net.Connect(endname, servername) -- connect a client to a server. 21 | // net.Enable(endname, enabled) -- enable/disable a client. 22 | // net.Reliable(bool) -- false means drop/delay messages 23 | // 24 | // end.Call("Raft.AppendEntries", &args, &reply) -- send an RPC, wait for reply. 25 | // the "Raft" is the name of the server struct to be called. 26 | // the "AppendEntries" is the name of the method to be called. 27 | // Call() returns true to indicate that the server executed the request 28 | // and the reply is valid. 29 | // Call() returns false if the network lost the request or reply 30 | // or the server is down. 31 | // It is OK to have multiple Call()s in progress at the same time on the 32 | // same ClientEnd. 33 | // Concurrent calls to Call() may be delivered to the server out of order, 34 | // since the network may re-order messages. 35 | // Call() is guaranteed to return (perhaps after a delay) *except* if the 36 | // handler function on the server side does not return. That is, there 37 | // is no need to implement your own timeouts around Call(). 38 | // the server RPC handler function must declare its args and reply arguments 39 | // as pointers, so that their types exactly match the types of the arguments 40 | // to Call(). 41 | // 42 | // srv := MakeServer() 43 | // srv.AddService(svc) -- a server can have multiple services, e.g. Raft and k/v 44 | // pass srv to net.AddServer() 45 | // 46 | // svc := MakeService(receiverObject) -- obj's methods will handle RPCs 47 | // much like Go's rpcs.Register() 48 | // pass svc to srv.AddService() 49 | // 50 | 51 | import "encoding/gob" 52 | import "bytes" 53 | import "reflect" 54 | import "sync" 55 | import "log" 56 | import "strings" 57 | import "math/rand" 58 | import "time" 59 | 60 | type reqMsg struct { 61 | endname interface{} // name of sending ClientEnd 62 | svcMeth string // e.g. "Raft.AppendEntries" 63 | argsType reflect.Type 64 | args []byte 65 | replyCh chan replyMsg 66 | } 67 | 68 | type replyMsg struct { 69 | ok bool 70 | reply []byte 71 | } 72 | 73 | type ClientEnd struct { 74 | endname interface{} // this end-point's name 75 | ch chan reqMsg // copy of Network.endCh 76 | } 77 | 78 | // send an RPC, wait for the reply. 79 | // the return value indicates success; false means the 80 | // server couldn't be contacted. 81 | func (e *ClientEnd) Call(svcMeth string, args interface{}, reply interface{}) bool { 82 | req := reqMsg{} 83 | req.endname = e.endname 84 | req.svcMeth = svcMeth 85 | req.argsType = reflect.TypeOf(args) 86 | req.replyCh = make(chan replyMsg) 87 | 88 | qb := new(bytes.Buffer) 89 | qe := gob.NewEncoder(qb) 90 | qe.Encode(args) 91 | req.args = qb.Bytes() 92 | 93 | e.ch <- req 94 | 95 | rep := <-req.replyCh 96 | if rep.ok { 97 | rb := bytes.NewBuffer(rep.reply) 98 | rd := gob.NewDecoder(rb) 99 | if err := rd.Decode(reply); err != nil { 100 | log.Fatalf("ClientEnd.Call(): decode reply: %v\n", err) 101 | } 102 | return true 103 | } else { 104 | return false 105 | } 106 | } 107 | 108 | type Network struct { 109 | mu sync.Mutex 110 | reliable bool 111 | longDelays bool // pause a long time on send on disabled connection 112 | longReordering bool // sometimes delay replies a long time 113 | ends map[interface{}]*ClientEnd // ends, by name 114 | enabled map[interface{}]bool // by end name 115 | servers map[interface{}]*Server // servers, by name 116 | connections map[interface{}]interface{} // endname -> servername 117 | endCh chan reqMsg 118 | } 119 | 120 | func MakeNetwork() *Network { 121 | rn := &Network{} 122 | rn.reliable = true 123 | rn.ends = map[interface{}]*ClientEnd{} 124 | rn.enabled = map[interface{}]bool{} 125 | rn.servers = map[interface{}]*Server{} 126 | rn.connections = map[interface{}](interface{}){} 127 | rn.endCh = make(chan reqMsg) 128 | 129 | // single goroutine to handle all ClientEnd.Call()s 130 | go func() { 131 | for xreq := range rn.endCh { 132 | go rn.ProcessReq(xreq) 133 | } 134 | }() 135 | 136 | return rn 137 | } 138 | 139 | func (rn *Network) Reliable(yes bool) { 140 | rn.mu.Lock() 141 | defer rn.mu.Unlock() 142 | 143 | rn.reliable = yes 144 | } 145 | 146 | func (rn *Network) LongReordering(yes bool) { 147 | rn.mu.Lock() 148 | defer rn.mu.Unlock() 149 | 150 | rn.longReordering = yes 151 | } 152 | 153 | func (rn *Network) LongDelays(yes bool) { 154 | rn.mu.Lock() 155 | defer rn.mu.Unlock() 156 | 157 | rn.longDelays = yes 158 | } 159 | 160 | func (rn *Network) ReadEndnameInfo(endname interface{}) (enabled bool, 161 | servername interface{}, server *Server, reliable bool, longreordering bool, 162 | ) { 163 | rn.mu.Lock() 164 | defer rn.mu.Unlock() 165 | 166 | enabled = rn.enabled[endname] 167 | servername = rn.connections[endname] 168 | if servername != nil { 169 | server = rn.servers[servername] 170 | } 171 | reliable = rn.reliable 172 | longreordering = rn.longReordering 173 | return 174 | } 175 | 176 | func (rn *Network) IsServerDead(endname interface{}, servername interface{}, server *Server) bool { 177 | rn.mu.Lock() 178 | defer rn.mu.Unlock() 179 | 180 | if rn.enabled[endname] == false || rn.servers[servername] != server { 181 | return true 182 | } 183 | return false 184 | } 185 | 186 | func (rn *Network) ProcessReq(req reqMsg) { 187 | enabled, servername, server, reliable, longreordering := rn.ReadEndnameInfo(req.endname) 188 | 189 | if enabled && servername != nil && server != nil { 190 | if reliable == false { 191 | // short delay 192 | ms := (rand.Int() % 27) 193 | time.Sleep(time.Duration(ms) * time.Millisecond) 194 | } 195 | 196 | if reliable == false && (rand.Int()%1000) < 100 { 197 | // drop the request, return as if timeout 198 | req.replyCh <- replyMsg{false, nil} 199 | return 200 | } 201 | 202 | // execute the request (call the RPC handler). 203 | // in a separate thread so that we can periodically check 204 | // if the server has been killed and the RPC should get a 205 | // failure reply. 206 | ech := make(chan replyMsg) 207 | go func() { 208 | r := server.dispatch(req) 209 | ech <- r 210 | }() 211 | 212 | // wait for handler to return, 213 | // but stop waiting if DeleteServer() has been called, 214 | // and return an error. 215 | var reply replyMsg 216 | replyOK := false 217 | serverDead := false 218 | for replyOK == false && serverDead == false { 219 | select { 220 | case reply = <-ech: 221 | replyOK = true 222 | case <-time.After(100 * time.Millisecond): 223 | serverDead = rn.IsServerDead(req.endname, servername, server) 224 | } 225 | } 226 | 227 | // do not reply if DeleteServer() has been called, i.e. 228 | // the server has been killed. this is needed to avoid 229 | // situation in which a client gets a positive reply 230 | // to an Append, but the server persisted the update 231 | // into the old Persister. config.go is careful to call 232 | // DeleteServer() before superseding the Persister. 233 | serverDead = rn.IsServerDead(req.endname, servername, server) 234 | 235 | if replyOK == false || serverDead == true { 236 | // server was killed while we were waiting; return error. 237 | req.replyCh <- replyMsg{false, nil} 238 | } else if reliable == false && (rand.Int()%1000) < 100 { 239 | // drop the reply, return as if timeout 240 | req.replyCh <- replyMsg{false, nil} 241 | } else if longreordering == true && rand.Intn(900) < 600 { 242 | // delay the response for a while 243 | ms := 200 + rand.Intn(1+rand.Intn(2000)) 244 | time.Sleep(time.Duration(ms) * time.Millisecond) 245 | req.replyCh <- reply 246 | } else { 247 | req.replyCh <- reply 248 | } 249 | } else { 250 | // simulate no reply and eventual timeout. 251 | ms := 0 252 | if rn.longDelays { 253 | // let Raft tests check that leader doesn't send 254 | // RPCs synchronously. 255 | ms = (rand.Int() % 7000) 256 | } else { 257 | // many kv tests require the client to try each 258 | // server in fairly rapid succession. 259 | ms = (rand.Int() % 100) 260 | } 261 | time.Sleep(time.Duration(ms) * time.Millisecond) 262 | req.replyCh <- replyMsg{false, nil} 263 | } 264 | 265 | } 266 | 267 | // create a client end-point. 268 | // start the thread that listens and delivers. 269 | func (rn *Network) MakeEnd(endname interface{}) *ClientEnd { 270 | rn.mu.Lock() 271 | defer rn.mu.Unlock() 272 | 273 | if _, ok := rn.ends[endname]; ok { 274 | log.Fatalf("MakeEnd: %v already exists\n", endname) 275 | } 276 | 277 | e := &ClientEnd{} 278 | e.endname = endname 279 | e.ch = rn.endCh 280 | rn.ends[endname] = e 281 | rn.enabled[endname] = false 282 | rn.connections[endname] = nil 283 | 284 | return e 285 | } 286 | 287 | func (rn *Network) AddServer(servername interface{}, rs *Server) { 288 | rn.mu.Lock() 289 | defer rn.mu.Unlock() 290 | 291 | rn.servers[servername] = rs 292 | } 293 | 294 | func (rn *Network) DeleteServer(servername interface{}) { 295 | rn.mu.Lock() 296 | defer rn.mu.Unlock() 297 | 298 | rn.servers[servername] = nil 299 | } 300 | 301 | // connect a ClientEnd to a server. 302 | // a ClientEnd can only be connected once in its lifetime. 303 | func (rn *Network) Connect(endname interface{}, servername interface{}) { 304 | rn.mu.Lock() 305 | defer rn.mu.Unlock() 306 | 307 | rn.connections[endname] = servername 308 | } 309 | 310 | // enable/disable a ClientEnd. 311 | func (rn *Network) Enable(endname interface{}, enabled bool) { 312 | rn.mu.Lock() 313 | defer rn.mu.Unlock() 314 | 315 | rn.enabled[endname] = enabled 316 | } 317 | 318 | // get a server's count of incoming RPCs. 319 | func (rn *Network) GetCount(servername interface{}) int { 320 | rn.mu.Lock() 321 | defer rn.mu.Unlock() 322 | 323 | svr := rn.servers[servername] 324 | return svr.GetCount() 325 | } 326 | 327 | // 328 | // a server is a collection of services, all sharing 329 | // the same rpc dispatcher. so that e.g. both a Raft 330 | // and a k/v server can listen to the same rpc endpoint. 331 | // 332 | type Server struct { 333 | mu sync.Mutex 334 | services map[string]*Service 335 | count int // incoming RPCs 336 | } 337 | 338 | func MakeServer() *Server { 339 | rs := &Server{} 340 | rs.services = map[string]*Service{} 341 | return rs 342 | } 343 | 344 | func (rs *Server) AddService(svc *Service) { 345 | rs.mu.Lock() 346 | defer rs.mu.Unlock() 347 | rs.services[svc.name] = svc 348 | } 349 | 350 | func (rs *Server) dispatch(req reqMsg) replyMsg { 351 | rs.mu.Lock() 352 | 353 | rs.count += 1 354 | 355 | // split Raft.AppendEntries into service and method 356 | dot := strings.LastIndex(req.svcMeth, ".") 357 | serviceName := req.svcMeth[:dot] 358 | methodName := req.svcMeth[dot+1:] 359 | 360 | service, ok := rs.services[serviceName] 361 | 362 | rs.mu.Unlock() 363 | 364 | if ok { 365 | return service.dispatch(methodName, req) 366 | } else { 367 | choices := []string{} 368 | for k, _ := range rs.services { 369 | choices = append(choices, k) 370 | } 371 | log.Fatalf("labrpc.Server.dispatch(): unknown service %v in %v.%v; expecting one of %v\n", 372 | serviceName, serviceName, methodName, choices) 373 | return replyMsg{false, nil} 374 | } 375 | } 376 | 377 | func (rs *Server) GetCount() int { 378 | rs.mu.Lock() 379 | defer rs.mu.Unlock() 380 | return rs.count 381 | } 382 | 383 | // an object with methods that can be called via RPC. 384 | // a single server may have more than one Service. 385 | type Service struct { 386 | name string 387 | rcvr reflect.Value 388 | typ reflect.Type 389 | methods map[string]reflect.Method 390 | } 391 | 392 | func MakeService(rcvr interface{}) *Service { 393 | svc := &Service{} 394 | svc.typ = reflect.TypeOf(rcvr) 395 | svc.rcvr = reflect.ValueOf(rcvr) 396 | svc.name = reflect.Indirect(svc.rcvr).Type().Name() 397 | svc.methods = map[string]reflect.Method{} 398 | 399 | for m := 0; m < svc.typ.NumMethod(); m++ { 400 | method := svc.typ.Method(m) 401 | mtype := method.Type 402 | mname := method.Name 403 | 404 | //fmt.Printf("%v pp %v ni %v 1k %v 2k %v no %v\n", 405 | // mname, method.PkgPath, mtype.NumIn(), mtype.In(1).Kind(), mtype.In(2).Kind(), mtype.NumOut()) 406 | 407 | if method.PkgPath != "" || // capitalized? 408 | mtype.NumIn() != 3 || 409 | //mtype.In(1).Kind() != reflect.Ptr || 410 | mtype.In(2).Kind() != reflect.Ptr || 411 | mtype.NumOut() != 0 { 412 | // the method is not suitable for a handler 413 | //fmt.Printf("bad method: %v\n", mname) 414 | } else { 415 | // the method looks like a handler 416 | svc.methods[mname] = method 417 | } 418 | } 419 | 420 | return svc 421 | } 422 | 423 | func (svc *Service) dispatch(methname string, req reqMsg) replyMsg { 424 | if method, ok := svc.methods[methname]; ok { 425 | // prepare space into which to read the argument. 426 | // the Value's type will be a pointer to req.argsType. 427 | args := reflect.New(req.argsType) 428 | 429 | // decode the argument. 430 | ab := bytes.NewBuffer(req.args) 431 | ad := gob.NewDecoder(ab) 432 | ad.Decode(args.Interface()) 433 | 434 | // allocate space for the reply. 435 | replyType := method.Type.In(2) 436 | replyType = replyType.Elem() 437 | replyv := reflect.New(replyType) 438 | 439 | // call the method. 440 | function := method.Func 441 | function.Call([]reflect.Value{svc.rcvr, args.Elem(), replyv}) 442 | 443 | // encode the reply. 444 | rb := new(bytes.Buffer) 445 | re := gob.NewEncoder(rb) 446 | re.EncodeValue(replyv) 447 | 448 | return replyMsg{true, rb.Bytes()} 449 | } else { 450 | choices := []string{} 451 | for k, _ := range svc.methods { 452 | choices = append(choices, k) 453 | } 454 | log.Fatalf("labrpc.Service.dispatch(): unknown method %v in %v; expecting one of %v\n", 455 | methname, req.svcMeth, choices) 456 | return replyMsg{false, nil} 457 | } 458 | } 459 | -------------------------------------------------------------------------------- /src/labrpc/test_test.go: -------------------------------------------------------------------------------- 1 | package labrpc 2 | 3 | import "testing" 4 | import "strconv" 5 | import "sync" 6 | import "runtime" 7 | import "time" 8 | import "fmt" 9 | 10 | type JunkArgs struct { 11 | X int 12 | } 13 | type JunkReply struct { 14 | X string 15 | } 16 | 17 | type JunkServer struct { 18 | mu sync.Mutex 19 | log1 []string 20 | log2 []int 21 | } 22 | 23 | func (js *JunkServer) Handler1(args string, reply *int) { 24 | js.mu.Lock() 25 | defer js.mu.Unlock() 26 | js.log1 = append(js.log1, args) 27 | *reply, _ = strconv.Atoi(args) 28 | } 29 | 30 | func (js *JunkServer) Handler2(args int, reply *string) { 31 | js.mu.Lock() 32 | defer js.mu.Unlock() 33 | js.log2 = append(js.log2, args) 34 | *reply = "handler2-" + strconv.Itoa(args) 35 | } 36 | 37 | func (js *JunkServer) Handler3(args int, reply *int) { 38 | js.mu.Lock() 39 | defer js.mu.Unlock() 40 | time.Sleep(20 * time.Second) 41 | *reply = -args 42 | } 43 | 44 | // args is a pointer 45 | func (js *JunkServer) Handler4(args *JunkArgs, reply *JunkReply) { 46 | reply.X = "pointer" 47 | } 48 | 49 | // args is a not pointer 50 | func (js *JunkServer) Handler5(args JunkArgs, reply *JunkReply) { 51 | reply.X = "no pointer" 52 | } 53 | 54 | func TestBasic(t *testing.T) { 55 | runtime.GOMAXPROCS(4) 56 | 57 | rn := MakeNetwork() 58 | 59 | e := rn.MakeEnd("end1-99") 60 | 61 | js := &JunkServer{} 62 | svc := MakeService(js) 63 | 64 | rs := MakeServer() 65 | rs.AddService(svc) 66 | rn.AddServer("server99", rs) 67 | 68 | rn.Connect("end1-99", "server99") 69 | rn.Enable("end1-99", true) 70 | 71 | { 72 | reply := "" 73 | e.Call("JunkServer.Handler2", 111, &reply) 74 | if reply != "handler2-111" { 75 | t.Fatalf("wrong reply from Handler2") 76 | } 77 | } 78 | 79 | { 80 | reply := 0 81 | e.Call("JunkServer.Handler1", "9099", &reply) 82 | if reply != 9099 { 83 | t.Fatalf("wrong reply from Handler1") 84 | } 85 | } 86 | } 87 | 88 | func TestTypes(t *testing.T) { 89 | runtime.GOMAXPROCS(4) 90 | 91 | rn := MakeNetwork() 92 | 93 | e := rn.MakeEnd("end1-99") 94 | 95 | js := &JunkServer{} 96 | svc := MakeService(js) 97 | 98 | rs := MakeServer() 99 | rs.AddService(svc) 100 | rn.AddServer("server99", rs) 101 | 102 | rn.Connect("end1-99", "server99") 103 | rn.Enable("end1-99", true) 104 | 105 | { 106 | var args JunkArgs 107 | var reply JunkReply 108 | // args must match type (pointer or not) of handler. 109 | e.Call("JunkServer.Handler4", &args, &reply) 110 | if reply.X != "pointer" { 111 | t.Fatalf("wrong reply from Handler4") 112 | } 113 | } 114 | 115 | { 116 | var args JunkArgs 117 | var reply JunkReply 118 | // args must match type (pointer or not) of handler. 119 | e.Call("JunkServer.Handler5", args, &reply) 120 | if reply.X != "no pointer" { 121 | t.Fatalf("wrong reply from Handler5") 122 | } 123 | } 124 | } 125 | 126 | // 127 | // does net.Enable(endname, false) really disconnect a client? 128 | // 129 | func TestDisconnect(t *testing.T) { 130 | runtime.GOMAXPROCS(4) 131 | 132 | rn := MakeNetwork() 133 | 134 | e := rn.MakeEnd("end1-99") 135 | 136 | js := &JunkServer{} 137 | svc := MakeService(js) 138 | 139 | rs := MakeServer() 140 | rs.AddService(svc) 141 | rn.AddServer("server99", rs) 142 | 143 | rn.Connect("end1-99", "server99") 144 | 145 | { 146 | reply := "" 147 | e.Call("JunkServer.Handler2", 111, &reply) 148 | if reply != "" { 149 | t.Fatalf("unexpected reply from Handler2") 150 | } 151 | } 152 | 153 | rn.Enable("end1-99", true) 154 | 155 | { 156 | reply := 0 157 | e.Call("JunkServer.Handler1", "9099", &reply) 158 | if reply != 9099 { 159 | t.Fatalf("wrong reply from Handler1") 160 | } 161 | } 162 | } 163 | 164 | // 165 | // test net.GetCount() 166 | // 167 | func TestCounts(t *testing.T) { 168 | runtime.GOMAXPROCS(4) 169 | 170 | rn := MakeNetwork() 171 | 172 | e := rn.MakeEnd("end1-99") 173 | 174 | js := &JunkServer{} 175 | svc := MakeService(js) 176 | 177 | rs := MakeServer() 178 | rs.AddService(svc) 179 | rn.AddServer(99, rs) 180 | 181 | rn.Connect("end1-99", 99) 182 | rn.Enable("end1-99", true) 183 | 184 | for i := 0; i < 17; i++ { 185 | reply := "" 186 | e.Call("JunkServer.Handler2", i, &reply) 187 | wanted := "handler2-" + strconv.Itoa(i) 188 | if reply != wanted { 189 | t.Fatalf("wrong reply %v from Handler1, expecting %v", reply, wanted) 190 | } 191 | } 192 | 193 | n := rn.GetCount(99) 194 | if n != 17 { 195 | t.Fatalf("wrong GetCount() %v, expected 17\n", n) 196 | } 197 | } 198 | 199 | // 200 | // test RPCs from concurrent ClientEnds 201 | // 202 | func TestConcurrentMany(t *testing.T) { 203 | runtime.GOMAXPROCS(4) 204 | 205 | rn := MakeNetwork() 206 | 207 | js := &JunkServer{} 208 | svc := MakeService(js) 209 | 210 | rs := MakeServer() 211 | rs.AddService(svc) 212 | rn.AddServer(1000, rs) 213 | 214 | ch := make(chan int) 215 | 216 | nclients := 20 217 | nrpcs := 10 218 | for ii := 0; ii < nclients; ii++ { 219 | go func(i int) { 220 | n := 0 221 | defer func() { ch <- n }() 222 | 223 | e := rn.MakeEnd(i) 224 | rn.Connect(i, 1000) 225 | rn.Enable(i, true) 226 | 227 | for j := 0; j < nrpcs; j++ { 228 | arg := i*100 + j 229 | reply := "" 230 | e.Call("JunkServer.Handler2", arg, &reply) 231 | wanted := "handler2-" + strconv.Itoa(arg) 232 | if reply != wanted { 233 | t.Fatalf("wrong reply %v from Handler1, expecting %v", reply, wanted) 234 | } 235 | n += 1 236 | } 237 | }(ii) 238 | } 239 | 240 | total := 0 241 | for ii := 0; ii < nclients; ii++ { 242 | x := <-ch 243 | total += x 244 | } 245 | 246 | if total != nclients*nrpcs { 247 | t.Fatalf("wrong number of RPCs completed, got %v, expected %v", total, nclients*nrpcs) 248 | } 249 | 250 | n := rn.GetCount(1000) 251 | if n != total { 252 | t.Fatalf("wrong GetCount() %v, expected %v\n", n, total) 253 | } 254 | } 255 | 256 | // 257 | // test unreliable 258 | // 259 | func TestUnreliable(t *testing.T) { 260 | runtime.GOMAXPROCS(4) 261 | 262 | rn := MakeNetwork() 263 | rn.Reliable(false) 264 | 265 | js := &JunkServer{} 266 | svc := MakeService(js) 267 | 268 | rs := MakeServer() 269 | rs.AddService(svc) 270 | rn.AddServer(1000, rs) 271 | 272 | ch := make(chan int) 273 | 274 | nclients := 300 275 | for ii := 0; ii < nclients; ii++ { 276 | go func(i int) { 277 | n := 0 278 | defer func() { ch <- n }() 279 | 280 | e := rn.MakeEnd(i) 281 | rn.Connect(i, 1000) 282 | rn.Enable(i, true) 283 | 284 | arg := i * 100 285 | reply := "" 286 | ok := e.Call("JunkServer.Handler2", arg, &reply) 287 | if ok { 288 | wanted := "handler2-" + strconv.Itoa(arg) 289 | if reply != wanted { 290 | t.Fatalf("wrong reply %v from Handler1, expecting %v", reply, wanted) 291 | } 292 | n += 1 293 | } 294 | }(ii) 295 | } 296 | 297 | total := 0 298 | for ii := 0; ii < nclients; ii++ { 299 | x := <-ch 300 | total += x 301 | } 302 | 303 | if total == nclients || total == 0 { 304 | t.Fatalf("all RPCs succeeded despite unreliable") 305 | } 306 | } 307 | 308 | // 309 | // test concurrent RPCs from a single ClientEnd 310 | // 311 | func TestConcurrentOne(t *testing.T) { 312 | runtime.GOMAXPROCS(4) 313 | 314 | rn := MakeNetwork() 315 | 316 | js := &JunkServer{} 317 | svc := MakeService(js) 318 | 319 | rs := MakeServer() 320 | rs.AddService(svc) 321 | rn.AddServer(1000, rs) 322 | 323 | e := rn.MakeEnd("c") 324 | rn.Connect("c", 1000) 325 | rn.Enable("c", true) 326 | 327 | ch := make(chan int) 328 | 329 | nrpcs := 20 330 | for ii := 0; ii < nrpcs; ii++ { 331 | go func(i int) { 332 | n := 0 333 | defer func() { ch <- n }() 334 | 335 | arg := 100 + i 336 | reply := "" 337 | e.Call("JunkServer.Handler2", arg, &reply) 338 | wanted := "handler2-" + strconv.Itoa(arg) 339 | if reply != wanted { 340 | t.Fatalf("wrong reply %v from Handler2, expecting %v", reply, wanted) 341 | } 342 | n += 1 343 | }(ii) 344 | } 345 | 346 | total := 0 347 | for ii := 0; ii < nrpcs; ii++ { 348 | x := <-ch 349 | total += x 350 | } 351 | 352 | if total != nrpcs { 353 | t.Fatalf("wrong number of RPCs completed, got %v, expected %v", total, nrpcs) 354 | } 355 | 356 | js.mu.Lock() 357 | defer js.mu.Unlock() 358 | if len(js.log2) != nrpcs { 359 | t.Fatalf("wrong number of RPCs delivered") 360 | } 361 | 362 | n := rn.GetCount(1000) 363 | if n != total { 364 | t.Fatalf("wrong GetCount() %v, expected %v\n", n, total) 365 | } 366 | } 367 | 368 | // 369 | // regression: an RPC that's delayed during Enabled=false 370 | // should not delay subsequent RPCs (e.g. after Enabled=true). 371 | // 372 | func TestRegression1(t *testing.T) { 373 | runtime.GOMAXPROCS(4) 374 | 375 | rn := MakeNetwork() 376 | 377 | js := &JunkServer{} 378 | svc := MakeService(js) 379 | 380 | rs := MakeServer() 381 | rs.AddService(svc) 382 | rn.AddServer(1000, rs) 383 | 384 | e := rn.MakeEnd("c") 385 | rn.Connect("c", 1000) 386 | 387 | // start some RPCs while the ClientEnd is disabled. 388 | // they'll be delayed. 389 | rn.Enable("c", false) 390 | ch := make(chan bool) 391 | nrpcs := 20 392 | for ii := 0; ii < nrpcs; ii++ { 393 | go func(i int) { 394 | ok := false 395 | defer func() { ch <- ok }() 396 | 397 | arg := 100 + i 398 | reply := "" 399 | // this call ought to return false. 400 | e.Call("JunkServer.Handler2", arg, &reply) 401 | ok = true 402 | }(ii) 403 | } 404 | 405 | time.Sleep(100 * time.Millisecond) 406 | 407 | // now enable the ClientEnd and check that an RPC completes quickly. 408 | t0 := time.Now() 409 | rn.Enable("c", true) 410 | { 411 | arg := 99 412 | reply := "" 413 | e.Call("JunkServer.Handler2", arg, &reply) 414 | wanted := "handler2-" + strconv.Itoa(arg) 415 | if reply != wanted { 416 | t.Fatalf("wrong reply %v from Handler2, expecting %v", reply, wanted) 417 | } 418 | } 419 | dur := time.Since(t0).Seconds() 420 | 421 | if dur > 0.03 { 422 | t.Fatalf("RPC took too long (%v) after Enable", dur) 423 | } 424 | 425 | for ii := 0; ii < nrpcs; ii++ { 426 | <-ch 427 | } 428 | 429 | js.mu.Lock() 430 | defer js.mu.Unlock() 431 | if len(js.log2) != 1 { 432 | t.Fatalf("wrong number (%v) of RPCs delivered, expected 1", len(js.log2)) 433 | } 434 | 435 | n := rn.GetCount(1000) 436 | if n != 1 { 437 | t.Fatalf("wrong GetCount() %v, expected %v\n", n, 1) 438 | } 439 | } 440 | 441 | // 442 | // if an RPC is stuck in a server, and the server 443 | // is killed with DeleteServer(), does the RPC 444 | // get un-stuck? 445 | // 446 | func TestKilled(t *testing.T) { 447 | runtime.GOMAXPROCS(4) 448 | 449 | rn := MakeNetwork() 450 | 451 | e := rn.MakeEnd("end1-99") 452 | 453 | js := &JunkServer{} 454 | svc := MakeService(js) 455 | 456 | rs := MakeServer() 457 | rs.AddService(svc) 458 | rn.AddServer("server99", rs) 459 | 460 | rn.Connect("end1-99", "server99") 461 | rn.Enable("end1-99", true) 462 | 463 | doneCh := make(chan bool) 464 | go func() { 465 | reply := 0 466 | ok := e.Call("JunkServer.Handler3", 99, &reply) 467 | doneCh <- ok 468 | }() 469 | 470 | time.Sleep(1000 * time.Millisecond) 471 | 472 | select { 473 | case <-doneCh: 474 | t.Fatalf("Handler3 should not have returned yet") 475 | case <-time.After(100 * time.Millisecond): 476 | } 477 | 478 | rn.DeleteServer("server99") 479 | 480 | select { 481 | case x := <-doneCh: 482 | if x != false { 483 | t.Fatalf("Handler3 returned successfully despite DeleteServer()") 484 | } 485 | case <-time.After(100 * time.Millisecond): 486 | t.Fatalf("Handler3 should return after DeleteServer()") 487 | } 488 | } 489 | 490 | func TestBenchmark(t *testing.T) { 491 | runtime.GOMAXPROCS(4) 492 | 493 | rn := MakeNetwork() 494 | 495 | e := rn.MakeEnd("end1-99") 496 | 497 | js := &JunkServer{} 498 | svc := MakeService(js) 499 | 500 | rs := MakeServer() 501 | rs.AddService(svc) 502 | rn.AddServer("server99", rs) 503 | 504 | rn.Connect("end1-99", "server99") 505 | rn.Enable("end1-99", true) 506 | 507 | t0 := time.Now() 508 | n := 100000 509 | for iters := 0; iters < n; iters++ { 510 | reply := "" 511 | e.Call("JunkServer.Handler2", 111, &reply) 512 | if reply != "handler2-111" { 513 | t.Fatalf("wrong reply from Handler2") 514 | } 515 | } 516 | fmt.Printf("%v for %v\n", time.Since(t0), n) 517 | // march 2016, rtm laptop, 22 microseconds per RPC 518 | } 519 | -------------------------------------------------------------------------------- /src/raft/config.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | // 4 | // support for Raft tester. 5 | // 6 | // we will use the original config.go to test your code for grading. 7 | // so, while you can modify this code to help you debug, please 8 | // test with the original before submitting. 9 | // 10 | 11 | import "labrpc" 12 | import "log" 13 | import "sync" 14 | import "testing" 15 | import "runtime" 16 | import crand "crypto/rand" 17 | import "encoding/base64" 18 | import "sync/atomic" 19 | import "time" 20 | import "fmt" 21 | 22 | func randstring(n int) string { 23 | b := make([]byte, 2*n) 24 | crand.Read(b) 25 | s := base64.URLEncoding.EncodeToString(b) 26 | return s[0:n] 27 | } 28 | 29 | type config struct { 30 | mu sync.Mutex 31 | t *testing.T 32 | net *labrpc.Network 33 | n int 34 | done int32 // tell internal threads to die 35 | rafts []*Raft 36 | applyErr []string // from apply channel readers 37 | connected []bool // whether each server is on the net 38 | saved []*Persister 39 | endnames [][]string // the port file names each sends to 40 | logs []map[int]int // copy of each server's committed entries 41 | } 42 | 43 | func make_config(t *testing.T, n int, unreliable bool) *config { 44 | runtime.GOMAXPROCS(4) 45 | cfg := &config{} 46 | cfg.t = t 47 | cfg.net = labrpc.MakeNetwork() 48 | cfg.n = n 49 | cfg.applyErr = make([]string, cfg.n) 50 | cfg.rafts = make([]*Raft, cfg.n) 51 | cfg.connected = make([]bool, cfg.n) 52 | cfg.saved = make([]*Persister, cfg.n) 53 | cfg.endnames = make([][]string, cfg.n) 54 | cfg.logs = make([]map[int]int, cfg.n) 55 | 56 | cfg.setunreliable(unreliable) 57 | 58 | cfg.net.LongDelays(true) 59 | 60 | // create a full set of Rafts. 61 | for i := 0; i < cfg.n; i++ { 62 | cfg.logs[i] = map[int]int{} 63 | cfg.start1(i) 64 | } 65 | 66 | // connect everyone 67 | for i := 0; i < cfg.n; i++ { 68 | cfg.connect(i) 69 | } 70 | 71 | return cfg 72 | } 73 | 74 | // shut down a Raft server but save its persistent state. 75 | func (cfg *config) crash1(i int) { 76 | cfg.disconnect(i) 77 | cfg.net.DeleteServer(i) // disable client connections to the server. 78 | 79 | cfg.mu.Lock() 80 | defer cfg.mu.Unlock() 81 | 82 | // a fresh persister, in case old instance 83 | // continues to update the Persister. 84 | // but copy old persister's content so that we always 85 | // pass Make() the last persisted state. 86 | if cfg.saved[i] != nil { 87 | cfg.saved[i] = cfg.saved[i].Copy() 88 | } 89 | 90 | rf := cfg.rafts[i] 91 | if rf != nil { 92 | cfg.mu.Unlock() 93 | rf.Kill() 94 | cfg.mu.Lock() 95 | cfg.rafts[i] = nil 96 | } 97 | 98 | if cfg.saved[i] != nil { 99 | raftlog := cfg.saved[i].ReadRaftState() 100 | cfg.saved[i] = &Persister{} 101 | cfg.saved[i].SaveRaftState(raftlog) 102 | } 103 | } 104 | 105 | // 106 | // start or re-start a Raft. 107 | // if one already exists, "kill" it first. 108 | // allocate new outgoing port file names, and a new 109 | // state persister, to isolate previous instance of 110 | // this server. since we cannot really kill it. 111 | // 112 | func (cfg *config) start1(i int) { 113 | cfg.crash1(i) 114 | 115 | // a fresh set of outgoing ClientEnd names. 116 | // so that old crashed instance's ClientEnds can't send. 117 | cfg.endnames[i] = make([]string, cfg.n) 118 | for j := 0; j < cfg.n; j++ { 119 | cfg.endnames[i][j] = randstring(20) 120 | } 121 | 122 | // a fresh set of ClientEnds. 123 | ends := make([]*labrpc.ClientEnd, cfg.n) 124 | for j := 0; j < cfg.n; j++ { 125 | ends[j] = cfg.net.MakeEnd(cfg.endnames[i][j]) 126 | cfg.net.Connect(cfg.endnames[i][j], j) 127 | } 128 | 129 | cfg.mu.Lock() 130 | 131 | // a fresh persister, so old instance doesn't overwrite 132 | // new instance's persisted state. 133 | // but copy old persister's content so that we always 134 | // pass Make() the last persisted state. 135 | if cfg.saved[i] != nil { 136 | cfg.saved[i] = cfg.saved[i].Copy() 137 | } else { 138 | cfg.saved[i] = MakePersister() 139 | } 140 | 141 | cfg.mu.Unlock() 142 | 143 | // listen to messages from Raft indicating newly committed messages. 144 | applyCh := make(chan ApplyMsg) 145 | go func() { 146 | for m := range applyCh { 147 | err_msg := "" 148 | if m.UseSnapshot { 149 | // ignore the snapshot 150 | } else if v, ok := (m.Command).(int); ok { 151 | cfg.mu.Lock() 152 | for j := 0; j < len(cfg.logs); j++ { 153 | if old, oldok := cfg.logs[j][m.Index]; oldok && old != v { 154 | // some server has already committed a different value for this entry! 155 | err_msg = fmt.Sprintf("commit index=%v server=%v %v != server=%v %v", 156 | m.Index, i, m.Command, j, old) 157 | } 158 | } 159 | _, prevok := cfg.logs[i][m.Index-1] 160 | cfg.logs[i][m.Index] = v 161 | cfg.mu.Unlock() 162 | 163 | if m.Index > 1 && prevok == false { 164 | err_msg = fmt.Sprintf("server %v apply out of order %v", i, m.Index) 165 | } 166 | } else { 167 | err_msg = fmt.Sprintf("committed command %v is not an int", m.Command) 168 | } 169 | 170 | if err_msg != "" { 171 | log.Fatalf("apply error: %v\n", err_msg) 172 | cfg.applyErr[i] = err_msg 173 | // keep reading after error so that Raft doesn't block 174 | // holding locks... 175 | } 176 | } 177 | }() 178 | 179 | rf := Make(ends, i, cfg.saved[i], applyCh) 180 | 181 | cfg.mu.Lock() 182 | cfg.rafts[i] = rf 183 | cfg.mu.Unlock() 184 | 185 | svc := labrpc.MakeService(rf) 186 | srv := labrpc.MakeServer() 187 | srv.AddService(svc) 188 | cfg.net.AddServer(i, srv) 189 | } 190 | 191 | func (cfg *config) cleanup() { 192 | for i := 0; i < len(cfg.rafts); i++ { 193 | if cfg.rafts[i] != nil { 194 | cfg.rafts[i].Kill() 195 | } 196 | } 197 | atomic.StoreInt32(&cfg.done, 1) 198 | } 199 | 200 | // attach server i to the net. 201 | func (cfg *config) connect(i int) { 202 | // fmt.Printf("connect(%d)\n", i) 203 | 204 | cfg.connected[i] = true 205 | 206 | // outgoing ClientEnds 207 | for j := 0; j < cfg.n; j++ { 208 | if cfg.connected[j] { 209 | endname := cfg.endnames[i][j] 210 | cfg.net.Enable(endname, true) 211 | } 212 | } 213 | 214 | // incoming ClientEnds 215 | for j := 0; j < cfg.n; j++ { 216 | if cfg.connected[j] { 217 | endname := cfg.endnames[j][i] 218 | cfg.net.Enable(endname, true) 219 | } 220 | } 221 | } 222 | 223 | // detach server i from the net. 224 | func (cfg *config) disconnect(i int) { 225 | // fmt.Printf("disconnect(%d)\n", i) 226 | 227 | cfg.connected[i] = false 228 | 229 | // outgoing ClientEnds 230 | for j := 0; j < cfg.n; j++ { 231 | if cfg.endnames[i] != nil { 232 | endname := cfg.endnames[i][j] 233 | cfg.net.Enable(endname, false) 234 | } 235 | } 236 | 237 | // incoming ClientEnds 238 | for j := 0; j < cfg.n; j++ { 239 | if cfg.endnames[j] != nil { 240 | endname := cfg.endnames[j][i] 241 | cfg.net.Enable(endname, false) 242 | } 243 | } 244 | } 245 | 246 | func (cfg *config) rpcCount(server int) int { 247 | return cfg.net.GetCount(server) 248 | } 249 | 250 | func (cfg *config) setunreliable(unrel bool) { 251 | cfg.net.Reliable(!unrel) 252 | } 253 | 254 | func (cfg *config) setlongreordering(longrel bool) { 255 | cfg.net.LongReordering(longrel) 256 | } 257 | 258 | // check that there's exactly one leader. 259 | // try a few times in case re-elections are needed. 260 | func (cfg *config) checkOneLeader() int { 261 | for iters := 0; iters < 10; iters++ { 262 | time.Sleep(500 * time.Millisecond) 263 | leaders := make(map[int][]int) 264 | for i := 0; i < cfg.n; i++ { 265 | if cfg.connected[i] { 266 | if t, leader := cfg.rafts[i].GetState(); leader { 267 | leaders[t] = append(leaders[t], i) 268 | } 269 | } 270 | } 271 | 272 | lastTermWithLeader := -1 273 | for t, leaders := range leaders { 274 | if len(leaders) > 1 { 275 | cfg.t.Fatalf("Term %d has %d (>1) leaders", t, len(leaders)) 276 | } 277 | if t > lastTermWithLeader { 278 | lastTermWithLeader = t 279 | } 280 | } 281 | 282 | if len(leaders) != 0 { 283 | return leaders[lastTermWithLeader][0] 284 | } 285 | } 286 | cfg.t.Fatalf("expected one leader, got none") 287 | return -1 288 | } 289 | 290 | // check that everyone agrees on the Term. 291 | func (cfg *config) checkTerms() int { 292 | term := -1 293 | for i := 0; i < cfg.n; i++ { 294 | if cfg.connected[i] { 295 | xterm, _ := cfg.rafts[i].GetState() 296 | if term == -1 { 297 | term = xterm 298 | } else if term != xterm { 299 | cfg.t.Fatalf("servers disagree on Term") 300 | } 301 | } 302 | } 303 | return term 304 | } 305 | 306 | // check that there's no leader 307 | func (cfg *config) checkNoLeader() { 308 | for i := 0; i < cfg.n; i++ { 309 | if cfg.connected[i] { 310 | _, is_leader := cfg.rafts[i].GetState() 311 | if is_leader { 312 | cfg.t.Fatalf("expected no leader, but %v claims to be leader", i) 313 | } 314 | } 315 | } 316 | } 317 | 318 | // how many servers think a log entry is committed? 319 | func (cfg *config) nCommitted(index int) (int, interface{}) { 320 | count := 0 321 | cmd := -1 322 | for i := 0; i < len(cfg.rafts); i++ { 323 | if cfg.applyErr[i] != "" { 324 | cfg.t.Fatal(cfg.applyErr[i]) 325 | } 326 | 327 | cfg.mu.Lock() 328 | cmd1, ok := cfg.logs[i][index] 329 | cfg.mu.Unlock() 330 | 331 | if ok { 332 | if count > 0 && cmd != cmd1 { 333 | cfg.t.Fatalf("committed values do not match: index %v, %v, %v\n", 334 | index, cmd, cmd1) 335 | } 336 | count += 1 337 | cmd = cmd1 338 | } 339 | } 340 | return count, cmd 341 | } 342 | 343 | // wait for at least n servers to commit. 344 | // but don't wait forever. 345 | func (cfg *config) wait(index int, n int, startTerm int) interface{} { 346 | to := 10 * time.Millisecond 347 | for iters := 0; iters < 30; iters++ { 348 | nd, _ := cfg.nCommitted(index) 349 | if nd >= n { 350 | break 351 | } 352 | time.Sleep(to) 353 | if to < time.Second { 354 | to *= 2 355 | } 356 | if startTerm > -1 { 357 | for _, r := range cfg.rafts { 358 | if t, _ := r.GetState(); t > startTerm { 359 | // someone has moved on 360 | // can no longer guarantee that we'll "win" 361 | return -1 362 | } 363 | } 364 | } 365 | } 366 | nd, cmd := cfg.nCommitted(index) 367 | if nd < n { 368 | cfg.t.Fatalf("only %d decided for index %d; wanted %d\n", 369 | nd, index, n) 370 | } 371 | return cmd 372 | } 373 | 374 | // do a complete agreement. 375 | // it might choose the wrong leader initially, 376 | // and have to re-submit after giving up. 377 | // entirely gives up after about 10 seconds. 378 | // indirectly checks that the servers agree on the 379 | // same value, since nCommitted() checks this, 380 | // as do the threads that read from applyCh. 381 | // returns index. 382 | func (cfg *config) one(cmd int, expectedServers int) int { 383 | t0 := time.Now() 384 | starts := 0 385 | for time.Since(t0).Seconds() < 10 { 386 | // try all the servers, maybe one is the leader. 387 | index := -1 388 | for si := 0; si < cfg.n; si++ { 389 | starts = (starts + 1) % cfg.n 390 | var rf *Raft 391 | cfg.mu.Lock() 392 | if cfg.connected[starts] { 393 | rf = cfg.rafts[starts] 394 | } 395 | cfg.mu.Unlock() 396 | if rf != nil { 397 | index1, _, ok := rf.Start(cmd) 398 | if ok { 399 | index = index1 400 | break 401 | } 402 | } 403 | } 404 | 405 | if index != -1 { 406 | // somebody claimed to be the leader and to have 407 | // submitted our command; wait a while for agreement. 408 | t1 := time.Now() 409 | for time.Since(t1).Seconds() < 2 { 410 | nd, cmd1 := cfg.nCommitted(index) 411 | if nd > 0 && nd >= expectedServers { 412 | // committed 413 | if cmd2, ok := cmd1.(int); ok && cmd2 == cmd { 414 | // and it was the command we submitted. 415 | return index 416 | } 417 | } 418 | time.Sleep(20 * time.Millisecond) 419 | } 420 | } else { 421 | time.Sleep(50 * time.Millisecond) 422 | } 423 | } 424 | cfg.t.Fatalf("one(%v) failed to reach agreement", cmd) 425 | return -1 426 | } 427 | -------------------------------------------------------------------------------- /src/raft/persister.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | // 4 | // support for Raft and kvraft to save persistent 5 | // Raft state (log &c) and k/v server snapshots. 6 | // 7 | // we will use the original persister.go to test your code for grading. 8 | // so, while you can modify this code to help you debug, please 9 | // test with the original before submitting. 10 | // 11 | 12 | import "sync" 13 | 14 | type Persister struct { 15 | mu sync.Mutex 16 | raftstate []byte 17 | snapshot []byte 18 | } 19 | 20 | func MakePersister() *Persister { 21 | return &Persister{} 22 | } 23 | 24 | func (ps *Persister) Copy() *Persister { 25 | ps.mu.Lock() 26 | defer ps.mu.Unlock() 27 | np := MakePersister() 28 | np.raftstate = ps.raftstate 29 | np.snapshot = ps.snapshot 30 | return np 31 | } 32 | 33 | func (ps *Persister) SaveRaftState(data []byte) { 34 | ps.mu.Lock() 35 | defer ps.mu.Unlock() 36 | ps.raftstate = data 37 | } 38 | 39 | func (ps *Persister) ReadRaftState() []byte { 40 | ps.mu.Lock() 41 | defer ps.mu.Unlock() 42 | return ps.raftstate 43 | } 44 | 45 | func (ps *Persister) RaftStateSize() int { 46 | ps.mu.Lock() 47 | defer ps.mu.Unlock() 48 | return len(ps.raftstate) 49 | } 50 | 51 | func (ps *Persister) SaveSnapshot(snapshot []byte) { 52 | ps.mu.Lock() 53 | defer ps.mu.Unlock() 54 | ps.snapshot = snapshot 55 | } 56 | 57 | func (ps *Persister) ReadSnapshot() []byte { 58 | ps.mu.Lock() 59 | defer ps.mu.Unlock() 60 | return ps.snapshot 61 | } 62 | -------------------------------------------------------------------------------- /src/raft/raft.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "labrpc" 7 | "math/rand" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | const ( 13 | FOLLOWER = 0 14 | CANDIDATE = 1 15 | LEADER = 2 16 | ) 17 | 18 | const ( 19 | electionTimeoutMin = 300 * time.Millisecond 20 | electionTimeoutMax = 600 * time.Millisecond 21 | heartbeatTimeout = 50 * time.Millisecond 22 | checkTimeout = 5 * time.Millisecond 23 | ) 24 | 25 | func randTime() time.Duration { 26 | diff := (electionTimeoutMax - electionTimeoutMin).Milliseconds() 27 | return electionTimeoutMin + time.Duration(rand.Intn(int(diff)))*time.Millisecond 28 | } 29 | 30 | func wait(n int, ch chan bool) { 31 | for i := 1; i < n; i++ { 32 | select { 33 | case <-ch: 34 | case <-time.After(checkTimeout): 35 | return 36 | } 37 | } 38 | } 39 | 40 | type ApplyMsg struct { 41 | Index int 42 | Command interface{} 43 | UseSnapshot bool 44 | Snapshot []byte 45 | } 46 | 47 | type LogItem struct { 48 | Term int 49 | Command interface{} 50 | } 51 | 52 | type Raft struct { 53 | mu sync.Mutex 54 | peers []*labrpc.ClientEnd 55 | persister *Persister 56 | me int 57 | 58 | // 修改以下三条必须持久化 59 | currentTerm int 60 | votedFor int 61 | log []LogItem 62 | 63 | commitIndex int 64 | lastApplied int 65 | 66 | nextIndex []int 67 | matchIndex []int 68 | 69 | role int 70 | votes int 71 | timer *time.Timer 72 | } 73 | 74 | func (rf *Raft) GetState() (int, bool) { 75 | rf.mu.Lock() 76 | defer rf.mu.Unlock() 77 | term := rf.currentTerm 78 | isLeader := rf.role == LEADER 79 | return term, isLeader 80 | } 81 | 82 | func (rf *Raft) persist() { 83 | w := new(bytes.Buffer) 84 | e := gob.NewEncoder(w) 85 | e.Encode(rf.currentTerm) 86 | e.Encode(rf.votedFor) 87 | e.Encode(rf.log) 88 | data := w.Bytes() 89 | rf.persister.SaveRaftState(data) 90 | } 91 | 92 | func (rf *Raft) readPersist(data []byte) { 93 | r := bytes.NewBuffer(data) 94 | d := gob.NewDecoder(r) 95 | d.Decode(&rf.currentTerm) 96 | d.Decode(&rf.votedFor) 97 | d.Decode(&rf.log) 98 | } 99 | 100 | func (rf *Raft) keepOrFollow(term int) { 101 | if term > rf.currentTerm { 102 | rf.currentTerm = term 103 | rf.votedFor = -1 // 重置投票 104 | rf.persist() 105 | rf.role = FOLLOWER 106 | } 107 | } 108 | 109 | type RequestVoteArgs struct { 110 | Term int 111 | CandidateId int 112 | LastLogIndex int 113 | LastLogTerm int 114 | } 115 | 116 | type RequestVoteReply struct { 117 | Term int 118 | VoteGranted bool 119 | } 120 | 121 | func (rf *Raft) RequestVote(args RequestVoteArgs, reply *RequestVoteReply) { 122 | rf.mu.Lock() 123 | defer rf.mu.Unlock() 124 | reply.Term = rf.currentTerm 125 | rf.keepOrFollow(args.Term) 126 | if args.Term >= rf.currentTerm && 127 | (rf.votedFor == -1 || rf.votedFor == args.CandidateId) && 128 | (args.LastLogTerm > rf.log[len(rf.log)-1].Term || 129 | args.LastLogTerm == rf.log[len(rf.log)-1].Term && 130 | args.LastLogIndex >= len(rf.log)-1) { // 日志至少一样新 131 | rf.votedFor = args.CandidateId 132 | rf.persist() 133 | rf.timer.Reset(randTime()) // 重置选举计时 134 | reply.VoteGranted = true 135 | } 136 | } 137 | 138 | func (rf *Raft) sendRequestVote(server int, args RequestVoteArgs, reply *RequestVoteReply, ch chan bool) { 139 | if !rf.peers[server].Call("Raft.RequestVote", args, reply) { 140 | return 141 | } 142 | rf.mu.Lock() 143 | rf.keepOrFollow(reply.Term) 144 | if rf.role == CANDIDATE && reply.VoteGranted { 145 | rf.votes += 1 146 | } 147 | rf.mu.Unlock() 148 | ch <- true 149 | } 150 | 151 | type AppendEntriesArgs struct { 152 | Term int 153 | LeaderId int 154 | PrevLogIndex int 155 | PrevLogTerm int 156 | Entries []LogItem 157 | LeaderCommit int 158 | } 159 | 160 | type AppendEntriesReply struct { 161 | Term int 162 | Success bool 163 | } 164 | 165 | func (rf *Raft) AppendEntries(args AppendEntriesArgs, reply *AppendEntriesReply) { 166 | rf.mu.Lock() 167 | defer rf.mu.Unlock() 168 | reply.Term = rf.currentTerm 169 | if args.Term == rf.currentTerm { 170 | rf.role = FOLLOWER 171 | rf.timer.Reset(randTime()) // term相同重置计时 172 | } 173 | rf.keepOrFollow(args.Term) 174 | if args.Term >= rf.currentTerm && 175 | args.PrevLogIndex < len(rf.log) && 176 | args.PrevLogTerm == rf.log[args.PrevLogIndex].Term { // term和log要匹配 177 | if args.PrevLogIndex+1 != len(rf.log) || args.Entries != nil { 178 | rf.log = append(rf.log[:args.PrevLogIndex+1], args.Entries...) // 删除不匹配并添加未持有的日志 179 | rf.persist() 180 | } 181 | if args.LeaderCommit > rf.commitIndex { 182 | if args.LeaderCommit < len(rf.log) { 183 | rf.commitIndex = args.LeaderCommit 184 | } else { 185 | rf.commitIndex = len(rf.log) 186 | } 187 | } 188 | reply.Success = true 189 | } 190 | } 191 | 192 | func (rf *Raft) sendAppendEntries(server int, args AppendEntriesArgs, reply *AppendEntriesReply, ch chan bool) { 193 | if !rf.peers[server].Call("Raft.AppendEntries", args, reply) { 194 | return 195 | } 196 | rf.mu.Lock() 197 | rf.keepOrFollow(reply.Term) 198 | if rf.role == LEADER && reply.Success { 199 | rf.nextIndex[server] = args.PrevLogIndex + len(args.Entries) + 1 200 | rf.matchIndex[server] = args.PrevLogIndex + len(args.Entries) 201 | } else if rf.role == LEADER && !reply.Success { 202 | rf.nextIndex[server]-- 203 | for rf.nextIndex[server] >= 0 && rf.log[rf.nextIndex[server]].Term == reply.Term { 204 | rf.nextIndex[server]-- // 跳过同一term的log 205 | } 206 | } 207 | rf.mu.Unlock() 208 | ch <- true 209 | } 210 | 211 | func (rf *Raft) Start(command interface{}) (int, int, bool) { 212 | rf.mu.Lock() 213 | defer rf.mu.Unlock() 214 | index := len(rf.log) 215 | term := rf.currentTerm 216 | isLeader := rf.role == LEADER 217 | if isLeader { 218 | rf.log = append(rf.log, LogItem{term, command}) 219 | rf.persist() 220 | } 221 | return index, term, isLeader 222 | } 223 | 224 | func (rf *Raft) Kill() { 225 | rf.mu.Lock() 226 | defer rf.mu.Unlock() 227 | rf.me = -1 228 | rf.timer.Reset(0) 229 | } 230 | 231 | func Make(peers []*labrpc.ClientEnd, me int, 232 | persister *Persister, applyCh chan ApplyMsg) *Raft { 233 | n := len(peers) 234 | rf := &Raft{} 235 | rf.peers = peers 236 | rf.persister = persister 237 | rf.me = me 238 | rf.votedFor = -1 239 | rf.log = make([]LogItem, 1) 240 | rf.nextIndex = make([]int, n) 241 | rf.matchIndex = make([]int, n) 242 | rf.role = FOLLOWER 243 | rf.timer = time.NewTimer(randTime()) 244 | rf.readPersist(persister.ReadRaftState()) 245 | 246 | go func() { 247 | for rf.me != -1 { 248 | time.Sleep(checkTimeout) 249 | rf.mu.Lock() 250 | for rf.me != -1 && rf.commitIndex > rf.lastApplied { 251 | rf.lastApplied++ 252 | applyCh <- ApplyMsg{Index: rf.lastApplied, Command: rf.log[rf.lastApplied].Command} 253 | } 254 | rf.mu.Unlock() 255 | } 256 | }() 257 | 258 | go func() { 259 | for rf.me != -1 { 260 | <-rf.timer.C 261 | switch rf.role { 262 | case FOLLOWER: 263 | rf.mu.Lock() 264 | rf.role = CANDIDATE 265 | rf.timer.Reset(0) 266 | rf.mu.Unlock() 267 | 268 | case CANDIDATE: 269 | rf.mu.Lock() 270 | rf.currentTerm++ 271 | rf.votedFor = me 272 | rf.persist() 273 | rf.votes = 1 274 | rf.timer.Reset(randTime()) 275 | m := len(rf.log) 276 | rf.mu.Unlock() 277 | 278 | ch := make(chan bool) 279 | for _, i := range rand.Perm(n) { 280 | rf.mu.Lock() 281 | if rf.me != -1 && rf.role == CANDIDATE && i != rf.me { 282 | args := RequestVoteArgs{rf.currentTerm, rf.me, 283 | m - 1, rf.log[m-1].Term} 284 | reply := RequestVoteReply{} 285 | go rf.sendRequestVote(i, args, &reply, ch) 286 | } 287 | rf.mu.Unlock() 288 | } 289 | 290 | wait(n, ch) // 等待所有发收成功或超时 291 | 292 | rf.mu.Lock() 293 | if rf.me != -1 && rf.role == CANDIDATE && 2*rf.votes > n { // 成功竞选 294 | rf.role = LEADER 295 | rf.timer.Reset(0) 296 | for i := 0; i < n; i++ { 297 | rf.nextIndex[i] = m 298 | } 299 | } 300 | rf.mu.Unlock() 301 | 302 | case LEADER: 303 | rf.mu.Lock() 304 | rf.timer.Reset(heartbeatTimeout) 305 | m := len(rf.log) 306 | rf.mu.Unlock() 307 | 308 | ch := make(chan bool) 309 | for _, i := range rand.Perm(n) { 310 | rf.mu.Lock() 311 | if rf.me != -1 && rf.role == LEADER && i != rf.me { 312 | args := AppendEntriesArgs{rf.currentTerm, rf.me, 313 | rf.nextIndex[i] - 1, rf.log[rf.nextIndex[i]-1].Term, 314 | nil, rf.commitIndex} 315 | if rf.nextIndex[i] < m { 316 | args.Entries = make([]LogItem, m-rf.nextIndex[i]) 317 | copy(args.Entries, rf.log[rf.nextIndex[i]:m]) 318 | } 319 | reply := AppendEntriesReply{} 320 | go rf.sendAppendEntries(i, args, &reply, ch) 321 | } 322 | rf.mu.Unlock() 323 | } 324 | 325 | wait(n, ch) // 等待所有发收成功或超时 326 | 327 | rf.mu.Lock() 328 | N := m - 1 329 | if rf.me != -1 && rf.role == LEADER && N > rf.commitIndex && rf.log[N].Term == rf.currentTerm { 330 | count := 1 331 | for i := 0; i < n; i++ { 332 | if i != me && rf.matchIndex[i] >= N { 333 | count++ 334 | } 335 | } 336 | if 2*count > n { // 确认提交 337 | rf.commitIndex = N 338 | } 339 | } 340 | rf.mu.Unlock() 341 | } 342 | } 343 | }() 344 | return rf 345 | } 346 | -------------------------------------------------------------------------------- /src/raft/test_test.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | // 4 | // Raft tests. 5 | // 6 | // we will use the original test_test.go to test your code for grading. 7 | // so, while you can modify this code to help you debug, please 8 | // test with the original before submitting. 9 | // 10 | 11 | import "testing" 12 | import "fmt" 13 | import "time" 14 | import "math/rand" 15 | import "sync" 16 | 17 | // The tester generously allows solutions to complete elections in one second 18 | // (much more than the paper's range of timeouts). 19 | const RaftElectionTimeout = 1000 * time.Millisecond 20 | 21 | func TestInitialElection(t *testing.T) { 22 | servers := 3 23 | cfg := make_config(t, servers, false) 24 | defer cfg.cleanup() 25 | 26 | fmt.Printf("Test: initial election ...\n") 27 | 28 | // is a leader elected? 29 | cfg.checkOneLeader() 30 | 31 | // does the leader+Term stay the same there is no failure? 32 | term1 := cfg.checkTerms() 33 | time.Sleep(2 * RaftElectionTimeout) 34 | term2 := cfg.checkTerms() 35 | if term1 != term2 { 36 | fmt.Printf("warning: Term changed even though there were no failures") 37 | } 38 | 39 | fmt.Printf(" ... Passed\n") 40 | } 41 | 42 | func TestReElection(t *testing.T) { 43 | servers := 3 44 | cfg := make_config(t, servers, false) 45 | defer cfg.cleanup() 46 | 47 | fmt.Printf("Test: election after network failure ...\n") 48 | 49 | leader1 := cfg.checkOneLeader() 50 | 51 | // if the leader disconnects, a new one should be elected. 52 | cfg.disconnect(leader1) 53 | cfg.checkOneLeader() 54 | 55 | // if the old leader rejoins, that shouldn't 56 | // disturb the old leader. 57 | cfg.connect(leader1) 58 | leader2 := cfg.checkOneLeader() 59 | 60 | // if there's no quorum, no leader should 61 | // be elected. 62 | cfg.disconnect(leader2) 63 | cfg.disconnect((leader2 + 1) % servers) 64 | time.Sleep(2 * RaftElectionTimeout) 65 | cfg.checkNoLeader() 66 | 67 | // if a quorum arises, it should elect a leader. 68 | cfg.connect((leader2 + 1) % servers) 69 | cfg.checkOneLeader() 70 | 71 | // re-join of last node shouldn't prevent leader from existing. 72 | cfg.connect(leader2) 73 | cfg.checkOneLeader() 74 | 75 | fmt.Printf(" ... Passed\n") 76 | } 77 | 78 | func TestBasicAgree(t *testing.T) { 79 | servers := 3 80 | cfg := make_config(t, servers, false) 81 | defer cfg.cleanup() 82 | 83 | fmt.Printf("Test: basic agreement ...\n") 84 | 85 | iters := 3 86 | for index := 1; index < iters+1; index++ { 87 | nd, _ := cfg.nCommitted(index) 88 | if nd > 0 { 89 | t.Fatalf("some have committed before Start()") 90 | } 91 | 92 | xindex := cfg.one(index*100, servers) 93 | if xindex != index { 94 | t.Fatalf("got index %v but expected %v", xindex, index) 95 | } 96 | } 97 | 98 | fmt.Printf(" ... Passed\n") 99 | } 100 | 101 | func TestFailAgree(t *testing.T) { 102 | servers := 3 103 | cfg := make_config(t, servers, false) 104 | defer cfg.cleanup() 105 | 106 | fmt.Printf("Test: agreement despite follower failure ...\n") 107 | 108 | cfg.one(101, servers) 109 | 110 | // follower network failure 111 | leader := cfg.checkOneLeader() 112 | cfg.disconnect((leader + 1) % servers) 113 | 114 | // agree despite one failed server? 115 | cfg.one(102, servers-1) 116 | cfg.one(103, servers-1) 117 | time.Sleep(RaftElectionTimeout) 118 | cfg.one(104, servers-1) 119 | cfg.one(105, servers-1) 120 | 121 | // failed server re-connected 122 | cfg.connect((leader + 1) % servers) 123 | 124 | // agree with full set of servers? 125 | cfg.one(106, servers) 126 | time.Sleep(RaftElectionTimeout) 127 | cfg.one(107, servers) 128 | 129 | fmt.Printf(" ... Passed\n") 130 | } 131 | 132 | func TestFailNoAgree(t *testing.T) { 133 | servers := 5 134 | cfg := make_config(t, servers, false) 135 | defer cfg.cleanup() 136 | 137 | fmt.Printf("Test: no agreement if too many followers fail ...\n") 138 | 139 | cfg.one(10, servers) 140 | 141 | // 3 of 5 followers disconnect 142 | leader := cfg.checkOneLeader() 143 | cfg.disconnect((leader + 1) % servers) 144 | cfg.disconnect((leader + 2) % servers) 145 | cfg.disconnect((leader + 3) % servers) 146 | 147 | index, _, ok := cfg.rafts[leader].Start(20) 148 | if ok != true { 149 | t.Fatalf("leader rejected Start()") 150 | } 151 | if index != 2 { 152 | t.Fatalf("expected index 2, got %v", index) 153 | } 154 | 155 | time.Sleep(2 * RaftElectionTimeout) 156 | 157 | n, _ := cfg.nCommitted(index) 158 | if n > 0 { 159 | t.Fatalf("%v committed but no majority", n) 160 | } 161 | 162 | // repair failures 163 | cfg.connect((leader + 1) % servers) 164 | cfg.connect((leader + 2) % servers) 165 | cfg.connect((leader + 3) % servers) 166 | 167 | // the disconnected majority may have chosen a leader from 168 | // among their own ranks, forgetting index 2. 169 | // or perhaps 170 | leader2 := cfg.checkOneLeader() 171 | index2, _, ok2 := cfg.rafts[leader2].Start(30) 172 | if ok2 == false { 173 | t.Fatalf("leader2 rejected Start()") 174 | } 175 | if index2 < 2 || index2 > 3 { 176 | t.Fatalf("unexpected index %v", index2) 177 | } 178 | 179 | cfg.one(1000, servers) 180 | 181 | fmt.Printf(" ... Passed\n") 182 | } 183 | 184 | func TestConcurrentStarts(t *testing.T) { 185 | servers := 3 186 | cfg := make_config(t, servers, false) 187 | defer cfg.cleanup() 188 | 189 | fmt.Printf("Test: concurrent Start()s ...\n") 190 | 191 | var success bool 192 | loop: 193 | for try := 0; try < 5; try++ { 194 | if try > 0 { 195 | // give solution some time to settle 196 | time.Sleep(3 * time.Second) 197 | } 198 | 199 | leader := cfg.checkOneLeader() 200 | _, term, ok := cfg.rafts[leader].Start(1) 201 | if !ok { 202 | // leader moved on really quickly 203 | continue 204 | } 205 | 206 | iters := 5 207 | var wg sync.WaitGroup 208 | is := make(chan int, iters) 209 | for ii := 0; ii < iters; ii++ { 210 | wg.Add(1) 211 | go func(i int) { 212 | defer wg.Done() 213 | i, term1, ok := cfg.rafts[leader].Start(100 + i) 214 | if term1 != term { 215 | return 216 | } 217 | if ok != true { 218 | return 219 | } 220 | is <- i 221 | }(ii) 222 | } 223 | 224 | wg.Wait() 225 | close(is) 226 | 227 | for j := 0; j < servers; j++ { 228 | if t, _ := cfg.rafts[j].GetState(); t != term { 229 | // Term changed -- can't expect low RPC counts 230 | continue loop 231 | } 232 | } 233 | 234 | failed := false 235 | cmds := []int{} 236 | for index := range is { 237 | cmd := cfg.wait(index, servers, term) 238 | if ix, ok := cmd.(int); ok { 239 | if ix == -1 { 240 | // peers have moved on to later terms 241 | // so we can't expect all Start()s to 242 | // have succeeded 243 | failed = true 244 | break 245 | } 246 | cmds = append(cmds, ix) 247 | } else { 248 | t.Fatalf("value %v is not an int", cmd) 249 | } 250 | } 251 | 252 | if failed { 253 | // avoid leaking goroutines 254 | go func() { 255 | for range is { 256 | } 257 | }() 258 | continue 259 | } 260 | 261 | for ii := 0; ii < iters; ii++ { 262 | x := 100 + ii 263 | ok := false 264 | for j := 0; j < len(cmds); j++ { 265 | if cmds[j] == x { 266 | ok = true 267 | } 268 | } 269 | if ok == false { 270 | t.Fatalf("Command %v missing in %v", x, cmds) 271 | } 272 | } 273 | 274 | success = true 275 | break 276 | } 277 | 278 | if !success { 279 | t.Fatalf("Term changed too often") 280 | } 281 | 282 | fmt.Printf(" ... Passed\n") 283 | } 284 | 285 | func TestRejoin(t *testing.T) { 286 | servers := 3 287 | cfg := make_config(t, servers, false) 288 | defer cfg.cleanup() 289 | 290 | fmt.Printf("Test: rejoin of partitioned leader ...\n") 291 | 292 | cfg.one(101, servers) 293 | 294 | // leader network failure 295 | leader1 := cfg.checkOneLeader() 296 | cfg.disconnect(leader1) 297 | 298 | // make old leader try to agree on some entries 299 | cfg.rafts[leader1].Start(102) 300 | cfg.rafts[leader1].Start(103) 301 | cfg.rafts[leader1].Start(104) 302 | 303 | // new leader commits, also for index=2 304 | cfg.one(103, 2) 305 | 306 | // new leader network failure 307 | leader2 := cfg.checkOneLeader() 308 | cfg.disconnect(leader2) 309 | 310 | // old leader connected again 311 | cfg.connect(leader1) 312 | 313 | cfg.one(104, 2) 314 | 315 | // all together now 316 | cfg.connect(leader2) 317 | 318 | cfg.one(105, servers) 319 | 320 | fmt.Printf(" ... Passed\n") 321 | } 322 | 323 | func TestBackup(t *testing.T) { 324 | servers := 5 325 | cfg := make_config(t, servers, false) 326 | defer cfg.cleanup() 327 | 328 | fmt.Printf("Test: leader backs up quickly over incorrect follower logs ...\n") 329 | 330 | cfg.one(rand.Int(), servers) 331 | 332 | // put leader and one follower in a partition 333 | leader1 := cfg.checkOneLeader() 334 | cfg.disconnect((leader1 + 2) % servers) 335 | cfg.disconnect((leader1 + 3) % servers) 336 | cfg.disconnect((leader1 + 4) % servers) 337 | 338 | // submit lots of commands that won't commit 339 | for i := 0; i < 50; i++ { 340 | cfg.rafts[leader1].Start(rand.Int()) 341 | } 342 | 343 | time.Sleep(RaftElectionTimeout / 2) 344 | 345 | cfg.disconnect((leader1 + 0) % servers) 346 | cfg.disconnect((leader1 + 1) % servers) 347 | 348 | // allow other partition to recover 349 | cfg.connect((leader1 + 2) % servers) 350 | cfg.connect((leader1 + 3) % servers) 351 | cfg.connect((leader1 + 4) % servers) 352 | 353 | // lots of successful commands to new group. 354 | for i := 0; i < 50; i++ { 355 | cfg.one(rand.Int(), 3) 356 | } 357 | 358 | // now another partitioned leader and one follower 359 | leader2 := cfg.checkOneLeader() 360 | other := (leader1 + 2) % servers 361 | if leader2 == other { 362 | other = (leader2 + 1) % servers 363 | } 364 | cfg.disconnect(other) 365 | 366 | // lots more commands that won't commit 367 | for i := 0; i < 50; i++ { 368 | cfg.rafts[leader2].Start(rand.Int()) 369 | } 370 | 371 | time.Sleep(RaftElectionTimeout / 2) 372 | 373 | // bring original leader back to life, 374 | for i := 0; i < servers; i++ { 375 | cfg.disconnect(i) 376 | } 377 | cfg.connect((leader1 + 0) % servers) 378 | cfg.connect((leader1 + 1) % servers) 379 | cfg.connect(other) 380 | 381 | // lots of successful commands to new group. 382 | for i := 0; i < 50; i++ { 383 | cfg.one(rand.Int(), 3) 384 | } 385 | 386 | // now everyone 387 | for i := 0; i < servers; i++ { 388 | cfg.connect(i) 389 | } 390 | cfg.one(rand.Int(), servers) 391 | 392 | fmt.Printf(" ... Passed\n") 393 | } 394 | 395 | func TestCount(t *testing.T) { 396 | servers := 3 397 | cfg := make_config(t, servers, false) 398 | defer cfg.cleanup() 399 | 400 | fmt.Printf("Test: RPC counts aren't too high ...\n") 401 | 402 | rpcs := func() (n int) { 403 | for j := 0; j < servers; j++ { 404 | n += cfg.rpcCount(j) 405 | } 406 | return 407 | } 408 | 409 | leader := cfg.checkOneLeader() 410 | 411 | total1 := rpcs() 412 | 413 | if total1 > 30 || total1 < 1 { 414 | t.Fatalf("too many or few RPCs (%v) to elect initial leader\n", total1) 415 | } 416 | 417 | var total2 int 418 | var success bool 419 | loop: 420 | for try := 0; try < 5; try++ { 421 | if try > 0 { 422 | // give solution some time to settle 423 | time.Sleep(3 * time.Second) 424 | } 425 | 426 | leader = cfg.checkOneLeader() 427 | total1 = rpcs() 428 | 429 | iters := 10 430 | starti, term, ok := cfg.rafts[leader].Start(1) 431 | if !ok { 432 | // leader moved on really quickly 433 | continue 434 | } 435 | cmds := []int{} 436 | for i := 1; i < iters+2; i++ { 437 | x := int(rand.Int31()) 438 | cmds = append(cmds, x) 439 | index1, term1, ok := cfg.rafts[leader].Start(x) 440 | if term1 != term { 441 | // Term changed while starting 442 | continue loop 443 | } 444 | if !ok { 445 | // No longer the leader, so Term has changed 446 | continue loop 447 | } 448 | if starti+i != index1 { 449 | t.Fatalf("Start() failed") 450 | } 451 | } 452 | 453 | for i := 1; i < iters+1; i++ { 454 | cmd := cfg.wait(starti+i, servers, term) 455 | if ix, ok := cmd.(int); ok == false || ix != cmds[i-1] { 456 | if ix == -1 { 457 | // Term changed -- try again 458 | continue loop 459 | } 460 | t.Fatalf("wrong value %v committed for index %v; expected %v\n", cmd, starti+i, cmds) 461 | } 462 | } 463 | 464 | failed := false 465 | total2 = 0 466 | for j := 0; j < servers; j++ { 467 | if t, _ := cfg.rafts[j].GetState(); t != term { 468 | // Term changed -- can't expect low RPC counts 469 | // need to keep going to update total2 470 | failed = true 471 | } 472 | total2 += cfg.rpcCount(j) 473 | } 474 | 475 | if failed { 476 | continue loop 477 | } 478 | 479 | if total2-total1 > (iters+1+3)*3 { 480 | t.Fatalf("too many RPCs (%v) for %v entries\n", total2-total1, iters) 481 | } 482 | 483 | success = true 484 | break 485 | } 486 | 487 | if !success { 488 | t.Fatalf("Term changed too often") 489 | } 490 | 491 | time.Sleep(RaftElectionTimeout) 492 | 493 | total3 := 0 494 | for j := 0; j < servers; j++ { 495 | total3 += cfg.rpcCount(j) 496 | } 497 | 498 | if total3-total2 > 3*20 { 499 | t.Fatalf("too many RPCs (%v) for 1 second of idleness\n", total3-total2) 500 | } 501 | 502 | fmt.Printf(" ... Passed\n") 503 | } 504 | 505 | func TestPersist1(t *testing.T) { 506 | servers := 3 507 | cfg := make_config(t, servers, false) 508 | defer cfg.cleanup() 509 | 510 | fmt.Printf("Test: basic persistence ...\n") 511 | 512 | cfg.one(11, servers) 513 | 514 | // crash and re-start all 515 | for i := 0; i < servers; i++ { 516 | cfg.start1(i) 517 | } 518 | for i := 0; i < servers; i++ { 519 | cfg.disconnect(i) 520 | cfg.connect(i) 521 | } 522 | 523 | cfg.one(12, servers) 524 | 525 | leader1 := cfg.checkOneLeader() 526 | cfg.disconnect(leader1) 527 | cfg.start1(leader1) 528 | cfg.connect(leader1) 529 | 530 | cfg.one(13, servers) 531 | 532 | leader2 := cfg.checkOneLeader() 533 | cfg.disconnect(leader2) 534 | cfg.one(14, servers-1) 535 | cfg.start1(leader2) 536 | cfg.connect(leader2) 537 | 538 | cfg.wait(4, servers, -1) // wait for leader2 to join before killing i3 539 | 540 | i3 := (cfg.checkOneLeader() + 1) % servers 541 | cfg.disconnect(i3) 542 | cfg.one(15, servers-1) 543 | cfg.start1(i3) 544 | cfg.connect(i3) 545 | 546 | cfg.one(16, servers) 547 | 548 | fmt.Printf(" ... Passed\n") 549 | } 550 | 551 | func TestPersist2(t *testing.T) { 552 | servers := 5 553 | cfg := make_config(t, servers, false) 554 | defer cfg.cleanup() 555 | 556 | fmt.Printf("Test: more persistence ...\n") 557 | 558 | index := 1 559 | for iters := 0; iters < 5; iters++ { 560 | cfg.one(10+index, servers) 561 | index++ 562 | 563 | leader1 := cfg.checkOneLeader() 564 | 565 | cfg.disconnect((leader1 + 1) % servers) 566 | cfg.disconnect((leader1 + 2) % servers) 567 | 568 | cfg.one(10+index, servers-2) 569 | index++ 570 | 571 | cfg.disconnect((leader1 + 0) % servers) 572 | cfg.disconnect((leader1 + 3) % servers) 573 | cfg.disconnect((leader1 + 4) % servers) 574 | 575 | cfg.start1((leader1 + 1) % servers) 576 | cfg.start1((leader1 + 2) % servers) 577 | cfg.connect((leader1 + 1) % servers) 578 | cfg.connect((leader1 + 2) % servers) 579 | 580 | time.Sleep(RaftElectionTimeout) 581 | 582 | cfg.start1((leader1 + 3) % servers) 583 | cfg.connect((leader1 + 3) % servers) 584 | 585 | cfg.one(10+index, servers-2) 586 | index++ 587 | 588 | cfg.connect((leader1 + 4) % servers) 589 | cfg.connect((leader1 + 0) % servers) 590 | } 591 | 592 | cfg.one(1000, servers) 593 | 594 | fmt.Printf(" ... Passed\n") 595 | } 596 | 597 | func TestPersist3(t *testing.T) { 598 | servers := 3 599 | cfg := make_config(t, servers, false) 600 | defer cfg.cleanup() 601 | 602 | fmt.Printf("Test: partitioned leader and one follower crash, leader restarts ...\n") 603 | 604 | cfg.one(101, 3) 605 | 606 | leader := cfg.checkOneLeader() 607 | cfg.disconnect((leader + 2) % servers) 608 | 609 | cfg.one(102, 2) 610 | 611 | cfg.crash1((leader + 0) % servers) 612 | cfg.crash1((leader + 1) % servers) 613 | cfg.connect((leader + 2) % servers) 614 | cfg.start1((leader + 0) % servers) 615 | cfg.connect((leader + 0) % servers) 616 | 617 | cfg.one(103, 2) 618 | 619 | cfg.start1((leader + 1) % servers) 620 | cfg.connect((leader + 1) % servers) 621 | 622 | cfg.one(104, servers) 623 | 624 | fmt.Printf(" ... Passed\n") 625 | } 626 | 627 | // 628 | // Test the scenarios described in Figure 8 of the extended Raft paper. Each 629 | // iteration asks a leader, if there is one, to insert a command in the Raft 630 | // log. If there is a leader, that leader will fail quickly with a high 631 | // probability (perhaps without committing the command), or crash after a while 632 | // with low probability (most likey committing the command). If the number of 633 | // alive servers isn't enough to form a majority, perhaps start a new server. 634 | // The leader in a new Term may try to finish replicating log entries that 635 | // haven't been committed yet. 636 | // 637 | //func TestFigure8(t *testing.T) { 638 | // servers := 5 639 | // cfg := make_config(t, servers, false) 640 | // defer cfg.cleanup() 641 | // 642 | // fmt.Printf("Test: Figure 8 ...\n") 643 | // 644 | // cfg.one(rand.Int(), 1) 645 | // 646 | // nup := servers 647 | // for iters := 0; iters < 1000; iters++ { 648 | // leader := -1 649 | // for i := 0; i < servers; i++ { 650 | // if cfg.rafts[i] != nil { 651 | // _, _, ok := cfg.rafts[i].Start(rand.Int()) 652 | // if ok { 653 | // leader = i 654 | // } 655 | // } 656 | // } 657 | // 658 | // if (rand.Int() % 1000) < 100 { 659 | // ms := rand.Int63() % (int64(RaftElectionTimeout/time.Millisecond) / 2) 660 | // time.Sleep(time.Duration(ms) * time.Millisecond) 661 | // } else { 662 | // ms := (rand.Int63() % 13) 663 | // time.Sleep(time.Duration(ms) * time.Millisecond) 664 | // } 665 | // 666 | // if leader != -1 { 667 | // cfg.crash1(leader) 668 | // nup -= 1 669 | // } 670 | // 671 | // if nup < 3 { 672 | // s := rand.Int() % servers 673 | // if cfg.rafts[s] == nil { 674 | // cfg.start1(s) 675 | // cfg.connect(s) 676 | // nup += 1 677 | // } 678 | // } 679 | // } 680 | // 681 | // for i := 0; i < servers; i++ { 682 | // if cfg.rafts[i] == nil { 683 | // cfg.start1(i) 684 | // cfg.connect(i) 685 | // } 686 | // } 687 | // 688 | // cfg.one(rand.Int(), servers) 689 | // 690 | // fmt.Printf(" ... Passed\n") 691 | //} 692 | // 693 | //func TestUnreliableAgree(t *testing.T) { 694 | // servers := 5 695 | // cfg := make_config(t, servers, true) 696 | // defer cfg.cleanup() 697 | // 698 | // fmt.Printf("Test: unreliable agreement ...\n") 699 | // 700 | // var wg sync.WaitGroup 701 | // 702 | // for iters := 1; iters < 50; iters++ { 703 | // for j := 0; j < 4; j++ { 704 | // wg.Add(1) 705 | // go func(iters, j int) { 706 | // defer wg.Done() 707 | // cfg.one((100*iters)+j, 1) 708 | // }(iters, j) 709 | // } 710 | // cfg.one(iters, 1) 711 | // } 712 | // 713 | // cfg.setunreliable(false) 714 | // 715 | // wg.Wait() 716 | // 717 | // cfg.one(100, servers) 718 | // 719 | // fmt.Printf(" ... Passed\n") 720 | //} 721 | // 722 | //func TestFigure8Unreliable(t *testing.T) { 723 | // servers := 5 724 | // cfg := make_config(t, servers, true) 725 | // defer cfg.cleanup() 726 | // 727 | // fmt.Printf("Test: Figure 8 (unreliable) ...\n") 728 | // 729 | // cfg.one(rand.Int()%10000, 1) 730 | // 731 | // nup := servers 732 | // for iters := 0; iters < 1000; iters++ { 733 | // if iters == 200 { 734 | // cfg.setlongreordering(true) 735 | // } 736 | // leader := -1 737 | // for i := 0; i < servers; i++ { 738 | // _, _, ok := cfg.rafts[i].Start(rand.Int() % 10000) 739 | // if ok && cfg.connected[i] { 740 | // leader = i 741 | // } 742 | // } 743 | // 744 | // if (rand.Int() % 1000) < 100 { 745 | // ms := rand.Int63() % (int64(RaftElectionTimeout/time.Millisecond) / 2) 746 | // time.Sleep(time.Duration(ms) * time.Millisecond) 747 | // } else { 748 | // ms := (rand.Int63() % 13) 749 | // time.Sleep(time.Duration(ms) * time.Millisecond) 750 | // } 751 | // 752 | // if leader != -1 && (rand.Int()%1000) < int(RaftElectionTimeout/time.Millisecond)/2 { 753 | // cfg.disconnect(leader) 754 | // nup -= 1 755 | // } 756 | // 757 | // if nup < 3 { 758 | // s := rand.Int() % servers 759 | // if cfg.connected[s] == false { 760 | // cfg.connect(s) 761 | // nup += 1 762 | // } 763 | // } 764 | // } 765 | // 766 | // for i := 0; i < servers; i++ { 767 | // if cfg.connected[i] == false { 768 | // cfg.connect(i) 769 | // } 770 | // } 771 | // 772 | // cfg.one(rand.Int()%10000, servers) 773 | // 774 | // fmt.Printf(" ... Passed\n") 775 | //} 776 | // 777 | //func internalChurn(t *testing.T, unreliable bool) { 778 | // 779 | // if unreliable { 780 | // fmt.Printf("Test: unreliable churn ...\n") 781 | // } else { 782 | // fmt.Printf("Test: churn ...\n") 783 | // } 784 | // 785 | // servers := 5 786 | // cfg := make_config(t, servers, unreliable) 787 | // defer cfg.cleanup() 788 | // 789 | // stop := int32(0) 790 | // 791 | // // create concurrent clients 792 | // cfn := func(me int, ch chan []int) { 793 | // var ret []int 794 | // ret = nil 795 | // defer func() { ch <- ret }() 796 | // values := []int{} 797 | // for atomic.LoadInt32(&stop) == 0 { 798 | // x := rand.Int() 799 | // index := -1 800 | // ok := false 801 | // for i := 0; i < servers; i++ { 802 | // // try them all, maybe one of them is a leader 803 | // cfg.mu.Lock() 804 | // rf := cfg.rafts[i] 805 | // cfg.mu.Unlock() 806 | // if rf != nil { 807 | // index1, _, ok1 := rf.Start(x) 808 | // if ok1 { 809 | // ok = ok1 810 | // index = index1 811 | // } 812 | // } 813 | // } 814 | // if ok { 815 | // // maybe leader will commit our value, maybe not. 816 | // // but don't wait forever. 817 | // for _, to := range []int{10, 20, 50, 100, 200} { 818 | // nd, cmd := cfg.nCommitted(index) 819 | // if nd > 0 { 820 | // if xx, ok := cmd.(int); ok { 821 | // if xx == x { 822 | // values = append(values, x) 823 | // } 824 | // } else { 825 | // cfg.t.Fatalf("wrong command type") 826 | // } 827 | // break 828 | // } 829 | // time.Sleep(time.Duration(to) * time.Millisecond) 830 | // } 831 | // } else { 832 | // time.Sleep(time.Duration(79+me*17) * time.Millisecond) 833 | // } 834 | // } 835 | // ret = values 836 | // } 837 | // 838 | // ncli := 3 839 | // cha := []chan []int{} 840 | // for i := 0; i < ncli; i++ { 841 | // cha = append(cha, make(chan []int)) 842 | // go cfn(i, cha[i]) 843 | // } 844 | // 845 | // for iters := 0; iters < 20; iters++ { 846 | // if (rand.Int() % 1000) < 200 { 847 | // i := rand.Int() % servers 848 | // cfg.disconnect(i) 849 | // } 850 | // 851 | // if (rand.Int() % 1000) < 500 { 852 | // i := rand.Int() % servers 853 | // if cfg.rafts[i] == nil { 854 | // cfg.start1(i) 855 | // } 856 | // cfg.connect(i) 857 | // } 858 | // 859 | // if (rand.Int() % 1000) < 200 { 860 | // i := rand.Int() % servers 861 | // if cfg.rafts[i] != nil { 862 | // cfg.crash1(i) 863 | // } 864 | // } 865 | // 866 | // // Make crash/restart infrequent enough that the peers can often 867 | // // keep up, but not so infrequent that everything has settled 868 | // // down from one change to the next. Pick a value smaller than 869 | // // the election timeout, but not hugely smaller. 870 | // time.Sleep((RaftElectionTimeout * 7) / 10) 871 | // } 872 | // 873 | // time.Sleep(RaftElectionTimeout) 874 | // cfg.setunreliable(false) 875 | // for i := 0; i < servers; i++ { 876 | // if cfg.rafts[i] == nil { 877 | // cfg.start1(i) 878 | // } 879 | // cfg.connect(i) 880 | // } 881 | // 882 | // atomic.StoreInt32(&stop, 1) 883 | // 884 | // values := []int{} 885 | // for i := 0; i < ncli; i++ { 886 | // vv := <-cha[i] 887 | // if vv == nil { 888 | // t.Fatal("client failed") 889 | // } 890 | // values = append(values, vv...) 891 | // } 892 | // 893 | // time.Sleep(RaftElectionTimeout) 894 | // 895 | // lastIndex := cfg.one(rand.Int(), servers) 896 | // 897 | // really := make([]int, lastIndex+1) 898 | // for index := 1; index <= lastIndex; index++ { 899 | // v := cfg.wait(index, servers, -1) 900 | // if vi, ok := v.(int); ok { 901 | // really = append(really, vi) 902 | // } else { 903 | // t.Fatalf("not an int") 904 | // } 905 | // } 906 | // 907 | // for _, v1 := range values { 908 | // ok := false 909 | // for _, v2 := range really { 910 | // if v1 == v2 { 911 | // ok = true 912 | // } 913 | // } 914 | // if ok == false { 915 | // cfg.t.Fatalf("didn't find a value") 916 | // } 917 | // } 918 | // 919 | // fmt.Printf(" ... Passed\n") 920 | //} 921 | // 922 | //func TestReliableChurn(t *testing.T) { 923 | // internalChurn(t, false) 924 | //} 925 | // 926 | //func TestUnreliableChurn(t *testing.T) { 927 | // internalChurn(t, true) 928 | //} 929 | -------------------------------------------------------------------------------- /src/raft/util.go: -------------------------------------------------------------------------------- 1 | package raft 2 | 3 | import "log" 4 | 5 | // Debugging 6 | const Debug = 0 7 | 8 | func DPrintf(format string, a ...interface{}) (n int, err error) { 9 | if Debug > 0 { 10 | log.Printf(format, a...) 11 | } 12 | return 13 | } 14 | --------------------------------------------------------------------------------