├── README.md └── src ├── .gitignore ├── diskv ├── client.go ├── common.go ├── server.go └── test_test.go ├── kvpaxos ├── client.go ├── common.go ├── server.go └── test_test.go ├── lockservice ├── client.go ├── common.go ├── server.go └── test_test.go ├── main ├── diskvd.go ├── kjv12.txt ├── lockc.go ├── lockd.go ├── mr-testout.txt ├── pbc.go ├── pbd.go ├── test-wc.sh ├── viewd.go └── wc.go ├── mapreduce ├── common.go ├── mapreduce.go ├── master.go ├── test_test.go └── worker.go ├── paxos ├── paxos.go └── test_test.go ├── pbservice ├── client.go ├── common.go ├── server.go └── test_test.go ├── shardkv ├── client.go ├── common.go ├── server.go └── test_test.go ├── shardmaster ├── client.go ├── common.go ├── server.go └── test_test.go └── viewservice ├── client.go ├── common.go ├── server.go └── test_test.go /README.md: -------------------------------------------------------------------------------- 1 | Please see the instructions for the labs [here](http://news.cs.nyu.edu/~jinyang/fa16-ds/labs/). 2 | -------------------------------------------------------------------------------- /src/.gitignore: -------------------------------------------------------------------------------- 1 | mrtmp.* 2 | /main/diff.out 3 | /mapreduce/x.txt 4 | /pbservice/x.txt 5 | /kvpaxos/x.txt 6 | -------------------------------------------------------------------------------- /src/diskv/client.go: -------------------------------------------------------------------------------- 1 | package diskv 2 | 3 | import "shardmaster" 4 | import "net/rpc" 5 | import "time" 6 | import "sync" 7 | import "fmt" 8 | import "crypto/rand" 9 | import "math/big" 10 | 11 | type Clerk struct { 12 | mu sync.Mutex // one RPC at a time 13 | sm *shardmaster.Clerk 14 | config shardmaster.Config 15 | // You'll have to modify Clerk. 16 | } 17 | 18 | func nrand() int64 { 19 | max := big.NewInt(int64(1) << 62) 20 | bigx, _ := rand.Int(rand.Reader, max) 21 | x := bigx.Int64() 22 | return x 23 | } 24 | 25 | func MakeClerk(shardmasters []string) *Clerk { 26 | ck := new(Clerk) 27 | ck.sm = shardmaster.MakeClerk(shardmasters) 28 | // You'll have to modify MakeClerk. 29 | return ck 30 | } 31 | 32 | // 33 | // call() sends an RPC to the rpcname handler on server srv 34 | // with arguments args, waits for the reply, and leaves the 35 | // reply in reply. the reply argument should be a pointer 36 | // to a reply structure. 37 | // 38 | // the return value is true if the server responded, and false 39 | // if call() was not able to contact the server. in particular, 40 | // the reply's contents are only valid if call() returned true. 41 | // 42 | // you should assume that call() will return an 43 | // error after a while if the server is dead. 44 | // don't provide your own time-out mechanism. 45 | // 46 | // please use call() to send all RPCs, in client.go and server.go. 47 | // please don't change this function. 48 | // 49 | func call(srv string, rpcname string, 50 | args interface{}, reply interface{}) bool { 51 | c, errx := rpc.Dial("unix", srv) 52 | if errx != nil { 53 | return false 54 | } 55 | defer c.Close() 56 | 57 | err := c.Call(rpcname, args, reply) 58 | if err == nil { 59 | return true 60 | } 61 | 62 | fmt.Println(err) 63 | return false 64 | } 65 | 66 | // 67 | // which shard is a key in? 68 | // please use this function, 69 | // and please do not change it. 70 | // 71 | func key2shard(key string) int { 72 | shard := 0 73 | if len(key) > 0 { 74 | shard = int(key[0]) 75 | } 76 | shard %= shardmaster.NShards 77 | return shard 78 | } 79 | 80 | // 81 | // fetch the current value for a key. 82 | // returns "" if the key does not exist. 83 | // keeps trying forever in the face of all other errors. 84 | // 85 | func (ck *Clerk) Get(key string) string { 86 | ck.mu.Lock() 87 | defer ck.mu.Unlock() 88 | 89 | // You'll have to modify Get(). 90 | 91 | for { 92 | shard := key2shard(key) 93 | 94 | gid := ck.config.Shards[shard] 95 | 96 | servers, ok := ck.config.Groups[gid] 97 | 98 | if ok { 99 | // try each server in the shard's replication group. 100 | for _, srv := range servers { 101 | args := &GetArgs{} 102 | args.Key = key 103 | var reply GetReply 104 | ok := call(srv, "DisKV.Get", args, &reply) 105 | if ok && (reply.Err == OK || reply.Err == ErrNoKey) { 106 | return reply.Value 107 | } 108 | if ok && (reply.Err == ErrWrongGroup) { 109 | break 110 | } 111 | } 112 | } 113 | 114 | time.Sleep(100 * time.Millisecond) 115 | 116 | // ask master for a new configuration. 117 | ck.config = ck.sm.Query(-1) 118 | } 119 | } 120 | 121 | // send a Put or Append request. 122 | func (ck *Clerk) PutAppend(key string, value string, op string) { 123 | ck.mu.Lock() 124 | defer ck.mu.Unlock() 125 | 126 | // You'll have to modify PutAppend(). 127 | 128 | for { 129 | shard := key2shard(key) 130 | 131 | gid := ck.config.Shards[shard] 132 | 133 | servers, ok := ck.config.Groups[gid] 134 | 135 | if ok { 136 | // try each server in the shard's replication group. 137 | for _, srv := range servers { 138 | args := &PutAppendArgs{} 139 | args.Key = key 140 | args.Value = value 141 | args.Op = op 142 | var reply PutAppendReply 143 | ok := call(srv, "DisKV.PutAppend", args, &reply) 144 | if ok && reply.Err == OK { 145 | return 146 | } 147 | if ok && (reply.Err == ErrWrongGroup) { 148 | break 149 | } 150 | } 151 | } 152 | 153 | time.Sleep(100 * time.Millisecond) 154 | 155 | // ask master for a new configuration. 156 | ck.config = ck.sm.Query(-1) 157 | } 158 | } 159 | 160 | func (ck *Clerk) Put(key string, value string) { 161 | ck.PutAppend(key, value, "Put") 162 | } 163 | func (ck *Clerk) Append(key string, value string) { 164 | ck.PutAppend(key, value, "Append") 165 | } 166 | -------------------------------------------------------------------------------- /src/diskv/common.go: -------------------------------------------------------------------------------- 1 | package diskv 2 | 3 | // 4 | // Sharded key/value server. 5 | // Lots of replica groups, each running op-at-a-time paxos. 6 | // Shardmaster decides which group serves each shard. 7 | // Shardmaster may change shard assignment from time to time. 8 | // 9 | // You will have to modify these definitions. 10 | // 11 | 12 | const ( 13 | OK = "OK" 14 | ErrNoKey = "ErrNoKey" 15 | ErrWrongGroup = "ErrWrongGroup" 16 | ) 17 | 18 | type Err string 19 | 20 | type PutAppendArgs struct { 21 | Key string 22 | Value string 23 | Op string // "Put" or "Append" 24 | // You'll have to add definitions here. 25 | // Field names must start with capital letters, 26 | // otherwise RPC will break. 27 | 28 | } 29 | 30 | type PutAppendReply struct { 31 | Err Err 32 | } 33 | 34 | type GetArgs struct { 35 | Key string 36 | // You'll have to add definitions here. 37 | } 38 | 39 | type GetReply struct { 40 | Err Err 41 | Value string 42 | } 43 | 44 | -------------------------------------------------------------------------------- /src/diskv/server.go: -------------------------------------------------------------------------------- 1 | package diskv 2 | 3 | import "net" 4 | import "fmt" 5 | import "net/rpc" 6 | import "log" 7 | import "time" 8 | import "paxos" 9 | import "sync" 10 | import "sync/atomic" 11 | import "os" 12 | import "syscall" 13 | import "encoding/gob" 14 | import "encoding/base32" 15 | import "math/rand" 16 | import "shardmaster" 17 | import "io/ioutil" 18 | import "strconv" 19 | 20 | 21 | const Debug = 0 22 | 23 | func DPrintf(format string, a ...interface{}) (n int, err error) { 24 | if Debug > 0 { 25 | log.Printf(format, a...) 26 | } 27 | return 28 | } 29 | 30 | 31 | type Op struct { 32 | // Your definitions here. 33 | } 34 | 35 | 36 | type DisKV struct { 37 | mu sync.Mutex 38 | l net.Listener 39 | me int 40 | dead int32 // for testing 41 | unreliable int32 // for testing 42 | sm *shardmaster.Clerk 43 | px *paxos.Paxos 44 | dir string // each replica has its own data directory 45 | 46 | gid int64 // my replica group ID 47 | 48 | // Your definitions here. 49 | } 50 | 51 | // 52 | // these are handy functions that might be useful 53 | // for reading and writing key/value files, and 54 | // for reading and writing entire shards. 55 | // puts the key files for each shard in a separate 56 | // directory. 57 | // 58 | 59 | func (kv *DisKV) shardDir(shard int) string { 60 | d := kv.dir + "/shard-" + strconv.Itoa(shard) + "/" 61 | // create directory if needed. 62 | _, err := os.Stat(d) 63 | if err != nil { 64 | if err := os.Mkdir(d, 0777); err != nil { 65 | log.Fatalf("Mkdir(%v): %v", d, err) 66 | } 67 | } 68 | return d 69 | } 70 | 71 | // cannot use keys in file names directly, since 72 | // they might contain troublesome characters like /. 73 | // base32-encode the key to get a file name. 74 | // base32 rather than base64 b/c Mac has case-insensitive 75 | // file names. 76 | func (kv *DisKV) encodeKey(key string) string { 77 | return base32.StdEncoding.EncodeToString([]byte(key)) 78 | } 79 | 80 | func (kv *DisKV) decodeKey(filename string) (string, error) { 81 | key, err := base32.StdEncoding.DecodeString(filename) 82 | return string(key), err 83 | } 84 | 85 | // read the content of a key's file. 86 | func (kv *DisKV) fileGet(shard int, key string) (string, error) { 87 | fullname := kv.shardDir(shard) + "/key-" + kv.encodeKey(key) 88 | content, err := ioutil.ReadFile(fullname) 89 | return string(content), err 90 | } 91 | 92 | // replace the content of a key's file. 93 | // uses rename() to make the replacement atomic with 94 | // respect to crashes. 95 | func (kv *DisKV) filePut(shard int, key string, content string) error { 96 | fullname := kv.shardDir(shard) + "/key-" + kv.encodeKey(key) 97 | tempname := kv.shardDir(shard) + "/temp-" + kv.encodeKey(key) 98 | if err := ioutil.WriteFile(tempname, []byte(content), 0666); err != nil { 99 | return err 100 | } 101 | if err := os.Rename(tempname, fullname); err != nil { 102 | return err 103 | } 104 | return nil 105 | } 106 | 107 | // return content of every key file in a given shard. 108 | func (kv *DisKV) fileReadShard(shard int) map[string]string { 109 | m := map[string]string{} 110 | d := kv.shardDir(shard) 111 | files, err := ioutil.ReadDir(d) 112 | if err != nil { 113 | log.Fatalf("fileReadShard could not read %v: %v", d, err) 114 | } 115 | for _, fi := range files { 116 | n1 := fi.Name() 117 | if n1[0:4] == "key-" { 118 | key, err := kv.decodeKey(n1[4:]) 119 | if err != nil { 120 | log.Fatalf("fileReadShard bad file name %v: %v", n1, err) 121 | } 122 | content, err := kv.fileGet(shard, key) 123 | if err != nil { 124 | log.Fatalf("fileReadShard fileGet failed for %v: %v", key, err) 125 | } 126 | m[key] = content 127 | } 128 | } 129 | return m 130 | } 131 | 132 | // replace an entire shard directory. 133 | func (kv *DisKV) fileReplaceShard(shard int, m map[string]string) { 134 | d := kv.shardDir(shard) 135 | os.RemoveAll(d) // remove all existing files from shard. 136 | for k, v := range m { 137 | kv.filePut(shard, k, v) 138 | } 139 | } 140 | 141 | 142 | func (kv *DisKV) Get(args *GetArgs, reply *GetReply) error { 143 | // Your code here. 144 | return nil 145 | } 146 | 147 | // RPC handler for client Put and Append requests 148 | func (kv *DisKV) PutAppend(args *PutAppendArgs, reply *PutAppendReply) error { 149 | // Your code here. 150 | return nil 151 | } 152 | 153 | // 154 | // Ask the shardmaster if there's a new configuration; 155 | // if so, re-configure. 156 | // 157 | func (kv *DisKV) tick() { 158 | // Your code here. 159 | } 160 | 161 | // tell the server to shut itself down. 162 | // please don't change these two functions. 163 | func (kv *DisKV) kill() { 164 | atomic.StoreInt32(&kv.dead, 1) 165 | kv.l.Close() 166 | kv.px.Kill() 167 | } 168 | 169 | // call this to find out if the server is dead. 170 | func (kv *DisKV) isdead() bool { 171 | return atomic.LoadInt32(&kv.dead) != 0 172 | } 173 | 174 | // please do not change these two functions. 175 | func (kv *DisKV) Setunreliable(what bool) { 176 | if what { 177 | atomic.StoreInt32(&kv.unreliable, 1) 178 | } else { 179 | atomic.StoreInt32(&kv.unreliable, 0) 180 | } 181 | } 182 | 183 | func (kv *DisKV) isunreliable() bool { 184 | return atomic.LoadInt32(&kv.unreliable) != 0 185 | } 186 | 187 | // 188 | // Start a shardkv server. 189 | // gid is the ID of the server's replica group. 190 | // shardmasters[] contains the ports of the 191 | // servers that implement the shardmaster. 192 | // servers[] contains the ports of the servers 193 | // in this replica group. 194 | // Me is the index of this server in servers[]. 195 | // dir is the directory name under which this 196 | // replica should store all its files. 197 | // each replica is passed a different directory. 198 | // restart is false the very first time this server 199 | // is started, and true to indicate a re-start 200 | // after a crash or after a crash with disk loss. 201 | // 202 | func StartServer(gid int64, shardmasters []string, 203 | servers []string, me int, dir string, restart bool) *DisKV { 204 | 205 | kv := new(DisKV) 206 | kv.me = me 207 | kv.gid = gid 208 | kv.sm = shardmaster.MakeClerk(shardmasters) 209 | kv.dir = dir 210 | 211 | // Your initialization code here. 212 | // Don't call Join(). 213 | 214 | // log.SetOutput(ioutil.Discard) 215 | 216 | gob.Register(Op{}) 217 | 218 | rpcs := rpc.NewServer() 219 | rpcs.Register(kv) 220 | 221 | kv.px = paxos.Make(servers, me, rpcs) 222 | 223 | // log.SetOutput(os.Stdout) 224 | 225 | 226 | 227 | os.Remove(servers[me]) 228 | l, e := net.Listen("unix", servers[me]) 229 | if e != nil { 230 | log.Fatal("listen error: ", e) 231 | } 232 | kv.l = l 233 | 234 | // please do not change any of the following code, 235 | // or do anything to subvert it. 236 | 237 | go func() { 238 | for kv.isdead() == false { 239 | conn, err := kv.l.Accept() 240 | if err == nil && kv.isdead() == false { 241 | if kv.isunreliable() && (rand.Int63()%1000) < 100 { 242 | // discard the request. 243 | conn.Close() 244 | } else if kv.isunreliable() && (rand.Int63()%1000) < 200 { 245 | // process the request but force discard of reply. 246 | c1 := conn.(*net.UnixConn) 247 | f, _ := c1.File() 248 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 249 | if err != nil { 250 | fmt.Printf("shutdown: %v\n", err) 251 | } 252 | go rpcs.ServeConn(conn) 253 | } else { 254 | go rpcs.ServeConn(conn) 255 | } 256 | } else if err == nil { 257 | conn.Close() 258 | } 259 | if err != nil && kv.isdead() == false { 260 | fmt.Printf("DisKV(%v) accept: %v\n", me, err.Error()) 261 | kv.kill() 262 | } 263 | } 264 | }() 265 | 266 | go func() { 267 | for kv.isdead() == false { 268 | kv.tick() 269 | time.Sleep(250 * time.Millisecond) 270 | } 271 | }() 272 | 273 | return kv 274 | } 275 | -------------------------------------------------------------------------------- /src/kvpaxos/client.go: -------------------------------------------------------------------------------- 1 | package kvpaxos 2 | 3 | import "net/rpc" 4 | import "crypto/rand" 5 | import "math/big" 6 | 7 | import "fmt" 8 | 9 | type Clerk struct { 10 | servers []string 11 | // You will have to modify this struct. 12 | } 13 | 14 | func nrand() int64 { 15 | max := big.NewInt(int64(1) << 62) 16 | bigx, _ := rand.Int(rand.Reader, max) 17 | x := bigx.Int64() 18 | return x 19 | } 20 | 21 | func MakeClerk(servers []string) *Clerk { 22 | ck := new(Clerk) 23 | ck.servers = servers 24 | // You'll have to add code here. 25 | return ck 26 | } 27 | 28 | // 29 | // call() sends an RPC to the rpcname handler on server srv 30 | // with arguments args, waits for the reply, and leaves the 31 | // reply in reply. the reply argument should be a pointer 32 | // to a reply structure. 33 | // 34 | // the return value is true if the server responded, and false 35 | // if call() was not able to contact the server. in particular, 36 | // the reply's contents are only valid if call() returned true. 37 | // 38 | // you should assume that call() will return an 39 | // error after a while if the server is dead. 40 | // don't provide your own time-out mechanism. 41 | // 42 | // please use call() to send all RPCs, in client.go and server.go. 43 | // please don't change this function. 44 | // 45 | func call(srv string, rpcname string, 46 | args interface{}, reply interface{}) bool { 47 | c, errx := rpc.Dial("unix", srv) 48 | if errx != nil { 49 | return false 50 | } 51 | defer c.Close() 52 | 53 | err := c.Call(rpcname, args, reply) 54 | if err == nil { 55 | return true 56 | } 57 | 58 | fmt.Println(err) 59 | return false 60 | } 61 | 62 | // 63 | // fetch the current value for a key. 64 | // returns "" if the key does not exist. 65 | // keeps trying forever in the face of all other errors. 66 | // 67 | func (ck *Clerk) Get(key string) string { 68 | // You will have to modify this function. 69 | return "" 70 | } 71 | 72 | // 73 | // shared by Put and Append. 74 | // 75 | func (ck *Clerk) PutAppend(key string, value string, op string) { 76 | // You will have to modify this function. 77 | } 78 | 79 | func (ck *Clerk) Put(key string, value string) { 80 | ck.PutAppend(key, value, "Put") 81 | } 82 | func (ck *Clerk) Append(key string, value string) { 83 | ck.PutAppend(key, value, "Append") 84 | } 85 | -------------------------------------------------------------------------------- /src/kvpaxos/common.go: -------------------------------------------------------------------------------- 1 | package kvpaxos 2 | 3 | const ( 4 | OK = "OK" 5 | ErrNoKey = "ErrNoKey" 6 | ) 7 | 8 | type Err string 9 | 10 | // Put or Append 11 | type PutAppendArgs struct { 12 | // You'll have to add definitions here. 13 | Key string 14 | Value string 15 | Op string // "Put" or "Append" 16 | // You'll have to add definitions here. 17 | // Field names must start with capital letters, 18 | // otherwise RPC will break. 19 | } 20 | 21 | type PutAppendReply struct { 22 | Err Err 23 | } 24 | 25 | type GetArgs struct { 26 | Key string 27 | // You'll have to add definitions here. 28 | } 29 | 30 | type GetReply struct { 31 | Err Err 32 | Value string 33 | } 34 | -------------------------------------------------------------------------------- /src/kvpaxos/server.go: -------------------------------------------------------------------------------- 1 | package kvpaxos 2 | 3 | import "net" 4 | import "fmt" 5 | import "net/rpc" 6 | import "log" 7 | import "paxos" 8 | import "sync" 9 | import "sync/atomic" 10 | import "os" 11 | import "syscall" 12 | import "encoding/gob" 13 | import "math/rand" 14 | 15 | 16 | const Debug = 0 17 | 18 | func DPrintf(format string, a ...interface{}) (n int, err error) { 19 | if Debug > 0 { 20 | log.Printf(format, a...) 21 | } 22 | return 23 | } 24 | 25 | 26 | type Op struct { 27 | // Your definitions here. 28 | // Field names must start with capital letters, 29 | // otherwise RPC will break. 30 | } 31 | 32 | type KVPaxos struct { 33 | mu sync.Mutex 34 | l net.Listener 35 | me int 36 | dead int32 // for testing 37 | unreliable int32 // for testing 38 | px *paxos.Paxos 39 | 40 | // Your definitions here. 41 | } 42 | 43 | 44 | func (kv *KVPaxos) Get(args *GetArgs, reply *GetReply) error { 45 | // Your code here. 46 | return nil 47 | } 48 | 49 | func (kv *KVPaxos) PutAppend(args *PutAppendArgs, reply *PutAppendReply) error { 50 | // Your code here. 51 | 52 | return nil 53 | } 54 | 55 | // tell the server to shut itself down. 56 | // please do not change these two functions. 57 | func (kv *KVPaxos) kill() { 58 | DPrintf("Kill(%d): die\n", kv.me) 59 | atomic.StoreInt32(&kv.dead, 1) 60 | kv.l.Close() 61 | kv.px.Kill() 62 | } 63 | 64 | // call this to find out if the server is dead. 65 | func (kv *KVPaxos) isdead() bool { 66 | return atomic.LoadInt32(&kv.dead) != 0 67 | } 68 | 69 | // please do not change these two functions. 70 | func (kv *KVPaxos) setunreliable(what bool) { 71 | if what { 72 | atomic.StoreInt32(&kv.unreliable, 1) 73 | } else { 74 | atomic.StoreInt32(&kv.unreliable, 0) 75 | } 76 | } 77 | 78 | func (kv *KVPaxos) isunreliable() bool { 79 | return atomic.LoadInt32(&kv.unreliable) != 0 80 | } 81 | 82 | // 83 | // servers[] contains the ports of the set of 84 | // servers that will cooperate via Paxos to 85 | // form the fault-tolerant key/value service. 86 | // me is the index of the current server in servers[]. 87 | // 88 | func StartServer(servers []string, me int) *KVPaxos { 89 | // call gob.Register on structures you want 90 | // Go's RPC library to marshall/unmarshall. 91 | gob.Register(Op{}) 92 | 93 | kv := new(KVPaxos) 94 | kv.me = me 95 | 96 | // Your initialization code here. 97 | 98 | rpcs := rpc.NewServer() 99 | rpcs.Register(kv) 100 | 101 | kv.px = paxos.Make(servers, me, rpcs) 102 | 103 | os.Remove(servers[me]) 104 | l, e := net.Listen("unix", servers[me]) 105 | if e != nil { 106 | log.Fatal("listen error: ", e) 107 | } 108 | kv.l = l 109 | 110 | 111 | // please do not change any of the following code, 112 | // or do anything to subvert it. 113 | 114 | go func() { 115 | for kv.isdead() == false { 116 | conn, err := kv.l.Accept() 117 | if err == nil && kv.isdead() == false { 118 | if kv.isunreliable() && (rand.Int63()%1000) < 100 { 119 | // discard the request. 120 | conn.Close() 121 | } else if kv.isunreliable() && (rand.Int63()%1000) < 200 { 122 | // process the request but force discard of reply. 123 | c1 := conn.(*net.UnixConn) 124 | f, _ := c1.File() 125 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 126 | if err != nil { 127 | fmt.Printf("shutdown: %v\n", err) 128 | } 129 | go rpcs.ServeConn(conn) 130 | } else { 131 | go rpcs.ServeConn(conn) 132 | } 133 | } else if err == nil { 134 | conn.Close() 135 | } 136 | if err != nil && kv.isdead() == false { 137 | fmt.Printf("KVPaxos(%v) accept: %v\n", me, err.Error()) 138 | kv.kill() 139 | } 140 | } 141 | }() 142 | 143 | return kv 144 | } 145 | -------------------------------------------------------------------------------- /src/kvpaxos/test_test.go: -------------------------------------------------------------------------------- 1 | package kvpaxos 2 | 3 | import "testing" 4 | import "runtime" 5 | import "strconv" 6 | import "os" 7 | import "time" 8 | import "fmt" 9 | import "math/rand" 10 | import "strings" 11 | import "sync/atomic" 12 | 13 | func check(t *testing.T, ck *Clerk, key string, value string) { 14 | v := ck.Get(key) 15 | if v != value { 16 | t.Fatalf("Get(%v) -> %v, expected %v", key, v, value) 17 | } 18 | } 19 | 20 | func port(tag string, host int) string { 21 | s := "/var/tmp/824-" 22 | s += strconv.Itoa(os.Getuid()) + "/" 23 | os.Mkdir(s, 0777) 24 | s += "kv-" 25 | s += strconv.Itoa(os.Getpid()) + "-" 26 | s += tag + "-" 27 | s += strconv.Itoa(host) 28 | return s 29 | } 30 | 31 | func cleanup(kva []*KVPaxos) { 32 | for i := 0; i < len(kva); i++ { 33 | if kva[i] != nil { 34 | kva[i].kill() 35 | } 36 | } 37 | } 38 | 39 | // predict effect of Append(k, val) if old value is prev. 40 | func NextValue(prev string, val string) string { 41 | return prev + val 42 | } 43 | 44 | func TestBasic(t *testing.T) { 45 | runtime.GOMAXPROCS(4) 46 | 47 | const nservers = 3 48 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 49 | var kvh []string = make([]string, nservers) 50 | defer cleanup(kva) 51 | 52 | for i := 0; i < nservers; i++ { 53 | kvh[i] = port("basic", i) 54 | } 55 | for i := 0; i < nservers; i++ { 56 | kva[i] = StartServer(kvh, i) 57 | } 58 | 59 | ck := MakeClerk(kvh) 60 | var cka [nservers]*Clerk 61 | for i := 0; i < nservers; i++ { 62 | cka[i] = MakeClerk([]string{kvh[i]}) 63 | } 64 | 65 | fmt.Printf("Test: Basic put/append/get ...\n") 66 | 67 | ck.Append("app", "x") 68 | ck.Append("app", "y") 69 | check(t, ck, "app", "xy") 70 | 71 | ck.Put("a", "aa") 72 | check(t, ck, "a", "aa") 73 | 74 | cka[1].Put("a", "aaa") 75 | 76 | check(t, cka[2], "a", "aaa") 77 | check(t, cka[1], "a", "aaa") 78 | check(t, ck, "a", "aaa") 79 | 80 | fmt.Printf(" ... Passed\n") 81 | 82 | fmt.Printf("Test: Concurrent clients ...\n") 83 | 84 | for iters := 0; iters < 20; iters++ { 85 | const npara = 15 86 | var ca [npara]chan bool 87 | for nth := 0; nth < npara; nth++ { 88 | ca[nth] = make(chan bool) 89 | go func(me int) { 90 | defer func() { ca[me] <- true }() 91 | ci := (rand.Int() % nservers) 92 | myck := MakeClerk([]string{kvh[ci]}) 93 | if (rand.Int() % 1000) < 500 { 94 | myck.Put("b", strconv.Itoa(rand.Int())) 95 | } else { 96 | myck.Get("b") 97 | } 98 | }(nth) 99 | } 100 | for nth := 0; nth < npara; nth++ { 101 | <-ca[nth] 102 | } 103 | var va [nservers]string 104 | for i := 0; i < nservers; i++ { 105 | va[i] = cka[i].Get("b") 106 | if va[i] != va[0] { 107 | t.Fatalf("mismatch") 108 | } 109 | } 110 | } 111 | 112 | fmt.Printf(" ... Passed\n") 113 | 114 | time.Sleep(1 * time.Second) 115 | } 116 | 117 | func TestDone(t *testing.T) { 118 | runtime.GOMAXPROCS(4) 119 | 120 | const nservers = 3 121 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 122 | var kvh []string = make([]string, nservers) 123 | defer cleanup(kva) 124 | 125 | for i := 0; i < nservers; i++ { 126 | kvh[i] = port("done", i) 127 | } 128 | for i := 0; i < nservers; i++ { 129 | kva[i] = StartServer(kvh, i) 130 | } 131 | ck := MakeClerk(kvh) 132 | var cka [nservers]*Clerk 133 | for pi := 0; pi < nservers; pi++ { 134 | cka[pi] = MakeClerk([]string{kvh[pi]}) 135 | } 136 | 137 | fmt.Printf("Test: server frees Paxos log memory...\n") 138 | 139 | ck.Put("a", "aa") 140 | check(t, ck, "a", "aa") 141 | 142 | runtime.GC() 143 | var m0 runtime.MemStats 144 | runtime.ReadMemStats(&m0) 145 | // rtm's m0.Alloc is 2 MB 146 | 147 | sz := 1000000 148 | items := 10 149 | 150 | for iters := 0; iters < 2; iters++ { 151 | for i := 0; i < items; i++ { 152 | key := strconv.Itoa(i) 153 | value := make([]byte, sz) 154 | for j := 0; j < len(value); j++ { 155 | value[j] = byte((rand.Int() % 100) + 1) 156 | } 157 | ck.Put(key, string(value)) 158 | check(t, cka[i%nservers], key, string(value)) 159 | } 160 | } 161 | 162 | // Put and Get to each of the replicas, in case 163 | // the Done information is piggybacked on 164 | // the Paxos proposer messages. 165 | for iters := 0; iters < 2; iters++ { 166 | for pi := 0; pi < nservers; pi++ { 167 | cka[pi].Put("a", "aa") 168 | check(t, cka[pi], "a", "aa") 169 | } 170 | } 171 | 172 | time.Sleep(1 * time.Second) 173 | 174 | runtime.GC() 175 | var m1 runtime.MemStats 176 | runtime.ReadMemStats(&m1) 177 | // rtm's m1.Alloc is 45 MB 178 | 179 | // fmt.Printf(" Memory: before %v, after %v\n", m0.Alloc, m1.Alloc) 180 | 181 | allowed := m0.Alloc + uint64(nservers*items*sz*2) 182 | if m1.Alloc > allowed { 183 | t.Fatalf("Memory use did not shrink enough (Used: %v, allowed: %v).\n", m1.Alloc, allowed) 184 | } 185 | 186 | fmt.Printf(" ... Passed\n") 187 | } 188 | 189 | func pp(tag string, src int, dst int) string { 190 | s := "/var/tmp/824-" 191 | s += strconv.Itoa(os.Getuid()) + "/" 192 | s += "kv-" + tag + "-" 193 | s += strconv.Itoa(os.Getpid()) + "-" 194 | s += strconv.Itoa(src) + "-" 195 | s += strconv.Itoa(dst) 196 | return s 197 | } 198 | 199 | func cleanpp(tag string, n int) { 200 | for i := 0; i < n; i++ { 201 | for j := 0; j < n; j++ { 202 | ij := pp(tag, i, j) 203 | os.Remove(ij) 204 | } 205 | } 206 | } 207 | 208 | func part(t *testing.T, tag string, npaxos int, p1 []int, p2 []int, p3 []int) { 209 | cleanpp(tag, npaxos) 210 | 211 | pa := [][]int{p1, p2, p3} 212 | for pi := 0; pi < len(pa); pi++ { 213 | p := pa[pi] 214 | for i := 0; i < len(p); i++ { 215 | for j := 0; j < len(p); j++ { 216 | ij := pp(tag, p[i], p[j]) 217 | pj := port(tag, p[j]) 218 | err := os.Link(pj, ij) 219 | if err != nil { 220 | t.Fatalf("os.Link(%v, %v): %v\n", pj, ij, err) 221 | } 222 | } 223 | } 224 | } 225 | } 226 | 227 | func TestPartition(t *testing.T) { 228 | runtime.GOMAXPROCS(4) 229 | 230 | tag := "partition" 231 | const nservers = 5 232 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 233 | defer cleanup(kva) 234 | defer cleanpp(tag, nservers) 235 | 236 | for i := 0; i < nservers; i++ { 237 | var kvh []string = make([]string, nservers) 238 | for j := 0; j < nservers; j++ { 239 | if j == i { 240 | kvh[j] = port(tag, i) 241 | } else { 242 | kvh[j] = pp(tag, i, j) 243 | } 244 | } 245 | kva[i] = StartServer(kvh, i) 246 | } 247 | defer part(t, tag, nservers, []int{}, []int{}, []int{}) 248 | 249 | var cka [nservers]*Clerk 250 | for i := 0; i < nservers; i++ { 251 | cka[i] = MakeClerk([]string{port(tag, i)}) 252 | } 253 | 254 | fmt.Printf("Test: No partition ...\n") 255 | 256 | part(t, tag, nservers, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 257 | cka[0].Put("1", "12") 258 | cka[2].Put("1", "13") 259 | check(t, cka[3], "1", "13") 260 | 261 | fmt.Printf(" ... Passed\n") 262 | 263 | fmt.Printf("Test: Progress in majority ...\n") 264 | 265 | part(t, tag, nservers, []int{2, 3, 4}, []int{0, 1}, []int{}) 266 | cka[2].Put("1", "14") 267 | check(t, cka[4], "1", "14") 268 | 269 | fmt.Printf(" ... Passed\n") 270 | 271 | fmt.Printf("Test: No progress in minority ...\n") 272 | 273 | done0 := make(chan bool) 274 | done1 := make(chan bool) 275 | go func() { 276 | cka[0].Put("1", "15") 277 | done0 <- true 278 | }() 279 | go func() { 280 | cka[1].Get("1") 281 | done1 <- true 282 | }() 283 | 284 | select { 285 | case <-done0: 286 | t.Fatalf("Put in minority completed") 287 | case <-done1: 288 | t.Fatalf("Get in minority completed") 289 | case <-time.After(time.Second): 290 | } 291 | 292 | check(t, cka[4], "1", "14") 293 | cka[3].Put("1", "16") 294 | check(t, cka[4], "1", "16") 295 | 296 | fmt.Printf(" ... Passed\n") 297 | 298 | fmt.Printf("Test: Completion after heal ...\n") 299 | 300 | part(t, tag, nservers, []int{0, 2, 3, 4}, []int{1}, []int{}) 301 | 302 | select { 303 | case <-done0: 304 | case <-time.After(30 * 100 * time.Millisecond): 305 | t.Fatalf("Put did not complete") 306 | } 307 | 308 | select { 309 | case <-done1: 310 | t.Fatalf("Get in minority completed") 311 | default: 312 | } 313 | 314 | check(t, cka[4], "1", "15") 315 | check(t, cka[0], "1", "15") 316 | 317 | part(t, tag, nservers, []int{0, 1, 2}, []int{3, 4}, []int{}) 318 | 319 | select { 320 | case <-done1: 321 | case <-time.After(100 * 100 * time.Millisecond): 322 | t.Fatalf("Get did not complete") 323 | } 324 | 325 | check(t, cka[1], "1", "15") 326 | 327 | fmt.Printf(" ... Passed\n") 328 | } 329 | 330 | func randclerk(kvh []string) *Clerk { 331 | sa := make([]string, len(kvh)) 332 | copy(sa, kvh) 333 | for i := range sa { 334 | j := rand.Intn(i + 1) 335 | sa[i], sa[j] = sa[j], sa[i] 336 | } 337 | return MakeClerk(sa) 338 | } 339 | 340 | // check that all known appends are present in a value, 341 | // and are in order for each concurrent client. 342 | func checkAppends(t *testing.T, v string, counts []int) { 343 | nclients := len(counts) 344 | for i := 0; i < nclients; i++ { 345 | lastoff := -1 346 | for j := 0; j < counts[i]; j++ { 347 | wanted := "x " + strconv.Itoa(i) + " " + strconv.Itoa(j) + " y" 348 | off := strings.Index(v, wanted) 349 | if off < 0 { 350 | t.Fatalf("missing element in Append result") 351 | } 352 | off1 := strings.LastIndex(v, wanted) 353 | if off1 != off { 354 | t.Fatalf("duplicate element in Append result") 355 | } 356 | if off <= lastoff { 357 | t.Fatalf("wrong order for element in Append result") 358 | } 359 | lastoff = off 360 | } 361 | } 362 | } 363 | 364 | func TestUnreliable(t *testing.T) { 365 | runtime.GOMAXPROCS(4) 366 | 367 | const nservers = 3 368 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 369 | var kvh []string = make([]string, nservers) 370 | defer cleanup(kva) 371 | 372 | for i := 0; i < nservers; i++ { 373 | kvh[i] = port("un", i) 374 | } 375 | for i := 0; i < nservers; i++ { 376 | kva[i] = StartServer(kvh, i) 377 | kva[i].setunreliable(true) 378 | } 379 | 380 | ck := MakeClerk(kvh) 381 | var cka [nservers]*Clerk 382 | for i := 0; i < nservers; i++ { 383 | cka[i] = MakeClerk([]string{kvh[i]}) 384 | } 385 | 386 | fmt.Printf("Test: Basic put/get, unreliable ...\n") 387 | 388 | ck.Put("a", "aa") 389 | check(t, ck, "a", "aa") 390 | 391 | cka[1].Put("a", "aaa") 392 | 393 | check(t, cka[2], "a", "aaa") 394 | check(t, cka[1], "a", "aaa") 395 | check(t, ck, "a", "aaa") 396 | 397 | fmt.Printf(" ... Passed\n") 398 | 399 | fmt.Printf("Test: Sequence of puts, unreliable ...\n") 400 | 401 | for iters := 0; iters < 6; iters++ { 402 | const ncli = 5 403 | var ca [ncli]chan bool 404 | for cli := 0; cli < ncli; cli++ { 405 | ca[cli] = make(chan bool) 406 | go func(me int) { 407 | ok := false 408 | defer func() { ca[me] <- ok }() 409 | myck := randclerk(kvh) 410 | key := strconv.Itoa(me) 411 | vv := myck.Get(key) 412 | myck.Append(key, "0") 413 | vv = NextValue(vv, "0") 414 | myck.Append(key, "1") 415 | vv = NextValue(vv, "1") 416 | myck.Append(key, "2") 417 | vv = NextValue(vv, "2") 418 | time.Sleep(100 * time.Millisecond) 419 | if myck.Get(key) != vv { 420 | t.Fatalf("wrong value") 421 | } 422 | if myck.Get(key) != vv { 423 | t.Fatalf("wrong value") 424 | } 425 | ok = true 426 | }(cli) 427 | } 428 | for cli := 0; cli < ncli; cli++ { 429 | x := <-ca[cli] 430 | if x == false { 431 | t.Fatalf("failure") 432 | } 433 | } 434 | } 435 | 436 | fmt.Printf(" ... Passed\n") 437 | 438 | fmt.Printf("Test: Concurrent clients, unreliable ...\n") 439 | 440 | for iters := 0; iters < 20; iters++ { 441 | const ncli = 15 442 | var ca [ncli]chan bool 443 | for cli := 0; cli < ncli; cli++ { 444 | ca[cli] = make(chan bool) 445 | go func(me int) { 446 | defer func() { ca[me] <- true }() 447 | myck := randclerk(kvh) 448 | if (rand.Int() % 1000) < 500 { 449 | myck.Put("b", strconv.Itoa(rand.Int())) 450 | } else { 451 | myck.Get("b") 452 | } 453 | }(cli) 454 | } 455 | for cli := 0; cli < ncli; cli++ { 456 | <-ca[cli] 457 | } 458 | 459 | var va [nservers]string 460 | for i := 0; i < nservers; i++ { 461 | va[i] = cka[i].Get("b") 462 | if va[i] != va[0] { 463 | t.Fatalf("mismatch; 0 got %v, %v got %v", va[0], i, va[i]) 464 | } 465 | } 466 | } 467 | 468 | fmt.Printf(" ... Passed\n") 469 | 470 | fmt.Printf("Test: Concurrent Append to same key, unreliable ...\n") 471 | 472 | ck.Put("k", "") 473 | 474 | ff := func(me int, ch chan int) { 475 | ret := -1 476 | defer func() { ch <- ret }() 477 | myck := randclerk(kvh) 478 | n := 0 479 | for n < 5 { 480 | myck.Append("k", "x "+strconv.Itoa(me)+" "+strconv.Itoa(n)+" y") 481 | n++ 482 | } 483 | ret = n 484 | } 485 | 486 | ncli := 5 487 | cha := []chan int{} 488 | for i := 0; i < ncli; i++ { 489 | cha = append(cha, make(chan int)) 490 | go ff(i, cha[i]) 491 | } 492 | 493 | counts := []int{} 494 | for i := 0; i < ncli; i++ { 495 | n := <-cha[i] 496 | if n < 0 { 497 | t.Fatal("client failed") 498 | } 499 | counts = append(counts, n) 500 | } 501 | 502 | vx := ck.Get("k") 503 | checkAppends(t, vx, counts) 504 | 505 | { 506 | for i := 0; i < nservers; i++ { 507 | vi := cka[i].Get("k") 508 | if vi != vx { 509 | t.Fatalf("mismatch; 0 got %v, %v got %v", vx, i, vi) 510 | } 511 | } 512 | } 513 | 514 | fmt.Printf(" ... Passed\n") 515 | 516 | time.Sleep(1 * time.Second) 517 | } 518 | 519 | func TestHole(t *testing.T) { 520 | runtime.GOMAXPROCS(4) 521 | 522 | fmt.Printf("Test: Tolerates holes in paxos sequence ...\n") 523 | 524 | tag := "hole" 525 | const nservers = 5 526 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 527 | defer cleanup(kva) 528 | defer cleanpp(tag, nservers) 529 | 530 | for i := 0; i < nservers; i++ { 531 | var kvh []string = make([]string, nservers) 532 | for j := 0; j < nservers; j++ { 533 | if j == i { 534 | kvh[j] = port(tag, i) 535 | } else { 536 | kvh[j] = pp(tag, i, j) 537 | } 538 | } 539 | kva[i] = StartServer(kvh, i) 540 | } 541 | defer part(t, tag, nservers, []int{}, []int{}, []int{}) 542 | 543 | for iters := 0; iters < 5; iters++ { 544 | part(t, tag, nservers, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 545 | 546 | ck2 := MakeClerk([]string{port(tag, 2)}) 547 | ck2.Put("q", "q") 548 | 549 | done := int32(0) 550 | const nclients = 10 551 | var ca [nclients]chan bool 552 | for xcli := 0; xcli < nclients; xcli++ { 553 | ca[xcli] = make(chan bool) 554 | go func(cli int) { 555 | ok := false 556 | defer func() { ca[cli] <- ok }() 557 | var cka [nservers]*Clerk 558 | for i := 0; i < nservers; i++ { 559 | cka[i] = MakeClerk([]string{port(tag, i)}) 560 | } 561 | key := strconv.Itoa(cli) 562 | last := "" 563 | cka[0].Put(key, last) 564 | for atomic.LoadInt32(&done) == 0 { 565 | ci := (rand.Int() % 2) 566 | if (rand.Int() % 1000) < 500 { 567 | nv := strconv.Itoa(rand.Int()) 568 | cka[ci].Put(key, nv) 569 | last = nv 570 | } else { 571 | v := cka[ci].Get(key) 572 | if v != last { 573 | t.Fatalf("%v: wrong value, key %v, wanted %v, got %v", 574 | cli, key, last, v) 575 | } 576 | } 577 | } 578 | ok = true 579 | }(xcli) 580 | } 581 | 582 | time.Sleep(3 * time.Second) 583 | 584 | part(t, tag, nservers, []int{2, 3, 4}, []int{0, 1}, []int{}) 585 | 586 | // can majority partition make progress even though 587 | // minority servers were interrupted in the middle of 588 | // paxos agreements? 589 | check(t, ck2, "q", "q") 590 | ck2.Put("q", "qq") 591 | check(t, ck2, "q", "qq") 592 | 593 | // restore network, wait for all threads to exit. 594 | part(t, tag, nservers, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 595 | atomic.StoreInt32(&done, 1) 596 | ok := true 597 | for i := 0; i < nclients; i++ { 598 | z := <-ca[i] 599 | ok = ok && z 600 | } 601 | if ok == false { 602 | t.Fatal("something is wrong") 603 | } 604 | check(t, ck2, "q", "qq") 605 | } 606 | 607 | fmt.Printf(" ... Passed\n") 608 | } 609 | 610 | func TestManyPartition(t *testing.T) { 611 | runtime.GOMAXPROCS(4) 612 | 613 | fmt.Printf("Test: Many clients, changing partitions ...\n") 614 | 615 | tag := "many" 616 | const nservers = 5 617 | var kva []*KVPaxos = make([]*KVPaxos, nservers) 618 | defer cleanup(kva) 619 | defer cleanpp(tag, nservers) 620 | 621 | for i := 0; i < nservers; i++ { 622 | var kvh []string = make([]string, nservers) 623 | for j := 0; j < nservers; j++ { 624 | if j == i { 625 | kvh[j] = port(tag, i) 626 | } else { 627 | kvh[j] = pp(tag, i, j) 628 | } 629 | } 630 | kva[i] = StartServer(kvh, i) 631 | kva[i].setunreliable(true) 632 | } 633 | defer part(t, tag, nservers, []int{}, []int{}, []int{}) 634 | part(t, tag, nservers, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 635 | 636 | done := int32(0) 637 | 638 | // re-partition periodically 639 | ch1 := make(chan bool) 640 | go func() { 641 | defer func() { ch1 <- true }() 642 | for atomic.LoadInt32(&done) == 0 { 643 | var a [nservers]int 644 | for i := 0; i < nservers; i++ { 645 | a[i] = (rand.Int() % 3) 646 | } 647 | pa := make([][]int, 3) 648 | for i := 0; i < 3; i++ { 649 | pa[i] = make([]int, 0) 650 | for j := 0; j < nservers; j++ { 651 | if a[j] == i { 652 | pa[i] = append(pa[i], j) 653 | } 654 | } 655 | } 656 | part(t, tag, nservers, pa[0], pa[1], pa[2]) 657 | time.Sleep(time.Duration(rand.Int63()%200) * time.Millisecond) 658 | } 659 | }() 660 | 661 | const nclients = 10 662 | var ca [nclients]chan bool 663 | for xcli := 0; xcli < nclients; xcli++ { 664 | ca[xcli] = make(chan bool) 665 | go func(cli int) { 666 | ok := false 667 | defer func() { ca[cli] <- ok }() 668 | sa := make([]string, nservers) 669 | for i := 0; i < nservers; i++ { 670 | sa[i] = port(tag, i) 671 | } 672 | for i := range sa { 673 | j := rand.Intn(i + 1) 674 | sa[i], sa[j] = sa[j], sa[i] 675 | } 676 | myck := MakeClerk(sa) 677 | key := strconv.Itoa(cli) 678 | last := "" 679 | myck.Put(key, last) 680 | for atomic.LoadInt32(&done) == 0 { 681 | if (rand.Int() % 1000) < 500 { 682 | nv := strconv.Itoa(rand.Int()) 683 | myck.Append(key, nv) 684 | last = NextValue(last, nv) 685 | } else { 686 | v := myck.Get(key) 687 | if v != last { 688 | t.Fatalf("%v: get wrong value, key %v, wanted %v, got %v", 689 | cli, key, last, v) 690 | } 691 | } 692 | } 693 | ok = true 694 | }(xcli) 695 | } 696 | 697 | time.Sleep(20 * time.Second) 698 | atomic.StoreInt32(&done, 1) 699 | <-ch1 700 | part(t, tag, nservers, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 701 | 702 | ok := true 703 | for i := 0; i < nclients; i++ { 704 | z := <-ca[i] 705 | ok = ok && z 706 | } 707 | 708 | if ok { 709 | fmt.Printf(" ... Passed\n") 710 | } 711 | } 712 | -------------------------------------------------------------------------------- /src/lockservice/client.go: -------------------------------------------------------------------------------- 1 | package lockservice 2 | 3 | import "net/rpc" 4 | import "fmt" 5 | 6 | 7 | // 8 | // the lockservice Clerk lives in the client 9 | // and maintains a little state. 10 | // 11 | type Clerk struct { 12 | servers [2]string // primary port, backup port 13 | // Your definitions here. 14 | } 15 | 16 | 17 | func MakeClerk(primary string, backup string) *Clerk { 18 | ck := new(Clerk) 19 | ck.servers[0] = primary 20 | ck.servers[1] = backup 21 | // Your initialization code here. 22 | return ck 23 | } 24 | 25 | // 26 | // call() sends an RPC to the rpcname handler on server srv 27 | // with arguments args, waits for the reply, and leaves the 28 | // reply in reply. the reply argument should be the address 29 | // of a reply structure. 30 | // 31 | // call() returns true if the server responded, and false 32 | // if call() was not able to contact the server. in particular, 33 | // reply's contents are valid if and only if call() returned true. 34 | // 35 | // you should assume that call() will return an 36 | // error after a while if the server is dead. 37 | // don't provide your own time-out mechanism. 38 | // 39 | // please use call() to send all RPCs, in client.go and server.go. 40 | // please don't change this function. 41 | // 42 | func call(srv string, rpcname string, 43 | args interface{}, reply interface{}) bool { 44 | c, errx := rpc.Dial("unix", srv) 45 | if errx != nil { 46 | return false 47 | } 48 | defer c.Close() 49 | 50 | err := c.Call(rpcname, args, reply) 51 | if err == nil { 52 | return true 53 | } 54 | 55 | fmt.Println(err) 56 | return false 57 | } 58 | 59 | // 60 | // ask the lock service for a lock. 61 | // returns true if the lock service 62 | // granted the lock, false otherwise. 63 | // 64 | // you will have to modify this function. 65 | // 66 | func (ck *Clerk) Lock(lockname string) bool { 67 | // prepare the arguments. 68 | args := &LockArgs{} 69 | args.Lockname = lockname 70 | var reply LockReply 71 | 72 | // send an RPC request, wait for the reply. 73 | ok := call(ck.servers[0], "LockServer.Lock", args, &reply) 74 | if ok == false { 75 | return false 76 | } 77 | 78 | return reply.OK 79 | } 80 | 81 | 82 | // 83 | // ask the lock service to unlock a lock. 84 | // returns true if the lock was previously held, 85 | // false otherwise. 86 | // 87 | 88 | func (ck *Clerk) Unlock(lockname string) bool { 89 | 90 | // Your code here. 91 | 92 | return false 93 | } 94 | -------------------------------------------------------------------------------- /src/lockservice/common.go: -------------------------------------------------------------------------------- 1 | package lockservice 2 | 3 | // 4 | // RPC definitions for a simple lock service. 5 | // 6 | // You will need to modify this file. 7 | // 8 | 9 | // 10 | // Lock(lockname) returns OK=true if the lock is not held. 11 | // If it is held, it returns OK=false immediately. 12 | // 13 | type LockArgs struct { 14 | // Go's net/rpc requires that these field 15 | // names start with upper case letters! 16 | Lockname string // lock name 17 | } 18 | 19 | type LockReply struct { 20 | OK bool 21 | } 22 | 23 | // 24 | // Unlock(lockname) returns OK=true if the lock was held. 25 | // It returns OK=false if the lock was not held. 26 | // 27 | type UnlockArgs struct { 28 | Lockname string 29 | } 30 | 31 | type UnlockReply struct { 32 | OK bool 33 | } 34 | -------------------------------------------------------------------------------- /src/lockservice/server.go: -------------------------------------------------------------------------------- 1 | package lockservice 2 | 3 | import "net" 4 | import "net/rpc" 5 | import "log" 6 | import "sync" 7 | import "fmt" 8 | import "os" 9 | import "io" 10 | import "time" 11 | 12 | type LockServer struct { 13 | mu sync.Mutex 14 | l net.Listener 15 | dead bool // for test_test.go 16 | dying bool // for test_test.go 17 | 18 | am_primary bool // am I the primary? 19 | backup string // backup's port 20 | 21 | // for each lock name, is it locked? 22 | locks map[string]bool 23 | } 24 | 25 | 26 | // 27 | // server Lock RPC handler. 28 | // 29 | // you will have to modify this function 30 | // 31 | func (ls *LockServer) Lock(args *LockArgs, reply *LockReply) error { 32 | ls.mu.Lock() 33 | defer ls.mu.Unlock() 34 | 35 | 36 | locked, _ := ls.locks[args.Lockname] 37 | 38 | if locked { 39 | reply.OK = false 40 | } else { 41 | reply.OK = true 42 | ls.locks[args.Lockname] = true 43 | } 44 | 45 | return nil 46 | } 47 | 48 | // 49 | // server Unlock RPC handler. 50 | // 51 | func (ls *LockServer) Unlock(args *UnlockArgs, reply *UnlockReply) error { 52 | 53 | // Your code here. 54 | 55 | return nil 56 | } 57 | 58 | // 59 | // tell the server to shut itself down. 60 | // for testing. 61 | // please don't change this. 62 | // 63 | func (ls *LockServer) kill() { 64 | ls.dead = true 65 | ls.l.Close() 66 | } 67 | 68 | // 69 | // hack to allow test_test.go to have primary process 70 | // an RPC but not send a reply. can't use the shutdown() 71 | // trick b/c that causes client to immediately get an 72 | // error and send to backup before primary does. 73 | // please don't change anything to do with DeafConn. 74 | // 75 | type DeafConn struct { 76 | c io.ReadWriteCloser 77 | } 78 | 79 | func (dc DeafConn) Write(p []byte) (n int, err error) { 80 | return len(p), nil 81 | } 82 | func (dc DeafConn) Close() error { 83 | return dc.c.Close() 84 | } 85 | func (dc DeafConn) Read(p []byte) (n int, err error) { 86 | return dc.c.Read(p) 87 | } 88 | 89 | func StartServer(primary string, backup string, am_primary bool) *LockServer { 90 | ls := new(LockServer) 91 | ls.backup = backup 92 | ls.am_primary = am_primary 93 | ls.locks = map[string]bool{} 94 | 95 | // Your initialization code here. 96 | 97 | 98 | me := "" 99 | if am_primary { 100 | me = primary 101 | } else { 102 | me = backup 103 | } 104 | 105 | // tell net/rpc about our RPC server and handlers. 106 | rpcs := rpc.NewServer() 107 | rpcs.Register(ls) 108 | 109 | // prepare to receive connections from clients. 110 | // change "unix" to "tcp" to use over a network. 111 | os.Remove(me) // only needed for "unix" 112 | l, e := net.Listen("unix", me) 113 | if e != nil { 114 | log.Fatal("listen error: ", e) 115 | } 116 | ls.l = l 117 | 118 | // please don't change any of the following code, 119 | // or do anything to subvert it. 120 | 121 | // create a thread to accept RPC connections from clients. 122 | go func() { 123 | for ls.dead == false { 124 | conn, err := ls.l.Accept() 125 | if err == nil && ls.dead == false { 126 | if ls.dying { 127 | // process the request but force discard of reply. 128 | 129 | // without this the connection is never closed, 130 | // b/c ServeConn() is waiting for more requests. 131 | // test_test.go depends on this two seconds. 132 | go func() { 133 | time.Sleep(2 * time.Second) 134 | conn.Close() 135 | }() 136 | ls.l.Close() 137 | 138 | // this object has the type ServeConn expects, 139 | // but discards writes (i.e. discards the RPC reply). 140 | deaf_conn := DeafConn{c: conn} 141 | 142 | rpcs.ServeConn(deaf_conn) 143 | 144 | ls.dead = true 145 | } else { 146 | go rpcs.ServeConn(conn) 147 | } 148 | } else if err == nil { 149 | conn.Close() 150 | } 151 | if err != nil && ls.dead == false { 152 | fmt.Printf("LockServer(%v) accept: %v\n", me, err.Error()) 153 | ls.kill() 154 | } 155 | } 156 | }() 157 | 158 | return ls 159 | } 160 | -------------------------------------------------------------------------------- /src/lockservice/test_test.go: -------------------------------------------------------------------------------- 1 | package lockservice 2 | 3 | import "testing" 4 | import "runtime" 5 | import "math/rand" 6 | import "os" 7 | import "strconv" 8 | import "time" 9 | import "fmt" 10 | 11 | func tl(t *testing.T, ck *Clerk, lockname string, expected bool) { 12 | x := ck.Lock(lockname) 13 | if x != expected { 14 | t.Fatalf("Lock(%v) returned %v; expected %v", lockname, x, expected) 15 | } 16 | } 17 | 18 | func tu(t *testing.T, ck *Clerk, lockname string, expected bool) { 19 | x := ck.Unlock(lockname) 20 | if x != expected { 21 | t.Fatalf("Unlock(%v) returned %v; expected %v", lockname, x, expected) 22 | } 23 | } 24 | 25 | // 26 | // cook up a unique-ish UNIX-domain socket name 27 | // in /var/tmp. can't use current directory since 28 | // AFS doesn't support UNIX-domain sockets. 29 | // 30 | func port(suffix string) string { 31 | s := "/var/tmp/824-" 32 | s += strconv.Itoa(os.Getuid()) + "/" 33 | os.Mkdir(s, 0777) 34 | s += strconv.Itoa(os.Getpid()) + "-" 35 | s += suffix 36 | return s 37 | } 38 | 39 | func TestBasic(t *testing.T) { 40 | fmt.Printf("Test: Basic lock/unlock ...\n") 41 | 42 | runtime.GOMAXPROCS(4) 43 | 44 | phost := port("p") 45 | bhost := port("b") 46 | p := StartServer(phost, bhost, true) // primary 47 | b := StartServer(phost, bhost, false) // backup 48 | 49 | ck := MakeClerk(phost, bhost) 50 | 51 | tl(t, ck, "a", true) 52 | tu(t, ck, "a", true) 53 | 54 | tl(t, ck, "a", true) 55 | tl(t, ck, "b", true) 56 | tu(t, ck, "a", true) 57 | tu(t, ck, "b", true) 58 | 59 | tl(t, ck, "a", true) 60 | tl(t, ck, "a", false) 61 | tu(t, ck, "a", true) 62 | tu(t, ck, "a", false) 63 | 64 | p.kill() 65 | b.kill() 66 | 67 | fmt.Printf(" ... Passed\n") 68 | } 69 | 70 | func TestPrimaryFail1(t *testing.T) { 71 | fmt.Printf("Test: Primary failure ...\n") 72 | runtime.GOMAXPROCS(4) 73 | 74 | phost := port("p") 75 | bhost := port("b") 76 | p := StartServer(phost, bhost, true) // primary 77 | b := StartServer(phost, bhost, false) // backup 78 | 79 | ck := MakeClerk(phost, bhost) 80 | 81 | tl(t, ck, "a", true) 82 | 83 | tl(t, ck, "b", true) 84 | tu(t, ck, "b", true) 85 | 86 | tl(t, ck, "c", true) 87 | tl(t, ck, "c", false) 88 | 89 | tl(t, ck, "d", true) 90 | tu(t, ck, "d", true) 91 | tl(t, ck, "d", true) 92 | 93 | p.kill() 94 | 95 | tl(t, ck, "a", false) 96 | tu(t, ck, "a", true) 97 | 98 | tu(t, ck, "b", false) 99 | tl(t, ck, "b", true) 100 | 101 | tu(t, ck, "c", true) 102 | 103 | tu(t, ck, "d", true) 104 | 105 | b.kill() 106 | fmt.Printf(" ... Passed\n") 107 | } 108 | 109 | func TestPrimaryFail2(t *testing.T) { 110 | fmt.Printf("Test: Primary failure just before reply #1 ...\n") 111 | runtime.GOMAXPROCS(4) 112 | 113 | phost := port("p") 114 | bhost := port("b") 115 | p := StartServer(phost, bhost, true) // primary 116 | b := StartServer(phost, bhost, false) // backup 117 | 118 | ck1 := MakeClerk(phost, bhost) 119 | ck2 := MakeClerk(phost, bhost) 120 | 121 | tl(t, ck1, "a", true) 122 | tl(t, ck1, "b", true) 123 | 124 | p.dying = true 125 | 126 | tl(t, ck2, "c", true) 127 | tl(t, ck1, "c", false) 128 | tu(t, ck2, "c", true) 129 | tl(t, ck1, "c", true) 130 | 131 | b.kill() 132 | fmt.Printf(" ... Passed\n") 133 | } 134 | 135 | func TestPrimaryFail3(t *testing.T) { 136 | fmt.Printf("Test: Primary failure just before reply #2 ...\n") 137 | runtime.GOMAXPROCS(4) 138 | 139 | phost := port("p") 140 | bhost := port("b") 141 | p := StartServer(phost, bhost, true) // primary 142 | b := StartServer(phost, bhost, false) // backup 143 | 144 | ck1 := MakeClerk(phost, bhost) 145 | 146 | tl(t, ck1, "a", true) 147 | tl(t, ck1, "b", true) 148 | 149 | p.dying = true 150 | 151 | tl(t, ck1, "b", false) 152 | 153 | b.kill() 154 | fmt.Printf(" ... Passed\n") 155 | } 156 | 157 | func TestPrimaryFail4(t *testing.T) { 158 | fmt.Printf("Test: Primary failure just before reply #3 ...\n") 159 | runtime.GOMAXPROCS(4) 160 | 161 | phost := port("p") 162 | bhost := port("b") 163 | p := StartServer(phost, bhost, true) // primary 164 | b := StartServer(phost, bhost, false) // backup 165 | 166 | ck1 := MakeClerk(phost, bhost) 167 | ck2 := MakeClerk(phost, bhost) 168 | 169 | tl(t, ck1, "a", true) 170 | tl(t, ck1, "b", true) 171 | 172 | p.dying = true 173 | 174 | tl(t, ck2, "b", false) 175 | 176 | b.kill() 177 | fmt.Printf(" ... Passed\n") 178 | } 179 | 180 | func TestPrimaryFail5(t *testing.T) { 181 | fmt.Printf("Test: Primary failure just before reply #4 ...\n") 182 | runtime.GOMAXPROCS(4) 183 | 184 | phost := port("p") 185 | bhost := port("b") 186 | p := StartServer(phost, bhost, true) // primary 187 | b := StartServer(phost, bhost, false) // backup 188 | 189 | ck1 := MakeClerk(phost, bhost) 190 | ck2 := MakeClerk(phost, bhost) 191 | 192 | tl(t, ck1, "a", true) 193 | tl(t, ck1, "b", true) 194 | tu(t, ck1, "b", true) 195 | 196 | p.dying = true 197 | 198 | tu(t, ck1, "b", false) 199 | tl(t, ck2, "b", true) 200 | 201 | b.kill() 202 | fmt.Printf(" ... Passed\n") 203 | } 204 | 205 | func TestPrimaryFail6(t *testing.T) { 206 | fmt.Printf("Test: Primary failure just before reply #5 ...\n") 207 | runtime.GOMAXPROCS(4) 208 | 209 | phost := port("p") 210 | bhost := port("b") 211 | p := StartServer(phost, bhost, true) // primary 212 | b := StartServer(phost, bhost, false) // backup 213 | 214 | ck1 := MakeClerk(phost, bhost) 215 | ck2 := MakeClerk(phost, bhost) 216 | 217 | tl(t, ck1, "a", true) 218 | tu(t, ck1, "a", true) 219 | tu(t, ck2, "a", false) 220 | tl(t, ck1, "b", true) 221 | 222 | p.dying = true 223 | 224 | tu(t, ck2, "b", true) 225 | tl(t, ck1, "b", true) 226 | 227 | b.kill() 228 | fmt.Printf(" ... Passed\n") 229 | } 230 | 231 | func TestPrimaryFail7(t *testing.T) { 232 | fmt.Printf("Test: Primary failure just before reply #6 ...\n") 233 | runtime.GOMAXPROCS(4) 234 | 235 | phost := port("p") 236 | bhost := port("b") 237 | p := StartServer(phost, bhost, true) // primary 238 | b := StartServer(phost, bhost, false) // backup 239 | 240 | ck1 := MakeClerk(phost, bhost) 241 | ck2 := MakeClerk(phost, bhost) 242 | 243 | tl(t, ck1, "a", true) 244 | tu(t, ck1, "a", true) 245 | tu(t, ck2, "a", false) 246 | tl(t, ck1, "b", true) 247 | 248 | p.dying = true 249 | 250 | ch := make(chan bool) 251 | go func() { 252 | ok := false 253 | defer func() { ch <- ok }() 254 | tu(t, ck2, "b", true) // 2 second delay until retry 255 | ok = true 256 | }() 257 | time.Sleep(1 * time.Second) 258 | tl(t, ck1, "b", true) 259 | 260 | ok := <-ch 261 | if ok == false { 262 | t.Fatalf("re-sent Unlock did not return true") 263 | } 264 | 265 | tu(t, ck1, "b", true) 266 | 267 | b.kill() 268 | fmt.Printf(" ... Passed\n") 269 | } 270 | 271 | func TestPrimaryFail8(t *testing.T) { 272 | fmt.Printf("Test: Primary failure just before reply #7 ...\n") 273 | runtime.GOMAXPROCS(4) 274 | 275 | phost := port("p") 276 | bhost := port("b") 277 | p := StartServer(phost, bhost, true) // primary 278 | b := StartServer(phost, bhost, false) // backup 279 | 280 | ck1 := MakeClerk(phost, bhost) 281 | ck2 := MakeClerk(phost, bhost) 282 | 283 | tl(t, ck1, "a", true) 284 | tu(t, ck1, "a", true) 285 | 286 | p.dying = true 287 | 288 | ch := make(chan bool) 289 | go func() { 290 | ok := false 291 | defer func() { ch <- ok }() 292 | tu(t, ck2, "a", false) // 2 second delay until retry 293 | ok = true 294 | }() 295 | time.Sleep(1 * time.Second) 296 | tl(t, ck1, "a", true) 297 | 298 | ok := <-ch 299 | if ok == false { 300 | t.Fatalf("re-sent Unlock did not return false") 301 | } 302 | 303 | tu(t, ck1, "a", true) 304 | 305 | b.kill() 306 | fmt.Printf(" ... Passed\n") 307 | } 308 | 309 | func TestBackupFail(t *testing.T) { 310 | fmt.Printf("Test: Backup failure ...\n") 311 | runtime.GOMAXPROCS(4) 312 | 313 | phost := port("p") 314 | bhost := port("b") 315 | p := StartServer(phost, bhost, true) // primary 316 | b := StartServer(phost, bhost, false) // backup 317 | 318 | ck := MakeClerk(phost, bhost) 319 | 320 | tl(t, ck, "a", true) 321 | 322 | tl(t, ck, "b", true) 323 | tu(t, ck, "b", true) 324 | 325 | tl(t, ck, "c", true) 326 | tl(t, ck, "c", false) 327 | 328 | tl(t, ck, "d", true) 329 | tu(t, ck, "d", true) 330 | tl(t, ck, "d", true) 331 | 332 | b.kill() 333 | 334 | tl(t, ck, "a", false) 335 | tu(t, ck, "a", true) 336 | 337 | tu(t, ck, "b", false) 338 | tl(t, ck, "b", true) 339 | 340 | tu(t, ck, "c", true) 341 | 342 | tu(t, ck, "d", true) 343 | 344 | p.kill() 345 | fmt.Printf(" ... Passed\n") 346 | } 347 | 348 | func TestMany(t *testing.T) { 349 | fmt.Printf("Test: Multiple clients with primary failure ...\n") 350 | runtime.GOMAXPROCS(4) 351 | 352 | phost := port("p") 353 | bhost := port("b") 354 | p := StartServer(phost, bhost, true) // primary 355 | b := StartServer(phost, bhost, false) // backup 356 | 357 | const nclients = 2 358 | const nlocks = 10 359 | done := false 360 | var state [nclients][nlocks]bool 361 | var acks [nclients]bool 362 | 363 | for xi := 0; xi < nclients; xi++ { 364 | go func(i int) { 365 | ck := MakeClerk(phost, bhost) 366 | rr := rand.New(rand.NewSource(int64(os.Getpid() + i))) 367 | for done == false { 368 | locknum := (rr.Int() % nlocks) 369 | lockname := strconv.Itoa(locknum + (i * 1000)) 370 | what := rr.Int() % 2 371 | if what == 0 { 372 | ck.Lock(lockname) 373 | state[i][locknum] = true 374 | } else { 375 | ck.Unlock(lockname) 376 | state[i][locknum] = false 377 | } 378 | } 379 | acks[i] = true 380 | }(xi) 381 | } 382 | 383 | time.Sleep(2 * time.Second) 384 | p.kill() 385 | time.Sleep(2 * time.Second) 386 | done = true 387 | time.Sleep(time.Second) 388 | ck := MakeClerk(phost, bhost) 389 | for xi := 0; xi < nclients; xi++ { 390 | if acks[xi] == false { 391 | t.Fatal("one client didn't complete") 392 | } 393 | for locknum := 0; locknum < nlocks; locknum++ { 394 | lockname := strconv.Itoa(locknum + (xi * 1000)) 395 | locked := !ck.Lock(lockname) 396 | if locked != state[xi][locknum] { 397 | t.Fatal("bad final state") 398 | } 399 | } 400 | } 401 | 402 | b.kill() 403 | fmt.Printf(" ... Passed\n") 404 | } 405 | 406 | func TestConcurrentCounts(t *testing.T) { 407 | fmt.Printf("Test: Multiple clients, single lock, primary failure ...\n") 408 | runtime.GOMAXPROCS(4) 409 | 410 | phost := port("p") 411 | bhost := port("b") 412 | p := StartServer(phost, bhost, true) // primary 413 | b := StartServer(phost, bhost, false) // backup 414 | 415 | const nclients = 2 416 | const nlocks = 1 417 | done := false 418 | var acks [nclients]bool 419 | var locks [nclients][nlocks]int 420 | var unlocks [nclients][nlocks]int 421 | 422 | for xi := 0; xi < nclients; xi++ { 423 | go func(i int) { 424 | ck := MakeClerk(phost, bhost) 425 | rr := rand.New(rand.NewSource(int64(os.Getpid() + i))) 426 | for done == false { 427 | locknum := rr.Int() % nlocks 428 | lockname := strconv.Itoa(locknum) 429 | what := rr.Int() % 2 430 | if what == 0 { 431 | if ck.Lock(lockname) { 432 | locks[i][locknum]++ 433 | } 434 | } else { 435 | if ck.Unlock(lockname) { 436 | unlocks[i][locknum]++ 437 | } 438 | } 439 | } 440 | acks[i] = true 441 | }(xi) 442 | } 443 | 444 | time.Sleep(2 * time.Second) 445 | p.kill() 446 | time.Sleep(2 * time.Second) 447 | done = true 448 | time.Sleep(time.Second) 449 | for xi := 0; xi < nclients; xi++ { 450 | if acks[xi] == false { 451 | t.Fatal("one client didn't complete") 452 | } 453 | } 454 | ck := MakeClerk(phost, bhost) 455 | for locknum := 0; locknum < nlocks; locknum++ { 456 | nl := 0 457 | nu := 0 458 | for xi := 0; xi < nclients; xi++ { 459 | nl += locks[xi][locknum] 460 | nu += unlocks[xi][locknum] 461 | } 462 | locked := ck.Unlock(strconv.Itoa(locknum)) 463 | // fmt.Printf("lock=%d nl=%d nu=%d locked=%v\n", 464 | // locknum, nl, nu, locked) 465 | if nl < nu || nl > nu+1 { 466 | t.Fatal("lock race 1") 467 | } 468 | if nl == nu && locked != false { 469 | t.Fatal("lock race 2") 470 | } 471 | if nl != nu && locked != true { 472 | t.Fatal("lock race 3") 473 | } 474 | } 475 | 476 | b.kill() 477 | fmt.Printf(" ... Passed\n") 478 | } 479 | -------------------------------------------------------------------------------- /src/main/diskvd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // 4 | // start a diskvd server. it's a member of some replica 5 | // group, which has other members, and it needs to know 6 | // how to talk to the members of the shardmaster service. 7 | // used by ../diskv/test_test.go 8 | // 9 | // arguments: 10 | // -g groupid 11 | // -m masterport1 -m masterport2 ... 12 | // -s replicaport1 -s replicaport2 ... 13 | // -i my-index-in-server-port-list 14 | // -u unreliable 15 | // -d directory 16 | // -r restart 17 | 18 | import "time" 19 | import "diskv" 20 | import "os" 21 | import "fmt" 22 | import "strconv" 23 | import "runtime" 24 | 25 | func usage() { 26 | fmt.Printf("Usage: diskvd -g gid -m master... -s server... -i my-index -d dir\n") 27 | os.Exit(1) 28 | } 29 | 30 | func main() { 31 | var gid int64 = -1 // my replica group ID 32 | masters := []string{} // ports of shardmasters 33 | replicas := []string{} // ports of servers in my replica group 34 | me := -1 // my index in replicas[] 35 | unreliable := false 36 | dir := "" // store persistent data here 37 | restart := false 38 | 39 | for i := 1; i+1 < len(os.Args); i += 2 { 40 | a0 := os.Args[i] 41 | a1 := os.Args[i+1] 42 | if a0 == "-g" { 43 | gid, _ = strconv.ParseInt(a1, 10, 64) 44 | } else if a0 == "-m" { 45 | masters = append(masters, a1) 46 | } else if a0 == "-s" { 47 | replicas = append(replicas, a1) 48 | } else if a0 == "-i" { 49 | me, _ = strconv.Atoi(a1) 50 | } else if a0 == "-u" { 51 | unreliable, _ = strconv.ParseBool(a1) 52 | } else if a0 == "-d" { 53 | dir = a1 54 | } else if a0 == "-r" { 55 | restart, _ = strconv.ParseBool(a1) 56 | } else { 57 | usage() 58 | } 59 | } 60 | 61 | if gid < 0 || me < 0 || len(masters) < 1 || me >= len(replicas) || dir == "" { 62 | usage() 63 | } 64 | 65 | runtime.GOMAXPROCS(4) 66 | 67 | srv := diskv.StartServer(gid, masters, replicas, me, dir, restart) 68 | srv.Setunreliable(unreliable) 69 | 70 | // for safety, force quit after 10 minutes. 71 | time.Sleep(10 * 60 * time.Second) 72 | mep, _ := os.FindProcess(os.Getpid()) 73 | mep.Kill() 74 | } 75 | -------------------------------------------------------------------------------- /src/main/lockc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // 4 | // see comments in lockd.go 5 | // 6 | 7 | import "lockservice" 8 | import "os" 9 | import "fmt" 10 | 11 | func usage() { 12 | fmt.Printf("Usage: lockc -l|-u primaryport backupport lockname\n") 13 | os.Exit(1) 14 | } 15 | 16 | func main() { 17 | if len(os.Args) == 5 { 18 | ck := lockservice.MakeClerk(os.Args[2], os.Args[3]) 19 | var ok bool 20 | if os.Args[1] == "-l" { 21 | ok = ck.Lock(os.Args[4]) 22 | } else if os.Args[1] == "-u" { 23 | ok = ck.Unlock(os.Args[4]) 24 | } else { 25 | usage() 26 | } 27 | fmt.Printf("reply: %v\n", ok) 28 | } else { 29 | usage() 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/lockd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // export GOPATH=~/golabs 4 | // go build lockd.go 5 | // go build lockc.go 6 | // ./lockd -p a b & 7 | // ./lockd -b a b & 8 | // ./lockc -l a b lx 9 | // ./lockc -u a b lx 10 | // 11 | // on Athena, use /tmp/myname-a and /tmp/myname-b 12 | // instead of a and b. 13 | 14 | import "time" 15 | import "lockservice" 16 | import "os" 17 | import "fmt" 18 | 19 | func main() { 20 | if len(os.Args) == 4 && os.Args[1] == "-p" { 21 | lockservice.StartServer(os.Args[2], os.Args[3], true) 22 | } else if len(os.Args) == 4 && os.Args[1] == "-b" { 23 | lockservice.StartServer(os.Args[2], os.Args[3], false) 24 | } else { 25 | fmt.Printf("Usage: lockd -p|-b primaryport backupport\n") 26 | os.Exit(1) 27 | } 28 | for { 29 | time.Sleep(100 * time.Second) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/mr-testout.txt: -------------------------------------------------------------------------------- 1 | unto: 8940 2 | he: 9666 3 | shall: 9760 4 | in: 12334 5 | that: 12577 6 | And: 12846 7 | to: 13384 8 | of: 34434 9 | and: 38850 10 | the: 62075 11 | -------------------------------------------------------------------------------- /src/main/pbc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // 4 | // pbservice client application 5 | // 6 | // export GOPATH=~/golabs 7 | // go build viewd.go 8 | // go build pbd.go 9 | // go build pbc.go 10 | // ./viewd /tmp/rtm-v & 11 | // ./pbd /tmp/rtm-v /tmp/rtm-1 & 12 | // ./pbd /tmp/rtm-v /tmp/rtm-2 & 13 | // ./pbc /tmp/rtm-v key1 value1 14 | // ./pbc /tmp/rtm-v key1 15 | // 16 | // change "rtm" to your user name. 17 | // start the pbd programs in separate windows and kill 18 | // and restart them to exercise fault tolerance. 19 | // 20 | 21 | import "pbservice" 22 | import "os" 23 | import "fmt" 24 | 25 | func usage() { 26 | fmt.Printf("Usage: pbc viewport key\n") 27 | fmt.Printf(" pbc viewport key value\n") 28 | os.Exit(1) 29 | } 30 | 31 | func main() { 32 | if len(os.Args) == 3 { 33 | // get 34 | ck := pbservice.MakeClerk(os.Args[1], "") 35 | v := ck.Get(os.Args[2]) 36 | fmt.Printf("%v\n", v) 37 | } else if len(os.Args) == 4 { 38 | // put 39 | ck := pbservice.MakeClerk(os.Args[1], "") 40 | ck.Put(os.Args[2], os.Args[3]) 41 | } else { 42 | usage() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/pbd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // 4 | // see directions in pbc.go 5 | // 6 | 7 | import "time" 8 | import "pbservice" 9 | import "os" 10 | import "fmt" 11 | 12 | func main() { 13 | if len(os.Args) != 3 { 14 | fmt.Printf("Usage: pbd viewport myport\n") 15 | os.Exit(1) 16 | } 17 | 18 | pbservice.StartServer(os.Args[1], os.Args[2]) 19 | 20 | for { 21 | time.Sleep(100 * time.Second) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/test-wc.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | go run wc.go master kjv12.txt sequential 3 | sort -n -k2 mrtmp.kjv12.txt | tail -10 | diff - mr-testout.txt > diff.out 4 | if [ -s diff.out ] 5 | then 6 | echo "Failed test. Output should be as in mr-testout.txt. Your output differs as follows (from diff.out):" 7 | cat diff.out 8 | else 9 | echo "Passed test" 10 | fi 11 | 12 | -------------------------------------------------------------------------------- /src/main/viewd.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | // 4 | // see directions in pbc.go 5 | // 6 | 7 | import "time" 8 | import "viewservice" 9 | import "os" 10 | import "fmt" 11 | 12 | func main() { 13 | if len(os.Args) != 2 { 14 | fmt.Printf("Usage: viewd port\n") 15 | os.Exit(1) 16 | } 17 | 18 | viewservice.StartServer(os.Args[1]) 19 | 20 | for { 21 | time.Sleep(100 * time.Second) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/wc.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "os" 4 | import "fmt" 5 | import "mapreduce" 6 | 7 | import "container/list" 8 | 9 | // our simplified version of MapReduce does not supply a 10 | // key to the Map function, as in the paper; only a value, 11 | // which is a part of the input file content. the return 12 | // value should be a list of key/value pairs, each represented 13 | // by a mapreduce.KeyValue. 14 | func Map(value string) *list.List { 15 | } 16 | 17 | // called once for each key generated by Map, with a list 18 | // of that key's string value. should return a single 19 | // output value for that key. 20 | func Reduce(key string, values *list.List) string { 21 | } 22 | 23 | // Can be run in 3 ways: 24 | // 1) Sequential (e.g., go run wc.go master x.txt sequential) 25 | // 2) Master (e.g., go run wc.go master x.txt localhost:7777) 26 | // 3) Worker (e.g., go run wc.go worker localhost:7777 localhost:7778 &) 27 | func main() { 28 | if len(os.Args) != 4 { 29 | fmt.Printf("%s: see usage comments in file\n", os.Args[0]) 30 | } else if os.Args[1] == "master" { 31 | if os.Args[3] == "sequential" { 32 | mapreduce.RunSingle(5, 3, os.Args[2], Map, Reduce) 33 | } else { 34 | mr := mapreduce.MakeMapReduce(5, 3, os.Args[2], os.Args[3]) 35 | // Wait until MR is done 36 | <-mr.DoneChannel 37 | } 38 | } else { 39 | mapreduce.RunWorker(os.Args[2], os.Args[3], Map, Reduce, 100) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/mapreduce/common.go: -------------------------------------------------------------------------------- 1 | package mapreduce 2 | 3 | import "fmt" 4 | import "net/rpc" 5 | 6 | const ( 7 | Map = "Map" 8 | Reduce = "Reduce" 9 | ) 10 | 11 | type JobType string 12 | 13 | // RPC arguments and replies. Field names must start with capital letters, 14 | // otherwise RPC will break. 15 | 16 | type DoJobArgs struct { 17 | File string 18 | Operation JobType 19 | JobNumber int // this job's number 20 | NumOtherPhase int // total number of jobs in other phase (map or reduce) 21 | } 22 | 23 | type DoJobReply struct { 24 | OK bool 25 | } 26 | 27 | type ShutdownArgs struct { 28 | } 29 | 30 | type ShutdownReply struct { 31 | Njobs int 32 | OK bool 33 | } 34 | 35 | type RegisterArgs struct { 36 | Worker string 37 | } 38 | 39 | type RegisterReply struct { 40 | OK bool 41 | } 42 | 43 | // 44 | // call() sends an RPC to the rpcname handler on server srv 45 | // with arguments args, waits for the reply, and leaves the 46 | // reply in reply. the reply argument should be the address 47 | // of a reply structure. 48 | // 49 | // call() returns true if the server responded, and false 50 | // if call() was not able to contact the server. in particular, 51 | // reply's contents are valid if and only if call() returned true. 52 | // 53 | // you should assume that call() will time out and return an 54 | // error after a while if it doesn't get a reply from the server. 55 | // 56 | // please use call() to send all RPCs, in master.go, mapreduce.go, 57 | // and worker.go. please don't change this function. 58 | // 59 | func call(srv string, rpcname string, 60 | args interface{}, reply interface{}) bool { 61 | c, errx := rpc.Dial("unix", srv) 62 | if errx != nil { 63 | return false 64 | } 65 | defer c.Close() 66 | 67 | err := c.Call(rpcname, args, reply) 68 | if err == nil { 69 | return true 70 | } 71 | 72 | fmt.Println(err) 73 | return false 74 | } 75 | -------------------------------------------------------------------------------- /src/mapreduce/mapreduce.go: -------------------------------------------------------------------------------- 1 | package mapreduce 2 | 3 | import "fmt" 4 | import "os" 5 | import "log" 6 | import "strconv" 7 | import "encoding/json" 8 | import "sort" 9 | import "container/list" 10 | import "net/rpc" 11 | import "net" 12 | import "bufio" 13 | import "hash/fnv" 14 | 15 | // import "os/exec" 16 | 17 | // A simple mapreduce library with a sequential implementation. 18 | // 19 | // The application provides an input file f, a Map and Reduce function, 20 | // and the number of nMap and nReduce tasks. 21 | // 22 | // Split() splits the file f in nMap input files: 23 | // f-0, f-1, ..., f- 24 | // one for each Map job. 25 | // 26 | // DoMap() runs Map on each map file and produces nReduce files for each 27 | // map file. Thus, there will be nMap x nReduce files after all map 28 | // jobs are done: 29 | // f-0-0, ..., f-0-0, f-0-, ..., 30 | // f--0, ... f--. 31 | // 32 | // DoReduce() collects reduce files from each map (f-*-), 33 | // and runs Reduce on those files. This produces result files, 34 | // which Merge() merges into a single output. 35 | 36 | // Debugging 37 | const Debug = 0 38 | 39 | func DPrintf(format string, a ...interface{}) (n int, err error) { 40 | if Debug > 0 { 41 | n, err = fmt.Printf(format, a...) 42 | } 43 | return 44 | } 45 | 46 | // Map and Reduce deal with pairs: 47 | type KeyValue struct { 48 | Key string 49 | Value string 50 | } 51 | 52 | type MapReduce struct { 53 | nMap int // Number of Map jobs 54 | nReduce int // Number of Reduce jobs 55 | file string // Name of input file 56 | MasterAddress string 57 | registerChannel chan string 58 | DoneChannel chan bool 59 | alive bool 60 | l net.Listener 61 | stats *list.List 62 | 63 | // Map of registered workers that you need to keep up to date 64 | Workers map[string]*WorkerInfo 65 | 66 | // add any additional state here 67 | } 68 | 69 | func InitMapReduce(nmap int, nreduce int, 70 | file string, master string) *MapReduce { 71 | mr := new(MapReduce) 72 | mr.nMap = nmap 73 | mr.nReduce = nreduce 74 | mr.file = file 75 | mr.MasterAddress = master 76 | mr.alive = true 77 | mr.registerChannel = make(chan string) 78 | mr.DoneChannel = make(chan bool) 79 | 80 | // initialize any additional state here 81 | return mr 82 | } 83 | 84 | func MakeMapReduce(nmap int, nreduce int, 85 | file string, master string) *MapReduce { 86 | mr := InitMapReduce(nmap, nreduce, file, master) 87 | mr.StartRegistrationServer() 88 | go mr.Run() 89 | return mr 90 | } 91 | 92 | func (mr *MapReduce) Register(args *RegisterArgs, res *RegisterReply) error { 93 | DPrintf("Register: worker %s\n", args.Worker) 94 | mr.registerChannel <- args.Worker 95 | res.OK = true 96 | return nil 97 | } 98 | 99 | func (mr *MapReduce) Shutdown(args *ShutdownArgs, res *ShutdownReply) error { 100 | DPrintf("Shutdown: registration server\n") 101 | mr.alive = false 102 | mr.l.Close() // causes the Accept to fail 103 | return nil 104 | } 105 | 106 | func (mr *MapReduce) StartRegistrationServer() { 107 | rpcs := rpc.NewServer() 108 | rpcs.Register(mr) 109 | os.Remove(mr.MasterAddress) // only needed for "unix" 110 | l, e := net.Listen("unix", mr.MasterAddress) 111 | if e != nil { 112 | log.Fatal("RegstrationServer", mr.MasterAddress, " error: ", e) 113 | } 114 | mr.l = l 115 | 116 | // now that we are listening on the master address, can fork off 117 | // accepting connections to another thread. 118 | go func() { 119 | for mr.alive { 120 | conn, err := mr.l.Accept() 121 | if err == nil { 122 | go func() { 123 | rpcs.ServeConn(conn) 124 | conn.Close() 125 | }() 126 | } else { 127 | DPrintf("RegistrationServer: accept error", err) 128 | break 129 | } 130 | } 131 | DPrintf("RegistrationServer: done\n") 132 | }() 133 | } 134 | 135 | // Name of the file that is the input for map job 136 | func MapName(fileName string, MapJob int) string { 137 | return "mrtmp." + fileName + "-" + strconv.Itoa(MapJob) 138 | } 139 | 140 | // Split bytes of input file into nMap splits, but split only on white space 141 | func (mr *MapReduce) Split(fileName string) { 142 | fmt.Printf("Split %s\n", fileName) 143 | infile, err := os.Open(fileName) 144 | if err != nil { 145 | log.Fatal("Split: ", err) 146 | } 147 | defer infile.Close() 148 | fi, err := infile.Stat() 149 | if err != nil { 150 | log.Fatal("Split: ", err) 151 | } 152 | size := fi.Size() 153 | nchunk := size / int64(mr.nMap) 154 | nchunk += 1 155 | 156 | outfile, err := os.Create(MapName(fileName, 0)) 157 | if err != nil { 158 | log.Fatal("Split: ", err) 159 | } 160 | writer := bufio.NewWriter(outfile) 161 | m := 1 162 | i := 0 163 | 164 | scanner := bufio.NewScanner(infile) 165 | for scanner.Scan() { 166 | if int64(i) > nchunk*int64(m) { 167 | writer.Flush() 168 | outfile.Close() 169 | outfile, err = os.Create(MapName(fileName, m)) 170 | writer = bufio.NewWriter(outfile) 171 | m += 1 172 | } 173 | line := scanner.Text() + "\n" 174 | writer.WriteString(line) 175 | i += len(line) 176 | } 177 | writer.Flush() 178 | outfile.Close() 179 | } 180 | 181 | func ReduceName(fileName string, MapJob int, ReduceJob int) string { 182 | return MapName(fileName, MapJob) + "-" + strconv.Itoa(ReduceJob) 183 | } 184 | 185 | func ihash(s string) uint32 { 186 | h := fnv.New32a() 187 | h.Write([]byte(s)) 188 | return h.Sum32() 189 | } 190 | 191 | // Read split for job, call Map for that split, and create nreduce 192 | // partitions. 193 | func DoMap(JobNumber int, fileName string, 194 | nreduce int, Map func(string) *list.List) { 195 | name := MapName(fileName, JobNumber) 196 | file, err := os.Open(name) 197 | if err != nil { 198 | log.Fatal("DoMap: ", err) 199 | } 200 | fi, err := file.Stat() 201 | if err != nil { 202 | log.Fatal("DoMap: ", err) 203 | } 204 | size := fi.Size() 205 | fmt.Printf("DoMap: read split %s %d\n", name, size) 206 | b := make([]byte, size) 207 | _, err = file.Read(b) 208 | if err != nil { 209 | log.Fatal("DoMap: ", err) 210 | } 211 | file.Close() 212 | res := Map(string(b)) 213 | // XXX a bit inefficient. could open r files and run over list once 214 | for r := 0; r < nreduce; r++ { 215 | file, err = os.Create(ReduceName(fileName, JobNumber, r)) 216 | if err != nil { 217 | log.Fatal("DoMap: create ", err) 218 | } 219 | enc := json.NewEncoder(file) 220 | for e := res.Front(); e != nil; e = e.Next() { 221 | kv := e.Value.(KeyValue) 222 | if ihash(kv.Key)%uint32(nreduce) == uint32(r) { 223 | err := enc.Encode(&kv) 224 | if err != nil { 225 | log.Fatal("DoMap: marshall ", err) 226 | } 227 | } 228 | } 229 | file.Close() 230 | } 231 | } 232 | 233 | func MergeName(fileName string, ReduceJob int) string { 234 | return "mrtmp." + fileName + "-res-" + strconv.Itoa(ReduceJob) 235 | } 236 | 237 | // Read map outputs for partition job, sort them by key, call reduce for each 238 | // key 239 | func DoReduce(job int, fileName string, nmap int, 240 | Reduce func(string, *list.List) string) { 241 | kvs := make(map[string]*list.List) 242 | for i := 0; i < nmap; i++ { 243 | name := ReduceName(fileName, i, job) 244 | fmt.Printf("DoReduce: read %s\n", name) 245 | file, err := os.Open(name) 246 | if err != nil { 247 | log.Fatal("DoReduce: ", err) 248 | } 249 | dec := json.NewDecoder(file) 250 | for { 251 | var kv KeyValue 252 | err = dec.Decode(&kv) 253 | if err != nil { 254 | break 255 | } 256 | _, ok := kvs[kv.Key] 257 | if !ok { 258 | kvs[kv.Key] = list.New() 259 | } 260 | kvs[kv.Key].PushBack(kv.Value) 261 | } 262 | file.Close() 263 | } 264 | var keys []string 265 | for k := range kvs { 266 | keys = append(keys, k) 267 | } 268 | sort.Strings(keys) 269 | p := MergeName(fileName, job) 270 | file, err := os.Create(p) 271 | if err != nil { 272 | log.Fatal("DoReduce: create ", err) 273 | } 274 | enc := json.NewEncoder(file) 275 | for _, k := range keys { 276 | res := Reduce(k, kvs[k]) 277 | enc.Encode(KeyValue{k, res}) 278 | } 279 | file.Close() 280 | } 281 | 282 | // Merge the results of the reduce jobs 283 | // XXX use merge sort 284 | func (mr *MapReduce) Merge() { 285 | DPrintf("Merge phase") 286 | kvs := make(map[string]string) 287 | for i := 0; i < mr.nReduce; i++ { 288 | p := MergeName(mr.file, i) 289 | fmt.Printf("Merge: read %s\n", p) 290 | file, err := os.Open(p) 291 | if err != nil { 292 | log.Fatal("Merge: ", err) 293 | } 294 | dec := json.NewDecoder(file) 295 | for { 296 | var kv KeyValue 297 | err = dec.Decode(&kv) 298 | if err != nil { 299 | break 300 | } 301 | kvs[kv.Key] = kv.Value 302 | } 303 | file.Close() 304 | } 305 | var keys []string 306 | for k := range kvs { 307 | keys = append(keys, k) 308 | } 309 | sort.Strings(keys) 310 | 311 | file, err := os.Create("mrtmp." + mr.file) 312 | if err != nil { 313 | log.Fatal("Merge: create ", err) 314 | } 315 | w := bufio.NewWriter(file) 316 | for _, k := range keys { 317 | fmt.Fprintf(w, "%s: %s\n", k, kvs[k]) 318 | } 319 | w.Flush() 320 | file.Close() 321 | } 322 | 323 | func RemoveFile(n string) { 324 | err := os.Remove(n) 325 | if err != nil { 326 | log.Fatal("CleanupFiles ", err) 327 | } 328 | } 329 | 330 | func (mr *MapReduce) CleanupFiles() { 331 | for i := 0; i < mr.nMap; i++ { 332 | RemoveFile(MapName(mr.file, i)) 333 | for j := 0; j < mr.nReduce; j++ { 334 | RemoveFile(ReduceName(mr.file, i, j)) 335 | } 336 | } 337 | for i := 0; i < mr.nReduce; i++ { 338 | RemoveFile(MergeName(mr.file, i)) 339 | } 340 | RemoveFile("mrtmp." + mr.file) 341 | } 342 | 343 | // Run jobs sequentially. 344 | func RunSingle(nMap int, nReduce int, file string, 345 | Map func(string) *list.List, 346 | Reduce func(string, *list.List) string) { 347 | mr := InitMapReduce(nMap, nReduce, file, "") 348 | mr.Split(mr.file) 349 | for i := 0; i < nMap; i++ { 350 | DoMap(i, mr.file, mr.nReduce, Map) 351 | } 352 | for i := 0; i < mr.nReduce; i++ { 353 | DoReduce(i, mr.file, mr.nMap, Reduce) 354 | } 355 | mr.Merge() 356 | } 357 | 358 | func (mr *MapReduce) CleanupRegistration() { 359 | args := &ShutdownArgs{} 360 | var reply ShutdownReply 361 | ok := call(mr.MasterAddress, "MapReduce.Shutdown", args, &reply) 362 | if ok == false { 363 | fmt.Printf("Cleanup: RPC %s error\n", mr.MasterAddress) 364 | } 365 | DPrintf("CleanupRegistration: done\n") 366 | } 367 | 368 | // Run jobs in parallel, assuming a shared file system 369 | func (mr *MapReduce) Run() { 370 | fmt.Printf("Run mapreduce job %s %s\n", mr.MasterAddress, mr.file) 371 | 372 | mr.Split(mr.file) 373 | mr.stats = mr.RunMaster() 374 | mr.Merge() 375 | mr.CleanupRegistration() 376 | 377 | fmt.Printf("%s: MapReduce done\n", mr.MasterAddress) 378 | 379 | mr.DoneChannel <- true 380 | } 381 | -------------------------------------------------------------------------------- /src/mapreduce/master.go: -------------------------------------------------------------------------------- 1 | package mapreduce 2 | 3 | import "container/list" 4 | import "fmt" 5 | 6 | 7 | type WorkerInfo struct { 8 | address string 9 | // You can add definitions here. 10 | } 11 | 12 | 13 | // Clean up all workers by sending a Shutdown RPC to each one of them Collect 14 | // the number of jobs each work has performed. 15 | func (mr *MapReduce) KillWorkers() *list.List { 16 | l := list.New() 17 | for _, w := range mr.Workers { 18 | DPrintf("DoWork: shutdown %s\n", w.address) 19 | args := &ShutdownArgs{} 20 | var reply ShutdownReply 21 | ok := call(w.address, "Worker.Shutdown", args, &reply) 22 | if ok == false { 23 | fmt.Printf("DoWork: RPC %s shutdown error\n", w.address) 24 | } else { 25 | l.PushBack(reply.Njobs) 26 | } 27 | } 28 | return l 29 | } 30 | 31 | func (mr *MapReduce) RunMaster() *list.List { 32 | // Your code here 33 | return mr.KillWorkers() 34 | } 35 | -------------------------------------------------------------------------------- /src/mapreduce/test_test.go: -------------------------------------------------------------------------------- 1 | package mapreduce 2 | 3 | import "testing" 4 | import "fmt" 5 | import "time" 6 | import "container/list" 7 | import "strings" 8 | import "os" 9 | import "bufio" 10 | import "log" 11 | import "sort" 12 | import "strconv" 13 | 14 | const ( 15 | nNumber = 100000 16 | nMap = 100 17 | nReduce = 50 18 | ) 19 | 20 | // Create input file with N numbers 21 | // Check if we have N numbers in output file 22 | 23 | // Split in words 24 | func MapFunc(value string) *list.List { 25 | DPrintf("Map %v\n", value) 26 | res := list.New() 27 | words := strings.Fields(value) 28 | for _, w := range words { 29 | kv := KeyValue{w, ""} 30 | res.PushBack(kv) 31 | } 32 | return res 33 | } 34 | 35 | // Just return key 36 | func ReduceFunc(key string, values *list.List) string { 37 | for e := values.Front(); e != nil; e = e.Next() { 38 | DPrintf("Reduce %s %v\n", key, e.Value) 39 | } 40 | return "" 41 | } 42 | 43 | // Checks input file agaist output file: each input number should show up 44 | // in the output file in string sorted order 45 | func check(t *testing.T, file string) { 46 | input, err := os.Open(file) 47 | if err != nil { 48 | log.Fatal("check: ", err) 49 | } 50 | defer input.Close() 51 | output, err := os.Open("mrtmp." + file) 52 | if err != nil { 53 | log.Fatal("check: ", err) 54 | } 55 | defer output.Close() 56 | 57 | var lines []string 58 | inputScanner := bufio.NewScanner(input) 59 | for inputScanner.Scan() { 60 | lines = append(lines, inputScanner.Text()) 61 | } 62 | 63 | sort.Strings(lines) 64 | 65 | outputScanner := bufio.NewScanner(output) 66 | i := 0 67 | for outputScanner.Scan() { 68 | var v1 int 69 | var v2 int 70 | text := outputScanner.Text() 71 | n, err := fmt.Sscanf(lines[i], "%d", &v1) 72 | if n == 1 && err == nil { 73 | n, err = fmt.Sscanf(text, "%d", &v2) 74 | } 75 | if err != nil || v1 != v2 { 76 | t.Fatalf("line %d: %d != %d err %v\n", i, v1, v2, err) 77 | } 78 | i += 1 79 | } 80 | if i != nNumber { 81 | t.Fatalf("Expected %d lines in output\n", nNumber) 82 | } 83 | } 84 | 85 | // Workers report back how many RPCs they have processed in the Shutdown reply. 86 | // Check that they processed at least 1 RPC. 87 | func checkWorker(t *testing.T, l *list.List) { 88 | for e := l.Front(); e != nil; e = e.Next() { 89 | if e.Value == 0 { 90 | t.Fatalf("Some worker didn't do any work\n") 91 | } 92 | } 93 | } 94 | 95 | // Make input file 96 | func makeInput() string { 97 | name := "824-mrinput.txt" 98 | file, err := os.Create(name) 99 | if err != nil { 100 | log.Fatal("mkInput: ", err) 101 | } 102 | w := bufio.NewWriter(file) 103 | for i := 0; i < nNumber; i++ { 104 | fmt.Fprintf(w, "%d\n", i) 105 | } 106 | w.Flush() 107 | file.Close() 108 | return name 109 | } 110 | 111 | // Cook up a unique-ish UNIX-domain socket name 112 | // in /var/tmp. can't use current directory since 113 | // AFS doesn't support UNIX-domain sockets. 114 | func port(suffix string) string { 115 | s := "/var/tmp/824-" 116 | s += strconv.Itoa(os.Getuid()) + "/" 117 | os.Mkdir(s, 0777) 118 | s += "mr" 119 | s += strconv.Itoa(os.Getpid()) + "-" 120 | s += suffix 121 | return s 122 | } 123 | 124 | func setup() *MapReduce { 125 | file := makeInput() 126 | master := port("master") 127 | mr := MakeMapReduce(nMap, nReduce, file, master) 128 | return mr 129 | } 130 | 131 | func cleanup(mr *MapReduce) { 132 | mr.CleanupFiles() 133 | RemoveFile(mr.file) 134 | } 135 | 136 | func TestBasic(t *testing.T) { 137 | fmt.Printf("Test: Basic mapreduce ...\n") 138 | mr := setup() 139 | for i := 0; i < 2; i++ { 140 | go RunWorker(mr.MasterAddress, port("worker"+strconv.Itoa(i)), 141 | MapFunc, ReduceFunc, -1) 142 | } 143 | // Wait until MR is done 144 | <-mr.DoneChannel 145 | check(t, mr.file) 146 | checkWorker(t, mr.stats) 147 | cleanup(mr) 148 | fmt.Printf(" ... Basic Passed\n") 149 | } 150 | 151 | func TestOneFailure(t *testing.T) { 152 | fmt.Printf("Test: One Failure mapreduce ...\n") 153 | mr := setup() 154 | // Start 2 workers that fail after 10 jobs 155 | go RunWorker(mr.MasterAddress, port("worker"+strconv.Itoa(0)), 156 | MapFunc, ReduceFunc, 10) 157 | go RunWorker(mr.MasterAddress, port("worker"+strconv.Itoa(1)), 158 | MapFunc, ReduceFunc, -1) 159 | // Wait until MR is done 160 | <-mr.DoneChannel 161 | check(t, mr.file) 162 | checkWorker(t, mr.stats) 163 | cleanup(mr) 164 | fmt.Printf(" ... One Failure Passed\n") 165 | } 166 | 167 | func TestManyFailures(t *testing.T) { 168 | fmt.Printf("Test: One ManyFailures mapreduce ...\n") 169 | mr := setup() 170 | i := 0 171 | done := false 172 | for !done { 173 | select { 174 | case done = <-mr.DoneChannel: 175 | check(t, mr.file) 176 | cleanup(mr) 177 | break 178 | default: 179 | // Start 2 workers each sec. The workers fail after 10 jobs 180 | w := port("worker" + strconv.Itoa(i)) 181 | go RunWorker(mr.MasterAddress, w, MapFunc, ReduceFunc, 10) 182 | i++ 183 | w = port("worker" + strconv.Itoa(i)) 184 | go RunWorker(mr.MasterAddress, w, MapFunc, ReduceFunc, 10) 185 | i++ 186 | time.Sleep(1 * time.Second) 187 | } 188 | } 189 | 190 | fmt.Printf(" ... Many Failures Passed\n") 191 | } 192 | -------------------------------------------------------------------------------- /src/mapreduce/worker.go: -------------------------------------------------------------------------------- 1 | package mapreduce 2 | 3 | import "fmt" 4 | import "os" 5 | import "log" 6 | import "net/rpc" 7 | import "net" 8 | import "container/list" 9 | 10 | // Worker is a server waiting for DoJob or Shutdown RPCs 11 | 12 | type Worker struct { 13 | name string 14 | Reduce func(string, *list.List) string 15 | Map func(string) *list.List 16 | nRPC int 17 | nJobs int 18 | l net.Listener 19 | } 20 | 21 | // The master sent us a job 22 | func (wk *Worker) DoJob(arg *DoJobArgs, res *DoJobReply) error { 23 | fmt.Printf("Dojob %s job %d file %s operation %v N %d\n", 24 | wk.name, arg.JobNumber, arg.File, arg.Operation, 25 | arg.NumOtherPhase) 26 | switch arg.Operation { 27 | case Map: 28 | DoMap(arg.JobNumber, arg.File, arg.NumOtherPhase, wk.Map) 29 | case Reduce: 30 | DoReduce(arg.JobNumber, arg.File, arg.NumOtherPhase, wk.Reduce) 31 | } 32 | res.OK = true 33 | return nil 34 | } 35 | 36 | // The master is telling us to shutdown. Report the number of Jobs we 37 | // have processed. 38 | func (wk *Worker) Shutdown(args *ShutdownArgs, res *ShutdownReply) error { 39 | DPrintf("Shutdown %s\n", wk.name) 40 | res.Njobs = wk.nJobs 41 | res.OK = true 42 | wk.nRPC = 1 // OK, because the same thread reads nRPC 43 | wk.nJobs-- // Don't count the shutdown RPC 44 | return nil 45 | } 46 | 47 | // Tell the master we exist and ready to work 48 | func Register(master string, me string) { 49 | args := &RegisterArgs{} 50 | args.Worker = me 51 | var reply RegisterReply 52 | ok := call(master, "MapReduce.Register", args, &reply) 53 | if ok == false { 54 | fmt.Printf("Register: RPC %s register error\n", master) 55 | } 56 | } 57 | 58 | // Set up a connection with the master, register with the master, 59 | // and wait for jobs from the master 60 | func RunWorker(MasterAddress string, me string, 61 | MapFunc func(string) *list.List, 62 | ReduceFunc func(string, *list.List) string, nRPC int) { 63 | DPrintf("RunWorker %s\n", me) 64 | wk := new(Worker) 65 | wk.name = me 66 | wk.Map = MapFunc 67 | wk.Reduce = ReduceFunc 68 | wk.nRPC = nRPC 69 | rpcs := rpc.NewServer() 70 | rpcs.Register(wk) 71 | os.Remove(me) // only needed for "unix" 72 | l, e := net.Listen("unix", me) 73 | if e != nil { 74 | log.Fatal("RunWorker: worker ", me, " error: ", e) 75 | } 76 | wk.l = l 77 | Register(MasterAddress, me) 78 | 79 | // DON'T MODIFY CODE BELOW 80 | for wk.nRPC != 0 { 81 | conn, err := wk.l.Accept() 82 | if err == nil { 83 | wk.nRPC -= 1 84 | go rpcs.ServeConn(conn) 85 | wk.nJobs += 1 86 | } else { 87 | break 88 | } 89 | } 90 | wk.l.Close() 91 | DPrintf("RunWorker %s exit\n", me) 92 | } 93 | -------------------------------------------------------------------------------- /src/paxos/paxos.go: -------------------------------------------------------------------------------- 1 | package paxos 2 | 3 | // 4 | // Paxos library, to be included in an application. 5 | // Multiple applications will run, each including 6 | // a Paxos peer. 7 | // 8 | // Manages a sequence of agreed-on values. 9 | // The set of peers is fixed. 10 | // Copes with network failures (partition, msg loss, &c). 11 | // Does not store anything persistently, so cannot handle crash+restart. 12 | // 13 | // The application interface: 14 | // 15 | // px = paxos.Make(peers []string, me string) 16 | // px.Start(seq int, v interface{}) -- start agreement on new instance 17 | // px.Status(seq int) (Fate, v interface{}) -- get info about an instance 18 | // px.Done(seq int) -- ok to forget all instances <= seq 19 | // px.Max() int -- highest instance seq known, or -1 20 | // px.Min() int -- instances before this seq have been forgotten 21 | // 22 | 23 | import "net" 24 | import "net/rpc" 25 | import "log" 26 | 27 | import "os" 28 | import "syscall" 29 | import "sync" 30 | import "sync/atomic" 31 | import "fmt" 32 | import "math/rand" 33 | 34 | 35 | // px.Status() return values, indicating 36 | // whether an agreement has been decided, 37 | // or Paxos has not yet reached agreement, 38 | // or it was agreed but forgotten (i.e. < Min()). 39 | type Fate int 40 | 41 | const ( 42 | Decided Fate = iota + 1 43 | Pending // not yet decided. 44 | Forgotten // decided but forgotten. 45 | ) 46 | 47 | type Paxos struct { 48 | mu sync.Mutex 49 | l net.Listener 50 | dead int32 // for testing 51 | unreliable int32 // for testing 52 | rpcCount int32 // for testing 53 | peers []string 54 | me int // index into peers[] 55 | 56 | 57 | // Your data here. 58 | } 59 | 60 | // 61 | // call() sends an RPC to the rpcname handler on server srv 62 | // with arguments args, waits for the reply, and leaves the 63 | // reply in reply. the reply argument should be a pointer 64 | // to a reply structure. 65 | // 66 | // the return value is true if the server responded, and false 67 | // if call() was not able to contact the server. in particular, 68 | // the replys contents are only valid if call() returned true. 69 | // 70 | // you should assume that call() will time out and return an 71 | // error after a while if it does not get a reply from the server. 72 | // 73 | // please use call() to send all RPCs, in client.go and server.go. 74 | // please do not change this function. 75 | // 76 | func call(srv string, name string, args interface{}, reply interface{}) bool { 77 | c, err := rpc.Dial("unix", srv) 78 | if err != nil { 79 | err1 := err.(*net.OpError) 80 | if err1.Err != syscall.ENOENT && err1.Err != syscall.ECONNREFUSED { 81 | fmt.Printf("paxos Dial() failed: %v\n", err1) 82 | } 83 | return false 84 | } 85 | defer c.Close() 86 | 87 | err = c.Call(name, args, reply) 88 | if err == nil { 89 | return true 90 | } 91 | 92 | fmt.Println(err) 93 | return false 94 | } 95 | 96 | 97 | // 98 | // the application wants paxos to start agreement on 99 | // instance seq, with proposed value v. 100 | // Start() returns right away; the application will 101 | // call Status() to find out if/when agreement 102 | // is reached. 103 | // 104 | func (px *Paxos) Start(seq int, v interface{}) { 105 | // Your code here. 106 | } 107 | 108 | // 109 | // the application on this machine is done with 110 | // all instances <= seq. 111 | // 112 | // see the comments for Min() for more explanation. 113 | // 114 | func (px *Paxos) Done(seq int) { 115 | // Your code here. 116 | } 117 | 118 | // 119 | // the application wants to know the 120 | // highest instance sequence known to 121 | // this peer. 122 | // 123 | func (px *Paxos) Max() int { 124 | // Your code here. 125 | return 0 126 | } 127 | 128 | // 129 | // Min() should return one more than the minimum among z_i, 130 | // where z_i is the highest number ever passed 131 | // to Done() on peer i. A peers z_i is -1 if it has 132 | // never called Done(). 133 | // 134 | // Paxos is required to have forgotten all information 135 | // about any instances it knows that are < Min(). 136 | // The point is to free up memory in long-running 137 | // Paxos-based servers. 138 | // 139 | // Paxos peers need to exchange their highest Done() 140 | // arguments in order to implement Min(). These 141 | // exchanges can be piggybacked on ordinary Paxos 142 | // agreement protocol messages, so it is OK if one 143 | // peers Min does not reflect another Peers Done() 144 | // until after the next instance is agreed to. 145 | // 146 | // The fact that Min() is defined as a minimum over 147 | // *all* Paxos peers means that Min() cannot increase until 148 | // all peers have been heard from. So if a peer is dead 149 | // or unreachable, other peers Min()s will not increase 150 | // even if all reachable peers call Done. The reason for 151 | // this is that when the unreachable peer comes back to 152 | // life, it will need to catch up on instances that it 153 | // missed -- the other peers therefor cannot forget these 154 | // instances. 155 | // 156 | func (px *Paxos) Min() int { 157 | // You code here. 158 | return 0 159 | } 160 | 161 | // 162 | // the application wants to know whether this 163 | // peer thinks an instance has been decided, 164 | // and if so what the agreed value is. Status() 165 | // should just inspect the local peer state; 166 | // it should not contact other Paxos peers. 167 | // 168 | func (px *Paxos) Status(seq int) (Fate, interface{}) { 169 | // Your code here. 170 | return Pending, nil 171 | } 172 | 173 | 174 | 175 | // 176 | // tell the peer to shut itself down. 177 | // for testing. 178 | // please do not change these two functions. 179 | // 180 | func (px *Paxos) Kill() { 181 | atomic.StoreInt32(&px.dead, 1) 182 | if px.l != nil { 183 | px.l.Close() 184 | } 185 | } 186 | 187 | // 188 | // has this peer been asked to shut down? 189 | // 190 | func (px *Paxos) isdead() bool { 191 | return atomic.LoadInt32(&px.dead) != 0 192 | } 193 | 194 | // please do not change these two functions. 195 | func (px *Paxos) setunreliable(what bool) { 196 | if what { 197 | atomic.StoreInt32(&px.unreliable, 1) 198 | } else { 199 | atomic.StoreInt32(&px.unreliable, 0) 200 | } 201 | } 202 | 203 | func (px *Paxos) isunreliable() bool { 204 | return atomic.LoadInt32(&px.unreliable) != 0 205 | } 206 | 207 | // 208 | // the application wants to create a paxos peer. 209 | // the ports of all the paxos peers (including this one) 210 | // are in peers[]. this servers port is peers[me]. 211 | // 212 | func Make(peers []string, me int, rpcs *rpc.Server) *Paxos { 213 | px := &Paxos{} 214 | px.peers = peers 215 | px.me = me 216 | 217 | 218 | // Your initialization code here. 219 | 220 | if rpcs != nil { 221 | // caller will create socket &c 222 | rpcs.Register(px) 223 | } else { 224 | rpcs = rpc.NewServer() 225 | rpcs.Register(px) 226 | 227 | // prepare to receive connections from clients. 228 | // change "unix" to "tcp" to use over a network. 229 | os.Remove(peers[me]) // only needed for "unix" 230 | l, e := net.Listen("unix", peers[me]) 231 | if e != nil { 232 | log.Fatal("listen error: ", e) 233 | } 234 | px.l = l 235 | 236 | // please do not change any of the following code, 237 | // or do anything to subvert it. 238 | 239 | // create a thread to accept RPC connections 240 | go func() { 241 | for px.isdead() == false { 242 | conn, err := px.l.Accept() 243 | if err == nil && px.isdead() == false { 244 | if px.isunreliable() && (rand.Int63()%1000) < 100 { 245 | // discard the request. 246 | conn.Close() 247 | } else if px.isunreliable() && (rand.Int63()%1000) < 200 { 248 | // process the request but force discard of reply. 249 | c1 := conn.(*net.UnixConn) 250 | f, _ := c1.File() 251 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 252 | if err != nil { 253 | fmt.Printf("shutdown: %v\n", err) 254 | } 255 | atomic.AddInt32(&px.rpcCount, 1) 256 | go rpcs.ServeConn(conn) 257 | } else { 258 | atomic.AddInt32(&px.rpcCount, 1) 259 | go rpcs.ServeConn(conn) 260 | } 261 | } else if err == nil { 262 | conn.Close() 263 | } 264 | if err != nil && px.isdead() == false { 265 | fmt.Printf("Paxos(%v) accept: %v\n", me, err.Error()) 266 | } 267 | } 268 | }() 269 | } 270 | 271 | 272 | return px 273 | } 274 | -------------------------------------------------------------------------------- /src/paxos/test_test.go: -------------------------------------------------------------------------------- 1 | package paxos 2 | 3 | import "testing" 4 | import "runtime" 5 | import "strconv" 6 | import "os" 7 | import "time" 8 | import "fmt" 9 | import "math/rand" 10 | import crand "crypto/rand" 11 | import "encoding/base64" 12 | import "sync/atomic" 13 | 14 | func randstring(n int) string { 15 | b := make([]byte, 2*n) 16 | crand.Read(b) 17 | s := base64.URLEncoding.EncodeToString(b) 18 | return s[0:n] 19 | } 20 | 21 | func port(tag string, host int) string { 22 | s := "/var/tmp/824-" 23 | s += strconv.Itoa(os.Getuid()) + "/" 24 | os.Mkdir(s, 0777) 25 | s += "px-" 26 | s += strconv.Itoa(os.Getpid()) + "-" 27 | s += tag + "-" 28 | s += strconv.Itoa(host) 29 | return s 30 | } 31 | 32 | func ndecided(t *testing.T, pxa []*Paxos, seq int) int { 33 | count := 0 34 | var v interface{} 35 | for i := 0; i < len(pxa); i++ { 36 | if pxa[i] != nil { 37 | decided, v1 := pxa[i].Status(seq) 38 | if decided == Decided { 39 | if count > 0 && v != v1 { 40 | t.Fatalf("decided values do not match; seq=%v i=%v v=%v v1=%v", 41 | seq, i, v, v1) 42 | } 43 | count++ 44 | v = v1 45 | } 46 | } 47 | } 48 | return count 49 | } 50 | 51 | func waitn(t *testing.T, pxa []*Paxos, seq int, wanted int) { 52 | to := 10 * time.Millisecond 53 | for iters := 0; iters < 30; iters++ { 54 | if ndecided(t, pxa, seq) >= wanted { 55 | break 56 | } 57 | time.Sleep(to) 58 | if to < time.Second { 59 | to *= 2 60 | } 61 | } 62 | nd := ndecided(t, pxa, seq) 63 | if nd < wanted { 64 | t.Fatalf("too few decided; seq=%v ndecided=%v wanted=%v", seq, nd, wanted) 65 | } 66 | } 67 | 68 | func waitmajority(t *testing.T, pxa []*Paxos, seq int) { 69 | waitn(t, pxa, seq, (len(pxa)/2)+1) 70 | } 71 | 72 | func checkmax(t *testing.T, pxa []*Paxos, seq int, max int) { 73 | time.Sleep(3 * time.Second) 74 | nd := ndecided(t, pxa, seq) 75 | if nd > max { 76 | t.Fatalf("too many decided; seq=%v ndecided=%v max=%v", seq, nd, max) 77 | } 78 | } 79 | 80 | func cleanup(pxa []*Paxos) { 81 | for i := 0; i < len(pxa); i++ { 82 | if pxa[i] != nil { 83 | pxa[i].Kill() 84 | } 85 | } 86 | } 87 | 88 | func noTestSpeed(t *testing.T) { 89 | runtime.GOMAXPROCS(4) 90 | 91 | const npaxos = 3 92 | var pxa []*Paxos = make([]*Paxos, npaxos) 93 | var pxh []string = make([]string, npaxos) 94 | defer cleanup(pxa) 95 | 96 | for i := 0; i < npaxos; i++ { 97 | pxh[i] = port("time", i) 98 | } 99 | for i := 0; i < npaxos; i++ { 100 | pxa[i] = Make(pxh, i, nil) 101 | } 102 | 103 | t0 := time.Now() 104 | 105 | for i := 0; i < 20; i++ { 106 | pxa[0].Start(i, "x") 107 | waitn(t, pxa, i, npaxos) 108 | } 109 | 110 | d := time.Since(t0) 111 | fmt.Printf("20 agreements %v seconds\n", d.Seconds()) 112 | } 113 | 114 | func TestBasic(t *testing.T) { 115 | runtime.GOMAXPROCS(4) 116 | 117 | const npaxos = 3 118 | var pxa []*Paxos = make([]*Paxos, npaxos) 119 | var pxh []string = make([]string, npaxos) 120 | defer cleanup(pxa) 121 | 122 | for i := 0; i < npaxos; i++ { 123 | pxh[i] = port("basic", i) 124 | } 125 | for i := 0; i < npaxos; i++ { 126 | pxa[i] = Make(pxh, i, nil) 127 | } 128 | 129 | fmt.Printf("Test: Single proposer ...\n") 130 | 131 | pxa[0].Start(0, "hello") 132 | waitn(t, pxa, 0, npaxos) 133 | 134 | fmt.Printf(" ... Passed\n") 135 | 136 | fmt.Printf("Test: Many proposers, same value ...\n") 137 | 138 | for i := 0; i < npaxos; i++ { 139 | pxa[i].Start(1, 77) 140 | } 141 | waitn(t, pxa, 1, npaxos) 142 | 143 | fmt.Printf(" ... Passed\n") 144 | 145 | fmt.Printf("Test: Many proposers, different values ...\n") 146 | 147 | pxa[0].Start(2, 100) 148 | pxa[1].Start(2, 101) 149 | pxa[2].Start(2, 102) 150 | waitn(t, pxa, 2, npaxos) 151 | 152 | fmt.Printf(" ... Passed\n") 153 | 154 | fmt.Printf("Test: Out-of-order instances ...\n") 155 | 156 | pxa[0].Start(7, 700) 157 | pxa[0].Start(6, 600) 158 | pxa[1].Start(5, 500) 159 | waitn(t, pxa, 7, npaxos) 160 | pxa[0].Start(4, 400) 161 | pxa[1].Start(3, 300) 162 | waitn(t, pxa, 6, npaxos) 163 | waitn(t, pxa, 5, npaxos) 164 | waitn(t, pxa, 4, npaxos) 165 | waitn(t, pxa, 3, npaxos) 166 | 167 | if pxa[0].Max() != 7 { 168 | t.Fatalf("wrong Max()") 169 | } 170 | 171 | fmt.Printf(" ... Passed\n") 172 | } 173 | 174 | func TestDeaf(t *testing.T) { 175 | runtime.GOMAXPROCS(4) 176 | 177 | const npaxos = 5 178 | var pxa []*Paxos = make([]*Paxos, npaxos) 179 | var pxh []string = make([]string, npaxos) 180 | defer cleanup(pxa) 181 | 182 | for i := 0; i < npaxos; i++ { 183 | pxh[i] = port("deaf", i) 184 | } 185 | for i := 0; i < npaxos; i++ { 186 | pxa[i] = Make(pxh, i, nil) 187 | } 188 | 189 | fmt.Printf("Test: Deaf proposer ...\n") 190 | 191 | pxa[0].Start(0, "hello") 192 | waitn(t, pxa, 0, npaxos) 193 | 194 | os.Remove(pxh[0]) 195 | os.Remove(pxh[npaxos-1]) 196 | 197 | pxa[1].Start(1, "goodbye") 198 | waitmajority(t, pxa, 1) 199 | time.Sleep(1 * time.Second) 200 | if ndecided(t, pxa, 1) != npaxos-2 { 201 | t.Fatalf("a deaf peer heard about a decision") 202 | } 203 | 204 | pxa[0].Start(1, "xxx") 205 | waitn(t, pxa, 1, npaxos-1) 206 | time.Sleep(1 * time.Second) 207 | if ndecided(t, pxa, 1) != npaxos-1 { 208 | t.Fatalf("a deaf peer heard about a decision") 209 | } 210 | 211 | pxa[npaxos-1].Start(1, "yyy") 212 | waitn(t, pxa, 1, npaxos) 213 | 214 | fmt.Printf(" ... Passed\n") 215 | } 216 | 217 | func TestForget(t *testing.T) { 218 | runtime.GOMAXPROCS(4) 219 | 220 | const npaxos = 6 221 | var pxa []*Paxos = make([]*Paxos, npaxos) 222 | var pxh []string = make([]string, npaxos) 223 | defer cleanup(pxa) 224 | 225 | for i := 0; i < npaxos; i++ { 226 | pxh[i] = port("gc", i) 227 | } 228 | for i := 0; i < npaxos; i++ { 229 | pxa[i] = Make(pxh, i, nil) 230 | } 231 | 232 | fmt.Printf("Test: Forgetting ...\n") 233 | 234 | // initial Min() correct? 235 | for i := 0; i < npaxos; i++ { 236 | m := pxa[i].Min() 237 | if m > 0 { 238 | t.Fatalf("wrong initial Min() %v", m) 239 | } 240 | } 241 | 242 | pxa[0].Start(0, "00") 243 | pxa[1].Start(1, "11") 244 | pxa[2].Start(2, "22") 245 | pxa[0].Start(6, "66") 246 | pxa[1].Start(7, "77") 247 | 248 | waitn(t, pxa, 0, npaxos) 249 | 250 | // Min() correct? 251 | for i := 0; i < npaxos; i++ { 252 | m := pxa[i].Min() 253 | if m != 0 { 254 | t.Fatalf("wrong Min() %v; expected 0", m) 255 | } 256 | } 257 | 258 | waitn(t, pxa, 1, npaxos) 259 | 260 | // Min() correct? 261 | for i := 0; i < npaxos; i++ { 262 | m := pxa[i].Min() 263 | if m != 0 { 264 | t.Fatalf("wrong Min() %v; expected 0", m) 265 | } 266 | } 267 | 268 | // everyone Done() -> Min() changes? 269 | for i := 0; i < npaxos; i++ { 270 | pxa[i].Done(0) 271 | } 272 | for i := 1; i < npaxos; i++ { 273 | pxa[i].Done(1) 274 | } 275 | for i := 0; i < npaxos; i++ { 276 | pxa[i].Start(8+i, "xx") 277 | } 278 | allok := false 279 | for iters := 0; iters < 12; iters++ { 280 | allok = true 281 | for i := 0; i < npaxos; i++ { 282 | s := pxa[i].Min() 283 | if s != 1 { 284 | allok = false 285 | } 286 | } 287 | if allok { 288 | break 289 | } 290 | time.Sleep(1 * time.Second) 291 | } 292 | if allok != true { 293 | t.Fatalf("Min() did not advance after Done()") 294 | } 295 | 296 | fmt.Printf(" ... Passed\n") 297 | } 298 | 299 | func TestManyForget(t *testing.T) { 300 | runtime.GOMAXPROCS(4) 301 | 302 | const npaxos = 3 303 | var pxa []*Paxos = make([]*Paxos, npaxos) 304 | var pxh []string = make([]string, npaxos) 305 | defer cleanup(pxa) 306 | 307 | for i := 0; i < npaxos; i++ { 308 | pxh[i] = port("manygc", i) 309 | } 310 | for i := 0; i < npaxos; i++ { 311 | pxa[i] = Make(pxh, i, nil) 312 | pxa[i].setunreliable(true) 313 | } 314 | 315 | fmt.Printf("Test: Lots of forgetting ...\n") 316 | 317 | const maxseq = 20 318 | 319 | go func() { 320 | na := rand.Perm(maxseq) 321 | for i := 0; i < len(na); i++ { 322 | seq := na[i] 323 | j := (rand.Int() % npaxos) 324 | v := rand.Int() 325 | pxa[j].Start(seq, v) 326 | runtime.Gosched() 327 | } 328 | }() 329 | 330 | done := make(chan bool) 331 | go func() { 332 | for { 333 | select { 334 | case <-done: 335 | return 336 | default: 337 | } 338 | seq := (rand.Int() % maxseq) 339 | i := (rand.Int() % npaxos) 340 | if seq >= pxa[i].Min() { 341 | decided, _ := pxa[i].Status(seq) 342 | if decided == Decided { 343 | pxa[i].Done(seq) 344 | } 345 | } 346 | runtime.Gosched() 347 | } 348 | }() 349 | 350 | time.Sleep(5 * time.Second) 351 | done <- true 352 | for i := 0; i < npaxos; i++ { 353 | pxa[i].setunreliable(false) 354 | } 355 | time.Sleep(2 * time.Second) 356 | 357 | for seq := 0; seq < maxseq; seq++ { 358 | for i := 0; i < npaxos; i++ { 359 | if seq >= pxa[i].Min() { 360 | pxa[i].Status(seq) 361 | } 362 | } 363 | } 364 | 365 | fmt.Printf(" ... Passed\n") 366 | } 367 | 368 | // 369 | // does paxos forgetting actually free the memory? 370 | // 371 | func TestForgetMem(t *testing.T) { 372 | runtime.GOMAXPROCS(4) 373 | 374 | fmt.Printf("Test: Paxos frees forgotten instance memory ...\n") 375 | 376 | const npaxos = 3 377 | var pxa []*Paxos = make([]*Paxos, npaxos) 378 | var pxh []string = make([]string, npaxos) 379 | defer cleanup(pxa) 380 | 381 | for i := 0; i < npaxos; i++ { 382 | pxh[i] = port("gcmem", i) 383 | } 384 | for i := 0; i < npaxos; i++ { 385 | pxa[i] = Make(pxh, i, nil) 386 | } 387 | 388 | pxa[0].Start(0, "x") 389 | waitn(t, pxa, 0, npaxos) 390 | 391 | runtime.GC() 392 | var m0 runtime.MemStats 393 | runtime.ReadMemStats(&m0) 394 | // m0.Alloc about a megabyte 395 | 396 | for i := 1; i <= 10; i++ { 397 | big := make([]byte, 1000000) 398 | for j := 0; j < len(big); j++ { 399 | big[j] = byte('a' + rand.Int()%26) 400 | } 401 | pxa[0].Start(i, string(big)) 402 | waitn(t, pxa, i, npaxos) 403 | } 404 | 405 | runtime.GC() 406 | var m1 runtime.MemStats 407 | runtime.ReadMemStats(&m1) 408 | // m1.Alloc about 90 megabytes 409 | 410 | for i := 0; i < npaxos; i++ { 411 | pxa[i].Done(10) 412 | } 413 | for i := 0; i < npaxos; i++ { 414 | pxa[i].Start(11+i, "z") 415 | } 416 | time.Sleep(3 * time.Second) 417 | for i := 0; i < npaxos; i++ { 418 | if pxa[i].Min() != 11 { 419 | t.Fatalf("expected Min() %v, got %v\n", 11, pxa[i].Min()) 420 | } 421 | } 422 | 423 | runtime.GC() 424 | var m2 runtime.MemStats 425 | runtime.ReadMemStats(&m2) 426 | // m2.Alloc about 10 megabytes 427 | 428 | if m2.Alloc > (m1.Alloc / 2) { 429 | t.Fatalf("memory use did not shrink enough") 430 | } 431 | 432 | again := make([]string, 10) 433 | for seq := 0; seq < npaxos && seq < 10; seq++ { 434 | again[seq] = randstring(20) 435 | for i := 0; i < npaxos; i++ { 436 | fate, _ := pxa[i].Status(seq) 437 | if fate != Forgotten { 438 | t.Fatalf("seq %d < Min() %d but not Forgotten", seq, pxa[i].Min()) 439 | } 440 | pxa[i].Start(seq, again[seq]) 441 | } 442 | } 443 | time.Sleep(1 * time.Second) 444 | for seq := 0; seq < npaxos && seq < 10; seq++ { 445 | for i := 0; i < npaxos; i++ { 446 | fate, v := pxa[i].Status(seq) 447 | if fate != Forgotten || v == again[seq] { 448 | t.Fatalf("seq %d < Min() %d but not Forgotten", seq, pxa[i].Min()) 449 | } 450 | } 451 | } 452 | 453 | fmt.Printf(" ... Passed\n") 454 | } 455 | 456 | // 457 | // does Max() work after Done()s? 458 | // 459 | func TestDoneMax(t *testing.T) { 460 | runtime.GOMAXPROCS(4) 461 | 462 | fmt.Printf("Test: Paxos Max() after Done()s ...\n") 463 | 464 | const npaxos = 3 465 | var pxa []*Paxos = make([]*Paxos, npaxos) 466 | var pxh []string = make([]string, npaxos) 467 | defer cleanup(pxa) 468 | 469 | for i := 0; i < npaxos; i++ { 470 | pxh[i] = port("donemax", i) 471 | } 472 | for i := 0; i < npaxos; i++ { 473 | pxa[i] = Make(pxh, i, nil) 474 | } 475 | 476 | pxa[0].Start(0, "x") 477 | waitn(t, pxa, 0, npaxos) 478 | 479 | for i := 1; i <= 10; i++ { 480 | pxa[0].Start(i, "y") 481 | waitn(t, pxa, i, npaxos) 482 | } 483 | 484 | for i := 0; i < npaxos; i++ { 485 | pxa[i].Done(10) 486 | } 487 | 488 | // Propagate messages so everyone knows about Done(10) 489 | for i := 0; i < npaxos; i++ { 490 | pxa[i].Start(10, "z") 491 | } 492 | time.Sleep(2 * time.Second) 493 | for i := 0; i < npaxos; i++ { 494 | mx := pxa[i].Max() 495 | if mx != 10 { 496 | t.Fatalf("Max() did not return correct result %d after calling Done(); returned %d", 10, mx) 497 | } 498 | } 499 | 500 | fmt.Printf(" ... Passed\n") 501 | } 502 | 503 | func TestRPCCount(t *testing.T) { 504 | runtime.GOMAXPROCS(4) 505 | 506 | fmt.Printf("Test: RPC counts aren't too high ...\n") 507 | 508 | const npaxos = 3 509 | var pxa []*Paxos = make([]*Paxos, npaxos) 510 | var pxh []string = make([]string, npaxos) 511 | defer cleanup(pxa) 512 | 513 | for i := 0; i < npaxos; i++ { 514 | pxh[i] = port("count", i) 515 | } 516 | for i := 0; i < npaxos; i++ { 517 | pxa[i] = Make(pxh, i, nil) 518 | } 519 | 520 | ninst1 := 5 521 | seq := 0 522 | for i := 0; i < ninst1; i++ { 523 | pxa[0].Start(seq, "x") 524 | waitn(t, pxa, seq, npaxos) 525 | seq++ 526 | } 527 | 528 | time.Sleep(2 * time.Second) 529 | 530 | total1 := int32(0) 531 | for j := 0; j < npaxos; j++ { 532 | total1 += atomic.LoadInt32(&pxa[j].rpcCount) 533 | } 534 | 535 | // per agreement: 536 | // 3 prepares 537 | // 3 accepts 538 | // 3 decides 539 | expected1 := int32(ninst1 * npaxos * npaxos) 540 | if total1 > expected1 { 541 | t.Fatalf("too many RPCs for serial Start()s; %v instances, got %v, expected %v", 542 | ninst1, total1, expected1) 543 | } 544 | 545 | ninst2 := 5 546 | for i := 0; i < ninst2; i++ { 547 | for j := 0; j < npaxos; j++ { 548 | go pxa[j].Start(seq, j+(i*10)) 549 | } 550 | waitn(t, pxa, seq, npaxos) 551 | seq++ 552 | } 553 | 554 | time.Sleep(2 * time.Second) 555 | 556 | total2 := int32(0) 557 | for j := 0; j < npaxos; j++ { 558 | total2 += atomic.LoadInt32(&pxa[j].rpcCount) 559 | } 560 | total2 -= total1 561 | 562 | // worst case per agreement: 563 | // Proposer 1: 3 prep, 3 acc, 3 decides. 564 | // Proposer 2: 3 prep, 3 acc, 3 prep, 3 acc, 3 decides. 565 | // Proposer 3: 3 prep, 3 acc, 3 prep, 3 acc, 3 prep, 3 acc, 3 decides. 566 | expected2 := int32(ninst2 * npaxos * 15) 567 | if total2 > expected2 { 568 | t.Fatalf("too many RPCs for concurrent Start()s; %v instances, got %v, expected %v", 569 | ninst2, total2, expected2) 570 | } 571 | 572 | fmt.Printf(" ... Passed\n") 573 | } 574 | 575 | // 576 | // many agreements (without failures) 577 | // 578 | func TestMany(t *testing.T) { 579 | runtime.GOMAXPROCS(4) 580 | 581 | fmt.Printf("Test: Many instances ...\n") 582 | 583 | const npaxos = 3 584 | var pxa []*Paxos = make([]*Paxos, npaxos) 585 | var pxh []string = make([]string, npaxos) 586 | defer cleanup(pxa) 587 | 588 | for i := 0; i < npaxos; i++ { 589 | pxh[i] = port("many", i) 590 | } 591 | for i := 0; i < npaxos; i++ { 592 | pxa[i] = Make(pxh, i, nil) 593 | pxa[i].Start(0, 0) 594 | } 595 | 596 | const ninst = 50 597 | for seq := 1; seq < ninst; seq++ { 598 | // only 5 active instances, to limit the 599 | // number of file descriptors. 600 | for seq >= 5 && ndecided(t, pxa, seq-5) < npaxos { 601 | time.Sleep(20 * time.Millisecond) 602 | } 603 | for i := 0; i < npaxos; i++ { 604 | pxa[i].Start(seq, (seq*10)+i) 605 | } 606 | } 607 | 608 | for { 609 | done := true 610 | for seq := 1; seq < ninst; seq++ { 611 | if ndecided(t, pxa, seq) < npaxos { 612 | done = false 613 | } 614 | } 615 | if done { 616 | break 617 | } 618 | time.Sleep(100 * time.Millisecond) 619 | } 620 | 621 | fmt.Printf(" ... Passed\n") 622 | } 623 | 624 | // 625 | // a peer starts up, with proposal, after others decide. 626 | // then another peer starts, without a proposal. 627 | // 628 | func TestOld(t *testing.T) { 629 | runtime.GOMAXPROCS(4) 630 | 631 | fmt.Printf("Test: Minority proposal ignored ...\n") 632 | 633 | const npaxos = 5 634 | var pxa []*Paxos = make([]*Paxos, npaxos) 635 | var pxh []string = make([]string, npaxos) 636 | defer cleanup(pxa) 637 | 638 | for i := 0; i < npaxos; i++ { 639 | pxh[i] = port("old", i) 640 | } 641 | 642 | pxa[1] = Make(pxh, 1, nil) 643 | pxa[2] = Make(pxh, 2, nil) 644 | pxa[3] = Make(pxh, 3, nil) 645 | pxa[1].Start(1, 111) 646 | 647 | waitmajority(t, pxa, 1) 648 | 649 | pxa[0] = Make(pxh, 0, nil) 650 | pxa[0].Start(1, 222) 651 | 652 | waitn(t, pxa, 1, 4) 653 | 654 | if false { 655 | pxa[4] = Make(pxh, 4, nil) 656 | waitn(t, pxa, 1, npaxos) 657 | } 658 | 659 | fmt.Printf(" ... Passed\n") 660 | } 661 | 662 | // 663 | // many agreements, with unreliable RPC 664 | // 665 | func TestManyUnreliable(t *testing.T) { 666 | runtime.GOMAXPROCS(4) 667 | 668 | fmt.Printf("Test: Many instances, unreliable RPC ...\n") 669 | 670 | const npaxos = 3 671 | var pxa []*Paxos = make([]*Paxos, npaxos) 672 | var pxh []string = make([]string, npaxos) 673 | defer cleanup(pxa) 674 | 675 | for i := 0; i < npaxos; i++ { 676 | pxh[i] = port("manyun", i) 677 | } 678 | for i := 0; i < npaxos; i++ { 679 | pxa[i] = Make(pxh, i, nil) 680 | pxa[i].setunreliable(true) 681 | pxa[i].Start(0, 0) 682 | } 683 | 684 | const ninst = 50 685 | for seq := 1; seq < ninst; seq++ { 686 | // only 3 active instances, to limit the 687 | // number of file descriptors. 688 | for seq >= 3 && ndecided(t, pxa, seq-3) < npaxos { 689 | time.Sleep(20 * time.Millisecond) 690 | } 691 | for i := 0; i < npaxos; i++ { 692 | pxa[i].Start(seq, (seq*10)+i) 693 | } 694 | } 695 | 696 | for { 697 | done := true 698 | for seq := 1; seq < ninst; seq++ { 699 | if ndecided(t, pxa, seq) < npaxos { 700 | done = false 701 | } 702 | } 703 | if done { 704 | break 705 | } 706 | time.Sleep(100 * time.Millisecond) 707 | } 708 | 709 | fmt.Printf(" ... Passed\n") 710 | } 711 | 712 | func pp(tag string, src int, dst int) string { 713 | s := "/var/tmp/824-" 714 | s += strconv.Itoa(os.Getuid()) + "/" 715 | s += "px-" + tag + "-" 716 | s += strconv.Itoa(os.Getpid()) + "-" 717 | s += strconv.Itoa(src) + "-" 718 | s += strconv.Itoa(dst) 719 | return s 720 | } 721 | 722 | func cleanpp(tag string, n int) { 723 | for i := 0; i < n; i++ { 724 | for j := 0; j < n; j++ { 725 | ij := pp(tag, i, j) 726 | os.Remove(ij) 727 | } 728 | } 729 | } 730 | 731 | func part(t *testing.T, tag string, npaxos int, p1 []int, p2 []int, p3 []int) { 732 | cleanpp(tag, npaxos) 733 | 734 | pa := [][]int{p1, p2, p3} 735 | for pi := 0; pi < len(pa); pi++ { 736 | p := pa[pi] 737 | for i := 0; i < len(p); i++ { 738 | for j := 0; j < len(p); j++ { 739 | ij := pp(tag, p[i], p[j]) 740 | pj := port(tag, p[j]) 741 | err := os.Link(pj, ij) 742 | if err != nil { 743 | // one reason this link can fail is if the 744 | // corresponding Paxos peer has prematurely quit and 745 | // deleted its socket file (e.g., called px.Kill()). 746 | t.Fatalf("os.Link(%v, %v): %v\n", pj, ij, err) 747 | } 748 | } 749 | } 750 | } 751 | } 752 | 753 | func TestPartition(t *testing.T) { 754 | runtime.GOMAXPROCS(4) 755 | 756 | tag := "partition" 757 | const npaxos = 5 758 | var pxa []*Paxos = make([]*Paxos, npaxos) 759 | defer cleanup(pxa) 760 | defer cleanpp(tag, npaxos) 761 | 762 | for i := 0; i < npaxos; i++ { 763 | var pxh []string = make([]string, npaxos) 764 | for j := 0; j < npaxos; j++ { 765 | if j == i { 766 | pxh[j] = port(tag, i) 767 | } else { 768 | pxh[j] = pp(tag, i, j) 769 | } 770 | } 771 | pxa[i] = Make(pxh, i, nil) 772 | } 773 | defer part(t, tag, npaxos, []int{}, []int{}, []int{}) 774 | 775 | seq := 0 776 | 777 | fmt.Printf("Test: No decision if partitioned ...\n") 778 | 779 | part(t, tag, npaxos, []int{0, 2}, []int{1, 3}, []int{4}) 780 | pxa[1].Start(seq, 111) 781 | checkmax(t, pxa, seq, 0) 782 | 783 | fmt.Printf(" ... Passed\n") 784 | 785 | fmt.Printf("Test: Decision in majority partition ...\n") 786 | 787 | part(t, tag, npaxos, []int{0}, []int{1, 2, 3}, []int{4}) 788 | time.Sleep(2 * time.Second) 789 | waitmajority(t, pxa, seq) 790 | 791 | fmt.Printf(" ... Passed\n") 792 | 793 | fmt.Printf("Test: All agree after full heal ...\n") 794 | 795 | pxa[0].Start(seq, 1000) // poke them 796 | pxa[4].Start(seq, 1004) 797 | part(t, tag, npaxos, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 798 | 799 | waitn(t, pxa, seq, npaxos) 800 | 801 | fmt.Printf(" ... Passed\n") 802 | 803 | fmt.Printf("Test: One peer switches partitions ...\n") 804 | 805 | for iters := 0; iters < 20; iters++ { 806 | seq++ 807 | 808 | part(t, tag, npaxos, []int{0, 1, 2}, []int{3, 4}, []int{}) 809 | pxa[0].Start(seq, seq*10) 810 | pxa[3].Start(seq, (seq*10)+1) 811 | waitmajority(t, pxa, seq) 812 | if ndecided(t, pxa, seq) > 3 { 813 | t.Fatalf("too many decided") 814 | } 815 | 816 | part(t, tag, npaxos, []int{0, 1}, []int{2, 3, 4}, []int{}) 817 | waitn(t, pxa, seq, npaxos) 818 | } 819 | 820 | fmt.Printf(" ... Passed\n") 821 | 822 | fmt.Printf("Test: One peer switches partitions, unreliable ...\n") 823 | 824 | for iters := 0; iters < 20; iters++ { 825 | seq++ 826 | 827 | for i := 0; i < npaxos; i++ { 828 | pxa[i].setunreliable(true) 829 | } 830 | 831 | part(t, tag, npaxos, []int{0, 1, 2}, []int{3, 4}, []int{}) 832 | for i := 0; i < npaxos; i++ { 833 | pxa[i].Start(seq, (seq*10)+i) 834 | } 835 | waitn(t, pxa, seq, 3) 836 | if ndecided(t, pxa, seq) > 3 { 837 | t.Fatalf("too many decided") 838 | } 839 | 840 | part(t, tag, npaxos, []int{0, 1}, []int{2, 3, 4}, []int{}) 841 | 842 | for i := 0; i < npaxos; i++ { 843 | pxa[i].setunreliable(false) 844 | } 845 | 846 | waitn(t, pxa, seq, 5) 847 | } 848 | 849 | fmt.Printf(" ... Passed\n") 850 | } 851 | 852 | func TestLots(t *testing.T) { 853 | runtime.GOMAXPROCS(4) 854 | 855 | fmt.Printf("Test: Many requests, changing partitions ...\n") 856 | 857 | tag := "lots" 858 | const npaxos = 5 859 | var pxa []*Paxos = make([]*Paxos, npaxos) 860 | defer cleanup(pxa) 861 | defer cleanpp(tag, npaxos) 862 | 863 | for i := 0; i < npaxos; i++ { 864 | var pxh []string = make([]string, npaxos) 865 | for j := 0; j < npaxos; j++ { 866 | if j == i { 867 | pxh[j] = port(tag, i) 868 | } else { 869 | pxh[j] = pp(tag, i, j) 870 | } 871 | } 872 | pxa[i] = Make(pxh, i, nil) 873 | pxa[i].setunreliable(true) 874 | } 875 | defer part(t, tag, npaxos, []int{}, []int{}, []int{}) 876 | 877 | done := int32(0) 878 | 879 | // re-partition periodically 880 | ch1 := make(chan bool) 881 | go func() { 882 | defer func() { ch1 <- true }() 883 | for atomic.LoadInt32(&done) == 0 { 884 | var a [npaxos]int 885 | for i := 0; i < npaxos; i++ { 886 | a[i] = (rand.Int() % 3) 887 | } 888 | pa := make([][]int, 3) 889 | for i := 0; i < 3; i++ { 890 | pa[i] = make([]int, 0) 891 | for j := 0; j < npaxos; j++ { 892 | if a[j] == i { 893 | pa[i] = append(pa[i], j) 894 | } 895 | } 896 | } 897 | part(t, tag, npaxos, pa[0], pa[1], pa[2]) 898 | time.Sleep(time.Duration(rand.Int63()%200) * time.Millisecond) 899 | } 900 | }() 901 | 902 | seq := int32(0) 903 | 904 | // periodically start a new instance 905 | ch2 := make(chan bool) 906 | go func() { 907 | defer func() { ch2 <- true }() 908 | for atomic.LoadInt32(&done) == 0 { 909 | // how many instances are in progress? 910 | nd := 0 911 | sq := int(atomic.LoadInt32(&seq)) 912 | for i := 0; i < sq; i++ { 913 | if ndecided(t, pxa, i) == npaxos { 914 | nd++ 915 | } 916 | } 917 | if sq-nd < 10 { 918 | for i := 0; i < npaxos; i++ { 919 | pxa[i].Start(sq, rand.Int()%10) 920 | } 921 | atomic.AddInt32(&seq, 1) 922 | } 923 | time.Sleep(time.Duration(rand.Int63()%300) * time.Millisecond) 924 | } 925 | }() 926 | 927 | // periodically check that decisions are consistent 928 | ch3 := make(chan bool) 929 | go func() { 930 | defer func() { ch3 <- true }() 931 | for atomic.LoadInt32(&done) == 0 { 932 | for i := 0; i < int(atomic.LoadInt32(&seq)); i++ { 933 | ndecided(t, pxa, i) 934 | } 935 | time.Sleep(time.Duration(rand.Int63()%300) * time.Millisecond) 936 | } 937 | }() 938 | 939 | time.Sleep(20 * time.Second) 940 | atomic.StoreInt32(&done, 1) 941 | <-ch1 942 | <-ch2 943 | <-ch3 944 | 945 | // repair, then check that all instances decided. 946 | for i := 0; i < npaxos; i++ { 947 | pxa[i].setunreliable(false) 948 | } 949 | part(t, tag, npaxos, []int{0, 1, 2, 3, 4}, []int{}, []int{}) 950 | time.Sleep(5 * time.Second) 951 | 952 | for i := 0; i < int(atomic.LoadInt32(&seq)); i++ { 953 | waitmajority(t, pxa, i) 954 | } 955 | 956 | fmt.Printf(" ... Passed\n") 957 | } 958 | -------------------------------------------------------------------------------- /src/pbservice/client.go: -------------------------------------------------------------------------------- 1 | package pbservice 2 | 3 | import "viewservice" 4 | import "net/rpc" 5 | import "fmt" 6 | 7 | import "crypto/rand" 8 | import "math/big" 9 | 10 | 11 | type Clerk struct { 12 | vs *viewservice.Clerk 13 | // Your declarations here 14 | } 15 | 16 | // this may come in handy. 17 | func nrand() int64 { 18 | max := big.NewInt(int64(1) << 62) 19 | bigx, _ := rand.Int(rand.Reader, max) 20 | x := bigx.Int64() 21 | return x 22 | } 23 | 24 | func MakeClerk(vshost string, me string) *Clerk { 25 | ck := new(Clerk) 26 | ck.vs = viewservice.MakeClerk(me, vshost) 27 | // Your ck.* initializations here 28 | 29 | return ck 30 | } 31 | 32 | 33 | // 34 | // call() sends an RPC to the rpcname handler on server srv 35 | // with arguments args, waits for the reply, and leaves the 36 | // reply in reply. the reply argument should be a pointer 37 | // to a reply structure. 38 | // 39 | // the return value is true if the server responded, and false 40 | // if call() was not able to contact the server. in particular, 41 | // the reply's contents are only valid if call() returned true. 42 | // 43 | // you should assume that call() will return an 44 | // error after a while if the server is dead. 45 | // don't provide your own time-out mechanism. 46 | // 47 | // please use call() to send all RPCs, in client.go and server.go. 48 | // please don't change this function. 49 | // 50 | func call(srv string, rpcname string, 51 | args interface{}, reply interface{}) bool { 52 | c, errx := rpc.Dial("unix", srv) 53 | if errx != nil { 54 | return false 55 | } 56 | defer c.Close() 57 | 58 | err := c.Call(rpcname, args, reply) 59 | if err == nil { 60 | return true 61 | } 62 | 63 | fmt.Println(err) 64 | return false 65 | } 66 | 67 | // 68 | // fetch a key's value from the current primary; 69 | // if they key has never been set, return "". 70 | // Get() must keep trying until it either the 71 | // primary replies with the value or the primary 72 | // says the key doesn't exist (has never been Put(). 73 | // 74 | func (ck *Clerk) Get(key string) string { 75 | 76 | // Your code here. 77 | 78 | return "???" 79 | } 80 | 81 | // 82 | // send a Put or Append RPC 83 | // 84 | func (ck *Clerk) PutAppend(key string, value string, op string) { 85 | 86 | // Your code here. 87 | } 88 | 89 | // 90 | // tell the primary to update key's value. 91 | // must keep trying until it succeeds. 92 | // 93 | func (ck *Clerk) Put(key string, value string) { 94 | ck.PutAppend(key, value, "Put") 95 | } 96 | 97 | // 98 | // tell the primary to append to key's value. 99 | // must keep trying until it succeeds. 100 | // 101 | func (ck *Clerk) Append(key string, value string) { 102 | ck.PutAppend(key, value, "Append") 103 | } 104 | -------------------------------------------------------------------------------- /src/pbservice/common.go: -------------------------------------------------------------------------------- 1 | package pbservice 2 | 3 | const ( 4 | OK = "OK" 5 | ErrNoKey = "ErrNoKey" 6 | ErrWrongServer = "ErrWrongServer" 7 | ) 8 | 9 | type Err string 10 | 11 | // Put or Append 12 | type PutAppendArgs struct { 13 | Key string 14 | Value string 15 | // You'll have to add definitions here. 16 | 17 | // Field names must start with capital letters, 18 | // otherwise RPC will break. 19 | } 20 | 21 | type PutAppendReply struct { 22 | Err Err 23 | } 24 | 25 | type GetArgs struct { 26 | Key string 27 | // You'll have to add definitions here. 28 | } 29 | 30 | type GetReply struct { 31 | Err Err 32 | Value string 33 | } 34 | 35 | 36 | // Your RPC definitions here. 37 | -------------------------------------------------------------------------------- /src/pbservice/server.go: -------------------------------------------------------------------------------- 1 | package pbservice 2 | 3 | import "net" 4 | import "fmt" 5 | import "net/rpc" 6 | import "log" 7 | import "time" 8 | import "viewservice" 9 | import "sync" 10 | import "sync/atomic" 11 | import "os" 12 | import "syscall" 13 | import "math/rand" 14 | 15 | 16 | 17 | type PBServer struct { 18 | mu sync.Mutex 19 | l net.Listener 20 | dead int32 // for testing 21 | unreliable int32 // for testing 22 | me string 23 | vs *viewservice.Clerk 24 | // Your declarations here. 25 | } 26 | 27 | 28 | func (pb *PBServer) Get(args *GetArgs, reply *GetReply) error { 29 | 30 | // Your code here. 31 | 32 | return nil 33 | } 34 | 35 | 36 | func (pb *PBServer) PutAppend(args *PutAppendArgs, reply *PutAppendReply) error { 37 | 38 | // Your code here. 39 | 40 | 41 | return nil 42 | } 43 | 44 | 45 | // 46 | // ping the viewserver periodically. 47 | // if view changed: 48 | // transition to new view. 49 | // manage transfer of state from primary to new backup. 50 | // 51 | func (pb *PBServer) tick() { 52 | 53 | // Your code here. 54 | } 55 | 56 | // tell the server to shut itself down. 57 | // please do not change these two functions. 58 | func (pb *PBServer) kill() { 59 | atomic.StoreInt32(&pb.dead, 1) 60 | pb.l.Close() 61 | } 62 | 63 | // call this to find out if the server is dead. 64 | func (pb *PBServer) isdead() bool { 65 | return atomic.LoadInt32(&pb.dead) != 0 66 | } 67 | 68 | // please do not change these two functions. 69 | func (pb *PBServer) setunreliable(what bool) { 70 | if what { 71 | atomic.StoreInt32(&pb.unreliable, 1) 72 | } else { 73 | atomic.StoreInt32(&pb.unreliable, 0) 74 | } 75 | } 76 | 77 | func (pb *PBServer) isunreliable() bool { 78 | return atomic.LoadInt32(&pb.unreliable) != 0 79 | } 80 | 81 | 82 | func StartServer(vshost string, me string) *PBServer { 83 | pb := new(PBServer) 84 | pb.me = me 85 | pb.vs = viewservice.MakeClerk(me, vshost) 86 | // Your pb.* initializations here. 87 | 88 | rpcs := rpc.NewServer() 89 | rpcs.Register(pb) 90 | 91 | os.Remove(pb.me) 92 | l, e := net.Listen("unix", pb.me) 93 | if e != nil { 94 | log.Fatal("listen error: ", e) 95 | } 96 | pb.l = l 97 | 98 | // please do not change any of the following code, 99 | // or do anything to subvert it. 100 | 101 | go func() { 102 | for pb.isdead() == false { 103 | conn, err := pb.l.Accept() 104 | if err == nil && pb.isdead() == false { 105 | if pb.isunreliable() && (rand.Int63()%1000) < 100 { 106 | // discard the request. 107 | conn.Close() 108 | } else if pb.isunreliable() && (rand.Int63()%1000) < 200 { 109 | // process the request but force discard of reply. 110 | c1 := conn.(*net.UnixConn) 111 | f, _ := c1.File() 112 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 113 | if err != nil { 114 | fmt.Printf("shutdown: %v\n", err) 115 | } 116 | go rpcs.ServeConn(conn) 117 | } else { 118 | go rpcs.ServeConn(conn) 119 | } 120 | } else if err == nil { 121 | conn.Close() 122 | } 123 | if err != nil && pb.isdead() == false { 124 | fmt.Printf("PBServer(%v) accept: %v\n", me, err.Error()) 125 | pb.kill() 126 | } 127 | } 128 | }() 129 | 130 | go func() { 131 | for pb.isdead() == false { 132 | pb.tick() 133 | time.Sleep(viewservice.PingInterval) 134 | } 135 | }() 136 | 137 | return pb 138 | } 139 | -------------------------------------------------------------------------------- /src/pbservice/test_test.go: -------------------------------------------------------------------------------- 1 | package pbservice 2 | 3 | import "viewservice" 4 | import "fmt" 5 | import "io" 6 | import "net" 7 | import "testing" 8 | import "time" 9 | import "log" 10 | import "runtime" 11 | import "math/rand" 12 | import "os" 13 | import "sync" 14 | import "strconv" 15 | import "strings" 16 | import "sync/atomic" 17 | 18 | func check(ck *Clerk, key string, value string) { 19 | v := ck.Get(key) 20 | if v != value { 21 | log.Fatalf("Get(%v) -> %v, expected %v", key, v, value) 22 | } 23 | } 24 | 25 | func port(tag string, host int) string { 26 | s := "/var/tmp/824-" 27 | s += strconv.Itoa(os.Getuid()) + "/" 28 | os.Mkdir(s, 0777) 29 | s += "pb-" 30 | s += strconv.Itoa(os.Getpid()) + "-" 31 | s += tag + "-" 32 | s += strconv.Itoa(host) 33 | return s 34 | } 35 | 36 | func TestBasicFail(t *testing.T) { 37 | runtime.GOMAXPROCS(4) 38 | 39 | tag := "basic" 40 | vshost := port(tag+"v", 1) 41 | vs := viewservice.StartServer(vshost) 42 | time.Sleep(time.Second) 43 | vck := viewservice.MakeClerk("", vshost) 44 | 45 | ck := MakeClerk(vshost, "") 46 | 47 | fmt.Printf("Test: Single primary, no backup ...\n") 48 | 49 | s1 := StartServer(vshost, port(tag, 1)) 50 | 51 | deadtime := viewservice.PingInterval * viewservice.DeadPings 52 | time.Sleep(deadtime * 2) 53 | if vck.Primary() != s1.me { 54 | t.Fatal("first primary never formed view") 55 | } 56 | 57 | ck.Put("111", "v1") 58 | check(ck, "111", "v1") 59 | 60 | ck.Put("2", "v2") 61 | check(ck, "2", "v2") 62 | 63 | ck.Put("1", "v1a") 64 | check(ck, "1", "v1a") 65 | 66 | ck.Append("ak", "hello") 67 | check(ck, "ak", "hello") 68 | ck.Put("ak", "xx") 69 | ck.Append("ak", "yy") 70 | check(ck, "ak", "xxyy") 71 | 72 | fmt.Printf(" ... Passed\n") 73 | 74 | // add a backup 75 | 76 | fmt.Printf("Test: Add a backup ...\n") 77 | 78 | s2 := StartServer(vshost, port(tag, 2)) 79 | for i := 0; i < viewservice.DeadPings*2; i++ { 80 | v, _ := vck.Get() 81 | if v.Backup == s2.me { 82 | break 83 | } 84 | time.Sleep(viewservice.PingInterval) 85 | } 86 | v, _ := vck.Get() 87 | if v.Backup != s2.me { 88 | t.Fatal("backup never came up") 89 | } 90 | 91 | ck.Put("3", "33") 92 | check(ck, "3", "33") 93 | 94 | // give the backup time to initialize 95 | time.Sleep(3 * viewservice.PingInterval) 96 | 97 | ck.Put("4", "44") 98 | check(ck, "4", "44") 99 | 100 | fmt.Printf(" ... Passed\n") 101 | 102 | fmt.Printf("Test: Count RPCs to viewserver ...\n") 103 | 104 | // verify that the client or server doesn't contact the 105 | // viewserver for every request -- i.e. that both client 106 | // and servers cache the current view and only refresh 107 | // it when something seems to be wrong. this test allows 108 | // each server to Ping() the viewserver 10 times / second. 109 | 110 | count1 := int(vs.GetRPCCount()) 111 | t1 := time.Now() 112 | for i := 0; i < 100; i++ { 113 | ck.Put("xk"+strconv.Itoa(i), strconv.Itoa(i)) 114 | } 115 | count2 := int(vs.GetRPCCount()) 116 | t2 := time.Now() 117 | dt := t2.Sub(t1) 118 | allowed := 2 * (dt / (100 * time.Millisecond)) // two servers tick()ing 10/second 119 | if (count2 - count1) > int(allowed)+20 { 120 | t.Fatal("too many viewserver RPCs") 121 | } 122 | 123 | fmt.Printf(" ... Passed\n") 124 | 125 | // kill the primary 126 | 127 | fmt.Printf("Test: Primary failure ...\n") 128 | 129 | s1.kill() 130 | for i := 0; i < viewservice.DeadPings*2; i++ { 131 | v, _ := vck.Get() 132 | if v.Primary == s2.me { 133 | break 134 | } 135 | time.Sleep(viewservice.PingInterval) 136 | } 137 | v, _ = vck.Get() 138 | if v.Primary != s2.me { 139 | t.Fatal("backup never switched to primary") 140 | } 141 | 142 | check(ck, "1", "v1a") 143 | check(ck, "3", "33") 144 | check(ck, "4", "44") 145 | 146 | fmt.Printf(" ... Passed\n") 147 | 148 | // kill solo server, start new server, check that 149 | // it does not start serving as primary 150 | 151 | fmt.Printf("Test: Kill last server, new one should not be active ...\n") 152 | 153 | s2.kill() 154 | s3 := StartServer(vshost, port(tag, 3)) 155 | time.Sleep(1 * time.Second) 156 | get_done := make(chan bool) 157 | go func() { 158 | ck.Get("1") 159 | get_done <- true 160 | }() 161 | 162 | select { 163 | case <-get_done: 164 | t.Fatalf("ck.Get() returned even though no initialized primary") 165 | case <-time.After(2 * time.Second): 166 | } 167 | 168 | fmt.Printf(" ... Passed\n") 169 | 170 | s1.kill() 171 | s2.kill() 172 | s3.kill() 173 | time.Sleep(time.Second) 174 | vs.Kill() 175 | time.Sleep(time.Second) 176 | } 177 | 178 | func TestAtMostOnce(t *testing.T) { 179 | runtime.GOMAXPROCS(4) 180 | 181 | tag := "tamo" 182 | vshost := port(tag+"v", 1) 183 | vs := viewservice.StartServer(vshost) 184 | time.Sleep(time.Second) 185 | vck := viewservice.MakeClerk("", vshost) 186 | 187 | fmt.Printf("Test: at-most-once Append; unreliable ...\n") 188 | 189 | const nservers = 1 190 | var sa [nservers]*PBServer 191 | for i := 0; i < nservers; i++ { 192 | sa[i] = StartServer(vshost, port(tag, i+1)) 193 | sa[i].setunreliable(true) 194 | } 195 | 196 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 197 | view, _ := vck.Get() 198 | if view.Primary != "" && view.Backup != "" { 199 | break 200 | } 201 | time.Sleep(viewservice.PingInterval) 202 | } 203 | 204 | // give p+b time to ack, initialize 205 | time.Sleep(viewservice.PingInterval * viewservice.DeadPings) 206 | 207 | ck := MakeClerk(vshost, "") 208 | k := "counter" 209 | val := "" 210 | for i := 0; i < 100; i++ { 211 | v := strconv.Itoa(i) 212 | ck.Append(k, v) 213 | val = val + v 214 | } 215 | 216 | v := ck.Get(k) 217 | if v != val { 218 | t.Fatalf("ck.Get() returned %v but expected %v\n", v, val) 219 | } 220 | 221 | fmt.Printf(" ... Passed\n") 222 | 223 | for i := 0; i < nservers; i++ { 224 | sa[i].kill() 225 | } 226 | time.Sleep(time.Second) 227 | vs.Kill() 228 | time.Sleep(time.Second) 229 | } 230 | 231 | // Put right after a backup dies. 232 | func TestFailPut(t *testing.T) { 233 | runtime.GOMAXPROCS(4) 234 | 235 | tag := "failput" 236 | vshost := port(tag+"v", 1) 237 | vs := viewservice.StartServer(vshost) 238 | time.Sleep(time.Second) 239 | vck := viewservice.MakeClerk("", vshost) 240 | 241 | s1 := StartServer(vshost, port(tag, 1)) 242 | time.Sleep(time.Second) 243 | s2 := StartServer(vshost, port(tag, 2)) 244 | time.Sleep(time.Second) 245 | s3 := StartServer(vshost, port(tag, 3)) 246 | 247 | for i := 0; i < viewservice.DeadPings*3; i++ { 248 | v, _ := vck.Get() 249 | if v.Primary != "" && v.Backup != "" { 250 | break 251 | } 252 | time.Sleep(viewservice.PingInterval) 253 | } 254 | time.Sleep(time.Second) // wait for backup initializion 255 | v1, _ := vck.Get() 256 | if v1.Primary != s1.me || v1.Backup != s2.me { 257 | t.Fatalf("wrong primary or backup") 258 | } 259 | 260 | ck := MakeClerk(vshost, "") 261 | 262 | ck.Put("a", "aa") 263 | ck.Put("b", "bb") 264 | ck.Put("c", "cc") 265 | check(ck, "a", "aa") 266 | check(ck, "b", "bb") 267 | check(ck, "c", "cc") 268 | 269 | // kill backup, then immediate Put 270 | fmt.Printf("Test: Put() immediately after backup failure ...\n") 271 | s2.kill() 272 | ck.Put("a", "aaa") 273 | check(ck, "a", "aaa") 274 | 275 | for i := 0; i < viewservice.DeadPings*3; i++ { 276 | v, _ := vck.Get() 277 | if v.Viewnum > v1.Viewnum && v.Primary != "" && v.Backup != "" { 278 | break 279 | } 280 | time.Sleep(viewservice.PingInterval) 281 | } 282 | time.Sleep(time.Second) // wait for backup initialization 283 | v2, _ := vck.Get() 284 | if v2.Primary != s1.me || v2.Backup != s3.me { 285 | t.Fatal("wrong primary or backup") 286 | } 287 | 288 | check(ck, "a", "aaa") 289 | fmt.Printf(" ... Passed\n") 290 | 291 | // kill primary, then immediate Put 292 | fmt.Printf("Test: Put() immediately after primary failure ...\n") 293 | s1.kill() 294 | ck.Put("b", "bbb") 295 | check(ck, "b", "bbb") 296 | 297 | for i := 0; i < viewservice.DeadPings*3; i++ { 298 | v, _ := vck.Get() 299 | if v.Viewnum > v2.Viewnum && v.Primary != "" { 300 | break 301 | } 302 | time.Sleep(viewservice.PingInterval) 303 | } 304 | time.Sleep(time.Second) 305 | 306 | check(ck, "a", "aaa") 307 | check(ck, "b", "bbb") 308 | check(ck, "c", "cc") 309 | fmt.Printf(" ... Passed\n") 310 | 311 | s1.kill() 312 | s2.kill() 313 | s3.kill() 314 | time.Sleep(viewservice.PingInterval * 2) 315 | vs.Kill() 316 | } 317 | 318 | // do a bunch of concurrent Put()s on the same key, 319 | // then check that primary and backup have identical values. 320 | // i.e. that they processed the Put()s in the same order. 321 | func TestConcurrentSame(t *testing.T) { 322 | runtime.GOMAXPROCS(4) 323 | 324 | tag := "cs" 325 | vshost := port(tag+"v", 1) 326 | vs := viewservice.StartServer(vshost) 327 | time.Sleep(time.Second) 328 | vck := viewservice.MakeClerk("", vshost) 329 | 330 | fmt.Printf("Test: Concurrent Put()s to the same key ...\n") 331 | 332 | const nservers = 2 333 | var sa [nservers]*PBServer 334 | for i := 0; i < nservers; i++ { 335 | sa[i] = StartServer(vshost, port(tag, i+1)) 336 | } 337 | 338 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 339 | view, _ := vck.Get() 340 | if view.Primary != "" && view.Backup != "" { 341 | break 342 | } 343 | time.Sleep(viewservice.PingInterval) 344 | } 345 | 346 | // give p+b time to ack, initialize 347 | time.Sleep(viewservice.PingInterval * viewservice.DeadPings) 348 | 349 | done := int32(0) 350 | 351 | view1, _ := vck.Get() 352 | const nclients = 3 353 | const nkeys = 2 354 | for xi := 0; xi < nclients; xi++ { 355 | go func(i int) { 356 | ck := MakeClerk(vshost, "") 357 | rr := rand.New(rand.NewSource(int64(os.Getpid() + i))) 358 | for atomic.LoadInt32(&done) == 0 { 359 | k := strconv.Itoa(rr.Int() % nkeys) 360 | v := strconv.Itoa(rr.Int()) 361 | ck.Put(k, v) 362 | } 363 | }(xi) 364 | } 365 | 366 | time.Sleep(5 * time.Second) 367 | atomic.StoreInt32(&done, 1) 368 | time.Sleep(time.Second) 369 | 370 | // read from primary 371 | ck := MakeClerk(vshost, "") 372 | var vals [nkeys]string 373 | for i := 0; i < nkeys; i++ { 374 | vals[i] = ck.Get(strconv.Itoa(i)) 375 | if vals[i] == "" { 376 | t.Fatalf("Get(%v) failed from primary", i) 377 | } 378 | } 379 | 380 | // kill the primary 381 | for i := 0; i < nservers; i++ { 382 | if view1.Primary == sa[i].me { 383 | sa[i].kill() 384 | break 385 | } 386 | } 387 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 388 | view, _ := vck.Get() 389 | if view.Primary == view1.Backup { 390 | break 391 | } 392 | time.Sleep(viewservice.PingInterval) 393 | } 394 | view2, _ := vck.Get() 395 | if view2.Primary != view1.Backup { 396 | t.Fatal("wrong Primary") 397 | } 398 | 399 | // read from old backup 400 | for i := 0; i < nkeys; i++ { 401 | z := ck.Get(strconv.Itoa(i)) 402 | if z != vals[i] { 403 | t.Fatalf("Get(%v) from backup; wanted %v, got %v", i, vals[i], z) 404 | } 405 | } 406 | 407 | fmt.Printf(" ... Passed\n") 408 | 409 | for i := 0; i < nservers; i++ { 410 | sa[i].kill() 411 | } 412 | time.Sleep(time.Second) 413 | vs.Kill() 414 | time.Sleep(time.Second) 415 | } 416 | 417 | // check that all known appends are present in a value, 418 | // and are in order for each concurrent client. 419 | func checkAppends(t *testing.T, v string, counts []int) { 420 | nclients := len(counts) 421 | for i := 0; i < nclients; i++ { 422 | lastoff := -1 423 | for j := 0; j < counts[i]; j++ { 424 | wanted := "x " + strconv.Itoa(i) + " " + strconv.Itoa(j) + " y" 425 | off := strings.Index(v, wanted) 426 | if off < 0 { 427 | t.Fatalf("missing element in Append result") 428 | } 429 | off1 := strings.LastIndex(v, wanted) 430 | if off1 != off { 431 | t.Fatalf("duplicate element in Append result") 432 | } 433 | if off <= lastoff { 434 | t.Fatalf("wrong order for element in Append result") 435 | } 436 | lastoff = off 437 | } 438 | } 439 | } 440 | 441 | // do a bunch of concurrent Append()s on the same key, 442 | // then check that primary and backup have identical values. 443 | // i.e. that they processed the Append()s in the same order. 444 | func TestConcurrentSameAppend(t *testing.T) { 445 | runtime.GOMAXPROCS(4) 446 | 447 | tag := "csa" 448 | vshost := port(tag+"v", 1) 449 | vs := viewservice.StartServer(vshost) 450 | time.Sleep(time.Second) 451 | vck := viewservice.MakeClerk("", vshost) 452 | 453 | fmt.Printf("Test: Concurrent Append()s to the same key ...\n") 454 | 455 | const nservers = 2 456 | var sa [nservers]*PBServer 457 | for i := 0; i < nservers; i++ { 458 | sa[i] = StartServer(vshost, port(tag, i+1)) 459 | } 460 | 461 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 462 | view, _ := vck.Get() 463 | if view.Primary != "" && view.Backup != "" { 464 | break 465 | } 466 | time.Sleep(viewservice.PingInterval) 467 | } 468 | 469 | // give p+b time to ack, initialize 470 | time.Sleep(viewservice.PingInterval * viewservice.DeadPings) 471 | 472 | view1, _ := vck.Get() 473 | 474 | // code for i'th concurrent client thread. 475 | ff := func(i int, ch chan int) { 476 | ret := -1 477 | defer func() { ch <- ret }() 478 | ck := MakeClerk(vshost, "") 479 | n := 0 480 | for n < 50 { 481 | v := "x " + strconv.Itoa(i) + " " + strconv.Itoa(n) + " y" 482 | ck.Append("k", v) 483 | n += 1 484 | } 485 | ret = n 486 | } 487 | 488 | // start the concurrent clients 489 | const nclients = 3 490 | chans := []chan int{} 491 | for i := 0; i < nclients; i++ { 492 | chans = append(chans, make(chan int)) 493 | go ff(i, chans[i]) 494 | } 495 | 496 | // wait for the clients, accumulate Append counts. 497 | counts := []int{} 498 | for i := 0; i < nclients; i++ { 499 | n := <-chans[i] 500 | if n < 0 { 501 | t.Fatalf("child failed") 502 | } 503 | counts = append(counts, n) 504 | } 505 | 506 | ck := MakeClerk(vshost, "") 507 | 508 | // check that primary's copy of the value has all 509 | // the Append()s. 510 | primaryv := ck.Get("k") 511 | checkAppends(t, primaryv, counts) 512 | 513 | // kill the primary so we can check the backup 514 | for i := 0; i < nservers; i++ { 515 | if view1.Primary == sa[i].me { 516 | sa[i].kill() 517 | break 518 | } 519 | } 520 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 521 | view, _ := vck.Get() 522 | if view.Primary == view1.Backup { 523 | break 524 | } 525 | time.Sleep(viewservice.PingInterval) 526 | } 527 | view2, _ := vck.Get() 528 | if view2.Primary != view1.Backup { 529 | t.Fatal("wrong Primary") 530 | } 531 | 532 | // check that backup's copy of the value has all 533 | // the Append()s. 534 | backupv := ck.Get("k") 535 | checkAppends(t, backupv, counts) 536 | 537 | if backupv != primaryv { 538 | t.Fatal("primary and backup had different values") 539 | } 540 | 541 | fmt.Printf(" ... Passed\n") 542 | 543 | for i := 0; i < nservers; i++ { 544 | sa[i].kill() 545 | } 546 | time.Sleep(time.Second) 547 | vs.Kill() 548 | time.Sleep(time.Second) 549 | } 550 | 551 | func TestConcurrentSameUnreliable(t *testing.T) { 552 | runtime.GOMAXPROCS(4) 553 | 554 | tag := "csu" 555 | vshost := port(tag+"v", 1) 556 | vs := viewservice.StartServer(vshost) 557 | time.Sleep(time.Second) 558 | vck := viewservice.MakeClerk("", vshost) 559 | 560 | fmt.Printf("Test: Concurrent Put()s to the same key; unreliable ...\n") 561 | 562 | const nservers = 2 563 | var sa [nservers]*PBServer 564 | for i := 0; i < nservers; i++ { 565 | sa[i] = StartServer(vshost, port(tag, i+1)) 566 | sa[i].setunreliable(true) 567 | } 568 | 569 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 570 | view, _ := vck.Get() 571 | if view.Primary != "" && view.Backup != "" { 572 | break 573 | } 574 | time.Sleep(viewservice.PingInterval) 575 | } 576 | 577 | // give p+b time to ack, initialize 578 | time.Sleep(viewservice.PingInterval * viewservice.DeadPings) 579 | 580 | { 581 | ck := MakeClerk(vshost, "") 582 | ck.Put("0", "x") 583 | ck.Put("1", "x") 584 | } 585 | 586 | done := int32(0) 587 | 588 | view1, _ := vck.Get() 589 | const nclients = 3 590 | const nkeys = 2 591 | cha := []chan bool{} 592 | for xi := 0; xi < nclients; xi++ { 593 | cha = append(cha, make(chan bool)) 594 | go func(i int, ch chan bool) { 595 | ok := false 596 | defer func() { ch <- ok }() 597 | ck := MakeClerk(vshost, "") 598 | rr := rand.New(rand.NewSource(int64(os.Getpid() + i))) 599 | for atomic.LoadInt32(&done) == 0 { 600 | k := strconv.Itoa(rr.Int() % nkeys) 601 | v := strconv.Itoa(rr.Int()) 602 | ck.Put(k, v) 603 | } 604 | ok = true 605 | }(xi, cha[xi]) 606 | } 607 | 608 | time.Sleep(5 * time.Second) 609 | atomic.StoreInt32(&done, 1) 610 | 611 | for i := 0; i < len(cha); i++ { 612 | ok := <-cha[i] 613 | if ok == false { 614 | t.Fatalf("child failed") 615 | } 616 | } 617 | 618 | // read from primary 619 | ck := MakeClerk(vshost, "") 620 | var vals [nkeys]string 621 | for i := 0; i < nkeys; i++ { 622 | vals[i] = ck.Get(strconv.Itoa(i)) 623 | if vals[i] == "" { 624 | t.Fatalf("Get(%v) failed from primary", i) 625 | } 626 | } 627 | 628 | // kill the primary 629 | for i := 0; i < nservers; i++ { 630 | if view1.Primary == sa[i].me { 631 | sa[i].kill() 632 | break 633 | } 634 | } 635 | for iters := 0; iters < viewservice.DeadPings*2; iters++ { 636 | view, _ := vck.Get() 637 | if view.Primary == view1.Backup { 638 | break 639 | } 640 | time.Sleep(viewservice.PingInterval) 641 | } 642 | view2, _ := vck.Get() 643 | if view2.Primary != view1.Backup { 644 | t.Fatal("wrong Primary") 645 | } 646 | 647 | // read from old backup 648 | for i := 0; i < nkeys; i++ { 649 | z := ck.Get(strconv.Itoa(i)) 650 | if z != vals[i] { 651 | t.Fatalf("Get(%v) from backup; wanted %v, got %v", i, vals[i], z) 652 | } 653 | } 654 | 655 | fmt.Printf(" ... Passed\n") 656 | 657 | for i := 0; i < nservers; i++ { 658 | sa[i].kill() 659 | } 660 | time.Sleep(time.Second) 661 | vs.Kill() 662 | time.Sleep(time.Second) 663 | } 664 | 665 | // constant put/get while crashing and restarting servers 666 | func TestRepeatedCrash(t *testing.T) { 667 | runtime.GOMAXPROCS(4) 668 | 669 | tag := "rc" 670 | vshost := port(tag+"v", 1) 671 | vs := viewservice.StartServer(vshost) 672 | time.Sleep(time.Second) 673 | vck := viewservice.MakeClerk("", vshost) 674 | 675 | fmt.Printf("Test: Repeated failures/restarts ...\n") 676 | 677 | const nservers = 3 678 | var sa [nservers]*PBServer 679 | samu := sync.Mutex{} 680 | for i := 0; i < nservers; i++ { 681 | sa[i] = StartServer(vshost, port(tag, i+1)) 682 | } 683 | 684 | for i := 0; i < viewservice.DeadPings; i++ { 685 | v, _ := vck.Get() 686 | if v.Primary != "" && v.Backup != "" { 687 | break 688 | } 689 | time.Sleep(viewservice.PingInterval) 690 | } 691 | 692 | // wait a bit for primary to initialize backup 693 | time.Sleep(viewservice.DeadPings * viewservice.PingInterval) 694 | 695 | done := int32(0) 696 | 697 | go func() { 698 | // kill and restart servers 699 | rr := rand.New(rand.NewSource(int64(os.Getpid()))) 700 | for atomic.LoadInt32(&done) == 0 { 701 | i := rr.Int() % nservers 702 | // fmt.Printf("%v killing %v\n", ts(), 5001+i) 703 | sa[i].kill() 704 | 705 | // wait long enough for new view to form, backup to be initialized 706 | time.Sleep(2 * viewservice.PingInterval * viewservice.DeadPings) 707 | 708 | sss := StartServer(vshost, port(tag, i+1)) 709 | samu.Lock() 710 | sa[i] = sss 711 | samu.Unlock() 712 | 713 | // wait long enough for new view to form, backup to be initialized 714 | time.Sleep(2 * viewservice.PingInterval * viewservice.DeadPings) 715 | } 716 | }() 717 | 718 | const nth = 2 719 | var cha [nth]chan bool 720 | for xi := 0; xi < nth; xi++ { 721 | cha[xi] = make(chan bool) 722 | go func(i int) { 723 | ok := false 724 | defer func() { cha[i] <- ok }() 725 | ck := MakeClerk(vshost, "") 726 | data := map[string]string{} 727 | rr := rand.New(rand.NewSource(int64(os.Getpid() + i))) 728 | for atomic.LoadInt32(&done) == 0 { 729 | k := strconv.Itoa((i * 1000000) + (rr.Int() % 10)) 730 | wanted, ok := data[k] 731 | if ok { 732 | v := ck.Get(k) 733 | if v != wanted { 734 | t.Fatalf("key=%v wanted=%v got=%v", k, wanted, v) 735 | } 736 | } 737 | nv := strconv.Itoa(rr.Int()) 738 | ck.Put(k, nv) 739 | data[k] = nv 740 | // if no sleep here, then server tick() threads do not get 741 | // enough time to Ping the viewserver. 742 | time.Sleep(10 * time.Millisecond) 743 | } 744 | ok = true 745 | }(xi) 746 | } 747 | 748 | time.Sleep(20 * time.Second) 749 | atomic.StoreInt32(&done, 1) 750 | 751 | fmt.Printf(" ... Put/Gets done ... \n") 752 | 753 | for i := 0; i < nth; i++ { 754 | ok := <-cha[i] 755 | if ok == false { 756 | t.Fatal("child failed") 757 | } 758 | } 759 | 760 | ck := MakeClerk(vshost, "") 761 | ck.Put("aaa", "bbb") 762 | if v := ck.Get("aaa"); v != "bbb" { 763 | t.Fatalf("final Put/Get failed") 764 | } 765 | 766 | fmt.Printf(" ... Passed\n") 767 | 768 | for i := 0; i < nservers; i++ { 769 | samu.Lock() 770 | sa[i].kill() 771 | samu.Unlock() 772 | } 773 | time.Sleep(time.Second) 774 | vs.Kill() 775 | time.Sleep(time.Second) 776 | } 777 | 778 | func TestRepeatedCrashUnreliable(t *testing.T) { 779 | runtime.GOMAXPROCS(4) 780 | 781 | tag := "rcu" 782 | vshost := port(tag+"v", 1) 783 | vs := viewservice.StartServer(vshost) 784 | time.Sleep(time.Second) 785 | vck := viewservice.MakeClerk("", vshost) 786 | 787 | fmt.Printf("Test: Repeated failures/restarts with concurrent updates to same key; unreliable ...\n") 788 | 789 | const nservers = 3 790 | var sa [nservers]*PBServer 791 | samu := sync.Mutex{} 792 | for i := 0; i < nservers; i++ { 793 | sa[i] = StartServer(vshost, port(tag, i+1)) 794 | sa[i].setunreliable(true) 795 | } 796 | 797 | for i := 0; i < viewservice.DeadPings; i++ { 798 | v, _ := vck.Get() 799 | if v.Primary != "" && v.Backup != "" { 800 | break 801 | } 802 | time.Sleep(viewservice.PingInterval) 803 | } 804 | 805 | // wait a bit for primary to initialize backup 806 | time.Sleep(viewservice.DeadPings * viewservice.PingInterval) 807 | 808 | done := int32(0) 809 | 810 | go func() { 811 | // kill and restart servers 812 | rr := rand.New(rand.NewSource(int64(os.Getpid()))) 813 | for atomic.LoadInt32(&done) == 0 { 814 | i := rr.Int() % nservers 815 | // fmt.Printf("%v killing %v\n", ts(), 5001+i) 816 | sa[i].kill() 817 | 818 | // wait long enough for new view to form, backup to be initialized 819 | time.Sleep(2 * viewservice.PingInterval * viewservice.DeadPings) 820 | 821 | sss := StartServer(vshost, port(tag, i+1)) 822 | samu.Lock() 823 | sa[i] = sss 824 | samu.Unlock() 825 | 826 | // wait long enough for new view to form, backup to be initialized 827 | time.Sleep(2 * viewservice.PingInterval * viewservice.DeadPings) 828 | } 829 | }() 830 | 831 | // concurrent client thread. 832 | ff := func(i int, ch chan int) { 833 | ret := -1 834 | defer func() { ch <- ret }() 835 | ck := MakeClerk(vshost, "") 836 | n := 0 837 | for atomic.LoadInt32(&done) == 0 { 838 | v := "x " + strconv.Itoa(i) + " " + strconv.Itoa(n) + " y" 839 | ck.Append("0", v) 840 | // if no sleep here, then server tick() threads do not get 841 | // enough time to Ping the viewserver. 842 | time.Sleep(10 * time.Millisecond) 843 | n++ 844 | } 845 | ret = n 846 | } 847 | 848 | const nth = 2 849 | var cha [nth]chan int 850 | for i := 0; i < nth; i++ { 851 | cha[i] = make(chan int) 852 | go ff(i, cha[i]) 853 | } 854 | 855 | time.Sleep(20 * time.Second) 856 | atomic.StoreInt32(&done, 1) 857 | 858 | fmt.Printf(" ... Appends done ... \n") 859 | 860 | counts := []int{} 861 | for i := 0; i < nth; i++ { 862 | n := <-cha[i] 863 | if n < 0 { 864 | t.Fatal("child failed") 865 | } 866 | counts = append(counts, n) 867 | } 868 | 869 | ck := MakeClerk(vshost, "") 870 | 871 | checkAppends(t, ck.Get("0"), counts) 872 | 873 | ck.Put("aaa", "bbb") 874 | if v := ck.Get("aaa"); v != "bbb" { 875 | t.Fatalf("final Put/Get failed") 876 | } 877 | 878 | fmt.Printf(" ... Passed\n") 879 | 880 | for i := 0; i < nservers; i++ { 881 | samu.Lock() 882 | sa[i].kill() 883 | samu.Unlock() 884 | } 885 | time.Sleep(time.Second) 886 | vs.Kill() 887 | time.Sleep(time.Second) 888 | } 889 | 890 | func proxy(t *testing.T, port string, delay *int32) { 891 | portx := port + "x" 892 | os.Remove(portx) 893 | if os.Rename(port, portx) != nil { 894 | t.Fatalf("proxy rename failed") 895 | } 896 | l, err := net.Listen("unix", port) 897 | if err != nil { 898 | t.Fatalf("proxy listen failed: %v", err) 899 | } 900 | go func() { 901 | defer l.Close() 902 | defer os.Remove(portx) 903 | defer os.Remove(port) 904 | for { 905 | c1, err := l.Accept() 906 | if err != nil { 907 | t.Fatalf("proxy accept failed: %v\n", err) 908 | } 909 | time.Sleep(time.Duration(atomic.LoadInt32(delay)) * time.Second) 910 | c2, err := net.Dial("unix", portx) 911 | if err != nil { 912 | t.Fatalf("proxy dial failed: %v\n", err) 913 | } 914 | 915 | go func() { 916 | for { 917 | buf := make([]byte, 1000) 918 | n, _ := c2.Read(buf) 919 | if n == 0 { 920 | break 921 | } 922 | n1, _ := c1.Write(buf[0:n]) 923 | if n1 != n { 924 | break 925 | } 926 | } 927 | }() 928 | for { 929 | buf := make([]byte, 1000) 930 | n, err := c1.Read(buf) 931 | if err != nil && err != io.EOF { 932 | t.Fatalf("proxy c1.Read: %v\n", err) 933 | } 934 | if n == 0 { 935 | break 936 | } 937 | n1, err1 := c2.Write(buf[0:n]) 938 | if err1 != nil || n1 != n { 939 | t.Fatalf("proxy c2.Write: %v\n", err1) 940 | } 941 | } 942 | 943 | c1.Close() 944 | c2.Close() 945 | } 946 | }() 947 | } 948 | 949 | func TestPartition1(t *testing.T) { 950 | runtime.GOMAXPROCS(4) 951 | 952 | tag := "part1" 953 | vshost := port(tag+"v", 1) 954 | vs := viewservice.StartServer(vshost) 955 | time.Sleep(time.Second) 956 | vck := viewservice.MakeClerk("", vshost) 957 | 958 | ck1 := MakeClerk(vshost, "") 959 | 960 | fmt.Printf("Test: Old primary does not serve Gets ...\n") 961 | 962 | vshosta := vshost + "a" 963 | os.Link(vshost, vshosta) 964 | 965 | s1 := StartServer(vshosta, port(tag, 1)) 966 | delay := int32(0) 967 | proxy(t, port(tag, 1), &delay) 968 | 969 | deadtime := viewservice.PingInterval * viewservice.DeadPings 970 | time.Sleep(deadtime * 2) 971 | if vck.Primary() != s1.me { 972 | t.Fatal("primary never formed initial view") 973 | } 974 | 975 | s2 := StartServer(vshost, port(tag, 2)) 976 | time.Sleep(deadtime * 2) 977 | v1, _ := vck.Get() 978 | if v1.Primary != s1.me || v1.Backup != s2.me { 979 | t.Fatal("backup did not join view") 980 | } 981 | 982 | ck1.Put("a", "1") 983 | check(ck1, "a", "1") 984 | 985 | os.Remove(vshosta) 986 | 987 | // start a client Get(), but use proxy to delay it long 988 | // enough that it won't reach s1 until after s1 is no 989 | // longer the primary. 990 | atomic.StoreInt32(&delay, 4) 991 | stale_get := make(chan bool) 992 | go func() { 993 | local_stale := false 994 | defer func() { stale_get <- local_stale }() 995 | x := ck1.Get("a") 996 | if x == "1" { 997 | local_stale = true 998 | } 999 | }() 1000 | 1001 | // now s1 cannot talk to viewserver, so view will change, 1002 | // and s1 won't immediately realize. 1003 | 1004 | for iter := 0; iter < viewservice.DeadPings*3; iter++ { 1005 | if vck.Primary() == s2.me { 1006 | break 1007 | } 1008 | time.Sleep(viewservice.PingInterval) 1009 | } 1010 | if vck.Primary() != s2.me { 1011 | t.Fatalf("primary never changed") 1012 | } 1013 | 1014 | // wait long enough that s2 is guaranteed to have Pinged 1015 | // the viewservice, and thus that s2 must know about 1016 | // the new view. 1017 | time.Sleep(2 * viewservice.PingInterval) 1018 | 1019 | // change the value (on s2) so it's no longer "1". 1020 | ck2 := MakeClerk(vshost, "") 1021 | ck2.Put("a", "111") 1022 | check(ck2, "a", "111") 1023 | 1024 | // wait for the background Get to s1 to be delivered. 1025 | select { 1026 | case x := <-stale_get: 1027 | if x { 1028 | t.Fatalf("Get to old primary succeeded and produced stale value") 1029 | } 1030 | case <-time.After(5 * time.Second): 1031 | } 1032 | 1033 | check(ck2, "a", "111") 1034 | 1035 | fmt.Printf(" ... Passed\n") 1036 | 1037 | s1.kill() 1038 | s2.kill() 1039 | vs.Kill() 1040 | } 1041 | 1042 | func TestPartition2(t *testing.T) { 1043 | runtime.GOMAXPROCS(4) 1044 | 1045 | tag := "part2" 1046 | vshost := port(tag+"v", 1) 1047 | vs := viewservice.StartServer(vshost) 1048 | time.Sleep(time.Second) 1049 | vck := viewservice.MakeClerk("", vshost) 1050 | 1051 | ck1 := MakeClerk(vshost, "") 1052 | 1053 | vshosta := vshost + "a" 1054 | os.Link(vshost, vshosta) 1055 | 1056 | s1 := StartServer(vshosta, port(tag, 1)) 1057 | delay := int32(0) 1058 | proxy(t, port(tag, 1), &delay) 1059 | 1060 | fmt.Printf("Test: Partitioned old primary does not complete Gets ...\n") 1061 | 1062 | deadtime := viewservice.PingInterval * viewservice.DeadPings 1063 | time.Sleep(deadtime * 2) 1064 | if vck.Primary() != s1.me { 1065 | t.Fatal("primary never formed initial view") 1066 | } 1067 | 1068 | s2 := StartServer(vshost, port(tag, 2)) 1069 | time.Sleep(deadtime * 2) 1070 | v1, _ := vck.Get() 1071 | if v1.Primary != s1.me || v1.Backup != s2.me { 1072 | t.Fatal("backup did not join view") 1073 | } 1074 | 1075 | ck1.Put("a", "1") 1076 | check(ck1, "a", "1") 1077 | 1078 | os.Remove(vshosta) 1079 | 1080 | // start a client Get(), but use proxy to delay it long 1081 | // enough that it won't reach s1 until after s1 is no 1082 | // longer the primary. 1083 | atomic.StoreInt32(&delay, 5) 1084 | stale_get := make(chan bool) 1085 | go func() { 1086 | local_stale := false 1087 | defer func() { stale_get <- local_stale }() 1088 | x := ck1.Get("a") 1089 | if x == "1" { 1090 | local_stale = true 1091 | } 1092 | }() 1093 | 1094 | // now s1 cannot talk to viewserver, so view will change. 1095 | 1096 | for iter := 0; iter < viewservice.DeadPings*3; iter++ { 1097 | if vck.Primary() == s2.me { 1098 | break 1099 | } 1100 | time.Sleep(viewservice.PingInterval) 1101 | } 1102 | if vck.Primary() != s2.me { 1103 | t.Fatalf("primary never changed") 1104 | } 1105 | 1106 | s3 := StartServer(vshost, port(tag, 3)) 1107 | for iter := 0; iter < viewservice.DeadPings*3; iter++ { 1108 | v, _ := vck.Get() 1109 | if v.Backup == s3.me && v.Primary == s2.me { 1110 | break 1111 | } 1112 | time.Sleep(viewservice.PingInterval) 1113 | } 1114 | v2, _ := vck.Get() 1115 | if v2.Primary != s2.me || v2.Backup != s3.me { 1116 | t.Fatalf("new backup never joined") 1117 | } 1118 | time.Sleep(2 * time.Second) 1119 | 1120 | ck2 := MakeClerk(vshost, "") 1121 | ck2.Put("a", "2") 1122 | check(ck2, "a", "2") 1123 | 1124 | s2.kill() 1125 | 1126 | // wait for delayed get to s1 to complete. 1127 | select { 1128 | case x := <-stale_get: 1129 | if x { 1130 | t.Fatalf("partitioned primary replied to a Get with a stale value") 1131 | } 1132 | case <-time.After(6 * time.Second): 1133 | } 1134 | 1135 | check(ck2, "a", "2") 1136 | 1137 | fmt.Printf(" ... Passed\n") 1138 | 1139 | s1.kill() 1140 | s2.kill() 1141 | s3.kill() 1142 | vs.Kill() 1143 | } 1144 | -------------------------------------------------------------------------------- /src/shardkv/client.go: -------------------------------------------------------------------------------- 1 | package shardkv 2 | 3 | import "shardmaster" 4 | import "net/rpc" 5 | import "time" 6 | import "sync" 7 | import "fmt" 8 | import "crypto/rand" 9 | import "math/big" 10 | 11 | type Clerk struct { 12 | mu sync.Mutex // one RPC at a time 13 | sm *shardmaster.Clerk 14 | config shardmaster.Config 15 | // You'll have to modify Clerk. 16 | } 17 | 18 | func nrand() int64 { 19 | max := big.NewInt(int64(1) << 62) 20 | bigx, _ := rand.Int(rand.Reader, max) 21 | x := bigx.Int64() 22 | return x 23 | } 24 | 25 | func MakeClerk(shardmasters []string) *Clerk { 26 | ck := new(Clerk) 27 | ck.sm = shardmaster.MakeClerk(shardmasters) 28 | // You'll have to modify MakeClerk. 29 | return ck 30 | } 31 | 32 | // 33 | // call() sends an RPC to the rpcname handler on server srv 34 | // with arguments args, waits for the reply, and leaves the 35 | // reply in reply. the reply argument should be a pointer 36 | // to a reply structure. 37 | // 38 | // the return value is true if the server responded, and false 39 | // if call() was not able to contact the server. in particular, 40 | // the reply's contents are only valid if call() returned true. 41 | // 42 | // you should assume that call() will return an 43 | // error after a while if the server is dead. 44 | // don't provide your own time-out mechanism. 45 | // 46 | // please use call() to send all RPCs, in client.go and server.go. 47 | // please don't change this function. 48 | // 49 | func call(srv string, rpcname string, 50 | args interface{}, reply interface{}) bool { 51 | c, errx := rpc.Dial("unix", srv) 52 | if errx != nil { 53 | return false 54 | } 55 | defer c.Close() 56 | 57 | err := c.Call(rpcname, args, reply) 58 | if err == nil { 59 | return true 60 | } 61 | 62 | fmt.Println(err) 63 | return false 64 | } 65 | 66 | // 67 | // which shard is a key in? 68 | // please use this function, 69 | // and please do not change it. 70 | // 71 | func key2shard(key string) int { 72 | shard := 0 73 | if len(key) > 0 { 74 | shard = int(key[0]) 75 | } 76 | shard %= shardmaster.NShards 77 | return shard 78 | } 79 | 80 | // 81 | // fetch the current value for a key. 82 | // returns "" if the key does not exist. 83 | // keeps trying forever in the face of all other errors. 84 | // 85 | func (ck *Clerk) Get(key string) string { 86 | ck.mu.Lock() 87 | defer ck.mu.Unlock() 88 | 89 | // You'll have to modify Get(). 90 | 91 | for { 92 | shard := key2shard(key) 93 | 94 | gid := ck.config.Shards[shard] 95 | 96 | servers, ok := ck.config.Groups[gid] 97 | 98 | if ok { 99 | // try each server in the shard's replication group. 100 | for _, srv := range servers { 101 | args := &GetArgs{} 102 | args.Key = key 103 | var reply GetReply 104 | ok := call(srv, "ShardKV.Get", args, &reply) 105 | if ok && (reply.Err == OK || reply.Err == ErrNoKey) { 106 | return reply.Value 107 | } 108 | if ok && (reply.Err == ErrWrongGroup) { 109 | break 110 | } 111 | } 112 | } 113 | 114 | time.Sleep(100 * time.Millisecond) 115 | 116 | // ask master for a new configuration. 117 | ck.config = ck.sm.Query(-1) 118 | } 119 | } 120 | 121 | // send a Put or Append request. 122 | func (ck *Clerk) PutAppend(key string, value string, op string) { 123 | ck.mu.Lock() 124 | defer ck.mu.Unlock() 125 | 126 | // You'll have to modify PutAppend(). 127 | 128 | for { 129 | shard := key2shard(key) 130 | 131 | gid := ck.config.Shards[shard] 132 | 133 | servers, ok := ck.config.Groups[gid] 134 | 135 | if ok { 136 | // try each server in the shard's replication group. 137 | for _, srv := range servers { 138 | args := &PutAppendArgs{} 139 | args.Key = key 140 | args.Value = value 141 | args.Op = op 142 | var reply PutAppendReply 143 | ok := call(srv, "ShardKV.PutAppend", args, &reply) 144 | if ok && reply.Err == OK { 145 | return 146 | } 147 | if ok && (reply.Err == ErrWrongGroup) { 148 | break 149 | } 150 | } 151 | } 152 | 153 | time.Sleep(100 * time.Millisecond) 154 | 155 | // ask master for a new configuration. 156 | ck.config = ck.sm.Query(-1) 157 | } 158 | } 159 | 160 | func (ck *Clerk) Put(key string, value string) { 161 | ck.PutAppend(key, value, "Put") 162 | } 163 | func (ck *Clerk) Append(key string, value string) { 164 | ck.PutAppend(key, value, "Append") 165 | } 166 | -------------------------------------------------------------------------------- /src/shardkv/common.go: -------------------------------------------------------------------------------- 1 | package shardkv 2 | 3 | // 4 | // Sharded key/value server. 5 | // Lots of replica groups, each running op-at-a-time paxos. 6 | // Shardmaster decides which group serves each shard. 7 | // Shardmaster may change shard assignment from time to time. 8 | // 9 | // You will have to modify these definitions. 10 | // 11 | 12 | const ( 13 | OK = "OK" 14 | ErrNoKey = "ErrNoKey" 15 | ErrWrongGroup = "ErrWrongGroup" 16 | ) 17 | 18 | type Err string 19 | 20 | type PutAppendArgs struct { 21 | Key string 22 | Value string 23 | Op string // "Put" or "Append" 24 | // You'll have to add definitions here. 25 | // Field names must start with capital letters, 26 | // otherwise RPC will break. 27 | 28 | } 29 | 30 | type PutAppendReply struct { 31 | Err Err 32 | } 33 | 34 | type GetArgs struct { 35 | Key string 36 | // You'll have to add definitions here. 37 | } 38 | 39 | type GetReply struct { 40 | Err Err 41 | Value string 42 | } 43 | 44 | -------------------------------------------------------------------------------- /src/shardkv/server.go: -------------------------------------------------------------------------------- 1 | package shardkv 2 | 3 | import "net" 4 | import "fmt" 5 | import "net/rpc" 6 | import "log" 7 | import "time" 8 | import "paxos" 9 | import "sync" 10 | import "sync/atomic" 11 | import "os" 12 | import "syscall" 13 | import "encoding/gob" 14 | import "math/rand" 15 | import "shardmaster" 16 | 17 | 18 | const Debug = 0 19 | 20 | func DPrintf(format string, a ...interface{}) (n int, err error) { 21 | if Debug > 0 { 22 | log.Printf(format, a...) 23 | } 24 | return 25 | } 26 | 27 | 28 | type Op struct { 29 | // Your definitions here. 30 | } 31 | 32 | 33 | type ShardKV struct { 34 | mu sync.Mutex 35 | l net.Listener 36 | me int 37 | dead int32 // for testing 38 | unreliable int32 // for testing 39 | sm *shardmaster.Clerk 40 | px *paxos.Paxos 41 | 42 | gid int64 // my replica group ID 43 | 44 | // Your definitions here. 45 | } 46 | 47 | 48 | func (kv *ShardKV) Get(args *GetArgs, reply *GetReply) error { 49 | // Your code here. 50 | return nil 51 | } 52 | 53 | // RPC handler for client Put and Append requests 54 | func (kv *ShardKV) PutAppend(args *PutAppendArgs, reply *PutAppendReply) error { 55 | // Your code here. 56 | return nil 57 | } 58 | 59 | // 60 | // Ask the shardmaster if there's a new configuration; 61 | // if so, re-configure. 62 | // 63 | func (kv *ShardKV) tick() { 64 | } 65 | 66 | // tell the server to shut itself down. 67 | // please don't change these two functions. 68 | func (kv *ShardKV) kill() { 69 | atomic.StoreInt32(&kv.dead, 1) 70 | kv.l.Close() 71 | kv.px.Kill() 72 | } 73 | 74 | // call this to find out if the server is dead. 75 | func (kv *ShardKV) isdead() bool { 76 | return atomic.LoadInt32(&kv.dead) != 0 77 | } 78 | 79 | // please do not change these two functions. 80 | func (kv *ShardKV) Setunreliable(what bool) { 81 | if what { 82 | atomic.StoreInt32(&kv.unreliable, 1) 83 | } else { 84 | atomic.StoreInt32(&kv.unreliable, 0) 85 | } 86 | } 87 | 88 | func (kv *ShardKV) isunreliable() bool { 89 | return atomic.LoadInt32(&kv.unreliable) != 0 90 | } 91 | 92 | // 93 | // Start a shardkv server. 94 | // gid is the ID of the server's replica group. 95 | // shardmasters[] contains the ports of the 96 | // servers that implement the shardmaster. 97 | // servers[] contains the ports of the servers 98 | // in this replica group. 99 | // Me is the index of this server in servers[]. 100 | // 101 | func StartServer(gid int64, shardmasters []string, 102 | servers []string, me int) *ShardKV { 103 | gob.Register(Op{}) 104 | 105 | kv := new(ShardKV) 106 | kv.me = me 107 | kv.gid = gid 108 | kv.sm = shardmaster.MakeClerk(shardmasters) 109 | 110 | // Your initialization code here. 111 | // Don't call Join(). 112 | 113 | rpcs := rpc.NewServer() 114 | rpcs.Register(kv) 115 | 116 | kv.px = paxos.Make(servers, me, rpcs) 117 | 118 | 119 | os.Remove(servers[me]) 120 | l, e := net.Listen("unix", servers[me]) 121 | if e != nil { 122 | log.Fatal("listen error: ", e) 123 | } 124 | kv.l = l 125 | 126 | // please do not change any of the following code, 127 | // or do anything to subvert it. 128 | 129 | go func() { 130 | for kv.isdead() == false { 131 | conn, err := kv.l.Accept() 132 | if err == nil && kv.isdead() == false { 133 | if kv.isunreliable() && (rand.Int63()%1000) < 100 { 134 | // discard the request. 135 | conn.Close() 136 | } else if kv.isunreliable() && (rand.Int63()%1000) < 200 { 137 | // process the request but force discard of reply. 138 | c1 := conn.(*net.UnixConn) 139 | f, _ := c1.File() 140 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 141 | if err != nil { 142 | fmt.Printf("shutdown: %v\n", err) 143 | } 144 | go rpcs.ServeConn(conn) 145 | } else { 146 | go rpcs.ServeConn(conn) 147 | } 148 | } else if err == nil { 149 | conn.Close() 150 | } 151 | if err != nil && kv.isdead() == false { 152 | fmt.Printf("ShardKV(%v) accept: %v\n", me, err.Error()) 153 | kv.kill() 154 | } 155 | } 156 | }() 157 | 158 | go func() { 159 | for kv.isdead() == false { 160 | kv.tick() 161 | time.Sleep(250 * time.Millisecond) 162 | } 163 | }() 164 | 165 | return kv 166 | } 167 | -------------------------------------------------------------------------------- /src/shardkv/test_test.go: -------------------------------------------------------------------------------- 1 | package shardkv 2 | 3 | import "testing" 4 | import "shardmaster" 5 | import "runtime" 6 | import "strconv" 7 | import "os" 8 | import "time" 9 | import "fmt" 10 | import "sync" 11 | import "sync/atomic" 12 | import "math/rand" 13 | 14 | // information about the servers of one replica group. 15 | type tGroup struct { 16 | gid int64 17 | servers []*ShardKV 18 | ports []string 19 | } 20 | 21 | // information about all the servers of a k/v cluster. 22 | type tCluster struct { 23 | t *testing.T 24 | masters []*shardmaster.ShardMaster 25 | mck *shardmaster.Clerk 26 | masterports []string 27 | groups []*tGroup 28 | } 29 | 30 | func port(tag string, host int) string { 31 | s := "/var/tmp/824-" 32 | s += strconv.Itoa(os.Getuid()) + "/" 33 | os.Mkdir(s, 0777) 34 | s += "skv-" 35 | s += strconv.Itoa(os.Getpid()) + "-" 36 | s += tag + "-" 37 | s += strconv.Itoa(host) 38 | return s 39 | } 40 | 41 | // 42 | // start a k/v replica server thread. 43 | // 44 | func (tc *tCluster) start1(gi int, si int, unreliable bool) { 45 | s := StartServer(tc.groups[gi].gid, tc.masterports, tc.groups[gi].ports, si) 46 | tc.groups[gi].servers[si] = s 47 | s.Setunreliable(unreliable) 48 | } 49 | 50 | func (tc *tCluster) cleanup() { 51 | for gi := 0; gi < len(tc.groups); gi++ { 52 | g := tc.groups[gi] 53 | for si := 0; si < len(g.servers); si++ { 54 | if g.servers[si] != nil { 55 | g.servers[si].kill() 56 | } 57 | } 58 | } 59 | 60 | for i := 0; i < len(tc.masters); i++ { 61 | if tc.masters[i] != nil { 62 | tc.masters[i].Kill() 63 | } 64 | } 65 | } 66 | 67 | func (tc *tCluster) shardclerk() *shardmaster.Clerk { 68 | return shardmaster.MakeClerk(tc.masterports) 69 | } 70 | 71 | func (tc *tCluster) clerk() *Clerk { 72 | return MakeClerk(tc.masterports) 73 | } 74 | 75 | func (tc *tCluster) join(gi int) { 76 | tc.mck.Join(tc.groups[gi].gid, tc.groups[gi].ports) 77 | } 78 | 79 | func (tc *tCluster) leave(gi int) { 80 | tc.mck.Leave(tc.groups[gi].gid) 81 | } 82 | 83 | func setup(t *testing.T, tag string, unreliable bool) *tCluster { 84 | runtime.GOMAXPROCS(4) 85 | 86 | const nmasters = 3 87 | const ngroups = 3 // replica groups 88 | const nreplicas = 3 // servers per group 89 | 90 | tc := &tCluster{} 91 | tc.t = t 92 | tc.masters = make([]*shardmaster.ShardMaster, nmasters) 93 | tc.masterports = make([]string, nmasters) 94 | 95 | for i := 0; i < nmasters; i++ { 96 | tc.masterports[i] = port(tag+"m", i) 97 | } 98 | for i := 0; i < nmasters; i++ { 99 | tc.masters[i] = shardmaster.StartServer(tc.masterports, i) 100 | } 101 | tc.mck = tc.shardclerk() 102 | 103 | tc.groups = make([]*tGroup, ngroups) 104 | 105 | for i := 0; i < ngroups; i++ { 106 | tc.groups[i] = &tGroup{} 107 | tc.groups[i].gid = int64(i + 100) 108 | tc.groups[i].servers = make([]*ShardKV, nreplicas) 109 | tc.groups[i].ports = make([]string, nreplicas) 110 | for j := 0; j < nreplicas; j++ { 111 | tc.groups[i].ports[j] = port(tag+"s", (i*nreplicas)+j) 112 | } 113 | for j := 0; j < nreplicas; j++ { 114 | tc.start1(i, j, unreliable) 115 | } 116 | } 117 | 118 | // return smh, gids, ha, sa, clean 119 | return tc 120 | } 121 | 122 | func TestBasic(t *testing.T) { 123 | tc := setup(t, "basic", false) 124 | defer tc.cleanup() 125 | 126 | fmt.Printf("Test: Basic Join/Leave ...\n") 127 | 128 | tc.join(0) 129 | 130 | ck := tc.clerk() 131 | 132 | ck.Put("a", "x") 133 | ck.Append("a", "b") 134 | if ck.Get("a") != "xb" { 135 | t.Fatalf("Get got wrong value") 136 | } 137 | 138 | keys := make([]string, 10) 139 | vals := make([]string, len(keys)) 140 | for i := 0; i < len(keys); i++ { 141 | keys[i] = strconv.Itoa(rand.Int()) 142 | vals[i] = strconv.Itoa(rand.Int()) 143 | ck.Put(keys[i], vals[i]) 144 | } 145 | 146 | // are keys still there after joins? 147 | for g := 1; g < len(tc.groups); g++ { 148 | tc.join(g) 149 | time.Sleep(1 * time.Second) 150 | for i := 0; i < len(keys); i++ { 151 | v := ck.Get(keys[i]) 152 | if v != vals[i] { 153 | t.Fatalf("joining; wrong value; g=%v k=%v wanted=%v got=%v", 154 | g, keys[i], vals[i], v) 155 | } 156 | vals[i] = strconv.Itoa(rand.Int()) 157 | ck.Put(keys[i], vals[i]) 158 | } 159 | } 160 | 161 | // are keys still there after leaves? 162 | for g := 0; g < len(tc.groups)-1; g++ { 163 | tc.leave(g) 164 | time.Sleep(1 * time.Second) 165 | for i := 0; i < len(keys); i++ { 166 | v := ck.Get(keys[i]) 167 | if v != vals[i] { 168 | t.Fatalf("leaving; wrong value; g=%v k=%v wanted=%v got=%v", 169 | g, keys[i], vals[i], v) 170 | } 171 | vals[i] = strconv.Itoa(rand.Int()) 172 | ck.Put(keys[i], vals[i]) 173 | } 174 | } 175 | 176 | fmt.Printf(" ... Passed\n") 177 | } 178 | 179 | func TestMove(t *testing.T) { 180 | tc := setup(t, "move", false) 181 | defer tc.cleanup() 182 | 183 | fmt.Printf("Test: Shards really move ...\n") 184 | 185 | tc.join(0) 186 | 187 | ck := tc.clerk() 188 | 189 | // insert one key per shard 190 | for i := 0; i < shardmaster.NShards; i++ { 191 | ck.Put(string('0'+i), string('0'+i)) 192 | } 193 | 194 | // add group 1. 195 | tc.join(1) 196 | time.Sleep(5 * time.Second) 197 | 198 | // check that keys are still there. 199 | for i := 0; i < shardmaster.NShards; i++ { 200 | if ck.Get(string('0'+i)) != string('0'+i) { 201 | t.Fatalf("missing key/value") 202 | } 203 | } 204 | 205 | // remove sockets from group 0. 206 | for _, port := range tc.groups[0].ports { 207 | os.Remove(port) 208 | } 209 | 210 | count := int32(0) 211 | var mu sync.Mutex 212 | for i := 0; i < shardmaster.NShards; i++ { 213 | go func(me int) { 214 | myck := tc.clerk() 215 | v := myck.Get(string('0' + me)) 216 | if v == string('0'+me) { 217 | mu.Lock() 218 | atomic.AddInt32(&count, 1) 219 | mu.Unlock() 220 | } else { 221 | t.Fatalf("Get(%v) yielded %v\n", me, v) 222 | } 223 | }(i) 224 | } 225 | 226 | time.Sleep(10 * time.Second) 227 | 228 | ccc := atomic.LoadInt32(&count) 229 | if ccc > shardmaster.NShards/3 && ccc < 2*(shardmaster.NShards/3) { 230 | fmt.Printf(" ... Passed\n") 231 | } else { 232 | t.Fatalf("%v keys worked after killing 1/2 of groups; wanted %v", 233 | ccc, shardmaster.NShards/2) 234 | } 235 | } 236 | 237 | func TestLimp(t *testing.T) { 238 | tc := setup(t, "limp", false) 239 | defer tc.cleanup() 240 | 241 | fmt.Printf("Test: Reconfiguration with some dead replicas ...\n") 242 | 243 | tc.join(0) 244 | 245 | ck := tc.clerk() 246 | 247 | ck.Put("a", "b") 248 | if ck.Get("a") != "b" { 249 | t.Fatalf("got wrong value") 250 | } 251 | 252 | // kill one server from each replica group. 253 | for gi := 0; gi < len(tc.groups); gi++ { 254 | sa := tc.groups[gi].servers 255 | ns := len(sa) 256 | sa[rand.Int()%ns].kill() 257 | } 258 | 259 | keys := make([]string, 10) 260 | vals := make([]string, len(keys)) 261 | for i := 0; i < len(keys); i++ { 262 | keys[i] = strconv.Itoa(rand.Int()) 263 | vals[i] = strconv.Itoa(rand.Int()) 264 | ck.Put(keys[i], vals[i]) 265 | } 266 | 267 | // are keys still there after joins? 268 | for g := 1; g < len(tc.groups); g++ { 269 | tc.join(g) 270 | time.Sleep(1 * time.Second) 271 | for i := 0; i < len(keys); i++ { 272 | v := ck.Get(keys[i]) 273 | if v != vals[i] { 274 | t.Fatalf("joining; wrong value; g=%v k=%v wanted=%v got=%v", 275 | g, keys[i], vals[i], v) 276 | } 277 | vals[i] = strconv.Itoa(rand.Int()) 278 | ck.Put(keys[i], vals[i]) 279 | } 280 | } 281 | 282 | // are keys still there after leaves? 283 | for gi := 0; gi < len(tc.groups)-1; gi++ { 284 | tc.leave(gi) 285 | time.Sleep(2 * time.Second) 286 | g := tc.groups[gi] 287 | for i := 0; i < len(g.servers); i++ { 288 | g.servers[i].kill() 289 | } 290 | for i := 0; i < len(keys); i++ { 291 | v := ck.Get(keys[i]) 292 | if v != vals[i] { 293 | t.Fatalf("leaving; wrong value; g=%v k=%v wanted=%v got=%v", 294 | g, keys[i], vals[i], v) 295 | } 296 | vals[i] = strconv.Itoa(rand.Int()) 297 | ck.Put(keys[i], vals[i]) 298 | } 299 | } 300 | 301 | fmt.Printf(" ... Passed\n") 302 | } 303 | 304 | func doConcurrent(t *testing.T, unreliable bool) { 305 | tc := setup(t, "concurrent-"+strconv.FormatBool(unreliable), unreliable) 306 | defer tc.cleanup() 307 | 308 | for i := 0; i < len(tc.groups); i++ { 309 | tc.join(i) 310 | } 311 | 312 | const npara = 11 313 | var ca [npara]chan bool 314 | for i := 0; i < npara; i++ { 315 | ca[i] = make(chan bool) 316 | go func(me int) { 317 | ok := true 318 | defer func() { ca[me] <- ok }() 319 | ck := tc.clerk() 320 | mymck := tc.shardclerk() 321 | key := strconv.Itoa(me) 322 | last := "" 323 | for iters := 0; iters < 3; iters++ { 324 | nv := strconv.Itoa(rand.Int()) 325 | ck.Append(key, nv) 326 | last = last + nv 327 | v := ck.Get(key) 328 | if v != last { 329 | ok = false 330 | t.Fatalf("Get(%v) expected %v got %v\n", key, last, v) 331 | } 332 | 333 | gi := rand.Int() % len(tc.groups) 334 | gid := tc.groups[gi].gid 335 | mymck.Move(rand.Int()%shardmaster.NShards, gid) 336 | 337 | time.Sleep(time.Duration(rand.Int()%30) * time.Millisecond) 338 | } 339 | }(i) 340 | } 341 | 342 | for i := 0; i < npara; i++ { 343 | x := <-ca[i] 344 | if x == false { 345 | t.Fatalf("something is wrong") 346 | } 347 | } 348 | } 349 | 350 | func TestConcurrent(t *testing.T) { 351 | fmt.Printf("Test: Concurrent Put/Get/Move ...\n") 352 | doConcurrent(t, false) 353 | fmt.Printf(" ... Passed\n") 354 | } 355 | 356 | func TestConcurrentUnreliable(t *testing.T) { 357 | fmt.Printf("Test: Concurrent Put/Get/Move (unreliable) ...\n") 358 | doConcurrent(t, true) 359 | fmt.Printf(" ... Passed\n") 360 | } 361 | -------------------------------------------------------------------------------- /src/shardmaster/client.go: -------------------------------------------------------------------------------- 1 | package shardmaster 2 | 3 | // 4 | // Shardmaster clerk. 5 | // Please don't change this file. 6 | // 7 | 8 | import "net/rpc" 9 | import "time" 10 | import "fmt" 11 | 12 | type Clerk struct { 13 | servers []string // shardmaster replicas 14 | } 15 | 16 | func MakeClerk(servers []string) *Clerk { 17 | ck := new(Clerk) 18 | ck.servers = servers 19 | return ck 20 | } 21 | 22 | // 23 | // call() sends an RPC to the rpcname handler on server srv 24 | // with arguments args, waits for the reply, and leaves the 25 | // reply in reply. the reply argument should be a pointer 26 | // to a reply structure. 27 | // 28 | // the return value is true if the server responded, and false 29 | // if call() was not able to contact the server. in particular, 30 | // the reply's contents are only valid if call() returned true. 31 | // 32 | // you should assume that call() will return an 33 | // error after a while if the server is dead. 34 | // don't provide your own time-out mechanism. 35 | // 36 | // please use call() to send all RPCs, in client.go and server.go. 37 | // please don't change this function. 38 | // 39 | func call(srv string, rpcname string, 40 | args interface{}, reply interface{}) bool { 41 | c, errx := rpc.Dial("unix", srv) 42 | if errx != nil { 43 | return false 44 | } 45 | defer c.Close() 46 | 47 | err := c.Call(rpcname, args, reply) 48 | if err == nil { 49 | return true 50 | } 51 | 52 | fmt.Println(err) 53 | return false 54 | } 55 | 56 | func (ck *Clerk) Query(num int) Config { 57 | for { 58 | // try each known server. 59 | for _, srv := range ck.servers { 60 | args := &QueryArgs{} 61 | args.Num = num 62 | var reply QueryReply 63 | ok := call(srv, "ShardMaster.Query", args, &reply) 64 | if ok { 65 | return reply.Config 66 | } 67 | } 68 | time.Sleep(100 * time.Millisecond) 69 | } 70 | } 71 | 72 | func (ck *Clerk) Join(gid int64, servers []string) { 73 | for { 74 | // try each known server. 75 | for _, srv := range ck.servers { 76 | args := &JoinArgs{} 77 | args.GID = gid 78 | args.Servers = servers 79 | var reply JoinReply 80 | ok := call(srv, "ShardMaster.Join", args, &reply) 81 | if ok { 82 | return 83 | } 84 | } 85 | time.Sleep(100 * time.Millisecond) 86 | } 87 | } 88 | 89 | func (ck *Clerk) Leave(gid int64) { 90 | for { 91 | // try each known server. 92 | for _, srv := range ck.servers { 93 | args := &LeaveArgs{} 94 | args.GID = gid 95 | var reply LeaveReply 96 | ok := call(srv, "ShardMaster.Leave", args, &reply) 97 | if ok { 98 | return 99 | } 100 | } 101 | time.Sleep(100 * time.Millisecond) 102 | } 103 | } 104 | 105 | func (ck *Clerk) Move(shard int, gid int64) { 106 | for { 107 | // try each known server. 108 | for _, srv := range ck.servers { 109 | args := &MoveArgs{} 110 | args.Shard = shard 111 | args.GID = gid 112 | var reply MoveReply 113 | ok := call(srv, "ShardMaster.Move", args, &reply) 114 | if ok { 115 | return 116 | } 117 | } 118 | time.Sleep(100 * time.Millisecond) 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/shardmaster/common.go: -------------------------------------------------------------------------------- 1 | package shardmaster 2 | 3 | // 4 | // Master shard server: assigns shards to replication groups. 5 | // 6 | // RPC interface: 7 | // Join(gid, servers) -- replica group gid is joining, give it some shards. 8 | // Leave(gid) -- replica group gid is retiring, hand off all its shards. 9 | // Move(shard, gid) -- hand off one shard from current owner to gid. 10 | // Query(num) -> fetch Config # num, or latest config if num==-1. 11 | // 12 | // A Config (configuration) describes a set of replica groups, and the 13 | // replica group responsible for each shard. Configs are numbered. Config 14 | // #0 is the initial configuration, with no groups and all shards 15 | // assigned to group 0 (the invalid group). 16 | // 17 | // A GID is a replica group ID. GIDs must be uniqe and > 0. 18 | // Once a GID joins, and leaves, it should never join again. 19 | // 20 | // Please don't change this file. 21 | // 22 | 23 | const NShards = 10 24 | 25 | type Config struct { 26 | Num int // config number 27 | Shards [NShards]int64 // shard -> gid 28 | Groups map[int64][]string // gid -> servers[] 29 | } 30 | 31 | type JoinArgs struct { 32 | GID int64 // unique replica group ID 33 | Servers []string // group server ports 34 | } 35 | 36 | type JoinReply struct { 37 | } 38 | 39 | type LeaveArgs struct { 40 | GID int64 41 | } 42 | 43 | type LeaveReply struct { 44 | } 45 | 46 | type MoveArgs struct { 47 | Shard int 48 | GID int64 49 | } 50 | 51 | type MoveReply struct { 52 | } 53 | 54 | type QueryArgs struct { 55 | Num int // desired config number 56 | } 57 | 58 | type QueryReply struct { 59 | Config Config 60 | } 61 | -------------------------------------------------------------------------------- /src/shardmaster/server.go: -------------------------------------------------------------------------------- 1 | package shardmaster 2 | 3 | import "net" 4 | import "fmt" 5 | import "net/rpc" 6 | import "log" 7 | 8 | import "paxos" 9 | import "sync" 10 | import "sync/atomic" 11 | import "os" 12 | import "syscall" 13 | import "encoding/gob" 14 | import "math/rand" 15 | 16 | type ShardMaster struct { 17 | mu sync.Mutex 18 | l net.Listener 19 | me int 20 | dead int32 // for testing 21 | unreliable int32 // for testing 22 | px *paxos.Paxos 23 | 24 | configs []Config // indexed by config num 25 | } 26 | 27 | 28 | type Op struct { 29 | // Your data here. 30 | } 31 | 32 | 33 | func (sm *ShardMaster) Join(args *JoinArgs, reply *JoinReply) error { 34 | // Your code here. 35 | 36 | return nil 37 | } 38 | 39 | func (sm *ShardMaster) Leave(args *LeaveArgs, reply *LeaveReply) error { 40 | // Your code here. 41 | 42 | return nil 43 | } 44 | 45 | func (sm *ShardMaster) Move(args *MoveArgs, reply *MoveReply) error { 46 | // Your code here. 47 | 48 | return nil 49 | } 50 | 51 | func (sm *ShardMaster) Query(args *QueryArgs, reply *QueryReply) error { 52 | // Your code here. 53 | 54 | return nil 55 | } 56 | 57 | // please don't change these two functions. 58 | func (sm *ShardMaster) Kill() { 59 | atomic.StoreInt32(&sm.dead, 1) 60 | sm.l.Close() 61 | sm.px.Kill() 62 | } 63 | 64 | // call this to find out if the server is dead. 65 | func (sm *ShardMaster) isdead() bool { 66 | return atomic.LoadInt32(&sm.dead) != 0 67 | } 68 | 69 | // please do not change these two functions. 70 | func (sm *ShardMaster) setunreliable(what bool) { 71 | if what { 72 | atomic.StoreInt32(&sm.unreliable, 1) 73 | } else { 74 | atomic.StoreInt32(&sm.unreliable, 0) 75 | } 76 | } 77 | 78 | func (sm *ShardMaster) isunreliable() bool { 79 | return atomic.LoadInt32(&sm.unreliable) != 0 80 | } 81 | 82 | // 83 | // servers[] contains the ports of the set of 84 | // servers that will cooperate via Paxos to 85 | // form the fault-tolerant shardmaster service. 86 | // me is the index of the current server in servers[]. 87 | // 88 | func StartServer(servers []string, me int) *ShardMaster { 89 | sm := new(ShardMaster) 90 | sm.me = me 91 | 92 | sm.configs = make([]Config, 1) 93 | sm.configs[0].Groups = map[int64][]string{} 94 | 95 | rpcs := rpc.NewServer() 96 | 97 | gob.Register(Op{}) 98 | rpcs.Register(sm) 99 | sm.px = paxos.Make(servers, me, rpcs) 100 | 101 | os.Remove(servers[me]) 102 | l, e := net.Listen("unix", servers[me]) 103 | if e != nil { 104 | log.Fatal("listen error: ", e) 105 | } 106 | sm.l = l 107 | 108 | // please do not change any of the following code, 109 | // or do anything to subvert it. 110 | 111 | go func() { 112 | for sm.isdead() == false { 113 | conn, err := sm.l.Accept() 114 | if err == nil && sm.isdead() == false { 115 | if sm.isunreliable() && (rand.Int63()%1000) < 100 { 116 | // discard the request. 117 | conn.Close() 118 | } else if sm.isunreliable() && (rand.Int63()%1000) < 200 { 119 | // process the request but force discard of reply. 120 | c1 := conn.(*net.UnixConn) 121 | f, _ := c1.File() 122 | err := syscall.Shutdown(int(f.Fd()), syscall.SHUT_WR) 123 | if err != nil { 124 | fmt.Printf("shutdown: %v\n", err) 125 | } 126 | go rpcs.ServeConn(conn) 127 | } else { 128 | go rpcs.ServeConn(conn) 129 | } 130 | } else if err == nil { 131 | conn.Close() 132 | } 133 | if err != nil && sm.isdead() == false { 134 | fmt.Printf("ShardMaster(%v) accept: %v\n", me, err.Error()) 135 | sm.Kill() 136 | } 137 | } 138 | }() 139 | 140 | return sm 141 | } 142 | -------------------------------------------------------------------------------- /src/shardmaster/test_test.go: -------------------------------------------------------------------------------- 1 | package shardmaster 2 | 3 | import "testing" 4 | import "runtime" 5 | import "strconv" 6 | import "os" 7 | 8 | // import "time" 9 | import "fmt" 10 | import "math/rand" 11 | 12 | func port(tag string, host int) string { 13 | s := "/var/tmp/824-" 14 | s += strconv.Itoa(os.Getuid()) + "/" 15 | os.Mkdir(s, 0777) 16 | s += "sm-" 17 | s += strconv.Itoa(os.Getpid()) + "-" 18 | s += tag + "-" 19 | s += strconv.Itoa(host) 20 | return s 21 | } 22 | 23 | func cleanup(sma []*ShardMaster) { 24 | for i := 0; i < len(sma); i++ { 25 | if sma[i] != nil { 26 | sma[i].Kill() 27 | } 28 | } 29 | } 30 | 31 | // 32 | // maybe should take a cka[] and find the server with 33 | // the highest Num. 34 | // 35 | func check(t *testing.T, groups []int64, ck *Clerk) { 36 | c := ck.Query(-1) 37 | if len(c.Groups) != len(groups) { 38 | t.Fatalf("wanted %v groups, got %v", len(groups), len(c.Groups)) 39 | } 40 | 41 | // are the groups as expected? 42 | for _, g := range groups { 43 | _, ok := c.Groups[g] 44 | if ok != true { 45 | t.Fatalf("missing group %v", g) 46 | } 47 | } 48 | 49 | // any un-allocated shards? 50 | if len(groups) > 0 { 51 | for s, g := range c.Shards { 52 | _, ok := c.Groups[g] 53 | if ok == false { 54 | t.Fatalf("shard %v -> invalid group %v", s, g) 55 | } 56 | } 57 | } 58 | 59 | // more or less balanced sharding? 60 | counts := map[int64]int{} 61 | for _, g := range c.Shards { 62 | counts[g] += 1 63 | } 64 | min := 257 65 | max := 0 66 | for g, _ := range c.Groups { 67 | if counts[g] > max { 68 | max = counts[g] 69 | } 70 | if counts[g] < min { 71 | min = counts[g] 72 | } 73 | } 74 | if max > min+1 { 75 | t.Fatalf("max %v too much larger than min %v", max, min) 76 | } 77 | } 78 | 79 | func TestBasic(t *testing.T) { 80 | runtime.GOMAXPROCS(4) 81 | 82 | const nservers = 3 83 | var sma []*ShardMaster = make([]*ShardMaster, nservers) 84 | var kvh []string = make([]string, nservers) 85 | defer cleanup(sma) 86 | 87 | for i := 0; i < nservers; i++ { 88 | kvh[i] = port("basic", i) 89 | } 90 | for i := 0; i < nservers; i++ { 91 | sma[i] = StartServer(kvh, i) 92 | } 93 | 94 | ck := MakeClerk(kvh) 95 | var cka [nservers]*Clerk 96 | for i := 0; i < nservers; i++ { 97 | cka[i] = MakeClerk([]string{kvh[i]}) 98 | } 99 | 100 | fmt.Printf("Test: Basic leave/join ...\n") 101 | 102 | cfa := make([]Config, 6) 103 | cfa[0] = ck.Query(-1) 104 | 105 | check(t, []int64{}, ck) 106 | 107 | var gid1 int64 = 1 108 | ck.Join(gid1, []string{"x", "y", "z"}) 109 | check(t, []int64{gid1}, ck) 110 | cfa[1] = ck.Query(-1) 111 | 112 | var gid2 int64 = 2 113 | ck.Join(gid2, []string{"a", "b", "c"}) 114 | check(t, []int64{gid1, gid2}, ck) 115 | cfa[2] = ck.Query(-1) 116 | 117 | ck.Join(gid2, []string{"a", "b", "c"}) 118 | check(t, []int64{gid1, gid2}, ck) 119 | cfa[3] = ck.Query(-1) 120 | 121 | cfx := ck.Query(-1) 122 | sa1 := cfx.Groups[gid1] 123 | if len(sa1) != 3 || sa1[0] != "x" || sa1[1] != "y" || sa1[2] != "z" { 124 | t.Fatalf("wrong servers for gid %v: %v\n", gid1, sa1) 125 | } 126 | sa2 := cfx.Groups[gid2] 127 | if len(sa2) != 3 || sa2[0] != "a" || sa2[1] != "b" || sa2[2] != "c" { 128 | t.Fatalf("wrong servers for gid %v: %v\n", gid2, sa2) 129 | } 130 | 131 | ck.Leave(gid1) 132 | check(t, []int64{gid2}, ck) 133 | cfa[4] = ck.Query(-1) 134 | 135 | ck.Leave(gid1) 136 | check(t, []int64{gid2}, ck) 137 | cfa[5] = ck.Query(-1) 138 | 139 | fmt.Printf(" ... Passed\n") 140 | 141 | fmt.Printf("Test: Historical queries ...\n") 142 | 143 | for i := 0; i < len(cfa); i++ { 144 | c := ck.Query(cfa[i].Num) 145 | if c.Num != cfa[i].Num { 146 | t.Fatalf("historical Num wrong") 147 | } 148 | if c.Shards != cfa[i].Shards { 149 | t.Fatalf("historical Shards wrong") 150 | } 151 | if len(c.Groups) != len(cfa[i].Groups) { 152 | t.Fatalf("number of historical Groups is wrong") 153 | } 154 | for gid, sa := range c.Groups { 155 | sa1, ok := cfa[i].Groups[gid] 156 | if ok == false || len(sa1) != len(sa) { 157 | t.Fatalf("historical len(Groups) wrong") 158 | } 159 | if ok && len(sa1) == len(sa) { 160 | for j := 0; j < len(sa); j++ { 161 | if sa[j] != sa1[j] { 162 | t.Fatalf("historical Groups wrong") 163 | } 164 | } 165 | } 166 | } 167 | } 168 | 169 | fmt.Printf(" ... Passed\n") 170 | 171 | fmt.Printf("Test: Move ...\n") 172 | { 173 | var gid3 int64 = 503 174 | ck.Join(gid3, []string{"3a", "3b", "3c"}) 175 | var gid4 int64 = 504 176 | ck.Join(gid4, []string{"4a", "4b", "4c"}) 177 | for i := 0; i < NShards; i++ { 178 | cf := ck.Query(-1) 179 | if i < NShards/2 { 180 | ck.Move(i, gid3) 181 | if cf.Shards[i] != gid3 { 182 | cf1 := ck.Query(-1) 183 | if cf1.Num <= cf.Num { 184 | t.Fatalf("Move should increase Config.Num") 185 | } 186 | } 187 | } else { 188 | ck.Move(i, gid4) 189 | if cf.Shards[i] != gid4 { 190 | cf1 := ck.Query(-1) 191 | if cf1.Num <= cf.Num { 192 | t.Fatalf("Move should increase Config.Num") 193 | } 194 | } 195 | } 196 | } 197 | cf2 := ck.Query(-1) 198 | for i := 0; i < NShards; i++ { 199 | if i < NShards/2 { 200 | if cf2.Shards[i] != gid3 { 201 | t.Fatalf("expected shard %v on gid %v actually %v", 202 | i, gid3, cf2.Shards[i]) 203 | } 204 | } else { 205 | if cf2.Shards[i] != gid4 { 206 | t.Fatalf("expected shard %v on gid %v actually %v", 207 | i, gid4, cf2.Shards[i]) 208 | } 209 | } 210 | } 211 | ck.Leave(gid3) 212 | ck.Leave(gid4) 213 | } 214 | fmt.Printf(" ... Passed\n") 215 | 216 | fmt.Printf("Test: Concurrent leave/join ...\n") 217 | 218 | const npara = 10 219 | gids := make([]int64, npara) 220 | var ca [npara]chan bool 221 | for xi := 0; xi < npara; xi++ { 222 | gids[xi] = int64(xi + 1) 223 | ca[xi] = make(chan bool) 224 | go func(i int) { 225 | defer func() { ca[i] <- true }() 226 | var gid int64 = gids[i] 227 | cka[(i+0)%nservers].Join(gid+1000, []string{"a", "b", "c"}) 228 | cka[(i+0)%nservers].Join(gid, []string{"a", "b", "c"}) 229 | cka[(i+1)%nservers].Leave(gid + 1000) 230 | }(xi) 231 | } 232 | for i := 0; i < npara; i++ { 233 | <-ca[i] 234 | } 235 | check(t, gids, ck) 236 | 237 | fmt.Printf(" ... Passed\n") 238 | 239 | fmt.Printf("Test: Min advances after joins ...\n") 240 | 241 | for i, sm := range sma { 242 | if sm.px.Min() <= 0 { 243 | t.Fatalf("Min() for %s did not advance", kvh[i]) 244 | } 245 | } 246 | 247 | fmt.Printf(" ... Passed\n") 248 | 249 | fmt.Printf("Test: Minimal transfers after joins ...\n") 250 | 251 | c1 := ck.Query(-1) 252 | for i := 0; i < 5; i++ { 253 | ck.Join(int64(npara+1+i), []string{"a", "b", "c"}) 254 | } 255 | c2 := ck.Query(-1) 256 | for i := int64(1); i <= npara; i++ { 257 | for j := 0; j < len(c1.Shards); j++ { 258 | if c2.Shards[j] == i { 259 | if c1.Shards[j] != i { 260 | t.Fatalf("non-minimal transfer after Join()s") 261 | } 262 | } 263 | } 264 | } 265 | 266 | fmt.Printf(" ... Passed\n") 267 | 268 | fmt.Printf("Test: Minimal transfers after leaves ...\n") 269 | 270 | for i := 0; i < 5; i++ { 271 | ck.Leave(int64(npara + 1 + i)) 272 | } 273 | c3 := ck.Query(-1) 274 | for i := int64(1); i <= npara; i++ { 275 | for j := 0; j < len(c1.Shards); j++ { 276 | if c2.Shards[j] == i { 277 | if c3.Shards[j] != i { 278 | t.Fatalf("non-minimal transfer after Leave()s") 279 | } 280 | } 281 | } 282 | } 283 | 284 | fmt.Printf(" ... Passed\n") 285 | } 286 | 287 | func TestUnreliable(t *testing.T) { 288 | runtime.GOMAXPROCS(4) 289 | 290 | const nservers = 3 291 | var sma []*ShardMaster = make([]*ShardMaster, nservers) 292 | var kvh []string = make([]string, nservers) 293 | defer cleanup(sma) 294 | 295 | for i := 0; i < nservers; i++ { 296 | kvh[i] = port("unrel", i) 297 | } 298 | for i := 0; i < nservers; i++ { 299 | sma[i] = StartServer(kvh, i) 300 | // don't turn on unreliable because the assignment 301 | // doesn't require the shardmaster to detect duplicate 302 | // client requests. 303 | // sma[i].setunreliable(true) 304 | } 305 | 306 | ck := MakeClerk(kvh) 307 | var cka [nservers]*Clerk 308 | for i := 0; i < nservers; i++ { 309 | cka[i] = MakeClerk([]string{kvh[i]}) 310 | } 311 | 312 | fmt.Printf("Test: Concurrent leave/join, failure ...\n") 313 | 314 | const npara = 20 315 | gids := make([]int64, npara) 316 | var ca [npara]chan bool 317 | for xi := 0; xi < npara; xi++ { 318 | gids[xi] = int64(xi + 1) 319 | ca[xi] = make(chan bool) 320 | go func(i int) { 321 | defer func() { ca[i] <- true }() 322 | var gid int64 = gids[i] 323 | cka[1+(rand.Int()%2)].Join(gid+1000, []string{"a", "b", "c"}) 324 | cka[1+(rand.Int()%2)].Join(gid, []string{"a", "b", "c"}) 325 | cka[1+(rand.Int()%2)].Leave(gid + 1000) 326 | // server 0 won't be able to hear any RPCs. 327 | os.Remove(kvh[0]) 328 | }(xi) 329 | } 330 | for i := 0; i < npara; i++ { 331 | <-ca[i] 332 | } 333 | check(t, gids, ck) 334 | 335 | fmt.Printf(" ... Passed\n") 336 | } 337 | 338 | func TestFreshQuery(t *testing.T) { 339 | runtime.GOMAXPROCS(4) 340 | 341 | const nservers = 3 342 | var sma []*ShardMaster = make([]*ShardMaster, nservers) 343 | var kvh []string = make([]string, nservers) 344 | defer cleanup(sma) 345 | 346 | for i := 0; i < nservers; i++ { 347 | kvh[i] = port("fresh", i) 348 | } 349 | for i := 0; i < nservers; i++ { 350 | sma[i] = StartServer(kvh, i) 351 | } 352 | 353 | ck1 := MakeClerk([]string{kvh[1]}) 354 | 355 | fmt.Printf("Test: Query() returns latest configuration ...\n") 356 | 357 | portx := kvh[0] + strconv.Itoa(rand.Int()) 358 | if os.Rename(kvh[0], portx) != nil { 359 | t.Fatalf("os.Rename() failed") 360 | } 361 | ck0 := MakeClerk([]string{portx}) 362 | 363 | ck1.Join(1001, []string{"a", "b", "c"}) 364 | c := ck0.Query(-1) 365 | _, ok := c.Groups[1001] 366 | if ok == false { 367 | t.Fatalf("Query(-1) produced a stale configuration") 368 | } 369 | 370 | fmt.Printf(" ... Passed\n") 371 | os.Remove(portx) 372 | } 373 | -------------------------------------------------------------------------------- /src/viewservice/client.go: -------------------------------------------------------------------------------- 1 | package viewservice 2 | 3 | import "net/rpc" 4 | import "fmt" 5 | 6 | // 7 | // the viewservice Clerk lives in the client 8 | // and maintains a little state. 9 | // 10 | type Clerk struct { 11 | me string // client's name (host:port) 12 | server string // viewservice's host:port 13 | } 14 | 15 | func MakeClerk(me string, server string) *Clerk { 16 | ck := new(Clerk) 17 | ck.me = me 18 | ck.server = server 19 | return ck 20 | } 21 | 22 | // 23 | // call() sends an RPC to the rpcname handler on server srv 24 | // with arguments args, waits for the reply, and leaves the 25 | // reply in reply. the reply argument should be a pointer 26 | // to a reply structure. 27 | // 28 | // the return value is true if the server responded, and false 29 | // if call() was not able to contact the server. in particular, 30 | // the reply's contents are only valid if call() returned true. 31 | // 32 | // you should assume that call() will return an 33 | // error after a while if the server is dead. 34 | // don't provide your own time-out mechanism. 35 | // 36 | // please use call() to send all RPCs, in client.go and server.go. 37 | // please don't change this function. 38 | // 39 | func call(srv string, rpcname string, 40 | args interface{}, reply interface{}) bool { 41 | c, errx := rpc.Dial("unix", srv) 42 | if errx != nil { 43 | return false 44 | } 45 | defer c.Close() 46 | 47 | err := c.Call(rpcname, args, reply) 48 | if err == nil { 49 | return true 50 | } 51 | 52 | fmt.Println(err) 53 | return false 54 | } 55 | 56 | func (ck *Clerk) Ping(viewnum uint) (View, error) { 57 | // prepare the arguments. 58 | args := &PingArgs{} 59 | args.Me = ck.me 60 | args.Viewnum = viewnum 61 | var reply PingReply 62 | 63 | // send an RPC request, wait for the reply. 64 | ok := call(ck.server, "ViewServer.Ping", args, &reply) 65 | if ok == false { 66 | return View{}, fmt.Errorf("Ping(%v) failed", viewnum) 67 | } 68 | 69 | return reply.View, nil 70 | } 71 | 72 | func (ck *Clerk) Get() (View, bool) { 73 | args := &GetArgs{} 74 | var reply GetReply 75 | ok := call(ck.server, "ViewServer.Get", args, &reply) 76 | if ok == false { 77 | return View{}, false 78 | } 79 | return reply.View, true 80 | } 81 | 82 | func (ck *Clerk) Primary() string { 83 | v, ok := ck.Get() 84 | if ok { 85 | return v.Primary 86 | } 87 | return "" 88 | } 89 | -------------------------------------------------------------------------------- /src/viewservice/common.go: -------------------------------------------------------------------------------- 1 | package viewservice 2 | 3 | import "time" 4 | 5 | // 6 | // This is a non-replicated view service for a simple 7 | // primary/backup system. 8 | // 9 | // The view service goes through a sequence of numbered 10 | // views, each with a primary and (if possible) a backup. 11 | // A view consists of a view number and the host:port of 12 | // the view's primary and backup p/b servers. 13 | // 14 | // The primary in a view is always either the primary 15 | // or the backup of the previous view (in order to ensure 16 | // that the p/b service's state is preserved). 17 | // 18 | // Each p/b server should send a Ping RPC once per PingInterval. 19 | // The view server replies with a description of the current 20 | // view. The Pings let the view server know that the p/b 21 | // server is still alive; inform the p/b server of the current 22 | // view; and inform the view server of the most recent view 23 | // that the p/b server knows about. 24 | // 25 | // The view server proceeds to a new view when either it hasn't 26 | // received a ping from the primary or backup for a while, or 27 | // if there was no backup and a new server starts Pinging. 28 | // 29 | // The view server will not proceed to a new view until 30 | // the primary from the current view acknowledges 31 | // that it is operating in the current view. This helps 32 | // ensure that there's at most one p/b primary operating at 33 | // a time. 34 | // 35 | 36 | type View struct { 37 | Viewnum uint 38 | Primary string 39 | Backup string 40 | } 41 | 42 | // clients should send a Ping RPC this often, 43 | // to tell the viewservice that the client is alive. 44 | const PingInterval = time.Millisecond * 100 45 | 46 | // the viewserver will declare a client dead if it misses 47 | // this many Ping RPCs in a row. 48 | const DeadPings = 5 49 | 50 | // 51 | // Ping(): called by a primary/backup server to tell the 52 | // view service it is alive, to indicate whether p/b server 53 | // has seen the latest view, and for p/b server to learn 54 | // the latest view. 55 | // 56 | // If Viewnum is zero, the caller is signalling that it is 57 | // alive and could become backup if needed. 58 | // 59 | 60 | type PingArgs struct { 61 | Me string // "host:port" 62 | Viewnum uint // caller's notion of current view # 63 | } 64 | 65 | type PingReply struct { 66 | View View 67 | } 68 | 69 | // 70 | // Get(): fetch the current view, without volunteering 71 | // to be a server. mostly for clients of the p/b service, 72 | // and for testing. 73 | // 74 | 75 | type GetArgs struct { 76 | } 77 | 78 | type GetReply struct { 79 | View View 80 | } 81 | -------------------------------------------------------------------------------- /src/viewservice/server.go: -------------------------------------------------------------------------------- 1 | package viewservice 2 | 3 | import "net" 4 | import "net/rpc" 5 | import "log" 6 | import "time" 7 | import "sync" 8 | import "fmt" 9 | import "os" 10 | import "sync/atomic" 11 | 12 | type ViewServer struct { 13 | mu sync.Mutex 14 | l net.Listener 15 | dead int32 // for testing 16 | rpccount int32 // for testing 17 | me string 18 | 19 | 20 | // Your declarations here. 21 | } 22 | 23 | // 24 | // server Ping RPC handler. 25 | // 26 | func (vs *ViewServer) Ping(args *PingArgs, reply *PingReply) error { 27 | 28 | // Your code here. 29 | 30 | return nil 31 | } 32 | 33 | // 34 | // server Get() RPC handler. 35 | // 36 | func (vs *ViewServer) Get(args *GetArgs, reply *GetReply) error { 37 | 38 | // Your code here. 39 | 40 | return nil 41 | } 42 | 43 | 44 | // 45 | // tick() is called once per PingInterval; it should notice 46 | // if servers have died or recovered, and change the view 47 | // accordingly. 48 | // 49 | func (vs *ViewServer) tick() { 50 | 51 | // Your code here. 52 | } 53 | 54 | // 55 | // tell the server to shut itself down. 56 | // for testing. 57 | // please don't change these two functions. 58 | // 59 | func (vs *ViewServer) Kill() { 60 | atomic.StoreInt32(&vs.dead, 1) 61 | vs.l.Close() 62 | } 63 | 64 | // 65 | // has this server been asked to shut down? 66 | // 67 | func (vs *ViewServer) isdead() bool { 68 | return atomic.LoadInt32(&vs.dead) != 0 69 | } 70 | 71 | // please don't change this function. 72 | func (vs *ViewServer) GetRPCCount() int32 { 73 | return atomic.LoadInt32(&vs.rpccount) 74 | } 75 | 76 | func StartServer(me string) *ViewServer { 77 | vs := new(ViewServer) 78 | vs.me = me 79 | // Your vs.* initializations here. 80 | 81 | // tell net/rpc about our RPC server and handlers. 82 | rpcs := rpc.NewServer() 83 | rpcs.Register(vs) 84 | 85 | // prepare to receive connections from clients. 86 | // change "unix" to "tcp" to use over a network. 87 | os.Remove(vs.me) // only needed for "unix" 88 | l, e := net.Listen("unix", vs.me) 89 | if e != nil { 90 | log.Fatal("listen error: ", e) 91 | } 92 | vs.l = l 93 | 94 | // please don't change any of the following code, 95 | // or do anything to subvert it. 96 | 97 | // create a thread to accept RPC connections from clients. 98 | go func() { 99 | for vs.isdead() == false { 100 | conn, err := vs.l.Accept() 101 | if err == nil && vs.isdead() == false { 102 | atomic.AddInt32(&vs.rpccount, 1) 103 | go rpcs.ServeConn(conn) 104 | } else if err == nil { 105 | conn.Close() 106 | } 107 | if err != nil && vs.isdead() == false { 108 | fmt.Printf("ViewServer(%v) accept: %v\n", me, err.Error()) 109 | vs.Kill() 110 | } 111 | } 112 | }() 113 | 114 | // create a thread to call tick() periodically. 115 | go func() { 116 | for vs.isdead() == false { 117 | vs.tick() 118 | time.Sleep(PingInterval) 119 | } 120 | }() 121 | 122 | return vs 123 | } 124 | -------------------------------------------------------------------------------- /src/viewservice/test_test.go: -------------------------------------------------------------------------------- 1 | package viewservice 2 | 3 | import "testing" 4 | import "runtime" 5 | import "time" 6 | import "fmt" 7 | import "os" 8 | import "strconv" 9 | 10 | func check(t *testing.T, ck *Clerk, p string, b string, n uint) { 11 | view, _ := ck.Get() 12 | if view.Primary != p { 13 | t.Fatalf("wanted primary %v, got %v", p, view.Primary) 14 | } 15 | if view.Backup != b { 16 | t.Fatalf("wanted backup %v, got %v", b, view.Backup) 17 | } 18 | if n != 0 && n != view.Viewnum { 19 | t.Fatalf("wanted viewnum %v, got %v", n, view.Viewnum) 20 | } 21 | if ck.Primary() != p { 22 | t.Fatalf("wanted primary %v, got %v", p, ck.Primary()) 23 | } 24 | } 25 | 26 | func port(suffix string) string { 27 | s := "/var/tmp/824-" 28 | s += strconv.Itoa(os.Getuid()) + "/" 29 | os.Mkdir(s, 0777) 30 | s += "viewserver-" 31 | s += strconv.Itoa(os.Getpid()) + "-" 32 | s += suffix 33 | return s 34 | } 35 | 36 | func Test1(t *testing.T) { 37 | runtime.GOMAXPROCS(4) 38 | 39 | vshost := port("v") 40 | vs := StartServer(vshost) 41 | 42 | ck1 := MakeClerk(port("1"), vshost) 43 | ck2 := MakeClerk(port("2"), vshost) 44 | ck3 := MakeClerk(port("3"), vshost) 45 | 46 | // 47 | 48 | if ck1.Primary() != "" { 49 | t.Fatalf("there was a primary too soon") 50 | } 51 | 52 | // very first primary 53 | fmt.Printf("Test: First primary ...\n") 54 | 55 | for i := 0; i < DeadPings*2; i++ { 56 | view, _ := ck1.Ping(0) 57 | if view.Primary == ck1.me { 58 | break 59 | } 60 | time.Sleep(PingInterval) 61 | } 62 | check(t, ck1, ck1.me, "", 1) 63 | fmt.Printf(" ... Passed\n") 64 | 65 | // very first backup 66 | fmt.Printf("Test: First backup ...\n") 67 | 68 | { 69 | vx, _ := ck1.Get() 70 | for i := 0; i < DeadPings*2; i++ { 71 | ck1.Ping(1) 72 | view, _ := ck2.Ping(0) 73 | if view.Backup == ck2.me { 74 | break 75 | } 76 | time.Sleep(PingInterval) 77 | } 78 | check(t, ck1, ck1.me, ck2.me, vx.Viewnum+1) 79 | } 80 | fmt.Printf(" ... Passed\n") 81 | 82 | // primary dies, backup should take over 83 | fmt.Printf("Test: Backup takes over if primary fails ...\n") 84 | 85 | { 86 | ck1.Ping(2) 87 | vx, _ := ck2.Ping(2) 88 | for i := 0; i < DeadPings*2; i++ { 89 | v, _ := ck2.Ping(vx.Viewnum) 90 | if v.Primary == ck2.me && v.Backup == "" { 91 | break 92 | } 93 | time.Sleep(PingInterval) 94 | } 95 | check(t, ck2, ck2.me, "", vx.Viewnum+1) 96 | } 97 | fmt.Printf(" ... Passed\n") 98 | 99 | // revive ck1, should become backup 100 | fmt.Printf("Test: Restarted server becomes backup ...\n") 101 | 102 | { 103 | vx, _ := ck2.Get() 104 | ck2.Ping(vx.Viewnum) 105 | for i := 0; i < DeadPings*2; i++ { 106 | ck1.Ping(0) 107 | v, _ := ck2.Ping(vx.Viewnum) 108 | if v.Primary == ck2.me && v.Backup == ck1.me { 109 | break 110 | } 111 | time.Sleep(PingInterval) 112 | } 113 | check(t, ck2, ck2.me, ck1.me, vx.Viewnum+1) 114 | } 115 | fmt.Printf(" ... Passed\n") 116 | 117 | // start ck3, kill the primary (ck2), the previous backup (ck1) 118 | // should become the server, and ck3 the backup. 119 | // this should happen in a single view change, without 120 | // any period in which there's no backup. 121 | fmt.Printf("Test: Idle third server becomes backup if primary fails ...\n") 122 | 123 | { 124 | vx, _ := ck2.Get() 125 | ck2.Ping(vx.Viewnum) 126 | for i := 0; i < DeadPings*2; i++ { 127 | ck3.Ping(0) 128 | v, _ := ck1.Ping(vx.Viewnum) 129 | if v.Primary == ck1.me && v.Backup == ck3.me { 130 | break 131 | } 132 | vx = v 133 | time.Sleep(PingInterval) 134 | } 135 | check(t, ck1, ck1.me, ck3.me, vx.Viewnum+1) 136 | } 137 | fmt.Printf(" ... Passed\n") 138 | 139 | // kill and immediately restart the primary -- does viewservice 140 | // conclude primary is down even though it's pinging? 141 | fmt.Printf("Test: Restarted primary treated as dead ...\n") 142 | 143 | { 144 | vx, _ := ck1.Get() 145 | ck1.Ping(vx.Viewnum) 146 | for i := 0; i < DeadPings*2; i++ { 147 | ck1.Ping(0) 148 | ck3.Ping(vx.Viewnum) 149 | v, _ := ck3.Get() 150 | if v.Primary != ck1.me { 151 | break 152 | } 153 | time.Sleep(PingInterval) 154 | } 155 | vy, _ := ck3.Get() 156 | if vy.Primary != ck3.me { 157 | t.Fatalf("expected primary=%v, got %v\n", ck3.me, vy.Primary) 158 | } 159 | } 160 | fmt.Printf(" ... Passed\n") 161 | 162 | fmt.Printf("Test: Dead backup is removed from view ...\n") 163 | 164 | // set up a view with just 3 as primary, 165 | // to prepare for the next test. 166 | { 167 | for i := 0; i < DeadPings*3; i++ { 168 | vx, _ := ck3.Get() 169 | ck3.Ping(vx.Viewnum) 170 | time.Sleep(PingInterval) 171 | } 172 | v, _ := ck3.Get() 173 | if v.Primary != ck3.me || v.Backup != "" { 174 | t.Fatalf("wrong primary or backup") 175 | } 176 | } 177 | fmt.Printf(" ... Passed\n") 178 | 179 | // does viewserver wait for ack of previous view before 180 | // starting the next one? 181 | fmt.Printf("Test: Viewserver waits for primary to ack view ...\n") 182 | 183 | { 184 | // set up p=ck3 b=ck1, but 185 | // but do not ack 186 | vx, _ := ck1.Get() 187 | for i := 0; i < DeadPings*3; i++ { 188 | ck1.Ping(0) 189 | ck3.Ping(vx.Viewnum) 190 | v, _ := ck1.Get() 191 | if v.Viewnum > vx.Viewnum { 192 | break 193 | } 194 | time.Sleep(PingInterval) 195 | } 196 | check(t, ck1, ck3.me, ck1.me, vx.Viewnum+1) 197 | vy, _ := ck1.Get() 198 | // ck3 is the primary, but it never acked. 199 | // let ck3 die. check that ck1 is not promoted. 200 | for i := 0; i < DeadPings*3; i++ { 201 | v, _ := ck1.Ping(vy.Viewnum) 202 | if v.Viewnum > vy.Viewnum { 203 | break 204 | } 205 | time.Sleep(PingInterval) 206 | } 207 | check(t, ck2, ck3.me, ck1.me, vy.Viewnum) 208 | } 209 | fmt.Printf(" ... Passed\n") 210 | 211 | // if old servers die, check that a new (uninitialized) server 212 | // cannot take over. 213 | fmt.Printf("Test: Uninitialized server can't become primary ...\n") 214 | 215 | { 216 | for i := 0; i < DeadPings*2; i++ { 217 | v, _ := ck1.Get() 218 | ck1.Ping(v.Viewnum) 219 | ck2.Ping(0) 220 | ck3.Ping(v.Viewnum) 221 | time.Sleep(PingInterval) 222 | } 223 | for i := 0; i < DeadPings*2; i++ { 224 | ck2.Ping(0) 225 | time.Sleep(PingInterval) 226 | } 227 | vz, _ := ck2.Get() 228 | if vz.Primary == ck2.me { 229 | t.Fatalf("uninitialized backup promoted to primary") 230 | } 231 | } 232 | fmt.Printf(" ... Passed\n") 233 | 234 | vs.Kill() 235 | } 236 | --------------------------------------------------------------------------------