├── Dockerfile.acquire_leader ├── Dockerfile.client_fast_reqs ├── Dockerfile.leader_client ├── Dockerfile.node1 ├── Dockerfile.node2 ├── Dockerfile.node3 ├── Dockerfile.node4 ├── Dockerfile.node5 ├── Dockerfile.simple_client ├── README.md ├── chubby ├── Makefile ├── api │ └── api.go ├── client │ ├── network.go │ └── session.go ├── cmd │ ├── acquire_Lock_Client.go │ ├── client_fast_reqs.go │ ├── leader_election.go │ ├── main.go │ ├── overload_leader_client.go │ ├── simple_client.go │ └── testLock_client.go ├── config │ └── config.go ├── server │ ├── handle.go │ ├── server.go │ └── session.go └── store │ └── store.go └── docker-compose.yml /Dockerfile.acquire_leader: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o chubby/acquire_lock_client chubby/cmd/acquire_Lock_Client.go 11 | 12 | # Run simple_client exec 13 | CMD ["chubby/acquire_lock_client"] 14 | -------------------------------------------------------------------------------- /Dockerfile.client_fast_reqs: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/client_fast_reqs chubby/cmd/client_fast_reqs.go 11 | 12 | # Run simple_client exec 13 | CMD ["bin/client_fast_reqs"] -------------------------------------------------------------------------------- /Dockerfile.leader_client: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o chubby/leader_election_client1 chubby/cmd/leader_election.go 11 | 12 | # Run simple_client exec 13 | CMD ["chubby/leader_election_client1"] 14 | -------------------------------------------------------------------------------- /Dockerfile.node1: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/chubby chubby/cmd/main.go 11 | 12 | # Run chubby exec 13 | CMD ["bin/chubby", "-id", "node1", "-listen", "172.20.128.1:5379", "-raftbind", "172.20.128.1:15379"] -------------------------------------------------------------------------------- /Dockerfile.node2: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/chubby chubby/cmd/main.go 11 | 12 | # Run chubby exec 13 | CMD ["bin/chubby", "-id", "node2", "-listen", "172.20.128.2:5379", "-raftbind", "172.20.128.2:15379", "-join", "172.20.128.1:5379"] -------------------------------------------------------------------------------- /Dockerfile.node3: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/chubby chubby/cmd/main.go 11 | 12 | # Run chubby exec 13 | CMD ["bin/chubby", "-id", "node3", "-listen", "172.20.128.3:5379", "-raftbind", "172.20.128.3:15379", "-join", "172.20.128.1:5379"] -------------------------------------------------------------------------------- /Dockerfile.node4: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/chubby chubby/cmd/main.go 11 | 12 | # Run chubby exec 13 | CMD ["bin/chubby", "-id", "node4", "-listen", "172.20.128.4:5379", "-raftbind", "172.20.128.4:15379", "-join", "172.20.128.1:5379"] -------------------------------------------------------------------------------- /Dockerfile.node5: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/chubby chubby/cmd/main.go 11 | 12 | # Run chubby exec 13 | CMD ["bin/chubby", "-id", "node5", "-listen", "172.20.128.5:5379", "-raftbind", "172.20.128.5:15379", "-join", "172.20.128.1:5379"] -------------------------------------------------------------------------------- /Dockerfile.simple_client: -------------------------------------------------------------------------------- 1 | FROM golang:1.12 2 | 3 | WORKDIR $GOPATH/src/cos518project/ 4 | COPY . . 5 | 6 | # Dependencies 7 | RUN go get -d -v ./... 8 | 9 | # Build executable 10 | RUN go build -o bin/simple_client chubby/cmd/simple_client.go 11 | 12 | # Run simple_client exec 13 | CMD ["bin/simple_client"] 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # chubby 2 | A (very simplified) implementation of [Chubby](https://static.googleusercontent.com/media/research.google.com/en//archive/chubby-osdi06.pdf), Google's distributed lock service, written for [COS 518, Spring 2019](http://www.cs.princeton.edu/courses/archive/spr19/cos518/index.html). 3 | 4 | ## Instructions 5 | To bring up five Chubby nodes in individual Docker containers, run `docker-compose up`. Superuser privileges may be required. 6 | 7 | Chubby nodes can also be run locally as individual processes. To build, `cd` into the `chubby` subdirectory and run `make chubby`. We can bring up three Chubby nodes as follows: 8 | 9 | 1. First node: `./chubby -id "node1" -raftdir ./node1 -listen ":5379" -raftbind ":15379"` 10 | 2. Second node: `./chubby -id "node2" -raftdir ./node2 -listen ":6379" -raftbind ":16379" -join "127.0.0.1:5379"` 11 | 3. Third node: `./chubby -id "node3" -raftdir ./node3 -listen ":7379" -raftbind ":17379" -join "127.0.0.1:5379"` 12 | 13 | Example Chubby clients can be found in the `cmd` folder. To run, build using `make [CLIENT NAME]`, then run the resulting executable (e.g., `make simple_client; ./simple_client`). 14 | -------------------------------------------------------------------------------- /chubby/Makefile: -------------------------------------------------------------------------------- 1 | all: chubby simple_client client_fast_reqs test_client over_load_client leader_election_client1 acquire_lock_client 2 | 3 | chubby: 4 | go build -o chubby cmd/main.go 5 | 6 | simple_client: 7 | go build -o simple_client cmd/simple_client.go 8 | 9 | client_fast_reqs: 10 | go build -o client_fast_reqs cmd/client_fast_reqs.go 11 | 12 | over_load_client: 13 | go build -o over_load_client cmd/overload_leader_client.go 14 | 15 | clean: 16 | rm chubby simple_client client_fast_reqs test_client over_load_client leader_election_client1 acquire_lock_client 17 | 18 | test_client: 19 | go build -o test_client cmd/testLock_client.go 20 | 21 | leader_election_client1: 22 | go build -o leader_election_client1 cmd/leader_election.go 23 | 24 | acquire_lock_client: 25 | go build -o acquire_lock_client cmd/acquire_Lock_Client.go 26 | -------------------------------------------------------------------------------- /chubby/api/api.go: -------------------------------------------------------------------------------- 1 | // Interfaces for RPC calls between clients and servers. 2 | 3 | package api 4 | 5 | import "time" 6 | 7 | /* 8 | * Shared types. 9 | */ 10 | 11 | type ClientID string 12 | type FilePath string 13 | 14 | // Mode of a lock 15 | type LockMode int 16 | const ( 17 | EXCLUSIVE LockMode = iota 18 | SHARED 19 | FREE 20 | ) 21 | 22 | /* 23 | * RPC interfaces. 24 | */ 25 | 26 | type InitSessionRequest struct { 27 | ClientID ClientID 28 | } 29 | 30 | type InitSessionResponse struct { 31 | } 32 | 33 | type KeepAliveRequest struct { 34 | ClientID ClientID 35 | // Session information: 36 | Locks map[FilePath]LockMode // Locks held by the client. 37 | } 38 | 39 | type KeepAliveResponse struct { 40 | LeaseLength time.Duration 41 | } 42 | 43 | // TODO: make all fields exported 44 | 45 | type OpenLockRequest struct { 46 | ClientID ClientID 47 | Filepath FilePath 48 | } 49 | 50 | type OpenLockResponse struct { 51 | 52 | } 53 | 54 | type DeleteLockRequest struct { 55 | ClientID ClientID 56 | Filepath FilePath 57 | } 58 | 59 | type DeleteLockResponse struct { 60 | 61 | } 62 | 63 | type TryAcquireLockRequest struct { 64 | ClientID ClientID 65 | Filepath FilePath 66 | Mode LockMode 67 | } 68 | 69 | type TryAcquireLockResponse struct { 70 | IsSuccessful bool 71 | } 72 | 73 | type ReleaseLockRequest struct { 74 | ClientID ClientID 75 | Filepath FilePath 76 | } 77 | 78 | type ReleaseLockResponse struct { 79 | 80 | } 81 | 82 | type ReadRequest struct { 83 | ClientID ClientID 84 | Filepath FilePath 85 | } 86 | 87 | type ReadResponse struct { 88 | Content string 89 | } 90 | 91 | type WriteRequest struct { 92 | ClientID ClientID 93 | Filepath FilePath 94 | Content string 95 | } 96 | 97 | type WriteResponse struct { 98 | IsSuccessful bool 99 | } -------------------------------------------------------------------------------- /chubby/client/network.go: -------------------------------------------------------------------------------- 1 | // Assumptions the client makes about addresses of Chubby servers. 2 | 3 | package client 4 | 5 | // We assume that any Chubby node must have one of these addresses. 6 | // Yes this is gross but we're doing it anyway because of time constraints 7 | var PossibleServerAddrs = map[string]bool { 8 | "172.20.128.1:5379": true, 9 | "172.20.128.2:5379": true, 10 | "172.20.128.3:5379": true, 11 | "172.20.128.4:5379": true, 12 | "172.20.128.5:5379": true, 13 | } -------------------------------------------------------------------------------- /chubby/client/session.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "cos518project/chubby/api" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "log" 9 | "net/rpc" 10 | "os" 11 | "time" 12 | ) 13 | 14 | type ClientSession struct { 15 | // Client ID 16 | clientID api.ClientID 17 | 18 | // Server address 19 | serverAddr string 20 | 21 | // RPC client 22 | rpcClient *rpc.Client 23 | 24 | // Record start time 25 | startTime time.Time 26 | 27 | // Local lease length 28 | leaseLength time.Duration 29 | 30 | // Locks held by the session 31 | locks map[api.FilePath]api.LockMode 32 | 33 | // Are we in jeopardy right now? 34 | jeopardyFlag bool 35 | 36 | // Channel for notifying if jeopardy has ended 37 | jeopardyChan chan struct{} 38 | 39 | // Did this session expire? 40 | expired bool 41 | 42 | // Logger 43 | logger *log.Logger 44 | } 45 | 46 | const DefaultLeaseDuration time.Duration = 12 * time.Second 47 | const JeopardyDuration time.Duration = 45 * time.Second 48 | 49 | // Set up a Chubby session and periodically send KeepAlives to the server. 50 | // This method should be run as a new goroutine by the client. 51 | func InitSession(clientID api.ClientID) (*ClientSession, error) { 52 | // Initialize a session. 53 | sess := &ClientSession{ 54 | clientID: clientID, 55 | startTime: time.Now(), 56 | leaseLength: DefaultLeaseDuration, 57 | locks: make(map[api.FilePath]api.LockMode), 58 | jeopardyFlag: false, 59 | jeopardyChan: make(chan struct{}, 2), 60 | expired: false, 61 | logger: log.New(os.Stderr, "[client] ", log.LstdFlags), 62 | } 63 | 64 | // Find leader by trying to establish a session with any of the 65 | // possible server addresses. 66 | for serverAddr := range PossibleServerAddrs { 67 | // Try to set up TCP connection to server. 68 | rpcClient, err := rpc.Dial("tcp", serverAddr) 69 | if err != nil { 70 | sess.logger.Printf("RPC Dial error: %s", err.Error()) 71 | continue 72 | } 73 | 74 | // Make RPC call. 75 | sess.logger.Printf("Sending InitSession request to server %s", serverAddr) 76 | req := api.InitSessionRequest{ClientID: clientID} 77 | resp := &api.InitSessionResponse{} 78 | err = rpcClient.Call("Handler.InitSession", req, resp) 79 | if err == io.ErrUnexpectedEOF { 80 | for { 81 | err = rpcClient.Call("Handler.InitSession", req, resp) 82 | if err != io.ErrUnexpectedEOF { 83 | break 84 | } 85 | } 86 | } 87 | if err != nil { 88 | sess.logger.Printf("InitSession with server %s failed with error %s", serverAddr, err.Error()) 89 | } else { 90 | sess.logger.Printf("Session with %s initialized at client", serverAddr) 91 | 92 | // Update session info. 93 | sess.serverAddr = serverAddr 94 | sess.rpcClient = rpcClient 95 | break 96 | } 97 | } 98 | 99 | if sess.serverAddr == "" { 100 | return nil, errors.New("Could not connect to any server.") 101 | } 102 | 103 | // Call MonitorSession. 104 | go sess.MonitorSession() 105 | 106 | return sess, nil 107 | } 108 | 109 | func (sess *ClientSession) MonitorSession() { 110 | sess.logger.Printf("Monitoring session with server %s", sess.serverAddr) 111 | for { 112 | // Make new keepAlive channel. 113 | // This should be ok because this loop only occurs every 12 seconds to 57 seconds. 114 | keepAliveChan := make(chan *api.KeepAliveResponse, 1) 115 | // Make a new channel to stop goroutines 116 | quitChan := make(chan struct{}) 117 | 118 | // Send a KeepAlive, waiting for a response from the master. 119 | go func() { 120 | defer func() { 121 | if r := recover(); r != nil { 122 | sess.logger.Printf("KeepAlive waiter encounted panic: %s; recovering", r) 123 | } 124 | }() 125 | 126 | req := api.KeepAliveRequest{ClientID: sess.clientID} 127 | resp := &api.KeepAliveResponse{} 128 | 129 | sess.logger.Printf("Sending KeepAlive to server %s", sess.serverAddr) 130 | sess.logger.Printf("Client ID is %s", string(sess.clientID)) 131 | err := sess.rpcClient.Call("Handler.KeepAlive", req, resp) 132 | if err != nil { 133 | sess.logger.Printf("rpc call error: %s", err.Error()) 134 | return // do not push anything onto channel -- session will time out 135 | } 136 | 137 | keepAliveChan <- resp 138 | }() 139 | 140 | // Set up timeout 141 | durationLeaseOver := time.Until(sess.startTime.Add(sess.leaseLength)) 142 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 143 | 144 | select { 145 | case resp := <- keepAliveChan: 146 | // Process master's response 147 | // The master's response should contain a new, extended lease timeout. 148 | sess.logger.Printf("KeepAlive response from %s received within lease timeout", sess.serverAddr) 149 | 150 | // Adjust new lease length. 151 | if (sess.leaseLength >= resp.LeaseLength) { 152 | sess.logger.Printf("WARNING: new lease length shorter than current lease length") 153 | } 154 | sess.leaseLength = resp.LeaseLength 155 | 156 | case <- time.After(durationLeaseOver): 157 | // Jeopardy period begins 158 | // If no response within local lease timeout, we have to block all RPCs 159 | // from the client until the jeopardy period is over. 160 | sess.jeopardyFlag = true 161 | sess.logger.Printf("session with %s in jeopardy", sess.serverAddr) 162 | 163 | // In a new goroutine, try to send KeepAlives to every server. 164 | // KeepAlive should check if the node is the master -> if not, ignore. 165 | // In KeepAlive request, eagerly send session information to server (leaseLength, locks) 166 | // Update session serverAddr. 167 | go func() { 168 | defer func() { 169 | if r := recover(); r != nil { 170 | sess.logger.Printf("KeepAlive waiter tried to send on closed channel: recovering") 171 | } 172 | }() 173 | 174 | // Jeopardy KeepAlives should allow client to eagerly send info 175 | // to help new leader rebuild in-mem structs 176 | req := api.KeepAliveRequest { 177 | ClientID: sess.clientID, 178 | Locks: make(map[api.FilePath]api.LockMode), 179 | } 180 | 181 | for filePath, lockMode := range sess.locks { 182 | sess.logger.Printf("Add lock %s to KeepAlive session info", filePath) 183 | req.Locks[filePath] = lockMode 184 | } 185 | 186 | resp := &api.KeepAliveResponse{} 187 | 188 | for { // Keep trying all servers: this way we can wait for cell to elect a new leader. 189 | select { 190 | case <- quitChan: 191 | return 192 | default: 193 | } 194 | for serverAddr := range PossibleServerAddrs { 195 | // Try to connect to server 196 | rpcClient, err := rpc.Dial("tcp", serverAddr) 197 | if err != nil { 198 | sess.logger.Printf("could not dial address %s", serverAddr) 199 | continue 200 | } 201 | 202 | // Try to send KeepAlive to server 203 | sess.logger.Printf("sending KeepAlive to server %s", serverAddr) 204 | err = rpcClient.Call("Handler.KeepAlive", req, resp) 205 | if err == nil { 206 | // Successfully contacted new leader! 207 | sess.logger.Printf("received KeepAlive resp from server %s", serverAddr) 208 | 209 | // Update session details 210 | sess.serverAddr = serverAddr 211 | sess.rpcClient = rpcClient 212 | sess.startTime = time.Now() 213 | sess.leaseLength = DefaultLeaseDuration 214 | 215 | // Send response onto channel 216 | sess.logger.Printf("Sending response onto keepAliveChan") 217 | keepAliveChan <- resp 218 | sess.logger.Printf("Sent response onto keepAliveChan") 219 | 220 | return // Avoid closing new rpc client 221 | } else { 222 | sess.logger.Printf("KeepAlive error from server at %s: %s", serverAddr, err.Error()) 223 | } 224 | 225 | rpcClient.Close() 226 | } 227 | } 228 | }() 229 | 230 | sess.logger.Printf("waiting for jeopardy responses") 231 | 232 | // Wait for responses. 233 | select { 234 | case resp := <- keepAliveChan: 235 | // Session is saved! 236 | sess.logger.Printf("session with %s safe", sess.serverAddr) 237 | 238 | // Process master's response 239 | if (sess.leaseLength == resp.LeaseLength) { 240 | // Tear down the session. 241 | sess.expired = true 242 | close(keepAliveChan) 243 | close(quitChan) // Stop waiting goroutines. 244 | err := sess.rpcClient.Close() 245 | if err != nil { 246 | sess.logger.Printf("rpc close error: %s", err.Error()) 247 | } 248 | sess.logger.Printf("session with %s torn down", sess.serverAddr) 249 | return 250 | } 251 | 252 | // Adjust new lease length. 253 | if (sess.leaseLength > resp.LeaseLength) { 254 | sess.logger.Printf("WARNING: new lease length shorter than current lease length") 255 | } 256 | sess.leaseLength = resp.LeaseLength 257 | 258 | // Unblock all requests. 259 | sess.jeopardyFlag = false 260 | sess.jeopardyChan <- struct{}{} 261 | 262 | case <- time.After(durationJeopardyOver): 263 | // Jeopardy period ends -- tear down the session 264 | sess.expired = true 265 | close(keepAliveChan) 266 | close(quitChan) // Stop waiting goroutines. 267 | err := sess.rpcClient.Close() 268 | if err != nil { 269 | sess.logger.Printf("rpc close error: %s", err.Error()) 270 | } 271 | sess.logger.Printf("session with %s expired", sess.serverAddr) 272 | return 273 | } 274 | } 275 | } 276 | } 277 | 278 | // Current plan is to implement a function for each Chubby library call. 279 | // Each function should check jeopardyFlag to see if call should be blocked. 280 | func (sess *ClientSession) OpenLock(filePath api.FilePath) error { 281 | if sess.jeopardyFlag { 282 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 283 | select { 284 | case <-sess.jeopardyChan: 285 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 286 | case <-time.After(durationJeopardyOver): 287 | return errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 288 | } 289 | } 290 | sess.logger.Printf("Sending OpenLock request to server %s", sess.serverAddr) 291 | req := api.OpenLockRequest{ClientID: sess.clientID, Filepath: filePath} 292 | resp := &api.OpenLockResponse{} 293 | 294 | var err error 295 | for { // If we get a connection problem, keep trying. 296 | err = sess.rpcClient.Call("Handler.OpenLock", req, resp) 297 | if err != io.ErrUnexpectedEOF { 298 | break 299 | } 300 | } 301 | 302 | if err != nil { 303 | sess.logger.Printf("OpenLock with server %s failed with error %s", sess.serverAddr, err.Error()) 304 | } else { 305 | sess.logger.Printf("Open Lock successfully at filepath %s in session with %s", filePath, sess.serverAddr) 306 | } 307 | return err 308 | } 309 | 310 | func (sess *ClientSession) DeleteLock(filePath api.FilePath) error { 311 | if sess.jeopardyFlag { 312 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 313 | select { 314 | case <-sess.jeopardyChan: 315 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 316 | case <-time.After(durationJeopardyOver): 317 | return errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 318 | } 319 | } 320 | sess.logger.Printf("Sending DeleteLock request to server %s", sess.serverAddr) 321 | req := api.DeleteLockRequest{ClientID: sess.clientID, Filepath: filePath} 322 | resp := &api.DeleteLockResponse{} 323 | 324 | var err error 325 | for { // If we get a connection problem, keep trying. 326 | err = sess.rpcClient.Call("Handler.DeleteLock", req, resp) 327 | if err != io.ErrUnexpectedEOF { 328 | break 329 | } 330 | } 331 | if err != nil { 332 | sess.logger.Printf("DeleteLock with server %s failed with error %s", sess.serverAddr, err.Error()) 333 | } else { 334 | sess.logger.Printf("Delete Lock successfully at filepath %s in session with %s", filePath, sess.serverAddr) 335 | } 336 | return err 337 | } 338 | 339 | func (sess *ClientSession) TryAcquireLock(filePath api.FilePath, mode api.LockMode) (bool, error) { 340 | if sess.jeopardyFlag { 341 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 342 | select { 343 | case <-sess.jeopardyChan: 344 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 345 | case <-time.After(durationJeopardyOver): 346 | return false, errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 347 | } 348 | } 349 | /*_, ok := sess.locks[filePath] 350 | if ok { 351 | return false, errors.New(fmt.Sprintf("Client already owns the lock %s", filePath)) 352 | }*/ 353 | 354 | //sess.logger.Printf("Sending TryAcquireLock request to server %s", sess.serverAddr) 355 | req := api.TryAcquireLockRequest{ClientID: sess.clientID, Filepath: filePath, Mode: mode} 356 | resp := &api.TryAcquireLockResponse{} 357 | 358 | var err error 359 | for { // If we get a connection problem, keep trying. 360 | err = sess.rpcClient.Call("Handler.TryAcquireLock", req, resp) 361 | if err != io.ErrUnexpectedEOF { 362 | break 363 | } 364 | } 365 | 366 | if resp.IsSuccessful { 367 | sess.locks[filePath] = mode 368 | } 369 | return resp.IsSuccessful, err 370 | } 371 | 372 | func (sess *ClientSession) ReleaseLock(filePath api.FilePath) error { 373 | if sess.jeopardyFlag { 374 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 375 | select { 376 | case <-sess.jeopardyChan: 377 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 378 | case <-time.After(durationJeopardyOver): 379 | return errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 380 | } 381 | } 382 | _, ok := sess.locks[filePath] 383 | if !ok { 384 | return errors.New(fmt.Sprintf("Client does not own the lock %s", filePath)) 385 | } 386 | 387 | //sess.logger.Printf("Sending ReleaseLock request to server %s", sess.serverAddr) 388 | req := api.ReleaseLockRequest{ClientID: sess.clientID, Filepath: filePath} 389 | resp := &api.ReleaseLockResponse{} 390 | 391 | var err error 392 | for { // If we get a connection problem, keep trying. 393 | err = sess.rpcClient.Call("Handler.ReleaseLock", req, resp) 394 | if err != io.ErrUnexpectedEOF { 395 | break 396 | } 397 | } 398 | 399 | if err == nil { 400 | delete(sess.locks, filePath) 401 | } 402 | return err 403 | } 404 | 405 | func (sess *ClientSession) ReadContent(filePath api.FilePath) (string,error) { 406 | if sess.jeopardyFlag { 407 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 408 | select { 409 | case <-sess.jeopardyChan: 410 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 411 | case <-time.After(durationJeopardyOver): 412 | return "", errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 413 | } 414 | } 415 | _, ok := sess.locks[filePath] 416 | if !ok { 417 | return "", errors.New(fmt.Sprintf("Client does not own the lock %s", filePath)) 418 | } 419 | 420 | //sess.logger.Printf("Sending ReleaseLock request to server %s", sess.serverAddr) 421 | req := api.ReadRequest{ClientID: sess.clientID, Filepath: filePath} 422 | resp := &api.ReadResponse{} 423 | 424 | var err error 425 | for { // If we get a connection problem, keep trying. 426 | err = sess.rpcClient.Call("Handler.ReadContent", req, resp) 427 | if err != io.ErrUnexpectedEOF { 428 | break 429 | } 430 | } 431 | 432 | return resp.Content, err 433 | } 434 | 435 | func (sess *ClientSession) WriteContent(filePath api.FilePath, content string) (bool,error) { 436 | if sess.jeopardyFlag { 437 | durationJeopardyOver := time.Until(sess.startTime.Add(sess.leaseLength + JeopardyDuration)) 438 | select { 439 | case <-sess.jeopardyChan: 440 | sess.logger.Printf("session with %s reestablished", sess.serverAddr) 441 | case <-time.After(durationJeopardyOver): 442 | return false, errors.New(fmt.Sprintf("session with %s expired", sess.serverAddr)) 443 | } 444 | } 445 | _, ok := sess.locks[filePath] 446 | if !ok { 447 | return false, errors.New(fmt.Sprintf("Client does not own the lock %s", filePath)) 448 | } 449 | 450 | //sess.logger.Printf("Sending ReleaseLock request to server %s", sess.serverAddr) 451 | req := api.WriteRequest{ClientID: sess.clientID, Filepath: filePath, Content: content} 452 | resp := &api.WriteResponse{} 453 | 454 | var err error 455 | for { // If we get a connection problem, keep trying. 456 | err = sess.rpcClient.Call("Handler.WriteContent", req, resp) 457 | if err != io.ErrUnexpectedEOF { 458 | break 459 | } 460 | } 461 | 462 | return resp.IsSuccessful, err 463 | } 464 | 465 | func (sess *ClientSession) IsExpired() bool { 466 | return sess.expired 467 | } 468 | -------------------------------------------------------------------------------- /chubby/cmd/acquire_Lock_Client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cos518project/chubby/api" 5 | "cos518project/chubby/client" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | "time" 13 | ) 14 | 15 | var acquireLock_clientID string // ID of this client. 16 | 17 | func init() { 18 | flag.StringVar(&acquireLock_clientID, "acquireLock_clientID1", "acquireLock_clientID", "ID of this client2") 19 | } 20 | 21 | func main() { 22 | // Parse flags from command line. 23 | flag.Parse() 24 | 25 | quitCh := make(chan os.Signal, 1) 26 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 27 | time.Sleep(5*time.Second) 28 | sess, err := client.InitSession(api.ClientID(acquireLock_clientID)) 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | errOpenLock := sess.OpenLock("Lock/Lock1") 33 | if errOpenLock != nil { 34 | log.Fatal(errOpenLock) 35 | } 36 | startTime := time.Now() 37 | for { 38 | isSuccessful, err := sess.TryAcquireLock("Lock/Lock1", api.EXCLUSIVE) 39 | if err != nil { 40 | log.Println(err) 41 | } 42 | if isSuccessful && err == nil { 43 | isSuccessful, err = sess.WriteContent("Lock/Lock1", acquireLock_clientID) 44 | if !isSuccessful { 45 | fmt.Println("Unexpected Error Writing to Lock") 46 | } 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | content, err := sess.ReadContent("Lock/Lock1") 51 | if err != nil { 52 | log.Fatal(err) 53 | } else { 54 | fmt.Printf("Read Content is %s\n",content) 55 | } 56 | if content == acquireLock_clientID { 57 | elapsed := time.Since(startTime) 58 | log.Printf("Successfully acquired lock after %s\n",elapsed) 59 | } 60 | return 61 | } 62 | } 63 | 64 | content, err := sess.ReadContent("Lock/Lock1") 65 | if err != nil { 66 | log.Fatal(err) 67 | } else { 68 | fmt.Printf("Read Content is %s\n",content) 69 | } 70 | if content == acquireLock_clientID { 71 | elapsed := time.Since(startTime) 72 | log.Printf("Successfully acquired lock after %s\n",elapsed) 73 | } 74 | 75 | // Exit on signal. 76 | <-quitCh 77 | } 78 | -------------------------------------------------------------------------------- /chubby/cmd/client_fast_reqs.go: -------------------------------------------------------------------------------- 1 | // This client continuously acquires and releases locks. 2 | 3 | package main 4 | 5 | import ( 6 | "cos518project/chubby/api" 7 | "cos518project/chubby/client" 8 | "flag" 9 | "fmt" 10 | "log" 11 | "os" 12 | "os/signal" 13 | "syscall" 14 | "time" 15 | ) 16 | 17 | var client_fast_reqs_id string // ID of this client. 18 | 19 | func init() { 20 | flag.StringVar(&client_fast_reqs_id, "clientID", "client_fast_reqs", "ID of this client") 21 | } 22 | 23 | // Adapted from: https://coderwall.com/p/cp5fya/measuring-execution-time-in-go 24 | /*func timeTrack(start time.Time, name string) { 25 | elapsed := time.Since(start) 26 | fmt.Printf("Start Time is %s\n", start.String()) 27 | fmt.Printf("%s latency: %s\n", name, elapsed) 28 | }*/ 29 | 30 | func main() { 31 | // Parse flags from command line. 32 | flag.Parse() 33 | 34 | quitCh := make(chan os.Signal, 1) 35 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 36 | 37 | sess, err := client.InitSession(api.ClientID(client_fast_reqs_id)) 38 | 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | var lockName api.FilePath = "lock" 44 | 45 | err = sess.OpenLock(lockName) 46 | if err != nil { 47 | fmt.Println("Failed to open lock. Exiting.") 48 | } 49 | 50 | fmt.Println("Begin acquiring and releasing locks.") 51 | 52 | //var startTime time.Time 53 | counter := 0 54 | go func(count *int) { 55 | for range time.Tick(time.Second) { 56 | fmt.Printf("We have performed %d operations in the last second\n", *count) 57 | *count = 0 58 | } 59 | }(&counter) 60 | for { 61 | select { 62 | case <- quitCh: 63 | fmt.Println("Exiting.") 64 | return 65 | 66 | default: 67 | //startTime = time.Now() 68 | ok, err := sess.TryAcquireLock(lockName, api.SHARED) 69 | //timeTrack(startTime, "TryAcquire") 70 | 71 | if err != nil { 72 | if sess.IsExpired() { 73 | sess, err = client.InitSession(api.ClientID(client_fast_reqs_id)) 74 | if err != nil { 75 | log.Fatal(err) 76 | } else { 77 | continue 78 | } 79 | } 80 | log.Printf("TryAcquire failed with error: %s\n", err.Error()) 81 | // Say we acquire a lock, then we pause that node 82 | // Release will fail with an error saying connection is closed 83 | // When the node is backup before jeopardy is over, the client 84 | // would hold the lock, and if we don't comment out continue, 85 | // the client will just keep getting the client already owns this lock 86 | // error 87 | //continue 88 | } 89 | if !ok { 90 | log.Println("Failed to acquire lock. Continuing.") 91 | //continue 92 | } 93 | counter += 1 94 | //startTime = time.Now() 95 | err = sess.ReleaseLock(lockName) 96 | //timeTrack(startTime, "Release") 97 | 98 | if err != nil { 99 | log.Printf("Release failed with error: %s\n", err.Error()) 100 | continue 101 | } 102 | counter += 1 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /chubby/cmd/leader_election.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cos518project/chubby/api" 5 | "cos518project/chubby/client" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "os" 10 | "os/signal" 11 | "syscall" 12 | ) 13 | 14 | var leader_election_id1 string // ID of this client. 15 | 16 | func init() { 17 | flag.StringVar(&leader_election_id1, "leader_clientID1", "leader_election_id1", "ID of this client1") 18 | } 19 | 20 | func main() { 21 | // Parse flags from command line. 22 | flag.Parse() 23 | 24 | quitCh := make(chan os.Signal, 1) 25 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 26 | 27 | sess, err := client.InitSession(api.ClientID(leader_election_id1)) 28 | if err != nil { 29 | log.Fatal(err) 30 | } 31 | errOpenLock := sess.OpenLock("Lock/Lock1") 32 | if errOpenLock != nil { 33 | log.Fatal(errOpenLock) 34 | } 35 | isSuccessful, err := sess.TryAcquireLock("Lock/Lock1", api.EXCLUSIVE) 36 | if !isSuccessful { 37 | fmt.Printf("Lock Acquire Unexpected Failure") 38 | } 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | isSuccessful, err = sess.WriteContent("Lock/Lock1", leader_election_id1) 43 | if !isSuccessful { 44 | fmt.Println("Unexpected Error Writing to Lock") 45 | } 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | content, err := sess.ReadContent("Lock/Lock1") 50 | if err != nil { 51 | log.Fatal(err) 52 | } else { 53 | fmt.Printf("Read Content is %s\n",content) 54 | } 55 | for { 56 | // Exit on signal. 57 | select { 58 | case <- quitCh: 59 | break 60 | default: 61 | continue 62 | } 63 | } 64 | } 65 | 66 | -------------------------------------------------------------------------------- /chubby/cmd/main.go: -------------------------------------------------------------------------------- 1 | // Adapted from Leto command interface: 2 | // https://github.com/yongman/leto/blob/master/cmd/main.go 3 | 4 | // Copyright (C) 2018 YanMing 5 | // 6 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 7 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 8 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 9 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 10 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 11 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 12 | // SOFTWARE. 13 | 14 | package main 15 | 16 | import ( 17 | "cos518project/chubby/config" 18 | "cos518project/chubby/server" 19 | "flag" 20 | "os" 21 | "os/signal" 22 | "syscall" 23 | ) 24 | 25 | var ( 26 | listen string // Server listen port. 27 | raftDir string // Raft data directory. 28 | raftBind string // Raft bus transport bind port. 29 | nodeId string // Node ID. 30 | join string // Address of existing cluster at which to join. 31 | inmem bool // If true, keep log and stable storage in memory. 32 | ) 33 | 34 | func init() { 35 | flag.StringVar(&listen, "listen", ":5379", "server listen port") 36 | flag.StringVar(&raftDir, "raftdir", "./", "raft data directory") 37 | flag.StringVar(&raftBind, "raftbind", ":15379", "raft bus transport bind port") 38 | flag.StringVar(&nodeId, "id", "", "node id") 39 | flag.StringVar(&join, "join", "", "join to existing cluster at this address") 40 | flag.BoolVar(&inmem, "inmem", false, "log and stable storage in memory") 41 | } 42 | 43 | func main() { 44 | // Parse flags from command line. 45 | flag.Parse() 46 | 47 | var ( 48 | c *config.Config 49 | ) 50 | 51 | // Create new Chubby config. 52 | c = config.NewConfig(listen, raftDir, raftBind, nodeId, join, inmem) 53 | //fmt.Println(c) 54 | 55 | quitCh := make(chan os.Signal, 1) 56 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 57 | 58 | // Run the server. 59 | go server.Run(c) 60 | // Exit on signal. 61 | <-quitCh 62 | } 63 | -------------------------------------------------------------------------------- /chubby/cmd/overload_leader_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cos518project/chubby/api" 5 | "cos518project/chubby/client" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | "time" 12 | ) 13 | func timeTrack(start time.Time, name string) { 14 | elapsed := time.Since(start) 15 | fmt.Printf("Start Time is %s\n", start.String()) 16 | fmt.Printf("%s latency: %s\n", name, elapsed) 17 | } 18 | 19 | func acquire_release(clientID string, sess *client.ClientSession) { 20 | var lockNames []string 21 | for i := 0; i < 100; i++ { 22 | lockName := fmt.Sprintf("lock_%d_%s",i, string(clientID)) 23 | err := sess.OpenLock(api.FilePath(lockName)) 24 | if err != nil { 25 | fmt.Println("Failed to open lock. Exiting.") 26 | } 27 | lockNames = append(lockNames, lockName) 28 | } 29 | for i := 0; i < 1000000; i++ { 30 | for j := 0; j < 100; j++ { 31 | ok, err := sess.TryAcquireLock(api.FilePath(lockNames[j]), api.EXCLUSIVE) 32 | if !ok { 33 | log.Println("Failed to acquire lock. Continuing.") 34 | } else { 35 | log.Println("Successfully acquired lock") 36 | } 37 | if err != nil { 38 | log.Fatal(err) 39 | } 40 | err = sess.ReleaseLock(api.FilePath(lockNames[j])) 41 | 42 | if err != nil { 43 | log.Printf("Release failed with error: %s\n", err.Error()) 44 | continue 45 | } else { 46 | log.Println("Successfully released lock") 47 | } 48 | } 49 | } 50 | } 51 | 52 | func main () { 53 | var clientIDs []string 54 | var sessions []*client.ClientSession 55 | quitCh := make(chan os.Signal, 1) 56 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 57 | 58 | for i := 0; i < 200; i++ { 59 | clientIDs = append(clientIDs, fmt.Sprintf("client_%d",i)) 60 | } 61 | for i := 0; i < 200; i++ { 62 | sess,err := client.InitSession(api.ClientID(clientIDs[i])) 63 | if err != nil { 64 | log.Fatal(err) 65 | } 66 | sessions = append(sessions, sess) 67 | } 68 | for i :=0; i < 199; i++ { 69 | go acquire_release(clientIDs[i], sessions[i]) 70 | } 71 | 72 | var lockNames []string 73 | for i := 0; i < 100; i++ { 74 | lockName := fmt.Sprintf("lock_%d_%s",i, string(clientIDs[99])) 75 | err := sessions[199].OpenLock(api.FilePath(lockName)) 76 | if err != nil { 77 | fmt.Println("Failed to open lock. Exiting.") 78 | } 79 | lockNames = append(lockNames, lockName) 80 | } 81 | 82 | counter := 0 83 | go func(count *int) { 84 | for range time.Tick(time.Second) { 85 | fmt.Printf("We have performed %d operations in the last second\n", *count) 86 | *count = 0 87 | } 88 | }(&counter) 89 | 90 | for i := 0; i < 1000000; i++ { 91 | for j := 0; j < 100; j++ { 92 | ok, err := sessions[199].TryAcquireLock(api.FilePath(lockNames[j]), api.EXCLUSIVE) 93 | if !ok { 94 | log.Println("Failed to acquire lock. Continuing.") 95 | } 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | err = sessions[199].ReleaseLock(api.FilePath(lockNames[j])) 100 | 101 | if err != nil { 102 | log.Printf("Release failed with error: %s\n", err.Error()) 103 | continue 104 | } 105 | if err == nil { 106 | counter += 1 107 | } 108 | 109 | } 110 | } 111 | 112 | 113 | // Exit on signal. 114 | <-quitCh 115 | 116 | } 117 | -------------------------------------------------------------------------------- /chubby/cmd/simple_client.go: -------------------------------------------------------------------------------- 1 | // This client does nothing but maintain a session with the Chubby server. 2 | 3 | package main 4 | 5 | import ( 6 | "cos518project/chubby/api" 7 | "cos518project/chubby/client" 8 | "flag" 9 | "log" 10 | "os" 11 | "os/signal" 12 | "syscall" 13 | ) 14 | 15 | var simple_client_id string // ID of this client. 16 | 17 | func init() { 18 | flag.StringVar(&simple_client_id, "clientID", "simple_client", "ID of this client") 19 | } 20 | 21 | func main() { 22 | // Parse flags from command line. 23 | flag.Parse() 24 | 25 | quitCh := make(chan os.Signal, 1) 26 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 27 | 28 | _, err := client.InitSession(api.ClientID(simple_client_id)) 29 | if err != nil { 30 | log.Fatal(err) 31 | } 32 | 33 | // Exit on signal. 34 | <-quitCh 35 | } 36 | -------------------------------------------------------------------------------- /chubby/cmd/testLock_client.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "cos518project/chubby/api" 5 | "cos518project/chubby/client" 6 | "flag" 7 | "log" 8 | "os" 9 | "os/signal" 10 | "syscall" 11 | ) 12 | 13 | var clientID1 string // ID of client 1 14 | var clientID2 string // ID of client 2 15 | 16 | func init() { 17 | flag.StringVar(&clientID1, "clientID1", "simple_client_1", "ID of client 1") 18 | flag.StringVar(&clientID2, "clientID2", "simple_client_2", "ID of client 2") 19 | } 20 | 21 | func main() { 22 | // Parse flags from command line. 23 | flag.Parse() 24 | 25 | quitCh := make(chan os.Signal, 1) 26 | signal.Notify(quitCh, os.Kill, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) 27 | 28 | // Establish two sessions 29 | sess1, err := client.InitSession(api.ClientID(clientID1)) 30 | sess2, err := client.InitSession(api.ClientID(clientID2)) 31 | 32 | // Test Open Locks 33 | errOpenLock1 := sess1.OpenLock("LOCK/Lock1") 34 | errOpenLock2 := sess2.OpenLock("LOCK/Lock2") 35 | 36 | if errOpenLock1 != nil { 37 | log.Printf("Session 1 has trouble opening lock ") 38 | log.Fatal(errOpenLock1) 39 | } else { 40 | log.Printf("Session 1 has opened lock successfully") 41 | } 42 | if errOpenLock2 != nil { 43 | log.Printf("Session 2 has trouble opening lock") 44 | log.Fatal(errOpenLock2) 45 | } else { 46 | log.Printf("Session 2 has opened lock successfully") 47 | } 48 | 49 | errOpenLock1 = sess1.OpenLock("LOCK/LockShared") 50 | if errOpenLock1 != nil { 51 | log.Printf("Session 1 has trouble opening lock") 52 | log.Fatal(errOpenLock1) 53 | } else { 54 | log.Printf("Session 1 has opened lock successfully") 55 | } 56 | 57 | // Test TryAcquire Lock 58 | isSuccessful, acquireErr := sess1.TryAcquireLock("LOCK/Lock1", api.EXCLUSIVE) 59 | if !isSuccessful { 60 | log.Printf("Try Acquire Lock failed when it should succeed") 61 | } 62 | if acquireErr != nil { 63 | log.Printf("Try Acquire Lock Unexpected Error") 64 | log.Fatal(acquireErr) 65 | } 66 | 67 | // Try Acquire a Shared Lock 68 | isSuccessful, acquireErr = sess1.TryAcquireLock("LOCK/LockShared", api.SHARED) 69 | if !isSuccessful { 70 | log.Printf("Try Acquire Shared Lock failed when it should succeed") 71 | } 72 | if acquireErr != nil { 73 | log.Printf("Try Acquire Lock Unexpected Error") 74 | log.Fatal(acquireErr) 75 | } 76 | 77 | isSuccessful, acquireErr = sess2.TryAcquireLock("LOCK/LockShared", api.SHARED) 78 | if !isSuccessful { 79 | log.Printf("Try Acquire Shared Lock failed when it should succeed") 80 | } 81 | if acquireErr != nil { 82 | log.Printf("Try Acquire Shared Lock Unexpected Error") 83 | log.Fatal(acquireErr) 84 | } 85 | 86 | // Should not be able to acquire a lock you already acquired 87 | isSuccessful, acquireErr = sess1.TryAcquireLock("LOCK/Lock1", api.EXCLUSIVE) 88 | if isSuccessful { 89 | log.Printf("Should fail because the lock we are trying to acquire is in exclusive mode") 90 | } 91 | if acquireErr == nil { 92 | log.Printf("Should fail because the lock we are trying to acquire is in exclusive mode") 93 | } 94 | 95 | // Should not be able to acquire a lock someone else acquired in exclusive mode 96 | isSuccessful, acquireErr = sess2.TryAcquireLock("LOCK/Lock1", api.EXCLUSIVE) 97 | if isSuccessful { 98 | log.Printf("Session 2 Should fail but successfuly because the lock we are trying to acquire is in exclusive mode") 99 | } 100 | 101 | // Should not be able to release a lock you don't own 102 | releaseErr := sess2.ReleaseLock("LOCK/Lock1") 103 | if releaseErr == nil { 104 | log.Printf("Should fail because the lock we are trying to release is a lock we don't own") 105 | } 106 | 107 | // Should not be able to delete a lock you don't own 108 | deleteErr := sess2.DeleteLock("LOCK/Lock1") 109 | if deleteErr == nil { 110 | log.Printf("Delete Lock Should Fail because %s is trying to delete a lock it doesn't own", clientID2) 111 | } 112 | 113 | // Test release lock 114 | releaseErr = sess1.ReleaseLock("LOCK/Lock1") 115 | if releaseErr != nil { 116 | log.Printf("Unexpected Lock release failure") 117 | log.Fatal(releaseErr) 118 | } 119 | 120 | // Test Delete Lock 121 | deleteErr = sess1.DeleteLock("LOCK/Lock1") 122 | if deleteErr == nil { 123 | log.Printf("Delete Lock Should Fail because %s is trying to delete a lock it doesn't hold", clientID1) 124 | } 125 | 126 | isSuccessful, acquireErr = sess1.TryAcquireLock("LOCK/Lock1", api.SHARED) 127 | 128 | if !isSuccessful { 129 | log.Printf("Unexpected Failure to Acquire Lock in Shared Mode") 130 | } 131 | if acquireErr != nil { 132 | log.Fatal(acquireErr) 133 | } 134 | // Test Delete Lock 135 | deleteErr = sess1.DeleteLock("LOCK/Lock1") 136 | if deleteErr == nil { 137 | log.Printf("Delete Lock Should Fail because %s is trying to delete a lock it holds in Shared mode", clientID1) 138 | } 139 | 140 | 141 | // Test release lock 142 | releaseErr = sess1.ReleaseLock("LOCK/Lock1") 143 | if releaseErr != nil { 144 | log.Printf("Unexpected Lock release failure") 145 | log.Fatal(releaseErr) 146 | } 147 | isSuccessful, acquireErr = sess1.TryAcquireLock("LOCK/Lock1", api.EXCLUSIVE) 148 | if !isSuccessful { 149 | log.Printf("Unexpected Exclusive Acquire Failure at lock path %s", "LOCK/Lock1") 150 | } 151 | if acquireErr != nil { 152 | log.Fatal(acquireErr) 153 | } 154 | deleteErr = sess1.DeleteLock("LOCK/Lock1") 155 | if deleteErr != nil { 156 | log.Printf("Unexpected Delete err %s", clientID1) 157 | log.Fatal(deleteErr) 158 | } 159 | 160 | // Test release lock 161 | releaseErr = sess1.ReleaseLock("LOCK/Lock1") 162 | if releaseErr == nil { 163 | log.Printf("Should fail because trying to release a lock that doesn't exist") 164 | } 165 | 166 | 167 | if err != nil { 168 | log.Fatal(err) 169 | } 170 | 171 | // Exit on signal. 172 | <-quitCh 173 | } 174 | -------------------------------------------------------------------------------- /chubby/config/config.go: -------------------------------------------------------------------------------- 1 | // Configuration for a Raft node. 2 | // 3 | // Adapted from leto config file: 4 | // https://github.com/yongman/leto/blob/master/config/config.go 5 | 6 | package config 7 | 8 | type Config struct { 9 | Listen string 10 | RaftDir string 11 | RaftBind string 12 | Join string 13 | NodeID string 14 | InMem bool 15 | } 16 | 17 | func NewConfig(listen, raftDir, raftBind, nodeId, join string, inmem bool) *Config { 18 | return &Config{ 19 | Listen: listen, 20 | RaftDir: raftDir, 21 | RaftBind: raftBind, 22 | NodeID: nodeId, 23 | Join: join, 24 | InMem: inmem, 25 | } 26 | } -------------------------------------------------------------------------------- /chubby/server/handle.go: -------------------------------------------------------------------------------- 1 | // Define RPC calls accepted by Chubby server. 2 | 3 | package server 4 | 5 | import ( 6 | "cos518project/chubby/api" 7 | "errors" 8 | "fmt" 9 | ) 10 | 11 | /* 12 | * Additional RPC interfaces available only to servers. 13 | */ 14 | 15 | type JoinRequest struct { 16 | RaftAddr string 17 | NodeID string 18 | } 19 | 20 | type JoinResponse struct { 21 | Error error 22 | } 23 | 24 | // RPC handler type 25 | type Handler int 26 | 27 | /* 28 | * Called by servers: 29 | */ 30 | 31 | // Join the caller server to our server. 32 | func (h *Handler) Join(req JoinRequest, res *JoinResponse) error { 33 | err := app.store.Join(req.NodeID, req.RaftAddr) 34 | res.Error = err 35 | return err 36 | } 37 | 38 | /* 39 | * Called by clients: 40 | */ 41 | 42 | // Initialize a client-server session. 43 | func (h *Handler) InitSession(req api.InitSessionRequest, res *api.InitSessionResponse) error { 44 | // If a non-leader node receives an InitSession, return error 45 | if app.store.RaftBind != string(app.store.Raft.Leader()) { 46 | return errors.New(fmt.Sprintf("Node %s is not the leader", app.address)) 47 | } 48 | 49 | _, err := CreateSession(api.ClientID(req.ClientID)) 50 | return err 51 | } 52 | 53 | // KeepAlive calls allow the client to extend the Chubby session. 54 | func (h *Handler) KeepAlive(req api.KeepAliveRequest, res *api.KeepAliveResponse) error { 55 | // If a non-leader node receives a KeepAlive, return error 56 | if app.store.RaftBind != string(app.store.Raft.Leader()) { 57 | return errors.New(fmt.Sprintf("Node %s is not the leader", app.address)) 58 | } 59 | 60 | var err error 61 | sess, ok := app.sessions[req.ClientID] 62 | if !ok { 63 | // Probably a jeopardy KeepAlive: create a new session for the client 64 | app.logger.Printf("Client %s sent jeopardy KeepAlive: creating new session", req.ClientID) 65 | 66 | // Note: this starts the local lease countdown 67 | // Should be ok to not call KeepAlive until later because lease TTL is pretty long (12s) 68 | sess, err = CreateSession(req.ClientID) 69 | if err != nil { 70 | // This shouldn't happen because session shouldn't be in app.sessions struct yet 71 | return err 72 | } 73 | 74 | app.logger.Printf("New session for client %s created", req.ClientID) 75 | 76 | // For each lock in the KeepAlive, try to acquire the lock 77 | // If any of the acquires fail, terminate the session. 78 | for filePath, lockMode := range(req.Locks) { 79 | ok, err := sess.TryAcquireLock(filePath, lockMode) 80 | if err != nil { 81 | // Don't return an error because the session won't terminate! 82 | app.logger.Printf("Error when client %s acquiring lock at %s: %s", req.ClientID, filePath, err.Error()) 83 | } 84 | if !ok { 85 | app.logger.Printf("Jeopardy client %s failed to acquire lock at %s", req.ClientID, filePath) 86 | // This should cause the KeepAlive response to return that session should end. 87 | sess.TerminateSession() 88 | 89 | return nil // Don't return an error because the session won't terminate! 90 | } 91 | app.logger.Printf("Lock %s reacquired successfully.", filePath) 92 | } 93 | 94 | app.logger.Printf("Finished jeopardy KeepAlive process for client %s", req.ClientID) 95 | } 96 | 97 | duration := sess.KeepAlive(req.ClientID) 98 | res.LeaseLength = duration 99 | return nil 100 | } 101 | 102 | // Open a lock. 103 | func (h *Handler) OpenLock(req api.OpenLockRequest, res *api.OpenLockResponse) error { 104 | sess, ok := app.sessions[req.ClientID] 105 | if !ok { 106 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 107 | } 108 | err := sess.OpenLock(req.Filepath) 109 | if err != nil { 110 | return err 111 | } 112 | return nil 113 | } 114 | 115 | // Delete a lock. 116 | func (h *Handler) DeleteLock(req api.DeleteLockRequest, res *api.DeleteLockResponse) error { 117 | sess, ok := app.sessions[req.ClientID] 118 | if !ok { 119 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 120 | } 121 | err := sess.DeleteLock(req.Filepath) 122 | if err != nil { 123 | return err 124 | } 125 | return nil 126 | } 127 | 128 | // Try to acquire a lock. 129 | func (h *Handler) TryAcquireLock(req api.TryAcquireLockRequest, res *api.TryAcquireLockResponse) error { 130 | sess, ok := app.sessions[req.ClientID] 131 | if !ok { 132 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 133 | } 134 | isSuccessful, err := sess.TryAcquireLock(req.Filepath, req.Mode) 135 | if err != nil { 136 | return err 137 | } 138 | res.IsSuccessful = isSuccessful 139 | return nil 140 | } 141 | 142 | // Release lock. 143 | func (h *Handler) ReleaseLock(req api.ReleaseLockRequest, res *api.ReleaseLockResponse) error { 144 | sess, ok := app.sessions[req.ClientID] 145 | if !ok { 146 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 147 | } 148 | err := sess.ReleaseLock(req.Filepath) 149 | if err != nil { 150 | return err 151 | } 152 | return nil 153 | } 154 | 155 | // Read Content 156 | func (h *Handler) ReadContent(req api.ReadRequest, res *api.ReadResponse) error { 157 | sess, ok := app.sessions[req.ClientID] 158 | if !ok { 159 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 160 | } 161 | content, err := sess.ReadContent(req.Filepath) 162 | if err != nil { 163 | return err 164 | } 165 | res.Content = content 166 | return nil 167 | } 168 | 169 | // Read Content 170 | func (h *Handler) WriteContent(req api.WriteRequest, res *api.WriteResponse) error { 171 | sess, ok := app.sessions[req.ClientID] 172 | if !ok { 173 | return errors.New(fmt.Sprintf("No session exists for %s", req.ClientID)) 174 | } 175 | err := sess.WriteContent (req.Filepath, req.Content) 176 | if err != nil { 177 | res.IsSuccessful = false 178 | return err 179 | } 180 | res.IsSuccessful = true 181 | return nil 182 | } -------------------------------------------------------------------------------- /chubby/server/server.go: -------------------------------------------------------------------------------- 1 | // Define Chubby server application. 2 | 3 | // Adapted from Leto server file: 4 | // https://github.com/yongman/leto/blob/master/server/server.go 5 | 6 | // Copyright (C) 2018 YanMing 7 | // 8 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 9 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 10 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 11 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 12 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 13 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 14 | // SOFTWARE. 15 | 16 | package server 17 | 18 | import ( 19 | "cos518project/chubby/config" 20 | "cos518project/chubby/store" 21 | "cos518project/chubby/api" 22 | "fmt" 23 | "log" 24 | "net" 25 | "net/rpc" 26 | "os" 27 | ) 28 | 29 | type App struct { 30 | listener net.Listener 31 | 32 | // wrapper and manager for db instance 33 | store *store.Store 34 | 35 | logger *log.Logger 36 | 37 | // Current Node's Address 38 | address string 39 | 40 | // In-memory struct of handles. 41 | // Maps handle IDs to handle metadata. 42 | // handles map[int]Handle 43 | 44 | // In-memory struct of locks. 45 | // Maps filepaths to Lock structs. 46 | locks map[api.FilePath]*Lock 47 | 48 | // In-memory struct of sessions. 49 | sessions map[api.ClientID]*Session 50 | } 51 | 52 | // No choice but to make this variable package-level :( 53 | var app *App 54 | 55 | func Run(conf *config.Config) { 56 | var err error 57 | 58 | // Init app struct. 59 | app = &App{ 60 | logger: log.New(os.Stderr, "[server] ", log.LstdFlags), 61 | store: store.New(conf.RaftDir, conf.RaftBind, conf.InMem), 62 | address: conf.Listen, 63 | locks: make(map[api.FilePath]*Lock), 64 | sessions: make(map[api.ClientID]*Session), 65 | } 66 | 67 | // Open the store. 68 | bootstrap := conf.Join == "" 69 | err = app.store.Open(bootstrap, conf.NodeID) 70 | if err != nil { 71 | log.Fatal(err) 72 | } 73 | 74 | if !bootstrap { 75 | // Set up TCP connection. 76 | client, err := rpc.Dial("tcp", conf.Join) 77 | if err != nil { 78 | log.Fatal(err) 79 | } 80 | 81 | app.logger.Printf("set up connection to %s", conf.Join) 82 | 83 | var req JoinRequest 84 | var resp JoinResponse 85 | 86 | req.RaftAddr = conf.RaftBind 87 | req.NodeID = conf.NodeID 88 | 89 | err = client.Call("Handler.Join", req, &resp) 90 | if err != nil { 91 | log.Fatal(err) 92 | } 93 | if resp.Error != nil { 94 | log.Fatal(err) 95 | } 96 | } 97 | 98 | // Listen for client connections. 99 | handler := new(Handler) 100 | err = rpc.Register(handler) 101 | 102 | app.listener, err = net.Listen("tcp", conf.Listen) 103 | app.logger.Printf("server listen in %s", conf.Listen) 104 | if err != nil { 105 | fmt.Println(err.Error()) 106 | } 107 | 108 | // Accept connections. 109 | rpc.Accept(app.listener) 110 | } 111 | -------------------------------------------------------------------------------- /chubby/server/session.go: -------------------------------------------------------------------------------- 1 | // Defines Chubby session metadata, as well as operations on locks that can 2 | // be performed as part of a Chubby session. 3 | 4 | package server 5 | 6 | import ( 7 | "cos518project/chubby/api" 8 | "errors" 9 | "fmt" 10 | "log" 11 | "sync" 12 | "time" 13 | ) 14 | 15 | const DefaultLeaseExt = 15 * time.Second 16 | 17 | // Session contains metadata for one Chubby session. 18 | // For simplicity, we say that each client can only init one session with 19 | // the Chubby servers. 20 | type Session struct { 21 | // Client to which this Session corresponds. 22 | clientID api.ClientID 23 | 24 | // Start time 25 | startTime time.Time 26 | 27 | // Length of the Lease 28 | leaseLength time.Duration 29 | 30 | //TTL Lock 31 | ttlLock sync.Mutex 32 | 33 | // Channel used to block KeepAlive 34 | ttlChannel chan struct{} 35 | 36 | // A data structure describing which locks the client holds. 37 | // Maps lock filepath -> Lock struct. 38 | locks map[api.FilePath]*Lock 39 | 40 | // Did we terminate this session? 41 | terminated bool 42 | 43 | // Terminated channel 44 | terminatedChan chan struct{} 45 | } 46 | 47 | // Lock describes information about a particular Chubby lock. 48 | type Lock struct { 49 | path api.FilePath // The path to this lock in the store. 50 | mode api.LockMode // api.SHARED or exclusive lock? 51 | owners map[api.ClientID]bool // Who is holding the lock? 52 | content string // The content of the file 53 | } 54 | 55 | /* Create Session struct. */ 56 | func CreateSession(clientID api.ClientID) (*Session, error) { 57 | sess, ok := app.sessions[clientID] 58 | 59 | if ok { 60 | return nil, errors.New(fmt.Sprintf("The client already has a session established with the master")) 61 | } 62 | 63 | 64 | 65 | app.logger.Printf("Creating session with client %s", clientID) 66 | 67 | // Create new session struct. 68 | sess = &Session{ 69 | clientID: clientID, 70 | startTime: time.Now(), 71 | leaseLength: DefaultLeaseExt, 72 | ttlChannel: make(chan struct{}, 2), 73 | locks: make(map[api.FilePath]*Lock), 74 | terminated: false, 75 | terminatedChan: make(chan struct{}, 2), 76 | } 77 | 78 | // Add the session to the sessions map. 79 | app.sessions[clientID] = sess 80 | 81 | // In a separate goroutine, periodically check if the lease is over 82 | go sess.MonitorSession() 83 | 84 | return sess, nil 85 | } 86 | 87 | func (sess *Session) MonitorSession() { 88 | app.logger.Printf("Monitoring session with client %s", sess.clientID) 89 | 90 | // At each second, check time until the lease is over. 91 | ticker := time.Tick(time.Second) 92 | for range ticker { 93 | timeLeaseOver := sess.startTime.Add(sess.leaseLength) 94 | 95 | var durationLeaseOver time.Duration = 0 96 | if timeLeaseOver.After(time.Now()) { 97 | durationLeaseOver = time.Until(timeLeaseOver) 98 | } 99 | 100 | if durationLeaseOver == 0 { 101 | // Lease expired: terminate the session 102 | app.logger.Printf("Lease with client %s expired: terminating session", sess.clientID) 103 | sess.TerminateSession() 104 | return 105 | } 106 | 107 | if durationLeaseOver <= (1 * time.Second) { 108 | // Trigger KeepAlive response 1 second before timeout 109 | sess.ttlChannel <- struct{}{} 110 | } 111 | } 112 | } 113 | 114 | // Terminate the session. 115 | func (sess *Session) TerminateSession() { 116 | // We can't justs delete the session from the app session map because 117 | // We cannot delete the session from the app session map because 118 | // Chubby could have experienced a failover event. 119 | sess.terminated = true 120 | close(sess.terminatedChan) 121 | 122 | // Release all the locks in the session. 123 | for filePath := range sess.locks { 124 | err := sess.ReleaseLock(filePath) 125 | if err != nil { 126 | app.logger.Printf( 127 | "error when client %s releasing lock at %s: %s", 128 | sess.clientID, 129 | filePath, 130 | err.Error()) 131 | } 132 | } 133 | 134 | app.logger.Printf("terminated session with client %s", sess.clientID) 135 | } 136 | 137 | // Extend Lease after receiving keepalive messages 138 | func (sess *Session) KeepAlive(clientID api.ClientID) (time.Duration) { 139 | // Block until shortly before lease expires 140 | select { 141 | case <- sess.terminatedChan: 142 | // Return early response saying that session should end. 143 | return sess.leaseLength 144 | 145 | case <- sess.ttlChannel: 146 | // Extend lease by 12 seconds 147 | sess.leaseLength = sess.leaseLength + DefaultLeaseExt 148 | 149 | app.logger.Printf( 150 | "session with client %s extended: lease length %s", 151 | sess.clientID, 152 | sess.leaseLength.String()) 153 | 154 | // Return new lease length. 155 | return sess.leaseLength 156 | } 157 | } 158 | 159 | // Create the lock if it does not exist. 160 | func (sess *Session) OpenLock(path api.FilePath) error { 161 | // Check if lock exists in persistent store 162 | _, err := app.store.Get(string(path)) 163 | if err != nil { 164 | // Add lock to persistent store: (key: LockPath, value: "") 165 | err = app.store.Set(string(path), "") 166 | if err != nil { 167 | return err 168 | } 169 | 170 | // Add lock to in-memory struct of locks 171 | lock := &Lock{ 172 | path: path, 173 | mode: api.FREE, 174 | owners: make(map[api.ClientID]bool), 175 | content: "", 176 | } 177 | app.locks[path] = lock 178 | sess.locks[path] = lock 179 | } 180 | 181 | return nil 182 | } 183 | 184 | // Delete the lock. Lock must be held in exclusive mode before calling DeleteLock. 185 | func (sess *Session) DeleteLock(path api.FilePath) error { 186 | // If we are not holding the lock, we cannot delete it. 187 | lock, exists := sess.locks[path] 188 | if !exists { 189 | return errors.New(fmt.Sprintf("Client does not hold the lock at path %s", path)) 190 | } 191 | 192 | // Check if we are holding the lock in exclusive mode 193 | if lock.mode != api.EXCLUSIVE { 194 | return errors.New(fmt.Sprintf("Client does not hold the lock at path %s in exclusive mode", path)) 195 | } 196 | 197 | // Check that the lock actually exists in the store. 198 | _, err := app.store.Get(string(path)) 199 | 200 | if err != nil { 201 | return errors.New(fmt.Sprintf("Lock at %s does not exist in persistent store", path)) 202 | } 203 | 204 | // Delete the lock from Session metadata. 205 | delete(sess.locks, path) 206 | 207 | // Delete the lock from in-memory struct of locks 208 | delete(app.locks, path) 209 | 210 | // Delete the lock from the store. 211 | err = app.store.Delete(string(path)) 212 | return err 213 | } 214 | 215 | // Try to acquire the lock, returning either success (true) or failure (false). 216 | func (sess *Session) TryAcquireLock (path api.FilePath, mode api.LockMode) (bool, error) { 217 | // Validate mode of the lock. 218 | if mode != api.EXCLUSIVE && mode != api.SHARED { 219 | return false, errors.New(fmt.Sprintf("Invalid mode.")) 220 | } 221 | 222 | // Do we already own the lock? Fail with error 223 | /*_, owned := app.locks[path] 224 | if owned { 225 | return false, errors.New(fmt.Sprintf("We already own the lock at %s", path)) 226 | }*/ 227 | 228 | // Check if lock exists in persistent store 229 | _, err := app.store.Get(string(path)) 230 | 231 | if err != nil { 232 | return false, errors.New(fmt.Sprintf("Lock at %s has not been opened", path)) 233 | } 234 | 235 | // Check if lock exists in in-mem struct 236 | lock, exists := app.locks[path] 237 | if !exists { 238 | // Assume that some failure has occurred 239 | // Lazily recover lock struct: add lock to in-memory struct of locks 240 | // TODO: check if this is correct? 241 | app.logger.Printf("Lock Doesn't Exist with Client ID", sess.clientID) 242 | 243 | lock = &Lock{ 244 | path: path, 245 | mode: api.FREE, 246 | owners: make(map[api.ClientID]bool), 247 | content: "", 248 | } 249 | app.locks[path] = lock 250 | sess.locks[path] = lock 251 | } 252 | 253 | // Check the mode of the lock 254 | switch lock.mode { 255 | case api.EXCLUSIVE: 256 | // Should fail: someone probably already owns the lock 257 | if len(lock.owners) == 0 { 258 | // Throw an error if there are no owners but lock.mode is api.EXCLUSIVE: 259 | // this means ReleaseLock was not implemented correctly 260 | return false, errors.New("Lock has EXCLUSIVE mode despite having no owners") 261 | } else if len(lock.owners) > 1 { 262 | return false, errors.New("Lock has EXCLUSIVE mode but has multiple owners") 263 | } else { 264 | // Fail with no error 265 | app.logger.Printf("Failed to acquire lock %s: already held in EXCLUSIVE mode", path) 266 | return false, nil 267 | } 268 | case api.SHARED: 269 | // If our mode is api.SHARED, then succeed; else fail 270 | if mode == api.EXCLUSIVE { 271 | app.logger.Printf("Failed to acquire lock %s in EXCLUSIVE mode: already held in SHARED mode", path) 272 | return false, nil 273 | } else { // mode == api.SHARED 274 | // Update lock owners 275 | lock.owners[sess.clientID] = true 276 | 277 | // Add lock to session lock struct 278 | sess.locks[path] = lock 279 | app.locks[path] = lock 280 | // Return success 281 | //app.logger.Printf("Lock %s acquired successfully with mode SHARED", path) 282 | return true, nil 283 | } 284 | case api.FREE: 285 | // If lock has owners, either TryAcquireLock or ReleaseLock was not implemented correctly 286 | if len(lock.owners) > 0 { 287 | return false, errors.New("Lock has FREE mode but is owned by 1 or more clients") 288 | } 289 | 290 | // Should succeed regardless of mode 291 | // Update lock owners 292 | lock.owners[sess.clientID] = true 293 | 294 | // Update lock mode 295 | lock.mode = mode 296 | 297 | // Add lock to session lock struct 298 | sess.locks[path] = lock 299 | 300 | // Update Lock Mode in the global Map 301 | app.locks[path] = lock 302 | 303 | // Return success 304 | //if mode == api.SHARED { 305 | // app.logger.Printf("Lock %s acquired successfully with mode SHARED", path) 306 | //} else { 307 | // app.logger.Printf("Lock %s acquired successfully with mode EXCLUSIVE", path) 308 | //} 309 | return true, nil 310 | default: 311 | return false, errors.New(fmt.Sprintf("Lock at %s has undefined mode %d", path, lock.mode)) 312 | } 313 | } 314 | 315 | // Release the lock. 316 | func (sess *Session) ReleaseLock (path api.FilePath) (error) { 317 | // Check if lock exists in persistent store 318 | _, err := app.store.Get(string(path)) 319 | 320 | if err != nil { 321 | return errors.New(fmt.Sprintf("Client with id %s: Lock at %s does not exist in persistent store", path, sess.clientID)) 322 | } 323 | 324 | // Grab lock struct from session locks map. 325 | lock, present := app.locks[path] 326 | 327 | // If not in session locks map, throw an error 328 | if !present || lock == nil { 329 | return errors.New(fmt.Sprintf("Lock at %s does not exist in session locks map", path)) 330 | } 331 | 332 | // Check that we are among the owners of the lock. 333 | _, present = lock.owners[sess.clientID] 334 | if !present || !lock.owners[sess.clientID] { 335 | return errors.New(fmt.Sprintf("Client %d does not own lock at path %s", sess.clientID, path)) 336 | } 337 | 338 | // Switch on lock mode. 339 | switch lock.mode { 340 | case api.FREE: 341 | // Throw an error: this means TryAcquire was not implemented correctly 342 | return errors.New(fmt.Sprint("Lock at %s has FREE mode: acquire not implemented correctly Client ID %s", path, sess.clientID)) 343 | case api.EXCLUSIVE: 344 | // Delete from lock owners 345 | delete(lock.owners, sess.clientID) 346 | 347 | // Set lock mode 348 | lock.mode = api.FREE 349 | 350 | // Delete lock from session locks map 351 | delete(sess.locks, path) 352 | app.locks[path] = lock 353 | log.Printf("Release lock at %s\n", path) 354 | // Return without error 355 | return nil 356 | case api.SHARED: 357 | // Delete from lock owners 358 | delete(lock.owners, sess.clientID) 359 | 360 | // Set lock mode if no more owners 361 | if len(lock.owners) == 0 { 362 | lock.mode = api.FREE 363 | } 364 | 365 | // Delete lock from session locks map 366 | delete(sess.locks, path) 367 | app.locks[path] = lock 368 | 369 | // Return without error 370 | return nil 371 | default: 372 | return errors.New(fmt.Sprintf("Lock at %s has undefined mode %d", path, lock.mode)) 373 | } 374 | } 375 | 376 | // Read the Content from a lockfile 377 | func (sess *Session) ReadContent (path api.FilePath) (string,error) { 378 | // Check if file exists in persistent store 379 | content, err := app.store.Get(string(path)) 380 | 381 | if err != nil { 382 | return "",errors.New(fmt.Sprintf("Client with id %s: File at %s does not exist in persistent store", path, sess.clientID)) 383 | } 384 | 385 | // Grab lock struct from session locks map. 386 | lock, present := app.locks[path] 387 | 388 | // If not in session locks map, throw an error 389 | if !present || lock == nil { 390 | return "",errors.New(fmt.Sprintf("Lock at %s does not exist in session locks map", path)) 391 | } 392 | 393 | // Check that we are among the owners of the lock. 394 | _, present = lock.owners[sess.clientID] 395 | if !present || !lock.owners[sess.clientID] { 396 | return "",errors.New(fmt.Sprintf("Client %d does not own lock at path %s", sess.clientID, path)) 397 | } 398 | 399 | return content, nil 400 | } 401 | 402 | // Write the Content to a lockfile 403 | func (sess *Session) WriteContent (path api.FilePath, content string) (error) { 404 | // Check if file exists in persistent store 405 | _, err := app.store.Get(string(path)) 406 | 407 | if err != nil { 408 | return errors.New(fmt.Sprintf("Client with id %s: File at %s does not exist in persistent store", path, sess.clientID)) 409 | } 410 | 411 | // Grab lock struct from session locks map. 412 | lock, present := app.locks[path] 413 | 414 | // If not in session locks map, throw an error 415 | if !present || lock == nil { 416 | return errors.New(fmt.Sprintf("Lock at %s does not exist in session locks map", path)) 417 | } 418 | 419 | // Check that we are among the owners of the lock. 420 | _, present = lock.owners[sess.clientID] 421 | if !present || !lock.owners[sess.clientID] { 422 | return errors.New(fmt.Sprintf("Client %d does not own lock at path %s", sess.clientID, path)) 423 | } 424 | 425 | err = app.store.Set(string(path), content) 426 | if err != nil { 427 | return errors.New(fmt.Sprintf("Write Error")) 428 | } 429 | return nil 430 | } 431 | 432 | -------------------------------------------------------------------------------- /chubby/store/store.go: -------------------------------------------------------------------------------- 1 | // Raft-backed distributed key-value store. 2 | 3 | // Adapted from hraftd (Hashicorp Raft reference example): 4 | // https://github.com/otoolep/hraftd/blob/master/store/store.go 5 | 6 | // Copyright (c) 2015-2018 Philip O'Toole 7 | // 8 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 9 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 10 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 11 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 12 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 13 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 14 | // SOFTWARE. 15 | 16 | package store 17 | 18 | import ( 19 | "encoding/json" 20 | "errors" 21 | "fmt" 22 | "io" 23 | "log" 24 | "net" 25 | "os" 26 | "path/filepath" 27 | "sync" 28 | "time" 29 | 30 | "github.com/hashicorp/raft" 31 | "github.com/hashicorp/raft-boltdb" 32 | ) 33 | 34 | const ( 35 | retainSnapshotCount = 2 36 | raftTimeout = 10 * time.Second 37 | ) 38 | 39 | type command struct { 40 | Op string `json:"op,omitempty"` 41 | Key string `json:"key,omitempty"` 42 | Value string `json:"value,omitempty"` 43 | } 44 | 45 | // Store defines a Raft-backed store. 46 | type Store struct { 47 | RaftDir string // Raft storage directory 48 | RaftBind string // Raft bind address 49 | Raft *raft.Raft // Raft instance 50 | inmem bool // Whether storage is in-memory 51 | 52 | mu sync.Mutex // Lock for synchronizing API operations 53 | m map[string]string // Key-value store for the system 54 | 55 | logger *log.Logger // Logger 56 | } 57 | 58 | // Returns a new store. 59 | func New(raftDir string, raftBind string, inmem bool) *Store { 60 | return &Store{ 61 | RaftDir: raftDir, 62 | RaftBind: raftBind, 63 | m: make(map[string]string), 64 | inmem: inmem, 65 | logger: log.New(os.Stderr, "[store] ", log.LstdFlags), 66 | } 67 | } 68 | 69 | // Open opens the store. If enableSingle is set, and there are no existing peers, 70 | // then this node becomes the first node, and therefore leader, of the cluster. 71 | // localID should be the server identifier for this node. 72 | func (s *Store) Open(enableSingle bool, localID string) error { 73 | 74 | // Setup Raft configuration. 75 | config := raft.DefaultConfig() 76 | config.LocalID = raft.ServerID(localID) 77 | 78 | // Setup Raft communication. 79 | addr, err := net.ResolveTCPAddr("tcp", s.RaftBind) 80 | if err != nil { 81 | return err 82 | } 83 | transport, err := raft.NewTCPTransport(s.RaftBind, addr, 3, 10*time.Second, os.Stderr) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | // Create the snapshot store. This allows the Raft to truncate the log. 89 | snapshots, err := raft.NewFileSnapshotStore(s.RaftDir, retainSnapshotCount, os.Stderr) 90 | if err != nil { 91 | return fmt.Errorf("file snapshot store: %s", err) 92 | } 93 | 94 | // Create the log store and stable store. 95 | var logStore raft.LogStore 96 | var stableStore raft.StableStore 97 | if s.inmem { 98 | logStore = raft.NewInmemStore() 99 | stableStore = raft.NewInmemStore() 100 | } else { 101 | boltDB, err := raftboltdb.NewBoltStore(filepath.Join(s.RaftDir, "raft.db")) 102 | if err != nil { 103 | return fmt.Errorf("new bolt store: %s", err) 104 | } 105 | logStore = boltDB 106 | stableStore = boltDB 107 | } 108 | 109 | // Instantiate the Raft systems. 110 | ra, err := raft.NewRaft(config, (*fsm)(s), logStore, stableStore, snapshots, transport) 111 | if err != nil { 112 | return fmt.Errorf("new raft: %s", err) 113 | } 114 | s.Raft = ra 115 | 116 | if enableSingle { 117 | configuration := raft.Configuration{ 118 | Servers: []raft.Server{ 119 | { 120 | ID: config.LocalID, 121 | Address: transport.LocalAddr(), 122 | }, 123 | }, 124 | } 125 | ra.BootstrapCluster(configuration) 126 | } 127 | 128 | return nil 129 | } 130 | 131 | // Get returns the value for the given key. 132 | func (s *Store) Get(key string) (string, error) { 133 | s.mu.Lock() 134 | defer s.mu.Unlock() 135 | 136 | val, exists := s.m[key] 137 | if !exists { 138 | return "", errors.New(fmt.Sprintf("key %s does not exist", key)) 139 | } 140 | return val, nil 141 | } 142 | 143 | // Set sets the value for the given key. 144 | func (s *Store) Set(key, value string) error { 145 | if s.Raft.State() != raft.Leader { 146 | return fmt.Errorf("not leader") 147 | } 148 | 149 | c := &command{ 150 | Op: "set", 151 | Key: key, 152 | Value: value, 153 | } 154 | b, err := json.Marshal(c) 155 | if err != nil { 156 | return err 157 | } 158 | 159 | f := s.Raft.Apply(b, raftTimeout) 160 | return f.Error() 161 | } 162 | 163 | // Delete deletes the given key. 164 | func (s *Store) Delete(key string) error { 165 | if s.Raft.State() != raft.Leader { 166 | return fmt.Errorf("not leader") 167 | } 168 | 169 | c := &command{ 170 | Op: "delete", 171 | Key: key, 172 | } 173 | b, err := json.Marshal(c) 174 | if err != nil { 175 | return err 176 | } 177 | 178 | f := s.Raft.Apply(b, raftTimeout) 179 | return f.Error() 180 | } 181 | 182 | // Join joins a node, identified by nodeID and located at addr, to this store. 183 | // The node must be ready to respond to Raft communications at that address. 184 | func (s *Store) Join(nodeID, addr string) error { 185 | s.logger.Printf("received join request for remote node %s at %s", nodeID, addr) 186 | 187 | configFuture := s.Raft.GetConfiguration() 188 | if err := configFuture.Error(); err != nil { 189 | s.logger.Printf("failed to get raft configuration: %v", err) 190 | return err 191 | } 192 | 193 | for _, srv := range configFuture.Configuration().Servers { 194 | // If a node already exists with either the joining node's ID or address, 195 | // that node may need to be removed from the config first. 196 | if srv.ID == raft.ServerID(nodeID) || srv.Address == raft.ServerAddress(addr) { 197 | // However if *both* the ID and the address are the same, then nothing -- not even 198 | // a join operation -- is needed. 199 | if srv.Address == raft.ServerAddress(addr) && srv.ID == raft.ServerID(nodeID) { 200 | s.logger.Printf("node %s at %s already member of cluster, ignoring join request", nodeID, addr) 201 | return nil 202 | } 203 | 204 | future := s.Raft.RemoveServer(srv.ID, 0, 0) 205 | if err := future.Error(); err != nil { 206 | return fmt.Errorf("error removing existing node %s at %s: %s", nodeID, addr, err) 207 | } 208 | } 209 | } 210 | 211 | f := s.Raft.AddVoter(raft.ServerID(nodeID), raft.ServerAddress(addr), 0, 0) 212 | if f.Error() != nil { 213 | return f.Error() 214 | } 215 | s.logger.Printf("node %s at %s joined successfully", nodeID, addr) 216 | return nil 217 | } 218 | 219 | // Implement interface for type FSM. 220 | type fsm Store 221 | 222 | // Apply applies a Raft log entry to the key-value store. 223 | func (f *fsm) Apply(l *raft.Log) interface{} { 224 | var c command 225 | if err := json.Unmarshal(l.Data, &c); err != nil { 226 | panic(fmt.Sprintf("failed to unmarshal command: %s", err.Error())) 227 | } 228 | 229 | switch c.Op { 230 | case "set": 231 | return f.applySet(c.Key, c.Value) 232 | case "delete": 233 | return f.applyDelete(c.Key) 234 | default: 235 | panic(fmt.Sprintf("unrecognized command op: %s", c.Op)) 236 | } 237 | } 238 | 239 | // Snapshot returns a snapshot of the key-value store. 240 | func (f *fsm) Snapshot() (raft.FSMSnapshot, error) { 241 | f.mu.Lock() 242 | defer f.mu.Unlock() 243 | 244 | // Clone the map. 245 | o := make(map[string]string) 246 | for k, v := range f.m { 247 | o[k] = v 248 | } 249 | return &fsmSnapshot{store: o}, nil 250 | } 251 | 252 | // Restore stores the key-value store to a previous state. 253 | func (f *fsm) Restore(rc io.ReadCloser) error { 254 | o := make(map[string]string) 255 | if err := json.NewDecoder(rc).Decode(&o); err != nil { 256 | return err 257 | } 258 | 259 | // Set the state from the snapshot, no lock required according to 260 | // Hashicorp docs. 261 | f.m = o 262 | return nil 263 | } 264 | 265 | func (f *fsm) applySet(key, value string) interface{} { 266 | f.mu.Lock() 267 | defer f.mu.Unlock() 268 | f.m[key] = value 269 | return nil 270 | } 271 | 272 | func (f *fsm) applyDelete(key string) interface{} { 273 | f.mu.Lock() 274 | defer f.mu.Unlock() 275 | delete(f.m, key) 276 | return nil 277 | } 278 | 279 | // Implement interface for type FSMSnapshot. 280 | type fsmSnapshot struct { 281 | store map[string]string 282 | } 283 | 284 | func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error { 285 | err := func() error { 286 | // Encode data. 287 | b, err := json.Marshal(f.store) 288 | if err != nil { 289 | return err 290 | } 291 | 292 | // Write data to sink. 293 | if _, err := sink.Write(b); err != nil { 294 | return err 295 | } 296 | 297 | // Close the sink. 298 | return sink.Close() 299 | }() 300 | 301 | if err != nil { 302 | sink.Cancel() 303 | } 304 | 305 | return err 306 | } 307 | 308 | func (f *fsmSnapshot) Release() {} 309 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | node1: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile.node1 7 | networks: 8 | static-network: 9 | ipv4_address: 172.20.128.1 10 | node2: 11 | build: 12 | context: . 13 | dockerfile: Dockerfile.node2 14 | restart: on-failure 15 | depends_on: 16 | - "node1" 17 | networks: 18 | static-network: 19 | ipv4_address: 172.20.128.2 20 | node3: 21 | build: 22 | context: . 23 | dockerfile: Dockerfile.node3 24 | restart: on-failure 25 | depends_on: 26 | - "node1" 27 | networks: 28 | static-network: 29 | ipv4_address: 172.20.128.3 30 | node4: 31 | build: 32 | context: . 33 | dockerfile: Dockerfile.node4 34 | restart: on-failure 35 | depends_on: 36 | - "node1" 37 | networks: 38 | static-network: 39 | ipv4_address: 172.20.128.4 40 | node5: 41 | build: 42 | context: . 43 | dockerfile: Dockerfile.node5 44 | restart: on-failure 45 | depends_on: 46 | - "node1" 47 | networks: 48 | static-network: 49 | ipv4_address: 172.20.128.5 50 | 51 | ################################################################## 52 | # Client docker containers: uncomment to build with server nodes # 53 | ################################################################## 54 | # client1: 55 | # build: 56 | # context: . 57 | # dockerfile: Dockerfile.leader_client 58 | # restart: on-failure 59 | # depends_on: 60 | # - "node1" 61 | # - "node2" 62 | # - "node3" 63 | # - "node4" 64 | # - "node5" 65 | # networks: 66 | # static-network: 67 | # ipv4_address: 172.20.192.1 68 | # client2: 69 | # build: 70 | # context: . 71 | # dockerfile: Dockerfile.acquire_leader 72 | # restart: on-failure 73 | # depends_on: 74 | # - "client1" 75 | # networks: 76 | # static-network: 77 | # ipv4_address: 172.20.193.1 78 | 79 | networks: 80 | static-network: 81 | ipam: 82 | config: 83 | - subnet: 172.20.0.0/16 84 | --------------------------------------------------------------------------------