├── LICENSE ├── MultiPaxos.tla ├── ParallelRaftCE.tla ├── ParallelRaftSE.tla ├── README.md ├── doc └── 2020.8-jos.pdf └── specification ├── MultiPaxos.pdf ├── MultiPaxos.tla ├── ParallelRaftCE.pdf ├── ParallelRaftCE.tla ├── ParallelRaftSE.pdf └── ParallelRaftSE.tla /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Xiaosong Gu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MultiPaxos.tla: -------------------------------------------------------------------------------- 1 | ----------------------------- MODULE MultiPaxos ----------------------------- 2 | 3 | EXTENDS Integers, FiniteSets 4 | 5 | CONSTANTS Acceptors,Nil,Value 6 | 7 | Ballots == Nat 8 | Instances == {0,1,2,3,4,5,6} 9 | Quorums == {Q \in SUBSET Acceptors : Cardinality(Q) > Cardinality(Acceptors) \div 2} 10 | Max(s) == CHOOSE x \in s : \forall y \in s : x \geq y 11 | 12 | ------------------------------------------------------------------------------- 13 | VARIABLES ballot, 14 | vote, 15 | leaderVote, 16 | 1amsgs, 17 | 1bmsgs, 18 | 2amsgs 19 | 20 | -------------------------------------------------------------------------------- 21 | Init == 22 | /\ ballot = [a \in Acceptors |-> 0] 23 | /\ vote = [a \in Acceptors |-> 24 | [i \in Instances |-> 25 | [b \in Ballots |-> Nil]]] 26 | /\ 1amsgs = {} 27 | /\ 1bmsgs = {} 28 | /\ 2amsgs = {} 29 | /\ leaderVote = [b \in Ballots |-> [i \in Instances |-> <<-1,Nil>>]] 30 | 31 | ---------------------------------------------------------------------------------- 32 | 33 | allEntries == {<> >> : i \in Instances, b \in Ballots \cup {-1}, v \in Value \cup {Nil}} 34 | 35 | TypeInv == 36 | /\ ballot \in [Acceptors -> {-1} \cup Ballots] 37 | /\ leaderVote \in [Ballots -> [Instances -> ({-1} \cup Ballots) \times ({Nil} \cup Value)]] 38 | /\ vote \in [Acceptors -> [Instances -> [Ballots -> ({Nil} \cup Value )]]] 39 | /\ 1amsgs \subseteq {<> : b \in Ballots} 40 | \* /\ 1bmsgs \subseteq {<> : b \in Ballots, a \in Acceptors, e \in SUBSET allEntries} 41 | /\ 2amsgs \subseteq {<>>> : i \in Instances, b \in Ballots, bb \in Ballots,v \in Value \cup {Nil}} 42 | /\ leaderVote \in [Ballots -> [Instances -> ((Ballots\cup {-1}) \times ({Nil} \cup Value))]] 43 | 44 | ------------------------------------------------------------------------------------ 45 | IncreaseBallot(a,b) == 46 | /\ ballot[a] 49 | 50 | Phase1a(b) == 51 | /\ 1amsgs' = 1amsgs \cup {<>} 52 | /\ UNCHANGED <> 53 | 54 | MaxAcceptorVote(a, i) == 55 | LET maxBallot == Max({b \in Ballots : vote[a][i][b] # Nil} \cup {-1}) 56 | v == IF maxBallot > -1 THEN vote[a][i][maxBallot] ELSE Nil 57 | IN <> 58 | 59 | Phase1b(a, b) == 60 | /\ ballot[a] \in 1amsgs 62 | /\ ballot' = [ballot EXCEPT ![a] = b] 63 | /\ 1bmsgs' = 1bmsgs \cup 64 | {<> : i \in Instances}, a>>} 65 | /\ UNCHANGED <> 66 | 67 | 1bMsgs(b, Q) == 68 | {m \in 1bmsgs : m[3] \in Q /\ m[1] = b} 69 | 70 | 71 | 72 | MaxVote(b, i, Q) == 73 | LET entries == UNION{m[2] : m \in 1bMsgs(b,Q)} 74 | ientries == {e \in entries : e[1] = i} 75 | maxBal == Max({e[2][1] : e \in ientries}) 76 | IN CHOOSE v \in Value \cup {Nil} : \E e \in ientries : 77 | /\ e[2][1] = maxBal /\ e[2][2] = v 78 | 79 | 80 | 81 | lastInstance(b,Q) == LET entries == UNION{m[2] : m \in 1bMsgs(b,Q)} 82 | valid == {e \in entries : e[2][1] /= -1} 83 | IN 84 | IF valid = {} THEN -1 ELSE Max({e[1] : e \in valid}) 85 | 86 | 87 | 88 | Merge(b) == /\ \E Q \in Quorums : 89 | /\ \A a \in Q : \E m \in 1bMsgs(b,Q) : m[3] = a 90 | /\ \E v \in Value : leaderVote' = [leaderVote EXCEPT ![b] = [i \in Instances |-> 91 | IF (i \in 0..lastInstance(b,Q) /\ leaderVote[b][i][1] = -1) 92 | \* THEN <> 93 | THEN IF MaxVote(b,i,Q) = Nil THEN <> 94 | ELSE <> 95 | ELSE leaderVote[b][i]]] 96 | /\ UNCHANGED <> 97 | 98 | 99 | Propose(b,i) == /\ leaderVote[b][i][1] = -1 100 | /\ \E Q \in Quorums : 101 | /\ \A a \in Q : \E m \in 1bMsgs(b,Q) : m[3] = a 102 | /\ \E v \in Value : leaderVote'=[leaderVote EXCEPT ![b][i] = IF MaxVote(b,i,Q) = Nil 103 | THEN <> 104 | ELSE <>] 105 | /\ UNCHANGED <> 106 | 107 | Phase2a(b, i) == 108 | /\ leaderVote[b][i][1] = b 109 | /\ 2amsgs' = 2amsgs \cup {<>} 110 | /\ UNCHANGED <> 111 | 112 | Vote(a, b, i) == 113 | /\ ballot[a] \leq b 114 | /\ ballot' = [ballot EXCEPT ![a] = b] 115 | /\ \E m \in 2amsgs : 116 | /\ m[2] = i /\ m[1] = b 117 | /\ vote' = [vote EXCEPT ![a][i][b] = m[3][2]] 118 | /\ UNCHANGED <> 119 | 120 | Next == 121 | \/ \E a \in Acceptors, b \in Ballots : IncreaseBallot(a,b) 122 | \/ \E b \in Ballots : Phase1a(b) 123 | \/ \E a \in Acceptors, b \in Ballots : Phase1b(a,b) 124 | \/ \E b \in Ballots : Merge(b) 125 | \/ \E b \in Ballots, i \in Instances : Propose(b,i) 126 | \/ \E b \in Ballots, i \in Instances : Phase2a(b,i) 127 | \/ \E a \in Acceptors, b \in Ballots, i \in Instances : Vote(a, b, i) 128 | 129 | Spec == Init /\ [][Next]_<> 130 | ---------------------------------------------------------------------------------------- 131 | Conservative(i,b) == 132 | \A a1,a2 \in Acceptors : 133 | LET v1 == vote[a1][i][b] 134 | v2 == vote[a2][i][b] 135 | IN (v1 # Nil /\ v2 # Nil) => v1 = v2 136 | 137 | ConservativeVoteArray == 138 | \A i \in Instances : \A b \in Ballots : 139 | Conservative(i,b) 140 | 141 | WellFormed == \A a \in Acceptors : \A i \in Instances : \A b \in Ballots : 142 | b > ballot[a] => vote[a][i][b] = Nil 143 | 144 | VotedFor(a,i,b,v) == vote[a][i][b] = v 145 | 146 | ChosenAt(i,b,v) == 147 | \E Q \in Quorums : \A a \in Q : VotedFor(a,i,b,v) 148 | 149 | Chosen(i,v) == 150 | \E b \in Ballots : ChosenAt(i,b,v) 151 | 152 | Choosable(v, i, b) == 153 | \E Q \in Quorums : \A a \in Q : ballot[a] > b => vote[a][i][b] = v 154 | 155 | SafeAt(v, i, b) == 156 | \A b2 \in Ballots : \A v2 \in Value : 157 | (b2 v = v2 159 | 160 | SafeInstanceVoteArray(i) == \A b \in Ballots : \A a \in Acceptors : 161 | LET v == vote[a][i][b] 162 | IN v # Nil => SafeAt(v, i, b) 163 | 164 | SafeVoteArray == \A i \in Instances : SafeInstanceVoteArray(i) 165 | 166 | Inv == TypeInv /\ WellFormed /\ SafeVoteArray /\ ConservativeVoteArray 167 | 168 | Correctness == 169 | \A i \in Instances : \A v1,v2 \in Value : 170 | Chosen(i, v1) /\ Chosen(i, v2) => v1 = v2 171 | 172 | ============================================================================= 173 | \* Modification History 174 | \* Last modified Wed Sep 09 15:26:35 CST 2020 by 15150 175 | 176 | -------------------------------------------------------------------------------- /ParallelRaftCE.tla: -------------------------------------------------------------------------------- 1 | --------------------------- MODULE ParallelRaftCE --------------------------- 2 | 3 | EXTENDS Integers,FiniteSets,Sequences,TLC,Naturals 4 | 5 | CONSTANTS Server,Follower,Candidate,Leader,LeaderCandidate,Nil,Value 6 | 7 | CONSTANTS RequestVoteRequest,RequestVoteResponse, 8 | RequestCommitRequest,RequestCommitResponse, 9 | RequestSyncRequest,RequestSyncResponse, 10 | UpdateSyncRequest,UpdateSyncResponse 11 | 12 | VARIABLE messages, 13 | currentTerm, 14 | currentState, 15 | votedFor, 16 | sync, 17 | endPoint 18 | 19 | serverVars == <> 20 | 21 | VARIABLE log 22 | logVars == <> 23 | 24 | 25 | 26 | VARIABLE syncTrack 27 | leaderVars == <> 28 | 29 | 30 | 31 | VARIABLE halfElections 32 | VARIABLE elections 33 | electionVars == <> 34 | 35 | 36 | VARIABLE allLogs 37 | VARIABLE allEntries 38 | VARIABLE allSynced 39 | 40 | 41 | 42 | vars == <> 43 | 44 | 45 | Quorums == {i \in SUBSET(Server) : Cardinality(i)*2 > Cardinality(Server)} 46 | 47 | Send(m) == messages' = messages \cup {m} 48 | 49 | 50 | Index == Nat 51 | Term == Nat 52 | 53 | Min(s) == IF s = {} THEN -1 ELSE CHOOSE i \in s : \A j \in s : j \geq i 54 | Max(s) == IF s = {} THEN -1 ELSE CHOOSE i \in s : \A j \in s : i \geq j 55 | 56 | 57 | 58 | InitServerVars == LET k == CHOOSE x \in Server : x \in Server 59 | IN 60 | /\ currentTerm = [i \in Server |-> 0] 61 | /\ sync = [i \in Server |-> 0] 62 | /\ currentState = [i \in Server |-> Follower] 63 | /\ endPoint = [i \in Server |-> [n \in Term |-> <<-1,-1>>]] 64 | /\ votedFor = [i \in Server |-> Nil] 65 | 66 | 67 | InitLeaderVars == /\ syncTrack = [i \in Server |-> [j \in Server |-> 0]] 68 | 69 | 70 | InitHistoryVars == /\ halfElections = {} 71 | /\ elections = {} 72 | /\ allLogs = {} 73 | /\ allEntries = {} 74 | /\ allSynced = {} 75 | 76 | 77 | InitLogVars == /\ log = [i \in Server |-> [n \in Index |-> [term |-> -1,date |-> -1, 78 | value |-> Nil, committed |-> FALSE]]] 79 | 80 | 81 | Init == /\ messages = {} 82 | /\ InitServerVars 83 | /\ InitLeaderVars 84 | /\ InitLogVars 85 | /\ InitHistoryVars 86 | 87 | 88 | Entries == [term : Term \cup {-1}, date : Term \cup {-1}, value : Value \cup {Nil} , committed : {TRUE,FALSE}] 89 | 90 | TypeSafety == /\ allLogs \in SUBSET (SUBSET allEntries) 91 | /\ currentTerm \in [Server -> Nat] 92 | /\ currentState \in [Server -> {Follower,Leader,LeaderCandidate,Candidate}] 93 | /\ votedFor \in [Server -> Server \cup {Nil}] 94 | /\ sync \in [Server -> Nat\cup{-1}] 95 | /\ endPoint \in [Server -> [Term -> [date : Term \cup {-1}, index : Index \cup {-1}]]] 96 | /\ endPoint \in [Server -> [Term -> ((Term \cup {-1}) \times (Index \cup {-1}))]] 97 | /\ log \in [Server -> [Index -> [term : Index \cup {-1}, date : Term \cup {-1}, 98 | value : Value \cup {Nil}, committed : {TRUE,FALSE}]]] 99 | /\ syncTrack \in [Server -> [Server -> Nat]] 100 | /\ halfElections \subseteq [eterm : Nat, eleaderCandidate : Server, esync : Nat, 101 | evotes : Quorums, elog : [Index -> Entries]] 102 | /\ elections \subseteq [eterm : Term, esync : Term, eleader : Server, evotes : Quorums, evoterLog : SUBSET [Index -> Entries],elog : [Index -> Entries]] 103 | 104 | 105 | 106 | 107 | logTail(s) == Max({i \in Index : s[i].term /= -1}) 108 | 109 | Restart(i) == 110 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 111 | /\ syncTrack' = [syncTrack EXCEPT ![i] = [j \in Server |-> 0]] 112 | /\ UNCHANGED <> 114 | 115 | Timeout(i) == 116 | /\ currentState[i] \in {Follower,Candidate} 117 | /\ currentState' = [currentState EXCEPT ![i] = Candidate] 118 | /\ currentTerm' = [currentTerm EXCEPT ![i] = currentTerm[i]+1] 119 | /\ currentTerm[i] + 1 \in Term 120 | /\ votedFor' = [votedFor EXCEPT ![i] = Nil] 121 | /\ UNCHANGED <> 123 | 124 | UpdateTerm(i) == 125 | /\ \E m \in messages : 126 | /\ m.mterm > currentTerm[i] 127 | /\ \/ m.mdest = i 128 | \/ m.mdest = Nil 129 | /\ currentTerm' = [currentTerm EXCEPT ![i] = m.mterm] 130 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 131 | /\ votedFor' = [votedFor EXCEPT ![i] = Nil] 132 | /\ UNCHANGED <> 133 | 134 | 135 | RequestVote(i) == 136 | /\ currentState[i] = Candidate 137 | /\ Send([ mtype |-> RequestVoteRequest, 138 | mterm |-> currentTerm[i], 139 | msync |-> sync[i], 140 | msource |-> i, 141 | mdest |-> Nil]) 142 | /\ UNCHANGED<> 143 | 144 | \* i: recipient 145 | HandleRequestVoteRequest(i) == 146 | /\ \E m \in messages : 147 | LET j == m.msource 148 | syncOK == /\ m.msync \geq sync[i] 149 | grant == /\ syncOK 150 | /\ votedFor[i] \in {Nil,j} 151 | /\ currentTerm[i] = m.mterm 152 | IN 153 | /\ m.mterm \leq currentTerm[i] 154 | /\ m.mtype = RequestVoteRequest 155 | /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] 156 | \/ \neg grant /\ UNCHANGED votedFor 157 | /\ Send([ mtype |-> RequestVoteResponse, 158 | mterm |-> currentTerm[i], 159 | mvoteGranted |-> grant, 160 | mlog |-> LET C == {n \in Index : log[i][n].term = sync[i]} 161 | IN {<> : n \in C}, 162 | mend |-> endPoint[i][m.msync], 163 | msource |-> i, 164 | mdest |-> j]) 165 | /\ UNCHANGED <> 167 | 168 | 169 | Merge(entries,term,date) == IF entries = {} THEN [term |-> term, 170 | date |-> date, 171 | value |-> Nil, 172 | committed |-> FALSE] 173 | ELSE 174 | LET 175 | committed == {e \in entries : e.committed = TRUE} 176 | chosen == 177 | CASE committed = {} -> CHOOSE x \in entries : 178 | \A y \in entries : x.date \geq y.date 179 | [] committed /= {} -> CHOOSE x \in committed : TRUE 180 | IN 181 | [ term |-> chosen.term, 182 | date |-> date, 183 | value |-> chosen.value, 184 | committed |-> chosen.committed] 185 | 186 | BecomeLeaderCandidate(i) == 187 | /\ currentState[i] = Candidate 188 | /\ \E P,Q \in Quorums : 189 | LET voteResponded == {m \in messages : /\ m.mtype = RequestVoteResponse 190 | /\ m.mdest = i 191 | /\ m.msource \in P 192 | /\ m.mterm = currentTerm[i]} 193 | voteGranted == {m \in voteResponded : /\ m.mvoteGranted = TRUE 194 | /\ m.msource \in Q} 195 | allLog == UNION {m.mlog : m \in voteResponded} 196 | end == LET allPoint == {m.mend : m \in voteResponded} 197 | e == CHOOSE e1 \in allPoint : (\A e2 \in allPoint : e1[1] \geq e2[1]) 198 | IN IF e[1] = -1 THEN Max({e1[1] : e1 \in allLog}) 199 | ELSE e[2] 200 | toRecover == {n \in 0..end : log[i][n].committed = FALSE} 201 | toSync == {<> 202 | : n \in toRecover} 203 | IN 204 | /\ \A q \in Q : \E m \in voteGranted : m.msource = q 205 | /\ log' = [log EXCEPT ![i] = IF end = -1 THEN [n \in Index |-> IF log[i][n].term = sync[i] THEN 206 | [term |-> -1, 207 | date |-> -1, 208 | value |-> Nil, 209 | committed |-> FALSE] 210 | ELSE log[i][n]] 211 | ELSE [n \in Index |-> IF n \in toRecover THEN 212 | (CHOOSE e \in toSync : e[1] = n)[2] 213 | ELSE IF (n > end) THEN 214 | [term |-> -1, 215 | date |-> -1, 216 | value |-> Nil, 217 | committed |-> FALSE] 218 | ELSE log[i][n]]] 219 | /\ endPoint' = [endPoint EXCEPT ![i][sync[i]] = <>] 220 | /\ halfElections' = halfElections \union {[ eterm |-> currentTerm[i], 221 | eleaderCandidate |-> i, 222 | esync |-> sync[i], 223 | evotes |-> Q, 224 | elog |-> log[i]]} 225 | /\ currentState' = [currentState EXCEPT ![i] = LeaderCandidate] 226 | /\ syncTrack' = [syncTrack EXCEPT ![i] = [j \in Server |-> sync[i]]] 227 | /\ UNCHANGED<> 228 | 229 | 230 | RequestSync(i) == 231 | /\ currentState[i] \in {LeaderCandidate,Leader} 232 | /\ \E s \in 0..sync[i] : 233 | LET start == Min({n \in Index : log[i][n].term = s}) 234 | end == Max({n \in Index : log[i][n].term = s}) 235 | IN 236 | /\ Send([ mtype |-> RequestSyncRequest, 237 | mterm |-> currentTerm[i], 238 | msync |-> s, 239 | mstart |-> start, 240 | mend |-> end, 241 | mentries |-> IF start = -1 THEN Nil ELSE [n \in start..end |-> log[i][n]], 242 | msource |-> i, 243 | mdest |-> Nil]) 244 | /\ UNCHANGED <> 245 | 246 | 247 | HandleRequestSyncRequest(i) == 248 | /\ \E m \in messages : 249 | LET j == m.msource 250 | grant == /\ m.mterm = currentTerm[i] 251 | /\ m.msync = sync[i] 252 | IN 253 | /\ m.mtype = RequestSyncRequest 254 | /\ m.mterm \leq currentTerm[i] 255 | /\ j /= i 256 | /\ \/ /\ grant 257 | /\ log' = [log EXCEPT ![i] = IF m.mstart = -1 THEN 258 | [n \in Index |-> IF log[i][n].term = sync[i] THEN 259 | [term |-> -1, 260 | date |-> -1, 261 | value |-> Nil, 262 | committed |-> FALSE] 263 | ELSE 264 | log[i][n]] 265 | ELSE 266 | [n \in Index |-> IF n < m.mstart THEN log[i][n] 267 | ELSE IF n \in m.mstart..m.mend 268 | THEN m.mentries[n] 269 | ELSE [term |-> -1, 270 | date |-> -1, 271 | value |-> Nil, 272 | committed |-> FALSE]]] 273 | /\ endPoint' = [endPoint EXCEPT ![i][sync[i]] = <>] 274 | \/ /\ \neg grant 275 | /\ UNCHANGED <> 276 | /\ Send([mtype |-> RequestSyncResponse, 277 | mterm |-> currentTerm[i], 278 | msyncGranted |-> grant, 279 | msync |-> sync[i], 280 | mstart |-> m.mstart, 281 | mend |-> m.mend, 282 | msource |-> i, 283 | mdest |-> j]) 284 | /\ UNCHANGED <> 285 | 286 | HandleRequestSyncResponse(i) == 287 | /\ \E m \in messages : 288 | LET j == m.msource IN 289 | /\ m.mtype = RequestSyncResponse 290 | /\ m.mdest = i 291 | /\ currentTerm[i] = m.mterm 292 | /\ currentState[i] \in {Leader,LeaderCandidate} 293 | /\ syncTrack' = [syncTrack EXCEPT ![i][j] = m.msync] 294 | /\ \/ /\ m.msyncGranted 295 | /\ m.msync < sync[i] 296 | /\ Send( [ mtype |-> UpdateSyncRequest, 297 | mterm |-> currentTerm[i], 298 | msync |-> Min({sync[i]} \union {k \in Nat : k > m.msync /\ 299 | Cardinality({n \in Index : log[i][n].term = k})>0}), 300 | msource |-> i, 301 | mdest |-> {j}]) 302 | \/ /\ \neg m.msyncGranted 303 | /\ UNCHANGED messages 304 | /\ UNCHANGED <> 305 | 306 | UpdateSync(i) == 307 | /\ currentState[i] = LeaderCandidate 308 | /\ \E Q \in Quorums : 309 | LET syncUpdated == {m \in messages : /\ m.mtype = RequestSyncResponse 310 | /\ m.mterm = currentTerm[i] 311 | /\ m.msyncGranted = TRUE 312 | /\ m.msync = sync[i] 313 | /\ m.msource \in Q 314 | /\ m.mdest = i} 315 | IN 316 | /\ \A q \in Q : (\E m \in syncUpdated : m.msource = q) \/ q = i 317 | /\ allSynced' = LET indexes == {n \in Index : log[i][n].term = sync[i]} 318 | entries == {< log[i][n].term, 319 | date |-> log[i][n].date, 320 | value |-> log[i][n].value, 321 | committed |-> TRUE]>> : n \in indexes} 322 | IN allSynced \cup {<>} 323 | /\ Send( [ mtype |-> UpdateSyncRequest, 324 | mterm |-> currentTerm[i], 325 | msync |-> currentTerm[i], 326 | msource |-> i, 327 | mdest |-> Q]) 328 | /\ UNCHANGED <> 329 | 330 | HandleUpdateSyncRequest(i) == 331 | \E m \in messages : 332 | LET grant == /\ currentTerm[i] = m.mterm 333 | /\ m.msync > sync[i] 334 | j == m.msource 335 | IN 336 | /\ m.mtype = UpdateSyncRequest 337 | /\ i \in m.mdest 338 | /\ m.mterm \leq currentTerm[i] 339 | /\ \/ /\ grant 340 | /\ sync' = [sync EXCEPT ![i] = m.msync] 341 | /\ log' = [log EXCEPT ![i] = [n \in Index |-> 342 | IF log[i][n].term = sync[i] THEN 343 | [term |-> log[i][n].term, 344 | date |-> log[i][n].date, 345 | value |-> log[i][n].value, 346 | committed |-> TRUE] 347 | ELSE log[i][n]]] 348 | \/ /\ \neg grant 349 | /\ UNCHANGED <> 350 | /\ Send([ mtype |-> UpdateSyncResponse, 351 | mterm |-> currentTerm[i], 352 | mupdateSyncGranted |-> grant, 353 | msync |-> sync'[i], 354 | msource |-> i, 355 | mdest |-> j]) 356 | /\ UNCHANGED <> 357 | 358 | HandleUpdateSyncResponse(i) == 359 | /\ \E m \in messages : 360 | LET j == m.msource IN 361 | /\ m.mtype = UpdateSyncResponse 362 | /\ m.mdest = i 363 | /\ currentTerm[i] = m.mterm 364 | /\ currentState[i] \in {Leader,LeaderCandidate} 365 | /\ \/ /\ m.mupdateSyncGranted 366 | /\ syncTrack' = [syncTrack EXCEPT ![i][j] = m.msync] 367 | \/ /\ \neg m.mupdateSyncGranted 368 | /\ UNCHANGED syncTrack 369 | /\ UNCHANGED <> 370 | 371 | BecomeLeader(i) == 372 | /\ currentState[i] = LeaderCandidate 373 | /\ \E Q \in Quorums : \A q \in Q : (q = i \/ syncTrack[i][q] = currentTerm[i]) 374 | /\ elections' = elections \union {[ eterm |-> currentTerm[i], 375 | esync |-> sync[i], 376 | eleader |-> i, 377 | evotes |-> Q, 378 | evoterLog |-> {log[k] : k \in Q}, 379 | elog |-> log[i]]} 380 | /\ sync' = [sync EXCEPT ![i] = currentTerm[i]] 381 | /\ currentState' = [currentState EXCEPT ![i] = Leader] 382 | /\ log' = [log EXCEPT ![i] = [n \in Index |-> 383 | IF log[i][n].term = sync[i] THEN 384 | [term |-> log[i][n].term, 385 | date |-> log[i][n].date, 386 | value |-> log[i][n].value, 387 | committed |-> TRUE] 388 | ELSE log[i][n]]] 389 | /\ UNCHANGED <> 390 | 391 | ClientRequest(i,v) == 392 | LET nextIndex == logTail(log[i]) + 1 393 | entry == [term |-> currentTerm[i], 394 | date |-> currentTerm[i], 395 | value |-> v, 396 | committed |-> FALSE] 397 | IN 398 | /\ currentState[i] = Leader 399 | /\ nextIndex \in Nat 400 | /\ log' = [log EXCEPT ![i][nextIndex] = entry] 401 | /\ UNCHANGED <> 402 | 403 | CommitEntry(i,n) == 404 | /\ \E Q \in Quorums: 405 | LET succ == {m \in messages : /\ m.type = RequestSyncResponse 406 | /\ m.msyncGranted = TRUE 407 | /\ m.mdest = i 408 | /\ m.mterm = currentTerm[i] 409 | /\ m.msource \in Q 410 | /\ n \in m.mstart..m.mend} 411 | IN /\ \A q \in Q : \E m \in succ : (m.msource = q \/ q = i) 412 | /\ log' = [log EXCEPT ![i][n].committed = TRUE] 413 | /\ currentState[i] = Leader 414 | /\ UNCHANGED <> 415 | 416 | 417 | Next == /\ 418 | \/ \E i \in Server: Restart(i) 419 | \/ \E i \in Server: Timeout(i) 420 | \/ \E i \in Server: UpdateTerm(i) 421 | \/ \E i \in Server: RequestVote(i) 422 | \/ \E i \in Server : HandleRequestVoteRequest(i) 423 | \/ \E i \in Server : BecomeLeaderCandidate(i) 424 | \/ \E i \in Server : BecomeLeader(i) 425 | \/ \E i \in Server, v \in Value : ClientRequest(i,v) 426 | \/ \E i,j \in Server : RequestSync(i) 427 | \/ \E i \in Server : HandleRequestSyncRequest(i) 428 | \/ \E i \in Server : HandleRequestSyncResponse(i) 429 | \/ \E i,j \in Server : UpdateSync(i) 430 | \/ \E i \in Server : HandleUpdateSyncRequest(i) 431 | \/ \E i \in Server : HandleUpdateSyncResponse(i) 432 | 433 | /\ allLogs' = allLogs \union {log[i] : i \in Server} 434 | /\ LET entries(i) == {<> : n \in Index} 435 | IN 436 | allEntries' = allEntries \union UNION {entries(i) : i \in Server} 437 | 438 | 439 | AllEntries(i) == {<> : n \in Index} 440 | 441 | Lemma1 == \A i \in Server : sync[i] \leq currentTerm[i] 442 | Lemma2 == \A i \in Server : currentState[i]=Leader => sync[i] = currentTerm[i] 443 | Lemma3 == \A e,f \in halfElections : e.eterm = f.eterm => e.eleaderCandidate = f.eleaderCandidate 444 | Lemma4 == \A e \in elections : \E f \in halfElections : e.eterm = f.eterm 445 | /\ e.eleader = f.eleaderCandidate 446 | Lemma5 == \A e,f \in elections : e.eterm = f.eterm => e.eleader = f.eleader 447 | Lemma6 == \forall i \in Server : currentState[i]=Leader => currentTerm[i] = sync[i] 448 | Lemma7 == \A e \in halfElections : e.esync < e.eterm 449 | Lemma8 == \A i,j \in Server, n \in Index : log[i][n].term = log[j][n].term => 450 | log[i][n].value = log[j][n].value 451 | Lemma9 == \A s1,s2 \in allSynced : s1[1]=s2[1] => s1=s2 452 | Lemma10 == \A e1,e2 \in elections : e1.eterm < e2.eterm => 453 | \E s \in allSynced : s[1] = e1.term 454 | Lemma11 == LET indexes(i,t) == {n \in Index : log[i][n].term=t} 455 | entries(i,t) == {<> : n \in indexes(i,t)} IN 456 | \A i \in Server : \A t \in Term : 457 | t < sync[i] /\ (\E e \in elections : e.eterm = t) => \E s \in allSynced : s[1] = t /\ 458 | entries(i,t) = s[3] 459 | Lemma12 == \A i \in Server : \A e \in AllEntries(i) : e[2].term \leq sync[i] 460 | Lemma13 == \A e \in halfElections : \A f \in elections : f.eterm \leq e.esync \/ f.eterm \geq e.eterm 461 | syncCompleteness == \A i,j \in Server : 462 | {e \in AllEntries(i) : e[2].term \geq 0 /\ e[2].term < Min({sync[i],sync[j]})} = 463 | {e \in AllEntries(j) : e[2].term \geq 0 /\ e[2].term < Min({sync[i],sync[j]})} 464 | 465 | 466 | Spec == Init/\ [][Next]_vars 467 | 468 | 469 | 470 | ============================================================================= 471 | \* Modification History 472 | \* Last modified Wed Sep 09 15:26:50 CST 2020 by 15150 473 | 474 | -------------------------------------------------------------------------------- /ParallelRaftSE.tla: -------------------------------------------------------------------------------- 1 | --------------------------- MODULE ParallelRaftSE --------------------------- 2 | 3 | EXTENDS Integers,FiniteSets,Sequences,TLC 4 | CONSTANTS Server,Follower,Candidate,Leader,LeaderCandidate,Nil,Value 5 | 6 | Quorums == {i \in SUBSET(Server) : Cardinality(i)*2 > Cardinality(Server)} 7 | Index == {0,1,2,3,4,5,6} 8 | Term == Nat 9 | 10 | ------------------------------------------------------------------------------ 11 | VARIABLE r1amsgs, 12 | r1bmsgs, 13 | r2amsgs, 14 | r2bmsgs, 15 | r3amsgs, 16 | negMsgs, 17 | currentTerm, 18 | currentState, 19 | vote, 20 | leaderLog, 21 | log 22 | 23 | serverVars == <> 24 | vars == <> 25 | 26 | --------------------------------------------------------------------------------- 27 | Max(s) == CHOOSE i \in s : \A j \in s : i \geq j 28 | 29 | lastIndex(i) == IF {b \in Index : log[i][b][1] /= -1} ={} 30 | THEN -1 31 | ELSE Max({b \in Index : log[i][b][1] /= -1}) 32 | 33 | allEntries == {<> : t \in Term \cup {-1}, v \in Value \cup {Nil}, b \in {TRUE,FALSE}} 34 | logEntries == {<> : i \in Index, e \in allEntries} 35 | 36 | TypeInv == /\ currentTerm \in [Server -> Nat] 37 | /\ currentState \in [Server -> {Follower,Leader,LeaderCandidate,Candidate}] 38 | /\ log \in [Server -> [Index -> (Term \cup {-1}) \times (Value \cup {Nil}) \times BOOLEAN]] 39 | /\ r1amsgs \subseteq {<> : t \in Term, i \in Server} 40 | /\ r1bmsgs \subseteq {<> : t \in Term, e \in SUBSET logEntries, i \in Server, j \in Server} 41 | /\ r2amsgs \subseteq {<> : t \in Term, n \in Index, e \in allEntries, i \in Server} 42 | /\ r2bmsgs \subseteq {<> : t \in Term, n \in Index, i \in Server, j \in Server} 43 | /\ r3amsgs \subseteq {<> : t \in Term, n \in Index, i \in Server} 44 | /\ negMsgs \subseteq {<> : t \in Term, i \in Server} 45 | /\ log \in [Server -> [Index -> allEntries]] 46 | /\ leaderLog \in [Term -> [Index -> allEntries]] 47 | /\ vote \in [Server -> [Index -> [Term -> Value \cup {Nil}]]] 48 | 49 | --------------------------------------------------------------------------------------- 50 | InitServerVars == LET k == CHOOSE x \in Server : x \in Server 51 | IN 52 | /\ currentTerm = [i \in Server |-> 0] 53 | /\ currentState = [i \in Server |-> Follower] 54 | 55 | InitLogVars == /\ log = [i \in Server |-> [j \in Index |-> <<-1,Nil,FALSE>>]] 56 | 57 | Init == /\ r1amsgs = {} 58 | /\ r1bmsgs = {} 59 | /\ r2amsgs = {} 60 | /\ r2bmsgs = {} 61 | /\ r3amsgs = {} 62 | /\ negMsgs = {} 63 | /\ vote = [i \in Server |-> [b \in Index |-> [t \in Term |-> Nil]]] 64 | /\ leaderLog = [i \in Term |-> [j \in Index |-> <<-1,Nil,FALSE>>]] 65 | /\ InitServerVars 66 | /\ InitLogVars 67 | 68 | ------------------------------------------------------------------------------------- 69 | Restart(i) == 70 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 71 | /\ UNCHANGED <> 73 | UpdateTerm(i,b) == 74 | /\ currentTerm[i] currentTerm[i] 81 | /\ m[2] = i 82 | /\ UpdateTerm(i,m[1]) 83 | /\ UNCHANGED <> 85 | 86 | Timeout(i) == 87 | /\ currentState[i] \in {Follower,Candidate} 88 | /\ currentTerm' = [currentTerm EXCEPT ![i] = currentTerm[i] + 1] 89 | /\ currentState' = [currentState EXCEPT ![i] = Candidate] 90 | /\ currentTerm[i] + 1 \in Nat 91 | /\ UNCHANGED <> 93 | 94 | RequestVote(i) == 95 | /\ currentState[i] = Candidate 96 | /\ r1amsgs' = r1amsgs \cup {<>} 97 | /\ UNCHANGED <> 99 | 100 | \* i: recipient 101 | HandleRequestVoteRequest(i) == 102 | /\ \E m \in r1amsgs : 103 | LET j == m[2] 104 | grant == m[1] > currentTerm[i] 105 | entries == {<> : n \in Index} 106 | IN 107 | \/ /\ grant 108 | /\ UpdateTerm(i,m[1]) 109 | /\ r1bmsgs' = r1bmsgs \cup {<>} 110 | /\ UNCHANGED negMsgs 111 | \/ /\ \neg grant 112 | /\ negMsgs' = negMsgs \cup {<>} 113 | /\ UNCHANGED <> 114 | /\ UNCHANGED <> 115 | 116 | 117 | Merge(entries,term,v) == 118 | LET 119 | committed == {e \in entries : e[3] = TRUE} 120 | chosen == 121 | CASE committed = {} -> CHOOSE x \in entries : \A y \in entries : x[1] \geq y[1] 122 | [] committed /= {} -> CHOOSE x \in committed : TRUE 123 | safe == IF chosen[2] = Nil THEN v ELSE chosen[2] 124 | IN <> 125 | 126 | BecomeLeaderCandidate(i) == 127 | /\ currentState[i] = Candidate 128 | /\ \E Q \in Quorums : 129 | LET voteGranted == {m \in r1bmsgs : m[4] = i /\ m[3] \in Q /\ m[1] = currentTerm[i]} 130 | allLog == UNION {m[2] : m \in voteGranted} 131 | valid == {e \in allLog : e[2][1] /= -1} 132 | end == IF valid = {} THEN -1 ELSE Max({e[1] : e \in valid}) 133 | IN 134 | /\ \A q \in Q : \E m \in voteGranted : m[3] = q 135 | /\ \E v \in Value : leaderLog' = [leaderLog EXCEPT ![currentTerm[i]] = [n \in Index |-> IF n \in 0..end THEN 136 | Merge({l[2] : l \in {t \in allLog : t[1] = n}},currentTerm[i],v) ELSE <<-1,Nil,FALSE>>]] 137 | /\ currentState' = [currentState EXCEPT ![i] = LeaderCandidate] 138 | /\ UNCHANGED<> 139 | 140 | RequestSync(i) == 141 | /\ currentState[i] \in {LeaderCandidate,Leader} 142 | /\ LET sync == {n \in Index : leaderLog[currentTerm[i]][n][1] /= -1} IN 143 | \E n \in sync : r2amsgs' = r2amsgs \cup {<>} 144 | /\ UNCHANGED <> 145 | 146 | HandleRequestSyncRequest(i) == 147 | /\ \E m \in r2amsgs : 148 | LET j == m[4] 149 | grant == m[1] \geq currentTerm[i] 150 | IN 151 | /\ \/ /\ m[1] > currentTerm[i] 152 | /\ UpdateTerm(i,m[1]) 153 | \/ /\ m[1] \leq currentTerm[i] 154 | /\ UNCHANGED <> 155 | /\ \/ /\ grant 156 | /\ log' = [log EXCEPT ![i][m[2]] = m[3]] 157 | /\ vote' = [vote EXCEPT ![i][m[2]][m[1]] = m[3][2]] 158 | /\ r2bmsgs ' = r2bmsgs \cup {<>} 159 | /\ UNCHANGED negMsgs 160 | \/ /\ \neg grant 161 | /\ negMsgs' = negMsgs \cup {<>} 162 | /\ UNCHANGED <> 163 | /\ UNCHANGED <> 164 | 165 | CommitEntry(i) == 166 | /\ \E index \in Index , Q \in Quorums : 167 | LET syncSuccess == { m \in r2bmsgs : m[4] = i /\ m[3] \in Q /\ 168 | m[1] = currentTerm[i] /\ m[2] = index} 169 | IN 170 | /\ currentState[i] \in {Leader,LeaderCandidate} 171 | /\ \A q \in Q : \E m \in syncSuccess : m[3] = q 172 | /\ leaderLog' = [leaderLog EXCEPT ![currentTerm[i]][index][3] = TRUE] 173 | /\ UNCHANGED <> 174 | 175 | RequestCommit(i) == 176 | /\ currentState[i] \in {Leader,LeaderCandidate} 177 | /\ LET committed == {n \in Index : leaderLog[currentTerm[i]][n][3] = TRUE} IN 178 | \E n \in committed : r3amsgs' = r3amsgs \cup {<>} 179 | /\ UNCHANGED <> 180 | 181 | HandleRequestCommitRequest(i) == 182 | /\ \E m \in r3amsgs : 183 | LET grant == currentTerm[i] \leq m[1] 184 | j == m[3] 185 | IN 186 | /\ \/ /\ m[1] > currentTerm[i] 187 | /\ UpdateTerm(i,m[1]) 188 | \/ /\ m[1] \leq currentTerm[i] 189 | /\ UNCHANGED <> 190 | /\ \/ /\ grant 191 | /\ log[i][m[2]][1] = m[1] 192 | /\ log' = [log EXCEPT ![i][m[2]][3] = TRUE] 193 | /\ UNCHANGED negMsgs 194 | \/ /\ \neg grant 195 | /\ negMsgs' = negMsgs \cup {<>} 196 | /\ UNCHANGED log 197 | /\ UNCHANGED <> 198 | 199 | BecomeLeader(i) == 200 | /\ currentState[i] = LeaderCandidate 201 | /\ currentState' = [currentState EXCEPT ![i] = Leader] 202 | /\ UNCHANGED <> 204 | 205 | ClientRequest(i) == 206 | LET ind == {b \in Index : leaderLog[currentTerm[i]][b][1] /= -1} 207 | nextIndex == IF ind ={} 208 | THEN 0 209 | ELSE Max(ind) + 1 210 | IN 211 | /\ currentState[i] = Leader 212 | /\ nextIndex \in Index 213 | /\ \E v \in Value :leaderLog' = [leaderLog EXCEPT ![currentTerm[i]][nextIndex] = 214 | <>] 215 | /\ UNCHANGED <> 216 | 217 | Next == \/ \E i \in Server: Restart(i) 218 | \/ \E i \in Server: Timeout(i) 219 | \/ \E i \in Server: ReceiveHighTerm(i) 220 | \/ \E i \in Server: RequestVote(i) 221 | \/ \E i \in Server : HandleRequestVoteRequest(i) 222 | \/ \E i \in Server : BecomeLeaderCandidate(i) 223 | \/ \E i \in Server : BecomeLeader(i) 224 | \/ \E i \in Server : CommitEntry(i) 225 | \/ \E i \in Server : ClientRequest(i) 226 | \/ \E i,j \in Server : RequestCommit(i) 227 | \/ \E i \in Server : HandleRequestCommitRequest(i) 228 | \/ \E i,j \in Server : RequestSync(i) 229 | \/ \E i \in Server : HandleRequestSyncRequest(i) 230 | 231 | 232 | Inv == /\ TypeInv 233 | 234 | ------------------------------------------------------------------------------------------------ 235 | Acceptors == Server 236 | Ballots == Term 237 | Instances == Index 238 | ballot == currentTerm 239 | leaderVote == [i \in Ballots |-> [j \in Index |-> <>]] 240 | 1amsgs == {<>: m \in r1amsgs} 241 | 1bmsgs == {<> >> : e \in m[2]},m[3]>> : m \in r1bmsgs} 242 | 2amsgs == {<> >>: m \in r2amsgs} 243 | 244 | Spec == Init /\ [][Next]_vars 245 | 246 | A == INSTANCE MultiPaxos 247 | 248 | THEOREM Refinement == Spec => A!Spec 249 | 250 | 251 | ============================================================================= 252 | \* Modification History 253 | \* Last modified Wed Sep 09 15:26:16 CST 2020 by 15150 254 | 255 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Parallel-Raft-tla 2 | ### Overview 3 | This project is devoted to provide formal specification and verification using TLA+ for the Parallel-Raft consensus protocol proposed in *PolarFS:An ultra-low latency and failure resilient distributed file system for shared storage cloud database*. 4 | 5 | The highlight of Parallel-Raft is that it enables "out-of-order executions". To better understand the relationship between Parallel-Raft and existing concensus algorithms(i.e. Raft,Multi-Paxos), we proposed ParallelRaft-SE(SE stands for "Sequential Execution"). Parallel-Raft is the same as Parallel-Raft except that it prohibits "out-of-order executions". In the way, we established the refinement mapping from ParallelRaft-SE to Multi-Paxos. 6 | 7 | We discovered that Parallel-Raft, according to its brief description, neglects the so-called "ghost log entries" phenomenon, which may violate the consistency among state machines. Therefore, based on ParallelRaft-SE, we proposed ParallelRaft-CE(CE stands for "Concurrent Executions"). ParallelRaft-CE avoids the "ghost log entries" by limiting parallelism in the commmitment of log entries. 8 | 9 | We provided the formal specifications of MultiPaxos,ParallelRaft-SE and ParallelRaft-CE and verified the refinement mapping from ParallelRaft-SE to Multi-Paxos and the correctness of ParallelRaft-CE using the TLC model checker. 10 | 11 | ### Papaers 12 | See [paper-parallelraft](https://github.com/HappyCS-Gu/Parallel-Raft-tla/blob/master/doc/2020.8-jos.pdf) 13 | ### How to Run 14 | Create and run TLA models in the usual way 15 | ### TLA+ Modules 16 | `MultiPaxos.tla`: specification of Multi-Paoxs 17 | 18 | `ParallelRaftSE`: specification of ParallelRaft-SE 19 | 20 | `ParallelRaftCE`: specification of ParallelRaft-CE 21 | -------------------------------------------------------------------------------- /doc/2020.8-jos.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HappyCS-Gu/Parallel-Raft-tla/567f3ee16432b778899d4802d9291bceea9306ad/doc/2020.8-jos.pdf -------------------------------------------------------------------------------- /specification/MultiPaxos.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HappyCS-Gu/Parallel-Raft-tla/567f3ee16432b778899d4802d9291bceea9306ad/specification/MultiPaxos.pdf -------------------------------------------------------------------------------- /specification/MultiPaxos.tla: -------------------------------------------------------------------------------- 1 | ----------------------------- MODULE MultiPaxos ----------------------------- 2 | 3 | EXTENDS Integers, FiniteSets 4 | 5 | CONSTANTS Acceptors,Nil,Value 6 | 7 | Ballots == Nat 8 | Instances == {0,1,2,3,4,5,6} 9 | Quorums == {Q \in SUBSET Acceptors : Cardinality(Q) > Cardinality(Acceptors) \div 2} 10 | Max(s) == CHOOSE x \in s : \forall y \in s : x \geq y 11 | 12 | ------------------------------------------------------------------------------- 13 | VARIABLES ballot, 14 | vote, 15 | leaderVote, 16 | 1amsgs, 17 | 1bmsgs, 18 | 2amsgs 19 | 20 | -------------------------------------------------------------------------------- 21 | Init == 22 | /\ ballot = [a \in Acceptors |-> 0] 23 | /\ vote = [a \in Acceptors |-> 24 | [i \in Instances |-> 25 | [b \in Ballots |-> Nil]]] 26 | /\ 1amsgs = {} 27 | /\ 1bmsgs = {} 28 | /\ 2amsgs = {} 29 | /\ leaderVote = [b \in Ballots |-> [i \in Instances |-> <<-1,Nil>>]] 30 | 31 | ---------------------------------------------------------------------------------- 32 | 33 | allEntries == {<> >> : i \in Instances, b \in Ballots \cup {-1}, v \in Value \cup {Nil}} 34 | 35 | TypeInv == 36 | /\ ballot \in [Acceptors -> {-1} \cup Ballots] 37 | /\ leaderVote \in [Ballots -> [Instances -> ({-1} \cup Ballots) \times ({Nil} \cup Value)]] 38 | /\ vote \in [Acceptors -> [Instances -> [Ballots -> ({Nil} \cup Value )]]] 39 | /\ 1amsgs \subseteq {<> : b \in Ballots} 40 | \* /\ 1bmsgs \subseteq {<> : b \in Ballots, a \in Acceptors, e \in SUBSET allEntries} 41 | /\ 2amsgs \subseteq {<>>> : i \in Instances, b \in Ballots, bb \in Ballots,v \in Value \cup {Nil}} 42 | /\ leaderVote \in [Ballots -> [Instances -> ((Ballots\cup {-1}) \times ({Nil} \cup Value))]] 43 | 44 | ------------------------------------------------------------------------------------ 45 | IncreaseBallot(a,b) == 46 | /\ ballot[a] 49 | 50 | Phase1a(b) == 51 | /\ 1amsgs' = 1amsgs \cup {<>} 52 | /\ UNCHANGED <> 53 | 54 | MaxAcceptorVote(a, i) == 55 | LET maxBallot == Max({b \in Ballots : vote[a][i][b] # Nil} \cup {-1}) 56 | v == IF maxBallot > -1 THEN vote[a][i][maxBallot] ELSE Nil 57 | IN <> 58 | 59 | Phase1b(a, b) == 60 | /\ ballot[a] \in 1amsgs 62 | /\ ballot' = [ballot EXCEPT ![a] = b] 63 | /\ 1bmsgs' = 1bmsgs \cup 64 | {<> : i \in Instances}, a>>} 65 | /\ UNCHANGED <> 66 | 67 | 1bMsgs(b, Q) == 68 | {m \in 1bmsgs : m[3] \in Q /\ m[1] = b} 69 | 70 | 71 | 72 | MaxVote(b, i, Q) == 73 | LET entries == UNION{m[2] : m \in 1bMsgs(b,Q)} 74 | ientries == {e \in entries : e[1] = i} 75 | maxBal == Max({e[2][1] : e \in ientries}) 76 | IN CHOOSE v \in Value \cup {Nil} : \E e \in ientries : 77 | /\ e[2][1] = maxBal /\ e[2][2] = v 78 | 79 | 80 | 81 | lastInstance(b,Q) == LET entries == UNION{m[2] : m \in 1bMsgs(b,Q)} 82 | valid == {e \in entries : e[2][1] /= -1} 83 | IN 84 | IF valid = {} THEN -1 ELSE Max({e[1] : e \in valid}) 85 | 86 | 87 | 88 | Merge(b) == /\ \E Q \in Quorums : 89 | /\ \A a \in Q : \E m \in 1bMsgs(b,Q) : m[3] = a 90 | /\ \E v \in Value : leaderVote' = [leaderVote EXCEPT ![b] = [i \in Instances |-> 91 | IF (i \in 0..lastInstance(b,Q) /\ leaderVote[b][i][1] = -1) 92 | \* THEN <> 93 | THEN IF MaxVote(b,i,Q) = Nil THEN <> 94 | ELSE <> 95 | ELSE leaderVote[b][i]]] 96 | /\ UNCHANGED <> 97 | 98 | 99 | Propose(b,i) == /\ leaderVote[b][i][1] = -1 100 | /\ \E Q \in Quorums : 101 | /\ \A a \in Q : \E m \in 1bMsgs(b,Q) : m[3] = a 102 | /\ \E v \in Value : leaderVote'=[leaderVote EXCEPT ![b][i] = IF MaxVote(b,i,Q) = Nil 103 | THEN <> 104 | ELSE <>] 105 | /\ UNCHANGED <> 106 | 107 | Phase2a(b, i) == 108 | /\ leaderVote[b][i][1] = b 109 | /\ 2amsgs' = 2amsgs \cup {<>} 110 | /\ UNCHANGED <> 111 | 112 | Vote(a, b, i) == 113 | /\ ballot[a] \leq b 114 | /\ ballot' = [ballot EXCEPT ![a] = b] 115 | /\ \E m \in 2amsgs : 116 | /\ m[2] = i /\ m[1] = b 117 | /\ vote' = [vote EXCEPT ![a][i][b] = m[3][2]] 118 | /\ UNCHANGED <> 119 | 120 | Next == 121 | \/ \E a \in Acceptors, b \in Ballots : IncreaseBallot(a,b) 122 | \/ \E b \in Ballots : Phase1a(b) 123 | \/ \E a \in Acceptors, b \in Ballots : Phase1b(a,b) 124 | \/ \E b \in Ballots : Merge(b) 125 | \/ \E b \in Ballots, i \in Instances : Propose(b,i) 126 | \/ \E b \in Ballots, i \in Instances : Phase2a(b,i) 127 | \/ \E a \in Acceptors, b \in Ballots, i \in Instances : Vote(a, b, i) 128 | 129 | Spec == Init /\ [][Next]_<> 130 | ---------------------------------------------------------------------------------------- 131 | Conservative(i,b) == 132 | \A a1,a2 \in Acceptors : 133 | LET v1 == vote[a1][i][b] 134 | v2 == vote[a2][i][b] 135 | IN (v1 # Nil /\ v2 # Nil) => v1 = v2 136 | 137 | ConservativeVoteArray == 138 | \A i \in Instances : \A b \in Ballots : 139 | Conservative(i,b) 140 | 141 | WellFormed == \A a \in Acceptors : \A i \in Instances : \A b \in Ballots : 142 | b > ballot[a] => vote[a][i][b] = Nil 143 | 144 | VotedFor(a,i,b,v) == vote[a][i][b] = v 145 | 146 | ChosenAt(i,b,v) == 147 | \E Q \in Quorums : \A a \in Q : VotedFor(a,i,b,v) 148 | 149 | Chosen(i,v) == 150 | \E b \in Ballots : ChosenAt(i,b,v) 151 | 152 | Choosable(v, i, b) == 153 | \E Q \in Quorums : \A a \in Q : ballot[a] > b => vote[a][i][b] = v 154 | 155 | SafeAt(v, i, b) == 156 | \A b2 \in Ballots : \A v2 \in Value : 157 | (b2 v = v2 159 | 160 | SafeInstanceVoteArray(i) == \A b \in Ballots : \A a \in Acceptors : 161 | LET v == vote[a][i][b] 162 | IN v # Nil => SafeAt(v, i, b) 163 | 164 | SafeVoteArray == \A i \in Instances : SafeInstanceVoteArray(i) 165 | 166 | Inv == TypeInv /\ WellFormed /\ SafeVoteArray /\ ConservativeVoteArray 167 | 168 | Correctness == 169 | \A i \in Instances : \A v1,v2 \in Value : 170 | Chosen(i, v1) /\ Chosen(i, v2) => v1 = v2 171 | 172 | ============================================================================= 173 | \* Modification History 174 | \* Last modified Wed Sep 09 15:26:35 CST 2020 by 15150 175 | 176 | -------------------------------------------------------------------------------- /specification/ParallelRaftCE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HappyCS-Gu/Parallel-Raft-tla/567f3ee16432b778899d4802d9291bceea9306ad/specification/ParallelRaftCE.pdf -------------------------------------------------------------------------------- /specification/ParallelRaftCE.tla: -------------------------------------------------------------------------------- 1 | --------------------------- MODULE ParallelRaftCE --------------------------- 2 | 3 | EXTENDS Integers,FiniteSets,Sequences,TLC,Naturals 4 | 5 | CONSTANTS Server,Follower,Candidate,Leader,LeaderCandidate,Nil,Value 6 | 7 | CONSTANTS RequestVoteRequest,RequestVoteResponse, 8 | RequestCommitRequest,RequestCommitResponse, 9 | RequestSyncRequest,RequestSyncResponse, 10 | UpdateSyncRequest,UpdateSyncResponse 11 | 12 | VARIABLE messages, 13 | currentTerm, 14 | currentState, 15 | votedFor, 16 | sync, 17 | endPoint 18 | 19 | serverVars == <> 20 | 21 | VARIABLE log 22 | logVars == <> 23 | 24 | 25 | 26 | VARIABLE syncTrack 27 | leaderVars == <> 28 | 29 | 30 | 31 | VARIABLE halfElections 32 | VARIABLE elections 33 | electionVars == <> 34 | 35 | 36 | VARIABLE allLogs 37 | VARIABLE allEntries 38 | VARIABLE allSynced 39 | 40 | 41 | 42 | vars == <> 43 | 44 | 45 | Quorums == {i \in SUBSET(Server) : Cardinality(i)*2 > Cardinality(Server)} 46 | 47 | Send(m) == messages' = messages \cup {m} 48 | 49 | 50 | Index == Nat 51 | Term == Nat 52 | 53 | Min(s) == IF s = {} THEN -1 ELSE CHOOSE i \in s : \A j \in s : j \geq i 54 | Max(s) == IF s = {} THEN -1 ELSE CHOOSE i \in s : \A j \in s : i \geq j 55 | 56 | 57 | 58 | InitServerVars == LET k == CHOOSE x \in Server : x \in Server 59 | IN 60 | /\ currentTerm = [i \in Server |-> 0] 61 | /\ sync = [i \in Server |-> 0] 62 | /\ currentState = [i \in Server |-> Follower] 63 | /\ endPoint = [i \in Server |-> [n \in Term |-> <<-1,-1>>]] 64 | /\ votedFor = [i \in Server |-> Nil] 65 | 66 | 67 | InitLeaderVars == /\ syncTrack = [i \in Server |-> [j \in Server |-> 0]] 68 | 69 | 70 | InitHistoryVars == /\ halfElections = {} 71 | /\ elections = {} 72 | /\ allLogs = {} 73 | /\ allEntries = {} 74 | /\ allSynced = {} 75 | 76 | 77 | InitLogVars == /\ log = [i \in Server |-> [n \in Index |-> [term |-> -1,date |-> -1, 78 | value |-> Nil, committed |-> FALSE]]] 79 | 80 | 81 | Init == /\ messages = {} 82 | /\ InitServerVars 83 | /\ InitLeaderVars 84 | /\ InitLogVars 85 | /\ InitHistoryVars 86 | 87 | 88 | Entries == [term : Term \cup {-1}, date : Term \cup {-1}, value : Value \cup {Nil} , committed : {TRUE,FALSE}] 89 | 90 | TypeSafety == /\ allLogs \in SUBSET (SUBSET allEntries) 91 | /\ currentTerm \in [Server -> Nat] 92 | /\ currentState \in [Server -> {Follower,Leader,LeaderCandidate,Candidate}] 93 | /\ votedFor \in [Server -> Server \cup {Nil}] 94 | /\ sync \in [Server -> Nat\cup{-1}] 95 | /\ endPoint \in [Server -> [Term -> [date : Term \cup {-1}, index : Index \cup {-1}]]] 96 | /\ endPoint \in [Server -> [Term -> ((Term \cup {-1}) \times (Index \cup {-1}))]] 97 | /\ log \in [Server -> [Index -> [term : Index \cup {-1}, date : Term \cup {-1}, 98 | value : Value \cup {Nil}, committed : {TRUE,FALSE}]]] 99 | /\ syncTrack \in [Server -> [Server -> Nat]] 100 | /\ halfElections \subseteq [eterm : Nat, eleaderCandidate : Server, esync : Nat, 101 | evotes : Quorums, elog : [Index -> Entries]] 102 | /\ elections \subseteq [eterm : Term, esync : Term, eleader : Server, evotes : Quorums, evoterLog : SUBSET [Index -> Entries],elog : [Index -> Entries]] 103 | 104 | 105 | 106 | 107 | logTail(s) == Max({i \in Index : s[i].term /= -1}) 108 | 109 | Restart(i) == 110 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 111 | /\ syncTrack' = [syncTrack EXCEPT ![i] = [j \in Server |-> 0]] 112 | /\ UNCHANGED <> 114 | 115 | Timeout(i) == 116 | /\ currentState[i] \in {Follower,Candidate} 117 | /\ currentState' = [currentState EXCEPT ![i] = Candidate] 118 | /\ currentTerm' = [currentTerm EXCEPT ![i] = currentTerm[i]+1] 119 | /\ currentTerm[i] + 1 \in Term 120 | /\ votedFor' = [votedFor EXCEPT ![i] = Nil] 121 | /\ UNCHANGED <> 123 | 124 | UpdateTerm(i) == 125 | /\ \E m \in messages : 126 | /\ m.mterm > currentTerm[i] 127 | /\ \/ m.mdest = i 128 | \/ m.mdest = Nil 129 | /\ currentTerm' = [currentTerm EXCEPT ![i] = m.mterm] 130 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 131 | /\ votedFor' = [votedFor EXCEPT ![i] = Nil] 132 | /\ UNCHANGED <> 133 | 134 | 135 | RequestVote(i) == 136 | /\ currentState[i] = Candidate 137 | /\ Send([ mtype |-> RequestVoteRequest, 138 | mterm |-> currentTerm[i], 139 | msync |-> sync[i], 140 | msource |-> i, 141 | mdest |-> Nil]) 142 | /\ UNCHANGED<> 143 | 144 | \* i: recipient 145 | HandleRequestVoteRequest(i) == 146 | /\ \E m \in messages : 147 | LET j == m.msource 148 | syncOK == /\ m.msync \geq sync[i] 149 | grant == /\ syncOK 150 | /\ votedFor[i] \in {Nil,j} 151 | /\ currentTerm[i] = m.mterm 152 | IN 153 | /\ m.mterm \leq currentTerm[i] 154 | /\ m.mtype = RequestVoteRequest 155 | /\ \/ grant /\ votedFor' = [votedFor EXCEPT ![i] = j] 156 | \/ \neg grant /\ UNCHANGED votedFor 157 | /\ Send([ mtype |-> RequestVoteResponse, 158 | mterm |-> currentTerm[i], 159 | mvoteGranted |-> grant, 160 | mlog |-> LET C == {n \in Index : log[i][n].term = sync[i]} 161 | IN {<> : n \in C}, 162 | mend |-> endPoint[i][m.msync], 163 | msource |-> i, 164 | mdest |-> j]) 165 | /\ UNCHANGED <> 167 | 168 | 169 | Merge(entries,term,date) == IF entries = {} THEN [term |-> term, 170 | date |-> date, 171 | value |-> Nil, 172 | committed |-> FALSE] 173 | ELSE 174 | LET 175 | committed == {e \in entries : e.committed = TRUE} 176 | chosen == 177 | CASE committed = {} -> CHOOSE x \in entries : 178 | \A y \in entries : x.date \geq y.date 179 | [] committed /= {} -> CHOOSE x \in committed : TRUE 180 | IN 181 | [ term |-> chosen.term, 182 | date |-> date, 183 | value |-> chosen.value, 184 | committed |-> chosen.committed] 185 | 186 | BecomeLeaderCandidate(i) == 187 | /\ currentState[i] = Candidate 188 | /\ \E P,Q \in Quorums : 189 | LET voteResponded == {m \in messages : /\ m.mtype = RequestVoteResponse 190 | /\ m.mdest = i 191 | /\ m.msource \in P 192 | /\ m.mterm = currentTerm[i]} 193 | voteGranted == {m \in voteResponded : /\ m.mvoteGranted = TRUE 194 | /\ m.msource \in Q} 195 | allLog == UNION {m.mlog : m \in voteResponded} 196 | end == LET allPoint == {m.mend : m \in voteResponded} 197 | e == CHOOSE e1 \in allPoint : (\A e2 \in allPoint : e1[1] \geq e2[1]) 198 | IN IF e[1] = -1 THEN Max({e1[1] : e1 \in allLog}) 199 | ELSE e[2] 200 | toRecover == {n \in 0..end : log[i][n].committed = FALSE} 201 | toSync == {<> 202 | : n \in toRecover} 203 | IN 204 | /\ \A q \in Q : \E m \in voteGranted : m.msource = q 205 | /\ log' = [log EXCEPT ![i] = IF end = -1 THEN [n \in Index |-> IF log[i][n].term = sync[i] THEN 206 | [term |-> -1, 207 | date |-> -1, 208 | value |-> Nil, 209 | committed |-> FALSE] 210 | ELSE log[i][n]] 211 | ELSE [n \in Index |-> IF n \in toRecover THEN 212 | (CHOOSE e \in toSync : e[1] = n)[2] 213 | ELSE IF (n > end) THEN 214 | [term |-> -1, 215 | date |-> -1, 216 | value |-> Nil, 217 | committed |-> FALSE] 218 | ELSE log[i][n]]] 219 | /\ endPoint' = [endPoint EXCEPT ![i][sync[i]] = <>] 220 | /\ halfElections' = halfElections \union {[ eterm |-> currentTerm[i], 221 | eleaderCandidate |-> i, 222 | esync |-> sync[i], 223 | evotes |-> Q, 224 | elog |-> log[i]]} 225 | /\ currentState' = [currentState EXCEPT ![i] = LeaderCandidate] 226 | /\ syncTrack' = [syncTrack EXCEPT ![i] = [j \in Server |-> sync[i]]] 227 | /\ UNCHANGED<> 228 | 229 | 230 | RequestSync(i) == 231 | /\ currentState[i] \in {LeaderCandidate,Leader} 232 | /\ \E s \in 0..sync[i] : 233 | LET start == Min({n \in Index : log[i][n].term = s}) 234 | end == Max({n \in Index : log[i][n].term = s}) 235 | IN 236 | /\ Send([ mtype |-> RequestSyncRequest, 237 | mterm |-> currentTerm[i], 238 | msync |-> s, 239 | mstart |-> start, 240 | mend |-> end, 241 | mentries |-> IF start = -1 THEN Nil ELSE [n \in start..end |-> log[i][n]], 242 | msource |-> i, 243 | mdest |-> Nil]) 244 | /\ UNCHANGED <> 245 | 246 | 247 | HandleRequestSyncRequest(i) == 248 | /\ \E m \in messages : 249 | LET j == m.msource 250 | grant == /\ m.mterm = currentTerm[i] 251 | /\ m.msync = sync[i] 252 | IN 253 | /\ m.mtype = RequestSyncRequest 254 | /\ m.mterm \leq currentTerm[i] 255 | /\ j /= i 256 | /\ \/ /\ grant 257 | /\ log' = [log EXCEPT ![i] = IF m.mstart = -1 THEN 258 | [n \in Index |-> IF log[i][n].term = sync[i] THEN 259 | [term |-> -1, 260 | date |-> -1, 261 | value |-> Nil, 262 | committed |-> FALSE] 263 | ELSE 264 | log[i][n]] 265 | ELSE 266 | [n \in Index |-> IF n < m.mstart THEN log[i][n] 267 | ELSE IF n \in m.mstart..m.mend 268 | THEN m.mentries[n] 269 | ELSE [term |-> -1, 270 | date |-> -1, 271 | value |-> Nil, 272 | committed |-> FALSE]]] 273 | /\ endPoint' = [endPoint EXCEPT ![i][sync[i]] = <>] 274 | \/ /\ \neg grant 275 | /\ UNCHANGED <> 276 | /\ Send([mtype |-> RequestSyncResponse, 277 | mterm |-> currentTerm[i], 278 | msyncGranted |-> grant, 279 | msync |-> sync[i], 280 | mstart |-> m.mstart, 281 | mend |-> m.mend, 282 | msource |-> i, 283 | mdest |-> j]) 284 | /\ UNCHANGED <> 285 | 286 | HandleRequestSyncResponse(i) == 287 | /\ \E m \in messages : 288 | LET j == m.msource IN 289 | /\ m.mtype = RequestSyncResponse 290 | /\ m.mdest = i 291 | /\ currentTerm[i] = m.mterm 292 | /\ currentState[i] \in {Leader,LeaderCandidate} 293 | /\ syncTrack' = [syncTrack EXCEPT ![i][j] = m.msync] 294 | /\ \/ /\ m.msyncGranted 295 | /\ m.msync < sync[i] 296 | /\ Send( [ mtype |-> UpdateSyncRequest, 297 | mterm |-> currentTerm[i], 298 | msync |-> Min({sync[i]} \union {k \in Nat : k > m.msync /\ 299 | Cardinality({n \in Index : log[i][n].term = k})>0}), 300 | msource |-> i, 301 | mdest |-> {j}]) 302 | \/ /\ \neg m.msyncGranted 303 | /\ UNCHANGED messages 304 | /\ UNCHANGED <> 305 | 306 | UpdateSync(i) == 307 | /\ currentState[i] = LeaderCandidate 308 | /\ \E Q \in Quorums : 309 | LET syncUpdated == {m \in messages : /\ m.mtype = RequestSyncResponse 310 | /\ m.mterm = currentTerm[i] 311 | /\ m.msyncGranted = TRUE 312 | /\ m.msync = sync[i] 313 | /\ m.msource \in Q 314 | /\ m.mdest = i} 315 | IN 316 | /\ \A q \in Q : (\E m \in syncUpdated : m.msource = q) \/ q = i 317 | /\ allSynced' = LET indexes == {n \in Index : log[i][n].term = sync[i]} 318 | entries == {< log[i][n].term, 319 | date |-> log[i][n].date, 320 | value |-> log[i][n].value, 321 | committed |-> TRUE]>> : n \in indexes} 322 | IN allSynced \cup {<>} 323 | /\ Send( [ mtype |-> UpdateSyncRequest, 324 | mterm |-> currentTerm[i], 325 | msync |-> currentTerm[i], 326 | msource |-> i, 327 | mdest |-> Q]) 328 | /\ UNCHANGED <> 329 | 330 | HandleUpdateSyncRequest(i) == 331 | \E m \in messages : 332 | LET grant == /\ currentTerm[i] = m.mterm 333 | /\ m.msync > sync[i] 334 | j == m.msource 335 | IN 336 | /\ m.mtype = UpdateSyncRequest 337 | /\ i \in m.mdest 338 | /\ m.mterm \leq currentTerm[i] 339 | /\ \/ /\ grant 340 | /\ sync' = [sync EXCEPT ![i] = m.msync] 341 | /\ log' = [log EXCEPT ![i] = [n \in Index |-> 342 | IF log[i][n].term = sync[i] THEN 343 | [term |-> log[i][n].term, 344 | date |-> log[i][n].date, 345 | value |-> log[i][n].value, 346 | committed |-> TRUE] 347 | ELSE log[i][n]]] 348 | \/ /\ \neg grant 349 | /\ UNCHANGED <> 350 | /\ Send([ mtype |-> UpdateSyncResponse, 351 | mterm |-> currentTerm[i], 352 | mupdateSyncGranted |-> grant, 353 | msync |-> sync'[i], 354 | msource |-> i, 355 | mdest |-> j]) 356 | /\ UNCHANGED <> 357 | 358 | HandleUpdateSyncResponse(i) == 359 | /\ \E m \in messages : 360 | LET j == m.msource IN 361 | /\ m.mtype = UpdateSyncResponse 362 | /\ m.mdest = i 363 | /\ currentTerm[i] = m.mterm 364 | /\ currentState[i] \in {Leader,LeaderCandidate} 365 | /\ \/ /\ m.mupdateSyncGranted 366 | /\ syncTrack' = [syncTrack EXCEPT ![i][j] = m.msync] 367 | \/ /\ \neg m.mupdateSyncGranted 368 | /\ UNCHANGED syncTrack 369 | /\ UNCHANGED <> 370 | 371 | BecomeLeader(i) == 372 | /\ currentState[i] = LeaderCandidate 373 | /\ \E Q \in Quorums : \A q \in Q : (q = i \/ syncTrack[i][q] = currentTerm[i]) 374 | /\ elections' = elections \union {[ eterm |-> currentTerm[i], 375 | esync |-> sync[i], 376 | eleader |-> i, 377 | evotes |-> Q, 378 | evoterLog |-> {log[k] : k \in Q}, 379 | elog |-> log[i]]} 380 | /\ sync' = [sync EXCEPT ![i] = currentTerm[i]] 381 | /\ currentState' = [currentState EXCEPT ![i] = Leader] 382 | /\ log' = [log EXCEPT ![i] = [n \in Index |-> 383 | IF log[i][n].term = sync[i] THEN 384 | [term |-> log[i][n].term, 385 | date |-> log[i][n].date, 386 | value |-> log[i][n].value, 387 | committed |-> TRUE] 388 | ELSE log[i][n]]] 389 | /\ UNCHANGED <> 390 | 391 | ClientRequest(i,v) == 392 | LET nextIndex == logTail(log[i]) + 1 393 | entry == [term |-> currentTerm[i], 394 | date |-> currentTerm[i], 395 | value |-> v, 396 | committed |-> FALSE] 397 | IN 398 | /\ currentState[i] = Leader 399 | /\ nextIndex \in Nat 400 | /\ log' = [log EXCEPT ![i][nextIndex] = entry] 401 | /\ UNCHANGED <> 402 | 403 | CommitEntry(i,n) == 404 | /\ \E Q \in Quorums: 405 | LET succ == {m \in messages : /\ m.type = RequestSyncResponse 406 | /\ m.msyncGranted = TRUE 407 | /\ m.mdest = i 408 | /\ m.mterm = currentTerm[i] 409 | /\ m.msource \in Q 410 | /\ n \in m.mstart..m.mend} 411 | IN /\ \A q \in Q : \E m \in succ : (m.msource = q \/ q = i) 412 | /\ log' = [log EXCEPT ![i][n].committed = TRUE] 413 | /\ currentState[i] = Leader 414 | /\ UNCHANGED <> 415 | 416 | 417 | Next == /\ 418 | \/ \E i \in Server: Restart(i) 419 | \/ \E i \in Server: Timeout(i) 420 | \/ \E i \in Server: UpdateTerm(i) 421 | \/ \E i \in Server: RequestVote(i) 422 | \/ \E i \in Server : HandleRequestVoteRequest(i) 423 | \/ \E i \in Server : BecomeLeaderCandidate(i) 424 | \/ \E i \in Server : BecomeLeader(i) 425 | \/ \E i \in Server, v \in Value : ClientRequest(i,v) 426 | \/ \E i,j \in Server : RequestSync(i) 427 | \/ \E i \in Server : HandleRequestSyncRequest(i) 428 | \/ \E i \in Server : HandleRequestSyncResponse(i) 429 | \/ \E i,j \in Server : UpdateSync(i) 430 | \/ \E i \in Server : HandleUpdateSyncRequest(i) 431 | \/ \E i \in Server : HandleUpdateSyncResponse(i) 432 | 433 | /\ allLogs' = allLogs \union {log[i] : i \in Server} 434 | /\ LET entries(i) == {<> : n \in Index} 435 | IN 436 | allEntries' = allEntries \union UNION {entries(i) : i \in Server} 437 | 438 | 439 | AllEntries(i) == {<> : n \in Index} 440 | 441 | Lemma1 == \A i \in Server : sync[i] \leq currentTerm[i] 442 | Lemma2 == \A i \in Server : currentState[i]=Leader => sync[i] = currentTerm[i] 443 | Lemma3 == \A e,f \in halfElections : e.eterm = f.eterm => e.eleaderCandidate = f.eleaderCandidate 444 | Lemma4 == \A e \in elections : \E f \in halfElections : e.eterm = f.eterm 445 | /\ e.eleader = f.eleaderCandidate 446 | Lemma5 == \A e,f \in elections : e.eterm = f.eterm => e.eleader = f.eleader 447 | Lemma6 == \forall i \in Server : currentState[i]=Leader => currentTerm[i] = sync[i] 448 | Lemma7 == \A e \in halfElections : e.esync < e.eterm 449 | Lemma8 == \A i,j \in Server, n \in Index : log[i][n].term = log[j][n].term => 450 | log[i][n].value = log[j][n].value 451 | Lemma9 == \A s1,s2 \in allSynced : s1[1]=s2[1] => s1=s2 452 | Lemma10 == \A e1,e2 \in elections : e1.eterm < e2.eterm => 453 | \E s \in allSynced : s[1] = e1.term 454 | Lemma11 == LET indexes(i,t) == {n \in Index : log[i][n].term=t} 455 | entries(i,t) == {<> : n \in indexes(i,t)} IN 456 | \A i \in Server : \A t \in Term : 457 | t < sync[i] /\ (\E e \in elections : e.eterm = t) => \E s \in allSynced : s[1] = t /\ 458 | entries(i,t) = s[3] 459 | Lemma12 == \A i \in Server : \A e \in AllEntries(i) : e[2].term \leq sync[i] 460 | Lemma13 == \A e \in halfElections : \A f \in elections : f.eterm \leq e.esync \/ f.eterm \geq e.eterm 461 | syncCompleteness == \A i,j \in Server : 462 | {e \in AllEntries(i) : e[2].term \geq 0 /\ e[2].term < Min({sync[i],sync[j]})} = 463 | {e \in AllEntries(j) : e[2].term \geq 0 /\ e[2].term < Min({sync[i],sync[j]})} 464 | 465 | 466 | Spec == Init/\ [][Next]_vars 467 | 468 | 469 | 470 | ============================================================================= 471 | \* Modification History 472 | \* Last modified Wed Sep 09 15:26:50 CST 2020 by 15150 473 | 474 | -------------------------------------------------------------------------------- /specification/ParallelRaftSE.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/HappyCS-Gu/Parallel-Raft-tla/567f3ee16432b778899d4802d9291bceea9306ad/specification/ParallelRaftSE.pdf -------------------------------------------------------------------------------- /specification/ParallelRaftSE.tla: -------------------------------------------------------------------------------- 1 | --------------------------- MODULE ParallelRaftSE --------------------------- 2 | 3 | EXTENDS Integers,FiniteSets,Sequences,TLC 4 | CONSTANTS Server,Follower,Candidate,Leader,LeaderCandidate,Nil,Value 5 | 6 | Quorums == {i \in SUBSET(Server) : Cardinality(i)*2 > Cardinality(Server)} 7 | Index == {0,1,2,3,4,5,6} 8 | Term == Nat 9 | 10 | ------------------------------------------------------------------------------ 11 | VARIABLE r1amsgs, 12 | r1bmsgs, 13 | r2amsgs, 14 | r2bmsgs, 15 | r3amsgs, 16 | negMsgs, 17 | currentTerm, 18 | currentState, 19 | vote, 20 | leaderLog, 21 | log 22 | 23 | serverVars == <> 24 | vars == <> 25 | 26 | --------------------------------------------------------------------------------- 27 | Max(s) == CHOOSE i \in s : \A j \in s : i \geq j 28 | 29 | lastIndex(i) == IF {b \in Index : log[i][b][1] /= -1} ={} 30 | THEN -1 31 | ELSE Max({b \in Index : log[i][b][1] /= -1}) 32 | 33 | allEntries == {<> : t \in Term \cup {-1}, v \in Value \cup {Nil}, b \in {TRUE,FALSE}} 34 | logEntries == {<> : i \in Index, e \in allEntries} 35 | 36 | TypeInv == /\ currentTerm \in [Server -> Nat] 37 | /\ currentState \in [Server -> {Follower,Leader,LeaderCandidate,Candidate}] 38 | /\ log \in [Server -> [Index -> (Term \cup {-1}) \times (Value \cup {Nil}) \times BOOLEAN]] 39 | /\ r1amsgs \subseteq {<> : t \in Term, i \in Server} 40 | /\ r1bmsgs \subseteq {<> : t \in Term, e \in SUBSET logEntries, i \in Server, j \in Server} 41 | /\ r2amsgs \subseteq {<> : t \in Term, n \in Index, e \in allEntries, i \in Server} 42 | /\ r2bmsgs \subseteq {<> : t \in Term, n \in Index, i \in Server, j \in Server} 43 | /\ r3amsgs \subseteq {<> : t \in Term, n \in Index, i \in Server} 44 | /\ negMsgs \subseteq {<> : t \in Term, i \in Server} 45 | /\ log \in [Server -> [Index -> allEntries]] 46 | /\ leaderLog \in [Term -> [Index -> allEntries]] 47 | /\ vote \in [Server -> [Index -> [Term -> Value \cup {Nil}]]] 48 | 49 | --------------------------------------------------------------------------------------- 50 | InitServerVars == LET k == CHOOSE x \in Server : x \in Server 51 | IN 52 | /\ currentTerm = [i \in Server |-> 0] 53 | /\ currentState = [i \in Server |-> Follower] 54 | 55 | InitLogVars == /\ log = [i \in Server |-> [j \in Index |-> <<-1,Nil,FALSE>>]] 56 | 57 | Init == /\ r1amsgs = {} 58 | /\ r1bmsgs = {} 59 | /\ r2amsgs = {} 60 | /\ r2bmsgs = {} 61 | /\ r3amsgs = {} 62 | /\ negMsgs = {} 63 | /\ vote = [i \in Server |-> [b \in Index |-> [t \in Term |-> Nil]]] 64 | /\ leaderLog = [i \in Term |-> [j \in Index |-> <<-1,Nil,FALSE>>]] 65 | /\ InitServerVars 66 | /\ InitLogVars 67 | 68 | ------------------------------------------------------------------------------------- 69 | Restart(i) == 70 | /\ currentState' = [currentState EXCEPT ![i] = Follower] 71 | /\ UNCHANGED <> 73 | UpdateTerm(i,b) == 74 | /\ currentTerm[i] currentTerm[i] 81 | /\ m[2] = i 82 | /\ UpdateTerm(i,m[1]) 83 | /\ UNCHANGED <> 85 | 86 | Timeout(i) == 87 | /\ currentState[i] \in {Follower,Candidate} 88 | /\ currentTerm' = [currentTerm EXCEPT ![i] = currentTerm[i] + 1] 89 | /\ currentState' = [currentState EXCEPT ![i] = Candidate] 90 | /\ currentTerm[i] + 1 \in Nat 91 | /\ UNCHANGED <> 93 | 94 | RequestVote(i) == 95 | /\ currentState[i] = Candidate 96 | /\ r1amsgs' = r1amsgs \cup {<>} 97 | /\ UNCHANGED <> 99 | 100 | \* i: recipient 101 | HandleRequestVoteRequest(i) == 102 | /\ \E m \in r1amsgs : 103 | LET j == m[2] 104 | grant == m[1] > currentTerm[i] 105 | entries == {<> : n \in Index} 106 | IN 107 | \/ /\ grant 108 | /\ UpdateTerm(i,m[1]) 109 | /\ r1bmsgs' = r1bmsgs \cup {<>} 110 | /\ UNCHANGED negMsgs 111 | \/ /\ \neg grant 112 | /\ negMsgs' = negMsgs \cup {<>} 113 | /\ UNCHANGED <> 114 | /\ UNCHANGED <> 115 | 116 | 117 | Merge(entries,term,v) == 118 | LET 119 | committed == {e \in entries : e[3] = TRUE} 120 | chosen == 121 | CASE committed = {} -> CHOOSE x \in entries : \A y \in entries : x[1] \geq y[1] 122 | [] committed /= {} -> CHOOSE x \in committed : TRUE 123 | safe == IF chosen[2] = Nil THEN v ELSE chosen[2] 124 | IN <> 125 | 126 | BecomeLeaderCandidate(i) == 127 | /\ currentState[i] = Candidate 128 | /\ \E Q \in Quorums : 129 | LET voteGranted == {m \in r1bmsgs : m[4] = i /\ m[3] \in Q /\ m[1] = currentTerm[i]} 130 | allLog == UNION {m[2] : m \in voteGranted} 131 | valid == {e \in allLog : e[2][1] /= -1} 132 | end == IF valid = {} THEN -1 ELSE Max({e[1] : e \in valid}) 133 | IN 134 | /\ \A q \in Q : \E m \in voteGranted : m[3] = q 135 | /\ \E v \in Value : leaderLog' = [leaderLog EXCEPT ![currentTerm[i]] = [n \in Index |-> IF n \in 0..end THEN 136 | Merge({l[2] : l \in {t \in allLog : t[1] = n}},currentTerm[i],v) ELSE <<-1,Nil,FALSE>>]] 137 | /\ currentState' = [currentState EXCEPT ![i] = LeaderCandidate] 138 | /\ UNCHANGED<> 139 | 140 | RequestSync(i) == 141 | /\ currentState[i] \in {LeaderCandidate,Leader} 142 | /\ LET sync == {n \in Index : leaderLog[currentTerm[i]][n][1] /= -1} IN 143 | \E n \in sync : r2amsgs' = r2amsgs \cup {<>} 144 | /\ UNCHANGED <> 145 | 146 | HandleRequestSyncRequest(i) == 147 | /\ \E m \in r2amsgs : 148 | LET j == m[4] 149 | grant == m[1] \geq currentTerm[i] 150 | IN 151 | /\ \/ /\ m[1] > currentTerm[i] 152 | /\ UpdateTerm(i,m[1]) 153 | \/ /\ m[1] \leq currentTerm[i] 154 | /\ UNCHANGED <> 155 | /\ \/ /\ grant 156 | /\ log' = [log EXCEPT ![i][m[2]] = m[3]] 157 | /\ vote' = [vote EXCEPT ![i][m[2]][m[1]] = m[3][2]] 158 | /\ r2bmsgs ' = r2bmsgs \cup {<>} 159 | /\ UNCHANGED negMsgs 160 | \/ /\ \neg grant 161 | /\ negMsgs' = negMsgs \cup {<>} 162 | /\ UNCHANGED <> 163 | /\ UNCHANGED <> 164 | 165 | CommitEntry(i) == 166 | /\ \E index \in Index , Q \in Quorums : 167 | LET syncSuccess == { m \in r2bmsgs : m[4] = i /\ m[3] \in Q /\ 168 | m[1] = currentTerm[i] /\ m[2] = index} 169 | IN 170 | /\ currentState[i] \in {Leader,LeaderCandidate} 171 | /\ \A q \in Q : \E m \in syncSuccess : m[3] = q 172 | /\ leaderLog' = [leaderLog EXCEPT ![currentTerm[i]][index][3] = TRUE] 173 | /\ UNCHANGED <> 174 | 175 | RequestCommit(i) == 176 | /\ currentState[i] \in {Leader,LeaderCandidate} 177 | /\ LET committed == {n \in Index : leaderLog[currentTerm[i]][n][3] = TRUE} IN 178 | \E n \in committed : r3amsgs' = r3amsgs \cup {<>} 179 | /\ UNCHANGED <> 180 | 181 | HandleRequestCommitRequest(i) == 182 | /\ \E m \in r3amsgs : 183 | LET grant == currentTerm[i] \leq m[1] 184 | j == m[3] 185 | IN 186 | /\ \/ /\ m[1] > currentTerm[i] 187 | /\ UpdateTerm(i,m[1]) 188 | \/ /\ m[1] \leq currentTerm[i] 189 | /\ UNCHANGED <> 190 | /\ \/ /\ grant 191 | /\ log[i][m[2]][1] = m[1] 192 | /\ log' = [log EXCEPT ![i][m[2]][3] = TRUE] 193 | /\ UNCHANGED negMsgs 194 | \/ /\ \neg grant 195 | /\ negMsgs' = negMsgs \cup {<>} 196 | /\ UNCHANGED log 197 | /\ UNCHANGED <> 198 | 199 | BecomeLeader(i) == 200 | /\ currentState[i] = LeaderCandidate 201 | /\ currentState' = [currentState EXCEPT ![i] = Leader] 202 | /\ UNCHANGED <> 204 | 205 | ClientRequest(i) == 206 | LET ind == {b \in Index : leaderLog[currentTerm[i]][b][1] /= -1} 207 | nextIndex == IF ind ={} 208 | THEN 0 209 | ELSE Max(ind) + 1 210 | IN 211 | /\ currentState[i] = Leader 212 | /\ nextIndex \in Index 213 | /\ \E v \in Value :leaderLog' = [leaderLog EXCEPT ![currentTerm[i]][nextIndex] = 214 | <>] 215 | /\ UNCHANGED <> 216 | 217 | Next == \/ \E i \in Server: Restart(i) 218 | \/ \E i \in Server: Timeout(i) 219 | \/ \E i \in Server: ReceiveHighTerm(i) 220 | \/ \E i \in Server: RequestVote(i) 221 | \/ \E i \in Server : HandleRequestVoteRequest(i) 222 | \/ \E i \in Server : BecomeLeaderCandidate(i) 223 | \/ \E i \in Server : BecomeLeader(i) 224 | \/ \E i \in Server : CommitEntry(i) 225 | \/ \E i \in Server : ClientRequest(i) 226 | \/ \E i,j \in Server : RequestCommit(i) 227 | \/ \E i \in Server : HandleRequestCommitRequest(i) 228 | \/ \E i,j \in Server : RequestSync(i) 229 | \/ \E i \in Server : HandleRequestSyncRequest(i) 230 | 231 | 232 | Inv == /\ TypeInv 233 | 234 | ------------------------------------------------------------------------------------------------ 235 | Acceptors == Server 236 | Ballots == Term 237 | Instances == Index 238 | ballot == currentTerm 239 | leaderVote == [i \in Ballots |-> [j \in Index |-> <>]] 240 | 1amsgs == {<>: m \in r1amsgs} 241 | 1bmsgs == {<> >> : e \in m[2]},m[3]>> : m \in r1bmsgs} 242 | 2amsgs == {<> >>: m \in r2amsgs} 243 | 244 | Spec == Init /\ [][Next]_vars 245 | 246 | A == INSTANCE MultiPaxos 247 | 248 | THEOREM Refinement == Spec => A!Spec 249 | 250 | 251 | ============================================================================= 252 | \* Modification History 253 | \* Last modified Wed Sep 09 15:26:16 CST 2020 by 15150 254 | 255 | --------------------------------------------------------------------------------