├── .gitignore
├── .idea
├── misc.xml
└── modules.xml
├── README.md
├── class-diagram-1.png
├── class-diagram-2.png
├── config
└── src
└── java
├── Paxos-log-replication-Java.iml
└── main
├── paxos
├── Replica.java
└── synod
│ ├── Acceptor.java
│ └── Leader.java
└── utilities
├── BallotComparator.java
├── Environment.java
├── PaxosMsgs.java
├── SingleNodeEnvironment.java
└── message_specification.proto
/.gitignore:
--------------------------------------------------------------------------------
1 | # Created by .ignore support plugin (hsz.mobi)
2 | ### Java template
3 | *.class
4 |
5 | # BlueJ files
6 | *.ctxt
7 |
8 | # Mobile Tools for Java (J2ME)
9 | .mtj.tmp/
10 |
11 | # Package Files #
12 | *.jar
13 | *.war
14 | *.ear
15 |
16 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
17 | hs_err_pid*
18 |
19 |
20 |
21 | # Created by https://www.gitignore.io/api/macos,intellij
22 |
23 | ### macOS ###
24 | *.DS_Store
25 | .AppleDouble
26 | .LSOverride
27 |
28 | # Icon must end with two \r
29 | Icon
30 | # Thumbnails
31 | ._*
32 | # Files that might appear in the root of a volume
33 | .DocumentRevisions-V100
34 | .fseventsd
35 | .Spotlight-V100
36 | .TemporaryItems
37 | .Trashes
38 | .VolumeIcon.icns
39 | .com.apple.timemachine.donotpresent
40 | # Directories potentially created on remote AFP share
41 | .AppleDB
42 | .AppleDesktop
43 | Network Trash Folder
44 | Temporary Items
45 | .apdisk
46 |
47 |
48 | ### Intellij ###
49 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
50 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
51 |
52 | # User-specific stuff:
53 | .idea/workspace.xml
54 | .idea/tasks.xml
55 |
56 | # Sensitive or high-churn files:
57 | .idea/dataSources/
58 | .idea/dataSources.ids
59 | .idea/dataSources.xml
60 | .idea/dataSources.local.xml
61 | .idea/sqlDataSources.xml
62 | .idea/dynamic.xml
63 | .idea/uiDesigner.xml
64 |
65 | # Gradle:
66 | .idea/gradle.xml
67 | .idea/libraries
68 |
69 | # Mongo Explorer plugin:
70 | .idea/mongoSettings.xml
71 |
72 | ## File-based project format:
73 | *.iws
74 |
75 | ## Plugin-specific files:
76 |
77 | # IntelliJ
78 | /out/
79 |
80 | # mpeltonen/sbt-idea plugin
81 | .idea_modules/
82 |
83 | # JIRA plugin
84 | atlassian-ide-plugin.xml
85 |
86 | # Crashlytics plugin (for Android Studio and IntelliJ)
87 | com_crashlytics_export_strings.xml
88 | crashlytics.properties
89 | crashlytics-build.properties
90 | fabric.properties
91 |
92 | ### Intellij Patch ###
93 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
94 |
95 | # *.iml
96 | # modules.xml
97 | # .idea/misc.xml
98 | # *.ipr
99 |
100 | # End of https://www.gitignore.io/api/macos,intellij
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Paxos Java implementation
2 |
3 | ## What is this?
4 |
5 | Implementation of multi-paxos protocol.
6 |
7 | ## How to demo this project?
8 |
9 | #### How to build?
10 |
11 | I use Intellij to manage my project. To build project, just
12 | import project to Intellij, don't forget to add google-protobuf library, then build.
13 |
14 | #### How to run?
15 |
16 | * First, in "config" file, decide the system topology(how many leaders, replicas and acceptors in the system).
17 |
18 | * The main method is in utilities.SingleNodeEnvironment. It accepts two args: the first is config file path, and the second is the number of requests(RSM commands) that need to be decided by Paxos protocol.
19 |
20 | * Run the program, you will see the result in console.
21 |
22 | ## Understand Paxos protocol
23 |
24 | ####Just read these papers
25 |
26 | * The Part-Time Parliament, by Leslie Lamport
27 | * Paxos Made Simple, by Leslie Lamport
28 | * Paxos Made Live, by Tushar Chandra, Robert Griesemer and Joshua Redstone
29 | * Paxos Made Moderately Complex, by ROBBERT VAN RENESSE and DENIZ ALTINBUKEN
30 |
31 | #### Basic distributed computing concepts related to Paxos
32 |
33 | * Time models -- synchronous, asynchronous, partial synchronous
34 | * Failure models -- fail-stop, byzantine
35 | * Definition of the consensus problem -- agreement, validity, termination
36 | * Replia state machine(RSM)
37 | * The FLP impossible result
38 | * And other RSM protocols -- Zab(Zookeeper), Viewstamp, Raft (Paxos, Viewstamp and Raft are based on similar ideas)
39 |
40 | An excellent must-read book -- Distributed Algorithms, by Nancy A. Lynch
41 |
42 | ## Understand the code
43 |
44 | Here is the class diagram of key classed in the project:
45 | 
46 |
47 | #### Some explanation
48 |
49 | * Replica, Leader and Acceptor are three key roles in Paxos protocol.
50 |
51 | * Do not confuse with 'Leader' here. The Leader in Paxos is the conductor of each ballot. You may think what is the difference between Leader here and Leader in Raft or Viewstamp.
52 | Leader in Raft or Viewstamp is more like the "prime" leader of all leaders in Paxos -- Raft and Viewstamp do a leader election. In Paxos, every leader can propose, while in Raft and Viewstamp, only the selected 'prime' leader can propose.
53 |
54 | * Scout and Commander are two helping roles in Leader. Scout is for phase 1 of protocol and Command for phase 2. They are designed as inner classes in Leader for simplicity.
55 |
56 |
57 | ##### For more infomation, pls see comments in code.
58 |
--------------------------------------------------------------------------------
/class-diagram-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SabaPing/Paxos-log-replication-Java/3fb39cd86ff786a85b88871814aa576d317735a4/class-diagram-1.png
--------------------------------------------------------------------------------
/class-diagram-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/SabaPing/Paxos-log-replication-Java/3fb39cd86ff786a85b88871814aa576d317735a4/class-diagram-2.png
--------------------------------------------------------------------------------
/config:
--------------------------------------------------------------------------------
1 | * Topology assumption:
2 | * every client has bidirectional channel to every replica,
3 | * every replica has bidirectional channel to every leader,
4 | * and every leader has bidirectional channel to every acceptor.
5 |
6 | * a pair of ip address and port# uniquely defines a role.
7 | * below define all roles, for system with f <= 1. A role is assigned to a host as:
8 |
9 | * Format:Role ID
10 |
11 | * replicas
12 | R 1
13 | R 2
14 |
15 | * leaders
16 | L 5
17 | L 6
18 |
19 | * acceptors
20 | A 9
21 | A 10
22 | A 11
23 |
24 |
--------------------------------------------------------------------------------
/src/java/Paxos-log-replication-Java.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/src/java/main/paxos/Replica.java:
--------------------------------------------------------------------------------
1 | package paxos;
2 |
3 | import utilities.Environment;
4 | import utilities.PaxosMsgs.*;
5 |
6 | import static utilities.PaxosMsgs.Paxos.Type.*;
7 |
8 | import java.util.*;
9 |
10 | /**
11 | * Invariants for replicas:
12 | * R1: There are no two different commands decided for the same slot.
13 | * R2: All commands up to slot out are in the set of decisions.
14 | * R3: For all replicas ρ, ρ.state is the result of applying
15 | * the commands⟨s,cs⟩∈ρ.decisions to initial_state for all s up to slot_out,
16 | * in order of slot number.
17 | * R4: For each ρ, the variable ρ.slot out cannot decrease over time.
18 | * R5: A replica proposes commands only for slots for which it knows the configuration.
19 | *
20 | * And R1 is ensured by synod(i.e., simple paxos)
21 | *
22 | * Did not implement reconfig function!!!
23 | */
24 | public class Replica extends Thread {
25 | //paxos related thread
26 | private int slot_in;
27 | private int slot_out;
28 | private final Queue requests;
29 | private final Map proposals;
30 | private final Map decisions;
31 | private final int ID;
32 |
33 | //for configuration change
34 | private final int WINDOW;
35 |
36 | //For sending and receiving messages
37 | private final Environment environment;
38 |
39 | public Replica(int ID, Environment environment) {
40 | super("Replica-" + ID);
41 | this.ID = ID;
42 | this.environment = environment;
43 | this.decisions = new HashMap<>();
44 | this.proposals = new HashMap<>();
45 | this.requests = new ArrayDeque<>();
46 | this.slot_in = 1;
47 | this.slot_out = 1;
48 |
49 | //???ease conflicts between ballot???
50 | this.WINDOW = 3;
51 | }
52 |
53 | /**
54 | * Transfer requests from requests to proposals
55 | */
56 | private void propose() {
57 | while ((slot_in < slot_out + WINDOW) && !requests.isEmpty()) {
58 | //if not already proposed for slot_in
59 | if (!decisions.containsKey(slot_in)) {
60 | Command polledCmd = requests.poll();
61 | proposals.put(slot_in, polledCmd);
62 |
63 | Paxos temp_propose = Paxos.newBuilder()
64 | .setType(PROPOSE)
65 | .setPropose(Propose.newBuilder()
66 | .setSlotIn(slot_in)
67 | .setC(polledCmd))
68 | .build();
69 |
70 | for (int leader : environment.getLeaders()) {
71 | environment.send(leader, temp_propose);
72 | }
73 | }
74 | slot_in++;
75 | }
76 | }
77 |
78 | /**
79 | * The function perform() is invoked with the same sequence of commands at all replicas.
80 | * Different replicas may end up proposing the same command for different slots,
81 | * and thus the same command may be decided multiple times.
82 | */
83 | private void perform(Command cmd) {
84 | //if it has already performed the command
85 | for (int i = 1; i < slot_out; i++) {
86 | if (decisions.get(i).equals(cmd)) {
87 | slot_out++;
88 | return;
89 | }
90 | }
91 |
92 | System.out.println("Replica " + ID + ": perform slot " + slot_out + " command " + cmd.getOperation());
93 | slot_out++;
94 |
95 | /**
96 | * for debugging and simplicity, no client here
97 | */
98 | // Paxos temp_response = Paxos.newBuilder()
99 | // .setType(RESPONSE)
100 | // .setResponse(Response.newBuilder()
101 | // .setCid(cmd.getCid())
102 | // .setResult("Performed"))
103 | // .build();
104 | //
105 | // environment.send(cmd.getClient(), temp_response);
106 | }
107 |
108 | public void run() {
109 | while (true) {
110 | Paxos incMsg = environment.receive(ID);
111 | switch (incMsg.getType()) {
112 | case REQUEST: {
113 | requests.offer(incMsg.getRequest().getC());
114 | break;
115 | }
116 |
117 | /**
118 | * Decisions may arrive out of order and multiple times
119 | */
120 | case DECISION: {
121 | //adds the decision to the set decisions
122 | Decision body = incMsg.getDecision();
123 | decisions.put(body.getSlotNum(), body.getC());
124 |
125 | /**
126 | * If there is a decision c′ corresponding to the current slot out,
127 | * the replica first checks to see if it has proposed a command c′′ for that slot.
128 | * If so, the replica removes ⟨slot out,c′′⟩ from the set proposals.
129 | * If c′′ ̸= c′, the replica returns c′′ to set requests
130 | */
131 | while (decisions.containsKey(slot_out)) {
132 | if (proposals.containsKey(slot_out)) {
133 | if (!proposals.get(slot_out).equals(decisions.get(slot_out))) {
134 | requests.offer(proposals.get(slot_out));
135 | }
136 | proposals.remove(slot_out);
137 | }
138 | perform(decisions.get(slot_out));
139 | }
140 | }
141 | }
142 | propose();
143 | }
144 | }
145 | }
146 |
--------------------------------------------------------------------------------
/src/java/main/paxos/synod/Acceptor.java:
--------------------------------------------------------------------------------
1 | package paxos.synod;
2 |
3 | import utilities.BallotComparator;
4 | import utilities.Environment;
5 |
6 | import java.util.*;
7 |
8 | import utilities.PaxosMsgs.*;
9 |
10 | import static utilities.PaxosMsgs.Paxos.Type.*;
11 |
12 | /**
13 | * If process is an acceptor, it only has a single thread.
14 | */
15 | public class Acceptor extends Thread {
16 |
17 | //Paxos related states
18 | private Ballot acceptorBallot;
19 | private final Set accepted;
20 | private final int ID;
21 |
22 | //For sending and receiving messages
23 | private final Environment environment;
24 |
25 | //Initial states
26 | public Acceptor(int id, Environment environment) {
27 | super("Acceptor-" + id);
28 | acceptorBallot = Ballot.newBuilder()
29 | .setPrefix(0)
30 | .setConductor(0)
31 | .build();
32 | accepted = new HashSet<>();
33 | this.environment = environment;
34 | this.ID = id;
35 | }
36 |
37 | @Override
38 | public void run() {
39 | while (true) {
40 | //For each message received, do something
41 | Paxos message = environment.receive(ID);
42 |
43 | //Acceptor can receive two types of messages
44 | switch (message.getType()) {
45 | //received a p1a message from a scout
46 | case P1A: {
47 | P1a body = message.getP1A();
48 |
49 | //if received ballot is larger than acceptor's ballot
50 | if (new BallotComparator().compare(body.getBallot(), acceptorBallot) > 0)
51 | acceptorBallot = body.getBallot();
52 |
53 | // reply to the scout
54 | environment.send(body.getFromLeader(),
55 | Paxos.newBuilder()
56 | .setType(P1B)
57 | .setP1B(P1b.newBuilder()
58 | .setFrom(ID)
59 | .setToScout(body.getFromScout())
60 | .setBallot(acceptorBallot)
61 | .setAccepted(Accepted.newBuilder()
62 | .addAllPvalue(accepted)))
63 | .build());
64 | break;
65 | }
66 | //p2a from a commander, code almost the same
67 | case P2A: {
68 | P2a body = message.getP2A();
69 | if (new BallotComparator().compare(body.getPvalue().getBallot(), acceptorBallot) == 0)
70 | accepted.add(body.getPvalue());
71 |
72 | environment.send(body.getFromLeader(),
73 | Paxos.newBuilder()
74 | .setType(P2B)
75 | .setP2B(P2b.newBuilder()
76 | .setFrom(ID)
77 | .setToCommander(body.getFromCommander())
78 | .setBallot(acceptorBallot))
79 | .build());
80 | break;
81 | }
82 | }
83 | }
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/src/java/main/paxos/synod/Leader.java:
--------------------------------------------------------------------------------
1 | package paxos.synod;
2 |
3 | import utilities.BallotComparator;
4 | import utilities.PaxosMsgs.*;
5 | import utilities.Environment;
6 |
7 | import java.util.*;
8 | import java.util.concurrent.ArrayBlockingQueue;
9 | import java.util.concurrent.BlockingQueue;
10 | import java.util.concurrent.ConcurrentHashMap;
11 | import java.util.concurrent.atomic.AtomicInteger;
12 | import static utilities.PaxosMsgs.Paxos.Type.*;
13 |
14 |
15 | /**
16 | * The Leader class has two Paxos related inner classes, which are Scout and Commander.
17 | * The leader is the main thread. Scouts and Commanders are launched by the leader.
18 | * Use inner classes to simplify state sharing.
19 | *
20 | * Inter threads communication is done by message queues and MessageHandler thread.
21 | * For one-to-one mapping between each thread and its message queue, each thread need a local id,
22 | * Use ThreadLocal to implement thread local id.
23 | *
24 | * All threads are executed in an almost-asynchronous manner,
25 | * the only very weak synchronized part are message queues.
26 | */
27 | public class Leader extends Thread{
28 |
29 | //Paxos related states, no one is shared, thus thread safe.
30 | private Ballot leaderBallot;
31 | private final Map proposals;
32 | private final int ID;
33 | private boolean active;
34 |
35 | //shared states for inter-threads communication, need thread-safe mechanism
36 | private final Map> messageQueues;
37 | private final BlockingQueue incomingMessages;
38 |
39 | //sending and receiving messages
40 | private final Environment environment;
41 |
42 | public Leader(int id, Environment environment) {
43 | super("Leader-" + id);
44 | ID = id;
45 | leaderBallot = Ballot.newBuilder().setPrefix(0).setConductor(id).build();
46 | active = false;
47 | proposals = new HashMap<>();
48 |
49 | this.environment = environment;
50 |
51 | //initial and register message queue
52 | messageQueues = new ConcurrentHashMap<>();
53 | this.incomingMessages = new ArrayBlockingQueue<>(100);
54 | }
55 |
56 | @Override
57 | public void run() {
58 | new MessageHandler(ThreadID.get()).start();
59 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
60 |
61 | //To ensure states is not shared, build a new ballot
62 | new Scout(Ballot.newBuilder(leaderBallot).build()).start();
63 |
64 | try {
65 | while (true)
66 | forever();
67 | } catch (InterruptedException e) {
68 | e.printStackTrace();
69 | } finally {
70 | //leader never end. not necessary to de-register its message Q.
71 | }
72 | }
73 |
74 | /**
75 | * this method transit shared states, need careful concurrent design
76 | * @throws InterruptedException
77 | */
78 | private void forever() throws InterruptedException{
79 | Paxos incMsg = incomingMessages.take();
80 | switch (incMsg.getType()) {
81 | case PROPOSE: {
82 | Propose temp_propose = incMsg.getPropose();
83 | if (!proposals.containsKey(temp_propose.getSlotIn())){
84 | proposals.put(temp_propose.getSlotIn(), temp_propose.getC());
85 | if (active) {
86 | /**
87 | * spawn a Commander
88 | * have some concerns! Is protobuf really immutable???
89 | * Assume it is!
90 | */
91 | new Commander(PValue.newBuilder()
92 | .setBallot(leaderBallot)
93 | .setSlotNum(temp_propose.getSlotIn())
94 | .setCmd(temp_propose.getC())
95 | .build()).start();
96 | }
97 | }
98 | }
99 | case ADOPTED: {
100 | Adopted temp_adopted = incMsg.getAdopted();
101 | if (leaderBallot.equals(temp_adopted.getBallot())){
102 | /**
103 | * updates the set of proposals, replacing for each slot number
104 | * the command corresponding to the maximum pvalue in pvals, if any.
105 | */
106 | Map pmax = new HashMap<>();
107 | for(PValue pv : temp_adopted.getPvalues().getPvalueList()){
108 | if(!pmax.containsKey(pv.getSlotNum()) ||
109 | new BallotComparator().compare(pmax.get(pv.getSlotNum()), pv.getBallot()) < 0)
110 | proposals.put(pv.getSlotNum(), pv.getCmd());
111 | }
112 | for(Map.Entry entry : proposals.entrySet()) {
113 | PValue temp_pv = PValue.newBuilder()
114 | .setBallot(leaderBallot)
115 | .setSlotNum(entry.getKey())
116 | .setCmd(entry.getValue())
117 | .build();
118 | new Commander(temp_pv).start();
119 | }
120 | active = true;
121 | }
122 | break;
123 | }
124 | case PREEMPTED: {
125 | Preempted temp_preempted = incMsg.getPreempted();
126 | if (new BallotComparator().compare(temp_preempted.getBallot(), leaderBallot) > 0) {
127 | active = false;
128 | leaderBallot = Ballot.newBuilder()
129 | .setPrefix(temp_preempted.getBallot().getPrefix() + 1)
130 | .setConductor(ID)
131 | .build();
132 | //ensure states are not shared
133 | new Scout(Ballot.newBuilder(leaderBallot).build()).start();
134 | }
135 | break;
136 | }
137 | }
138 | }
139 |
140 | /**
141 | * 这里scouts由leader id + leaderBallot 唯一确定，commanders由leader+leaderBallot+slot唯一确定。
142 | * 对每个不同的scout和commander threads，里面都要有个blocking Q。
143 | * MessageHandler 必须知道msg type，leaderBallot，和proposal，把对应的msg put到对应的Q里。
144 | * 以上保证了reliable。
145 | * 注意：见paper笔记。需要在msg里加额外的from，to fields
146 | * And give scout and commander 一个额外的 int id, use ThreadLocal
147 | */
148 | private class MessageHandler extends Thread{
149 |
150 | /**
151 | * message handler must know its leader's id.
152 | * No need to know scouts and commanders, their IDs are in message body.
153 | */
154 | private final int leadLocalId;
155 |
156 | public MessageHandler (int leadLocalId) {
157 | super("MessageHandler");
158 | this.leadLocalId = leadLocalId;
159 | }
160 |
161 | @Override
162 | public void run() {
163 | while(true) {
164 |
165 | //This statement will be blocked if there is no incoming message.
166 | Paxos message = environment.receive(ID);
167 |
168 | try {
169 | switch (message.getType()) {
170 | case PROPOSE: {
171 | messageQueues.get(leadLocalId).put(message);
172 | break;
173 | }
174 | case P1B: {
175 | int scoutId = message.getP1B().getToScout();
176 | if (!messageQueues.containsKey(scoutId)) break;
177 | messageQueues.get(scoutId).put(message);
178 | break;
179 | }
180 | case P2B: {
181 | int commanderId = message.getP2B().getToCommander();
182 | if (!messageQueues.containsKey(commanderId)) break;
183 | messageQueues.get(commanderId).put(message);
184 | break;
185 | }
186 | }
187 | }catch (InterruptedException e){
188 | e.printStackTrace();
189 | }
190 | }
191 | }
192 |
193 | }
194 |
195 | /**
196 | * inner class Scout
197 | * inter-thread communication -- use message queue
198 | * do synod phase 1
199 | */
200 | private class Scout extends Thread{
201 |
202 | //Paxos related states, must ensure not shared.
203 | private final Set pValueSet;
204 | private final Ballot scoutBallot;
205 | private final List waitforAccetpors;
206 |
207 | private final BlockingQueue incomingMessages;
208 |
209 | public Scout(Ballot ballot) {
210 | super("Scout in " + ID + " with Ballot " + ballot);
211 | scoutBallot = ballot;
212 | pValueSet = new HashSet<>();
213 | this.incomingMessages = new ArrayBlockingQueue<>(100);
214 |
215 | //fuck you ThreadLocal, in
216 | waitforAccetpors = new ArrayList<>(environment.getAcceptors());
217 | }
218 |
219 | @Override
220 | public void run() {
221 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
222 |
223 | Paxos p1a = Paxos.newBuilder()
224 | .setType(P1A)
225 | .setP1A(P1a.newBuilder()
226 | .setFromLeader(ID)
227 | .setFromScout(ThreadID.get())
228 | .setBallot(scoutBallot))
229 | .build();
230 |
231 | for(int acceptor : waitforAccetpors){
232 | environment.send(acceptor, p1a);
233 | }
234 |
235 | try {
236 | while(true){
237 |
238 | //because of message handler, scout can only get p1b msg
239 | P1b incMsg = incomingMessages.take().getP1B();
240 |
241 | if(incMsg.getBallot().equals(scoutBallot)){
242 | pValueSet.addAll(incMsg.getAccepted().getPvalueList());
243 | //use Integer.valueOf to use internal cache
244 | waitforAccetpors.remove(Integer.valueOf(incMsg.getFrom()));
245 |
246 | if(waitforAccetpors.size() < environment.getAcceptors().size()/2){
247 | //偷懒实现了，由于inner class知道leader的Q。。。
248 | Leader.this.incomingMessages.put(Paxos.newBuilder()
249 | .setType(ADOPTED)
250 | .setAdopted(Adopted.newBuilder()
251 | .setBallot(scoutBallot)
252 | .setPvalues(Accepted.newBuilder()
253 | .addAllPvalue(pValueSet)))
254 | .build());
255 | //thread exit
256 | return;
257 | }
258 | } else {
259 | Leader.this.incomingMessages.put(Paxos.newBuilder()
260 | .setType(PREEMPTED)
261 | .setPreempted(Preempted.newBuilder()
262 | .setBallot(incMsg.getBallot()))
263 | .build());
264 | return;
265 | }
266 |
267 | }
268 | } catch (InterruptedException e) {
269 | e.printStackTrace();
270 | } finally {
271 | //clean up, remove its message queue from map.
272 | messageQueues.remove(ThreadID.get());
273 | }
274 | }
275 | }
276 |
277 | /**
278 | * inner class Commander
279 | * do synod phase 2
280 | */
281 | private class Commander extends Thread{
282 | private final List waitforAccetpors;
283 | private final PValue pvalue;
284 |
285 | private final BlockingQueue incomingMessages;
286 |
287 | public Commander(PValue pvalue) {
288 | super("Commander in " + ID + " with pvalue " + pvalue);
289 | this.pvalue = pvalue;
290 | waitforAccetpors = new ArrayList<>(environment.getAcceptors());
291 | incomingMessages = new ArrayBlockingQueue<>(100);
292 | }
293 |
294 | @Override
295 | public void run() {
296 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
297 | Paxos p2a = Paxos.newBuilder()
298 | .setType(P2A)
299 | .setP2A(P2a.newBuilder()
300 | .setFromLeader(ID)
301 | .setFromCommander(ThreadID.get())
302 | .setPvalue(pvalue))
303 | .build();
304 |
305 | for(int acceptor : waitforAccetpors){
306 | //de-register its message queue
307 | environment.send(acceptor, p2a);
308 | }
309 |
310 | try {
311 | while(true){
312 | //because of message handler, scout can only get p2b msg
313 | P2b incMsg = incomingMessages.take().getP2B();
314 |
315 | if(incMsg.getBallot().equals(pvalue.getBallot())){
316 | //use Integer.valueOf to use internal cache
317 | waitforAccetpors.remove(Integer.valueOf(incMsg.getFrom()));
318 | if(waitforAccetpors.size() < environment.getAcceptors().size()/2){
319 | Paxos decisionToReplicas = Paxos.newBuilder()
320 | .setType(DECISION)
321 | .setDecision(Decision.newBuilder()
322 | .setSlotNum(pvalue.getSlotNum())
323 | .setC(pvalue.getCmd()))
324 | .build();
325 |
326 | for(int replica : environment.getReplicas()) {
327 | environment.send(replica, decisionToReplicas);
328 | }
329 | return;
330 | }
331 | } else {
332 | Leader.this.incomingMessages.put(Paxos.newBuilder()
333 | .setType(PREEMPTED)
334 | .setPreempted(Preempted.newBuilder()
335 | .setBallot(incMsg.getBallot()))
336 | .build());
337 | return;
338 | }
339 | }
340 | } catch (InterruptedException e) {
341 | e.printStackTrace();
342 | } finally {
343 | messageQueues.remove(ThreadID.get());
344 | }
345 | }
346 | }
347 |
348 | /**
349 | * really got fucked when call threadlocal get in Thread's constructor.
350 | * The value returned by get() is not for this thread, but for the thread who call the constructor.
351 | * Target thread has not stared when its constructor is being called.
352 | */
353 | public static class ThreadID {
354 | private static final AtomicInteger nextID = new AtomicInteger(0);
355 |
356 | /**
357 | * 所以这里把ThreadLocal encapsulate的目的是实现auto-increment
358 | * note: 整个class是thread safe的，遵循了书上将的模式
359 | *
360 | * 仔细想想，下面这个obj，其实只有一个，所有threads的threadlocalmap的key都是指向这个obj的
361 | * 知道lead.class不被gc，那么他就是strongly reachable.
362 | */
363 | private static final ThreadLocal threadID = new ThreadLocal(){
364 | @Override
365 | protected Integer initialValue() {
366 | return nextID.getAndIncrement();
367 | }
368 | };
369 |
370 | public static int get() {
371 | return threadID.get();
372 | }
373 | }
374 | }
--------------------------------------------------------------------------------
/src/java/main/utilities/BallotComparator.java:
--------------------------------------------------------------------------------
1 | package utilities;
2 |
3 | import utilities.PaxosMsgs;
4 |
5 | import java.util.Comparator;
6 |
7 |
8 | public class BallotComparator implements Comparator{
9 | @Override
10 | public int compare(PaxosMsgs.Ballot o1, PaxosMsgs.Ballot o2) {
11 | String b1 = "" + o1.getPrefix() + o1.getConductor();
12 | String b2 = "" + o2.getPrefix() + o2.getConductor();
13 | return b1.compareTo(b2);
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/java/main/utilities/Environment.java:
--------------------------------------------------------------------------------
1 | package utilities;
2 |
3 | import java.util.List;
4 |
5 | /**
6 | * This interface provide network abstraction.
7 | * Except for methods in this interface, the whole system know nothing about network.
8 | *
9 | * Not assume FIFO, only reliability.
10 | *
11 | * Classes implementing this interface store ID-address mappings,
12 | * which from the config file.
13 | *
14 | * Message boundary design -- 1 byte length
15 | *
16 | */
17 | public interface Environment {
18 |
19 | public List getReplicas();
20 |
21 | public List getLeaders();
22 |
23 | public List getAcceptors();
24 |
25 | public void send(int toID, PaxosMsgs.Paxos msg);
26 |
27 | public PaxosMsgs.Paxos receive(int id);
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/src/java/main/utilities/SingleNodeEnvironment.java:
--------------------------------------------------------------------------------
1 | package utilities;
2 |
3 |
4 | import java.io.IOException;
5 | import java.nio.file.Files;
6 | import java.nio.file.Path;
7 | import java.nio.file.Paths;
8 | import java.util.ArrayList;
9 | import java.util.List;
10 | import java.util.Map;
11 | import java.util.Scanner;
12 | import java.util.concurrent.*;
13 |
14 | import paxos.Replica;
15 | import paxos.synod.Acceptor;
16 | import paxos.synod.Leader;
17 | import utilities.PaxosMsgs.Paxos;
18 |
19 | /**
20 | * non-distributed mode implementation.
21 | * is used for debugging.
22 | */
23 | public class SingleNodeEnvironment implements Environment {
24 |
25 | private final List replicas;
26 | private final List leaders;
27 | private final List acceptors;
28 |
29 | private final Map> msgQueueMap;
30 |
31 | private final BlockingQueue result;
32 |
33 | public SingleNodeEnvironment(String config) {
34 | replicas = new ArrayList<>();
35 | leaders = new ArrayList<>();
36 | acceptors = new ArrayList<>();
37 | msgQueueMap = new ConcurrentHashMap<>();
38 | result = new ArrayBlockingQueue<>(100);
39 | parseConfig(config);
40 | buildMsgQueue();
41 | }
42 |
43 | private void parseConfig(String filePath) {
44 | Path config = Paths.get(filePath);
45 | try (Scanner sc = new Scanner(Files.newBufferedReader(config))) {
46 | while (sc.hasNextLine()) {
47 | String line = sc.nextLine();
48 | if (line.length() == 0 || line.charAt(0) == '*') continue;
49 | char role = line.charAt(0);
50 | int id = Integer.parseInt(line.substring(2));
51 |
52 | switch (role) {
53 | case 'R': {
54 | replicas.add(id);
55 | break;
56 | }
57 | case 'L': {
58 | leaders.add(id);
59 | break;
60 | }
61 | case 'A': {
62 | acceptors.add(id);
63 | break;
64 | }
65 | }
66 | }
67 | } catch (IOException e) {
68 | System.out.println("Cannot parse config file");
69 | e.printStackTrace();
70 | }
71 | }
72 |
73 | private void buildMsgQueue() {
74 | for (int i : replicas) {
75 | msgQueueMap.put(i, new ArrayBlockingQueue<>(100));
76 | }
77 | for (int i : leaders) {
78 | msgQueueMap.put(i, new ArrayBlockingQueue<>(100));
79 | }
80 | for (int i : acceptors) {
81 | msgQueueMap.put(i, new ArrayBlockingQueue<>(100));
82 | }
83 | }
84 |
85 | @Override
86 | public List getReplicas() {
87 | return replicas;
88 | }
89 |
90 | @Override
91 | public List getLeaders() {
92 | return leaders;
93 | }
94 |
95 | @Override
96 | public List getAcceptors() {
97 | return acceptors;
98 | }
99 |
100 | @Override
101 | public void send(int toID, PaxosMsgs.Paxos msg) {
102 | try {
103 | msgQueueMap.get(toID).put(msg);
104 | } catch (InterruptedException e) {
105 | e.printStackTrace();
106 | }
107 | }
108 |
109 | @Override
110 | public Paxos receive(int who) {
111 | //todo null is bad, need to define a default Paxos msg
112 | Paxos msg = null;
113 | try {
114 | msg = msgQueueMap.get(who).take();
115 | } catch (InterruptedException e) {
116 | e.printStackTrace();
117 | }
118 | return msg;
119 | }
120 |
121 | /**
122 | * needs two args: config file and # of requests
123 | *
124 | * @param args
125 | */
126 | public static void main(String[] args) {
127 | Environment env = new SingleNodeEnvironment(args[0]);
128 |
129 | //here, using fixedThreadPool is wrong. Need an unbound pool.
130 | // ExecutorService threadPool = Executors.newCachedThreadPool();
131 |
132 | //start all nodes
133 | for (int i : env.getReplicas()) {
134 | new Replica(i, env).start();
135 | }
136 | for (int i : env.getAcceptors()) {
137 | new Acceptor(i, env).start();
138 | }
139 | for (int i : env.getLeaders()) {
140 | new Leader(i, env).start();
141 | }
142 |
143 | //sending some requests, assume client id is 99
144 | int numOfReq = Integer.parseInt(args[1]);
145 | for (int i = 1; i <= numOfReq; i++) {
146 | Paxos tempMsg = Paxos.newBuilder()
147 | .setType(Paxos.Type.REQUEST)
148 | .setRequest(PaxosMsgs.Request.newBuilder()
149 | .setC(PaxosMsgs.Command.newBuilder()
150 | .setClient(99)
151 | .setCid(i)
152 | .setOperation("Request # " + i)))
153 | .build();
154 | for (int r : env.getReplicas()) {
155 | env.send(r, tempMsg);
156 | }
157 |
158 | try {
159 | Thread.sleep(1000);
160 | } catch (InterruptedException e) {
161 | e.printStackTrace();
162 | }
163 | }
164 | }
165 |
166 | }
167 |
--------------------------------------------------------------------------------
/src/java/main/utilities/message_specification.proto:
--------------------------------------------------------------------------------
1 | syntax = "proto2";
2 | package paxos;
3 |
4 | option java_outer_classname = "PaxosMsgs";
5 |
6 | //wrapper. Add a protocol header to every message
7 | message Paxos {
8 | enum Type {
9 | P1A = 1;
10 | P1B = 2;
11 | P2A = 3;
12 | P2B = 4;
13 |
14 | REQUEST = 5;
15 | RESPONSE = 6;
16 |
17 | PROPOSE = 7;
18 | DECISION = 8;
19 |
20 | ADOPTED = 9;
21 | PREEMPTED = 10;
22 | }
23 |
24 | required Type type = 1;
25 |
26 | optional P1a p1a = 2;
27 | optional P1b p1b = 3;
28 | optional P2a p2a = 4;
29 | optional P2b p2b = 5;
30 | optional Request request = 6;
31 | optional Response response = 7;
32 | optional Propose propose = 8;
33 | optional Decision decision = 9;
34 | optional Adopted adopted = 10;
35 | optional Preempted preempted = 11;
36 | }
37 |
38 | //message in synod
39 | message P1a {
40 | required int32 fromLeader = 1;
41 | required int32 fromScout = 2;
42 | required Ballot ballot = 3;
43 | }
44 |
45 | message P1b {
46 | required int32 from = 1;
47 | required int32 toScout = 2;
48 | required Ballot ballot = 3;
49 | required Accepted accepted = 4;
50 | }
51 |
52 | message P2a {
53 | required int32 fromLeader = 1;
54 | required int32 fromCommander = 2;
55 | required PValue pvalue = 3;
56 | }
57 |
58 | message P2b {
59 | required int32 from = 1;
60 | required int32 toCommander = 2;
61 | required Ballot ballot = 3;
62 | }
63 |
64 | //message between replicas and clients
65 | message Request {
66 | required Command c = 1;
67 | }
68 |
69 | message Response {
70 | required int32 cid = 1;
71 | required string result = 2;
72 | }
73 |
74 | //messages between replicas and synod
75 | message Propose {
76 | required int32 slot_in = 1;
77 | required Command c = 2;
78 | }
79 |
80 | message Decision {
81 | required int32 slotNum = 1;
82 | required Command c = 2;
83 | }
84 |
85 | //messages for IPC in leader
86 | message Adopted {
87 | required Ballot ballot = 1;
88 | required Accepted pvalues = 2;
89 | }
90 |
91 | message Preempted {
92 | required Ballot ballot = 1;
93 | }
94 |
95 |
96 |
97 |
98 |
99 | //helper structure
100 | message Ballot {
101 | required int32 prefix = 1;
102 | required int32 conductor = 2;
103 |
104 | }
105 |
106 | message PValue {
107 | required Ballot ballot = 1;
108 | required int32 slotNum = 2;
109 | required Command cmd = 3;
110 |
111 | }
112 |
113 | message Accepted {
114 | repeated PValue pvalue = 1;
115 | }
116 |
117 | message Command {
118 | required int32 client = 1;
119 | required int32 cid = 2;
120 | required string operation = 3;
121 | }
--------------------------------------------------------------------------------