getAcceptors() {
97 | return acceptors;
98 | }
99 |
100 | @Override
101 | public void send(int toID, PaxosMsgs.Paxos msg) {
102 | try {
103 | msgQueueMap.get(toID).put(msg);
104 | } catch (InterruptedException e) {
105 | e.printStackTrace();
106 | }
107 | }
108 |
109 | @Override
110 | public Paxos receive(int who) {
111 | //todo null is bad, need to define a default Paxos msg
112 | Paxos msg = null;
113 | try {
114 | msg = msgQueueMap.get(who).take();
115 | } catch (InterruptedException e) {
116 | e.printStackTrace();
117 | }
118 | return msg;
119 | }
120 |
121 | /**
122 | * needs two args: config file and # of requests
123 | *
124 | * @param args
125 | */
126 | public static void main(String[] args) {
127 | Environment env = new SingleNodeEnvironment(args[0]);
128 |
129 | //here, using fixedThreadPool is wrong. Need an unbound pool.
130 | // ExecutorService threadPool = Executors.newCachedThreadPool();
131 |
132 | //start all nodes
133 | for (int i : env.getReplicas()) {
134 | new Replica(i, env).start();
135 | }
136 | for (int i : env.getAcceptors()) {
137 | new Acceptor(i, env).start();
138 | }
139 | for (int i : env.getLeaders()) {
140 | new Leader(i, env).start();
141 | }
142 |
143 | //sending some requests, assume client id is 99
144 | int numOfReq = Integer.parseInt(args[1]);
145 | for (int i = 1; i <= numOfReq; i++) {
146 | Paxos tempMsg = Paxos.newBuilder()
147 | .setType(Paxos.Type.REQUEST)
148 | .setRequest(PaxosMsgs.Request.newBuilder()
149 | .setC(PaxosMsgs.Command.newBuilder()
150 | .setClient(99)
151 | .setCid(i)
152 | .setOperation("Request # " + i)))
153 | .build();
154 | for (int r : env.getReplicas()) {
155 | env.send(r, tempMsg);
156 | }
157 |
158 | try {
159 | Thread.sleep(1000);
160 | } catch (InterruptedException e) {
161 | e.printStackTrace();
162 | }
163 | }
164 | }
165 |
166 | }
167 |
--------------------------------------------------------------------------------
/src/java/main/paxos/Replica.java:
--------------------------------------------------------------------------------
1 | package paxos;
2 |
3 | import utilities.Environment;
4 | import utilities.PaxosMsgs.*;
5 |
6 | import static utilities.PaxosMsgs.Paxos.Type.*;
7 |
8 | import java.util.*;
9 |
10 | /**
11 | * Invariants for replicas:
12 | * R1: There are no two different commands decided for the same slot.
13 | * R2: All commands up to slot out are in the set of decisions.
14 | * R3: For all replicas ρ, ρ.state is the result of applying
15 | * the commands⟨s,cs⟩∈ρ.decisions to initial_state for all s up to slot_out,
16 | * in order of slot number.
17 | * R4: For each ρ, the variable ρ.slot out cannot decrease over time.
18 | * R5: A replica proposes commands only for slots for which it knows the configuration.
19 | *
20 | * And R1 is ensured by synod(i.e., simple paxos)
21 | *
22 | * Did not implement reconfig function!!!
23 | */
24 | public class Replica extends Thread {
25 | //paxos related thread
26 | private int slot_in;
27 | private int slot_out;
28 | private final Queue requests;
29 | private final Map proposals;
30 | private final Map decisions;
31 | private final int ID;
32 |
33 | //for configuration change
34 | private final int WINDOW;
35 |
36 | //For sending and receiving messages
37 | private final Environment environment;
38 |
39 | public Replica(int ID, Environment environment) {
40 | super("Replica-" + ID);
41 | this.ID = ID;
42 | this.environment = environment;
43 | this.decisions = new HashMap<>();
44 | this.proposals = new HashMap<>();
45 | this.requests = new ArrayDeque<>();
46 | this.slot_in = 1;
47 | this.slot_out = 1;
48 |
49 | //???ease conflicts between ballot???
50 | this.WINDOW = 3;
51 | }
52 |
53 | /**
54 | * Transfer requests from requests to proposals
55 | */
56 | private void propose() {
57 | while ((slot_in < slot_out + WINDOW) && !requests.isEmpty()) {
58 | //if not already proposed for slot_in
59 | if (!decisions.containsKey(slot_in)) {
60 | Command polledCmd = requests.poll();
61 | proposals.put(slot_in, polledCmd);
62 |
63 | Paxos temp_propose = Paxos.newBuilder()
64 | .setType(PROPOSE)
65 | .setPropose(Propose.newBuilder()
66 | .setSlotIn(slot_in)
67 | .setC(polledCmd))
68 | .build();
69 |
70 | for (int leader : environment.getLeaders()) {
71 | environment.send(leader, temp_propose);
72 | }
73 | }
74 | slot_in++;
75 | }
76 | }
77 |
78 | /**
79 | * The function perform() is invoked with the same sequence of commands at all replicas.
80 | * Different replicas may end up proposing the same command for different slots,
81 | * and thus the same command may be decided multiple times.
82 | */
83 | private void perform(Command cmd) {
84 | //if it has already performed the command
85 | for (int i = 1; i < slot_out; i++) {
86 | if (decisions.get(i).equals(cmd)) {
87 | slot_out++;
88 | return;
89 | }
90 | }
91 |
92 | System.out.println("Replica " + ID + ": perform slot " + slot_out + " command " + cmd.getOperation());
93 | slot_out++;
94 |
95 | /**
96 | * for debugging and simplicity, no client here
97 | */
98 | // Paxos temp_response = Paxos.newBuilder()
99 | // .setType(RESPONSE)
100 | // .setResponse(Response.newBuilder()
101 | // .setCid(cmd.getCid())
102 | // .setResult("Performed"))
103 | // .build();
104 | //
105 | // environment.send(cmd.getClient(), temp_response);
106 | }
107 |
108 | public void run() {
109 | while (true) {
110 | Paxos incMsg = environment.receive(ID);
111 | switch (incMsg.getType()) {
112 | case REQUEST: {
113 | requests.offer(incMsg.getRequest().getC());
114 | break;
115 | }
116 |
117 | /**
118 | * Decisions may arrive out of order and multiple times
119 | */
120 | case DECISION: {
121 | //adds the decision to the set decisions
122 | Decision body = incMsg.getDecision();
123 | decisions.put(body.getSlotNum(), body.getC());
124 |
125 | /**
126 | * If there is a decision c′ corresponding to the current slot out,
127 | * the replica first checks to see if it has proposed a command c′′ for that slot.
128 | * If so, the replica removes ⟨slot out,c′′⟩ from the set proposals.
129 | * If c′′ ̸= c′, the replica returns c′′ to set requests
130 | */
131 | while (decisions.containsKey(slot_out)) {
132 | if (proposals.containsKey(slot_out)) {
133 | if (!proposals.get(slot_out).equals(decisions.get(slot_out))) {
134 | requests.offer(proposals.get(slot_out));
135 | }
136 | proposals.remove(slot_out);
137 | }
138 | perform(decisions.get(slot_out));
139 | }
140 | }
141 | }
142 | propose();
143 | }
144 | }
145 | }
146 |
--------------------------------------------------------------------------------
/src/java/main/paxos/synod/Leader.java:
--------------------------------------------------------------------------------
1 | package paxos.synod;
2 |
3 | import utilities.BallotComparator;
4 | import utilities.PaxosMsgs.*;
5 | import utilities.Environment;
6 |
7 | import java.util.*;
8 | import java.util.concurrent.ArrayBlockingQueue;
9 | import java.util.concurrent.BlockingQueue;
10 | import java.util.concurrent.ConcurrentHashMap;
11 | import java.util.concurrent.atomic.AtomicInteger;
12 | import static utilities.PaxosMsgs.Paxos.Type.*;
13 |
14 |
15 | /**
16 | * The Leader class has two Paxos related inner classes, which are Scout and Commander.
17 | * The leader is the main thread. Scouts and Commanders are launched by the leader.
18 | * Use inner classes to simplify state sharing.
19 | *
20 | * Inter threads communication is done by message queues and MessageHandler thread.
21 | * For one-to-one mapping between each thread and its message queue, each thread need a local id,
22 | * Use ThreadLocal to implement thread local id.
23 | *
24 | * All threads are executed in an almost-asynchronous manner,
25 | * the only very weak synchronized part are message queues.
26 | */
27 | public class Leader extends Thread{
28 |
29 | //Paxos related states, no one is shared, thus thread safe.
30 | private Ballot leaderBallot;
31 | private final Map proposals;
32 | private final int ID;
33 | private boolean active;
34 |
35 | //shared states for inter-threads communication, need thread-safe mechanism
36 | private final Map> messageQueues;
37 | private final BlockingQueue incomingMessages;
38 |
39 | //sending and receiving messages
40 | private final Environment environment;
41 |
42 | public Leader(int id, Environment environment) {
43 | super("Leader-" + id);
44 | ID = id;
45 | leaderBallot = Ballot.newBuilder().setPrefix(0).setConductor(id).build();
46 | active = false;
47 | proposals = new HashMap<>();
48 |
49 | this.environment = environment;
50 |
51 | //initial and register message queue
52 | messageQueues = new ConcurrentHashMap<>();
53 | this.incomingMessages = new ArrayBlockingQueue<>(100);
54 | }
55 |
56 | @Override
57 | public void run() {
58 | new MessageHandler(ThreadID.get()).start();
59 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
60 |
61 | //To ensure states is not shared, build a new ballot
62 | new Scout(Ballot.newBuilder(leaderBallot).build()).start();
63 |
64 | try {
65 | while (true)
66 | forever();
67 | } catch (InterruptedException e) {
68 | e.printStackTrace();
69 | } finally {
70 | //leader never end. not necessary to de-register its message Q.
71 | }
72 | }
73 |
74 | /**
75 | * this method transit shared states, need careful concurrent design
76 | * @throws InterruptedException
77 | */
78 | private void forever() throws InterruptedException{
79 | Paxos incMsg = incomingMessages.take();
80 | switch (incMsg.getType()) {
81 | case PROPOSE: {
82 | Propose temp_propose = incMsg.getPropose();
83 | if (!proposals.containsKey(temp_propose.getSlotIn())){
84 | proposals.put(temp_propose.getSlotIn(), temp_propose.getC());
85 | if (active) {
86 | /**
87 | * spawn a Commander
88 | * have some concerns! Is protobuf really immutable???
89 | * Assume it is!
90 | */
91 | new Commander(PValue.newBuilder()
92 | .setBallot(leaderBallot)
93 | .setSlotNum(temp_propose.getSlotIn())
94 | .setCmd(temp_propose.getC())
95 | .build()).start();
96 | }
97 | }
98 | }
99 | case ADOPTED: {
100 | Adopted temp_adopted = incMsg.getAdopted();
101 | if (leaderBallot.equals(temp_adopted.getBallot())){
102 | /**
103 | * updates the set of proposals, replacing for each slot number
104 | * the command corresponding to the maximum pvalue in pvals, if any.
105 | */
106 | Map pmax = new HashMap<>();
107 | for(PValue pv : temp_adopted.getPvalues().getPvalueList()){
108 | if(!pmax.containsKey(pv.getSlotNum()) ||
109 | new BallotComparator().compare(pmax.get(pv.getSlotNum()), pv.getBallot()) < 0)
110 | proposals.put(pv.getSlotNum(), pv.getCmd());
111 | }
112 | for(Map.Entry entry : proposals.entrySet()) {
113 | PValue temp_pv = PValue.newBuilder()
114 | .setBallot(leaderBallot)
115 | .setSlotNum(entry.getKey())
116 | .setCmd(entry.getValue())
117 | .build();
118 | new Commander(temp_pv).start();
119 | }
120 | active = true;
121 | }
122 | break;
123 | }
124 | case PREEMPTED: {
125 | Preempted temp_preempted = incMsg.getPreempted();
126 | if (new BallotComparator().compare(temp_preempted.getBallot(), leaderBallot) > 0) {
127 | active = false;
128 | leaderBallot = Ballot.newBuilder()
129 | .setPrefix(temp_preempted.getBallot().getPrefix() + 1)
130 | .setConductor(ID)
131 | .build();
132 | //ensure states are not shared
133 | new Scout(Ballot.newBuilder(leaderBallot).build()).start();
134 | }
135 | break;
136 | }
137 | }
138 | }
139 |
140 | /**
141 | * 这里scouts由leader id + leaderBallot 唯一确定，commanders由leader+leaderBallot+slot唯一确定。
142 | * 对每个不同的scout和commander threads，里面都要有个blocking Q。
143 | * MessageHandler 必须知道msg type，leaderBallot，和proposal，把对应的msg put到对应的Q里。
144 | * 以上保证了reliable。
145 | * 注意：见paper笔记。需要在msg里加额外的from，to fields
146 | * And give scout and commander 一个额外的 int id, use ThreadLocal
147 | */
148 | private class MessageHandler extends Thread{
149 |
150 | /**
151 | * message handler must know its leader's id.
152 | * No need to know scouts and commanders, their IDs are in message body.
153 | */
154 | private final int leadLocalId;
155 |
156 | public MessageHandler (int leadLocalId) {
157 | super("MessageHandler");
158 | this.leadLocalId = leadLocalId;
159 | }
160 |
161 | @Override
162 | public void run() {
163 | while(true) {
164 |
165 | //This statement will be blocked if there is no incoming message.
166 | Paxos message = environment.receive(ID);
167 |
168 | try {
169 | switch (message.getType()) {
170 | case PROPOSE: {
171 | messageQueues.get(leadLocalId).put(message);
172 | break;
173 | }
174 | case P1B: {
175 | int scoutId = message.getP1B().getToScout();
176 | if (!messageQueues.containsKey(scoutId)) break;
177 | messageQueues.get(scoutId).put(message);
178 | break;
179 | }
180 | case P2B: {
181 | int commanderId = message.getP2B().getToCommander();
182 | if (!messageQueues.containsKey(commanderId)) break;
183 | messageQueues.get(commanderId).put(message);
184 | break;
185 | }
186 | }
187 | }catch (InterruptedException e){
188 | e.printStackTrace();
189 | }
190 | }
191 | }
192 |
193 | }
194 |
195 | /**
196 | * inner class Scout
197 | * inter-thread communication -- use message queue
198 | * do synod phase 1
199 | */
200 | private class Scout extends Thread{
201 |
202 | //Paxos related states, must ensure not shared.
203 | private final Set pValueSet;
204 | private final Ballot scoutBallot;
205 | private final List waitforAccetpors;
206 |
207 | private final BlockingQueue incomingMessages;
208 |
209 | public Scout(Ballot ballot) {
210 | super("Scout in " + ID + " with Ballot " + ballot);
211 | scoutBallot = ballot;
212 | pValueSet = new HashSet<>();
213 | this.incomingMessages = new ArrayBlockingQueue<>(100);
214 |
215 | //fuck you ThreadLocal, in
216 | waitforAccetpors = new ArrayList<>(environment.getAcceptors());
217 | }
218 |
219 | @Override
220 | public void run() {
221 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
222 |
223 | Paxos p1a = Paxos.newBuilder()
224 | .setType(P1A)
225 | .setP1A(P1a.newBuilder()
226 | .setFromLeader(ID)
227 | .setFromScout(ThreadID.get())
228 | .setBallot(scoutBallot))
229 | .build();
230 |
231 | for(int acceptor : waitforAccetpors){
232 | environment.send(acceptor, p1a);
233 | }
234 |
235 | try {
236 | while(true){
237 |
238 | //because of message handler, scout can only get p1b msg
239 | P1b incMsg = incomingMessages.take().getP1B();
240 |
241 | if(incMsg.getBallot().equals(scoutBallot)){
242 | pValueSet.addAll(incMsg.getAccepted().getPvalueList());
243 | //use Integer.valueOf to use internal cache
244 | waitforAccetpors.remove(Integer.valueOf(incMsg.getFrom()));
245 |
246 | if(waitforAccetpors.size() < environment.getAcceptors().size()/2){
247 | //偷懒实现了，由于inner class知道leader的Q。。。
248 | Leader.this.incomingMessages.put(Paxos.newBuilder()
249 | .setType(ADOPTED)
250 | .setAdopted(Adopted.newBuilder()
251 | .setBallot(scoutBallot)
252 | .setPvalues(Accepted.newBuilder()
253 | .addAllPvalue(pValueSet)))
254 | .build());
255 | //thread exit
256 | return;
257 | }
258 | } else {
259 | Leader.this.incomingMessages.put(Paxos.newBuilder()
260 | .setType(PREEMPTED)
261 | .setPreempted(Preempted.newBuilder()
262 | .setBallot(incMsg.getBallot()))
263 | .build());
264 | return;
265 | }
266 |
267 | }
268 | } catch (InterruptedException e) {
269 | e.printStackTrace();
270 | } finally {
271 | //clean up, remove its message queue from map.
272 | messageQueues.remove(ThreadID.get());
273 | }
274 | }
275 | }
276 |
277 | /**
278 | * inner class Commander
279 | * do synod phase 2
280 | */
281 | private class Commander extends Thread{
282 | private final List waitforAccetpors;
283 | private final PValue pvalue;
284 |
285 | private final BlockingQueue incomingMessages;
286 |
287 | public Commander(PValue pvalue) {
288 | super("Commander in " + ID + " with pvalue " + pvalue);
289 | this.pvalue = pvalue;
290 | waitforAccetpors = new ArrayList<>(environment.getAcceptors());
291 | incomingMessages = new ArrayBlockingQueue<>(100);
292 | }
293 |
294 | @Override
295 | public void run() {
296 | messageQueues.putIfAbsent(ThreadID.get(), this.incomingMessages);
297 | Paxos p2a = Paxos.newBuilder()
298 | .setType(P2A)
299 | .setP2A(P2a.newBuilder()
300 | .setFromLeader(ID)
301 | .setFromCommander(ThreadID.get())
302 | .setPvalue(pvalue))
303 | .build();
304 |
305 | for(int acceptor : waitforAccetpors){
306 | //de-register its message queue
307 | environment.send(acceptor, p2a);
308 | }
309 |
310 | try {
311 | while(true){
312 | //because of message handler, scout can only get p2b msg
313 | P2b incMsg = incomingMessages.take().getP2B();
314 |
315 | if(incMsg.getBallot().equals(pvalue.getBallot())){
316 | //use Integer.valueOf to use internal cache
317 | waitforAccetpors.remove(Integer.valueOf(incMsg.getFrom()));
318 | if(waitforAccetpors.size() < environment.getAcceptors().size()/2){
319 | Paxos decisionToReplicas = Paxos.newBuilder()
320 | .setType(DECISION)
321 | .setDecision(Decision.newBuilder()
322 | .setSlotNum(pvalue.getSlotNum())
323 | .setC(pvalue.getCmd()))
324 | .build();
325 |
326 | for(int replica : environment.getReplicas()) {
327 | environment.send(replica, decisionToReplicas);
328 | }
329 | return;
330 | }
331 | } else {
332 | Leader.this.incomingMessages.put(Paxos.newBuilder()
333 | .setType(PREEMPTED)
334 | .setPreempted(Preempted.newBuilder()
335 | .setBallot(incMsg.getBallot()))
336 | .build());
337 | return;
338 | }
339 | }
340 | } catch (InterruptedException e) {
341 | e.printStackTrace();
342 | } finally {
343 | messageQueues.remove(ThreadID.get());
344 | }
345 | }
346 | }
347 |
348 | /**
349 | * really got fucked when call threadlocal get in Thread's constructor.
350 | * The value returned by get() is not for this thread, but for the thread who call the constructor.
351 | * Target thread has not stared when its constructor is being called.
352 | */
353 | public static class ThreadID {
354 | private static final AtomicInteger nextID = new AtomicInteger(0);
355 |
356 | /**
357 | * 所以这里把ThreadLocal encapsulate的目的是实现auto-increment
358 | * note: 整个class是thread safe的，遵循了书上将的模式
359 | *
360 | * 仔细想想，下面这个obj，其实只有一个，所有threads的threadlocalmap的key都是指向这个obj的
361 | * 知道lead.class不被gc，那么他就是strongly reachable.
362 | */
363 | private static final ThreadLocal threadID = new ThreadLocal(){
364 | @Override
365 | protected Integer initialValue() {
366 | return nextID.getAndIncrement();
367 | }
368 | };
369 |
370 | public static int get() {
371 | return threadID.get();
372 | }
373 | }
374 | }
--------------------------------------------------------------------------------