37 |
38 |
39 |
--------------------------------------------------------------------------------
/Lectures/LEC03/6.824 Lab 2: Raft_files/vomnibar_data/clipboard.js:
--------------------------------------------------------------------------------
1 | // Generated by CoffeeScript 1.9.3
2 | (function() {
3 | var Clipboard, root;
4 |
5 | Clipboard = {
6 | _createTextArea: function(tagName) {
7 | var textArea;
8 | if (tagName == null) {
9 | tagName = "textarea";
10 | }
11 | textArea = document.createElement(tagName);
12 | textArea.style.position = "absolute";
13 | textArea.style.left = "-100%";
14 | textArea.contentEditable = "true";
15 | return textArea;
16 | },
17 | copy: function(arg) {
18 | var data, textArea;
19 | data = arg.data;
20 | textArea = this._createTextArea();
21 | textArea.value = data;
22 | document.body.appendChild(textArea);
23 | textArea.select();
24 | document.execCommand("Copy");
25 | return document.body.removeChild(textArea);
26 | },
27 | paste: function() {
28 | var textArea, value;
29 | textArea = this._createTextArea("div");
30 | document.body.appendChild(textArea);
31 | textArea.focus();
32 | document.execCommand("Paste");
33 | value = textArea.innerText;
34 | document.body.removeChild(textArea);
35 | return value;
36 | }
37 | };
38 |
39 | root = typeof exports !== "undefined" && exports !== null ? exports : (window.root != null ? window.root : window.root = {});
40 |
41 | root.Clipboard = Clipboard;
42 |
43 | if (typeof exports === "undefined" || exports === null) {
44 | extend(window, root);
45 | }
46 |
47 | }).call(this);
48 |
--------------------------------------------------------------------------------
/Lectures/LEC03/6.824 Lab 2: Raft_files/vomnibar_data/ui_component_server.js:
--------------------------------------------------------------------------------
1 | // Generated by CoffeeScript 1.9.3
2 | (function() {
3 | var UIComponentServer, registerPort, root;
4 |
5 | window.addEventListener("message", registerPort = function(event) {
6 | return chrome.storage.local.get("vimiumSecret", function(arg) {
7 | var secret;
8 | secret = arg.vimiumSecret;
9 | if (!(event.source === window.parent && event.data === secret)) {
10 | return;
11 | }
12 | UIComponentServer.portOpen(event.ports[0]);
13 | return window.removeEventListener("message", registerPort);
14 | });
15 | });
16 |
17 | UIComponentServer = {
18 | ownerPagePort: null,
19 | handleMessage: null,
20 | portOpen: function(ownerPagePort) {
21 | this.ownerPagePort = ownerPagePort;
22 | this.ownerPagePort.onmessage = (function(_this) {
23 | return function(event) {
24 | return typeof _this.handleMessage === "function" ? _this.handleMessage(event) : void 0;
25 | };
26 | })(this);
27 | return this.registerIsReady();
28 | },
29 | registerHandler: function(handleMessage) {
30 | this.handleMessage = handleMessage;
31 | },
32 | postMessage: function(message) {
33 | var ref;
34 | return (ref = this.ownerPagePort) != null ? ref.postMessage(message) : void 0;
35 | },
36 | hide: function() {
37 | return this.postMessage("hide");
38 | },
39 | registerIsReady: (function() {
40 | var uiComponentIsReadyCount;
41 | uiComponentIsReadyCount = document.readyState === "loading" ? (window.addEventListener("DOMContentLoaded", function() {
42 | return UIComponentServer.registerIsReady();
43 | }), 0) : 1;
44 | return function() {
45 | if (++uiComponentIsReadyCount === 2) {
46 | if (window.frameId != null) {
47 | this.postMessage({
48 | name: "setIframeFrameId",
49 | iframeFrameId: window.frameId
50 | });
51 | }
52 | return this.postMessage("uiComponentIsReady");
53 | }
54 | };
55 | })()
56 | };
57 |
58 | root = typeof exports !== "undefined" && exports !== null ? exports : window;
59 |
60 | root.UIComponentServer = UIComponentServer;
61 |
62 | root.isVimiumUIComponent = true;
63 |
64 | }).call(this);
65 |
--------------------------------------------------------------------------------
/Lectures/LEC03/6.824 Lab 2: Raft_files/vomnibar_data/vomnibar.css:
--------------------------------------------------------------------------------
1 |
2 | /* Vomnibar CSS */
3 |
4 | #vomnibar ol, #vomnibar ul {
5 | list-style: none;
6 | display: none;
7 | }
8 |
9 | #vomnibar {
10 | display: block;
11 | position: fixed;
12 | width: calc(100% - 20px); /* adjusted to keep border radius and box-shadow visible*/
13 | /*min-width: 400px;
14 | top: 70px;
15 | left: 50%;*/
16 | top: 8px;
17 | left: 8px;
18 | /*margin: 0 0 0 -40%;*/
19 | font-family: sans-serif;
20 |
21 | background: #F1F1F1;
22 | text-align: left;
23 | border-radius: 4px;
24 | box-shadow: 0px 2px 10px rgba(0, 0, 0, 0.8);
25 | border: 1px solid #aaa;
26 | z-index: 2139999999; /* One less than hint markers and the help dialog (see ../content_scripts/vimium.css). */
27 | }
28 |
29 | #vomnibar input {
30 | color: #000;
31 | font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif;
32 | font-size: 20px;
33 | height: 34px;
34 | margin-bottom: 0;
35 | padding: 4px;
36 | background-color: white;
37 | border-radius: 3px;
38 | border: 1px solid #E8E8E8;
39 | box-shadow: #444 0px 0px 1px;
40 | width: 100%;
41 | outline: none;
42 | box-sizing: border-box;
43 | }
44 |
45 | #vomnibar .vomnibarSearchArea {
46 | display: block;
47 | padding: 10px;
48 | background-color: #F1F1F1;
49 | border-radius: 4px 4px 0 0;
50 | border-bottom: 1px solid #C6C9CE;
51 | }
52 |
53 | #vomnibar ul {
54 | background-color: white;
55 | border-radius: 0 0 4px 4px;
56 | list-style: none;
57 | padding: 10px 0;
58 | padding-top: 0;
59 | }
60 |
61 | #vomnibar li {
62 | border-bottom: 1px solid #ddd;
63 | line-height: 1.1em;
64 | padding: 7px 10px;
65 | font-size: 16px;
66 | color: black;
67 | position: relative;
68 | display: list-item;
69 | margin: auto;
70 | }
71 |
72 | #vomnibar li:last-of-type {
73 | border-bottom: none;
74 | }
75 |
76 | #vomnibar li .vomnibarTopHalf, #vomnibar li .vomnibarBottomHalf {
77 | display: block;
78 | overflow: hidden;
79 | }
80 |
81 | #vomnibar li .vomnibarBottomHalf {
82 | font-size: 15px;
83 | margin-top: 3px;
84 | padding: 2px 0;
85 | }
86 |
87 | #vomnibar li .vomnibarIcon {
88 | background-position-y: center;
89 | background-size: 16px;
90 | background-repeat: no-repeat;
91 | padding-left: 20px;
92 | }
93 |
94 | #vomnibar li .vomnibarSource {
95 | color: #777;
96 | margin-right: 4px;
97 | }
98 | #vomnibar li .vomnibarRelevancy {
99 | position: absolute;
100 | right: 0;
101 | top: 0;
102 | padding: 5px;
103 | background-color: white;
104 | color: black;
105 | font-family: monospace;
106 | width: 100px;
107 | overflow: hidden;
108 | }
109 |
110 | #vomnibar li .vomnibarUrl {
111 | white-space: nowrap;
112 | color: #224684;
113 | }
114 |
115 | #vomnibar li .vomnibarMatch {
116 | font-weight: bold;
117 | color: black;
118 | }
119 |
120 | #vomnibar li em, #vomnibar li .vomnibarTitle {
121 | color: black;
122 | margin-left: 4px;
123 | font-weight: normal;
124 | }
125 | #vomnibar li em { font-style: italic; }
126 | #vomnibar li em .vomnibarMatch, #vomnibar li .vomnibarTitle .vomnibarMatch {
127 | color: #333;
128 | }
129 |
130 | #vomnibar li.vomnibarSelected {
131 | background-color: #BBCEE9;
132 | font-weight: normal;
133 | }
134 |
135 | #vomnibarInput::selection {
136 | /* This is the light grey color of the vomnibar border. */
137 | /* background-color: #F1F1F1; */
138 |
139 | /* This is the light blue color of the vomnibar selected item. */
140 | /* background-color: #BBCEE9; */
141 |
142 | /* This is a considerably lighter blue than Vimium blue, which seems softer
143 | * on the eye for this purpose. */
144 | background-color: #E6EEFB;
145 | }
146 |
147 | .vomnibarInsertText {
148 | }
149 |
150 | .vomnibarNoInsertText {
151 | visibility: hidden;
152 | }
153 |
--------------------------------------------------------------------------------
/Lectures/LEC03/Bolosky.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC03/Bolosky.pdf
--------------------------------------------------------------------------------
/Lectures/LEC03/README.md:
--------------------------------------------------------------------------------
1 | # 课程 03
2 |
3 | ## 课前思考
4 |
5 | 阅读论文 [GFS(2003)](gfs.pdf)，可以思考以下问题:
6 |
7 | 1. Why is atomic record append at-least-once, rather than exactly once?
8 | 1. How does an application know what sections of a chunk consist of padding and duplicate records?
9 | 1. How can clients find their data given that atomic record append writes it at an unpredictable offset in the file?
10 | 1. The paper mentions reference counts -- what are they?
11 | 1. If an application uses the standard POSIX file APIs, would it need to be modified in order to use GFS?
12 | 1. How does GFS determine the location of the nearest replica?
13 | 1. Does Google still use GFS?
14 | 1. Won't the master be a performance bottleneck?
15 | 1. How acceptable is it that GFS trades correctness for performance and simplicity?
16 | 1. What if the master fails?
17 |
18 | 问题的答案在[这里](gfs-faq.txt.md)
19 |
20 | 在阅读[讲义](l-gfs-short.txt.md)请先思考以下问题：
21 |
22 | - Why are we reading this paper?
23 | - What is consistency?
24 | - "Ideal" consistency model
25 | - Challenges to achieving ideal consistency
26 | - GFS goals:
27 | - High-level design / Reads / Writes
28 | - Record append
29 | - Housekeeping
30 | - Failures
31 | - Does GFS achieve "ideal" consistency?
32 | - Authors claims weak consistency is not a big problems for apps
33 | - Performance (Figure 3)
34 |
35 | ## Lab 02
36 |
37 | [Lab 02 的上机说明](6.824 Lab 2: Raft.html)
38 |
39 | ### Lab 02 准备工作
40 |
41 | - 阅读 [extended Raft paper](raft-extended.pdf)
42 | - 阅读 [LEC 05](../LEC05/README.md) 和 [LEC 06](../LEC06/README.md)
43 | - 阅读
44 | - 阅读 [Students' Guide to Raft](https://thesquareplanet.com/blog/students-guide-to-raft/)
45 | - 阅读 [Raft Locking Advice](raft-locking.txt.md)
46 | - 阅读 [Raft Structure Advice](raft-structure.txt.md)
47 | - 选读 [Paxos Replicated State Machines as the Basis of a High-Performance Data Store](Bolosky.pdf)
48 |
--------------------------------------------------------------------------------
/Lectures/LEC03/gfs.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC03/gfs.pdf
--------------------------------------------------------------------------------
/Lectures/LEC03/raft-extended.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC03/raft-extended.pdf
--------------------------------------------------------------------------------
/Lectures/LEC03/raft-structure.txt.md:
--------------------------------------------------------------------------------
1 | Raft Structure Advice
2 |
3 | A Raft instance has to deal with the arrival of external events
4 | (Start() calls, AppendEntries and RequestVote RPCs, and RPC replies),
5 | and it has to execute periodic tasks (elections and heart-beats).
6 | There are many ways to structure your Raft code to manage these
7 | activities; this document outlines a few ideas.
8 |
9 | Each Raft instance has a bunch of state (the log, the current index,
10 | &c) which must be updated in response to events arising in concurrent
11 | goroutines. The Go documentation points out that the goroutines can
12 | perform the updates directly using shared data structures and locks,
13 | or by passing messages on channels, perhaps to a central goroutine
14 | that manages state changes. For Raft, we recommend shared data
15 | structures and locks.
16 |
17 | A Raft instance has two time-driven activities: the leader must send
18 | heart-beats, and others must start an election if too much time has
19 | passed since hearing from the leader. It's probably best to drive each
20 | of these activities with a dedicated long-running goroutine, rather
21 | than combining multiple activities into a single goroutine.
22 |
23 | The management of the election timeout is a common source of
24 | headaches. Perhaps the simplest plan is to maintain a variable in the
25 | Raft struct containing the last time at which the peer heard from the
26 | leader, and to have the election timeout goroutine periodically check
27 | to see whether the time since then is greater than the timeout period.
28 | It's easiest to use time.Sleep() with a small constant argument to
29 | drive the periodic checks; time.Ticker and Time.Timer are difficult to
30 | use correctly.
31 |
32 | You'll want to have a separate long-running goroutine that sends
33 | committed log entries in order on the applyCh. It must be separate,
34 | since sending on the applyCh can block; and it must be a single
35 | goroutine, since otherwise it may be hard to ensure that you send log
36 | entries in log order. The code that advances commitIndex will need to
37 | kick the apply goroutine; it's probably easiest to use a condition
38 | variable (Go's sync.Cond) for this.
39 |
40 | Each RPC should probably be sent (and its reply processed) in its own
41 | goroutine, for two reasons: so that unreachable peers don't delay the
42 | collection of a majority of replies, and so that the heartbeat and
43 | election timers can continue to tick at all times. It's easiest to do
44 | the RPC reply processing in the same goroutine, rather than sending
45 | reply information over a channel.
46 |
47 | Keep in mind that the network can delay RPCs and RPC replies, and when
48 | you send concurrent RPCs, the network can re-order requests and
49 | replies. Figure 2 is pretty good about pointing out places where RPC
50 | handlers have to be careful about this (e.g. an RPC handler should
51 | ignore RPCs with old terms). Figure 2 is not always explicit about RPC
52 | reply processing. The leader has to be careful when processing
53 | replies; it must check that the term hasn't changed since sending the
54 | RPC, and must account for the possibility that replies from concurrent
55 | RPCs to the same follower have changed the leader's state (e.g.
56 | nextIndex).
57 |
--------------------------------------------------------------------------------
/Lectures/LEC04/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 4: Primary/Backup Replication
2 |
3 | ## 课前阅读
4 |
5 | [The Design of a for Practical System Practical Virtual System Machines for Fault-Tolerant Fault-Tolerant Virtual Machines](vm-ft.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. The introduction says that it is more difficult to ensure deterministic execution on physical servers than on VMs. Why is this the case?
10 | 1. What is a hypervisor?
11 | 1. Both GFS and VMware FT provide fault tolerance. How should we think about when one or the other is better?
12 | 1. How do Section 3.4's bounce buffers help avoid races?
13 | 1. What is "an atomic test-and-set operation on the shared storage"?
14 | 1. How much performance is lost by following the Output Rule?
15 | 1. What if the application calls a random number generator? Won't that yield different results on primary and backup and cause the executions to diverge?
16 | 1. How were the creators certain that they captured all possible forms of non-determinism?
17 | 1. What happens if the primary fails just after it sends output to the external world?
18 | 1. Section 3.4 talks about disk I/Os that are outstanding on the primary when a failure happens; it says "Instead, we re-issue the pending I/Os during the go-live process of the backup VM." Where are the pending I/Os located/stored, and how far back does the re-issuing need to go?
19 | 1. How secure is this system?
20 | 1. Is it reasonable to address only the fail-stop failures? What are other type of failures?
21 |
22 | ## 课堂讲义
23 |
24 | [讲义](l-vm-ft.txt)
25 |
26 | [FAQ ANSWERS](vm-ft-faq.txt)
27 |
28 | ## 作业
29 |
30 | How does [VM FT](vm-ft.pdf) handle network partitions? That is, is it possible that if the primary and the backup end up in different network partitions that the backup will become a primary too and the system will run with two primaries?
--------------------------------------------------------------------------------
/Lectures/LEC04/vm-ft.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC04/vm-ft.pdf
--------------------------------------------------------------------------------
/Lectures/LEC05/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 5: Raft (1)
2 |
3 | ## 课前阅读
4 |
5 | [In Search of an Understandable Consensus Algorithm (Extended Version)]( ../LEC03/raft-extended.pdf), to end of Section 5
6 |
7 | ## FAQ
8 |
9 | 1. Does Raft sacrifice anything for simplicity?
10 | 1. Is raft used in real-world software, or do companies generally roll their own flavor of Paxos (or use a different consensus protocol)?
11 | 1. What is Paxos? In what sense is Raft simpler?
12 | 1. How long had Paxos existed before the authors created Raft? How widespread is Raft's usage in production now?
13 | 1. How does Raft's performance compare to Paxos in real-world applications?
14 | 1. Why are we learning/implementing Raft instead of Paxos?
15 | 1. Are there systems like Raft that can survive and continue to operate when only a minority of the cluster is active?
16 | 1. In Raft, the service which is being replicated is not available to the clients during an election process. In practice how much of a problem does this cause?
17 | 1. Are there other consensus systems that don't have leader-election pauses?
18 | 1. How are Raft and VMware FT related?
19 | 1. Why can't a malicious person take over a Raft server, or forge incorrect Raft messages?
20 | 1. The paper mentions that Raft works under all non-Byzantine conditions. What are Byzantine conditions and why could they make Raft fail?
21 | 1. In Figure 1, what does the interface between client and server look like?
22 | 1. What if a client sends a request to a leader, the the leader crashes before sending the client request to all followers, and the new leader doesn't have the request in its log? Won't that cause the client request to be lost?
23 | 1. If there's a network partition, can Raft end up with two leaders and split brain?
24 | 1. Suppose a new leader is elected while the network is partitioned, but the old leader is in a different partition. How will the old leader know to stop committing new entries?
25 | 1. When some servers have failed, does "majority" refer to a majority of the live servers, or a majority of all servers (even the dead ones)?
26 | 1. What if the election timeout is too short? Will that cause Raft to malfunction?
27 | 1. Why randomize election timeouts?
28 | 1. Can a candidate declare itself the leader as soon as it receives votes from a majority, and not bother waiting for further RequestVote replies?
29 | 1. Can a leader in Raft ever stop being a leader except by crashing?
30 | 1. When are followers' log entries sent to their state machines?
31 | 1. Should the leader wait for replies to AppendEntries RPCs?
32 | 1. What happens if more than half of the servers die?
33 | 1. Why is the Raft log 1-indexed?
34 | 1. When network partition happens, wouldn't client requests in minority partitions be lost?
35 | 1. Is the argument in 5.4.3 a complete proof?
36 |
37 | ## 课堂讲义
38 |
39 | [讲义](l-raft.txt)
40 |
41 | [FAQ ANSWERS](raft-faq.txt)
42 |
43 | ## 作业
44 |
45 | Suppose we have the scenario shown in the Raft paper's Figure 7: a cluster of seven servers, with the log contents shown. The first server crashes (the one at the top of the figure), and cannot be contacted. A leader election ensues. For each of the servers marked (a), (d), and (f), could that server be elected? If yes, which servers would vote for it? If no, what specific Raft mechanism(s) would prevent it from being elected?
--------------------------------------------------------------------------------
/Lectures/LEC06/2015-osr-raft.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC06/2015-osr-raft.pdf
--------------------------------------------------------------------------------
/Lectures/LEC07/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 7: Raft (3) -- Snapshots, Linearizability, Duplicate Detection
2 |
3 | ## 课前阅读
4 |
5 | [spinnaker](spinnaker.pdf)，全文
6 |
7 | ## FAQ
8 |
9 | 1. What is timeline consistency?
10 | 1. When there is only 1 node up in the cohort, the paper says it鈥檚 still timeline consistency; how is that possible?
11 | 1. What are the trade-offs of the different consistency levels?
12 | 1. How does Spinnaker implement timeline reads, as opposed to consistent reads?
13 | 1. Why are Spinnaker's timeline reads faster than its consistent reads, in Figure 8?
14 | 1. Why do Spinnaker's consistent reads have much higher performance than Cassandra's quorum reads, in Figure 8?
15 | 1. What is the CAP theorem about?
16 | 1. Where does Spinnaker sit in the CAP scheme?
17 | 1. Could Spinnaker use Raft as the replication protocol rather than Paxos?
18 | 1. The paper mentions Paxos hadn't previously been used for database replication. What was it used for?
19 | 1. What is the reason for logical truncation?
20 | 1. What exactly is the key range (i.e. 'r') defined in the paper for leader election?
21 | 1. Is there a more detailed description of Spinnaker's replication protocol somewhere?
22 | 1. How does Spinnaker's leader election ensure there is at most one leader?
23 | 1. Does Spinnaker have something corresponding to Raft's terms?
24 | 1. Section 9.1 says that a Spinnaker leader replies to a consistent read without consulting the followers. How does the leader ensure that it is still the leader, so that it doesn't reply to a consistent read with stale data?
25 | 1. Would it be possible to replace the use of Zookeeper with a Raft-like leader election protocol?
26 | 1. What is the difference between a forced and a non-forced log write?
27 | 1. Step 6 of Figure 7 seems to say that the candidate with the longest long gets to be the next leader. But in Raft we saw that this rule doesn't work, and that Raft has to use the more elaborate Election Restriction. Why can Spinnaker safely use longest log?
28 |
29 | ## 课堂讲义
30 |
31 | [讲义](l-spinnaker.txt)
32 |
33 | ## 作业
34 |
35 | Please read the paper's Appendices. In Spinnaker a leader to responds to a client request after the leader and one follower have written a log record for the request on persistent storage. Why is this sufficient to guarantee strong consistency even after the leader or the one follower fail?
36 |
37 | (This paper relies on Zookeeper, which we will read later.)
--------------------------------------------------------------------------------
/Lectures/LEC07/spinnaker.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC07/spinnaker.pdf
--------------------------------------------------------------------------------
/Lectures/LEC08/6.824 Lab 3_ Fault-tolerant Key_Value Service_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC08/6.824 Lab 3_ Fault-tolerant Key_Value Service_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC08/6.824 Lab 3_ Fault-tolerant Key_Value Service_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC08/6.824 Lab 3_ Fault-tolerant Key_Value Service_files/style.css:
--------------------------------------------------------------------------------
1 | body { max-width: 45em; }
2 | body pre { overflow-x: auto; }
3 |
4 | body {
5 | color: black;
6 | background-color: white;
7 | font-family: sans-serif;
8 | }
9 |
10 | .title {
11 | text-align: center
12 | }
13 | .subtitle {
14 | text-align: center;
15 | font-style: italic;
16 | }
17 | .author {
18 | text-align: center;
19 | }
20 |
21 | ul.hints, .note, .challenge, .todo, pre {
22 | margin: 1em;
23 | border: 1px dashed;
24 | padding: 1em;
25 | }
26 |
27 | ul.hints { color: #50A02D; }
28 | ul.hints li { margin-left: 1em; }
29 | ul.hints li::before {
30 | content: "Hint: ";
31 | font-weight: bold;
32 | }
33 |
34 | .important {
35 | margin: 1em;
36 | padding: 1em;
37 | background-color: #990000;
38 | color: #fff;
39 | }
40 | .important::before {
41 | content: "Important: ";
42 | background-color: #550000;
43 | width: 100%;
44 | display: block;
45 | margin: -1em -1em 1em -1em;
46 | padding: 1em;
47 | font-weight: bold;
48 | }
49 | .note { color: #4682B4; }
50 | .note::before {
51 | content: "Note: ";
52 | font-weight: bold;
53 | }
54 |
55 | .challenge, .todo { border-style: solid; }
56 | .challenge::before, .todo::before {
57 | float: right;
58 | font-weight: bold;
59 | color: white;
60 | margin-right: -1em;
61 | margin-top: -1em;
62 | margin-bottom: .5em;
63 | margin-left: 1em;
64 | padding: .5em 1em;
65 | }
66 | .todo { color: #B22222; }
67 | .todo::before {
68 | content: "TASK";
69 | background: #B22222;
70 | }
71 | .challenge { color: #8B4513; }
72 | .challenge::before {
73 | content: "CHALLENGE";
74 | background: #8B4513;
75 | }
76 |
77 |
78 | tt, code {
79 | font-family: monospace;
80 | border-radius: 3px;
81 | font-size: 110%;
82 | color: #657b83;
83 | background-color: #fdf6e3;
84 | padding: 0.2em;
85 | word-wrap: break-word;
86 | }
87 |
88 | pre {
89 | font-size: 100%;
90 | color: #839496;
91 | background: #002b36;
92 | }
93 |
94 | .classic {
95 | color: black;
96 | }
97 |
98 |
99 | div.required .header {
100 | font-weight: bold;
101 | }
102 |
103 | div.challenge .header {
104 | font-style: italic;
105 | }
106 |
107 | div.required {
108 | background-color: #eeeeff;
109 | }
110 |
111 |
--------------------------------------------------------------------------------
/Lectures/LEC08/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 8: Zookeeper Case Study
2 |
3 | ## 课前阅读
4 |
5 | [zookeeper](zookeeper.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. Why are only update requests A-linearizable?
10 | 1. How does linearizability differ from serializability?
11 | 1. What is pipelining?
12 | 1. What about Zookeeper's use case makes wait-free better than locking?
13 | 1. What does wait-free mean?
14 | 1. What is the reason for implementing 'fuzzy snapshots'? How can state changes be idempotent?
15 | 1. How does ZooKeeper choose leaders?
16 | 1. How does Zookeeper's performance compare to other systems such as Paxos?
17 | 1. How does the ordering guarantee solve the race conditions in Section 2.3?
18 | 1. How big is the ZooKeeper database? It seems like the server must have a lot of memory.
19 | 1. What's a universal object?
20 | 1. How does a client know when to leave a barrier (top of page 7)?
21 | 1. Is it possible to add more servers into an existing ZooKeeper without taking the service down for a period of time?
22 | 1. How are watches implemented in the client library?
23 |
24 | ## 课堂讲义
25 |
26 | [讲义](l-zookeeper.txt)
27 |
28 | [FAQ](zookeeper-faq.txt)
29 |
30 | ## 作业
31 |
32 | One use of [Zookeeper](zookeeper.pdf) is a fault-tolerant lock service (see the section "Simple locks" on page 6). Why isn't possible that two clients can acquire the same lock? In particular, how does Zookeeper decide if a client has failed and it can give the lock to some other client?
33 |
34 | ## LAB 3
35 |
36 | [LAB 3 说明](6.824 Lab 3_ Fault-tolerant Key_Value Service.html)
--------------------------------------------------------------------------------
/Lectures/LEC08/atc12-final74.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC08/atc12-final74.pdf
--------------------------------------------------------------------------------
/Lectures/LEC08/p124-herlihy.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC08/p124-herlihy.pdf
--------------------------------------------------------------------------------
/Lectures/LEC08/zookeeper.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC08/zookeeper.pdf
--------------------------------------------------------------------------------
/Lectures/LEC09/README.md:
--------------------------------------------------------------------------------
1 | # LECTURE 09 Guest lecturer on Go (Russ Cox Google/Go)
2 |
3 | ## FAQ
4 |
5 | 1. Can I stop these complaints about my unused variable/import?
6 | 1. Is the defer keyword in other languages?
7 | 1. Why is the type after the variable declaration, unlike C languages?
8 | 1. Why not adopt classes and OOP like in C++ and Java?
9 | 1. Why does struct require a trailing comma on a multiline definition?
10 | 1. Why are list definitions inconsistent, where some need commas and some do not?
11 | 1. Why does Go name its while loops "for"?
12 | 1. There seem to be a lot of new languages emerging these days, including Rust, D, Swift and Nim, among probably others. Are there any lessons you've learned from these other languages and their communities that you wish you'd been able to incorporate into Go?
13 | 1. Why the focus on concurrency and goroutines?
14 |
15 | ## 课堂内容
16 |
17 | [PPT](gopattern.pdf)
18 |
19 | [FAQ 答案](go-faq.txt)
20 |
21 | ## 作业
22 |
23 | Russ Cox is one of the leads on the Go project. What do you like best about Go? Why? Would you want to change anything in the language? If so, what and why?
--------------------------------------------------------------------------------
/Lectures/LEC09/go-faq.txt:
--------------------------------------------------------------------------------
1 | Q: Can I stop these complaints about my unused variable/import?
2 |
3 | A: There's a good explanation at https://golang.org/doc/faq#unused_variables_and_imports.
4 |
5 | Q: Is the defer keyword in other languages?
6 |
7 | A: Defer was new in Go. We originally added it to provide a way to
8 | recover from panics (see "recover" in the spec), but it turned out
9 | to be very useful for idioms like "defer mu.Unlock()" as well.
10 | Later, Swift added a defer statement too. It seems clearly inspired
11 | by Go but I'm not sure how close the details are.
12 |
13 | Q: Why is the type after the variable declaration, unlike C languages?
14 |
15 | A: There's a good explanation at https://blog.golang.org/gos-declaration-syntax.
16 |
17 | Q: Why not adopt classes and OOP like in C++ and Java?
18 |
19 | A: We believe that Go's approach to object-oriented programming,
20 | which is closer to Smalltalk than to Java/C++/Simula, is more
21 | lightweight and makes it easier to adapt large programs. I talked
22 | about this at Google I/O in 2010. See
23 | https://github.com/golang/go/wiki/GoTalks#go-programming for links
24 | to the video and slides.
25 |
26 | Q: Why does struct require a trailing comma on a multiline definition?
27 |
28 | A: Originally it didn't, but all statements were terminated by
29 | semicolons. We made semicolons optional shortly after the public
30 | release of Go. When we did that, we tried to avoid Javascript's
31 | mistake of making the semicolon rules very complex and error-prone.
32 | Instead we have a simple rule: every line ends in an implicit
33 | semicolon unless the final token is something that cannot possibly
34 | end a statement (for example, a plus sign, or a comma). One effect
35 | of this is that if you don't put the trailing comma on the line,
36 | it gets an implicit semicolon, which doesn't parse well. It's
37 | unfortunate, and it wasn't that way before the semicolon rules, but
38 | we're so happy about not typing semicolons all the time that we'll
39 | live with it. The original proposal for semicolon insertion is at
40 | https://groups.google.com/d/msg/golang-nuts/XuMrWI0Q8uk/kXcBb4W3rH8J.
41 | See the next answer also.
42 |
43 | Q: Why are list definitions inconsistent, where some need commas and some do not?
44 |
45 | A: The ones that don't need commas need semicolons, but those
46 | semicolons are being inserted automatically (see previous answer).
47 | The rule is that statements are separated by semicolons and smaller
48 | pieces of syntax by commas:
49 |
50 | import "x";
51 | import "y";
52 |
53 | var x = []int{
54 | 1,
55 | 2,
56 | 3,
57 | }
58 |
59 | When you factor out a group of imports, you still have semicolons:
60 |
61 | import (
62 | "x";
63 | "y";
64 | )
65 |
66 | var x = []int{
67 | 1,
68 | 2,
69 | 3,
70 | }
71 |
72 | But then when we made semicolons optional, the semicolons disappeared
73 | from the statement blocks leaving the commas behind:
74 |
75 | import (
76 | "x"
77 | "y"
78 | )
79 |
80 | var x = []int{
81 | 1,
82 | 2,
83 | 3,
84 | }
85 |
86 | Now the distinction is between nothing and something, instead of
87 | two different characters, and it's more pronounced. If we had known
88 | from the start that semicolons would be optional I think we might
89 | have used them in more syntactic forms, or maybe made some forms
90 | accept either commas or semicolons. At this point that seems
91 | unlikely, though.
92 |
93 | Q: Why does Go name its while loops "for"?
94 |
95 | A: C has both while(cond) {} and for(;cond;) {}. It didn't seem
96 | like Go needed two keywords for the same thing.
97 |
98 | Q: There seem to be a lot of new languages emerging these days,
99 | including Rust, D, Swift and Nim, among probably others. Are there
100 | any lessons you've learned from these other languages and their
101 | communities that you wish you'd been able to incorporate into Go?
102 |
103 | A: I do watch those languages for developments. I think they've
104 | learned things from Go and I hope we've also learned things from
105 | them. Some day I'd like the Go compiler to do a better job of
106 | inferring ownership rules, or maybe even having lightweight ownership
107 | expressions in the type system. Javari, Midori, Pony, and Rust are
108 | inspirations here. I wrote a bit more about this at
109 | https://research.swtch.com/go2017.
110 |
111 | Q: Why the focus on concurrency and goroutines?
112 |
113 | A: We knew from past experience that good concurrency support using
114 | channels and lightweight processes would make writing the kinds of
115 | systems we built at Google a lot easier, as I hope the lecture
116 | showed. There's a bit more explanation at https://golang.org/doc/faq#csp,
117 | and some background about our earlier experiences at
118 | https://swtch.com/~rsc/thread/.
119 |
--------------------------------------------------------------------------------
/Lectures/LEC09/gopattern.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC09/gopattern.pdf
--------------------------------------------------------------------------------
/Lectures/LEC10/9.1.5-9.1.6.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC10/9.1.5-9.1.6.pdf
--------------------------------------------------------------------------------
/Lectures/LEC10/9.5.2-9.5.3.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC10/9.5.2-9.5.3.pdf
--------------------------------------------------------------------------------
/Lectures/LEC10/9.6.3.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC10/9.6.3.pdf
--------------------------------------------------------------------------------
/Lectures/LEC10/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 10: Distributed Transactions
2 |
3 | ## 预习
4 |
5 | Read [6.033 book Chapter 9](atomicity_open_5_0.pdf) , just [9.1.5, 9.1.6](9.1.5-9.1.6.pdf), [9.5.2, 9.5.3](9.5.2-9.5.3.pdf), [9.6.3](9.6.3.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. How does this material fit into 6.824?
10 | 1. Why is it so important for transactions to be atomic?
11 | 1. Could one use Raft instead of two-phase commit?
12 | 1. In two-phase commit, why would a worker send an abort message, rather than a PREPARED message?
13 | 1. Can two-phase locking generate deadlock?
14 | 1. Why does it matter whether locks are held until after a transaction commits or aborts?
15 | 1. What is the point of the two-phase locking rule that says a transaction isn't allowed to acquire any locks after the first time that it releases a lock?
16 | 1. Does two-phase commit solve the dilemma of the two generals described in the reading's Section 9.6.4?
17 | 1. Are the locks exclusive, or can they allow multiple readers to have simultaneous access?
18 | 1. How should one decide between pessimistic and optimistic concurrency control?
19 | 1. What should two-phase commit workers do if the transaction coordinator crashes?
20 | 1. Why don't people use three-phase commit, which allows workers to commit or abort even if the coordinator crashes?
21 |
22 | ## 课堂
23 |
24 | [讲义](l-2pc.txt)
25 |
26 | [FAQ 答案](chapter9-faq.txt)
27 |
28 | ## 作业
29 |
30 | [6.033 Book](https://ocw.mit.edu/resources/res-6-004-principles-of-computer-system-design-an-introduction-spring-2009/online-textbook/). Read just these parts of Chapter 9: 9.1.5, 9.1.6, 9.5.2, 9.5.3, 9.6.3. The last two sections (on two-phase locking and distributed two-phase commit) are the most important. The Question: describe a situation where Two-Phase Locking yields higher performance than Simple Locking.
--------------------------------------------------------------------------------
/Lectures/LEC10/atomicity_open_5_0_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC10/atomicity_open_5_0_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC11/farm-2015_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC11/farm-2015_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC12/6.824 Lab 4_ Sharded Key_Value Service_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC12/6.824 Lab 4_ Sharded Key_Value Service_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC12/6.824 Lab 4_ Sharded Key_Value Service_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC12/6.824 Lab 4_ Sharded Key_Value Service_files/style.css:
--------------------------------------------------------------------------------
1 | body { max-width: 45em; }
2 | body pre { overflow-x: auto; }
3 |
4 | body {
5 | color: black;
6 | background-color: white;
7 | font-family: sans-serif;
8 | }
9 |
10 | .title {
11 | text-align: center
12 | }
13 | .subtitle {
14 | text-align: center;
15 | font-style: italic;
16 | }
17 | .author {
18 | text-align: center;
19 | }
20 |
21 | ul.hints, .note, .challenge, .todo, pre {
22 | margin: 1em;
23 | border: 1px dashed;
24 | padding: 1em;
25 | }
26 |
27 | ul.hints { color: #50A02D; }
28 | ul.hints li { margin-left: 1em; }
29 | ul.hints li::before {
30 | content: "Hint: ";
31 | font-weight: bold;
32 | }
33 |
34 | .important {
35 | margin: 1em;
36 | padding: 1em;
37 | background-color: #990000;
38 | color: #fff;
39 | }
40 | .important::before {
41 | content: "Important: ";
42 | background-color: #550000;
43 | width: 100%;
44 | display: block;
45 | margin: -1em -1em 1em -1em;
46 | padding: 1em;
47 | font-weight: bold;
48 | }
49 | .note { color: #4682B4; }
50 | .note::before {
51 | content: "Note: ";
52 | font-weight: bold;
53 | }
54 |
55 | .challenge, .todo { border-style: solid; }
56 | .challenge::before, .todo::before {
57 | float: right;
58 | font-weight: bold;
59 | color: white;
60 | margin-right: -1em;
61 | margin-top: -1em;
62 | margin-bottom: .5em;
63 | margin-left: 1em;
64 | padding: .5em 1em;
65 | }
66 | .todo { color: #B22222; }
67 | .todo::before {
68 | content: "TASK";
69 | background: #B22222;
70 | }
71 | .challenge { color: #8B4513; }
72 | .challenge::before {
73 | content: "CHALLENGE";
74 | background: #8B4513;
75 | }
76 |
77 |
78 | tt, code {
79 | font-family: monospace;
80 | border-radius: 3px;
81 | font-size: 110%;
82 | color: #657b83;
83 | background-color: #fdf6e3;
84 | padding: 0.2em;
85 | word-wrap: break-word;
86 | }
87 |
88 | pre {
89 | font-size: 100%;
90 | color: #839496;
91 | background: #002b36;
92 | }
93 |
94 | .classic {
95 | color: black;
96 | }
97 |
98 |
99 | div.required .header {
100 | font-weight: bold;
101 | }
102 |
103 | div.challenge .header {
104 | font-style: italic;
105 | }
106 |
107 | div.required {
108 | background-color: #eeeeff;
109 | }
110 |
111 |
--------------------------------------------------------------------------------
/Lectures/LEC12/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 12: Spark Case Study
2 |
3 | ## 预习
4 |
5 | [Resilient Distributed Datasets: A Fault-Tolerant Abstraction forIn-Memory Cluster Computing](zaharia-spark_cropped.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. Is Spark currently in use in any major applications?
10 | 1. How common is it for PhD students to create something on the scale of Spark?
11 | 1. Should we view Spark as being similar to MapReduce?
12 | 1. Why are RDDs called immutable if they allow for transformations?
13 | 1. Do distributed systems designers worry about energy efficiency?
14 | 1. How do applications figure out the location of an RDD?
15 | 1. How does Spark achieve fault tolerance?
16 | 1. Why is Spark developed using Scala? What's special about the language?
17 | 1. Does anybody still use MapReduce rather than Spark, since Spark seems to be strictly superior? If so, why do people still use MR?
18 | 1. Is the RDD concept implemented in any systems other than Spark?
19 |
20 | ## 上课
21 |
22 | [讲义](l-spark.txt)
23 |
24 | [FAQ 答案](spark-faq.txt)
25 |
26 | ## 作业
27 |
28 | Resilient Distributed Datasets: A Fault-Tolerant Abstraction for In-Memory Cluster Computing What applications can Spark support well that MapReduce/Hadoop cannot support?
29 |
30 | ## LAB 4
31 |
32 | [LAB 4 说明](6.824 Lab 4_ Sharded Key_Value Service.html)
--------------------------------------------------------------------------------
/Lectures/LEC12/zaharia-spark_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC12/zaharia-spark_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC13/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC13/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC13/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC13/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC13/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 13: Naiad Case Study
2 |
3 | ## 预习
4 |
5 | [Naiad: A Timely Dataflow System](naiad_cropped.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. Is Naiad/timely dataflow in use in production environments anywhere? If not, and what would be barriers to adoption?
10 | 1. The paper says that Naiad performs better than Spark. In what cases would one prefer which one over the other?
11 | 1. What was the reasoning for implmenting Naiad in C#? It seems like this language choice introduced lots of pain, and performance would be better in C/C++.
12 | 1. I'm confused by the OnRecv/OnNotify/SendBy/NotifyAt API. Does the user need to tell the system when to notify another vertex?
13 | 1. What do the authors mean when they state systems like Spark "requires centralized modifications to the dataflow graph, which introduce substantial overhead"? Why does Spark perform worse than Naiad for iterative computations?
14 |
15 | ## 课堂
16 |
17 | [讲义](l-naiad.txt)
18 |
19 | [FAQ 答案](naiad-faq.txt)
20 |
21 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC13/naiad_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC13/naiad_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC14/6.824 Spring 2018 Paper Questions.html:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC14/6.824 Spring 2018 Paper Questions.html
--------------------------------------------------------------------------------
/Lectures/LEC14/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC14/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC14/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC14/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC14/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 14: Parameter Server Case Study
2 |
3 | ## 预习
4 |
5 | [Scaling Distributed Machine Learning with the Parameter Server](parameter.pdf)
6 |
7 | ## FAQ
8 |
9 | Is the parameter server model useful for deep neural networks? It looks like this paper was written in 2014, which was before deep learning suddenly became very popular.
10 |
11 | ## 课堂
12 |
13 | [讲义](l-parameter.txt)
14 |
15 | [FAQ 答案](parameter-faq.txt)
16 |
17 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC14/parameter-faq.txt:
--------------------------------------------------------------------------------
1 | Q: Is the parameter server model useful for deep neural networks? It looks like this paper was written in 2014, which was before deep learning suddenly became very popular.
2 |
3 | A: Yes! Neural networks tend to have many (millions) of parameters, so they're a good fit for a parameter server model. Indeed, the parameter server model has been quite influential in distributed deep learning frameworks. For example, TensorFlow's distributed execution uses parameter servers: https://www.tensorflow.org/deploy/distributed (the "ps" hosts are parameter servers).
4 |
5 | Even though neural networks do not always have the kind of sparse parameter spaces that the examples in the paper use, parameter servers are still beneficial for them. Each PS aggregate gradient changes from multiple workers; if the workers had to do all-to-all exchange of gradient changes, this would require much higher network bandwidth. See this example:
6 |
7 | https://stackoverflow.com/questions/39559183/what-is-the-reason-to-use-parameter-server-in-distributed-tensorflow-learning
8 |
--------------------------------------------------------------------------------
/Lectures/LEC14/parameter.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC14/parameter.pdf
--------------------------------------------------------------------------------
/Lectures/LEC14/parameter_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC14/parameter_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC15/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC15/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC15/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC15/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC15/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 15: Frangipani
2 |
3 | ## 预习
4 |
5 | [Frangipani:A ScalableDistributedFileSystem](thekkath-frangipani.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. Why are we reading this paper?
10 | 1. How does Frangipani differ from GFS?
11 | 1. Why do the Petal servers in the Frangipani system have a block interface? Why not have file servers (like AFS), that know about things like directories and files?
12 | 1. Can a workstation running Frangipani break security?
13 | 1. What is Digital Equipment?
14 | 1. What does the comment "File creation takes longer..." in Section 9.2 mean?
15 | 1. Frangipani is over 20 years old now; what's the state-of-the-art in distributed file systems?
16 | 1. The paper says Frangipani only does crash recovery for its own file system meta-data (i-nodes, directories, free bitmaps), but not for users' file content. What does that mean, and why is it OK?
17 | 1. What's the difference between the log stored in the Frangipani workstation and the log stored on Petal?
18 | 1. How does Petal take efficient snapshots of the large virtual disk that it represents?
19 | 1. The paper says that Frangipani doesn't immediately send new log entries to Petal. What happens if a workstation crashes after a system call completes, but before it send the corresponding log entry to Petal?
20 | 1. What does it mean to stripe a file? Is this similar to sharding?
21 | 1. What is the "false sharing" problem mentioned in the paper?
22 |
23 | ## 课堂
24 |
25 | [讲义](l-frangipani.txt)
26 |
27 | [FAQ 答案](frangipani-faq.txt)
28 |
29 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC15/thekkath-frangipani.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC15/thekkath-frangipani.pdf
--------------------------------------------------------------------------------
/Lectures/LEC15/thekkath-frangipani_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC15/thekkath-frangipani_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC16/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC16/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC16/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC16/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC16/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 16: Scaling Memcache at Facebook
2 |
3 | ## 预习
4 |
5 | [Scaling Memcache at Facebook](memcache-fb.pdf)
6 |
7 | ## FAQ
8 |
9 | 无
10 |
11 | ## 课堂
12 |
13 | [讲义](l-memcached.txt)
14 |
15 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC16/memcache-fb.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC16/memcache-fb.pdf
--------------------------------------------------------------------------------
/Lectures/LEC16/memcache-fb_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC16/memcache-fb_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC17/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 17: Eventual Consistency, Bayou
2 |
3 | ## 预习
4 |
5 | [Managing Update Conflicts in Bayou,a Weakly Connected Replicated Storage Syste](bayou-conflicts.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. A lot of Bayou's design is driven by the desire to support disconnected operation. Is that still important today?
10 | 1. Doesn't widely-available wireless Internet mean everyone is connected all the time?
11 | 1. Bayou supports direct synchronization of one device to another, e.g. over Bluetooth or infrared, without going through the Internet or a server. Is that important?
12 | 1. Does anyone use Bayou today? If not, why are we reading this paper?
13 | 1. Has the idea of applications performing conflict resolution been used in other distributed systems?
14 | 1. Do companies like Dropbox use protocols similar to Bayou?
15 | 1. What does it mean for data to be weakly consistent?
16 | 1. Is eventual consistency the best you can do if you want to support disconnected operation?
17 | 1. It seems like writing dependency checks and merge procedures for a variety of operations could be a tough interface for programmers to handle. Is there anything I'm missing there?
18 | 1. Is the primary replica a single point of failure?
19 | 1. How do dependency checks detect Write-Write conflicts? The paper says "Such conflicts can be detected by having the dependency check query the current values of any data items being updated and ensure that they have not changed from the values they had at the time the Write was submitted", but I don't quite understand in this case what the expected result of the dependency check is.
20 | 1. When are dependency checks called?
21 | 1. If two clients make conflicting calendar reservations on partitioned servers, do the dependency checks get called when those two servers communicate?
22 | 1. It looks like the logic in the dependency check would take place when you're first inserting a write operation, but you wouldn't find any conflicts from partitioned servers.
23 | 1. What are anti-entropy sessions?
24 | 1. What is an epidemic algorithm?
25 | 1. Why are Write exchange sessions called anti-entropy sessions?
26 | 1. In order to know if writes are stabilized, does a server have to contact all other servers?
27 | 1. How much time could it take for a Write to reach all servers?
28 | 1. In what case is automatic resolution not possible? Does it only depend on the application, or is it the case that for any application, it's possible for automatic resolution to fail?
29 | 1. What are examples of good (quick convergence) and not-so-good anti-entropy policies?
30 | 1. I don't understand why "tentative deletion may result in a tuple that appears in the committed view but not in the full view." (very beginning of page 8)
31 | 1. Bayou introduces a lot of new ideas, but it's not clear which ideas are most important for performance.
32 | 1. What kind of information does the Undo Log contain? (e.g. does it contain a snapshot of changed files from a Write, or the reverse operation?) Or is this more of an implementation detail?
33 | 1. How is a particular server designated as the primary?
34 | 1. What if communication fails in the middle of an anti-entropy session?
35 | 1. Does Bayou cache?
36 | 1. What are the session guarantees mentioned by the paper?
37 |
38 | ## 课堂
39 |
40 | [讲义](l-bayou.txt)
41 |
42 | [FAQ 答案](bayou-faq.txt)
43 |
44 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC17/bayou-conflicts.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC17/bayou-conflicts.pdf
--------------------------------------------------------------------------------
/Lectures/LEC17/bayou-conflicts_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC17/bayou-conflicts_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC18/README.md:
--------------------------------------------------------------------------------
1 | # Guest lecturer: Frank Dabek of Google
2 |
--------------------------------------------------------------------------------
/Lectures/LEC19/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 19: P2P, DHTs, and Chord
2 |
3 | ## 预习
4 |
5 | [Chord:A Scalable Peer-to-peerLookupServiceforInternetApplications](stoica-chord.pdf) 和 [Trackerless Bittorrent](bep_0005.rst_post.html)
6 |
7 | ## FAQ
8 |
9 | 1. Is hashing across machines a good way to get load balanced sharding? Why not explicitly divide up the key space so it's evenly split?
10 | 1. Does BitTorrent use Chord?
11 | 1. If you want to add fault-tolerance to a Chord-based system should you replicate each Chord node using Raft?
12 | 1. What if Chord DHT nodes are malicious?
13 | 1. Is Chord used anywhere in practice?
14 | 1. Could the performance be improved if the nodes knew more about network locality?
15 | 1. Is it possible to design a DHT in which lookups take less than log(N) hops?
16 | 1. Does consistent hashing of keys still guarantee load balanced nodes if keys are not evenly distributed?
17 | 1. In the case of concurrent joins and failures, Chord pauses when a get fails to find the key it was looking for. If there's constant activity, how can Chord distinguish between the system not being stable and the key not actually existing?
18 | 1. If I introduce a malicious peer in Chord that keeps returning wrong values or inexistent addresses how disruptive can it be to the whole DHT? How does Bittorrent deal with particular issue?
19 | 1. Why isn’t there a danger of improper load balancing if some keys are simply used more than others?
20 |
21 | ## 课堂
22 |
23 | [讲义](l-dht.txt)
24 |
25 | [FAQ 答案](chord-faq.txt)
26 |
27 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC19/bep_0005.rst_post_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC19/bep_0005.rst_post_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC19/bep_0005.rst_post_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC19/stoica-chord.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC19/stoica-chord.pdf
--------------------------------------------------------------------------------
/Lectures/LEC19/stoica-chord_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC19/stoica-chord_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC20/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC20/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC20/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC20/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC20/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 20: Dynamo
2 |
3 | ## 预习
4 |
5 | [Dynamo: Amazon’s Highly Available Key-value Store](dynamo.pdf)
6 |
7 | ## FAQ
8 |
9 | Q: What's a typical number of nodes in a Dynamo instance? Large enough that vector clocks will become impractical large?
10 | Q: How can deleted items resurface in a shopping cart (Section 4.4)?
11 | Q: How does Dynamo recover from permanent failure -- what is anti-entropy using Merkle trees?
12 | Q: What are virtual nodes?
13 | Q: Will Dynamo's use of a DHT prevent it from scaling?
14 | Q: What's a gossip-based protocol?
15 |
16 | ## 课堂
17 |
18 | [讲义](l-dynamo.txt)
19 |
20 | [FAQ 答案](dynamo-faq.txt)
21 |
22 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC20/dynamo-faq.txt:
--------------------------------------------------------------------------------
1 | Dynamo FAQ
2 |
3 | Q: What's a typical number of nodes in a Dynamo instance? Large enough that
4 | vector clocks will become impractical large?
5 |
6 | A: I don't know how big Amazon's Dynamo instances are, but I expect they are
7 | sizeable. As long as there are no failures the same node will do the puts and
8 | thus the VV stays small (1 item). With weird failure patterns it may grow large,
9 | but if the VV grows larger than 10 items, Dynamo throws out the
10 | least-recently-used entry. (This means that there could be cases that Dynamo
11 | thinks there is a conflict, but if it would have remembered all entries, there
12 | wasn't an actual conflict.)
13 |
14 | Q: How can deleted items resurface in a shopping cart (Section 4.4)?
15 |
16 | A: Suppose there are two copies of the shopping cart, each in a different data
17 | center. Suppose both copies are identical and have one item in them. Now suppose
18 | a user deletes the item, but the second copy is unreachable, maybe because of a
19 | network partition. Dynamo will update the first copy, but not the second; the
20 | user will see that the shopping cart is empty. Now the network reconnects, and
21 | the two copies must be reconciled. The application using Dynamo is responsible
22 | for doing the reconciliation since it knows the semantics of the objects. In the
23 | case of a shopping cart, the application will take the union of the shopping
24 | carts' contents (which is less sophisticated than Bayou but simpler). Now if the
25 | user looks at the shopping cart again, the deleted item has re-appeared.
26 |
27 | Q: How does Dynamo recover from permanent failure -- what is anti-entropy using
28 | Merkle trees?
29 |
30 | A: In this paper, anti-entropy is a big word for synchronizing two replicas. To
31 | determine what is different between two replicas, Dynamo traverses a Merkle
32 | representation of the two replicas. If a top-level node matches, then Dynamo
33 | doesn't descend that branch. If a node in the tree don't match, they copy the
34 | branch of the new version to the old one. The use of Merkle trees allows the
35 | authors to copy only the parts of the tree that are different. Wikipedia has a
36 | picture to illustrate: https://en.wikipedia.org/wiki/Merkle_tree.
37 |
38 | Q: What are virtual nodes?
39 |
40 | A: They are a scheme to balance the keys nicely across all nodes. If you throw N
41 | balls in B bins, you get on average B/N balls in a bin but some bins may have
42 | many balls. To ensure that all bins are close to the average, you can place
43 | multiple virtual bins in each real bin. The average of multiple random variables
44 | has lower variance than a single random variable.
45 |
46 | Q: Will Dynamo's use of a DHT prevent it from scaling?
47 |
48 | A: It is pretty-well understood now how to build DHTs that scale to large number
49 | of nodes, even O(1) DHTs (e.g., see
50 | http://www.news.cs.nyu.edu/~jinyang/pub/nsdi05-accordion.pdf). The Dynamo
51 | solution as described is an O(1) DHT, with ok scalability. I think the authors
52 | figured that they will use a solution from the literature once they actually
53 | have a scaling problem.
54 |
55 | Q: What's a gossip-based protocol?
56 |
57 | A: Any system that doesn't have a master that knows about all participants in
58 | the system typically has a protocol to find other members; often such protocols
59 | are called gossip protocols, because the participants need to gossip (gather
60 | information from other nodes) to determine who is part of the system. More
61 | generally, gossip refers to information spreading over a whole system via pairs
62 | of computers exchanging what they know. You can learn more about gossip
63 | protocols from Wikipedia: https://en.wikipedia.org/wiki/Gossip_protocol.
64 |
--------------------------------------------------------------------------------
/Lectures/LEC20/dynamo.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC20/dynamo.pdf
--------------------------------------------------------------------------------
/Lectures/LEC20/dynamo_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC20/dynamo_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC21/6.824 Spring 2018 Paper Questions_files/1153146:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC21/6.824 Spring 2018 Paper Questions_files/1153146
--------------------------------------------------------------------------------
/Lectures/LEC21/6.824 Spring 2018 Paper Questions_files/common.js:
--------------------------------------------------------------------------------
1 | function autofill_email() {
2 | var spans = document.getElementsByTagName("span");
3 | for (i = 0; i < spans.length; i++) {
4 | if (spans[i].className != "email_autofill")
5 | continue;
6 | spans[i].innerHTML = spans[i].innerHTML.replace(/ at /, "@");
7 | }
8 | }
9 |
10 | function parse_query() {
11 | var parts = location.search.substr(1).split("&");
12 | var query = {};
13 |
14 | for (var i = 0; i < parts.length; i++) {
15 | var variable, value;
16 | var index = parts[i].indexOf("=");
17 | if (index < 0) {
18 | variable = decodeURIComponent(parts[i]);
19 | value = "";
20 | } else {
21 | variable = decodeURIComponent(parts[i].substr(0, index));
22 | value = decodeURIComponent(parts[i].substr(index + 1));
23 | }
24 | query[variable] = value;
25 | }
26 | return query;
27 | }
28 |
29 | function init_questions(query) {
30 | var questions = document.getElementById('questions');
31 | if (!questions)
32 | return;
33 |
34 | var id = query["q"];
35 | if (id) {
36 | var qlist = questions.getElementsByTagName('div');
37 | for (var i = 0; i < qlist.length; i++) {
38 | if (qlist[i].id != id) {
39 | questions.removeChild(qlist[i]);
40 | i--;
41 | } else if (query["lec"]) {
42 | // Append the lecture number if we have it.
43 | var p = document.createElement("p");
44 | var b = document.createElement("b");
45 | if ("textContent" in b)
46 | b.textContent = "Lecture " + query["lec"];
47 | else
48 | b.innerText = "Lecture " + query["lec"];
49 | p.appendChild(b);
50 | qlist[i].insertBefore(p, qlist[i].firstChild);
51 | }
52 | }
53 | }
54 | }
55 |
56 | function date_to_txt(d) {
57 | var month = ["jan", "feb", "mar", "apr", "may", "jun",
58 | "jul", "aug", "sep", "oct", "nov", "dec"];
59 | return month[d.getMonth()] + ' ' + d.getDate();
60 | }
61 |
62 | function find_date_txt(t) {
63 | var els = document.getElementsByTagName('font');
64 | for (var i = 0; i < els.length; i++) {
65 | if (els[i].className == "date" && els[i].innerHTML == t)
66 | return els[i];
67 | }
68 | return null;
69 | }
70 |
71 | function highlight_date() {
72 | var d = new Date();
73 | if (d.getFullYear() != 2015) {
74 | return;
75 | }
76 |
77 | for (var fwd = 0; fwd < 3; fwd++) {
78 | var t = date_to_txt(d);
79 | var e = find_date_txt(t);
80 | if (e) {
81 | e.parentNode.style.border = '2px solid blue';
82 | break;
83 | }
84 | d.setTime(d.getTime() + 24*60*60*1000);
85 | }
86 | }
87 |
88 | function page_onload() {
89 | var query = parse_query();
90 | autofill_email();
91 | init_questions(query);
92 | highlight_date();
93 | }
94 |
--------------------------------------------------------------------------------
/Lectures/LEC21/6.824 Spring 2018 Paper Questions_files/saved_resource.html:
--------------------------------------------------------------------------------
1 |
2 |
3 |
--------------------------------------------------------------------------------
/Lectures/LEC21/README.md:
--------------------------------------------------------------------------------
1 | # Lecture 21: Bitcoin
2 |
3 | ## 预习
4 |
5 | [Bitcoin: A Peer-to-Peer Electronic Cash Syste](bitcoin.pdf)
6 |
7 | ## FAQ
8 |
9 | 1. I don't understand why the blockchain is so important. Isn't the requirement for the owner's signature on each transaction enough to prevent bitcoins from being stolen?
10 | 1. Why does Bitcoin need to define a new currency? Wouldn't it be more convenient to use an existing currency like dollars?
11 | 1. Why is the purpose of proof-of-work?
12 | 1. Could a Bitcoin-like system use something less wasteful than proof-of-work?
13 | 1. Can Alice spend the same coin twice by sending "pay Bob" and "pay Charlie" to different subsets of miners?
14 | 1. It takes an average of 10 minutes for a Bitcoin block to be validated. Does this mean that the parties involved aren't sure if the transaction really happened until 10 minutes later?
15 | 1. What can be done to speed up transactions on the blockchain?
16 | 1. The entire blockchain needs to be downloaded before a node can participate in the network. Won't that take an impractically long time as the blockchain grows?
17 | 1. Is it feasible for an attacker to gain a majority of the computing power among peers?
18 | 1. Are there any ways for Bitcoin mining to do useful work, beyond simply brute-force calculating SHA-256 hashes?
19 | 1. There is hardware specifically designed to mine Bitcoin. How does this type of hardware differ from the type of hardware in a laptop?
20 | 1. The paper estimates that the disk space required to store the block chain will by 4.2 megabytes per year. That seems very low!
21 | 1. Would the advent of quantum computing break the bitcoin system?
22 | 1. Bitcoin uses the hash of the transaction record to identify the transaction, so it can be named in future transactions. Is this guaranteed to lead to unique IDs?
23 | 1. It sounds like anyone can create new Bitcoins. Why is that OK? Won't it lead to forgery or inflation?
24 | 1. The paper mentions that some amount of fraud is admissible; where does this fraud come from?
25 | 1. Has there been fraudulent use of Bitcoin?
26 | 1. Satoshi's paper mentions that each transaction has its own transaction fees that are given to whoever mined the block. Why would a miner not simply try to mine blocks with transactions with the highest transaction fees?
27 | 1. Why would a miner bother including transactions that yield no fee?
28 | 1. How are transaction fees determined/advertised?
29 | 1. What are some techniques for storing my personal bitcoins, in particular the private keys needed to spend my bitcoins? I've heard of people printing out the keys, replicating them on USB, etc. Does a secure online repository exist?
30 | 1. What other kinds of virtual currency were there before and after Bitcoin (I know the paper mentioned hashcash)? What was different about Bitcoin that led it to have more success than its predecessors?
31 | 1. What happens when more (or fewer) people mine Bitcoin?
32 | 1. Is there any way to make Bitcoin completely anonymous?
33 | 1. If I lose the private key(s) associated with the bitcoins I own, how can I get my money back?
34 | 1. What do people buy and sell with bitcoins?
35 | 1. Why is bitcoin illegal in some countries?
36 | 1. Why do bitcoins have any value at all? Why do people accept it as money?
37 | 1. How is the price of Bitcoin determined?
38 | 1. Why is the price of bitcoin so volatile?
39 |
40 | ## 课堂
41 |
42 | [讲义](l-bitcoin.txt)
43 |
44 | [FAQ 答案](bitcoin-faq.txt)
45 |
46 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC21/bitcoin.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC21/bitcoin.pdf
--------------------------------------------------------------------------------
/Lectures/LEC21/bitcoin_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC21/bitcoin_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/LEC22/README.md:
--------------------------------------------------------------------------------
1 | # LEC 22: Project demos
2 |
3 | ## 预习
4 |
5 | [Experiences with a Distributed, Scalable,Methodological File System: AnalogicFS](katabi-analogicfs.pdf)
6 |
7 | ## FAQ
8 |
9 | Why are we reading this paper?
10 |
11 | ## 课堂
12 |
13 | [FAQ 答案](analogicfs-faq.txt)
14 |
15 | ## [作业](6.824 Spring 2018 Paper Questions.html)
--------------------------------------------------------------------------------
/Lectures/LEC22/analogicfs-faq.txt:
--------------------------------------------------------------------------------
1 | Q: Why are we reading this paper?
2 |
3 | A: Because it is a fun, intriguing read. This paper was "written" (see
4 | https://pdos.csail.mit.edu/archive/scigen/) by a 6.824 TAs many years ago, and
5 | assigning it has become a 6.824 tradition. We enjoy reading the answers to the
6 | posted question. The assignment also has a slightly serious undertone: be
7 | careful with what you read and believe.
8 |
9 |
--------------------------------------------------------------------------------
/Lectures/LEC22/katabi-analogicfs.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC22/katabi-analogicfs.pdf
--------------------------------------------------------------------------------
/Lectures/LEC22/katabi-analogicfs_cropped.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/Lectures/LEC22/katabi-analogicfs_cropped.pdf
--------------------------------------------------------------------------------
/Lectures/README.md:
--------------------------------------------------------------------------------
1 | # 6.824 课程安排
2 |
3 | [课程表](6.824 Schedule_ Spring 2018.html)
4 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | # This is the Makefile helping you submit the labs.
2 | # Just create 6.824/api.key with your API key in it,
3 | # and submit your lab with the following command:
4 | # $ make [lab1|lab2a|lab2b|lab2c|lab3a|lab3b|lab4a|lab4b]
5 |
6 | LABS=" lab1 lab2a lab2b lab2c lab3a lab3b lab4a lab4b "
7 |
8 | %:
9 | @echo "Preparing $@-handin.tar.gz"
10 | @echo "Checking for committed temporary files..."
11 | @if git ls-files | grep -E 'mrtmp|mrinput' > /dev/null; then \
12 | echo "" ; \
13 | echo "OBS! You have committed some large temporary files:" ; \
14 | echo "" ; \
15 | git ls-files | grep -E 'mrtmp|mrinput' | sed 's/^/\t/' ; \
16 | echo "" ; \
17 | echo "Follow the instructions at http://stackoverflow.com/a/308684/472927" ; \
18 | echo "to remove them, and then run make again." ; \
19 | echo "" ; \
20 | exit 1 ; \
21 | fi
22 | @if echo $(LABS) | grep -q " $@ " ; then \
23 | echo "Tarring up your submission..." ; \
24 | tar cvzf $@-handin.tar.gz \
25 | "--exclude=src/main/pg-*.txt" \
26 | "--exclude=src/main/diskvd" \
27 | "--exclude=src/mapreduce/824-mrinput-*.txt" \
28 | "--exclude=mrtmp.*" \
29 | "--exclude=src/main/diff.out" \
30 | Makefile src; \
31 | if ! test -e api.key ; then \
32 | echo "Missing $(PWD)/api.key. Please create the file with your key in it or submit the $@-handin.tar.gz via the web interface."; \
33 | else \
34 | echo "Are you sure you want to submit $@? Enter 'yes' to continue:"; \
35 | read line; \
36 | if test "$$line" != "yes" ; then echo "Giving up submission"; exit; fi; \
37 | if test `stat -c "%s" "$@-handin.tar.gz" 2>/dev/null || stat -f "%z" "$@-handin.tar.gz"` -ge 20971520 ; then echo "File exceeds 20MB."; exit; fi; \
38 | mv api.key api.key.fix ; \
39 | cat api.key.fix | tr -d '\n' > api.key ; \
40 | rm api.key.fix ; \
41 | curl -F file=@$@-handin.tar.gz -F "key= /dev/null || { \
43 | echo ; \
44 | echo "Submit seems to have failed."; \
45 | echo "Please upload the tarball manually on the submission website."; } \
46 | fi; \
47 | else \
48 | echo "Bad target $@. Usage: make [$(LABS)]"; \
49 | fi
50 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | #
2 |
3 | ## 课程内容
4 |
5 | - [LEC 01: Introduction](Lectures/LEC01)
6 | - [LEC 02: RPC and Threads, Crawler, K/V](Lectures/LEC02)
7 | - [LEC 03: GFS](Lectures/LEC03)
8 | - [LEC 04: Primary-Backup Replication](Lectures/LEC04)
9 | - [LEC 05: Fault Tolerance: Raft](Lectures/LEC05)
10 | - [LEC 06: Fault Tolerance: Raft](Lectures/LEC06)
11 | - [LEC 07: Spinnaker](Lectures/LEC07)
12 | - [LEC 08: Zookeeper](Lectures/LEC08)
13 | - [LEC 09: Guest lecturer on Go (Russ Cox Google/Go)](Lectures/LEC09)
14 | - [LEC 10: Distributed Transactions](Lectures/LEC10)
15 | - [LEC 11: Optimistic Concurrency Control](Lectures/LEC11)
16 | - [LEC 12: Big Data: Spark](Lectures/LEC12)
17 | - [LEC 13: Big Data: Naiad](Lectures/LEC13)
18 | - [LEC 14: Distributed Machine Learning: Parameter Server](Lectures/LEC14)
19 | - [LEC 15: Cache Consistency: Frangipani](Lectures/LEC15)
20 | - [LEC 16: Cache Consistency: Memcached at Facebook](Lectures/LEC16)
21 | - [LEC 17: Disconnected Operation, Eventual Consistency](Lectures/LEC17)
22 | - [LEC 18: Guest lecturer: Frank Dabek of Google](Lectures/LEC18)
23 | - [LEC 19: Peer-to-peer, DHTs](Lectures/LEC19)
24 | - [LEC 20: Dynamo](Lectures/LEC20)
25 | - [LEC 21: Peer-to-peer: Bitcoin](Lectures/LEC21)
26 | - [LEC 22: Project demos](Lectures/LEC22)
27 |
--------------------------------------------------------------------------------
/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-cayman
--------------------------------------------------------------------------------
/pkg/linux_amd64/labgob.a:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/pkg/linux_amd64/labgob.a
--------------------------------------------------------------------------------
/pkg/linux_amd64/labrpc.a:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/pkg/linux_amd64/labrpc.a
--------------------------------------------------------------------------------
/pkg/linux_amd64/raft.a:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/pkg/linux_amd64/raft.a
--------------------------------------------------------------------------------
/src/6.824.zip:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/aQuaYi/MIT-6.824-Distributed-Systems/a49acb477d4be0cadacac89691550e4c2636bc33/src/6.824.zip
--------------------------------------------------------------------------------
/src/README.md:
--------------------------------------------------------------------------------
1 | # Labs 上机实验
2 |
--------------------------------------------------------------------------------
/src/kvraft/client.go:
--------------------------------------------------------------------------------
1 | package raftkv
2 |
3 | import "labrpc"
4 | import "crypto/rand"
5 | import "math/big"
6 |
7 |
8 | type Clerk struct {
9 | servers []*labrpc.ClientEnd
10 | // You will have to modify this struct.
11 | }
12 |
13 | func nrand() int64 {
14 | max := big.NewInt(int64(1) << 62)
15 | bigx, _ := rand.Int(rand.Reader, max)
16 | x := bigx.Int64()
17 | return x
18 | }
19 |
20 | func MakeClerk(servers []*labrpc.ClientEnd) *Clerk {
21 | ck := new(Clerk)
22 | ck.servers = servers
23 | // You'll have to add code here.
24 | return ck
25 | }
26 |
27 | //
28 | // fetch the current value for a key.
29 | // returns "" if the key does not exist.
30 | // keeps trying forever in the face of all other errors.
31 | //
32 | // you can send an RPC with code like this:
33 | // ok := ck.servers[i].Call("KVServer.Get", &args, &reply)
34 | //
35 | // the types of args and reply (including whether they are pointers)
36 | // must match the declared types of the RPC handler function's
37 | // arguments. and reply must be passed as a pointer.
38 | //
39 | func (ck *Clerk) Get(key string) string {
40 |
41 | // You will have to modify this function.
42 | return ""
43 | }
44 |
45 | //
46 | // shared by Put and Append.
47 | //
48 | // you can send an RPC with code like this:
49 | // ok := ck.servers[i].Call("KVServer.PutAppend", &args, &reply)
50 | //
51 | // the types of args and reply (including whether they are pointers)
52 | // must match the declared types of the RPC handler function's
53 | // arguments. and reply must be passed as a pointer.
54 | //
55 | func (ck *Clerk) PutAppend(key string, value string, op string) {
56 | // You will have to modify this function.
57 | }
58 |
59 | func (ck *Clerk) Put(key string, value string) {
60 | ck.PutAppend(key, value, "Put")
61 | }
62 | func (ck *Clerk) Append(key string, value string) {
63 | ck.PutAppend(key, value, "Append")
64 | }
65 |
--------------------------------------------------------------------------------
/src/kvraft/common.go:
--------------------------------------------------------------------------------
1 | package raftkv
2 |
3 | const (
4 | OK = "OK"
5 | ErrNoKey = "ErrNoKey"
6 | )
7 |
8 | type Err string
9 |
10 | // Put or Append
11 | type PutAppendArgs struct {
12 | Key string
13 | Value string
14 | Op string // "Put" or "Append"
15 | // You'll have to add definitions here.
16 | // Field names must start with capital letters,
17 | // otherwise RPC will break.
18 | }
19 |
20 | type PutAppendReply struct {
21 | WrongLeader bool
22 | Err Err
23 | }
24 |
25 | type GetArgs struct {
26 | Key string
27 | // You'll have to add definitions here.
28 | }
29 |
30 | type GetReply struct {
31 | WrongLeader bool
32 | Err Err
33 | Value string
34 | }
35 |
--------------------------------------------------------------------------------
/src/kvraft/server.go:
--------------------------------------------------------------------------------
1 | package raftkv
2 |
3 | import (
4 | "labgob"
5 | "labrpc"
6 | "log"
7 | "raft"
8 | "sync"
9 | )
10 |
11 | const Debug = 0
12 |
13 | func DPrintf(format string, a ...interface{}) (n int, err error) {
14 | if Debug > 0 {
15 | log.Printf(format, a...)
16 | }
17 | return
18 | }
19 |
20 | type Op struct {
21 | // Your definitions here.
22 | // Field names must start with capital letters,
23 | // otherwise RPC will break.
24 | }
25 |
26 | type KVServer struct {
27 | mu sync.Mutex
28 | me int
29 | rf *raft.Raft
30 | applyCh chan raft.ApplyMsg
31 |
32 | maxraftstate int // snapshot if log grows this big
33 |
34 | // Your definitions here.
35 | }
36 |
37 | func (kv *KVServer) Get(args *GetArgs, reply *GetReply) {
38 | // TODO: Your code here.
39 | }
40 |
41 | func (kv *KVServer) PutAppend(args *PutAppendArgs, reply *PutAppendReply) {
42 | // TODO: Your code here.
43 | }
44 |
45 | //
46 | // the tester calls Kill() when a KVServer instance won't
47 | // be needed again. you are not required to do anything
48 | // in Kill(), but it might be convenient to (for example)
49 | // turn off debug output from this instance.
50 | //
51 | func (kv *KVServer) Kill() {
52 | kv.rf.Kill()
53 | // TODO: Your code here, if desired.
54 | }
55 |
56 | //
57 | // servers[] contains the ports of the set of
58 | // servers that will cooperate via Raft to
59 | // form the fault-tolerant key/value service.
60 | // me is the index of the current server in servers[].
61 | // the k/v server should store snapshots with persister.SaveSnapshot(),
62 | // and Raft should save its state (including log) with persister.SaveRaftState().
63 | // the k/v server should snapshot when Raft's saved state exceeds maxraftstate bytes,
64 | // in order to allow Raft to garbage-collect its log. if maxraftstate is -1,
65 | // you don't need to snapshot.
66 | // StartKVServer() must return quickly, so it should start goroutines
67 | // for any long-running work.
68 | //
69 | func StartKVServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister, maxraftstate int) *KVServer {
70 | // call labgob.Register on structures you want
71 | // Go's RPC library to marshall/unmarshall.
72 | labgob.Register(Op{})
73 |
74 | kv := new(KVServer)
75 | kv.me = me
76 | kv.maxraftstate = maxraftstate
77 |
78 | // You may need initialization code here.
79 |
80 | kv.applyCh = make(chan raft.ApplyMsg)
81 | kv.rf = raft.Make(servers, me, persister, kv.applyCh)
82 |
83 | // You may need initialization code here.
84 |
85 | return kv
86 | }
87 |
--------------------------------------------------------------------------------
/src/labgob/labgob.go:
--------------------------------------------------------------------------------
1 | package labgob
2 |
3 | //
4 | // trying to send non-capitalized fields over RPC produces a range of
5 | // misbehavior, including both mysterious incorrect computation and
6 | // outright crashes. so this wrapper around Go's encoding/gob warns
7 | // about non-capitalized field names.
8 | //
9 |
10 | import "encoding/gob"
11 | import "io"
12 | import "reflect"
13 | import "fmt"
14 | import "sync"
15 | import "unicode"
16 | import "unicode/utf8"
17 |
18 | var mu sync.Mutex
19 | var errorCount int // for TestCapital
20 | var checked map[reflect.Type]bool
21 |
22 | type LabEncoder struct {
23 | gob *gob.Encoder
24 | }
25 |
26 | func NewEncoder(w io.Writer) *LabEncoder {
27 | enc := &LabEncoder{}
28 | enc.gob = gob.NewEncoder(w)
29 | return enc
30 | }
31 |
32 | func (enc *LabEncoder) Encode(e interface{}) error {
33 | checkValue(e)
34 | return enc.gob.Encode(e)
35 | }
36 |
37 | func (enc *LabEncoder) EncodeValue(value reflect.Value) error {
38 | checkValue(value.Interface())
39 | return enc.gob.EncodeValue(value)
40 | }
41 |
42 | type LabDecoder struct {
43 | gob *gob.Decoder
44 | }
45 |
46 | func NewDecoder(r io.Reader) *LabDecoder {
47 | dec := &LabDecoder{}
48 | dec.gob = gob.NewDecoder(r)
49 | return dec
50 | }
51 |
52 | func (dec *LabDecoder) Decode(e interface{}) error {
53 | checkValue(e)
54 | checkDefault(e)
55 | return dec.gob.Decode(e)
56 | }
57 |
58 | func Register(value interface{}) {
59 | checkValue(value)
60 | gob.Register(value)
61 | }
62 |
63 | func RegisterName(name string, value interface{}) {
64 | checkValue(value)
65 | gob.RegisterName(name, value)
66 | }
67 |
68 | func checkValue(value interface{}) {
69 | checkType(reflect.TypeOf(value))
70 | }
71 |
72 | func checkType(t reflect.Type) {
73 | k := t.Kind()
74 |
75 | mu.Lock()
76 | // only complain once, and avoid recursion.
77 | if checked == nil {
78 | checked = map[reflect.Type]bool{}
79 | }
80 | if checked[t] {
81 | mu.Unlock()
82 | return
83 | }
84 | checked[t] = true
85 | mu.Unlock()
86 |
87 | switch k {
88 | case reflect.Struct:
89 | for i := 0; i < t.NumField(); i++ {
90 | f := t.Field(i)
91 | rune, _ := utf8.DecodeRuneInString(f.Name)
92 | if unicode.IsUpper(rune) == false {
93 | // ta da
94 | fmt.Printf("labgob error: lower-case field %v of %v in RPC or persist/snapshot will break your Raft\n",
95 | f.Name, t.Name())
96 | mu.Lock()
97 | errorCount += 1
98 | mu.Unlock()
99 | }
100 | checkType(f.Type)
101 | }
102 | return
103 | case reflect.Slice, reflect.Array, reflect.Ptr:
104 | checkType(t.Elem())
105 | return
106 | case reflect.Map:
107 | checkType(t.Elem())
108 | checkType(t.Key())
109 | return
110 | default:
111 | return
112 | }
113 | }
114 |
115 | //
116 | // warn if the value contains non-default values,
117 | // as it would if one sent an RPC but the reply
118 | // struct was already modified. if the RPC reply
119 | // contains default values, GOB won't overwrite
120 | // the non-default value.
121 | //
122 | func checkDefault(value interface{}) {
123 | if value == nil {
124 | return
125 | }
126 | checkDefault1(reflect.ValueOf(value), 1, "")
127 | }
128 |
129 | func checkDefault1(value reflect.Value, depth int, name string) {
130 | if depth > 3 {
131 | return
132 | }
133 |
134 | t := value.Type()
135 | k := t.Kind()
136 |
137 | switch k {
138 | case reflect.Struct:
139 | for i := 0; i < t.NumField(); i++ {
140 | vv := value.Field(i)
141 | name1 := t.Field(i).Name
142 | if name != "" {
143 | name1 = name + "." + name1
144 | }
145 | checkDefault1(vv, depth+1, name1)
146 | }
147 | return
148 | case reflect.Ptr:
149 | if value.IsNil() {
150 | return
151 | }
152 | checkDefault1(value.Elem(), depth+1, name)
153 | return
154 | case reflect.Bool,
155 | reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
156 | reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
157 | reflect.Uintptr, reflect.Float32, reflect.Float64,
158 | reflect.String:
159 | if reflect.DeepEqual(reflect.Zero(t).Interface(), value.Interface()) == false {
160 | mu.Lock()
161 | if errorCount < 1 {
162 | what := name
163 | if what == "" {
164 | what = t.Name()
165 | }
166 | fmt.Printf("labgob warning: Decoding into a non-default variable/field %v may not work\n",
167 | what)
168 | }
169 | errorCount += 1
170 | mu.Unlock()
171 | }
172 | return
173 | }
174 | }
175 |
--------------------------------------------------------------------------------
/src/labgob/test_test.go:
--------------------------------------------------------------------------------
1 | package labgob
2 |
3 | import "testing"
4 |
5 | import "bytes"
6 |
7 | type T1 struct {
8 | T1int0 int
9 | T1int1 int
10 | T1string0 string
11 | T1string1 string
12 | }
13 |
14 | type T2 struct {
15 | T2slice []T1
16 | T2map map[int]*T1
17 | T2t3 interface{}
18 | }
19 |
20 | type T3 struct {
21 | T3int999 int
22 | }
23 |
24 | //
25 | // test that we didn't break GOB.
26 | //
27 | func TestGOB(t *testing.T) {
28 | e0 := errorCount
29 |
30 | w := new(bytes.Buffer)
31 |
32 | Register(T3{})
33 |
34 | {
35 | x0 := 0
36 | x1 := 1
37 | t1 := T1{}
38 | t1.T1int1 = 1
39 | t1.T1string1 = "6.824"
40 | t2 := T2{}
41 | t2.T2slice = []T1{T1{}, t1}
42 | t2.T2map = map[int]*T1{}
43 | t2.T2map[99] = &T1{1, 2, "x", "y"}
44 | t2.T2t3 = T3{999}
45 |
46 | e := NewEncoder(w)
47 | e.Encode(x0)
48 | e.Encode(x1)
49 | e.Encode(t1)
50 | e.Encode(t2)
51 | }
52 | data := w.Bytes()
53 |
54 | {
55 | var x0 int
56 | var x1 int
57 | var t1 T1
58 | var t2 T2
59 |
60 | r := bytes.NewBuffer(data)
61 | d := NewDecoder(r)
62 | if d.Decode(&x0) != nil ||
63 | d.Decode(&x1) != nil ||
64 | d.Decode(&t1) != nil ||
65 | d.Decode(&t2) != nil {
66 | t.Fatalf("Decode failed")
67 | }
68 |
69 | if x0 != 0 {
70 | t.Fatalf("wrong x0 %v\n", x0)
71 | }
72 | if x1 != 1 {
73 | t.Fatalf("wrong x1 %v\n", x1)
74 | }
75 | if t1.T1int0 != 0 {
76 | t.Fatalf("wrong t1.T1int0 %v\n", t1.T1int0)
77 | }
78 | if t1.T1int1 != 1 {
79 | t.Fatalf("wrong t1.T1int1 %v\n", t1.T1int1)
80 | }
81 | if t1.T1string0 != "" {
82 | t.Fatalf("wrong t1.T1string0 %v\n", t1.T1string0)
83 | }
84 | if t1.T1string1 != "6.824" {
85 | t.Fatalf("wrong t1.T1string1 %v\n", t1.T1string1)
86 | }
87 | if len(t2.T2slice) != 2 {
88 | t.Fatalf("wrong t2.T2slice len %v\n", len(t2.T2slice))
89 | }
90 | if t2.T2slice[1].T1int1 != 1 {
91 | t.Fatalf("wrong slice value\n")
92 | }
93 | if len(t2.T2map) != 1 {
94 | t.Fatalf("wrong t2.T2map len %v\n", len(t2.T2map))
95 | }
96 | if t2.T2map[99].T1string1 != "y" {
97 | t.Fatalf("wrong map value\n")
98 | }
99 | t3 := (t2.T2t3).(T3)
100 | if t3.T3int999 != 999 {
101 | t.Fatalf("wrong t2.T2t3.T3int999\n")
102 | }
103 | }
104 |
105 | if errorCount != e0 {
106 | t.Fatalf("there were errors, but should not have been")
107 | }
108 | }
109 |
110 | type T4 struct {
111 | Yes int
112 | no int
113 | }
114 |
115 | //
116 | // make sure we check capitalization
117 | // labgob prints one warning during this test.
118 | //
119 | func TestCapital(t *testing.T) {
120 | e0 := errorCount
121 |
122 | v := []map[*T4]int{}
123 |
124 | w := new(bytes.Buffer)
125 | e := NewEncoder(w)
126 | e.Encode(v)
127 | data := w.Bytes()
128 |
129 | var v1 []map[T4]int
130 | r := bytes.NewBuffer(data)
131 | d := NewDecoder(r)
132 | d.Decode(&v1)
133 |
134 | if errorCount != e0+1 {
135 | t.Fatalf("failed to warn about lower-case field")
136 | }
137 | }
138 |
139 | //
140 | // check that we warn when someone sends a default value over
141 | // RPC but the target into which we're decoding holds a non-default
142 | // value, which GOB seems not to overwrite as you'd expect.
143 | //
144 | // labgob does not print a warning.
145 | //
146 | func TestDefault(t *testing.T) {
147 | e0 := errorCount
148 |
149 | type DD struct {
150 | X int
151 | }
152 |
153 | // send a default value...
154 | dd1 := DD{}
155 |
156 | w := new(bytes.Buffer)
157 | e := NewEncoder(w)
158 | e.Encode(dd1)
159 | data := w.Bytes()
160 |
161 | // and receive it into memory that already
162 | // holds non-default values.
163 | reply := DD{99}
164 |
165 | r := bytes.NewBuffer(data)
166 | d := NewDecoder(r)
167 | d.Decode(&reply)
168 |
169 | if errorCount != e0+1 {
170 | t.Fatalf("failed to warn about decoding into non-default value")
171 | }
172 | }
173 |
--------------------------------------------------------------------------------
/src/linearizability/bitset.go:
--------------------------------------------------------------------------------
1 | package linearizability
2 |
3 | type bitset []uint64
4 |
5 | // data layout:
6 | // bits 0-63 are in data[0], the next are in data[1], etc.
7 |
8 | func newBitset(bits uint) bitset {
9 | extra := uint(0)
10 | if bits%64 != 0 {
11 | extra = 1
12 | }
13 | chunks := bits/64 + extra
14 | return bitset(make([]uint64, chunks))
15 | }
16 |
17 | func (b bitset) clone() bitset {
18 | dataCopy := make([]uint64, len(b))
19 | copy(dataCopy, b)
20 | return bitset(dataCopy)
21 | }
22 |
23 | func bitsetIndex(pos uint) (uint, uint) {
24 | return pos / 64, pos % 64
25 | }
26 |
27 | func (b bitset) set(pos uint) bitset {
28 | major, minor := bitsetIndex(pos)
29 | b[major] |= (1 << minor)
30 | return b
31 | }
32 |
33 | func (b bitset) clear(pos uint) bitset {
34 | major, minor := bitsetIndex(pos)
35 | b[major] &^= (1 << minor)
36 | return b
37 | }
38 |
39 | func (b bitset) get(pos uint) bool {
40 | major, minor := bitsetIndex(pos)
41 | return b[major]&(1<> 1)
48 | v = (v & 0x3333333333333333) + ((v & 0xCCCCCCCCCCCCCCCC) >> 2)
49 | v = (v & 0x0F0F0F0F0F0F0F0F) + ((v & 0xF0F0F0F0F0F0F0F0) >> 4)
50 | v *= 0x0101010101010101
51 | total += uint((v >> 56) & 0xFF)
52 | }
53 | return total
54 | }
55 |
56 | func (b bitset) hash() uint64 {
57 | hash := uint64(b.popcnt())
58 | for _, v := range b {
59 | hash ^= v
60 | }
61 | return hash
62 | }
63 |
64 | func (b bitset) equals(b2 bitset) bool {
65 | if len(b) != len(b2) {
66 | return false
67 | }
68 | for i := range b {
69 | if b[i] != b2[i] {
70 | return false
71 | }
72 | }
73 | return true
74 | }
75 |
--------------------------------------------------------------------------------
/src/linearizability/model.go:
--------------------------------------------------------------------------------
1 | package linearizability
2 |
3 | type Operation struct {
4 | Input interface{}
5 | Call int64 // invocation time
6 | Output interface{}
7 | Return int64 // response time
8 | }
9 |
10 | type EventKind bool
11 |
12 | const (
13 | CallEvent EventKind = false
14 | ReturnEvent EventKind = true
15 | )
16 |
17 | type Event struct {
18 | Kind EventKind
19 | Value interface{}
20 | Id uint
21 | }
22 |
23 | type Model struct {
24 | // Partition functions, such that a history is linearizable if an only
25 | // if each partition is linearizable. If you don't want to implement
26 | // this, you can always use the `NoPartition` functions implemented
27 | // below.
28 | Partition func(history []Operation) [][]Operation
29 | PartitionEvent func(history []Event) [][]Event
30 | // Initial state of the system.
31 | Init func() interface{}
32 | // Step function for the system. Returns whether or not the system
33 | // could take this step with the given inputs and outputs and also
34 | // returns the new state. This should not mutate the existing state.
35 | Step func(state interface{}, input interface{}, output interface{}) (bool, interface{})
36 | // Equality on states. If you are using a simple data type for states,
37 | // you can use the `ShallowEqual` function implemented below.
38 | Equal func(state1, state2 interface{}) bool
39 | }
40 |
41 | func NoPartition(history []Operation) [][]Operation {
42 | return [][]Operation{history}
43 | }
44 |
45 | func NoPartitionEvent(history []Event) [][]Event {
46 | return [][]Event{history}
47 | }
48 |
49 | func ShallowEqual(state1, state2 interface{}) bool {
50 | return state1 == state2
51 | }
52 |
--------------------------------------------------------------------------------
/src/linearizability/models.go:
--------------------------------------------------------------------------------
1 | package linearizability
2 |
3 | // kv model
4 |
5 | type KvInput struct {
6 | Op uint8 // 0 => get, 1 => put, 2 => append
7 | Key string
8 | Value string
9 | }
10 |
11 | type KvOutput struct {
12 | Value string
13 | }
14 |
15 | func KvModel() Model {
16 | return Model {
17 | Partition: func(history []Operation) [][]Operation {
18 | m := make(map[string][]Operation)
19 | for _, v := range history {
20 | key := v.Input.(KvInput).Key
21 | m[key] = append(m[key], v)
22 | }
23 | var ret [][]Operation
24 | for _, v := range m {
25 | ret = append(ret, v)
26 | }
27 | return ret
28 | },
29 | Init: func() interface{} {
30 | // note: we are modeling a single key's value here;
31 | // we're partitioning by key, so this is okay
32 | return ""
33 | },
34 | Step: func(state, input, output interface{}) (bool, interface{}) {
35 | inp := input.(KvInput)
36 | out := output.(KvOutput)
37 | st := state.(string)
38 | if inp.Op == 0 {
39 | // get
40 | return out.Value == st, state
41 | } else if inp.Op == 1 {
42 | // put
43 | return true, inp.Value
44 | } else {
45 | // append
46 | return true, (st + inp.Value)
47 | }
48 | },
49 | Equal: ShallowEqual,
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/main/diskvd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //
4 | // start a diskvd server. it's a member of some replica
5 | // group, which has other members, and it needs to know
6 | // how to talk to the members of the shardmaster service.
7 | // used by ../diskv/test_test.go
8 | //
9 | // arguments:
10 | // -g groupid
11 | // -m masterport1 -m masterport2 ...
12 | // -s replicaport1 -s replicaport2 ...
13 | // -i my-index-in-server-port-list
14 | // -u unreliable
15 | // -d directory
16 | // -r restart
17 |
18 | import "time"
19 | import "diskv"
20 | import "os"
21 | import "fmt"
22 | import "strconv"
23 | import "runtime"
24 |
25 | func usage() {
26 | fmt.Printf("Usage: diskvd -g gid -m master... -s server... -i my-index -d dir\n")
27 | os.Exit(1)
28 | }
29 |
30 | func main() {
31 | var gid int64 = -1 // my replica group ID
32 | masters := []string{} // ports of shardmasters
33 | replicas := []string{} // ports of servers in my replica group
34 | me := -1 // my index in replicas[]
35 | unreliable := false
36 | dir := "" // store persistent data here
37 | restart := false
38 |
39 | for i := 1; i+1 < len(os.Args); i += 2 {
40 | a0 := os.Args[i]
41 | a1 := os.Args[i+1]
42 | if a0 == "-g" {
43 | gid, _ = strconv.ParseInt(a1, 10, 64)
44 | } else if a0 == "-m" {
45 | masters = append(masters, a1)
46 | } else if a0 == "-s" {
47 | replicas = append(replicas, a1)
48 | } else if a0 == "-i" {
49 | me, _ = strconv.Atoi(a1)
50 | } else if a0 == "-u" {
51 | unreliable, _ = strconv.ParseBool(a1)
52 | } else if a0 == "-d" {
53 | dir = a1
54 | } else if a0 == "-r" {
55 | restart, _ = strconv.ParseBool(a1)
56 | } else {
57 | usage()
58 | }
59 | }
60 |
61 | if gid < 0 || me < 0 || len(masters) < 1 || me >= len(replicas) || dir == "" {
62 | usage()
63 | }
64 |
65 | runtime.GOMAXPROCS(4)
66 |
67 | srv := diskv.StartServer(gid, masters, replicas, me, dir, restart)
68 | srv.Setunreliable(unreliable)
69 |
70 | // for safety, force quit after 10 minutes.
71 | time.Sleep(10 * 60 * time.Second)
72 | mep, _ := os.FindProcess(os.Getpid())
73 | mep.Kill()
74 | }
75 |
--------------------------------------------------------------------------------
/src/main/ii.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import "os"
4 | import "fmt"
5 | import "mapreduce"
6 |
7 | // The mapping function is called once for each piece of the input.
8 | // In this framework, the key is the name of the file that is being processed,
9 | // and the value is the file's contents. The return value should be a slice of
10 | // key/value pairs, each represented by a mapreduce.KeyValue.
11 | func mapF(document string, value string) (res []mapreduce.KeyValue) {
12 | // TODO: Your code here (Part V).
13 | }
14 |
15 | // The reduce function is called once for each key generated by Map, with a
16 | // list of that key's string value (merged across all inputs). The return value
17 | // should be a single output value for that key.
18 | func reduceF(key string, values []string) string {
19 | // TODO: Your code here (Part V).
20 | }
21 |
22 | // Can be run in 3 ways:
23 | // 1) Sequential (e.g., go run wc.go master sequential x1.txt .. xN.txt)
24 | // 2) Master (e.g., go run wc.go master localhost:7777 x1.txt .. xN.txt)
25 | // 3) Worker (e.g., go run wc.go worker localhost:7777 localhost:7778 &)
26 | func main() {
27 | if len(os.Args) < 4 {
28 | fmt.Printf("%s: see usage comments in file\n", os.Args[0])
29 | } else if os.Args[1] == "master" {
30 | var mr *mapreduce.Master
31 | if os.Args[2] == "sequential" {
32 | mr = mapreduce.Sequential("iiseq", os.Args[3:], 3, mapF, reduceF)
33 | } else {
34 | mr = mapreduce.Distributed("iiseq", os.Args[3:], 3, os.Args[2])
35 | }
36 | mr.Wait()
37 | } else {
38 | mapreduce.RunWorker(os.Args[2], os.Args[3], mapF, reduceF, 100, nil)
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/lockc.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //
4 | // see comments in lockd.go
5 | //
6 |
7 | import "lockservice"
8 | import "os"
9 | import "fmt"
10 |
11 | func usage() {
12 | fmt.Printf("Usage: lockc -l|-u primaryport backupport lockname\n")
13 | os.Exit(1)
14 | }
15 |
16 | func main() {
17 | if len(os.Args) == 5 {
18 | ck := lockservice.MakeClerk(os.Args[2], os.Args[3])
19 | var ok bool
20 | if os.Args[1] == "-l" {
21 | ok = ck.Lock(os.Args[4])
22 | } else if os.Args[1] == "-u" {
23 | ok = ck.Unlock(os.Args[4])
24 | } else {
25 | usage()
26 | }
27 | fmt.Printf("reply: %v\n", ok)
28 | } else {
29 | usage()
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/lockd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | // export GOPATH=~/6.824
4 | // go build lockd.go
5 | // go build lockc.go
6 | // ./lockd -p a b &
7 | // ./lockd -b a b &
8 | // ./lockc -l a b lx
9 | // ./lockc -u a b lx
10 | //
11 | // on Athena, use /tmp/myname-a and /tmp/myname-b
12 | // instead of a and b.
13 |
14 | import "time"
15 | import "lockservice"
16 | import "os"
17 | import "fmt"
18 |
19 | func main() {
20 | if len(os.Args) == 4 && os.Args[1] == "-p" {
21 | lockservice.StartServer(os.Args[2], os.Args[3], true)
22 | } else if len(os.Args) == 4 && os.Args[1] == "-b" {
23 | lockservice.StartServer(os.Args[2], os.Args[3], false)
24 | } else {
25 | fmt.Printf("Usage: lockd -p|-b primaryport backupport\n")
26 | os.Exit(1)
27 | }
28 | for {
29 | time.Sleep(100 * time.Second)
30 | }
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/mr-challenge.txt:
--------------------------------------------------------------------------------
1 | www: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
2 | year: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
3 | years: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
4 | yesterday: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
5 | yet: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
6 | you: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
7 | young: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
8 | your: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
9 | yourself: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
10 | zip: 8 pg-being_ernest.txt,pg-dorian_gray.txt,pg-frankenstein.txt,pg-grimm.txt,pg-huckleberry_finn.txt,pg-metamorphosis.txt,pg-sherlock_holmes.txt,pg-tom_sawyer.txt
11 |
--------------------------------------------------------------------------------
/src/main/mr-testout.txt:
--------------------------------------------------------------------------------
1 | that: 7871
2 | it: 7987
3 | in: 8415
4 | was: 8578
5 | a: 13382
6 | of: 13536
7 | I: 14296
8 | to: 16079
9 | and: 23612
10 | the: 29748
11 |
--------------------------------------------------------------------------------
/src/main/pbc.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //
4 | // pbservice client application
5 | //
6 | // export GOPATH=~/6.824
7 | // go build viewd.go
8 | // go build pbd.go
9 | // go build pbc.go
10 | // ./viewd /tmp/rtm-v &
11 | // ./pbd /tmp/rtm-v /tmp/rtm-1 &
12 | // ./pbd /tmp/rtm-v /tmp/rtm-2 &
13 | // ./pbc /tmp/rtm-v key1 value1
14 | // ./pbc /tmp/rtm-v key1
15 | //
16 | // change "rtm" to your user name.
17 | // start the pbd programs in separate windows and kill
18 | // and restart them to exercise fault tolerance.
19 | //
20 |
21 | import "pbservice"
22 | import "os"
23 | import "fmt"
24 |
25 | func usage() {
26 | fmt.Printf("Usage: pbc viewport key\n")
27 | fmt.Printf(" pbc viewport key value\n")
28 | os.Exit(1)
29 | }
30 |
31 | func main() {
32 | if len(os.Args) == 3 {
33 | // get
34 | ck := pbservice.MakeClerk(os.Args[1], "")
35 | v := ck.Get(os.Args[2])
36 | fmt.Printf("%v\n", v)
37 | } else if len(os.Args) == 4 {
38 | // put
39 | ck := pbservice.MakeClerk(os.Args[1], "")
40 | ck.Put(os.Args[2], os.Args[3])
41 | } else {
42 | usage()
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/src/main/pbd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //
4 | // see directions in pbc.go
5 | //
6 |
7 | import "time"
8 | import "pbservice"
9 | import "os"
10 | import "fmt"
11 |
12 | func main() {
13 | if len(os.Args) != 3 {
14 | fmt.Printf("Usage: pbd viewport myport\n")
15 | os.Exit(1)
16 | }
17 |
18 | pbservice.StartServer(os.Args[1], os.Args[2])
19 |
20 | for {
21 | time.Sleep(100 * time.Second)
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/test-ii.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | go run ii.go master sequential pg-*.txt
3 |
4 | # cause sort to be case sensitive.
5 | # on Ubuntu (Athena) it's otherwise insensitive.
6 | LC_ALL=C
7 | export LC_ALL
8 |
9 | sort -k1,1 mrtmp.iiseq | sort -snk2,2 | grep -v '16' | tail -10 | diff - mr-challenge.txt > diff.out
10 | if [ -s diff.out ]
11 | then
12 | echo "Failed test. Output should be as in mr-challenge.txt. Your output differs as follows (from diff.out):" > /dev/stderr
13 | cat diff.out
14 | else
15 | echo "Passed test" > /dev/stderr
16 | fi
17 |
18 |
--------------------------------------------------------------------------------
/src/main/test-mr.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | here=$(dirname "$0")
3 | [[ "$here" = /* ]] || here="$PWD/$here"
4 | export GOPATH="$here/../../"
5 | echo ""
6 | echo "==> Part I"
7 | go test -run Sequential mapreduce/...
8 | echo ""
9 | echo "==> Part II"
10 | (cd "$here" && sh ./test-wc.sh > /dev/null)
11 | echo ""
12 | echo "==> Part III"
13 | go test -run TestParallel mapreduce/...
14 | echo ""
15 | echo "==> Part IV"
16 | go test -run Failure mapreduce/...
17 | echo ""
18 | echo "==> Part V (inverted index)"
19 | (cd "$here" && sh ./test-ii.sh > /dev/null)
20 |
21 | rm "$here"/mrtmp.* "$here"/diff.out
22 |
--------------------------------------------------------------------------------
/src/main/test-wc.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | go run wc.go master sequential pg-*.txt
3 | sort -n -k2 mrtmp.wcseq | tail -10 | diff - mr-testout.txt > diff.out
4 | if [ -s diff.out ]
5 | then
6 | echo "Failed test. Output should be as in mr-testout.txt. Your output differs as follows (from diff.out):" > /dev/stderr
7 | cat diff.out
8 | else
9 | echo "Passed test" > /dev/stderr
10 | fi
11 |
12 |
--------------------------------------------------------------------------------
/src/main/viewd.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | //
4 | // see directions in pbc.go
5 | //
6 |
7 | import "time"
8 | import "viewservice"
9 | import "os"
10 | import "fmt"
11 |
12 | func main() {
13 | if len(os.Args) != 2 {
14 | fmt.Printf("Usage: viewd port\n")
15 | os.Exit(1)
16 | }
17 |
18 | viewservice.StartServer(os.Args[1])
19 |
20 | for {
21 | time.Sleep(100 * time.Second)
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/wc.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "fmt"
5 | "mapreduce"
6 | "os"
7 | "strconv"
8 | "strings"
9 | "unicode"
10 | )
11 |
12 | //
13 | // The map function is called once for each file of input. The first
14 | // argument is the name of the input file, and the second is the
15 | // file's complete contents. You should ignore the input file name,
16 | // and look only at the contents argument. The return value is a slice
17 | // of key/value pairs.
18 | //
19 | func mapF(filename string, contents string) []mapreduce.KeyValue {
20 | // 把 contents 划分成单词
21 | f := func(c rune) bool {
22 | return !unicode.IsLetter(c)
23 | }
24 | words := strings.FieldsFunc(contents, f)
25 |
26 | // 统计 filename 中每个单词的次数
27 | counter := make(map[string]int, 1024)
28 | for i := range words {
29 | counter[words[i]]++
30 | }
31 |
32 | res := make([]mapreduce.KeyValue, 0, len(counter))
33 | for w, c := range counter {
34 | res = append(res, mapreduce.KeyValue{
35 | Key: w,
36 | // 把 w 和 c 绑定在一起
37 | // reduce 任务才知道 value 是对应的哪个 w 的值
38 | Value: fmt.Sprintf("%s:%d", w, c),
39 | })
40 | }
41 |
42 | return res
43 | }
44 |
45 | //
46 | // The reduce function is called once for each key generated by the
47 | // map tasks, with a list of all the values created for that key by
48 | // any map task.
49 | func reduceF(key string, values []string) string {
50 | pre := key + ":"
51 | n := len(pre)
52 | res := 0
53 | for i := range values {
54 | // 只有 key 对应的 value 才会被处理
55 | if !strings.HasPrefix(values[i], pre) {
56 | continue
57 | }
58 | // 把 value 转换成数字，并累加到结果上
59 | num, _ := strconv.Atoi(values[i][n:])
60 | res += num
61 | }
62 |
63 | return strconv.Itoa(res)
64 | }
65 |
66 | // Can be run in 3 ways:
67 | // 1) Sequential (e.g., go run wc.go master sequential x1.txt .. xN.txt)
68 | // 2) Master (e.g., go run wc.go master localhost:7777 x1.txt .. xN.txt)
69 | // 3) Worker (e.g., go run wc.go worker localhost:7777 localhost:7778 &)
70 | func main() {
71 | if len(os.Args) < 4 {
72 | fmt.Printf("%s: see usage comments in file\n", os.Args[0])
73 | } else if os.Args[1] == "master" {
74 | var mr *mapreduce.Master
75 | if os.Args[2] == "sequential" {
76 | mr = mapreduce.Sequential("wcseq", os.Args[3:], 3, mapF, reduceF)
77 | } else {
78 | mr = mapreduce.Distributed("wcseq", os.Args[3:], 3, os.Args[2])
79 | }
80 | mr.Wait()
81 | } else {
82 | mapreduce.RunWorker(os.Args[2], os.Args[3], mapF, reduceF, 100, nil)
83 | }
84 | }
85 |
--------------------------------------------------------------------------------
/src/mapreduce/common.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "fmt"
5 | "strconv"
6 | )
7 |
8 | // Debugging enabled?
9 | const debugEnabled = false
10 |
11 | // debug() will only print if debugEnabled is true
12 | func debug(format string, a ...interface{}) (n int, err error) {
13 | if debugEnabled {
14 | n, err = fmt.Printf(format, a...)
15 | }
16 | return
17 | }
18 |
19 | // jobPhase indicates whether a task is scheduled as a map or reduce task.
20 | type jobPhase string
21 |
22 | const (
23 | mapPhase jobPhase = "mapPhase"
24 | reducePhase = "reducePhase"
25 | )
26 |
27 | // KeyValue is a type used to hold the key/value pairs passed to the map and
28 | // reduce functions.
29 | type KeyValue struct {
30 | Key string
31 | Value string
32 | }
33 |
34 | // reduceName constructs the name of the intermediate file which map task
35 | // produces for reduce task .
36 | func reduceName(jobName string, mapTask int, reduceTask int) string {
37 | return "mrtmp." + jobName + "-" + strconv.Itoa(mapTask) + "-" + strconv.Itoa(reduceTask)
38 | }
39 |
40 | // mergeName constructs the name of the output file of reduce task
41 | func mergeName(jobName string, reduceTask int) string {
42 | return "mrtmp." + jobName + "-res-" + strconv.Itoa(reduceTask)
43 | }
44 |
--------------------------------------------------------------------------------
/src/mapreduce/common_map.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "encoding/json"
5 | "hash/fnv"
6 | "io/ioutil"
7 | "log"
8 | "os"
9 | )
10 |
11 | func doMap(
12 | jobName string, // the name of the MapReduce job
13 | mapTask int, // which map task this is
14 | inFile string,
15 | nReduce int, // the number of reduce task that will be run ("R" in the paper)
16 | mapF func(filename string, contents string) []KeyValue,
17 | ) {
18 |
19 | // doMap manages one map task: it should read one of the input files
20 | // (inFile), call the user-defined map function (mapF) for that file's
21 | // contents, and partition mapF's output into nReduce intermediate files.
22 |
23 | // There is one intermediate file per reduce task. The file name
24 | // includes both the map task number and the reduce task number. Use
25 | // the filename generated by reduceName(jobName, mapTask, r)
26 | // as the intermediate file for reduce task r. Call ihash() (see below) on each key, mod nReduce, to pick r for a key/value pair.
27 |
28 | // mapF() is the map function provided by the application. The first
29 | // argument should be the input file name, though the map function
30 | // typically ignores it. The second argument should be the entire
31 | // input file contents. mapF() returns a slice containing the
32 | // key/value pairs for reduce; see common.go for the definition of
33 | // KeyValue.
34 |
35 | // Look at Go's ioutil and os packages for functions to read
36 | // and write files.
37 |
38 | // Coming up with a scheme for how to format the key/value pairs on
39 | // disk can be tricky, especially when taking into account that both
40 | // keys and values could contain newlines, quotes, and any other
41 | // character you can think of.
42 |
43 | // One format often used for serializing data to a byte stream that the
44 | // other end can correctly reconstruct is JSON. You are not required to
45 | // use JSON, but as the output of the reduce tasks *must* be JSON,
46 | // familiarizing yourself with it here may prove useful. You can write
47 | // out a data structure as a JSON string to a file using the commented
48 | // code below. The corresponding decoding functions can be found in
49 | // common_reduce.go.
50 |
51 | // enc := json.NewEncoder(file)
52 | // for _, kv := ... {
53 | // err := enc.Encode(&kv)
54 |
55 | // Remember to close the file after you have written all the values!
56 |
57 | // 读取 inFile 文件
58 | data, err := ioutil.ReadFile(inFile)
59 | if err != nil {
60 | log.Fatal(err)
61 | }
62 |
63 | // 利用 mapF 函数处理 inFile 文件内容，得到 kvs
64 | kvs := mapF(inFile, string(data))
65 |
66 | // 把 kvs 按照 R 任务的需求，划分成 nReduce 份
67 | for i := 0; i < nReduce; i++ {
68 | // Reduce 任务 i ，创建文件
69 | filename := reduceName(jobName, mapTask, i)
70 | file, err := os.Create(filename)
71 | if err != nil {
72 | log.Fatal(err)
73 | }
74 |
75 | // 把 file 作为 json 流式编码器的输出目的地
76 | enc := json.NewEncoder(file)
77 |
78 | for _, kv := range kvs {
79 | if ihash(kv.Key)%nReduce != i {
80 | // 根据题意，
81 | // 删除掉不需要 Reduce 任务 i 处理的 kv
82 | continue
83 | }
84 |
85 | // 对 kv 进行编码后，发送到 file
86 | err := enc.Encode(&kv)
87 | if err != nil {
88 | log.Fatal(err)
89 | }
90 | }
91 |
92 | // Map 任务 m 为 Reduce 任务 i 准备处理的内容
93 | // 已经全部收集完毕，所以，可以关闭 file 了
94 | file.Close()
95 | }
96 | }
97 |
98 | func ihash(s string) int {
99 | h := fnv.New32a()
100 | h.Write([]byte(s))
101 | return int(h.Sum32() & 0x7fffffff)
102 | }
103 |
--------------------------------------------------------------------------------
/src/mapreduce/common_reduce.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "encoding/json"
5 | "io"
6 | "log"
7 | "os"
8 | "sort"
9 | )
10 |
11 | func doReduce(
12 | jobName string, // the name of the whole MapReduce job
13 | reduceTask int, // which reduce task this is
14 | outFile string, // write the output here
15 | nMap int, // the number of map tasks that were run ("M" in the paper)
16 | reduceF func(key string, values []string) string,
17 | ) {
18 |
19 | // doReduce manages one reduce task: it should read the intermediate
20 | // files for the task, sort the intermediate key/value pairs by key,
21 | // call the user-defined reduce function (reduceF) for each key, and
22 | // write reduceF's output to disk.
23 |
24 | // You'll need to read one intermediate file from each map task;
25 | // reduceName(jobName, m, reduceTask) yields the file
26 | // name from map task m.
27 |
28 | // Your doMap() encoded the key/value pairs in the intermediate
29 | // files, so you will need to decode them. If you used JSON, you can
30 | // read and decode by creating a decoder and repeatedly calling
31 | // .Decode(&kv) on it until it returns an error.
32 |
33 | // You may find the first example in the golang sort package
34 | // documentation useful.
35 |
36 | // reduceF() is the application's reduce function. You should
37 | // call it once per distinct key, with a slice of all the values
38 | // for that key. reduceF() returns the reduced value for that key.
39 |
40 | // You should write the reduce output as JSON encoded KeyValue
41 | // objects to the file named outFile. We require you to use JSON
42 | // because that is what the merger than combines the output
43 | // from all the reduce tasks expects. There is nothing special about
44 | // JSON -- it is just the marshalling format we chose to use. Your
45 | // output code will look something like this:
46 |
47 | // enc := json.NewEncoder(file)
48 | // for key := ... {
49 | // enc.Encode(KeyValue{key, reduceF(...)})
50 | // }
51 | // file.Close()
52 |
53 | // 从 m*r 个 Map 任务生成的中间文件中，读取此 Reduce 任务需要的 m 个文件
54 |
55 | // Reduce 任务需要读取 Map 任务生成的 intermediate file 作为输入
56 | // kvs 是存在这些输入的容器
57 | kvs := make([]*KeyValue, 0, 2048)
58 | // 每个 Map 任务，都为此 Reduce 任务生成了一个 intermediate file
59 | // 所以，此 Reduce 任务总共需要读取 nMap 个文件
60 | for m := 0; m < nMap; m++ {
61 | // 打开需要读取的文件
62 | filename := reduceName(jobName, m, reduceTask)
63 | file, err := os.Open(filename)
64 | if err != nil {
65 | log.Fatal(err)
66 | }
67 |
68 | // 创建流式解码器
69 | dec := json.NewDecoder(file)
70 | for {
71 | var kv KeyValue
72 | // dec 读取一行，并解码到 kv 中
73 | err := dec.Decode(&kv)
74 | if err != nil {
75 | if err == io.EOF {
76 | // 读取到末尾
77 | break
78 | }
79 | log.Fatal(err)
80 | }
81 | // 把新获取的值保存到 kvs 中
82 | kvs = append(kvs, &kv)
83 | }
84 |
85 | file.Close()
86 | }
87 |
88 | // kvs 按照 key 的升序排列
89 | sort.Slice(kvs, func(i int, j int) bool {
90 | return kvs[i].Key < kvs[j].Key
91 | })
92 |
93 | // 集合所有的 values
94 | values := make([]string, len(kvs))
95 | for i := range values {
96 | values[i] = kvs[i].Value
97 | }
98 |
99 | // 创建输出文件
100 | file, err := os.Create(outFile)
101 | if err != nil {
102 | log.Fatal(err)
103 | }
104 | defer file.Close()
105 |
106 | // 创建流式编码器，每次被编码的内容，都会输出到 file 的一个新行
107 | enc := json.NewEncoder(file)
108 |
109 | // 对每个 key 都运用 reduceF
110 | for i := range kvs {
111 | key := kvs[i].Key
112 | err := enc.Encode(KeyValue{key, reduceF(key, values)})
113 | if err != nil {
114 | log.Fatal(err)
115 | }
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/src/mapreduce/common_rpc.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "fmt"
5 | "net/rpc"
6 | )
7 |
8 | // What follows are RPC types and methods.
9 | // Field names must start with capital letters, otherwise RPC will break.
10 |
11 | // DoTaskArgs holds the arguments that are passed to a worker when a job is
12 | // scheduled on it.
13 | type DoTaskArgs struct {
14 | JobName string
15 | File string // only for map, the input file
16 | Phase jobPhase // are we in mapPhase or reducePhase?
17 | TaskNumber int // this task's index in the current phase
18 |
19 | // NumOtherPhase is the total number of tasks in other phase; mappers
20 | // need this to compute the number of output bins, and reducers needs
21 | // this to know how many input files to collect.
22 | NumOtherPhase int
23 | }
24 |
25 | // ShutdownReply is the response to a WorkerShutdown.
26 | // It holds the number of tasks this worker has processed since it was started.
27 | type ShutdownReply struct {
28 | Ntasks int
29 | }
30 |
31 | // RegisterArgs is the argument passed when a worker registers with the master.
32 | type RegisterArgs struct {
33 | Worker string // the worker's UNIX-domain socket name, i.e. its RPC address
34 | }
35 |
36 | // call() sends an RPC to the rpcname handler on server srv
37 | // with arguments args, waits for the reply, and leaves the
38 | // reply in reply. the reply argument should be the address
39 | // of a reply structure.
40 | //
41 | // call() returns true if the server responded, and false if call()
42 | // received no reply from the server. reply's contents are valid if
43 | // and only if call() returned true.
44 | //
45 | // you should assume that call() will time out and return
46 | // false after a while if it doesn't get a reply from the server.
47 | //
48 | // please use call() to send all RPCs. please don't change this
49 | // function.
50 | //
51 | func call(srv string, rpcname string,
52 | args interface{}, reply interface{}) bool {
53 | c, errx := rpc.Dial("unix", srv)
54 | if errx != nil {
55 | return false
56 | }
57 | defer c.Close()
58 |
59 | err := c.Call(rpcname, args, reply)
60 | if err == nil {
61 | return true
62 | }
63 |
64 | fmt.Println(err)
65 | return false
66 | }
67 |
--------------------------------------------------------------------------------
/src/mapreduce/master_rpc.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "fmt"
5 | "log"
6 | "net"
7 | "net/rpc"
8 | "os"
9 | )
10 |
11 | // Shutdown is an RPC method that shuts down the Master's RPC server.
12 | func (mr *Master) Shutdown(_, _ *struct{}) error {
13 | debug("Shutdown: registration server\n")
14 | close(mr.shutdown)
15 | mr.l.Close() // causes the Accept to fail
16 | return nil
17 | }
18 |
19 | // startRPCServer starts the Master's RPC server. It continues accepting RPC
20 | // calls (Register in particular) for as long as the worker is alive.
21 | func (mr *Master) startRPCServer() {
22 | rpcs := rpc.NewServer()
23 | rpcs.Register(mr)
24 | os.Remove(mr.address) // only needed for "unix"
25 | l, e := net.Listen("unix", mr.address)
26 | if e != nil {
27 | log.Fatal("RegstrationServer", mr.address, " error: ", e)
28 | }
29 | mr.l = l
30 |
31 | // now that we are listening on the master address, can fork off
32 | // accepting connections to another thread.
33 | go func() {
34 | loop:
35 | for {
36 | select {
37 | case <-mr.shutdown:
38 | break loop
39 | default:
40 | }
41 | conn, err := mr.l.Accept()
42 | if err == nil {
43 | go func() {
44 | rpcs.ServeConn(conn)
45 | conn.Close()
46 | }()
47 | } else {
48 | debug("RegistrationServer: accept error", err)
49 | break
50 | }
51 | }
52 | debug("RegistrationServer: done\n")
53 | }()
54 | }
55 |
56 | // stopRPCServer stops the master RPC server.
57 | // This must be done through an RPC to avoid race conditions between the RPC
58 | // server thread and the current thread.
59 | func (mr *Master) stopRPCServer() {
60 | var reply ShutdownReply
61 | ok := call(mr.address, "Master.Shutdown", new(struct{}), &reply)
62 | if ok == false {
63 | fmt.Printf("Cleanup: RPC %s error\n", mr.address)
64 | }
65 | debug("cleanupRegistration: done\n")
66 | }
67 |
--------------------------------------------------------------------------------
/src/mapreduce/master_splitmerge.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "bufio"
5 | "encoding/json"
6 | "fmt"
7 | "log"
8 | "os"
9 | "sort"
10 | )
11 |
12 | // merge combines the results of the many reduce jobs into a single output file
13 | // XXX use merge sort
14 | func (mr *Master) merge() {
15 | debug("Merge phase")
16 | kvs := make(map[string]string)
17 | for i := 0; i < mr.nReduce; i++ {
18 | p := mergeName(mr.jobName, i)
19 | fmt.Printf("Merge: read %s\n", p)
20 | file, err := os.Open(p)
21 | if err != nil {
22 | log.Fatal("Merge: ", err)
23 | }
24 | dec := json.NewDecoder(file)
25 | for {
26 | var kv KeyValue
27 | err = dec.Decode(&kv)
28 | if err != nil {
29 | break
30 | }
31 | kvs[kv.Key] = kv.Value
32 | }
33 | file.Close()
34 | }
35 | var keys []string
36 | for k := range kvs {
37 | keys = append(keys, k)
38 | }
39 | sort.Strings(keys)
40 |
41 | file, err := os.Create("mrtmp." + mr.jobName)
42 | if err != nil {
43 | log.Fatal("Merge: create ", err)
44 | }
45 | w := bufio.NewWriter(file)
46 | for _, k := range keys {
47 | fmt.Fprintf(w, "%s: %s\n", k, kvs[k])
48 | }
49 | w.Flush()
50 | file.Close()
51 | }
52 |
53 | // removeFile is a simple wrapper around os.Remove that logs errors.
54 | func removeFile(n string) {
55 | err := os.Remove(n)
56 | if err != nil {
57 | log.Fatal("CleanupFiles ", err)
58 | }
59 | }
60 |
61 | // CleanupFiles removes all intermediate files produced by running mapreduce.
62 | func (mr *Master) CleanupFiles() {
63 | for i := range mr.files {
64 | for j := 0; j < mr.nReduce; j++ {
65 | removeFile(reduceName(mr.jobName, i, j))
66 | }
67 | }
68 | for i := 0; i < mr.nReduce; i++ {
69 | removeFile(mergeName(mr.jobName, i))
70 | }
71 | removeFile("mrtmp." + mr.jobName)
72 | }
73 |
--------------------------------------------------------------------------------
/src/mapreduce/schedule.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | import (
4 | "fmt"
5 | "sync"
6 | )
7 |
8 | //
9 | // schedule() starts and waits for all tasks in the given phase (mapPhase
10 | // or reducePhase). the mapFiles argument holds the names of the files that
11 | // are the inputs to the map phase, one per map task. nReduce is the
12 | // number of reduce tasks. the registerChan argument yields a stream
13 | // of registered workers; each item is the worker's RPC address,
14 | // suitable for passing to call(). registerChan will yield all
15 | // existing registered workers (if any) and new ones as they register.
16 | //
17 | func schedule(jobName string, mapFiles []string, nReduce int, phase jobPhase, registerChan chan string) {
18 | var ntasks int
19 | var nOther int // number of inputs (for reduce) or outputs (for map)
20 | switch phase {
21 | case mapPhase:
22 | ntasks = len(mapFiles)
23 | nOther = nReduce
24 | case reducePhase:
25 | ntasks = nReduce
26 | nOther = len(mapFiles)
27 | }
28 |
29 | fmt.Printf("Schedule: %d %v tasks (%d I/Os)\n", ntasks, phase, nOther)
30 |
31 | // All ntasks tasks have to be scheduled on workers. Once all tasks
32 | // have completed successfully, schedule() should return.
33 |
34 | // 要等待所有的任务完成后，才能结束这个函数，所以，添加 wg
35 | var wg sync.WaitGroup
36 | wg.Add(ntasks)
37 |
38 | // 所有的任务都通过 taskChan 发送
39 | taskChan := make(chan int, 3)
40 | go func() {
41 | for i := 0; i < ntasks; i++ {
42 | taskChan <- i
43 | }
44 | }()
45 |
46 | go func() {
47 |
48 | for {
49 | // 从 registerChan 获取服务器的地址
50 | srv := <-registerChan
51 | i := <-taskChan
52 |
53 | // 为将要执行的任务准备相关参数
54 | args := DoTaskArgs{
55 | JobName: jobName,
56 | Phase: phase,
57 | TaskNumber: i,
58 | NumOtherPhase: nOther,
59 | }
60 | if phase == mapPhase {
61 | // 只有 mapPhase 才需要设置 .File 属性
62 | args.File = mapFiles[i]
63 | }
64 |
65 | go func() {
66 | // 把任务发送给 srv
67 | if call(srv, "Worker.DoTask", args, nil) {
68 | // 任务成功的话，标记任务结束
69 | wg.Done()
70 | // 任务结束后，srv 闲置，srv 进入 registerChan 等待下一次分配任务
71 | registerChan <- srv
72 | } else {
73 | // 任务失败了，把未完成的任务号，重新放入 taskChan 中，等待下一次分配
74 | taskChan <- args.TaskNumber
75 | }
76 | }()
77 | }
78 | }()
79 |
80 | wg.Wait()
81 |
82 | fmt.Printf("Schedule: %v done\n", phase)
83 | }
84 |
--------------------------------------------------------------------------------
/src/mapreduce/worker.go:
--------------------------------------------------------------------------------
1 | package mapreduce
2 |
3 | //
4 | // Please do not modify this file.
5 | //
6 |
7 | import (
8 | "fmt"
9 | "log"
10 | "net"
11 | "net/rpc"
12 | "os"
13 | "sync"
14 | "time"
15 | )
16 |
17 | // Parallelism track whether workers executed in parallel.
18 | type Parallelism struct {
19 | mu sync.Mutex
20 | now int32
21 | max int32
22 | }
23 |
24 | // Worker holds the state for a server waiting for DoTask or Shutdown RPCs
25 | type Worker struct {
26 | sync.Mutex
27 |
28 | name string
29 | Map func(string, string) []KeyValue
30 | Reduce func(string, []string) string
31 | nRPC int // quit after this many RPCs; protected by mutex
32 | nTasks int // total tasks executed; protected by mutex
33 | concurrent int // number of parallel DoTasks in this worker; mutex
34 | l net.Listener
35 | parallelism *Parallelism
36 | }
37 |
38 | // DoTask is called by the master when a new task is being scheduled on this
39 | // worker.
40 | func (wk *Worker) DoTask(arg *DoTaskArgs, _ *struct{}) error {
41 | fmt.Printf("%s: given %v task #%d on file %s (nios: %d)\n",
42 | wk.name, arg.Phase, arg.TaskNumber, arg.File, arg.NumOtherPhase)
43 |
44 | wk.Lock()
45 | wk.nTasks += 1
46 | wk.concurrent += 1
47 | nc := wk.concurrent
48 | wk.Unlock()
49 |
50 | if nc > 1 {
51 | // schedule() should never issue more than one RPC at a
52 | // time to a given worker.
53 | log.Fatal("Worker.DoTask: more than one DoTask sent concurrently to a single worker\n")
54 | }
55 |
56 | pause := false
57 | if wk.parallelism != nil {
58 | wk.parallelism.mu.Lock()
59 | wk.parallelism.now += 1
60 | if wk.parallelism.now > wk.parallelism.max {
61 | wk.parallelism.max = wk.parallelism.now
62 | }
63 | if wk.parallelism.max < 2 {
64 | pause = true
65 | }
66 | wk.parallelism.mu.Unlock()
67 | }
68 |
69 | if pause {
70 | // give other workers a chance to prove that
71 | // they are executing in parallel.
72 | time.Sleep(time.Second)
73 | }
74 |
75 | switch arg.Phase {
76 | case mapPhase:
77 | doMap(arg.JobName, arg.TaskNumber, arg.File, arg.NumOtherPhase, wk.Map)
78 | case reducePhase:
79 | doReduce(arg.JobName, arg.TaskNumber, mergeName(arg.JobName, arg.TaskNumber), arg.NumOtherPhase, wk.Reduce)
80 | }
81 |
82 | wk.Lock()
83 | wk.concurrent -= 1
84 | wk.Unlock()
85 |
86 | if wk.parallelism != nil {
87 | wk.parallelism.mu.Lock()
88 | wk.parallelism.now -= 1
89 | wk.parallelism.mu.Unlock()
90 | }
91 |
92 | fmt.Printf("%s: %v task #%d done\n", wk.name, arg.Phase, arg.TaskNumber)
93 | return nil
94 | }
95 |
96 | // Shutdown is called by the master when all work has been completed.
97 | // We should respond with the number of tasks we have processed.
98 | func (wk *Worker) Shutdown(_ *struct{}, res *ShutdownReply) error {
99 | debug("Shutdown %s\n", wk.name)
100 | wk.Lock()
101 | defer wk.Unlock()
102 | res.Ntasks = wk.nTasks
103 | wk.nRPC = 1
104 | return nil
105 | }
106 |
107 | // Tell the master we exist and ready to work
108 | func (wk *Worker) register(master string) {
109 | args := new(RegisterArgs)
110 | args.Worker = wk.name
111 | ok := call(master, "Master.Register", args, new(struct{}))
112 | if ok == false {
113 | fmt.Printf("Register: RPC %s register error\n", master)
114 | }
115 | }
116 |
117 | // RunWorker sets up a connection with the master, registers its address, and
118 | // waits for tasks to be scheduled.
119 | func RunWorker(MasterAddress string, me string,
120 | MapFunc func(string, string) []KeyValue,
121 | ReduceFunc func(string, []string) string,
122 | nRPC int, parallelism *Parallelism,
123 | ) {
124 | debug("RunWorker %s\n", me)
125 | wk := new(Worker)
126 | wk.name = me
127 | wk.Map = MapFunc
128 | wk.Reduce = ReduceFunc
129 | wk.nRPC = nRPC
130 | wk.parallelism = parallelism
131 | rpcs := rpc.NewServer()
132 | rpcs.Register(wk)
133 | os.Remove(me) // only needed for "unix"
134 | l, e := net.Listen("unix", me)
135 | if e != nil {
136 | log.Fatal("RunWorker: worker ", me, " error: ", e)
137 | }
138 | wk.l = l
139 | wk.register(MasterAddress)
140 |
141 | // DON'T MODIFY CODE BELOW
142 | for {
143 | wk.Lock()
144 | if wk.nRPC == 0 {
145 | wk.Unlock()
146 | break
147 | }
148 | wk.Unlock()
149 | conn, err := wk.l.Accept()
150 | if err == nil {
151 | wk.Lock()
152 | wk.nRPC--
153 | wk.Unlock()
154 | go rpcs.ServeConn(conn)
155 | } else {
156 | break
157 | }
158 | }
159 | wk.l.Close()
160 | debug("RunWorker %s exit\n", me)
161 | }
162 |
--------------------------------------------------------------------------------
/src/raft/applyMsg.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import (
4 | "fmt"
5 | "sort"
6 | )
7 |
8 | // ApplyMsg 是发送消息
9 | // as each Raft peer becomes aware that successive log entries are
10 | // committed, the peer should send an ApplyMsg to the service (or
11 | // tester) on the same server, via the applyCh passed to Make(). set
12 | // CommandValid to true to indicate that the ApplyMsg contains a newly
13 | // committed log entry.
14 | //
15 | // in Lab 3 you'll want to send other kinds of messages (e.g.,
16 | // snapshots) on the applyCh; at that point you can add fields to
17 | // ApplyMsg, but set CommandValid to false for these other uses.
18 | //
19 | type ApplyMsg struct {
20 | CommandValid bool // TODO: 弄清楚这个干嘛的
21 | CommandIndex int // Command zd Raft.logs 中的索引号
22 | Command interface{} // Command 的具体内容
23 | }
24 |
25 | func (a ApplyMsg) String() string {
26 | return fmt.Sprintf("ApplyMsg{idx:%d,cmd:%v}", a.CommandIndex, a.Command)
27 | }
28 |
29 | // 每当 rf.logs 或 rf.commitIndex 有变化时，就收到通知
30 | // 然后，检查发现有可以 commit 的 entry 的话
31 | // 就通过 applyCh 发送 ApplyMsg 给 replication state machine 进行 commit
32 | func (rf *Raft) checkApplyLoop(applyCh chan ApplyMsg) {
33 | rf.shutdownWG.Add(1)
34 | isChanged := false
35 |
36 | for {
37 | select {
38 | case <-rf.toCheckApplyChan:
39 | debugPrintf(" S#%d 在 checkApplyLoop 的 case <- rf.toCheckApplyChan，收到信号。将要检查是否有新的 entry 可以 commit", rf.me)
40 | case <-rf.shutdownChan:
41 | debugPrintf(" S#%d 在 checkApplyLoop 的 case <- rf.shutdownChan，收到信号。关闭 checkApplyLoop", rf.me)
42 | rf.shutdownWG.Done()
43 | return
44 | }
45 |
46 | rf.rwmu.Lock()
47 |
48 | // 如果 rf 是 LEADER
49 | // 先检查能否更新 rf.commitIndex
50 | if rf.state == LEADER {
51 | idx := maxMajorityIndex(rf.matchIndex)
52 | // paper 5.4.2, only log entries from the leader's current term are committed by counting replicas
53 | if idx > rf.commitIndex &&
54 | rf.logs[idx].LogTerm == rf.currentTerm {
55 | rf.commitIndex = idx
56 | debugPrintf("%s 发现了新的 maxMajorityIndex==%d, 已经更新 rf.commitIndex", rf, idx)
57 | isChanged = true
58 | }
59 | }
60 |
61 | if rf.lastApplied < rf.commitIndex {
62 | isChanged = true
63 | debugPrintf("%s 有新的 log 可以 commit ，因为 lastApplied(%d) < commitIndex(%d)", rf, rf.lastApplied, rf.commitIndex)
64 | }
65 |
66 | for rf.lastApplied < rf.commitIndex {
67 | rf.lastApplied++
68 | applyMsg := ApplyMsg{
69 | CommandValid: true, // TODO: ApplyMsg.CommandValid 属性是做什么用的
70 | Command: rf.logs[rf.lastApplied].Command,
71 | CommandIndex: rf.lastApplied}
72 | debugPrintf("%s apply %s", rf, applyMsg)
73 | applyCh <- applyMsg
74 | }
75 |
76 | if isChanged {
77 | rf.persist()
78 | isChanged = false
79 | }
80 |
81 | rf.rwmu.Unlock()
82 | }
83 | }
84 |
85 | // 返回 matchIndex 中超过半数的 Index
86 | // 例如
87 | // matchIndex == {8,7,6,5,4}
88 | // temp == {4,5,6,7,8}
89 | // i = (5-1)/2 = 2
90 | // 超过半数的 server 拥有 {4,5,6}
91 | // 其中 temp[i] == 6 是最大值
92 | func maxMajorityIndex(matchIndex []int) int {
93 | temp := make([]int, len(matchIndex))
94 | copy(temp, matchIndex)
95 | sort.Ints(temp)
96 | i := (len(matchIndex) - 1) / 2
97 | return temp[i]
98 | }
99 |
--------------------------------------------------------------------------------
/src/raft/applyMsg_test.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import "testing"
4 |
5 | func Test_maxMajorityIndex(t *testing.T) {
6 | type args struct {
7 | matchIndex []int
8 | }
9 | tests := []struct {
10 | name string
11 | args args
12 | want int
13 | }{
14 |
15 | {
16 | "找到 max majority index 0",
17 | args{matchIndex: []int{1, 1, 0, 0, 0}},
18 | 0,
19 | },
20 |
21 | {
22 | "找到 max majority index 6",
23 | args{matchIndex: []int{8, 7, 6, 5}},
24 | 6,
25 | },
26 |
27 | {
28 | "找到 max majority index 6",
29 | args{matchIndex: []int{8, 7, 6, 5, 4}},
30 | 6,
31 | },
32 |
33 | //
34 | }
35 | for _, tt := range tests {
36 | t.Run(tt.name, func(t *testing.T) {
37 | if got := maxMajorityIndex(tt.args.matchIndex); got != tt.want {
38 | t.Errorf("maxMajorityIndex() = %v, want %v", got, tt.want)
39 | }
40 | })
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/raft/fsm.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import "fmt"
4 |
5 | type fsmState int
6 | type fsmEvent string
7 | type fsmHandler func(*Raft, interface{}) fsmState
8 |
9 | // 规定了 server 所需的 3 种状态
10 | const (
11 | LEADER fsmState = iota
12 | CANDIDATE
13 | FOLLOWER
14 | )
15 |
16 | func (s fsmState) String() string {
17 | switch s {
18 | case LEADER:
19 | return "Leader"
20 | case CANDIDATE:
21 | return "Candidate"
22 | case FOLLOWER:
23 | return "Follower"
24 | default:
25 | panic("出现了第4种 server state")
26 | }
27 | }
28 |
29 | func (rf *Raft) addHandler(state fsmState, event fsmEvent, handler fsmHandler) {
30 | if _, ok := rf.handlers[state]; !ok {
31 | rf.handlers[state] = make(map[fsmEvent]fsmHandler, 10)
32 | }
33 | if _, ok := rf.handlers[state][event]; ok {
34 | debugPrintf("[警告] FSM 的状态 (%s) 的事件 (%s) 的处理方法，被覆盖", state, event)
35 | }
36 | rf.handlers[state][event] = handler
37 | }
38 |
39 | func (rf *Raft) call(event fsmEvent, args interface{}) {
40 | rf.rwmu.Lock()
41 |
42 | oldState := rf.state
43 |
44 | if rf.handlers[oldState] == nil ||
45 | rf.handlers[oldState][event] == nil {
46 | msg := fmt.Sprintf("%s 的状态 (%s) 没有事件 (%s) 的转换 handler", rf, oldState, event)
47 | panic(msg)
48 | }
49 |
50 | rf.state = rf.handlers[oldState][event](rf, args)
51 |
52 | debugPrintf("%s 事件(%s)，已从 (%s) → (%s)", rf, event, oldState, rf.state)
53 |
54 | rf.rwmu.Unlock()
55 |
56 | }
57 |
58 | // 全部的事件
59 | var (
60 | // election timer 超时
61 | electionTimeOutEvent = fsmEvent("election timeout")
62 | // CANDIDATE 获取 majority 选票的时候，会触发此事件
63 | winElectionEvent = fsmEvent("win this term election")
64 | // server 在 RPC 通讯中，会首先检查对方的 term
65 | // 如果发现对方的 term 数比自己的高，会触发此事件
66 | discoverNewTermEvent = fsmEvent("discover new term")
67 | // server 在接收到 appendEntriesArgs 后，发现对方的 term
68 | discoverNewLeaderEvent = fsmEvent("disover a leader with higher term")
69 | )
70 |
71 | // 添加 rf 转换状态时的处理函数
72 | func (rf *Raft) addHandlers() {
73 |
74 | // 添加 FOLLOWER 状态下的处理函数
75 | rf.addHandler(FOLLOWER, electionTimeOutEvent, fsmHandler(startNewElection))
76 | rf.addHandler(FOLLOWER, discoverNewTermEvent, fsmHandler(toFollower))
77 | rf.addHandler(FOLLOWER, discoverNewLeaderEvent, fsmHandler(toFollower))
78 |
79 | // 添加 CANDIDATE 状态下的处理函数
80 | rf.addHandler(CANDIDATE, electionTimeOutEvent, fsmHandler(startNewElection))
81 | rf.addHandler(CANDIDATE, winElectionEvent, fsmHandler(comeToPower))
82 | rf.addHandler(CANDIDATE, discoverNewTermEvent, fsmHandler(toFollower))
83 | rf.addHandler(CANDIDATE, discoverNewLeaderEvent, fsmHandler(toFollower))
84 |
85 | // 添加 LEADER 状态下的处理函数
86 | rf.addHandler(LEADER, discoverNewTermEvent, fsmHandler(toFollower))
87 | rf.addHandler(LEADER, discoverNewLeaderEvent, fsmHandler(toFollower))
88 |
89 | }
90 |
--------------------------------------------------------------------------------
/src/raft/logEntry.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | // LogEntry is log entry
4 | type LogEntry struct {
5 | LogIndex int // 此 log 在 LEADER.logs 中的索引号
6 | LogTerm int // LEADER 在生成此 log 时的 LEADER.currentTerm
7 | Command interface{} // 具体的命令内容
8 | }
9 |
10 | // TODO: 注释 truncateLog
11 | func truncateLog(lastIncludedIndex int, lastIncludedTerm int, log []LogEntry) []LogEntry {
12 | var newLogEntries []LogEntry
13 | newLogEntries = append(newLogEntries, LogEntry{LogIndex: lastIncludedIndex, LogTerm: lastIncludedTerm})
14 |
15 | for index := len(log) - 1; index >= 0; index-- {
16 | if log[index].LogIndex == lastIncludedIndex && log[index].LogTerm == lastIncludedTerm {
17 | newLogEntries = append(newLogEntries, log[index+1:]...)
18 | break
19 | }
20 | }
21 |
22 | return newLogEntries
23 | }
24 |
--------------------------------------------------------------------------------
/src/raft/persister.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | //
4 | // support for Raft and kvraft to save persistent
5 | // Raft state (log &c) and k/v server snapshots.
6 | //
7 | // we will use the original persister.go to test your code for grading.
8 | // so, while you can modify this code to help you debug, please
9 | // test with the original before submitting.
10 | //
11 |
12 | import "sync"
13 |
14 | // Persister is
15 | type Persister struct {
16 | mu sync.Mutex
17 | raftstate []byte
18 | snapshot []byte
19 | }
20 |
21 | // MakePersister is
22 | func MakePersister() *Persister {
23 | return &Persister{}
24 | }
25 |
26 | // Copy is
27 | func (ps *Persister) Copy() *Persister {
28 | ps.mu.Lock()
29 | defer ps.mu.Unlock()
30 | np := MakePersister()
31 | np.raftstate = ps.raftstate
32 | np.snapshot = ps.snapshot
33 | return np
34 | }
35 |
36 | // SaveRaftState is
37 | func (ps *Persister) SaveRaftState(state []byte) {
38 | ps.mu.Lock()
39 | defer ps.mu.Unlock()
40 | ps.raftstate = state
41 | }
42 |
43 | // ReadRaftState is
44 | func (ps *Persister) ReadRaftState() []byte {
45 | ps.mu.Lock()
46 | defer ps.mu.Unlock()
47 | return ps.raftstate
48 | }
49 |
50 | // RaftStateSize is
51 | func (ps *Persister) RaftStateSize() int {
52 | ps.mu.Lock()
53 | defer ps.mu.Unlock()
54 | return len(ps.raftstate)
55 | }
56 |
57 | // SaveStateAndSnapshot save both Raft state and K/V snapshot as a single atomic action,
58 | // to help avoid them getting out of sync.
59 | func (ps *Persister) SaveStateAndSnapshot(state []byte, snapshot []byte) {
60 | ps.mu.Lock()
61 | defer ps.mu.Unlock()
62 | ps.raftstate = state
63 | ps.snapshot = snapshot
64 | }
65 |
66 | // ReadSnapshot 读取快照
67 | func (ps *Persister) ReadSnapshot() []byte {
68 | ps.mu.Lock()
69 | defer ps.mu.Unlock()
70 | return ps.snapshot
71 | }
72 |
73 | // SnapshotSize 快照尺寸
74 | func (ps *Persister) SnapshotSize() int {
75 | ps.mu.Lock()
76 | defer ps.mu.Unlock()
77 | return len(ps.snapshot)
78 | }
79 |
--------------------------------------------------------------------------------
/src/raft/raft_main.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | //
4 | // this is an outline of the API that raft must expose to
5 | // the service (or tester). see comments below for
6 | // each of these functions for more details.
7 | //
8 | // rf = Make(...)
9 | // create a new Raft server.
10 | // rf.Start(command interface{}) (index, term, isleader)
11 | // start agreement on a new log entry
12 | // rf.GetState() (term, isLeader)
13 | // ask a Raft for its current term, and whether it thinks it is leader
14 | // ApplyMsg
15 | // each time a new entry is committed to the log, each Raft peer
16 | // should send an ApplyMsg to the service (or tester)
17 | // in the same server.
18 | //
19 |
20 | import (
21 | "labrpc"
22 | )
23 |
24 | const (
25 | // NOBODY 表示没有投票给任何人
26 | NOBODY = -1
27 | )
28 |
29 | // Make is
30 | // the service or tester wants to create a Raft server. the ports
31 | // of all the Raft servers (including this one) are in peers[]. this
32 | // server's port is peers[me]. all the servers' peers[] arrays
33 | // have the same order. persister is a place for this server to
34 | // save its persistent state, and also initially holds the most
35 | // recent saved state, if any. applyCh is a channel on which the
36 | // tester or service expects Raft to send ApplyMsg messages.
37 | // Make() must return quickly, so it should start goroutines
38 | // for any long-running work.
39 | //
40 |
41 | // Make is
42 | func Make(peers []*labrpc.ClientEnd, me int, persister *Persister, applyCh chan ApplyMsg) *Raft {
43 | rf := newRaft(peers, me, persister)
44 |
45 | go rf.checkApplyLoop(applyCh)
46 |
47 | // initialize from state persisted before a crash
48 | rf.readPersist(persister.ReadRaftState())
49 |
50 | return rf
51 | }
52 |
--------------------------------------------------------------------------------
/src/raft/raft_new.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import (
4 | "fmt"
5 | "labrpc"
6 | "sync"
7 | "time"
8 | )
9 |
10 | // Raft implements a single Raft peer.
11 | type Raft struct {
12 | rwmu sync.RWMutex // Lock to protect shared access to this peer's state
13 | peers []*labrpc.ClientEnd // RPC end points of all peers
14 |
15 | // Persistent state on all servers
16 | // (Updated on stable storage before responding to RPCs)
17 | // This implementation doesn't use disk; ti will save and restore
18 | // persistent state from a Persister object
19 | // Raft should initialize its state from Persister,
20 | // and should use it to save its persistent state each tiem the state changes
21 | // Use ReadRaftState() and SaveRaftState
22 | persister *Persister // Object to hold this peer's persisted state
23 | me int // this peer's index into peers[]
24 |
25 | // Your data here (2A, 2B, 2C).
26 | // Look at the paper's Figure 2 for a description of what
27 | // state a Raft server must maintain.
28 |
29 | // from Figure 2
30 |
31 | // Persistent state on call servers
32 | currentTerm int // 此 server 当前所处的 term 编号
33 | votedFor int // 此 server 在此 term 中投票给了谁，是 peers 中的索引号
34 | logs []LogEntry // 此 server 中保存的 logs
35 |
36 | // Volatile state on all servers:
37 | commitIndex int // logs 中已经 commited 的 log 的最大索引号
38 | lastApplied int // logs 中最新元素的索引号
39 |
40 | // Volatile state on leaders:
41 | nextIndex []int // 下一个要发送给 follower 的 log 的索引号
42 | matchIndex []int // leader 与 follower 共有的 log 的最大的索引号
43 |
44 | // Raft 作为 FSM 管理自身状态所需的属性
45 | state fsmState
46 | handlers map[fsmState]map[fsmEvent]fsmHandler
47 |
48 | // 超时，就由 FOLLOWER 变 CANDIDATE
49 | electionTimer *time.Timer
50 |
51 | // 用于通知 raft 已经关闭的信息
52 | shutdownChan chan struct{}
53 | shutdownWG sync.WaitGroup
54 |
55 | // 当 rf 接收到合格的 rpc 信号时，会通过 resetElectionTimerChan 发送信号
56 | resetElectionTimerChan chan struct{}
57 |
58 | // candidate 或 leader 中途转变为 follower 的话，就关闭这个 channel 来发送信号
59 | // 因为，同一个 rf 不可能既是 candidate 又是 leader
60 | // 所以，用来通知的 channel 只要有一个就好了
61 | convertToFollowerChan chan struct{}
62 |
63 | // logs 中添加了新的 entries 以后，会通过这个发送信号
64 | toCheckApplyChan chan struct{}
65 |
66 | // 关闭，则表示需要终结此次 election
67 | endElectionChan chan struct{}
68 |
69 | //
70 | }
71 |
72 | func (rf *Raft) String() string {
73 | postfix := ""
74 | if rf.state == LEADER {
75 | postfix = fmt.Sprintf(", nextIndex%v, matchIndex%v", rf.nextIndex, rf.matchIndex)
76 | }
77 | return fmt.Sprintf("@@ S#%d:T%d:L%d:%s:%2d, commitIndex:%d, lastApplied:%d, logs:%v%s @@",
78 | rf.me, rf.currentTerm, len(rf.logs), rf.state, rf.votedFor,
79 | rf.commitIndex, rf.lastApplied, rf.logs, postfix)
80 | }
81 |
82 | func newRaft(peers []*labrpc.ClientEnd, me int, persister *Persister) *Raft {
83 | rf := &Raft{
84 | peers: peers,
85 | persister: persister,
86 | me: me,
87 | currentTerm: 0,
88 | votedFor: NOBODY,
89 |
90 | // logs 的序列号从 1 开始
91 | logs: make([]LogEntry, 1),
92 | commitIndex: 0,
93 | lastApplied: 0,
94 |
95 | // 初始状态都是 FOLLOWER
96 | state: FOLLOWER,
97 |
98 | handlers: make(map[fsmState]map[fsmEvent]fsmHandler, 3),
99 |
100 | // 并不会等 1 秒，很快就会被重置
101 | electionTimer: time.NewTimer(time.Second),
102 |
103 | // 靠关闭来传递信号，所以，不设置缓冲
104 | shutdownChan: make(chan struct{}),
105 | // endElectionChan 需要用到的时候，再赋值
106 |
107 | // 靠数据来传递信号，所以, 设置缓冲
108 | resetElectionTimerChan: make(chan struct{}, 3),
109 | toCheckApplyChan: make(chan struct{}, 3),
110 | }
111 |
112 | rf.addHandlers()
113 |
114 | go electionLoop(rf)
115 |
116 | return rf
117 | }
118 |
119 | // 触发 election timer 超时，就开始新的选举
120 | func electionLoop(rf *Raft) {
121 | rf.shutdownWG.Add(1)
122 |
123 | for {
124 | rf.resetElectionTimer()
125 |
126 | select {
127 | case <-rf.electionTimer.C:
128 | debugPrintf("%s 在 electionLoop 中，从 case <-rf.electionTimer.C 收到信号, 将要开始 term(%d) 的 election", rf, rf.currentTerm+1)
129 | rf.call(electionTimeOutEvent, nil)
130 | case <-rf.resetElectionTimerChan:
131 | debugPrintf("%s 在 electionLoop 中，从 case <-rf.resetElectionTimerChan 收到信号", rf)
132 | case <-rf.shutdownChan:
133 | debugPrintf(" S#%d 在 electionLoop 的 case <- rf.shutdownChan，收到信号。关闭 electionLoop", rf.me)
134 | rf.shutdownWG.Done()
135 | return
136 | }
137 | }
138 | }
139 |
--------------------------------------------------------------------------------
/src/raft/raft_new_test.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import "testing"
4 |
5 | func Test_state_String(t *testing.T) {
6 | tests := []struct {
7 | name string
8 | s fsmState
9 | want string
10 | }{
11 |
12 | {
13 | "print leader",
14 | LEADER,
15 | "Leader",
16 | },
17 |
18 | {
19 | "print candidate",
20 | CANDIDATE,
21 | "Candidate",
22 | },
23 |
24 | {
25 | "print follower",
26 | FOLLOWER,
27 | "Follower",
28 | },
29 |
30 | //
31 | }
32 | for _, tt := range tests {
33 | t.Run(tt.name, func(t *testing.T) {
34 | if got := tt.s.String(); got != tt.want {
35 | t.Errorf("state.String() = %v, want %v", got, tt.want)
36 | }
37 | })
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/raft/rpc_snapshot.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import (
4 | "bytes"
5 | "encoding/gob"
6 | )
7 |
8 | // InstallSnapshotArgs is
9 | type InstallSnapshotArgs struct {
10 | Term int
11 | LeaderID int
12 | LastIncludedIndex int
13 | LastIncludedTerm int
14 | Data []byte
15 | }
16 |
17 | // InstallSnapshotReply is
18 | type InstallSnapshotReply struct {
19 | Term int
20 | }
21 |
22 | // InstallSnapshot is
23 | func (rf *Raft) InstallSnapshot(args InstallSnapshotArgs, reply *InstallSnapshotReply) {
24 | // Your code here.
25 | rf.rwmu.Lock()
26 | defer rf.rwmu.Unlock()
27 | if args.Term < rf.currentTerm {
28 | reply.Term = rf.currentTerm
29 | return
30 | }
31 | // rf.chanHeartbeat <- struct{}{}
32 | rf.state = FOLLOWER
33 |
34 | // FIXME:
35 | // rf.currentTerm = rf.currentTerm
36 |
37 | // FIXME: don't have this method
38 | // rf.persister.SaveSnapshot(args.Data)
39 |
40 | rf.logs = truncateLog(args.LastIncludedIndex, args.LastIncludedTerm, rf.logs)
41 |
42 | // msg := ApplyMsg{UseSnapshot: true, Snapshot: args.Data}
43 |
44 | rf.lastApplied = args.LastIncludedIndex
45 | rf.commitIndex = args.LastIncludedIndex
46 |
47 | rf.persist()
48 |
49 | // rf.chanApply <- msg
50 | }
51 | func (rf *Raft) sendInstallSnapshot(server int, args InstallSnapshotArgs, reply *InstallSnapshotReply) bool {
52 | ok := rf.peers[server].Call("Raft.InstallSnapshot", args, reply)
53 | if ok {
54 | if reply.Term > rf.currentTerm {
55 | rf.currentTerm = reply.Term
56 | rf.state = FOLLOWER
57 | rf.votedFor = -1
58 | return ok
59 | }
60 |
61 | rf.nextIndex[server] = args.LastIncludedIndex + 1
62 | rf.matchIndex[server] = args.LastIncludedIndex
63 | }
64 | return ok
65 | }
66 |
67 | func (rf *Raft) readSnapshot(data []byte) {
68 |
69 | rf.readPersist(rf.persister.ReadRaftState())
70 |
71 | if len(data) == 0 {
72 | return
73 | }
74 |
75 | r := bytes.NewBuffer(data)
76 | d := gob.NewDecoder(r)
77 |
78 | var LastIncludedIndex int
79 | var LastIncludedTerm int
80 |
81 | d.Decode(&LastIncludedIndex)
82 | d.Decode(&LastIncludedTerm)
83 |
84 | rf.commitIndex = LastIncludedIndex
85 | rf.lastApplied = LastIncludedIndex
86 |
87 | rf.logs = truncateLog(LastIncludedIndex, LastIncludedTerm, rf.logs)
88 |
89 | // msg := ApplyMsg{UseSnapshot: true, Snapshot: data}
90 |
91 | go func() {
92 | // rf.chanApply <- msg
93 | }()
94 | }
95 |
--------------------------------------------------------------------------------
/src/raft/util.go:
--------------------------------------------------------------------------------
1 | package raft
2 |
3 | import "log"
4 |
5 | func init() {
6 | log.SetFlags(log.LstdFlags | log.Lmicroseconds)
7 | debugPrintf("程序开始运行")
8 | }
9 |
10 | // needDebug for Debugging
11 | const needDebug = false
12 |
13 | // debugPrintf 根据设置打印输出
14 | func debugPrintf(format string, a ...interface{}) {
15 | if needDebug {
16 | log.Printf(format, a...)
17 | }
18 | }
19 |
20 | func max(a, b int) int {
21 | if a > b {
22 | return a
23 | }
24 | return b
25 | }
26 |
27 | func min(a, b int) int {
28 | if a < b {
29 | return a
30 | }
31 | return b
32 | }
33 |
--------------------------------------------------------------------------------
/src/shardkv/client.go:
--------------------------------------------------------------------------------
1 | package shardkv
2 |
3 | //
4 | // client code to talk to a sharded key/value service.
5 | //
6 | // the client first talks to the shardmaster to find out
7 | // the assignment of shards (keys) to groups, and then
8 | // talks to the group that holds the key's shard.
9 | //
10 |
11 | import "labrpc"
12 | import "crypto/rand"
13 | import "math/big"
14 | import "shardmaster"
15 | import "time"
16 |
17 | //
18 | // which shard is a key in?
19 | // please use this function,
20 | // and please do not change it.
21 | //
22 | func key2shard(key string) int {
23 | shard := 0
24 | if len(key) > 0 {
25 | shard = int(key[0])
26 | }
27 | shard %= shardmaster.NShards
28 | return shard
29 | }
30 |
31 | func nrand() int64 {
32 | max := big.NewInt(int64(1) << 62)
33 | bigx, _ := rand.Int(rand.Reader, max)
34 | x := bigx.Int64()
35 | return x
36 | }
37 |
38 | type Clerk struct {
39 | sm *shardmaster.Clerk
40 | config shardmaster.Config
41 | make_end func(string) *labrpc.ClientEnd
42 | // You will have to modify this struct.
43 | }
44 |
45 | //
46 | // the tester calls MakeClerk.
47 | //
48 | // masters[] is needed to call shardmaster.MakeClerk().
49 | //
50 | // make_end(servername) turns a server name from a
51 | // Config.Groups[gid][i] into a labrpc.ClientEnd on which you can
52 | // send RPCs.
53 | //
54 | func MakeClerk(masters []*labrpc.ClientEnd, make_end func(string) *labrpc.ClientEnd) *Clerk {
55 | ck := new(Clerk)
56 | ck.sm = shardmaster.MakeClerk(masters)
57 | ck.make_end = make_end
58 | // You'll have to add code here.
59 | return ck
60 | }
61 |
62 | //
63 | // fetch the current value for a key.
64 | // returns "" if the key does not exist.
65 | // keeps trying forever in the face of all other errors.
66 | // You will have to modify this function.
67 | //
68 | func (ck *Clerk) Get(key string) string {
69 | args := GetArgs{}
70 | args.Key = key
71 |
72 | for {
73 | shard := key2shard(key)
74 | gid := ck.config.Shards[shard]
75 | if servers, ok := ck.config.Groups[gid]; ok {
76 | // try each server for the shard.
77 | for si := 0; si < len(servers); si++ {
78 | srv := ck.make_end(servers[si])
79 | var reply GetReply
80 | ok := srv.Call("ShardKV.Get", &args, &reply)
81 | if ok && reply.WrongLeader == false && (reply.Err == OK || reply.Err == ErrNoKey) {
82 | return reply.Value
83 | }
84 | if ok && (reply.Err == ErrWrongGroup) {
85 | break
86 | }
87 | }
88 | }
89 | time.Sleep(100 * time.Millisecond)
90 | // ask master for the latest configuration.
91 | ck.config = ck.sm.Query(-1)
92 | }
93 |
94 | return ""
95 | }
96 |
97 | //
98 | // shared by Put and Append.
99 | // You will have to modify this function.
100 | //
101 | func (ck *Clerk) PutAppend(key string, value string, op string) {
102 | args := PutAppendArgs{}
103 | args.Key = key
104 | args.Value = value
105 | args.Op = op
106 |
107 |
108 | for {
109 | shard := key2shard(key)
110 | gid := ck.config.Shards[shard]
111 | if servers, ok := ck.config.Groups[gid]; ok {
112 | for si := 0; si < len(servers); si++ {
113 | srv := ck.make_end(servers[si])
114 | var reply PutAppendReply
115 | ok := srv.Call("ShardKV.PutAppend", &args, &reply)
116 | if ok && reply.WrongLeader == false && reply.Err == OK {
117 | return
118 | }
119 | if ok && reply.Err == ErrWrongGroup {
120 | break
121 | }
122 | }
123 | }
124 | time.Sleep(100 * time.Millisecond)
125 | // ask master for the latest configuration.
126 | ck.config = ck.sm.Query(-1)
127 | }
128 | }
129 |
130 | func (ck *Clerk) Put(key string, value string) {
131 | ck.PutAppend(key, value, "Put")
132 | }
133 | func (ck *Clerk) Append(key string, value string) {
134 | ck.PutAppend(key, value, "Append")
135 | }
136 |
--------------------------------------------------------------------------------
/src/shardkv/common.go:
--------------------------------------------------------------------------------
1 | package shardkv
2 |
3 | //
4 | // Sharded key/value server.
5 | // Lots of replica groups, each running op-at-a-time paxos.
6 | // Shardmaster decides which group serves each shard.
7 | // Shardmaster may change shard assignment from time to time.
8 | //
9 | // You will have to modify these definitions.
10 | //
11 |
12 | const (
13 | OK = "OK"
14 | ErrNoKey = "ErrNoKey"
15 | ErrWrongGroup = "ErrWrongGroup"
16 | )
17 |
18 | type Err string
19 |
20 | // Put or Append
21 | type PutAppendArgs struct {
22 | // You'll have to add definitions here.
23 | Key string
24 | Value string
25 | Op string // "Put" or "Append"
26 | // You'll have to add definitions here.
27 | // Field names must start with capital letters,
28 | // otherwise RPC will break.
29 | }
30 |
31 | type PutAppendReply struct {
32 | WrongLeader bool
33 | Err Err
34 | }
35 |
36 | type GetArgs struct {
37 | Key string
38 | // You'll have to add definitions here.
39 | }
40 |
41 | type GetReply struct {
42 | WrongLeader bool
43 | Err Err
44 | Value string
45 | }
46 |
--------------------------------------------------------------------------------
/src/shardkv/server.go:
--------------------------------------------------------------------------------
1 | package shardkv
2 |
3 | // import "shardmaster"
4 | import "labrpc"
5 | import "raft"
6 | import "sync"
7 | import "labgob"
8 |
9 | type Op struct {
10 | // Your definitions here.
11 | // Field names must start with capital letters,
12 | // otherwise RPC will break.
13 | }
14 |
15 | type ShardKV struct {
16 | mu sync.Mutex
17 | me int
18 | rf *raft.Raft
19 | applyCh chan raft.ApplyMsg
20 | make_end func(string) *labrpc.ClientEnd
21 | gid int
22 | masters []*labrpc.ClientEnd
23 | maxraftstate int // snapshot if log grows this big
24 |
25 | // Your definitions here.
26 | }
27 |
28 | func (kv *ShardKV) Get(args *GetArgs, reply *GetReply) {
29 | // TODO: Your code here.
30 | }
31 |
32 | func (kv *ShardKV) PutAppend(args *PutAppendArgs, reply *PutAppendReply) {
33 | // TODO: Your code here.
34 | }
35 |
36 | //
37 | // the tester calls Kill() when a ShardKV instance won't
38 | // be needed again. you are not required to do anything
39 | // in Kill(), but it might be convenient to (for example)
40 | // turn off debug output from this instance.
41 | //
42 | func (kv *ShardKV) Kill() {
43 | kv.rf.Kill()
44 | // TODO: Your code here, if desired.
45 | }
46 |
47 | //
48 | // servers[] contains the ports of the servers in this group.
49 | //
50 | // me is the index of the current server in servers[].
51 | //
52 | // the k/v server should store snapshots with
53 | // persister.SaveSnapshot(), and Raft should save its state (including
54 | // log) with persister.SaveRaftState().
55 | //
56 | // the k/v server should snapshot when Raft's saved state exceeds
57 | // maxraftstate bytes, in order to allow Raft to garbage-collect its
58 | // log. if maxraftstate is -1, you don't need to snapshot.
59 | //
60 | // gid is this group's GID, for interacting with the shardmaster.
61 | //
62 | // pass masters[] to shardmaster.MakeClerk() so you can send
63 | // RPCs to the shardmaster.
64 | //
65 | // make_end(servername) turns a server name from a
66 | // Config.Groups[gid][i] into a labrpc.ClientEnd on which you can
67 | // send RPCs. You'll need this to send RPCs to other groups.
68 | //
69 | // look at client.go for examples of how to use masters[]
70 | // and make_end() to send RPCs to the group owning a specific shard.
71 | //
72 | // StartServer() must return quickly, so it should start goroutines
73 | // for any long-running work.
74 | //
75 | func StartServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister, maxraftstate int, gid int, masters []*labrpc.ClientEnd, make_end func(string) *labrpc.ClientEnd) *ShardKV {
76 | // call labgob.Register on structures you want
77 | // Go's RPC library to marshall/unmarshall.
78 | labgob.Register(Op{})
79 |
80 | kv := new(ShardKV)
81 | kv.me = me
82 | kv.maxraftstate = maxraftstate
83 | kv.make_end = make_end
84 | kv.gid = gid
85 | kv.masters = masters
86 |
87 | // Your initialization code here.
88 |
89 | // Use something like this to talk to the shardmaster:
90 | // kv.mck = shardmaster.MakeClerk(kv.masters)
91 |
92 | kv.applyCh = make(chan raft.ApplyMsg)
93 | kv.rf = raft.Make(servers, me, persister, kv.applyCh)
94 |
95 | return kv
96 | }
97 |
--------------------------------------------------------------------------------
/src/shardmaster/client.go:
--------------------------------------------------------------------------------
1 | package shardmaster
2 |
3 | //
4 | // Shardmaster clerk.
5 | //
6 |
7 | import "labrpc"
8 | import "time"
9 | import "crypto/rand"
10 | import "math/big"
11 |
12 | type Clerk struct {
13 | servers []*labrpc.ClientEnd
14 | // Your data here.
15 | }
16 |
17 | func nrand() int64 {
18 | max := big.NewInt(int64(1) << 62)
19 | bigx, _ := rand.Int(rand.Reader, max)
20 | x := bigx.Int64()
21 | return x
22 | }
23 |
24 | func MakeClerk(servers []*labrpc.ClientEnd) *Clerk {
25 | ck := new(Clerk)
26 | ck.servers = servers
27 | // TODO: Your code here.
28 | return ck
29 | }
30 |
31 | func (ck *Clerk) Query(num int) Config {
32 | args := &QueryArgs{}
33 | // TODO: Your code here.
34 | args.Num = num
35 | for {
36 | // try each known server.
37 | for _, srv := range ck.servers {
38 | var reply QueryReply
39 | ok := srv.Call("ShardMaster.Query", args, &reply)
40 | if ok && reply.WrongLeader == false {
41 | return reply.Config
42 | }
43 | }
44 | time.Sleep(100 * time.Millisecond)
45 | }
46 | }
47 |
48 | func (ck *Clerk) Join(servers map[int][]string) {
49 | args := &JoinArgs{}
50 | // TODO: Your code here.
51 | args.Servers = servers
52 |
53 | for {
54 | // try each known server.
55 | for _, srv := range ck.servers {
56 | var reply JoinReply
57 | ok := srv.Call("ShardMaster.Join", args, &reply)
58 | if ok && reply.WrongLeader == false {
59 | return
60 | }
61 | }
62 | time.Sleep(100 * time.Millisecond)
63 | }
64 | }
65 |
66 | func (ck *Clerk) Leave(gids []int) {
67 | args := &LeaveArgs{}
68 | // TODO: Your code here.
69 | args.GIDs = gids
70 |
71 | for {
72 | // try each known server.
73 | for _, srv := range ck.servers {
74 | var reply LeaveReply
75 | ok := srv.Call("ShardMaster.Leave", args, &reply)
76 | if ok && reply.WrongLeader == false {
77 | return
78 | }
79 | }
80 | time.Sleep(100 * time.Millisecond)
81 | }
82 | }
83 |
84 | func (ck *Clerk) Move(shard int, gid int) {
85 | args := &MoveArgs{}
86 | // TODO: Your code here.
87 | args.Shard = shard
88 | args.GID = gid
89 |
90 | for {
91 | // try each known server.
92 | for _, srv := range ck.servers {
93 | var reply MoveReply
94 | ok := srv.Call("ShardMaster.Move", args, &reply)
95 | if ok && reply.WrongLeader == false {
96 | return
97 | }
98 | }
99 | time.Sleep(100 * time.Millisecond)
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/src/shardmaster/common.go:
--------------------------------------------------------------------------------
1 | package shardmaster
2 |
3 | //
4 | // Master shard server: assigns shards to replication groups.
5 | //
6 | // RPC interface:
7 | // Join(servers) -- add a set of groups (gid -> server-list mapping).
8 | // Leave(gids) -- delete a set of groups.
9 | // Move(shard, gid) -- hand off one shard from current owner to gid.
10 | // Query(num) -> fetch Config # num, or latest config if num==-1.
11 | //
12 | // A Config (configuration) describes a set of replica groups, and the
13 | // replica group responsible for each shard. Configs are numbered. Config
14 | // #0 is the initial configuration, with no groups and all shards
15 | // assigned to group 0 (the invalid group).
16 | //
17 | // You will need to add fields to the RPC argument structs.
18 | //
19 |
20 | // The number of shards.
21 | const NShards = 10
22 |
23 | // A configuration -- an assignment of shards to groups.
24 | // Please don't change this.
25 | type Config struct {
26 | Num int // config number
27 | Shards [NShards]int // shard -> gid
28 | Groups map[int][]string // gid -> servers[]
29 | }
30 |
31 | const (
32 | OK = "OK"
33 | )
34 |
35 | type Err string
36 |
37 | type JoinArgs struct {
38 | Servers map[int][]string // new GID -> servers mappings
39 | }
40 |
41 | type JoinReply struct {
42 | WrongLeader bool
43 | Err Err
44 | }
45 |
46 | type LeaveArgs struct {
47 | GIDs []int
48 | }
49 |
50 | type LeaveReply struct {
51 | WrongLeader bool
52 | Err Err
53 | }
54 |
55 | type MoveArgs struct {
56 | Shard int
57 | GID int
58 | }
59 |
60 | type MoveReply struct {
61 | WrongLeader bool
62 | Err Err
63 | }
64 |
65 | type QueryArgs struct {
66 | Num int // desired config number
67 | }
68 |
69 | type QueryReply struct {
70 | WrongLeader bool
71 | Err Err
72 | Config Config
73 | }
74 |
--------------------------------------------------------------------------------
/src/shardmaster/server.go:
--------------------------------------------------------------------------------
1 | package shardmaster
2 |
3 | import "raft"
4 | import "labrpc"
5 | import "sync"
6 | import "labgob"
7 |
8 | type ShardMaster struct {
9 | mu sync.Mutex
10 | me int
11 | rf *raft.Raft
12 | applyCh chan raft.ApplyMsg
13 |
14 | // Your data here.
15 |
16 | configs []Config // indexed by config num
17 | }
18 |
19 | type Op struct {
20 | // Your data here.
21 | }
22 |
23 | func (sm *ShardMaster) Join(args *JoinArgs, reply *JoinReply) {
24 | // TODO: Your code here.
25 | }
26 |
27 | func (sm *ShardMaster) Leave(args *LeaveArgs, reply *LeaveReply) {
28 | // TODO: Your code here.
29 | }
30 |
31 | func (sm *ShardMaster) Move(args *MoveArgs, reply *MoveReply) {
32 | // TODO: Your code here.
33 | }
34 |
35 | func (sm *ShardMaster) Query(args *QueryArgs, reply *QueryReply) {
36 | // TODO: Your code here.
37 | }
38 |
39 | //
40 | // the tester calls Kill() when a ShardMaster instance won't
41 | // be needed again. you are not required to do anything
42 | // in Kill(), but it might be convenient to (for example)
43 | // turn off debug output from this instance.
44 | //
45 | func (sm *ShardMaster) Kill() {
46 | sm.rf.Kill()
47 | // TODO: Your code here, if desired.
48 | }
49 |
50 | // needed by shardkv tester
51 | func (sm *ShardMaster) Raft() *raft.Raft {
52 | return sm.rf
53 | }
54 |
55 | //
56 | // servers[] contains the ports of the set of
57 | // servers that will cooperate via Paxos to
58 | // form the fault-tolerant shardmaster service.
59 | // me is the index of the current server in servers[].
60 | //
61 | func StartServer(servers []*labrpc.ClientEnd, me int, persister *raft.Persister) *ShardMaster {
62 | sm := new(ShardMaster)
63 | sm.me = me
64 |
65 | sm.configs = make([]Config, 1)
66 | sm.configs[0].Groups = map[int][]string{}
67 |
68 | labgob.Register(Op{})
69 | sm.applyCh = make(chan raft.ApplyMsg)
70 | sm.rf = raft.Make(servers, me, persister, sm.applyCh)
71 |
72 | // TODO: Your code here.
73 |
74 | return sm
75 | }
76 |
--------------------------------------------------------------------------------