stack_;
33 | Node nodes_[kMaxNodes];
34 | int cur_node_;
35 |
36 | Column* new_column(int n = 0)
37 | {
38 | assert(cur_node_ < kMaxNodes);
39 | Column* c = &nodes_[cur_node_++];
40 | memset(c, 0, sizeof(Column));
41 | c->left = c;
42 | c->right = c;
43 | c->up = c;
44 | c->down = c;
45 | c->col = c;
46 | c->name = n;
47 | return c;
48 | }
49 |
50 | void append_column(int n)
51 | {
52 | assert(columns_[n] == NULL);
53 |
54 | Column* c = new_column(n);
55 | put_left(root_, c);
56 | columns_[n] = c;
57 | }
58 |
59 | Node* new_row(int col)
60 | {
61 | assert(columns_[col] != NULL);
62 | assert(cur_node_ < kMaxNodes);
63 |
64 | Node* r = &nodes_[cur_node_++];
65 |
66 | //Node* r = new Node;
67 | memset(r, 0, sizeof(Node));
68 | r->left = r;
69 | r->right = r;
70 | r->up = r;
71 | r->down = r;
72 | r->name = col;
73 | r->col = columns_[col];
74 | put_up(r->col, r);
75 | return r;
76 | }
77 |
78 | int get_row_col(int row, int val)
79 | {
80 | return kRow+row*10+val;
81 | }
82 |
83 | int get_col_col(int col, int val)
84 | {
85 | return kCol+col*10+val;
86 | }
87 |
88 | int get_box_col(int box, int val)
89 | {
90 | return kBox+box*10+val;
91 | }
92 |
93 | Dance(int inout[81]) : inout_(inout), cur_node_(0)
94 | {
95 | stack_.reserve(100);
96 |
97 | root_ = new_column();
98 | root_->left = root_->right = root_;
99 | memset(columns_, 0, sizeof(columns_));
100 |
101 | bool rows[N][10] = {false};
102 | bool cols[N][10] = {false};
103 | bool boxes[N][10] = {false};
104 |
105 | for (int i = 0; i < N; ++i) {
106 | int row = i / 9;
107 | int col = i % 9;
108 | int box = row/3*3 + col/3;
109 | int val = inout[i];
110 | rows[row][val] = true;
111 | cols[col][val] = true;
112 | boxes[box][val] = true;
113 | }
114 |
115 | for (int i = 0; i < N; ++i) {
116 | if (inout[i] == 0) {
117 | append_column(i);
118 | }
119 | }
120 |
121 | for (int i = 0; i < 9; ++i) {
122 | for (int v = 1; v < 10; ++v) {
123 | if (!rows[i][v])
124 | append_column(get_row_col(i, v));
125 | if (!cols[i][v])
126 | append_column(get_col_col(i, v));
127 | if (!boxes[i][v])
128 | append_column(get_box_col(i, v));
129 | }
130 | }
131 |
132 | for (int i = 0; i < N; ++i) {
133 | if (inout[i] == 0) {
134 | int row = i / 9;
135 | int col = i % 9;
136 | int box = row/3*3 + col/3;
137 | //int val = inout[i];
138 | for (int v = 1; v < 10; ++v) {
139 | if (!(rows[row][v] || cols[col][v] || boxes[box][v])) {
140 | Node* n0 = new_row(i);
141 | Node* nr = new_row(get_row_col(row, v));
142 | Node* nc = new_row(get_col_col(col, v));
143 | Node* nb = new_row(get_box_col(box, v));
144 | put_left(n0, nr);
145 | put_left(n0, nc);
146 | put_left(n0, nb);
147 | }
148 | }
149 | }
150 | }
151 | }
152 |
153 | Column* get_min_column()
154 | {
155 | Column* c = root_->right;
156 | int min_size = c->size;
157 | if (min_size > 1) {
158 | for (Column* cc = c->right; cc != root_; cc = cc->right) {
159 | if (min_size > cc->size) {
160 | c = cc;
161 | min_size = cc->size;
162 | if (min_size <= 1)
163 | break;
164 | }
165 | }
166 | }
167 | return c;
168 | }
169 |
170 | void cover(Column* c)
171 | {
172 | c->right->left = c->left;
173 | c->left->right = c->right;
174 | for (Node* row = c->down; row != c; row = row->down) {
175 | for (Node* j = row->right; j != row; j = j->right) {
176 | j->down->up = j->up;
177 | j->up->down = j->down;
178 | j->col->size--;
179 | }
180 | }
181 | }
182 |
183 | void uncover(Column* c)
184 | {
185 | for (Node* row = c->up; row != c; row = row->up) {
186 | for (Node* j = row->left; j != row; j = j->left) {
187 | j->col->size++;
188 | j->down->up = j;
189 | j->up->down = j;
190 | }
191 | }
192 | c->right->left = c;
193 | c->left->right = c;
194 | }
195 |
196 | bool solve()
197 | {
198 | if (root_->left == root_) {
199 | for (size_t i = 0; i < stack_.size(); ++i) {
200 | Node* n = stack_[i];
201 | int cell = -1;
202 | int val = -1;
203 | while (cell == -1 || val == -1) {
204 | if (n->name < 100)
205 | cell = n->name;
206 | else
207 | val = n->name % 10;
208 | n = n->right;
209 | }
210 |
211 | //assert(cell != -1 && val != -1);
212 | inout_[cell] = val;
213 | }
214 | return true;
215 | }
216 |
217 | Column* const col = get_min_column();
218 | cover(col);
219 | for (Node* row = col->down; row != col; row = row->down) {
220 | stack_.push_back(row);
221 | for (Node* j = row->right; j != row; j = j->right) {
222 | cover(j->col);
223 | }
224 | if (solve()) {
225 | return true;
226 | }
227 | stack_.pop_back();
228 | for (Node* j = row->left; j != row; j = j->left) {
229 | uncover(j->col);
230 | }
231 | }
232 | uncover(col);
233 | return false;
234 | }
235 |
236 | void put_left(Column* old, Column* nnew)
237 | {
238 | nnew->left = old->left;
239 | nnew->right = old;
240 | old->left->right = nnew;
241 | old->left = nnew;
242 | }
243 |
244 | void put_up(Column* old, Node* nnew)
245 | {
246 | nnew->up = old->up;
247 | nnew->down = old;
248 | old->up->down = nnew;
249 | old->up = nnew;
250 | old->size++;
251 | nnew->col = old;
252 | }
253 | };
254 |
255 | bool solve_sudoku_dancing_links(int unused)
256 | {
257 | Dance d(board);
258 | return d.solve();
259 | }
260 |
--------------------------------------------------------------------------------
/Lab1/src/Sudoku/Lab1.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #~/test_data/advanced_test (include test1 test1000)
3 | if [ $# != 2 ] ; then
4 | echo "参数数量不对"
5 | echo "参数1:包含测试文件组的文件 例如 \"test_group\" (其中可能包含 ./test1 ./test1000) "
6 | echo "参数2:包含测试文件组的文件 例如 \"answer_group\" (其中可能包含 ./answer1 ./answer1000) "
7 | echo "完整命令列如 : $0 test_group"
8 | exit 1;
9 | fi
10 |
11 | test_data_txt=$1
12 | answer_date_txt=$2
13 | GROUP_NAME="group1"
14 | Current_Folder=$(cd `dirname $0`; pwd)
15 | Record_Time="${test_data_txt}_Folder/Record_Time.csv"
16 | Error_Teams="${test_data_txt}_Folder/Error_Teams.csv"
17 |
18 | if [ -e ${test_data_txt}_Folder ];then
19 | echo "${test_data_txt}_Folder 无需创建，已存在"
20 | else
21 | mkdir ${test_data_txt}_Folder
22 | echo "创建${test_data_txt}_Folder"
23 | fi
24 |
25 | touch ${Record_Time}
26 |
27 | echo "Team_Name,Use_Time(ms),Creat_Time,Modify_Time" > ${Record_Time}
28 |
29 | #$1 team_name $2 Result_name $3 ${Answer_Line} $4 diff_line
30 | Cul_Run_Time(){
31 | # stat $2
32 |
33 | INODE=$(ls -i $2 |awk '{print $1}')
34 | FILENAME=$2
35 |
36 | string=$(pwd)
37 | array=(${string//\// })
38 | root_Directory=${array[0]};
39 | # echo "root_Directory=@$root_Directory@"
40 |
41 | DISK=$(df -h |grep ${root_Directory} |awk '{print $1}')
42 | # echo "DISK=@$DISK@"
43 | if [[ -z ${DISK} ]];then
44 | DISK=$(df -h |grep "/$" |awk '{print $1}')
45 | fi
46 | # 文件系统非ext4则退出
47 | [[ "`df -T | grep ${DISK} |awk '{print $2}'`" != "ext4" ]] && { echo ${DISK} is not mount on type ext4! Only ext4 file system support!;exit 2; }
48 |
49 | Create_Time_t=$(sudo debugfs -R "stat <${INODE}>" ${DISK} | grep "crtime:" | awk '{printf $7}')
50 |
51 | Access_Time_t=$(stat $2 | grep "Access: 2" | awk '{printf $3}')
52 | Modify_Time_t=$(stat $2 | grep Modify | awk '{printf $3}')
53 |
54 | if [[ -z ${Modify_Time_t} ]];then
55 | echo "推测为中文系统"
56 | Modify_Time_t=$(stat $2 | grep "最近改动" | awk '{printf $2}')
57 | else
58 | echo "推测为英文系统"
59 | fi
60 | # echo "Access_Time_t:$Access_Time_t Modify_Time_t:$Modify_Time_t"
61 | echo "Create_Time_t:$Create_Time_t Modify_Time_t:$Modify_Time_t"
62 | # Access_Time=$(date -d ${Access_Time_t} +%s%N)
63 | Create_Time_t=$(date -d ${Create_Time_t} +%s%N)
64 | Modify_Time=$(date -d ${Modify_Time_t} +%s%N)
65 | # echo "$1 A is $Access_Time M is $Modify_Time"
66 | # Use_Time=`echo $Modify_Time $Access_Time | awk '{ print ($1 - $2) / 1000000}'`
67 | echo "$1 Cr is $Create_Time_t M is $Modify_Time"
68 | Use_Time=`echo $Modify_Time $Create_Time_t | awk '{ print ($1 - $2) / 1000000}'`
69 | # Check_Answer $1 $2 $3
70 | echo -e "\033[32m $1 Use_Time: $Use_Time ms \033[0m"
71 |
72 | Team_Answer=$(cat $2 | wc -l)
73 | if [[ ${Team_Answer} -ge $3 ]];then
74 | echo -e "\033[32m Up to standard Team_Answer:${Team_Answer} Answer_Line:$3 \033[0m"
75 | else
76 | echo -e "\033[31m Not up to standard Team_Answer:${Team_Answer} Answer_Line:$3 \033[0m"
77 | echo "$1 Not up to standard Team_Answer:${Team_Answer} Answer_Line:$3" >> ${Error_Teams}
78 | fi
79 |
80 | right_rate=$(echo "scale=2;1 - $4 / $3" | bc)
81 | # echo "$1,${right_rate},$Use_Time,${Team_Answer},$3,,$Access_Time_t,$Modify_Time_t"
82 | # echo "$1,${right_rate},$Use_Time,${Team_Answer},$3,,$Access_Time_t,$Modify_Time_t" >> ${Record_Time}
83 | }
84 |
85 | Basic_Test(){
86 | # $1 team_name $2 test_data_file_name $3 answer_date_file_name
87 | test_data=$(cat $2) # ./test1 kml
88 | Answer_Line=0
89 | Exe_Prog=""
90 | cd ${Current_Folder}
91 | Result="${test_data_txt}_Folder/Basic_Result"
92 | Answer="${test_data_txt}_Folder/Basic_Answer"
93 |
94 | if [[ -e sudoku_solve ]];then
95 | echo "找到执行文件sudoku_solve"
96 | Exe_Prog='sudoku_solve'
97 | elif [[ -e sudoku ]];then
98 | echo "找到执行文件sudoku"
99 | Exe_Prog='sudoku'
100 | else
101 | echo "未找到命名为sudoku或sudoku_solve的执行文件"
102 | echo "进行编译"
103 | make
104 | if [[ -e sudoku_solve ]];then
105 | echo "找到执行文件sudoku_solve"
106 | Exe_Prog='sudoku_solve'
107 | elif [[ -e sudoku ]];then
108 | echo "找到执行文件sudoku"
109 | Exe_Prog='sudoku'
110 | else
111 | echo "编译后仍未找到相应执行文件"
112 | exit 1;
113 | fi
114 | fi
115 |
116 | if [[ -e ./${Answer} ]];then
117 | rm ./${Answer}
118 | touch ./${Answer}
119 | else
120 | touch ./${Answer}
121 | fi
122 | sudo chmod 777 ./${Answer}
123 | screen -S $1 -X quit
124 | screen -dmS $1
125 |
126 | if [[ -e ./${Result} ]];then
127 | rm ./${Result}
128 | touch ./${Result}
129 | else
130 | touch ./${Result}
131 | fi
132 |
133 | screen -S $1 -X stuff "stdbuf -oL ./${Exe_Prog} > ./${Result}^M"
134 | while read -r test_data
135 | do
136 | Answer_Line=`expr ${Answer_Line} + $(cat ${test_data} | wc -l)`
137 | screen -S $1 -X stuff "${test_data}^M"
138 | done < $2
139 |
140 | while read -r answer_data
141 | do
142 | cat ${Current_Folder}/${answer_data} >> ${Current_Folder}/${Answer}
143 | done < $3
144 |
145 | sleep 30
146 | screen -S $1 -X stuff "^C^M"
147 | screen -S $1 -X quit
148 |
149 | diff_line=$(diff -ab ./${Result} ./${Answer} | wc -l)
150 | echo -e "\033[32m 错误个数:$diff_line / ${Answer_Line} \033[0m"
151 | Cul_Run_Time $1 ${Current_Folder}/${Result} ${Answer_Line} ${diff_line}
152 | }
153 |
154 | Advanced_Test(){
155 | # $1 team_name $2 test_data_file_name
156 | test_data=$(cat $2) # ./test10000
157 | Answer_Line=0
158 | Exe_Prog=""
159 | cd ${Current_Folder}
160 | Result="${test_data_txt}_Folder/Advanced_Result"
161 | Answer="${test_data_txt}_Folder/Advanced_Answer"
162 |
163 | if [[ -e sudoku_solve ]];then
164 | echo "找到执行文件sudoku_solve"
165 | Exe_Prog='sudoku_solve'
166 | elif [[ -e sudoku ]];then
167 | echo "找到执行文件sudoku"
168 | Exe_Prog='sudoku'
169 | else
170 | echo "未找到命名为sudoku或sudoku_solve的执行文件"
171 | echo "进行编译"
172 | make
173 | if [[ -e sudoku_solve ]];then
174 | echo "找到执行文件sudoku_solve"
175 | Exe_Prog='sudoku_solve'
176 | elif [[ -e sudoku ]];then
177 | echo "找到执行文件sudoku"
178 | Exe_Prog='sudoku'
179 | else
180 | echo "编译后仍未找到相应执行文件"
181 | exit 1;
182 | fi
183 | fi
184 |
185 | if [[ -e ./${Answer} ]];then
186 | rm ./${Answer}
187 | touch ./${Answer}
188 | else
189 | touch ./${Answer}
190 | fi
191 | sudo chmod 777 ./${Answer}
192 | screen -S $1 -X quit
193 | screen -dmS $1
194 |
195 | if [[ -e ./${Result} ]];then
196 | rm ./${Result}
197 | touch ./${Result}
198 | else
199 | touch ./${Result}
200 | fi
201 |
202 | screen -S $1 -X stuff "stdbuf -oL ./${Exe_Prog} > ./${Result}^M"
203 | while read -r test_data
204 | do
205 | Answer_Line=`expr ${Answer_Line} + $(cat ${test_data} | wc -l)`
206 | screen -S $1 -X stuff "${test_data}^M"
207 | done < $2
208 |
209 | while read -r answer_data
210 | do
211 | cat ${Current_Folder}/${answer_data} >> ${Current_Folder}/${Answer}
212 | done < $3
213 |
214 | sleep 30
215 | screen -S $1 -X stuff "^C^M"
216 | screen -S $1 -X quit
217 |
218 | diff_line=$(diff -ab ./${Result} ./${Answer} | wc -l)
219 | echo -e "\033[32m 错误个数:$diff_line / ${Answer_Line} \033[0m"
220 | Cul_Run_Time $1 ${Current_Folder}/${Result} ${Answer_Line} ${diff_line}
221 | }
222 |
223 |
224 | if [ -e ${test_data_txt} ];then
225 | Running_Version=$(cat ${test_data_txt} | wc -l)
226 | echo "测试样例：$Running_Version 个"
227 | cat ${test_data_txt}
228 | if [[ $Running_Version == 1 ]];then
229 | echo -e "\033[32m --------------${GROUP_NAME} is running Basic_Test-------------- \033[0m"
230 | Basic_Test ${GROUP_NAME} ${test_data_txt} ${answer_date_txt}
231 | elif [[ $Running_Version == 0 ]];then
232 | echo "请检查输入格式，确保文件末尾有一个换行,参照实验指导书"
233 | else
234 | echo -e "\033[32m --------------${GROUP_NAME} is running Advanced_Test--------------\033[0m"
235 | Advanced_Test ${GROUP_NAME} ${test_data_txt} ${answer_date_txt}
236 | fi
237 | else
238 | echo "文件$1不存在"
239 | exit 1;
240 | fi
241 |
242 |
--------------------------------------------------------------------------------
/Lab4/README.md:
--------------------------------------------------------------------------------
1 | # 云计算Lab4
2 |
3 | ## **1. Overview**
4 |
5 | 在 Lab 2 中，我们完成了一个简化的 HTTP/1.1 Web 服务器；
6 |
7 | 在 Lab 3 中，我们完成了一个简化的 分布式数据库系统。
8 |
9 | 现在让我们尝试将两者组合并完善，构建一个高性能的分布式选课系统。
10 |
11 | 我们的目标：
12 |
13 | - 运用课程知识与过去的实验项目，构建一个分布式选课系统
14 | - 学习如何提高一个系统的健壮性和可扩展性，尽可能提升其性能
15 |
16 | **注意：如果你选择完成 Lab 4，那么将不再需要完成 Lab 3，因为 Lab 4 已经涵盖了 Lab 3 的要求，其中数据库方面的要求，请查看Lab3，我们会根据你的 Lab 4 完成情况对 Lab 3 进行打分。**
17 |
18 | ---
19 |
20 | ## 2. Background
21 |
22 | 你所需要的有关 Http Server 与 Distributed Database 的基础知识都在过去的实验中，如果你忘记了或者想要复习，可以重新查看 [Lab 2](https://github.com/1989chenguo/CloudComputingLabs/tree/master/Lab2) 和 [Lab 3](https://github.com/1989chenguo/CloudComputingLabs/tree/master/Lab3)。
23 |
24 | ### Dividing databases
25 |
26 | 随着近些年信息化大发展，越来越多的数据存入了数据库中，而物理服务器的CPU、内存、存储、连接数等资源是有限的，某个时段大量连接同时执行操作，会导致数据库在处理上遇到性能瓶颈。为了解决这个问题，行业先驱门充分发扬了`分而治之`的思想，对大表进行分割，然后实施更好的控制和管理，同时使用多台机器的CPU、内存、存储，提供更好的性能。
27 |
28 | 其中一种简单的逻辑分割是`垂直分库`。比如我们的数据库中有商品表Products、还有对订单表Orders，还有积分表Scores。接下来我们就可以创建三个数据库，一个数据库存放商品，一个数据库存放订单，一个数据库存放积分。如下图所示：
29 |
30 | 
31 |
32 | 这样做有以下几个优点：
33 |
34 | - 跟随业务进行分割，和最近流行的微服务概念相似，方便解耦之后的管理及扩展。
35 | - 高并发的场景下，垂直拆分使用多台服务器的CPU、I/O、内存能提升性能，同时对单机数据库连接数、一些资源限制也得到了提升。
36 | - 能实现冷热数据的分离。
37 |
38 | 当然，分库分表在实际应用中是一个十分复杂的设计，在低并发的应用程序中，分库分表的效果并不明显，因此需要根据业务需求，并发承载能力等综合考虑。为了简化这一过程，在实验中，我们仅要求对数据进行垂直分库，即将课程数据和学生信息分别存储到不同的节点中，以此来实现课程数据访问和学生信息访问互不影响的效果。
39 |
40 | ### Load Balancing
41 |
42 | 负载平衡（英语：load balancing）是一种电子计算机技术，用来在多个计算机（计算机集群）、网络连接、CPU、磁盘驱动器或其他资源中分配负载，以达到优化资源使用、最大化吞吐率、最小化响应时间、同时避免过载的目的。 使用带有负载平衡的多个服务器组件，取代单一的组件，可以通过冗余提高可靠性。负载平衡服务通常是由专用软件和硬件来完成。 主要作用是将大量作业合理地分摊到多个操作单元上进行执行，用于解决互联网架构中的高并发和高可用的问题。
43 |
44 | 负载均衡最重要的一个应用是利用多台服务器提供单一服务，对于互联网服务，负载平衡器通常是一个软件程序，这个程序侦听一个外部端口，互联网用户可以通过这个端口来访问服务，而作为负载平衡器的软件会将用户的请求转发给后台内网服务器，内网服务器将请求的响应返回给负载平衡器，负载平衡器再将响应发送到用户，这样就向互联网用户隐藏了内网结构，阻止了用户直接访问后台（内网）服务器，使得服务器更加安全，可以阻止对核心网络栈和运行在其它端口服务的攻击。
45 |
46 | 当所有后台服务器出现故障时，有些负载平衡器会提供一些特殊的功能来处理这种情况。例如转发请求到一个备用的负载平衡器、显示一条关于服务中断的消息等。负载平衡器使得IT团队可以显著提高容错能力。它可以自动提供大量的容量以处理任何应用程序流量的增加或减少。
47 |
48 | 为了更好的实现负载均衡器，你可能需要选择一种或几种合适的调度算法。
49 |
50 | ---
51 |
52 | ## **3. Your Tasks**
53 |
54 | **Basic Version**
55 |
56 | 
57 |
58 | 在 **basic** 版本中，你只需要运行一个 Web 服务器，与其相连的是多个存储器集群。
59 |
60 | 对于Web服务器，我们要求：支持高并发，可以同时响应多个客户端发送的请求。
61 |
62 | 对于存储器集群，我们要求：集群数量不小于2，单个集群之间的通信可以使用 2PC 或 RAFT 协议实现。
63 |
64 | > 当存储器集群数量增多时，你可以对存储的数据进行分库存储，如课程中的数据可以存储在 Cluster 1 中，而学生个人信息相关的数据可以存储在 Cluster 2 中。
65 | >
66 |
67 | **Advanced Version**
68 |
69 | 
70 |
71 | 在advance版本中，你需要运行一个负载均衡器，多个Web服务器以及多个存储器集群。
72 |
73 | 对于负载均衡器，我们要求：支持高并发，能够动态地选择活跃并且可用的Web服务器，将请求发送至相应节点进行处理。
74 |
75 | 对于Web服务器和存储器集群，我们要求等同于Basic版本。
76 |
77 | ## 4. Standardized Format
78 |
79 | 为了更好的进行测试与评阅，我们将对实验过程中的一些数据格式进行标准规定，请确保你实现的最终系统在这些对外接口上是我们的标准一致的。
80 |
81 | 在请求体和响应体中，我们采用JSON格式进行通信。请求体根据请求内容决定，而响应体包含了特定的数据结构，分别为`code`，`data`和`msg`。其中`code`表示状态码，200为正常响应，其他为错误响应，可以根据实际情况返回不同的非零值。`data`表示响应对象，里面包含响应结果，错误情况下为空。`msg`表示响应消息，正常情况下一般为ok，错误情况下根据实际值返回错误消息。
82 |
83 | > 除了特定的 api 之外，Web 服务器仍需要支持静态文件访问等基本功能（如支持显示 html 页面，非法 path 返回 404 页面等），在此略去介绍，具体信息可参考 Lab 2。
84 | >
85 |
86 | > 我们对 api 的设计进行了非常多的简化，现实中将会远比它们复杂，你可以使用要求以外的 path 扩展你的系统，如：你可以自行设计报错信息。
87 | >
88 |
89 | ### 4.1 Web Server
90 |
91 | 一个完整的选课系统需要包含众多功能，为了降低复杂性，我们将其抽象为查询课程，选中课程和退选课程这几个模块。
92 |
93 | 1. `/api/search/course`
94 |
95 | > 介绍：主要用于查询课程信息，相关的参数使用 `query` 携带，查询后给出的数据至少需要包含课程的 编号，名称，容量和已选人数。
96 | >
97 |
98 | > 方法：GET
99 | >
100 |
101 | > 参数：id=[course_id]
102 | >
103 | >
104 | > > 课程编号仅由 `[a-zA-Z0-9]`组成，不含有特殊字符。
105 | > >
106 |
107 | > 响应体：
108 | >
109 | >
110 | > ```json
111 | > {
112 | > "code": 200,
113 | > "data": {
114 | > "id": course_id, // 课程id
115 | > "name": course_name , // 课程
116 | > "capacity": capacity , // 课程容量
117 | > "selected": selected // 已选人数
118 | > },
119 | > "msg": "ok"
120 | > }
121 | > ```
122 | >
123 |
124 | 2. `/api/search/all`
125 |
126 | > 介绍：与查询单个课程类似，但其 `data`字段的值从单个课程信息变为了包含所有课程信息的数组。
127 | >
128 |
129 | > 方法：GET
130 | >
131 |
132 | > 响应体：
133 | >
134 | >
135 | > ```json
136 | > {
137 | > "code": 200,
138 | > "data": [{
139 | > "id": course_id, // 课程id
140 | > "name": course_name , // 课程
141 | > "capacity": capacity , // 课程容量
142 | > "selected": selected // 已选人数
143 | > },{
144 | > "id": course_id,
145 | > "name": course_name , // 课程
146 | > "capacity": capacity , // 课程容量
147 | > "selected": selected // 已选人数
148 | > },
149 | > ...
150 | > ],
151 | > "msg": "ok"
152 | > }
153 | > ```
154 | >
155 |
156 | 3. `/api/search/student`
157 |
158 | > 介绍：主要用于查询学生所选课程信息，相关的参数使用 `query` 携带
159 | >
160 |
161 | > 方法：GET
162 | >
163 |
164 | > 参数：stuid=[student_id]
165 | >
166 | >
167 | > > 学生学号仅由 `[a-zA-Z0-9]`组成，不含有特殊字符。
168 | > >
169 |
170 | > 响应体：
171 | >
172 | >
173 | > ```json
174 | > {
175 | > "code": 200,
176 | > "data": {
177 | > "stuid": student_id, // 学号
178 | > "name": student_name , // 学生姓名
179 | > "course": [{ // 学生选的课程数组
180 | > "id": course_id,
181 | > "name": course_name , // 课程
182 | > },{
183 | > "id": course_id,
184 | > "name": course_name , // 课程
185 | > },
186 | > ...
187 | > ]
188 | > },
189 | > "msg": "ok"
190 | > }
191 | > ```
192 | >
193 | > > 如果学生未选课程， `data.course = []`即可。
194 | > >
195 |
196 | 4. `/api/choose`
197 |
198 | > 介绍：主要用于进行选课操作，相关的参数使用 `http request body`携带
199 | >
200 |
201 | > 方法：POST
202 | >
203 |
204 | > 请求体：
205 | >
206 | >
207 | > ```json
208 | > {
209 | > "stuid": student_id, // 学生学号
210 | > "course_id": course_id // 课程编号
211 | > }
212 | > ```
213 | >
214 | > > 当需要进行选课操作时，`payload` 应至少包含 学生学号，课程编号。
215 | > >
216 |
217 | > 响应体：
218 | >
219 | >
220 | > ```json
221 | > {
222 | > "code": 200,
223 | > "data": [],
224 | > "msg": "ok"
225 | > }
226 | > ```
227 | >
228 | > > 对于 payload 格式或数据非法，无法查询到相关信息，课程已满 等错误情况，返回包含错误信息的 JSON 对象即可。
229 | > >
230 |
231 | 5. `/api/drop`
232 |
233 | > 介绍：主要用于进行退选操作，相关的参数使用 `http request body`携带
234 | >
235 |
236 | > 方法：POST
237 | >
238 |
239 | > 请求体：
240 | >
241 | >
242 | > ```json
243 | > {
244 | > "stuid": student_id, // 学生学号
245 | > "course_id": course_id // 课程编号
246 | > }
247 | > ```
248 | >
249 | > > 当需要进行选课操作时，`payload` 应至少包含 学生学号，课程编号。
250 | > >
251 |
252 | > 响应体：
253 | >
254 | >
255 | > ```json
256 | > {
257 | > "code": 200,
258 | > "data": [],
259 | > "msg": "ok"
260 | > }
261 | > ```
262 | >
263 | > > 对于 payload 格式或数据非法，无法查询到相关信息，该学生没有选择这门课等错误情况，返回包含错误信息的 JSON 对象即可。
264 | > >
265 |
266 | ### 4.2 Store Server
267 |
268 | ### Command Lines Arguments
269 |
270 | 由于我们系统使用了默认数据，所以你的存储器应该在启动时，对这些数据进行读取。
271 |
272 | > 默认数据可以在 data/ 文件夹中找到。
273 | >
274 |
275 | > 其余说明请参考 Lab 3。
276 | >
277 |
278 | ### Database Tables
279 |
280 | 数据库的基础格式是关系型数据库，你也可以根据需要增添修改数据的类型，也可以选择设计新类型的数据库，但必须保证通过数据库可以正常完成需要的功能。
281 |
282 | > 默认数据可以在 data/ 文件夹中找到。
283 | >
284 |
285 | ### Course
286 |
287 | | id(key) - string | name - string | capacity - int | selected - int |
288 | | --- | --- | --- | --- |
289 | | "CS06142" | "云计算技术" | 120 | 120 |
290 |
291 | ### Student
292 |
293 | | id(key) - string | name - string |
294 | | --- | --- |
295 | | "211926010111" | "张三" |
296 |
297 | ### Course Selection
298 |
299 | | Course id - string | Student id - string |
300 | | --- | --- |
301 | | "CS06142" | "211926010111" |
302 |
303 | ### Commands
304 |
305 | 正常情况下，用户访问数据仅通过前端的 HTTP Web 服务器，而数据查询由 Web 服务器与数据库集群进行交互。
306 |
307 | 以下给出的数据库命令**仅供参考**，你可以选择自己实现一个满足相同基本功能的命令集。
308 |
309 | ### Check Courses Capacity
310 |
311 | 发送 `GET Course [course id]`, e.g., `GET Course CS06142`；
312 |
313 | 返回 `[course id] [course name] [course capacity] [selected number]`, e.g., `CS06142 云计算技术 120 120`
314 |
315 | 特别的，可以一次查询所有的课程最大容量信息:
316 |
317 | 发送 `GET Course all`, e.g., `GET Course all`；
318 |
319 | 返回 **多行** `[course id] [course name] [course capacity] [selected number]`, e.g., `CS04008 计算机网络 90 80\nCS06142 云计算技术 120 120`。
320 |
321 | ### Check Student Selected Courses
322 |
323 | 发送 `GET Student Courses [student id]`, e.g., `GET Student Courses 211926010111`；
324 |
325 | 返回 **多行** `[course id] [course name]`, e.g., `CS04008 计算机网络\nCS06142 云计算技术`。
326 |
327 | ### Choose a Course
328 |
329 | 发送 `ADD Student Course [student id] [course id]`, e.g., `ADD Student Course 211926010111 CS06142`；
330 |
331 | 如果成功，返回 `+OK`; 如果失败，返回 `-ERROR`。
332 |
333 | ### Drop a Course
334 |
335 | 发送 `DEL Student Course [student id] [course id]`, e.g., `DEL Student Course 211926010111 CS06142`；
336 |
337 | 如果成功，返回 `+OK`; 如果失败，返回 `-ERROR`。
338 |
339 | ### 4.3 **Load Balancer**
340 |
341 | 为了更好地统一测试，我们需要你的负载均衡器采用标准的配置文件启动，例如，你可以这样将所有的Web服务的地址和端口传给负载均衡器，由负载均衡器动态的转发请求到这些Web服务上：
342 |
343 | ```
344 | # web_servers.conf
345 | # 每一行都是一个内部web服务进程的监听地址和端口
346 | 127.0.0.1 8081
347 | 127.0.0.1 8082
348 | 127.0.0.1 8083
349 | 127.0.0.1 8084
350 | ```
351 |
352 | ## 5. Test
353 |
354 | 默认情况下，我们将在 Linux Ubuntu 20.04 x64 服务器进行测试。
355 |
356 | 我们预期的测试和评分项目有：
357 |
358 | - API功能实现：上述每个API均为1分（**5分**）
359 | - 存储节点容灾测试：随机终止存储节点服务，接口运行情况（**3分**）
360 | - 负载均衡实现：通过负载均衡器对外提供接口服务，并且Web服务能够容灾（**1分**）
361 | - 性能测试：在高并发下、动态切换的情况下，系统的承载能力，根据所有小组的测试情况进行内部评比得分（1**分**）
362 |
363 | 为了标准化测试，你的系统在运行上，需要满足一下条件：
364 |
365 | 1. 注意文件结构，不同模块源代码处于不同文件夹中（我们会检查你的源代码）
366 | 2. 项目的根目录下可以使用 `make` 命令直接编译所有模块
367 | 3. 可执行文件分为 Web 服务器 和 存储器集群两个部分，其中
368 | - Web 服务器可执行文件为 `web-server`，支持 `-ip`, `-port`, `-config_path` 等命令行参数，例如`./web-server --ip 127.0.0.1 --port 8080 --config_path ./conf/store_servers.conf`
369 | - 存储器 可执行文件为 `store-server`，支持 `-config_path` 等命令行参数，并使用配置文件 `coordinator.conf` 或 `participant.conf` 进行配置，具体细节请查看Lab3。测试中，每个集群的存储服务器数量为 4，2pc和raft任选一种实现，例如：
370 |
371 | ```
372 | # 启动存储服务器集群1（该集群主要存储 学生-选课 数据） - 2pc
373 | $ ./store-server --config_path ./conf/coordinator1.conf
374 | $ ./store-server --config_path ./conf/participant101.conf
375 | $ ./store-server --config_path ./conf/participant102.conf
376 | $ ./store-server --config_path ./conf/participant103.conf
377 |
378 | # 启动存储服务器集群2（该集群主要存储 课程-选课人数 数据） - raft
379 | $ ./store-server --config_path ./conf/store201.conf
380 | $ ./store-server --config_path ./conf/store202.conf
381 | $ ./store-server --config_path ./conf/store203.conf
382 | $ ./store-server --config_path ./conf/store204.conf
383 | ```
384 |
385 | - 如果你实现了 `Load Balancer`，其可执行文件请命名为 `load-balancer`，支持 `-ip --port --web_config_path -store_config_path` 等命令行参数。例如
386 |
387 | ```
388 | # 启动负载均衡器
389 | $ ./load-balancer --ip 127.0.0.1 --port 8080 --store_config_path ./conf/store_servers.conf --web_config_path ./conf/web_servers.conf
390 |
391 | # 启动web服务器集群
392 | $ ./web-server01 --ip 127.0.0.1 --port 8081 --config_path ./conf/store_servers.conf
393 | $ ./web-server02 --ip 127.0.0.1 --port 8082 --config_path ./conf/store_servers.conf
394 | $ ./web-server02 --ip 127.0.0.1 --port 8083 --config_path ./conf/store_servers.conf
395 | $ ./web-server02 --ip 127.0.0.1 --port 8084 --config_path ./conf/store_servers.conf
396 | ```
397 |
398 |
399 | > 可执行文件与配置文件的要求可以参考 Lab2, Lab 3。
400 | >
401 |
402 | 4.每次运行系统，都将先启动多个存储器集群中的每个存储服务器，随后启动多个 Web 服务器，最后再启动 Load Balancer（如果需要），请保证每个可执行文件都能正常运行。
403 |
404 | 类似之前的实验，我们提供了[测试程序](https://github.com/LabCloudComputing)，请仔细阅读 Lab 4 实验文档和测试程序文档后使用。
405 |
406 | ---
407 |
408 | ## 6. Lab submission
409 |
410 | 将你的代码提交到 `/Lab4/` 文件夹，并编写 `Makefile` 确保可以直接使用 `make` 命令编译所有需要的可执行文件。
411 |
412 | 考虑到 `Lab 4` 实现完整系统较为复杂，验收可能会存在额外问题，请在提交代码的同时，编写一份 `intro.md` ，并添加一些说明，如：如何编译，如何运行你的程序；是否运用了其他依赖库，如何安装依赖；如何管理数据库，实现数据分块等。
413 |
414 | ---
415 |
416 | ## 7. Grading standards
417 |
418 | 完成所有 Basic 版本的要求，你将获得 8 分；
419 |
420 | 完成所有 Advanced 版本的要求，你将获得 10 分。
421 |
422 | 如果你有部分功能没有实现，将根据你的完成情况进行细化打分。
423 |
424 | **此分数不包含 Lab 3，Lab 3 的分数会根据 Lab 4 完成情况另外获得。**
425 |
--------------------------------------------------------------------------------
/Lab1/README.md:
--------------------------------------------------------------------------------
1 | # Lab 1: “Super-fast” Sudoku Solving
2 |
3 | Enter in the folder you have cloned from our lab git repo, and pull the latest commit.
4 |
5 | `git pull`
6 |
7 | You can find this lab1's instruction in `Lab1/README.md`
8 |
9 | All materials of lab1 are in folder `Lab1/`
10 |
11 | ## 1. Overview
12 |
13 | Implement a Sudoku solving program, using multiple threads or multiple processes, running on a single machine. Try to **utilize all your CPU cores** and make your program **run as fast as possible**!
14 |
15 | ### Goals
16 |
17 | * Practice basic parallel programming skills, such as using multiple threads/processes;
18 | * Get familiar with Unix OS environment development (eg., file I/O, get timestamp);
19 | * Get familiar with source code version control tools (git), and learn to collaborate with others using github;
20 |
21 | ## 2. Background
22 |
23 | ### 2.1 Introduction to Sudoku
24 |
25 | Sudoku (originally called Number Place) is a logic-based combinatorial number-placement puzzle.
26 |
27 | You are given a 9×9 board of 81 squares logically separated into 9 columsn, 9 rows, and 9 3×3 subsquares. The goal is, given a board with some initial numbers filled in (we call it a **Sudoku puzzle**), fill in the rest of the board so that every column, row, and subsquare have all the digits from 1 to 9 and each digit appears only once (we call it a **Sudoku solution**).
28 |
29 |
30 | An example Sudoku puzzle:
31 |
32 | 
33 |
34 | An example Sudoku solution to above puzzle:
35 |
36 | 
37 |
38 | ### 2.2 Some useful resources
39 |
40 | If you have no idea about what algorithms can be used to solve Sudoku puzzles, we suggest you read [this](https://rafal.io/posts/solving-sudoku-with-dancing-links.html). To simplify your work, we have provided a simple [implementation](src/Sudoku/) `(Lab1/src/Sudoku`) of 4 Sudoku solving algorithms (some are slow, some are fast), but without using multiple threads/processes. The two files *test1* and *test1000* contain many puzzles for you to test.
41 |
42 | Of course, you are always encouraged (not mandatory) to implement those algorithms by yourselves and even your own algorithms (if you have time).
43 |
44 | ## 3. Your Lab Task
45 |
46 | ### 3.1 Write a program
47 |
48 | Implement a program which satisfies the following requirements:
49 |
50 | #### 3.1.1 Program input and output
51 |
52 | ##### **Input**
53 |
54 | 1. When executing the compiled program from the code you submit, it must **not** take any parameters and the program must be named ***sudoku_solve***. In other words, your program can be run normally by typing `./sudoku_solve` in the terminal.
55 | 2. But after start, your program needs to be able to read multiple strings from ***stdin***, which are separated by line breaks. The content of these strings are **the paths of the Sudoku puzzles files**.
56 | 3. In each input file, **each line** of the file represents a Sudoku puzzle that needs to be solved. Each line contains 81 digits, where the first 9 digits represent the first row of the Sudoku puzzle, the 10th to the 18th digits represent the second row of the puzzle, and so on. The digit 0 represents a blank that needs to be filled in when solving the puzzle.
57 |
58 | **3.1.1.1 Puzzle file example**
59 |
60 | In the following image, we have printed three Sudoku puzzle **files** in the terminal. They contain one, two, and three puzzles, and their names are test1, test2, and test3.
61 |
62 | 
63 |
64 | **3.1.1.2 Example input from terminal**
65 |
66 | In the following code snippet, we showed how to input Sudoku puzzle files that need to be solved from the terminal after your program starts running:
67 |
68 | ```
69 | ./test1
70 | ./test2
71 | ./test3
72 | ```
73 |
74 | ##### Output
75 |
76 | For each Sudoku puzzle in file, your program should only output the solution to the puzzle to ***stdout***. The format of the solution should be the same as the puzzle, except that all 0s in the puzzle should be replaced with the correct numbers. Also, remember that the order of the answers should be consistent with the order of the input Sudoku puzzles.
77 |
78 | **3.1.1.3 Example output**
79 |
80 | ```
81 | 312647985786953241945128367854379126273461859691285473437592618569814732128736594
82 | 693784512487512936125963874932651487568247391741398625319475268856129743274836159
83 | 869725413512934687374168529798246135231857946456319872683571294925483761147692358
84 | 693784512487512936125963874932651487568247391741398625319475268856129743274836159
85 | 364978512152436978879125634738651429691247385245389167923764851486512793517893246
86 | 378694512564218397291753684643125978712869453859437261435971826186542739927386145
87 | ```
88 |
89 | **3.1.1.4 Output order requirement**
90 |
91 | Taking the three test files above as an example, your program needs to output the result of `./test1` first, then `./test2`, and finally `./test3`. Additionally, the order of answers corresponding to each file should be consistent with the order of puzzles in the file.
92 |
93 | In the example output, the 1st line corresponds to the solution of the Sudoku puzzle in `./test1`. The 2nd and 3rd lines correspond to the 1st and 2nd puzzles in `./test2`. The 4th, 5th, and 6th lines correspond to the 1st, 2nd, and 3rd puzzles in test3.
94 |
95 | #### 3.1.2 Implementation requirements
96 |
97 | ##### 3.1.2.1 Basic version
98 |
99 | Your program should be able to:
100 |
101 | 1. Accept **one** input file name, and the size of the input file is smaller than 100MB.
102 | 2. Successfully solve the puzzles in the input file, and output the results in the format described before.
103 | 3. Use multiple threads/processes to make use of most of your machine's CPU cores.
104 |
105 | \[Tips\]:
106 |
107 | 1. Use an event queue to dispatch tasks and merge results to/from worker threads.
108 | 2. Dynamically detect how many CPU cores your machine has to determine how many threads/processes your program should use.
109 | 3. Be careful about contention between multiple threads/processes.
110 |
111 | ##### 3.1.2.2 Advanced version
112 |
113 | Your program should be able to:
114 |
115 | 1. Complete all the requirements in the basic version.
116 | 2. Accept **any number of** input file names, and the size of input file can be **any large** (as long as it can be stored on your disk)
117 | 3. When the program is solving puzzles in the previously input file(s), the program can **meanwhile accept more input file names from *stdin***.
118 |
119 | \[Tips\]:
120 |
121 | 1. Use a dedicated thread to accept input.
122 | 2. To avoid exhausting all memory, read different parts of the file into memory and solve them one by one.
123 | 3. You are encouraged to try more optimizations, such as cache coherency processing.
124 |
125 | ### 3.2. Test script
126 |
127 | We have provided a script for you to use so that you can test your program before submitting, you can use this shell script on most Linux systems.
128 |
129 | It will verify the correctness of the program based on the output, and show you the time your program takes.
130 |
131 | ##### 3.2.1 Get prepared
132 |
133 | 1. This script requires you to provide **two input files**. These two files need to be placed in the **same directory as the script**. The following describes how to prepare these two input files:
134 |
135 | a. The content of the **first file** is a list of Sudoku puzzle files which will be input to your program. For example, we name the first file as `test_group`, and we prepared 2 puzzle files(see in 3.1.1.1) to test our program, the first file is named as "test1" and it has 1 puzzle to be solved, and the other file is named "test1000" and it has 1000 puzzles to be solved. Then the content format of `test_group` is as follows:
136 |
137 | 
138 |
139 | b. The content of the **second file** is a list of Sudoku solution files to the puzzles you prepared. Take the example of the first file above, we name the second file as `answer_group`. Same to the `test_group` above, `answer_group` should also contain 2 answer files, the first answer file corresponds to ""test1"" in `test_group`, it's named "answer1" and it has 1 solution, the second answer file corresponds to "test1000" in `test_group`, it's name is "answer1000" and it has 1000 solutions. Then the content format of `answer_group` is as follows:
140 |
141 | 
142 |
143 | You can create your own test files and put them in the test group and answer group if you want.
144 |
145 | 2. Using this script requires installing the `screen` command. You can use the `which screen` command to check if `screen` is installed on your local machine. If it is not installed, you can install it using the following command:
146 | ```shell
147 | ubuntu(debian): sudo apt-get install screen #This or apt
148 | sudo apt install screen #This or apt-get
149 | centos(redhat): yum install screen 2.
150 | ```
151 | 3. The compiled executable file should be named **sudoku_solve**, and it need to be placed in the **same directory as this script**.
152 |
153 | 4. If your file system is not in the **ext4** format, this script may not be able to do performance tests successfully and can only do basic tests. You can use the following command to check which paths are mounted on an exf4-formatted disk:
154 |
155 | ```shell
156 | df -h --type=ext4
157 | ```
158 |
159 | 
160 |
161 | 5. The script will generate a sub-folder contains two files named Basic(Advanced)_Result and Basic(Advanced)_Answer, the Result file contains your program's output and the Answer file contains reference output. The script will delete these two folders every time it runs, so if you need the data, make sure to copy them before running the script.
162 |
163 | 6. The script presents the time spent as relative, and there is a relatively fixed processing time for each run. For example, if your program takes 1300ms to solve a Sudoku problem, but the script takes 800ms for preparation, the final display will show 2100 milliseconds. Additionally, everyone's configuration and performance are different, resulting in different times. This means that you only need to keep optimizing on your own machine instead of comparing with others. Rest assured that the same environment will be used for the final scoring.
164 |
165 | 7. (**Very important**) If you want to get a high score (script test), it is better to flush the data in the buffer in time.
166 |
167 | ##### 3.2.2 Script Usage
168 |
169 | ```shell
170 | sudo ./Lab1.sh test_group answer_group
171 | ```
172 |
173 | The script takes two parameters, they are the 2 prepared files in 3.2.1.
174 |
175 | Note that this output below means that you are missing a line break at the end of your answer group, which will not be recognized. Of course, the path of your puzzle file should be in the same order as the answer path.
176 |
177 | 
178 |
179 | An example：
180 |
181 | 
182 |
183 | ## 4. Lab submission
184 |
185 | Please put all your code in folder `Lab1` and write a `Makefile` so that we **can compile your code in one single command** `make`. The compiled runnable executable binary should be named `sudoku_solve` and located in folder `Lab1`. If you do not know how to write `Makefile`, you can find a simple example in `Lab1/src/Sudoku`. Please carefully following above rules so that TAs can automatically test your code!!!
186 |
187 | Please submit your lab program and performance test report following the guidance in the [Overall Lab Instructions](../README.md) (`../README.md`)
188 |
189 | ## 5. Grading standards
190 |
191 | 1. You can get 38 points if you can: 1) finish all the requirements of the basic version, and 2) your performance test report has finished the two requirements described before. If you missed some parts, you will get part of the points depending how much you finished
192 | 2. You can get 40 points (full score) if you can: 1) finish all the requirements of the advanced version, and 2) your performance test report has finished the two requirements described before. If you missed some parts, you will get part of the points depending how much you finished.
193 |
194 |
--------------------------------------------------------------------------------
/Lab2/background.md:
--------------------------------------------------------------------------------
1 | ## 2. Background
2 |
3 | ### 2.1 Hypertext Transport Protocol
4 |
5 | The Hypertext Transport Protocol (HTTP) is the most commonly used application protocol on the Internet today.
6 |
7 | Like many network protocols, HTTP uses a client-server model. An HTTP client opens a network connection to an HTTP server and sends an HTTP request message. Then, the server replies with an HTTP response message, which usually contains some resource (file, text, binary data) that was requested by the client.
8 |
9 | We briefly introduce the HTTP message format and structure in this section for your convenience. Detailed specification of HTTP/1.1 can be found in [RFC 2616 - Hypertext Transfer Protocol -- HTTP/1.1](https://tools.ietf.org/html/rfc2616).
10 |
11 | ### 2.2 HTTP Messages
12 |
13 | HTTP messages are simple, formatted blocks of data.
14 |
15 | All HTTP messages fall into two types: **request** messages and **response** messages.
16 |
17 | * Request messages request an action from a web server.
18 |
19 | * Response messages carry results of a request back to a client.
20 |
21 | Both request and response messages have the same basic message structure.
22 |
23 | #### 2.2.1 Message Format
24 |
25 | HTTP request and response messages consist of 3 components:
26 |
27 | - a start line describing the message,
28 | - a block of headers containing attributes,
29 | - and an optional body containing data.
30 |
31 | Each component has the format as following
32 |
33 | ##### 2.2.1.1 Start Line
34 |
35 | All HTTP messages begin with a start line. The start line for a request message says *what to do*. The start line for a response message says *what happened*.
36 |
37 | Specifically, the start line is also called ***Request line*** in *Request messages* and ***Response line*** in *Response messages*.
38 |
39 | * **Request line**
40 | * contains a method describing what operation the server should perform and a request URL describing the resource on which to perform the method.
41 | * also includes an HTTP version tells the server what dialect of HTTP the client is speaking. All of these fields are separated by whitespace.
42 | * eg. `GET /index.html HTTP/1.1`
43 |
44 | * **Response line**
45 | * contains the HTTP version that the response message is using, a numeric status code, and a textual reason phrase describing the status of the operation.
46 | * eg. `HTTP/1.1 200 OK`
47 |
48 | ##### 2.2.1.2 Header
49 |
50 | Following the start line comes a list of zero, one, or many HTTP header fields.
51 |
52 | HTTP header fields add additional information to request and response messages. They are basically just lists of name/value pairs.
53 |
54 | Each HTTP header has a simple syntax: a name, followed by a colon `:`, followed by optional whitespace, followed by the field value, followed by a `CRLF`.
55 |
56 | HTTP headers are classified into:
57 | * General headers
58 | * Request headers
59 | * Response headers
60 | * Entity headers
61 | * Extension headers.
62 |
63 | Note that request-header fields are different from the response-header fields. We will not introduce those fields in details and you are not required to implement in this lab.
64 |
65 | You can find them in [RFC 2616 - Hypertext Transfer Protocol -- HTTP/1.1](https://tools.ietf.org/html/rfc2616).
66 |
67 | Example of headers in a request:
68 |
69 | ```
70 | Host: 127.0.0.1:8888
71 | User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0
72 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
73 | Accept-Language: en-US,en;q=0.5
74 | Accept-Encoding: gzip, deflate
75 | Connection: keep-alive
76 | Upgrade-Insecure-Requests: 1
77 | Cache-Control: max-age=0
78 | // CRLF
79 | ```
80 |
81 | Example of headers in a response:
82 |
83 | ```
84 | Server: Guo's Web Server
85 | Content-length: 248
86 | Content-type: text/html
87 | // CRLF
88 | ```
89 |
90 | ##### 2.2.1.3 Entity Body
91 |
92 | The third part of an HTTP message is the optional entity body. Entity bodies are the payload of HTTP messages. They are the things that HTTP was designed to transmit.
93 |
94 | HTTP messages can carry many kinds of digital data: images, video, HTML documents, software applications, credit card transactions, electronic mail, and so on.
95 |
96 | Example of entity body:
97 |
98 | ```
99 |
100 | CS06142
101 |
102 | CS06142
103 | Welcome to Cloud Computing Course.
104 |
105 |
106 | Http Server at ip-127-0-0-1 Port 8888
107 |
108 | ```
109 |
110 | #### 2.2.2 Structure of HTTP Request
111 |
112 | A HTTP request message contains:
113 |
114 | * an HTTP request line:
115 | * method,
116 | * a query string,
117 | * the HTTP protocol version,
118 | * zero or more HTTP header lines,
119 | * a blank line (i.e. a `CRLF` by itself),
120 | * a optional content that request need to carry.
121 |
122 | Example of HTTP request message:
123 |
124 | ```
125 | GET /index.html HTTP/1.1
126 | Host: 127.0.0.1:8888
127 | User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0
128 | Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
129 | Accept-Language: en-US,en;q=0.5
130 | Accept-Encoding: gzip, deflate
131 | Connection: keep-alive
132 | Upgrade-Insecure-Requests: 1
133 | Cache-Control: max-age=0
134 | // CRLF
135 | ```
136 |
137 | #### 2.2.3 Structure of HTTP Response
138 |
139 | A HTTP response message contains:
140 |
141 | * an HTTP response status line:
142 | * the HTTP protocol version,
143 | * the status code
144 | * a description of the status code),
145 | * zero or more HTTP header lines,
146 | * a blank line (i.e. a `CRLF` by itself)
147 | * the content requested by the HTTP request.
148 |
149 | Example of HTTP response message:
150 |
151 | ```
152 | HTTP/1.1 200 OK
153 | Server: Tiny Web Server
154 | Content-length: 248
155 | Content-type: text/html
156 | // CRLF
157 |
158 | CS06142
159 |
160 | CS06142
161 | Welcome to Cloud Computing Course.
162 |
163 |
164 | Http Server at ip-127-0-0-1 Port 8888
165 |
166 | ```
167 |
168 | The `Host` request header specifies the host and port number of the server to which the request is being sent. If no port is included, the default port for the service requested is implied (e.g., 443 for an HTTPS URL, and 80 for an HTTP URL).
169 |
170 | The `Content-Length` header indicates the size of the message body, in bytes, sent to the recipient.
171 |
172 | The `Content-Type` representation header is used to indicate the original media type of the resource.
173 |
174 | Here are some examples:
175 |
176 | | file extension | MIME Type |
177 | | --- | --- |
178 | | *.html | text/html |
179 | | *.js | text/javascript |
180 | | *.css| text/css |
181 | | *.txt | text/plain |
182 | | *.json | application/json |
183 | | *.png | image/png |
184 | | *.jpg | image/jpg |
185 |
186 | ### 2.3 HTTP Proxy
187 |
188 | HTTP proxy servers are intermediaries. Proxies sit between clients and servers and act as "middlemen", shuffling HTTP messages back and forth between the parties.
189 |
190 | HTTP proxy servers are middlemen that fulfill transactions on the client's behalf. Without a HTTP proxy, HTTP clients talk directly to HTTP servers. With a HTTP proxy, the client instead talks to the proxy, which itself communicates with the server on the client's behalf.
191 |
192 | HTTP proxy servers are both web servers and web clients. Because HTTP clients send request messages to proxies, the proxy server must properly handle the requests and the connections and return responses, just like a web server. At the same time, the proxy itself sends requests to servers, so it must also behave like a correct HTTP client, sending requests and receiving responses.
193 |
194 | The working pattern of HTTP proxy is shown in the following figure:
195 |
196 | ```
197 | +-----------+ +-----------+
198 | | | | |
199 | +----------+ Request | | Request | |
200 | | |+---------------> |+--------------> |
201 | | Client | | Proxy | | Server |
202 | | <---------------+| <--------------+| |
203 | +----------+ Response | | Response | |
204 | | | | |
205 | +-----------+ +-----------+
206 | ```
207 |
208 | ## 2.4 JSON
209 |
210 | JSON (JavaScript Object Notation) is a lightweight data-interchange format. It is easy for humans to read and write. It is easy for machines to parse and generate. It is based on a subset of the JavaScript Programming Language Standard ECMA-262 3rd Edition - December 1999. JSON is a text format that is completely language independent but uses conventions that are familiar to programmers of the C-family of languages, including C, C++, C#, Java, JavaScript, Perl, Python, and many others. These properties make JSON an ideal data-interchange language.
211 |
212 | JSON is built on two structures:
213 |
214 | * A collection of name/value pairs. In various languages, this is realized as an object, record, struct, dictionary, hash table, keyed list, or associative array.
215 |
216 | * An ordered list of values. In most languages, this is realized as an array, vector, list, or sequence.
217 |
218 | These are universal data structures. Virtually all modern programming languages support them in one form or another. It makes sense that a data format that is interchangeable with programming languages also be based on these structures.
219 |
220 | In JSON, they take on these forms:
221 |
222 | * An object is an unordered set of name/value pairs. An object begins with `{` and ends with `}`. Each name is followed by `:`and the name/value pairs are separated by `,`.
223 |
224 | * An array is an ordered collection of values. An array begins with `[` and ends with `]`. Values are separated by `,`.
225 |
226 | * A value can be a string in double quotes, or a number, or true or false or null, or an object or an array. These structures can be nested.
227 |
228 | * A string is a sequence of zero or more Unicode characters, wrapped in double quotes, using backslash escapes. A character is represented as a single character string. A string is very much like a C or Java string.
229 |
230 | * A number is very much like a C or Java number, except that the octal and hexadecimal formats are not used.
231 |
232 | * Whitespace can be inserted between any pair of tokens. Excepting a few encoding details, that completely describes the language.
233 |
234 | For example:
235 |
236 | ```json
237 | {
238 | "status": {
239 | "code": 200,
240 | "text": "OK"
241 | },
242 | "data" : [
243 | {
244 | "id": "S1",
245 | "name": "Foo"
246 | },
247 | "Bar\r\n",
248 | 5.0,
249 | true,
250 | null
251 | ]
252 | }
253 | ```
254 |
255 | ## 2.5 CURL
256 |
257 | `curl` is a tool to transfer data from or to a server, using one of the supported protocols (DICT, FILE, FTP, FTPS, GOPHER, HTTP, HTTPS, IMAP, IMAPS, LDAP, LDAPS, MQTT, POP3, POP3S, RTMP, RTMPS, RTSP, SCP, SFTP, SMB, SMBS, SMTP, SMTPS, TELNET and TFTP). The command is designed to work without user interaction.
258 |
259 | `curl` offers a busload of useful tricks like proxy support, user authentication, FTP upload, HTTP post, SSL connections, cookies, file transfer resume, Metalink, and more. As you will see below, the number of features will make your head spin!
260 |
261 | `curl` is powered by libcurl for all transfer-related features. See libcurl(3) for details.
262 |
263 | ### 2.5.1 source code
264 |
265 | You can check out the latest [source code](https://github.com/curl/curl) from GitHub.
266 |
267 | ### 2.5.2 install
268 |
269 | On many operating systems, curl is already installed by default. If not, you can use the system's package management tool to install it. E.g., `sudo apt install curl` for Ubuntu.
270 |
271 | ### 2.5.3 tutorials
272 |
273 | For more tutorials, please check [curl tutorial](https://curl.se/docs/manual.html) or use `man curl` after installing.
274 |
275 | Here we give some examples that you might need in your lab.
276 |
277 | #### 2.5.3.1 GET
278 |
279 | ##### access static resouces
280 |
281 | ```shell
282 | curl -i -X GET http://localhost:8080/index.html
283 | ```
284 |
285 | ##### access a web service
286 |
287 | ```shell
288 | curl -i -X GET http://localhost:8080/api/search
289 | ```
290 |
291 | ###### with query string
292 |
293 | ```shell
294 | curl -i -G -d 'id=1&name=Foo' -X GET http://localhost:8080/api/search
295 | ```
296 |
297 | ##### 2.5.3.2 POST
298 |
299 | ##### send data like html
300 |
301 | ```shell
302 | curl -i -d 'id=1&name=Foo'
303 | -H 'Content-Type: application/x-www-form-urlencoded'
304 | -X POST http://localhost:8080/api/upload
305 | ```
306 |
307 | ##### send data use form-data
308 |
309 | ```shell
310 | curl -i -F "id=1" -F "name=Foo"
311 | -X POST http://localhost:8080/api/upload
312 | ```
313 | ##### send data use json
314 |
315 | ```shell
316 | curl -i -d '{"id":"1","name":"Foo"}'
317 | -H 'Content-Type: application/json'
318 | -X POST http://localhost:8080/api/upload
319 | ```
320 |
--------------------------------------------------------------------------------
/Lab4/data/courses.txt:
--------------------------------------------------------------------------------
1 | AR03015 中国建筑史 63
2 | AR03024 专业美术I 27
3 | AR03034 建筑阴影透视 61
4 | AR03035 设计基础II 30
5 | AR03037 环境心理学 55
6 | AR03038 专业美术II 32
7 | AR04009 居住区规划 32
8 | AR04010 居住建筑设计原理 175
9 | AR04029 景观建筑学 32
10 | AR04034 公共建筑设计原理 93
11 | AR04035 建筑设计II 35
12 | AR05014 建筑设计（4） 30
13 | AR05023 建筑设计（6） 31
14 | AR05034 城市规划管理与法规 51
15 | AR05035 道路交通规划 51
16 | AR05036 城市规划原理(2) 51
17 | AR05039 控制性详细规划与设计 22
18 | AR05041 城市社会与心理导论 51
19 | AR05042 城乡基础设施规划 51
20 | AR05052 建筑力学 121
21 | AR05053 建筑材料 130
22 | AR05071 城乡信息系统应用 53
23 | AR06010 城市开发与房地产经济 45
24 | AR06012 城市设计 32
25 | AR06015 村镇规划 23
26 | AR06029 建筑安全 62
27 | AR06037 建筑经济 127
28 | AR06049N 建筑物理II（声环境） 127
29 | AR06050N 建筑师业务与建筑法规概论 126
30 | AR06051 建筑施工 127
31 | AR06056 可持续建筑技术B 121
32 | AR06058 历史建筑保护及更新 124
33 | AR06077 中国近现代建筑 127
34 | AR06091 乡村互助自建住宅 124
35 | AR06106 历史建筑保护与更新 45
36 | AR06138 设计艺术史 121
37 | AR10010 建筑构造课程设计 30
38 | AR10016 毕业实习 139
39 | AR10017 毕业设计（论文） 141
40 | BA04020 财务管理学 28
41 | BA04030 运筹学 59
42 | BA04032 组织行为学 63
43 | BA04033 基础会计学 30
44 | BA04041 数据挖掘与商务智能决策 33
45 | BA04042 应用统计学 59
46 | BA04044 计量经济学 63
47 | BA04045 金融工程学 59
48 | BA04048 企业战略管理 28
49 | BA04052 中级财务会计学（下） 27
50 | BA04054 会计学 33
51 | BA04055 管理运筹学 60
52 | BA04056 大数据基础 33
53 | BA04057 数据库原理与应用 63
54 | BA04059 运营管理 28
55 | BA05052S 财务报告（FR） 21
56 | BA05056 公司报告P2 19
57 | BA05081 企业战略管理 89
58 | BA05081E 企业战略管理（E） 24
59 | BA05082E 人力资源管理（E） 24
60 | BA05086 人员测评管理 89
61 | BA05098 物流与供应链管理（S） 21
62 | BA05099 网络营销学 21
63 | BA05100 电子商务运营管理 21
64 | BA05104 网络金融学 21
65 | BA05111S 定价决策（S） 24
66 | BA05114 服务营销（S） 24
67 | BA05117 金融风险管理 59
68 | BA05119 投资学 59
69 | BA05121 金融衍生工具（S） 59
70 | BA05123 成本会计学 60
71 | BA05126 财务分析 15
72 | BA05127 国际会计学（S） 24
73 | BA05128 审计学 56
74 | BA05131 高级财务管理学（S） 15
75 | BA05132 计量经济学 15
76 | BA05144 信息系统开发工具(S) 33
77 | BA05145 信息系统分析与设计(S) 33
78 | BA05146 信息网络工程设计 33
79 | BA05147 企业资源计划 33
80 | BA05218 战略商务领袖SBL（I） 19
81 | BA06251 战略商务领袖SBL（II） 19
82 | BA06355 税务（TX） 21
83 | BA06377 公司法与商法（LW） 21
84 | BA10005 毕业实习 16
85 | BA10006 毕业论文 16
86 | BA10031 毕业论文（设计） 27
87 | BA10032 毕业实习 27
88 | BA10033 导师课程 27
89 | BS04005 有机化学实验B 30
90 | BS04011 生物化学实验(2) 17
91 | BS04022 生物化学(2) 41
92 | BS04023 微生物学 41
93 | BS04027 细胞生物学 41
94 | BS04042 材料科学导论 44
95 | BS04043 生物医学统计 44
96 | BS04045 有机化学B 60
97 | BS05024 酶与发酵工程 47
98 | BS05025 细胞工程 47
99 | BS05026 基因工程 47
100 | BS05030 组织工程学 39
101 | BS05031 生物医学影像与图像处理 39
102 | BS05032 生物医学影像与图像处理实验 14
103 | BS10013 毕业论文 45
104 | BS10016 微生物学实验 17
105 | BS10020 细胞生物学实验 22
106 | CA05005 税收管理学 53
107 | CA05007 税收筹划 53
108 | CA05008 国际税收学 53
109 | CE04003A 理论力学（中文） 61
110 | CE04005A 结构力学I（中文） 49
111 | CE04022 机械设计基础A 24
112 | CE04024 传热学B 70
113 | CE04037 土木工程测量 25
114 | CE04041 流体力学A 24
115 | CE04048 建筑力学(2) 78
116 | CE04049 流体力学B 28
117 | CE04054 工程图学与CAD（2） 30
118 | CE04059 物理化学B 28
119 | CE04061 有机化学B 25
120 | CE04062 工程测量 25
121 | CE04063 工程力学B 88
122 | CE04074 会计学 33
123 | CE04075 房屋建筑学 33
124 | CE05006 建筑给水排水工程 87
125 | CE05007 水工程经济与概预算 87
126 | CE05009 水工艺设备、仪表与控制 87
127 | CE05012 工程造价 35
128 | CE05021 建筑环境学 73
129 | CE05022 空调用制冷技术 73
130 | CE05025 建筑设备自动化 73
131 | CE05026 供热工程 73
132 | CE05036 基础工程 53
133 | CE05070 防灾工程学 53
134 | CE05072 空气调节 73
135 | CE05074 建筑能源供应系统 73
136 | CE05076 工程施工技术 35
137 | CE05078 施工组织学 35
138 | CE05079 工程项目管理 35
139 | CE05083 水质物理化学净化工程 87
140 | CE05084 水质生物净化工程 87
141 | CE05086 水处理实验技术 87
142 | CE05087 弹性力学 53
143 | CE05089 工程作用与结构设计原则 48
144 | CE05102 土木工程概论 89
145 | CE05103A 土木工程材料（中文） 33
146 | CE06077 隧道工程 44
147 | CE06100 交通工程 40
148 | CE06102 道路施工组织与概预算 40
149 | CE06111N 多层及高层建筑结构设计 53
150 | CE06115 路基路面工程 40
151 | CE06119 混凝土及砌体结构房屋设计 53
152 | CE06120 建筑施工技术与管理 53
153 | CE06121 钢及组合结构房屋设计 54
154 | CE06135 桥梁工程（2） 61
155 | CE06136 桥梁施工、维护技术与概预算 61
156 | CE06138 地下结构设计 44
157 | CE06141 地基处理 44
158 | CE06201 钢及组合结构桥梁 61
159 | CE06202 边坡与支挡结构 44
160 | CE06203 建筑工程施工组织与概预算 107
161 | CE10004 水工程经济评价课程设计 87
162 | CE10023 建筑给水排水课程设计 87
163 | CE10029 混凝土结构设计原理课程设计 61
164 | CE10038 钢结构设计原理课程设计 61
165 | CE10050 毕业实习 75
166 | CE10051 毕业设计（论文） 79
167 | CE10071 毕业设计（论文） 28
168 | CE10072 毕业实习 26
169 | CE10073 道路规划设计课程设计 40
170 | CE10075N 建筑学原理课程设计 53
171 | CE10090 工程造价课程设计 35
172 | CE10091 项目管理课程设计 35
173 | CE10092 毕业设计（论文） 66
174 | CE10096 导师课程 79
175 | CH04035 基础分析化学实验(2) 26
176 | CH04037 基础物理化学实验(2) 26
177 | CH04050 基础有机化学实验A 15
178 | CH04051 基础物理化学实验A（1） 25
179 | CH04053 基础分析化学实验A 30
180 | CH04061 分析化学（1） 30
181 | CH04063 物理化学（1） 49
182 | CH04067 物理化学A(2) 51
183 | CH04069 有机化学A（2） 25
184 | CH04071 分析化学B 30
185 | CH04073 化工原理（2） 66
186 | CH04074 化工原理实验（1） 25
187 | CH04076 基础无机化学实验（2） 27
188 | CH04078 基础有机化学实验（2） 26
189 | CH05006 高分子化学 27
190 | CH05018 化学工程基础实验 29
191 | CH05042 化学工程基础 82
192 | CH05049 金属腐蚀及防护理论 29
193 | CH05054 化工设计 44
194 | CH05056 化工分离工程 64
195 | CH05057 化工设备机械基础 15
196 | CH05058 化工过程分析与合成 64
197 | CH05060 化工仪表与自动化 64
198 | CH06050 波谱分析* 28
199 | CH06051 波谱分析实验* 26
200 | CH10013 化学专业实验（中级） 26
201 | CH10015 毕业设计（论文） 37
202 | CH10023 毕业设计（论文） 25
203 | CL03001 写作学概论 85
204 | CL03002 语言学概论 30
205 | CL03004 文学概论 85
206 | CL04002 中国古代文学（2） 73
207 | CL04004 中国古代文学（4） 85
208 | CL04005 古代汉语（1） 73
209 | CL04009 中国现当代文学（2） 73
210 | CL04011 外国文学（2） 85
211 | CL04013 现代汉语（2） 73
212 | CL05034 汉语史 85
213 | CL10005 毕业论文 82
214 | CL10007 实践见习 81
215 | CS04026 网络系统实验 26
216 | CS04030 离散数学 92
217 | CS04031 数据结构与算法 92
218 | CS04033 计算机系统 67
219 | CS04034 操作系统 100
220 | CS05014N 嵌入式计算机系统 47
221 | CS05062 数据库系统 30
222 | CS05063 计算机网络 30
223 | CS05064 数字媒体系统实验 30
224 | CS05065 计算机网络技术 52
225 | CS05067 编译技术 52
226 | CS05074 计算机网络 59
227 | CS05076 数据库原理 60
228 | CS05078 计算机网络系统结构 47
229 | CS05082 计算机网络 66
230 | CS05083 数据库系统 66
231 | CS05084 安全系统实验 66
232 | CS05103 计算机网络 66
233 | CS05104 数据库系统 67
234 | CS10023N 毕业实习 117
235 | CS10024 数据库系统设计 30
236 | CS10026 毕业设计（含导师课程） 119
237 | CS10035 交换机/路由器实验 215
238 | CS10049 电子测试平台与工具2 166
239 | DT03016 影像设计与传播 28
240 | DT03018 三维形体与空间 28
241 | DT03021 智能设计方法 52
242 | DT04014 设计批评 48
243 | DT04026 视觉设计基础 28
244 | DT04027 工业设计模型 28
245 | DT04028 工业设计史 55
246 | DT04066 设计研究方法 25
247 | DT05021 复杂系统设计 51
248 | DT05027 互联技术与服务设计 49
249 | DT05028 人机工程与设计心理学 26
250 | DT10010 毕业实习 109
251 | DT10011 毕业论文（设计） 114
252 | EC04015 模拟电子技术实验 519
253 | EC04017 数字电子技术实验 519
254 | EC04025 传感与检测技术 60
255 | EC04028 导师课程 239
256 | EC04030 模拟电子技术基础 66
257 | EC04031 数字电子技术基础 33
258 | EC05007 过程控制与仪表 67
259 | EC05012 通信电路 102
260 | EC05013 通信原理 102
261 | EC05014 数字信号处理 102
262 | EC05015 测控电路 39
263 | EC05016 电气测量技术 39
264 | EC05019 电力系统基础 31
265 | EC05020 计算机控制技术 67
266 | EC05021 电机与拖动基础 137
267 | EC06108 电子系统创新设计与工程实践(2) 20
268 | EC10006 毕业实习 267
269 | EC10008 微机应用系统综合设计 29
270 | EC10015 毕业设计 91
271 | EI04033 电路 60
272 | EI04038 电磁场与电磁波 31
273 | EI04039 量子力学 63
274 | EI04040 计算机组成与设计 63
275 | EI04041 信息与通信中的数学基础 119
276 | EI04043 信号与系统 63
277 | EI04049 信号与系统 63
278 | EI04057 电路实验 119
279 | EI05005N 通信原理 60
280 | EI05031 电子器件基础 64
281 | EI05034 半导体集成电路 132
282 | EI05037 固体物理 31
283 | EI05041 模拟电子线路 63
284 | EI10001 专业综合实验（1） 132
285 | EI10022 认识实习 60
286 | EI10051 毕业设计（论文） 27
287 | EI10054 科研论文写作 27
288 | EI10056 数字系统综合实验 129
289 | EM04009 机械设计基础B 52
290 | EM04011 材料力学A（2） 52
291 | EM04012 固体力学基础 52
292 | EN04002 金融学 30
293 | EN04005 应用统计学 65
294 | EN04009 财政学 67
295 | EN04011 计量经济学 66
296 | EN04014 博弈论 65
297 | EN04015 计量经济学 59
298 | EN04016 中国经济改革与发展专题 95
299 | EN04019 数理经济学 30
300 | EN04020 现代市场经济运行导论 30
301 | EN04022 中级宏观经济学 30
302 | EN05002 西方经济学流派 56
303 | EN05008 劳动经济学 56
304 | EN05010 产业经济学 56
305 | EN10019 毕业实习 44
306 | EN10020 毕业论文 44
307 | EP04011 机械设计A 45
308 | EP04013 材料力学A 45
309 | EP04015 工程热力学 45
310 | EP05012 内燃机学 46
311 | EP10019 毕业实习 50
312 | EP10020 毕业论文（设计） 51
313 | ES04004 流体力学 73
314 | ES04008 基础化学实验（物化） 27
315 | ES05003 物理化学A 25
316 | ES05009 环境工程微生物 76
317 | ES05011 大气污染控制工程 76
318 | ES05012 水污染控制工程 76
319 | ES10006 环工原理课程设计 76
320 | ES10010 毕业论文（设计） 84
321 | ES10011 毕业实习 84
322 | ET05004 国际金融 99
323 | ET05010 国际结算（双语） 60
324 | ET05011 电子商务 60
325 | ET05012 世界经济概论 50
326 | FI03001 数据库原理及应用 29
327 | FI04005 国际金融学 36
328 | FI04007 计量经济学 74
329 | FI04008X 动态规划与随机控制 44
330 | FI04010 风险管理（保险方向） 26
331 | FI04012 投资学（证券投资） 52
332 | FI04013 管理学 43
333 | FI04014 C++程序设计语言 57
334 | FI05005N 保险精算学（寿险Ⅰ） 30
335 | FI05006N 保险精算学（非寿险） 26
336 | FI05009 中央银行学 73
337 | FI05014 商业银行管理学 77
338 | FI05015 技术经济与项目评估 68
339 | FI05016 外汇交易理论与实务 36
340 | FI05020 运筹学 74
341 | FI05024N 保险法 26
342 | FI05025 博弈论 71
343 | FI05026 金融工程学 68
344 | FI05027 金融计量学 44
345 | FI05030 金融衍生品定价与管理 44
346 | FI05038 人寿与健康保险 30
347 | FI05040 寿险精算I 35
348 | FI05041 非寿险精算 35
349 | FI10025 毕业论文（导师指导课） 31
350 | FI10026 毕业实习 34
351 | FI10041 毕业论文（导师指导课） 9
352 | FL03002 综合英语 II 25
353 | FL03021 英语语音与配音II 25
354 | FL03029 基础日语（2） 28
355 | FL03031 文学理论基础 77
356 | FL03033 英语写作I 25
357 | FL03040 现代汉语 104
358 | FL04023 高级日语(2) 24
359 | FL04024 英语写作III 29
360 | FL04027 英汉语言对比 28
361 | FL04029 英语散文赏析 25
362 | FL04031 英语诗歌赏析 29
363 | FL04035 普通语言学 52
364 | FL04038 基础日语（4） 18
365 | FL04041 日语语言与文化 28
366 | FL04042 日语实用写作 28
367 | FL05057 口译基础 29
368 | FL05058 英美文学史 23
369 | FL05060 跨文化交际(适用于日语专业) 24
370 | FL05062 日汉翻译 63
371 | FL05064 日本文学史 24
372 | FL05067 笔译基础 29
373 | FL05076 古典日语语法 62
374 | FL05078 汉日笔译理论与实践 62
375 | FL05080 日本近现代文学 18
376 | FL05084 日语时文阅读 18
377 | FL10005 毕业论文 91
378 | FL10008 毕业实习 91
379 | FL10025 日语视听集训（2） 62
380 | FL10034 专业导论（2） 104
381 | FL10035 日语专业导论（2） 77
382 | FL10037 创新创业（2）专业导师研讨课 110
383 | FL10045 英语听力集训（2） 29
384 | FL10049 英语经典学期读书报告（2） 104
385 | FL10051 英语经典学期读书报告（4） 110
386 | FL10056 日语经典学期读书报告（2） 77
387 | FL10058 日语经典学期读书报告（4） 62
388 | GE01065 二外（3） 4
389 | GE01090 体育（2） 23
390 | GE01092 体育（4） 12
391 | GE01093 计算机导论A 51
392 | GE01101 毛泽东思想和中国特色社会主义理论体系 62
393 | GE01102 中国近现代史纲要 35
394 | GE01104 马克思主义基本原理（2） 100
395 | GE01112 心理素质与生涯发展(6) 168
396 | GE01115 大学英语A（2） 40
397 | GE01150 毛泽东思想和中国特色社会主义理论体系概论 87
398 | GE01151 思政实践 201
399 | GE01152 思想道德修养与法律基础 102
400 | GE01153 中国近现代史纲要 92
401 | GE01154 马克思主义基本原理 102
402 | GE01156 形势与政策(2) 31
403 | GE01158 形势与政策(4) 85
404 | GE01160 形势与政策(6) 26
405 | GE01163 计算与人工智能概论 58
406 | GE01164 写作与沟通 31
407 | GE01172 习近平新时代中国特色社会主义思想概论 98
408 | GE01174 习近平新时代中国特色社会主义思想概论 63
409 | GE01180 大学英语B2 40
410 | GE02018 土木工程制图 73
411 | GE02042 城市道路与交通 127
412 | GE02059 电工电子学 46
413 | GE03003 线性代数A 119
414 | GE03006 普通物理A（2） 56
415 | GE03007 普通物理实验A（1） 29
416 | GE03010 普通物理实验B 28
417 | GE03013 线性代数B 95
418 | GE03036 数学分析B（2） 58
419 | GE03039 微观经济学 99
420 | GE03040 宏观经济学 61
421 | GE03052 化学原理(2) 31
422 | GE03057 Advanced Calculus II 31
423 | GE03058 University Physics I 31
424 | GE03060 普通物理实验I 22
425 | GE03061 管理学 28
426 | GE03063 管理学 57
427 | GE03071 宏观经济学 61
428 | GE03073 经济学原理 30
429 | GE03116 机器人基础物理（1） 58
430 | GE04001 常微分方程 59
431 | GE09021 电工电子实习A 188
432 | GE09037 电工电子实习B 101
433 | GE09048 军事理论 242
434 | GE09051 机械工程训练 143
435 | GE09053 机电技术创新实训 129
436 | GE09054 工程训练A 220
437 | GE09055 电工电子实训 70
438 | GE09056 工程认知 212
439 | GE09059 机械制造创新实训 45
440 | HC0418 中国文学流变（2） 29
441 | HE10008 六艺（一） 30
442 | HEC0101 《易经》研读 29
443 | HEC0103 《尚书》研读 25
444 | HEC0106 《论语》研读 29
445 | HEC0107 《孟子》研读 25
446 | HEC0201 《史记》研读 29
447 | HEC0205 《通典》研读 30
448 | HEC0222 中国通史(2) 29
449 | HEC0302 《庄子》研读 25
450 | HEC0304 《朱子语类》研读 30
451 | HEC0305 《金刚经》《坛经》研读 30
452 | HEC0319 中国哲学源流（2） 29
453 | HEC0401 《楚辞》研读 25
454 | HEC0403 《文心雕龙》研读 25
455 | HEM0002 中国化马克思主义经典著作选读 29
456 | HEM0006 《毛泽东选集》研读及调研 25
457 | HES0001 自然科学经典选读 30
458 | HEW0001 西方汉学经典选读 29
459 | HEW0003 西方哲学经典选读 25
460 | HEW0005 古希腊哲学经典研读 30
461 | HS03003 历史文选Ⅱ 30
462 | HS03004 历史文献学 33
463 | HS03007 考古学通论 30
464 | HS04006 中华人民共和国史 29
465 | HS04008 世界中古史 30
466 | HS04013 中国古代史（下） 30
467 | HS04015 中国现代史 33
468 | HS04018 世界现代史 33
469 | HS05018 中国史学史 29
470 | HS05019 西方史学史 29
471 | HS05024 中国经学史 29
472 | HS05025 中国政治制度史 29
473 | HS10009 毕业实习 25
474 | HS10012 毕业论文 25
475 | HS10017 导师课程 25
476 | JM03007 媒介经营管理概论 132
477 | JM03013 社会心理学 139
478 | JM03019 政治学原理 139
479 | JM04013 传播学概论 132
480 | JM04023 外国新闻传播史 132
481 | JM04027 新闻采访与写作 67
482 | JM04029 广告策划与创意 132
483 | JM04030 市场调查与分析 71
484 | JM05029 马克思主义新闻思想 87
485 | JM05033 新闻评论 87
486 | JM05035 广告文案写作 47
487 | JM05037 数字广告设计与制作 47
488 | JM05039 品牌传播与管理 47
489 | JM10014 毕业设计（论文） 45
490 | JM10015 毕业实习 45
491 | LA03022 行政法 92
492 | LA03042 刑事诉讼法 180
493 | LA03044 刑法总论 70
494 | LA03046 国际私法 2
495 | LA04006 刑事诉讼法 180
496 | LA04016 经济法学 92
497 | LA04018 婚姻家庭与继承法 184
498 | LA04020 合同法 70
499 | LA04022 商法概论 70
500 | LA05003 国际私法 89
501 | LA05011 环境与资源保护法 89
502 | LA05012 劳动与社会保障法 180
503 | LA06002 公司法 178
504 | LA10006N 论文写作指导 180
505 | LA10009 学年论文 180
506 | LA10010 毕业实习 168
507 | LA10011 毕业论文 171
508 | LA10014 毕业实习（第二学位） 2
509 | MA03005 高等代数（2） 64
510 | MA03010 数学分析（2） 64
511 | MA04002 概率论 56
512 | MA04004 复变函数 56
513 | MA04007 偏微分方程 36
514 | MA05003 实变函数 56
515 | MA05015 数理统计 35
516 | MA05016 数值分析（双语） 37
517 | MA05019 新生导论课 124
518 | MA10007 毕业论文 48
519 | MA10012 学科创新训练 123
520 | MA10019 代数学基础训练(2) 60
521 | MA10021 分析学基础训练（2） 60
522 | ME04002 机械工程图学（2） 64
523 | ME04022 流体力学 62
524 | ME04025 机械设计 66
525 | ME04028 管理学 64
526 | ME04029 经济学 66
527 | ME04031 机械设计基础 64
528 | ME04032 流体力学 25
529 | ME04034 材料力学B 61
530 | ME04035 热工学基础 33
531 | ME04036 机械原理 61
532 | ME04038 经济学 64
533 | ME05002 汽车理论 33
534 | ME05003 汽车设计 67
535 | ME05033 机械制造技术 65
536 | ME05034 机械制造装备 65
537 | ME05037 微机原理 65
538 | ME05039 汽车构造及发动机原理 62
539 | ME05040 汽车电子技术 66
540 | ME05041 汽车制造工艺 34
541 | ME05048 质量管理与可靠性 66
542 | ME05049 管理信息系统 66
543 | ME05050 有限元法 25
544 | ME05051 复合材料力学 25
545 | ME05054 液压与气压传动 61
546 | ME06128 设施规划与物流分析 66
547 | ME06150 人工智能 33
548 | ME06152 先进设计方法 33
549 | ME06154 工业网络原理及应用 33
550 | ME10005 机电控制综合实验 46
551 | ME10009 机械原理课程设计 32
552 | ME10012 计算机辅助生产管理课程设计 33
553 | ME10021 毕业实习 58
554 | ME10031 机械综合实验(1) 65
555 | ME10033 机械设计课程设计 32
556 | ME10037 毕业设计（论文） 242
557 | ME10041 设施规划与物流课程设计 33
558 | ME10044 机械设计基础课程设计 33
559 | ME10048 毕业设计（论文） 58
560 | MS04018 材料物理化学（1） 90
561 | MS04106 材料科学基础 46
562 | MS04202 冶金传输原理 40
563 | MS05104 先进装备材料 73
564 | MS05105 功能材料 46
565 | MS05106 工程材料学 73
566 | MS05202 材料测试技术B 40
567 | MS05204 金属塑性加工原理 40
568 | MS06144 材料模拟计算 75
569 | MS10106 毕业(实习)及论文 157
570 | MS10107 创新实践 197
571 | MS10208 毕业（实习）及论文 44
572 | MS10211 材料学科基础实验(3) 148
573 | MS10215 材料学科金相平台实验（1） 27
574 | PA04002 社会学概论 57
575 | PA04010 心理学导论 57
576 | PA04015 公共行政学说史（双语） 59
577 | PA04016 公共管理统计学应用 59
578 | PA05011 行政组织学 59
579 | PA05013 行政法 50
580 | PA05016 社会保障学 59
581 | PA05018 城市管理学 59
582 | PA05019 社会组织管理 59
583 | PA06001 电子政务 50
584 | PA10015 毕业实习 44
585 | PA10019 毕业论文 44
586 | PH03001 线性代数 31
587 | PH05004 光学 64
588 | PH05020 光电子学基础 59
589 | PH05021 物理学与光电子学前沿讲座 59
590 | PH05022 专业综合实验(1) 59
591 | PH05101 热学 31
592 | PH05102 电磁学 31
593 | PH05104 电路分析 33
594 | PH05107 电动力学 31
595 | PH05108 量子力学 64
596 | PH10011 专业综合课程设计(1) 60
597 | PH10101 创新创业（科学素养与方法1） 64
598 | PR01002 语文(2) 68
599 | PR01004 文科数学(2) 20
600 | PR01006 理科数学（2） 48
601 | PR01008 英语（2） 20
602 | PR01012 普通话（2） 20
603 | PR01014 计算机基础（2） 20
604 | PR05002 应用写作（2） 20
605 | PR05004 中国文化概论（2） 20
606 | PR05006 经典文学作品选读（2） 20
607 | PR05009 基础物理 48
608 | PR05012 习近平新时代中国特色社会主义思想（2） 20
609 | PS04003 行政学说史 50
610 | PS05012 行政领导学原理 50
611 | RO04002 数字电路与系统设计 59
612 | RO04004 "控制原理与控制系统 " 59
613 | RO10004 "机器人工程中级实践 " 59
614 | SC10003 科学研究训练（3） 22
615 | SC10006 科学研究训练（6） 26
616 | ST04006 统计学 9
617 | ST04007 计量经济学A 73
618 | ST04008 数理统计 73
619 | ST04009 统计学概论 30
620 | ST04010 概率论 86
621 | ST05004 随机过程 74
622 | ST05005 国民经济统计学 71
623 | ST05006 时间序列分析 69
624 | ST05007 质量管理统计学 71
625 | ST05010 统计软件 134
626 | TF03020 文化概论 51
627 | TF03021 传播伦理与法规 50
628 | TF03025 中国文学作品选读 61
629 | TF04011 视听语言 31
630 | TF04013 中外广播电视史 54
631 | TF04016 广播电视作品评析 31
632 | TF04018 现代汉语 54
633 | TF04020 新闻学原理 30
634 | TF04021 新闻采访与写作 54
635 | TF04024 新闻传播伦理与法规 139
636 | TF04029 摄影摄像基础 25
637 | TF05019 电视艺术概论 31
638 | TF05031 纪录片创作 30
639 | TF05032 电视艺术片创作 29
640 | TF05060 新闻播音 25
641 | TF05062 文艺作品演播 21
642 | TF05066 电视节目主持（2） 21
643 | TF05070 电视综艺节目制作 29
644 | TF05071 网络编辑 30
645 | TF05090 播音主持语音与发声1、2 30
646 | TF06081 化妆与形象设计 30
647 | TF10001 专业见习 31
648 | TF10026 毕业论文（设计） 30
649 | TF10052 毕业实习 30
650 |
--------------------------------------------------------------------------------
/Lab2/README.md:
--------------------------------------------------------------------------------
1 | # Lab2: Your Own HTTP Server
2 |
3 | *Some materials are from Homework 2 of CS162 2019 at UC Berkeley.* *Thanks to CS162!*
4 |
5 | Enter in the folder you have cloned from our lab git repo, and pull the latest commit - use `git pull`.
6 |
7 | You can find this lab2's instruction in `Lab2/README.md`
8 |
9 | All materials of lab2 are in folder `Lab2/`.
10 |
11 | # 1. Overview
12 |
13 | Implement an HTTP server based on HTTP/1.1 from scratch by your own, using network programming knowledges learned from our class.
14 |
15 | Also, try to use high concurrency programming skills learned from the class to guarantee the web server's performance.
16 |
17 | **Our Goals:**
18 |
19 | * Practice basic network programming skills, such as using socket API, parsing packets;
20 | * Get familiar with robust and high-performance concurrent programming.
21 |
22 | # 2. Backround
23 |
24 | **Please check [background.md](./background.md) first to learn some basics about `HTTP`, `HTTP messages` , `HTTP proxy`, `JSON` & `curl`.**
25 |
26 | # 3. Your Lab Task
27 |
28 | ## Implement your own HTTP Server
29 |
30 | In this Lab, we won't provide any basic code. You should implement an HTTP server based on HTTP/1.1, from scratch which satisfies the following requirements:
31 |
32 | **HTTP Server Outline**
33 |
34 | From a network standpoint, your HTTP server should implement the following:
35 |
36 | 1. Create a listening socket and bind it to a port;
37 | 1. Wait a client to connect to the port;
38 | 1. Accept the client and obtain a new connection socket;
39 | 1. Read in and parse the HTTP request;
40 | 1. Start delivering services (*some are optional*):
41 | * Handle HTTP GET/POST requests and return responses.
42 | * Proxy the request to another HTTP server.
43 |
44 | The server will be in either non-proxy mode or proxy mode (we have introduced the proxy in background section `2.3`). It does not do both things at the same time.
45 |
46 | For better testing and scoring, we have made **some functional requirements** for your submitted works.
47 |
48 | **ATTENTION**: Lab 2 is a pre-lab of Lab 4. If you plan to complete Lab 4, please complete **the advanced version** of Lab 2.
49 |
50 | ### 3.1 Handle HTTP request & send HTTP response
51 |
52 | In this Lab, **you just need to implement the GET method and the POST method in your HTTP server**.
53 |
54 | For any other methods, your server should send a response with 501 status code (see `2.2`).
55 | That is to say, if your HTTP server receive an HTTP request but the request method is neither GET nor POST, the HTTP server just need to return a 501 Not Implemented error message (a response message with Response line having status code to be 501, see `2.2`).
56 |
57 | See examples in section [3.7](#37-access-your-http-server).
58 |
59 | #### 3.1.1 Handle HTTP GET request
60 |
61 | The HTTP server should be able to handle HTTP GET requests for specific resources, e.g., web service data & static files.
62 |
63 | For a GET request, the server needs to check whether the path of request corresponds to a web service or an existed static file.
64 |
65 | What you need to do:
66 |
67 | **basic version**:
68 |
69 | * A search service: use a specific url to get some data from the server that does not come from a file stored on the file system.
70 | * The url don't have query string, e.g. `http://localhost:8080/api/check`;
71 | * Send response with plain text content.
72 |
73 | * Static files access: use a specific url to access a text file stored on the file system.
74 | * Clients only request `*.html`;
75 | * Clients may request a file in subdirectories, e.g. `http://localhost:8080/test/index.html`;
76 | * Send response with the full content of html file.
77 |
78 | * If the path is invalid or the static file does not exist
79 | * Send response with the full content of `404 Not Found` page.
80 |
81 | **advanced version**:
82 |
83 | * A search service: use a specific url to get some data from the server that does not come from a file stored on the file system.
84 | * The url may have query string, e.g. `http://localhost:8080/search?id=1&name=foo`;
85 | * The query string contains some key-value pairs. Keys are `id` & `name`;
86 | * If the query string is valid, send response with json type content.
87 | * Or if invalid, send response with json type content, including error messages;
88 |
89 | * Static files access: use a specific url to access a text file stored on the file system.
90 | * Clients may request `*.html`, `*.js`, `*.css`, `*.json` and other plain text files;
91 | * Clients may request a file in subdirectories, e.g. `http://localhost:8080/test/index.html`;
92 | * Send Response with the correct `Content-Type` & the full content of the file.
93 |
94 | * If the path is invalid or the static file does not exist
95 | * Send response with the full content of `404 Not Found` page.
96 |
97 | *You don't need to implement to transmit base64 encoded binary files, such as image files like `*.png`.*
98 |
99 | #### 3.1.2 Handle HTTP POST request
100 |
101 | The HTTP server should be able to handle HTTP POST requests.
102 |
103 | For a POST request, the server needs to check whether the path of request corresponds to a web service.
104 |
105 | > For special characters, like `%`, you may need google about **percent encoding**.
106 |
107 | What you need to do:
108 |
109 | **basic version**:
110 |
111 | * A upload service: use a specific url to upload some data to the server
112 | * The content type is `application/x-www-form-urlencoded`;
113 | * The payload contains key-value pairs, keys are `id` & `name`;
114 | * If the payload is valid, send response with `200 OK`, `Content-Type: text/plain` & data;
115 | * Or if the payload invalid, send response with send response with `404 Not Found`, `Content-Type: text/plain` & error messages;
116 |
117 | * If the path is invalid
118 | * Send response with the full content of `404 Not Found` page.
119 |
120 | **advanced version**:
121 |
122 | * A upload service: use a specific url to upload some data from the server that does not come from a file stored on the file system.
123 | * The content type is `application/x-www-form-urlencoded`, `application/json`;
124 | * The payload contains key-value pairs, keys are `id` & `name`;
125 | * If the payload is valid, send response with `200 OK`, `Content-Type: application/json` & data;
126 | * Or if the payload invalid, send response with send response with `404 Not Found`, `Content-Type: application/json` & error messages;
127 |
128 | * If the path is invalid
129 | * Send response with the full content of `404 Not Found` page.
130 |
131 | If you are not familiered with `application/x-www-form-urlencoded`, check [MDN docs](https://developer.mozilla.org/en-US/docs/Learn/Forms/Sending_and_retrieving_form_data).
132 |
133 | #### 3.1.3 Other request
134 |
135 | Just return 501 Not Implemented page for other request method (e.g. DELETE, PUT, etc.).
136 |
137 | ### 3.2 Implement a proxy server (optional for advanced version)
138 |
139 | Enable your server to proxy HTTP requests to another HTTP server and forward the responses to the clients.
140 |
141 | 1. You should use the value of the `--proxy` command line argument, which contains the address and port number of the upstream HTTP server.
142 |
143 | 2. Your proxy server should wait for new data on both sockets (the HTTP client file descriptor, and the upstream HTTP server file descriptor). When data arrives, you should immediately read it to a buffer and then write it to the other socket. You are essentially maintaining 2-way communication between the HTTP client and the upstream HTTP server. Note that your proxy must support multiple requests/responses.
144 |
145 | 3. If either of the sockets closes, communication cannot continue, so you should close the other socket and exit the child process.
146 |
147 | Hints: 1) This is more tricky than writing to a file or reading from stdin, since you do not know which side of the 2-way stream will write data first, or whether they will write more data after receiving a response. 2) You should again use threads for this task. For example, consider using two threads to facilitate the two-way communication, one from A to B and the other from B to A.
148 |
149 | > Most websites now use HTTPS and will check your HTTP header lines, your proxy server may not be suitable for any web server.
150 | >
151 | > It is not our task to implement protocols other than HTTP.
152 | >
153 | > They often involve encryption algorithms and are difficult to implement directly using socket api.
154 | >
155 | > However, if you are interested in protocols such as HTTPS, you can try to use other libraries, e.g. openssl.
156 |
157 | ### 3.3 Use multi-thread to increase concurrency
158 |
159 | Your HTTP server should use multiple threads to handle as many concurrent clients' requests as possible. You have at least the following three options to architect your multi-thread server:
160 |
161 | - **On-demand threads**. You can can create a new thread whenever a new client comes in, and use that thread to handle all that clients' task, including parsing the HTTP request, fetching page files, and sending response. The thread can be destroyed after that client finishes, e.g, detect through TCP `recv()`. However,it may not be easy to detect client finish in the HTTP layer.
162 |
163 | - **A pool of always-on threads**. You can use a fixed-sized thread pool in your HTTP server program for handling multiple client requests concurrently. If there are no tasks, those threads are in a waiting state. If a new client comes in, assign a thread to handle the client's request and send response to it. If the assigned thread is busy, you can use a work queue to buffer the request, and let the thread process it later.
164 |
165 | - **Combined**. Combine above two styles together. For example, you can use thread pool to receive request and send response, and use on-demand threads to fetch large page files.
166 |
167 | Feel free to choose any one from the above three, or use other multi-thread architecture that you think is cool.
168 |
169 | ### 3.4 Support HTTP pipelining
170 |
171 | In the basic version, you have **only one request per TCP connection going on at the same time**. The client waits for response, and when it gets response, perhaps reuses the TCP connection for a new request (or use a new TCP connection). This is also what normal HTTP server supports.
172 |
173 | In the advanced version, **multiple http requests can be fired concurrently on one TCP connection**. This is also called HTTP pipelining which is supported by many real browsers and servers (such as Chrome). Note that HTTP requests that come from the same TCP connection should be responded in the same order. So take care the order problem when using complex multi-thread styles.
174 |
175 | ### 3.5 Specify arguments
176 |
177 | Your program should enable long options to accept arguments and specify those arguments during start.
178 |
179 | They are:
180 |
181 | | arguments | descriptions |
182 | | --- | --- |
183 | | -i, --ip \ | Specify the server IP address. |
184 | | -p, --port \ | Selects which port the HTTP server listens on for incoming connections. |
185 | | --proxy \ | Selects an "upstream" HTTP server to proxy. |
186 | | -t, --threads \ | Number of threads if you use multi-thread. |
187 |
188 | The `--proxy` can have a schema before `://` and a port number after a colon (e.g. `http://www.example.com:80`). If a port number is not specified, port 80 is the default for HTTP.
189 |
190 | If you have no idea about *long options*, you can read [this](https://www.gnu.org/software/libc/manual/html_node/Argument-Syntax.html#Argument-Syntax). And you may need to use some functions like `getopt_long()`, `getopt_long_only()`, `getopt()` and so on. Check those function's usage with the `man` command.
191 |
192 | ### 3.6 Run your HTTP Server
193 |
194 | **For advanced version**:
195 |
196 | Your program should be able to start at terminal. If your program is called *http-server*, just typing:
197 |
198 | in the non-proxy mode:
199 |
200 | `./http-server --ip 127.0.0.1 --port 8888 --threads 8`
201 |
202 | It means that your HTTP server's IP address is 127.0.0.1 and service port is 8888. The --number-thread indicates that there are 8 threads in the thread pool for handling multiple client request concurrently.
203 |
204 | in the proxy mode:
205 |
206 | `./http-server --ip 127.0.0.1 --port 8888 --threads 8 --proxy http://www.example.com:80`
207 |
208 | It means that this is an HTTP proxy server. This proxy's IP address is 127.0.0.1 and service port is 8888. And the proxy has a thread pool with 8 threads. The --proxy indicates that the "upstream" HTTP server is `http://www.example.com:80`. So, if you send a request message to this proxy (i.e. `127.0.0.1:8888`), it will forward this request message to the "upstream" HTTP server (i.e. `http://www.example.com:80`) and forward the response message to the client.
209 |
210 | > If you want to access this server from other host, maybe you should open the firewall corresponding to the port and bind the ip to 0.0.0.0.
211 |
212 | When you run the command above, your HTTP server should run correctly.
213 |
214 | ### 3.7 Access Your HTTP Server
215 |
216 | We assume that the ip the server bound is `127.0.0.1` and the port is `8080`. If the proxy feature is used, the remote server is `www.example.com`.
217 |
218 | For more efficient testing and scoring, we need you to implement some of the interfaces described below, and make sure that the content of the response is consistent with our expectations.
219 |
220 | **That's why we provide some standard static files. You can check those files in `./static/` & `./data/`.**
221 |
222 | Please make sure your server supports accessing them, and **do not modify their content or relative paths**.
223 |
224 | #### 3.7.1 Using GET method
225 |
226 | **1) access static files**
227 |
228 | For **basic** version:
229 |
230 | | path | files in local fs | status code | content type |
231 | | --- | --- | --- | --- |
232 | | /, /index.html | /`{static dir}`/index.html | 200 | text/html |
233 | | /404.html | /`{static dir}`/404.html | 404 | text/html |
234 | | /501.html | /`{static dir}`/501.html | 501 | text/html |
235 | | any other error paths | /`{static dir}`/404.html | 404 |text/html |
236 |
237 | E.g.:
238 |
239 | ```shell
240 | user@linux:~/http-server$ curl -i -X GET http://localhost:8080/index.html
241 | HTTP/1.1 200 OK
242 | Content-Type: text/html
243 | Content-Length: 210
244 |
245 |
246 |
247 |
248 |
249 |
250 | Http Server
251 |
252 |
253 | index.html
254 |
255 |
256 | ```
257 |
258 | For **advanced** version:
259 |
260 | | path | files in local fs | status code | content type |
261 | | --- | --- | --- | --- |
262 | | /, /index.html | /`{static dir}`/index.html | 200 | text/html |
263 | | /404.html | /`{static dir}`/404.html | 404 | text/html |
264 | | /501.html | /`{static dir}`/501.html | 501 | text/html |
265 | | [/\*]/\*.html | /`{static dir}`[/\*]/\*.html | 200 | text/html |
266 | | [/\*]/\*.js | /`{static dir}`[/\*]/\*.js | 200 | text/javascript |
267 | | [/\*]/\*.css | /`{static dir}`[/\*]/\*.css | 200 | text/css |
268 | | [/\*]/\*.json | /`{static dir}`[/\*]/\*.json | 200 | application/json |
269 | | any other error paths | /`{static dir}`/404.html | 404 | text/html |
270 |
271 | ```
272 | user@linux:~/http-server$ curl -i -X GET http://localhost:8080/data.json
273 | HTTP/1.1 200 OK
274 | Content-Type: application/json
275 | Content-Length: 93
276 |
277 | [{"id":1,"name":"Foo"},{"id":2,"name":"Bar"},{"id":3,"name":"Foo Bar"},{"id":4,"name":"Foo"}]
278 | ```
279 |
280 | > For standardized testing, output JSON data in one line without unnecessary spaces.
281 |
282 | **2) access web services for getting data**
283 |
284 | For **basic** version:
285 |
286 | | path | get data | status code | content type |
287 | | --- | --- | --- | --- |
288 | | /api/check | strings in `data.txt` | 200 | text/plain |
289 | | any other error paths | /{static files}/404.html | 404 | text/html |
290 |
291 | E.g.:
292 |
293 | ```shell
294 | user@linux:~/http-server$ curl -i -X GET http://localhost:8080/api/check
295 | HTTP/1.1 200 OK
296 | Content-Type: text/plain
297 | Content-Length: 13
298 |
299 | id=1&name=Foo
300 | ```
301 |
302 | For **advanced** version:
303 |
304 | | path | get data | status code | content type |
305 | | --- | --- | --- | --- |
306 | | /api/list | all objects in `data.json` | 200 | application/json |
307 | | /api/search?[id=`value1`&name=`value2`] | all objects that match
`*.id == value1 && *.name == value2`
in `/data/data.json` | 200 | application/json |
308 | | /api/search?[id=`value1`&name=`value2`] | if no object matches, return all objects
in `/data/not_found.json` | 404 |application/json |
309 | | any other error paths | /{static files}/404.html | 404 | text/html |
310 |
311 | > The query string format is a regular expression: `id=[0-9]+&name=[a-zA-Z0-9]+`.
312 |
313 | > If you do not want to parse json file, you could store the data in memory instead of reading file every time.
314 |
315 | E.g.:
316 |
317 | ```shell
318 | user@linux:~/http-server$ curl -i -G -d 'id=1&name=Foo' \
319 | > -X GET http://localhost:8080/api/search
320 | HTTP/1.1 200 OK
321 | Content-Type: application/json
322 | Content-Length: 23
323 |
324 | [{"id":1,"name":"Foo"}]
325 | ```
326 |
327 | > For standardized testing, output JSON data in one line without unnecessary spaces.
328 |
329 | #### 3.7.2 Using POST method
330 |
331 | **access web services for uploading data**
332 |
333 | All data you need to upload are two key/value pairs: `id: {value1}` & `name: {value2}`.
334 |
335 | > For standardized testing, output in the order of `id`, `name`.
336 |
337 | Your server should check the data. In default case, the server should echo the data from the client.
338 |
339 | If the data format is error, e.g., `id=1&name`, `id=1` or `id=1&foo=a`, send content error message.
340 |
341 | For **basic** version:
342 |
343 | The client should send requests with `Content-Type: application/x-www-form-urlencoded`.
344 |
345 | > The echo data format is a regular expression: `id=[0-9]+&name=[a-zA-Z0-9]+`.
346 |
347 | After handling, send response:
348 |
349 | | path | content | status code | content type |
350 | | --- | --- | --- | --- |
351 | | /api/echo | echo the data received | 200 | application/x-www-form-urlencoded |
352 | | /api/echo | if the data format is error, send strings
in `/data/error.txt` | 404 | text/plain |
353 | | any other error paths | /{static files}/404.html | 404 | text/html |
354 |
355 | E.g.:
356 |
357 | ```shell
358 | user@linux:~/http-server$ curl -i -d 'id=1&name=Foo' \
359 | > -H 'Content-Type: application/x-www-form-urlencoded' \
360 | > -X POST http://localhost:8080/api/echo
361 | HTTP/1.1 200 OK
362 | Content-Type: application/x-www-form-urlencoded
363 | Content-Length: 13
364 |
365 | id=1&name=Foo
366 | ```
367 |
368 | For **advanced** version:
369 |
370 | The client should send requests with `Content-Type`:
371 | * `application/x-www-form-urlencoded`,
372 | * `application/json`.
373 |
374 | After handling, send response:
375 |
376 | | path | content | status code | content type |
377 | | --- | --- | --- | --- |
378 | | /api/upload | echo the data received | 200 | application/json |
379 | | /api/upload | if the data format is error, send strings
in `/data/error.json` | 404 | application/json |
380 | | any other error paths | /{static files}/404.html | 404 | text/html |
381 |
382 | E.g.:
383 |
384 | ```shell
385 | user@linux:~/http-server$ curl -i -d '{"id":1,"name":"Foo"}' \
386 | > -H 'Content-Type: application/json' \
387 | > -X POST http://localhost:8080/api/upload
388 | HTTP/1.1 200 OK
389 | Content-Type: application/json
390 | Content-Length: 21
391 |
392 | {"id":1,"name":"Foo"}
393 | ```
394 |
395 | > For standardized testing, output JSON data in one line without unnecessary spaces.
396 |
397 | #### 3.7.3 Other method
398 |
399 | The HTTP server will not handle HTTP requests except GET requests and POST requests.
400 |
401 | If you send an HTTP request with `DELETE` (or `PUT`, `HEAD`, etc.) to delete the specified resource, your server should send `/{static dir}/501.html`:
402 |
403 | ### 3.8 Implementation requirements
404 |
405 | **Version**
406 |
407 | * **Basic version**
408 |
409 | Complete all the tasks of **the basic version** described in section `3.1.1~3.1.7` except `3.1.3` & `3.1.4`.
410 |
411 | * **Advanced version**
412 |
413 | Complete all the tasks of **the advanced version** described in section `3.1.1~3.1.7` **including `3.1.3` & `3.1.4`**.
414 |
415 | # 4. Tester
416 |
417 | We provide some tools for testing.
418 |
419 | You can check the repos of the [tester](https://github.com/LabCloudComputing/http-server-tester).
420 |
421 | Of course you can also use your own browser and other web tools like curl.
422 |
423 | # 5. Lab submission
424 |
425 | Please put all your code in folder `Lab2` and write a `Makefile` so that we **can compile your code in one single command** `make`. The compiled runnable executable binary should be named `http-server` and located in folder `Lab2`. Please carefully following above rules so that TAs can automatically test your code!!!
426 |
427 | Please submit your lab program following the guidance in the [Overall Lab Instructions](../README.md) (`../README.md`)
428 |
429 | # 6. Grading standards
430 |
431 | * You can get 28 points if you can:
432 | * finish all the requirements of the basic version
433 |
434 | * You can get 30 points (full score) if you can:
435 | * finish all the requirements of the advanced version
436 |
437 | If you missed some parts, you will get part of the points depending how much you finished.
438 |
--------------------------------------------------------------------------------
/Lab4/data/students.txt:
--------------------------------------------------------------------------------
1 | 202208010101 学生1
2 | 202208010102 学生2
3 | 202208010103 学生3
4 | 202208010104 学生4
5 | 202208010105 学生5
6 | 202208010106 学生6
7 | 202208010107 学生7
8 | 202208010108 学生8
9 | 202208010109 学生9
10 | 202208010110 学生10
11 | 202208010111 学生11
12 | 202208010112 学生12
13 | 202208010113 学生13
14 | 202208010114 学生14
15 | 202208010115 学生15
16 | 202208010116 学生16
17 | 202208010117 学生17
18 | 202208010118 学生18
19 | 202208010119 学生19
20 | 202208010120 学生20
21 | 202208010121 学生21
22 | 202208010122 学生22
23 | 202208010123 学生23
24 | 202208010124 学生24
25 | 202208010125 学生25
26 | 202208010126 学生26
27 | 202208010127 学生27
28 | 202208010128 学生28
29 | 202208010129 学生29
30 | 202208010130 学生30
31 | 202208010131 学生31
32 | 202208010132 学生32
33 | 202208010133 学生33
34 | 202208010134 学生34
35 | 202208010135 学生35
36 | 202208010136 学生36
37 | 202208010137 学生37
38 | 202208010138 学生38
39 | 202208010139 学生39
40 | 202208010140 学生40
41 | 202208010141 学生41
42 | 202208010142 学生42
43 | 202208010143 学生43
44 | 202208010144 学生44
45 | 202208010145 学生45
46 | 202208010146 学生46
47 | 202208010147 学生47
48 | 202208010148 学生48
49 | 202208010149 学生49
50 | 202208010150 学生50
51 | 202208010201 学生51
52 | 202208010202 学生52
53 | 202208010203 学生53
54 | 202208010204 学生54
55 | 202208010205 学生55
56 | 202208010206 学生56
57 | 202208010207 学生57
58 | 202208010208 学生58
59 | 202208010209 学生59
60 | 202208010210 学生60
61 | 202208010211 学生61
62 | 202208010212 学生62
63 | 202208010213 学生63
64 | 202208010214 学生64
65 | 202208010215 学生65
66 | 202208010216 学生66
67 | 202208010217 学生67
68 | 202208010218 学生68
69 | 202208010219 学生69
70 | 202208010220 学生70
71 | 202208010221 学生71
72 | 202208010222 学生72
73 | 202208010223 学生73
74 | 202208010224 学生74
75 | 202208010225 学生75
76 | 202208010226 学生76
77 | 202208010227 学生77
78 | 202208010228 学生78
79 | 202208010229 学生79
80 | 202208010230 学生80
81 | 202208010231 学生81
82 | 202208010232 学生82
83 | 202208010233 学生83
84 | 202208010234 学生84
85 | 202208010235 学生85
86 | 202208010236 学生86
87 | 202208010237 学生87
88 | 202208010238 学生88
89 | 202208010239 学生89
90 | 202208010240 学生90
91 | 202208010241 学生91
92 | 202208010242 学生92
93 | 202208010243 学生93
94 | 202208010244 学生94
95 | 202208010245 学生95
96 | 202208010246 学生96
97 | 202208010247 学生97
98 | 202208010248 学生98
99 | 202208010249 学生99
100 | 202208010250 学生100
101 | 202208010301 学生101
102 | 202208010302 学生102
103 | 202208010303 学生103
104 | 202208010304 学生104
105 | 202208010305 学生105
106 | 202208010306 学生106
107 | 202208010307 学生107
108 | 202208010308 学生108
109 | 202208010309 学生109
110 | 202208010310 学生110
111 | 202208010311 学生111
112 | 202208010312 学生112
113 | 202208010313 学生113
114 | 202208010314 学生114
115 | 202208010315 学生115
116 | 202208010316 学生116
117 | 202208010317 学生117
118 | 202208010318 学生118
119 | 202208010319 学生119
120 | 202208010320 学生120
121 | 202208010321 学生121
122 | 202208010322 学生122
123 | 202208010323 学生123
124 | 202208010324 学生124
125 | 202208010325 学生125
126 | 202208010326 学生126
127 | 202208010327 学生127
128 | 202208010328 学生128
129 | 202208010329 学生129
130 | 202208010330 学生130
131 | 202208010331 学生131
132 | 202208010332 学生132
133 | 202208010333 学生133
134 | 202208010334 学生134
135 | 202208010335 学生135
136 | 202208010336 学生136
137 | 202208010337 学生137
138 | 202208010338 学生138
139 | 202208010339 学生139
140 | 202208010340 学生140
141 | 202208010341 学生141
142 | 202208010342 学生142
143 | 202208010343 学生143
144 | 202208010344 学生144
145 | 202208010345 学生145
146 | 202208010346 学生146
147 | 202208010347 学生147
148 | 202208010348 学生148
149 | 202208010349 学生149
150 | 202208010350 学生150
151 | 202208010401 学生151
152 | 202208010402 学生152
153 | 202208010403 学生153
154 | 202208010404 学生154
155 | 202208010405 学生155
156 | 202208010406 学生156
157 | 202208010407 学生157
158 | 202208010408 学生158
159 | 202208010409 学生159
160 | 202208010410 学生160
161 | 202208010411 学生161
162 | 202208010412 学生162
163 | 202208010413 学生163
164 | 202208010414 学生164
165 | 202208010415 学生165
166 | 202208010416 学生166
167 | 202208010417 学生167
168 | 202208010418 学生168
169 | 202208010419 学生169
170 | 202208010420 学生170
171 | 202208010421 学生171
172 | 202208010422 学生172
173 | 202208010423 学生173
174 | 202208010424 学生174
175 | 202208010425 学生175
176 | 202208010426 学生176
177 | 202208010427 学生177
178 | 202208010428 学生178
179 | 202208010429 学生179
180 | 202208010430 学生180
181 | 202208010431 学生181
182 | 202208010432 学生182
183 | 202208010433 学生183
184 | 202208010434 学生184
185 | 202208010435 学生185
186 | 202208010436 学生186
187 | 202208010437 学生187
188 | 202208010438 学生188
189 | 202208010439 学生189
190 | 202208010440 学生190
191 | 202208010441 学生191
192 | 202208010442 学生192
193 | 202208010443 学生193
194 | 202208010444 学生194
195 | 202208010445 学生195
196 | 202208010446 学生196
197 | 202208010447 学生197
198 | 202208010448 学生198
199 | 202208010449 学生199
200 | 202208010450 学生200
201 | 202208010501 学生201
202 | 202208010502 学生202
203 | 202208010503 学生203
204 | 202208010504 学生204
205 | 202208010505 学生205
206 | 202208010506 学生206
207 | 202208010507 学生207
208 | 202208010508 学生208
209 | 202208010509 学生209
210 | 202208010510 学生210
211 | 202208010511 学生211
212 | 202208010512 学生212
213 | 202208010513 学生213
214 | 202208010514 学生214
215 | 202208010515 学生215
216 | 202208010516 学生216
217 | 202208010517 学生217
218 | 202208010518 学生218
219 | 202208010519 学生219
220 | 202208010520 学生220
221 | 202208010521 学生221
222 | 202208010522 学生222
223 | 202208010523 学生223
224 | 202208010524 学生224
225 | 202208010525 学生225
226 | 202208010526 学生226
227 | 202208010527 学生227
228 | 202208010528 学生228
229 | 202208010529 学生229
230 | 202208010530 学生230
231 | 202208010531 学生231
232 | 202208010532 学生232
233 | 202208010533 学生233
234 | 202208010534 学生234
235 | 202208010535 学生235
236 | 202208010536 学生236
237 | 202208010537 学生237
238 | 202208010538 学生238
239 | 202208010539 学生239
240 | 202208010540 学生240
241 | 202208010541 学生241
242 | 202208010542 学生242
243 | 202208010543 学生243
244 | 202208010544 学生244
245 | 202208010545 学生245
246 | 202208010546 学生246
247 | 202208010547 学生247
248 | 202208010548 学生248
249 | 202208010549 学生249
250 | 202208010550 学生250
251 | 202208010601 学生251
252 | 202208010602 学生252
253 | 202208010603 学生253
254 | 202208010604 学生254
255 | 202208010605 学生255
256 | 202208010606 学生256
257 | 202208010607 学生257
258 | 202208010608 学生258
259 | 202208010609 学生259
260 | 202208010610 学生260
261 | 202208010611 学生261
262 | 202208010612 学生262
263 | 202208010613 学生263
264 | 202208010614 学生264
265 | 202208010615 学生265
266 | 202208010616 学生266
267 | 202208010617 学生267
268 | 202208010618 学生268
269 | 202208010619 学生269
270 | 202208010620 学生270
271 | 202208010621 学生271
272 | 202208010622 学生272
273 | 202208010623 学生273
274 | 202208010624 学生274
275 | 202208010625 学生275
276 | 202208010626 学生276
277 | 202208010627 学生277
278 | 202208010628 学生278
279 | 202208010629 学生279
280 | 202208010630 学生280
281 | 202208010631 学生281
282 | 202208010632 学生282
283 | 202208010633 学生283
284 | 202208010634 学生284
285 | 202208010635 学生285
286 | 202208010636 学生286
287 | 202208010637 学生287
288 | 202208010638 学生288
289 | 202208010639 学生289
290 | 202208010640 学生290
291 | 202208010641 学生291
292 | 202208010642 学生292
293 | 202208010643 学生293
294 | 202208010644 学生294
295 | 202208010645 学生295
296 | 202208010646 学生296
297 | 202208010647 学生297
298 | 202208010648 学生298
299 | 202208010649 学生299
300 | 202208010650 学生300
301 | 202208010701 学生301
302 | 202208010702 学生302
303 | 202208010703 学生303
304 | 202208010704 学生304
305 | 202208010705 学生305
306 | 202208010706 学生306
307 | 202208010707 学生307
308 | 202208010708 学生308
309 | 202208010709 学生309
310 | 202208010710 学生310
311 | 202208010711 学生311
312 | 202208010712 学生312
313 | 202208010713 学生313
314 | 202208010714 学生314
315 | 202208010715 学生315
316 | 202208010716 学生316
317 | 202208010717 学生317
318 | 202208010718 学生318
319 | 202208010719 学生319
320 | 202208010720 学生320
321 | 202208010721 学生321
322 | 202208010722 学生322
323 | 202208010723 学生323
324 | 202208010724 学生324
325 | 202208010725 学生325
326 | 202208010726 学生326
327 | 202208010727 学生327
328 | 202208010728 学生328
329 | 202208010729 学生329
330 | 202208010730 学生330
331 | 202208010731 学生331
332 | 202208010732 学生332
333 | 202208010733 学生333
334 | 202208010734 学生334
335 | 202208010735 学生335
336 | 202208010736 学生336
337 | 202208010737 学生337
338 | 202208010738 学生338
339 | 202208010739 学生339
340 | 202208010740 学生340
341 | 202208010741 学生341
342 | 202208010742 学生342
343 | 202208010743 学生343
344 | 202208010744 学生344
345 | 202208010745 学生345
346 | 202208010746 学生346
347 | 202208010747 学生347
348 | 202208010748 学生348
349 | 202208010749 学生349
350 | 202208010750 学生350
351 | 202208010801 学生351
352 | 202208010802 学生352
353 | 202208010803 学生353
354 | 202208010804 学生354
355 | 202208010805 学生355
356 | 202208010806 学生356
357 | 202208010807 学生357
358 | 202208010808 学生358
359 | 202208010809 学生359
360 | 202208010810 学生360
361 | 202208010811 学生361
362 | 202208010812 学生362
363 | 202208010813 学生363
364 | 202208010814 学生364
365 | 202208010815 学生365
366 | 202208010816 学生366
367 | 202208010817 学生367
368 | 202208010818 学生368
369 | 202208010819 学生369
370 | 202208010820 学生370
371 | 202208010821 学生371
372 | 202208010822 学生372
373 | 202208010823 学生373
374 | 202208010824 学生374
375 | 202208010825 学生375
376 | 202208010826 学生376
377 | 202208010827 学生377
378 | 202208010828 学生378
379 | 202208010829 学生379
380 | 202208010830 学生380
381 | 202208010831 学生381
382 | 202208010832 学生382
383 | 202208010833 学生383
384 | 202208010834 学生384
385 | 202208010835 学生385
386 | 202208010836 学生386
387 | 202208010837 学生387
388 | 202208010838 学生388
389 | 202208010839 学生389
390 | 202208010840 学生390
391 | 202208010841 学生391
392 | 202208010842 学生392
393 | 202208010843 学生393
394 | 202208010844 学生394
395 | 202208010845 学生395
396 | 202208010846 学生396
397 | 202208010847 学生397
398 | 202208010848 学生398
399 | 202208010849 学生399
400 | 202208010850 学生400
401 | 202208010901 学生401
402 | 202208010902 学生402
403 | 202208010903 学生403
404 | 202208010904 学生404
405 | 202208010905 学生405
406 | 202208010906 学生406
407 | 202208010907 学生407
408 | 202208010908 学生408
409 | 202208010909 学生409
410 | 202208010910 学生410
411 | 202208010911 学生411
412 | 202208010912 学生412
413 | 202208010913 学生413
414 | 202208010914 学生414
415 | 202208010915 学生415
416 | 202208010916 学生416
417 | 202208010917 学生417
418 | 202208010918 学生418
419 | 202208010919 学生419
420 | 202208010920 学生420
421 | 202208010921 学生421
422 | 202208010922 学生422
423 | 202208010923 学生423
424 | 202208010924 学生424
425 | 202208010925 学生425
426 | 202208010926 学生426
427 | 202208010927 学生427
428 | 202208010928 学生428
429 | 202208010929 学生429
430 | 202208010930 学生430
431 | 202208010931 学生431
432 | 202208010932 学生432
433 | 202208010933 学生433
434 | 202208010934 学生434
435 | 202208010935 学生435
436 | 202208010936 学生436
437 | 202208010937 学生437
438 | 202208010938 学生438
439 | 202208010939 学生439
440 | 202208010940 学生440
441 | 202208010941 学生441
442 | 202208010942 学生442
443 | 202208010943 学生443
444 | 202208010944 学生444
445 | 202208010945 学生445
446 | 202208010946 学生446
447 | 202208010947 学生447
448 | 202208010948 学生448
449 | 202208010949 学生449
450 | 202208010950 学生450
451 | 202208011001 学生451
452 | 202208011002 学生452
453 | 202208011003 学生453
454 | 202208011004 学生454
455 | 202208011005 学生455
456 | 202208011006 学生456
457 | 202208011007 学生457
458 | 202208011008 学生458
459 | 202208011009 学生459
460 | 202208011010 学生460
461 | 202208011011 学生461
462 | 202208011012 学生462
463 | 202208011013 学生463
464 | 202208011014 学生464
465 | 202208011015 学生465
466 | 202208011016 学生466
467 | 202208011017 学生467
468 | 202208011018 学生468
469 | 202208011019 学生469
470 | 202208011020 学生470
471 | 202208011021 学生471
472 | 202208011022 学生472
473 | 202208011023 学生473
474 | 202208011024 学生474
475 | 202208011025 学生475
476 | 202208011026 学生476
477 | 202208011027 学生477
478 | 202208011028 学生478
479 | 202208011029 学生479
480 | 202208011030 学生480
481 | 202208011031 学生481
482 | 202208011032 学生482
483 | 202208011033 学生483
484 | 202208011034 学生484
485 | 202208011035 学生485
486 | 202208011036 学生486
487 | 202208011037 学生487
488 | 202208011038 学生488
489 | 202208011039 学生489
490 | 202208011040 学生490
491 | 202208011041 学生491
492 | 202208011042 学生492
493 | 202208011043 学生493
494 | 202208011044 学生494
495 | 202208011045 学生495
496 | 202208011046 学生496
497 | 202208011047 学生497
498 | 202208011048 学生498
499 | 202208011049 学生499
500 | 202208011050 学生500
501 | 202208011101 学生501
502 | 202208011102 学生502
503 | 202208011103 学生503
504 | 202208011104 学生504
505 | 202208011105 学生505
506 | 202208011106 学生506
507 | 202208011107 学生507
508 | 202208011108 学生508
509 | 202208011109 学生509
510 | 202208011110 学生510
511 | 202208011111 学生511
512 | 202208011112 学生512
513 | 202208011113 学生513
514 | 202208011114 学生514
515 | 202208011115 学生515
516 | 202208011116 学生516
517 | 202208011117 学生517
518 | 202208011118 学生518
519 | 202208011119 学生519
520 | 202208011120 学生520
521 | 202208011121 学生521
522 | 202208011122 学生522
523 | 202208011123 学生523
524 | 202208011124 学生524
525 | 202208011125 学生525
526 | 202208011126 学生526
527 | 202208011127 学生527
528 | 202208011128 学生528
529 | 202208011129 学生529
530 | 202208011130 学生530
531 | 202208011131 学生531
532 | 202208011132 学生532
533 | 202208011133 学生533
534 | 202208011134 学生534
535 | 202208011135 学生535
536 | 202208011136 学生536
537 | 202208011137 学生537
538 | 202208011138 学生538
539 | 202208011139 学生539
540 | 202208011140 学生540
541 | 202208011141 学生541
542 | 202208011142 学生542
543 | 202208011143 学生543
544 | 202208011144 学生544
545 | 202208011145 学生545
546 | 202208011146 学生546
547 | 202208011147 学生547
548 | 202208011148 学生548
549 | 202208011149 学生549
550 | 202208011150 学生550
551 | 202208011201 学生551
552 | 202208011202 学生552
553 | 202208011203 学生553
554 | 202208011204 学生554
555 | 202208011205 学生555
556 | 202208011206 学生556
557 | 202208011207 学生557
558 | 202208011208 学生558
559 | 202208011209 学生559
560 | 202208011210 学生560
561 | 202208011211 学生561
562 | 202208011212 学生562
563 | 202208011213 学生563
564 | 202208011214 学生564
565 | 202208011215 学生565
566 | 202208011216 学生566
567 | 202208011217 学生567
568 | 202208011218 学生568
569 | 202208011219 学生569
570 | 202208011220 学生570
571 | 202208011221 学生571
572 | 202208011222 学生572
573 | 202208011223 学生573
574 | 202208011224 学生574
575 | 202208011225 学生575
576 | 202208011226 学生576
577 | 202208011227 学生577
578 | 202208011228 学生578
579 | 202208011229 学生579
580 | 202208011230 学生580
581 | 202208011231 学生581
582 | 202208011232 学生582
583 | 202208011233 学生583
584 | 202208011234 学生584
585 | 202208011235 学生585
586 | 202208011236 学生586
587 | 202208011237 学生587
588 | 202208011238 学生588
589 | 202208011239 学生589
590 | 202208011240 学生590
591 | 202208011241 学生591
592 | 202208011242 学生592
593 | 202208011243 学生593
594 | 202208011244 学生594
595 | 202208011245 学生595
596 | 202208011246 学生596
597 | 202208011247 学生597
598 | 202208011248 学生598
599 | 202208011249 学生599
600 | 202208011250 学生600
601 | 202208011301 学生601
602 | 202208011302 学生602
603 | 202208011303 学生603
604 | 202208011304 学生604
605 | 202208011305 学生605
606 | 202208011306 学生606
607 | 202208011307 学生607
608 | 202208011308 学生608
609 | 202208011309 学生609
610 | 202208011310 学生610
611 | 202208011311 学生611
612 | 202208011312 学生612
613 | 202208011313 学生613
614 | 202208011314 学生614
615 | 202208011315 学生615
616 | 202208011316 学生616
617 | 202208011317 学生617
618 | 202208011318 学生618
619 | 202208011319 学生619
620 | 202208011320 学生620
621 | 202208011321 学生621
622 | 202208011322 学生622
623 | 202208011323 学生623
624 | 202208011324 学生624
625 | 202208011325 学生625
626 | 202208011326 学生626
627 | 202208011327 学生627
628 | 202208011328 学生628
629 | 202208011329 学生629
630 | 202208011330 学生630
631 | 202208011331 学生631
632 | 202208011332 学生632
633 | 202208011333 学生633
634 | 202208011334 学生634
635 | 202208011335 学生635
636 | 202208011336 学生636
637 | 202208011337 学生637
638 | 202208011338 学生638
639 | 202208011339 学生639
640 | 202208011340 学生640
641 | 202208011341 学生641
642 | 202208011342 学生642
643 | 202208011343 学生643
644 | 202208011344 学生644
645 | 202208011345 学生645
646 | 202208011346 学生646
647 | 202208011347 学生647
648 | 202208011348 学生648
649 | 202208011349 学生649
650 | 202208011350 学生650
651 | 202208011401 学生651
652 | 202208011402 学生652
653 | 202208011403 学生653
654 | 202208011404 学生654
655 | 202208011405 学生655
656 | 202208011406 学生656
657 | 202208011407 学生657
658 | 202208011408 学生658
659 | 202208011409 学生659
660 | 202208011410 学生660
661 | 202208011411 学生661
662 | 202208011412 学生662
663 | 202208011413 学生663
664 | 202208011414 学生664
665 | 202208011415 学生665
666 | 202208011416 学生666
667 | 202208011417 学生667
668 | 202208011418 学生668
669 | 202208011419 学生669
670 | 202208011420 学生670
671 | 202208011421 学生671
672 | 202208011422 学生672
673 | 202208011423 学生673
674 | 202208011424 学生674
675 | 202208011425 学生675
676 | 202208011426 学生676
677 | 202208011427 学生677
678 | 202208011428 学生678
679 | 202208011429 学生679
680 | 202208011430 学生680
681 | 202208011431 学生681
682 | 202208011432 学生682
683 | 202208011433 学生683
684 | 202208011434 学生684
685 | 202208011435 学生685
686 | 202208011436 学生686
687 | 202208011437 学生687
688 | 202208011438 学生688
689 | 202208011439 学生689
690 | 202208011440 学生690
691 | 202208011441 学生691
692 | 202208011442 学生692
693 | 202208011443 学生693
694 | 202208011444 学生694
695 | 202208011445 学生695
696 | 202208011446 学生696
697 | 202208011447 学生697
698 | 202208011448 学生698
699 | 202208011449 学生699
700 | 202208011450 学生700
701 | 202208011501 学生701
702 | 202208011502 学生702
703 | 202208011503 学生703
704 | 202208011504 学生704
705 | 202208011505 学生705
706 | 202208011506 学生706
707 | 202208011507 学生707
708 | 202208011508 学生708
709 | 202208011509 学生709
710 | 202208011510 学生710
711 | 202208011511 学生711
712 | 202208011512 学生712
713 | 202208011513 学生713
714 | 202208011514 学生714
715 | 202208011515 学生715
716 | 202208011516 学生716
717 | 202208011517 学生717
718 | 202208011518 学生718
719 | 202208011519 学生719
720 | 202208011520 学生720
721 | 202208011521 学生721
722 | 202208011522 学生722
723 | 202208011523 学生723
724 | 202208011524 学生724
725 | 202208011525 学生725
726 | 202208011526 学生726
727 | 202208011527 学生727
728 | 202208011528 学生728
729 | 202208011529 学生729
730 | 202208011530 学生730
731 | 202208011531 学生731
732 | 202208011532 学生732
733 | 202208011533 学生733
734 | 202208011534 学生734
735 | 202208011535 学生735
736 | 202208011536 学生736
737 | 202208011537 学生737
738 | 202208011538 学生738
739 | 202208011539 学生739
740 | 202208011540 学生740
741 | 202208011541 学生741
742 | 202208011542 学生742
743 | 202208011543 学生743
744 | 202208011544 学生744
745 | 202208011545 学生745
746 | 202208011546 学生746
747 | 202208011547 学生747
748 | 202208011548 学生748
749 | 202208011549 学生749
750 | 202208011550 学生750
751 | 202208011601 学生751
752 | 202208011602 学生752
753 | 202208011603 学生753
754 | 202208011604 学生754
755 | 202208011605 学生755
756 | 202208011606 学生756
757 | 202208011607 学生757
758 | 202208011608 学生758
759 | 202208011609 学生759
760 | 202208011610 学生760
761 | 202208011611 学生761
762 | 202208011612 学生762
763 | 202208011613 学生763
764 | 202208011614 学生764
765 | 202208011615 学生765
766 | 202208011616 学生766
767 | 202208011617 学生767
768 | 202208011618 学生768
769 | 202208011619 学生769
770 | 202208011620 学生770
771 | 202208011621 学生771
772 | 202208011622 学生772
773 | 202208011623 学生773
774 | 202208011624 学生774
775 | 202208011625 学生775
776 | 202208011626 学生776
777 | 202208011627 学生777
778 | 202208011628 学生778
779 | 202208011629 学生779
780 | 202208011630 学生780
781 | 202208011631 学生781
782 | 202208011632 学生782
783 | 202208011633 学生783
784 | 202208011634 学生784
785 | 202208011635 学生785
786 | 202208011636 学生786
787 | 202208011637 学生787
788 | 202208011638 学生788
789 | 202208011639 学生789
790 | 202208011640 学生790
791 | 202208011641 学生791
792 | 202208011642 学生792
793 | 202208011643 学生793
794 | 202208011644 学生794
795 | 202208011645 学生795
796 | 202208011646 学生796
797 | 202208011647 学生797
798 | 202208011648 学生798
799 | 202208011649 学生799
800 | 202208011650 学生800
801 | 202208011701 学生801
802 | 202208011702 学生802
803 | 202208011703 学生803
804 | 202208011704 学生804
805 | 202208011705 学生805
806 | 202208011706 学生806
807 | 202208011707 学生807
808 | 202208011708 学生808
809 | 202208011709 学生809
810 | 202208011710 学生810
811 | 202208011711 学生811
812 | 202208011712 学生812
813 | 202208011713 学生813
814 | 202208011714 学生814
815 | 202208011715 学生815
816 | 202208011716 学生816
817 | 202208011717 学生817
818 | 202208011718 学生818
819 | 202208011719 学生819
820 | 202208011720 学生820
821 | 202208011721 学生821
822 | 202208011722 学生822
823 | 202208011723 学生823
824 | 202208011724 学生824
825 | 202208011725 学生825
826 | 202208011726 学生826
827 | 202208011727 学生827
828 | 202208011728 学生828
829 | 202208011729 学生829
830 | 202208011730 学生830
831 | 202208011731 学生831
832 | 202208011732 学生832
833 | 202208011733 学生833
834 | 202208011734 学生834
835 | 202208011735 学生835
836 | 202208011736 学生836
837 | 202208011737 学生837
838 | 202208011738 学生838
839 | 202208011739 学生839
840 | 202208011740 学生840
841 | 202208011741 学生841
842 | 202208011742 学生842
843 | 202208011743 学生843
844 | 202208011744 学生844
845 | 202208011745 学生845
846 | 202208011746 学生846
847 | 202208011747 学生847
848 | 202208011748 学生848
849 | 202208011749 学生849
850 | 202208011750 学生850
851 | 202208011801 学生851
852 | 202208011802 学生852
853 | 202208011803 学生853
854 | 202208011804 学生854
855 | 202208011805 学生855
856 | 202208011806 学生856
857 | 202208011807 学生857
858 | 202208011808 学生858
859 | 202208011809 学生859
860 | 202208011810 学生860
861 | 202208011811 学生861
862 | 202208011812 学生862
863 | 202208011813 学生863
864 | 202208011814 学生864
865 | 202208011815 学生865
866 | 202208011816 学生866
867 | 202208011817 学生867
868 | 202208011818 学生868
869 | 202208011819 学生869
870 | 202208011820 学生870
871 | 202208011821 学生871
872 | 202208011822 学生872
873 | 202208011823 学生873
874 | 202208011824 学生874
875 | 202208011825 学生875
876 | 202208011826 学生876
877 | 202208011827 学生877
878 | 202208011828 学生878
879 | 202208011829 学生879
880 | 202208011830 学生880
881 | 202208011831 学生881
882 | 202208011832 学生882
883 | 202208011833 学生883
884 | 202208011834 学生884
885 | 202208011835 学生885
886 | 202208011836 学生886
887 | 202208011837 学生887
888 | 202208011838 学生888
889 | 202208011839 学生889
890 | 202208011840 学生890
891 | 202208011841 学生891
892 | 202208011842 学生892
893 | 202208011843 学生893
894 | 202208011844 学生894
895 | 202208011845 学生895
896 | 202208011846 学生896
897 | 202208011847 学生897
898 | 202208011848 学生898
899 | 202208011849 学生899
900 | 202208011850 学生900
901 | 202208011901 学生901
902 | 202208011902 学生902
903 | 202208011903 学生903
904 | 202208011904 学生904
905 | 202208011905 学生905
906 | 202208011906 学生906
907 | 202208011907 学生907
908 | 202208011908 学生908
909 | 202208011909 学生909
910 | 202208011910 学生910
911 | 202208011911 学生911
912 | 202208011912 学生912
913 | 202208011913 学生913
914 | 202208011914 学生914
915 | 202208011915 学生915
916 | 202208011916 学生916
917 | 202208011917 学生917
918 | 202208011918 学生918
919 | 202208011919 学生919
920 | 202208011920 学生920
921 | 202208011921 学生921
922 | 202208011922 学生922
923 | 202208011923 学生923
924 | 202208011924 学生924
925 | 202208011925 学生925
926 | 202208011926 学生926
927 | 202208011927 学生927
928 | 202208011928 学生928
929 | 202208011929 学生929
930 | 202208011930 学生930
931 | 202208011931 学生931
932 | 202208011932 学生932
933 | 202208011933 学生933
934 | 202208011934 学生934
935 | 202208011935 学生935
936 | 202208011936 学生936
937 | 202208011937 学生937
938 | 202208011938 学生938
939 | 202208011939 学生939
940 | 202208011940 学生940
941 | 202208011941 学生941
942 | 202208011942 学生942
943 | 202208011943 学生943
944 | 202208011944 学生944
945 | 202208011945 学生945
946 | 202208011946 学生946
947 | 202208011947 学生947
948 | 202208011948 学生948
949 | 202208011949 学生949
950 | 202208011950 学生950
951 | 202208012001 学生951
952 | 202208012002 学生952
953 | 202208012003 学生953
954 | 202208012004 学生954
955 | 202208012005 学生955
956 | 202208012006 学生956
957 | 202208012007 学生957
958 | 202208012008 学生958
959 | 202208012009 学生959
960 | 202208012010 学生960
961 | 202208012011 学生961
962 | 202208012012 学生962
963 | 202208012013 学生963
964 | 202208012014 学生964
965 | 202208012015 学生965
966 | 202208012016 学生966
967 | 202208012017 学生967
968 | 202208012018 学生968
969 | 202208012019 学生969
970 | 202208012020 学生970
971 | 202208012021 学生971
972 | 202208012022 学生972
973 | 202208012023 学生973
974 | 202208012024 学生974
975 | 202208012025 学生975
976 | 202208012026 学生976
977 | 202208012027 学生977
978 | 202208012028 学生978
979 | 202208012029 学生979
980 | 202208012030 学生980
981 | 202208012031 学生981
982 | 202208012032 学生982
983 | 202208012033 学生983
984 | 202208012034 学生984
985 | 202208012035 学生985
986 | 202208012036 学生986
987 | 202208012037 学生987
988 | 202208012038 学生988
989 | 202208012039 学生989
990 | 202208012040 学生990
991 | 202208012041 学生991
992 | 202208012042 学生992
993 | 202208012043 学生993
994 | 202208012044 学生994
995 | 202208012045 学生995
996 | 202208012046 学生996
997 | 202208012047 学生997
998 | 202208012048 学生998
999 | 202208012049 学生999
1000 | 202208012050 学生1000
1001 |
--------------------------------------------------------------------------------
/Lab3/README.md:
--------------------------------------------------------------------------------
1 | # Lab 3: A Simple Distributed Key-Value Store
2 |
3 | Enter in the folder you have cloned from our lab git repo, and pull the latest commit.
4 |
5 | `git pull`
6 |
7 | You can find this lab3's instruction in `Lab3/README.md`
8 |
9 | All materials of lab3 are in folder `Lab3/`
10 |
11 | ## 1. Overview
12 |
13 | Implement a simple distributed in-memory key-value database (KV store) by your own, using two-phase commit protocol or raft protocol to guarantee the consistency and robustness of your KV store.
14 |
15 | ### Goals
16 |
17 | * Get to know the problems and difficulties in building a distributed system;
18 | * Know how to use a simple protocol to maintain consistency between multiple distributed servers;
19 | * Use some simple schemes to handle server failures.
20 |
21 | ## 2. Background
22 |
23 | ### 2.1 KV store
24 |
25 | An in-memory key-value database (also called **key-value store**, or **KV store** in this document) is a database that keeps all its data in the main memory and uses a simple key-value data structure.
26 |
27 | Typically, a KV-store exposes the following methods for users to store/delete/read data to/from the databases:
28 |
29 | - **set(key, value)**: stores the value "value" with the key "key" (some KV stores also call this command as "**put**").
30 | - **del(key)**: deletes any record associated with the key "key".
31 | - **value=get(key)**: retrieves and returns the value associated with the key "key".
32 |
33 | In practice, in-memory KV store serves most user requests only with data stored in the main memory, meanwhile, it often backups its data in the hard disks thus to be durable. However, in this lab, you are not required to consider the data durability (i.e., you can only store the data in main memory).
34 |
35 | ### 2.2 Distributed KV store
36 |
37 | In large-scale cloud services, KV stores are often built in distributed manner, for the sake of **performance and robustness**.
38 |
39 | For example, a KV store may distribute its data in multiple server machines, and simultaneously use those servers to serve user requests. As such, user requests can be evenly balanced to multiple servers when the load is high, and the database can survive from the failure of some servers.
40 |
41 | However, when the KV database is distributed, many problems arise. Among those problems, **consistency issue** may be most notorious and toughest one. For example, assume that a KV database uses two servers A and B to hold data and serve user requests. When user 1 put some key at server A, and then after a very short time user 2 and 3 each posts a get request of this key to server A and B, respectively, then how to make sure that user 2 and 3 both **get the same latest value of this key**? What is more challenging, failures can happen on servers and networks when users are posting requests.
42 |
43 | Next, we will introduce a very simple but classic protocol to ensure consistency among distributed database servers.
44 |
45 | ### 2.3 Two-phase commit
46 |
47 | Now we describe possibly the simplest (but it works!) consensus protocol called **two-phase commit**, or **2PC**. This protocol can ensure the data consistency when the database is located on multiple distributed servers, that is, **two or more machines agree to do something, or not do it, atomically**. Note that this is only a very short introduction. You are encouraged to search more materials if you have further questions (e.g., [this](https://www.bilibili.com/video/BV1at411y7iQ) is a good video about 2PC).
48 |
49 | Frist, 2PC maintains a **commit log** on each machine, which keeps track of whether commit has happened. Log is used to guarantee that all machines either commit or don’t.
50 |
51 | Second, there is **one global coordinator**. It receives transactions from clients (e.g., set/delete/get) and reply results to clients, and coordinates multiple database servers, to ensure either all servers commit the transaction or all abort it.
52 |
53 | Based on above architecture, 2PC operates in two distinct phases:
54 |
55 | 1) The first phase is **prepared phase**. The global coordinator requests that all database servers (called participants) will promise to commit or rollback the transaction (e.g., set or delete a key) . Participants record promise in log, then acknowledge. If anyone votes to abort, coordinator writes "Abort" in its log and tells everyone to abort; each records "Abort" in log.
56 |
57 | 2) The second phase is **commit-or-abort phase**. After all participants respond that they are prepared, then the coordinator writes "Commit" to its log. Then the coordinator asks all participants to commit; All participants respond with ACK. After receive ACKs, the coordinator writes "Got Commit" to log.
58 |
59 | 
60 |
61 | The detailed steps in 2PC are as follows:
62 |
63 | ***Prepared phase:***
64 |
65 | 1) To commit the transaction, the coordinator starts by sending a **REQUEST-TO-PREPARE** message to each
66 | participant.
67 |
68 | 2) The coordinator waits for all participants to "vote" on the request.
69 |
70 | 3) In response to receiving a **REQUEST-TO-PREPARE** message, each participant votes by sending a message back to the coordinator, as follows:
71 |
72 | - It votes **PREPARED** if it is prepared to commit.
73 | - It may vote **NO** for any reason, usually because it cannot prepare the transaction due to a local failure.
74 | - It may delay voting indefinitely, for example, because the network drops the vote.
75 |
76 | ***Commit-Or-Abort phase:***
77 |
78 | 1) If the coordinator receives **PREPARED** messages from all participants, it decides to commit. The transaction is now officially committed. Otherwise, it either received a **NO** message or gave up waiting for some participant, so it decides to abort.
79 |
80 | 2) The coordinator sends its decision to all participants (i.e.,**COMMIT** or **ABORT**).
81 |
82 | 3) Participants acknowledge receipt of the commit or abort by replying **DONE**.
83 |
84 | 4) After receiving **DONE** from all participants, the coordinator can reply clients with the transaction result (either success or failed), and *forget* the transaction, meaning that it can deallocate any memory it was using to keep track of information about the transaction.
85 |
86 | ### 2.4 raft
87 |
88 | 2PC protocol may not be able to deal with clients' requests even when coordinator fails, thus some extend methods are necessary, such as [Raft algorithm](https://raft.github.io/).
89 |
90 | You can also find more details in [Section 4.2](#42-advanced-version20-points)
91 |
92 | ## 3. Your Lab Task
93 |
94 | ### 3.1 Task overview
95 |
96 | 
97 |
98 | You need to implement a simple distributed KV store. Your KV store should run as a network service on remote machine(s), which can receive clients' operations on the database and return results to clients, through network messages. To cope with failures, your are required to implement your KV store services on multiple machines (which are connected with each other through network). Use 2PC protocol to maintain database consistency among multiple machines.
99 |
100 | To simplify the task, your KV store data is only stored in main memory. Also, the log in 2PC protocol is also only stored in main memory.
101 |
102 | Detailed lab requirements are discussed below.
103 |
104 | ### 3.2 KV store command formats
105 |
106 | Your KV store servers are only required to support three database commands: `SET`, `GET` and `DEL` commands (case sensitive). These commands (including arguments) and the return results are encapsulated into network messages using dedicated message format. For simplicity, all the keys and values are stored as strings in your KV store.
107 |
108 | Further details are as follows:
109 |
110 | #### 3.2.1 Network message format
111 |
112 | In this lab, you should use a **simplified version** of RESP (REdis Serialization Protocol) message format to pass KV store commands from clients to servers (called request message) and pass command results from server to clients (called response message) . Specifically:
113 |
114 | ##### 3.2.1.1 Client request message
115 |
116 | Clients send commands to the server using RESP Arrays (more details see `section 3.2.2`). RESP Arrays consist of:
117 |
118 | - A `*` character as the first byte, followed by the number of elements in the array as a decimal number, followed by CRLF.
119 | - Arbitrary number of bulk strings (up to 512 MB in length).
120 |
121 | The bulk string consist of:
122 |
123 | - A `$` byte followed by the number of bytes composing the string (a prefixed length), terminated by CRLF.
124 | - The actual string data.
125 | - A final CRLF.
126 |
127 | For example, the string `CS06142` is encoded as follows:
128 |
129 | `$7\r\nCS06142\r\n`
130 |
131 | The bulk string `$7\r\nCS06142\r\n` start with a `$` byte, and the following number 7 indicate that the length of string `CS06142` is 7. Then, the terminated CRLF, actual string data `CS06142` and the final CRLF are next, consecutively.
132 |
133 | ##### 3.2.1.2 Server response message
134 |
135 | 1) Success message:
136 |
137 | Success messages are encoded in the following way: a plus '+' character, followed by a string that cannot contain a CR or LF character (no newlines are allowed), terminated by CRLF (that is "\r\n").
138 |
139 | For example, the `SET` command reply with just "OK" on success (more details see `section 3.2.2`):
140 |
141 | `+OK\r\n`
142 |
143 | 2) Error message:
144 |
145 | Like success messages, error messages consist of: a minus '-' character, followed by a string, terminated by CRLF.
146 |
147 | For example, if an error occurs, just return (more details see `section 3.2.2`):
148 |
149 | `-ERROR\r\n`
150 |
151 | 3) RESP Arrays message:
152 |
153 | Your server should return an RESP Arrays message when the `GET` command executed successfully (more detail see `section 3.2.2`).
154 |
155 | For example:
156 |
157 | `*2\r\n$5\r\nCloud\r\n$9\r\nComputing\r\n`
158 |
159 | 4) Integer message:
160 |
161 | Some commands need to return an integer (e.g., `DEL` command, more details see `section 3.2.2`). An integer message is just a CRLF terminated string representing an integer, prefixed by a ":" byte.
162 |
163 | An integer message example:
164 |
165 | `:1\r\n`
166 |
167 | #### 3.2.2 Database commands
168 |
169 | ##### 3.2.2.1 SET command
170 |
171 | `SET key value`
172 |
173 | The function of the `SET` command is to set **key** to hold the string **value**. If key already holds a value, it is overwritten. For example, if you want to set key `CS06142` to hold the string `Cloud Computing`, the command would be:
174 |
175 | `SET CS06142 "Cloud Computing"`
176 |
177 | According to the message format we have discussed in `section 3.2.1`, this command would be encoded as a RESP message format:
178 |
179 | `*4\r\n$3\r\nSET\r\n$7\r\nCS06142\r\n$5\r\nCloud\r\n$9\r\nComputing\r\n`
180 |
181 | **Note:** The encoded message start with a `*` byte. The following number 4 indicate that there are 4 bulk strings in this message. These bulk strings are `$3\r\nSET\r\n`, `$7\r\nCS06142\r\n`, `$5\r\nCloud\r\n`, and `$9\r\nComputing\r\n`.
182 |
183 | If the `SET` operation succeeds, the server will return a success message; otherwise, return an error message.
184 |
185 | For example, if the `SET` operation succeeds, the server just returns:
186 |
187 | `+OK\r\n`
188 |
189 | Otherwise, it returns:
190 |
191 | `-ERROR\r\n`
192 |
193 | ##### 3.2.2.2 GET command
194 |
195 | `GET key`
196 |
197 | `GET` command can get the **value** of **key**. If the key does not exist the special value `nil` is returned.
198 |
199 | For example, if you want to check the value of the key `CS06142`, construct the command like this:
200 |
201 | `GET CS06142`
202 |
203 | Like the `GET` command, this command should be encoded as a specific message format before sending to the server.
204 |
205 | `*2\r\n$3\r\nGET\r\n$7\r\nCS06142\r\n`
206 |
207 | If the `GET` command was executed correctly, the value (encoded as RESP Arrays format) of the key will be returned.
208 |
209 | For example, if the command above executed correctly, return:
210 |
211 | `*2\r\n$5\r\nCloud\r\n$9\r\nComputing\r\n`
212 |
213 | , assuming the value of the key `CS06142` is`Cloud Computing`.
214 |
215 | If the key does no exist, just return:
216 |
217 | `*1\r\n$3\r\nnil\r\n`
218 |
219 | If an error occurs, return an error message:
220 |
221 | `-ERROR\r\n`
222 |
223 | ##### 3.2.2.3 DEL command
224 |
225 | `DEL key1 key2 ...`
226 |
227 | The `DEL` command is used for removing one or more specified **keys** (arbitrary number, up to 512 MB message length). A key is ignored if it does not exist.
228 |
229 | The `DEL` command should return the number of keys that were removed.
230 |
231 | For example, if you want to delete key `CS06142` and key `CS162`, you can construct the `DEL` command:
232 |
233 | `DEL CS06142 CS162`
234 |
235 | Similar to the `SET` and `GET` commands, the `DEL` command will be encoded in RESP message as follows:
236 |
237 | `*3\r\n$3\r\nDEL\r\n$7\r\nCS06142\r\n$5\r\nCS162\r\n`
238 |
239 | The server will return the number of keys that were removed.
240 |
241 | For example, if the `DEL` command above executed, return an integer message:
242 |
243 | `:1\r\n`
244 |
245 | **note:** Because we only set the key `CS06142` to hold a value. As for key `CS162`, it will be ignored because it does no exist. So, the number in the integer message is 1.
246 |
247 | If an error occurs, return an error message:
248 |
249 | `-ERROR\r\n`
250 |
251 | ### 3.3 Use 2PC protocol to build a KV store on multiple servers(Basic Version)
252 |
253 | You should implement the coordinator and participant program.
254 |
255 | The **coordinator** does not store any data. It only receives and parses KV commands from clients, runs 2PC protocol to coordinates participants to conduct the KV commands consistently, and reply command results to clients. *There is only one coordinator in the whole system*.
256 |
257 | Each **participant** maintains a KV database in its main memory, and conduct KV commands sent from the coordinator, and return results to the coordinator.
258 |
259 | You can use any message format for communication between the coordinator and participants. For example, you can also use the RESP format introduced before, or use some other RPC library.
260 |
261 | ### 3.4 Use Raft protocol to build a KV store on multiple servers(Advanced Version)
262 |
263 | Before implementing the advanced version, please carefully read the introduction to the raft algorithm in section 2.4.
264 |
265 | Several points to note here, because the test script needs to simulate communication between the client and your storage system, and since the generation of system leader is random, the test script is initially unknown. Therefore, the test script will randomly select a service to send a request. If the service is not a leader, the request needs to be rejected and the correct leader information returned, and the test script will initiate a request for the leader again.
266 |
267 | You can use any message format for communication between the leader and followers. For example, you can also use the RESP format introduced before, or use some other RPC library.
268 |
269 | ### 3.5 Run your program
270 |
271 | #### 3.5.1 Program arguments
272 |
273 | Enable long options to accept arguments in your program, just like lab2. There should be one and only one argument for your program: `--config_path`, which specifies the path of the configuration file. All the detailed configurations are written in the configuration file. Your program should read and parse the configuration file, and run as coordinator or participant accordingly.
274 |
275 | ***For basic version***, your program is called **kvstore2pcsystem**:
276 |
277 | run the **coordinator** process, just typing (`./src/coordinator.conf` is the coordinator's configuration file)
278 |
279 | `./kvstore2pcsystem --config_path ./src/coordinator.conf`
280 |
281 | run the **participant** process, just typing (`./src/participant.conf` is the participant's configuration file)
282 |
283 | `./kvstore2pcsystem --config_path ./src/participant.conf`
284 |
285 | When you run the command above, your program should run correctly without any further inputs.
286 |
287 | ***For advanced version(specifically refers to the version implemented with raft)***, your program is called **kvstoreraftsystem**:
288 |
289 | run the process with configuration file:
290 |
291 | `./kvstoreraftsystem --config_path ./src/follower.conf`
292 |
293 | When you run the command above, your program should run correctly without any further inputs.
294 |
295 | #### 3.5.2 Configuration file format
296 |
297 | ***For basic version***, a configuration file consists of two kinds of lines: 1) parameter line, and 2) comment line.
298 |
299 | - **A comment line** starts with a character '!'. The whole comment line are not parsed by the program.
300 | - **A parameter line** starts with a *parameter*, followed by a *value*. The parameter and value are separated by a whitespace. Parameter lines specify the necessary information that the coordinator or participants should know before running. There are three valid parameters: `mode`, `coordinator_info`, and `participant_info`.
301 | - The parameter `mode` specifies that whether the program runs as a coordinator or a participant. Its value should only be either `coordinator` or `participant`. `mode` line is always *the first parameter line* in the configuration file.
302 | - The parameter `coordinator_info` specifies the network address that the coordinator is listening on. Its value consists of the IP address and port number (separated by character ':'). Clients and participants can communicate with the coordinator using this network address. Since there is only one coordinator, there is only one `coordinator_info` line in both coordinator's and participants' configuration file.
303 | - The parameter `participant_info` consists of the network address that participant process is listening on. Its value consists of the IP address and port number (separated by character ':'). The coordinator can communicate with the participant using this network address. For participants, there is only one `participant_info` line in the configuration file, specifying its own network address; For the coordinator, there can be multiple `participant_info` lines in the configuration file, specifying the network addresses of all participants.
304 |
305 | Sample coordinator configuration file(**basic**):
306 |
307 | ```tsconfig
308 | !
309 | ! Coordinator configuration
310 | ! 2023/05/05 11:25:33
311 | !
312 | ! The argument name and value are separated by whitespace in the configuration file.
313 | !
314 | ! Mode of process, coordinator OR participant
315 | mode coordinator
316 | !
317 | ! The address and port the coordinator process is listening on.
318 | ! Note that the address and port are separated by character ':'.
319 | coordinator_info 127.0.0.1:8001
320 | !
321 | ! Address and port information of all participants.
322 | ! Three lines specifies three participants' addresses.
323 | participant_info 127.0.0.1:8002
324 | participant_info 127.0.0.1:8003
325 | participant_info 127.0.0.1:8004
326 | ```
327 |
328 | Sample participant configuration file(**basic**):
329 |
330 | ```tsconfig
331 | !
332 | ! Participant configuration
333 | ! 2023/05/05 11:25:33
334 | !
335 | ! The argument name and value are separated by whitespace in the configuration file.
336 | !
337 | ! Mode of process, coordinator OR participant
338 | mode participant
339 | !
340 | ! The address and port the participant process is listening on.
341 | participant_info 127.0.0.1:8002
342 | !
343 | ! The address and port the coordinator process is listening on.
344 | coordinator_info 127.0.0.1:8001
345 | ```
346 |
347 | ***For advanced version(specifically refers to the version implemented with raft)***, the parameter line of the configuration file is consistent with the basic version, except that the mode line is no longer required.
348 |
349 | The configuration files for each service are similar. In addition to specifying their own configuration information, they also provide configuration information for all other services in the cluster (this is required because they all need to communicate with each other, which is different from the implementation of 2PC).
350 |
351 | Sample configuration file implemented with raft(**advanced**):
352 |
353 | ```tsconfig
354 | !
355 | ! configuration of advanced version
356 | ! 2023/05/05 11:25:33
357 | !
358 | ! The argument name and value are separated by whitespace in the configuration file.
359 | !
360 | ! The address and port the follower process is listening on.
361 | follower_info 127.0.0.1:8001
362 | !
363 | ! The address and port the other follower processes are listening on.
364 | follower_info 127.0.0.1:8002
365 | follower_info 127.0.0.1:8003
366 | ```
367 |
368 | ## 4 Implementation requirements
369 |
370 | ### 4.1 Basic version (18 points totally)
371 |
372 | In order to finish the new Lab4, you should finish 2nd level of Basic version in Lab3 at least.
373 |
374 | If you want to get excellent score in Lab4, please try to finish [Section 4.2](#42-advanced-version20-points).
375 |
376 | #### 4.1.1 1st level of Basic version(15 points)
377 |
378 | Your program should complete all the tasks described in `section 3.1-3.4`. Your system is required to correctly receive and conduct the KV commands, and reply the corresponding results, as described before.
379 |
380 | In the 1st level of basic version, there will be **no participant failures**. Also, we will **not inject any network failures**. However, the network may still drop packets occasionally. You can use TCP to handle such occasional drops.
381 |
382 | **The coordinator process may be killed and restart at any time for multiple times**. So do not store any database data in the coordinator. The coordinator will deal with clients' KV commands when it is working. When the coordinator is killed, your system is not required to reply any client's commands. Clients will keep retransmit its KV commands until it gets success response from the coordinator. The coordinator remembers no history (except for the information in the configuration file), so it will deal with all commands as new ones after it restarts.
383 |
384 | Your program should run correctly with 3 or more participants.
385 |
386 | #### 4.1.2 2nd level of Basic version(16 points)
387 |
388 | Your program should complete all the tasks described in `section 3.1-3.4`. Your system is required to correctly receive and conduct the KV commands, and reply the corresponding results, as described before.
389 |
390 | In the 2nd level of Basic version, **participants may fail, and the network links may fail**. However, the participant and network link **failures are one-shot**, that is, if they fail, they will never come back again. Also, **the coordinator process may be killed and restart at any time for multiple times**. The coordinator will deal with clients' KV commands when it is working. When the coordinator is killed, your system is not required to reply any client's commands.
391 |
392 | When the coordinator is working, it should be able to detect the failure of participants. You can use some periodical heartbeat messages to detect the participant failure (e.g., no reply after certain number of heartbeats). **Once a participant is dead, the coordinator should be able to remove it from the system, and correctly receive/conduct/reply clients' KV commands with the rest participants**. If all participants fail, the coordinator will always reply ERROR to the clients' KV commands. The coordinator remembers no history (except for the information in the configuration file), so it need to redetect all the participants' liveness after restart.
393 |
394 | Your program should run correctly with 3 or more participants.
395 |
396 | **NOTE**: **Groups that have registered for demo 3 should at least finish the advanced version.**
397 |
398 | #### 4.1.3 3rd level of Basic version(18 points)
399 |
400 | Your program should complete all the tasks described in `section 3.1-3.4`. Your system is required to correctly receive and conduct the KV commands, and reply the corresponding results, as described before.
401 |
402 | In the 3rd level of Basic version, **participants may fail, and the network links may fail**. The participant and network link **failures can be both permanent or transient**, that is, if they fail, they may come back again at any time. Also, **the coordinator process may be killed and restart at any time for multiple times**. The coordinator will deal with clients' KV commands when it is working. When the coordinator is killed, your system is not required to reply any client's commands.
403 |
404 | When the coordinator is working, it should be able to detect **both failure and recovery** of participants. Once a participant is dead, the coordinator should be able to remove it from the system; Once a participant is recovered, the coordinator should be able to add it back to the system. Note that to keep database consistent, after a participant recovers from failure, you should copy the latest KV store database from some working participants to this recovered participant. We will not provide any copy protocols for you. You are encouraged to devise your own schemes as long as it is correct. You should be very careful about the consistency issue since the failures can be very complex and random. For example, considering the case that there are two participants A and B, A fails first and then A comes back and B fails later. During this procedure, client's may keep sending KV commands, so A needs to sync those new KV commands after coming back, otherwise it cannot server client's request correctly after B fails.
405 |
406 | We will randomly inject failure and recovery to all the participants and network links. To ensure the database can be successfully copied, during our test, after a participant (or its network link) is recovered from failure, we will ensure that *no failures happen in the following 10 seconds* (no coordinator/participant/network failure). Your coordinator should be able to copy the latest database to the newly recovered participant in this 10 seconds. Moreover, we will always ensure that *there is at least one working participant* in the system. As such, the coordinator should be able to correctly receive/conduct/reply clients' KV commands during the failures and recoveries.
407 |
408 | Your program should run correctly with 3 or more participants.
409 |
410 | **NOTE**: **Groups that have registered for demo 4 should at least finish the 3rd level of Basic version**
411 |
412 | ### **4.2 Advanced version(20 points)**
413 |
414 | Since the coordinator may permanently fail, your system should also **be able to deal with clients' requests even when coordinator fails**. 2PC is not able to handle this problem. So in this version, you will not use 2PC protocol, but use some advanced consensus protocol to handle this case.
415 |
416 | You can implement multiple KV store servers, where each server can receive requests from clients, stores data, and reply responses. Clients are preconfigured with all servers' addresses, and may send KV commands to any of the server, randomly. To keep consistency, normally there is only one leader server that deal with all the clients' requests, and backup the data in other servers. Clients' commands to other servers are all redirected to the leader. The consensus protocol can help servers to detect the failure of the leader server, and reelect a new leader. Also, the consensus protocol can help to maintain database consistency among multiple servers.
417 |
418 | [Raft](https://raft.github.io/) is a very good consensus protocol for this purpose. You may want to read its paper by yourself and use raft to implement this version (there are many open-sourced raft implementation that you can borrow). Sorry I'm not going to teach you this :) Of course, it is always good to use other consensus protocols or even your own schemes.
419 |
420 | **Tips for testing**:
421 | **When doing Advanced version, you should give [coordinator-type](#342-configuration-file-format) configuration files to all servers.**
422 |
423 | **P.S.**
424 | **If you want to finish Lab4 excellently, `Raft` is a better choice than `2PC`.**
425 |
426 | **NOTE**: **This version is very difficult, so it is not compulsory but just a challenge. Have fun!**
427 |
428 | ## 5. Lab submission
429 |
430 | Please put all your code in folder `Lab3` and write a `Makefile` so that we **can compile your code in one single command** `make`. The compiled runnable executable binary should be named `kvstore2pcsystem` and located in folder `Lab3`. Please carefully following above rules so that TAs can automatically test your code!!!
431 |
432 | You can use any available code or library for this lab. Please search the Internet. However, do not copy other teams' code. No performance test report is required for this lab. Enjoy the lab :)
433 |
434 | Please submit your lab program following the guidance in the [Overall Lab Instructions](../README.md) (`../README.md`)
435 |
436 | ## 6. Lab3 tester
437 |
438 | You can find it in [lab3_tester](https://github.com/LabCloudComputing/CloudComputing_Lab3_tester).
439 |
440 | Test script of lab3 will update later in order to test Advance version.
441 |
442 | ## 7. Grading standards
443 |
444 | * You can get 18 points if you can:
445 |
446 | * finish all the requirements of the basic version
447 |
448 | * You can get 20 points (full score) if you can:
449 |
450 | * finish all the requirements of the advanced version
451 |
452 | If you missed some parts, you will get part of the points depending how much you finished.
453 |
--------------------------------------------------------------------------------