├── .gitignore ├── GNUmakefile ├── README.md ├── client.txt ├── config.cc ├── config.h ├── extent_client.cc ├── extent_client.h ├── extent_protocol.h ├── extent_server.cc ├── extent_server.h ├── extent_smain.cc ├── fuse.cc ├── gettime.cc ├── gettime.h ├── handle.cc ├── handle.h ├── lang ├── algorithm.h └── verify.h ├── lock_client.cc ├── lock_client.h ├── lock_client_cache.cc ├── lock_client_cache.h ├── lock_client_cache_rsm.cc ├── lock_client_cache_rsm.h ├── lock_demo.cc ├── lock_protocol.h ├── lock_server.cc ├── lock_server.h ├── lock_server_cache.cc ├── lock_server_cache.h ├── lock_server_cache_rsm.cc ├── lock_server_cache_rsm.h ├── lock_smain.cc ├── lock_tester.cc ├── locks.cc ├── locks.h ├── log.cc ├── log.h ├── mkfs.sh ├── paxos.cc ├── paxos.h ├── paxos_protocol.h ├── raii.cc ├── raii.h ├── rpc ├── connection.cc ├── connection.h ├── fifo.h ├── jsl_log.cc ├── jsl_log.h ├── marshall.h ├── method_thread.h ├── pollmgr.cc ├── pollmgr.h ├── rpc.cc ├── rpc.h ├── rpctest.cc ├── slock.h ├── thr_pool.cc └── thr_pool.h ├── rsm.cc ├── rsm.h ├── rsm_client.cc ├── rsm_client.h ├── rsm_protocol.h ├── rsm_state_transfer.h ├── rsm_tester.cc ├── rsm_tester.pl ├── rsmtest_client.cc ├── rsmtest_client.h ├── start.sh ├── stop.sh ├── test-lab-2-a.pl ├── test-lab-2-b.pl ├── test-lab-3-a.pl ├── test-lab-3-b.c ├── test-lab-3-c.c ├── tprintf.h ├── unittest.cc ├── yfs_client.cc └── yfs_client.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by https://www.gitignore.io 2 | 3 | ### C++ ### 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | 22 | # Compiled Static libraries 23 | *.lai 24 | *.la 25 | *.a 26 | *.lib 27 | 28 | # Executables 29 | *.exe 30 | *.out 31 | *.app 32 | 33 | # Created by https://www.gitignore.io 34 | 35 | ### C ### 36 | # Object files 37 | *.o 38 | *.ko 39 | *.obj 40 | *.elf 41 | 42 | # Precompiled Headers 43 | *.gch 44 | *.pch 45 | 46 | # Libraries 47 | *.lib 48 | *.a 49 | *.la 50 | *.lo 51 | 52 | # Shared objects (inc. Windows DLLs) 53 | *.dll 54 | *.so 55 | *.so.* 56 | *.dylib 57 | 58 | # Executables 59 | *.exe 60 | *.out 61 | *.app 62 | *.i*86 63 | *.x86_64 64 | *.hex 65 | 66 | # Debug files 67 | *.dSYM/ 68 | 69 | # Created by https://www.gitignore.io 70 | 71 | ### Linux ### 72 | *~ 73 | 74 | # KDE directory preferences 75 | .directory 76 | 77 | # Linux trash folder which might appear on any partition or disk 78 | .Trash-* 79 | 80 | # Created by https://www.gitignore.io 81 | 82 | ### Vim ### 83 | [._]*.s[a-w][a-z] 84 | [._]s[a-w][a-z] 85 | *.un~ 86 | Session.vim 87 | .netrwhist 88 | *~ 89 | 90 | .idea/ 91 | CMakeLists.txt 92 | *.d 93 | 94 | 95 | 96 | extent_server 97 | lock_demo 98 | lock_server 99 | lock_tester 100 | rpc/rpctest 101 | yfs_client 102 | yfs_client.cc.orig 103 | 104 | 105 | core 106 | test-lab-3-b 107 | test-lab-3-c 108 | # Created by https://www.gitignore.io 109 | 110 | ### CMake ### 111 | CMakeCache.txt 112 | CMakeFiles 113 | Makefile 114 | cmake_install.cmake 115 | install_manifest.txt 116 | 117 | -------------------------------------------------------------------------------- /GNUmakefile: -------------------------------------------------------------------------------- 1 | LAB=7 2 | SOL=0 3 | RPC=./rpc 4 | LAB2GE=$(shell expr $(LAB) \>\= 2) 5 | LAB3GE=$(shell expr $(LAB) \>\= 3) 6 | LAB4GE=$(shell expr $(LAB) \>\= 4) 7 | LAB5GE=$(shell expr $(LAB) \>\= 5) 8 | LAB6GE=$(shell expr $(LAB) \>\= 6) 9 | LAB7GE=$(shell expr $(LAB) \>\= 7) 10 | CXXFLAGS = -g -MMD -Wall -I. -I$(RPC) -DLAB=$(LAB) -DSOL=$(SOL) -D_FILE_OFFSET_BITS=64 -std=c++11 11 | FUSEFLAGS= -D_FILE_OFFSET_BITS=64 -DFUSE_USE_VERSION=25 -I/usr/local/include/fuse -I/usr/include/fuse 12 | ifeq ($(shell uname -s),Darwin) 13 | MACFLAGS= -D__FreeBSD__=10 14 | else 15 | MACFLAGS= 16 | endif 17 | LDFLAGS = -L. -L/usr/local/lib 18 | LDLIBS = -lpthread 19 | ifeq ($(LAB2GE),1) 20 | ifeq ($(shell uname -s),Darwin) 21 | ifeq ($(shell sw_vers -productVersion | sed -e "s/.*\(10\.[0-9]\).*/\1/"),10.6) 22 | LDLIBS += -lfuse_ino64 23 | else 24 | LDLIBS += -lfuse 25 | endif 26 | else 27 | LDLIBS += -lfuse 28 | endif 29 | endif 30 | LDLIBS += $(shell test -f `gcc -print-file-name=librt.so` && echo -lrt) 31 | LDLIBS += $(shell test -f `gcc -print-file-name=libdl.so` && echo -ldl) 32 | CC = g++ 33 | CXX = g++ 34 | 35 | lab: lab$(LAB) 36 | lab1: rpc/rpctest lock_server lock_tester lock_demo 37 | lab2: rpc/rpctest lock_server lock_tester lock_demo yfs_client extent_server 38 | lab3: yfs_client extent_server lock_server test-lab-3-b test-lab-3-c 39 | lab4: yfs_client extent_server lock_server lock_tester test-lab-3-b\ 40 | test-lab-3-c 41 | lab5: yfs_client extent_server lock_server test-lab-3-b test-lab-3-c 42 | lab6: lock_server rsm_tester 43 | lab7: lock_tester lock_server rsm_tester 44 | 45 | hfiles1=rpc/fifo.h rpc/connection.h rpc/rpc.h rpc/marshall.h rpc/method_thread.h\ 46 | rpc/thr_pool.h rpc/pollmgr.h rpc/jsl_log.h rpc/slock.h rpc/rpctest.cc\ 47 | lock_protocol.h lock_server.h lock_client.h gettime.h gettime.cc lang/verify.h \ 48 | lang/algorithm.h locks.h 49 | hfiles2=yfs_client.h extent_client.h extent_protocol.h extent_server.h 50 | hfiles3=lock_client_cache.h lock_server_cache.h handle.h tprintf.h 51 | hfiles4=log.h rsm.h rsm_protocol.h config.h paxos.h paxos_protocol.h rsm_state_transfer.h rsmtest_client.h tprintf.h 52 | hfiles5=rsm_state_transfer.h rsm_client.h 53 | rsm_files = rsm.cc paxos.cc config.cc log.cc handle.cc 54 | 55 | rpclib=rpc/rpc.cc rpc/connection.cc rpc/pollmgr.cc rpc/thr_pool.cc rpc/jsl_log.cc gettime.cc 56 | rpc/librpc.a: $(patsubst %.cc,%.o,$(rpclib)) 57 | rm -f $@ 58 | ar cq $@ $^ 59 | ranlib rpc/librpc.a 60 | 61 | rpc/rpctest=rpc/rpctest.cc 62 | rpc/rpctest: $(patsubst %.cc,%.o,$(rpctest)) rpc/librpc.a 63 | 64 | lock_demo=lock_demo.cc lock_client.cc 65 | lock_demo : $(patsubst %.cc,%.o,$(lock_demo)) rpc/librpc.a 66 | 67 | lock_tester=lock_tester.cc lock_client.cc 68 | ifeq ($(LAB4GE),1) 69 | lock_tester += lock_client_cache.cc 70 | endif 71 | ifeq ($(LAB7GE),1) 72 | lock_tester+=rsm_client.cc handle.cc lock_client_cache_rsm.cc 73 | endif 74 | lock_tester : $(patsubst %.cc,%.o,$(lock_tester)) rpc/librpc.a 75 | 76 | lock_server=lock_server.cc lock_smain.cc locks.cc 77 | ifeq ($(LAB4GE),1) 78 | lock_server+=lock_server_cache.cc handle.cc 79 | endif 80 | ifeq ($(LAB6GE),1) 81 | lock_server+= $(rsm_files) 82 | endif 83 | ifeq ($(LAB7GE),1) 84 | lock_server+= lock_server_cache_rsm.cc 85 | endif 86 | 87 | lock_server : $(patsubst %.cc,%.o,$(lock_server)) rpc/librpc.a 88 | 89 | yfs_client=yfs_client.cc extent_client.cc fuse.cc 90 | ifeq ($(LAB3GE),1) 91 | yfs_client += lock_client.cc 92 | endif 93 | ifeq ($(LAB7GE),1) 94 | yfs_client += rsm_client.cc lock_client_cache_rsm.cc 95 | endif 96 | ifeq ($(LAB4GE),1) 97 | yfs_client += lock_client_cache.cc 98 | endif 99 | yfs_client : $(patsubst %.cc,%.o,$(yfs_client)) rpc/librpc.a 100 | 101 | extent_server=extent_server.cc extent_smain.cc 102 | extent_server : $(patsubst %.cc,%.o,$(extent_server)) rpc/librpc.a 103 | 104 | test-lab-3-b=test-lab-3-b.c 105 | test-lab-3-b: $(patsubst %.c,%.o,$(test_lab_4-b)) rpc/librpc.a 106 | 107 | test-lab-3-c=test-lab-3-c.c 108 | test-lab-4-c: $(patsubst %.c,%.o,$(test_lab_4-c)) rpc/librpc.a 109 | 110 | rsm_tester=rsm_tester.cc rsmtest_client.cc 111 | rsm_tester: $(patsubst %.cc,%.o,$(rsm_tester)) rpc/librpc.a 112 | 113 | %.o: %.cc 114 | $(CXX) $(CXXFLAGS) -c $< -o $@ 115 | 116 | fuse.o: fuse.cc 117 | $(CXX) -c $(CXXFLAGS) $(FUSEFLAGS) $(MACFLAGS) $< 118 | 119 | # mklab.inc is needed by 6.824 staff only. Just ignore it. 120 | -include mklab.inc 121 | 122 | -include *.d 123 | -include rpc/*.d 124 | 125 | clean_files=rpc/rpctest rpc/*.o rpc/*.d rpc/librpc.a *.o *.d yfs_client extent_server lock_server lock_tester lock_demo rpctest test-lab-3-b test-lab-3-c rsm_tester 126 | .PHONY: clean handin 127 | clean: 128 | rm $(clean_files) -rf 129 | 130 | handin_ignore=$(clean_files) core* *log 131 | handin_file=$(shell whoami)-lab$(LAB).tgz 132 | labdir=$(shell basename $(PWD)) 133 | handin: 134 | @if test -f stop.sh; then ./stop.sh > /dev/null 2>&1 | echo ""; fi 135 | @bash -c "cd ../; tar -X <(tr ' ' '\n' < <(echo '$(handin_ignore)')) -czvf $(handin_file) $(labdir); mv $(handin_file) $(labdir); cd $(labdir)" 136 | @echo Please email $(handin_file) to 6.824-submit@pdos.csail.mit.edu 137 | @echo Thanks! 138 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **Yet another File System**, MIT 6.824 *Distrituted System* course project, written in C++. 2 | 3 | [Lab infomation](http://pdos.csail.mit.edu/6.824-2012/labs/index.html) 4 | 5 | -------------------------------------------------------------------------------- /client.txt: -------------------------------------------------------------------------------- 1 | acquire 2 | ------- 3 | NONE -> ACQUIRING -> LOCKED/RETRY->NONE 4 | FREE -> LOCKED 5 | LOCKED -> lock_wait -> LOCKED/ACQUIRING->LOCKED 6 | ACQURING-> lock_wait -> LOCKED/ACQUIRING->LOCKED 7 | RELEASING-> lock_wait -> ACQUIRING 8 | 9 | 10 | release 11 | ------- 12 | NONE -> NONE/ERROR 13 | FREE -> FREE/ERROR 14 | LOCKED -> FREE/RELEASING->revoke_notify->NONE 15 | ACQURING-> ERROR 16 | RELEASING-> ERROR 17 | 18 | revoke 19 | ------ 20 | NONE -> NONE 21 | FREE -> NONE 22 | LOCKED -> ++nrevoke -> revoke_wait -> RELEASING->NONE 23 | ACQURING-> ++nrevoke -> revoke_wait -> RELEASING->NONE 24 | RELEASING-> ERROR 25 | 26 | retry 27 | ----- 28 | NONE -> ACQURING -> ++nretry -> retry_notify -> ACQURING 29 | FREE -> ERROR 30 | LOCKED -> ERROR 31 | ACQURING-> ERROR 32 | RELEASING-> ++nretry -> retry_notify -> ACQURING 33 | 34 | -------------------------------------------------------------------------------- /config.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "config.h" 5 | #include "paxos.h" 6 | #include "handle.h" 7 | #include "tprintf.h" 8 | #include "lang/verify.h" 9 | 10 | // The config module maintains views. As a node joins or leaves a 11 | // view, the next view will be the same as previous view, except with 12 | // the new node added or removed. The first view contains only node 13 | // 1. If node 2 joins after the first node (it will download the views 14 | // from node 1), it will learn about view 1 with the first node as the 15 | // only member. It will then invoke Paxos to create the next view. 16 | // It will tell Paxos to ask the nodes in view 1 to agree on the value 17 | // {1, 2}. If Paxos returns success, then it moves to view 2 with 18 | // {1,2} as the members. When node 3 joins, the config module runs 19 | // Paxos with the nodes in view 2 and the proposed value to be 20 | // {1,2,3}. And so on. When a node discovers that some node of the 21 | // current view is not responding, it kicks off Paxos to propose a new 22 | // value (the current view minus the node that isn't responding). The 23 | // config module uses Paxos to create a total order of views, and it 24 | // is ensured that the majority of the previous view agrees to the 25 | // next view. The Paxos log contains all the values (i.e., views) 26 | // agreed on. 27 | // 28 | // The RSM module informs config to add nodes. The config module 29 | // runs a heartbeater thread that checks in with nodes. If a node 30 | // doesn't respond, the config module will invoke Paxos's proposer to 31 | // remove the node. Higher layers will learn about this change when a 32 | // Paxos acceptor accepts the new proposed value through 33 | // paxos_commit(). 34 | // 35 | // To be able to bring other nodes up to date to the latest formed 36 | // view, each node will have a complete history of all view numbers 37 | // and their values that it knows about. At any time a node can reboot 38 | // and when it re-joins, it may be many views behind; by remembering 39 | // all views, the other nodes can bring this re-joined node up to 40 | // date. 41 | 42 | static void * 43 | heartbeatthread(void *x) 44 | { 45 | config *r = (config *) x; 46 | r->heartbeater(); 47 | return 0; 48 | } 49 | 50 | config::config(std::string _first, std::string _me, config_view_change *_vc) 51 | : myvid (0), first (_first), me (_me), vc (_vc) 52 | { 53 | VERIFY (pthread_mutex_init(&cfg_mutex, NULL) == 0); 54 | VERIFY(pthread_cond_init(&config_cond, NULL) == 0); 55 | 56 | std::ostringstream ost; 57 | ost << me; 58 | 59 | acc = new acceptor(this, me == _first, me, ost.str()); 60 | pro = new proposer(this, acc, me); 61 | 62 | // XXX hack; maybe should have its own port number 63 | pxsrpc = acc->get_rpcs(); 64 | pxsrpc->reg(paxos_protocol::heartbeat, this, &config::heartbeat); 65 | 66 | { 67 | ScopedLock ml(&cfg_mutex); 68 | 69 | reconstruct(); 70 | 71 | pthread_t th; 72 | VERIFY (pthread_create(&th, NULL, &heartbeatthread, (void *) this) == 0); 73 | } 74 | } 75 | 76 | void 77 | config::restore(std::string s) 78 | { 79 | ScopedLock ml(&cfg_mutex); 80 | acc->restore(s); 81 | reconstruct(); 82 | } 83 | 84 | std::vector 85 | config::get_view(unsigned instance) 86 | { 87 | ScopedLock ml(&cfg_mutex); 88 | return get_view_wo(instance); 89 | } 90 | 91 | // caller should hold cfg_mutex 92 | std::vector 93 | config::get_view_wo(unsigned instance) 94 | { 95 | std::string value = acc->value(instance); 96 | tprintf("get_view(%d): returns %s\n", instance, value.c_str()); 97 | return members(value); 98 | } 99 | 100 | std::vector 101 | config::members(std::string value) 102 | { 103 | std::istringstream ist(value); 104 | std::string m; 105 | std::vector view; 106 | while (ist >> m) { 107 | view.push_back(m); 108 | } 109 | return view; 110 | } 111 | 112 | std::string 113 | config::value(std::vector m) 114 | { 115 | std::ostringstream ost; 116 | for (unsigned i = 0; i < m.size(); i++) { 117 | ost << m[i]; 118 | ost << " "; 119 | } 120 | return ost.str(); 121 | } 122 | 123 | // caller should hold cfg_mutex 124 | void 125 | config::reconstruct() 126 | { 127 | if (acc->instance() > 0) { 128 | std::string m; 129 | myvid = acc->instance(); 130 | mems = get_view_wo(myvid); 131 | tprintf("config::reconstruct: %d %s\n", myvid, print_members(mems).c_str()); 132 | } 133 | } 134 | 135 | // Called by Paxos's acceptor. 136 | void 137 | config::paxos_commit(unsigned instance, std::string value) 138 | { 139 | std::string m; 140 | std::vector newmem; 141 | ScopedLock ml(&cfg_mutex); 142 | 143 | newmem = members(value); 144 | tprintf("config::paxos_commit: %d: %s\n", instance, 145 | print_members(newmem).c_str()); 146 | 147 | for (unsigned i = 0; i < mems.size(); i++) { 148 | tprintf("config::paxos_commit: is %s still a member?\n", mems[i].c_str()); 149 | if (!isamember(mems[i], newmem) && me != mems[i]) { 150 | tprintf("config::paxos_commit: delete %s\n", mems[i].c_str()); 151 | mgr.delete_handle(mems[i]); 152 | } 153 | } 154 | 155 | mems = newmem; 156 | myvid = instance; 157 | if (vc) { 158 | unsigned vid = myvid; 159 | VERIFY(pthread_mutex_unlock(&cfg_mutex)==0); 160 | vc->commit_change(vid); 161 | VERIFY(pthread_mutex_lock(&cfg_mutex)==0); 162 | } 163 | } 164 | 165 | bool 166 | config::ismember(std::string m, unsigned vid) 167 | { 168 | bool r; 169 | ScopedLock ml(&cfg_mutex); 170 | std::vector v = get_view_wo(vid); 171 | r = isamember(m, v); 172 | return r; 173 | } 174 | 175 | bool 176 | config::add(std::string new_m, unsigned vid) 177 | { 178 | std::vector m; 179 | std::vector curm; 180 | ScopedLock ml(&cfg_mutex); 181 | if (vid != myvid) 182 | return false; 183 | tprintf("config::add %s\n", new_m.c_str()); 184 | m = mems; 185 | m.push_back(new_m); 186 | curm = mems; 187 | std::string v = value(m); 188 | int nextvid = myvid + 1; 189 | VERIFY(pthread_mutex_unlock(&cfg_mutex)==0); 190 | bool r = pro->run(nextvid, curm, v); 191 | VERIFY(pthread_mutex_lock(&cfg_mutex)==0); 192 | if (r) { 193 | tprintf("config::add: proposer returned success\n"); 194 | } else { 195 | tprintf("config::add: proposer returned failure\n"); 196 | } 197 | return r; 198 | } 199 | 200 | // caller should hold cfg_mutex 201 | bool 202 | config::remove_wo(std::string m) 203 | { 204 | tprintf("config::remove: myvid %d remove? %s\n", myvid, m.c_str()); 205 | std::vector n; 206 | for (unsigned i = 0; i < mems.size(); i++) { 207 | if (mems[i] != m) n.push_back(mems[i]); 208 | } 209 | std::string v = value(n); 210 | std::vector cmems = mems; 211 | int nextvid = myvid + 1; 212 | VERIFY(pthread_mutex_unlock(&cfg_mutex)==0); 213 | bool r = pro->run(nextvid, cmems, v); 214 | VERIFY(pthread_mutex_lock(&cfg_mutex)==0); 215 | if (r) { 216 | tprintf("config::remove: proposer returned success\n"); 217 | } else { 218 | tprintf("config::remove: proposer returned failure\n"); 219 | } 220 | return r; 221 | } 222 | 223 | void 224 | config::heartbeater() 225 | { 226 | struct timeval now; 227 | struct timespec next_timeout; 228 | std::string m; 229 | heartbeat_t h; 230 | bool stable; 231 | unsigned vid; 232 | std::vector cmems; 233 | ScopedLock ml(&cfg_mutex); 234 | 235 | while (1) { 236 | 237 | gettimeofday(&now, NULL); 238 | next_timeout.tv_sec = now.tv_sec + 3; 239 | next_timeout.tv_nsec = 0; 240 | tprintf("heartbeater: go to sleep\n"); 241 | pthread_cond_timedwait(&config_cond, &cfg_mutex, &next_timeout); 242 | 243 | stable = true; 244 | vid = myvid; 245 | cmems = get_view_wo(vid); 246 | tprintf("heartbeater: current membership %s\n", print_members(cmems).c_str()); 247 | 248 | if (!isamember(me, cmems)) { 249 | tprintf("heartbeater: not member yet; skip hearbeat\n"); 250 | continue; 251 | } 252 | 253 | //find the node with the smallest id 254 | m = me; 255 | for (unsigned i = 0; i < cmems.size(); i++) { 256 | if (m > cmems[i]) 257 | m = cmems[i]; 258 | } 259 | 260 | if (m == me) { 261 | //if i am the one with smallest id, ping the rest of the nodes 262 | for (unsigned i = 0; i < cmems.size(); i++) { 263 | if (cmems[i] != me) { 264 | if ((h = doheartbeat(cmems[i])) != OK) { 265 | stable = false; 266 | m = cmems[i]; 267 | break; 268 | } 269 | } 270 | } 271 | } else { 272 | //the rest of the nodes ping the one with smallest id 273 | if ((h = doheartbeat(m)) != OK) 274 | stable = false; 275 | } 276 | 277 | if (!stable && vid == myvid) { 278 | remove_wo(m); 279 | } 280 | } 281 | } 282 | 283 | paxos_protocol::status 284 | config::heartbeat(std::string m, unsigned vid, int &r) 285 | { 286 | ScopedLock ml(&cfg_mutex); 287 | int ret = paxos_protocol::ERR; 288 | r = (int) myvid; 289 | tprintf("heartbeat from %s(%d) myvid %d\n", m.c_str(), vid, myvid); 290 | if (vid == myvid) { 291 | ret = paxos_protocol::OK; 292 | } else if (pro->isrunning()) { 293 | VERIFY (vid == myvid + 1 || vid + 1 == myvid); 294 | ret = paxos_protocol::OK; 295 | } else { 296 | ret = paxos_protocol::ERR; 297 | } 298 | return ret; 299 | } 300 | 301 | config::heartbeat_t 302 | config::doheartbeat(std::string m) 303 | { 304 | int ret = rpc_const::timeout_failure; 305 | int r; 306 | unsigned vid = myvid; 307 | heartbeat_t res = OK; 308 | 309 | tprintf("doheartbeater to %s (%d)\n", m.c_str(), vid); 310 | handle h(m); 311 | VERIFY(pthread_mutex_unlock(&cfg_mutex)==0); 312 | rpcc *cl = h.safebind(); 313 | if (cl) { 314 | ret = cl->call(paxos_protocol::heartbeat, me, vid, r, 315 | rpcc::to(1000)); 316 | } 317 | VERIFY(pthread_mutex_lock(&cfg_mutex)==0); 318 | if (ret != paxos_protocol::OK) { 319 | if (ret == rpc_const::atmostonce_failure || 320 | ret == rpc_const::oldsrv_failure) { 321 | mgr.delete_handle(m); 322 | } else { 323 | tprintf("doheartbeat: problem with %s (%d) my vid %d his vid %d\n", 324 | m.c_str(), ret, vid, r); 325 | if (ret < 0) res = FAILURE; 326 | else res = VIEWERR; 327 | } 328 | } 329 | tprintf("doheartbeat done %d\n", res); 330 | return res; 331 | } 332 | 333 | -------------------------------------------------------------------------------- /config.h: -------------------------------------------------------------------------------- 1 | #ifndef config_h 2 | #define config_h 3 | 4 | #include 5 | #include 6 | #include "paxos.h" 7 | 8 | class config_view_change { 9 | public: 10 | virtual void commit_change(unsigned vid) = 0; 11 | virtual ~config_view_change() {}; 12 | }; 13 | 14 | class config : public paxos_change { 15 | private: 16 | acceptor *acc; 17 | proposer *pro; 18 | rpcs *pxsrpc; 19 | unsigned myvid; 20 | std::string first; 21 | std::string me; 22 | config_view_change *vc; 23 | std::vector mems; 24 | pthread_mutex_t cfg_mutex; 25 | pthread_cond_t heartbeat_cond; 26 | pthread_cond_t config_cond; 27 | paxos_protocol::status heartbeat(std::string m, unsigned instance, int &r); 28 | std::string value(std::vector mems); 29 | std::vector members(std::string v); 30 | std::vector get_view_wo(unsigned instance); 31 | bool remove_wo(std::string); 32 | void reconstruct(); 33 | typedef enum { 34 | OK, // response and same view # 35 | VIEWERR, // response but different view # 36 | FAILURE, // no response 37 | } heartbeat_t; 38 | heartbeat_t doheartbeat(std::string m); 39 | public: 40 | config(std::string _first, std::string _me, config_view_change *_vc); 41 | unsigned vid() { return myvid; } 42 | std::string myaddr() { return me; }; 43 | std::string dump() { return acc->dump(); }; 44 | std::vector get_view(unsigned instance); 45 | void restore(std::string s); 46 | bool add(std::string, unsigned vid); 47 | bool ismember(std::string m, unsigned vid); 48 | void heartbeater(void); 49 | void paxos_commit(unsigned instance, std::string v); 50 | rpcs *get_rpcs() { return acc->get_rpcs(); } 51 | void breakpoint(int b) { pro->breakpoint(b); } 52 | }; 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /extent_client.cc: -------------------------------------------------------------------------------- 1 | // RPC stubs for clients to talk to extent_server 2 | 3 | #include "extent_client.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | namespace { 11 | 12 | inline bool isfile(extent_protocol::extentid_t id) { 13 | if (id & 0x80000000) { 14 | return true; 15 | } 16 | return false; 17 | } 18 | 19 | inline bool isdir(extent_protocol::extentid_t id) { 20 | return !isfile(id); 21 | } 22 | 23 | } 24 | 25 | 26 | // The calls assume that the caller holds a lock on the extent 27 | 28 | extent_client::extent_client(std::string dst) { 29 | sockaddr_in dstsock; 30 | make_sockaddr(dst.c_str(), &dstsock); 31 | cl = new rpcc(dstsock); 32 | if (cl->bind() != 0) { 33 | printf("extent_client: bind failed\n"); 34 | } 35 | } 36 | 37 | extent_protocol::status 38 | extent_client::get(extent_protocol::extentid_t eid, std::string &buf) { 39 | decltype(cache_.begin()) iter; 40 | { 41 | std::unique_lock m_(cache_mtx_); 42 | iter = cache_.find(eid); 43 | if (cache_.end() != iter && (iter->second.status == extent_protocol::ALL_CACHED) ) { 44 | buf = iter->second.buf; 45 | iter->second.attr.atime = std::time(nullptr); 46 | return extent_protocol::OK; 47 | } 48 | } 49 | 50 | cl->call(extent_protocol::get, eid, buf); 51 | 52 | { 53 | extent_protocol::attr attr; 54 | getattr(eid, attr); 55 | std::unique_lock m_(cache_mtx_); 56 | if (cache_.end() != iter) { 57 | iter->second.buf = buf; 58 | iter->second.attr.atime = std::time(nullptr); 59 | iter->second.status = extent_protocol::ALL_CACHED; 60 | } else { 61 | cache_[eid] = {eid, attr, buf, false, extent_protocol::ALL_CACHED}; 62 | } 63 | 64 | } 65 | 66 | 67 | return extent_protocol::OK; 68 | } 69 | 70 | extent_protocol::status 71 | extent_client::getattr(extent_protocol::extentid_t eid, 72 | extent_protocol::attr &attr) { 73 | if (isdir(eid)) { 74 | return cl->call(extent_protocol::getattr, eid, attr); 75 | } 76 | decltype(cache_.begin()) iter; 77 | { 78 | std::unique_lock m_(cache_mtx_); 79 | iter = cache_.find(eid); 80 | if (cache_.end() != iter && (iter->second.status & extent_protocol::ATTR_CACHED) ) { 81 | attr = iter->second.attr; 82 | return extent_protocol::OK; 83 | } 84 | } 85 | 86 | extent_protocol::status ret = extent_protocol::OK; 87 | ret = cl->call(extent_protocol::getattr, eid, attr); 88 | 89 | { 90 | std::unique_lock m_(cache_mtx_); 91 | if (cache_.end() != iter) { 92 | iter->second.attr = attr; 93 | iter->second.status |= extent_protocol::ATTR_CACHED; 94 | } else { 95 | cache_[eid] = {eid, attr, std::string(), false, extent_protocol::ATTR_CACHED}; 96 | } 97 | 98 | } 99 | 100 | return ret; 101 | } 102 | 103 | extent_protocol::status 104 | extent_client::put(extent_protocol::extentid_t eid, std::string buf) { 105 | decltype(cache_.begin()) iter; 106 | std::unique_lock m_(cache_mtx_); 107 | iter = cache_.find(eid); 108 | if (cache_.end() != iter && (iter->second.status == extent_protocol::ALL_CACHED)) { 109 | iter->second.buf = buf; 110 | iter->second.attr.mtime = iter->second.attr.atime = std::time(nullptr); 111 | iter->second.attr.size = buf.size(); 112 | iter->second.modified = true; 113 | return extent_protocol::OK; 114 | } 115 | 116 | extent_protocol::status ret = extent_protocol::OK; 117 | extent_protocol::attr attr; 118 | attr.atime = attr.mtime = attr.ctime = std::time(nullptr); 119 | attr.size = buf.size(); 120 | cache_[eid] = {eid, attr, buf, true, extent_protocol::ALL_CACHED}; 121 | return ret; 122 | } 123 | 124 | extent_protocol::status 125 | extent_client::remove(extent_protocol::extentid_t eid) { 126 | { 127 | std::unique_lock m_(cache_mtx_); 128 | auto iter = cache_.find(eid); 129 | if (cache_.end() != iter) { 130 | cache_.erase(iter); 131 | } 132 | } 133 | extent_protocol::status ret = extent_protocol::OK; 134 | int r; 135 | ret = cl->call(extent_protocol::remove, eid, r); 136 | return ret; 137 | } 138 | 139 | 140 | extent_protocol::status extent_client::create(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t id){ 141 | int r; 142 | return cl->call(extent_protocol::create, pid, name, id, r); 143 | } 144 | 145 | extent_protocol::status extent_client::lookup(extent_protocol::extentid_t pid, std::string name, 146 | extent_protocol::extentid_t &id) { 147 | return cl->call(extent_protocol::lookup, pid, name, id); 148 | } 149 | extent_protocol::status extent_client::readdir(extent_protocol::extentid_t pid, std::map &ents){ 150 | return cl->call(extent_protocol::readdir, pid, ents); 151 | } 152 | 153 | extent_protocol::status extent_client::flush(extent_protocol::extentid_t id) { 154 | if (isdir(id) ) { 155 | return extent_protocol::OK; 156 | } 157 | 158 | extent_cache cache; 159 | { 160 | std::unique_lock m_(cache_mtx_); 161 | auto iter = cache_.find(id); 162 | if (cache_.end() != iter) { 163 | cache = iter->second; 164 | cache_.erase(iter); 165 | } else { 166 | return extent_protocol::OK; 167 | } 168 | } 169 | 170 | if (cache.modified) { 171 | int r; 172 | cl->call(extent_protocol::flush, id, cache.buf, cache.attr, cache.status, r); 173 | 174 | } 175 | 176 | return extent_protocol::OK; 177 | } 178 | -------------------------------------------------------------------------------- /extent_client.h: -------------------------------------------------------------------------------- 1 | // extent client interface. 2 | 3 | #ifndef extent_client_h 4 | #define extent_client_h 5 | 6 | #include 7 | #include 8 | #include 9 | #include "extent_protocol.h" 10 | #include "rpc.h" 11 | 12 | class yfs_client; 13 | 14 | class extent_client { 15 | rpcc *cl; 16 | 17 | struct extent_cache { 18 | extent_protocol::extentid_t id; 19 | extent_protocol::attr attr; 20 | std::string buf; 21 | bool modified; 22 | int status; 23 | }; 24 | 25 | std::map cache_; 26 | std::mutex cache_mtx_; 27 | 28 | public: 29 | extent_client(std::string dst); 30 | 31 | extent_protocol::status get(extent_protocol::extentid_t eid, 32 | std::string &buf); 33 | 34 | extent_protocol::status getattr(extent_protocol::extentid_t eid, 35 | extent_protocol::attr &a); 36 | 37 | extent_protocol::status put(extent_protocol::extentid_t eid, std::string buf); 38 | 39 | extent_protocol::status remove(extent_protocol::extentid_t eid); 40 | 41 | extent_protocol::status flush(extent_protocol::extentid_t eid); 42 | // extent_protocol::status create(bool is_dir, extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t &id); 43 | extent_protocol::status create(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t id); 44 | extent_protocol::status lookup(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t &id); 45 | extent_protocol::status readdir(extent_protocol::extentid_t , std::map &); 46 | }; 47 | 48 | #endif 49 | 50 | -------------------------------------------------------------------------------- /extent_protocol.h: -------------------------------------------------------------------------------- 1 | // extent wire protocol 2 | 3 | #ifndef extent_protocol_h 4 | #define extent_protocol_h 5 | 6 | #include "rpc.h" 7 | 8 | class extent_protocol { 9 | public: 10 | typedef int status; 11 | typedef unsigned long long extentid_t; 12 | enum xxstatus { OK, RPCERR, NOENT, IOERR, EXIST }; 13 | 14 | enum cache_status { 15 | BUF_CACHED = 1, 16 | ATTR_CACHED = 2, 17 | ALL_CACHED = 3 18 | }; 19 | 20 | enum rpc_numbers { 21 | put = 0x6001, 22 | get, 23 | getattr, 24 | remove, 25 | create, 26 | lookup, 27 | readdir, 28 | flush 29 | }; 30 | 31 | struct attr { 32 | unsigned int atime; 33 | unsigned int mtime; 34 | unsigned int ctime; 35 | unsigned int size; 36 | }; 37 | 38 | }; 39 | 40 | 41 | 42 | 43 | inline unmarshall & 44 | operator>>(unmarshall &u, extent_protocol::attr &a) 45 | { 46 | u >> a.atime; 47 | u >> a.mtime; 48 | u >> a.ctime; 49 | u >> a.size; 50 | return u; 51 | } 52 | 53 | inline marshall & 54 | operator<<(marshall &m, extent_protocol::attr a) 55 | { 56 | m << a.atime; 57 | m << a.mtime; 58 | m << a.ctime; 59 | m << a.size; 60 | return m; 61 | } 62 | 63 | 64 | 65 | 66 | #endif 67 | -------------------------------------------------------------------------------- /extent_server.cc: -------------------------------------------------------------------------------- 1 | // the extent server implementation 2 | 3 | #include "extent_server.h" 4 | #include "rpc/marshall.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | extent_server::extent_server(): rand_(std::random_device()()){ 15 | extents_[1] = std::make_shared(1, 0, "", 0); 16 | } 17 | 18 | 19 | int extent_server::put(extent_protocol::extentid_t id, std::string buf, int &ret) { 20 | // You fill this in for Lab 2. 21 | printf("=== PUT for %llu: %s\n", id, buf.data()); 22 | std::unique_lock m_(mtx_); 23 | auto iter = extents_.find(id); 24 | if (iter != extents_.end()){ 25 | // update name and mtime, atime, size 26 | iter->second->buf = buf; 27 | iter->second->attr.mtime = iter->second->attr.atime = std::time(nullptr); 28 | iter->second->attr.size = iter->second->buf.size(); 29 | ret = iter->second->buf.size(); 30 | return extent_protocol::OK; 31 | } 32 | return extent_protocol::NOENT; 33 | } 34 | 35 | int extent_server::get(extent_protocol::extentid_t id, std::string &buf) { 36 | // You fill this in for Lab 2. 37 | printf("=== GET for %llu \n", id); 38 | std::unique_lock m_(mtx_); 39 | auto iter = extents_.find(id); 40 | if (iter != extents_.end()){ 41 | buf = iter->second->buf; 42 | iter->second->attr.atime = std::time(nullptr); 43 | return extent_protocol::OK; 44 | } 45 | return extent_protocol::NOENT; 46 | } 47 | 48 | int extent_server::getattr(extent_protocol::extentid_t id, extent_protocol::attr &a) { 49 | // You fill this in for Lab 2. 50 | // You replace this with a real implementation. We send a phony response 51 | // for now because it's difficult to get FUSE to do anything (including 52 | // unmount) if getattr fails. 53 | std::unique_lock m_(mtx_); 54 | auto iter = extents_.find(id); 55 | if (iter != extents_.end()){ 56 | a = iter->second->attr; 57 | return extent_protocol::OK; 58 | } 59 | return extent_protocol::NOENT; 60 | } 61 | 62 | int extent_server::remove(extent_protocol::extentid_t id, int &) { 63 | // You fill this in for Lab 2. 64 | std::unique_lock m_(mtx_); 65 | auto it = extents_.find(id); 66 | if (it != extents_.end()){ 67 | auto piter = extents_.find(it->second->parent_id); 68 | std::list &chdlst = piter->second->chd; 69 | for (auto lstiter = chdlst.begin(); lstiter != chdlst.end(); ++lstiter){ 70 | if (lstiter->eid == id){ 71 | chdlst.erase(lstiter); 72 | break; 73 | } 74 | } 75 | piter->second->attr.mtime = piter->second->attr.ctime = std::time(nullptr); 76 | extents_.erase(it); 77 | return extent_protocol::OK; 78 | } 79 | return extent_protocol::NOENT; 80 | } 81 | int extent_server::create(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t id, int &){ 82 | std::unique_lock m_(mtx_); 83 | 84 | printf("CREATE:: %s, %llu\n", name.data(), id); 85 | 86 | auto piter = extents_.find(pid); 87 | if (piter == extents_.end() || isfile(pid)){ 88 | return extent_protocol::IOERR; 89 | } 90 | 91 | std::list &chdlst = piter->second->chd; 92 | auto lstiter = chdlst.cbegin(); 93 | for (; lstiter != chdlst.cend(); ++lstiter){ 94 | if (lstiter->name == name){ 95 | return extent_protocol::EXIST; 96 | } else if (lstiter->name > name){ 97 | break; 98 | } 99 | } 100 | chdlst.insert(lstiter, dir_ent(name, id)); 101 | piter->second->attr.mtime = piter->second->attr.ctime = std::time(nullptr); 102 | extents_[id] = std::make_shared(id, pid, name, 0); 103 | return extent_protocol::OK; 104 | } 105 | 106 | int extent_server::lookup( extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t &ret) { 107 | 108 | ret = 0; 109 | std::unique_lock m_(mtx_); 110 | auto piter = extents_.find(pid); 111 | if (piter == extents_.end()){ 112 | return extent_protocol::IOERR; 113 | } 114 | std::list &chdlst = piter->second->chd; 115 | auto lstiter = chdlst.cbegin(); 116 | for (; lstiter != chdlst.cend(); ++lstiter){ 117 | if (lstiter->name == name){ 118 | ret = lstiter->eid; 119 | return extent_protocol::OK; 120 | } else if (lstiter->name > name){ 121 | return extent_protocol::NOENT; 122 | } 123 | } 124 | return extent_protocol::NOENT; 125 | } 126 | 127 | int extent_server::readdir(extent_protocol::extentid_t pid, std::map &ents){ 128 | std::unique_lock m_(mtx_); 129 | auto piter = extents_.find(pid); 130 | if (piter == extents_.end() || isfile(pid)){ 131 | return extent_protocol::NOENT; 132 | } 133 | 134 | std::list &chdlst = piter->second->chd; 135 | for (auto &item : chdlst){ 136 | ents[item.name] = item.eid; 137 | } 138 | 139 | return extent_protocol::OK; 140 | } 141 | 142 | int extent_server::flush(extent_protocol::extentid_t id, std::string buff, extent_protocol::attr attr, int status, int &) { 143 | std::unique_lock m_(mtx_); 144 | auto iter = extents_.find(id); 145 | if (iter != extents_.end()){ 146 | // update name and mtime, atime, size 147 | if (status & extent_protocol::BUF_CACHED) { 148 | iter->second->buf = buff; 149 | } 150 | if (status & extent_protocol::ATTR_CACHED) { 151 | iter->second->attr = attr; 152 | } 153 | return extent_protocol::OK; 154 | } 155 | return extent_protocol::NOENT; 156 | } 157 | -------------------------------------------------------------------------------- /extent_server.h: -------------------------------------------------------------------------------- 1 | // this is the extent server 2 | 3 | #ifndef extent_server_h 4 | #define extent_server_h 5 | 6 | #include 7 | 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "extent_protocol.h" 15 | #include "yfs_client.h" 16 | 17 | struct dir_ent{ 18 | std::string name; 19 | extent_protocol::extentid_t eid; 20 | 21 | dir_ent(const std::string &nm, extent_protocol::extentid_t id): name(nm), eid(id){} 22 | dir_ent(dir_ent &&rhs): name(std::move(rhs.name)), eid(rhs.eid){} 23 | }; 24 | 25 | struct extent_entry { 26 | extent_protocol::extentid_t eid; 27 | extent_protocol::extentid_t parent_id; // 0 for not set, 1 for '/' 28 | // @name is not essential, will be removed. 29 | std::string name; 30 | extent_protocol::attr attr; 31 | 32 | std::list chd; 33 | std::string buf; 34 | int n_chd; 35 | 36 | extent_entry (const extent_entry &rhs):eid(rhs.eid), parent_id(rhs.parent_id), name(rhs.name), attr(rhs.attr), n_chd(0){} 37 | extent_entry (extent_entry &&rhs):eid(rhs.eid), parent_id(rhs.parent_id), name(std::move(rhs.name)), attr(rhs.attr), n_chd(0){} 38 | extent_entry& operator=(extent_entry &&rhs){ 39 | if (&rhs != this){ 40 | eid = rhs.eid; 41 | parent_id = rhs.parent_id; 42 | name = std::move(rhs.name); 43 | attr = rhs.attr; 44 | n_chd = rhs.n_chd; 45 | } 46 | return *this; 47 | } 48 | extent_entry () = default; 49 | extent_entry (extent_protocol::extentid_t id, extent_protocol::extentid_t pid, std::string nm, int sz = 0) 50 | : eid(id), parent_id(pid), name(nm), attr({0,0,0,0}), buf(""), n_chd(0){ 51 | attr.atime = attr.ctime = attr.mtime = std::time(nullptr); 52 | attr.size = sz; 53 | } 54 | }; 55 | 56 | class extent_server { 57 | std::mutex mtx_; 58 | std::map > extents_; 59 | std::mt19937 rand_; 60 | 61 | 62 | bool isfile(extent_protocol::extentid_t id){ 63 | return (id & 0x80000000) != 0; 64 | } 65 | public: 66 | extent_server(); 67 | 68 | int put(extent_protocol::extentid_t id, std::string, int &); 69 | 70 | int get(extent_protocol::extentid_t id, std::string &); 71 | 72 | int getattr(extent_protocol::extentid_t id, extent_protocol::attr &); 73 | 74 | int remove(extent_protocol::extentid_t id, int &); 75 | 76 | int flush(extent_protocol::extentid_t, std::string, extent_protocol::attr, int status, int &); 77 | 78 | 79 | int create(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t id, int &); 80 | 81 | int lookup(extent_protocol::extentid_t pid, std::string name, extent_protocol::extentid_t &ret); 82 | 83 | int readdir(extent_protocol::extentid_t pid, std::map &ents); 84 | 85 | }; 86 | 87 | #endif 88 | 89 | 90 | -------------------------------------------------------------------------------- /extent_smain.cc: -------------------------------------------------------------------------------- 1 | #include "rpc.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "extent_server.h" 7 | 8 | // Main loop of extent server 9 | 10 | int 11 | main(int argc, char *argv[]) { 12 | int count = 0; 13 | 14 | if (argc != 2) { 15 | fprintf(stderr, "Usage: %s port\n", argv[0]); 16 | exit(1); 17 | } 18 | 19 | setvbuf(stdout, NULL, _IONBF, 0); 20 | 21 | char *count_env = getenv("RPC_COUNT"); 22 | if (count_env != NULL) { 23 | count = atoi(count_env); 24 | } 25 | 26 | rpcs server(atoi(argv[1]), count); 27 | extent_server ls; 28 | 29 | server.reg(extent_protocol::get, &ls, &extent_server::get); 30 | server.reg(extent_protocol::getattr, &ls, &extent_server::getattr); 31 | server.reg(extent_protocol::put, &ls, &extent_server::put); 32 | server.reg(extent_protocol::remove, &ls, &extent_server::remove); 33 | server.reg(extent_protocol::create, &ls, &extent_server::create); 34 | server.reg(extent_protocol::lookup, &ls, &extent_server::lookup); 35 | server.reg(extent_protocol::readdir, &ls, &extent_server::readdir); 36 | server.reg(extent_protocol::flush, &ls, &extent_server::flush); 37 | 38 | while (1) 39 | sleep(1000); 40 | } 41 | -------------------------------------------------------------------------------- /gettime.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c), MM Weiss 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, 6 | * are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright notice, 12 | * this list of conditions and the following disclaimer in the documentation 13 | * and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the MM Weiss nor the names of its contributors 16 | * may be used to endorse or promote products derived from this software without 17 | * specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 20 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 22 | * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 24 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 26 | * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | /* 31 | * clock_gettime_stub.c 32 | * gcc -Wall -c clock_gettime_stub.c 33 | * posix realtime functions; MacOS user space glue 34 | */ 35 | 36 | /* @comment 37 | * other possible implementation using intel builtin rdtsc 38 | * rdtsc-workaround: http://www.mcs.anl.gov/~kazutomo/rdtsc.html 39 | * 40 | * we could get the ticks by doing this 41 | * 42 | * __asm __volatile("mov %%ebx, %%esi\n\t" 43 | * "cpuid\n\t" 44 | * "xchg %%esi, %%ebx\n\t" 45 | * "rdtsc" 46 | * : "=a" (a), 47 | * "=d" (d) 48 | * ); 49 | 50 | * we could even replace our tricky sched_yield call by assembly code to get a better accurency, 51 | * anyway the following C stub will satisfy 99% of apps using posix clock_gettime call, 52 | * moreover, the setter version (clock_settime) could be easly written using mach primitives: 53 | * http://www.opensource.apple.com/source/xnu/xnu-${VERSION}/osfmk/man/ (clock_[set|get]_time) 54 | * 55 | * hackers don't be crackers, don't you use a flush toilet? 56 | * 57 | * 58 | * @see draft: ./posix-realtime-stub/posix-realtime-stub.c 59 | * 60 | */ 61 | 62 | 63 | #ifdef __APPLE__ 64 | 65 | #pragma weak clock_gettime 66 | 67 | #include 68 | #include 69 | #include 70 | #include 71 | #include 72 | #include 73 | #include 74 | #include 75 | 76 | typedef enum { 77 | CLOCK_REALTIME, 78 | CLOCK_MONOTONIC, 79 | CLOCK_PROCESS_CPUTIME_ID, 80 | CLOCK_THREAD_CPUTIME_ID 81 | } clockid_t; 82 | 83 | static mach_timebase_info_data_t __clock_gettime_inf; 84 | 85 | int clock_gettime(clockid_t clk_id, struct timespec *tp) { 86 | kern_return_t ret; 87 | clock_serv_t clk; 88 | clock_id_t clk_serv_id; 89 | mach_timespec_t tm; 90 | 91 | uint64_t start, end, delta, nano; 92 | 93 | task_basic_info_data_t tinfo; 94 | task_thread_times_info_data_t ttinfo; 95 | mach_msg_type_number_t tflag; 96 | 97 | int retval = -1; 98 | switch (clk_id) { 99 | case CLOCK_REALTIME: 100 | case CLOCK_MONOTONIC: 101 | clk_serv_id = clk_id == CLOCK_REALTIME ? CALENDAR_CLOCK : SYSTEM_CLOCK; 102 | if (KERN_SUCCESS == (ret = host_get_clock_service(mach_host_self(), clk_serv_id, &clk))) { 103 | if (KERN_SUCCESS == (ret = clock_get_time(clk, &tm))) { 104 | tp->tv_sec = tm.tv_sec; 105 | tp->tv_nsec = tm.tv_nsec; 106 | retval = 0; 107 | } 108 | } 109 | if (KERN_SUCCESS != ret) { 110 | errno = EINVAL; 111 | retval = -1; 112 | } 113 | break; 114 | case CLOCK_PROCESS_CPUTIME_ID: 115 | case CLOCK_THREAD_CPUTIME_ID: 116 | start = mach_absolute_time(); 117 | if (clk_id == CLOCK_PROCESS_CPUTIME_ID) { 118 | getpid(); 119 | } else { 120 | sched_yield(); 121 | } 122 | end = mach_absolute_time(); 123 | delta = end - start; 124 | if (0 == __clock_gettime_inf.denom) { 125 | mach_timebase_info(&__clock_gettime_inf); 126 | } 127 | nano = delta * __clock_gettime_inf.numer / __clock_gettime_inf.denom; 128 | tp->tv_sec = nano * 1e-9; 129 | tp->tv_nsec = nano - (tp->tv_sec * 1e9); 130 | retval = 0; 131 | break; 132 | default: 133 | errno = EINVAL; 134 | retval = -1; 135 | } 136 | return retval; 137 | } 138 | 139 | #endif // __APPLE__ 140 | -------------------------------------------------------------------------------- /gettime.h: -------------------------------------------------------------------------------- 1 | #ifndef gettime_h 2 | #define gettime_h 3 | 4 | #ifdef __APPLE__ 5 | typedef enum { 6 | CLOCK_REALTIME, 7 | CLOCK_MONOTONIC, 8 | CLOCK_PROCESS_CPUTIME_ID, 9 | CLOCK_THREAD_CPUTIME_ID 10 | } clockid_t; 11 | 12 | int clock_gettime(clockid_t clk_id, struct timespec *tp); 13 | #endif 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /handle.cc: -------------------------------------------------------------------------------- 1 | #include "handle.h" 2 | #include 3 | #include "tprintf.h" 4 | 5 | handle_mgr mgr; 6 | 7 | handle::handle(std::string m) 8 | { 9 | h = mgr.get_handle(m); 10 | } 11 | 12 | rpcc * 13 | handle::safebind() 14 | { 15 | if (!h) 16 | return NULL; 17 | ScopedLock ml(&h->cl_mutex); 18 | if (h->del) 19 | return NULL; 20 | if (h->cl) 21 | return h->cl; 22 | sockaddr_in dstsock; 23 | make_sockaddr(h->m.c_str(), &dstsock); 24 | rpcc *cl = new rpcc(dstsock); 25 | //tprintf("handler_mgr::get_handle trying to bind...%s\n", h->m.c_str()); 26 | int ret; 27 | // Starting with lab 6, our test script assumes that the failure 28 | // can be detected by paxos and rsm layer within few seconds. We have 29 | // to set the timeout with a small value to support the assumption. 30 | // 31 | // Note: with RPC_LOSSY=5, your lab would failed to pass the tests of 32 | // lab 6 and lab 7 because the rpc layer may delay your RPC request, 33 | // and cause a time out failure. Please make sure RPC_LOSSY is set to 0. 34 | ret = cl->bind(rpcc::to(1000)); 35 | if (ret < 0) { 36 | tprintf("handle_mgr::get_handle bind failure! %s %d\n", h->m.c_str(), ret); 37 | delete cl; 38 | h->del = true; 39 | } else { 40 | //tprintf("handle_mgr::get_handle bind succeeded %s\n", h->m.c_str()); 41 | h->cl = cl; 42 | } 43 | return h->cl; 44 | } 45 | 46 | handle::~handle() 47 | { 48 | if (h) mgr.done_handle(h); 49 | } 50 | 51 | handle_mgr::handle_mgr() 52 | { 53 | VERIFY (pthread_mutex_init(&handle_mutex, NULL) == 0); 54 | } 55 | 56 | struct hinfo * 57 | handle_mgr::get_handle(std::string m) 58 | { 59 | ScopedLock ml(&handle_mutex); 60 | struct hinfo *h = 0; 61 | if (hmap.find(m) == hmap.end()) { 62 | h = new hinfo; 63 | h->cl = NULL; 64 | h->del = false; 65 | h->refcnt = 1; 66 | h->m = m; 67 | pthread_mutex_init(&h->cl_mutex, NULL); 68 | hmap[m] = h; 69 | } else if (!hmap[m]->del) { 70 | h = hmap[m]; 71 | h->refcnt ++; 72 | } 73 | return h; 74 | } 75 | 76 | void 77 | handle_mgr::done_handle(struct hinfo *h) 78 | { 79 | ScopedLock ml(&handle_mutex); 80 | h->refcnt--; 81 | if (h->refcnt == 0 && h->del) 82 | delete_handle_wo(h->m); 83 | } 84 | 85 | void 86 | handle_mgr::delete_handle(std::string m) 87 | { 88 | ScopedLock ml(&handle_mutex); 89 | delete_handle_wo(m); 90 | } 91 | 92 | // Must be called with handle_mutex locked. 93 | void 94 | handle_mgr::delete_handle_wo(std::string m) 95 | { 96 | if (hmap.find(m) == hmap.end()) { 97 | tprintf("handle_mgr::delete_handle_wo: cl %s isn't in cl list\n", m.c_str()); 98 | } else { 99 | tprintf("handle_mgr::delete_handle_wo: cl %s refcnt %d\n", m.c_str(), 100 | hmap[m]->refcnt); 101 | struct hinfo *h = hmap[m]; 102 | if (h->refcnt == 0) { 103 | if (h->cl) { 104 | h->cl->cancel(); 105 | delete h->cl; 106 | } 107 | pthread_mutex_destroy(&h->cl_mutex); 108 | hmap.erase(m); 109 | delete h; 110 | } else { 111 | h->del = true; 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /handle.h: -------------------------------------------------------------------------------- 1 | // manage a cache of RPC connections. 2 | // assuming cid is a std::string holding the 3 | // host:port of the RPC server you want 4 | // to talk to: 5 | // 6 | // handle h(cid); 7 | // rpcc *cl = h.safebind(); 8 | // if(cl){ 9 | // ret = cl->call(...); 10 | // } else { 11 | // bind() failed 12 | // } 13 | // 14 | // if the calling program has not contacted 15 | // cid before, safebind() will create a new 16 | // connection, call bind(), and return 17 | // an rpcc*, or 0 if bind() failed. if the 18 | // program has previously contacted cid, 19 | // safebind() just returns the previously 20 | // created rpcc*. best not to hold any 21 | // mutexes while calling safebind(). 22 | 23 | #ifndef handle_h 24 | #define handle_h 25 | 26 | #include 27 | #include 28 | #include "rpc.h" 29 | 30 | struct hinfo { 31 | rpcc *cl; 32 | int refcnt; 33 | bool del; 34 | std::string m; 35 | pthread_mutex_t cl_mutex; 36 | }; 37 | 38 | class handle { 39 | private: 40 | struct hinfo *h; 41 | public: 42 | handle(std::string m); 43 | ~handle(); 44 | /* safebind will try to bind with the rpc server on the first call. 45 | * Since bind may block, the caller probably should not hold a mutex 46 | * when calling safebind. 47 | * 48 | * return: 49 | * if the first safebind succeeded, all later calls would return 50 | * a rpcc object; otherwise, all later calls would return NULL. 51 | * 52 | * Example: 53 | * handle h(dst); 54 | * XXX_protocol::status ret; 55 | * if (h.safebind()) { 56 | * ret = h.safebind()->call(...); 57 | * } 58 | * if (!h.safebind() || ret != XXX_protocol::OK) { 59 | * // handle failure 60 | * } 61 | */ 62 | rpcc *safebind(); 63 | }; 64 | 65 | class handle_mgr { 66 | private: 67 | pthread_mutex_t handle_mutex; 68 | std::map hmap; 69 | public: 70 | handle_mgr(); 71 | struct hinfo *get_handle(std::string m); 72 | void done_handle(struct hinfo *h); 73 | void delete_handle(std::string m); 74 | void delete_handle_wo(std::string m); 75 | }; 76 | 77 | extern class handle_mgr mgr; 78 | 79 | #endif 80 | -------------------------------------------------------------------------------- /lang/algorithm.h: -------------------------------------------------------------------------------- 1 | // compile time version of min and max 2 | 3 | #ifndef algorithm_h 4 | #define algorithm_h 5 | 6 | template 7 | struct static_max 8 | { 9 | static const int value = A > B ? A : B; 10 | }; 11 | 12 | template 13 | struct static_min 14 | { 15 | static const int value = A < B ? A : B; 16 | }; 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /lang/verify.h: -------------------------------------------------------------------------------- 1 | // safe assertions. 2 | 3 | #ifndef verify_client_h 4 | #define verify_client_h 5 | 6 | #include 7 | #include 8 | 9 | #ifdef NDEBUG 10 | #define VERIFY(expr) do { if (!(expr)) abort(); } while (0) 11 | #else 12 | #define VERIFY(expr) assert(expr) 13 | #endif 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /lock_client.cc: -------------------------------------------------------------------------------- 1 | // RPC stubs for clients to talk to lock_server 2 | 3 | #include "lock_client.h" 4 | #include "rpc.h" 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | lock_client::lock_client(std::string dst) 12 | { 13 | sockaddr_in dstsock; 14 | make_sockaddr(dst.c_str(), &dstsock); 15 | cl = new rpcc(dstsock); 16 | if (cl->bind() < 0) { 17 | printf("lock_client: call bind\n"); 18 | } 19 | } 20 | 21 | int 22 | lock_client::stat(lock_protocol::lockid_t lid) 23 | { 24 | int r; 25 | lock_protocol::status ret = cl->call(lock_protocol::stat, cl->id(), lid, r); 26 | VERIFY (ret == lock_protocol::OK); 27 | return r; 28 | } 29 | 30 | lock_protocol::status 31 | lock_client::acquire(lock_protocol::lockid_t lid) 32 | { 33 | int r; 34 | lock_protocol::status ret = cl->call(lock_protocol::acquire, cl->id(), lid, r); 35 | VERIFY(ret == lock_protocol::OK); 36 | return r; 37 | } 38 | 39 | lock_protocol::status 40 | lock_client::release(lock_protocol::lockid_t lid) 41 | { 42 | int r; 43 | lock_protocol::status ret = cl->call(lock_protocol::release, cl->id(), lid, r); 44 | VERIFY(ret == lock_protocol::OK); 45 | return r; 46 | } 47 | 48 | -------------------------------------------------------------------------------- /lock_client.h: -------------------------------------------------------------------------------- 1 | // lock client interface. 2 | 3 | #ifndef lock_client_h 4 | #define lock_client_h 5 | 6 | #include 7 | #include "lock_protocol.h" 8 | #include "rpc.h" 9 | #include 10 | 11 | // Client interface to the lock server 12 | class lock_client { 13 | protected: 14 | rpcc *cl; 15 | public: 16 | lock_client(std::string d); 17 | 18 | virtual ~lock_client() { }; 19 | 20 | virtual lock_protocol::status acquire(lock_protocol::lockid_t); 21 | 22 | virtual lock_protocol::status release(lock_protocol::lockid_t); 23 | 24 | virtual lock_protocol::status stat(lock_protocol::lockid_t); 25 | }; 26 | 27 | class lc_guard { 28 | lock_client *lc; 29 | lock_protocol::lockid_t lid; 30 | public: 31 | lc_guard(lock_client *l, lock_protocol::lockid_t id) : lc(l), lid(id) { 32 | lc->acquire(lid); 33 | } 34 | 35 | ~lc_guard() { 36 | unlock(); 37 | } 38 | 39 | void lock() { 40 | lc->acquire(lid); 41 | } 42 | 43 | void unlock() { 44 | lc->release(lid); 45 | } 46 | }; 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /lock_client_cache.cc: -------------------------------------------------------------------------------- 1 | // RPC stubs for clients to talk to lock_server, and cache the locks 2 | // see lock_client.cache.h for protocol details. 3 | 4 | #include "lock_client_cache.h" 5 | #include "rpc.h" 6 | #include 7 | #include 8 | #include 9 | #include "tprintf.h" 10 | 11 | 12 | lock_client_cache::lock_client_cache(std::string xdst, 13 | class lock_release_user *_lu) 14 | : lock_client(xdst), lu(_lu) { 15 | rpcs *rlsrpc = new rpcs(0); 16 | rlsrpc->reg(rlock_protocol::revoke, this, &lock_client_cache::revoke_handler); 17 | rlsrpc->reg(rlock_protocol::retry, this, &lock_client_cache::retry_handler); 18 | 19 | const char *hname; 20 | hname = "127.0.0.1"; 21 | std::ostringstream host; 22 | host << hname << ":" << rlsrpc->port(); 23 | rlock_port = rlsrpc->port(); 24 | id = host.str(); 25 | } 26 | 27 | // Caller host the little lock 28 | lock_protocol::status 29 | lock_client_cache::racquire(client_lock *rlk, std::unique_lock &m_) { 30 | int r; 31 | rlk->status = client_lock::ACQUIRING; 32 | for (;;){ 33 | m_.unlock(); 34 | auto ret = cl->call(lock_protocol::acquire, rlk->lid, id, r); 35 | m_.lock(); 36 | if (lock_protocol::OK == ret){ 37 | rlk->status = client_lock::LOCKED; 38 | return lock_protocol::OK; 39 | } else if (lock_protocol::RETRY == ret) { 40 | if (rlk->nretry){ 41 | rlk->nretry = 0; 42 | continue; 43 | } 44 | rlk->retry_cv.wait(m_, [&](){ return rlk->nretry;}); 45 | rlk->nretry = 0; 46 | } else { 47 | return ret; 48 | } 49 | 50 | } 51 | } 52 | 53 | lock_protocol::status 54 | lock_client_cache::acquire(lock_protocol::lockid_t lid) { 55 | int ret = lock_protocol::OK; 56 | std::unique_lock mtx_(mtxtb); 57 | client_lock *rlk = nullptr; 58 | auto iter = rlocktb.find(lid); 59 | if (iter == rlocktb.end()){ 60 | rlk = new client_lock(lid); 61 | rlocktb[lid] = rlk; 62 | } else { 63 | rlk = iter->second; 64 | } 65 | 66 | 67 | std::unique_lock m_(rlk->mtx); 68 | mtx_.unlock(); 69 | 70 | // LOCKED, ACQUIRING, RELEASING wait on lock_cv 71 | if (client_lock::LOCKED == rlk->status || client_lock::ACQUIRING == rlk->status || client_lock::RELEASING == rlk->status){ 72 | rlk->lock_cv.wait(m_, [&](){ return client_lock::FREE == rlk->status || client_lock::NONE == rlk->status;}); 73 | } 74 | 75 | if (client_lock::NONE == rlk->status) { 76 | return racquire(rlk, m_); 77 | } 78 | if (client_lock::FREE == rlk->status) { 79 | rlk->status = client_lock::LOCKED; 80 | return lock_protocol::OK; 81 | } 82 | 83 | return ret; 84 | } 85 | 86 | lock_protocol::status 87 | lock_client_cache::release(lock_protocol::lockid_t lid) { 88 | std::unique_lock mtx_(mtxtb); 89 | auto iter = rlocktb.find(lid); 90 | if (iter == rlocktb.end()){ 91 | return lock_protocol::OK; 92 | } 93 | client_lock *rlk = iter->second; 94 | 95 | std::unique_lock m_(rlk->mtx); 96 | mtx_.unlock(); 97 | 98 | 99 | if (client_lock::LOCKED != rlk->status){ 100 | return lock_protocol::IOERR; 101 | } 102 | 103 | if (rlk->nrevoke){ 104 | lu->dorelease(lid); 105 | rlk->status = client_lock::RELEASING; 106 | rlk->nrevoke--; 107 | rlk->revoke_cv.notify_all(); 108 | } else { 109 | rlk->status = client_lock::FREE; 110 | rlk->lock_cv.notify_all(); 111 | } 112 | return lock_protocol::OK; 113 | } 114 | 115 | rlock_protocol::status 116 | lock_client_cache::revoke_handler(lock_protocol::lockid_t lid, 117 | int &) { 118 | 119 | std::unique_lock mtx_(mtxtb); 120 | auto iter = rlocktb.find(lid); 121 | if (iter == rlocktb.end()){ 122 | return lock_protocol::OK; 123 | } 124 | client_lock *rlk = iter->second; 125 | 126 | 127 | if (client_lock::FREE == rlk->status || client_lock::NONE == rlk->status){ 128 | lu->dorelease(rlk->lid); 129 | rlk->status = client_lock::NONE; 130 | return lock_protocol::OK; 131 | } 132 | 133 | 134 | // Wait when LOCKED or ACQUIRING 135 | rlk->nrevoke++; 136 | std::unique_lock m_(rlk->mtx); 137 | mtx_.unlock(); 138 | rlk->revoke_cv.wait(m_, [&](){ return client_lock::RELEASING == rlk->status;}); 139 | 140 | rlk->status = client_lock::NONE; 141 | rlk->lock_cv.notify_all(); 142 | return lock_protocol::OK; 143 | } 144 | 145 | rlock_protocol::status 146 | lock_client_cache::retry_handler(lock_protocol::lockid_t lid, 147 | int &) { 148 | int ret = rlock_protocol::OK; 149 | std::unique_lock mtx_(mtxtb); 150 | 151 | auto iter = rlocktb.find(lid); 152 | if (iter == rlocktb.end()){ 153 | return lock_protocol::OK; 154 | } 155 | client_lock *rlk = iter->second; 156 | 157 | std::unique_lock m_(rlk->mtx); 158 | mtx_.unlock(); 159 | 160 | // rlk->status = client_lock::NONE; 161 | rlk->nretry++; 162 | // rlk->status = client_lock::ACQUIRING; 163 | rlk->retry_cv.notify_one(); 164 | 165 | return ret; 166 | } 167 | 168 | lock_client_cache::~lock_client_cache(){ 169 | std::unique_lock mtx_(mtxtb); 170 | client_lock *rlk = nullptr; 171 | for (auto iter = rlocktb.begin(); iter != rlocktb.end(); ++iter) { 172 | rlk = iter->second; 173 | int r; 174 | tprintf("REMOTE RELEASE %llu\n", rlk->lid); 175 | lu->dorelease(rlk->lid); 176 | cl->call(lock_protocol::release, rlk->lid, id, r); 177 | delete rlk; 178 | } 179 | } 180 | 181 | -------------------------------------------------------------------------------- /lock_client_cache.h: -------------------------------------------------------------------------------- 1 | // lock client interface. 2 | 3 | #ifndef lock_client_cache_h 4 | 5 | #define lock_client_cache_h 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "lock_protocol.h" 12 | #include "rpc.h" 13 | #include "lock_client.h" 14 | #include "lang/verify.h" 15 | #include "extent_client.h" 16 | 17 | 18 | // Classes that inherit lock_release_user can override dorelease so that 19 | // that they will be called when lock_client releases a lock. 20 | // You will not need to do anything with this class until Lab 5. 21 | class lock_release_user { 22 | public: 23 | virtual void dorelease(lock_protocol::lockid_t) = 0; 24 | 25 | virtual ~lock_release_user() { }; 26 | }; 27 | 28 | class lock_release: public lock_release_user { 29 | extent_client *ec; 30 | public: 31 | lock_release(extent_client *e): ec(e) {} 32 | void dorelease(lock_protocol::lockid_t lid) override { 33 | ec->flush(lid); 34 | } 35 | 36 | }; 37 | 38 | class lock_client_cache : public lock_client { 39 | private: 40 | class lock_release_user *lu; 41 | // rpcs *rlsrpc; 42 | 43 | int rlock_port; 44 | std::string hostname; 45 | std::string id; 46 | 47 | struct client_lock { 48 | enum xxxstatus { 49 | FREE, 50 | LOCKED, 51 | NONE, 52 | ACQUIRING, 53 | RELEASING, 54 | RETRYING, 55 | }; 56 | lock_protocol::lockid_t lid; 57 | int status = NONE; 58 | int nrevoke = 0; 59 | int nretry = 0; 60 | pthread_t owner = 0; 61 | std::mutex mtx; 62 | std::condition_variable lock_cv; 63 | std::condition_variable revoke_cv; 64 | std::condition_variable retry_cv; 65 | client_lock(lock_protocol::lockid_t id): lid(id), status(NONE), nrevoke(0), nretry(0){} 66 | }; 67 | 68 | std::map rlocktb; 69 | std::mutex mtxtb; 70 | 71 | lock_protocol::status racquire(client_lock *rlk, std::unique_lock &m_); 72 | 73 | 74 | public: 75 | lock_client_cache(std::string xdst, class lock_release_user *l = 0); 76 | 77 | virtual ~lock_client_cache(); 78 | 79 | lock_protocol::status acquire(lock_protocol::lockid_t); 80 | 81 | lock_protocol::status release(lock_protocol::lockid_t); 82 | 83 | rlock_protocol::status revoke_handler(lock_protocol::lockid_t, 84 | int &); 85 | 86 | rlock_protocol::status retry_handler(lock_protocol::lockid_t, 87 | int &); 88 | }; 89 | 90 | 91 | #endif 92 | -------------------------------------------------------------------------------- /lock_client_cache_rsm.cc: -------------------------------------------------------------------------------- 1 | // RPC stubs for clients to talk to lock_server, and cache the locks 2 | // see lock_client.cache.h for protocol details. 3 | 4 | #include "lock_client_cache_rsm.h" 5 | #include "rpc.h" 6 | #include 7 | #include 8 | #include 9 | #include "tprintf.h" 10 | #include "marshall.h" 11 | 12 | #include "rsm_client.h" 13 | 14 | #include 15 | 16 | static void * 17 | releasethread(void *x) 18 | { 19 | lock_client_cache_rsm *cc = (lock_client_cache_rsm *) x; 20 | cc->releaser(); 21 | return 0; 22 | } 23 | 24 | int lock_client_cache_rsm::last_port = 0; 25 | 26 | lock_client_cache_rsm::lock_client_cache_rsm(std::string xdst, 27 | class lock_release_user *_lu) 28 | : lock_client(xdst), lu(_lu) 29 | { 30 | srand(time(NULL)^last_port); 31 | rlock_port = ((rand()%32000) | (0x1 << 10)); 32 | const char *hname; 33 | // VERIFY(gethostname(hname, 100) == 0); 34 | hname = "127.0.0.1"; 35 | std::ostringstream host; 36 | host << hname << ":" << rlock_port; 37 | id = host.str(); 38 | last_port = rlock_port; 39 | rpcs *rlsrpc = new rpcs(rlock_port); 40 | rlsrpc->reg(rlock_protocol::revoke, this, &lock_client_cache_rsm::revoke_handler); 41 | rlsrpc->reg(rlock_protocol::retry, this, &lock_client_cache_rsm::retry_handler); 42 | xid = 0; 43 | // You fill this in Step Two, Lab 7 44 | // - Create rsmc, and use the object to do RPC 45 | // calls instead of the rpcc object of lock_client 46 | rsmc = new rsm_client(xdst); 47 | //pthread_t th; 48 | int r = pthread_create(&th, NULL, &releasethread, (void *) this); 49 | VERIFY (r == 0); 50 | } 51 | 52 | // Caller held the little lock 53 | lock_protocol::status 54 | lock_client_cache_rsm::racquire(client_lock *rlk, std::unique_lock &m_) { 55 | int r; 56 | rlk->status = client_lock::ACQUIRING; 57 | 58 | lock_protocol::xid_t xid_ = ++xid; 59 | 60 | for (;;){ 61 | m_.unlock(); 62 | auto ret = rsmc->call(lock_protocol::acquire, rlk->lid, id, xid_, r); 63 | m_.lock(); 64 | if (lock_protocol::OK == ret){ 65 | rlk->status = client_lock::LOCKED; 66 | rlk->xid = xid_; 67 | return lock_protocol::OK; 68 | } else if (lock_protocol::RETRY == ret) { 69 | if (rlk->nretry){ 70 | rlk->nretry = 0; 71 | continue; 72 | } 73 | rlk->retry_cv.wait_for(m_, std::chrono::seconds(3), [&](){ return rlk->nretry;}); 74 | rlk->nretry = 0; 75 | } else { 76 | return ret; 77 | } 78 | 79 | } 80 | } 81 | 82 | void 83 | lock_client_cache_rsm::releaser() 84 | { 85 | 86 | // This method should be a continuous loop, waiting to be notified of 87 | // freed locks that have been revoked by the server, so that it can 88 | // send a release RPC. 89 | client_lock *rlk; 90 | for (;;) { 91 | release_fifo.deq(&rlk); 92 | if (rlk->nrevoke < 0) { 93 | return; 94 | } 95 | int r; 96 | int ret = lock_protocol::OK; 97 | if (lu) { 98 | lu->dorelease(rlk->lid); 99 | } 100 | ret = rsmc->call(lock_protocol::release, rlk->lid, id, rlk->xid, r); 101 | std::lock_guard m_(rlk->mtx); 102 | rlk->status = client_lock::NONE; 103 | rlk->lock_cv.notify_all(); 104 | } 105 | 106 | } 107 | 108 | 109 | lock_protocol::status 110 | lock_client_cache_rsm::acquire(lock_protocol::lockid_t lid) 111 | { 112 | //tprintf("%s ACQUIRE for %llu\n", id.data(), lid); 113 | int ret = lock_protocol::OK; 114 | std::unique_lock mtx_(mtxtb); 115 | client_lock *rlk = nullptr; 116 | auto iter = rlocktb.find(lid); 117 | if (iter == rlocktb.end()){ 118 | rlk = new client_lock(lid); 119 | rlocktb[lid] = rlk; 120 | } else { 121 | rlk = iter->second; 122 | } 123 | 124 | 125 | std::unique_lock m_(rlk->mtx); 126 | mtx_.unlock(); 127 | 128 | // LOCKED, ACQUIRING, RELEASING wait on lock_cv 129 | if (client_lock::LOCKED == rlk->status || client_lock::ACQUIRING == rlk->status || client_lock::RELEASING == rlk->status){ 130 | rlk->lock_cv.wait(m_, [&](){ return client_lock::FREE == rlk->status || client_lock::NONE == rlk->status;}); 131 | } 132 | 133 | if (client_lock::NONE == rlk->status) { 134 | return racquire(rlk, m_); 135 | } 136 | if (client_lock::FREE == rlk->status) { 137 | rlk->status = client_lock::LOCKED; 138 | return lock_protocol::OK; 139 | } 140 | 141 | return ret; 142 | } 143 | 144 | lock_protocol::status 145 | lock_client_cache_rsm::release(lock_protocol::lockid_t lid) 146 | { 147 | std::unique_lock mtx_(mtxtb); 148 | //tprintf("%s RELEASE for %llu\n", id.data(), lid); 149 | auto iter = rlocktb.find(lid); 150 | if (iter == rlocktb.end()){ 151 | return lock_protocol::OK; 152 | } 153 | client_lock *rlk = iter->second; 154 | 155 | std::unique_lock m_(rlk->mtx); 156 | mtx_.unlock(); 157 | 158 | 159 | if (client_lock::LOCKED != rlk->status){ 160 | return lock_protocol::IOERR; 161 | } 162 | 163 | //tprintf("RELEASE Phase 1\n"); 164 | if (rlk->nrevoke){ 165 | rlk->status = client_lock::RELEASING; 166 | --(rlk->nrevoke); 167 | release_fifo.enq(rlk); 168 | //tprintf("RELEASE Phase 2\n"); 169 | } else { 170 | rlk->status = client_lock::FREE; 171 | rlk->lock_cv.notify_all(); 172 | //tprintf("RELEASE Phase 3\n"); 173 | } 174 | return lock_protocol::OK; 175 | 176 | } 177 | 178 | 179 | rlock_protocol::status 180 | lock_client_cache_rsm::revoke_handler(lock_protocol::lockid_t lid, 181 | lock_protocol::xid_t xid, int &) 182 | { 183 | std::unique_lock mtx_(mtxtb); 184 | auto iter = rlocktb.find(lid); 185 | if (iter == rlocktb.end()){ 186 | return lock_protocol::OK; 187 | } 188 | client_lock *rlk = iter->second; 189 | 190 | if (xid < rlk->xid) { 191 | return lock_protocol::RPCERR; 192 | } 193 | 194 | if (client_lock::FREE == rlk->status || client_lock::NONE == rlk->status){ 195 | rlk->status = client_lock::RELEASING; 196 | release_fifo.enq(rlk); 197 | return lock_protocol::OK; 198 | } 199 | 200 | 201 | // Wait when LOCKED or ACQUIRING 202 | rlk->nrevoke++; 203 | return lock_protocol::OK; 204 | } 205 | 206 | rlock_protocol::status 207 | lock_client_cache_rsm::retry_handler(lock_protocol::lockid_t lid, 208 | lock_protocol::xid_t xid, int &) 209 | { 210 | int ret = rlock_protocol::OK; 211 | std::unique_lock mtx_(mtxtb); 212 | 213 | auto iter = rlocktb.find(lid); 214 | if (iter == rlocktb.end()){ 215 | return lock_protocol::OK; 216 | } 217 | client_lock *rlk = iter->second; 218 | 219 | std::unique_lock m_(rlk->mtx); 220 | mtx_.unlock(); 221 | 222 | // rlk->status = client_lock::NONE; 223 | rlk->nretry++; 224 | // rlk->status = client_lock::ACQUIRING; 225 | rlk->retry_cv.notify_one(); 226 | 227 | return ret; 228 | } 229 | 230 | 231 | lock_client_cache_rsm::~lock_client_cache_rsm(){ 232 | std::unique_lock mtx_(mtxtb); 233 | client_lock *rlk = nullptr; 234 | for (auto iter = rlocktb.begin(); iter != rlocktb.end(); ++iter) { 235 | rlk = iter->second; 236 | std::lock_guard m_(rlk->mtx); 237 | if (rlk->status == client_lock::RELEASING || rlk->status == client_lock::NONE) { 238 | continue; 239 | } 240 | release_fifo.enq(rlk); 241 | } 242 | rlk = new client_lock(0); 243 | rlk->nrevoke = -1; 244 | release_fifo.enq(rlk); 245 | 246 | pthread_join(th, NULL); 247 | delete rlk; 248 | } 249 | -------------------------------------------------------------------------------- /lock_client_cache_rsm.h: -------------------------------------------------------------------------------- 1 | // lock client interface. 2 | 3 | #ifndef lock_client_cache_rsm_h 4 | 5 | #define lock_client_cache_rsm_h 6 | 7 | #include 8 | #include "lock_protocol.h" 9 | #include "rpc.h" 10 | #include "fifo.h" 11 | #include "lock_client.h" 12 | #include "lang/verify.h" 13 | 14 | #include "rsm_client.h" 15 | #include "extent_client.h" 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | // Classes that inherit lock_release_user can override dorelease so that 23 | // that they will be called when lock_client releases a lock. 24 | // You will not need to do anything with this class until Lab 5. 25 | class lock_release_user { 26 | public: 27 | virtual void dorelease(lock_protocol::lockid_t) = 0; 28 | virtual ~lock_release_user() {}; 29 | }; 30 | 31 | class lock_release: public lock_release_user { 32 | extent_client *ec; 33 | public: 34 | lock_release(extent_client *e): ec(e) {} 35 | void dorelease(lock_protocol::lockid_t lid) override { 36 | ec->flush(lid); 37 | } 38 | }; 39 | 40 | 41 | class lock_client_cache_rsm; 42 | 43 | // Clients that caches locks. The server can revoke locks using 44 | // lock_revoke_server. 45 | class lock_client_cache_rsm : public lock_client { 46 | private: 47 | rsm_client *rsmc; 48 | class lock_release_user *lu; 49 | int rlock_port; 50 | std::string hostname; 51 | std::string id; 52 | std::atomic xid; 53 | 54 | struct client_lock { 55 | enum xxxstatus { 56 | FREE, 57 | LOCKED, 58 | NONE, 59 | ACQUIRING, 60 | RELEASING, 61 | RETRYING, 62 | }; 63 | lock_protocol::lockid_t lid; 64 | lock_protocol::xid_t xid; 65 | int status = NONE; 66 | int nrevoke = 0; 67 | int nretry = 0; 68 | pthread_t owner = 0; 69 | std::mutex mtx; 70 | std::condition_variable lock_cv; 71 | std::condition_variable revoke_cv; 72 | std::condition_variable retry_cv; 73 | client_lock(lock_protocol::lockid_t id): lid(id), xid(0), status(NONE), nrevoke(0), nretry(0){} 74 | }; 75 | 76 | std::map rlocktb; 77 | std::mutex mtxtb; 78 | 79 | lock_protocol::status racquire(client_lock *rlk, std::unique_lock &m); 80 | 81 | struct release_info { 82 | lock_protocol::lockid_t lid; 83 | lock_protocol::xid_t xid; 84 | }; 85 | fifo release_fifo; 86 | 87 | pthread_t th; 88 | 89 | public: 90 | static int last_port; 91 | lock_client_cache_rsm(std::string xdst, class lock_release_user *l = 0); 92 | virtual ~lock_client_cache_rsm(); 93 | lock_protocol::status acquire(lock_protocol::lockid_t); 94 | virtual lock_protocol::status release(lock_protocol::lockid_t); 95 | void releaser(); 96 | rlock_protocol::status revoke_handler(lock_protocol::lockid_t, 97 | lock_protocol::xid_t, int &); 98 | rlock_protocol::status retry_handler(lock_protocol::lockid_t, 99 | lock_protocol::xid_t, int &); 100 | }; 101 | 102 | 103 | #endif 104 | -------------------------------------------------------------------------------- /lock_demo.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Lock demo 3 | // 4 | 5 | #include "lock_protocol.h" 6 | #include "lock_client.h" 7 | #include "rpc.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | std::string dst; 14 | lock_client *lc; 15 | 16 | int 17 | main(int argc, char *argv[]) 18 | { 19 | int r; 20 | 21 | if(argc != 2){ 22 | fprintf(stderr, "Usage: %s [host:]port\n", argv[0]); 23 | exit(1); 24 | } 25 | 26 | dst = argv[1]; 27 | lc = new lock_client(dst); 28 | r = lc->stat(1); 29 | printf ("stat returned %d\n", r); 30 | return 0; 31 | } 32 | -------------------------------------------------------------------------------- /lock_protocol.h: -------------------------------------------------------------------------------- 1 | // lock protocol 2 | 3 | #ifndef lock_protocol_h 4 | #define lock_protocol_h 5 | 6 | #include "rpc.h" 7 | 8 | class lock_protocol { 9 | public: 10 | enum xxstatus { OK, RETRY, RPCERR, NOENT, IOERR }; 11 | typedef int status; 12 | typedef unsigned long long lockid_t; 13 | typedef unsigned long long xid_t; 14 | enum rpc_numbers { 15 | acquire = 0x7001, 16 | release, 17 | stat 18 | }; 19 | }; 20 | 21 | class rlock_protocol { 22 | public: 23 | enum xxstatus { OK, RPCERR }; 24 | typedef int status; 25 | enum rpc_numbers { 26 | revoke = 0x8001, 27 | retry = 0x8002 28 | }; 29 | }; 30 | #endif 31 | -------------------------------------------------------------------------------- /lock_server.cc: -------------------------------------------------------------------------------- 1 | // the lock server implementation 2 | 3 | #include "lock_server.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | lock_server::lock_server(): 10 | nacquire (0) 11 | { 12 | } 13 | 14 | lock_protocol::status 15 | lock_server::stat(int clt, lock_protocol::lockid_t lid, int &r) 16 | { 17 | lock_protocol::status ret = lock_protocol::OK; 18 | printf("stat request from clt %d\n", clt); 19 | r = nacquire; 20 | return ret; 21 | } 22 | 23 | lock_protocol::status 24 | lock_server::acquire(int clt, lock_protocol::lockid_t lid, int &r) { 25 | printf("acquire request for lock %llu from clt %d\n", lid, clt); 26 | locks_.lock(lid); 27 | return lock_protocol::OK; 28 | } 29 | 30 | 31 | lock_protocol::status 32 | lock_server::release(int clt, lock_protocol::lockid_t lid, int &r){ 33 | printf("release request for lock %llu from clt %d\n", lid, clt); 34 | locks_.unlock(lid); 35 | return lock_protocol::OK; 36 | } 37 | -------------------------------------------------------------------------------- /lock_server.h: -------------------------------------------------------------------------------- 1 | // this is the lock server 2 | // the lock client has a similar interface 3 | 4 | #ifndef lock_server_h 5 | #define lock_server_h 6 | 7 | #include 8 | #include "lock_protocol.h" 9 | #include "lock_client.h" 10 | #include "locks.h" 11 | #include "rpc.h" 12 | 13 | class lock_server { 14 | 15 | protected: 16 | int nacquire; 17 | locks locks_; 18 | 19 | public: 20 | lock_server(); 21 | 22 | ~lock_server() { }; 23 | 24 | lock_protocol::status stat(int clt, lock_protocol::lockid_t lid, int &); 25 | lock_protocol::status acquire(int clt, lock_protocol::lockid_t lid, int &r); 26 | lock_protocol::status release(int clt, lock_protocol::lockid_t lid, int &r); 27 | 28 | }; 29 | 30 | #endif 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /lock_server_cache.cc: -------------------------------------------------------------------------------- 1 | // the caching lock server implementation 2 | 3 | #include "lock_server_cache.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "lang/verify.h" 9 | #include "handle.h" 10 | #include "tprintf.h" 11 | 12 | 13 | lock_server_cache::lock_server_cache() : nacquire(0) { 14 | } 15 | 16 | 17 | int lock_server_cache::acquire(lock_protocol::lockid_t lid, std::string id, 18 | int &) { 19 | tprintf("%s acquire for %llu\n", id.data(), lid); 20 | std::unique_lock mtx_(mtxtb); 21 | auto iter = locktb.find(lid); 22 | server_lock *rlk = nullptr; 23 | if (iter == locktb.end()) { 24 | rlk = new server_lock(lid); 25 | locktb[lid] = rlk; 26 | } else { 27 | rlk = iter->second; 28 | } 29 | 30 | if (server_lock::FREE == rlk->status) { 31 | rlk->status = server_lock::LOCKED; 32 | rlk->owner = id; 33 | return lock_protocol::OK; 34 | } 35 | 36 | if (server_lock::LOCKED == rlk->status){ 37 | handle h(rlk->owner); 38 | rpcc *cl = h.safebind(); 39 | if (cl){ 40 | int r; 41 | rlk->status = server_lock::REVOKE_SENT; 42 | mtx_.unlock(); 43 | auto ret = cl->call(rlock_protocol::revoke, lid, r); 44 | mtx_.lock(); 45 | if (lock_protocol::OK == ret){ 46 | rlk->status = server_lock::LOCKED; 47 | rlk->owner = id; 48 | } else { 49 | return ret; 50 | } 51 | if (!rlk->retry.empty()){ 52 | std::string cid = rlk->retry.front(); 53 | rlk->retry.pop(); 54 | mtx_.unlock(); 55 | handle hretry(cid); 56 | cl = hretry.safebind(); 57 | if (cl) { 58 | cl->call(rlock_protocol::retry, lid, r); 59 | } 60 | } 61 | return lock_protocol::OK; 62 | } else { 63 | return lock_protocol::RPCERR; 64 | } 65 | } else { 66 | rlk->retry.push(id); 67 | return lock_protocol::RETRY; 68 | } 69 | 70 | } 71 | 72 | int 73 | lock_server_cache::release(lock_protocol::lockid_t lid, std::string id, 74 | int &r) { 75 | 76 | printf("%s release %llu\n", id.data(), lid); 77 | std::unique_lock mtx_(mtxtb); 78 | 79 | auto iter = locktb.find(lid); 80 | server_lock *rlk = nullptr; 81 | if (iter == locktb.end()) { 82 | return lock_protocol::IOERR; 83 | } else { 84 | rlk = iter->second; 85 | } 86 | 87 | if (id == rlk->owner && server_lock::LOCKED == rlk->status){ 88 | rlk->status = server_lock::FREE; 89 | } 90 | return lock_protocol::OK; 91 | } 92 | 93 | lock_protocol::status 94 | lock_server_cache::stat(lock_protocol::lockid_t lid, int &r) { 95 | tprintf("stat request\n"); 96 | r = nacquire; 97 | return lock_protocol::OK; 98 | } 99 | 100 | -------------------------------------------------------------------------------- /lock_server_cache.h: -------------------------------------------------------------------------------- 1 | #ifndef lock_server_cache_h 2 | #define lock_server_cache_h 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "lock_protocol.h" 9 | #include "rpc.h" 10 | #include "lock_server.h" 11 | 12 | 13 | class lock_server_cache { 14 | private: 15 | int nacquire; 16 | 17 | struct server_lock { 18 | enum xxxstatus { 19 | FREE, 20 | LOCKED, 21 | REVOKE_SENT, 22 | }; 23 | lock_protocol::lockid_t lid; 24 | int status; 25 | std::string owner; 26 | std::queue retry; 27 | 28 | server_lock(lock_protocol::lockid_t id): lid(id), status(FREE){} 29 | }; 30 | 31 | std::map locktb; 32 | std::mutex mtxtb; 33 | 34 | public: 35 | lock_server_cache(); 36 | 37 | lock_protocol::status stat(lock_protocol::lockid_t, int &); 38 | 39 | int acquire(lock_protocol::lockid_t, std::string id, int &); 40 | 41 | int release(lock_protocol::lockid_t, std::string id, int &); 42 | }; 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /lock_server_cache_rsm.cc: -------------------------------------------------------------------------------- 1 | // the caching lock server implementation 2 | 3 | #include "lock_server_cache_rsm.h" 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include "lang/verify.h" 9 | #include "handle.h" 10 | #include "tprintf.h" 11 | #include "marshall.h" 12 | 13 | 14 | static void * 15 | revokethread(void *x) 16 | { 17 | lock_server_cache_rsm *sc = (lock_server_cache_rsm *) x; 18 | sc->revoker(); 19 | return 0; 20 | } 21 | 22 | static void * 23 | retrythread(void *x) 24 | { 25 | lock_server_cache_rsm *sc = (lock_server_cache_rsm *) x; 26 | sc->retryer(); 27 | return 0; 28 | } 29 | 30 | lock_server_cache_rsm::lock_server_cache_rsm(class rsm *_rsm) 31 | : rsm (_rsm) 32 | { 33 | pthread_t th; 34 | int r = pthread_create(&th, NULL, &revokethread, (void *) this); 35 | VERIFY (r == 0); 36 | r = pthread_create(&th, NULL, &retrythread, (void *) this); 37 | VERIFY (r == 0); 38 | rsm->set_state_transfer(this); 39 | } 40 | 41 | void 42 | lock_server_cache_rsm::revoker() 43 | { 44 | 45 | // This method should be a continuous loop, that sends revoke 46 | // messages to lock holders whenever another client wants the 47 | // same lock 48 | server_lock *rlk = nullptr; 49 | for (;;) { 50 | revoke_fifo.deq(&rlk); 51 | if (!rsm->amiprimary()) { 52 | continue; 53 | } 54 | handle h(rlk->owner); 55 | auto cl = h.safebind(); 56 | if (cl){ 57 | int r; 58 | tprintf("send revoke %llu to %s\n", rlk->lid, rlk->owner.data()); 59 | cl->call(rlock_protocol::revoke, rlk->lid, rlk->xid, r); 60 | } 61 | } 62 | } 63 | 64 | 65 | void 66 | lock_server_cache_rsm::retryer() 67 | { 68 | 69 | // This method should be a continuous loop, waiting for locks 70 | // to be released and then sending retry messages to those who 71 | // are waiting for it. 72 | re_info info; 73 | for (;;) { 74 | retry_fifo.deq(&info); 75 | if (!rsm->amiprimary()) { 76 | continue; 77 | } 78 | handle h(info.dst); 79 | auto cl = h.safebind(); 80 | if (cl) { 81 | int r; 82 | tprintf("send retry %llu to %s\n", info.lid, info.dst.data()); 83 | cl->call(rlock_protocol::retry, info.lid, info.xid, r); 84 | } 85 | } 86 | } 87 | 88 | 89 | int lock_server_cache_rsm::acquire(lock_protocol::lockid_t lid, std::string id, 90 | lock_protocol::xid_t xid, int &) 91 | { 92 | tprintf("%s acquire for %llu\n", id.data(), lid); 93 | std::unique_lock mtx_(mtxtb); 94 | auto iter = locktb.find(lid); 95 | server_lock *rlk = nullptr; 96 | if (iter == locktb.end()) { 97 | rlk = new server_lock(lid); 98 | locktb[lid] = rlk; 99 | } else { 100 | rlk = iter->second; 101 | } 102 | 103 | if (rlk->owner == id) { 104 | if (xid < rlk->xid) { 105 | return lock_protocol::RPCERR; 106 | } 107 | 108 | if (xid == rlk->xid && server_lock::LOCKED == rlk->status) { 109 | return lock_protocol::OK; 110 | } 111 | } 112 | 113 | if (server_lock::FREE == rlk->status) { 114 | rlk->status = server_lock::LOCKED; 115 | rlk->owner = id; 116 | rlk->xid = xid; 117 | if (!rlk->retry.empty()) { 118 | re_info info; 119 | info.lid = lid; 120 | info.dst = rlk->retry.front(); 121 | rlk->retry.pop(); 122 | retry_fifo.enq(info); 123 | } 124 | return lock_protocol::OK; 125 | } 126 | 127 | rlk->retry.push(id); 128 | if (server_lock::LOCKED == rlk->status) { 129 | rlk->status = server_lock::REVOKE_SENT; 130 | revoke_fifo.enq(rlk); 131 | } 132 | return lock_protocol::RETRY; 133 | } 134 | 135 | int 136 | lock_server_cache_rsm::release(lock_protocol::lockid_t lid, std::string id, 137 | lock_protocol::xid_t xid, int &r) 138 | { 139 | tprintf("%s release %llu\n", id.data(), lid); 140 | std::unique_lock mtx_(mtxtb); 141 | 142 | auto iter = locktb.find(lid); 143 | server_lock *rlk = nullptr; 144 | 145 | if (iter == locktb.end()) { 146 | return lock_protocol::IOERR; 147 | } else { 148 | rlk = iter->second; 149 | } 150 | 151 | if (id != rlk->owner) { 152 | return lock_protocol::IOERR; 153 | } 154 | 155 | if (xid != rlk->xid) { 156 | return lock_protocol::RPCERR; 157 | } 158 | 159 | rlk->status = server_lock::FREE; 160 | rlk->owner.clear(); 161 | rlk->xid = 0; 162 | if (!rlk->retry.empty()) { 163 | re_info info; 164 | info.lid = lid; 165 | info.dst = rlk->retry.front(); 166 | info.xid = rlk->xid; 167 | rlk->retry.pop(); 168 | retry_fifo.enq(info); 169 | } 170 | return lock_protocol::OK; 171 | } 172 | 173 | 174 | std::string 175 | lock_server_cache_rsm::marshal_state() 176 | { 177 | //std::ostringstream ost; 178 | //std::string r; 179 | marshall rep; 180 | using ull = unsigned long long; 181 | std::lock_guard m_(mtxtb); 182 | rep << static_cast(locktb.size()); 183 | for (auto &lock : locktb) { 184 | ull lid = lock.second->lid; 185 | int status = lock.second->status; 186 | std::string owner = lock.second->owner; 187 | rep << lid << status << owner; 188 | ull retrysize = lock.second->retry.size(); 189 | rep << retrysize; 190 | for (size_t i = 0; i < retrysize; ++i) { 191 | std::string retryer = lock.second->retry.front(); 192 | rep << retryer; 193 | lock.second->retry.pop(); 194 | lock.second->retry.push(retryer); 195 | } 196 | ull xid = lock.second->xid; 197 | rep << xid; 198 | } 199 | return rep.str(); 200 | } 201 | 202 | void 203 | lock_server_cache_rsm::unmarshal_state(std::string state) 204 | { 205 | unmarshall rep(state); 206 | using ull = unsigned long long; 207 | std::lock_guard m_(mtxtb); 208 | locktb.clear(); 209 | ull tbsize; 210 | rep >> tbsize; 211 | for (size_t i = 0; i < tbsize; ++i) { 212 | ull lid; 213 | int status; 214 | std::string owner; 215 | ull retrysize; 216 | rep >> lid >> status >> owner >> retrysize; 217 | auto rlk = new server_lock(lid); 218 | rlk->status = status; 219 | rlk->owner = owner; 220 | std::string retryer; 221 | for (size_t i = 0; i < retrysize; ++i) { 222 | rep >> retryer; 223 | rlk->retry.push(retryer); 224 | } 225 | ull xid; 226 | rep >> xid; 227 | rlk->xid = xid; 228 | locktb[lid] = rlk; 229 | } 230 | } 231 | 232 | lock_protocol::status 233 | lock_server_cache_rsm::stat(lock_protocol::lockid_t lid, int &r) 234 | { 235 | printf("stat request\n"); 236 | r = nacquire; 237 | return lock_protocol::OK; 238 | } 239 | 240 | -------------------------------------------------------------------------------- /lock_server_cache_rsm.h: -------------------------------------------------------------------------------- 1 | #ifndef lock_server_cache_rsm_h 2 | #define lock_server_cache_rsm_h 3 | 4 | #include 5 | 6 | #include "lock_protocol.h" 7 | #include "rpc.h" 8 | #include "rsm_state_transfer.h" 9 | #include "rsm.h" 10 | #include "fifo.h" 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | class lock_server_cache_rsm : public rsm_state_transfer { 17 | private: 18 | int nacquire; 19 | class rsm *rsm; 20 | struct server_lock { 21 | enum xxxstatus { 22 | FREE, 23 | LOCKED, 24 | REVOKE_SENT, 25 | }; 26 | lock_protocol::lockid_t lid; 27 | int status; 28 | std::string owner; 29 | std::queue retry; 30 | lock_protocol::xid_t xid; 31 | 32 | server_lock(lock_protocol::lockid_t id): lid(id), status(FREE), xid(0){} 33 | }; 34 | std::map locktb; 35 | std::mutex mtxtb; 36 | 37 | struct re_info { 38 | lock_protocol::xid_t xid; 39 | lock_protocol::lockid_t lid; 40 | std::string dst; 41 | }; 42 | 43 | fifo revoke_fifo; 44 | fifo retry_fifo; 45 | 46 | public: 47 | lock_server_cache_rsm(class rsm *rsm = 0); 48 | lock_protocol::status stat(lock_protocol::lockid_t, int &); 49 | void revoker(); 50 | void retryer(); 51 | std::string marshal_state(); 52 | void unmarshal_state(std::string state); 53 | int acquire(lock_protocol::lockid_t, std::string id, 54 | lock_protocol::xid_t, int &); 55 | int release(lock_protocol::lockid_t, std::string id, lock_protocol::xid_t, 56 | int &); 57 | }; 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /lock_smain.cc: -------------------------------------------------------------------------------- 1 | #include "rpc.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "lock_server_cache_rsm.h" 8 | #include "paxos.h" 9 | #include "rsm.h" 10 | 11 | #include "jsl_log.h" 12 | 13 | // Main loop of lock_server 14 | 15 | static void 16 | force_exit(int) { 17 | exit(0); 18 | } 19 | 20 | int 21 | main(int argc, char *argv[]) 22 | { 23 | int count = 0; 24 | 25 | // Force the lock_server to exit after 20 minutes 26 | signal(SIGALRM, force_exit); 27 | alarm(20 * 60); 28 | 29 | setvbuf(stdout, NULL, _IONBF, 0); 30 | setvbuf(stderr, NULL, _IONBF, 0); 31 | 32 | srandom(getpid()); 33 | 34 | if(argc != 3){ 35 | fprintf(stderr, "Usage: %s [master:]port [me:]port\n", argv[0]); 36 | exit(1); 37 | } 38 | 39 | //jsl_set_debug(2); 40 | // Comment out the next line to switch between the ordinary lock 41 | // server and the RSM. In Lab 6, we disable the lock server and 42 | // implement Paxos. In Lab 7, we will make the lock server use your 43 | // RSM layer. 44 | #define RSM 45 | #ifdef RSM 46 | // You must comment out the next line once you are done with Step One. 47 | //#define STEP_ONE 48 | #ifdef STEP_ONE 49 | rpcs server(atoi(argv[1])); 50 | lock_server_cache_rsm ls; 51 | server.reg(lock_protocol::acquire, &ls, &lock_server_cache_rsm::acquire); 52 | server.reg(lock_protocol::release, &ls, &lock_server_cache_rsm::release); 53 | server.reg(lock_protocol::stat, &ls, &lock_server_cache_rsm::stat); 54 | #else 55 | rsm rsm(argv[1], argv[2]); 56 | lock_server_cache_rsm ls(&rsm); 57 | rsm.set_state_transfer((rsm_state_transfer *)&ls); 58 | rsm.reg(lock_protocol::acquire, &ls, &lock_server_cache_rsm::acquire); 59 | rsm.reg(lock_protocol::release, &ls, &lock_server_cache_rsm::release); 60 | rsm.reg(lock_protocol::stat, &ls, &lock_server_cache_rsm::stat); 61 | #endif // STEP_ONE 62 | #endif // RSM 63 | 64 | 65 | 66 | while (1) 67 | sleep(1000); 68 | } 69 | -------------------------------------------------------------------------------- /lock_tester.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Lock server tester 3 | // 4 | 5 | #include "lock_protocol.h" 6 | #include "lock_client.h" 7 | #include "rpc.h" 8 | #include "jsl_log.h" 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "lang/verify.h" 16 | #include "lock_client_cache_rsm.h" 17 | 18 | // must be >= 2 19 | int nt = 6; //XXX: lab1's rpc handlers are blocking. Since rpcs uses a thread pool of 10 threads, we cannot test more than 10 blocking rpc. 20 | std::string dst; 21 | lock_client_cache_rsm **lc = new lock_client_cache_rsm * [nt]; 22 | lock_protocol::lockid_t a = 1; 23 | lock_protocol::lockid_t b = 2; 24 | lock_protocol::lockid_t c = 3; 25 | 26 | // check_grant() and check_release() check that the lock server 27 | // doesn't grant the same lock to both clients. 28 | // it assumes that lock names are distinct in the first byte. 29 | int ct[256]; 30 | pthread_mutex_t count_mutex; 31 | 32 | void 33 | check_grant(lock_protocol::lockid_t lid) 34 | { 35 | ScopedLock ml(&count_mutex); 36 | int x = lid & 0xff; 37 | if(ct[x] != 0){ 38 | fprintf(stderr, "error: server granted %016llx twice\n", lid); 39 | fprintf(stdout, "error: server granted %016llx twice\n", lid); 40 | exit(1); 41 | } 42 | ct[x] += 1; 43 | } 44 | 45 | void 46 | check_release(lock_protocol::lockid_t lid) 47 | { 48 | ScopedLock ml(&count_mutex); 49 | int x = lid & 0xff; 50 | if(ct[x] != 1){ 51 | fprintf(stderr, "error: client released un-held lock %016llx\n", lid); 52 | exit(1); 53 | } 54 | ct[x] -= 1; 55 | } 56 | 57 | void 58 | test1(void) 59 | { 60 | printf ("acquire a release a acquire a release a\n"); 61 | lc[0]->acquire(a); 62 | check_grant(a); 63 | lc[0]->release(a); 64 | check_release(a); 65 | lc[0]->acquire(a); 66 | check_grant(a); 67 | lc[0]->release(a); 68 | check_release(a); 69 | 70 | printf ("acquire a acquire b release b release a\n"); 71 | lc[0]->acquire(a); 72 | check_grant(a); 73 | lc[0]->acquire(b); 74 | check_grant(b); 75 | lc[0]->release(b); 76 | check_release(b); 77 | lc[0]->release(a); 78 | check_release(a); 79 | } 80 | 81 | void * 82 | test2(void *x) 83 | { 84 | int i = * (int *) x; 85 | 86 | printf ("test2: client %d acquire a release a\n", i); 87 | lc[i]->acquire(a); 88 | printf ("test2: client %d acquire done\n", i); 89 | check_grant(a); 90 | sleep(1); 91 | printf ("test2: client %d release\n", i); 92 | check_release(a); 93 | lc[i]->release(a); 94 | printf ("test2: client %d release done\n", i); 95 | return 0; 96 | } 97 | 98 | void * 99 | test3(void *x) 100 | { 101 | int i = * (int *) x; 102 | 103 | printf ("test3: client %d acquire a release a concurrent\n", i); 104 | for (int j = 0; j < 10; j++) { 105 | lc[i]->acquire(a); 106 | check_grant(a); 107 | printf ("test3: client %d got lock\n", i); 108 | check_release(a); 109 | lc[i]->release(a); 110 | } 111 | return 0; 112 | } 113 | 114 | void * 115 | test4(void *x) 116 | { 117 | int i = * (int *) x; 118 | 119 | printf ("test4: thread %d acquire a release a concurrent; same clnt\n", i); 120 | for (int j = 0; j < 10; j++) { 121 | lc[0]->acquire(a); 122 | check_grant(a); 123 | printf ("test4: thread %d on client 0 got lock\n", i); 124 | check_release(a); 125 | lc[0]->release(a); 126 | } 127 | return 0; 128 | } 129 | 130 | void * 131 | test5(void *x) 132 | { 133 | int i = * (int *) x; 134 | 135 | printf ("test5: client %d acquire a release a concurrent; same and diff clnt\n", i); 136 | for (int j = 0; j < 10; j++) { 137 | if (i < 5) lc[0]->acquire(a); 138 | else lc[1]->acquire(a); 139 | check_grant(a); 140 | printf ("test5: client %d got lock\n", i); 141 | check_release(a); 142 | if (i < 5) lc[0]->release(a); 143 | else lc[1]->release(a); 144 | } 145 | return 0; 146 | } 147 | 148 | static void 149 | force_exit(int) { 150 | exit(0); 151 | } 152 | 153 | int 154 | main(int argc, char *argv[]) 155 | { 156 | int r; 157 | pthread_t th[nt]; 158 | int test = 0; 159 | 160 | // Force the lock_server to exit after 20 minutes 161 | signal(SIGALRM, force_exit); 162 | alarm(20 * 60); 163 | 164 | setvbuf(stdout, NULL, _IONBF, 0); 165 | setvbuf(stderr, NULL, _IONBF, 0); 166 | srandom(getpid()); 167 | 168 | //jsl_set_debug(2); 169 | 170 | if(argc < 2) { 171 | fprintf(stderr, "Usage: %s [host:]port [test]\n", argv[0]); 172 | exit(1); 173 | } 174 | 175 | dst = argv[1]; 176 | 177 | if (argc > 2) { 178 | test = atoi(argv[2]); 179 | if(test < 1 || test > 5){ 180 | printf("Test number must be between 1 and 5\n"); 181 | exit(1); 182 | } 183 | } 184 | 185 | VERIFY(pthread_mutex_init(&count_mutex, NULL) == 0); 186 | printf("cache lock client\n"); 187 | for (int i = 0; i < nt; i++) lc[i] = new lock_client_cache_rsm(dst); 188 | 189 | if(!test || test == 1){ 190 | test1(); 191 | } 192 | 193 | if(!test || test == 2){ 194 | // test2 195 | for (int i = 0; i < nt; i++) { 196 | int *a = new int (i); 197 | r = pthread_create(&th[i], NULL, test2, (void *) a); 198 | VERIFY (r == 0); 199 | } 200 | for (int i = 0; i < nt; i++) { 201 | pthread_join(th[i], NULL); 202 | } 203 | } 204 | 205 | if(!test || test == 3){ 206 | printf("test 3\n"); 207 | 208 | // test3 209 | for (int i = 0; i < nt; i++) { 210 | int *a = new int (i); 211 | r = pthread_create(&th[i], NULL, test3, (void *) a); 212 | VERIFY (r == 0); 213 | } 214 | for (int i = 0; i < nt; i++) { 215 | pthread_join(th[i], NULL); 216 | } 217 | } 218 | 219 | if(!test || test == 4){ 220 | printf("test 4\n"); 221 | 222 | // test 4 223 | for (int i = 0; i < 2; i++) { 224 | int *a = new int (i); 225 | r = pthread_create(&th[i], NULL, test4, (void *) a); 226 | VERIFY (r == 0); 227 | } 228 | for (int i = 0; i < 2; i++) { 229 | pthread_join(th[i], NULL); 230 | } 231 | } 232 | 233 | if(!test || test == 5){ 234 | printf("test 5\n"); 235 | 236 | // test 5 237 | 238 | for (int i = 0; i < nt; i++) { 239 | int *a = new int (i); 240 | r = pthread_create(&th[i], NULL, test5, (void *) a); 241 | VERIFY (r == 0); 242 | } 243 | for (int i = 0; i < nt; i++) { 244 | pthread_join(th[i], NULL); 245 | } 246 | } 247 | for (int i = 0; i < nt; i++) delete lc[i]; 248 | 249 | printf ("%s: passed all tests successfully\n", argv[0]); 250 | 251 | } 252 | -------------------------------------------------------------------------------- /locks.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mactavish on 15-5-6. 3 | // 4 | 5 | #include "locks.h" 6 | #include "raii.h" 7 | 8 | 9 | locks::state locks::lookup(lock_protocol::lockid_t lockid) { 10 | std::lock_guard smtx_(mtx); 11 | try { 12 | return table.at(lockid); 13 | } catch (std::out_of_range &e) { 14 | return locks::FREE; 15 | } 16 | } 17 | 18 | bool locks::lock(lock_protocol::lockid_t lockid) { 19 | std::unique_lock lk(mtx); 20 | auto ptb = table.find(lockid); 21 | if (ptb == table.end()){ 22 | table[lockid] = locks::LOCKED; 23 | return true; 24 | } 25 | 26 | cv.wait(lk, [&](){return locks::FREE == table[lockid];}); 27 | 28 | table[lockid] = locks::LOCKED; 29 | return true; 30 | } 31 | 32 | bool locks::unlock(lock_protocol::lockid_t lockid) { 33 | std::unique_lock lk(mtx); 34 | 35 | auto ptb = table.find(lockid); 36 | if (ptb == table.end() || locks::FREE == ptb->second){ 37 | return false; 38 | } 39 | 40 | table[lockid] = locks::FREE; 41 | cv.notify_all(); 42 | return true; 43 | } 44 | -------------------------------------------------------------------------------- /locks.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mactavish on 15-5-6. 3 | // 4 | 5 | #ifndef MIT_6_824_2012_LOCK_H 6 | #define MIT_6_824_2012_LOCK_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include "lock_protocol.h" 12 | 13 | class locks { 14 | public: 15 | enum state { 16 | FREE, 17 | LOCKED 18 | }; 19 | locks(){}; 20 | ~locks(){}; 21 | locks::state lookup(lock_protocol::lockid_t lockid); 22 | bool lock(lock_protocol::lockid_t lockid); 23 | bool unlock(lock_protocol::lockid_t lockid); 24 | private: 25 | std::map table; 26 | std::mutex mtx; 27 | std::condition_variable cv; 28 | 29 | 30 | }; 31 | 32 | #endif //MIT_6_824_2012_LOCK_H 33 | -------------------------------------------------------------------------------- /log.cc: -------------------------------------------------------------------------------- 1 | #include "paxos.h" 2 | #include 3 | #include 4 | 5 | // Paxos must maintain some durable state (i.e., that survives power 6 | // failures) to run Paxos correct. This module implements a log with 7 | // all durable state to run Paxos. Since the values chosen correspond 8 | // to views, the log contains all views since the beginning of time. 9 | 10 | log::log(acceptor *_acc, std::string _me) 11 | : pxs (_acc) 12 | { 13 | name = "paxos-" + _me + ".log"; 14 | logread(); 15 | } 16 | 17 | void 18 | log::logread(void) 19 | { 20 | std::ifstream from; 21 | std::string type; 22 | unsigned instance; 23 | 24 | from.open(name.c_str()); 25 | printf ("logread\n"); 26 | while (from >> type) { 27 | if (type == "done") { 28 | std::string v; 29 | from >> instance; 30 | from.get(); 31 | getline(from, v); 32 | pxs->values[instance] = v; 33 | pxs->instance_h = instance; 34 | printf ("logread: instance: %d w. v = %s\n", instance, 35 | pxs->values[instance].c_str()); 36 | pxs->v_a.clear(); 37 | pxs->n_h.n = 0; 38 | pxs->n_a.n = 0; 39 | } else if (type == "propseen") { 40 | from >> pxs->n_h.n; 41 | from >> pxs->n_h.m; 42 | printf("logread: high update: %d(%s)\n", pxs->n_h.n, pxs->n_h.m.c_str()); 43 | } else if (type == "accepted") { 44 | std::string v; 45 | from >> pxs->n_a.n; 46 | from >> pxs->n_a.m; 47 | from.get(); 48 | getline(from, v); 49 | pxs->v_a = v; 50 | printf("logread: prop update %d(%s) with v = %s\n", pxs->n_a.n, 51 | pxs->n_a.m.c_str(), pxs->v_a.c_str()); 52 | } else { 53 | printf("logread: unknown log record\n"); 54 | VERIFY(0); 55 | } 56 | } 57 | from.close(); 58 | } 59 | 60 | std::string 61 | log::dump() 62 | { 63 | std::ifstream from; 64 | std::string res; 65 | std::string v; 66 | from.open(name.c_str()); 67 | while (getline(from, v)) { 68 | res = res + v + "\n"; 69 | } 70 | from.close(); 71 | return res; 72 | } 73 | 74 | void 75 | log::restore(std::string s) 76 | { 77 | std::ofstream f; 78 | printf("restore: %s\n", s.c_str()); 79 | f.open(name.c_str(), std::ios::trunc); 80 | f << s; 81 | f.close(); 82 | } 83 | 84 | // XXX should be an atomic operation 85 | void 86 | log::loginstance(unsigned instance, std::string v) 87 | { 88 | std::ofstream f; 89 | f.open(name.c_str(), std::ios::app); 90 | f << "done"; 91 | f << " "; 92 | f << instance; 93 | f << " "; 94 | f << v; 95 | f << "\n"; 96 | f.close(); 97 | } 98 | 99 | // an acceptor should call logprop(n_h) when it 100 | // receives a prepare to which it responds prepare_ok(). 101 | void 102 | log::logprop(prop_t n_h) 103 | { 104 | std::ofstream f; 105 | f.open(name.c_str(), std::ios::app); 106 | f << "propseen"; 107 | f << " "; 108 | f << n_h.n; 109 | f << " "; 110 | f << n_h.m; 111 | f << "\n"; 112 | f.close(); 113 | } 114 | 115 | // an acceptor should call logaccept(n_a, v_a) when it 116 | // receives an accept RPC to which it replies accept_ok(). 117 | void 118 | log::logaccept(prop_t n, std::string v) 119 | { 120 | std::ofstream f; 121 | f.open(name.c_str(), std::ios::app); 122 | f << "accepted"; 123 | f << " "; 124 | f << n.n; 125 | f << " "; 126 | f << n.m; 127 | f << " "; 128 | f << v; 129 | f << "\n"; 130 | f.close(); 131 | } 132 | 133 | -------------------------------------------------------------------------------- /log.h: -------------------------------------------------------------------------------- 1 | #ifndef log_h 2 | #define log_h 3 | 4 | #include 5 | #include 6 | 7 | 8 | class acceptor; 9 | 10 | class log { 11 | private: 12 | std::string name; 13 | acceptor *pxs; 14 | public: 15 | log (acceptor*, std::string _me); 16 | std::string dump(); 17 | void restore(std::string s); 18 | void logread(void); 19 | /* Log a committed paxos instance*/ 20 | void loginstance(unsigned instance, std::string v); 21 | /* Log the highest proposal number that the local paxos acceptor has ever seen */ 22 | void logprop(prop_t n_h); 23 | /* Log the proposal (proposal number and value) that the local paxos acceptor 24 | accept has ever accepted */ 25 | void logaccept(prop_t n_a, std::string v); 26 | }; 27 | 28 | #endif /* log_h */ 29 | -------------------------------------------------------------------------------- /mkfs.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | rm -rf ID/* 4 | mkdir -p ID 5 | echo -n > ID/0000000000000001 6 | -------------------------------------------------------------------------------- /paxos.cc: -------------------------------------------------------------------------------- 1 | #include "paxos.h" 2 | #include "handle.h" 3 | // #include 4 | #include 5 | #include "tprintf.h" 6 | #include "lang/verify.h" 7 | 8 | // This module implements the proposer and acceptor of the Paxos 9 | // distributed algorithm as described by Lamport's "Paxos Made 10 | // Simple". To kick off an instance of Paxos, the caller supplies a 11 | // list of nodes, a proposed value, and invokes the proposer. If the 12 | // majority of the nodes agree on the proposed value after running 13 | // this instance of Paxos, the acceptor invokes the upcall 14 | // paxos_commit to inform higher layers of the agreed value for this 15 | // instance. 16 | 17 | 18 | bool 19 | operator>(const prop_t &a, const prop_t &b) { 20 | return (a.n > b.n || (a.n == b.n && a.m > b.m)); 21 | } 22 | 23 | bool 24 | operator>=(const prop_t &a, const prop_t &b) { 25 | return (a.n > b.n || (a.n == b.n && a.m >= b.m)); 26 | } 27 | 28 | std::string 29 | print_members(const std::vector &nodes) { 30 | std::string s; 31 | s.clear(); 32 | for (unsigned i = 0; i < nodes.size(); i++) { 33 | s += nodes[i]; 34 | if (i < (nodes.size() - 1)) 35 | s += ","; 36 | } 37 | return s; 38 | } 39 | 40 | bool isamember(std::string m, const std::vector &nodes) { 41 | for (unsigned i = 0; i < nodes.size(); i++) { 42 | if (nodes[i] == m) return 1; 43 | } 44 | return 0; 45 | } 46 | 47 | bool 48 | proposer::isrunning() { 49 | bool r; 50 | ScopedLock ml(&pxs_mutex); 51 | r = !stable; 52 | return r; 53 | } 54 | 55 | // check if the servers in l2 contains a majority of servers in l1 56 | bool 57 | proposer::majority(const std::vector &l1, 58 | const std::vector &l2) { 59 | unsigned n = 0; 60 | 61 | for (unsigned i = 0; i < l1.size(); i++) { 62 | if (isamember(l1[i], l2)) 63 | n++; 64 | } 65 | return n >= (l1.size() >> 1) + 1; 66 | } 67 | 68 | proposer::proposer(class paxos_change *_cfg, class acceptor *_acceptor, 69 | std::string _me) 70 | : cfg(_cfg), acc(_acceptor), me(_me), break1(false), break2(false), 71 | stable(true) { 72 | VERIFY (pthread_mutex_init(&pxs_mutex, NULL) == 0); 73 | my_n.n = 0; 74 | my_n.m = me; 75 | } 76 | 77 | void 78 | proposer::setn() { 79 | my_n.n = acc->get_n_h().n + 1 > my_n.n + 1 ? acc->get_n_h().n + 1 : my_n.n + 1; 80 | } 81 | 82 | bool 83 | proposer::run(int instance, std::vector cur_nodes, std::string newv) { 84 | std::vector accepts; 85 | std::vector nodes; 86 | std::string v; 87 | bool r = false; 88 | 89 | ScopedLock ml(&pxs_mutex); 90 | tprintf("start: initiate paxos for %s w. i=%d v=%s stable=%d\n", 91 | print_members(cur_nodes).c_str(), instance, newv.c_str(), stable); 92 | if (!stable) { // already running proposer? 93 | tprintf("proposer::run: already running\n"); 94 | return false; 95 | } 96 | stable = false; 97 | setn(); 98 | accepts.clear(); 99 | v.clear(); 100 | if (prepare(instance, accepts, cur_nodes, v)) { 101 | 102 | if (majority(cur_nodes, accepts)) { 103 | tprintf("paxos::manager: received a majority of prepare responses\n"); 104 | 105 | if (v.size() == 0) 106 | v = newv; 107 | 108 | breakpoint1(); 109 | 110 | nodes = accepts; 111 | accepts.clear(); 112 | accept(instance, accepts, nodes, v); 113 | 114 | if (majority(cur_nodes, accepts)) { 115 | tprintf("paxos::manager: received a majority of accept responses\n"); 116 | 117 | breakpoint2(); 118 | 119 | decide(instance, accepts, v); 120 | r = true; 121 | } else { 122 | tprintf("paxos::manager: no majority of accept responses\n"); 123 | } 124 | } else { 125 | tprintf("paxos::manager: no majority of prepare responses\n"); 126 | } 127 | } else { 128 | tprintf("paxos::manager: prepare is rejected %d\n", stable); 129 | } 130 | stable = true; 131 | return r; 132 | } 133 | 134 | // proposer::prepare() calls prepare to send prepare RPCs to nodes 135 | // and collect responses. if one of those nodes 136 | // replies with an oldinstance, return false. 137 | // otherwise fill in accepts with set of nodes that accepted, 138 | // set v to the v_a with the highest n_a, and return true. 139 | bool 140 | proposer::prepare(unsigned instance, std::vector &accepts, 141 | std::vector nodes, 142 | std::string &v) { 143 | // You fill this in for Lab 6 144 | // Note: if got an "oldinstance" reply, commit the instance using 145 | // acc->commit(...), and return false. 146 | paxos_protocol::preparearg preparearg; 147 | preparearg.instance = instance; 148 | preparearg.n = my_n; 149 | paxos_protocol::prepareres prepareres; 150 | prop_t _na; 151 | _na.n = 0; 152 | _na.m = "0"; 153 | std::string _va; 154 | _va.clear(); 155 | for (auto &node : nodes) { 156 | handle h(node); 157 | int ret = paxos_protocol::OK; 158 | if (!h.safebind()) { 159 | continue; 160 | } 161 | ret = h.safebind()->call(paxos_protocol::preparereq, me, preparearg, prepareres, rpcc::to(1000)); 162 | if (paxos_protocol::OK == ret) { 163 | if (prepareres.oldinstance) { 164 | acc->commit(instance, prepareres.v_a); 165 | return false; 166 | } 167 | accepts.push_back(node); 168 | if (prepareres.n_a > _na) { 169 | _na = prepareres.n_a; 170 | _va = prepareres.v_a; 171 | } 172 | } 173 | } 174 | if (_va.size()) { 175 | v = _va; 176 | } 177 | return true; 178 | } 179 | 180 | // run() calls this to send out accept RPCs to accepts. 181 | // fill in accepts with list of nodes that accepted. 182 | void 183 | proposer::accept(unsigned instance, std::vector &accepts, 184 | std::vector nodes, std::string v) { 185 | // You fill this in for Lab 6 186 | paxos_protocol::acceptarg acceptarg; 187 | acceptarg.instance = instance; 188 | acceptarg.n = my_n; 189 | acceptarg.v = v; 190 | bool r = false; 191 | for (auto &node : nodes) { 192 | handle h(node); 193 | int ret = paxos_protocol::OK; 194 | if (!h.safebind()) { 195 | continue; 196 | } 197 | ret = h.safebind()->call(paxos_protocol::acceptreq, me, acceptarg, r, rpcc::to(1000)); 198 | 199 | if (paxos_protocol::OK == ret && r) { 200 | accepts.push_back(node); 201 | } 202 | 203 | } 204 | } 205 | 206 | void 207 | proposer::decide(unsigned instance, std::vector accepts, 208 | std::string v) { 209 | // You fill this in for Lab 6 210 | paxos_protocol::decidearg decidearg; 211 | decidearg.instance = instance; 212 | decidearg.v = v; 213 | int r; 214 | for (auto &node : accepts) { 215 | handle h(node); 216 | if (h.safebind()) { 217 | h.safebind()->call(paxos_protocol::decidereq, me, decidearg, r, rpcc::to(1000)); 218 | } 219 | } 220 | } 221 | 222 | acceptor::acceptor(class paxos_change *_cfg, bool _first, std::string _me, 223 | std::string _value) 224 | : cfg(_cfg), me(_me), instance_h(0) { 225 | VERIFY (pthread_mutex_init(&pxs_mutex, NULL) == 0); 226 | 227 | n_h.n = 0; 228 | n_h.m = me; 229 | n_a.n = 0; 230 | n_a.m = me; 231 | v_a.clear(); 232 | 233 | l = new log(this, me); 234 | 235 | if (instance_h == 0 && _first) { 236 | values[1] = _value; 237 | l->loginstance(1, _value); 238 | instance_h = 1; 239 | } 240 | 241 | pxs = new rpcs(atoi(_me.c_str())); 242 | pxs->reg(paxos_protocol::preparereq, this, &acceptor::preparereq); 243 | pxs->reg(paxos_protocol::acceptreq, this, &acceptor::acceptreq); 244 | pxs->reg(paxos_protocol::decidereq, this, &acceptor::decidereq); 245 | } 246 | 247 | paxos_protocol::status 248 | acceptor::preparereq(std::string src, paxos_protocol::preparearg a, 249 | paxos_protocol::prepareres &r) { 250 | // You fill this in for Lab 6 251 | // Remember to initialize *BOTH* r.accept and r.oldinstance appropriately. 252 | // Remember to *log* the proposal if the proposal is accepted. 253 | r.accept = r.oldinstance = false; 254 | if (a.instance <= instance_h) { 255 | r.oldinstance = true; 256 | r.v_a = value(a.instance); 257 | return paxos_protocol::OK; 258 | } 259 | 260 | if (a.n >= n_h) { 261 | n_h = a.n; 262 | r.n_a = n_a; 263 | r.v_a = v_a; 264 | l->logprop(n_h); 265 | return paxos_protocol::OK; 266 | } 267 | return paxos_protocol::ERR; 268 | } 269 | 270 | // the src argument is only for debug purpose 271 | paxos_protocol::status 272 | acceptor::acceptreq(std::string src, paxos_protocol::acceptarg a, bool &r) { 273 | // You fill this in for Lab 6 274 | // Remember to *log* the accept if the proposal is accepted. 275 | r = false; 276 | if (n_h > a.n) { 277 | return paxos_protocol::ERR; 278 | } 279 | 280 | n_a = a.n; 281 | v_a = a.v; 282 | r = true; 283 | l->logaccept(n_a, v_a); 284 | 285 | return paxos_protocol::OK; 286 | } 287 | 288 | // the src argument is only for debug purpose 289 | paxos_protocol::status 290 | acceptor::decidereq(std::string src, paxos_protocol::decidearg a, int &r) { 291 | ScopedLock ml(&pxs_mutex); 292 | tprintf("decidereq for accepted instance %d (my instance %d) v=%s\n", 293 | a.instance, instance_h, v_a.c_str()); 294 | if (a.instance == instance_h + 1) { 295 | VERIFY(v_a == a.v); 296 | commit_wo(a.instance, v_a); 297 | } else if (a.instance <= instance_h) { 298 | // we are ahead ignore. 299 | } else { 300 | // we are behind 301 | VERIFY(0); 302 | } 303 | return paxos_protocol::OK; 304 | } 305 | 306 | void 307 | acceptor::commit_wo(unsigned instance, std::string value) { 308 | //assume pxs_mutex is held 309 | tprintf("acceptor::commit: instance=%d has v= %s\n", instance, value.c_str()); 310 | if (instance > instance_h) { 311 | tprintf("commit: highestaccepteinstance = %d\n", instance); 312 | values[instance] = value; 313 | l->loginstance(instance, value); 314 | instance_h = instance; 315 | n_h.n = 0; 316 | n_h.m = me; 317 | n_a.n = 0; 318 | n_a.m = me; 319 | v_a.clear(); 320 | if (cfg) { 321 | pthread_mutex_unlock(&pxs_mutex); 322 | cfg->paxos_commit(instance, value); 323 | pthread_mutex_lock(&pxs_mutex); 324 | } 325 | } 326 | } 327 | 328 | void 329 | acceptor::commit(unsigned instance, std::string value) { 330 | ScopedLock ml(&pxs_mutex); 331 | commit_wo(instance, value); 332 | } 333 | 334 | std::string 335 | acceptor::dump() { 336 | return l->dump(); 337 | } 338 | 339 | void 340 | acceptor::restore(std::string s) { 341 | l->restore(s); 342 | l->logread(); 343 | } 344 | 345 | 346 | 347 | // For testing purposes 348 | 349 | // Call this from your code between phases prepare and accept of proposer 350 | void 351 | proposer::breakpoint1() { 352 | if (break1) { 353 | tprintf("Dying at breakpoint 1!\n"); 354 | exit(1); 355 | } 356 | } 357 | 358 | // Call this from your code between phases accept and decide of proposer 359 | void 360 | proposer::breakpoint2() { 361 | if (break2) { 362 | tprintf("Dying at breakpoint 2!\n"); 363 | exit(1); 364 | } 365 | } 366 | 367 | void 368 | proposer::breakpoint(int b) { 369 | if (b == 3) { 370 | tprintf("Proposer: breakpoint 1\n"); 371 | break1 = true; 372 | } else if (b == 4) { 373 | tprintf("Proposer: breakpoint 2\n"); 374 | break2 = true; 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /paxos.h: -------------------------------------------------------------------------------- 1 | #ifndef paxos_h 2 | #define paxos_h 3 | 4 | #include 5 | #include 6 | #include "rpc.h" 7 | #include "paxos_protocol.h" 8 | #include "log.h" 9 | 10 | 11 | class paxos_change { 12 | public: 13 | virtual void paxos_commit(unsigned instance, std::string v) = 0; 14 | 15 | virtual ~paxos_change() { }; 16 | }; 17 | 18 | class acceptor { 19 | private: 20 | log *l; 21 | rpcs *pxs; 22 | paxos_change *cfg; 23 | std::string me; 24 | pthread_mutex_t pxs_mutex; 25 | 26 | // Acceptor state 27 | prop_t n_h; // number of the highest proposal seen in a prepare 28 | prop_t n_a; // number of highest proposal accepted 29 | std::string v_a; // value of highest proposal accepted 30 | unsigned instance_h; // number of the highest instance we have decided 31 | std::map values; // vals of each instance 32 | 33 | void commit_wo(unsigned instance, std::string v); 34 | 35 | paxos_protocol::status preparereq(std::string src, 36 | paxos_protocol::preparearg a, 37 | paxos_protocol::prepareres &r); 38 | 39 | paxos_protocol::status acceptreq(std::string src, 40 | paxos_protocol::acceptarg a, bool &r); 41 | 42 | paxos_protocol::status decidereq(std::string src, 43 | paxos_protocol::decidearg a, int &r); 44 | 45 | friend class log; 46 | 47 | public: 48 | acceptor(class paxos_change *cfg, bool _first, std::string _me, 49 | std::string _value); 50 | 51 | ~acceptor() { }; 52 | 53 | void commit(unsigned instance, std::string v); 54 | 55 | unsigned instance() { return instance_h; } 56 | 57 | std::string value(unsigned instance) { return values[instance]; } 58 | 59 | std::string dump(); 60 | 61 | void restore(std::string); 62 | 63 | rpcs *get_rpcs() { return pxs; }; 64 | 65 | prop_t get_n_h() { return n_h; }; 66 | 67 | unsigned get_instance_h() { return instance_h; }; 68 | }; 69 | 70 | extern bool isamember(std::string m, const std::vector &nodes); 71 | 72 | extern std::string print_members(const std::vector &nodes); 73 | 74 | class proposer { 75 | private: 76 | log *l; 77 | paxos_change *cfg; 78 | acceptor *acc; 79 | std::string me; 80 | bool break1; 81 | bool break2; 82 | 83 | pthread_mutex_t pxs_mutex; 84 | 85 | // Proposer state 86 | bool stable; 87 | prop_t my_n; // number of the last proposal used in this instance 88 | 89 | void setn(); 90 | 91 | bool prepare(unsigned instance, std::vector &accepts, 92 | std::vector nodes, 93 | std::string &v); 94 | 95 | void accept(unsigned instance, std::vector &accepts, 96 | std::vector nodes, std::string v); 97 | 98 | void decide(unsigned instance, std::vector accepts, 99 | std::string v); 100 | 101 | void breakpoint1(); 102 | 103 | void breakpoint2(); 104 | 105 | bool majority(const std::vector &l1, const std::vector &l2); 106 | 107 | friend class log; 108 | 109 | public: 110 | proposer(class paxos_change *cfg, class acceptor *_acceptor, std::string _me); 111 | 112 | ~proposer() { }; 113 | 114 | bool run(int instance, std::vector cnodes, std::string v); 115 | 116 | bool isrunning(); 117 | 118 | void breakpoint(int b); 119 | }; 120 | 121 | 122 | #endif /* paxos_h */ 123 | -------------------------------------------------------------------------------- /paxos_protocol.h: -------------------------------------------------------------------------------- 1 | #ifndef paxos_protocol_h 2 | #define paxos_protocol_h 3 | 4 | #include "rpc.h" 5 | 6 | struct prop_t { 7 | unsigned n; 8 | std::string m; 9 | }; 10 | 11 | class paxos_protocol { 12 | public: 13 | enum xxstatus { 14 | OK, ERR 15 | }; 16 | typedef int status; 17 | enum rpc_numbers { 18 | preparereq = 0x11001, 19 | acceptreq, 20 | decidereq, 21 | heartbeat, 22 | }; 23 | 24 | struct preparearg { 25 | unsigned instance; 26 | prop_t n; 27 | }; 28 | 29 | struct prepareres { 30 | bool oldinstance; 31 | bool accept; 32 | prop_t n_a; 33 | std::string v_a; 34 | }; 35 | 36 | struct acceptarg { 37 | unsigned instance; 38 | prop_t n; 39 | std::string v; 40 | }; 41 | 42 | struct decidearg { 43 | unsigned instance; 44 | std::string v; 45 | }; 46 | 47 | }; 48 | 49 | inline unmarshall & 50 | operator>>(unmarshall &u, prop_t &a) { 51 | u >> a.n; 52 | u >> a.m; 53 | return u; 54 | } 55 | 56 | inline marshall & 57 | operator<<(marshall &m, prop_t a) { 58 | m << a.n; 59 | m << a.m; 60 | return m; 61 | } 62 | 63 | inline unmarshall & 64 | operator>>(unmarshall &u, paxos_protocol::preparearg &a) { 65 | u >> a.instance; 66 | u >> a.n; 67 | return u; 68 | } 69 | 70 | inline marshall & 71 | operator<<(marshall &m, paxos_protocol::preparearg a) { 72 | m << a.instance; 73 | m << a.n; 74 | return m; 75 | } 76 | 77 | inline unmarshall & 78 | operator>>(unmarshall &u, paxos_protocol::prepareres &r) { 79 | u >> r.oldinstance; 80 | u >> r.accept; 81 | u >> r.n_a; 82 | u >> r.v_a; 83 | return u; 84 | } 85 | 86 | inline marshall & 87 | operator<<(marshall &m, paxos_protocol::prepareres r) { 88 | m << r.oldinstance; 89 | m << r.accept; 90 | m << r.n_a; 91 | m << r.v_a; 92 | return m; 93 | } 94 | 95 | inline unmarshall & 96 | operator>>(unmarshall &u, paxos_protocol::acceptarg &a) { 97 | u >> a.instance; 98 | u >> a.n; 99 | u >> a.v; 100 | return u; 101 | } 102 | 103 | inline marshall & 104 | operator<<(marshall &m, paxos_protocol::acceptarg a) { 105 | m << a.instance; 106 | m << a.n; 107 | m << a.v; 108 | return m; 109 | } 110 | 111 | inline unmarshall & 112 | operator>>(unmarshall &u, paxos_protocol::decidearg &a) { 113 | u >> a.instance; 114 | u >> a.v; 115 | return u; 116 | } 117 | 118 | inline marshall & 119 | operator<<(marshall &m, paxos_protocol::decidearg a) { 120 | m << a.instance; 121 | m << a.v; 122 | return m; 123 | } 124 | 125 | #endif 126 | -------------------------------------------------------------------------------- /raii.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mactavish on 15-5-6. 3 | // 4 | 5 | #include "raii.h" 6 | 7 | -------------------------------------------------------------------------------- /raii.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mactavish on 15-5-6. 3 | // 4 | 5 | #ifndef MIT_6_824_2012_RAII_H 6 | #define MIT_6_824_2012_RAII_H 7 | 8 | #include "lock_client.h" 9 | #include 10 | 11 | class scope_lock{ 12 | public: 13 | scope_lock(lock_client *cl, lock_protocol::lockid_t lid): cl_(cl), lid_(lid) { 14 | cl_->acquire(lid); 15 | } 16 | void unlock(){ 17 | cl_->release(lid_); 18 | } 19 | ~scope_lock(){ 20 | cl_->release(lid_); 21 | } 22 | private: 23 | lock_client *cl_; 24 | lock_protocol::lockid_t lid_; 25 | }; 26 | 27 | 28 | #endif //MIT_6_824_2012_RAII_H 29 | -------------------------------------------------------------------------------- /rpc/connection.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "method_thread.h" 10 | #include "connection.h" 11 | #include "slock.h" 12 | #include "pollmgr.h" 13 | #include "jsl_log.h" 14 | #include "gettime.h" 15 | #include "lang/verify.h" 16 | 17 | #define MAX_PDU (10<<20) //maximum PDF is 10M 18 | 19 | 20 | connection::connection(chanmgr *m1, int f1, int l1) 21 | : mgr_(m1), fd_(f1), dead_(false),waiters_(0), refno_(1),lossy_(l1) 22 | { 23 | 24 | int flags = fcntl(fd_, F_GETFL, NULL); 25 | flags |= O_NONBLOCK; 26 | fcntl(fd_, F_SETFL, flags); 27 | 28 | signal(SIGPIPE, SIG_IGN); 29 | VERIFY(pthread_mutex_init(&m_,0)==0); 30 | VERIFY(pthread_mutex_init(&ref_m_,0)==0); 31 | VERIFY(pthread_cond_init(&send_wait_,0)==0); 32 | VERIFY(pthread_cond_init(&send_complete_,0)==0); 33 | 34 | VERIFY(gettimeofday(&create_time_, NULL) == 0); 35 | 36 | PollMgr::Instance()->add_callback(fd_, CB_RDONLY, this); 37 | } 38 | 39 | connection::~connection() 40 | { 41 | VERIFY(dead_); 42 | VERIFY(pthread_mutex_destroy(&m_)== 0); 43 | VERIFY(pthread_mutex_destroy(&ref_m_)== 0); 44 | VERIFY(pthread_cond_destroy(&send_wait_) == 0); 45 | VERIFY(pthread_cond_destroy(&send_complete_) == 0); 46 | if (rpdu_.buf) 47 | free(rpdu_.buf); 48 | VERIFY(!wpdu_.buf); 49 | close(fd_); 50 | } 51 | 52 | void 53 | connection::incref() 54 | { 55 | ScopedLock ml(&ref_m_); 56 | refno_++; 57 | } 58 | 59 | bool 60 | connection::isdead() 61 | { 62 | ScopedLock ml(&m_); 63 | return dead_; 64 | } 65 | 66 | void 67 | connection::closeconn() 68 | { 69 | { 70 | ScopedLock ml(&m_); 71 | if (!dead_) { 72 | dead_ = true; 73 | shutdown(fd_,SHUT_RDWR); 74 | }else{ 75 | return; 76 | } 77 | } 78 | //after block_remove_fd, select will never wait on fd_ 79 | //and no callbacks will be active 80 | PollMgr::Instance()->block_remove_fd(fd_); 81 | } 82 | 83 | void 84 | connection::decref() 85 | { 86 | VERIFY(pthread_mutex_lock(&ref_m_)==0); 87 | refno_ --; 88 | VERIFY(refno_>=0); 89 | if (refno_==0) { 90 | VERIFY(pthread_mutex_lock(&m_)==0); 91 | if (dead_) { 92 | VERIFY(pthread_mutex_unlock(&ref_m_)==0); 93 | VERIFY(pthread_mutex_unlock(&m_)==0); 94 | delete this; 95 | return; 96 | } 97 | VERIFY(pthread_mutex_unlock(&m_)==0); 98 | } 99 | pthread_mutex_unlock(&ref_m_); 100 | } 101 | 102 | int 103 | connection::ref() 104 | { 105 | ScopedLock rl(&ref_m_); 106 | return refno_; 107 | } 108 | 109 | int 110 | connection::compare(connection *another) 111 | { 112 | if (create_time_.tv_sec > another->create_time_.tv_sec) 113 | return 1; 114 | if (create_time_.tv_sec < another->create_time_.tv_sec) 115 | return -1; 116 | if (create_time_.tv_usec > another->create_time_.tv_usec) 117 | return 1; 118 | if (create_time_.tv_usec < another->create_time_.tv_usec) 119 | return -1; 120 | return 0; 121 | } 122 | 123 | bool 124 | connection::send(char *b, int sz) 125 | { 126 | ScopedLock ml(&m_); 127 | waiters_++; 128 | while (!dead_ && wpdu_.buf) { 129 | VERIFY(pthread_cond_wait(&send_wait_, &m_)==0); 130 | } 131 | waiters_--; 132 | if (dead_) { 133 | return false; 134 | } 135 | wpdu_.buf = b; 136 | wpdu_.sz = sz; 137 | wpdu_.solong = 0; 138 | 139 | if (lossy_) { 140 | if ((random()%100) < lossy_) { 141 | jsl_log(JSL_DBG_1, "connection::send LOSSY TEST shutdown fd_ %d\n", fd_); 142 | shutdown(fd_,SHUT_RDWR); 143 | } 144 | } 145 | 146 | if (!writepdu()) { 147 | dead_ = true; 148 | VERIFY(pthread_mutex_unlock(&m_) == 0); 149 | PollMgr::Instance()->block_remove_fd(fd_); 150 | VERIFY(pthread_mutex_lock(&m_) == 0); 151 | }else{ 152 | if (wpdu_.solong == wpdu_.sz) { 153 | }else{ 154 | //should be rare to need to explicitly add write callback 155 | PollMgr::Instance()->add_callback(fd_, CB_WRONLY, this); 156 | while (!dead_ && wpdu_.solong >= 0 && wpdu_.solong < wpdu_.sz) { 157 | VERIFY(pthread_cond_wait(&send_complete_,&m_) == 0); 158 | } 159 | } 160 | } 161 | bool ret = (!dead_ && wpdu_.solong == wpdu_.sz); 162 | wpdu_.solong = wpdu_.sz = 0; 163 | wpdu_.buf = NULL; 164 | if (waiters_ > 0) 165 | pthread_cond_broadcast(&send_wait_); 166 | return ret; 167 | } 168 | 169 | //fd_ is ready to be written 170 | void 171 | connection::write_cb(int s) 172 | { 173 | ScopedLock ml(&m_); 174 | VERIFY(!dead_); 175 | VERIFY(fd_ == s); 176 | if (wpdu_.sz == 0) { 177 | PollMgr::Instance()->del_callback(fd_,CB_WRONLY); 178 | return; 179 | } 180 | if (!writepdu()) { 181 | PollMgr::Instance()->del_callback(fd_, CB_RDWR); 182 | dead_ = true; 183 | }else{ 184 | VERIFY(wpdu_.solong >= 0); 185 | if (wpdu_.solong < wpdu_.sz) { 186 | return; 187 | } 188 | } 189 | pthread_cond_signal(&send_complete_); 190 | } 191 | 192 | //fd_ is ready to be read 193 | void 194 | connection::read_cb(int s) 195 | { 196 | ScopedLock ml(&m_); 197 | VERIFY(fd_ == s); 198 | if (dead_) { 199 | return; 200 | } 201 | 202 | bool succ = true; 203 | if (!rpdu_.buf || rpdu_.solong < rpdu_.sz) { 204 | succ = readpdu(); 205 | } 206 | 207 | if (!succ) { 208 | PollMgr::Instance()->del_callback(fd_,CB_RDWR); 209 | dead_ = true; 210 | pthread_cond_signal(&send_complete_); 211 | } 212 | 213 | if (rpdu_.buf && rpdu_.sz == rpdu_.solong) { 214 | if (mgr_->got_pdu(this, rpdu_.buf, rpdu_.sz)) { 215 | //chanmgr has successfully consumed the pdu 216 | rpdu_.buf = NULL; 217 | rpdu_.sz = rpdu_.solong = 0; 218 | } 219 | } 220 | } 221 | 222 | bool 223 | connection::writepdu() 224 | { 225 | VERIFY(wpdu_.solong >= 0); 226 | if (wpdu_.solong == wpdu_.sz) 227 | return true; 228 | 229 | if (wpdu_.solong == 0) { 230 | int sz = htonl(wpdu_.sz); 231 | bcopy(&sz,wpdu_.buf,sizeof(sz)); 232 | } 233 | int n = write(fd_, wpdu_.buf + wpdu_.solong, (wpdu_.sz-wpdu_.solong)); 234 | if (n < 0) { 235 | if (errno != EAGAIN) { 236 | jsl_log(JSL_DBG_1, "connection::writepdu fd_ %d failure errno=%d\n", fd_, errno); 237 | wpdu_.solong = -1; 238 | wpdu_.sz = 0; 239 | } 240 | return (errno == EAGAIN); 241 | } 242 | wpdu_.solong += n; 243 | return true; 244 | } 245 | 246 | bool 247 | connection::readpdu() 248 | { 249 | if (!rpdu_.sz) { 250 | int sz, sz1; 251 | int n = read(fd_, &sz1, sizeof(sz1)); 252 | 253 | if (n == 0) { 254 | return false; 255 | } 256 | 257 | if (n < 0) { 258 | VERIFY(errno!=EAGAIN); 259 | return false; 260 | } 261 | 262 | if (n >0 && n!= sizeof(sz)) { 263 | jsl_log(JSL_DBG_OFF, "connection::readpdu short read of sz\n"); 264 | return false; 265 | } 266 | 267 | sz = ntohl(sz1); 268 | 269 | if (sz > MAX_PDU) { 270 | char *tmpb = (char *)&sz1; 271 | jsl_log(JSL_DBG_2, "connection::readpdu read pdu TOO BIG %d network order=%x %x %x %x %x\n", sz, 272 | sz1, tmpb[0],tmpb[1],tmpb[2],tmpb[3]); 273 | return false; 274 | } 275 | 276 | rpdu_.sz = sz; 277 | VERIFY(rpdu_.buf == NULL); 278 | rpdu_.buf = (char *)malloc(sz+sizeof(sz)); 279 | VERIFY(rpdu_.buf); 280 | bcopy(&sz1,rpdu_.buf,sizeof(sz)); 281 | rpdu_.solong = sizeof(sz); 282 | } 283 | 284 | int n = read(fd_, rpdu_.buf + rpdu_.solong, rpdu_.sz - rpdu_.solong); 285 | if (n <= 0) { 286 | if (errno == EAGAIN) 287 | return true; 288 | if (rpdu_.buf) 289 | free(rpdu_.buf); 290 | rpdu_.buf = NULL; 291 | rpdu_.sz = rpdu_.solong = 0; 292 | return (errno == EAGAIN); 293 | } 294 | rpdu_.solong += n; 295 | return true; 296 | } 297 | 298 | tcpsconn::tcpsconn(chanmgr *m1, int port, int lossytest) 299 | : mgr_(m1), lossy_(lossytest) 300 | { 301 | 302 | VERIFY(pthread_mutex_init(&m_,NULL) == 0); 303 | 304 | struct sockaddr_in sin; 305 | memset(&sin, 0, sizeof(sin)); 306 | sin.sin_family = AF_INET; 307 | sin.sin_port = htons(port); 308 | 309 | tcp_ = socket(AF_INET, SOCK_STREAM, 0); 310 | if(tcp_ < 0){ 311 | perror("tcpsconn::tcpsconn accept_loop socket:"); 312 | VERIFY(0); 313 | } 314 | 315 | int yes = 1; 316 | setsockopt(tcp_, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); 317 | setsockopt(tcp_, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); 318 | 319 | if(bind(tcp_, (sockaddr *)&sin, sizeof(sin)) < 0){ 320 | perror("accept_loop tcp bind:"); 321 | VERIFY(0); 322 | } 323 | 324 | if(listen(tcp_, 1000) < 0) { 325 | perror("tcpsconn::tcpsconn listen:"); 326 | VERIFY(0); 327 | } 328 | 329 | socklen_t addrlen = sizeof(sin); 330 | VERIFY(getsockname(tcp_, (sockaddr *)&sin, &addrlen) == 0); 331 | port_ = ntohs(sin.sin_port); 332 | 333 | jsl_log(JSL_DBG_2, "tcpsconn::tcpsconn listen on %d %d\n", port_, 334 | sin.sin_port); 335 | 336 | if (pipe(pipe_) < 0) { 337 | perror("accept_loop pipe:"); 338 | VERIFY(0); 339 | } 340 | 341 | int flags = fcntl(pipe_[0], F_GETFL, NULL); 342 | flags |= O_NONBLOCK; 343 | fcntl(pipe_[0], F_SETFL, flags); 344 | 345 | VERIFY((th_ = method_thread(this, false, &tcpsconn::accept_conn)) != 0); 346 | } 347 | 348 | tcpsconn::~tcpsconn() 349 | { 350 | VERIFY(close(pipe_[1]) == 0); 351 | VERIFY(pthread_join(th_, NULL) == 0); 352 | 353 | //close all the active connections 354 | std::map::iterator i; 355 | for (i = conns_.begin(); i != conns_.end(); i++) { 356 | i->second->closeconn(); 357 | i->second->decref(); 358 | } 359 | } 360 | 361 | void 362 | tcpsconn::process_accept() 363 | { 364 | sockaddr_in sin; 365 | socklen_t slen = sizeof(sin); 366 | int s1 = accept(tcp_, (sockaddr *)&sin, &slen); 367 | if (s1 < 0) { 368 | perror("tcpsconn::accept_conn error"); 369 | pthread_exit(NULL); 370 | } 371 | 372 | jsl_log(JSL_DBG_2, "accept_loop got connection fd=%d %s:%d\n", 373 | s1, inet_ntoa(sin.sin_addr), ntohs(sin.sin_port)); 374 | connection *ch = new connection(mgr_, s1, lossy_); 375 | 376 | // garbage collect all dead connections with refcount of 1 377 | std::map::iterator i; 378 | for (i = conns_.begin(); i != conns_.end();) { 379 | if (i->second->isdead() && i->second->ref() == 1) { 380 | jsl_log(JSL_DBG_2, "accept_loop garbage collected fd=%d\n", 381 | i->second->channo()); 382 | i->second->decref(); 383 | // Careful not to reuse i right after erase. (i++) will 384 | // be evaluated before the erase call because in C++, 385 | // there is a sequence point before a function call. 386 | // See http://en.wikipedia.org/wiki/Sequence_point. 387 | conns_.erase(i++); 388 | } else 389 | ++i; 390 | } 391 | 392 | conns_[ch->channo()] = ch; 393 | } 394 | 395 | void 396 | tcpsconn::accept_conn() 397 | { 398 | fd_set rfds; 399 | int max_fd = pipe_[0] > tcp_ ? pipe_[0] : tcp_; 400 | 401 | while (1) { 402 | FD_ZERO(&rfds); 403 | FD_SET(pipe_[0], &rfds); 404 | FD_SET(tcp_, &rfds); 405 | 406 | int ret = select(max_fd+1, &rfds, NULL, NULL, NULL); 407 | 408 | if (ret < 0) { 409 | if (errno == EINTR) { 410 | continue; 411 | } else { 412 | perror("accept_conn select:"); 413 | jsl_log(JSL_DBG_OFF, "tcpsconn::accept_conn failure errno %d\n",errno); 414 | VERIFY(0); 415 | } 416 | } 417 | 418 | if (FD_ISSET(pipe_[0], &rfds)) { 419 | close(pipe_[0]); 420 | close(tcp_); 421 | return; 422 | } 423 | else if (FD_ISSET(tcp_, &rfds)) { 424 | process_accept(); 425 | } else { 426 | VERIFY(0); 427 | } 428 | } 429 | } 430 | 431 | connection * 432 | connect_to_dst(const sockaddr_in &dst, chanmgr *mgr, int lossy) 433 | { 434 | int s= socket(AF_INET, SOCK_STREAM, 0); 435 | int yes = 1; 436 | setsockopt(s, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); 437 | if(connect(s, (sockaddr*)&dst, sizeof(dst)) < 0) { 438 | jsl_log(JSL_DBG_1, "rpcc::connect_to_dst failed to %s:%d\n", 439 | inet_ntoa(dst.sin_addr), (int)ntohs(dst.sin_port)); 440 | close(s); 441 | return NULL; 442 | } 443 | jsl_log(JSL_DBG_2, "connect_to_dst fd=%d to dst %s:%d\n", 444 | s, inet_ntoa(dst.sin_addr), (int)ntohs(dst.sin_port)); 445 | return new connection(mgr, s, lossy); 446 | } 447 | 448 | 449 | -------------------------------------------------------------------------------- /rpc/connection.h: -------------------------------------------------------------------------------- 1 | #ifndef connection_h 2 | #define connection_h 1 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include "pollmgr.h" 13 | 14 | class connection; 15 | 16 | class chanmgr { 17 | public: 18 | virtual bool got_pdu(connection *c, char *b, int sz) = 0; 19 | virtual ~chanmgr() {} 20 | }; 21 | 22 | class connection : public aio_callback { 23 | public: 24 | struct charbuf { 25 | charbuf(): buf(NULL), sz(0), solong(0) {} 26 | charbuf (char *b, int s) : buf(b), sz(s), solong(0){} 27 | char *buf; 28 | int sz; 29 | int solong; //amount of bytes written or read so far 30 | }; 31 | 32 | connection(chanmgr *m1, int f1, int lossytest=0); 33 | ~connection(); 34 | 35 | int channo() { return fd_; } 36 | bool isdead(); 37 | void closeconn(); 38 | 39 | bool send(char *b, int sz); 40 | void write_cb(int s); 41 | void read_cb(int s); 42 | 43 | void incref(); 44 | void decref(); 45 | int ref(); 46 | 47 | int compare(connection *another); 48 | private: 49 | 50 | bool readpdu(); 51 | bool writepdu(); 52 | 53 | chanmgr *mgr_; 54 | const int fd_; 55 | bool dead_; 56 | 57 | charbuf wpdu_; 58 | charbuf rpdu_; 59 | 60 | struct timeval create_time_; 61 | 62 | int waiters_; 63 | int refno_; 64 | const int lossy_; 65 | 66 | pthread_mutex_t m_; 67 | pthread_mutex_t ref_m_; 68 | pthread_cond_t send_complete_; 69 | pthread_cond_t send_wait_; 70 | }; 71 | 72 | class tcpsconn { 73 | public: 74 | tcpsconn(chanmgr *m1, int port, int lossytest=0); 75 | ~tcpsconn(); 76 | inline int port() { return port_; } 77 | void accept_conn(); 78 | private: 79 | int port_; 80 | pthread_mutex_t m_; 81 | pthread_t th_; 82 | int pipe_[2]; 83 | 84 | int tcp_; //file desciptor for accepting connection 85 | chanmgr *mgr_; 86 | int lossy_; 87 | std::map conns_; 88 | 89 | void process_accept(); 90 | }; 91 | 92 | struct bundle { 93 | bundle(chanmgr *m, int s, int l):mgr(m),tcp(s),lossy(l) {} 94 | chanmgr *mgr; 95 | int tcp; 96 | int lossy; 97 | }; 98 | 99 | void start_accept_thread(chanmgr *mgr, int port, pthread_t *th, int *fd = NULL, int lossy=0); 100 | connection *connect_to_dst(const sockaddr_in &dst, chanmgr *mgr, int lossy=0); 101 | #endif 102 | -------------------------------------------------------------------------------- /rpc/fifo.h: -------------------------------------------------------------------------------- 1 | #ifndef fifo_h 2 | #define fifo_h 3 | 4 | // fifo template 5 | // blocks enq() and deq() when queue is FULL or EMPTY 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "slock.h" 13 | #include "lang/verify.h" 14 | 15 | template 16 | class fifo { 17 | public: 18 | fifo(int m=0); 19 | ~fifo(); 20 | bool enq(T, bool blocking=true); 21 | void deq(T *); 22 | bool size(); 23 | 24 | private: 25 | std::list q_; 26 | pthread_mutex_t m_; 27 | pthread_cond_t non_empty_c_; // q went non-empty 28 | pthread_cond_t has_space_c_; // q is not longer overfull 29 | unsigned int max_; //maximum capacity of the queue, block enq threads if exceeds this limit 30 | }; 31 | 32 | template 33 | fifo::fifo(int limit) : max_(limit) 34 | { 35 | VERIFY(pthread_mutex_init(&m_, 0) == 0); 36 | VERIFY(pthread_cond_init(&non_empty_c_, 0) == 0); 37 | VERIFY(pthread_cond_init(&has_space_c_, 0) == 0); 38 | } 39 | 40 | template 41 | fifo::~fifo() 42 | { 43 | //fifo is to be deleted only when no threads are using it! 44 | VERIFY(pthread_mutex_destroy(&m_)==0); 45 | VERIFY(pthread_cond_destroy(&non_empty_c_) == 0); 46 | VERIFY(pthread_cond_destroy(&has_space_c_) == 0); 47 | } 48 | 49 | template bool 50 | fifo::size() 51 | { 52 | ScopedLock ml(&m_); 53 | return q_.size(); 54 | } 55 | 56 | template bool 57 | fifo::enq(T e, bool blocking) 58 | { 59 | ScopedLock ml(&m_); 60 | while (1) { 61 | if (!max_ || q_.size() < max_) { 62 | q_.push_back(e); 63 | break; 64 | } 65 | if (blocking) 66 | VERIFY(pthread_cond_wait(&has_space_c_, &m_) == 0); 67 | else 68 | return false; 69 | } 70 | VERIFY(pthread_cond_signal(&non_empty_c_) == 0); 71 | return true; 72 | } 73 | 74 | template void 75 | fifo::deq(T *e) 76 | { 77 | ScopedLock ml(&m_); 78 | 79 | while(1) { 80 | if(q_.empty()){ 81 | VERIFY (pthread_cond_wait(&non_empty_c_, &m_) == 0); 82 | } else { 83 | *e = q_.front(); 84 | q_.pop_front(); 85 | if (max_ && q_.size() < max_) { 86 | VERIFY(pthread_cond_signal(&has_space_c_)==0); 87 | } 88 | break; 89 | } 90 | } 91 | return; 92 | } 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /rpc/jsl_log.cc: -------------------------------------------------------------------------------- 1 | #include "jsl_log.h" 2 | 3 | int JSL_DEBUG_LEVEL = 0; 4 | void 5 | jsl_set_debug(int level) { 6 | JSL_DEBUG_LEVEL = level; 7 | } 8 | 9 | 10 | -------------------------------------------------------------------------------- /rpc/jsl_log.h: -------------------------------------------------------------------------------- 1 | #ifndef __JSL_LOG_H__ 2 | #define __JSL_LOG_H__ 1 3 | 4 | enum dbcode { 5 | JSL_DBG_OFF = 0, 6 | JSL_DBG_1 = 1, // Critical 7 | JSL_DBG_2 = 2, // Error 8 | JSL_DBG_3 = 3, // Info 9 | JSL_DBG_4 = 4, // Debugging 10 | }; 11 | 12 | extern int JSL_DEBUG_LEVEL; 13 | 14 | #define jsl_log(level,...) \ 15 | do { \ 16 | if(JSL_DEBUG_LEVEL < abs(level)) \ 17 | {;} \ 18 | else { \ 19 | { printf(__VA_ARGS__);} \ 20 | } \ 21 | } while(0) 22 | 23 | void jsl_set_debug(int level); 24 | 25 | #endif // __JSL_LOG_H__ 26 | -------------------------------------------------------------------------------- /rpc/marshall.h: -------------------------------------------------------------------------------- 1 | #ifndef marshall_h 2 | #define marshall_h 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "lang/verify.h" 14 | #include "lang/algorithm.h" 15 | 16 | struct req_header { 17 | req_header(int x=0, int p=0, int c = 0, int s = 0, int xi = 0): 18 | xid(x), proc(p), clt_nonce(c), srv_nonce(s), xid_rep(xi) {} 19 | int xid; 20 | int proc; 21 | unsigned int clt_nonce; 22 | unsigned int srv_nonce; 23 | int xid_rep; 24 | }; 25 | 26 | struct reply_header { 27 | reply_header(int x=0, int r=0): xid(x), ret(r) {} 28 | int xid; 29 | int ret; 30 | }; 31 | 32 | typedef uint64_t rpc_checksum_t; 33 | typedef int rpc_sz_t; 34 | 35 | enum { 36 | //size of initial buffer allocation 37 | DEFAULT_RPC_SZ = 1024, 38 | #if RPC_CHECKSUMMING 39 | //size of rpc_header includes a 4-byte int to be filled by tcpchan and uint64_t checksum 40 | RPC_HEADER_SZ = static_max::value + sizeof(rpc_sz_t) + sizeof(rpc_checksum_t) 41 | #else 42 | RPC_HEADER_SZ = static_max::value + sizeof(rpc_sz_t) 43 | #endif 44 | }; 45 | 46 | class marshall { 47 | private: 48 | char *_buf; // Base of the raw bytes buffer (dynamically readjusted) 49 | int _capa; // Capacity of the buffer 50 | int _ind; // Read/write head position 51 | 52 | public: 53 | marshall() { 54 | _buf = (char *) malloc(sizeof(char)*DEFAULT_RPC_SZ); 55 | VERIFY(_buf); 56 | _capa = DEFAULT_RPC_SZ; 57 | _ind = RPC_HEADER_SZ; 58 | } 59 | 60 | ~marshall() { 61 | if (_buf) 62 | free(_buf); 63 | } 64 | 65 | int size() { return _ind;} 66 | char *cstr() { return _buf;} 67 | 68 | void rawbyte(unsigned char); 69 | void rawbytes(const char *, int); 70 | 71 | // Return the current content (excluding header) as a string 72 | std::string get_content() { 73 | return std::string(_buf+RPC_HEADER_SZ,_ind-RPC_HEADER_SZ); 74 | } 75 | 76 | // Return the current content (excluding header) as a string 77 | std::string str() { 78 | return get_content(); 79 | } 80 | 81 | void pack(int i); 82 | 83 | void pack_req_header(const req_header &h) { 84 | int saved_sz = _ind; 85 | //leave the first 4-byte empty for channel to fill size of pdu 86 | _ind = sizeof(rpc_sz_t); 87 | #if RPC_CHECKSUMMING 88 | _ind += sizeof(rpc_checksum_t); 89 | #endif 90 | pack(h.xid); 91 | pack(h.proc); 92 | pack((int)h.clt_nonce); 93 | pack((int)h.srv_nonce); 94 | pack(h.xid_rep); 95 | _ind = saved_sz; 96 | } 97 | 98 | void pack_reply_header(const reply_header &h) { 99 | int saved_sz = _ind; 100 | //leave the first 4-byte empty for channel to fill size of pdu 101 | _ind = sizeof(rpc_sz_t); 102 | #if RPC_CHECKSUMMING 103 | _ind += sizeof(rpc_checksum_t); 104 | #endif 105 | pack(h.xid); 106 | pack(h.ret); 107 | _ind = saved_sz; 108 | } 109 | 110 | void take_buf(char **b, int *s) { 111 | *b = _buf; 112 | *s = _ind; 113 | _buf = NULL; 114 | _ind = 0; 115 | return; 116 | } 117 | }; 118 | marshall& operator<<(marshall &, bool); 119 | marshall& operator<<(marshall &, unsigned int); 120 | marshall& operator<<(marshall &, int); 121 | marshall& operator<<(marshall &, unsigned char); 122 | marshall& operator<<(marshall &, char); 123 | marshall& operator<<(marshall &, unsigned short); 124 | marshall& operator<<(marshall &, short); 125 | marshall& operator<<(marshall &, unsigned long long); 126 | marshall& operator<<(marshall &, const std::string &); 127 | 128 | class unmarshall { 129 | private: 130 | char *_buf; 131 | int _sz; 132 | int _ind; 133 | bool _ok; 134 | public: 135 | unmarshall(): _buf(NULL),_sz(0),_ind(0),_ok(false) {} 136 | unmarshall(char *b, int sz): _buf(b),_sz(sz),_ind(),_ok(true) {} 137 | unmarshall(const std::string &s) : _buf(NULL),_sz(0),_ind(0),_ok(false) 138 | { 139 | //take the content which does not exclude a RPC header from a string 140 | take_content(s); 141 | } 142 | ~unmarshall() { 143 | if (_buf) free(_buf); 144 | } 145 | 146 | //take contents from another unmarshall object 147 | void take_in(unmarshall &another); 148 | 149 | //take the content which does not exclude a RPC header from a string 150 | void take_content(const std::string &s) { 151 | _sz = s.size()+RPC_HEADER_SZ; 152 | _buf = (char *)realloc(_buf,_sz); 153 | VERIFY(_buf); 154 | _ind = RPC_HEADER_SZ; 155 | memcpy(_buf+_ind, s.data(), s.size()); 156 | _ok = true; 157 | } 158 | 159 | bool ok() { return _ok; } 160 | char *cstr() { return _buf;} 161 | bool okdone(); 162 | unsigned int rawbyte(); 163 | void rawbytes(std::string &s, unsigned int n); 164 | 165 | int ind() { return _ind;} 166 | int size() { return _sz;} 167 | void unpack(int *); //non-const ref 168 | void take_buf(char **b, int *sz) { 169 | *b = _buf; 170 | *sz = _sz; 171 | _sz = _ind = 0; 172 | _buf = NULL; 173 | } 174 | 175 | void unpack_req_header(req_header *h) { 176 | //the first 4-byte is for channel to fill size of pdu 177 | _ind = sizeof(rpc_sz_t); 178 | #if RPC_CHECKSUMMING 179 | _ind += sizeof(rpc_checksum_t); 180 | #endif 181 | unpack(&h->xid); 182 | unpack(&h->proc); 183 | unpack((int *)&h->clt_nonce); 184 | unpack((int *)&h->srv_nonce); 185 | unpack(&h->xid_rep); 186 | _ind = RPC_HEADER_SZ; 187 | } 188 | 189 | void unpack_reply_header(reply_header *h) { 190 | //the first 4-byte is for channel to fill size of pdu 191 | _ind = sizeof(rpc_sz_t); 192 | #if RPC_CHECKSUMMING 193 | _ind += sizeof(rpc_checksum_t); 194 | #endif 195 | unpack(&h->xid); 196 | unpack(&h->ret); 197 | _ind = RPC_HEADER_SZ; 198 | } 199 | }; 200 | 201 | unmarshall& operator>>(unmarshall &, bool &); 202 | unmarshall& operator>>(unmarshall &, unsigned char &); 203 | unmarshall& operator>>(unmarshall &, char &); 204 | unmarshall& operator>>(unmarshall &, unsigned short &); 205 | unmarshall& operator>>(unmarshall &, short &); 206 | unmarshall& operator>>(unmarshall &, unsigned int &); 207 | unmarshall& operator>>(unmarshall &, int &); 208 | unmarshall& operator>>(unmarshall &, unsigned long long &); 209 | unmarshall& operator>>(unmarshall &, std::string &); 210 | 211 | template marshall & 212 | operator<<(marshall &m, std::vector v) 213 | { 214 | m << (unsigned int) v.size(); 215 | for(unsigned i = 0; i < v.size(); i++) 216 | m << v[i]; 217 | return m; 218 | } 219 | 220 | template unmarshall & 221 | operator>>(unmarshall &u, std::vector &v) 222 | { 223 | v.clear(); 224 | unsigned n; 225 | u >> n; 226 | for(unsigned i = 0; i < n; i++){ 227 | C z; 228 | u >> z; 229 | v.push_back(z); 230 | } 231 | return u; 232 | } 233 | 234 | template marshall & 235 | operator<<(marshall &m, const std::map &d) { 236 | typename std::map::const_iterator i; 237 | 238 | m << (unsigned int) d.size(); 239 | 240 | for (i = d.begin(); i != d.end(); i++) { 241 | m << i->first << i->second; 242 | } 243 | return m; 244 | } 245 | 246 | template unmarshall & 247 | operator>>(unmarshall &u, std::map &d) { 248 | unsigned int n; 249 | u >> n; 250 | 251 | d.clear(); 252 | 253 | for (unsigned int lcv = 0; lcv < n; lcv++) { 254 | A a; 255 | B b; 256 | u >> a >> b; 257 | d[a] = b; 258 | } 259 | return u; 260 | } 261 | 262 | #endif 263 | -------------------------------------------------------------------------------- /rpc/method_thread.h: -------------------------------------------------------------------------------- 1 | #ifndef method_thread_h 2 | #define method_thread_h 3 | 4 | // method_thread(): start a thread that runs an object method. 5 | // returns a pthread_t on success, and zero on error. 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "lang/verify.h" 12 | 13 | static pthread_t 14 | method_thread_parent(void *(*fn)(void *), void *arg, bool detach) 15 | { 16 | pthread_t th; 17 | pthread_attr_t attr; 18 | pthread_attr_init(&attr); 19 | // set stack size to 100K, so we don't run out of memory 20 | pthread_attr_setstacksize(&attr, 100*1024); 21 | int err = pthread_create(&th, &attr, fn, arg); 22 | pthread_attr_destroy(&attr); 23 | if (err != 0) { 24 | fprintf(stderr, "pthread_create ret %d %s\n", err, strerror(err)); 25 | exit(1); 26 | } 27 | 28 | if (detach) { 29 | // don't keep thread state around after exit, to avoid 30 | // running out of threads. set detach==false if you plan 31 | // to pthread_join. 32 | VERIFY(pthread_detach(th) == 0); 33 | } 34 | 35 | return th; 36 | } 37 | 38 | static void 39 | method_thread_child() 40 | { 41 | // defer pthread_cancel() by default. check explicitly by 42 | // enabling then pthread_testcancel(). 43 | int oldstate, oldtype; 44 | VERIFY(pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &oldstate) == 0); 45 | VERIFY(pthread_setcanceltype(PTHREAD_CANCEL_DEFERRED, &oldtype) == 0); 46 | } 47 | 48 | template pthread_t 49 | method_thread(C *o, bool detach, void (C::*m)()) 50 | { 51 | class XXX { 52 | public: 53 | C *o; 54 | void (C::*m)(); 55 | static void *yyy(void *vvv) { 56 | XXX *x = (XXX*)vvv; 57 | C *o = x->o; 58 | void (C::*m)() = x->m; 59 | delete x; 60 | method_thread_child(); 61 | (o->*m)(); 62 | return 0; 63 | } 64 | }; 65 | XXX *x = new XXX; 66 | x->o = o; 67 | x->m = m; 68 | return method_thread_parent(&XXX::yyy, (void *) x, detach); 69 | } 70 | 71 | template pthread_t 72 | method_thread(C *o, bool detach, void (C::*m)(A), A a) 73 | { 74 | class XXX { 75 | public: 76 | C *o; 77 | void (C::*m)(A a); 78 | A a; 79 | static void *yyy(void *vvv) { 80 | XXX *x = (XXX*)vvv; 81 | C *o = x->o; 82 | void (C::*m)(A ) = x->m; 83 | A a = x->a; 84 | delete x; 85 | method_thread_child(); 86 | (o->*m)(a); 87 | return 0; 88 | } 89 | }; 90 | XXX *x = new XXX; 91 | x->o = o; 92 | x->m = m; 93 | x->a = a; 94 | return method_thread_parent(&XXX::yyy, (void *) x, detach); 95 | } 96 | 97 | namespace { 98 | // ~xavid: this causes a bizzare compile error on OS X.5 when 99 | // it's declared in the function, so I moved it out here. 100 | template 101 | class XXX { 102 | public: 103 | C *o; 104 | void (C::*m)(A1 a1, A2 a2); 105 | A1 a1; 106 | A2 a2; 107 | static void *yyy(void *vvv) { 108 | XXX *x = (XXX*)vvv; 109 | C *o = x->o; 110 | void (C::*m)(A1 , A2 ) = x->m; 111 | A1 a1 = x->a1; 112 | A2 a2 = x->a2; 113 | delete x; 114 | method_thread_child(); 115 | (o->*m)(a1, a2); 116 | return 0; 117 | } 118 | }; 119 | } 120 | 121 | template pthread_t 122 | method_thread(C *o, bool detach, void (C::*m)(A1 , A2 ), A1 a1, A2 a2) 123 | { 124 | XXX *x = new XXX; 125 | x->o = o; 126 | x->m = m; 127 | x->a1 = a1; 128 | x->a2 = a2; 129 | return method_thread_parent(&XXX::yyy, (void *) x, detach); 130 | } 131 | 132 | template pthread_t 133 | method_thread(C *o, bool detach, void (C::*m)(A1 , A2, A3 ), A1 a1, A2 a2, A3 a3) 134 | { 135 | class XXX { 136 | public: 137 | C *o; 138 | void (C::*m)(A1 a1, A2 a2, A3 a3); 139 | A1 a1; 140 | A2 a2; 141 | A3 a3; 142 | static void *yyy(void *vvv) { 143 | XXX *x = (XXX*)vvv; 144 | C *o = x->o; 145 | void (C::*m)(A1 , A2 , A3 ) = x->m; 146 | A1 a1 = x->a1; 147 | A2 a2 = x->a2; 148 | A3 a3 = x->a3; 149 | delete x; 150 | method_thread_child(); 151 | (o->*m)(a1, a2, a3); 152 | return 0; 153 | } 154 | }; 155 | XXX *x = new XXX; 156 | x->o = o; 157 | x->m = m; 158 | x->a1 = a1; 159 | x->a2 = a2; 160 | x->a3 = a3; 161 | return method_thread_parent(&XXX::yyy, (void *) x, detach); 162 | } 163 | 164 | #endif 165 | -------------------------------------------------------------------------------- /rpc/pollmgr.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "slock.h" 7 | #include "jsl_log.h" 8 | #include "method_thread.h" 9 | #include "lang/verify.h" 10 | #include "pollmgr.h" 11 | 12 | PollMgr *PollMgr::instance = NULL; 13 | static pthread_once_t pollmgr_is_initialized = PTHREAD_ONCE_INIT; 14 | 15 | void 16 | PollMgrInit() 17 | { 18 | PollMgr::instance = new PollMgr(); 19 | } 20 | 21 | PollMgr * 22 | PollMgr::Instance() 23 | { 24 | pthread_once(&pollmgr_is_initialized, PollMgrInit); 25 | return instance; 26 | } 27 | 28 | PollMgr::PollMgr() : pending_change_(false) 29 | { 30 | bzero(callbacks_, MAX_POLL_FDS*sizeof(void *)); 31 | aio_ = new SelectAIO(); 32 | //aio_ = new EPollAIO(); 33 | 34 | VERIFY(pthread_mutex_init(&m_, NULL) == 0); 35 | VERIFY(pthread_cond_init(&changedone_c_, NULL) == 0); 36 | VERIFY((th_ = method_thread(this, false, &PollMgr::wait_loop)) != 0); 37 | } 38 | 39 | PollMgr::~PollMgr() 40 | { 41 | //never kill me!!! 42 | VERIFY(0); 43 | } 44 | 45 | void 46 | PollMgr::add_callback(int fd, poll_flag flag, aio_callback *ch) 47 | { 48 | VERIFY(fd < MAX_POLL_FDS); 49 | 50 | ScopedLock ml(&m_); 51 | aio_->watch_fd(fd, flag); 52 | 53 | VERIFY(!callbacks_[fd] || callbacks_[fd]==ch); 54 | callbacks_[fd] = ch; 55 | } 56 | 57 | //remove all callbacks related to fd 58 | //the return guarantees that callbacks related to fd 59 | //will never be called again 60 | void 61 | PollMgr::block_remove_fd(int fd) 62 | { 63 | ScopedLock ml(&m_); 64 | aio_->unwatch_fd(fd, CB_RDWR); 65 | pending_change_ = true; 66 | VERIFY(pthread_cond_wait(&changedone_c_, &m_)==0); 67 | callbacks_[fd] = NULL; 68 | } 69 | 70 | void 71 | PollMgr::del_callback(int fd, poll_flag flag) 72 | { 73 | ScopedLock ml(&m_); 74 | if (aio_->unwatch_fd(fd, flag)) { 75 | callbacks_[fd] = NULL; 76 | } 77 | } 78 | 79 | bool 80 | PollMgr::has_callback(int fd, poll_flag flag, aio_callback *c) 81 | { 82 | ScopedLock ml(&m_); 83 | if (!callbacks_[fd] || callbacks_[fd]!=c) 84 | return false; 85 | 86 | return aio_->is_watched(fd, flag); 87 | } 88 | 89 | void 90 | PollMgr::wait_loop() 91 | { 92 | 93 | std::vector readable; 94 | std::vector writable; 95 | 96 | while (1) { 97 | { 98 | ScopedLock ml(&m_); 99 | if (pending_change_) { 100 | pending_change_ = false; 101 | VERIFY(pthread_cond_broadcast(&changedone_c_)==0); 102 | } 103 | } 104 | readable.clear(); 105 | writable.clear(); 106 | aio_->wait_ready(&readable,&writable); 107 | 108 | if (!readable.size() && !writable.size()) { 109 | continue; 110 | } 111 | //no locking of m_ 112 | //because no add_callback() and del_callback should 113 | //modify callbacks_[fd] while the fd is not dead 114 | for (unsigned int i = 0; i < readable.size(); i++) { 115 | int fd = readable[i]; 116 | if (callbacks_[fd]) 117 | callbacks_[fd]->read_cb(fd); 118 | } 119 | 120 | for (unsigned int i = 0; i < writable.size(); i++) { 121 | int fd = writable[i]; 122 | if (callbacks_[fd]) 123 | callbacks_[fd]->write_cb(fd); 124 | } 125 | } 126 | } 127 | 128 | SelectAIO::SelectAIO() : highfds_(0) 129 | { 130 | FD_ZERO(&rfds_); 131 | FD_ZERO(&wfds_); 132 | 133 | VERIFY(pipe(pipefd_) == 0); 134 | FD_SET(pipefd_[0], &rfds_); 135 | highfds_ = pipefd_[0]; 136 | 137 | int flags = fcntl(pipefd_[0], F_GETFL, NULL); 138 | flags |= O_NONBLOCK; 139 | fcntl(pipefd_[0], F_SETFL, flags); 140 | 141 | VERIFY(pthread_mutex_init(&m_, NULL) == 0); 142 | } 143 | 144 | SelectAIO::~SelectAIO() 145 | { 146 | VERIFY(pthread_mutex_destroy(&m_) == 0); 147 | } 148 | 149 | void 150 | SelectAIO::watch_fd(int fd, poll_flag flag) 151 | { 152 | ScopedLock ml(&m_); 153 | if (highfds_ <= fd) 154 | highfds_ = fd; 155 | 156 | if (flag == CB_RDONLY) { 157 | FD_SET(fd,&rfds_); 158 | }else if (flag == CB_WRONLY) { 159 | FD_SET(fd,&wfds_); 160 | }else { 161 | FD_SET(fd,&rfds_); 162 | FD_SET(fd,&wfds_); 163 | } 164 | 165 | char tmp = 1; 166 | VERIFY(write(pipefd_[1], &tmp, sizeof(tmp))==1); 167 | } 168 | 169 | bool 170 | SelectAIO::is_watched(int fd, poll_flag flag) 171 | { 172 | ScopedLock ml(&m_); 173 | if (flag == CB_RDONLY) { 174 | return FD_ISSET(fd,&rfds_); 175 | }else if (flag == CB_WRONLY) { 176 | return FD_ISSET(fd,&wfds_); 177 | }else{ 178 | return (FD_ISSET(fd,&rfds_) && FD_ISSET(fd,&wfds_)); 179 | } 180 | } 181 | 182 | bool 183 | SelectAIO::unwatch_fd(int fd, poll_flag flag) 184 | { 185 | ScopedLock ml(&m_); 186 | if (flag == CB_RDONLY) { 187 | FD_CLR(fd, &rfds_); 188 | }else if (flag == CB_WRONLY) { 189 | FD_CLR(fd, &wfds_); 190 | }else if (flag == CB_RDWR) { 191 | FD_CLR(fd, &wfds_); 192 | FD_CLR(fd, &rfds_); 193 | }else{ 194 | VERIFY(0); 195 | } 196 | 197 | if (!FD_ISSET(fd,&rfds_) && !FD_ISSET(fd,&wfds_)) { 198 | if (fd == highfds_) { 199 | int newh = pipefd_[0]; 200 | for (int i = 0; i <= highfds_; i++) { 201 | if (FD_ISSET(i, &rfds_)) { 202 | newh = i; 203 | }else if (FD_ISSET(i, &wfds_)) { 204 | newh = i; 205 | } 206 | } 207 | highfds_ = newh; 208 | } 209 | } 210 | if (flag == CB_RDWR) { 211 | char tmp = 1; 212 | VERIFY(write(pipefd_[1], &tmp, sizeof(tmp))==1); 213 | } 214 | return (!FD_ISSET(fd, &rfds_) && !FD_ISSET(fd, &wfds_)); 215 | } 216 | 217 | void 218 | SelectAIO::wait_ready(std::vector *readable, std::vector *writable) 219 | { 220 | fd_set trfds, twfds; 221 | int high; 222 | 223 | { 224 | ScopedLock ml(&m_); 225 | trfds = rfds_; 226 | twfds = wfds_; 227 | high = highfds_; 228 | 229 | } 230 | 231 | int ret = select(high+1, &trfds, &twfds, NULL, NULL); 232 | 233 | if (ret < 0) { 234 | if (errno == EINTR) { 235 | return; 236 | } else { 237 | perror("select:"); 238 | jsl_log(JSL_DBG_OFF, "PollMgr::select_loop failure errno %d\n",errno); 239 | VERIFY(0); 240 | } 241 | } 242 | 243 | for (int fd = 0; fd <= high; fd++) { 244 | if (fd == pipefd_[0] && FD_ISSET(fd, &trfds)) { 245 | char tmp; 246 | VERIFY (read(pipefd_[0],&tmp,sizeof(tmp))==1); 247 | VERIFY(tmp==1); 248 | }else { 249 | if (FD_ISSET(fd, &twfds)) { 250 | writable->push_back(fd); 251 | } 252 | if (FD_ISSET(fd, &trfds)) { 253 | readable->push_back(fd); 254 | } 255 | } 256 | } 257 | } 258 | 259 | #ifdef __linux__ 260 | 261 | EPollAIO::EPollAIO() 262 | { 263 | pollfd_ = epoll_create(MAX_POLL_FDS); 264 | VERIFY(pollfd_ >= 0); 265 | bzero(fdstatus_, sizeof(int)*MAX_POLL_FDS); 266 | } 267 | 268 | EPollAIO::~EPollAIO() 269 | { 270 | close(pollfd_); 271 | } 272 | 273 | static inline 274 | int poll_flag_to_event(poll_flag flag) 275 | { 276 | int f; 277 | if (flag == CB_RDONLY) { 278 | f = EPOLLIN; 279 | }else if (flag == CB_WRONLY) { 280 | f = EPOLLOUT; 281 | }else { //flag == CB_RDWR 282 | f = EPOLLIN | EPOLLOUT; 283 | } 284 | return f; 285 | } 286 | 287 | void 288 | EPollAIO::watch_fd(int fd, poll_flag flag) 289 | { 290 | VERIFY(fd < MAX_POLL_FDS); 291 | 292 | struct epoll_event ev; 293 | int op = fdstatus_[fd]? EPOLL_CTL_MOD : EPOLL_CTL_ADD; 294 | fdstatus_[fd] |= (int)flag; 295 | 296 | ev.events = EPOLLET; 297 | ev.data.fd = fd; 298 | 299 | if (fdstatus_[fd] & CB_RDONLY) { 300 | ev.events |= EPOLLIN; 301 | } 302 | if (fdstatus_[fd] & CB_WRONLY) { 303 | ev.events |= EPOLLOUT; 304 | } 305 | 306 | if (flag == CB_RDWR) { 307 | VERIFY(ev.events == (uint32_t)(EPOLLET | EPOLLIN | EPOLLOUT)); 308 | } 309 | 310 | VERIFY(epoll_ctl(pollfd_, op, fd, &ev) == 0); 311 | } 312 | 313 | bool 314 | EPollAIO::unwatch_fd(int fd, poll_flag flag) 315 | { 316 | VERIFY(fd < MAX_POLL_FDS); 317 | fdstatus_[fd] &= ~(int)flag; 318 | 319 | struct epoll_event ev; 320 | int op = fdstatus_[fd]? EPOLL_CTL_MOD : EPOLL_CTL_DEL; 321 | 322 | ev.events = EPOLLET; 323 | ev.data.fd = fd; 324 | 325 | if (fdstatus_[fd] & CB_RDONLY) { 326 | ev.events |= EPOLLIN; 327 | } 328 | if (fdstatus_[fd] & CB_WRONLY) { 329 | ev.events |= EPOLLOUT; 330 | } 331 | 332 | if (flag == CB_RDWR) { 333 | VERIFY(op == EPOLL_CTL_DEL); 334 | } 335 | VERIFY(epoll_ctl(pollfd_, op, fd, &ev) == 0); 336 | return (op == EPOLL_CTL_DEL); 337 | } 338 | 339 | bool 340 | EPollAIO::is_watched(int fd, poll_flag flag) 341 | { 342 | VERIFY(fd < MAX_POLL_FDS); 343 | return ((fdstatus_[fd] & CB_MASK) == flag); 344 | } 345 | 346 | void 347 | EPollAIO::wait_ready(std::vector *readable, std::vector *writable) 348 | { 349 | int nfds = epoll_wait(pollfd_, ready_, MAX_POLL_FDS, -1); 350 | for (int i = 0; i < nfds; i++) { 351 | if (ready_[i].events & EPOLLIN) { 352 | readable->push_back(ready_[i].data.fd); 353 | } 354 | if (ready_[i].events & EPOLLOUT) { 355 | writable->push_back(ready_[i].data.fd); 356 | } 357 | } 358 | } 359 | 360 | #endif 361 | -------------------------------------------------------------------------------- /rpc/pollmgr.h: -------------------------------------------------------------------------------- 1 | #ifndef pollmgr_h 2 | #define pollmgr_h 3 | 4 | #include 5 | #include 6 | 7 | #ifdef __linux__ 8 | #include 9 | #endif 10 | 11 | #define MAX_POLL_FDS 128 12 | 13 | typedef enum { 14 | CB_NONE = 0x0, 15 | CB_RDONLY = 0x1, 16 | CB_WRONLY = 0x10, 17 | CB_RDWR = 0x11, 18 | CB_MASK = ~0x11, 19 | } poll_flag; 20 | 21 | class aio_mgr { 22 | public: 23 | virtual void watch_fd(int fd, poll_flag flag) = 0; 24 | virtual bool unwatch_fd(int fd, poll_flag flag) = 0; 25 | virtual bool is_watched(int fd, poll_flag flag) = 0; 26 | virtual void wait_ready(std::vector *readable, std::vector *writable) = 0; 27 | virtual ~aio_mgr() {} 28 | }; 29 | 30 | class aio_callback { 31 | public: 32 | virtual void read_cb(int fd) = 0; 33 | virtual void write_cb(int fd) = 0; 34 | virtual ~aio_callback() {} 35 | }; 36 | 37 | class PollMgr { 38 | public: 39 | PollMgr(); 40 | ~PollMgr(); 41 | 42 | static PollMgr *Instance(); 43 | static PollMgr *CreateInst(); 44 | 45 | void add_callback(int fd, poll_flag flag, aio_callback *ch); 46 | void del_callback(int fd, poll_flag flag); 47 | bool has_callback(int fd, poll_flag flag, aio_callback *ch); 48 | void block_remove_fd(int fd); 49 | void wait_loop(); 50 | 51 | 52 | static PollMgr *instance; 53 | static int useful; 54 | static int useless; 55 | 56 | private: 57 | pthread_mutex_t m_; 58 | pthread_cond_t changedone_c_; 59 | pthread_t th_; 60 | 61 | aio_callback *callbacks_[MAX_POLL_FDS]; 62 | aio_mgr *aio_; 63 | bool pending_change_; 64 | 65 | }; 66 | 67 | class SelectAIO : public aio_mgr { 68 | public : 69 | 70 | SelectAIO(); 71 | ~SelectAIO(); 72 | void watch_fd(int fd, poll_flag flag); 73 | bool unwatch_fd(int fd, poll_flag flag); 74 | bool is_watched(int fd, poll_flag flag); 75 | void wait_ready(std::vector *readable, std::vector *writable); 76 | 77 | private: 78 | 79 | fd_set rfds_; 80 | fd_set wfds_; 81 | int highfds_; 82 | int pipefd_[2]; 83 | 84 | pthread_mutex_t m_; 85 | 86 | }; 87 | 88 | #ifdef __linux__ 89 | class EPollAIO : public aio_mgr { 90 | public: 91 | EPollAIO(); 92 | ~EPollAIO(); 93 | void watch_fd(int fd, poll_flag flag); 94 | bool unwatch_fd(int fd, poll_flag flag); 95 | bool is_watched(int fd, poll_flag flag); 96 | void wait_ready(std::vector *readable, std::vector *writable); 97 | 98 | private: 99 | int pollfd_; 100 | struct epoll_event ready_[MAX_POLL_FDS]; 101 | int fdstatus_[MAX_POLL_FDS]; 102 | 103 | }; 104 | #endif /* __linux */ 105 | 106 | #endif /* pollmgr_h */ 107 | 108 | -------------------------------------------------------------------------------- /rpc/slock.h: -------------------------------------------------------------------------------- 1 | #ifndef __SCOPED_LOCK__ 2 | #define __SCOPED_LOCK__ 3 | 4 | #include 5 | #include "lang/verify.h" 6 | struct ScopedLock { 7 | private: 8 | pthread_mutex_t *m_; 9 | public: 10 | ScopedLock(pthread_mutex_t *m): m_(m) { 11 | VERIFY(pthread_mutex_lock(m_)==0); 12 | } 13 | ~ScopedLock() { 14 | VERIFY(pthread_mutex_unlock(m_)==0); 15 | } 16 | }; 17 | #endif /*__SCOPED_LOCK__*/ 18 | -------------------------------------------------------------------------------- /rpc/thr_pool.cc: -------------------------------------------------------------------------------- 1 | #include "slock.h" 2 | #include "thr_pool.h" 3 | #include 4 | #include 5 | #include "lang/verify.h" 6 | 7 | static void * 8 | do_worker(void *arg) 9 | { 10 | ThrPool *tp = (ThrPool *)arg; 11 | while (1) { 12 | ThrPool::job_t j; 13 | if (!tp->takeJob(&j)) 14 | break; //die 15 | 16 | (void)(j.f)(j.a); 17 | } 18 | pthread_exit(NULL); 19 | } 20 | 21 | //if blocking, then addJob() blocks when queue is full 22 | //otherwise, addJob() simply returns false when queue is full 23 | ThrPool::ThrPool(int sz, bool blocking) 24 | : nthreads_(sz),blockadd_(blocking),jobq_(100*sz) 25 | { 26 | pthread_attr_init(&attr_); 27 | pthread_attr_setstacksize(&attr_, 128<<10); 28 | 29 | for (int i = 0; i < sz; i++) { 30 | pthread_t t; 31 | VERIFY(pthread_create(&t, &attr_, do_worker, (void *)this) ==0); 32 | th_.push_back(t); 33 | } 34 | } 35 | 36 | //IMPORTANT: this function can be called only when no external thread 37 | //will ever use this thread pool again or is currently blocking on it 38 | ThrPool::~ThrPool() 39 | { 40 | for (int i = 0; i < nthreads_; i++) { 41 | job_t j; 42 | j.f = (void *(*)(void *))NULL; //poison pill to tell worker threads to exit 43 | jobq_.enq(j); 44 | } 45 | 46 | for (int i = 0; i < nthreads_; i++) { 47 | VERIFY(pthread_join(th_[i], NULL)==0); 48 | } 49 | 50 | VERIFY(pthread_attr_destroy(&attr_)==0); 51 | } 52 | 53 | bool 54 | ThrPool::addJob(void *(*f)(void *), void *a) 55 | { 56 | job_t j; 57 | j.f = f; 58 | j.a = a; 59 | 60 | return jobq_.enq(j,blockadd_); 61 | } 62 | 63 | bool 64 | ThrPool::takeJob(job_t *j) 65 | { 66 | jobq_.deq(j); 67 | return (j->f!=NULL); 68 | } 69 | 70 | -------------------------------------------------------------------------------- /rpc/thr_pool.h: -------------------------------------------------------------------------------- 1 | #ifndef __THR_POOL__ 2 | #define __THR_POOL__ 3 | 4 | #include 5 | #include 6 | 7 | #include "fifo.h" 8 | 9 | class ThrPool { 10 | 11 | 12 | public: 13 | struct job_t { 14 | void *(*f)(void *); //function point 15 | void *a; //function arguments 16 | }; 17 | 18 | ThrPool(int sz, bool blocking=true); 19 | ~ThrPool(); 20 | template bool addObjJob(C *o, void (C::*m)(A), A a); 21 | void waitDone(); 22 | 23 | bool takeJob(job_t *j); 24 | 25 | private: 26 | pthread_attr_t attr_; 27 | int nthreads_; 28 | bool blockadd_; 29 | 30 | 31 | fifo jobq_; 32 | std::vector th_; 33 | 34 | bool addJob(void *(*f)(void *), void *a); 35 | }; 36 | 37 | template bool 38 | ThrPool::addObjJob(C *o, void (C::*m)(A), A a) 39 | { 40 | 41 | class objfunc_wrapper { 42 | public: 43 | C *o; 44 | void (C::*m)(A a); 45 | A a; 46 | static void *func(void *vvv) { 47 | objfunc_wrapper *x = (objfunc_wrapper*)vvv; 48 | C *o = x->o; 49 | void (C::*m)(A ) = x->m; 50 | A a = x->a; 51 | (o->*m)(a); 52 | delete x; 53 | return 0; 54 | } 55 | }; 56 | 57 | objfunc_wrapper *x = new objfunc_wrapper; 58 | x->o = o; 59 | x->m = m; 60 | x->a = a; 61 | return addJob(&objfunc_wrapper::func, (void *)x); 62 | } 63 | 64 | 65 | #endif 66 | 67 | -------------------------------------------------------------------------------- /rsm.h: -------------------------------------------------------------------------------- 1 | // replicated state machine interface. 2 | 3 | #ifndef rsm_h 4 | #define rsm_h 5 | 6 | #include 7 | #include 8 | #include "rsm_protocol.h" 9 | #include "rsm_state_transfer.h" 10 | #include "rpc.h" 11 | #include 12 | #include "config.h" 13 | 14 | 15 | class rsm : public config_view_change { 16 | private: 17 | void reg1(int proc, handler *); 18 | protected: 19 | std::map procs; 20 | config *cfg; 21 | class rsm_state_transfer *stf; 22 | rpcs *rsmrpc; 23 | // On slave: expected viewstamp of next invoke request 24 | // On primary: viewstamp for the next request from rsm_client 25 | viewstamp myvs; 26 | viewstamp last_myvs; // Viewstamp of the last executed request 27 | std::string primary; 28 | bool insync; 29 | bool inviewchange; 30 | unsigned vid_commit; // Latest view id that is known to rsm layer 31 | unsigned vid_insync; // The view id that this node is synchronizing for 32 | std::vector backups; // A list of unsynchronized backups 33 | 34 | // For testing purposes 35 | rpcs *testsvr; 36 | bool partitioned; 37 | bool dopartition; 38 | bool break1; 39 | bool break2; 40 | 41 | 42 | rsm_client_protocol::status client_members(int i, 43 | std::vector &r); 44 | rsm_protocol::status invoke(int proc, viewstamp vs, std::string mreq, 45 | int &dummy); 46 | rsm_protocol::status transferreq(std::string src, viewstamp last, unsigned vid, 47 | rsm_protocol::transferres &r); 48 | rsm_protocol::status transferdonereq(std::string m, unsigned vid, int &); 49 | rsm_protocol::status joinreq(std::string src, viewstamp last, 50 | rsm_protocol::joinres &r); 51 | rsm_test_protocol::status test_net_repairreq(int heal, int &r); 52 | rsm_test_protocol::status breakpointreq(int b, int &r); 53 | 54 | pthread_mutex_t rsm_mutex; 55 | pthread_mutex_t invoke_mutex; 56 | pthread_cond_t recovery_cond; 57 | pthread_cond_t sync_cond; 58 | 59 | void execute(int procno, std::string req, std::string &r); 60 | rsm_client_protocol::status client_invoke(int procno, std::string req, 61 | std::string &r); 62 | bool statetransfer(std::string m); 63 | bool statetransferdone(std::string m); 64 | bool join(std::string m); 65 | void set_primary(unsigned vid); 66 | std::string find_highest(viewstamp &vs, std::string &m, unsigned &vid); 67 | bool sync_with_backups(); 68 | bool sync_with_primary(); 69 | void net_repair_wo(bool heal); 70 | void breakpoint1(); 71 | void breakpoint2(); 72 | void partition1(); 73 | void commit_change_wo(unsigned vid); 74 | public: 75 | rsm (std::string _first, std::string _me); 76 | ~rsm() {}; 77 | 78 | bool amiprimary(); 79 | void set_state_transfer(rsm_state_transfer *_stf) { stf = _stf; }; 80 | void recovery(); 81 | void commit_change(unsigned vid); 82 | 83 | template 84 | void reg(int proc, S*, int (S::*meth)(const A1 a1, R &)); 85 | template 86 | void reg(int proc, S*, int (S::*meth)(const A1 a1, const A2 a2, R &)); 87 | template 88 | void reg(int proc, S*, int (S::*meth)(const A1 a1, const A2 a2, 89 | const A3 a3, R &)); 90 | template 91 | void reg(int proc, S*, int (S::*meth)(const A1 a1, const A2 a2, 92 | const A3 a3, const A4 a4, R &)); 93 | template 94 | void reg(int proc, S*, int (S::*meth)(const A1 a1, const A2 a2, 95 | const A3 a3, const A4 a4, 96 | const A5 a5, R &)); 97 | }; 98 | 99 | template void 100 | rsm::reg(int proc, S*sob, int (S::*meth)(const A1 a1, R & r)) 101 | { 102 | class h1 : public handler { 103 | private: 104 | S * sob; 105 | int (S::*meth)(const A1 a1, R & r); 106 | public: 107 | h1(S *xsob, int (S::*xmeth)(const A1 a1, R & r)) 108 | : sob(xsob), meth(xmeth) { } 109 | int fn(unmarshall &args, marshall &ret) { 110 | A1 a1; 111 | R r; 112 | args >> a1; 113 | VERIFY(args.okdone()); 114 | int b = (sob->*meth)(a1,r); 115 | ret << r; 116 | return b; 117 | } 118 | }; 119 | reg1(proc, new h1(sob, meth)); 120 | } 121 | 122 | template void 123 | rsm::reg(int proc, S*sob, int (S::*meth)(const A1 a1, const A2 a2, R & r)) 124 | { 125 | class h1 : public handler { 126 | private: 127 | S * sob; 128 | int (S::*meth)(const A1 a1, const A2 a2, R & r); 129 | public: 130 | h1(S *xsob, int (S::*xmeth)(const A1 a1, const A2 a2, R & r)) 131 | : sob(xsob), meth(xmeth) { } 132 | int fn(unmarshall &args, marshall &ret) { 133 | A1 a1; 134 | A2 a2; 135 | R r; 136 | args >> a1; 137 | args >> a2; 138 | VERIFY(args.okdone()); 139 | int b = (sob->*meth)(a1,a2,r); 140 | ret << r; 141 | return b; 142 | } 143 | }; 144 | reg1(proc, new h1(sob, meth)); 145 | } 146 | 147 | template void 148 | rsm::reg(int proc, S*sob, int (S::*meth)(const A1 a1, const A2 a2, 149 | const A3 a3, R & r)) 150 | { 151 | class h1 : public handler { 152 | private: 153 | S * sob; 154 | int (S::*meth)(const A1 a1, const A2 a2, const A3 a3, R & r); 155 | public: 156 | h1(S *xsob, int (S::*xmeth)(const A1 a1, const A2 a2, const A3 a3, R & r)) 157 | : sob(xsob), meth(xmeth) { } 158 | int fn(unmarshall &args, marshall &ret) { 159 | A1 a1; 160 | A2 a2; 161 | A3 a3; 162 | R r; 163 | args >> a1; 164 | args >> a2; 165 | args >> a3; 166 | VERIFY(args.okdone()); 167 | int b = (sob->*meth)(a1,a2,a3,r); 168 | ret << r; 169 | return b; 170 | } 171 | }; 172 | reg1(proc, new h1(sob, meth)); 173 | } 174 | 175 | template void 176 | rsm::reg(int proc, S*sob, int (S::*meth)(const A1 a1, const A2 a2, 177 | const A3 a3, const A4 a4, R & r)) 178 | { 179 | class h1 : public handler { 180 | private: 181 | S * sob; 182 | int (S::*meth)(const A1 a1, const A2 a2, const A3 a3, const A4 a4, R & r); 183 | public: 184 | h1(S *xsob, int (S::*xmeth)(const A1 a1, const A2 a2, const A3 a3, 185 | const A4 a4, R & r)) 186 | : sob(xsob), meth(xmeth) { } 187 | int fn(unmarshall &args, marshall &ret) { 188 | A1 a1; 189 | A2 a2; 190 | A3 a3; 191 | A4 a4; 192 | R r; 193 | args >> a1; 194 | args >> a2; 195 | args >> a3; 196 | args >> a4; 197 | VERIFY(args.okdone()); 198 | int b = (sob->*meth)(a1,a2,a3,a4,r); 199 | ret << r; 200 | return b; 201 | } 202 | }; 203 | reg1(proc, new h1(sob, meth)); 204 | } 205 | 206 | 207 | template void 208 | rsm::reg(int proc, S*sob, int (S::*meth)(const A1 a1, const A2 a2, 209 | const A3 a3, const A4 a4, 210 | const A5 a5, R & r)) 211 | { 212 | class h1 : public handler { 213 | private: 214 | S * sob; 215 | int (S::*meth)(const A1 a1, const A2 a2, const A3 a3, const A4 a4, 216 | const A5 a5, R & r); 217 | public: 218 | h1(S *xsob, int (S::*xmeth)(const A1 a1, const A2 a2, const A3 a3, 219 | const A4 a4, const A5 a5, R & r)) 220 | : sob(xsob), meth(xmeth) { } 221 | int fn(unmarshall &args, marshall &ret) { 222 | A1 a1; 223 | A2 a2; 224 | A3 a3; 225 | A4 a4; 226 | A5 a5; 227 | R r; 228 | args >> a1; 229 | args >> a2; 230 | args >> a3; 231 | args >> a4; 232 | VERIFY(args.okdone()); 233 | int b = (sob->*meth)(a1,a2,a3,a4,a5,r); 234 | ret << r; 235 | return b; 236 | } 237 | }; 238 | reg1(proc, new h1(sob, meth)); 239 | } 240 | 241 | #endif /* rsm_h */ 242 | -------------------------------------------------------------------------------- /rsm_client.cc: -------------------------------------------------------------------------------- 1 | #include "rsm_client.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "lang/verify.h" 8 | 9 | 10 | rsm_client::rsm_client(std::string dst) 11 | { 12 | printf("create rsm_client\n"); 13 | std::vector mems; 14 | 15 | pthread_mutex_init(&rsm_client_mutex, NULL); 16 | sockaddr_in dstsock; 17 | make_sockaddr(dst.c_str(), &dstsock); 18 | primary = dst; 19 | 20 | { 21 | ScopedLock ml(&rsm_client_mutex); 22 | VERIFY (init_members()); 23 | } 24 | printf("rsm_client: done\n"); 25 | } 26 | 27 | // Assumes caller holds rsm_client_mutex 28 | void 29 | rsm_client::primary_failure() 30 | { 31 | // You fill this in for Lab 7 32 | VERIFY(!known_mems.empty()); 33 | if (!known_mems.empty()){ 34 | primary = known_mems.back(); 35 | known_mems.pop_back(); 36 | } 37 | } 38 | 39 | rsm_protocol::status 40 | rsm_client::invoke(int proc, std::string req, std::string &rep) 41 | { 42 | int ret; 43 | ScopedLock ml(&rsm_client_mutex); 44 | while (1) { 45 | printf("rsm_client::invoke proc %x primary %s\n", proc, primary.c_str()); 46 | handle h(primary); 47 | 48 | VERIFY(pthread_mutex_unlock(&rsm_client_mutex)==0); 49 | rpcc *cl = h.safebind(); 50 | if (cl) { 51 | ret = cl->call(rsm_client_protocol::invoke, proc, req, 52 | rep, rpcc::to(5000)); 53 | } 54 | VERIFY(pthread_mutex_lock(&rsm_client_mutex)==0); 55 | 56 | if (!cl) { 57 | goto prim_fail; 58 | } 59 | 60 | printf("rsm_client::invoke proc %x primary %s ret %d\n", proc, 61 | primary.c_str(), ret); 62 | if (ret == rsm_client_protocol::OK) { 63 | break; 64 | } 65 | if (ret == rsm_client_protocol::BUSY) { 66 | printf("rsm is busy %s\n", primary.c_str()); 67 | sleep(3); 68 | continue; 69 | } 70 | if (ret == rsm_client_protocol::NOTPRIMARY) { 71 | printf("primary %s isn't the primary--let's get a complete list of mems\n", 72 | primary.c_str()); 73 | if (init_members()) 74 | continue; 75 | } 76 | prim_fail: 77 | printf("primary %s failed ret %d\n", primary.c_str(), ret); 78 | primary_failure(); 79 | printf ("rsm_client::invoke: retry new primary %s\n", primary.c_str()); 80 | } 81 | return ret; 82 | } 83 | 84 | bool 85 | rsm_client::init_members() 86 | { 87 | printf("rsm_client::init_members get members!\n"); 88 | handle h(primary); 89 | std::vector new_view; 90 | VERIFY(pthread_mutex_unlock(&rsm_client_mutex)==0); 91 | int ret; 92 | rpcc *cl = h.safebind(); 93 | if (cl) { 94 | ret = cl->call(rsm_client_protocol::members, 0, new_view, 95 | rpcc::to(1000)); 96 | } 97 | VERIFY(pthread_mutex_lock(&rsm_client_mutex)==0); 98 | if (cl == 0 || ret != rsm_protocol::OK) 99 | return false; 100 | if (new_view.size() < 1) { 101 | printf("rsm_client::init_members do not know any members!\n"); 102 | VERIFY(0); 103 | } 104 | known_mems = new_view; 105 | primary = known_mems.back(); 106 | known_mems.pop_back(); 107 | 108 | printf("rsm_client::init_members: primary %s\n", primary.c_str()); 109 | 110 | return true; 111 | } 112 | 113 | -------------------------------------------------------------------------------- /rsm_client.h: -------------------------------------------------------------------------------- 1 | #ifndef rsm_client_h 2 | #define rsm_client_h 3 | 4 | #include "rpc.h" 5 | #include "rsm_protocol.h" 6 | #include 7 | #include 8 | 9 | 10 | // 11 | // rsm client interface. 12 | // 13 | // The client stubs package up an rpc, and then call the invoke procedure 14 | // on the replicated state machine passing the RPC as an argument. This way 15 | // the replicated state machine isn't service specific; any server can use it. 16 | // 17 | 18 | class rsm_client { 19 | 20 | protected: 21 | std::string primary; 22 | std::vector known_mems; 23 | pthread_mutex_t rsm_client_mutex; 24 | void primary_failure(); 25 | bool init_members(); 26 | public: 27 | rsm_client(std::string dst); 28 | rsm_protocol::status invoke(int proc, std::string req, std::string &rep); 29 | 30 | template 31 | int call(unsigned int proc, const A1 & a1, R &r); 32 | 33 | template 34 | int call(unsigned int proc, const A1 & a1, const A2 & a2, R &r); 35 | 36 | template 37 | int call(unsigned int proc, const A1 & a1, const A2 & a2, const A3 & a3, 38 | R &r); 39 | 40 | template 41 | int call(unsigned int proc, const A1 & a1, const A2 & a2, const A3 & a3, 42 | const A4 & a4, R &r); 43 | 44 | template 45 | int call(unsigned int proc, const A1 & a1, const A2 & a2, const A3 & a3, 46 | const A4 & a4, const A5 & a5, R &r); 47 | private: 48 | template int call_m(unsigned int proc, marshall &req, R &r); 49 | }; 50 | 51 | template int 52 | rsm_client::call_m(unsigned int proc, marshall &req, R &r) 53 | { 54 | std::string rep; 55 | std::string res; 56 | int intret = invoke(proc, req.str(), rep); 57 | VERIFY( intret == rsm_client_protocol::OK ); 58 | unmarshall u(rep); 59 | u >> intret; 60 | if (intret < 0) return intret; 61 | u >> res; 62 | if (!u.okdone()) { 63 | fprintf(stderr, "rsm_client::call_m: failed to unmarshall the reply.\n" 64 | "You probably forgot to set the reply string in " 65 | "rsm::client_invoke, or you may call RPC 0x%x with wrong return " 66 | "type\n", proc); 67 | VERIFY(0); 68 | return rpc_const::unmarshal_reply_failure; 69 | } 70 | unmarshall u1(res); 71 | u1 >> r; 72 | if(!u1.okdone()) { 73 | fprintf(stderr, "rsm_client::call_m: failed to unmarshall the reply.\n" 74 | "You are probably calling RPC 0x%x with wrong return " 75 | "type.\n", proc); 76 | VERIFY(0); 77 | return rpc_const::unmarshal_reply_failure; 78 | } 79 | return intret; 80 | } 81 | 82 | template int 83 | rsm_client::call(unsigned int proc, const A1 & a1, R & r) 84 | { 85 | marshall m; 86 | m << a1; 87 | return call_m(proc, m, r); 88 | } 89 | 90 | template int 91 | rsm_client::call(unsigned int proc, const A1 & a1, const A2 & a2, R & r) 92 | { 93 | marshall m; 94 | m << a1; 95 | m << a2; 96 | return call_m(proc, m, r); 97 | } 98 | 99 | template int 100 | rsm_client::call(unsigned int proc, const A1 & a1, 101 | const A2 & a2, const A3 & a3, R & r) 102 | { 103 | marshall m; 104 | std::string rep; 105 | std::string res; 106 | m << a1; 107 | m << a2; 108 | m << a3; 109 | return call_m(proc, m, r); 110 | } 111 | 112 | template int 113 | rsm_client::call(unsigned int proc, const A1 & a1, 114 | const A2 & a2, const A3 & a3, const A4 & a4, R & r) 115 | { 116 | marshall m; 117 | std::string rep; 118 | std::string res; 119 | m << a1; 120 | m << a2; 121 | m << a3; 122 | m << a4; 123 | return call_m(proc, m, r); 124 | } 125 | 126 | template int 127 | rsm_client::call(unsigned int proc, const A1 & a1, 128 | const A2 & a2, const A3 & a3, const A4 & a4, const A5 & a5, 129 | R & r) 130 | { 131 | marshall m; 132 | std::string rep; 133 | std::string res; 134 | m << a1; 135 | m << a2; 136 | m << a3; 137 | m << a4; 138 | m << a5; 139 | return call_m(proc, m, r); 140 | } 141 | 142 | #endif 143 | -------------------------------------------------------------------------------- /rsm_protocol.h: -------------------------------------------------------------------------------- 1 | #ifndef rsm_protocol_h 2 | #define rsm_protocol_h 3 | 4 | #include "rpc.h" 5 | 6 | 7 | class rsm_client_protocol { 8 | public: 9 | enum xxstatus { OK, ERR, NOTPRIMARY, BUSY}; 10 | typedef int status; 11 | enum rpc_numbers { 12 | invoke = 0x9001, 13 | members, 14 | }; 15 | }; 16 | 17 | 18 | struct viewstamp { 19 | viewstamp (unsigned int _vid = 0, unsigned int _seqno = 0) { 20 | vid = _vid; 21 | seqno = _seqno; 22 | }; 23 | unsigned int vid; 24 | unsigned int seqno; 25 | }; 26 | 27 | class rsm_protocol { 28 | public: 29 | enum xxstatus { OK, ERR, BUSY}; 30 | typedef int status; 31 | enum rpc_numbers { 32 | invoke = 0x10001, 33 | transferreq, 34 | transferdonereq, 35 | joinreq, 36 | }; 37 | 38 | struct transferres { 39 | std::string state; 40 | viewstamp last; 41 | }; 42 | 43 | struct joinres { 44 | std::string log; 45 | }; 46 | }; 47 | 48 | inline bool operator==(viewstamp a, viewstamp b) { 49 | return a.vid == b.vid && a.seqno == b.seqno; 50 | } 51 | 52 | inline bool operator>(viewstamp a, viewstamp b) { 53 | return (a.vid > b.vid) || ((a.vid == b.vid) && a.seqno > b.seqno); 54 | } 55 | 56 | inline bool operator!=(viewstamp a, viewstamp b) { 57 | return a.vid != b.vid || a.seqno != b.seqno; 58 | } 59 | 60 | inline marshall& operator<<(marshall &m, viewstamp v) 61 | { 62 | m << v.vid; 63 | m << v.seqno; 64 | return m; 65 | } 66 | 67 | inline unmarshall& operator>>(unmarshall &u, viewstamp &v) { 68 | u >> v.vid; 69 | u >> v.seqno; 70 | return u; 71 | } 72 | 73 | inline marshall & 74 | operator<<(marshall &m, rsm_protocol::transferres r) 75 | { 76 | m << r.state; 77 | m << r.last; 78 | return m; 79 | } 80 | 81 | inline unmarshall & 82 | operator>>(unmarshall &u, rsm_protocol::transferres &r) 83 | { 84 | u >> r.state; 85 | u >> r.last; 86 | return u; 87 | } 88 | 89 | inline marshall & 90 | operator<<(marshall &m, rsm_protocol::joinres r) 91 | { 92 | m << r.log; 93 | return m; 94 | } 95 | 96 | inline unmarshall & 97 | operator>>(unmarshall &u, rsm_protocol::joinres &r) 98 | { 99 | u >> r.log; 100 | return u; 101 | } 102 | 103 | class rsm_test_protocol { 104 | public: 105 | enum xxstatus { OK, ERR}; 106 | typedef int status; 107 | enum rpc_numbers { 108 | net_repair = 0x12001, 109 | breakpoint = 0x12002, 110 | }; 111 | }; 112 | 113 | #endif 114 | -------------------------------------------------------------------------------- /rsm_state_transfer.h: -------------------------------------------------------------------------------- 1 | #ifndef rsm_state_transfer_h 2 | #define rsm_state_transfer_h 3 | 4 | class rsm_state_transfer { 5 | public: 6 | virtual std::string marshal_state() = 0; 7 | virtual void unmarshal_state(std::string) = 0; 8 | virtual ~rsm_state_transfer() {}; 9 | }; 10 | 11 | #endif 12 | -------------------------------------------------------------------------------- /rsm_tester.cc: -------------------------------------------------------------------------------- 1 | // 2 | // RSM test client 3 | // 4 | 5 | #include "rsm_protocol.h" 6 | #include "rsmtest_client.h" 7 | #include "rpc.h" 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | using namespace std; 14 | 15 | rsmtest_client *lc; 16 | 17 | int 18 | main(int argc, char *argv[]) 19 | { 20 | int r; 21 | 22 | if(argc != 4){ 23 | fprintf(stderr, "Usage: %s [host:]port [partition] arg\n", argv[0]); 24 | exit(1); 25 | } 26 | 27 | lc = new rsmtest_client(argv[1]); 28 | string command(argv[2]); 29 | if (command == "partition") { 30 | r = lc->net_repair(atoi(argv[3])); 31 | printf ("net_repair returned %d\n", r); 32 | } else if (command == "breakpoint") { 33 | int b = atoi(argv[3]); 34 | r = lc->breakpoint(b); 35 | printf ("breakpoint %d returned %d\n", b, r); 36 | } else { 37 | fprintf(stderr, "Unknown command %s\n", argv[2]); 38 | } 39 | exit(0); 40 | } 41 | -------------------------------------------------------------------------------- /rsmtest_client.cc: -------------------------------------------------------------------------------- 1 | // RPC stubs for clients to talk to rsmtest_server 2 | 3 | #include "rsmtest_client.h" 4 | #include "rpc.h" 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | rsmtest_client::rsmtest_client(std::string dst) 12 | { 13 | sockaddr_in dstsock; 14 | make_sockaddr(dst.c_str(), &dstsock); 15 | cl = new rpcc(dstsock); 16 | if (cl->bind() < 0) { 17 | printf("rsmtest_client: call bind\n"); 18 | } 19 | } 20 | 21 | int 22 | rsmtest_client::net_repair(int heal) 23 | { 24 | int r; 25 | int ret = cl->call(rsm_test_protocol::net_repair, heal, r); 26 | VERIFY (ret == rsm_test_protocol::OK); 27 | return r; 28 | } 29 | 30 | int 31 | rsmtest_client::breakpoint(int b) 32 | { 33 | int r; 34 | int ret = cl->call(rsm_test_protocol::breakpoint, b, r); 35 | VERIFY (ret == rsm_test_protocol::OK); 36 | return r; 37 | } 38 | 39 | 40 | -------------------------------------------------------------------------------- /rsmtest_client.h: -------------------------------------------------------------------------------- 1 | // rsmtest client interface. 2 | 3 | #ifndef rsmtest_client_h 4 | #define rsmtest_client_h 5 | 6 | #include 7 | #include "rsm_protocol.h" 8 | #include "rpc.h" 9 | 10 | // Client interface to the rsmtest server 11 | class rsmtest_client { 12 | protected: 13 | rpcc *cl; 14 | public: 15 | rsmtest_client(std::string d); 16 | virtual ~rsmtest_client() {}; 17 | virtual rsm_test_protocol::status net_repair(int heal); 18 | virtual rsm_test_protocol::status breakpoint(int b); 19 | }; 20 | #endif 21 | -------------------------------------------------------------------------------- /start.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ulimit -c unlimited 4 | 5 | LOSSY=$1 6 | NUM_LS=$2 7 | 8 | if [ -z $NUM_LS ]; then 9 | NUM_LS=0 10 | fi 11 | 12 | BASE_PORT=$RANDOM 13 | BASE_PORT=$[BASE_PORT+2000] 14 | EXTENT_PORT=$BASE_PORT 15 | YFS1_PORT=$[BASE_PORT+2] 16 | YFS2_PORT=$[BASE_PORT+4] 17 | LOCK_PORT=$[BASE_PORT+6] 18 | 19 | YFSDIR1=$PWD/yfs1 20 | YFSDIR2=$PWD/yfs2 21 | 22 | if [ "$LOSSY" ]; then 23 | export RPC_LOSSY=$LOSSY 24 | fi 25 | 26 | if [ $NUM_LS -gt 1 ]; then 27 | x=0 28 | rm config 29 | while [ $x -lt $NUM_LS ]; do 30 | port=$[LOCK_PORT+2*x] 31 | x=$[x+1] 32 | echo $port >> config 33 | done 34 | x=0 35 | while [ $x -lt $NUM_LS ]; do 36 | port=$[LOCK_PORT+2*x] 37 | x=$[x+1] 38 | echo "starting ./lock_server $LOCK_PORT $port > /tmp/lock_server$x.log 2>&1 &" 39 | ./lock_server $LOCK_PORT $port > /tmp/lock_server$x.log 2>&1 & 40 | sleep 1 41 | done 42 | else 43 | echo "starting ./lock_server $LOCK_PORT > /tmp/lock_server.log 2>&1 &" 44 | ./lock_server $LOCK_PORT > /tmp/lock_server.log 2>&1 & 45 | sleep 1 46 | fi 47 | 48 | unset RPC_LOSSY 49 | 50 | echo "starting ./extent_server $EXTENT_PORT > /tmp/extent_server.log 2>&1 &" 51 | ./extent_server $EXTENT_PORT > /tmp/extent_server.log 2>&1 & 52 | sleep 1 53 | 54 | rm -rf $YFSDIR1 55 | mkdir $YFSDIR1 || exit 1 56 | sleep 1 57 | echo "starting ./yfs_client $YFSDIR1 $EXTENT_PORT $LOCK_PORT > yfs_client1.log 2>&1 &" 58 | ./yfs_client $YFSDIR1 $EXTENT_PORT $LOCK_PORT > /tmp/yfs_client1.log 2>&1 & 59 | sleep 1 60 | 61 | rm -rf $YFSDIR2 62 | mkdir $YFSDIR2 || exit 1 63 | sleep 1 64 | echo "starting ./yfs_client $YFSDIR2 $EXTENT_PORT $LOCK_PORT > yfs_client2.log 2>&1 &" 65 | ./yfs_client $YFSDIR2 $EXTENT_PORT $LOCK_PORT > /tmp/yfs_client2.log 2>&1 & 66 | 67 | sleep 2 68 | 69 | # make sure FUSE is mounted where we expect 70 | pwd=`pwd -P` 71 | if [ `mount | grep "$pwd/yfs1" | grep -v grep | wc -l` -ne 1 ]; then 72 | sh stop.sh 73 | echo "Failed to mount YFS properly at ./yfs1" 74 | exit -1 75 | fi 76 | 77 | # make sure FUSE is mounted where we expect 78 | if [ `mount | grep "$pwd/yfs2" | grep -v grep | wc -l` -ne 1 ]; then 79 | sh stop.sh 80 | echo "Failed to mount YFS properly at ./yfs2" 81 | exit -1 82 | fi 83 | -------------------------------------------------------------------------------- /stop.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | YFSDIR1=$PWD/yfs1 4 | YFSDIR2=$PWD/yfs2 5 | 6 | export PATH=$PATH:/usr/local/bin 7 | UMOUNT="umount" 8 | if [ -f "/usr/local/bin/fusermount" -o -f "/usr/bin/fusermount" -o -f "/bin/fusermount" ]; then 9 | UMOUNT="fusermount -u"; 10 | fi 11 | $UMOUNT $YFSDIR1 12 | $UMOUNT $YFSDIR2 13 | killall extent_server 14 | killall yfs_client 15 | killall lock_server 16 | -------------------------------------------------------------------------------- /test-lab-2-a.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | # test CREATE/MKNOD, LOOKUP, READDIR. 4 | 5 | use strict; 6 | 7 | if($#ARGV != 0){ 8 | print STDERR "Usage: test-lab-2-a.pl directory\n"; 9 | exit(1); 10 | } 11 | my $dir = $ARGV[0]; 12 | 13 | my $seq = 0; 14 | 15 | my $files = { }; 16 | my @dead; 17 | 18 | for(my $iters = 0; $iters < 200; $iters++){ 19 | createone(); 20 | } 21 | 22 | for(my $iters = 0; $iters < 100; $iters++){ 23 | if(rand() < 0.1){ 24 | livecheck(); 25 | } 26 | if(rand() < 0.1){ 27 | deadcheck(); 28 | } 29 | if(rand() < 0.02){ 30 | dircheck(); 31 | } 32 | if(rand() < 0.5){ 33 | createone(); 34 | } 35 | } 36 | 37 | dircheck(); 38 | printf "Passed all tests!\n"; 39 | exit(0); 40 | 41 | sub createone { 42 | my $name = "file-"; 43 | for(my $i = 0; $i < 40; $i++){ 44 | $name .= sprintf("%c", ord('a') + int(rand(26))); 45 | } 46 | $name .= "-$$-" . $seq; 47 | $seq = $seq + 1; 48 | my $contents = rand(); 49 | print "create $name\n"; 50 | if(!open(F, ">$dir/$name")){ 51 | print STDERR "test-lab-2-a: cannot create $dir/$name : $!\n"; 52 | exit(1); 53 | } 54 | close(F); 55 | $files->{$name} = $contents; 56 | } 57 | 58 | sub createagain { 59 | my @a = keys(%$files); 60 | return if $#a < 0; 61 | my $i = int(rand($#a + 1)); 62 | my $k = $a[$i]; 63 | print "re-create $k\n"; 64 | if(!open(F, ">$dir/$k")){ 65 | print STDERR "test-lab-2-a: cannot re-create $dir/$k : $!\n"; 66 | exit(1); 67 | } 68 | close(F); 69 | } 70 | 71 | # make sure all the live files are there, 72 | # and that all the dead files are not there. 73 | sub dircheck { 74 | print "dircheck\n"; 75 | opendir(D, $dir); 76 | my %h; 77 | my $f; 78 | while(defined($f = readdir(D))){ 79 | if(!defined($h{$f})){ 80 | $h{$f} = 0; 81 | } 82 | $h{$f} = $h{$f} + 1; 83 | } 84 | closedir(D); 85 | 86 | foreach $f (keys(%$files)){ 87 | if(!defined($h{$f})){ 88 | print STDERR "test-lab-2-a.pl: $f is not in the directory listing\n"; 89 | exit(1); 90 | } 91 | if($h{$f} > 1){ 92 | print STDERR "test-lab-2-a.pl: $f appears more than once in the directory\n"; 93 | exit(1); 94 | } 95 | } 96 | 97 | foreach $f (@dead){ 98 | if(defined($h{$f})){ 99 | print STDERR "test-lab-2-a.pl: $f is dead but in directory listing\n"; 100 | exit(1); 101 | } 102 | } 103 | } 104 | 105 | sub livecheck { 106 | my @a = keys(%$files); 107 | return if $#a < 0; 108 | my $i = int(rand($#a + 1)); 109 | my $k = $a[$i]; 110 | print "livecheck $k\n"; 111 | if(!open(F, "$dir/$k")){ 112 | print STDERR "test-lab-2-a: cannot open $dir/$k : $!\n"; 113 | exit(1); 114 | } 115 | close(F); 116 | if( ! -f "$dir/$k" ){ 117 | print STDERR "test-lab-2-a: $dir/$k is not of type file\n"; 118 | exit(1); 119 | } 120 | if(open(F, ">$dir/$k/xx")){ 121 | print STDERR "test-lab-2-a: $dir/$k acts like a directory, not a file\n"; 122 | exit(1); 123 | } 124 | } 125 | 126 | sub deadcheck { 127 | my $name = "file-$$-" . $seq; 128 | $seq = $seq + 1; 129 | print "check-not-there $name\n"; 130 | if(open(F, "$dir/$name")){ 131 | print STDERR "test-lab-2-a: $dir/$name exists but should not\n"; 132 | exit(1); 133 | } 134 | } 135 | 136 | sub deleteone { 137 | my @a = keys(%$files); 138 | return 0 if $#a < 0; 139 | my $i = int(rand($#a + 1)); 140 | my $k = $a[$i]; 141 | print "delete $k\n"; 142 | if(unlink($dir . "/" . $k) == 0){ 143 | print STDERR "test-lab-2-a: unlink $k failed: $!\n"; 144 | exit(1); 145 | } 146 | delete $files->{$k}; 147 | push(@dead, $k); 148 | return 1; 149 | } 150 | -------------------------------------------------------------------------------- /test-lab-2-b.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | # Case 1: Single file system 4 | # 1. create new files 5 | # 2. open an existing file 6 | # 3. write to a file 7 | # - write to the middle 8 | # - append 9 | # - write from middle and beyond file length 10 | # - seek beyond file length and write 11 | # 4. read file content 12 | # from each case above 13 | # 14 | # Make sure file handles/file type for new files are correct. 15 | 16 | # Case 2: Two file systems mounted under same root dir. 17 | # 0. Start two fs with same rootdir 18 | # 1. create files in dir1 19 | # 2. read the files from dir2 20 | # 21 | # 22 | 23 | use strict; 24 | $| = 1; 25 | 26 | if($#ARGV != 1){ 27 | print STDERR "Usage: test-lab-2-b.pl directory1 directory2\n"; 28 | exit(1); 29 | } 30 | my $dir1 = $ARGV[0]; 31 | my $dir2 = $ARGV[1]; 32 | 33 | my $f1 = "a$$"; 34 | my $f2 = "b$$"; 35 | 36 | my $files = { }; 37 | 38 | print "Write and read one file: "; 39 | writeone($dir1, $f1, 600); 40 | checkcontent($dir1, $f1); 41 | print "OK\n"; 42 | 43 | print "Write and read a second file: "; 44 | writeone($dir1, $f2, 4111); 45 | checkcontent($dir1, $f2); 46 | checkcontent($dir1, $f1); 47 | print "OK\n"; 48 | 49 | print "Overwrite an existing file: "; 50 | writeone($dir1, $f1, 275); # shorter than before... 51 | checkcontent($dir1, $f1); 52 | checkcontent($dir1, $f2); 53 | print "OK\n"; 54 | 55 | print "Append to an existing file: "; 56 | writeone($dir1, $f1, 8192); 57 | append($dir1, $f1, 7007); 58 | checkcontent($dir1, $f1); 59 | print "OK\n"; 60 | 61 | print "Write into the middle of an existing file: "; 62 | writeat($dir1, $f1, 190); 63 | checkcontent($dir1, $f1); 64 | print "OK\n"; 65 | 66 | print "Write beyond the end of an existing file: "; 67 | writeat($dir1, $f1, 65536); 68 | checkcontent($dir1, $f1); 69 | print "OK\n"; 70 | 71 | print "Check that one cannot open non-existant file: "; 72 | checknot($dir1, "z-$$-z"); 73 | print "OK\n"; 74 | 75 | print "Check directory listing: "; 76 | dircheck($dir1); 77 | print "OK\n"; 78 | 79 | print "Read files via second server: "; 80 | checkcontent($dir2, $f1); 81 | checkcontent($dir2, $f2); 82 | print "OK\n"; 83 | 84 | print "Check directory listing on second server: "; 85 | dircheck($dir2); 86 | print "OK\n"; 87 | 88 | print "Passed all tests\n"; 89 | 90 | sub writeone { 91 | my($d, $name, $len) = @_; 92 | my $contents = ""; 93 | 94 | my $f = $d . "/" . $name; 95 | 96 | use FileHandle; 97 | sysopen F, $f, O_TRUNC|O_RDWR|O_CREAT 98 | or die "cannot create $f\n"; 99 | 100 | while(length($contents) < $len){ 101 | $contents .= rand(); 102 | } 103 | $contents = substr($contents, 0, $len); 104 | $files->{$name} = $contents; 105 | 106 | syswrite F, $files->{$name}, length($files->{$name}) 107 | or die "cannot write to $f"; 108 | close(F); 109 | } 110 | 111 | sub checkcontent { 112 | my($d, $name) = @_; 113 | 114 | my $f = $d . "/" . $name; 115 | 116 | open F, "$f" or die "could not open $f for reading"; 117 | my $c2 = ""; 118 | while() { 119 | $c2 .= $_; 120 | } 121 | close(F); 122 | $files->{$name} eq $c2 or die "content of $f is incorrect\n"; 123 | } 124 | 125 | sub checknot { 126 | my($d, $name) = @_; 127 | 128 | my $f = $d . "/" . $name; 129 | 130 | my $x = open(F, $f); 131 | if(defined($x)){ 132 | print STDERR "$x exists but should not\n"; 133 | exit(1); 134 | } 135 | } 136 | 137 | sub append { 138 | my($d, $name, $n) = @_; 139 | 140 | my $f = $d . "/" . $name; 141 | 142 | use FileHandle; 143 | sysopen F, "$f", O_RDWR 144 | or die "cannot open $f for append\n"; 145 | 146 | my $contents = ""; 147 | while(length($contents) < $n){ 148 | $contents .= rand(); 149 | } 150 | $contents = substr($contents, 0, $n); 151 | $files->{$name} .= $contents; ## Append the file content 152 | 153 | seek(F, 0, 2); ## goto end of file 154 | syswrite(F, $contents, length($contents), 0) or die "cannot append to $f"; 155 | close(F); 156 | } 157 | 158 | sub writeat { 159 | my($d, $name, $off) = @_; 160 | 161 | my $f = $d . "/" . $name; 162 | 163 | use FileHandle; 164 | sysopen F, "$f", O_RDWR 165 | or die "cannot open $f for read/write\n"; 166 | 167 | my $contents = rand(); 168 | 169 | my $x = $files->{$name}; 170 | if (length($x) < $off + length($contents)) { 171 | my $nappend = $off + length($contents) - length($x); 172 | for (my $i=0; $i < $nappend; $i++) { 173 | $x .= "\0"; 174 | } 175 | } 176 | substr($x, $off, length($contents)) = $contents; 177 | $files->{$name} = $x; 178 | 179 | seek(F, $off, 0); 180 | syswrite(F, $contents, length($contents), 0) 181 | or die "cannot write $f at offset $off"; 182 | close(F); 183 | } 184 | 185 | sub dircheck { 186 | my($dir) = @_; 187 | 188 | opendir(D, $dir); 189 | my %h; 190 | my $f; 191 | while(defined($f = readdir(D))){ 192 | if(defined($h{$f})){ 193 | print STDERR "$f appears more than once in directory $dir\n"; 194 | exit(1); 195 | } 196 | $h{$f} = 1; 197 | } 198 | closedir(D); 199 | 200 | foreach $f (keys(%$files)){ 201 | if(!defined($h{$f})){ 202 | print STDERR "$f is missing from directory $dir\n"; 203 | exit(1); 204 | } 205 | } 206 | } 207 | 208 | exit(0); 209 | 210 | -------------------------------------------------------------------------------- /test-lab-3-a.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl -w 2 | 3 | sub oops { 4 | my($msg) = @_; 5 | print STDERR "test-lab-3-a.pl error: $msg : $!\n"; 6 | exit(1); 7 | } 8 | 9 | sub oops1 { 10 | my($msg) = @_; 11 | print STDERR "test-lab-3-a.pl error: $msg\n"; 12 | exit(1); 13 | } 14 | 15 | if($#ARGV != 0){ 16 | print STDERR "Usage: test-lab-3-a.pl directory\n"; 17 | exit(1); 18 | } 19 | 20 | my $seq = 0; 21 | my $root = $ARGV[0]; 22 | my $dir = $root . "/d" . $$; 23 | print "mkdir $dir\n"; 24 | if(mkdir($dir, 0777) == 0){ 25 | oops("mkdir $dir"); 26 | } 27 | 28 | my $files = { }; 29 | my @dead; 30 | 31 | createone(); 32 | deleteone(); 33 | createone(); 34 | checkmtime(); 35 | checkdirmtime(); 36 | 37 | for($iters = 0; $iters < 10; $iters++){ 38 | createone(); 39 | } 40 | 41 | for($iters = 0; $iters < 50; $iters++){ 42 | if(rand() < 0.2){ 43 | deadcheck(); 44 | } 45 | if(rand() < 0.2){ 46 | livecheck(); 47 | } 48 | if(rand() < 0.02){ 49 | dircheck(); 50 | } 51 | if(rand() < 0.1){ 52 | checkdirmtime(); 53 | } 54 | if(rand() < 0.1){ 55 | checkmtime(); 56 | } 57 | if(rand() < 0.5){ 58 | createone(); 59 | } 60 | if(rand() < 0.5){ 61 | deleteone(); 62 | } 63 | } 64 | 65 | dircheck(); 66 | cleanup(); 67 | dircheck(); 68 | printf "Passed all tests!\n"; 69 | exit(0); 70 | 71 | sub createone { 72 | my $name = "x-" . $seq; 73 | $seq = $seq + 1; 74 | my $contents = rand(); 75 | print "create $name\n"; 76 | if(!open(F, ">$dir/$name")){ 77 | oops("cannot create $name"); 78 | } 79 | print F "$contents"; 80 | close(F); 81 | $files->{$name} = $contents; 82 | } 83 | 84 | # make sure all the live files are there, 85 | # and that all the dead files are not there. 86 | sub dircheck { 87 | print "dircheck\n"; 88 | opendir(D, $dir); 89 | my %h; 90 | my $f; 91 | while(defined($f = readdir(D))){ 92 | if(defined($h{$f})){ 93 | oops1("$f occurs twice in directory"); 94 | } 95 | $h{$f} = 1; 96 | } 97 | closedir(D); 98 | 99 | foreach $f (keys(%$files)){ 100 | if(!defined($h{$f})){ 101 | oops1("$f is not in directory listing"); 102 | } 103 | } 104 | 105 | foreach $f (@dead){ 106 | if(defined($h{$f})){ 107 | oops1("$f was removed but is in directory listing"); 108 | } 109 | } 110 | 111 | foreach $f (keys(%h)){ 112 | next if ($f eq "." or $f eq ".."); 113 | if(!defined($files->{$f})){ 114 | oops1("unexpected file $f in directory listing"); 115 | } 116 | } 117 | } 118 | 119 | sub livecheck { 120 | my @a = keys(%$files); 121 | return if $#a < 0; 122 | my $i = int(rand($#a + 1)); 123 | my $k = $a[$i]; 124 | print "livecheck $k\n"; 125 | oops("cannot open $k") if !open(F, "$dir/$k"); 126 | my $z = ; 127 | if($z ne $files->{$k}){ 128 | oops1("file $k wrong contents"); 129 | } 130 | close(F); 131 | } 132 | 133 | sub deadcheck { 134 | return if $#dead < 0; 135 | my $i = int(rand($#dead + 1)); 136 | my $k = $dead[$i]; 137 | return if defined($files->{$k}); # ??? 138 | print "deadcheck $k\n"; 139 | if(rand(1.0) < 0.5){ 140 | if(open(F, $dir . "/" . $k)){ 141 | oops1("dead file $k is readable"); 142 | } 143 | } else { 144 | if(unlink($dir . "/" . $k)){ 145 | oops1("dead file $k was removable"); 146 | } 147 | } 148 | } 149 | 150 | sub deleteone { 151 | my @a = keys(%$files); 152 | return 0 if $#a < 0; 153 | my $i = int(rand($#a + 1)); 154 | my $k = $a[$i]; 155 | print "delete $k\n"; 156 | if(unlink($dir . "/" . $k) == 0){ 157 | oops("unlink $k failed"); 158 | } 159 | delete $files->{$k}; 160 | push(@dead, $k); 161 | return 1; 162 | } 163 | 164 | sub checkdirmtime { 165 | print "checkdirmtime\n"; 166 | opendir(D, $dir); 167 | closedir(D); 168 | my @st1 = stat($dir . "/."); 169 | sleep(2); 170 | my $op; 171 | if(rand() < 0.75){ 172 | return if deleteone() == 0; 173 | $op = "delete"; 174 | } else { 175 | createone(); 176 | $op = "create"; 177 | } 178 | opendir(D, $dir); 179 | closedir(D); 180 | my @st2 = stat($dir . "/."); 181 | if($st1[9] == $st2[9]){ 182 | print $st2[9], " ", $st2[9], "\n"; 183 | oops1("$op did not change directory mtime"); 184 | } 185 | if($st1[10] == $st2[10]){ 186 | oops1("$op did not change directory ctime"); 187 | } 188 | } 189 | 190 | sub checkmtime { 191 | my @a = keys(%$files); 192 | return if $#a < 0; 193 | my $i = int(rand($#a + 1)); 194 | my $k = $a[$i]; 195 | print "checkmtime $k\n"; 196 | 197 | my @st1 = stat("$dir/$k"); 198 | sleep(2); 199 | if(!open(F, ">$dir/$k")){ 200 | oops("cannot re-create $dir/$k"); 201 | } 202 | my @st2 = stat("$dir/$k"); 203 | sleep(2); 204 | print F $files->{$k}; 205 | close(F); 206 | if(!open(F, "$dir/$k")){ 207 | oops("cannot open $dir/$k"); 208 | } 209 | close(F); 210 | my @st3 = stat("$dir/$k"); 211 | 212 | if($st1[9] == $st2[9]){ 213 | oops1("CREATE did not change mtime"); 214 | } 215 | if($st2[9] == $st3[9]){ 216 | oops1("WRITE did not change mtime"); 217 | } 218 | } 219 | 220 | sub cleanup { 221 | while(deleteone()){ 222 | } 223 | } 224 | -------------------------------------------------------------------------------- /test-lab-3-c.c: -------------------------------------------------------------------------------- 1 | /* 2 | * test-lab-5 dir1 dir2 3 | * 4 | * Creates and deletes files in different directories 5 | * on the same underlying file system. These operations 6 | * should not require much put/get or lock traffic in 7 | * a yfs with a write-back cache and lazy lock release. 8 | */ 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | char d1[512], d2[512]; 23 | extern int errno; 24 | 25 | void 26 | create1(const char *d, const char *f, const char *in) 27 | { 28 | int fd; 29 | char n[512]; 30 | 31 | /* 32 | * The FreeBSD NFS client only invalidates its caches 33 | * cache if the mtime changes by a whole second. 34 | */ 35 | sleep(1); 36 | 37 | sprintf(n, "%s/%s", d, f); 38 | fd = creat(n, 0666); 39 | if(fd < 0){ 40 | fprintf(stderr, "test-lab-3-c: create(%s): %s\n", 41 | n, strerror(errno)); 42 | exit(1); 43 | } 44 | if(write(fd, in, strlen(in)) != strlen(in)){ 45 | fprintf(stderr, "test-lab-3-c: write(%s): %s\n", 46 | n, strerror(errno)); 47 | exit(1); 48 | } 49 | if(close(fd) != 0){ 50 | fprintf(stderr, "test-lab-3-c: close(%s): %s\n", 51 | n, strerror(errno)); 52 | exit(1); 53 | } 54 | } 55 | 56 | void 57 | check1(const char *d, const char *f, const char *in) 58 | { 59 | int fd, cc; 60 | char n[512], buf[512]; 61 | 62 | sprintf(n, "%s/%s", d, f); 63 | fd = open(n, 0); 64 | if(fd < 0){ 65 | fprintf(stderr, "test-lab-3-c: open(%s): %s\n", 66 | n, strerror(errno)); 67 | exit(1); 68 | } 69 | errno = 0; 70 | cc = read(fd, buf, sizeof(buf) - 1); 71 | if(cc != strlen(in)){ 72 | fprintf(stderr, "test-lab-3-c: read(%s) returned too little %d%s%s\n", 73 | n, 74 | cc, 75 | errno ? ": " : "", 76 | errno ? strerror(errno) : ""); 77 | exit(1); 78 | } 79 | close(fd); 80 | buf[cc] = '\0'; 81 | if(strncmp(buf, in, strlen(n)) != 0){ 82 | fprintf(stderr, "test-lab-3-c: read(%s) got \"%s\", not \"%s\"\n", 83 | n, buf, in); 84 | exit(1); 85 | } 86 | } 87 | 88 | void 89 | unlink1(const char *d, const char *f) 90 | { 91 | char n[512]; 92 | 93 | sleep(1); 94 | 95 | sprintf(n, "%s/%s", d, f); 96 | if(unlink(n) != 0){ 97 | fprintf(stderr, "test-lab-3-c: unlink(%s): %s\n", 98 | n, strerror(errno)); 99 | exit(1); 100 | } 101 | } 102 | 103 | void 104 | checknot(const char *d, const char *f) 105 | { 106 | int fd; 107 | char n[512]; 108 | 109 | sprintf(n, "%s/%s", d, f); 110 | fd = open(n, 0); 111 | if(fd >= 0){ 112 | fprintf(stderr, "test-lab-3-c: open(%s) succeeded for deleted file\n", n); 113 | exit(1); 114 | } 115 | } 116 | 117 | void 118 | append1(const char *d, const char *f, const char *in) 119 | { 120 | int fd; 121 | char n[512]; 122 | 123 | sleep(1); 124 | 125 | sprintf(n, "%s/%s", d, f); 126 | fd = open(n, O_WRONLY|O_APPEND); 127 | if(fd < 0){ 128 | fprintf(stderr, "test-lab-3-c: append open(%s): %s\n", 129 | n, strerror(errno)); 130 | exit(1); 131 | } 132 | if(write(fd, in, strlen(in)) != strlen(in)){ 133 | fprintf(stderr, "test-lab-3-c: append write(%s): %s\n", 134 | n, strerror(errno)); 135 | exit(1); 136 | } 137 | if(close(fd) != 0){ 138 | fprintf(stderr, "test-lab-3-c: append close(%s): %s\n", 139 | n, strerror(errno)); 140 | exit(1); 141 | } 142 | } 143 | 144 | void 145 | createn(const char *d, const char *prefix, int nf) 146 | { 147 | int fd, i; 148 | char n[512]; 149 | 150 | /* 151 | * The FreeBSD NFS client only invalidates its caches 152 | * cache if the mtime changes by a whole second. 153 | */ 154 | sleep(1); 155 | 156 | for(i = 0; i < nf; i++){ 157 | sprintf(n, "%s/%s-%d", d, prefix, i); 158 | fd = creat(n, 0666); 159 | if(fd < 0){ 160 | fprintf(stderr, "test-lab-3-c: create(%s): %s\n", 161 | n, strerror(errno)); 162 | exit(1); 163 | } 164 | if(write(fd, &i, sizeof(i)) != sizeof(i)){ 165 | fprintf(stderr, "test-lab-3-c: write(%s): %s\n", 166 | n, strerror(errno)); 167 | exit(1); 168 | } 169 | if(close(fd) != 0){ 170 | fprintf(stderr, "test-lab-3-c: close(%s): %s\n", 171 | n, strerror(errno)); 172 | exit(1); 173 | } 174 | } 175 | } 176 | 177 | void 178 | checkn(const char *d, const char *prefix, int nf) 179 | { 180 | int fd, i, cc, j; 181 | char n[512]; 182 | 183 | for(i = 0; i < nf; i++){ 184 | sprintf(n, "%s/%s-%d", d, prefix, i); 185 | fd = open(n, 0); 186 | if(fd < 0){ 187 | fprintf(stderr, "test-lab-3-c: open(%s): %s\n", 188 | n, strerror(errno)); 189 | exit(1); 190 | } 191 | j = -1; 192 | cc = read(fd, &j, sizeof(j)); 193 | if(cc != sizeof(j)){ 194 | fprintf(stderr, "test-lab-3-c: read(%s) returned too little %d%s%s\n", 195 | n, 196 | cc, 197 | errno ? ": " : "", 198 | errno ? strerror(errno) : ""); 199 | exit(1); 200 | } 201 | if(j != i){ 202 | fprintf(stderr, "test-lab-3-c: checkn %s contained %d not %d\n", 203 | n, j, i); 204 | exit(1); 205 | } 206 | close(fd); 207 | } 208 | } 209 | 210 | void 211 | unlinkn(const char *d, const char *prefix, int nf) 212 | { 213 | char n[512]; 214 | int i; 215 | 216 | sleep(1); 217 | 218 | for(i = 0; i < nf; i++){ 219 | sprintf(n, "%s/%s-%d", d, prefix, i); 220 | if(unlink(n) != 0){ 221 | fprintf(stderr, "test-lab-3-c: unlink(%s): %s\n", 222 | n, strerror(errno)); 223 | exit(1); 224 | } 225 | } 226 | } 227 | 228 | int 229 | compar(const void *xa, const void *xb) 230 | { 231 | char *a = *(char**)xa; 232 | char *b = *(char**)xb; 233 | return strcmp(a, b); 234 | } 235 | 236 | void 237 | dircheck(const char *d, int nf) 238 | { 239 | DIR *dp; 240 | struct dirent *e; 241 | char *names[1000]; 242 | int nnames = 0, i; 243 | 244 | dp = opendir(d); 245 | if(dp == 0){ 246 | fprintf(stderr, "test-lab-3-c: opendir(%s): %s\n", d, strerror(errno)); 247 | exit(1); 248 | } 249 | while((e = readdir(dp))){ 250 | if(e->d_name[0] != '.'){ 251 | if(nnames >= sizeof(names)/sizeof(names[0])){ 252 | fprintf(stderr, "warning: too many files in %s\n", d); 253 | } 254 | names[nnames] = (char *) malloc(strlen(e->d_name) + 1); 255 | strcpy(names[nnames], e->d_name); 256 | nnames++; 257 | } 258 | } 259 | closedir(dp); 260 | 261 | if(nf != nnames){ 262 | fprintf(stderr, "test-lab-3-c: wanted %d dir entries, got %d\n", nf, nnames); 263 | exit(1); 264 | } 265 | 266 | /* check for duplicate entries */ 267 | qsort(names, nnames, sizeof(names[0]), compar); 268 | for(i = 0; i < nnames-1; i++){ 269 | if(strcmp(names[i], names[i+1]) == 0){ 270 | fprintf(stderr, "test-lab-3-c: duplicate directory entry for %s\n", names[i]); 271 | exit(1); 272 | } 273 | } 274 | 275 | for(i = 0; i < nnames; i++) 276 | free(names[i]); 277 | } 278 | 279 | void 280 | reap (int pid) 281 | { 282 | int wpid, status; 283 | wpid = waitpid (pid, &status, 0); 284 | if (wpid < 0) { 285 | perror("waitpid"); 286 | exit(1); 287 | } 288 | if (wpid != pid) { 289 | fprintf(stderr, "unexpected pid reaped: %d\n", wpid); 290 | exit(1); 291 | } 292 | if(!WIFEXITED(status) || WEXITSTATUS(status) != 0) { 293 | fprintf(stderr, "child exited unhappily\n"); 294 | exit(1); 295 | } 296 | } 297 | 298 | int 299 | main(int argc, char *argv[]) 300 | { 301 | int pid; 302 | 303 | if(argc != 3){ 304 | fprintf(stderr, "Usage: test-lab-3-c dir1 dir2\n"); 305 | exit(1); 306 | } 307 | 308 | sprintf(d1, "%s/da%d", argv[1], getpid()); 309 | if(mkdir(d1, 0777) != 0){ 310 | fprintf(stderr, "test-lab-3-c: failed: mkdir(%s): %s\n", 311 | d1, strerror(errno)); 312 | exit(1); 313 | } 314 | sprintf(d2, "%s/db%d", argv[2], getpid()); 315 | if(mkdir(d2, 0777) != 0){ 316 | fprintf(stderr, "test-lab-3-c: failed: mkdir(%s): %s\n", 317 | d2, strerror(errno)); 318 | exit(1); 319 | } 320 | 321 | { 322 | char dd[512]; 323 | sprintf(dd, "%s/da%d", argv[2], getpid()); 324 | if(access(dd, 0) != 0){ 325 | fprintf(stderr, "test-lab-3-c: failed: access(%s) after mkdir %s: %s\n", 326 | dd, d1, strerror(errno)); 327 | exit(1); 328 | } 329 | } 330 | 331 | setbuf(stdout, 0); 332 | 333 | printf("Create/delete in separate directories: "); 334 | 335 | pid = fork(); 336 | if(pid < 0){ 337 | perror("test-lab-3-c: fork"); 338 | exit(1); 339 | } 340 | if(pid == 0){ 341 | createn(d2, "xx", 100); 342 | unlinkn(d2, "xx", 99); 343 | exit(0); 344 | } 345 | createn(d1, "yy", 100); 346 | unlinkn(d1, "yy", 99); 347 | sleep(4); 348 | reap(pid); 349 | dircheck(d1, 1); 350 | dircheck(d2, 1); 351 | 352 | printf("tests completed OK\n"); 353 | 354 | exit(0); 355 | return(0); 356 | } 357 | 358 | -------------------------------------------------------------------------------- /tprintf.h: -------------------------------------------------------------------------------- 1 | #ifndef TPRINTF_H 2 | #define TPRINTF_H 3 | 4 | #if 1 5 | 6 | #define tprintf(args...) do { \ 7 | struct timeval tv; \ 8 | gettimeofday(&tv, 0); \ 9 | printf( "%lu:\t", tv.tv_sec * 1000 + tv.tv_usec / 1000);\ 10 | printf( args); \ 11 | } while (0); 12 | 13 | #else 14 | 15 | #define tprintf(args...) 16 | 17 | #endif 18 | #endif 19 | -------------------------------------------------------------------------------- /unittest.cc: -------------------------------------------------------------------------------- 1 | // 2 | // Created by mactavish on 15-5-13. 3 | // 4 | 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "extent_server.h" 11 | 12 | 13 | -------------------------------------------------------------------------------- /yfs_client.cc: -------------------------------------------------------------------------------- 1 | // yfs client. implements FS operations using extent and lock server 2 | #include "yfs_client.h" 3 | #include "extent_client.h" 4 | #include "lock_client.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | 16 | yfs_client::yfs_client(std::string extent_dst, std::string lock_dst): mt(std::random_device()()) { 17 | ec = new extent_client(extent_dst); 18 | lu = new lock_release(ec); 19 | lc = new lock_client_cache(lock_dst, lu); 20 | } 21 | 22 | yfs_client::~yfs_client() { 23 | delete lc; 24 | delete lu; 25 | delete ec; 26 | } 27 | 28 | yfs_client::inum 29 | yfs_client::n2i(std::string n) { 30 | std::istringstream ist(n); 31 | unsigned long long finum; 32 | ist >> finum; 33 | return finum; 34 | } 35 | 36 | std::string 37 | yfs_client::filename(inum inum) { 38 | std::ostringstream ost; 39 | ost << inum; 40 | return ost.str(); 41 | } 42 | 43 | bool 44 | yfs_client::isfile(inum inum) { 45 | if (inum & 0x80000000) 46 | return true; 47 | return false; 48 | } 49 | 50 | bool 51 | yfs_client::isdir(inum inum) { 52 | return !isfile(inum); 53 | } 54 | 55 | int 56 | yfs_client::getfile(inum inum, fileinfo &fin) { 57 | int r = OK; 58 | lc_guard lc_(lc, inum); 59 | 60 | printf("getfile %016llx\n", inum); 61 | extent_protocol::attr a; 62 | if (ec->getattr(inum, a) != extent_protocol::OK) { 63 | r = IOERR; 64 | goto release; 65 | } 66 | 67 | fin.atime = a.atime; 68 | fin.mtime = a.mtime; 69 | fin.ctime = a.ctime; 70 | fin.size = a.size; 71 | printf("getfile %016llx -> sz %llu\n", inum, fin.size); 72 | 73 | release: 74 | 75 | return r; 76 | } 77 | 78 | int 79 | yfs_client::getdir(inum inum, dirinfo &din) { 80 | int r = OK; 81 | lc_guard lc_(lc, inum); 82 | 83 | printf("getdir %016llx\n", inum); 84 | extent_protocol::attr a; 85 | if (ec->getattr(inum, a) != extent_protocol::OK) { 86 | r = IOERR; 87 | goto release; 88 | } 89 | din.atime = a.atime; 90 | din.mtime = a.mtime; 91 | din.ctime = a.ctime; 92 | 93 | release: 94 | return r; 95 | } 96 | 97 | yfs_client::status 98 | yfs_client::create(inum parent, std::string name, inum &ino, bool is_dir) { 99 | ino = 0; 100 | lc_guard(lc, parent); 101 | auto ret = lookup(parent, name, ino); 102 | if (OK == ret){ 103 | return EXIST; 104 | } 105 | 106 | extent_protocol::attr attr; 107 | 108 | for (int i = 0; i < 10; ++i){ 109 | if (is_dir){ 110 | ino = mt() & 0x7FFFFFFF; 111 | } else { 112 | ino = mt() | 0x80000000; 113 | } 114 | if (NOENT == ec->getattr(ino, attr)){ 115 | return ec->create(parent, name, ino); 116 | } 117 | } 118 | return IOERR; 119 | } 120 | 121 | yfs_client::status 122 | yfs_client::mkdir(inum parent, std::string name, inum &ino) { 123 | return create(parent, name, ino, true); 124 | } 125 | 126 | yfs_client::status 127 | yfs_client::unlink(inum parent, std::string name){ 128 | inum ino; 129 | lc_guard lc_(lc, parent); 130 | auto ret = lookup(parent, name, ino); 131 | if (OK != ret){ 132 | return ret; 133 | } 134 | printf("UNLINK== his ino is %llu\n", ino); 135 | if (isdir(ino)){ 136 | return IOERR; 137 | } 138 | return ec->remove(ino); 139 | } 140 | 141 | yfs_client::status 142 | yfs_client::lookup(inum parent, std::string name, inum &ino) { 143 | return ec->lookup(parent, name, ino); 144 | } 145 | 146 | yfs_client::status 147 | yfs_client::readdir(inum parent, std::map &ents){ 148 | return ec->readdir(parent, ents); 149 | } 150 | 151 | 152 | yfs_client::status yfs_client::read(yfs_client::inum ino, std::size_t size, std::size_t off, std::string &buf) { 153 | std::string tmp; 154 | auto ret = ec->get(ino, tmp); 155 | if (extent_protocol::OK != ret){ 156 | return ret; 157 | } 158 | buf = std::move(tmp.substr(off, size)); 159 | return ret; 160 | } 161 | 162 | yfs_client::status yfs_client::write(yfs_client::inum ino, std::size_t size, std::size_t off, const char *cbuf) { 163 | lc_guard lc_(lc, ino); 164 | 165 | std::string tmp; 166 | auto ret = ec->get(ino, tmp); 167 | if (extent_protocol::OK != ret){ 168 | return ret; 169 | } 170 | 171 | tmp.resize(std::max(tmp.size(), off + size), '\0'); 172 | for (std::size_t i = off, j = 0; j < size; ++i,++j){ 173 | tmp[i] = cbuf[j]; 174 | } 175 | return ec->put(ino, tmp); 176 | } 177 | 178 | yfs_client::status yfs_client::setattr(inum ino, struct stat *st) { 179 | // Update mtime, atime 180 | lc_guard(lc, ino); 181 | std::string tmp; 182 | auto ret = ec->get(ino, tmp); 183 | if (OK != ret){ 184 | return ret; 185 | } 186 | if (st->st_size < (int)tmp.size()){ 187 | tmp = tmp.substr(0, st->st_size); 188 | } else { 189 | tmp += std::string('\0', st->st_size - tmp.size()); 190 | } 191 | return ec->put(ino, tmp); 192 | } 193 | -------------------------------------------------------------------------------- /yfs_client.h: -------------------------------------------------------------------------------- 1 | #ifndef yfs_client_h 2 | #define yfs_client_h 3 | 4 | #include 5 | #include "extent_client.h" 6 | #include "raii.h" 7 | #include "lock_client_cache.h" 8 | #include 9 | #include 10 | 11 | #include "lock_protocol.h" 12 | #include "lock_client.h" 13 | 14 | 15 | class yfs_client { 16 | extent_client *ec; 17 | lock_release_user *lu; 18 | lock_client_cache *lc; 19 | std::mt19937 mt; 20 | 21 | public: 22 | 23 | typedef unsigned long long inum; 24 | enum xxstatus { 25 | OK, RPCERR, NOENT, IOERR, EXIST 26 | }; 27 | typedef int status; 28 | 29 | struct fileinfo { 30 | unsigned long long size; 31 | unsigned long atime; 32 | unsigned long mtime; 33 | unsigned long ctime; 34 | }; 35 | struct dirinfo { 36 | unsigned long atime; 37 | unsigned long mtime; 38 | unsigned long ctime; 39 | }; 40 | struct dirent { 41 | std::string name; 42 | yfs_client::inum inum; 43 | }; 44 | 45 | private: 46 | static std::string filename(inum); 47 | 48 | static inum n2i(std::string); 49 | 50 | public: 51 | 52 | yfs_client(std::string, std::string); 53 | ~yfs_client(); 54 | 55 | bool isfile(inum); 56 | 57 | bool isdir(inum); 58 | 59 | int getfile(inum, fileinfo &); 60 | 61 | int getdir(inum, dirinfo &); 62 | 63 | yfs_client::status create(inum parent, std::string name, inum &id, bool is_dir = false); 64 | yfs_client::status mkdir(inum parent, std::string name, inum &id); 65 | yfs_client::status unlink(inum parent, std::string name); 66 | yfs_client::status lookup(inum parent, std::string name, inum &id); 67 | yfs_client::status readdir(inum parent, std::map &); 68 | yfs_client::status read(inum ino, std::size_t size, std::size_t off, std::string &buf); 69 | yfs_client::status write(inum ino, std::size_t size, std::size_t off, const char *buf); 70 | yfs_client::status setattr(inum ino, struct stat *st); 71 | }; 72 | 73 | #endif 74 | 75 | --------------------------------------------------------------------------------