├── TODO
├── tests
    ├── .gitignore
    ├── testrunner.sh
    ├── MockPaxosLearner.h
    ├── MockPaxosPeer.h
    ├── MockPaxosState.h
    ├── MockPosixOps.h
    ├── Makefile
    ├── PaxosBrainTest.cpp
    ├── PaxosFileLoggerTest.cpp
    └── PaxosPeerTest.cpp
├── .gitignore
├── PaxosLearner.h
├── PaxosStateLogger.h
├── PaxosThriftServer.h
├── PaxosPeer.h
├── EchoLearner.h
├── PaxosServiceHandler.h
├── PaxosStateIf.h
├── PaxosStateLoggerException.h
├── PaxosServiceHandler.cpp
├── PaxosClient.h
├── PaxosTestServer.cpp
├── PaxosTypes.h
├── PaxosBrain.h
├── PaxosState.h
├── PaxosTestClient.cpp
├── util
    ├── ThriftSerializer.h
    ├── PosixOpsIf.h
    └── PosixOps.h
├── PaxosThriftServer.cpp
├── thrift
    └── Paxos.thrift
├── PaxosThriftPeer.h
├── README.md
├── PaxosFileLogger.h
├── PaxosState.cpp
├── Makefile
├── PaxosBrain.cpp
├── PaxosFileLogReader.cpp
├── PaxosThriftPeer.cpp
├── PaxosClient.cpp
└── PaxosFileLogger.cpp


/TODO:
--------------------------------------------------------------------------------
1 | - Add checksum to the log
2 | - Make the sends nonblocking
3 | 


--------------------------------------------------------------------------------
/tests/.gitignore:
--------------------------------------------------------------------------------
1 | PaxosTest
2 | PaxosFileLoggerTest
3 | PaxosBrainTest
4 | 


--------------------------------------------------------------------------------
/tests/testrunner.sh:
--------------------------------------------------------------------------------
1 | #!/bin/sh
2 | 
3 | echo $@
4 | 
5 | for i in $@
6 | do
7 |   ./$i
8 | done
9 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | PaxosTestServer
2 | PaxosTestClient
3 | PaxosFileLogReader
4 | *.o
5 | libpaxos.a
6 | *.swp
7 | 
8 | 


--------------------------------------------------------------------------------
/PaxosLearner.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_LEARNER_H__
 2 | #define __PAXOS_LEARNER_H__
 3 | 
 4 | namespace Paxos {
 5 | class PaxosLearner {
 6 | public:
 7 | 	virtual void learn(const std::string& val) = 0;
 8 | };
 9 | }
10 | #endif
11 | 


--------------------------------------------------------------------------------
/tests/MockPaxosLearner.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MOCK_PAXOS_LEARNER_H__
 2 | #define __MOCK_PAXOS_LEARNER_H__
 3 | 
 4 | #include "gmock/gmock.h"
 5 | #include "PaxosLearner.h"
 6 | 
 7 | namespace Paxos {
 8 | class MockPaxosLearner : public PaxosLearner {
 9 | public:
10 |   MOCK_METHOD1(learn, void(const std::string&));
11 | };
12 | }
13 | 
14 | #endif
15 | 


--------------------------------------------------------------------------------
/PaxosStateLogger.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_STATE_LOGGER_H__
 2 | #define __PAXOS_STATE_LOGGER_H__
 3 | 
 4 | #include "Paxos_types.h"
 5 | #include "PaxosStateLoggerException.h"
 6 | 
 7 | namespace Paxos {
 8 | class PaxosStateLogger {
 9 | public:
10 |   virtual void log(const PaxosTransaction&) = 0;
11 |   virtual bool getLatestTransaction(PaxosTransaction&) = 0;
12 |   virtual void commitLatestTransaction() = 0;
13 | };
14 | }
15 | 
16 | #endif
17 | 


--------------------------------------------------------------------------------
/PaxosThriftServer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_THRIFT_SERVER_H__
 2 | #define __PAXOS_THRIFT_SERVER_H__
 3 | 
 4 | #include "PaxosBrain.h"
 5 | 
 6 | #include <thread>
 7 | 
 8 | namespace Paxos {
 9 | class PaxosThriftServer {
10 | public:
11 | 	PaxosThriftServer(PaxosBrain&, int port = 9090);
12 | 	~PaxosThriftServer();
13 | 	void start();
14 | 
15 | private:
16 | 	std::thread	serverThread_;
17 | 	PaxosBrain& brain_;
18 | 	int port_;
19 | };
20 | }
21 | 
22 | #endif
23 | 


--------------------------------------------------------------------------------
/PaxosPeer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_PEER_H__
 2 | #define __PAXOS_PEER_H__
 3 | #include "Paxos_types.h"
 4 | #include <iostream>
 5 | 
 6 | namespace Paxos {
 7 | class PaxosPeer {
 8 | public:
 9 | 	virtual void initialize() {}
10 |   virtual void sendPropose(const PaxosProposeArgs& p, PaxosProposeResult& r) = 0; 
11 |   virtual void sendAccept(const PaxosAcceptArgs& a, PaxosAcceptResult& r) = 0;
12 |   virtual void sendLearn(const std::string& value) = 0;
13 |   virtual int64_t getHighestProposalSeen() = 0;
14 | };
15 | }
16 | #endif
17 | 


--------------------------------------------------------------------------------
/EchoLearner.h:
--------------------------------------------------------------------------------
 1 | #include "PaxosLearner.h"
 2 | 
 3 | #include <vector>
 4 | 
 5 | using namespace Paxos;
 6 | 
 7 | class EchoLearner : public PaxosLearner {
 8 | public:
 9 | 	virtual void learn(const std::string& val) {
10 | 		std::cout << "Learned " << val << std::endl;
11 |     values_.push_back(val);
12 | 	}
13 | 
14 |   void dumpVals() {
15 |     for (auto i = values_.begin(); i != values_.end(); ++i) {
16 |       std::cout << *i << std::endl;
17 |     }
18 |   }
19 | 
20 | private:
21 |   std::vector<std::string> values_;
22 | };
23 | 
24 | 


--------------------------------------------------------------------------------
/tests/MockPaxosPeer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MOCK_PAXOS_PEER_H__
 2 | #define __MOCK_PAXOS_PEER_H__
 3 | 
 4 | #include "gmock/gmock.h"
 5 | #include "PaxosPeer.h"
 6 | 
 7 | namespace Paxos {
 8 | class MockPaxosPeer : public PaxosPeer {
 9 | public:
10 |   MOCK_METHOD2(sendPropose, void(const PaxosProposeArgs&, PaxosProposeResult&));
11 |   MOCK_METHOD2(sendAccept, void(const PaxosAcceptArgs&, PaxosAcceptResult&));
12 |   MOCK_METHOD1(sendLearn, void(const std::string&));
13 |   MOCK_METHOD0(getHighestProposalSeen, int64_t());
14 | };
15 | }
16 | 
17 | #endif
18 | 


--------------------------------------------------------------------------------
/PaxosServiceHandler.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_SERVICE_HANDLER_H__
 2 | #define __PAXOS_SERVICE_HANDLER_H__
 3 | 
 4 | #include "PaxosService.h"
 5 | #include "PaxosBrain.h"
 6 | 
 7 | namespace Paxos {
 8 | class PaxosServiceHandler : public PaxosServiceIf {
 9 | public:
10 |   PaxosServiceHandler(PaxosBrain&);
11 |   void    propose(PaxosProposeResult&, const PaxosProposeArgs&);
12 |   void    accept(PaxosAcceptResult&, const PaxosAcceptArgs&);
13 |   void    learn(const std::string& value);
14 |   int64_t getHighestProposalSeen();
15 | 
16 | private:
17 |   PaxosBrain& brain_;
18 | };
19 | }
20 | 
21 | #endif
22 | 


--------------------------------------------------------------------------------
/tests/MockPaxosState.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MOCK_PAXOS_STATE_H__
 2 | #define __MOCK_PAXOS_STATE_H__
 3 | 
 4 | #include "PaxosStateIf.h"
 5 | 
 6 | namespace Paxos {
 7 | class MockPaxosState : public PaxosStateIf {
 8 | public:
 9 |   MOCK_METHOD0(getHighestProposalSeen, int64_t());
10 |   MOCK_METHOD1(setHighestProposalSeen, void(int64_t));
11 |   MOCK_METHOD0(isTransactionInProgress, bool());
12 |   MOCK_METHOD0(getPendingTransaction, PaxosTransaction());
13 |   MOCK_METHOD1(setPendingTransaction, void(const PaxosTransaction&));
14 |   MOCK_METHOD0(clearPendingTransaction, void());
15 | };
16 | }
17 | 
18 | #endif
19 | 


--------------------------------------------------------------------------------
/PaxosStateIf.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_STATE_IF_H__
 2 | #define __PAXOS_STATE_IF_H__
 3 | 
 4 | #include "Paxos_types.h"
 5 | 
 6 | namespace Paxos {
 7 | class PaxosStateIf {
 8 | public:
 9 |   virtual int64_t           getHighestProposalSeen() = 0;
10 |   virtual void              setHighestProposalSeen(int64_t) = 0;
11 | 
12 |   virtual bool              isTransactionInProgress() = 0;
13 | 
14 |   virtual PaxosTransaction  getPendingTransaction() = 0;
15 |   virtual void              setPendingTransaction(const PaxosTransaction&) = 0;
16 |   virtual void              clearPendingTransaction() = 0;
17 | };
18 | }
19 | #endif
20 | 


--------------------------------------------------------------------------------
/PaxosStateLoggerException.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_STATE_LOGGER_EXCEPTION_H__
 2 | #define __PAXOS_STATE_LOGGER_EXCEPTION_H__
 3 | 
 4 | namespace Paxos {
 5 | class PaxosStateLoggerException : public std::exception {
 6 | public:
 7 |   static const int LOG_NOT_FOUND = 0;
 8 |   static const int LOG_CORRUPT = 1;
 9 |   static const int IO_ERROR = 3;
10 | 
11 |   PaxosStateLoggerException(int errorcode, const std::string what) : 
12 |     errcode_(errorcode), what_(what) {
13 |   }
14 | 
15 |   int errcode() { return errcode_; }
16 |   const char * what() { return what_.c_str(); }
17 | 
18 | private:
19 |   const int         errcode_;
20 |   const std::string what_;
21 | };
22 | }
23 | #endif
24 | 


--------------------------------------------------------------------------------
/PaxosServiceHandler.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosServiceHandler.h"
 2 | 
 3 | using namespace Paxos;
 4 | 
 5 | PaxosServiceHandler::PaxosServiceHandler(PaxosBrain& brain) : brain_(brain) {
 6 | }
 7 | 
 8 | void PaxosServiceHandler::propose(PaxosProposeResult& res, const PaxosProposeArgs& args) {
 9 |   res = brain_.recvPropose(args);
10 | }
11 | 
12 | void PaxosServiceHandler::accept(PaxosAcceptResult& res, const PaxosAcceptArgs& args) {
13 |   res = brain_.recvAccept(args);
14 | 	brain_.sentAcceptResponse();
15 | }
16 | 
17 | void PaxosServiceHandler::learn(const std::string& value) {
18 |   brain_.learn(value);
19 | }
20 | 
21 | int64_t PaxosServiceHandler::getHighestProposalSeen() {
22 |   return brain_.getHighestProposalSeen();
23 | }
24 | 
25 | 


--------------------------------------------------------------------------------
/PaxosClient.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_CLIENT_H__
 2 | #define __PAXOS_CLIENT_H__
 3 | 
 4 | #include "PaxosPeer.h"
 5 | #include "Paxos_types.h"
 6 | 
 7 | #include <vector>
 8 | #include <mutex>
 9 | 
10 | namespace Paxos {
11 | class PaxosClient {
12 | public:
13 |   PaxosClient(std::vector<PaxosPeer *>& peers);
14 |   void initialize();
15 |   void setMaxRetries(int);
16 |   bool submit(std::string& val);
17 | 
18 | private:
19 |   const int   MAX_RETRIES = 3;
20 | 
21 |   bool sendAccept(PaxosTransaction&);
22 |   void sendLearn(std::string&);
23 | 
24 |   int                      maxTriesPerSubmit_;
25 |   std::vector<PaxosPeer *> peers_;
26 |   int64_t                  highestProposalSeen_;
27 |   std::mutex               clientLock_;
28 | };
29 | }
30 | #endif
31 | 


--------------------------------------------------------------------------------
/PaxosTestServer.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosThriftServer.h"
 2 | #include "EchoLearner.h"
 3 | #include "PaxosFileLogger.h"
 4 | 
 5 | #include "util/PosixOps.h"
 6 | 
 7 | #include <vector>
 8 | #include <thread>
 9 | #include <iostream>
10 | #include <sstream>
11 | 
12 | using namespace std;
13 | using namespace Paxos;
14 | 
15 | int main(int argc, char** argv) {
16 |   char buf[255];
17 |   sprintf(buf, "/tmp/paxos_log_%s", argv[1]);
18 | 
19 |   EchoLearner learner;
20 |   PaxosFileLogger logger(buf, "log_");
21 | 
22 |   PaxosBrain brain(logger, learner);
23 |   int port = atoi(argv[1]);
24 |   PaxosThriftServer server(brain, port);
25 | 
26 |   cout << "Starting server on port " << port << endl;
27 |   server.start();	
28 | 
29 |   cout << "Back in main thread!!" << endl;
30 | 
31 |   return 0;
32 | }
33 | 
34 | 


--------------------------------------------------------------------------------
/PaxosTypes.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_TYPES_H__
 2 | #define __PAXOS_TYPES_H__
 3 | #include <sys/types.h>
 4 | 
 5 | #include <mutex>
 6 | 
 7 | namespace Paxos {
 8 | struct PaxosTransaction {
 9 |   int64_t       proposal;
10 |   std::string   value;
11 | };
12 | 
13 | enum ProposeStatus {
14 |   PROMISE,
15 |   PROMISED_HIGHER_VERSION,
16 |   HAS_UNFINISHED_TRANSACTION,
17 | };
18 | 
19 | struct PaxosProposeArgs {
20 |   int64_t     proposal;
21 | };
22 | 
23 | struct PaxosProposeResult {
24 |   ProposeStatus           status;
25 |   PaxosTransaction        pendingTxn;
26 | };
27 | 
28 | enum PaxosAcceptStatus {
29 |   ACCEPTED,
30 |   REJECTED,
31 | };
32 | 
33 | struct PaxosAcceptArgs {
34 |   PaxosTransaction  txn;
35 | };
36 | 
37 | struct PaxosAcceptResult {
38 |   PaxosAcceptStatus status;
39 | };
40 | }
41 | #endif
42 | 


--------------------------------------------------------------------------------
/PaxosBrain.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_BRAIN_H__
 2 | #define __PAXOS_BRAIN_H__
 3 | 
 4 | #include "Paxos_types.h"
 5 | #include "PaxosThriftPeer.h"
 6 | #include "PaxosStateIf.h"
 7 | #include "PaxosStateLogger.h"
 8 | #include "PaxosLearner.h"
 9 | 
10 | namespace Paxos {
11 | class PaxosBrain {
12 | private:
13 | 	PaxosLearner&						      learner_;
14 |   std::shared_ptr<PaxosStateIf> state_;
15 | 
16 | public:
17 |   PaxosBrain(PaxosStateLogger&, PaxosLearner&);
18 |   PaxosBrain(std::shared_ptr<PaxosStateIf> state, PaxosLearner&);
19 | 
20 |   PaxosProposeResult  recvPropose(const PaxosProposeArgs&);
21 |   PaxosAcceptResult   recvAccept(const PaxosAcceptArgs&);
22 |   void                sentAcceptResponse();
23 |   void                learn(const std::string&);
24 |   int64_t             getHighestProposalSeen();
25 | };
26 | }
27 | #endif
28 | 


--------------------------------------------------------------------------------
/tests/MockPosixOps.h:
--------------------------------------------------------------------------------
 1 | #ifndef __MOCK_POSIX_OPS_H__
 2 | #define __MOCK_POSIX_OPS_H__
 3 | 
 4 | #include "gmock/gmock.h"
 5 | #include "util/PosixOpsIf.h"
 6 | 
 7 | class MockPosixOps : public PosixOpsIf {
 8 | public:
 9 |   MOCK_METHOD2(mkdir, int(const char* path, mode_t mode));
10 |   MOCK_METHOD3(open, int(const char *path, int flags, mode_t mode));
11 |   MOCK_METHOD3(lseek, int64_t(int fd, off_t offset, int whence));
12 |   MOCK_METHOD2(fstat, int(int fd, struct stat* buf));
13 |   MOCK_METHOD3(read, int(int fd, void* buf, size_t size));
14 |   MOCK_METHOD3(readv, int(int fd, struct iovec* iov, int count));
15 |   MOCK_METHOD3(write, int(int fd, void* buf, size_t size));
16 |   MOCK_METHOD3(writev, int(int fd, struct iovec* iov, int count));
17 |   MOCK_METHOD1(close, int(int fd));
18 |   MOCK_METHOD0(getErrno, int());
19 | };
20 | #endif
21 | 


--------------------------------------------------------------------------------
/PaxosState.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_STATE_H__
 2 | #define __PAXOS_STATE_H__
 3 | 
 4 | #include "Paxos_types.h"
 5 | #include "PaxosStateIf.h"
 6 | #include "PaxosStateLogger.h"
 7 | 
 8 | #include <mutex>
 9 | 
10 | namespace Paxos {
11 | class PaxosState : public PaxosStateIf {
12 | private:
13 |   std::mutex        stateLock_;
14 |   int64_t           highestProposalSeen_;
15 |   bool              txnInProgress_;
16 |   PaxosTransaction  pendingTxn_;
17 | 
18 |   PaxosStateLogger& logger_;
19 | 
20 | public:
21 |   PaxosState(PaxosStateLogger&);
22 |   int64_t             getHighestProposalSeen();
23 |   void                setHighestProposalSeen(int64_t);
24 | 
25 |   bool                isTransactionInProgress();
26 | 
27 |   PaxosTransaction    getPendingTransaction();
28 |   void                setPendingTransaction(const PaxosTransaction&);
29 |   void                clearPendingTransaction();
30 | };
31 | }  
32 | #endif
33 | 


--------------------------------------------------------------------------------
/PaxosTestClient.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosThriftPeer.h"
 2 | #include "PaxosThriftServer.h"
 3 | #include "EchoLearner.h"
 4 | #include "PaxosClient.h"
 5 | #include "PaxosFileLogger.h"
 6 | 
 7 | #include "util/PosixOps.h"
 8 | 
 9 | #include <vector>
10 | #include <thread>
11 | #include <iostream>
12 | #include <sstream>
13 | 
14 | using namespace std;
15 | using namespace Paxos;
16 | 
17 | int main(int argc, char** argv) {
18 |   std::vector<PaxosPeer *> peers;
19 |   for (int port = 9090; port < 9093; ++port) {
20 |     PaxosThriftPeer *p = new PaxosThriftPeer("localhost", port);
21 |     peers.push_back(p);
22 |   }
23 | 
24 |   PaxosClient client(peers);
25 |   client.initialize();
26 | 
27 |   for (int i = 0; i < 10000; ++i) {
28 |     stringstream ss;
29 |     string s;
30 |     ss << i;
31 |     s = ss.str();
32 |     cout << "Submitting " << s << endl;
33 |     client.submit(s);
34 |   }
35 | 
36 |   return 0;
37 | }
38 | 
39 | 


--------------------------------------------------------------------------------
/util/ThriftSerializer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __THRIFT_SERIALIZER_H__
 2 | #define __THRIFT_SERIALIZER_H__
 3 | 
 4 | #include "protocol/TCompactProtocol.h"
 5 | #include "transport/TBufferTransports.h"
 6 | 
 7 | #include <boost/shared_ptr.hpp>
 8 | 
 9 | using namespace apache;
10 | using namespace apache::thrift;
11 | using namespace apache::thrift::transport;
12 | using namespace apache::thrift::protocol;
13 | 
14 | template <typename T>
15 | class ThriftSerializer {
16 | private:
17 |   boost::shared_ptr<TMemoryBuffer>   buffer_;
18 | 
19 | public:
20 |   ThriftSerializer() : buffer_(new TMemoryBuffer()) {
21 |   }
22 | 
23 |   void serialize(const T& object, uint8_t** buf, uint32_t* size) {
24 |     buffer_->resetBuffer();
25 |     TCompactProtocol protocol(buffer_);
26 |     object.write(&protocol);
27 | 
28 |     buffer_->getBuffer(buf, size);
29 |   }
30 | 
31 |   void deserialize(T& object, uint8_t* buf, uint32_t size) {
32 |     buffer_->resetBuffer(buf, size);
33 |     TCompactProtocol protocol(buffer_);
34 |     
35 |     object.read(&protocol);
36 |   }
37 | };
38 | 
39 | #endif
40 | 


--------------------------------------------------------------------------------
/PaxosThriftServer.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosThriftServer.h"
 2 | #include "PaxosServiceHandler.h"
 3 | 
 4 | #include <thrift/protocol/TBinaryProtocol.h>
 5 | #include <thrift/server/TNonblockingServer.h>
 6 | 
 7 | using namespace apache::thrift;
 8 | using namespace apache::thrift::protocol;
 9 | using namespace apache::thrift::transport;
10 | using namespace apache::thrift::server;
11 | 
12 | using namespace Paxos;
13 | 
14 | PaxosThriftServer::PaxosThriftServer(PaxosBrain& brain, int port) : brain_(brain), port_(port) {
15 | }
16 | 
17 | void PaxosThriftServer::start() {
18 | 	boost::shared_ptr<TProtocolFactory> protocolFactory(
19 | 		new TBinaryProtocolFactory());
20 | 	boost::shared_ptr<PaxosServiceHandler>	handler(new PaxosServiceHandler(brain_));
21 | 	boost::shared_ptr<TProcessor> processor(new PaxosServiceProcessor(handler));
22 | 
23 | 	TNonblockingServer server(processor, protocolFactory, port_);
24 | 	serverThread_ = std::thread(std::bind(&TNonblockingServer::serve, server));
25 | }
26 | 
27 | PaxosThriftServer::~PaxosThriftServer() {
28 | 	serverThread_.join();
29 | }
30 | 
31 | 


--------------------------------------------------------------------------------
/thrift/Paxos.thrift:
--------------------------------------------------------------------------------
 1 | namespace cpp Paxos
 2 | 
 3 | struct PaxosTransaction {
 4 |   1: i64      proposal,
 5 |   2: binary   value,
 6 |   3: bool     committed,
 7 | }
 8 | 
 9 | enum PaxosProposeStatus {
10 |   PROMISE,
11 |   PROMISED_HIGHER_VERSION,
12 |   HAS_UNFINISHED_TRANSACTION,
13 | }
14 | 
15 | struct PaxosProposeArgs {
16 |   1: i64 proposal,
17 | }
18 | 
19 | struct PaxosProposeResult {
20 |   1: PaxosProposeStatus             status,
21 |   2: optional i64                   higherProposal,                        
22 |   3: optional PaxosTransaction      pendingTxn,
23 | }
24 | 
25 | enum PaxosAcceptStatus {
26 |   ACCEPTED,
27 |   REJECTED,
28 | }
29 | 
30 | struct PaxosAcceptArgs {
31 |   1: PaxosTransaction    txn,
32 | }
33 | 
34 | struct PaxosAcceptResult {
35 |   1: PaxosAcceptStatus   status,
36 | }
37 | 
38 | service PaxosService {
39 |   PaxosProposeResult  propose(1: PaxosProposeArgs pArgs),
40 |   PaxosAcceptResult   accept(1: PaxosAcceptArgs aArgs),
41 |   void                learn(1: binary value);
42 |   i64                 getHighestProposalSeen(),
43 | }
44 |  
45 | 
46 | 


--------------------------------------------------------------------------------
/util/PosixOpsIf.h:
--------------------------------------------------------------------------------
 1 | #ifndef __POSIX_OPS_IF_H__
 2 | #define __POSIX_OPS_IF_H__
 3 | 
 4 | // large file support
 5 | #ifndef _LARGEFILE64_SOURCE
 6 | #define _LARGEFILE64_SOURCE
 7 | #endif
 8 | 
 9 | #include <sys/types.h>
10 | #include <sys/stat.h>
11 | #include <sys/uio.h>
12 | #include <fcntl.h>
13 | 
14 | class PosixOpsIf {
15 | public:
16 |   static const mode_t defaultFileMode_ = S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH;
17 |   static const mode_t defaultDirMode_ = 
18 |     S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH; 
19 |   virtual int mkdir(const char* path, 
20 |     mode_t mode = PosixOpsIf::defaultDirMode_) = 0;
21 |   virtual int open(const char *path, int flags, 
22 |     mode_t mode = PosixOpsIf::defaultFileMode_) = 0;
23 |   virtual int64_t lseek(int fd, off_t offset, int whence) = 0;
24 |   virtual int fstat(int fd, struct stat* buf) = 0;
25 |   virtual int read(int fd, void* buf, size_t size) = 0;
26 |   virtual int readv(int fd, struct iovec* iov, int count) = 0;
27 |   virtual int write(int fd, void* buf, size_t size) = 0;
28 |   virtual int writev(int fd, struct iovec* iov, int count) = 0;
29 |   virtual int close(int fd) = 0;
30 |   virtual int getErrno() = 0;
31 | 
32 |   virtual ~PosixOpsIf() {}
33 | };
34 | #endif
35 | 


--------------------------------------------------------------------------------
/PaxosThriftPeer.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_THRIFT_PEER_H__
 2 | #define __PAXOS_THRIFT_PEER_H__
 3 | 
 4 | #include "PaxosPeer.h"
 5 | #include "PaxosService.h"
 6 | 
 7 | #include <transport/TSocket.h>
 8 | #include <transport/TBufferTransports.h>
 9 | #include <protocol/TBinaryProtocol.h>
10 | 
11 | #include <mutex>
12 | 
13 | namespace Paxos {
14 | class PaxosThriftPeer : public PaxosPeer {
15 | public:
16 | 	PaxosThriftPeer(std::string hostname, int port);
17 | 	~PaxosThriftPeer();
18 | 	void initialize();
19 | 	virtual void sendPropose(const PaxosProposeArgs& args, PaxosProposeResult& res);
20 | 	virtual void sendAccept(const PaxosAcceptArgs& args, PaxosAcceptResult& res);
21 |   virtual void sendLearn(const std::string& val);
22 |   virtual int64_t getHighestProposalSeen();
23 | 
24 | 	void setConnectTimeout(int timeoutInMs);
25 | 	void setSendTimeout(int timeoutInMs);
26 | 	void setReceiveTimeout(int timeoutInMs);
27 | 
28 | private:
29 | 	std::mutex clientLock_;
30 | 	std::shared_ptr<PaxosServiceClient> client_;
31 | 	std::string hostname_;
32 | 	int port_;
33 | 	int connectTimeout_;
34 | 	int readTimeout_;
35 | 	int writeTimeout_;
36 | 
37 | 	const int MAX_RETRIES = 3;
38 | 	const int DEFAULT_CONNECT_TIMEOUT = 100;
39 | 	const int DEFAULT_READ_TIMEOUT = 100;
40 | 	const int DEFAULT_WRITE_TIMEOUT = 100;
41 | };
42 | }
43 | 
44 | #endif
45 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | #Paxos
 2 | 
 3 | A simple Paxos implementation in C++. For more information on the Paxos protocol, go here: 
 4 | http://en.wikipedia.org/wiki/Paxos_%28computer_science%29
 5 | 
 6 | ##Key Classes##
 7 | The key classes in this library are:
 8 | 
 9 | ###<code>PaxosBrain</code>###
10 | This class implements the core of the Paxos protocol. It plays the role of the acceptor in the protocol description.
11 | 
12 | ###<code>PaxosPeer</code>###
13 | This interface represents a connection to a replica in the Paxos quorum. <code>PaxosThriftClient</code> is an implementation of this interface in thrift.
14 | 
15 | ###<code>PaxosClient</code>###
16 | This class implements the logic required to reach consensus. It implements the roles of the client and the proposer.
17 | 
18 | ###<code>PaxosLearner</code>###
19 | This interface implements the role of the learner. Typically an application will implement this interface to "learn" of events that have been agreed upon.
20 | 
21 | ###<code>PaxosStateLogger</code>###
22 | This interface defines the interface that any class that implements persistent state logging must implement. <code>PaxosFileLogger</code> implements this interface and logs all transactions as a log of events. 
23 | 
24 | ##Building the library##
25 | 
26 |     make thrift
27 |     make
28 |     make runtests   #to run tests
29 | 


--------------------------------------------------------------------------------
/tests/Makefile:
--------------------------------------------------------------------------------
 1 | CXX=g++
 2 | DEFINES=-DHAVE_CONFIG_H
 3 | FLAGS=-Wall -Werror -g -std=gnu++11
 4 | PAXOS_LIB=-lpaxos -lthrift
 5 | PAXOS_LIB_PATH=-L..
 6 | 
 7 | GTEST_DIR=/home/rni/gmock-svn/gtest
 8 | GMOCK_DIR=/home/rni/gmock-svn/
 9 | GMOCK_INCLUDES=-isystem $(GTEST_DIR)/include -isystem $(GMOCK_DIR)/include
10 | GMOCK_LIB_DIR=-L$(GMOCK_DIR)
11 | GMOCK_LIB=-lgmock 
12 | GMOCK_DEP_LIBS=-pthread
13 | 
14 | INCLUDES=-I.. -I ../thrift/gen-cpp -I. -I/usr/local/include/thrift
15 | 
16 | TESTS=PaxosPeerTest PaxosFileLoggerTest PaxosBrainTest
17 | 
18 | all: $(TESTS)
19 | 
20 | PaxosPeerTest: PaxosPeerTest.cpp
21 | 	$(CXX) $(FLAGS) $(DEFINES) $(INCLUDES) $(PAXOS_LIB_PATH) \
22 |     $(GMOCK_INCLUDES) $(GMOCK_LIB_DIR) $(GMOCK_DEP_LIBS) \
23 |     PaxosPeerTest.cpp $(GMOCK_LIB) -o PaxosPeerTest $(PAXOS_LIB)
24 | 
25 | PaxosFileLoggerTest: PaxosFileLoggerTest.cpp
26 | 	$(CXX) $(FLAGS) $(DEFINES) $(INCLUDES) $(PAXOS_LIB_PATH) \
27 |     $(GMOCK_INCLUDES) $(GMOCK_LIB_DIR) $(GMOCK_DEP_LIBS) \
28 |     PaxosFileLoggerTest.cpp $(GMOCK_LIB) -o PaxosFileLoggerTest $(PAXOS_LIB)
29 | 
30 | PaxosBrainTest: PaxosBrainTest.cpp
31 | 	$(CXX) $(FLAGS) $(DEFINES) $(INCLUDES) $(PAXOS_LIB_PATH) \
32 |     $(GMOCK_INCLUDES) $(GMOCK_LIB_DIR) $(GMOCK_DEP_LIBS) \
33 |     PaxosBrainTest.cpp $(GMOCK_LIB) -o PaxosBrainTest $(PAXOS_LIB)
34 | 
35 | .PHONY : clean
36 | clean:
37 | 	rm -f *.o $(TESTS)
38 | 
39 | .PHONY : runtests
40 | runtests: all
41 | 	./testrunner.sh $(TESTS)
42 | 
43 | 


--------------------------------------------------------------------------------
/util/PosixOps.h:
--------------------------------------------------------------------------------
 1 | #ifndef __POSIX_OPS_H__
 2 | #define __POSIX_OPS_H__
 3 | 
 4 | // large file support
 5 | #ifndef _LARGEFILE64_SOURCE
 6 | #define _LARGEFILE64_SOURCE
 7 | #endif
 8 | 
 9 | #include <sys/types.h>
10 | #include <sys/stat.h>
11 | #include <sys/uio.h>
12 | #include <fcntl.h>
13 | 
14 | #include "PosixOpsIf.h"
15 | 
16 | class PosixOps : public PosixOpsIf {
17 | public:
18 |   virtual inline int mkdir(const char* path, 
19 |       mode_t mode = PosixOpsIf::defaultDirMode_) {
20 |     return ::mkdir(path, mode);
21 |   }
22 | 
23 |   inline int open(const char *path, 
24 |       int flags, 
25 |       mode_t mode = PosixOpsIf::defaultFileMode_) {
26 |     return ::open(path, flags, mode);
27 |   }
28 | 
29 |   inline int64_t lseek(int fd, off_t offset, int whence) {
30 |     return (int64_t)::lseek(fd, offset, whence);
31 |   }
32 | 
33 |   inline int fstat(int fd, struct stat* buf) {
34 |     return ::fstat(fd, buf);
35 |   }
36 |   
37 |   inline int read(int fd, void* buf, size_t size) {
38 |     return ::read(fd, buf, size);
39 |   }
40 | 
41 |   inline int readv(int fd, struct iovec* iov, int count) {
42 |     return ::readv(fd, iov, count);
43 |   }
44 | 
45 |   inline int write(int fd, void* buf, size_t size) {
46 |     return ::write(fd, buf, size);
47 |   }
48 | 
49 |   inline int writev(int fd, struct iovec* iov, int count) {
50 |     return ::writev(fd, iov, count);
51 |   }
52 |  
53 |   inline int close(int fd) {
54 |     return ::close(fd);
55 |   }
56 |   
57 |   inline int getErrno() {
58 |     return errno;
59 |   }
60 | };
61 | #endif
62 | 


--------------------------------------------------------------------------------
/PaxosFileLogger.h:
--------------------------------------------------------------------------------
 1 | #ifndef __PAXOS_FILE_LOGGER_H__
 2 | #define __PAXOS_FILE_LOGGER_H__
 3 | 
 4 | #include "PaxosStateLogger.h"
 5 | #include "util/ThriftSerializer.h"
 6 | #include "util/PosixOpsIf.h"
 7 | 
 8 | namespace Paxos {
 9 | struct LogEntry {
10 |   uint8_t*  buffer;
11 |   uint8_t   committed;
12 |   uint8_t*  checksum;
13 | };
14 |    
15 | class PaxosFileLogger : public PaxosStateLogger {
16 | public:
17 |   PaxosFileLogger(const std::string dir, 
18 |     const std::string prefix,
19 |     PosixOpsIf* posix = NULL);
20 |   void log(const PaxosTransaction&);
21 |   bool getLatestTransaction(PaxosTransaction&);
22 |   void commitLatestTransaction();
23 | 
24 |   void setMaxLogFileSize(uint32_t);
25 |   ~PaxosFileLogger();
26 | 
27 |   static const uint64_t RECORD_MARKER = 0xcafebabe;
28 | 
29 | private:
30 |   static const uint64_t MAX_FILE_SIZE = 1024 * 1024 * 1024; // 1G
31 | 
32 |   void openLatestLogFile();
33 |   void openLogFile(unsigned long);
34 |   void recordNextLogFile(std::string&);
35 |   unsigned long getLatestLogFile();
36 |   void rotateLogFileIfNeeded();
37 | 
38 |   const std::string   dir_;
39 |   const std::string   prefix_;
40 |   unsigned long       currentSuffix_;
41 |   int                 currentFd_;
42 |   std::string         currentFile_;
43 |   uint32_t            maxSize_;
44 |   
45 |   // XXX: This is dirty, but will have to do for now.
46 |   // It's the only way I can think of to mock out the posix 
47 |   // ops for better testing
48 |   PosixOpsIf*         posix_;
49 |   bool                shouldDeallocatePosix_;
50 | };
51 | }
52 | #endif
53 | 


--------------------------------------------------------------------------------
/PaxosState.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosState.h"
 2 | 
 3 | using namespace std;
 4 | using namespace Paxos;
 5 | 
 6 | PaxosState::PaxosState(PaxosStateLogger& logger) : logger_(logger) {
 7 |   highestProposalSeen_ = -1;
 8 |   txnInProgress_ = false;
 9 | 
10 |   PaxosTransaction txn;
11 |   
12 |   if (logger_.getLatestTransaction(txn)) {
13 |     try {
14 |       highestProposalSeen_ = txn.proposal;
15 |       if (!txn.committed) {
16 |         pendingTxn_ = txn;
17 |         txnInProgress_ = true;
18 |       }
19 |     } catch (PaxosStateLoggerException& e) {
20 |       if (e.errcode() != PaxosStateLoggerException::LOG_NOT_FOUND) {
21 |         cerr << e.what() << endl;
22 |         throw;
23 |       }
24 |     }
25 |   }
26 | }
27 | 
28 | int64_t PaxosState::getHighestProposalSeen() {
29 |   std::lock_guard<std::mutex> g(stateLock_);
30 |   return highestProposalSeen_;
31 | }
32 | 
33 | void PaxosState::setHighestProposalSeen(int64_t proposal) {
34 |   std::lock_guard<std::mutex> g(stateLock_);
35 |   highestProposalSeen_ = proposal;
36 | }
37 | 
38 | bool PaxosState::isTransactionInProgress() {
39 |   std::lock_guard<std::mutex> g(stateLock_);
40 |   return txnInProgress_;
41 | }
42 | 
43 | PaxosTransaction PaxosState::getPendingTransaction() {
44 |   std::lock_guard<std::mutex> g(stateLock_);
45 |   return pendingTxn_;
46 | }
47 | 
48 | void PaxosState::setPendingTransaction(const PaxosTransaction& txn) {
49 |   std::lock_guard<std::mutex> g(stateLock_);
50 |   pendingTxn_ = txn;
51 |   logger_.log(txn);
52 |   txnInProgress_ = true;
53 | }
54 | 
55 | 
56 | void PaxosState::clearPendingTransaction() {
57 |   std::lock_guard<std::mutex> g(stateLock_);
58 |   logger_.commitLatestTransaction();
59 |   txnInProgress_ = false;
60 | }
61 | 


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | CXX=g++
 2 | AR=ar
 3 | THRIFT=/usr/local/bin/thrift
 4 | INCLUDES=-I. -I./thrift/gen-cpp -I/usr/local/include/thrift
 5 | FLAGS=-Wall -Werror -g -std=gnu++11
 6 | DEFINES=-DHAVE_CONFIG_H
 7 | LIBRARY_INCLUDES=-L/usr/local/lib -L.
 8 | THRIFTDIR=./thrift
 9 | 
10 | PAXOS_LIBNAME=libpaxos.a
11 | 
12 | COMMON_LIBS=-lthrift
13 | SERVER_LIBS=$(COMMON_LIBS) -lthriftnb -levent
14 | PAXOS_LIB=-lpaxos
15 | 
16 | THRIFT_OBJS=thrift/gen-cpp/PaxosService.o thrift/gen-cpp/Paxos_types.o \
17 | 	thrift/gen-cpp/Paxos_constants.o
18 | OBJS=$(THRIFT_OBJS) PaxosThriftServer.o PaxosServiceHandler.o \
19 | 	PaxosBrain.o PaxosFileLogger.o PaxosState.o PaxosThriftPeer.o PaxosClient.o 
20 | 
21 | all: libpaxos.a PaxosTestServer PaxosTestClient PaxosFileLogReader
22 | 
23 | PaxosTestClient: libpaxos.a PaxosTestClient.o
24 | 	$(CXX) $(INCLUDES) $(FLAGS) $(DEFINES) $(LIBRARY_INCLUDES) \
25 |     -o PaxosTestClient PaxosTestClient.o $(PAXOS_LIB) $(SERVER_LIBS)
26 | 
27 | PaxosTestServer: libpaxos.a PaxosTestServer.o
28 | 	$(CXX) $(INCLUDES) $(FLAGS) $(DEFINES) $(LIBRARY_INCLUDES) \
29 |     -o PaxosTestServer PaxosTestServer.o $(PAXOS_LIB) $(SERVER_LIBS)
30 | 
31 | PaxosFileLogReader: PaxosFileLogReader.o
32 | 	$(CXX) $(INCLUDES) $(FLAGS) $(DEFINES) $(LIBRARY_INCLUDES) \
33 |     -o PaxosFileLogReader PaxosFileLogReader.o $(PAXOS_LIB) $(SERVER_LIBS)
34 | 
35 | %.o: %.cpp
36 | 	$(CXX) $(INCLUDES) $(FLAGS) $(DEFINES) -c $< -o $@
37 | 
38 | libpaxos.a: $(OBJS)
39 | 	$(AR) rcs libpaxos.a $(OBJS)
40 | 
41 | .PHONY: runtests
42 | runtests: libpaxos.a
43 | 	make -C ./tests clean
44 | 	make -C ./tests runtests
45 | 
46 | .PHONY: thrift
47 | thrift:
48 | 	$(THRIFT) --gen cpp -o $(THRIFTDIR) $(THRIFTDIR)/Paxos.thrift
49 | 	
50 | .PHONY : clean
51 | clean:
52 | 	rm -rf $(THRIFTDIR)/gen-cpp
53 | 	rm -f *.o main libpaxos.a
54 | 


--------------------------------------------------------------------------------
/PaxosBrain.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosBrain.h"
 2 | #include "PaxosState.h"
 3 | 
 4 | using namespace Paxos;
 5 | 
 6 | PaxosBrain::PaxosBrain(PaxosStateLogger& logger, 
 7 |     PaxosLearner& learner) : learner_(learner) {
 8 |     state_.reset(new PaxosState(logger));
 9 | }
10 | 
11 | PaxosBrain::PaxosBrain(std::shared_ptr<PaxosStateIf> state, 
12 |     PaxosLearner& learner) : learner_(learner), state_(state) {
13 | }
14 |   
15 | PaxosProposeResult PaxosBrain::recvPropose(const PaxosProposeArgs& args) {
16 |   PaxosProposeResult res;
17 | 
18 |   int64_t highestProposal = state_->getHighestProposalSeen();
19 |   if (args.proposal <= highestProposal) {
20 |     // We have agreed to a higher proposal
21 |     res.status = PaxosProposeStatus::PROMISED_HIGHER_VERSION;
22 |     res.higherProposal = highestProposal;
23 |   } else if (state_->isTransactionInProgress()) {
24 |     // Proposer crashed before we could respond
25 |     res.pendingTxn = state_->getPendingTransaction();
26 |     res.status = PaxosProposeStatus::HAS_UNFINISHED_TRANSACTION;
27 |   } else {
28 |     // It's all good. Let's promise not to honor any lower proposals
29 |     state_->setHighestProposalSeen(args.proposal);
30 |     res.status = PaxosProposeStatus::PROMISE;
31 |   }
32 |  
33 |   return res;
34 | }
35 | 
36 | PaxosAcceptResult PaxosBrain::recvAccept(const PaxosAcceptArgs& args) {
37 |   PaxosAcceptResult res;
38 | 
39 |   if (args.txn.proposal < state_->getHighestProposalSeen()) {
40 |     res.status = PaxosAcceptStatus::REJECTED;
41 |   } else {
42 |     state_->setPendingTransaction(args.txn);
43 |     res.status = PaxosAcceptStatus::ACCEPTED;
44 |   }
45 |   return res;
46 | }
47 | 
48 | void PaxosBrain::sentAcceptResponse() {
49 |   // Successfully sent the response to the proposer. Clear state
50 |   state_->clearPendingTransaction();
51 | }
52 | 
53 | void PaxosBrain::learn(const std::string& value) {
54 |   learner_.learn(value);
55 | }
56 | 
57 | int64_t PaxosBrain::getHighestProposalSeen() {
58 |   return state_->getHighestProposalSeen();
59 | }
60 | 
61 | 


--------------------------------------------------------------------------------
/PaxosFileLogReader.cpp:
--------------------------------------------------------------------------------
 1 | #ifndef _LARGEFILE64_SOURCE
 2 | #define _LARGEFILE64_SOURCE
 3 | #endif
 4 | 
 5 | #include "util/ThriftSerializer.h"
 6 | #include "Paxos_types.h"
 7 | #include "PaxosFileLogger.h"
 8 | 
 9 | #include <iostream>
10 | 
11 | #include <sys/mman.h>
12 | #include <sys/types.h>
13 | #include <sys/stat.h>
14 | #include <fcntl.h>
15 | #include <unistd.h>
16 | 
17 | using namespace std;
18 | using namespace Paxos;
19 | 
20 | void usage(char* name) {
21 |   cout << name << " <log file name>" << endl;
22 | }
23 | 
24 | void dumpLogs(void* addr, uint32_t len) {
25 |   uint8_t* p = (uint8_t *)addr + len;
26 | 
27 |   while (p > addr) {
28 |     uint64_t marker;
29 |     PaxosTransaction txn;
30 |     uint8_t committed;
31 |     uint32_t size;
32 |     uint32_t txn_size;
33 | 
34 |     p -= sizeof(size);
35 |     size = *((uint32_t *)p);
36 |     size = ntohl(size);
37 |     p -= size;
38 | 
39 |     uint8_t* record = p;
40 |     marker = *((uint64_t *)record);
41 |     record += sizeof(marker);
42 | 
43 |     txn_size = size - sizeof(marker) - sizeof(committed);
44 |     
45 |     ThriftSerializer<PaxosTransaction> s;
46 |     s.deserialize(txn, record, txn_size);
47 |     record += txn_size;
48 | 
49 |     committed = *record;
50 |     record += sizeof(committed);
51 | 
52 |     cout << "Marker: " << marker << endl;
53 |     cout << "Proposal: " << txn.proposal << endl;
54 |     cout << "Value: " << txn.value << endl;
55 |     cout << "Committed: " << (bool)committed << endl;
56 |     cout << "Txn size: " << txn_size << endl;
57 |     cout << "Size: " << size << endl;
58 | 
59 |     cout << endl;
60 |   }
61 | }
62 | 
63 | int main(int argc, char** argv) {
64 |   if (argc != 2) {
65 |     usage(argv[0]);
66 |     return -1;
67 |   }
68 | 
69 |   int fd;
70 |   if ((fd = open(argv[1], O_RDONLY)) < 0) {
71 |     cerr << "Could not open file " << argv[1] << endl;
72 |     return -1;
73 |   }
74 | 
75 |   struct stat s;
76 |   if (fstat(fd, &s) < 0) {
77 |     cerr << "Could not stat file " << argv[1] << endl;
78 |     return -1;
79 |   }
80 |   
81 |   cout << (uint32_t)s.st_size << endl;
82 | 
83 |   void* addr;
84 |   addr = mmap(NULL, s.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
85 |   if (addr == (void *)-1) {
86 |     perror("mmap failed");
87 |     return -1;
88 |   }
89 | 
90 |   dumpLogs(addr, s.st_size);
91 | 
92 |   munmap(addr, s.st_size);
93 |   close(fd);
94 | 
95 |   return 0;
96 | }
97 | 
98 | 


--------------------------------------------------------------------------------
/PaxosThriftPeer.cpp:
--------------------------------------------------------------------------------
 1 | #include "PaxosThriftPeer.h"
 2 | 
 3 | #include <sys/socket.h>
 4 | 
 5 | using namespace apache::thrift;
 6 | using namespace apache::thrift::transport;
 7 | using namespace apache::thrift::protocol;
 8 | 
 9 | using namespace Paxos;
10 | 
11 | PaxosThriftPeer::PaxosThriftPeer(std::string host, int port) {
12 |   hostname_ = host;
13 |   port_ = port;
14 |   connectTimeout_ = DEFAULT_CONNECT_TIMEOUT;
15 |   readTimeout_ = DEFAULT_READ_TIMEOUT;
16 |   writeTimeout_ = DEFAULT_WRITE_TIMEOUT;
17 | 
18 |   initialize();
19 | }
20 | 
21 | // NOTE: This function is not thread safe. It must be called with the 
22 | // clientLock_ held.
23 | void PaxosThriftPeer::initialize() {
24 |   boost::shared_ptr<TSocket> socket(new TSocket(hostname_, port_));
25 |   socket->open();
26 |   
27 |   socket->setConnTimeout(connectTimeout_);
28 |   socket->setRecvTimeout(readTimeout_);
29 |   socket->setSendTimeout(writeTimeout_);
30 |   
31 |   boost::shared_ptr<TTransport> transport(new TFramedTransport(socket));
32 |   boost::shared_ptr<TProtocol> protocol(new TBinaryProtocol(transport));
33 | 
34 |   client_.reset(new PaxosServiceClient(protocol));
35 | }
36 | 
37 | PaxosThriftPeer::~PaxosThriftPeer() {
38 |   client_->getInputProtocol()->getTransport()->close();
39 | }
40 | 
41 | void PaxosThriftPeer::sendPropose(const PaxosProposeArgs& args, PaxosProposeResult& res) {
42 |   std::lock_guard<std::mutex> g(clientLock_);
43 |   
44 |   for (int i = 0; i < MAX_RETRIES; ++i) {
45 |     try {
46 |       client_->propose(res, args);
47 |     } catch (...) {
48 |       if (i == MAX_RETRIES - 1) {
49 |         throw;
50 |       } else {
51 |         initialize();
52 |       }
53 |       continue;
54 |     }
55 |     break;
56 |   }
57 | }
58 | 
59 | void PaxosThriftPeer::sendAccept(const PaxosAcceptArgs& args, PaxosAcceptResult& res) {
60 |   std::lock_guard<std::mutex> g(clientLock_);
61 |   for (int i = 0; i < MAX_RETRIES; ++i) {
62 |     try {
63 |       client_->accept(res, args);
64 |     } catch (...) {
65 |       if (i == MAX_RETRIES - 1) {
66 |         throw;
67 |       } else {
68 |         initialize();
69 |       }
70 |       continue;
71 |     }
72 |     break;
73 |   }
74 | }
75 | 
76 | void PaxosThriftPeer::sendLearn(const std::string& val) {
77 |   std::lock_guard<std::mutex> g(clientLock_);
78 |   for (int i = 0; i < MAX_RETRIES; ++i) {
79 |     try {
80 |       client_->learn(val);
81 |     } catch (...) {
82 |       if (i == MAX_RETRIES - 1) {
83 |         throw;
84 |       } else {
85 |         initialize();
86 |       }
87 |       continue;
88 |     }
89 |     break;
90 |   }
91 | }
92 | 
93 | int64_t PaxosThriftPeer::getHighestProposalSeen() {
94 |   std::lock_guard<std::mutex> g(clientLock_);
95 |   return client_->getHighestProposalSeen();
96 | }
97 | 


--------------------------------------------------------------------------------
/tests/PaxosBrainTest.cpp:
--------------------------------------------------------------------------------
  1 | #include "gtest/gtest.h"
  2 | #include "gmock/gmock.h"
  3 | 
  4 | #include "PaxosBrain.h"
  5 | #include "MockPaxosLearner.h"
  6 | #include "MockPaxosState.h"
  7 | 
  8 | using namespace std;
  9 | using namespace testing;
 10 | using namespace Paxos;
 11 | 
 12 | TEST(SimpleProposeTest, PaxosBrainTest) {
 13 |   PaxosProposeArgs args;
 14 |   args.proposal = 1;
 15 | 
 16 |   MockPaxosLearner learner;
 17 |   MockPaxosState* state_ptr = new MockPaxosState();
 18 |   shared_ptr<PaxosStateIf> state(state_ptr);
 19 |   PaxosBrain brain(state, learner);
 20 | 
 21 |   EXPECT_CALL(*state_ptr, getHighestProposalSeen())
 22 |     .Times(1)
 23 |     .WillOnce(Return(0));
 24 | 
 25 |   EXPECT_CALL(*state_ptr, setHighestProposalSeen(args.proposal))
 26 |     .Times(1);
 27 | 
 28 |   PaxosProposeResult res = brain.recvPropose(args);
 29 | 
 30 |   EXPECT_EQ(PaxosProposeStatus::PROMISE, res.status);
 31 | }
 32 | 
 33 | TEST(HigherProposeSeenTest, PaxosBrainTest) {
 34 |   PaxosProposeArgs args;
 35 |   args.proposal = 1;
 36 | 
 37 |   MockPaxosLearner learner;
 38 |   MockPaxosState* state_ptr = new MockPaxosState();
 39 |   shared_ptr<PaxosStateIf> state(state_ptr);
 40 |   PaxosBrain brain(state, learner);
 41 | 
 42 |   EXPECT_CALL(*state_ptr, getHighestProposalSeen())
 43 |     .Times(1)
 44 |     .WillOnce(Return(2));
 45 | 
 46 |   PaxosProposeResult res = brain.recvPropose(args);
 47 | 
 48 |   EXPECT_EQ(PaxosProposeStatus::PROMISED_HIGHER_VERSION, res.status);
 49 |   EXPECT_EQ(2, res.higherProposal);
 50 | }
 51 | 
 52 | TEST(UncommittedTransactionTest, PaxosBrainTest) {
 53 |   PaxosProposeArgs args;
 54 |   args.proposal = 2;
 55 | 
 56 |   PaxosTransaction txn;
 57 |   txn.proposal = 1;
 58 |   txn.value = "haha";
 59 | 
 60 |   MockPaxosLearner learner;
 61 |   MockPaxosState* state_ptr = new MockPaxosState();
 62 |   shared_ptr<PaxosStateIf> state(state_ptr);
 63 |   PaxosBrain brain(state, learner);
 64 | 
 65 |   EXPECT_CALL(*state_ptr, getHighestProposalSeen())
 66 |     .Times(1)
 67 |     .WillOnce(Return(1));
 68 | 
 69 |   EXPECT_CALL(*state_ptr, isTransactionInProgress())
 70 |     .Times(1)
 71 |     .WillOnce(Return(true));
 72 | 
 73 |   EXPECT_CALL(*state_ptr, getPendingTransaction())
 74 |     .Times(1)
 75 |     .WillOnce(Return(txn));
 76 | 
 77 |   PaxosProposeResult res = brain.recvPropose(args);
 78 | 
 79 |   EXPECT_EQ(PaxosProposeStatus::HAS_UNFINISHED_TRANSACTION, res.status);
 80 |   EXPECT_EQ(1, res.pendingTxn.proposal);
 81 |   EXPECT_EQ(0, txn.value.compare(res.pendingTxn.value));
 82 | }
 83 | 
 84 | TEST(AcceptFailTest, PaxosBrainTest) {
 85 |   PaxosAcceptArgs args;
 86 |   args.txn.proposal = 1;
 87 |   args.txn.value = "haha";
 88 | 
 89 |   MockPaxosLearner learner;
 90 |   MockPaxosState* state_ptr = new MockPaxosState();
 91 |   shared_ptr<PaxosStateIf> state(state_ptr);
 92 |   PaxosBrain brain(state, learner);
 93 | 
 94 |   EXPECT_CALL(*state_ptr, getHighestProposalSeen())
 95 |     .Times(1)
 96 |     .WillOnce(Return(2));
 97 | 
 98 |   PaxosAcceptResult res = brain.recvAccept(args);
 99 | 
100 |   EXPECT_EQ(PaxosAcceptStatus::REJECTED, res.status);
101 | }
102 | 
103 | TEST(AcceptSuccessTest, PaxosBrainTest) {
104 |   PaxosAcceptArgs args;
105 |   args.txn.proposal = 1;
106 |   args.txn.value = "haha";
107 | 
108 |   MockPaxosLearner learner;
109 |   MockPaxosState* state_ptr = new MockPaxosState();
110 |   shared_ptr<PaxosStateIf> state(state_ptr);
111 |   PaxosBrain brain(state, learner);
112 | 
113 |   EXPECT_CALL(*state_ptr, getHighestProposalSeen())
114 |     .Times(1)
115 |     .WillOnce(Return(args.txn.proposal));
116 | 
117 |   EXPECT_CALL(*state_ptr, setPendingTransaction(args.txn))
118 |     .Times(1);
119 | 
120 |   PaxosAcceptResult res = brain.recvAccept(args);
121 | 
122 |   EXPECT_EQ(PaxosAcceptStatus::ACCEPTED, res.status);
123 | }
124 | 
125 | int main(int argc, char **argv) {
126 |   ::testing::InitGoogleTest(&argc, argv);
127 |   return RUN_ALL_TESTS();
128 | }
129 | 
130 | 


--------------------------------------------------------------------------------
/tests/PaxosFileLoggerTest.cpp:
--------------------------------------------------------------------------------
  1 | #include "MockPosixOps.h"
  2 | #include "PaxosFileLogger.h"
  3 | 
  4 | using namespace testing;
  5 | using namespace Paxos;
  6 | 
  7 | typedef int CopyBufferMethod(int, void*, unsigned long);
  8 | 
  9 | class CopyBufferAction : public ActionInterface<CopyBufferMethod> {
 10 | public:
 11 |   CopyBufferAction(void* buf) {
 12 |     buf_ = buf;
 13 |   }
 14 | 
 15 |   virtual int Perform(const std::tr1::tuple<int, void*, unsigned long>& args) {
 16 |     void* dest = std::tr1::get<1>(args);
 17 |     int len = std::tr1::get<2>(args);
 18 | 
 19 |     memcpy(dest, buf_, len);
 20 |     return len;
 21 |   }
 22 | 
 23 | private:
 24 |   void* buf_;
 25 | };
 26 | 
 27 | Action<CopyBufferMethod> CopyBuffer(void* buf) {
 28 |   return MakeAction(new CopyBufferAction(buf));
 29 | }
 30 | 
 31 | TEST(ConstructorMkdirFailTest, PaxosFileLoggerTest) {
 32 |   MockPosixOps posix;
 33 | 
 34 |   EXPECT_CALL(posix, mkdir(StrEq("/tmp/paxos"), _))
 35 |     .Times(1)
 36 |     .WillOnce(Return(-1));
 37 | 
 38 |   EXPECT_THROW(PaxosFileLogger logger("/tmp/paxos", "log_", &posix), 
 39 |     PaxosStateLoggerException);
 40 | }
 41 | 
 42 | TEST(ConstructorMkdirExistsTest, PaxosFileLoggerTest) {
 43 |   MockPosixOps posix;
 44 | 
 45 |   EXPECT_CALL(posix, mkdir(StrEq("/tmp/paxos"), _))
 46 |     .Times(1)
 47 |     .WillOnce(Return(-1));
 48 | 
 49 |   EXPECT_CALL(posix, getErrno())
 50 |     .Times(1)
 51 |     .WillOnce(Return(EEXIST));
 52 | 
 53 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/CurrentFile"), O_RDWR | O_CREAT, _))
 54 |     .Times(1)
 55 |     .WillOnce(Return(-1));
 56 | 
 57 |   EXPECT_THROW(PaxosFileLogger logger("/tmp/paxos", "log_", &posix), 
 58 |     PaxosStateLoggerException);
 59 | }
 60 | 
 61 | TEST(ConstructorOpenFailTest, PaxosFileLoggerTest) {
 62 |   MockPosixOps posix;
 63 | 
 64 |   EXPECT_CALL(posix, mkdir(StrEq("/tmp/paxos"), _))
 65 |     .Times(1)
 66 |     .WillOnce(Return(3));
 67 | 
 68 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/CurrentFile"), O_RDWR | O_CREAT, _))
 69 |     .Times(1)
 70 |     .WillOnce(Return(-1));
 71 | 
 72 |   EXPECT_THROW(PaxosFileLogger logger("/tmp/paxos", "log_", &posix), 
 73 |     PaxosStateLoggerException);
 74 | }
 75 | 
 76 | 
 77 | TEST(ConstructorReadFailTest, PaxosFileLoggerTest) {
 78 |   MockPosixOps posix;
 79 | 
 80 |   EXPECT_CALL(posix, mkdir(StrEq("/tmp/paxos"), _))
 81 |     .Times(1)
 82 |     .WillOnce(Return(3));
 83 | 
 84 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/CurrentFile"), O_RDWR | O_CREAT, _))
 85 |     .Times(1)
 86 |     .WillOnce(Return(4));
 87 | 
 88 |   EXPECT_CALL(posix, read(4, _, _))
 89 |     .Times(1)
 90 |     .WillOnce(Return(-1));
 91 | 
 92 |   EXPECT_THROW(PaxosFileLogger logger("/tmp/paxos", "log_", &posix), 
 93 |     PaxosStateLoggerException);
 94 | }
 95 | 
 96 | 
 97 | int writePtrMatcher(void* buf) {
 98 |   return !memcmp(buf, (void*)"0000", 5);
 99 | }
100 | 
101 | TEST(ConstructorSuccessTest, PaxosFileLoggerTest) {
102 |   MockPosixOps posix;
103 | 
104 |   EXPECT_CALL(posix, mkdir(StrEq("/tmp/paxos"), _))
105 |     .Times(1)
106 |     .WillOnce(Return(3));
107 | 
108 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/CurrentFile"), O_RDWR | O_CREAT, _))
109 |     .Times(1)
110 |     .WillOnce(Return(4));
111 | 
112 |   char buf[] = "0000";
113 |   EXPECT_CALL(posix, read(4, _, _))
114 |     .Times(1)
115 |     .WillOnce(CopyBuffer(buf));
116 | 
117 |   EXPECT_CALL(posix, close(4))
118 |     .Times(2);
119 | 
120 |   EXPECT_CALL(posix, close(-1))
121 |     .Times(1);
122 | 
123 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/CurrentFile"), O_WRONLY, _))
124 |     .Times(1)
125 |     .WillOnce(Return(4));
126 | 
127 |   EXPECT_CALL(posix, write(4, Truly(writePtrMatcher), 5))
128 |     .Times(1)
129 |     .WillOnce(Return(5));
130 | 
131 |   EXPECT_CALL(posix, open(StrEq("/tmp/paxos/log__0000"), O_CREAT | O_RDWR, _))
132 |     .Times(1)
133 |     .WillOnce(Return(5));
134 | 
135 |   // Destructor will close it
136 |   EXPECT_CALL(posix, close(5))
137 |     .Times(1);
138 | 
139 |   PaxosFileLogger logger("/tmp/paxos", "log_", &posix);
140 | }
141 | int main(int argc, char** argv) {
142 |   ::testing::InitGoogleTest(&argc, argv);
143 | 
144 |   return RUN_ALL_TESTS();
145 | }
146 | 
147 | 


--------------------------------------------------------------------------------
/PaxosClient.cpp:
--------------------------------------------------------------------------------
  1 | #include "PaxosClient.h"
  2 | using namespace std;
  3 | using namespace Paxos;
  4 | 
  5 | PaxosClient::PaxosClient(vector<PaxosPeer *>& peers) {
  6 |   peers_ = peers;
  7 |   highestProposalSeen_ = 0;
  8 |   maxTriesPerSubmit_ = MAX_RETRIES;
  9 | }
 10 | 
 11 | void PaxosClient::initialize() {
 12 |   std::lock_guard<std::mutex> g(clientLock_);
 13 |   for (auto i = peers_.begin(); i != peers_.end(); ++i) {
 14 |     (*i)->initialize();
 15 |     
 16 |     int highestProposalSeen = (*i)->getHighestProposalSeen();
 17 |     if (highestProposalSeen > highestProposalSeen_) {
 18 |       highestProposalSeen_ = highestProposalSeen;
 19 |     }
 20 |   }
 21 | }
 22 | 
 23 | void PaxosClient::setMaxRetries(int max_retries) {
 24 |   maxTriesPerSubmit_ = max_retries + 1;
 25 | }
 26 | 
 27 | bool PaxosClient::sendAccept(PaxosTransaction& p) {
 28 |   PaxosAcceptArgs args;
 29 |   args.txn = p;
 30 |   std::vector<PaxosAcceptResult> a_res;
 31 | 
 32 |   for (auto i = peers_.begin(); i < peers_.end(); ++i) {
 33 |     try {
 34 |       PaxosAcceptResult par;
 35 |       (*i)->sendAccept(args, par);
 36 |       a_res.push_back(par);
 37 |     } catch (...) {
 38 |       continue;
 39 |     }
 40 |   }
 41 | 
 42 |   if (a_res.size() <= peers_.size() / 2) {
 43 |     return false;
 44 |   }
 45 | 
 46 |   for (auto i = a_res.begin(); i < a_res.end(); ++i) {
 47 |     if (i->status == PaxosAcceptStatus::REJECTED) {
 48 |       return false;
 49 |     }
 50 |   }
 51 | 
 52 |   return true;
 53 | }
 54 | 
 55 | void PaxosClient::sendLearn(std::string& val) {
 56 |   for (auto i = peers_.begin(); i < peers_.end(); ++i) {
 57 |     try {
 58 |       (*i)->sendLearn(val);
 59 |     } catch (...) {
 60 |       continue;
 61 |     }
 62 |   }
 63 | }
 64 | 
 65 | bool PaxosClient::submit(std::string& val) {
 66 |   bool success = false;
 67 |   int tries = 0;
 68 | 
 69 |   while (!success && tries < maxTriesPerSubmit_) {
 70 |     ++tries;
 71 |     PaxosProposeArgs p_args;
 72 | 
 73 |     p_args.proposal = ++highestProposalSeen_;
 74 | 
 75 |     std::vector<PaxosProposeResult> p_res;
 76 |     for (auto i = peers_.begin(); i != peers_.end(); ++i) {
 77 |       try {
 78 |         PaxosProposeResult ppr;
 79 |         (*i)->sendPropose(p_args, ppr);
 80 |         p_res.push_back(ppr);
 81 |       } catch (...) {
 82 |         continue;
 83 |       }
 84 |     }
 85 | 
 86 |     // Did enough peers respond?
 87 |     if (p_res.size() <= peers_.size() / 2) {
 88 |       // Nope... do over!
 89 |       continue;
 90 |     }
 91 | 
 92 |     unsigned int numPromises = 0;
 93 |     bool shouldSendAccept = true;
 94 |     bool isReplaying = false;
 95 |     PaxosTransaction highestPendingTxn;
 96 |     highestPendingTxn.proposal = -1;
 97 |     for (auto i = p_res.begin(); i != p_res.end(); ++i) {
 98 |       if (i->status == PaxosProposeStatus::PROMISE) {
 99 |         numPromises++;
100 |         continue;
101 |       } else if (i->status == PaxosProposeStatus::PROMISED_HIGHER_VERSION) {
102 |         // Someone had a higher proposal. Bail...
103 |         shouldSendAccept = false;
104 |         if (highestProposalSeen_ < i->higherProposal) {
105 |           // This can happen if multiple client respond with 
106 |           // PROMISED_HIGHER_VERSION
107 |           highestProposalSeen_ = i->higherProposal;
108 |         }
109 |         break;
110 |       } else {
111 |         // Some peer had an uncommitted transaction
112 |         if (i->pendingTxn.proposal > highestPendingTxn.proposal) {
113 |           highestPendingTxn = i->pendingTxn;
114 |         }
115 | 
116 |         isReplaying = true;
117 |       }
118 |     }
119 | 
120 |     if (!shouldSendAccept) {
121 |       // There is a higher proposal than ours. Do over
122 |       continue;
123 |     }
124 | 
125 |     if (highestPendingTxn.proposal == -1) {
126 |       // No pending proposals!
127 |       highestPendingTxn.proposal = p_args.proposal;
128 |       highestPendingTxn.value = val;
129 |     }
130 |       
131 |     bool status = sendAccept(highestPendingTxn);
132 |   
133 |     if (status) {
134 |       sendLearn(highestPendingTxn.value);
135 |     }
136 | 
137 |     if (isReplaying) {
138 |       // We recovered a previous transaction, so success stays false; Do over
139 |       isReplaying = false;
140 |     } else {
141 |       if (status && highestPendingTxn.proposal == p_args.proposal) {
142 |         // We successfully agreed upon our value
143 |         success = true;
144 |       }
145 |     }
146 |   }
147 | 
148 |   return success;
149 | }
150 | 
151 | 


--------------------------------------------------------------------------------
/PaxosFileLogger.cpp:
--------------------------------------------------------------------------------
  1 | #include "PaxosFileLogger.h"
  2 | #include "PaxosStateLoggerException.h"
  3 | #include "util/PosixOps.h"
  4 | 
  5 | #include <sys/types.h>
  6 | #include <sys/stat.h>
  7 | #include <fcntl.h>
  8 | #include <stdio.h>
  9 | 
 10 | #include <iostream>
 11 | #include <cassert>
 12 | #include <memory>
 13 | 
 14 | using namespace std;
 15 | using namespace Paxos;
 16 | 
 17 | PaxosFileLogger::PaxosFileLogger(const string dir, 
 18 |     const string prefix,
 19 |     PosixOpsIf* posix) :
 20 |   dir_(dir), prefix_(prefix) {
 21 |   currentFile_ = "";
 22 |   currentFd_ = -1;
 23 |   currentSuffix_ = 0;
 24 |   maxSize_ = MAX_FILE_SIZE;
 25 | 
 26 |   if (posix) {
 27 |     posix_ = posix;
 28 |     shouldDeallocatePosix_ = false;
 29 |   } else {
 30 |     posix_ = new PosixOps();
 31 |     shouldDeallocatePosix_ = true;
 32 |   }
 33 | 
 34 |   openLatestLogFile();
 35 | }
 36 | 
 37 | void PaxosFileLogger::openLatestLogFile() {
 38 |   // Create the directory if it doesn't exist
 39 |   int ret = posix_->mkdir(dir_.c_str());
 40 |   if (ret < 0 && posix_->getErrno() != EEXIST) {
 41 |     perror(__FILE__);
 42 |     throw PaxosStateLoggerException(
 43 |       PaxosStateLoggerException::LOG_CORRUPT,
 44 |       "Could not create directory " + dir_);
 45 |   }
 46 |    
 47 |   currentSuffix_ = getLatestLogFile(); 
 48 |   openLogFile(currentSuffix_);
 49 | }
 50 | 
 51 | unsigned long PaxosFileLogger::getLatestLogFile() {
 52 |   string curFile = dir_ + "/CurrentFile";
 53 | 
 54 |   // Open CurrentFile; create if it doesn't exist
 55 |   int fd = posix_->open(curFile.c_str(), O_CREAT | O_RDWR);
 56 |   if (fd < 0) {
 57 |     perror(__FILE__);
 58 |     throw PaxosStateLoggerException(
 59 |       PaxosStateLoggerException::LOG_CORRUPT,
 60 |       "Could not open 'CurrentFile'");
 61 |   }
 62 | 
 63 |   // Read in the suffix
 64 |   unsigned long suffix = 0;
 65 |   char buf[8];
 66 |   int size;
 67 |   if ((size = posix_->read(fd, buf, sizeof(buf))) < 0) {
 68 |     posix_->close(fd);
 69 |     perror(__FILE__);
 70 |     throw PaxosStateLoggerException(
 71 |       PaxosStateLoggerException::LOG_CORRUPT,
 72 |       "Error reading 'CurrentFile'");
 73 |   }
 74 | 
 75 |   posix_->close(fd);
 76 | 
 77 |   if (size != 0) {
 78 |     suffix = atoi(buf);
 79 |   }
 80 | 
 81 |   return suffix;
 82 | }
 83 | 
 84 | void PaxosFileLogger::recordNextLogFile(string& suffix) {
 85 |   string curFile = dir_ + "/CurrentFile";
 86 | 
 87 |   int fd = posix_->open(curFile.c_str(), O_WRONLY);
 88 |   if (fd < 0) {
 89 |     perror(__FILE__);
 90 |     throw PaxosStateLoggerException(
 91 |       PaxosStateLoggerException::LOG_CORRUPT,
 92 |       "Could not open 'CurrentFile'");
 93 |   }
 94 | 
 95 |   // Write the name of the current log file
 96 |   if (posix_->write(fd, (void *)suffix.c_str(), suffix.size() + 1) < 0) {
 97 |     posix_->close(fd);
 98 |     perror(__FILE__);
 99 |     throw PaxosStateLoggerException(
100 |       PaxosStateLoggerException::LOG_CORRUPT,
101 |       "Error writing 'CurrentFile'");
102 |   }
103 |   
104 |   posix_->close(fd);
105 | }
106 | 
107 | void PaxosFileLogger::openLogFile(unsigned long suffix) {
108 |   char buf[8];
109 | 
110 |   sprintf(buf, "%04lu", suffix);
111 |   string str_suffix(buf);
112 | 
113 |   string filename = dir_ + "/";
114 |   filename += prefix_;
115 |   filename += "_";
116 |   filename += str_suffix;
117 | 
118 |   posix_->close(currentFd_);
119 | 
120 |   recordNextLogFile(str_suffix);
121 |  
122 |   // Open the file
123 |   currentFd_ = posix_->open(filename.c_str(), O_CREAT | O_RDWR);
124 |   if (currentFd_ < 0) {
125 |     perror(__FILE__);
126 |     throw PaxosStateLoggerException(
127 |       PaxosStateLoggerException::LOG_CORRUPT,
128 |       "Could not open log file");
129 |   }
130 | 
131 |   currentFile_ = filename;
132 | }
133 | 
134 | void PaxosFileLogger::rotateLogFileIfNeeded() {
135 |   struct stat s;
136 |   if (posix_->fstat(currentFd_, &s) < 0) {
137 |     throw PaxosStateLoggerException(
138 |       PaxosStateLoggerException::IO_ERROR,
139 |       "Could not stat current log file '" + currentFile_ + "'");
140 |   }
141 | 
142 |   if ((uint32_t)s.st_size > maxSize_) {
143 |     currentSuffix_++;
144 |     openLogFile(currentSuffix_++);
145 |   }
146 | }
147 |  
148 | void PaxosFileLogger::log(const PaxosTransaction& txn) {
149 |   uint8_t committed;
150 |   uint8_t *buf;
151 |   uint32_t buf_size;
152 |   uint32_t record_size;
153 |   uint64_t marker;
154 | 
155 |   // Rotate the log file if it's too big
156 |   rotateLogFileIfNeeded();
157 |  
158 |   // Seek to the end of the file. This is needed as commit might have
159 |   // left the file pointer before the end.
160 |   if (posix_->lseek(currentFd_, 0, SEEK_END) < 0) {
161 |     throw PaxosStateLoggerException(
162 |       PaxosStateLoggerException::IO_ERROR,
163 |       "Could not seek to the end of log file '" + 
164 |         currentFile_ + "'");
165 |   }
166 | 
167 |   ThriftSerializer<PaxosTransaction> serializer;
168 |   serializer.serialize(txn, &buf, &buf_size);
169 | 
170 |   marker = PaxosFileLogger::RECORD_MARKER;
171 |   committed = (uint8_t)txn.committed;
172 |   record_size = sizeof(marker) + sizeof(committed) + buf_size;
173 |   record_size = htonl(record_size);
174 | 
175 |   struct iovec iov[4];
176 |   iov[0].iov_base = &marker;
177 |   iov[0].iov_len = sizeof(marker);
178 | 
179 |   iov[1].iov_base = (void *)buf;
180 |   iov[1].iov_len = buf_size;  
181 |   
182 |   iov[2].iov_base = &committed;
183 |   iov[2].iov_len = sizeof(committed);
184 | 
185 |   iov[3].iov_base = &record_size;
186 |   iov[3].iov_len = sizeof(record_size);
187 | 
188 |   if (posix_->writev(currentFd_, iov, 4) < 0) {
189 |     throw PaxosStateLoggerException(
190 |       PaxosStateLoggerException::IO_ERROR,
191 |       "Could not write to log file '" + currentFile_ + "'");
192 |   }
193 | 
194 |   fsync(currentFd_);
195 | }
196 | 
197 | bool PaxosFileLogger::getLatestTransaction(PaxosTransaction& txn) {
198 |   uint32_t size = 0;
199 |   
200 |   struct stat s;
201 |   if (posix_->fstat(currentFd_, &s) < 0) {
202 |     throw PaxosStateLoggerException(
203 |       PaxosStateLoggerException::IO_ERROR,
204 |       "Could not stat current log file '" + currentFile_ + "'");
205 |   }
206 | 
207 |   if (!s.st_size) {
208 |     return false;
209 |   }
210 | 
211 |   if (posix_->lseek(currentFd_, -1 * sizeof(size), SEEK_END) < 0) {
212 |     throw PaxosStateLoggerException(
213 |       PaxosStateLoggerException::IO_ERROR,
214 |       "Could not seek to latest transaction size in log file '" + 
215 |         currentFile_ + "'");
216 |   }
217 | 
218 |   if (posix_->read(currentFd_, &size, sizeof(size)) < 0) {
219 |     throw PaxosStateLoggerException(
220 |       PaxosStateLoggerException::IO_ERROR,
221 |       "Could not read transaction size from log file '" + currentFile_ + "'");
222 |   }
223 | 
224 |   size = ntohl(size);
225 | 
226 |   assert(size);
227 | 
228 |   if (posix_->lseek(currentFd_, -1 * (size + sizeof(size)), SEEK_END) < 0) {
229 |     throw PaxosStateLoggerException(
230 |       PaxosStateLoggerException::IO_ERROR,
231 |       "Could not read latest transaction in log file '" + currentFile_ + "'");
232 |   }
233 | 
234 |   uint8_t committed;
235 |   uint32_t read_size;
236 |   uint64_t marker;
237 |   
238 |   uint32_t txn_size = size - sizeof(marker) - sizeof(committed);
239 |   auto deleter = [] (uint8_t* p) { free ((void *)p); };
240 |   unique_ptr<uint8_t, decltype(deleter)> buf((uint8_t *)malloc(txn_size), 
241 |     deleter);
242 | 
243 |   struct iovec iov[4];
244 |   iov[0].iov_base = &marker;
245 |   iov[0].iov_len = sizeof(marker);
246 | 
247 |   iov[1].iov_base = buf.get();
248 |   iov[1].iov_len = txn_size;
249 |   
250 |   iov[2].iov_base = &committed;
251 |   iov[2].iov_len = sizeof(committed);
252 |   
253 |   iov[3].iov_base = &read_size;
254 |   iov[3].iov_len = sizeof(read_size);
255 | 
256 |   if (posix_->readv(currentFd_, iov, 4) < 0) {
257 |     throw PaxosStateLoggerException(
258 |       PaxosStateLoggerException::IO_ERROR,
259 |       "Could not read transaction from log file '" + currentFile_ + "'");
260 |   }
261 | 
262 |   assert(size == ntohl(read_size)); 
263 |   assert(marker == PaxosFileLogger::RECORD_MARKER);
264 | 
265 |   ThriftSerializer<PaxosTransaction> serializer;
266 |   serializer.deserialize(txn, buf.get(), txn_size);
267 |   txn.committed = (bool)committed;
268 | 
269 |   return true;
270 | }
271 | 
272 | void PaxosFileLogger::commitLatestTransaction() {
273 |   if (posix_->lseek(currentFd_, -1 * (sizeof(uint8_t) + 
274 |     sizeof(uint32_t)), SEEK_END) < 0) {
275 |     throw PaxosStateLoggerException(
276 |       PaxosStateLoggerException::IO_ERROR,
277 |       "Could not seek to latest transaction size in log file '" + 
278 |         currentFile_ + "'");
279 |   }
280 | 
281 |   uint8_t committed = (uint8_t)true;
282 |   if (posix_->write(currentFd_, &committed, sizeof(committed)) < 0) {
283 |     throw PaxosStateLoggerException(
284 |       PaxosStateLoggerException::IO_ERROR,
285 |       "Could not write commited state to latest transaction in log file '" +
286 |         currentFile_ + "'");
287 |   }
288 | }
289 | 
290 | void PaxosFileLogger::setMaxLogFileSize(uint32_t size) {
291 |   maxSize_ = size;
292 | }
293 | 
294 | PaxosFileLogger::~PaxosFileLogger() {
295 |   posix_->close(currentFd_);
296 |   if (shouldDeallocatePosix_) {
297 |     delete posix_;
298 |   }
299 | }
300 | 
301 | 


--------------------------------------------------------------------------------
/tests/PaxosPeerTest.cpp:
--------------------------------------------------------------------------------
  1 | #include "gtest/gtest.h"
  2 | 
  3 | #include "MockPaxosPeer.h"
  4 | #include "MockPaxosLearner.h"
  5 | #include "PaxosClient.h"
  6 | 
  7 | #include <vector>
  8 | #include <string>
  9 | 
 10 | #define MAX_TRIES 3
 11 | 
 12 | using namespace std;
 13 | using namespace testing;
 14 | using namespace Paxos;
 15 | 
 16 | class BasicFixture : public Test {
 17 | protected:
 18 |   virtual void SetUpExpectations() = 0;
 19 |   virtual void SetUp() {
 20 |     SetUpExpectations();
 21 | 
 22 |     peers.push_back(&peer1);
 23 |     peers.push_back(&peer2);
 24 |     peers.push_back(&peer3);
 25 |   }
 26 | 
 27 |   MockPaxosPeer peer1;
 28 |   MockPaxosPeer peer2;
 29 |   MockPaxosPeer peer3;
 30 | 
 31 |   vector<PaxosPeer *> peers;
 32 | 
 33 | };
 34 | 
 35 | class SimpleSuccessFixture : public BasicFixture {
 36 |   virtual void SetUpExpectations() {
 37 |     PaxosProposeResult pRes;
 38 |     PaxosAcceptResult aRes;
 39 | 
 40 |     pRes.status = PaxosProposeStatus::PROMISE;
 41 | 
 42 |     EXPECT_CALL(peer1, sendPropose(_, _))
 43 |       .Times(1)
 44 |       .WillOnce(SetArgReferee<1>(pRes));
 45 | 
 46 |     EXPECT_CALL(peer2, sendPropose(_, _))
 47 |       .Times(1)
 48 |       .WillOnce(SetArgReferee<1>(pRes));
 49 | 
 50 |     EXPECT_CALL(peer3, sendPropose(_, _))
 51 |       .Times(1)
 52 |       .WillOnce(SetArgReferee<1>(pRes));
 53 | 
 54 |     aRes.status = PaxosAcceptStatus::ACCEPTED;
 55 | 
 56 |     EXPECT_CALL(peer1, sendAccept(_, _))
 57 |       .Times(1)
 58 |       .WillOnce(SetArgReferee<1>(aRes));
 59 | 
 60 |     EXPECT_CALL(peer2, sendAccept(_, _))
 61 |       .Times(1)
 62 |       .WillOnce(SetArgReferee<1>(aRes));
 63 | 
 64 |     EXPECT_CALL(peer3, sendAccept(_, _))
 65 |       .Times(1)
 66 |       .WillOnce(SetArgReferee<1>(aRes));
 67 | 
 68 |     EXPECT_CALL(peer1, sendLearn(_))
 69 |       .Times(1);
 70 | 
 71 |     EXPECT_CALL(peer2, sendLearn(_))
 72 |       .Times(1);
 73 | 
 74 |     EXPECT_CALL(peer3, sendLearn(_))
 75 |       .Times(1);
 76 |   }
 77 | };
 78 | 
 79 | class ProposeFailureFixture : public BasicFixture {
 80 |   virtual void SetUpExpectations() {
 81 |     PaxosProposeResult pRes;
 82 | 
 83 |     pRes.status = PaxosProposeStatus::PROMISED_HIGHER_VERSION;
 84 | 
 85 |     EXPECT_CALL(peer1, sendPropose(_, _))
 86 |       .Times(MAX_TRIES)
 87 |       .WillRepeatedly(SetArgReferee<1>(pRes));
 88 | 
 89 |     EXPECT_CALL(peer2, sendPropose(_, _))
 90 |       .Times(MAX_TRIES)
 91 |       .WillRepeatedly(SetArgReferee<1>(pRes));
 92 | 
 93 |     EXPECT_CALL(peer3, sendPropose(_, _))
 94 |       .Times(MAX_TRIES)
 95 |       .WillRepeatedly(SetArgReferee<1>(pRes));
 96 | 
 97 |     EXPECT_CALL(peer1, sendAccept(_, _))
 98 |       .Times(0);
 99 | 
100 |     EXPECT_CALL(peer2, sendAccept(_, _))
101 |       .Times(0);
102 | 
103 |     EXPECT_CALL(peer3, sendAccept(_, _))
104 |       .Times(0);
105 | 
106 |     EXPECT_CALL(peer1, sendLearn(_))
107 |       .Times(0);
108 | 
109 |     EXPECT_CALL(peer2, sendLearn(_))
110 |       .Times(0);
111 | 
112 |     EXPECT_CALL(peer3, sendLearn(_))
113 |       .Times(0);
114 |   }
115 | };
116 | 
117 | class AcceptFailureFixture : public BasicFixture {
118 |   virtual void SetUpExpectations() {
119 |     PaxosProposeResult pRes;
120 |     PaxosAcceptResult aRes;
121 | 
122 |     pRes.status = PaxosProposeStatus::PROMISE;
123 | 
124 |     EXPECT_CALL(peer1, sendPropose(_, _))
125 |       .Times(MAX_TRIES)
126 |       .WillRepeatedly(SetArgReferee<1>(pRes));
127 | 
128 |     EXPECT_CALL(peer2, sendPropose(_, _))
129 |       .Times(MAX_TRIES)
130 |       .WillRepeatedly(SetArgReferee<1>(pRes));
131 | 
132 |     EXPECT_CALL(peer3, sendPropose(_, _))
133 |       .Times(MAX_TRIES)
134 |       .WillRepeatedly(SetArgReferee<1>(pRes));
135 | 
136 |     aRes.status = PaxosAcceptStatus::REJECTED;
137 | 
138 |     EXPECT_CALL(peer1, sendAccept(_, _))
139 |       .Times(MAX_TRIES)
140 |       .WillRepeatedly(SetArgReferee<1>(aRes));
141 | 
142 |     EXPECT_CALL(peer2, sendAccept(_, _))
143 |       .Times(MAX_TRIES)
144 |       .WillRepeatedly(SetArgReferee<1>(aRes));
145 | 
146 |     EXPECT_CALL(peer3, sendAccept(_, _))
147 |       .Times(MAX_TRIES)
148 |       .WillRepeatedly(SetArgReferee<1>(aRes));
149 | 
150 |     EXPECT_CALL(peer1, sendLearn(_))
151 |       .Times(0);
152 | 
153 |     EXPECT_CALL(peer2, sendLearn(_))
154 |       .Times(0);
155 | 
156 |     EXPECT_CALL(peer3, sendLearn(_))
157 |       .Times(0);
158 |   }
159 | };
160 | 
161 | class PartialProposeFailureFixture : public BasicFixture {
162 |   virtual void SetUpExpectations() {
163 |     PaxosProposeResult pFailRes;
164 |     PaxosProposeResult pSuccessRes;
165 | 
166 |     pFailRes.status = PaxosProposeStatus::PROMISED_HIGHER_VERSION;
167 |     pSuccessRes.status = PaxosProposeStatus::PROMISE;
168 | 
169 |     EXPECT_CALL(peer1, sendPropose(_, _))
170 |       .Times(MAX_TRIES)
171 |       .WillRepeatedly(SetArgReferee<1>(pSuccessRes));
172 | 
173 |     EXPECT_CALL(peer2, sendPropose(_, _))
174 |       .Times(MAX_TRIES)
175 |       .WillRepeatedly(SetArgReferee<1>(pFailRes));
176 | 
177 |     EXPECT_CALL(peer3, sendPropose(_, _))
178 |       .Times(MAX_TRIES)
179 |       .WillRepeatedly(SetArgReferee<1>(pFailRes));
180 | 
181 |     EXPECT_CALL(peer1, sendAccept(_, _))
182 |       .Times(0);
183 | 
184 |     EXPECT_CALL(peer2, sendAccept(_, _))
185 |       .Times(0);
186 | 
187 |     EXPECT_CALL(peer3, sendAccept(_, _))
188 |       .Times(0);
189 | 
190 |     EXPECT_CALL(peer1, sendLearn(_))
191 |       .Times(0);
192 | 
193 |     EXPECT_CALL(peer2, sendLearn(_))
194 |       .Times(0);
195 | 
196 |     EXPECT_CALL(peer3, sendLearn(_))
197 |       .Times(0);
198 |   }
199 | };
200 | 
201 | class HigherVersionFailureFixture : public BasicFixture {
202 |   virtual void SetUpExpectations() {
203 |     PaxosProposeResult pFailRes;
204 |     PaxosProposeResult pSuccessRes;
205 | 
206 |     pFailRes.status = PaxosProposeStatus::PROMISED_HIGHER_VERSION;
207 |     pSuccessRes.status = PaxosProposeStatus::PROMISE;
208 | 
209 |     EXPECT_CALL(peer1, sendPropose(_, _))
210 |       .Times(1)
211 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
212 | 
213 |     EXPECT_CALL(peer2, sendPropose(_, _))
214 |       .Times(1)
215 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
216 | 
217 |     EXPECT_CALL(peer3, sendPropose(_, _))
218 |       .Times(1)
219 |       .WillOnce(SetArgReferee<1>(pFailRes));
220 | 
221 |     PaxosAcceptResult aRes;
222 |     aRes.status = PaxosAcceptStatus::REJECTED;
223 | 
224 |     EXPECT_CALL(peer1, sendAccept(_, _))
225 |       .Times(0);
226 | 
227 |     EXPECT_CALL(peer2, sendAccept(_, _))
228 |       .Times(0);
229 | 
230 |     EXPECT_CALL(peer3, sendAccept(_, _))
231 |       .Times(0);
232 | 
233 |     EXPECT_CALL(peer1, sendLearn(_))
234 |       .Times(0);
235 | 
236 |     EXPECT_CALL(peer2, sendLearn(_))
237 |       .Times(0);
238 | 
239 |     EXPECT_CALL(peer3, sendLearn(_))
240 |       .Times(0);
241 |   }
242 | };
243 | 
244 | class PendingTransactionFixture : public BasicFixture {
245 |   virtual void SetUpExpectations() {
246 |     PaxosProposeResult pFailRes;
247 |     PaxosProposeResult pSuccessRes;
248 | 
249 |     PaxosTransaction txn;
250 |     txn.proposal = 1;
251 |     txn.value = "haha0";
252 | 
253 |     pFailRes.status = PaxosProposeStatus::HAS_UNFINISHED_TRANSACTION;
254 |     pFailRes.pendingTxn = txn;
255 | 
256 |     pSuccessRes.status = PaxosProposeStatus::PROMISE;
257 | 
258 |     EXPECT_CALL(peer1, sendPropose(_, _))
259 |       .Times(2)
260 |       .WillOnce(SetArgReferee<1>(pFailRes))
261 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
262 | 
263 |     EXPECT_CALL(peer2, sendPropose(_, _))
264 |       .Times(2)
265 |       .WillOnce(SetArgReferee<1>(pFailRes))
266 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
267 | 
268 |     EXPECT_CALL(peer3, sendPropose(_, _))
269 |       .Times(2)
270 |       .WillOnce(SetArgReferee<1>(pFailRes))
271 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
272 | 
273 |     PaxosAcceptResult aSuccessRes;
274 |     aSuccessRes.status = PaxosAcceptStatus::ACCEPTED;
275 | 
276 |     EXPECT_CALL(peer1, sendAccept(_, _))
277 |       .Times(2)
278 |       .WillRepeatedly(SetArgReferee<1>(aSuccessRes));
279 | 
280 |     EXPECT_CALL(peer2, sendAccept(_, _))
281 |       .Times(2)
282 |       .WillRepeatedly(SetArgReferee<1>(aSuccessRes));
283 | 
284 |     EXPECT_CALL(peer3, sendAccept(_, _))
285 |       .Times(2)
286 |       .WillRepeatedly(SetArgReferee<1>(aSuccessRes));
287 | 
288 |     EXPECT_CALL(peer1, sendLearn(_))
289 |       .Times(2);
290 | 
291 |     EXPECT_CALL(peer2, sendLearn(_))
292 |       .Times(2);
293 | 
294 |     EXPECT_CALL(peer3, sendLearn(_))
295 |       .Times(2);
296 |   }
297 | };
298 | 
299 | class PendingTransactionFailureFixture : public BasicFixture {
300 |   virtual void SetUpExpectations() {
301 |     PaxosProposeResult pFailRes;
302 |     PaxosProposeResult pSuccessRes;
303 | 
304 |     PaxosTransaction txn;
305 |     txn.proposal = 1;
306 |     txn.value = "haha0";
307 | 
308 |     pFailRes.status = PaxosProposeStatus::HAS_UNFINISHED_TRANSACTION;
309 |     pFailRes.pendingTxn = txn;
310 | 
311 |     pSuccessRes.status = PaxosProposeStatus::PROMISE;
312 | 
313 |     EXPECT_CALL(peer1, sendPropose(_, _))
314 |       .Times(2)
315 |       .WillRepeatedly(SetArgReferee<1>(pSuccessRes));
316 | 
317 |     EXPECT_CALL(peer2, sendPropose(_, _))
318 |       .Times(2)
319 |       .WillRepeatedly(SetArgReferee<1>(pSuccessRes));
320 | 
321 |     EXPECT_CALL(peer3, sendPropose(_, _))
322 |       .Times(2)
323 |       .WillOnce(SetArgReferee<1>(pFailRes))
324 |       .WillOnce(SetArgReferee<1>(pSuccessRes));
325 | 
326 |     PaxosAcceptResult aSuccessRes;
327 |     aSuccessRes.status = PaxosAcceptStatus::ACCEPTED;
328 | 
329 |     PaxosAcceptResult aFailRes;
330 |     aFailRes.status = PaxosAcceptStatus::REJECTED;
331 | 
332 |     EXPECT_CALL(peer1, sendAccept(_, _))
333 |       .Times(2)
334 |       .WillOnce(SetArgReferee<1>(aFailRes))
335 |       .WillOnce(SetArgReferee<1>(aSuccessRes));
336 | 
337 |     EXPECT_CALL(peer2, sendAccept(_, _))
338 |       .Times(2)
339 |       .WillOnce(SetArgReferee<1>(aFailRes))
340 |       .WillOnce(SetArgReferee<1>(aSuccessRes));
341 | 
342 |     EXPECT_CALL(peer3, sendAccept(_, _))
343 |       .Times(2)
344 |       .WillRepeatedly(SetArgReferee<1>(aSuccessRes));
345 | 
346 |     EXPECT_CALL(peer1, sendLearn(StrEq("haha")))
347 |       .Times(1);
348 | 
349 |     EXPECT_CALL(peer2, sendLearn(StrEq("haha")))
350 |       .Times(1);
351 | 
352 |     EXPECT_CALL(peer3, sendLearn(StrEq("haha")))
353 |       .Times(1);
354 |   }
355 | };
356 | 
357 | TEST_F(SimpleSuccessFixture, BasicTest) {
358 |   string testString = "haha";
359 | 
360 |   PaxosClient brain(peers);
361 |   EXPECT_TRUE(brain.submit(testString));
362 | }
363 | 
364 | TEST_F(ProposeFailureFixture, BasicTest) {
365 |   string testString = "haha";
366 | 
367 |   PaxosClient brain(peers);
368 |   brain.setMaxRetries(MAX_TRIES - 1);
369 |   EXPECT_FALSE(brain.submit(testString));
370 | }
371 | 
372 | TEST_F(AcceptFailureFixture, BasicTest) {
373 |   string testString = "haha";
374 | 
375 |   PaxosClient brain(peers);
376 |   brain.setMaxRetries(MAX_TRIES - 1);
377 |   EXPECT_FALSE(brain.submit(testString));
378 | }
379 | 
380 | TEST_F(PartialProposeFailureFixture, BasicTest) {
381 |   string testString = "haha";
382 | 
383 |   PaxosClient brain(peers);
384 |   brain.setMaxRetries(MAX_TRIES - 1);
385 |   EXPECT_FALSE(brain.submit(testString));
386 | }
387 | 
388 | TEST_F(HigherVersionFailureFixture, BasicTest) {
389 |   string testString = "haha";
390 | 
391 |   PaxosClient brain(peers);
392 |   brain.setMaxRetries(0);
393 |   EXPECT_FALSE(brain.submit(testString));
394 | }
395 | 
396 | TEST_F(PendingTransactionFixture, BasicTest) {
397 |   string testString = "haha1";
398 | 
399 |   PaxosClient brain(peers);
400 |   EXPECT_TRUE(brain.submit(testString));
401 | }
402 | 
403 | TEST_F(PendingTransactionFailureFixture, BasicTest) {
404 |   string testString = "haha";
405 | 
406 |   PaxosClient brain(peers);
407 |   EXPECT_TRUE(brain.submit(testString));
408 | }
409 | 
410 | int main(int argc, char **argv) {
411 |   ::testing::InitGoogleTest(&argc, argv);
412 |   return RUN_ALL_TESTS();
413 | }
414 | 
415 | 


--------------------------------------------------------------------------------