├── .gitignore ├── apps └── memkv │ ├── README.md │ ├── compile_proto.sh │ ├── src │ ├── logging.h │ ├── slice.h │ ├── CMakeLists.txt │ ├── status.cc │ ├── status.h │ ├── memkv_store.h │ ├── pb │ │ └── memkv.proto │ ├── memkv_service.h │ ├── db.h │ ├── testing.h │ ├── memkv_server.cc │ ├── memkv_store_test.cc │ ├── memkv_service.cc │ ├── memkv_store.cc │ └── db.cc │ ├── CMakeLists.txt │ └── run.sh ├── .travis.yml ├── compile_proto.sh ├── .gitmodules ├── clang-format.sh ├── include └── consensus │ ├── base │ ├── slice.h │ ├── endianness.h │ ├── glog_logger.h │ ├── errno.h │ ├── task_queue.h │ ├── env_util.h │ ├── background_worker.h │ ├── random.h │ ├── status.h │ ├── simple_channel.h │ ├── logging.h │ ├── stl_container_utils.h │ ├── testing.h │ └── coding.h │ ├── rpc │ └── cluster.h │ ├── ready_flusher.h │ ├── raft_timer.h │ ├── pb │ └── raft_server.proto │ ├── raft_service.h │ ├── wal │ └── wal.h │ └── replicated_log.h ├── install_dependencies.sh ├── src ├── base │ ├── endianness.cc │ ├── port.h │ ├── random.cc │ ├── errno.cc │ ├── glog_logger.cc │ ├── status.cc │ ├── background_worker_test.cc │ ├── task_queue.cc │ ├── testing.cc │ ├── mock_env.h │ ├── env_util.cc │ ├── random_test.cc │ ├── background_worker.cc │ ├── env_test.cc │ └── coding.cc ├── wal │ ├── segment_meta.cc │ ├── segment_meta.h │ ├── mock_wal.h │ ├── format.h │ ├── wal.cc │ ├── readable_log_segment.h │ ├── wal_bench.cc │ ├── log_writer.h │ ├── log_manager.h │ ├── log_writer.cc │ ├── readable_log_segment.cc │ ├── log_writer_test.cc │ └── log_manager_test.cc ├── rpc │ ├── mock_cluster.h │ ├── cluster.cc │ ├── peer.h │ ├── peer.cc │ └── raft_client.h ├── raft_task_executor.cc ├── raft_task_executor_test.h ├── raft_task_executor.h ├── raft_timer_test.cc ├── wal_commit_observer.h ├── raft_service_test.cc ├── raft_task_executor_test.cc ├── replicated_log.cc ├── wal_commit_observer.cc ├── replicated_log_test.cc ├── raft_timer.cc ├── raft_service.cc ├── ready_flusher.cc ├── CMakeLists.txt └── replicated_log_impl.h ├── cmake_modules ├── FindGLog.cmake ├── FindGflags.cmake ├── FindUBSan.cmake ├── FindASan.cmake ├── FindMSan.cmake ├── FindTSan.cmake └── FindSanitizers.cmake ├── CMakeLists.txt ├── run.sh ├── .clang-format ├── README.md └── deps_definitions.sh /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | cmake-build-debug/ 3 | .idea 4 | third_parties/ 5 | output/ 6 | sofa-pbrpc/ 7 | src/pb/raft_server.pb.cc 8 | src/pb/raft_server.pb.h -------------------------------------------------------------------------------- /apps/memkv/README.md: -------------------------------------------------------------------------------- 1 | # memkv 2 | 3 | **memkv** is an experimental storage system based on consensus-yaraft. 4 | It provides chubby-like naming structure. 5 | 6 | ## APIs 7 | 8 | It supports these operations: 9 | 10 | - Write 11 | - Delete 12 | - Get -------------------------------------------------------------------------------- /apps/memkv/compile_proto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SCRIPT_DIR=$(realpath `dirname $0`) 4 | CONSENSUS_YARAFT_DIR=$(dirname `dirname $SCRIPT_DIR`) 5 | PROTOC=$CONSENSUS_YARAFT_DIR/yaraft/build/third_parties/bin/protoc 6 | 7 | $PROTOC -I $SCRIPT_DIR/src/pb \ 8 | --cpp_out=$SCRIPT_DIR/src/pb \ 9 | $SCRIPT_DIR/src/pb/memkv.proto -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | sudo: required 3 | dist: trusty 4 | compiler: 5 | - clang 6 | - g++ 7 | 8 | branches: 9 | only: 10 | - master 11 | 12 | install: 13 | - sudo apt-get -y install libboost-all-dev libsnappy-dev realpath 14 | - bash install_dependencies.sh 15 | - bash compile_proto.sh 16 | - bash run.sh build 17 | 18 | script: 19 | - cd cmake-build-debug && cmake .. && make -j4 20 | - cd ../ && bash run.sh test -------------------------------------------------------------------------------- /compile_proto.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | SCRIPT_DIR=$(cd `dirname $0` | pwd) 4 | PROTOC=$SCRIPT_DIR/yaraft/build/third_parties/bin/protoc 5 | CONSENSUS_PROTO_DIR=$SCRIPT_DIR/include/consensus/pb 6 | $PROTOC -I $SCRIPT_DIR/yaraft/build/third_parties/include \ 7 | -I $CONSENSUS_PROTO_DIR \ 8 | -I $SCRIPT_DIR/build/third_parties/include \ 9 | --cpp_out=$CONSENSUS_PROTO_DIR \ 10 | $CONSENSUS_PROTO_DIR/raft_server.proto -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "yaraft"] 2 | path = yaraft 3 | url = https://github.com/neverchanje/yaraft.git 4 | [submodule "concurrentqueue"] 5 | path = src/concurrentqueue 6 | url = https://github.com/cameron314/concurrentqueue.git 7 | [submodule "readerwriterqueue"] 8 | path = src/readerwriterqueue 9 | url = https://github.com/cameron314/readerwriterqueue.git 10 | [submodule "brpc"] 11 | path = brpc 12 | url = https://github.com/brpc/brpc.git -------------------------------------------------------------------------------- /clang-format.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # format each of the modified files comparing to the latest git version. 4 | git status -s | while read LINE; do 5 | IFS=' ' 6 | PARAMS=(${LINE}) 7 | if [[ ${#PARAMS[@]} == 2 ]]; then 8 | if [[ ${PARAMS[0]} == 'M' || ${PARAMS[0]} == 'AM' ]]; then 9 | FILE=${PARAMS[1]} 10 | EXTENSION=${FILE##*.} 11 | if [[ ${EXTENSION} == 'h' || ${EXTENSION} == 'cc' || ${EXTENSION} == 'cpp' ]]; then 12 | clang-format -i ${FILE} 13 | fi 14 | fi 15 | fi 16 | done -------------------------------------------------------------------------------- /apps/memkv/src/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "status.h" 18 | 19 | #include -------------------------------------------------------------------------------- /apps/memkv/src/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace memkv { 20 | 21 | using silly::Slice; 22 | 23 | } // namespace memkv -------------------------------------------------------------------------------- /include/consensus/base/slice.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | using silly::Slice; 22 | 23 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/endianness.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | DECLARE_bool(is_little_endian); 22 | 23 | } // namespace consensus 24 | -------------------------------------------------------------------------------- /install_dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | source deps_definitions.sh 6 | 7 | echo "Installing necessary dependencies for building..." 8 | 9 | # build yaraft 10 | build_yaraft 11 | 12 | # build brpc 13 | if [ ! -f $TP_STAMP_DIR/$BRPC_NAME ]; then 14 | build_brpc 15 | make_stamp $BRPC_NAME 16 | fi 17 | 18 | # install other dependencies 19 | cd $PROJECT_DIR 20 | mkdir -p $TP_DIR 21 | mkdir -p $TP_STAMP_DIR 22 | mkdir -p $BUILD_DIR 23 | 24 | install_if_necessary(){ 25 | local depName=$1 26 | if [ ! -d $TP_DIR/$depName ]; then 27 | fetch_and_expand $depName.zip 28 | fi 29 | if [ ! -f $TP_STAMP_DIR/$depName ]; then 30 | $2 31 | make_stamp $depName 32 | fi 33 | } 34 | 35 | cd $TP_DIR 36 | 37 | # install google benchmark 38 | install_if_necessary $GOOGLE_BENCH_NAME build_google_bench 39 | 40 | install_if_necessary $GFLAG_NAME build_gflag 41 | install_if_necessary $GLOG_NAME build_glog -------------------------------------------------------------------------------- /include/consensus/base/glog_logger.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | class GLogLogger : public yaraft::Logger { 22 | public: 23 | void Log(yaraft::LogLevel level, int line, const char* file, const yaraft::Slice& log) override; 24 | }; 25 | 26 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/endianness.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/endianness.h" 16 | 17 | #include 18 | #include 19 | 20 | namespace consensus { 21 | 22 | #if defined(PROTOBUF_LITTLE_ENDIAN) 23 | DEFINE_bool(is_little_endian, true, ""); 24 | #else 25 | DEFINE_bool(is_little_endian, false, ""); 26 | #endif 27 | 28 | } // namespace consensus 29 | -------------------------------------------------------------------------------- /src/wal/segment_meta.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "wal/segment_meta.h" 16 | #include "base/logging.h" 17 | #include "wal/readable_log_segment.h" 18 | 19 | namespace consensus { 20 | namespace wal { 21 | 22 | std::string SegmentFileName(uint64_t segmentId, uint64_t firstIdx) { 23 | return fmt::format("{}-{}.wal", segmentId, firstIdx); 24 | } 25 | 26 | } // namespace wal 27 | } // namespace consensus -------------------------------------------------------------------------------- /src/rpc/mock_cluster.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "rpc/cluster.h" 18 | 19 | namespace consensus { 20 | namespace rpc { 21 | 22 | class MockCluster : public Cluster { 23 | public: 24 | virtual ~MockCluster() = default; 25 | 26 | virtual Status Pass(std::vector& mails) override { 27 | return Status::OK(); 28 | } 29 | }; 30 | 31 | } // namespace rpc 32 | } // namespace consensus 33 | -------------------------------------------------------------------------------- /src/raft_task_executor.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/simple_channel.h" 16 | 17 | #include "raft_task_executor.h" 18 | 19 | namespace consensus { 20 | 21 | yaraft::Ready *RaftTaskExecutor::GetReady() { 22 | yaraft::Ready *rd; 23 | Barrier barrier; 24 | Submit([&](yaraft::RawNode *node) { 25 | rd = node->GetReady(); 26 | barrier.Signal(); 27 | }); 28 | barrier.Wait(); 29 | return rd; 30 | } 31 | 32 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/errno.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | namespace consensus { 21 | 22 | void ErrnoToCString(int err, char *buf, size_t buf_len); 23 | 24 | // Return a string representing an errno. 25 | inline static std::string ErrnoToString(int err) { 26 | char buf[512]; 27 | ErrnoToCString(err, buf, sizeof(buf)); 28 | return std::string(buf); 29 | } 30 | 31 | } // namespace consensus -------------------------------------------------------------------------------- /src/rpc/cluster.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "peer.h" 16 | 17 | namespace consensus { 18 | namespace rpc { 19 | 20 | Cluster *Cluster::Default(const std::map &initialCluster) { 21 | std::map peerMap; 22 | for (const auto &e : initialCluster) { 23 | peerMap[e.first] = new Peer(e.second); 24 | } 25 | auto p = new PeerManager(std::move(peerMap)); 26 | return p; 27 | } 28 | 29 | } // namespace rpc 30 | } // namespace consensus 31 | -------------------------------------------------------------------------------- /include/consensus/base/task_queue.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | // TaskQueue is a Multi-Producer-Single-Consumer queue running in the background thread, 22 | // waiting to consume tasks. 23 | class TaskQueue { 24 | public: 25 | TaskQueue(); 26 | 27 | ~TaskQueue(); 28 | 29 | void Enqueue(std::function task); 30 | 31 | private: 32 | class Impl; 33 | std::unique_ptr impl_; 34 | }; 35 | 36 | } // namespace consensus -------------------------------------------------------------------------------- /src/wal/segment_meta.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/status.h" 18 | 19 | namespace consensus { 20 | namespace wal { 21 | 22 | using silly::operator""_sl; 23 | static constexpr Slice kLogSegmentHeaderMagic = "yaraft_log"_sl; 24 | 25 | struct SegmentMetaData { 26 | std::string fileName; 27 | size_t numEntries; 28 | 29 | SegmentMetaData() : numEntries(0){}; 30 | }; 31 | 32 | std::string SegmentFileName(uint64_t segmentId, uint64_t firstIdx); 33 | 34 | } // namespace wal 35 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/rpc/cluster.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "consensus/base/status.h" 20 | 21 | #include 22 | 23 | namespace consensus { 24 | namespace rpc { 25 | 26 | class Cluster { 27 | public: 28 | virtual ~Cluster() = default; 29 | 30 | virtual Status Pass(std::vector& mails) = 0; 31 | 32 | static Cluster* Default(const std::map& initialCluster); 33 | }; 34 | 35 | } // namespace rpc 36 | } // namespace consensus 37 | -------------------------------------------------------------------------------- /src/wal/mock_wal.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/status.h" 18 | #include "wal/wal.h" 19 | 20 | namespace consensus { 21 | namespace wal { 22 | 23 | class MockWriteAheadLog : public WriteAheadLog { 24 | public: 25 | virtual ~MockWriteAheadLog() = default; 26 | 27 | virtual Status AppendEntries(const PBEntryVec& entries) override { 28 | return Status::OK(); 29 | } 30 | 31 | virtual Status GC(CompactionHint* hint) override { 32 | return Status::OK(); 33 | } 34 | }; 35 | 36 | } // namespace wal 37 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/port.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | // 18 | // GCC can be told that a certain branch is not likely to be taken (for 19 | // instance, a CHECK failure), and use that information in static analysis. 20 | // Giving it this information can help it optimize for the common case in 21 | // the absence of better information (ie. -fprofile-arcs). 22 | // 23 | #if defined(__GNUC__) 24 | #define PREDICT_FALSE(x) (__builtin_expect(x, 0)) 25 | #define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1)) 26 | #else 27 | #define PREDICT_FALSE(x) x 28 | #define PREDICT_TRUE(x) x 29 | #endif 30 | -------------------------------------------------------------------------------- /apps/memkv/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | set(MEMKV_LINK_LIBS 2 | ${Boost_LIBRARIES} 3 | ${FMT_LIBRARY} 4 | ${GFLAGS_STATIC_LIB} 5 | ${BRPC_LIBRARY} 6 | ${ZLIB_LIBRARIES} 7 | ${PROTOBUF_STATIC_LIBRARY} 8 | ${GLOG_STATIC_LIB} 9 | ${OPENSSL_LIBRARIES} 10 | ${LEVELDB_LIBRARY} 11 | ${CONSENSUS_YARAFT_LIBRARY} 12 | ${YARAFT_LIBRARY} 13 | pthread 14 | dl 15 | ) 16 | 17 | set(MEMKV_SOURCES 18 | memkv_store.cc 19 | slice.h 20 | status.cc 21 | memkv_service.cc 22 | logging.h 23 | testing.h 24 | db.cc 25 | pb/memkv.pb.cc) 26 | 27 | add_library(memkv ${MEMKV_SOURCES}) 28 | target_link_libraries(memkv ${MEMKV_LINK_LIBS}) 29 | set(MEMKV_LINK_LIBS memkv ${MEMKV_LINK_LIBS}) 30 | 31 | function(ADD_TEST TEST_NAME) 32 | add_executable(${TEST_NAME} ${TEST_NAME}.cc) 33 | target_link_libraries(${TEST_NAME} ${MEMKV_LINK_LIBS} ${GTEST_LIB} ${GTEST_MAIN_LIB}) 34 | endfunction() 35 | 36 | ADD_TEST(memkv_store_test) 37 | 38 | add_executable(memkv_server memkv_server.cc) 39 | target_link_libraries(memkv_server ${MEMKV_LINK_LIBS}) 40 | 41 | install(TARGETS memkv_server DESTINATION bin) -------------------------------------------------------------------------------- /include/consensus/ready_flusher.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | // ReadyFlusher is a single background thread for asynchronously flushing the Ready-s, 22 | // so that the FSM thread can be free from stalls every time when it generates a Ready. 23 | 24 | class ReplicatedLogImpl; 25 | class ReadyFlusher { 26 | public: 27 | ReadyFlusher(); 28 | 29 | ~ReadyFlusher(); 30 | 31 | void Register(ReplicatedLogImpl* log); 32 | 33 | private: 34 | class Impl; 35 | std::unique_ptr impl_; 36 | }; 37 | 38 | } // namespace consensus 39 | -------------------------------------------------------------------------------- /src/base/random.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "base/random.h" 19 | 20 | namespace consensus { 21 | 22 | void RandomString(void* dest, size_t n, Random* rng) { 23 | size_t i = 0; 24 | uint32_t random = rng->Next(); 25 | char* cdest = static_cast(dest); 26 | static const size_t sz = sizeof(random); 27 | if (n >= sz) { 28 | for (i = 0; i <= n - sz; i += sz) { 29 | memcpy(&cdest[i], &random, sizeof(random)); 30 | random = rng->Next(); 31 | } 32 | } 33 | memcpy(cdest + i, &random, n - i); 34 | } 35 | 36 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/src/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "status.h" 16 | 17 | #include 18 | 19 | namespace memkv { 20 | 21 | #define ERROR_TO_STRING(err) \ 22 | case (err): \ 23 | return #err 24 | 25 | std::string Error::toString(unsigned int c) { 26 | auto code = static_cast(c); 27 | switch (code) { 28 | ERROR_TO_STRING(OK); 29 | ERROR_TO_STRING(InvalidArgument); 30 | ERROR_TO_STRING(NodeNotExist); 31 | ERROR_TO_STRING(ConsensusError); 32 | default: 33 | LOG(FATAL) << "invalid error code: " << c; 34 | assert(false); 35 | } 36 | } 37 | 38 | } // namespace memkv -------------------------------------------------------------------------------- /include/consensus/base/env_util.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "consensus/base/slice.h" 21 | #include "consensus/base/status.h" 22 | 23 | namespace consensus { 24 | 25 | class RandomAccessFile; 26 | 27 | namespace env_util { 28 | 29 | // Read the full content of `file` into `scratch`. 30 | Status ReadFully(RandomAccessFile *file, uint64_t offset, size_t n, Slice *result, char *scratch); 31 | 32 | // Read data into an unallocated buffer. 33 | Status ReadFullyToBuffer(const Slice &fname, Slice *result, char **scratch); 34 | 35 | } // namespace env_util 36 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/src/status.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace memkv { 20 | 21 | class Error { 22 | public: 23 | enum ErrorCodes { 24 | OK, 25 | 26 | InvalidArgument, 27 | NodeNotExist, 28 | ConsensusError, 29 | }; 30 | 31 | static std::string toString(unsigned int code); 32 | }; 33 | 34 | typedef silly::Status Status; 35 | 36 | template 37 | using StatusWith = silly::StatusWith; 38 | 39 | #ifdef FMT_Status 40 | #undef FMT_Status 41 | #endif 42 | #define FMT_Status(code, msg, args...) Status::Make(Error::code, fmt::format(msg, ##args)) 43 | 44 | } // namespace memkv -------------------------------------------------------------------------------- /include/consensus/raft_timer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | // RaftTimer is the background timer that ticks for every 1ms. 22 | // In our implementation, it doesn't generate a task every 1ms, instead, 23 | // it request RaftTaskExecutor to run RawNode::Tick 100 times for every 100ms. 24 | 25 | class RaftTaskExecutor; 26 | class RaftTimer { 27 | public: 28 | RaftTimer(); 29 | 30 | ~RaftTimer(); 31 | 32 | // Thread-safe 33 | void Register(RaftTaskExecutor* executor); 34 | 35 | private: 36 | class Impl; 37 | std::unique_ptr impl_; 38 | }; 39 | 40 | } // namespace consensus 41 | -------------------------------------------------------------------------------- /apps/memkv/src/memkv_store.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "slice.h" 18 | #include "status.h" 19 | 20 | namespace memkv { 21 | 22 | // MemKvStore is the internal in-memory storage of memkv. It's thread-safe. 23 | // A request will first go through DB, after WAL committed, it finally applies 24 | // in MemKvStore. 25 | class MemKvStore { 26 | public: 27 | Status Write(const Slice &path, const Slice &value); 28 | 29 | Status Delete(const Slice &path); 30 | 31 | Status Get(const Slice &path, std::string *data); 32 | 33 | MemKvStore(); 34 | 35 | ~MemKvStore(); 36 | 37 | private: 38 | class Impl; 39 | std::unique_ptr impl_; 40 | }; 41 | 42 | } // namespace memkv -------------------------------------------------------------------------------- /include/consensus/pb/raft_server.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package consensus.pb; 4 | 5 | import "yaraft/pb/raftpb.proto"; 6 | 7 | option cc_generic_services = true; 8 | option java_generic_services = true; 9 | option py_generic_services = true; 10 | 11 | enum StatusCode { 12 | OK = 0; 13 | 14 | StepLocalMsg = 1; 15 | StepPeerNotFound = 2; 16 | } 17 | 18 | message StepRequest { 19 | // The message that drives the RaftServer to perform RawNode::Step. 20 | required yaraft.pb.Message message = 1; 21 | } 22 | 23 | message StepResponse { 24 | required StatusCode code = 1; 25 | } 26 | 27 | message StatusRequest { 28 | } 29 | 30 | message StatusResponse { 31 | // leader is the member ID which the responding member believes is the current leader. 32 | optional uint64 leader = 1; 33 | // raftIndex is the current raft index of the responding member. 34 | optional uint64 raftIndex = 2; 35 | // raftTerm is the current raft term of the responding member. 36 | optional uint64 raftTerm = 3; 37 | // raftCommit is the current raft committed index of the responding member. 38 | optional uint64 raftCommit = 4; 39 | } 40 | 41 | service RaftService { 42 | rpc Step (StepRequest) returns (StepResponse); 43 | rpc Status (StatusRequest) returns (StatusResponse); 44 | } -------------------------------------------------------------------------------- /apps/memkv/src/pb/memkv.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package memkv.pb; 4 | 5 | option cc_generic_services = true; 6 | option java_generic_services = true; 7 | option py_generic_services = true; 8 | 9 | enum ErrCode { 10 | OK = 0; 11 | InvalidArgument = 1; 12 | NodeNotExist = 2; 13 | ConsensusError = 3; 14 | } 15 | 16 | message ReadRequest { 17 | optional string path = 1; 18 | 19 | // is stale read allowed 20 | // if false, requests will get rejected when current node is not leader. 21 | optional bool stale = 2; 22 | } 23 | 24 | message ReadResult { 25 | optional ErrCode errorCode = 1; 26 | optional string errorMessage = 2; 27 | 28 | optional bytes value = 3; 29 | } 30 | 31 | message WriteRequest { 32 | optional string path = 1; 33 | optional string value = 2; 34 | } 35 | 36 | message WriteResult { 37 | optional ErrCode errorCode = 1; 38 | optional string errorMessage = 2; 39 | } 40 | 41 | message DeleteRequest { 42 | optional string path = 1; 43 | } 44 | 45 | message DeleteResult { 46 | optional ErrCode errorCode = 1; 47 | optional string errorMessage = 2; 48 | } 49 | 50 | service MemKVService { 51 | rpc Write (WriteRequest) returns (WriteResult); 52 | rpc Read (ReadRequest) returns (ReadResult); 53 | rpc Delete (DeleteRequest) returns (DeleteResult); 54 | } -------------------------------------------------------------------------------- /src/raft_task_executor_test.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/testing.h" 18 | #include "raft_task_executor.h" 19 | 20 | #include 21 | 22 | namespace consensus { 23 | 24 | class RaftTaskExecutorTest : public ::testing::Test { 25 | public: 26 | virtual void SetUp() override { 27 | conf_ = new yaraft::Config; 28 | conf_->id = 1; 29 | conf_->peers = {1, 2, 3}; 30 | conf_->electionTick = 1000; 31 | conf_->heartbeatTick = 100; 32 | 33 | memstore_ = new yaraft::MemoryStorage; 34 | conf_->storage = memstore_; 35 | taskQueue_ = new TaskQueue; 36 | } 37 | 38 | protected: 39 | yaraft::Config *conf_; 40 | yaraft::MemoryStorage *memstore_; 41 | TaskQueue *taskQueue_; 42 | }; 43 | 44 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/errno.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "errno.h" 16 | #include "base/logging.h" 17 | 18 | namespace consensus { 19 | 20 | void ErrnoToCString(int err, char *buf, size_t buf_len) { 21 | CHECK_GT(buf_len, 0); 22 | #if !defined(__GLIBC__) || \ 23 | ((_POSIX_C_SOURCE >= 200112 || _XOPEN_SOURCE >= 600) && !defined(_GNU_SOURCE)) 24 | // Using POSIX version 'int strerror_r(...)'. 25 | int ret = strerror_r(err, buf, buf_len); 26 | if (ret && ret != ERANGE && ret != EINVAL) { 27 | strncpy(buf, "unknown error", buf_len); 28 | buf[buf_len - 1] = '\0'; 29 | } 30 | #else 31 | // Using GLIBC version 32 | char* ret = strerror_r(err, buf, buf_len); 33 | if (ret != buf) { 34 | strncpy(buf, ret, buf_len); 35 | buf[buf_len - 1] = '\0'; 36 | } 37 | #endif 38 | } 39 | 40 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/background_worker.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "consensus/base/status.h" 21 | 22 | #include 23 | 24 | namespace consensus { 25 | 26 | class BackgroundWorker { 27 | __DISALLOW_COPYING__(BackgroundWorker); 28 | 29 | public: 30 | BackgroundWorker(); 31 | 32 | ~BackgroundWorker(); 33 | 34 | // Start looping on the given recurring task infinitely, until users call Stop(). 35 | Status StartLoop(std::function recurringTask); 36 | 37 | // Stop looping. The function returns error status when the background thread 38 | // is already stopped. 39 | Status Stop(); 40 | 41 | bool Stopped() const; 42 | 43 | private: 44 | class Impl; 45 | std::unique_ptr impl_; 46 | }; 47 | 48 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/random.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | namespace consensus { 21 | 22 | class Random { 23 | public: 24 | explicit Random(uint32_t seed) : gen_(seed) {} 25 | 26 | // Generates the next random number 27 | uint32_t Next() { 28 | return static_cast(gen_()); 29 | } 30 | 31 | // Returns a uniformly distributed value in the range [0..n-1] 32 | // REQUIRES: n > 0 33 | uint32_t Uniform(uint32_t n) { 34 | return Next() % n; 35 | } 36 | 37 | private: 38 | std::mt19937 gen_; 39 | }; 40 | 41 | void RandomString(void* dest, size_t n, Random* rng); 42 | 43 | inline std::string RandomString(size_t n, Random* rng) { 44 | std::string s; 45 | s.resize(n); 46 | RandomString(&s[0], n, rng); 47 | return s; 48 | } 49 | 50 | } // namespace consensus -------------------------------------------------------------------------------- /src/rpc/peer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/status.h" 18 | #include "pb/raft_server.pb.h" 19 | #include "rpc/cluster.h" 20 | 21 | #include 22 | 23 | namespace consensus { 24 | namespace rpc { 25 | 26 | class AsyncRaftClient; 27 | class Peer { 28 | public: 29 | explicit Peer(const std::string& url); 30 | 31 | void AsyncSend(yaraft::pb::Message* msg); 32 | 33 | private: 34 | std::unique_ptr client_; 35 | }; 36 | 37 | class PeerManager : public Cluster { 38 | public: 39 | explicit PeerManager(std::map&& peerMap) : peerMap_(peerMap) {} 40 | 41 | ~PeerManager() override; 42 | 43 | Status Pass(std::vector& mails) override; 44 | 45 | private: 46 | std::map peerMap_; 47 | }; 48 | 49 | } // namespace rpc 50 | } // namespace consensus 51 | -------------------------------------------------------------------------------- /src/base/glog_logger.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "consensus/base/glog_logger.h" 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | void GLogLogger::Log(yaraft::LogLevel level, int line, const char *file, const yaraft::Slice &log) { 22 | google::LogSeverity severity; 23 | switch (level) { 24 | case yaraft::INFO: 25 | severity = google::INFO; 26 | break; 27 | case yaraft::WARNING: 28 | severity = google::WARNING; 29 | break; 30 | case yaraft::ERROR: 31 | severity = google::ERROR; 32 | break; 33 | case yaraft::FATAL: 34 | severity = google::FATAL; 35 | break; 36 | default: 37 | fprintf(stderr, "unsupported level of log: %d\n", static_cast(level)); 38 | assert(false); 39 | } 40 | 41 | google::LogMessage(file, line, severity).stream() << log; 42 | } 43 | 44 | } // namespace consensus 45 | -------------------------------------------------------------------------------- /include/consensus/base/status.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "consensus/base/slice.h" 18 | 19 | #include 20 | 21 | namespace consensus { 22 | 23 | class Error { 24 | public: 25 | enum ErrorCodes { 26 | OK, 27 | 28 | IOError, 29 | NotSupported, 30 | Corruption, 31 | LogCompacted, 32 | OutOfBound, 33 | YARaftError, 34 | BadConfig, 35 | RpcError, 36 | IllegalState, 37 | RuntimeError, 38 | InvalidArgument, 39 | WalWriteToNonLeader, 40 | }; 41 | 42 | static std::string toString(unsigned int errorCode); 43 | }; 44 | 45 | typedef silly::Status Status; 46 | 47 | template 48 | using StatusWith = silly::StatusWith; 49 | 50 | #ifdef FMT_Status 51 | #undef FMT_Status 52 | #endif 53 | #define FMT_Status(code, msg, args...) Status::Make(Error::code, fmt::format(msg, ##args)) 54 | 55 | } // namespace consensus 56 | -------------------------------------------------------------------------------- /src/rpc/peer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "rpc/peer.h" 16 | #include "base/logging.h" 17 | #include "base/stl_container_utils.h" 18 | #include "rpc/raft_client.h" 19 | 20 | namespace consensus { 21 | namespace rpc { 22 | 23 | Peer::Peer(const std::string& url) : client_(new AsyncRaftClient(url)) {} 24 | 25 | void Peer::AsyncSend(yaraft::pb::Message* msg) { 26 | client_->Step(msg); 27 | } 28 | 29 | Status PeerManager::Pass(std::vector& mails) { 30 | for (const auto& m : mails) { 31 | CHECK(m.to() != 0); 32 | CHECK(peerMap_.find(m.to()) != peerMap_.end()); 33 | 34 | auto newMsg = new yaraft::pb::Message(std::move(m)); 35 | peerMap_[m.to()]->AsyncSend(newMsg); 36 | } 37 | return Status::OK(); 38 | } 39 | 40 | PeerManager::~PeerManager() { 41 | STLDeleteContainerPairSecondPointers(peerMap_.begin(), peerMap_.end()); 42 | } 43 | 44 | } // namespace rpc 45 | } // namespace consensus 46 | -------------------------------------------------------------------------------- /src/wal/format.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | namespace wal { 21 | 22 | // Format of the encoded log batch: 23 | // 24 | // LogBatch := LogHeader Record+ 25 | // LogHeader := Crc32 Length 26 | // Record := Type VarString 27 | // 28 | // Crc32 -> 4 bytes, checksum of fields followed in the log block 29 | // Type -> 1 byte, RecordType 30 | // VarString -> varint32 + bytes, encoded log entry or encoded hard state 31 | // 32 | // Each segment composes of a series of log entries: 33 | // 34 | // Segment := SegmentHeader LogBlock* SegmentFooter 35 | // SegmentHeader := Magic 36 | // SegmentFooter := 37 | // 38 | 39 | constexpr static size_t kLogBatchHeaderSize = 4 + 4; 40 | constexpr static size_t kRecordHeaderSize = 1; 41 | 42 | enum RecordType { 43 | kHardStateType = 1, 44 | kLogEntryType = 2, 45 | }; 46 | 47 | } // namespace wal 48 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/status.h" 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | #define CONVERT_ERROR_TO_STRING(err) \ 22 | case (err): \ 23 | return #err 24 | 25 | std::string Error::toString(unsigned int code) { 26 | switch (code) { 27 | CONVERT_ERROR_TO_STRING(IOError); 28 | CONVERT_ERROR_TO_STRING(NotSupported); 29 | CONVERT_ERROR_TO_STRING(Corruption); 30 | CONVERT_ERROR_TO_STRING(LogCompacted); 31 | CONVERT_ERROR_TO_STRING(OutOfBound); 32 | CONVERT_ERROR_TO_STRING(YARaftError); 33 | CONVERT_ERROR_TO_STRING(BadConfig); 34 | CONVERT_ERROR_TO_STRING(RpcError); 35 | CONVERT_ERROR_TO_STRING(IllegalState); 36 | CONVERT_ERROR_TO_STRING(RuntimeError); 37 | CONVERT_ERROR_TO_STRING(InvalidArgument); 38 | CONVERT_ERROR_TO_STRING(WalWriteToNonLeader); 39 | default: 40 | return fmt::format("Unknown error codes: {}", code); 41 | } 42 | } 43 | 44 | } // namespace consensus 45 | -------------------------------------------------------------------------------- /src/raft_task_executor.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/task_queue.h" 18 | 19 | #include 20 | 21 | namespace consensus { 22 | 23 | // RaftTaskExecutor provides a task model for callers to submitting tasks to 24 | // manipulate the RawNode(aka FSM), all in asynchronous way. 25 | // The tasks may be submitted from RaftTimer (for ticking), RaftService::Step, ReplicatedLog::Write, 26 | // currently they will all be executed sequentially, the underlying worker is a single thread. 27 | // 28 | class RaftTaskExecutor { 29 | public: 30 | RaftTaskExecutor(yaraft::RawNode* node, TaskQueue* taskQueue) : node_(node), queue_(taskQueue) {} 31 | 32 | typedef std::function RaftTask; 33 | 34 | void Submit(RaftTask task) { 35 | queue_->Enqueue(std::bind(task, node_)); 36 | } 37 | 38 | yaraft::Ready* GetReady(); 39 | 40 | private: 41 | yaraft::RawNode* node_; 42 | std::shared_ptr queue_; 43 | }; 44 | 45 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/background_worker_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/background_worker.h" 16 | #include "base/testing.h" 17 | 18 | using namespace consensus; 19 | 20 | // This test verifies BackgroundWorker::Stop will stop the background thread only when it's 21 | // stoppable, otherwise it generates fault prompt. 22 | TEST(BackgroundWorkerTest, Stop) { 23 | BackgroundWorker worker; 24 | auto func = []() { usleep(1000 * 1); }; 25 | 26 | ASSERT_EQ(worker.Stop().Code(), Error::RuntimeError); 27 | 28 | { 29 | worker.StartLoop(func); 30 | ASSERT_OK(worker.Stop()); 31 | } 32 | 33 | { 34 | worker.StartLoop(func); 35 | ASSERT_OK(worker.Stop()); 36 | ASSERT_EQ(worker.Stop().Code(), Error::RuntimeError); 37 | } 38 | } 39 | 40 | TEST(BackgroundWorkerTest, Start) { 41 | BackgroundWorker worker; 42 | 43 | { 44 | auto func = []() { usleep(1000 * 1); }; 45 | worker.StartLoop(func); 46 | ASSERT_EQ(worker.StartLoop(func).Code(), Error::RuntimeError); 47 | worker.Stop(); 48 | } 49 | } -------------------------------------------------------------------------------- /src/raft_timer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/simple_channel.h" 16 | #include "raft_task_executor_test.h" 17 | #include "raft_timer.h" 18 | 19 | using namespace consensus; 20 | 21 | class RaftTimerTest : public RaftTaskExecutorTest {}; 22 | 23 | // This test verifies that RaftTimer sends ticks regularly to RaftTaskExecutor, which 24 | // is supposed to start election after election timeout. 25 | TEST_F(RaftTimerTest, Timeout) { 26 | conf_->peers = {1}; 27 | conf_->electionTick = 200; 28 | yaraft::RawNode node(conf_); 29 | RaftTaskExecutor executor(&node, taskQueue_); 30 | RaftTimer timer; 31 | timer.Register(&executor); 32 | 33 | sleep(2); 34 | 35 | uint64_t currentTerm = 0; 36 | uint64_t lastIndex = 0; 37 | 38 | Barrier barrier; 39 | executor.Submit([&](yaraft::RawNode *n) { 40 | currentTerm = n->CurrentTerm(); 41 | lastIndex = n->LastIndex(); 42 | 43 | barrier.Signal(); 44 | }); 45 | barrier.Wait(); 46 | 47 | ASSERT_GE(currentTerm, 1); 48 | ASSERT_GE(lastIndex, 1); 49 | } -------------------------------------------------------------------------------- /src/wal_commit_observer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/simple_channel.h" 18 | #include "base/status.h" 19 | 20 | #include 21 | 22 | namespace consensus { 23 | 24 | // WalCommitObserver observes updates of the committedIndex. If the Ready 25 | // object generated by RawNode contains committedIndex updates, the observers 26 | // will be informed, and all the registered listeners with ranges 27 | // covering the new committedIndex will be notified. 28 | // 29 | // Thread-Safe 30 | class WalCommitObserver { 31 | __DISALLOW_COPYING__(WalCommitObserver); 32 | 33 | public: 34 | WalCommitObserver(); 35 | 36 | // ONLY the wal write thread is allowed to call this function. 37 | void Register(std::pair range, SimpleChannel *channel); 38 | 39 | // ONLY the flusher thread is allowed to call this function. 40 | void Notify(uint64_t commitIndex); 41 | 42 | ~WalCommitObserver(); 43 | 44 | private: 45 | class Impl; 46 | std::unique_ptr impl_; 47 | }; 48 | 49 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/raft_service.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | namespace consensus { 20 | 21 | class RaftTaskExecutor; 22 | 23 | class RaftServiceImpl : public pb::RaftService { 24 | public: 25 | explicit RaftServiceImpl(RaftTaskExecutor *executor) : executor_(executor) {} 26 | 27 | ~RaftServiceImpl() = default; 28 | 29 | // RaftService::Step handles each request by calling RawNode::Step. If the request message 30 | // is invalid, the RaftService will respond with an error code. 31 | // @param `request` may be mutated after Step. 32 | void Step(google::protobuf::RpcController *controller, const pb::StepRequest *request, 33 | pb::StepResponse *response, google::protobuf::Closure *done) override; 34 | 35 | void Status(::google::protobuf::RpcController *controller, const pb::StatusRequest *request, 36 | pb::StatusResponse *response, ::google::protobuf::Closure *done) override; 37 | 38 | private: 39 | RaftTaskExecutor *executor_; 40 | }; 41 | 42 | } // namespace consensus 43 | -------------------------------------------------------------------------------- /src/raft_service_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "raft_service.h" 16 | #include "raft_task_executor_test.h" 17 | 18 | using namespace consensus; 19 | 20 | class RaftServiceTest : public RaftTaskExecutorTest { 21 | public: 22 | }; 23 | 24 | TEST_F(RaftServiceTest, Step) { 25 | yaraft::RawNode node(conf_); 26 | RaftTaskExecutor executor(&node, taskQueue_); 27 | RaftServiceImpl service(&executor); 28 | 29 | pb::StepResponse response; 30 | pb::StepRequest request; 31 | 32 | auto msg = new yaraft::pb::Message; 33 | msg->set_type(yaraft::pb::MsgHup); 34 | request.set_allocated_message(msg); 35 | auto done = google::protobuf::NewCallback([]() {}); 36 | service.Step(nullptr, &request, &response, done); 37 | ASSERT_EQ(response.code(), pb::StepLocalMsg); 38 | 39 | msg = new yaraft::pb::Message; 40 | msg->set_from(111); 41 | msg->set_type(yaraft::pb::MsgHeartbeatResp); 42 | request.set_allocated_message(msg); 43 | done = google::protobuf::NewCallback([]() {}); 44 | service.Step(nullptr, &request, &response, done); 45 | ASSERT_EQ(response.code(), pb::StepPeerNotFound); 46 | } -------------------------------------------------------------------------------- /apps/memkv/src/memkv_service.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "db.h" 20 | #include "memkv_store.h" 21 | #include "pb/memkv.pb.h" 22 | 23 | namespace memkv { 24 | 25 | class MemKVServiceImpl : public pb::MemKVService { 26 | public: 27 | void Write(::google::protobuf::RpcController* controller, 28 | const ::memkv::pb::WriteRequest* request, ::memkv::pb::WriteResult* response, 29 | ::google::protobuf::Closure* done) override; 30 | 31 | void Read(::google::protobuf::RpcController* controller, const ::memkv::pb::ReadRequest* request, 32 | ::memkv::pb::ReadResult* response, ::google::protobuf::Closure* done) override; 33 | 34 | void Delete(::google::protobuf::RpcController* controller, 35 | const ::memkv::pb::DeleteRequest* request, ::memkv::pb::DeleteResult* response, 36 | ::google::protobuf::Closure* done) override; 37 | 38 | explicit MemKVServiceImpl(DB* db); 39 | 40 | ~MemKVServiceImpl() override; 41 | 42 | private: 43 | std::shared_ptr db_; 44 | }; 45 | 46 | } // namespace memkv -------------------------------------------------------------------------------- /include/consensus/base/simple_channel.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace consensus { 8 | 9 | // SimpleChannel is a golang-like channel implementation that provides inter-thread communication. 10 | // 11 | // On the sender side: 12 | // auto s = Status::Make(Error::Corruption, "something wrong"); 13 | // SimpleChannel *ch; 14 | // (*ch) <<= s; 15 | // 16 | // On the receiver side: 17 | // Status s; 18 | // SimpleChannel *ch; 19 | // (*ch) >>= s; 20 | // 21 | // SimpleChannel is just a wrapper of the std::future and std::promise. 22 | // 23 | 24 | template 25 | class SimpleChannel { 26 | __DISALLOW_COPYING__(SimpleChannel); 27 | 28 | public: 29 | SimpleChannel() { 30 | future_ = promise_.get_future(); 31 | } 32 | 33 | SimpleChannel(SimpleChannel &&chan) 34 | : promise_(std::move(chan.promise_)), future_(std::move(chan.future_)) {} 35 | 36 | void operator<<=(const V &value) { 37 | promise_.set_value(value); 38 | } 39 | 40 | void operator<<=(V &&value) { 41 | promise_.set_value(value); 42 | } 43 | 44 | void operator>>=(V &value) { 45 | future_.wait(); 46 | value = future_.get(); 47 | } 48 | 49 | private: 50 | std::promise promise_; 51 | std::future future_; 52 | }; 53 | 54 | class Barrier { 55 | __DISALLOW_COPYING__(Barrier); 56 | 57 | public: 58 | Barrier() { 59 | future_ = promise_.get_future(); 60 | } 61 | 62 | void Signal() { 63 | promise_.set_value(); 64 | } 65 | 66 | void Wait() { 67 | future_.wait(); 68 | } 69 | 70 | private: 71 | std::promise promise_; 72 | std::future future_; 73 | }; 74 | 75 | } // namespace consensus -------------------------------------------------------------------------------- /src/wal/wal.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "wal/wal.h" 16 | #include "base/logging.h" 17 | #include "wal/log_manager.h" 18 | 19 | namespace consensus { 20 | namespace wal { 21 | 22 | Status WriteAheadLog::Default(const WriteAheadLogOptions& options, WriteAheadLogUPtr* wal, 23 | yaraft::MemStoreUptr* memstore) { 24 | LogManagerUPtr lm; 25 | RETURN_NOT_OK(LogManager::Recover(options, memstore, &lm)); 26 | LOG_ASSERT(lm != nullptr); 27 | 28 | wal->reset(lm.release()); 29 | return Status::OK(); 30 | } 31 | 32 | WriteAheadLogOptions::WriteAheadLogOptions() 33 | : verify_checksum(true), log_segment_size(64 * 1024 * 1024) {} 34 | 35 | WriteAheadLogUPtr TEST_CreateWalStore(const std::string& testDir, yaraft::MemStoreUptr* pMemstore) { 36 | WriteAheadLogOptions options; 37 | options.log_dir = testDir; 38 | options.verify_checksum = true; 39 | 40 | WriteAheadLogUPtr wal; 41 | FATAL_NOT_OK(WriteAheadLog::Default(options, &wal, pMemstore), "WriteAheadLog::Default"); 42 | wal->Write({yaraft::pb::Entry()}); 43 | 44 | return std::move(wal); 45 | } 46 | 47 | } // namespace wal 48 | } // namespace consensus -------------------------------------------------------------------------------- /cmake_modules/FindGLog.cmake: -------------------------------------------------------------------------------- 1 | 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, 13 | # software distributed under the License is distributed on an 14 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | # KIND, either express or implied. See the License for the 16 | # specific language governing permissions and limitations 17 | # under the License. 18 | 19 | # - Find GLOG (logging.h, libglog.a, libglog.so, and libglog.so.0) 20 | # This module defines 21 | # GLOG_INCLUDE_DIR, directory containing headers 22 | # GLOG_SHARED_LIB, path to libglog's shared library 23 | # GLOG_STATIC_LIB, path to libglog's static library 24 | # GLOG_FOUND, whether glog has been found 25 | 26 | find_path(GLOG_INCLUDE_DIR glog/logging.h 27 | # make sure we don't accidentally pick up a different version 28 | NO_CMAKE_SYSTEM_PATH 29 | NO_SYSTEM_ENVIRONMENT_PATH) 30 | find_library(GLOG_SHARED_LIB glog 31 | NO_CMAKE_SYSTEM_PATH 32 | NO_SYSTEM_ENVIRONMENT_PATH) 33 | find_library(GLOG_STATIC_LIB libglog.a 34 | NO_CMAKE_SYSTEM_PATH 35 | NO_SYSTEM_ENVIRONMENT_PATH) 36 | 37 | include(FindPackageHandleStandardArgs) 38 | find_package_handle_standard_args(GLOG REQUIRED_VARS 39 | GLOG_SHARED_LIB GLOG_STATIC_LIB GLOG_INCLUDE_DIR) -------------------------------------------------------------------------------- /src/base/task_queue.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/task_queue.h" 16 | #include "base/background_worker.h" 17 | #include "base/logging.h" 18 | #include "concurrentqueue/blockingconcurrentqueue.h" 19 | 20 | namespace consensus { 21 | 22 | class TaskQueue::Impl { 23 | using Runnable = std::function; 24 | 25 | public: 26 | void Enqueue(Runnable task) { 27 | queue_.enqueue(task); 28 | } 29 | 30 | void Start() { 31 | FATAL_NOT_OK(worker_.StartLoop([&]() { 32 | Runnable task; 33 | if (queue_.wait_dequeue_timed(task, std::chrono::milliseconds(50))) { 34 | task(); 35 | } 36 | }), 37 | "TaskQueue::Start"); 38 | } 39 | 40 | void Stop() { 41 | FATAL_NOT_OK(worker_.Stop(), "TaskQueue::Stop"); 42 | } 43 | 44 | private: 45 | moodycamel::BlockingConcurrentQueue queue_; 46 | BackgroundWorker worker_; 47 | }; 48 | 49 | void TaskQueue::Enqueue(std::function task) { 50 | impl_->Enqueue(task); 51 | } 52 | 53 | TaskQueue::TaskQueue() : impl_(new Impl()) { 54 | impl_->Start(); 55 | } 56 | 57 | TaskQueue::~TaskQueue() { 58 | impl_->Stop(); 59 | } 60 | 61 | } // namespace consensus -------------------------------------------------------------------------------- /cmake_modules/FindGflags.cmake: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | 18 | # - Find GFLAGS (gflags.h, libgflags.a, libgflags.so, and libgflags.so.0) 19 | # This module defines 20 | # GFLAGS_INCLUDE_DIR, directory containing headers 21 | # GFLAGS_SHARED_LIB, path to libgflags shared library 22 | # GFLAGS_STATIC_LIB, path to libgflags static library 23 | # GFLAGS_FOUND, whether gflags has been found 24 | 25 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 26 | # make sure we don't accidentally pick up a different version 27 | NO_CMAKE_SYSTEM_PATH 28 | NO_SYSTEM_ENVIRONMENT_PATH) 29 | find_library(GFLAGS_SHARED_LIB gflags 30 | NO_CMAKE_SYSTEM_PATH 31 | NO_SYSTEM_ENVIRONMENT_PATH) 32 | find_library(GFLAGS_STATIC_LIB libgflags.a 33 | NO_CMAKE_SYSTEM_PATH 34 | NO_SYSTEM_ENVIRONMENT_PATH) 35 | 36 | include(FindPackageHandleStandardArgs) 37 | find_package_handle_standard_args(GFLAGS REQUIRED_VARS 38 | GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR) 39 | -------------------------------------------------------------------------------- /src/raft_task_executor_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "raft_task_executor_test.h" 16 | #include "base/simple_channel.h" 17 | 18 | #include 19 | #include 20 | 21 | using namespace consensus; 22 | 23 | // This test verifies the tasks submitted to RaftTaskExecutor will be executed sequentially. 24 | TEST_F(RaftTaskExecutorTest, TasksInSequence) { 25 | yaraft::RawNode node(conf_); 26 | RaftTaskExecutor executor(&node, taskQueue_); 27 | 28 | boost::thread_group group; 29 | boost::latch latch(3); 30 | 31 | // appending a character into std::string is absolutely a non-atomic operation,. 32 | std::string s; 33 | std::atomic_int count(0); 34 | 35 | Barrier barrier; 36 | for (int i = 0; i < 3; i++) { 37 | group.create_thread([&]() { 38 | latch.count_down_and_wait(); 39 | 40 | for (int k = 0; k < 100; k++) { 41 | executor.Submit([&](yaraft::RawNode *n) { 42 | s.push_back('a'); 43 | if (++count == 300) { 44 | barrier.Signal(); 45 | } 46 | }); 47 | } 48 | }); 49 | } 50 | barrier.Wait(); 51 | 52 | ASSERT_EQ(s.length(), 300); 53 | ASSERT_EQ(s, std::string(300, 'a')); 54 | } -------------------------------------------------------------------------------- /apps/memkv/src/db.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | 19 | #include "slice.h" 20 | #include "status.h" 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | namespace memkv { 27 | 28 | // Abstract handle to particular state of a DB. 29 | // A Snapshot is an immutable object and can therefore be safely 30 | // accessed from multiple threads without any external synchronization. 31 | class Snapshot { 32 | protected: 33 | virtual ~Snapshot(); 34 | }; 35 | 36 | class DBOptions { 37 | public: 38 | uint64_t member_id; 39 | std::string wal_dir; 40 | std::map initial_cluster; 41 | }; 42 | 43 | class DB { 44 | public: 45 | static StatusWith Bootstrap(const DBOptions &options); 46 | 47 | Status Write(const Slice &path, const Slice &value); 48 | 49 | Status Delete(const Slice &path); 50 | 51 | Status Get(const Slice &path, bool stale, std::string *data); 52 | 53 | consensus::pb::RaftService *CreateRaftServiceInstance() const; 54 | 55 | DB(); 56 | 57 | ~DB(); 58 | 59 | private: 60 | class Impl; 61 | std::unique_ptr impl_; 62 | }; 63 | 64 | } // namespace memkv -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(consensus-yaraft) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 5 | set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake_modules") 6 | 7 | set(THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/build/third_parties) 8 | set(YARAFT_THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/yaraft/build/third_parties) 9 | set(BRPC_THIRDPARTY_DIR ${CMAKE_CURRENT_SOURCE_DIR}/brpc/output) 10 | 11 | # Look in thirdparty prefix paths before anywhere else for system dependencies. 12 | set(CMAKE_PREFIX_PATH ${THIRDPARTY_DIR} ${YARAFT_THIRDPARTY_DIR} ${BRPC_THIRDPARTY_DIR} ${CMAKE_PREFIX_PATH}) 13 | 14 | find_package(Protobuf REQUIRED) 15 | 16 | find_package(GLog REQUIRED) 17 | 18 | find_package(Gflags REQUIRED) 19 | 20 | find_package(Sanitizers REQUIRED) 21 | 22 | find_library(GTEST_LIB gtest PATH ${DEPS_LIB_DIR}) 23 | find_library(GTEST_MAIN_LIB gtest_main PATH ${DEPS_LIB_DIR}) 24 | message("-- Found ${GTEST_LIB}") 25 | message("-- Found ${GTEST_MAIN_LIB}") 26 | 27 | find_package(Boost COMPONENTS system filesystem thread REQUIRED) 28 | 29 | find_package(OpenSSL REQUIRED) 30 | 31 | find_library(FMT_LIBRARY fmt PATH ${DEPS_LIB_DIR}) 32 | message("-- Found ${FMT_LIBRARY}") 33 | 34 | find_library(YARAFT_LIBRARY yaraft PATH ${DEPS_LIB_DIR}) 35 | message("-- Found ${YARAFT_LIBRARY}") 36 | 37 | find_library(BRPC_LIBRARY libbrpc.a brpc) 38 | message("-- Found ${BRPC_LIBRARY}") 39 | find_library(LEVELDB_LIBRARY leveldb) 40 | message("-- Found ${LEVELDB_LIBRARY}") 41 | find_package(ZLIB REQUIRED) 42 | 43 | include_directories(${THIRDPARTY_DIR}/include) 44 | include_directories(${YARAFT_THIRDPARTY_DIR}/include) 45 | include_directories(${BRPC_THIRDPARTY_DIR}/include) 46 | include_directories(src) 47 | include_directories(include) 48 | include_directories(include/consensus) 49 | 50 | add_subdirectory(src) -------------------------------------------------------------------------------- /src/base/testing.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "base/env.h" 18 | #include "base/testing.h" 19 | #include "base/random.h" 20 | 21 | namespace consensus { 22 | 23 | uint32_t BaseTest::SeedRandom() { 24 | int seed; 25 | if (FLAGS_gtest_random_seed) { 26 | seed = FLAGS_gtest_random_seed; 27 | } else { 28 | seed = static_cast(std::chrono::system_clock::now().time_since_epoch().count() / 1e9); 29 | } 30 | LOG(INFO) << "Using random seed: " << seed; 31 | return static_cast(seed); 32 | } 33 | 34 | WritableFile* BaseTest::OpenFileForWrite(const std::string& fname, Env::CreateMode mode, 35 | bool sync_on_close) { 36 | auto sw = Env::Default()->NewWritableFile(fname, mode, sync_on_close); 37 | CHECK(sw.GetStatus().IsOK()) << sw.GetStatus(); 38 | return sw.GetValue(); 39 | } 40 | 41 | void BaseTest::WriteTestFile(const Slice& path, size_t size, std::string* testData, Random* rng) { 42 | auto sw = Env::Default()->NewWritableFile(path); 43 | ASSERT_OK(sw.GetStatus()); 44 | std::unique_ptr wf(sw.GetValue()); 45 | 46 | (*testData) = RandomString(size, rng); 47 | ASSERT_OK(wf->Append(*testData)); 48 | ASSERT_OK(wf->Close()); 49 | } 50 | 51 | } // namespace consensus -------------------------------------------------------------------------------- /cmake_modules/FindUBSan.cmake: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 4 | # 2013 Matthew Arsenault 5 | # 2015-2016 RWTH Aachen University, Federal Republic of Germany 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | option(SANITIZE_UNDEFINED 26 | "Enable UndefinedBehaviorSanitizer for sanitized targets." Off) 27 | 28 | set(FLAG_CANDIDATES 29 | "-g -fsanitize=undefined" 30 | ) 31 | 32 | 33 | include(sanitize-helpers) 34 | 35 | if (SANITIZE_UNDEFINED) 36 | sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" 37 | "UndefinedBehaviorSanitizer" "UBSan") 38 | endif () 39 | 40 | function (add_sanitize_undefined TARGET) 41 | if (NOT SANITIZE_UNDEFINED) 42 | return() 43 | endif () 44 | 45 | saitizer_add_flags(${TARGET} "UndefinedBehaviorSanitizer" "UBSan") 46 | endfunction () 47 | -------------------------------------------------------------------------------- /src/base/mock_env.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/env.h" 16 | 17 | namespace consensus { 18 | namespace wal { 19 | 20 | class MockWritableFile : public WritableFile { 21 | public: 22 | MockWritableFile() {} 23 | 24 | ~MockWritableFile() {} 25 | 26 | Status Append(const Slice& data) override { 27 | buf_.append(data.data(), data.size()); 28 | return Status::OK(); 29 | } 30 | 31 | Status Truncate(uint64_t size) override { 32 | buf_.resize(size); 33 | return Status::OK(); 34 | } 35 | 36 | Status Close() override { 37 | return Status::OK(); 38 | } 39 | 40 | Status Flush(FlushMode mode) override { 41 | return Status::OK(); 42 | } 43 | 44 | Status Sync() override { 45 | return Status::OK(); 46 | } 47 | 48 | uint64_t Size() const override { 49 | return buf_.size(); 50 | } 51 | 52 | Status PreAllocate(uint64_t size) override { 53 | return Status::Make(Error::NotSupported); 54 | } 55 | 56 | const std::string& filename() const override { 57 | return fileName_; 58 | } 59 | 60 | std::string Data() const { 61 | return buf_; 62 | } 63 | 64 | private: 65 | std::string buf_; 66 | std::string fileName_; 67 | }; 68 | 69 | } // namespace wal 70 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | /// @brief Emit a warning if @c to_call returns a bad status. 21 | #define WARN_NOT_OK(to_call, warning_prefix) \ 22 | do { \ 23 | const auto& _s = (to_call); \ 24 | if (UNLIKELY(!_s.IsOK())) { \ 25 | LOG(WARNING) << (warning_prefix) << ": " << _s.ToString(); \ 26 | } \ 27 | } while (0); 28 | 29 | /// @brief Emit a fatal error if @c to_call returns a bad status. 30 | #define FATAL_NOT_OK(to_call, fatal_prefix) \ 31 | do { \ 32 | const auto& _s = (to_call); \ 33 | if (UNLIKELY(!_s.IsOK())) { \ 34 | LOG(FATAL) << (fatal_prefix) << ": " << _s.ToString(); \ 35 | } \ 36 | } while (0); 37 | 38 | #define FMT_LOG(level, formatStr, args...) LOG(level) << fmt::format(formatStr, ##args) 39 | 40 | #define FMT_SLOG(level, formatStr, args...) LOG(level) << fmt::sprintf(formatStr, ##args) -------------------------------------------------------------------------------- /src/base/env_util.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/env_util.h" 16 | #include "base/env.h" 17 | #include "base/logging.h" 18 | 19 | #include 20 | #include 21 | 22 | namespace consensus { 23 | namespace env_util { 24 | 25 | Status ReadFully(RandomAccessFile *file, uint64_t offset, size_t n, Slice *result, char *scratch) { 26 | size_t remain = n; 27 | char *dst = scratch; 28 | while (remain > 0) { 29 | Slice this_result; 30 | RETURN_NOT_OK(file->Read(offset, remain, &this_result, dst)); 31 | DCHECK_LE(this_result.size(), remain); 32 | if (this_result.size() == 0) { 33 | return Status::Make(Error::IOError, 34 | fmt::format("EOF trying to read {} bytes at offset {}", n, offset)); 35 | } 36 | dst += this_result.size(); 37 | remain -= this_result.size(); 38 | offset += this_result.size(); 39 | } 40 | *result = Slice(scratch, n - remain); 41 | return Status::OK(); 42 | } 43 | 44 | Status ReadFullyToBuffer(const Slice &fname, Slice *result, char **scratch) { 45 | RandomAccessFile *raf; 46 | ASSIGN_IF_OK(Env::Default()->NewRandomAccessFile(fname), raf); 47 | 48 | size_t n; 49 | ASSIGN_IF_OK(raf->Size(), n); 50 | (*scratch) = new char[n]; 51 | 52 | return ReadFully(raf, 0, n, result, *scratch); 53 | } 54 | 55 | } // namespace env_util 56 | } // namespace consensus -------------------------------------------------------------------------------- /src/wal/readable_log_segment.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/status.h" 18 | #include "wal/segment_meta.h" 19 | 20 | #include 21 | 22 | namespace consensus { 23 | namespace wal { 24 | 25 | extern Status ReadSegmentIntoMemoryStorage(const Slice &fname, yaraft::MemoryStorage *memstore, 26 | SegmentMetaData *metaData, bool verifyChecksum); 27 | 28 | // ReadableLogSegment reads the data of a segment into memory all at once. 29 | // It's sufficient because it's only used in wal recovery. 30 | class ReadableLogSegment { 31 | public: 32 | ReadableLogSegment(const Slice &scratch, yaraft::MemoryStorage *memStore, 33 | SegmentMetaData *metaData, bool verifyChecksum) 34 | : remain_(scratch.size()), 35 | buf_(scratch.data()), 36 | metaData_(metaData), 37 | memStore_(memStore), 38 | verifyChecksum_(verifyChecksum) {} 39 | 40 | Status ReadHeader(); 41 | 42 | Status ReadRecord(); 43 | 44 | bool Eof(); 45 | 46 | private: 47 | Status checkRemain(size_t need); 48 | 49 | void advance(size_t size); 50 | 51 | private: 52 | const char *buf_; 53 | size_t remain_; 54 | yaraft::MemoryStorage *memStore_; 55 | SegmentMetaData *metaData_; 56 | 57 | const bool verifyChecksum_; 58 | }; 59 | 60 | } // namespace wal 61 | } // namespace consensus -------------------------------------------------------------------------------- /src/rpc/raft_client.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/logging.h" 18 | #include "pb/raft_server.pb.h" 19 | 20 | #include 21 | 22 | namespace consensus { 23 | namespace rpc { 24 | 25 | static void doneCallBack(pb::StepResponse* response, brpc::Controller* cntl) { 26 | if (cntl->Failed()) { 27 | FMT_SLOG(ERROR, "request failed: %s", cntl->ErrorText().c_str()); 28 | } else { 29 | delete response; 30 | } 31 | delete cntl; 32 | } 33 | 34 | class AsyncRaftClient { 35 | public: 36 | explicit AsyncRaftClient(const std::string& url) { 37 | brpc::ChannelOptions options; 38 | options.max_retry = 0; // no retry 39 | options.connect_timeout_ms = 2000; 40 | channel_.Init(url.c_str(), &options); 41 | } 42 | 43 | // Asynchronously sending request to specified url. 44 | void Step(yaraft::pb::Message* msg) { 45 | // -- prepare parameters -- 46 | 47 | auto cntl = new brpc::Controller; 48 | cntl->set_timeout_ms(3000); 49 | 50 | pb::StepRequest request; 51 | request.set_allocated_message(msg); 52 | auto response = new pb::StepResponse; 53 | 54 | // -- request -- 55 | 56 | pb::RaftService_Stub stub(&channel_); 57 | stub.Step(cntl, &request, response, brpc::NewCallback(&doneCallBack, response, cntl)); 58 | } 59 | 60 | private: 61 | brpc::Channel channel_; 62 | }; 63 | 64 | } // namespace rpc 65 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | project(memkv) 4 | 5 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 6 | 7 | get_filename_component(CONSENSUS_YARAFT_DIR ${PROJECT_SOURCE_DIR} PATH) 8 | get_filename_component(CONSENSUS_YARAFT_DIR ${CONSENSUS_YARAFT_DIR} PATH) 9 | 10 | set(YARAFT_THIRDPARTY_DIR ${CONSENSUS_YARAFT_DIR}/yaraft/build/third_parties) 11 | set(CONSENSUS_YARAFT_THIRDPARTY_DIR ${CONSENSUS_YARAFT_DIR}/build/third_parties) 12 | set(CONSENSUS_YARAFT_OUTPUT ${CONSENSUS_YARAFT_DIR}/output) 13 | set(BRPC_THIRDPARTY_DIR ${CONSENSUS_YARAFT_DIR}/brpc/output) 14 | 15 | set(CMAKE_PREFIX_PATH 16 | ${CONSENSUS_YARAFT_OUTPUT} 17 | ${CONSENSUS_YARAFT_THIRDPARTY_DIR} 18 | ${YARAFT_THIRDPARTY_DIR} 19 | ${BRPC_THIRDPARTY_DIR} 20 | ${CMAKE_PREFIX_PATH}) 21 | set(CMAKE_MODULE_PATH "${CONSENSUS_YARAFT_DIR}/cmake_modules") 22 | 23 | find_package(Boost COMPONENTS system filesystem thread REQUIRED) 24 | find_package(Protobuf REQUIRED) 25 | find_package(Gflags REQUIRED) 26 | 27 | find_library(FMT_LIBRARY fmt) 28 | message("-- Found ${FMT_LIBRARY}") 29 | 30 | find_library(GTEST_LIB gtest) 31 | find_library(GTEST_MAIN_LIB gtest_main) 32 | message("-- Found ${GTEST_LIB}") 33 | message("-- Found ${GTEST_MAIN_LIB}") 34 | 35 | find_library(BRPC_LIBRARY libbrpc.a brpc) 36 | message("-- Found ${BRPC_LIBRARY}") 37 | 38 | find_package(ZLIB REQUIRED) 39 | find_package(GLog REQUIRED) 40 | find_package(OpenSSL REQUIRED) 41 | 42 | find_library(LEVELDB_LIBRARY leveldb) 43 | message("-- Found ${LEVELDB_LIBRARY}") 44 | 45 | find_library(CONSENSUS_YARAFT_LIBRARY consensus_yaraft) 46 | message("-- Found ${CONSENSUS_YARAFT_LIBRARY}") 47 | find_library(YARAFT_LIBRARY yaraft) 48 | message("-- Found ${YARAFT_LIBRARY}") 49 | 50 | include_directories(${YARAFT_THIRDPARTY_DIR}/include) 51 | include_directories(${CONSENSUS_YARAFT_THIRDPARTY_DIR}/include) 52 | include_directories(${CONSENSUS_YARAFT_DIR}/output/include) 53 | include_directories(${BRPC_THIRDPARTY_DIR}/include) 54 | include_directories(src) 55 | 56 | add_subdirectory(src) -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | PROJECT_DIR=`pwd` 6 | 7 | function usage() 8 | { 9 | echo "usage: run.sh []" 10 | echo 11 | echo "Command list:" 12 | echo " help print the help info" 13 | echo " build build the system" 14 | echo " test run unit test" 15 | echo 16 | echo "Command 'run.sh -h' will print help for subcommands." 17 | } 18 | 19 | ##################### 20 | ## build 21 | ##################### 22 | 23 | function run_build() { 24 | pushd ${PROJECT_DIR} 25 | mkdir -p ${PROJECT_DIR}/cmake-build-debug 26 | mkdir -p ${PROJECT_DIR}/output 27 | cd ${PROJECT_DIR}/cmake-build-debug 28 | cmake .. -DCMAKE_INSTALL_PREFIX=${PROJECT_DIR}/output -DCMAKE_BUILD_TYPE=Debug 29 | make -j8 && make install 30 | popd 31 | } 32 | 33 | ##################### 34 | ## unit test 35 | ##################### 36 | 37 | TEST_DIR=cmake-build-debug/src 38 | 39 | function unit_test() 40 | { 41 | echo "===========" $1 "===========" 42 | ./$TEST_DIR/$1 43 | if [ $? -ne 0 ]; then 44 | echo "TEST FAILED!!!" 45 | exit 1 46 | fi 47 | } 48 | 49 | 50 | function run_test() { 51 | unit_test env_test 52 | unit_test coding_test 53 | unit_test background_worker_test 54 | unit_test random_test 55 | 56 | unit_test log_writer_test 57 | unit_test log_manager_test 58 | 59 | unit_test raft_service_test 60 | unit_test raft_timer_test 61 | unit_test raft_task_executor_test 62 | # unit_test replicated_log_test 63 | } 64 | 65 | #################################################################### 66 | 67 | if [ $# -eq 0 ]; then 68 | usage 69 | exit 0 70 | fi 71 | 72 | cmd=$1 73 | case $cmd in 74 | help) 75 | usage 76 | ;; 77 | build) 78 | shift 79 | run_build $* 80 | ;; 81 | test) 82 | shift 83 | run_test $* 84 | ;; 85 | *) 86 | echo "ERROR: unknown command $cmd" 87 | echo 88 | usage 89 | exit -1 90 | esac 91 | -------------------------------------------------------------------------------- /src/wal/wal_bench.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "base/logging.h" 18 | #include "base/testing.h" 19 | #include "wal/wal.h" 20 | 21 | #include 22 | 23 | using yaraft::EntryVec; 24 | 25 | using namespace consensus; 26 | using namespace consensus::wal; 27 | 28 | void WalBench(benchmark::State& state) { 29 | TestDirectoryHelper dirHelper("/tmp/consensus-wal-bench"); 30 | WriteAheadLogUPtr wal(TEST_CreateWalStore(dirHelper.GetTestDir())); 31 | 32 | size_t per_size = state.range(0); 33 | int num_entries = state.range(1); 34 | 35 | // prepare data for writing 36 | size_t totalBytes = 0; 37 | std::string data = std::string(per_size, 'a'); 38 | 39 | EntryVec entries; 40 | for (uint64_t i = 0; i < num_entries; i++) { 41 | entries.push_back(yaraft::PBEntry().Index(i + 1).Term(1).Data(data).v); 42 | totalBytes += entries.back().ByteSize(); 43 | } 44 | 45 | // start benchmark 46 | while (state.KeepRunning()) { 47 | FATAL_NOT_OK(wal->Write(entries), "WriteAheadLog::Write"); 48 | } 49 | 50 | state.SetBytesProcessed(state.iterations() * totalBytes); 51 | } 52 | 53 | BENCHMARK(WalBench) 54 | ->Args({1000, 1}) 55 | ->Args({1000, 10}) 56 | ->Args({1000, 100}) 57 | ->Args({1000, 500}) 58 | ->Args({1000, 1000}) 59 | ->Args({10000, 1}) 60 | ->Args({10000, 10}) 61 | ->Args({10000, 100}) 62 | ->Args({10000, 500}) 63 | ->Args({10000, 1000}) 64 | ->Unit(benchmark::kMillisecond); 65 | 66 | BENCHMARK_MAIN(); -------------------------------------------------------------------------------- /src/base/random_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/random.h" 16 | #include "base/testing.h" 17 | 18 | using namespace consensus; 19 | 20 | class RandomTest : public BaseTest { 21 | protected: 22 | RandomTest() : rng_(SeedRandom()) {} 23 | 24 | Random rng_; 25 | 26 | static const int kLenMax = 100; 27 | static const int kNumTrials = 100; 28 | }; 29 | 30 | namespace { 31 | 32 | // Checks string defined at start is set to \0 everywhere but [from, to) 33 | void CheckEmpty(char* start, int from, int to, int stop) { 34 | DCHECK_LE(0, from); 35 | DCHECK_LE(from, to); 36 | DCHECK_LE(to, stop); 37 | for (int j = 0; (j == from ? j = to : j) < stop; ++j) { 38 | CHECK_EQ(start[j], '\0') << "Index " << j << " not null after defining" 39 | << "indices [" << from << "," << to << ") of " 40 | << "a nulled string [0," << stop << ")."; 41 | } 42 | } 43 | 44 | } // anonymous namespace 45 | 46 | // Makes sure that RandomString only writes the specified amount 47 | TEST_F(RandomTest, TestRandomString) { 48 | char start[kLenMax]; 49 | 50 | for (int i = 0; i < kNumTrials; ++i) { 51 | memset(start, '\0', kLenMax); 52 | int to = rng_.Uniform(kLenMax + 1); 53 | int from = rng_.Uniform(to + 1); 54 | RandomString(start + from, to - from, &rng_); 55 | CheckEmpty(start, from, to, kLenMax); 56 | } 57 | 58 | // Corner case 59 | memset(start, '\0', kLenMax); 60 | RandomString(start, 0, &rng_); 61 | CheckEmpty(start, 0, 0, kLenMax); 62 | } 63 | -------------------------------------------------------------------------------- /src/replicated_log.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "raft_service.h" 18 | #include "raft_task_executor.h" 19 | #include "replicated_log.h" 20 | #include "replicated_log_impl.h" 21 | 22 | namespace consensus { 23 | 24 | Status ReplicatedLog::Write(const Slice &log) { 25 | Status s; 26 | SimpleChannel chan = impl_->AsyncWrite(log); 27 | chan >>= s; 28 | return s; 29 | } 30 | 31 | StatusWith ReplicatedLog::New(const ReplicatedLogOptions &options) { 32 | return ReplicatedLogImpl::New(options); 33 | } 34 | 35 | RaftTaskExecutor *ReplicatedLog::RaftTaskExecutorInstance() const { 36 | return impl_->executor_.get(); 37 | } 38 | 39 | ReplicatedLog::~ReplicatedLog() {} 40 | 41 | SimpleChannel ReplicatedLog::AsyncWrite(const Slice &log) { 42 | return impl_->AsyncWrite(log); 43 | } 44 | 45 | uint64_t ReplicatedLog::Id() const { 46 | return impl_->Id(); 47 | } 48 | 49 | Status ReplicatedLogOptions::Validate() const { 50 | #define ConfigNotNull(var) \ 51 | if ((var) == nullptr) \ 52 | return FMT_Status(BadConfig, "ReplicatedLogOptions::" #var " should not be null"); 53 | ConfigNotNull(wal); 54 | 55 | // memstore is allowed to be null, when no log exists. 56 | 57 | return Status::OK(); 58 | } 59 | 60 | ReplicatedLogOptions::ReplicatedLogOptions() 61 | : heartbeat_interval(100), 62 | election_timeout(10 * 1000), 63 | taskQueue(nullptr), 64 | flusher(nullptr), 65 | timer(nullptr), 66 | wal(nullptr), 67 | memstore(nullptr) {} 68 | 69 | } // namespace consensus 70 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | --- 2 | Language: Cpp 3 | # BasedOnStyle: Google 4 | AccessModifierOffset: -1 5 | AlignAfterOpenBracket: true 6 | AlignConsecutiveAssignments: false 7 | AlignEscapedNewlinesLeft: true 8 | AlignOperands: true 9 | AlignTrailingComments: true 10 | AllowAllParametersOfDeclarationOnNextLine: true 11 | AllowShortBlocksOnASingleLine: false 12 | AllowShortCaseLabelsOnASingleLine: false 13 | AllowShortFunctionsOnASingleLine: Empty 14 | AllowShortIfStatementsOnASingleLine: false 15 | AllowShortLoopsOnASingleLine: false 16 | AlwaysBreakAfterDefinitionReturnType: None 17 | AlwaysBreakBeforeMultilineStrings: true 18 | AlwaysBreakTemplateDeclarations: true 19 | BinPackArguments: true 20 | BinPackParameters: true 21 | BreakBeforeBinaryOperators: None 22 | BreakBeforeBraces: Attach 23 | BreakBeforeTernaryOperators: true 24 | BreakConstructorInitializersBeforeComma: false 25 | ColumnLimit: 100 26 | CommentPragmas: '' 27 | ConstructorInitializerAllOnOneLineOrOnePerLine: true 28 | ConstructorInitializerIndentWidth: 4 29 | ContinuationIndentWidth: 4 30 | Cpp11BracedListStyle: true 31 | DerivePointerAlignment: true 32 | DisableFormat: false 33 | ExperimentalAutoDetectBinPacking: false 34 | ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] 35 | IndentCaseLabels: true 36 | IndentWidth: 2 37 | IndentWrappedFunctionNames: false 38 | KeepEmptyLinesAtTheStartOfBlocks: false 39 | MacroBlockBegin: '' 40 | MacroBlockEnd: '' 41 | MaxEmptyLinesToKeep: 1 42 | NamespaceIndentation: None 43 | ObjCBlockIndentWidth: 2 44 | ObjCSpaceAfterProperty: false 45 | ObjCSpaceBeforeProtocolList: false 46 | PenaltyBreakBeforeFirstCallParameter: 1 47 | PenaltyBreakComment: 300 48 | PenaltyBreakFirstLessLess: 120 49 | PenaltyBreakString: 1000 50 | PenaltyExcessCharacter: 1000000 51 | PenaltyReturnTypeOnItsOwnLine: 200 52 | PointerAlignment: Left 53 | SpaceAfterCStyleCast: false 54 | SpaceBeforeAssignmentOperators: true 55 | SpaceBeforeParens: ControlStatements 56 | SpaceInEmptyParentheses: false 57 | SpacesBeforeTrailingComments: 2 58 | SpacesInAngles: false 59 | SpacesInContainerLiterals: true 60 | SpacesInCStyleCastParentheses: false 61 | SpacesInParentheses: false 62 | SpacesInSquareBrackets: false 63 | Standard: Cpp11 64 | TabWidth: 8 65 | UseTab: Never 66 | -------------------------------------------------------------------------------- /src/wal_commit_observer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "base/logging.h" 19 | 20 | #include "wal_commit_observer.h" 21 | 22 | namespace consensus { 23 | 24 | class WalCommitObserver::Impl { 25 | public: 26 | void Register(std::pair range, SimpleChannel *channel) { 27 | std::lock_guard g(mu_); 28 | 29 | WalWritesMap::const_iterator it = writes_.find(range); 30 | if (it != writes_.end()) { 31 | FMT_LOG(ERROR, 32 | "WalCommitObserver::Register: duplicate calls for a range of entries in [{}, {}]", 33 | range.first, range.second); 34 | } else { 35 | writes_[range] = channel; 36 | } 37 | } 38 | 39 | void Notify(uint64_t commitIndex) { 40 | std::lock_guard g(mu_); 41 | 42 | auto it = writes_.begin(); 43 | for (; it != writes_.end(); it++) { 44 | if (commitIndex >= it->first.second) { 45 | (*it->second) <<= Status::OK(); 46 | } 47 | writes_.erase(it); 48 | } 49 | } 50 | 51 | private: 52 | typedef std::map, SimpleChannel *> WalWritesMap; 53 | WalWritesMap writes_; 54 | 55 | std::mutex mu_; 56 | }; 57 | 58 | void WalCommitObserver::Register(std::pair range, 59 | SimpleChannel *channel) { 60 | impl_->Register(range, channel); 61 | } 62 | 63 | void WalCommitObserver::Notify(uint64_t commitIndex) { 64 | impl_->Notify(commitIndex); 65 | } 66 | 67 | WalCommitObserver::~WalCommitObserver() = default; 68 | 69 | WalCommitObserver::WalCommitObserver() : impl_(new Impl) {} 70 | 71 | } // namespace consensus -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # consensus-yaraft 2 | [![Build Status](https://travis-ci.org/neverchanje/consensus-yaraft.svg)](https://travis-ci.org/neverchanje/consensus-yaraft) 3 | 4 | consensus-yaraft is an embeddable distributed log storage library that provides strong consistency based 5 | on the Raft algorithm. 6 | 7 | ## Features 8 | 9 | - **Fault tolerance & Strong consistency**: every single log will be synchronously replicated through the [Raft](raft) state machine. Failure of minority doesn't impede progress. 10 | - **Multi Raft**: A process creates 100 raft nodes doesn't have to create 100 threads. The background timer, the disk io thread pool (`ReadyFlusher`), even the FSM task queue, can be shared between raft nodes. 11 | - **Simple**: the basic operation for writing a slice of log includes only `ReplicatedLog::Write`. 12 | 13 | [raft]: https://raft.github.io/ 14 | 15 | ## Limitation 16 | 17 | - The library doesn't provide with client-interaction support. 18 | - Writes can only be applied to the leader. Any write operations to a non-leader will get rejected. 19 | 20 | ## Installation 21 | 22 | consensus-yaraft is written in C++11, please ensure a compiler with C++11 support is installed. 23 | 24 | Ensure you have cmake, unzip, libtool, autoconf installed on your system. 25 | 26 | Ensure you have leveldb, zlib, openssl installed, because 27 | [brpc relies on them](https://github.com/brpc/brpc/blob/master/docs/cn/getting_started.md). 28 | 29 | On Ubuntu 14.04 30 | 31 | ```sh 32 | sudo apt install zlib1g-dev libboost-all-dev libsnappy-dev 33 | bash install_dependencies.sh 34 | bash compile_proto.sh 35 | bash run.sh build 36 | ``` 37 | 38 | Once the building becomes success, the library would be installed in the directory `output/`. 39 | 40 | ## Internals 41 | 42 | For more details about the architecture and design of consensus-yaraft, please read 43 | [this article](https://github.com/neverchanje/consensus-yaraft/wiki). 44 | Currently we relies on [brpc](brpc) to implement network communication. 45 | 46 | ## MemKV 47 | 48 | [apps/memkv](apps/memkv) is a prototype of using consensus-yaraft to implement a raft-based in-memory key-value store. 49 | 50 | ## License 51 | 52 | consensus-yaraft is under the Apache 2.0 license. See the LICENSE file for details. 53 | 54 | [brpc]: https://github.com/brpc/brpc 55 | [raft]: https://raft.github.io 56 | -------------------------------------------------------------------------------- /apps/memkv/src/testing.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "logging.h" 18 | 19 | #include 20 | 21 | #define ASSERT_OK(status) \ 22 | do { \ 23 | const auto &_s = status; \ 24 | if (_s.IsOK()) { \ 25 | SUCCEED(); \ 26 | } else { \ 27 | FAIL() << "Bad status: " << _s.ToString(); \ 28 | } \ 29 | } while (0) 30 | 31 | #define ASSIGN_IF_ASSERT_OK(statusWith, var) \ 32 | do { \ 33 | auto _sw = statusWith; \ 34 | ASSERT_OK(_sw); \ 35 | (var) = _sw.GetValue(); \ 36 | } while (0) 37 | 38 | #define ASSERT_ERROR(status, err) \ 39 | do { \ 40 | const auto &_s = status; \ 41 | if (_s.Code() == err) { \ 42 | SUCCEED(); \ 43 | } else { \ 44 | FAIL() << "expected error is: " << Error::toString(static_cast(err)) \ 45 | << ", actual error is: " << _s.ToString(); \ 46 | } \ 47 | } while (0) 48 | -------------------------------------------------------------------------------- /src/replicated_log_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/testing.h" 16 | 17 | #include "replicated_log_impl.h" 18 | 19 | using namespace consensus; 20 | 21 | class ReplicatedLogTest : public BaseTest { 22 | public: 23 | void SetUp() override { 24 | options.initial_cluster[1] = "127.0.0.1:12321"; 25 | options.id = 1; 26 | options.heartbeat_interval = 100; 27 | options.election_timeout = 1000; 28 | options.memstore = new yaraft::MemoryStorage; 29 | 30 | yaraft::MemStoreUptr memstore; 31 | options.wal = wal::TEST_CreateWalStore(GetTestDir(), &memstore).release(); 32 | options.memstore = memstore.release(); 33 | } 34 | 35 | void TearDown() override { 36 | delete replicatedLog; 37 | delete options.wal; 38 | } 39 | 40 | protected: 41 | ReplicatedLog* replicatedLog; 42 | ReplicatedLogOptions options; 43 | }; 44 | 45 | TEST_F(ReplicatedLogTest, WriteToNonLeader) { 46 | TestDirGuard g(CreateTestDirGuard()); 47 | ASSIGN_IF_ASSERT_OK(ReplicatedLog::New(options), replicatedLog); 48 | 49 | Status s = replicatedLog->Write("abc"); 50 | ASSERT_EQ(s.Code(), Error::WalWriteToNonLeader); 51 | } 52 | 53 | TEST_F(ReplicatedLogTest, WriteLeader) { 54 | TestDirGuard g(CreateTestDirGuard()); 55 | ASSIGN_IF_ASSERT_OK(ReplicatedLog::New(options), replicatedLog); 56 | 57 | while (replicatedLog->GetInfo().currentTerm != 1) { 58 | sleep(1); 59 | } 60 | 61 | for (int i = 0; i < 4; i++) { 62 | Status s = replicatedLog->Write("abc"); 63 | ASSERT_OK(s); 64 | } 65 | 66 | ASSERT_EQ(replicatedLog->GetInfo().logIndex, 5); 67 | ASSERT_EQ(replicatedLog->GetInfo().commitIndex, 5); 68 | ASSERT_EQ(replicatedLog->GetInfo().currentTerm, 1); 69 | ASSERT_EQ(replicatedLog->GetInfo().currentLeader, 1); 70 | } -------------------------------------------------------------------------------- /cmake_modules/FindASan.cmake: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 4 | # 2013 Matthew Arsenault 5 | # 2015-2016 RWTH Aachen University, Federal Republic of Germany 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | option(SANITIZE_ADDRESS "Enable AddressSanitizer for sanitized targets." Off) 26 | 27 | set(FLAG_CANDIDATES 28 | # Clang 3.2+ use this version. The no-omit-frame-pointer option is optional. 29 | "-g -fsanitize=address -fno-omit-frame-pointer" 30 | "-g -fsanitize=address" 31 | 32 | # Older deprecated flag for ASan 33 | "-g -faddress-sanitizer" 34 | ) 35 | 36 | 37 | if (SANITIZE_ADDRESS AND (SANITIZE_THREAD OR SANITIZE_MEMORY)) 38 | message(FATAL_ERROR "AddressSanitizer is not compatible with " 39 | "ThreadSanitizer or MemorySanitizer.") 40 | endif () 41 | 42 | 43 | include(sanitize-helpers) 44 | 45 | if (SANITIZE_ADDRESS) 46 | sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "AddressSanitizer" 47 | "ASan") 48 | 49 | find_program(ASan_WRAPPER "asan-wrapper" PATHS ${CMAKE_MODULE_PATH}) 50 | mark_as_advanced(ASan_WRAPPER) 51 | endif () 52 | 53 | function (add_sanitize_address TARGET) 54 | if (NOT SANITIZE_ADDRESS) 55 | return() 56 | endif () 57 | 58 | saitizer_add_flags(${TARGET} "AddressSanitizer" "ASan") 59 | endfunction () 60 | -------------------------------------------------------------------------------- /src/base/background_worker.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "base/background_worker.h" 19 | 20 | namespace consensus { 21 | 22 | class BackgroundWorker::Impl { 23 | public: 24 | explicit Impl() : stopped_(true) {} 25 | 26 | Status StartLoop(std::function recurringTask) { 27 | if (bgWorkerThread_.joinable()) { 28 | return Status::Make(Error::RuntimeError, 29 | "BackgroundWorker::StartLoop: start an already started thread"); 30 | } 31 | 32 | if (!recurringTask) { 33 | return Status::Make(Error::InvalidArgument, 34 | "BackgroundWorker::StartLoop: the given task has no target"); 35 | } 36 | 37 | stopped_.store(false); 38 | bgWorkerThread_ = std::thread([=]() { 39 | while (!stopped_.load()) { 40 | recurringTask(); 41 | } 42 | }); 43 | return Status::OK(); 44 | } 45 | 46 | Status Stop() { 47 | if (!bgWorkerThread_.joinable()) { 48 | return Status::Make(Error::RuntimeError, 49 | "BackgroundWorker::Stop: stop a not-yet-started thread"); 50 | } 51 | 52 | stopped_.store(true); 53 | bgWorkerThread_.join(); 54 | return Status::OK(); 55 | } 56 | 57 | bool Stopped() const { 58 | return stopped_.load(); 59 | } 60 | 61 | private: 62 | std::thread bgWorkerThread_; 63 | std::atomic_bool stopped_; 64 | }; 65 | 66 | BackgroundWorker::BackgroundWorker() : impl_(new Impl) {} 67 | 68 | Status BackgroundWorker::StartLoop(std::function recurringTask) { 69 | return impl_->StartLoop(recurringTask); 70 | } 71 | 72 | Status BackgroundWorker::Stop() { 73 | return impl_->Stop(); 74 | } 75 | 76 | bool BackgroundWorker::Stopped() const { 77 | return impl_->Stopped(); 78 | } 79 | 80 | BackgroundWorker::~BackgroundWorker() {} 81 | 82 | } // namespace consensus -------------------------------------------------------------------------------- /cmake_modules/FindMSan.cmake: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 4 | # 2013 Matthew Arsenault 5 | # 2015-2016 RWTH Aachen University, Federal Republic of Germany 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | option(SANITIZE_MEMORY "Enable MemorySanitizer for sanitized targets." Off) 26 | 27 | set(FLAG_CANDIDATES 28 | "-g -fsanitize=memory" 29 | ) 30 | 31 | 32 | include(sanitize-helpers) 33 | 34 | if (SANITIZE_MEMORY) 35 | if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux") 36 | message(WARNING "MemorySanitizer disabled for target ${TARGET} because " 37 | "MemorySanitizer is supported for Linux systems only.") 38 | set(SANITIZE_MEMORY Off CACHE BOOL 39 | "Enable MemorySanitizer for sanitized targets." FORCE) 40 | elseif (NOT ${CMAKE_SIZEOF_VOID_P} EQUAL 8) 41 | message(WARNING "MemorySanitizer disabled for target ${TARGET} because " 42 | "MemorySanitizer is supported for 64bit systems only.") 43 | set(SANITIZE_MEMORY Off CACHE BOOL 44 | "Enable MemorySanitizer for sanitized targets." FORCE) 45 | else () 46 | sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "MemorySanitizer" 47 | "MSan") 48 | endif () 49 | endif () 50 | 51 | function (add_sanitize_memory TARGET) 52 | if (NOT SANITIZE_MEMORY) 53 | return() 54 | endif () 55 | 56 | saitizer_add_flags(${TARGET} "MemorySanitizer" "MSan") 57 | endfunction () 58 | -------------------------------------------------------------------------------- /include/consensus/base/stl_container_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2002 Google Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | namespace consensus { 18 | 19 | // STLDeleteContainerPairSecondPointers() 20 | // For a range within a container of pairs, calls delete 21 | // (non-array version) on the SECOND item in the pairs. 22 | // NOTE: Like STLDeleteContainerPointers, deleting behind the iterator. 23 | // Deleting the value does not always invalidate the iterator, but it may 24 | // do so if the key is a pointer into the value object. 25 | // NOTE: If you're calling this on an entire container, you probably want 26 | // to call STLDeleteValues(&container) instead, or use ValueDeleter. 27 | template 28 | void STLDeleteContainerPairSecondPointers(ForwardIterator begin, ForwardIterator end) { 29 | while (begin != end) { 30 | ForwardIterator temp = begin; 31 | ++begin; 32 | delete temp->second; 33 | } 34 | } 35 | 36 | // STLDeleteContainerPointers() 37 | // For a range within a container of pointers, calls delete 38 | // (non-array version) on these pointers. 39 | // NOTE: for these three functions, we could just implement a DeleteObject 40 | // functor and then call for_each() on the range and functor, but this 41 | // requires us to pull in all of , which seems expensive. 42 | // For hash_[multi]set, it is important that this deletes behind the iterator 43 | // because the hash_set may call the hash function on the iterator when it is 44 | // advanced, which could result in the hash function trying to deference a 45 | // stale pointer. 46 | // NOTE: If you're calling this on an entire container, you probably want 47 | // to call STLDeleteElements(&container) instead, or use an ElementDeleter. 48 | template 49 | void STLDeleteContainerPointers(ForwardIterator begin, ForwardIterator end) { 50 | while (begin != end) { 51 | ForwardIterator temp = begin; 52 | ++begin; 53 | delete *temp; 54 | } 55 | } 56 | 57 | } // namespace consensus -------------------------------------------------------------------------------- /src/raft_timer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "raft_task_executor.h" 18 | #include "raft_timer.h" 19 | 20 | #include "base/background_worker.h" 21 | #include "base/logging.h" 22 | #include "base/simple_channel.h" 23 | 24 | #include 25 | #include 26 | 27 | namespace consensus { 28 | 29 | // Timeout granularity of timer in milli-seconds. 30 | static uint32_t kTimerGranularity = 100; 31 | 32 | class RaftTimer::Impl { 33 | public: 34 | Impl() : timer_(io_service_) { 35 | FMT_LOG(INFO, "Set up raft timer with timeout granularity: {}ms", kTimerGranularity); 36 | } 37 | 38 | void Stop() { 39 | FATAL_NOT_OK(worker_.Stop(), "RaftTimer::Stop"); 40 | 41 | io_service_.stop(); 42 | } 43 | 44 | void Start() { 45 | io_service_.run(); 46 | 47 | worker_.StartLoop([&]() { 48 | timer_.expires_from_now(boost::posix_time::milliseconds(kTimerGranularity)); 49 | timer_.wait(); 50 | 51 | std::lock_guard g(mu_); 52 | for (auto& executor : executors_) { 53 | Barrier channel; 54 | executor->Submit([&](yaraft::RawNode* node) { 55 | for (uint32_t i = 0; i < kTimerGranularity; i++) { 56 | node->Tick(); 57 | } 58 | channel.Signal(); 59 | }); 60 | channel.Wait(); 61 | } 62 | }); 63 | } 64 | 65 | void Register(RaftTaskExecutor* executor) { 66 | std::lock_guard g(mu_); 67 | executors_.push_back(executor); 68 | } 69 | 70 | private: 71 | std::vector executors_; 72 | std::mutex mu_; 73 | 74 | boost::asio::io_service io_service_; 75 | boost::asio::deadline_timer timer_; 76 | 77 | BackgroundWorker worker_; 78 | }; 79 | 80 | RaftTimer::RaftTimer() : impl_(new Impl) { 81 | impl_->Start(); 82 | } 83 | 84 | RaftTimer::~RaftTimer() { 85 | impl_->Stop(); 86 | } 87 | 88 | void RaftTimer::Register(RaftTaskExecutor* executor) { 89 | impl_->Register(executor); 90 | } 91 | 92 | } // namespace consensus 93 | -------------------------------------------------------------------------------- /cmake_modules/FindTSan.cmake: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 4 | # 2013 Matthew Arsenault 5 | # 2015-2016 RWTH Aachen University, Federal Republic of Germany 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | option(SANITIZE_THREAD "Enable ThreadSanitizer for sanitized targets." Off) 26 | 27 | set(FLAG_CANDIDATES 28 | "-g -fsanitize=thread" 29 | ) 30 | 31 | 32 | # ThreadSanitizer is not compatible with MemorySanitizer. 33 | if (SANITIZE_THREAD AND SANITIZE_MEMORY) 34 | message(FATAL_ERROR "ThreadSanitizer is not compatible with " 35 | "MemorySanitizer.") 36 | endif () 37 | 38 | 39 | include(sanitize-helpers) 40 | 41 | if (SANITIZE_THREAD) 42 | if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Linux") 43 | message(WARNING "ThreadSanitizer disabled for target ${TARGET} because " 44 | "ThreadSanitizer is supported for Linux systems only.") 45 | set(SANITIZE_THREAD Off CACHE BOOL 46 | "Enable ThreadSanitizer for sanitized targets." FORCE) 47 | elseif (NOT ${CMAKE_SIZEOF_VOID_P} EQUAL 8) 48 | message(WARNING "ThreadSanitizer disabled for target ${TARGET} because " 49 | "ThreadSanitizer is supported for 64bit systems only.") 50 | set(SANITIZE_THREAD Off CACHE BOOL 51 | "Enable ThreadSanitizer for sanitized targets." FORCE) 52 | else () 53 | sanitizer_check_compiler_flags("${FLAG_CANDIDATES}" "ThreadSanitizer" 54 | "TSan") 55 | endif () 56 | endif () 57 | 58 | function (add_sanitize_thread TARGET) 59 | if (NOT SANITIZE_THREAD) 60 | return() 61 | endif () 62 | 63 | saitizer_add_flags(${TARGET} "ThreadSanitizer" "TSan") 64 | endfunction () 65 | -------------------------------------------------------------------------------- /src/raft_service.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "raft_service.h" 16 | #include "raft_task_executor.h" 17 | #include "raft_timer.h" 18 | 19 | #include "base/logging.h" 20 | #include "base/simple_channel.h" 21 | 22 | #include 23 | 24 | namespace consensus { 25 | 26 | static pb::StatusCode yaraftErrorCodeToRpcStatusCode(yaraft::Error::ErrorCodes code) { 27 | switch (code) { 28 | case yaraft::Error::StepLocalMsg: 29 | return pb::StepLocalMsg; 30 | case yaraft::Error::StepPeerNotFound: 31 | return pb::StepPeerNotFound; 32 | default: 33 | LOG(FATAL) << "Unexpected error code: " << yaraft::Error::toString(code); 34 | return pb::OK; 35 | } 36 | } 37 | 38 | void RaftServiceImpl::Step(google::protobuf::RpcController *controller, 39 | const pb::StepRequest *request, pb::StepResponse *response, 40 | google::protobuf::Closure *done) { 41 | yaraft::pb::Message *msg = const_cast(request)->mutable_message(); 42 | 43 | response->set_code(pb::OK); 44 | 45 | Barrier barrier; 46 | executor_->Submit(std::bind( 47 | [&](yaraft::RawNode *node) { 48 | auto s = node->Step(*msg); 49 | if (UNLIKELY(!s.IsOK())) { 50 | response->set_code(yaraftErrorCodeToRpcStatusCode(s.Code())); 51 | } 52 | barrier.Signal(); 53 | }, 54 | std::placeholders::_1)); 55 | barrier.Wait(); 56 | 57 | done->Run(); 58 | } 59 | 60 | void RaftServiceImpl::Status(::google::protobuf::RpcController *controller, 61 | const pb::StatusRequest *request, pb::StatusResponse *response, 62 | ::google::protobuf::Closure *done) { 63 | Barrier barrier; 64 | executor_->Submit(std::bind( 65 | [&](yaraft::RawNode *node) { 66 | response->set_leader(node->LeaderHint()); 67 | response->set_raftindex(node->LastIndex()); 68 | response->set_raftterm(node->CurrentTerm()); 69 | 70 | barrier.Signal(); 71 | }, 72 | std::placeholders::_1)); 73 | barrier.Wait(); 74 | 75 | done->Run(); 76 | } 77 | 78 | } // namespace consensus 79 | -------------------------------------------------------------------------------- /include/consensus/wal/wal.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "consensus/base/status.h" 18 | 19 | #include 20 | #include 21 | 22 | namespace consensus { 23 | namespace wal { 24 | 25 | using PBEntryVec = std::vector; 26 | using PBEntriesIterator = PBEntryVec::iterator; 27 | using ConstPBEntriesIterator = PBEntryVec::const_iterator; 28 | 29 | class WriteAheadLogOptions { 30 | public: 31 | // Approximate size of wal packed per segment. 32 | // Default: 64MB 33 | size_t log_segment_size; 34 | 35 | // Whether to verify the checksum of each entry during recovery. 36 | // Default: true 37 | bool verify_checksum; 38 | 39 | std::string log_dir; 40 | 41 | WriteAheadLogOptions(); 42 | }; 43 | 44 | class WriteAheadLog; 45 | using WriteAheadLogUPtr = std::unique_ptr; 46 | 47 | // WriteAheadLog provides an abstraction for writing log entries and raft state 48 | // into the underlying storage. 49 | class WriteAheadLog { 50 | public: 51 | virtual ~WriteAheadLog() = default; 52 | 53 | // Save log entries and raft state into underlying storage. 54 | virtual Status Write(const PBEntryVec& vec, const yaraft::pb::HardState* hs) = 0; 55 | 56 | // write log entries only 57 | Status Write(const PBEntryVec& vec) { 58 | return Write(vec, nullptr); 59 | } 60 | 61 | // write hard state only 62 | Status Write(const yaraft::pb::HardState* hs) { 63 | return Write(PBEntryVec(), hs); 64 | } 65 | 66 | virtual Status Sync() = 0; 67 | 68 | virtual Status Close() = 0; 69 | 70 | struct CompactionHint {}; 71 | 72 | // Abandon the unused logs. 73 | virtual Status GC(CompactionHint* hint) = 0; 74 | 75 | // Default implementation of WAL. 76 | static Status Default(const WriteAheadLogOptions& options, WriteAheadLogUPtr* wal, 77 | yaraft::MemStoreUptr* memstore); 78 | }; 79 | 80 | extern WriteAheadLogUPtr TEST_CreateWalStore(const std::string& testDir, yaraft::MemStoreUptr* pMemstore); 81 | 82 | inline WriteAheadLogUPtr TEST_CreateWalStore(const std::string& testDir) { 83 | yaraft::MemStoreUptr memstore; 84 | return TEST_CreateWalStore(testDir, &memstore); 85 | } 86 | 87 | } // namespace wal 88 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/src/memkv_server.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "db.h" 16 | #include "logging.h" 17 | #include "memkv_service.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | using namespace memkv; 24 | 25 | DEFINE_uint64(id, 1, "one of the values in {1, 2, 3}"); 26 | DEFINE_string(wal_dir, "", "directory to store wal"); 27 | DEFINE_int32(server_count, 3, "number of servers in the cluster"); 28 | DEFINE_string(memkv_log_dir, "", 29 | "If specified, logfiles are written into this directory instead " 30 | "of the default logging directory."); 31 | 32 | void InitLogging(const char* argv0) { 33 | google::InitGoogleLogging(argv0); 34 | FLAGS_log_dir = FLAGS_memkv_log_dir; 35 | if (!FLAGS_log_dir.empty()) { 36 | FATAL_NOT_OK(consensus::Env::Default()->CreateDirIfMissing(FLAGS_log_dir), FLAGS_log_dir); 37 | } else { 38 | // print to stderr when log_dir is not specified. 39 | FLAGS_logtostderr = true; 40 | } 41 | yaraft::SetLogger(boost::make_unique()); 42 | } 43 | 44 | int main(int argc, char* argv[]) { 45 | gflags::ParseCommandLineFlags(&argc, &argv, true); 46 | InitLogging(argv[0]); 47 | 48 | // 49 | // -- create DB instance -- 50 | // 51 | DBOptions options; 52 | options.member_id = FLAGS_id; 53 | options.wal_dir = FLAGS_wal_dir; 54 | for (int i = 1; i <= FLAGS_server_count; i++) { 55 | // TODO: initial_cluster should be configured by user 56 | options.initial_cluster[i] = fmt::format("127.0.0.1:{}", 12320 + i); 57 | } 58 | auto sw = DB::Bootstrap(options); 59 | if (!sw.IsOK()) { 60 | LOG(FATAL) << sw.GetStatus(); 61 | } 62 | DB* db = sw.GetValue(); 63 | 64 | // 65 | // -- start memkv server -- 66 | // 67 | FMT_LOG(INFO, "Starting memkv server {} at {}", FLAGS_id, options.initial_cluster[FLAGS_id]); 68 | FMT_LOG(INFO, "--wal_dir: {}", FLAGS_wal_dir); 69 | brpc::ServerOptions opts; 70 | opts.num_threads = 1; 71 | brpc::Server server; 72 | server.AddService(new MemKVServiceImpl(db), brpc::SERVER_OWNS_SERVICE); 73 | server.AddService(db->CreateRaftServiceInstance(), brpc::SERVER_OWNS_SERVICE); 74 | server.Start(options.initial_cluster[FLAGS_id].c_str(), &opts); 75 | server.RunUntilAskedToQuit(); 76 | 77 | return 0; 78 | } -------------------------------------------------------------------------------- /include/consensus/replicated_log.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | #include "consensus/base/simple_channel.h" 21 | #include "consensus/base/slice.h" 22 | #include "consensus/base/status.h" 23 | #include "consensus/base/task_queue.h" 24 | #include "consensus/raft_timer.h" 25 | #include "consensus/ready_flusher.h" 26 | #include "consensus/wal/wal.h" 27 | 28 | #include 29 | 30 | namespace consensus { 31 | 32 | struct ReplicatedLogOptions { 33 | // id -> IP 34 | std::map initial_cluster; 35 | 36 | uint64_t id; 37 | 38 | // time (in milliseconds) of a heartbeat interval. 39 | uint32_t heartbeat_interval; 40 | 41 | // time (in milliseconds) for an election to timeout. 42 | uint32_t election_timeout; 43 | 44 | // dedicated worker of the raft node. 45 | // there may have multiple instances sharing the same queue. 46 | TaskQueue* taskQueue; 47 | 48 | // the global timer 49 | RaftTimer* timer; 50 | 51 | // the global ready flusher 52 | ReadyFlusher* flusher; 53 | 54 | wal::WriteAheadLog* wal; 55 | yaraft::MemoryStorage* memstore; 56 | 57 | ReplicatedLogOptions(); 58 | 59 | Status Validate() const; 60 | }; 61 | 62 | // A ReplicatedLog is a distributed log storage with strong consistency. Every single 63 | // write will be replicated to a majority of nodes if successes. 64 | // If the current node is not leader, the write fails immediately. 65 | // 66 | // Not Thread Safe 67 | class ReplicatedLog { 68 | __DISALLOW_COPYING__(ReplicatedLog); 69 | 70 | public: 71 | static StatusWith New(const ReplicatedLogOptions&); 72 | 73 | // Write a slice of log in synchronous way. 74 | // Returns error `WalWriteToNonLeader` if the current node is not leader. 75 | Status Write(const Slice& log); 76 | 77 | // Asynchronously write a slice of log into storage, the call will returns immediately 78 | // with a SimpleChannel that's used to wait for the commit of this write. 79 | SimpleChannel AsyncWrite(const Slice& log); 80 | 81 | RaftTaskExecutor* RaftTaskExecutorInstance() const; 82 | 83 | uint64_t Id() const; 84 | 85 | ~ReplicatedLog(); 86 | 87 | private: 88 | ReplicatedLog() = default; 89 | 90 | private: 91 | friend class ReplicatedLogTest; 92 | 93 | friend class ReplicatedLogImpl; 94 | std::unique_ptr impl_; 95 | }; 96 | 97 | } // namespace consensus 98 | -------------------------------------------------------------------------------- /src/wal/log_writer.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/env.h" 18 | #include "base/logging.h" 19 | #include "wal/format.h" 20 | #include "wal/log_manager.h" 21 | #include "wal/segment_meta.h" 22 | 23 | #include 24 | #include 25 | 26 | namespace consensus { 27 | namespace wal { 28 | 29 | class LogWriter { 30 | public: 31 | // Create a log writer for the new log segment. 32 | static StatusWith New(LogManager *manager) { 33 | uint64_t newSegId = manager->files_.size() + 1; 34 | uint64_t newSegStart = manager->lastIndex_ + 1; 35 | std::string fname = manager->options_.log_dir + "/" + SegmentFileName(newSegId, newSegStart); 36 | FMT_LOG(INFO, "creating new segment segId: {}, firstId: {}", newSegId, newSegStart); 37 | 38 | WritableFile *wf; 39 | ASSIGN_IF_OK(Env::Default()->NewWritableFile(fname, Env::CREATE_NON_EXISTING), wf); 40 | 41 | return new LogWriter(wf, fname, manager->options_.log_segment_size); 42 | } 43 | 44 | LogWriter(WritableFile *wf, const std::string &fname, size_t logSegmentSize) 45 | : file_(wf), empty_(true), logSegmentSize_(logSegmentSize) { 46 | meta_.fileName = fname; 47 | } 48 | 49 | // Append log entries in range [begin, end) & hard state into the underlying segment. 50 | // If the current write is beyond the configured segment size, it returns a 51 | // iterator points at the next entry to be appended. 52 | StatusWith Append(ConstPBEntriesIterator begin, 53 | ConstPBEntriesIterator end, 54 | const yaraft::pb::HardState *hs = nullptr); 55 | 56 | Status Sync() { 57 | return file_->Sync(); 58 | } 59 | 60 | Status Finish(SegmentMetaData *meta) { 61 | RETURN_NOT_OK(file_->Sync()); 62 | RETURN_NOT_OK(file_->Close()); 63 | 64 | *meta = meta_; 65 | return Status::OK(); 66 | } 67 | 68 | private: 69 | void saveHardState(const yaraft::pb::HardState &hs, char *dest, size_t *offset); 70 | 71 | void saveEntries(ConstPBEntriesIterator begin, ConstPBEntriesIterator end, char *dest, 72 | size_t *offset); 73 | 74 | private: 75 | friend class LogWriterTest; 76 | 77 | private: 78 | std::unique_ptr file_; 79 | SegmentMetaData meta_; 80 | 81 | const size_t logSegmentSize_; 82 | 83 | bool empty_; 84 | }; 85 | 86 | } // namespace wal 87 | } // namespace consensus -------------------------------------------------------------------------------- /src/wal/log_manager.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/status.h" 18 | #include "wal/segment_meta.h" 19 | #include "wal/wal.h" 20 | 21 | #include 22 | 23 | namespace consensus { 24 | 25 | class WritableFile; 26 | 27 | namespace wal { 28 | 29 | class LogWriter; 30 | 31 | class LogManager; 32 | using LogManagerUPtr = std::unique_ptr; 33 | 34 | // Not-Thread-Safe 35 | class LogManager : public WriteAheadLog { 36 | public: 37 | explicit LogManager(const WriteAheadLogOptions& options); 38 | 39 | // Recover from existing wal files. 40 | // The options.log_dir will be created when it's not existed. 41 | // All of the uncompacted log entries will be read into `memstore`. 42 | // 43 | // ASSERT: *memstore == null 44 | static Status Recover(const WriteAheadLogOptions& options, yaraft::MemStoreUptr* memstore, 45 | LogManagerUPtr* pLogManager); 46 | 47 | ~LogManager() override; 48 | 49 | // Required: no holes between logs and msg.entries. 50 | Status Write(const PBEntryVec& vec, const yaraft::pb::HardState* hs) override; 51 | 52 | // naive implementation: delete all committed segments. 53 | Status GC(WriteAheadLog::CompactionHint* hint) override; 54 | 55 | Status Sync() override; 56 | 57 | Status Close() override; 58 | 59 | // the number of log segments 60 | size_t SegmentNum() const { 61 | return files_.size() + static_cast(bool(current_)); 62 | } 63 | 64 | private: 65 | Status doWrite(ConstPBEntriesIterator begin, ConstPBEntriesIterator end, 66 | const yaraft::pb::HardState* hs); 67 | 68 | void finishCurrentWriter(); 69 | 70 | private: 71 | friend class LogManagerTest; 72 | friend class LogWriter; 73 | 74 | // current_ always writes to the last log segment when it's not null. 75 | std::unique_ptr current_; 76 | 77 | // metadata of the immutable segments 78 | // new segment will be appended when the current writer finishes 79 | std::vector files_; 80 | 81 | uint64_t lastIndex_; 82 | bool empty_; 83 | 84 | const WriteAheadLogOptions options_; 85 | }; 86 | 87 | Status AppendToMemStore(yaraft::pb::Entry& e, yaraft::MemoryStorage* memstore); 88 | 89 | inline Status AppendToMemStore(yaraft::EntryVec& vec, yaraft::MemoryStorage* memstore) { 90 | for (auto& e : vec) { 91 | auto s = AppendToMemStore(e, memstore); 92 | if (!s.IsOK()) 93 | return s; 94 | } 95 | return Status::OK(); 96 | } 97 | 98 | } // namespace wal 99 | } // namespace consensus -------------------------------------------------------------------------------- /src/wal/log_writer.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "wal/log_writer.h" 16 | #include "base/coding.h" 17 | 18 | #include 19 | 20 | namespace consensus { 21 | namespace wal { 22 | 23 | StatusWith LogWriter::Append(ConstPBEntriesIterator begin, 24 | ConstPBEntriesIterator end, 25 | const yaraft::pb::HardState *hs) { 26 | if (empty_) { 27 | RETURN_NOT_OK(file_->Append(kLogSegmentHeaderMagic)); 28 | empty_ = false; 29 | } 30 | 31 | ssize_t remains = logSegmentSize_ - file_->Size(); 32 | size_t totalSize = kLogBatchHeaderSize; 33 | 34 | if (hs) { 35 | totalSize += kRecordHeaderSize + hs->ByteSize(); 36 | } 37 | 38 | bool writeEntries = false; 39 | auto newBegin = begin; 40 | if (totalSize < remains && begin != end) { 41 | writeEntries = true; 42 | for (; newBegin != end; newBegin++) { 43 | if (totalSize < remains) { 44 | totalSize += kRecordHeaderSize + VarintLength(newBegin->ByteSize()) + newBegin->ByteSize(); 45 | } else { 46 | break; 47 | } 48 | } 49 | } 50 | 51 | std::string scratch(totalSize, '\0'); 52 | size_t offset = kLogBatchHeaderSize; 53 | 54 | if (hs) { 55 | saveHardState(*hs, &scratch[offset], &offset); 56 | } 57 | 58 | if (writeEntries) { 59 | saveEntries(begin, newBegin, &scratch[offset], &offset); 60 | } 61 | 62 | size_t dataLen = totalSize - kLogBatchHeaderSize; 63 | 64 | // len field 65 | EncodeFixed32(&scratch[4], static_cast(dataLen)); 66 | 67 | // crc field 68 | boost::crc_32_type crc; 69 | crc.process_bytes(&scratch[kLogBatchHeaderSize], dataLen); 70 | EncodeFixed32(&scratch[0], static_cast(crc.checksum())); 71 | 72 | RETURN_NOT_OK(file_->Append(scratch)); 73 | 74 | meta_.numEntries += std::distance(begin, newBegin); 75 | return newBegin; 76 | } 77 | 78 | void LogWriter::saveHardState(const yaraft::pb::HardState &hs, char *dest, size_t *offset) { 79 | char *p = dest; 80 | p[0] = static_cast(kHardStateType); 81 | 82 | p = EncodeVarint32(p + 1, hs.ByteSize()); 83 | hs.SerializeToArray(p, hs.ByteSize()); 84 | 85 | (*offset) += p - dest + hs.ByteSize(); 86 | } 87 | 88 | void LogWriter::saveEntries(ConstPBEntriesIterator begin, ConstPBEntriesIterator end, char *dest, 89 | size_t *offset) { 90 | char *p = dest; 91 | char type = static_cast(kLogEntryType); 92 | 93 | for (auto it = begin; it != end; it++) { 94 | p[0] = type; 95 | p = EncodeVarint32(p + 1, it->ByteSize()); 96 | it->SerializeToArray(p, it->ByteSize()); 97 | p += it->ByteSize(); 98 | } 99 | (*offset) += p - dest; 100 | } 101 | 102 | } // namespace wal 103 | } // namespace consensus -------------------------------------------------------------------------------- /src/ready_flusher.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/background_worker.h" 16 | 17 | #include "raft_task_executor.h" 18 | #include "ready_flusher.h" 19 | #include "replicated_log_impl.h" 20 | 21 | #include 22 | 23 | namespace consensus { 24 | 25 | class ReadyFlusher::Impl { 26 | public: 27 | Impl() = default; 28 | 29 | void Register(ReplicatedLogImpl *log) { 30 | std::lock_guard g(mu_); 31 | logs_.push_back(log); 32 | } 33 | 34 | void Start() { 35 | FATAL_NOT_OK(worker_.StartLoop(std::bind(&Impl::flushRound, this)), 36 | "ReadyFlusher::Impl::Start"); 37 | } 38 | 39 | void Stop() { 40 | FATAL_NOT_OK(worker_.Stop(), "ReadyFlusher::Impl::Stop"); 41 | } 42 | 43 | private: 44 | void flushRound() { 45 | mu_.lock(); 46 | std::vector logs = logs_; 47 | mu_.unlock(); 48 | 49 | if (logs.empty()) { 50 | return; 51 | } 52 | 53 | for (auto rl : logs) { 54 | yaraft::Ready *rd = rl->executor_->GetReady(); 55 | if (rd) { 56 | std::async(std::bind(&Impl::flushReady, this, rl, rd)); 57 | } 58 | } 59 | } 60 | 61 | void flushReady(ReplicatedLogImpl *rl, yaraft::Ready *rd) { 62 | yaraft::pb::HardState *hs = nullptr; 63 | std::unique_ptr g(rd); 64 | 65 | // the leader can write to its disk in parallel with replicating to the followers and them 66 | // writing to their disks. 67 | // For more details, check raft thesis 10.2.1 68 | if (rd->currentLeader == rl->Id()) { 69 | if (!rd->messages.empty()) { 70 | rl->cluster_->Pass(rd->messages); 71 | rd->messages.clear(); 72 | } 73 | } 74 | 75 | if (rd->hardState) { 76 | hs = rd->hardState.get(); 77 | } 78 | 79 | if (!rd->entries.empty()) { 80 | FATAL_NOT_OK(rl->wal_->Write(rd->entries, hs), "Wal::Write"); 81 | } else { 82 | FATAL_NOT_OK(rl->wal_->Write(hs), "Wal::Write"); 83 | } 84 | 85 | // committedIndex has changed 86 | if (rd->hardState && rd->hardState->has_commit()) { 87 | rl->walCommitObserver_->Notify(rd->hardState->commit()); 88 | } 89 | 90 | // states have already been persisted. 91 | rd->Advance(rl->memstore_); 92 | 93 | // followers should respond only after state persisted 94 | if (rd->currentLeader != rl->Id()) { 95 | if (!rd->messages.empty()) { 96 | rl->cluster_->Pass(rd->messages); 97 | rd->messages.clear(); 98 | } 99 | } 100 | } 101 | 102 | private: 103 | std::vector logs_; 104 | std::mutex mu_; 105 | 106 | BackgroundWorker worker_; 107 | }; 108 | 109 | void ReadyFlusher::Register(ReplicatedLogImpl *log) { 110 | impl_->Register(log); 111 | } 112 | 113 | ReadyFlusher::ReadyFlusher() : impl_(new Impl) { 114 | impl_->Start(); 115 | } 116 | 117 | ReadyFlusher::~ReadyFlusher() { 118 | impl_->Stop(); 119 | } 120 | 121 | } // namespace consensus 122 | -------------------------------------------------------------------------------- /cmake_modules/FindSanitizers.cmake: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | # 3 | # Copyright (c) 4 | # 2013 Matthew Arsenault 5 | # 2015-2016 RWTH Aachen University, Federal Republic of Germany 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy 8 | # of this software and associated documentation files (the "Software"), to deal 9 | # in the Software without restriction, including without limitation the rights 10 | # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | # copies of the Software, and to permit persons to whom the Software is 12 | # furnished to do so, subject to the following conditions: 13 | # 14 | # The above copyright notice and this permission notice shall be included in all 15 | # copies or substantial portions of the Software. 16 | # 17 | # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | # SOFTWARE. 24 | 25 | # If any of the used compiler is a GNU compiler, add a second option to static 26 | # link against the sanitizers. 27 | option(SANITIZE_LINK_STATIC "Try to link static against sanitizers." Off) 28 | 29 | 30 | 31 | 32 | set(FIND_QUIETLY_FLAG "") 33 | if (DEFINED Sanitizers_FIND_QUIETLY) 34 | set(FIND_QUIETLY_FLAG "QUIET") 35 | endif () 36 | 37 | find_package(ASan ${FIND_QUIETLY_FLAG}) 38 | find_package(TSan ${FIND_QUIETLY_FLAG}) 39 | find_package(MSan ${FIND_QUIETLY_FLAG}) 40 | find_package(UBSan ${FIND_QUIETLY_FLAG}) 41 | 42 | 43 | 44 | 45 | function(sanitizer_add_blacklist_file FILE) 46 | if(NOT IS_ABSOLUTE ${FILE}) 47 | set(FILE "${CMAKE_CURRENT_SOURCE_DIR}/${FILE}") 48 | endif() 49 | get_filename_component(FILE "${FILE}" REALPATH) 50 | 51 | sanitizer_check_compiler_flags("-fsanitize-blacklist=${FILE}" 52 | "SanitizerBlacklist" "SanBlist") 53 | endfunction() 54 | 55 | function(add_sanitizers ...) 56 | # If no sanitizer is enabled, return immediately. 57 | if (NOT (SANITIZE_ADDRESS OR SANITIZE_MEMORY OR SANITIZE_THREAD OR 58 | SANITIZE_UNDEFINED)) 59 | return() 60 | endif () 61 | 62 | foreach (TARGET ${ARGV}) 63 | # Check if this target will be compiled by exactly one compiler. Other- 64 | # wise sanitizers can't be used and a warning should be printed once. 65 | sanitizer_target_compilers(${TARGET} TARGET_COMPILER) 66 | list(LENGTH TARGET_COMPILER NUM_COMPILERS) 67 | if (NUM_COMPILERS GREATER 1) 68 | message(WARNING "Can't use any sanitizers for target ${TARGET}, " 69 | "because it will be compiled by incompatible compilers. " 70 | "Target will be compiled without sanitzers.") 71 | return() 72 | 73 | # If the target is compiled by no known compiler, ignore it. 74 | elseif (NUM_COMPILERS EQUAL 0) 75 | message(WARNING "Can't use any sanitizers for target ${TARGET}, " 76 | "because it uses an unknown compiler. Target will be " 77 | "compiled without sanitzers.") 78 | return() 79 | endif () 80 | 81 | # Add sanitizers for target. 82 | add_sanitize_address(${TARGET}) 83 | add_sanitize_thread(${TARGET}) 84 | add_sanitize_memory(${TARGET}) 85 | add_sanitize_undefined(${TARGET}) 86 | endforeach () 87 | endfunction(add_sanitizers) 88 | -------------------------------------------------------------------------------- /include/consensus/base/testing.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "consensus/base/env.h" 18 | #include "consensus/base/logging.h" 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | namespace consensus { 25 | 26 | using ::testing::FLAGS_gtest_random_seed; 27 | 28 | class WritableFile; 29 | class Random; 30 | 31 | #define ASSERT_OK(status) \ 32 | do { \ 33 | const auto &_s = status; \ 34 | if (_s.IsOK()) { \ 35 | SUCCEED(); \ 36 | } else { \ 37 | FAIL() << "Bad status: " << _s.ToString(); \ 38 | } \ 39 | } while (0) 40 | 41 | #define ASSIGN_IF_ASSERT_OK(statusWith, var) \ 42 | do { \ 43 | auto _sw = statusWith; \ 44 | ASSERT_OK(_sw); \ 45 | (var) = _sw.GetValue(); \ 46 | } while (0) 47 | 48 | class TestDirectoryHelper { 49 | public: 50 | explicit TestDirectoryHelper(const std::string &testDir) : dir_(testDir) { 51 | acquire(); 52 | } 53 | 54 | ~TestDirectoryHelper() { 55 | release(); 56 | } 57 | 58 | std::string GetTestDir() const { 59 | return dir_; 60 | } 61 | 62 | private: 63 | void acquire() { 64 | FATAL_NOT_OK(Env::Default()->CreateDirIfMissing(dir_), "Env::CreateDirIfMissing"); 65 | } 66 | 67 | void release() { 68 | FATAL_NOT_OK(Env::Default()->DeleteRecursively(dir_), "Env::DeleteRecursively"); 69 | } 70 | 71 | private: 72 | std::string dir_; 73 | }; 74 | 75 | class BaseTest : public ::testing::Test { 76 | public: 77 | BaseTest() { 78 | initTestDir(); 79 | } 80 | 81 | virtual ~BaseTest() = default; 82 | 83 | uint32_t SeedRandom(); 84 | 85 | WritableFile *OpenFileForWrite(const std::string &fname, 86 | Env::CreateMode mode = Env::CREATE_IF_NON_EXISTING_TRUNCATE, 87 | bool sync_on_close = false); 88 | 89 | // Write 'size' bytes of data to a file, with a simple pattern stored in it. 90 | void WriteTestFile(const Slice &path, size_t size, std::string *testData, Random *rng); 91 | 92 | TestDirectoryHelper *CreateTestDirGuard() const { 93 | return new TestDirectoryHelper(test_dir_); 94 | } 95 | 96 | std::string GetTestDir() const { 97 | return test_dir_; 98 | } 99 | 100 | private: 101 | void initTestDir() { 102 | test_dir_ = fmt::format("/tmp/consensus-yaraft.{}.{}", testInfo()->test_case_name(), 103 | testInfo()->name()); 104 | } 105 | 106 | const ::testing::TestInfo *const testInfo() { 107 | return ::testing::UnitTest::GetInstance()->current_test_info(); 108 | } 109 | 110 | private: 111 | std::string test_dir_; 112 | }; 113 | 114 | typedef std::unique_ptr TestDirGuard; 115 | 116 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/src/memkv_store_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | 17 | #include "memkv_store.h" 18 | #include "testing.h" 19 | 20 | #include 21 | 22 | using namespace memkv; 23 | 24 | class TestMemKV : public testing::Test { 25 | public: 26 | }; 27 | 28 | // This test ensures that the same keys can always read 29 | // the same value. 30 | TEST_F(TestMemKV, WriteAndGet) { 31 | struct TestData { 32 | std::string key; 33 | std::string testKey; 34 | 35 | Error::ErrorCodes wcode; 36 | } tests[] = { 37 | {"/tmp/xiaomi/ads/pegasus-1/table-num-counter", "/tmp/xiaomi/ads/pegasus-1/table-num-counter", 38 | Error::OK}, 39 | {"/tmp/xiaomi/ads/pegasus-1/table-num-counter", "/var", Error::NodeNotExist}, 40 | {"tmp/xiaomi", "///tmp/xiaomi", Error::OK}, 41 | 42 | // can not pass an empty path as parameter 43 | {"tmp", "", Error::InvalidArgument}, 44 | }; 45 | 46 | for (auto t : tests) { 47 | MemKvStore kv; 48 | 49 | std::string expected = "test_value"; 50 | ASSERT_OK(kv.Write(t.key, expected)); 51 | std::string actual; 52 | ASSERT_ERROR(kv.Get(t.testKey, &actual), t.wcode); 53 | if (t.wcode == Error::OK) { 54 | ASSERT_EQ(actual, expected); 55 | } 56 | } 57 | } 58 | 59 | std::string randomString(size_t len, consensus::Random *rnd) { 60 | std::string result; 61 | result.resize(len); 62 | for (size_t i = 0; i < len; i++) { 63 | result[i] = static_cast('a' + rnd->Uniform(26)); 64 | } 65 | return result; 66 | } 67 | 68 | TEST_F(TestMemKV, MultiReadWrite) { 69 | std::unordered_map data_map; 70 | consensus::Random rnd(0); 71 | 72 | for (int i = 0; i < 2; i++) { 73 | std::string key = randomString(10, &rnd); 74 | data_map[key] = randomString(20, &rnd); 75 | LOG(INFO) << key << " " << data_map[key]; 76 | } 77 | 78 | MemKvStore store; 79 | for (auto &kv : data_map) { 80 | store.Write(kv.first, kv.second); 81 | } 82 | 83 | for (auto &kv : data_map) { 84 | std::string value; 85 | ASSERT_OK(store.Get(kv.first, &value)); 86 | ASSERT_EQ(value, kv.second); 87 | } 88 | } 89 | 90 | TEST_F(TestMemKV, Delete) { 91 | MemKvStore kv; 92 | std::string key = "/tmp", expected = "1"; 93 | ASSERT_OK(kv.Write(key, expected)); 94 | 95 | std::string actual; 96 | ASSERT_OK(kv.Get(key, &actual)); 97 | ASSERT_EQ(actual, expected); 98 | 99 | ASSERT_OK(kv.Delete(key)); 100 | ASSERT_ERROR(kv.Get(key, &actual), Error::NodeNotExist); 101 | } 102 | 103 | TEST_F(TestMemKV, DeleteRecursively) { 104 | MemKvStore kv; 105 | std::string expected = "1"; 106 | ASSERT_OK(kv.Write("/tmp/xiaomi/ads/pegasus-1/table-num-counter", expected)); 107 | 108 | std::string actual; 109 | ASSERT_OK(kv.Get("/tmp/xiaomi/ads/pegasus-1/table-num-counter", &actual)); 110 | ASSERT_EQ(actual, expected); 111 | 112 | ASSERT_OK(kv.Delete("/tmp/xiaomi/ads/pegasus-1")); 113 | ASSERT_OK(kv.Get("/tmp/xiaomi/ads", &actual)); 114 | ASSERT_ERROR(kv.Get("/tmp/xiaomi/ads/pegasus-1", &actual), Error::NodeNotExist); 115 | } -------------------------------------------------------------------------------- /apps/memkv/src/memkv_service.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "memkv_service.h" 16 | #include "logging.h" 17 | 18 | namespace memkv { 19 | 20 | static inline pb::ErrCode memkvErrorToRpcErrno(Error::ErrorCodes code) { 21 | switch (code) { 22 | case Error::OK: 23 | return pb::OK; 24 | case Error::InvalidArgument: 25 | return pb::InvalidArgument; 26 | case Error::NodeNotExist: 27 | return pb::NodeNotExist; 28 | case Error::ConsensusError: 29 | return pb::ConsensusError; 30 | default: 31 | LOG(FATAL) << "Unexpected error code: " << Error::toString(code); 32 | return pb::OK; 33 | } 34 | } 35 | 36 | // write request via http goes like this: 37 | // http 'URL:PORT/Write/abc/a?value=hello' 38 | // if ok, path "/abc/a" will be set to "hello" 39 | void MemKVServiceImpl::Write(::google::protobuf::RpcController *controller, 40 | const ::memkv::pb::WriteRequest *request, 41 | ::memkv::pb::WriteResult *response, 42 | ::google::protobuf::Closure *done) { 43 | auto cntl = static_cast(controller); 44 | Status s; 45 | if (cntl->has_http_request()) { 46 | Slice path(cntl->http_request().unresolved_path()); 47 | Slice value(*cntl->http_request().uri().GetQuery("value")); 48 | s = db_->Write(path, value); 49 | } else { 50 | s = db_->Write(request->path(), request->value()); 51 | } 52 | 53 | response->set_errorcode(memkvErrorToRpcErrno(s.Code())); 54 | if (!s.IsOK()) { 55 | response->set_errormessage(s.ToString()); 56 | } 57 | done->Run(); 58 | } 59 | 60 | void MemKVServiceImpl::Read(::google::protobuf::RpcController *controller, 61 | const ::memkv::pb::ReadRequest *request, 62 | ::memkv::pb::ReadResult *response, ::google::protobuf::Closure *done) { 63 | auto cntl = static_cast(controller); 64 | auto result = new std::string; 65 | Status s; 66 | if (cntl->has_http_request()) { 67 | // by default brpc will encode string to base64 68 | // we don't need this feature here. 69 | cntl->set_pb_bytes_to_base64(false); 70 | 71 | Slice path(cntl->http_request().unresolved_path()); 72 | bool stale = false; 73 | if (cntl->http_request().uri().GetQuery("stale") != nullptr) { 74 | stale = true; 75 | } 76 | s = db_->Get(path, stale, result); 77 | } else { 78 | s = db_->Get(request->path(), request->stale(), result); 79 | } 80 | 81 | response->set_allocated_value(result); 82 | response->set_errorcode(memkvErrorToRpcErrno(s.Code())); 83 | if (!s.IsOK()) { 84 | response->set_errormessage(s.ToString()); 85 | } 86 | done->Run(); 87 | } 88 | 89 | void MemKVServiceImpl::Delete(::google::protobuf::RpcController *controller, 90 | const ::memkv::pb::DeleteRequest *request, 91 | ::memkv::pb::DeleteResult *response, 92 | ::google::protobuf::Closure *done) {} 93 | 94 | MemKVServiceImpl::MemKVServiceImpl(DB *db) : db_(db) {} 95 | 96 | MemKVServiceImpl::~MemKVServiceImpl() = default; 97 | 98 | } // namespace memkv -------------------------------------------------------------------------------- /src/base/env_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/env.h" 16 | #include "base/env_util.h" 17 | #include "base/random.h" 18 | #include "base/testing.h" 19 | 20 | using namespace consensus; 21 | using namespace std; 22 | 23 | class TestEnv : public BaseTest { 24 | public: 25 | TestEnv() : rng_(SeedRandom()) {} 26 | 27 | void TestAppendVector(size_t num_slices, size_t slice_size) { 28 | const string kTestPath = GetTestDir() + "/test_env_appendvec_read_append"; 29 | 30 | unique_ptr wf(OpenFileForWrite(kTestPath)); 31 | 32 | std::string slices; 33 | LOG(INFO) << fmt::format( 34 | "appending a vector of slices(number of slices={}, size of slice={} b)", num_slices, 35 | slice_size); 36 | 37 | vector dataSet; 38 | RandomDataSet(num_slices, slice_size, &dataSet); 39 | 40 | for (int i = 0; i < num_slices; i++) { 41 | slices += dataSet[i]; 42 | } 43 | 44 | ASSERT_OK(wf->Append(slices)); 45 | ASSERT_OK(wf->Close()); 46 | 47 | string testData; 48 | for (string& s : dataSet) { 49 | testData += s; 50 | } 51 | ASSERT_EQ(testData.size(), num_slices * slice_size); 52 | 53 | ReadAndVerifyTestData(kTestPath, testData); 54 | } 55 | 56 | void ReadAndVerifyTestData(const string& filePath, const string& testData) { 57 | Slice s; 58 | char* scratch; 59 | ASSERT_OK(env_util::ReadFullyToBuffer(filePath, &s, &scratch)); 60 | 61 | ASSERT_EQ(s.ToString(), testData); 62 | } 63 | 64 | void RandomDataSet(size_t num_slices, size_t slice_size, vector* dataSet) { 65 | dataSet->resize(num_slices); 66 | for (int i = 0; i < num_slices; i++) { 67 | (*dataSet)[i] = RandomString(slice_size, &rng_); 68 | } 69 | } 70 | 71 | protected: 72 | Random rng_; 73 | }; 74 | 75 | TEST_F(TestEnv, ReadFully) { 76 | TestDirGuard g(CreateTestDirGuard()); 77 | 78 | const int kFileSize = 64 * 1024; 79 | const int kTrialNum = 1000; 80 | const string kTestPath = GetTestDir() + "/test_" + fmt::format("{}", rng_.Next()); 81 | 82 | for (int i = 0; i < kTrialNum; i++) { 83 | string testData; 84 | WriteTestFile(kTestPath, kFileSize, &testData, &rng_); 85 | ReadAndVerifyTestData(kTestPath, testData); 86 | Env::Default()->DeleteFile(kTestPath); 87 | ASSERT_NO_FATAL_FAILURE(); 88 | } 89 | } 90 | 91 | TEST_F(TestEnv, AppendVector) { 92 | TestDirGuard g(CreateTestDirGuard()); 93 | TestAppendVector(2000, 1024); 94 | } 95 | 96 | TEST_F(TestEnv, GetChildren) { 97 | TestDirGuard g(CreateTestDirGuard()); 98 | const int kFileNum = 10; 99 | 100 | uint32_t randNum = rng_.Next(); 101 | 102 | vector files; 103 | for (int i = 0; i < kFileNum; i++) { 104 | // create empty test files 105 | string fileName(fmt::format("test_{}_{:02d}", randNum, i)); 106 | unique_ptr wf(OpenFileForWrite(GetTestDir() + "/" + fileName)); 107 | wf->Close(); 108 | files.push_back(fileName); 109 | } 110 | 111 | vector result; 112 | ASSERT_OK(Env::Default()->GetChildren(GetTestDir(), &result)); 113 | 114 | std::sort(files.begin(), files.end()); 115 | std::sort(result.begin(), result.end()); 116 | 117 | ASSERT_EQ(files, result); 118 | } -------------------------------------------------------------------------------- /src/wal/readable_log_segment.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "wal/readable_log_segment.h" 16 | #include "base/coding.h" 17 | #include "base/env_util.h" 18 | #include "base/logging.h" 19 | #include "wal/format.h" 20 | 21 | #include 22 | 23 | namespace consensus { 24 | namespace wal { 25 | 26 | Status ReadSegmentIntoMemoryStorage(const Slice &fname, yaraft::MemoryStorage *memStore, 27 | SegmentMetaData *metaData, bool verifyChecksum) { 28 | LOG_ASSERT(memStore != nullptr); 29 | 30 | char *buf; 31 | Slice s; 32 | RETURN_NOT_OK(env_util::ReadFullyToBuffer(fname, &s, &buf)); 33 | 34 | ReadableLogSegment seg(s, memStore, metaData, verifyChecksum); 35 | RETURN_NOT_OK_APPEND(seg.ReadHeader(), fmt::format(" [segment: {}] ", fname.ToString())); 36 | while (!seg.Eof()) { 37 | RETURN_NOT_OK_APPEND(seg.ReadRecord(), fmt::format(" [segment: {}] ", fname.ToString())); 38 | } 39 | 40 | return Status::OK(); 41 | } 42 | 43 | Status ReadableLogSegment::ReadHeader() { 44 | // check magic 45 | RETURN_NOT_OK_APPEND(checkRemain(kLogSegmentHeaderMagic.size()), "[bad magic length]"); 46 | Slice magic(buf_, kLogSegmentHeaderMagic.size()); 47 | if (UNLIKELY(kLogSegmentHeaderMagic.Compare(magic) != 0)) { 48 | return FMT_Status(Corruption, "bad header magic: {}", magic.ToString()); 49 | } 50 | advance(kLogSegmentHeaderMagic.size()); 51 | 52 | return Status::OK(); 53 | } 54 | 55 | Status ReadableLogSegment::ReadRecord() { 56 | RETURN_NOT_OK_APPEND(checkRemain(kLogBatchHeaderSize), " [bad batch header] "); 57 | 58 | uint32_t crc = DecodeFixed32(buf_); 59 | uint32_t len = DecodeFixed32(buf_ + 4); 60 | advance(kLogBatchHeaderSize); 61 | 62 | RETURN_NOT_OK_APPEND(checkRemain(len), " [bad batch length] "); 63 | 64 | if (verifyChecksum_) { 65 | boost::crc_32_type crc32; 66 | crc32.process_bytes(buf_, len); 67 | if (crc32.checksum() != crc) { 68 | return FMT_Status(Corruption, "bad checksum"); 69 | } 70 | } 71 | 72 | Slice record(buf_, len); 73 | while (record.Len() > 0) { 74 | auto type = static_cast(record[0]); 75 | record.Skip(kRecordHeaderSize); 76 | 77 | Slice data; 78 | if (UNLIKELY(!GetLengthPrefixedSlice(&record, &data))) { 79 | return Status::Make(Error::Corruption, "bad record"); 80 | } 81 | 82 | if (type == kLogEntryType) { 83 | yaraft::pb::Entry e; 84 | e.ParseFromArray(data.RawData(), data.Len()); 85 | memStore_->Append(e); 86 | metaData_->numEntries++; 87 | } else if (type == kHardStateType) { 88 | yaraft::pb::HardState hs; 89 | hs.ParseFromArray(data.RawData(), data.Len()); 90 | memStore_->SetHardState(hs); 91 | } 92 | } 93 | advance(len); 94 | 95 | return Status::OK(); 96 | } 97 | 98 | bool ReadableLogSegment::Eof() { 99 | return remain_ == 0; 100 | } 101 | 102 | Status ReadableLogSegment::checkRemain(size_t need) { 103 | if (UNLIKELY(remain_ < need)) { 104 | return FMT_Status(Corruption, "segment is too small to contain {} number of bytes", need); 105 | } 106 | return Status::OK(); 107 | } 108 | 109 | void ReadableLogSegment::advance(size_t size) { 110 | remain_ -= size; 111 | buf_ += size; 112 | } 113 | 114 | } // namespace wal 115 | } // namespace consensus -------------------------------------------------------------------------------- /include/consensus/base/coding.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // Endian-neutral encoding: 6 | // * Fixed-length numbers are encoded with least-significant byte first 7 | // * In addition we support variable length "varint" encoding 8 | // * Strings are encoded prefixed by their length in varint format 9 | 10 | #pragma once 11 | 12 | #include 13 | #include 14 | #include 15 | 16 | #include "consensus/base/endianness.h" 17 | #include "consensus/base/slice.h" 18 | 19 | namespace consensus { 20 | 21 | // Standard Put... routines append to a string 22 | extern void PutFixed32(std::string* dst, uint32_t value); 23 | extern void PutFixed64(std::string* dst, uint64_t value); 24 | extern void PutVarint32(std::string* dst, uint32_t value); 25 | extern void PutVarint64(std::string* dst, uint64_t value); 26 | extern void PutLengthPrefixedSlice(std::string* dst, const Slice& value); 27 | 28 | // Standard Get... routines parse a value from the beginning of a Slice 29 | // and advance the slice past the parsed value. 30 | extern bool GetVarint32(Slice* input, uint32_t* value); 31 | extern bool GetVarint64(Slice* input, uint64_t* value); 32 | extern bool GetLengthPrefixedSlice(Slice* input, Slice* result); 33 | 34 | // Pointer-based variants of GetVarint... These either store a value 35 | // in *v and return a pointer just past the parsed value, or return 36 | // NULL on error. These routines only look at bytes in the range 37 | // [p..limit-1] 38 | extern const char* GetVarint32Ptr(const char* p, const char* limit, uint32_t* v); 39 | extern const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* v); 40 | 41 | // Returns the length of the varint32 or varint64 encoding of "v" 42 | extern int VarintLength(uint64_t v); 43 | 44 | // Lower-level versions of Put... that write directly into a character buffer 45 | // REQUIRES: dst has enough space for the value being written 46 | extern void EncodeFixed32(char* dst, uint32_t value); 47 | extern void EncodeFixed64(char* dst, uint64_t value); 48 | 49 | // Lower-level versions of Put... that write directly into a character buffer 50 | // and return a pointer just past the last byte written. 51 | // REQUIRES: dst has enough space for the value being written 52 | extern char* EncodeVarint32(char* dst, uint32_t value); 53 | extern char* EncodeVarint64(char* dst, uint64_t value); 54 | 55 | // Lower-level versions of Get... that read directly from a character buffer 56 | // without any bounds checking. 57 | 58 | inline uint32_t DecodeFixed32(const char* ptr) { 59 | if (FLAGS_is_little_endian) { 60 | // Load the raw bytes 61 | uint32_t result; 62 | memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load 63 | return result; 64 | } else { 65 | return ((static_cast(static_cast(ptr[0]))) | 66 | (static_cast(static_cast(ptr[1])) << 8) | 67 | (static_cast(static_cast(ptr[2])) << 16) | 68 | (static_cast(static_cast(ptr[3])) << 24)); 69 | } 70 | } 71 | 72 | inline uint64_t DecodeFixed64(const char* ptr) { 73 | if (FLAGS_is_little_endian) { 74 | // Load the raw bytes 75 | uint64_t result; 76 | memcpy(&result, ptr, sizeof(result)); // gcc optimizes this to a plain load 77 | return result; 78 | } else { 79 | uint64_t lo = DecodeFixed32(ptr); 80 | uint64_t hi = DecodeFixed32(ptr + 4); 81 | return (hi << 32) | lo; 82 | } 83 | } 84 | 85 | // Internal routine for use by fallback path of GetVarint32Ptr 86 | extern const char* GetVarint32PtrFallback(const char* p, const char* limit, uint32_t* value); 87 | inline const char* GetVarint32Ptr(const char* p, const char* limit, uint32_t* value) { 88 | if (p < limit) { 89 | uint32_t result = *(reinterpret_cast(p)); 90 | if ((result & 128) == 0) { 91 | *value = result; 92 | return p + 1; 93 | } 94 | } 95 | return GetVarint32PtrFallback(p, limit, value); 96 | } 97 | 98 | } // namespace consensus 99 | -------------------------------------------------------------------------------- /src/wal/log_writer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/coding.h" 16 | #include "base/mock_env.h" 17 | #include "base/random.h" 18 | #include "base/testing.h" 19 | #include "wal/format.h" 20 | #include "wal/log_writer.h" 21 | #include "wal/readable_log_segment.h" 22 | 23 | namespace consensus { 24 | namespace wal { 25 | 26 | using namespace yaraft; 27 | 28 | class LogWriterTest : public BaseTest { 29 | public: 30 | LogWriterTest() = default; 31 | 32 | // Initialize the max-segment-size to make it holds exactly `entriesInSegment` number of entries. 33 | void InitLogSegment(size_t entriesInSegment) { 34 | entries.clear(); 35 | logSegmentSize = kLogSegmentHeaderMagic.size() + kLogBatchHeaderSize; 36 | for (uint64_t i = 1; i <= entriesInSegment; i++) { 37 | entries.push_back(PBEntry().Index(i).Term(i).v); 38 | logSegmentSize += 39 | kRecordHeaderSize + VarintLength(entries.back().ByteSize()) + entries.back().ByteSize(); 40 | } 41 | } 42 | 43 | void TestAppendEntries(size_t entriesInSegment) { 44 | InitLogSegment(entriesInSegment); 45 | 46 | // write one more entry than the segment can hold 47 | entries.push_back(PBEntry().Index(entriesInSegment + 1).Term(entriesInSegment + 1).v); 48 | 49 | auto wf = new MockWritableFile; 50 | LogWriter writer(wf, "test-seg", logSegmentSize); 51 | ConstPBEntriesIterator it; 52 | ASSIGN_IF_ASSERT_OK(writer.Append(entries.begin(), entries.end()), it); 53 | 54 | ASSERT_EQ(std::distance(entries.cbegin(), it), entriesInSegment); 55 | ASSERT_EQ(wf->Data().size(), logSegmentSize); 56 | 57 | SegmentMetaData metaData; 58 | ASSERT_OK(writer.Finish(&metaData)); 59 | ASSERT_EQ(metaData.numEntries, entriesInSegment); 60 | } 61 | 62 | void TestEncodeAndDecode(size_t entriesInSegment) { 63 | InitLogSegment(entriesInSegment); 64 | bool verifyChecksum = true; 65 | 66 | auto wf = new MockWritableFile; 67 | LogWriter writer(wf, "test-seg", logSegmentSize); 68 | ASSERT_OK(writer.Append(entries.begin(), entries.end())); 69 | 70 | std::string fileData = wf->Data(); 71 | ASSERT_EQ(fileData.size(), logSegmentSize); 72 | 73 | SegmentMetaData meta; 74 | yaraft::MemoryStorage memStore; 75 | ReadableLogSegment seg(fileData, &memStore, &meta, verifyChecksum); 76 | 77 | // read from segment 78 | ASSERT_OK(seg.ReadHeader()); 79 | while (!seg.Eof()) { 80 | ASSERT_OK(seg.ReadRecord()); 81 | } 82 | 83 | EntryVec actualEntries(memStore.TEST_Entries().begin() + 1, memStore.TEST_Entries().end()); 84 | ASSERT_EQ(actualEntries.size(), entries.size()); 85 | for (int i = 0; i < actualEntries.size(); i++) { 86 | ASSERT_EQ(actualEntries[i].DebugString(), entries[i].DebugString()); 87 | } 88 | } 89 | 90 | private: 91 | EntryVec entries; 92 | size_t logSegmentSize; 93 | }; 94 | 95 | // LogWriterTest.AppendEntries ensures that LogWriter::AppendEntries stops writing 96 | // when entries is out of bound. 97 | TEST_F(LogWriterTest, AppendEntries) { 98 | TestAppendEntries(10); 99 | TestAppendEntries(50); 100 | TestAppendEntries(100); 101 | TestAppendEntries(200); 102 | TestAppendEntries(500); 103 | TestAppendEntries(1000); 104 | } 105 | 106 | // This test verifies that data being written can be decoded correctly. 107 | TEST_F(LogWriterTest, EncodeAndDecode) { 108 | TestEncodeAndDecode(10); 109 | TestEncodeAndDecode(200); 110 | TestEncodeAndDecode(500); 111 | TestEncodeAndDecode(1000); 112 | TestEncodeAndDecode(2000); 113 | TestEncodeAndDecode(10000); 114 | } 115 | 116 | } // namespace wal 117 | } // namespace consensus -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_library(GOOGLE_BENCH_LIB benchmark) 2 | 3 | set(CONSENSUS_LINK_LIBS 4 | ${FMT_LIBRARY} 5 | ${GLOG_STATIC_LIB} 6 | ${GFLAGS_STATIC_LIB} 7 | ${Boost_LIBRARIES} 8 | ${YARAFT_LIBRARY} 9 | ${PROTOBUF_STATIC_LIBRARY} 10 | ${PROTOBUF_PROTOC_STATIC_LIBRARY} 11 | ${BRPC_LIBRARY} 12 | ${OPENSSL_LIBRARIES} 13 | ${LEVELDB_LIBRARY} 14 | ${ZLIB_LIBRARIES} 15 | pthread 16 | dl) 17 | 18 | ##------------------- Base -------------------## 19 | 20 | set(BASE_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/base) 21 | 22 | set(BASE_SOURCES 23 | ${BASE_SOURCE_DIR}/env_posix.cc 24 | ${BASE_SOURCE_DIR}/errno.cc 25 | ${BASE_SOURCE_DIR}/random.cc 26 | ${BASE_SOURCE_DIR}/status.cc 27 | ${BASE_SOURCE_DIR}/testing.cc 28 | ${BASE_SOURCE_DIR}/env_util.cc 29 | ${BASE_SOURCE_DIR}/coding.cc 30 | ${BASE_SOURCE_DIR}/glog_logger.cc 31 | ${BASE_SOURCE_DIR}/endianness.cc 32 | ${BASE_SOURCE_DIR}/background_worker.cc 33 | ${BASE_SOURCE_DIR}/task_queue.cc) 34 | 35 | add_library(consensus_base ${BASE_SOURCES}) 36 | target_link_libraries(consensus_base ${CONSENSUS_LINK_LIBS}) 37 | set(CONSENSUS_LINK_LIBS ${CONSENSUS_LINK_LIBS} consensus_base) 38 | 39 | function(ADD_BASE_TEST TEST_NAME) 40 | add_executable(${TEST_NAME} ${BASE_SOURCE_DIR}/${TEST_NAME}.cc) 41 | target_link_libraries(${TEST_NAME} ${CONSENSUS_LINK_LIBS} ${GTEST_LIB} ${GTEST_MAIN_LIB}) 42 | endfunction() 43 | 44 | ADD_BASE_TEST(env_test) 45 | 46 | ADD_BASE_TEST(random_test) 47 | 48 | ADD_BASE_TEST(coding_test) 49 | 50 | ADD_BASE_TEST(background_worker_test) 51 | 52 | ##------------------- WAL -------------------## 53 | 54 | set(WAL_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/wal) 55 | 56 | set(WAL_SOURCES 57 | ${WAL_SOURCE_DIR}/segment_meta.cc 58 | ${WAL_SOURCE_DIR}/wal.cc 59 | ${WAL_SOURCE_DIR}/log_writer.cc 60 | ${WAL_SOURCE_DIR}/log_manager.cc 61 | ${WAL_SOURCE_DIR}/readable_log_segment.cc) 62 | 63 | add_library(consensus_wal ${WAL_SOURCES}) 64 | target_link_libraries(consensus_wal ${CONSENSUS_LINK_LIBS}) 65 | set(CONSENSUS_LINK_LIBS ${CONSENSUS_LINK_LIBS} consensus_wal) 66 | 67 | function(ADD_WAL_TEST TEST_NAME) 68 | add_executable(${TEST_NAME} ${WAL_SOURCE_DIR}/${TEST_NAME}.cc) 69 | target_link_libraries(${TEST_NAME} ${CONSENSUS_LINK_LIBS} ${GTEST_LIB} ${GTEST_MAIN_LIB}) 70 | endfunction() 71 | 72 | ADD_WAL_TEST(log_writer_test) 73 | 74 | ADD_WAL_TEST(log_manager_test) 75 | 76 | add_executable(wal_bench wal/wal_bench.cc) 77 | target_link_libraries(wal_bench ${GOOGLE_BENCH_LIB} ${CONSENSUS_LINK_LIBS}) 78 | 79 | ##------------------- RPC -------------------## 80 | 81 | set(RPC_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src/rpc) 82 | 83 | set(RPC_SOURCES 84 | ${RPC_SOURCE_DIR}/peer.cc 85 | ${RPC_SOURCE_DIR}/cluster.cc 86 | ${RPC_SOURCE_DIR}/raft_client.h 87 | ${PROJECT_SOURCE_DIR}/include/consensus/pb/raft_server.pb.cc 88 | ) 89 | 90 | add_library(consensus_rpc ${RPC_SOURCES}) 91 | target_link_libraries(consensus_rpc ${CONSENSUS_LINK_LIBS}) 92 | set(CONSENSUS_LINK_LIBS ${CONSENSUS_LINK_LIBS} consensus_rpc) 93 | 94 | ##------------------- consensus-all -------------------## 95 | 96 | set(CONSENSUS_SOURCE_DIR ${PROJECT_SOURCE_DIR}/src) 97 | 98 | set(ALL_SOURCES 99 | ${CONSENSUS_SOURCE_DIR}/replicated_log.cc 100 | ${CONSENSUS_SOURCE_DIR}/replicated_log_impl.h 101 | ${CONSENSUS_SOURCE_DIR}/ready_flusher.cc 102 | ${CONSENSUS_SOURCE_DIR}/raft_timer.cc 103 | ${CONSENSUS_SOURCE_DIR}/raft_task_executor.cc 104 | ${CONSENSUS_SOURCE_DIR}/wal_commit_observer.cc 105 | ${CONSENSUS_SOURCE_DIR}/raft_service.cc 106 | ${RPC_SOURCES} 107 | ${WAL_SOURCES} 108 | ${BASE_SOURCES}) 109 | add_library(consensus_yaraft ${ALL_SOURCES}) 110 | target_link_libraries(consensus_yaraft ${CONSENSUS_LINK_LIBS}) 111 | set(CONSENSUS_LINK_LIBS ${CONSENSUS_LINK_LIBS} consensus_yaraft) 112 | 113 | function(ADD_CONSENSUS_TEST TEST_NAME) 114 | add_executable(${TEST_NAME} ${CONSENSUS_SOURCE_DIR}/${TEST_NAME}.cc) 115 | target_link_libraries(${TEST_NAME} ${CONSENSUS_LINK_LIBS} ${GTEST_LIB} ${GTEST_MAIN_LIB}) 116 | endfunction() 117 | 118 | ADD_CONSENSUS_TEST(raft_task_executor_test) 119 | ADD_CONSENSUS_TEST(raft_timer_test) 120 | ADD_CONSENSUS_TEST(raft_service_test) 121 | # ADD_CONSENSUS_TEST(replicated_log_test) 122 | 123 | install(TARGETS consensus_yaraft DESTINATION lib) 124 | install(DIRECTORY ${PROJECT_SOURCE_DIR}/include/consensus DESTINATION include) -------------------------------------------------------------------------------- /deps_definitions.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PROJECT_DIR=`pwd` 4 | BUILD_DIR=$PROJECT_DIR/build 5 | TP_DIR=$PROJECT_DIR/third_parties 6 | TP_BUILD_DIR="$BUILD_DIR/third_parties" 7 | TP_STAMP_DIR=$TP_BUILD_DIR/stamp 8 | 9 | SNAPPY_VERSION=1.1.3 10 | SNAPPY_NAME=snappy-$SNAPPY_VERSION 11 | SNAPPY_SOURCE=$TP_DIR/$SNAPPY_NAME 12 | 13 | BRPC_VERSION=`git rev-parse @:brpc | cut -c 1-7` 14 | BRPC_NAME=brpc-$BRPC_VERSION 15 | BRPC_SOURCE=$PROJECT_DIR/brpc 16 | 17 | GOOGLE_BENCH_VERSION=1.2.0 18 | GOOGLE_BENCH_NAME=benchmark-$GOOGLE_BENCH_VERSION 19 | GOOGLE_BENCH_SOURCE=$TP_DIR/$GOOGLE_BENCH_NAME 20 | 21 | GLOG_VERSION=0.3.5 22 | GLOG_NAME=glog-$GLOG_VERSION 23 | GLOG_SOURCE=$TP_DIR/$GLOG_NAME 24 | 25 | GFLAG_VERSION=2.2.0 26 | GFLAG_NAME=gflags-$GFLAG_VERSION 27 | GFLAG_SOURCE=$TP_DIR/$GFLAG_NAME 28 | 29 | QINIU_CDN_URL_PREFIX=http://onnzg1pyx.bkt.clouddn.com 30 | 31 | make_stamp() { 32 | touch $TP_STAMP_DIR/$1 33 | } 34 | 35 | fetch_and_expand() { 36 | local FILENAME=$1 37 | if [ -z "$FILENAME" ]; then 38 | echo "Error: Must specify file to fetch" 39 | exit 1 40 | fi 41 | 42 | TAR_CMD=tar 43 | if [[ "$OSTYPE" == "darwin"* ]] && which gtar &>/dev/null; then 44 | TAR_CMD=gtar 45 | fi 46 | 47 | FULL_URL="${QINIU_CDN_URL_PREFIX}/${FILENAME}" 48 | SUCCESS=0 49 | # Loop in case we encounter a corrupted archive and we need to re-download it. 50 | for attempt in 1 2; do 51 | if [ -r "$FILENAME" ]; then 52 | echo "Archive $FILENAME already exists. Not re-downloading archive." 53 | else 54 | echo "Fetching $FILENAME from $FULL_URL" 55 | wget "$FULL_URL" 56 | fi 57 | 58 | echo "Unpacking $FILENAME" 59 | if [[ "$FILENAME" =~ \.zip$ ]]; then 60 | if ! unzip -q "$FILENAME"; then 61 | echo "Error unzipping $FILENAME, removing file" 62 | rm "$FILENAME" 63 | continue 64 | fi 65 | elif [[ "$FILENAME" =~ \.(tar\.gz|tgz)$ ]]; then 66 | if ! $TAR_CMD xf "$FILENAME"; then 67 | echo "Error untarring $FILENAME, removing file" 68 | rm "$FILENAME" 69 | continue 70 | fi 71 | else 72 | echo "Error: unknown file format: $FILENAME" 73 | exit 1 74 | fi 75 | 76 | SUCCESS=1 77 | break 78 | done 79 | 80 | if [ $SUCCESS -ne 1 ]; then 81 | echo "Error: failed to fetch and unpack $FILENAME" 82 | exit 1 83 | fi 84 | 85 | # Allow for not removing previously-downloaded artifacts. 86 | # Useful on a low-bandwidth connection. 87 | if [ -z "$NO_REMOVE_THIRDPARTY_ARCHIVES" ]; then 88 | echo "Removing $FILENAME" 89 | rm $FILENAME 90 | fi 91 | echo 92 | } 93 | 94 | build_brpc() { 95 | pushd $BRPC_SOURCE 96 | bash $PROJECT_DIR/build_brpc.sh 97 | popd 98 | } 99 | 100 | build_google_bench() { 101 | mkdir -p $GOOGLE_BENCH_SOURCE/build 102 | pushd $GOOGLE_BENCH_SOURCE/build 103 | rm -rf CMakeCache.txt CMakeFiles/ 104 | cmake \ 105 | -DCMAKE_BUILD_TYPE=Release \ 106 | -DCMAKE_POSITION_INDEPENDENT_CODE=On \ 107 | -DCMAKE_INSTALL_PREFIX=$TP_BUILD_DIR \ 108 | -DBUILD_SHARED_LIBS=On \ 109 | -DBUILD_STATIC_LIBS=On \ 110 | -DREGISTER_INSTALL_PREFIX=Off \ 111 | $GOOGLE_BENCH_SOURCE 112 | make -j8 install 113 | popd 114 | } 115 | 116 | build_gflag() { 117 | GFLAGS_BDIR=$GFLAG_SOURCE/build 118 | mkdir -p $GFLAGS_BDIR 119 | pushd $GFLAGS_BDIR 120 | rm -rf CMakeCache.txt CMakeFiles/ 121 | cmake \ 122 | -DCMAKE_BUILD_TYPE=Release \ 123 | -DCMAKE_POSITION_INDEPENDENT_CODE=On \ 124 | -DCMAKE_INSTALL_PREFIX=$TP_BUILD_DIR \ 125 | -DBUILD_SHARED_LIBS=On \ 126 | -DBUILD_STATIC_LIBS=On \ 127 | -DREGISTER_INSTALL_PREFIX=Off \ 128 | $GFLAG_SOURCE 129 | make install 130 | popd 131 | } 132 | 133 | # Due to some sort of historical reason, glog seems conflicting with gflags. 134 | # If we build glog with gflags, some undefined reference problems will come up. 135 | build_glog() { 136 | GLOG_BDIR=$GLOG_SOURCE/build 137 | mkdir -p $GLOG_BDIR 138 | pushd ${GLOG_BDIR} 139 | cmake \ 140 | -DCMAKE_INSTALL_PREFIX=$TP_BUILD_DIR \ 141 | -DBUILD_SHARED_LIBS=Off \ 142 | -DWITH_GFLAGS=OFF \ 143 | -DBUILD_TESTING=OFF \ 144 | $GLOG_SOURCE 145 | make -j4 && make install 146 | popd 147 | } 148 | 149 | build_yaraft() { 150 | YARAFT_SOURCE=$PROJECT_DIR/yaraft 151 | YARAFT_BDIR=$YARAFT_SOURCE/build 152 | mkdir -p $YARAFT_BDIR 153 | pushd $YARAFT_SOURCE 154 | bash $YARAFT_SOURCE/install_deps_if_necessary.sh 155 | popd 156 | pushd $YARAFT_BDIR 157 | cmake \ 158 | -DCMAKE_INSTALL_PREFIX=$TP_BUILD_DIR \ 159 | -DBUILD_TEST=OFF \ 160 | -DCMAKE_BUILD_TYPE=Debug \ 161 | $YARAFT_SOURCE 162 | make -j4 && make install 163 | popd 164 | } -------------------------------------------------------------------------------- /src/replicated_log_impl.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #pragma once 16 | 17 | #include "base/env.h" 18 | #include "base/logging.h" 19 | #include "rpc/peer.h" 20 | #include "wal/wal.h" 21 | 22 | #include "raft_service.h" 23 | #include "raft_task_executor.h" 24 | #include "raft_timer.h" 25 | #include "ready_flusher.h" 26 | #include "replicated_log.h" 27 | #include "wal_commit_observer.h" 28 | 29 | #include 30 | 31 | namespace consensus { 32 | 33 | class ReplicatedLogImpl { 34 | friend class ReplicatedLog; 35 | 36 | public: 37 | static StatusWith New(const ReplicatedLogOptions &oldOptions) { 38 | // The construction order is: 39 | // - RawNode 40 | // - RaftTaskExecutor (depends on RawNode) 41 | // - RaftTimer, (depends on RaftTaskExecutor) 42 | // - ReadyFlusher (depends on WalCommitObserver, WAL, RPC) 43 | // - ReplicatedLog 44 | ReplicatedLogOptions options = oldOptions; 45 | RETURN_NOT_OK(options.Validate()); 46 | 47 | auto impl = new ReplicatedLogImpl; 48 | 49 | // -- RawNode -- 50 | auto conf = new yaraft::Config; 51 | conf->electionTick = options.election_timeout; 52 | conf->heartbeatTick = options.heartbeat_interval; 53 | conf->id = options.id; 54 | if (!options.memstore) { 55 | options.memstore = new yaraft::MemoryStorage; 56 | } 57 | conf->storage = options.memstore; 58 | for (const auto &e : options.initial_cluster) { 59 | conf->peers.push_back(e.first); 60 | } 61 | impl->node_.reset(new yaraft::RawNode(conf)); 62 | 63 | // -- RaftTaskExecutor -- 64 | TaskQueue *taskQueue = options.taskQueue; 65 | if (!taskQueue) { 66 | taskQueue = new TaskQueue; 67 | } 68 | impl->executor_.reset(new RaftTaskExecutor(impl->node_.get(), taskQueue)); 69 | 70 | // -- RaftTimer -- 71 | impl->timer_.reset(options.timer); 72 | if (!impl->timer_) { 73 | impl->timer_.reset(new RaftTimer); 74 | } 75 | impl->timer_->Register(impl->executor_.get()); 76 | 77 | // -- ReadyFlusher -- 78 | impl->wal_ = options.wal; 79 | impl->walCommitObserver_.reset(new WalCommitObserver); 80 | impl->memstore_ = options.memstore; 81 | impl->cluster_.reset(rpc::Cluster::Default(options.initial_cluster)); 82 | impl->flusher_.reset(options.flusher); 83 | if (!impl->flusher_) { 84 | impl->flusher_.reset(new ReadyFlusher); 85 | } 86 | impl->flusher_->Register(impl); 87 | 88 | auto rl = new ReplicatedLog; 89 | rl->impl_.reset(impl); 90 | return rl; 91 | } 92 | 93 | ~ReplicatedLogImpl() = default; 94 | 95 | SimpleChannel AsyncWrite(const Slice &log) { 96 | SimpleChannel channel; 97 | 98 | executor_->Submit([&](yaraft::RawNode *node) { 99 | uint64_t id = Id(); 100 | if (!node->IsLeader()) { 101 | channel <<= 102 | FMT_Status(WalWriteToNonLeader, "writing to a non-leader node, [id: {}, leader: {}]", 103 | id, node->LeaderHint()); 104 | return; 105 | } 106 | 107 | yaraft::Status s = node->Propose(log); 108 | if (UNLIKELY(!s.IsOK())) { 109 | channel <<= Status::Make(Error::YARaftError, s.ToString()); 110 | return; 111 | } 112 | 113 | // listening for the committedIndex to forward to the newly-appended log. 114 | uint64_t newIndex = node->LastIndex(); 115 | walCommitObserver_->Register(std::make_pair(newIndex, newIndex), &channel); 116 | }); 117 | 118 | return channel; 119 | } 120 | 121 | uint64_t Id() const { 122 | return node_->Id(); 123 | } 124 | 125 | private: 126 | friend class ReadyFlusher; 127 | 128 | std::unique_ptr node_; 129 | 130 | std::unique_ptr executor_; 131 | 132 | std::unique_ptr walCommitObserver_; 133 | 134 | std::unique_ptr cluster_; 135 | 136 | std::shared_ptr timer_; 137 | 138 | std::shared_ptr flusher_; 139 | 140 | yaraft::MemoryStorage *memstore_; 141 | 142 | wal::WriteAheadLog *wal_; 143 | }; 144 | 145 | } // namespace consensus -------------------------------------------------------------------------------- /apps/memkv/src/memkv_store.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | #include "logging.h" 19 | #include "memkv_store.h" 20 | 21 | #include 22 | 23 | namespace boost { 24 | template <> 25 | struct range_iterator { 26 | typedef memkv::Slice::const_iterator type; 27 | }; 28 | } // namespace boost 29 | 30 | namespace memkv { 31 | 32 | struct Node { 33 | using ChildrenTable = std::unordered_map; 34 | 35 | ~Node() { 36 | for (auto &child : children) { 37 | delete child.second; 38 | } 39 | } 40 | 41 | ChildrenTable children; 42 | std::string data; 43 | }; 44 | 45 | class MemKvStore::Impl { 46 | public: 47 | Status Write(const Slice &path, const Slice &value) { 48 | std::lock_guard d(mu_); 49 | 50 | std::vector pathVec; 51 | ASSIGN_IF_OK(validatePath(path), pathVec); 52 | 53 | Node *n = &root_; 54 | for (const Slice &seg : pathVec) { 55 | // ignore empty segment 56 | if (seg.Len() == 0) { 57 | continue; 58 | } 59 | 60 | auto result = n->children.emplace(std::make_pair(seg.ToString(), nullptr)); 61 | bool nodeNotExist = result.second; 62 | 63 | Node **pNode = &result.first->second; 64 | if (nodeNotExist) { 65 | *pNode = new Node(); 66 | } 67 | 68 | n = *pNode; 69 | } 70 | 71 | n->data = value.ToString(); 72 | return Status::OK(); 73 | } 74 | 75 | Status Delete(const Slice &path) { 76 | std::lock_guard d(mu_); 77 | 78 | std::vector pathVec; 79 | ASSIGN_IF_OK(validatePath(path), pathVec); 80 | 81 | Node *n = &root_; 82 | for (auto segIt = pathVec.begin(); segIt != pathVec.end(); segIt++) { 83 | Slice seg = *segIt; 84 | 85 | if (seg.Len() == 0) { 86 | continue; 87 | } 88 | 89 | auto it = n->children.find(seg.ToString()); 90 | if (it == n->children.end()) { 91 | // the given path is deleted 92 | return Status::OK(); 93 | } 94 | 95 | // we have found the matched node, delete it now. 96 | if (std::next(segIt) == pathVec.end()) { 97 | n->children.erase(it); 98 | return Status::OK(); 99 | } 100 | 101 | n = it->second; 102 | } 103 | 104 | return Status::Make(Error::InvalidArgument, "cannot delete root directory"); 105 | } 106 | 107 | Status Get(const Slice &path, std::string *data) { 108 | std::lock_guard d(mu_); 109 | 110 | std::vector pathVec; 111 | ASSIGN_IF_OK(validatePath(path), pathVec); 112 | 113 | Node *n = &root_; 114 | for (const Slice &seg : pathVec) { 115 | if (seg.Len() == 0) { 116 | continue; 117 | } 118 | 119 | auto it = n->children.find(seg.ToString()); 120 | if (it == n->children.end()) { 121 | return FMT_Status(NodeNotExist, "node does not exist on path {}", path.data()); 122 | } 123 | 124 | n = it->second; 125 | } 126 | 127 | *data = n->data; 128 | return Status::OK(); 129 | } 130 | 131 | private: 132 | StatusWith> validatePath(const Slice &p) { 133 | Slice path = p; 134 | path.TrimSpace(); 135 | if (UNLIKELY(path.Len() == 0)) { 136 | return Status::Make(Error::InvalidArgument, "path is empty"); 137 | } 138 | 139 | for (size_t i = 0; i < path.size(); i++) { 140 | if (path[i] == '\0') { 141 | return FMT_Status(InvalidArgument, "path contains NUL at index {}", i); 142 | } 143 | } 144 | 145 | std::vector result; 146 | boost::split(result, path, [](char c) { return c == '/'; }); 147 | return result; 148 | } 149 | 150 | private: 151 | Node root_; 152 | 153 | mutable std::mutex mu_; 154 | }; 155 | 156 | Status MemKvStore::Write(const Slice &path, const Slice &value) { 157 | return impl_->Write(path, value); 158 | } 159 | 160 | Status MemKvStore::Delete(const Slice &path) { 161 | return impl_->Delete(path); 162 | } 163 | 164 | Status MemKvStore::Get(const Slice &path, std::string *data) { 165 | return impl_->Get(path, data); 166 | } 167 | 168 | MemKvStore::MemKvStore() : impl_(new Impl) {} 169 | 170 | MemKvStore::~MemKvStore() = default; 171 | 172 | } // namespace memkv -------------------------------------------------------------------------------- /apps/memkv/src/db.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "db.h" 16 | #include "logging.h" 17 | #include "memkv_service.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace memkv { 24 | 25 | enum OpType { 26 | kWrite = 1, 27 | kDelete = 2, 28 | }; 29 | 30 | // Log format: 31 | // - WRITE: type path value 32 | // type = 1 byte 33 | // path, value = varstring 34 | // - DELETE: type path 35 | // 36 | static std::string LogEncode(OpType type, const Slice &path, const Slice &value) { 37 | using consensus::VarintLength; 38 | using consensus::PutLengthPrefixedSlice; 39 | 40 | std::string result; 41 | result.push_back(static_cast(type)); 42 | 43 | PutLengthPrefixedSlice(&result, path); 44 | if (type == kWrite) { 45 | PutLengthPrefixedSlice(&result, value); 46 | } 47 | return result; 48 | } 49 | 50 | class DB::Impl { 51 | public: 52 | explicit Impl(consensus::ReplicatedLog *replicatedLog) 53 | : kv_(new MemKvStore), log_(replicatedLog) {} 54 | 55 | Status Get(const Slice &path, bool stale, std::string *data) { 56 | bool allowed = false; 57 | 58 | // TODO: use read/write lock to protect `leader`, it's more efficient than using task queue. 59 | uint64_t currentLeader = 0; 60 | consensus::Barrier barrier; 61 | log_->RaftTaskExecutorInstance()->Submit([&](yaraft::RawNode *node) { 62 | currentLeader = node->LeaderHint(); 63 | barrier.Signal(); 64 | }); 65 | barrier.Wait(); 66 | 67 | uint64_t id = log_->Id(); 68 | if (currentLeader != id && !stale) { 69 | return FMT_Status(ConsensusError, 70 | "read from a non-leader node, stale read not allowed [id: {}, leader: {}]", 71 | id, currentLeader); 72 | } 73 | return kv_->Get(path, data); 74 | } 75 | 76 | Status Delete(const Slice &path) { 77 | std::string log = LogEncode(OpType::kDelete, path, nullptr); 78 | 79 | consensus::Status s = log_->Write(log); 80 | if (!s.IsOK()) { 81 | return Status::Make(Error::ConsensusError, s.ToString()); 82 | } 83 | 84 | RETURN_NOT_OK(kv_->Delete(path)); 85 | return Status::OK(); 86 | } 87 | 88 | Status Write(const Slice &path, const Slice &value) { 89 | std::string log = LogEncode(OpType::kWrite, path, value); 90 | 91 | consensus::Status s = log_->Write(log); 92 | if (!s.IsOK()) { 93 | return Status::Make(Error::ConsensusError, s.ToString()); 94 | } 95 | 96 | RETURN_NOT_OK(kv_->Write(path, value)); 97 | return Status::OK(); 98 | } 99 | 100 | private: 101 | friend class DB; 102 | 103 | std::unique_ptr log_; 104 | std::unique_ptr kv_; 105 | }; 106 | 107 | StatusWith DB::Bootstrap(const DBOptions &options) { 108 | using consensus::ReplicatedLogOptions; 109 | using consensus::ReplicatedLog; 110 | using namespace consensus::wal; 111 | 112 | ReplicatedLogOptions rlogOptions; 113 | rlogOptions.id = options.member_id; 114 | rlogOptions.heartbeat_interval = 100; 115 | rlogOptions.election_timeout = 1000; 116 | rlogOptions.initial_cluster = options.initial_cluster; 117 | 118 | WriteAheadLogOptions walOptions; 119 | walOptions.log_dir = options.wal_dir; 120 | 121 | WriteAheadLogUPtr wal; 122 | yaraft::MemStoreUptr memstore; 123 | consensus::Status s = WriteAheadLog::Default(walOptions, &wal, &memstore); 124 | if (!s.IsOK()) { 125 | return Status::Make(Error::ConsensusError, s.ToString()) << " [WriteAheadLog::Default]"; 126 | } 127 | rlogOptions.wal = wal.release(); 128 | rlogOptions.memstore = memstore.release(); 129 | 130 | consensus::StatusWith sw = ReplicatedLog::New(rlogOptions); 131 | if (!sw.IsOK()) { 132 | return Status::Make(Error::ConsensusError, sw.ToString()) << " [ReplicatedLog::New]"; 133 | } 134 | 135 | ReplicatedLog *rlog = sw.GetValue(); 136 | auto db = new DB(); 137 | db->impl_.reset(new DB::Impl(rlog)); 138 | return db; 139 | } 140 | 141 | Status DB::Get(const Slice &path, bool stale, std::string *data) { 142 | return impl_->Get(path, stale, data); 143 | } 144 | 145 | Status DB::Delete(const Slice &path) { 146 | return impl_->Delete(path); 147 | } 148 | 149 | Status DB::Write(const Slice &path, const Slice &value) { 150 | return impl_->Write(path, value); 151 | } 152 | 153 | DB::DB() {} 154 | 155 | DB::~DB() = default; 156 | 157 | consensus::pb::RaftService *DB::CreateRaftServiceInstance() const { 158 | return new consensus::RaftServiceImpl(impl_->log_->RaftTaskExecutorInstance()); 159 | } 160 | 161 | } // namespace memkv -------------------------------------------------------------------------------- /src/wal/log_manager_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Wu Tao 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include "base/testing.h" 16 | #include "wal/log_manager.h" 17 | #include "wal/readable_log_segment.h" 18 | 19 | namespace consensus { 20 | namespace wal { 21 | 22 | using yaraft::EntryVec; 23 | using yaraft::PBEntry; 24 | using yaraft::MemoryStorage; 25 | using yaraft::pb::Entry; 26 | 27 | inline bool operator==(EntryVec v1, EntryVec v2) { 28 | if (v1.size() != v2.size()) 29 | return false; 30 | for (int i = 0; i < v1.size(); i++) { 31 | if (v1[i].DebugString() != v2[i].DebugString()) { 32 | return false; 33 | } 34 | } 35 | return true; 36 | } 37 | 38 | class LogManagerTest : public BaseTest {}; 39 | 40 | TEST_F(LogManagerTest, AppendToOneSegment) { 41 | using namespace yaraft; 42 | 43 | struct TestData { 44 | size_t totalEntries; 45 | } tests[] = {{10}, {50}, {100}, {1000}}; 46 | 47 | for (auto t : tests) { 48 | TestDirGuard g(CreateTestDirGuard()); 49 | WriteAheadLogUPtr wal(TEST_CreateWalStore(GetTestDir())); 50 | 51 | EntryVec expected; 52 | for (uint64_t i = 1; i <= t.totalEntries; i++) { 53 | expected.push_back(PBEntry().Index(i).Term(i).v); 54 | } 55 | ASSERT_OK(wal->Write(expected)); 56 | 57 | // flush data into file 58 | ASSERT_OK(wal->Close()); 59 | 60 | yaraft::MemoryStorage memStore; 61 | SegmentMetaData meta; 62 | ASSERT_OK(ReadSegmentIntoMemoryStorage(GetTestDir() + "/" + SegmentFileName(1, 1), &memStore, 63 | &meta, true)); 64 | 65 | EntryVec actual(memStore.TEST_Entries().begin() + 1, memStore.TEST_Entries().end()); 66 | ASSERT_EQ(actual.size(), t.totalEntries); 67 | for (int i = 0; i < t.totalEntries; i++) { 68 | ASSERT_EQ(actual[i].DebugString(), expected[i].DebugString()); 69 | } 70 | } 71 | } 72 | 73 | TEST_F(LogManagerTest, AppendToMemStore) { 74 | using E = PBEntry; 75 | 76 | struct TestData { 77 | EntryVec vec; 78 | 79 | Error::ErrorCodes code; 80 | EntryVec expected; 81 | } tests[] = { 82 | {{E().Index(2).Term(1).v, E().Index(3).Term(1).v}, 83 | Error::OK, 84 | {E().Index(2).Term(1).v, E().Index(3).Term(1).v}}, 85 | 86 | {{E().Index(2).Term(2).v, E().Index(3).Term(1).v}, Error::YARaftError, {}}, 87 | 88 | {{E().Index(2).Term(2).v, E().Index(2).Term(2).v}, Error::OK, {E().Index(2).Term(2).v}}, 89 | 90 | {{E().Index(2).Term(2).v, E().Index(3).Term(3).v, E().Index(2).Term(4).v}, 91 | Error::OK, 92 | {E().Index(2).Term(4).v}}, 93 | 94 | {{E().Index(2).Term(1).v, E().Index(3).Term(1).v, E().Index(4).Term(2).v, 95 | E().Index(5).Term(3).v, E().Index(6).Term(3).v, E().Index(7).Term(3).v, 96 | E().Index(5).Term(4).v}, 97 | Error::OK, 98 | {E().Index(2).Term(1).v, E().Index(3).Term(1).v, E().Index(4).Term(2).v, 99 | E().Index(5).Term(4).v}}, 100 | }; 101 | 102 | for (auto t : tests) { 103 | MemoryStorage memstore; 104 | memstore.TEST_Entries().clear(); 105 | memstore.TEST_Entries().emplace_back(E().Index(1).Term(1).v); 106 | 107 | auto s = AppendToMemStore(t.vec, &memstore); 108 | ASSERT_EQ(s.Code(), t.code); 109 | 110 | if (s.IsOK()) { 111 | EntryVec& actual = memstore.TEST_Entries(); 112 | std::move(std::next(actual.begin()), actual.end(), actual.begin()); 113 | actual.pop_back(); 114 | 115 | ASSERT_TRUE(t.expected == memstore.TEST_Entries()); 116 | } 117 | } 118 | } 119 | 120 | // This test verifies that no logs will be loaded when LogManager recovers from empty directory. 121 | TEST_F(LogManagerTest, RecoverFromEmtpyDirectory) { 122 | TestDirGuard g(CreateTestDirGuard()); 123 | yaraft::MemStoreUptr memstore; 124 | WriteAheadLogUPtr d(TEST_CreateWalStore(GetTestDir(), &memstore)); 125 | 126 | ASSERT_EQ(memstore, nullptr); 127 | } 128 | 129 | TEST_F(LogManagerTest, Recover) { 130 | struct TestData { 131 | int logsNum; 132 | } tests[] = { 133 | {10}, {100}, {1000}, 134 | }; 135 | 136 | for (auto t : tests) { 137 | TestDirGuard g(CreateTestDirGuard()); 138 | size_t segNum; 139 | 140 | // prepare data 141 | EntryVec expected; 142 | for (uint64_t i = 1; i <= t.logsNum; i++) { 143 | expected.push_back(PBEntry().Index(i).Term(i).v); 144 | } 145 | { 146 | WriteAheadLogUPtr w(TEST_CreateWalStore(GetTestDir())); 147 | ASSERT_OK(w->Write(expected)); 148 | ASSERT_OK(w->Close()); 149 | 150 | auto m = dynamic_cast(w.get()); 151 | segNum = m->SegmentNum(); 152 | } 153 | 154 | WriteAheadLogOptions options; 155 | options.log_dir = GetTestDir(); 156 | options.log_segment_size = 1024; 157 | 158 | yaraft::MemStoreUptr memstore; 159 | LogManagerUPtr m; 160 | ASSERT_OK(LogManager::Recover(options, &memstore, &m)); 161 | 162 | ASSERT_EQ(segNum, m->SegmentNum()); 163 | 164 | EntryVec actual(memstore->TEST_Entries().begin() + 1, memstore->TEST_Entries().end()); 165 | for (int i = 1; i < actual.size(); i++) { 166 | ASSERT_TRUE(expected == actual); 167 | } 168 | } 169 | } 170 | 171 | } // namespace wal 172 | } // namespace consensus -------------------------------------------------------------------------------- /src/base/coding.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "base/coding.h" 6 | 7 | namespace consensus { 8 | 9 | void EncodeFixed32(char* buf, uint32_t value) { 10 | if (FLAGS_is_little_endian) { 11 | memcpy(buf, &value, sizeof(value)); 12 | } else { 13 | buf[0] = value & 0xff; 14 | buf[1] = (value >> 8) & 0xff; 15 | buf[2] = (value >> 16) & 0xff; 16 | buf[3] = (value >> 24) & 0xff; 17 | } 18 | } 19 | 20 | void EncodeFixed64(char* buf, uint64_t value) { 21 | if (FLAGS_is_little_endian) { 22 | memcpy(buf, &value, sizeof(value)); 23 | } else { 24 | buf[0] = value & 0xff; 25 | buf[1] = (value >> 8) & 0xff; 26 | buf[2] = (value >> 16) & 0xff; 27 | buf[3] = (value >> 24) & 0xff; 28 | buf[4] = (value >> 32) & 0xff; 29 | buf[5] = (value >> 40) & 0xff; 30 | buf[6] = (value >> 48) & 0xff; 31 | buf[7] = (value >> 56) & 0xff; 32 | } 33 | } 34 | 35 | void PutFixed32(std::string* dst, uint32_t value) { 36 | char buf[sizeof(value)]; 37 | EncodeFixed32(buf, value); 38 | dst->append(buf, sizeof(buf)); 39 | } 40 | 41 | void PutFixed64(std::string* dst, uint64_t value) { 42 | char buf[sizeof(value)]; 43 | EncodeFixed64(buf, value); 44 | dst->append(buf, sizeof(buf)); 45 | } 46 | 47 | char* EncodeVarint32(char* dst, uint32_t v) { 48 | // Operate on characters as unsigneds 49 | unsigned char* ptr = reinterpret_cast(dst); 50 | static const int B = 128; 51 | if (v < (1 << 7)) { 52 | *(ptr++) = v; 53 | } else if (v < (1 << 14)) { 54 | *(ptr++) = v | B; 55 | *(ptr++) = v >> 7; 56 | } else if (v < (1 << 21)) { 57 | *(ptr++) = v | B; 58 | *(ptr++) = (v >> 7) | B; 59 | *(ptr++) = v >> 14; 60 | } else if (v < (1 << 28)) { 61 | *(ptr++) = v | B; 62 | *(ptr++) = (v >> 7) | B; 63 | *(ptr++) = (v >> 14) | B; 64 | *(ptr++) = v >> 21; 65 | } else { 66 | *(ptr++) = v | B; 67 | *(ptr++) = (v >> 7) | B; 68 | *(ptr++) = (v >> 14) | B; 69 | *(ptr++) = (v >> 21) | B; 70 | *(ptr++) = v >> 28; 71 | } 72 | return reinterpret_cast(ptr); 73 | } 74 | 75 | void PutVarint32(std::string* dst, uint32_t v) { 76 | char buf[5]; 77 | char* ptr = EncodeVarint32(buf, v); 78 | dst->append(buf, ptr - buf); 79 | } 80 | 81 | char* EncodeVarint64(char* dst, uint64_t v) { 82 | static const int B = 128; 83 | unsigned char* ptr = reinterpret_cast(dst); 84 | while (v >= B) { 85 | *(ptr++) = (v & (B - 1)) | B; 86 | v >>= 7; 87 | } 88 | *(ptr++) = static_cast(v); 89 | return reinterpret_cast(ptr); 90 | } 91 | 92 | void PutVarint64(std::string* dst, uint64_t v) { 93 | char buf[10]; 94 | char* ptr = EncodeVarint64(buf, v); 95 | dst->append(buf, ptr - buf); 96 | } 97 | 98 | void PutLengthPrefixedSlice(std::string* dst, const Slice& value) { 99 | PutVarint32(dst, value.size()); 100 | dst->append(value.data(), value.size()); 101 | } 102 | 103 | int VarintLength(uint64_t v) { 104 | int len = 1; 105 | while (v >= 128) { 106 | v >>= 7; 107 | len++; 108 | } 109 | return len; 110 | } 111 | 112 | const char* GetVarint32PtrFallback(const char* p, const char* limit, uint32_t* value) { 113 | uint32_t result = 0; 114 | for (uint32_t shift = 0; shift <= 28 && p < limit; shift += 7) { 115 | uint32_t byte = *(reinterpret_cast(p)); 116 | p++; 117 | if (byte & 128) { 118 | // More bytes are present 119 | result |= ((byte & 127) << shift); 120 | } else { 121 | result |= (byte << shift); 122 | *value = result; 123 | return reinterpret_cast(p); 124 | } 125 | } 126 | return NULL; 127 | } 128 | 129 | bool GetVarint32(Slice* input, uint32_t* value) { 130 | const char* p = input->data(); 131 | const char* limit = p + input->size(); 132 | const char* q = GetVarint32Ptr(p, limit, value); 133 | if (q == NULL) { 134 | return false; 135 | } else { 136 | *input = Slice(q, limit - q); 137 | return true; 138 | } 139 | } 140 | 141 | const char* GetVarint64Ptr(const char* p, const char* limit, uint64_t* value) { 142 | uint64_t result = 0; 143 | for (uint32_t shift = 0; shift <= 63 && p < limit; shift += 7) { 144 | uint64_t byte = *(reinterpret_cast(p)); 145 | p++; 146 | if (byte & 128) { 147 | // More bytes are present 148 | result |= ((byte & 127) << shift); 149 | } else { 150 | result |= (byte << shift); 151 | *value = result; 152 | return reinterpret_cast(p); 153 | } 154 | } 155 | return NULL; 156 | } 157 | 158 | bool GetVarint64(Slice* input, uint64_t* value) { 159 | const char* p = input->data(); 160 | const char* limit = p + input->size(); 161 | const char* q = GetVarint64Ptr(p, limit, value); 162 | if (q == NULL) { 163 | return false; 164 | } else { 165 | *input = Slice(q, limit - q); 166 | return true; 167 | } 168 | } 169 | 170 | const char* GetLengthPrefixedSlice(const char* p, const char* limit, Slice* result) { 171 | uint32_t len; 172 | p = GetVarint32Ptr(p, limit, &len); 173 | if (p == NULL) 174 | return NULL; 175 | if (p + len > limit) 176 | return NULL; 177 | *result = Slice(p, len); 178 | return p + len; 179 | } 180 | 181 | bool GetLengthPrefixedSlice(Slice* input, Slice* result) { 182 | uint32_t len; 183 | if (GetVarint32(input, &len) && input->size() >= len) { 184 | *result = Slice(input->data(), len); 185 | input->Skip(len); 186 | return true; 187 | } else { 188 | return false; 189 | } 190 | } 191 | 192 | } // namespace consensus 193 | -------------------------------------------------------------------------------- /apps/memkv/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | 5 | PROJECT_DIR=`pwd` 6 | 7 | function usage() 8 | { 9 | echo "usage: run.sh []" 10 | echo 11 | echo "Command list:" 12 | echo " help print the help info" 13 | echo " build build the system" 14 | echo 15 | echo " start_onebox start memkv onebox" 16 | echo " stop_onebox stop memkv onebox" 17 | echo " list_onebox list memkv onebox" 18 | echo 19 | echo " test run unit test" 20 | echo 21 | echo "Command 'run.sh -h' will print help for subcommands." 22 | } 23 | 24 | function run_build() { 25 | pushd ${PROJECT_DIR} 26 | mkdir -p ${PROJECT_DIR}/cmake-build-debug 27 | mkdir -p ${PROJECT_DIR}/output 28 | cd ${PROJECT_DIR}/cmake-build-debug 29 | cmake \ 30 | -DCMAKE_INSTALL_PREFIX=${PROJECT_DIR}/output \ 31 | -DCMAKE_BUILD_TYPE=Debug\ 32 | .. 33 | make -j8 && make install 34 | popd 35 | } 36 | 37 | ##################### 38 | ## start_onebox 39 | ##################### 40 | 41 | function usage_start_onebox() 42 | { 43 | echo "Options for subcommand 'start_onebox':" 44 | echo " -h|--help print the help info" 45 | echo " -m|--server_count " 46 | echo " server count, default is 3" 47 | } 48 | 49 | function run_start_onebox() { 50 | cd ${PROJECT_DIR} 51 | SERVER_COUNT=3 52 | while [[ $# > 0 ]]; do 53 | key="$1" 54 | case $key in 55 | -h|--help) 56 | usage_start_onebox 57 | exit 0 58 | ;; 59 | -s|--server_count) 60 | SERVER_COUNT="$2" 61 | shift 62 | ;; 63 | *) 64 | echo "ERROR: unknown option \"$key\"" 65 | echo 66 | usage_start_onebox 67 | exit -1 68 | ;; 69 | esac 70 | shift 71 | done 72 | 73 | port=12320 74 | for i in $(seq ${SERVER_COUNT}) 75 | do 76 | # restart from empty wal dir 77 | SERVER_DIR=${PROJECT_DIR}/output/server${i}.consensus 78 | if [ -d $SERVER_DIR ]; then 79 | echo "rm -rf $SERVER_DIR" 80 | rm -rf $SERVER_DIR 81 | fi 82 | echo "./output/bin/memkv_server --id=${i} --server_count=${SERVER_COUNT} --wal_dir=$SERVER_DIR/wal --memkv_log_dir=$SERVER_DIR/log &> /dev/null &" 83 | ./output/bin/memkv_server \ 84 | --id=${i} \ 85 | --server_count=${SERVER_COUNT} \ 86 | --wal_dir=$SERVER_DIR/wal \ 87 | --memkv_log_dir=$SERVER_DIR/log &> /dev/null & 88 | done 89 | } 90 | 91 | ##################### 92 | ## stop_onebox 93 | ##################### 94 | 95 | function run_stop_onebox() { 96 | ps -ef | grep output/bin/memkv_server | grep 'server_count' | awk '{print $2}' | xargs kill &>/dev/null 97 | } 98 | 99 | ############################ 100 | ## list_onebox 101 | ############################ 102 | 103 | function run_list_onebox() { 104 | ps -ef | grep output/bin/memkv_server | grep 'server_count' | sort -k11 105 | } 106 | 107 | ##################### 108 | ## stop_onebox_instance 109 | ##################### 110 | 111 | function usage_stop_onebox_instance() 112 | { 113 | echo "Options for subcommand 'stop_onebox_instance':" 114 | echo " -h|--help print the help info" 115 | echo " -s|--server_id " 116 | echo " raft server id" 117 | } 118 | 119 | function run_stop_onebox_instance() { 120 | SERVER_ID=0 121 | while [[ $# > 0 ]]; do 122 | key="$1" 123 | case $key in 124 | -h|--help) 125 | usage_stop_onebox_instance 126 | exit 0 127 | ;; 128 | -s|--server_id) 129 | SERVER_ID="$2" 130 | shift 131 | ;; 132 | *) 133 | echo "ERROR: unknown option \"$key\"" 134 | echo 135 | usage_stop_onebox_instance 136 | exit -1 137 | ;; 138 | esac 139 | shift 140 | done 141 | if [ ${SERVER_ID} != "0" ]; then 142 | ps -ef | grep output/bin/memkv_server | grep id=${SERVER_ID} | awk '{print $2}' | xargs kill &>/dev/null 143 | fi 144 | } 145 | 146 | ##################### 147 | ## unit test 148 | ##################### 149 | 150 | TEST_DIR=cmake-build-debug/src 151 | 152 | function unit_test() 153 | { 154 | echo "===========" $1 "===========" 155 | ./$TEST_DIR/$1 156 | if [ $? -ne 0 ]; then 157 | echo "TEST FAILED!!!" 158 | exit 1 159 | fi 160 | } 161 | 162 | 163 | function run_test() { 164 | unit_test memkv_store_test 165 | } 166 | 167 | #################################################################### 168 | 169 | if [ $# -eq 0 ]; then 170 | usage 171 | exit 0 172 | fi 173 | 174 | cmd=$1 175 | case $cmd in 176 | help) 177 | usage 178 | ;; 179 | build) 180 | shift 181 | run_build $* 182 | ;; 183 | start_onebox) 184 | shift 185 | run_start_onebox $* 186 | ;; 187 | stop_onebox) 188 | shift 189 | run_stop_onebox $* 190 | ;; 191 | list_onebox) 192 | shift 193 | run_list_onebox $* 194 | ;; 195 | stop_onebox_instance) 196 | shift 197 | run_stop_onebox_instance $* 198 | ;; 199 | test) 200 | shift 201 | run_test $* 202 | ;; 203 | *) 204 | echo "ERROR: unknown command $cmd" 205 | echo 206 | usage 207 | exit -1 208 | esac 209 | --------------------------------------------------------------------------------