├── .gitignore ├── .travis.yml ├── AUTHORS ├── CMakeLists.txt ├── LICENSE ├── README.md ├── build_in_travis_ci.sh ├── client ├── CMakeLists.txt ├── client.h ├── pidb.pb.cc ├── pidb.pb.h ├── pidb.proto ├── test_get.cpp ├── test_iterator.cpp ├── test_pushfile.cpp ├── test_put.cpp ├── test_snapshot.cpp └── test_write_batch.cpp ├── demo ├── CMakeLists.txt ├── client.cpp ├── cmake_install.cmake ├── pidb.pb.cc ├── pidb.pb.h ├── pidb.proto ├── run_client.sh ├── run_server.sh ├── server.cpp ├── shflags └── stop.sh ├── shflags ├── src ├── README.md ├── include │ └── pidb │ │ ├── options.h │ │ └── status.h ├── libs │ └── json11 │ │ ├── .gitignore │ │ ├── CMakeLists.txt │ │ ├── LICENSE.txt │ │ ├── Makefile │ │ ├── README.md │ │ ├── json11.cpp │ │ ├── json11.hpp │ │ ├── json11.pc.in │ │ └── test.cpp ├── master │ ├── CMakeLists.old.txt │ ├── CMakeLists.txt │ ├── head │ │ ├── guard_dog.cpp │ │ ├── guard_dog.h │ │ ├── json11.cpp │ │ ├── json11.hpp │ │ ├── master.cpp │ │ ├── master.h │ │ ├── raft_manage.cpp │ │ ├── raft_manage.h │ │ ├── route_table.cpp │ │ ├── route_table.h │ │ ├── route_table_bad.json │ │ ├── route_table_ok.json │ │ ├── store_heartbeat.cpp │ │ ├── store_heartbeat.h │ │ └── store_table.json │ ├── master.proto │ ├── master_main.cpp │ ├── run_client.sh │ ├── run_server.sh │ ├── shflags │ ├── test_client.cpp │ └── test_main.cpp ├── port │ ├── README │ ├── atomic_pointer.h │ ├── port.h │ ├── port_config.h.in │ ├── port_example.h │ ├── port_stdcxx.h │ ├── thread_annotations.h │ └── win │ │ └── stdint.h ├── server │ ├── RouteTableTest.cpp │ ├── context_cache.cpp │ ├── context_cache.h │ ├── main.cpp │ ├── master.proto │ ├── master_service_impl.cpp │ ├── master_service_impl.h │ ├── pidb.proto │ ├── pidb_service_impl.cpp │ ├── pidb_service_impl.h │ ├── raftnode.cpp │ ├── raftnode.h │ ├── remote_file_send.cpp │ ├── remote_file_send.h │ ├── route_table.cpp │ ├── route_table.h │ ├── server.cpp │ ├── server.h │ ├── shareddb.h │ └── shflags └── util │ └── status.cc ├── tags ├── test ├── CMakeLists.txt ├── README.md ├── test_context_cache.cpp ├── test_push_file.cpp ├── test_route_table.cpp └── test_timer.cpp └── utils └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | 19 | # Compiled Static libraries 20 | *.lai 21 | *.la 22 | *.a 23 | *.lib 24 | 25 | # Executables 26 | *.exe 27 | *.out 28 | *.app 29 | 30 | #cmake 31 | *.cmake 32 | 33 | 34 | /build 35 | 36 | /buildserver 37 | /*build*/ 38 | /*bld*/ 39 | client/*build*/ 40 | */bld/ 41 | .DS_Store 42 | .idea 43 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | dist: xenial 3 | sudo: required 4 | 5 | compiler: 6 | - gcc 7 | 8 | env: 9 | - PURPOSE=compile 10 | # - PURPOSE=unittest 11 | 12 | install: 13 | - sudo apt-get install -qq realpath libgflags-dev libprotobuf-dev libprotoc-dev protobuf-compiler libgoogle-perftools-dev 14 | - sudo apt-get install libgtest-dev && cd /usr/src/gtest && sudo env "PATH=$PATH" cmake . && sudo make && sudo mv libgtest* /usr/lib/ && cd - 15 | - sudo apt-get install -y gdb 16 | - git clone https://github.com/dmclNewbee302/leveldb.git && mkdir -p leveldb/bld && cd leveldb/bld && cmake .. -DBUILD_SHARED_LIBS=1 -DBUILD_STATIC_LIBS=1 && make -j4 && sudo make install && cd - 17 | - git clone https://github.com/brpc/brpc.git && mkdir -p brpc/bld && cd brpc/bld && cmake .. && make -j4 && sudo make install && cd - 18 | - git clone https://github.com/brpc/braft.git && mkdir -p braft/bld && cd braft/bld && cmake .. && make -j4 && sudo make install && cd - 19 | 20 | before_script: 21 | - ulimit -c unlimited -S 22 | 23 | script: 24 | - sh build_in_travis_ci.sh 25 | 26 | after_failure: 27 | - COREFILE=$(find . -maxdepth 1 -name "core*" | head -n 1) 28 | - if [[ -f "$COREFILE" ]]; then gdb -c "$COREFILE" example -ex "thread apply all bt" -ex "set pagination 0" -batch; fi 29 | 30 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # Names should be added to this file like so: 2 | # Name or Organization 3 | 4 | UESTC DMCL RESEARCH GROUP. 5 | 6 | ehds 7 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.09) 2 | project(pidb C CXX) 3 | 4 | option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" ON) 5 | option(LINK_TCMALLOC "Link tcmalloc if possible" ON) 6 | option(BUILD_UNITTEST "Build Test" OFF) 7 | # execute_process( 8 | # COMMAND bash -c "find /usr/local/ -type d -path \"*include/levledb\" | xargs dirname | xargs dirname | tr -d '\n'" 9 | # OUTPUT_VARIABLE OUTPUT_PATH 10 | #) 11 | 12 | set(CMAKE_PROJECT_VERSION 0) 13 | set(CMAKE_PROJECT_VERSION_MAJOR 0) 14 | set(CMAKE_PROJECT_VERSION_MAJOR 1) 15 | 16 | set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) 17 | 18 | include(FindThreads) 19 | include(FindProtobuf) 20 | 21 | include_directories( 22 | "${PROJECT_SOURCE_DIR}" 23 | ) 24 | 25 | set(PIDB_PUBLIC_INCLUDE_DIR "include/pidb") 26 | set(PIDB_SERVER_DIR "src/server") 27 | if (NOT PROTOBUF_PROTOC_EXECUTABLE) 28 | get_filename_component(PROTO_LIB_DIR ${PROTOBUF_LIBRARY} DIRECTORY) 29 | set (PROTOBUF_PROTOC_EXECUTABLE "${PROTO_LIB_DIR}/../bin/protoc") 30 | endif() 31 | 32 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER "${PROJECT_SOURCE_DIR}/src/server/pidb.proto") 33 | protobuf_generate_cpp(MASTER_PROTO_SRC MASTER_PROTO_HEADER "${PROJECT_SOURCE_DIR}/src/server/master.proto") 34 | # include PROTO_HEADER 35 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 36 | 37 | find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) 38 | if(EXAMPLE_LINK_SO) 39 | find_library(BRPC_LIB NAMES brpc) 40 | find_library(BRAFT_LIB NAMES braft) 41 | else() 42 | find_library(BRPC_LIB NAMES libbrpc.a brpc) 43 | find_library(BRAFT_LIB NAMES libbraft.a braft) 44 | endif() 45 | 46 | if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) 47 | message(FATAL_ERROR "Fail to find brpc") 48 | endif() 49 | include_directories(${BRPC_INCLUDE_PATH}) 50 | 51 | find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) 52 | if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) 53 | message (FATAL_ERROR "Fail to find braft") 54 | endif() 55 | 56 | include_directories(${BRAFT_INCLUDE_PATH}) 57 | 58 | 59 | find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) 60 | find_library(GFLAGS_LIBRARY NAMES gflags libgflags) 61 | 62 | if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) 63 | message(FATAL_ERROR "Fail to find gflags") 64 | endif() 65 | include_directories(${GFLAGS_INCLUDE_PATH}) 66 | 67 | execute_process( 68 | COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" 69 | OUTPUT_VARIABLE GFLAGS_NS 70 | ) 71 | if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") 72 | execute_process( 73 | COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" 74 | OUTPUT_VARIABLE GFLAGS_NS 75 | ) 76 | endif() 77 | 78 | if (LINK_TCMALLOC) 79 | find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h) 80 | find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler) 81 | if (GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_LIBRARIES) 82 | set(CMAKE_CXX_FLAGS "-DBRPC_ENABLE_CPU_PROFILER") 83 | include_directories(${GPERFTOOLS_INCLUDE_DIR}) 84 | else () 85 | set (GPERFTOOLS_LIBRARIES "") 86 | endif () 87 | endif () 88 | 89 | set(CMAKE_CPP_FLAGS "-DGFLAGS_NS=${GFLAGS_NS}") 90 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") 91 | 92 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 93 | # require at least gcc 4.8 94 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) 95 | message(FATAL_ERROR "GCC is too old, please install a newer version supporting C++11") 96 | endif() 97 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 98 | # require at least clang 3.3 99 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) 100 | message(FATAL_ERROR "Clang is too old, please install a newer version supporting C++11") 101 | endif() 102 | else() 103 | message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") 104 | endif() 105 | 106 | if(CMAKE_VERSION VERSION_LESS "3.1.3") 107 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 108 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 109 | endif() 110 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 111 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 112 | endif() 113 | else() 114 | set(CMAKE_CXX_STANDARD 11) 115 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 116 | endif() 117 | 118 | find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) 119 | find_library(LEVELDB_LIB NAMES leveldb) 120 | if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) 121 | message(FATAL_ERROR "Fail to find leveldb") 122 | endif() 123 | 124 | 125 | 126 | add_subdirectory("${PROJECT_SOURCE_DIR}/src/libs/json11") 127 | include_directories("${PROJECT_SOURCE_DIR}/src/libs") 128 | include_directories(${LEVELDB_INCLUDE_PATH}) 129 | include_directories("${PROJECT_SOURCE_DIR}/src") 130 | include_directories("${PROJECT_SOURCE_DIR}/src/include") 131 | include_directories("${PROJECT_SOURCE_DIR}/src/server") 132 | 133 | 134 | add_library(pidb SHARED "") 135 | add_executable(pidb_server src/server/main.cpp) 136 | target_sources(pidb PUBLIC 137 | ${PROTO_SRC} 138 | ${PROTO_HEADER} 139 | ${MASTER_PROTO_SRC} 140 | ${MASTER_PROTO_HEADER} 141 | "${PROJECT_SOURCE_DIR}/src/server/route_table.cpp" 142 | "${PROJECT_SOURCE_DIR}/src/server/route_table.h" 143 | "${PROJECT_SOURCE_DIR}/src/port/port_stdcxx.h" 144 | "${PROJECT_SOURCE_DIR}/src/port/port.h" 145 | "${PROJECT_SOURCE_DIR}/src/include/pidb/status.h" 146 | "${PROJECT_SOURCE_DIR}/src/util/status.cc" 147 | "${PROJECT_SOURCE_DIR}/src/port/port_stdcxx.h" 148 | 149 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/server.h" 150 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/server.cpp" 151 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/raftnode.h" 152 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/raftnode.cpp" 153 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/shareddb.h" 154 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/pidb_service_impl.h" 155 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/pidb_service_impl.cpp" 156 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/master_service_impl.h" 157 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/master_service_impl.cpp" 158 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/context_cache.h" 159 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/context_cache.cpp" 160 | 161 | "${PROJECT_SOURCE_DIR}/src/include/pidb/options.h" 162 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/remote_file_send.h" 163 | "${PROJECT_SOURCE_DIR}/${PIDB_SERVER_DIR}/remote_file_send.cpp" 164 | 165 | 166 | ) 167 | 168 | target_include_directories(pidb_server 169 | PUBLIC 170 | $ 171 | $ 172 | ) 173 | 174 | set(DYNAMIC_LIB 175 | ${CMAKE_THREAD_LIBS_INIT} 176 | ${GFLAGS_LIBRARY} 177 | ${PROTOBUF_LIBRARY} 178 | ${GPERFTOOLS_LIBRARIES} 179 | ${LEVELDB_LIB} 180 | ${BRAFT_LIB} 181 | ${BRPC_LIB} 182 | json11 183 | rt 184 | ssl 185 | crypto 186 | dl 187 | z 188 | ) 189 | 190 | if (BUILD_UNITTEST) 191 | enable_testing() 192 | add_subdirectory(test) 193 | endif () 194 | 195 | 196 | target_link_libraries(pidb 197 | "-Xlinker \"-(\"" 198 | ${DYNAMIC_LIB} 199 | "-Xlinker \"-)\"") 200 | 201 | target_link_libraries(pidb_server pidb dl) 202 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 The PiDB Authors. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of DSC Lab of UESTC. nor the names of its 14 | contributors may be used to endorse or promote products derived from 15 | this software without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PiDB-distributed leveldb 2 | [![Build Status](https://travis-ci.com/CDDSCLab/PiDB.svg?branch=master)](https://travis-ci.com/github/CDDSCLab/PiDB) 3 | ## 所依赖的库 4 | 1. [brpc](https://github.com/apache/incubator-brpc) 5 | 2. [braft](https://github.com/brpc/braft) 6 | 7 | ## 关于测试 8 | 1. 请将所有的单元测试写在根目录的test文件夹下 9 | 2. 使用gtest的测试框架，具体请参考gtest的用法 10 | 3. 注意一些包的依赖，比如我要测试pidb server的一些功能， 11 | 要链接pidb的库文件 12 | 13 | 14 | ## 关于client 15 | 1. client最为和pidb的一个独立的库的存在 16 | 所以编译文件和测试都是单独的 17 | 18 | ## TODO 19 | 1. Iterator 的操作 20 | 1. 依照目前已经实现的Snapshot形式，返回id的形式标识不同用户的iterator 21 | 2. 具体功能与leveldb的功能一样（next，pre，seek） 22 | 3. 客户端需要根据当前遍历的结果做出调整，因为数据库是分region的，每个Server的region的key是不连续的 23 | 所以每个region会有一个iterator(smallest_key,largest_key),客户端需要根据当前region是否已经到头，选择下一个 24 | region等 25 | 4. 优化：每次next的时候可以获得一个batch的大小，减少访问次数 26 | 27 | 28 | 29 | ## 第一个实例 30 | 实现了一个极简的demo，能够实现leveldb的Put和Write 31 | 32 | demo演示 33 | 在demo文件夹下 34 | ``` 35 | mkdir bld && cd bld && cmake .. && make 36 | ``` 37 | 然后运行 38 | ``` 39 | run_server.sh 和 run_client.sh 40 | ``` 41 | -------------------------------------------------------------------------------- /build_in_travis_ci.sh: -------------------------------------------------------------------------------- 1 | if [ -z "$PURPOSE" ]; then 2 | echo "PURPOSE must be set" 3 | exit 1 4 | fi 5 | if [ -z "$CXX" ]; then 6 | echo "CXX must be set" 7 | exit 1 8 | fi 9 | if [ -z "$CC" ]; then 10 | echo "CC must be set" 11 | exit 1 12 | fi 13 | 14 | runcmd(){ 15 | eval $@ 16 | [[ $? != 0 ]] && { 17 | exit 1 18 | } 19 | return 0 20 | } 21 | 22 | echo "build combination: PURPOSE=$PURPOSE CXX=$CXX CC=$CC" 23 | 24 | rm -rf bld && mkdir bld && cd bld 25 | if [ "$PURPOSE" = "compile" ]; then 26 | if ! cmake ..; then 27 | echo "Fail to generate Makefile by cmake" 28 | exit 1 29 | fi 30 | make -j4 31 | elif [ "$PURPOSE" = "unittest" ]; then 32 | if ! cmake -DBUILD_UNIT_TESTS=ON ..; then 33 | echo "Fail to generate Makefile by cmake" 34 | exit 1 35 | fi 36 | make -j4 && cd test && sh ../../test/run_tests.sh && cd ../ 37 | else 38 | echo "Unknown purpose=\"$PURPOSE\"" 39 | fi 40 | -------------------------------------------------------------------------------- /client/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.9) 2 | project(block C CXX) 3 | 4 | option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" OFF) 5 | option(LINK_TCMALLOC "Link tcmalloc if possible" ON) 6 | 7 | # execute_process( 8 | # COMMAND bash -c "find ${CMAKE_SOURCE_DIR}/../.. -type d -path \"*output/include/braft\" | xargs dirname | xargs dirname | tr -d '\n'" 9 | # OUTPUT_VARIABLE OUTPUT_PATH 10 | #) 11 | 12 | set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) 13 | 14 | include(FindThreads) 15 | include(FindProtobuf) 16 | 17 | 18 | include_directories(${Protobuf_INCLUDE_DIRS}) 19 | include_directories("../src/server") 20 | link_directories("../cmake-build-debug") 21 | 22 | if (NOT PROTOBUF_PROTOC_EXECUTABLE) 23 | get_filename_component(PROTO_LIB_DIR ${PROTOBUF_LIBRARY} DIRECTORY) 24 | set (PROTOBUF_PROTOC_EXECUTABLE "${PROTO_LIB_DIR}/../bin/protoc") 25 | endif() 26 | 27 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER pidb.proto) 28 | # include PROTO_HEADER 29 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 30 | 31 | find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) 32 | if(EXAMPLE_LINK_SO) 33 | find_library(BRPC_LIB NAMES brpc) 34 | find_library(BRAFT_LIB NAMES braft) 35 | else() 36 | find_library(BRPC_LIB NAMES libbrpc.a brpc) 37 | find_library(BRAFT_LIB NAMES libbraft.a braft) 38 | endif() 39 | 40 | if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) 41 | message(FATAL_ERROR "Fail to find brpc") 42 | endif() 43 | include_directories(${BRPC_INCLUDE_PATH}) 44 | 45 | find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) 46 | if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) 47 | message (FATAL_ERROR "Fail to find braft") 48 | endif() 49 | include_directories(${BRAFT_INCLUDE_PATH}) 50 | 51 | find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) 52 | find_library(GFLAGS_LIBRARY NAMES gflags libgflags) 53 | 54 | if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) 55 | message(FATAL_ERROR "Fail to find gflags") 56 | endif() 57 | include_directories(${GFLAGS_INCLUDE_PATH}) 58 | 59 | execute_process( 60 | COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" 61 | OUTPUT_VARIABLE GFLAGS_NS 62 | ) 63 | if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") 64 | execute_process( 65 | COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" 66 | OUTPUT_VARIABLE GFLAGS_NS 67 | ) 68 | endif() 69 | 70 | if (LINK_TCMALLOC) 71 | find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h) 72 | find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler) 73 | if (GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_LIBRARIES) 74 | set(CMAKE_CXX_FLAGS "-DBRPC_ENABLE_CPU_PROFILER") 75 | include_directories(${GPERFTOOLS_INCLUDE_DIR}) 76 | else () 77 | set (GPERFTOOLS_LIBRARIES "") 78 | endif () 79 | endif () 80 | 81 | set(CMAKE_CPP_FLAGS "-DGFLAGS_NS=${GFLAGS_NS}") 82 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") 83 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 84 | # require at least gcc 4.8 85 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) 86 | message(FATAL_ERROR "GCC is too old, please install a newer version supporting C++11") 87 | endif() 88 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 89 | # require at least clang 3.3 90 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) 91 | message(FATAL_ERROR "Clang is too old, please install a newer version supporting C++11") 92 | endif() 93 | else() 94 | message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") 95 | endif() 96 | 97 | if(CMAKE_VERSION VERSION_LESS "3.1.3") 98 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 99 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 100 | endif() 101 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 102 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 103 | endif() 104 | else() 105 | set(CMAKE_CXX_STANDARD 11) 106 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 107 | endif() 108 | 109 | find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) 110 | find_library(LEVELDB_LIB NAMES leveldb) 111 | if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) 112 | message(FATAL_ERROR "Fail to find leveldb") 113 | endif() 114 | include_directories(${LEVELDB_INCLUDE_PATH}) 115 | 116 | include_directories($(PROJECT_SOURCE_DIR)) 117 | 118 | set(DYNAMIC_LIB 119 | ${CMAKE_THREAD_LIBS_INIT} 120 | ${GFLAGS_LIBRARY} 121 | ${PROTOBUF_LIBRARY} 122 | ${GPERFTOOLS_LIBRARIES} 123 | ${LEVELDB_LIB} 124 | ${BRPC_LIB} 125 | rt 126 | ssl 127 | pidb 128 | crypto 129 | dl 130 | z 131 | ) 132 | 133 | file(GLOB TEST_PIDB_SRCS "test_*.cpp") 134 | 135 | foreach(PIDB_UT ${TEST_PIDB_SRCS}) 136 | get_filename_component(PIDB_UT_WE ${PIDB_UT} NAME_WE) 137 | add_executable(${PIDB_UT_WE} ${PIDB_UT} ${PROTO_SRC} ${PROTO_HEADER}) 138 | target_link_libraries(${PIDB_UT_WE} 139 | "-Xlinker \"-(\"" 140 | ${DYNAMIC_LIB} 141 | "-Xlinker \"-)\"" 142 | ) 143 | endforeach() 144 | 145 | 146 | -------------------------------------------------------------------------------- /client/client.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-23. 3 | // 4 | 5 | #ifndef BLOCK_CLIENT_H 6 | #define BLOCK_CLIENT_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "pidb.pb.h" 15 | 16 | DEFINE_string(attachment, "", "Carry this along with requests"); 17 | DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto"); 18 | DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short"); 19 | DEFINE_string(server, "127.0.1.1:8100", "IP Address of server"); 20 | DEFINE_string(load_balancer, "", "The algorithm for load balancing"); 21 | DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds"); 22 | DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 23 | DEFINE_int32(interval_ms, 1000, "Milliseconds between consecutive requests"); 24 | 25 | 26 | #endif //BLOCK_CLIENT_H 27 | -------------------------------------------------------------------------------- /client/pidb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pidb; 3 | option cc_generic_services = true; 4 | 5 | message Empty{} 6 | 7 | message Success{ 8 | required bool success = 1; 9 | optional string message = 2; 10 | } 11 | message PiDBSnapshot{ 12 | required int32 id = 1; //snapshot id 13 | } 14 | 15 | 16 | message PiDBRequest{ 17 | required string key = 1; 18 | optional string value = 2; 19 | optional PiDBSnapshot snapshot = 3; //读清求的snapshot 20 | } 21 | 22 | message PiDBOperator{ 23 | required int32 op = 1; //0: unknown 0:Put 1:Delete 2:Write 完善 24 | required string key=2; 25 | optional string value = 3; 26 | } 27 | 28 | message PiDBWriteBatch{ 29 | repeated PiDBOperator WriteBatch = 1; 30 | } 31 | 32 | message PiDBResponse{ 33 | required bool success = 1; 34 | optional string old_value = 2; 35 | optional string new_value = 3; 36 | optional string redirect = 4; 37 | } 38 | 39 | //Iterator的相关信息 40 | message PiDBIterator{ 41 | required int32 id = 1; // Iterator的标识 42 | optional string start = 2; // Iterator的起始位置 暂时为 optional 43 | optional string stop = 3; // Iterator的终点位置 44 | } 45 | service PiDBService { 46 | rpc Put(PiDBRequest) returns (PiDBResponse); 47 | rpc Get(PiDBRequest) returns (PiDBResponse); 48 | rpc Write(PiDBWriteBatch) returns (PiDBResponse); 49 | rpc GetSnapshot(Empty) returns (PiDBSnapshot); 50 | rpc ReleaseSnapshot(PiDBSnapshot) returns (Success); 51 | rpc GetIterator(PiDBIterator) returns (PiDBIterator); 52 | rpc Iterate (PiDBIterator) returns (PiDBResponse); //Iterate 暂时默认为只能next 53 | } 54 | 55 | -------------------------------------------------------------------------------- /client/test_get.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-18. 3 | // 4 | #include "client.h" 5 | 6 | int main(int argc, char* argv[]) { 7 | // Parse gflags. We recommend you to use gflags as well. 8 | GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); 9 | 10 | butil::CreateDirectory(butil::FilePath("./data")); 11 | // A Channel represents a communication line to a Server. Notice that 12 | // Channel is thread-safe and can be shared by all threads in your program. 13 | brpc::Channel channel; 14 | 15 | // Initialize the channel, NULL means using default options. 16 | brpc::ChannelOptions options; 17 | options.protocol = FLAGS_protocol; 18 | options.connection_type = FLAGS_connection_type; 19 | options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/; 20 | options.max_retry = FLAGS_max_retry; 21 | if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) { 22 | LOG(ERROR) << "Fail to initialize channel"; 23 | return -1; 24 | } 25 | 26 | //初始化stub 27 | pidb::PiDBService_Stub stub(&channel); 28 | 29 | // Send a request and wait for the response every 1 second. 30 | int log_id = 0; 31 | 32 | 33 | brpc::Controller cntl; 34 | pidb::PiDBResponse response; 35 | 36 | //-----BATCH OPERATION 37 | // pidb::PiDBWriteBatch batch; 38 | // pidb::PiDBOperator *oper; 39 | // 40 | // 41 | // oper = batch.add_writebatch(); 42 | // //设置一个batch的操作 put key1->value1 43 | // oper->set_op(2); 44 | // oper->set_key("key4"); 45 | // oper->set_value("value4"); 46 | // 47 | // oper=batch.add_writebatch(); 48 | // oper->set_op(2); 49 | // oper->set_key("key5"); 50 | // oper->set_value("value5"); 51 | // //oper->set_value("value2"); //删除操作不需要设置alue 52 | // 53 | // cntl.set_log_id(log_id ++); // set by user 54 | // // Set attachment which is wired to network directly instead of 55 | // // being serialized into protobuf messages. 56 | // //cntl.request_attachment().append(FLAGS_attachment); 57 | // 58 | // // Because `done'(last parameter) is NULL, this function waits until 59 | // // the response comes back or error occurs(including timedout). 60 | // 61 | // stub.Write(&cntl, &batch, &response, NULL); 62 | // if (!cntl.Failed()) { 63 | // LOG(INFO) << "Received response from " << cntl.remote_side() 64 | // << " to " << cntl.local_side() 65 | // << ": " << response.success() << " (attached=" 66 | // << cntl.response_attachment() << ")" 67 | // << " latency=" << cntl.latency_us() << "us"; 68 | // if(response.success()) 69 | // LOG(INFO)<<"Put operation success"; 70 | // } else { 71 | // LOG(WARNING) << cntl.ErrorText(); 72 | // } 73 | // cntl.Reset(); 74 | // //获得snapshot 75 | // pidb::PiDBSnapshot *snapshot = new pidb::PiDBSnapshot(); 76 | // // pidb::PiDBSnapshot snapshot ; 77 | // pidb::Empty empty; 78 | // stub.GetSnapshot(&cntl,&empty,snapshot,NULL); 79 | // if(!cntl.Failed()){ 80 | // LOG(INFO)<id(); 81 | // } else{ 82 | // LOG(ERROR)< 7 | int main(int argc, char* argv[]) { 8 | // Parse gflags. We recommend you to use gflags as well. 9 | GFLAGS_NS::ParseCommandLineFlags(&argc, &argv, true); 10 | 11 | butil::CreateDirectory(butil::FilePath("./data")); 12 | // A Channel represents a communication line to a Server. Notice that 13 | // Channel is thread-safe and can be shared by all threads in your program. 14 | brpc::Channel channel; 15 | 16 | // Initialize the channel, NULL means using default options. 17 | brpc::ChannelOptions options; 18 | options.protocol = FLAGS_protocol; 19 | options.connection_type = FLAGS_connection_type; 20 | options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/; 21 | options.max_retry = FLAGS_max_retry; 22 | if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) { 23 | LOG(ERROR) << "Fail to initialize channel"; 24 | return -1; 25 | } 26 | 27 | //出使花stub 28 | pidb::PiDBService_Stub stub(&channel); 29 | 30 | // Send a request and wait for the response every 1 second. 31 | int log_id = 0; 32 | 33 | 34 | brpc::Controller cntl; 35 | pidb::PiDBResponse response; 36 | 37 | //pidb::PiDBSnapshot *snapshot = new pidb::PiDBSnapshot(); 38 | // pidb::PiDBSnapshot snapshot ; 39 | std::unique_ptr snapshot(new pidb::PiDBSnapshot()); 40 | cntl.set_log_id(log_id++); 41 | pidb::Empty empty; 42 | stub.GetSnapshot(&cntl,&empty,snapshot.get(),NULL); 43 | if(!cntl.Failed()){ 44 | LOG(INFO)<<"Get Snapshopt success id :"<id(); 45 | } else{ 46 | LOG(ERROR)<id()<<" Success"; 53 | }else{ 54 | LOG(INFO)<<"Release Snapshopt id:"<id()<<" Failed"; 55 | } 56 | 57 | 58 | pidb::PiDBRequest request; 59 | 60 | request.set_key("name"); 61 | request.set_value("uestc"); 62 | cntl.Reset(); 63 | stub.Put(&cntl,&request,&response,NULL); 64 | if (!cntl.Failed()) { 65 | LOG(INFO) << "Received response from " << cntl.remote_side() 66 | << " to " << cntl.local_side() 67 | << ": " << response.success() << " (attached=" 68 | << cntl.response_attachment() << ")" 69 | << " latency=" << cntl.latency_us() << "us"; 70 | LOG(INFO)<<"Put Key: "<id(); 78 | 79 | request.set_key("name"); 80 | request.set_value("dscl"); 81 | cntl.Reset(); 82 | stub.Put(&cntl,&request,&response,NULL); 83 | if (!cntl.Failed()) { 84 | LOG(INFO) << "Received response from " << cntl.remote_side() 85 | << " to " << cntl.local_side() 86 | << ": " << response.success() << " (attached=" 87 | << cntl.response_attachment() << ")" 88 | << " latency=" << cntl.latency_us() << "us"; 89 | LOG(INFO)<<"Put Key: "<id()<<"Key:"<value1 45 | oper->set_op(2); 46 | oper->set_key("key4"); 47 | oper->set_value("value4"); 48 | 49 | oper=batch.add_writebatch(); 50 | oper->set_op(2); 51 | oper->set_key("key5"); 52 | oper->set_value("value_new"); 53 | //oper->set_value("value2"); //删除操作不需要设置alue 54 | 55 | cntl.set_log_id(log_id ++); // set by user 56 | stub.Write(&cntl, &batch, &response, NULL); 57 | 58 | if (!cntl.Failed()) { 59 | LOG(INFO) << "Received response from " << cntl.remote_side() 60 | << " to " << cntl.local_side() 61 | << ": " << response.success() << " (attached=" 62 | << cntl.response_attachment() << ")" 63 | << " latency=" << cntl.latency_us() << "us"; 64 | LOG(INFO)<<"Write batch success"; 65 | } else { 66 | LOG(WARNING) << cntl.ErrorText(); 67 | } 68 | } -------------------------------------------------------------------------------- /demo/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8.10) 2 | project(block C CXX) 3 | 4 | option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" OFF) 5 | option(LINK_TCMALLOC "Link tcmalloc if possible" ON) 6 | 7 | execute_process( 8 | COMMAND bash -c "find ${CMAKE_SOURCE_DIR}/../../.. -type d -path \"*output/include/braft\" | xargs dirname | xargs dirname | tr -d '\n'" 9 | OUTPUT_VARIABLE OUTPUT_PATH 10 | ) 11 | execute_process( 12 | COMMAND bash -c "cp ${CMAKE_SOURCE_DIR}/*.sh ${CMAKE_CURRENT_BINARY_DIR}" 13 | ) 14 | set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) 15 | 16 | include(FindThreads) 17 | include(FindProtobuf) 18 | 19 | if (NOT PROTOBUF_PROTOC_EXECUTABLE) 20 | get_filename_component(PROTO_LIB_DIR ${PROTOBUF_LIBRARY} DIRECTORY) 21 | set (PROTOBUF_PROTOC_EXECUTABLE "${PROTO_LIB_DIR}/../bin/protoc") 22 | endif() 23 | 24 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER pidb.proto) 25 | # include PROTO_HEADER 26 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 27 | 28 | find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) 29 | if(EXAMPLE_LINK_SO) 30 | find_library(BRPC_LIB NAMES brpc) 31 | find_library(BRAFT_LIB NAMES braft) 32 | else() 33 | find_library(BRPC_LIB NAMES libbrpc.a brpc) 34 | find_library(BRAFT_LIB NAMES libbraft.a braft) 35 | endif() 36 | 37 | if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) 38 | message(FATAL_ERROR "Fail to find brpc") 39 | endif() 40 | include_directories(${BRPC_INCLUDE_PATH}) 41 | 42 | find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) 43 | if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) 44 | message (FATAL_ERROR "Fail to find braft") 45 | endif() 46 | include_directories(${BRAFT_INCLUDE_PATH}) 47 | 48 | find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) 49 | find_library(GFLAGS_LIBRARY NAMES gflags libgflags) 50 | if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) 51 | message(FATAL_ERROR "Fail to find gflags") 52 | endif() 53 | include_directories(${GFLAGS_INCLUDE_PATH}) 54 | 55 | execute_process( 56 | COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" 57 | OUTPUT_VARIABLE GFLAGS_NS 58 | ) 59 | if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") 60 | execute_process( 61 | COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" 62 | OUTPUT_VARIABLE GFLAGS_NS 63 | ) 64 | endif() 65 | 66 | if (LINK_TCMALLOC) 67 | find_path(GPERFTOOLS_INCLUDE_DIR NAMES gperftools/heap-profiler.h) 68 | find_library(GPERFTOOLS_LIBRARIES NAMES tcmalloc_and_profiler) 69 | if (GPERFTOOLS_INCLUDE_DIR AND GPERFTOOLS_LIBRARIES) 70 | set(CMAKE_CXX_FLAGS "-DBRPC_ENABLE_CPU_PROFILER") 71 | include_directories(${GPERFTOOLS_INCLUDE_DIR}) 72 | else () 73 | set (GPERFTOOLS_LIBRARIES "") 74 | endif () 75 | endif () 76 | 77 | set(CMAKE_CPP_FLAGS "-DGFLAGS_NS=${GFLAGS_NS}") 78 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") 79 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 80 | # require at least gcc 4.8 81 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) 82 | message(FATAL_ERROR "GCC is too old, please install a newer version supporting C++11") 83 | endif() 84 | elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 85 | # require at least clang 3.3 86 | if(CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.3) 87 | message(FATAL_ERROR "Clang is too old, please install a newer version supporting C++11") 88 | endif() 89 | else() 90 | message(WARNING "You are using an unsupported compiler! Compilation has only been tested with Clang and GCC.") 91 | endif() 92 | 93 | if(CMAKE_VERSION VERSION_LESS "3.1.3") 94 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 95 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 96 | endif() 97 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 98 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 99 | endif() 100 | else() 101 | set(CMAKE_CXX_STANDARD 11) 102 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 103 | endif() 104 | 105 | find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) 106 | find_library(LEVELDB_LIB NAMES leveldb) 107 | if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) 108 | message(FATAL_ERROR "Fail to find leveldb") 109 | endif() 110 | include_directories(${LEVELDB_INCLUDE_PATH}) 111 | 112 | add_executable(db_client client.cpp ${PROTO_SRC} ${PROTO_HEADER}) 113 | add_executable(db_server server.cpp ${PROTO_SRC} ${PROTO_HEADER}) 114 | 115 | set(DYNAMIC_LIB 116 | ${CMAKE_THREAD_LIBS_INIT} 117 | ${GFLAGS_LIBRARY} 118 | ${PROTOBUF_LIBRARY} 119 | ${GPERFTOOLS_LIBRARIES} 120 | ${LEVELDB_LIB} 121 | ${BRAFT_LIB} 122 | ${BRPC_LIB} 123 | rt 124 | ssl 125 | crypto 126 | dl 127 | z 128 | ) 129 | 130 | target_link_libraries(db_client 131 | "-Xlinker \"-(\"" 132 | ${DYNAMIC_LIB} 133 | "-Xlinker \"-)\"") 134 | target_link_libraries(db_server 135 | "-Xlinker \"-(\"" 136 | ${DYNAMIC_LIB} 137 | "-Xlinker \"-)\"") 138 | -------------------------------------------------------------------------------- /demo/client.cpp: -------------------------------------------------------------------------------- 1 | // author:ehds 2 | // date: 2019-04-03 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "pidb.pb.h" 12 | 13 | DEFINE_bool(log_each_request, false, "Print log for each request"); 14 | DEFINE_bool(use_bthread, false, "Use bthread to send requests"); 15 | DEFINE_int32(block_size, 64 * 1024 * 1024u, "Size of block"); 16 | DEFINE_int32(request_size, 1024, "Size of each requst"); 17 | DEFINE_int32(thread_num, 1, "Number of threads sending requests"); 18 | DEFINE_int32(timeout_ms, 500, "Timeout for each request"); 19 | DEFINE_int32(write_percentage, 20, "Percentage of fetch_add"); 20 | DEFINE_string(conf, "", "Configuration of the raft group"); 21 | DEFINE_string(group, "Block", "Id of the replication group"); 22 | DEFINE_string(key,"name","key of operation"); 23 | DEFINE_string(value,"hedongseng","value of operation"); 24 | 25 | bvar::LatencyRecorder g_latency_recorder("block_client"); 26 | 27 | static void* sender(void* arg) { 28 | while (!brpc::IsAskedToQuit()) { 29 | braft::PeerId leader; 30 | // Select leader of the target group from RouteTable 31 | if (braft::rtb::select_leader(FLAGS_group, &leader) != 0) { 32 | // Leader is unknown in RouteTable. Ask RouteTable to refresh leader 33 | // by sending RPCs. 34 | butil::Status st = braft::rtb::refresh_leader( 35 | FLAGS_group, FLAGS_timeout_ms); 36 | if (!st.ok()) { 37 | // Not sure about the leader, sleep for a while and the ask again. 38 | LOG(WARNING) << "Fail to refresh_leader : " << st; 39 | bthread_usleep(FLAGS_timeout_ms * 1000L); 40 | } 41 | continue; 42 | } 43 | 44 | // Now we known who is the leader, construct Stub and then sending 45 | // rpc 46 | brpc::Channel channel; 47 | if (channel.Init(leader.addr, NULL) != 0) { 48 | LOG(ERROR) << "Fail to init channel to " << leader; 49 | bthread_usleep(FLAGS_timeout_ms * 1000L); 50 | continue; 51 | } 52 | pidb::PiDBService_Stub stub(&channel); 53 | 54 | brpc::Controller cntl; 55 | cntl.set_timeout_ms(FLAGS_timeout_ms); 56 | // Randomly select which request we want send; 57 | pidb::PiDBRequest request; 58 | pidb::PiDBResponse response; 59 | const char* op = NULL; 60 | if (butil::fast_rand_less_than(100) < (size_t)FLAGS_write_percentage) { 61 | op = "write"; 62 | request.set_key(FLAGS_key); 63 | request.set_value(FLAGS_value); 64 | cntl.request_attachment().resize(FLAGS_request_size, 'a'); 65 | stub.write(&cntl, &request, &response, NULL); 66 | } else { 67 | op = "read"; 68 | request.set_key(FLAGS_key); 69 | request.set_value(FLAGS_value); 70 | stub.read(&cntl, &request, &response, NULL); 71 | } 72 | if (cntl.Failed()) { 73 | LOG(WARNING) << "Fail to send request to " << leader 74 | << " : " << cntl.ErrorText(); 75 | // Clear leadership since this RPC failed. 76 | braft::rtb::update_leader(FLAGS_group, braft::PeerId()); 77 | bthread_usleep(FLAGS_timeout_ms * 1000L); 78 | continue; 79 | } 80 | if (!response.success()) { 81 | LOG(WARNING) << "Fail to send request to " << leader 82 | << ", redirecting to " 83 | << (response.has_redirect() 84 | ? response.redirect() : "nowhere"); 85 | // Update route table since we have redirect information 86 | braft::rtb::update_leader(FLAGS_group, response.redirect()); 87 | continue; 88 | } 89 | if(std::string(op)=="read"){ 90 | LOG(INFO)<<"READ DATA FROM DB key is "< tids; 121 | tids.resize(FLAGS_thread_num); 122 | if (!FLAGS_use_bthread) { 123 | for (int i = 0; i < FLAGS_thread_num; ++i) { 124 | if (pthread_create(&tids[i], NULL, sender, NULL) != 0) { 125 | LOG(ERROR) << "Fail to create pthread"; 126 | return -1; 127 | } 128 | } 129 | } else { 130 | for (int i = 0; i < FLAGS_thread_num; ++i) { 131 | if (bthread_start_background(&tids[i], NULL, sender, NULL) != 0) { 132 | LOG(ERROR) << "Fail to create bthread"; 133 | return -1; 134 | } 135 | } 136 | } 137 | 138 | while (!brpc::IsAskedToQuit()) { 139 | sleep(10); 140 | LOG_IF(INFO, !FLAGS_log_each_request) 141 | << "Sending Request to " << FLAGS_group 142 | << " (" << FLAGS_conf << ')' 143 | << " at qps=" << g_latency_recorder.qps(1) 144 | << " latency=" << g_latency_recorder.latency(1); 145 | } 146 | 147 | LOG(INFO) << "Block client is going to quit"; 148 | for (int i = 0; i < FLAGS_thread_num; ++i) { 149 | if (!FLAGS_use_bthread) { 150 | pthread_join(tids[i], NULL); 151 | } else { 152 | bthread_join(tids[i], NULL); 153 | } 154 | } 155 | 156 | return 0; 157 | } 158 | -------------------------------------------------------------------------------- /demo/cmake_install.cmake: -------------------------------------------------------------------------------- 1 | # Install script for directory: /home/ehds/Project/braft-master/example/PiDB 2 | 3 | # Set the install prefix 4 | if(NOT DEFINED CMAKE_INSTALL_PREFIX) 5 | set(CMAKE_INSTALL_PREFIX "/usr/local") 6 | endif() 7 | string(REGEX REPLACE "/$" "" CMAKE_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") 8 | 9 | # Set the install configuration name. 10 | if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME) 11 | if(BUILD_TYPE) 12 | string(REGEX REPLACE "^[^A-Za-z0-9_]+" "" 13 | CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}") 14 | else() 15 | set(CMAKE_INSTALL_CONFIG_NAME "") 16 | endif() 17 | message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"") 18 | endif() 19 | 20 | # Set the component getting installed. 21 | if(NOT CMAKE_INSTALL_COMPONENT) 22 | if(COMPONENT) 23 | message(STATUS "Install component: \"${COMPONENT}\"") 24 | set(CMAKE_INSTALL_COMPONENT "${COMPONENT}") 25 | else() 26 | set(CMAKE_INSTALL_COMPONENT) 27 | endif() 28 | endif() 29 | 30 | # Install shared libraries without execute permission? 31 | if(NOT DEFINED CMAKE_INSTALL_SO_NO_EXE) 32 | set(CMAKE_INSTALL_SO_NO_EXE "0") 33 | endif() 34 | 35 | # Is this installation the result of a crosscompile? 36 | if(NOT DEFINED CMAKE_CROSSCOMPILING) 37 | set(CMAKE_CROSSCOMPILING "FALSE") 38 | endif() 39 | 40 | if(CMAKE_INSTALL_COMPONENT) 41 | set(CMAKE_INSTALL_MANIFEST "install_manifest_${CMAKE_INSTALL_COMPONENT}.txt") 42 | else() 43 | set(CMAKE_INSTALL_MANIFEST "install_manifest.txt") 44 | endif() 45 | 46 | string(REPLACE ";" "\n" CMAKE_INSTALL_MANIFEST_CONTENT 47 | "${CMAKE_INSTALL_MANIFEST_FILES}") 48 | file(WRITE "/home/ehds/Project/braft-master/example/PiDB/${CMAKE_INSTALL_MANIFEST}" 49 | "${CMAKE_INSTALL_MANIFEST_CONTENT}") 50 | -------------------------------------------------------------------------------- /demo/pidb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pidb; 3 | option cc_generic_services = true; 4 | 5 | message PiDBRequest{ 6 | required string key = 1; 7 | optional string value = 2; 8 | } 9 | 10 | message PiDBResponse{ 11 | required bool success = 1; 12 | optional string old_value = 2; 13 | optional string new_value = 3; 14 | optional string redirect = 4; 15 | } 16 | 17 | service PiDBService { 18 | rpc write(PiDBRequest) returns (PiDBResponse); 19 | rpc read(PiDBRequest) returns (PiDBResponse); 20 | } 21 | -------------------------------------------------------------------------------- /demo/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright (c) 2018 Baidu.com, Inc. All Rights Reserved 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # source shflags from current directory 18 | mydir="${BASH_SOURCE%/*}" 19 | if [[ ! -d "$mydir" ]]; then mydir="$PWD"; fi 20 | . $mydir/../shflags 21 | 22 | 23 | # define command-line flags 24 | DEFINE_boolean clean 1 'Remove old "runtime" dir before running' 25 | DEFINE_integer write_percentage 10 'Percentage of write operation' 26 | DEFINE_integer bthread_concurrency '8' 'Number of worker pthreads' 27 | DEFINE_integer server_port 8200 "Port of the first server" 28 | DEFINE_integer server_num '3' 'Number of servers' 29 | DEFINE_integer thread_num 1 'Number of sending thread' 30 | DEFINE_string crash_on_fatal 'true' 'Crash on fatal log' 31 | DEFINE_string log_each_request 'false' 'Print log for each request' 32 | DEFINE_string valgrind 'false' 'Run in valgrind' 33 | DEFINE_string use_bthread "true" "Use bthread to send request" 34 | DEFINE_string key 'name' "key of operation" 35 | DEFINE_string value 'hale' "value of operation" 36 | 37 | FLAGS "$@" || exit 1 38 | 39 | # hostname prefers ipv6 40 | IP=`hostname -i | awk '{print $NF}'` 41 | 42 | if [ "$FLAGS_valgrind" == "true" ] && [ $(which valgrind) ] ; then 43 | VALGRIND="valgrind --tool=memcheck --leak-check=full" 44 | fi 45 | 46 | raft_peers="" 47 | for ((i=0; i<$FLAGS_server_num; ++i)); do 48 | raft_peers="${raft_peers}${IP}:$((${FLAGS_server_port}+i)):0," 49 | done 50 | 51 | export TCMALLOC_SAMPLE_PARAMETER=524288 52 | 53 | ${VALGRIND} ./db_client\ 54 | --write_percentage=${FLAGS_write_percentage} \ 55 | --bthread_concurrency=${FLAGS_bthread_concurrency} \ 56 | --conf="${raft_peers}" \ 57 | --crash_on_fatal_log=${FLAGS_crash_on_fatal} \ 58 | --log_each_request=${FLAGS_log_each_request} \ 59 | --thread_num=${FLAGS_thread_num} \ 60 | --use_bthread=${FLAGS_use_bthread} \ 61 | 62 | -------------------------------------------------------------------------------- /demo/run_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | mydir="${BASH_SOURCE%/*}" 3 | if [[ ! -d "$mydir" ]]; then mydir="$PWD"; fi 4 | . $mydir/../shflags 5 | 6 | # define command-line flags 7 | DEFINE_string crash_on_fatal 'true' 'Crash on fatal log' 8 | DEFINE_integer bthread_concurrency '18' 'Number of worker pthreads' 9 | DEFINE_string sync 'true' 'fsync each time' 10 | DEFINE_string valgrind 'false' 'Run in valgrind' 11 | DEFINE_integer max_segment_size '8388608' 'Max segment size' 12 | DEFINE_integer server_num '3' 'Number of servers' 13 | DEFINE_boolean clean 1 'Remove old "runtime" dir before running' 14 | DEFINE_integer port 8200 "Port of the first server" 15 | DEFINE_string db_path './db' "Path of leveldb" 16 | # parse the command-line 17 | FLAGS "$@" || exit 1 18 | eval set -- "${FLAGS_ARGV}" 19 | 20 | # The alias for printing to stderr 21 | alias error=">&2 echo block: " 22 | 23 | # hostname prefers ipv6 24 | IP=`hostname -i | awk '{print $NF}'` 25 | 26 | if [ "$FLAGS_valgrind" == "true" ] && [ $(which valgrind) ] ; then 27 | VALGRIND="valgrind --tool=memcheck --leak-check=full" 28 | fi 29 | 30 | raft_peers="" 31 | for ((i=0; i<$FLAGS_server_num; ++i)); do 32 | raft_peers="${raft_peers}${IP}:$((${FLAGS_port}+$i)):0," 33 | done 34 | echo ${raft_peers} 35 | 36 | if [ "$FLAGS_clean" == "0" ]; then 37 | rm -rf runtime 38 | fi 39 | 40 | export TCMALLOC_SAMPLE_PARAMETER=524288 41 | 42 | for ((i=0; i<$FLAGS_server_num; ++i)); do 43 | mkdir -p runtime/$i 44 | cp ./db_server runtime/$i 45 | cd runtime/$i 46 | ${VALGRIND} ./db_server\ 47 | -bthread_concurrency=${FLAGS_bthread_concurrency}\ 48 | -crash_on_fatal_log=${FLAGS_crash_on_fatal} \ 49 | -raft_max_segment_size=${FLAGS_max_segment_size} \ 50 | -raft_sync=${FLAGS_sync} \ 51 | -port=$((${FLAGS_port}+i)) -conf="${raft_peers}" > std.log 2>&1 & 52 | 53 | if [[ ! -d "./db" ]]; then mkdir db; fi 54 | 55 | cd ../.. 56 | done 57 | -------------------------------------------------------------------------------- /demo/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | killall -9 db_server 4 | -------------------------------------------------------------------------------- /src/README.md: -------------------------------------------------------------------------------- 1 | # server目录 2 | 3 | 完成功能 4 | ## 一 server端 5 | 1. server的心跳（收集信息发给PD) 6 | 2. server对raft的管理 7 | 3. routetable 8 | 9 | ## levedb功能补充 10 | 1. 增加range的dumpfile功能 11 | 12 | ## raft 13 | 1. leveldb的写操作 14 | 2. iterator 15 | 3. snapshot 16 | 17 | # 关于测试 18 | 1. 请将所有的单元测试写在根目录的test文件夹下 19 | 2. 使用gtest的测试框架，具体请参考gtest的用法 20 | 3. 注意一些包的依赖，比如我要测试pidb server的一些功能， 21 | 要链接pidb的库文件 22 | 23 | -------------------------------------------------------------------------------- /src/include/pidb/options.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-4. 3 | // 4 | 5 | #ifndef PIDB_OPTIONS_H 6 | #define PIDB_OPTIONS_H 7 | 8 | #include 9 | #include 10 | 11 | namespace pidb { 12 | 13 | struct RaftOption 14 | { 15 | std::string group; 16 | std::string conf; 17 | std::string data_path; 18 | int32_t port; 19 | RaftOption():data_path("./data"){} 20 | 21 | }; 22 | 23 | struct ServerOption{ 24 | std::string data_path; 25 | int32_t port; 26 | int heartbeat_timeout_ms; 27 | //默认的配置 28 | ServerOption(const std::string &path,int32_t p) 29 | :data_path(path),port(p){}; 30 | ServerOption():data_path("./data"),port(8100),heartbeat_timeout_ms(5000){} 31 | }; 32 | 33 | } 34 | 35 | 36 | #endif //PIDB_OPTIONS_H 37 | -------------------------------------------------------------------------------- /src/include/pidb/status.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-4. 3 | // 4 | 5 | #ifndef BLOCK_STATUS_H 6 | #define BLOCK_STATUS_H 7 | 8 | #include 9 | #include 10 | #include "leveldb/slice.h" 11 | 12 | namespace pidb { 13 | using leveldb::Slice; 14 | class Status { 15 | public: 16 | // Create a success status. 17 | Status() noexcept : state_(nullptr) { } 18 | ~Status() { delete[] state_; } 19 | 20 | Status(const Status& rhs); 21 | Status& operator=(const Status& rhs); 22 | 23 | Status(Status&& rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } 24 | Status& operator=(Status&& rhs) noexcept; 25 | 26 | // Return a success status. 27 | static Status OK() { return Status(); } 28 | 29 | // Return error status of an appropriate type. 30 | static Status NotFound(const Slice& msg, const Slice& msg2 = Slice()) { 31 | return Status(kNotFound, msg, msg2); 32 | } 33 | static Status Corruption(const Slice& msg, const Slice& msg2 = Slice()) { 34 | return Status(kCorruption, msg, msg2); 35 | } 36 | static Status NotSupported(const Slice& msg, const Slice& msg2 = Slice()) { 37 | return Status(kNotSupported, msg, msg2); 38 | } 39 | static Status InvalidArgument(const Slice& msg, const Slice& msg2 = Slice()) { 40 | return Status(kInvalidArgument, msg, msg2); 41 | } 42 | static Status IOError(const Slice& msg, const Slice& msg2 = Slice()) { 43 | return Status(kIOError, msg, msg2); 44 | } 45 | 46 | // Returns true iff the status indicates success. 47 | bool ok() const { return (state_ == nullptr); } 48 | 49 | // Returns true iff the status indicates a NotFound error. 50 | bool IsNotFound() const { return code() == kNotFound; } 51 | 52 | // Returns true iff the status indicates a Corruption error. 53 | bool IsCorruption() const { return code() == kCorruption; } 54 | 55 | // Returns true iff the status indicates an IOError. 56 | bool IsIOError() const { return code() == kIOError; } 57 | 58 | // Returns true iff the status indicates a NotSupportedError. 59 | bool IsNotSupportedError() const { return code() == kNotSupported; } 60 | 61 | // Returns true iff the status indicates an InvalidArgument. 62 | bool IsInvalidArgument() const { return code() == kInvalidArgument; } 63 | 64 | // Return a string representation of this status suitable for printing. 65 | // Returns the string "OK" for success. 66 | std::string ToString() const; 67 | 68 | private: 69 | // OK status has a null state_. Otherwise, state_ is a new[] array 70 | // of the following form: 71 | // state_[0..3] == length of message 72 | // state_[4] == code 73 | // state_[5..] == message 74 | const char* state_; 75 | 76 | enum Code { 77 | kOk = 0, 78 | kNotFound = 1, 79 | kCorruption = 2, 80 | kNotSupported = 3, 81 | kInvalidArgument = 4, 82 | kIOError = 5 83 | }; 84 | 85 | Code code() const { 86 | return (state_ == nullptr) ? kOk : static_cast(state_[4]); 87 | } 88 | 89 | Status(Code code, const Slice& msg, const Slice& msg2); 90 | static const char* CopyState(const char* s); 91 | }; 92 | 93 | inline Status::Status(const Status& rhs) { 94 | state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); 95 | } 96 | inline Status& Status::operator=(const Status& rhs) { 97 | // The following condition catches both aliasing (when this == &rhs), 98 | // and the common case where both rhs and *this are ok. 99 | if (state_ != rhs.state_) { 100 | delete[] state_; 101 | state_ = (rhs.state_ == nullptr) ? nullptr : CopyState(rhs.state_); 102 | } 103 | return *this; 104 | } 105 | inline Status& Status::operator=(Status&& rhs) noexcept { 106 | std::swap(state_, rhs.state_); 107 | return *this; 108 | } 109 | 110 | } // namespace pidb 111 | 112 | 113 | #endif //BLOCK_STATUS_H 114 | -------------------------------------------------------------------------------- /src/libs/json11/.gitignore: -------------------------------------------------------------------------------- 1 | # generated files 2 | test 3 | libjson11.a 4 | json11.pc 5 | 6 | # Cmake 7 | CMakeCache.txt 8 | CTestTestfile.cmake 9 | CMakeFiles 10 | CMakeScripts 11 | cmake_install.cmake 12 | install_manifest.txt -------------------------------------------------------------------------------- /src/libs/json11/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | if (CMAKE_VERSION VERSION_LESS "3") 3 | project(json11 CXX) 4 | else() 5 | cmake_policy(SET CMP0048 NEW) 6 | project(json11 VERSION 1.0.0 LANGUAGES CXX) 7 | endif() 8 | 9 | enable_testing() 10 | 11 | option(JSON11_BUILD_TESTS "Build unit tests" OFF) 12 | option(JSON11_ENABLE_DR1467_CANARY "Enable canary test for DR 1467" OFF) 13 | 14 | if(CMAKE_VERSION VERSION_LESS "3") 15 | add_definitions(-std=c++11) 16 | else() 17 | set(CMAKE_CXX_STANDARD 11) 18 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 19 | endif() 20 | 21 | if(CMAKE_INSTALL_PREFIX_INITIALIZED_TO_DEFAULT) 22 | set(CMAKE_INSTALL_PREFIX /usr) 23 | endif() 24 | 25 | add_library(json11 json11.cpp) 26 | target_include_directories(json11 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}) 27 | target_compile_options(json11 28 | PRIVATE -fPIC -fno-rtti -fno-exceptions -Wall) 29 | 30 | # Set warning flags, which may vary per platform 31 | include(CheckCXXCompilerFlag) 32 | set(_possible_warnings_flags /W4 /WX -Wextra -Werror) 33 | foreach(_warning_flag ${_possible_warnings_flags}) 34 | unset(_flag_supported) 35 | CHECK_CXX_COMPILER_FLAG(${_warning_flag} _flag_supported) 36 | if(${_flag_supported}) 37 | target_compile_options(json11 PRIVATE ${_warning_flag}) 38 | endif() 39 | endforeach() 40 | 41 | configure_file("json11.pc.in" "json11.pc" @ONLY) 42 | 43 | if (JSON11_BUILD_TESTS) 44 | 45 | # enable test for DR1467, described here: https://llvm.org/bugs/show_bug.cgi?id=23812 46 | if(JSON11_ENABLE_DR1467_CANARY) 47 | add_definitions(-D JSON11_ENABLE_DR1467_CANARY=1) 48 | else() 49 | add_definitions(-D JSON11_ENABLE_DR1467_CANARY=0) 50 | endif() 51 | 52 | add_executable(json11_test test.cpp) 53 | target_link_libraries(json11_test json11) 54 | endif() 55 | 56 | install(TARGETS json11 DESTINATION lib/${CMAKE_LIBRARY_ARCHITECTURE}) 57 | install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/json11.hpp" DESTINATION include/${CMAKE_LIBRARY_ARCHITECTURE}) 58 | install(FILES "${CMAKE_CURRENT_BINARY_DIR}/json11.pc" DESTINATION lib/${CMAKE_LIBRARY_ARCHITECTURE}/pkgconfig) 59 | -------------------------------------------------------------------------------- /src/libs/json11/LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Dropbox, Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/libs/json11/Makefile: -------------------------------------------------------------------------------- 1 | # Environment variable to enable or disable code which demonstrates the behavior change 2 | # in Xcode 7 / Clang 3.7, introduced by DR1467 and described here: 3 | # https://llvm.org/bugs/show_bug.cgi?id=23812 4 | # Defaults to on in order to act as a warning to anyone who's unaware of the issue. 5 | ifneq ($(JSON11_ENABLE_DR1467_CANARY),) 6 | CANARY_ARGS = -DJSON11_ENABLE_DR1467_CANARY=$(JSON11_ENABLE_DR1467_CANARY) 7 | endif 8 | 9 | test: json11.cpp json11.hpp test.cpp 10 | $(CXX) $(CANARY_ARGS) -O -std=c++11 json11.cpp test.cpp -o test -fno-rtti -fno-exceptions 11 | 12 | clean: 13 | if [ -e test ]; then rm test; fi 14 | 15 | .PHONY: clean 16 | -------------------------------------------------------------------------------- /src/libs/json11/README.md: -------------------------------------------------------------------------------- 1 | json11 2 | ------ 3 | 4 | json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. 5 | 6 | The core object provided by the library is json11::Json. A Json object represents any JSON 7 | value: null, bool, number (int or double), string (std::string), array (std::vector), or 8 | object (std::map). 9 | 10 | Json objects act like values. They can be assigned, copied, moved, compared for equality or 11 | order, and so on. There are also helper methods Json::dump, to serialize a Json to a string, and 12 | Json::parse (static) to parse a std::string as a Json object. 13 | 14 | It's easy to make a JSON object with C++11's new initializer syntax: 15 | 16 | Json my_json = Json::object { 17 | { "key1", "value1" }, 18 | { "key2", false }, 19 | { "key3", Json::array { 1, 2, 3 } }, 20 | }; 21 | std::string json_str = my_json.dump(); 22 | 23 | There are also implicit constructors that allow standard and user-defined types to be 24 | automatically converted to JSON. For example: 25 | 26 | class Point { 27 | public: 28 | int x; 29 | int y; 30 | Point (int x, int y) : x(x), y(y) {} 31 | Json to_json() const { return Json::array { x, y }; } 32 | }; 33 | 34 | std::vector points = { { 1, 2 }, { 10, 20 }, { 100, 200 } }; 35 | std::string points_json = Json(points).dump(); 36 | 37 | JSON values can have their values queried and inspected: 38 | 39 | Json json = Json::array { Json::object { { "k", "v" } } }; 40 | std::string str = json[0]["k"].string_value(); 41 | 42 | For more documentation see json11.hpp. 43 | 44 | __Maintenance note:__ This repo is stable but no longer actively maintained. No further development 45 | is planned, and no new feature PRs will be merged. Bug fixes may be merged on a volunteer basis. 46 | If you have questions or want to talk to other users of Djinni, you can join the Slack community 47 | via the link at the end of this document. 48 | -------------------------------------------------------------------------------- /src/libs/json11/json11.hpp: -------------------------------------------------------------------------------- 1 | /* json11 2 | * 3 | * json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. 4 | * 5 | * The core object provided by the library is json11::Json. A Json object represents any JSON 6 | * value: null, bool, number (int or double), string (std::string), array (std::vector), or 7 | * object (std::map). 8 | * 9 | * Json objects act like values: they can be assigned, copied, moved, compared for equality or 10 | * order, etc. There are also helper methods Json::dump, to serialize a Json to a string, and 11 | * Json::parse (static) to parse a std::string as a Json object. 12 | * 13 | * Internally, the various types of Json object are represented by the JsonValue class 14 | * hierarchy. 15 | * 16 | * A note on numbers - JSON specifies the syntax of number formatting but not its semantics, 17 | * so some JSON implementations distinguish between integers and floating-point numbers, while 18 | * some don't. In json11, we choose the latter. Because some JSON implementations (namely 19 | * Javascript itself) treat all numbers as the same type, distinguishing the two leads 20 | * to JSON that will be *silently* changed by a round-trip through those implementations. 21 | * Dangerous! To avoid that risk, json11 stores all numbers as double internally, but also 22 | * provides integer helpers. 23 | * 24 | * Fortunately, double-precision IEEE754 ('double') can precisely store any integer in the 25 | * range +/-2^53, which includes every 'int' on most systems. (Timestamps often use int64 26 | * or long long to avoid the Y2038K problem; a double storing microseconds since some epoch 27 | * will be exact for +/- 275 years.) 28 | */ 29 | 30 | /* Copyright (c) 2013 Dropbox, Inc. 31 | * 32 | * Permission is hereby granted, free of charge, to any person obtaining a copy 33 | * of this software and associated documentation files (the "Software"), to deal 34 | * in the Software without restriction, including without limitation the rights 35 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 36 | * copies of the Software, and to permit persons to whom the Software is 37 | * furnished to do so, subject to the following conditions: 38 | * 39 | * The above copyright notice and this permission notice shall be included in 40 | * all copies or substantial portions of the Software. 41 | * 42 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 43 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 44 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 45 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 46 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 47 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 48 | * THE SOFTWARE. 49 | */ 50 | 51 | #pragma once 52 | 53 | #include 54 | #include 55 | #include 56 | #include 57 | #include 58 | 59 | #ifdef _MSC_VER 60 | #if _MSC_VER <= 1800 // VS 2013 61 | #ifndef noexcept 62 | #define noexcept throw() 63 | #endif 64 | 65 | #ifndef snprintf 66 | #define snprintf _snprintf_s 67 | #endif 68 | #endif 69 | #endif 70 | 71 | namespace json11 { 72 | 73 | enum JsonParse { 74 | STANDARD, COMMENTS 75 | }; 76 | 77 | class JsonValue; 78 | 79 | class Json final { 80 | public: 81 | // Types 82 | enum Type { 83 | NUL, NUMBER, BOOL, STRING, ARRAY, OBJECT 84 | }; 85 | 86 | // Array and object typedefs 87 | typedef std::vector array; 88 | typedef std::map object; 89 | 90 | // Constructors for the various types of JSON value. 91 | Json() noexcept; // NUL 92 | Json(std::nullptr_t) noexcept; // NUL 93 | Json(double value); // NUMBER 94 | Json(int value); // NUMBER 95 | Json(bool value); // BOOL 96 | Json(const std::string &value); // STRING 97 | Json(std::string &&value); // STRING 98 | Json(const char * value); // STRING 99 | Json(const array &values); // ARRAY 100 | Json(array &&values); // ARRAY 101 | Json(const object &values); // OBJECT 102 | Json(object &&values); // OBJECT 103 | 104 | // Implicit constructor: anything with a to_json() function. 105 | template 106 | Json(const T & t) : Json(t.to_json()) {} 107 | 108 | // Implicit constructor: map-like objects (std::map, std::unordered_map, etc) 109 | template ().begin()->first)>::value 111 | && std::is_constructible().begin()->second)>::value, 112 | int>::type = 0> 113 | Json(const M & m) : Json(object(m.begin(), m.end())) {} 114 | 115 | // Implicit constructor: vector-like objects (std::list, std::vector, std::set, etc) 116 | template ().begin())>::value, 118 | int>::type = 0> 119 | Json(const V & v) : Json(array(v.begin(), v.end())) {} 120 | 121 | // This prevents Json(some_pointer) from accidentally producing a bool. Use 122 | // Json(bool(some_pointer)) if that behavior is desired. 123 | Json(void *) = delete; 124 | 125 | // Accessors 126 | Type type() const; 127 | 128 | bool is_null() const { return type() == NUL; } 129 | bool is_number() const { return type() == NUMBER; } 130 | bool is_bool() const { return type() == BOOL; } 131 | bool is_string() const { return type() == STRING; } 132 | bool is_array() const { return type() == ARRAY; } 133 | bool is_object() const { return type() == OBJECT; } 134 | 135 | // Return the enclosed value if this is a number, 0 otherwise. Note that json11 does not 136 | // distinguish between integer and non-integer numbers - number_value() and int_value() 137 | // can both be applied to a NUMBER-typed object. 138 | double number_value() const; 139 | int int_value() const; 140 | 141 | // Return the enclosed value if this is a boolean, false otherwise. 142 | bool bool_value() const; 143 | // Return the enclosed string if this is a string, "" otherwise. 144 | const std::string &string_value() const; 145 | // Return the enclosed std::vector if this is an array, or an empty vector otherwise. 146 | const array &array_items() const; 147 | // Return the enclosed std::map if this is an object, or an empty map otherwise. 148 | const object &object_items() const; 149 | 150 | // Return a reference to arr[i] if this is an array, Json() otherwise. 151 | const Json & operator[](size_t i) const; 152 | // Return a reference to obj[key] if this is an object, Json() otherwise. 153 | const Json & operator[](const std::string &key) const; 154 | 155 | // Serialize. 156 | void dump(std::string &out) const; 157 | std::string dump() const { 158 | std::string out; 159 | dump(out); 160 | return out; 161 | } 162 | 163 | // Parse. If parse fails, return Json() and assign an error message to err. 164 | static Json parse(const std::string & in, 165 | std::string & err, 166 | JsonParse strategy = JsonParse::STANDARD); 167 | static Json parse(const char * in, 168 | std::string & err, 169 | JsonParse strategy = JsonParse::STANDARD) { 170 | if (in) { 171 | return parse(std::string(in), err, strategy); 172 | } else { 173 | err = "null input"; 174 | return nullptr; 175 | } 176 | } 177 | // Parse multiple objects, concatenated or separated by whitespace 178 | static std::vector parse_multi( 179 | const std::string & in, 180 | std::string::size_type & parser_stop_pos, 181 | std::string & err, 182 | JsonParse strategy = JsonParse::STANDARD); 183 | 184 | static inline std::vector parse_multi( 185 | const std::string & in, 186 | std::string & err, 187 | JsonParse strategy = JsonParse::STANDARD) { 188 | std::string::size_type parser_stop_pos; 189 | return parse_multi(in, parser_stop_pos, err, strategy); 190 | } 191 | 192 | bool operator== (const Json &rhs) const; 193 | bool operator< (const Json &rhs) const; 194 | bool operator!= (const Json &rhs) const { return !(*this == rhs); } 195 | bool operator<= (const Json &rhs) const { return !(rhs < *this); } 196 | bool operator> (const Json &rhs) const { return (rhs < *this); } 197 | bool operator>= (const Json &rhs) const { return !(*this < rhs); } 198 | 199 | /* has_shape(types, err) 200 | * 201 | * Return true if this is a JSON object and, for each item in types, has a field of 202 | * the given type. If not, return false and set err to a descriptive message. 203 | */ 204 | typedef std::initializer_list> shape; 205 | bool has_shape(const shape & types, std::string & err) const; 206 | 207 | private: 208 | std::shared_ptr m_ptr; 209 | }; 210 | 211 | // Internal class hierarchy - JsonValue objects are not exposed to users of this API. 212 | class JsonValue { 213 | protected: 214 | friend class Json; 215 | friend class JsonInt; 216 | friend class JsonDouble; 217 | virtual Json::Type type() const = 0; 218 | virtual bool equals(const JsonValue * other) const = 0; 219 | virtual bool less(const JsonValue * other) const = 0; 220 | virtual void dump(std::string &out) const = 0; 221 | virtual double number_value() const; 222 | virtual int int_value() const; 223 | virtual bool bool_value() const; 224 | virtual const std::string &string_value() const; 225 | virtual const Json::array &array_items() const; 226 | virtual const Json &operator[](size_t i) const; 227 | virtual const Json::object &object_items() const; 228 | virtual const Json &operator[](const std::string &key) const; 229 | virtual ~JsonValue() {} 230 | }; 231 | 232 | } // namespace json11 233 | -------------------------------------------------------------------------------- /src/libs/json11/json11.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | libdir=${prefix}/lib/@CMAKE_LIBRARY_ARCHITECTURE@ 3 | includedir=${prefix}/include/@CMAKE_LIBRARY_ARCHITECTURE@ 4 | 5 | Name: @PROJECT_NAME@ 6 | Description: json11 is a tiny JSON library for C++11, providing JSON parsing and serialization. 7 | Version: @PROJECT_VERSION@ 8 | Libs: -L${libdir} -ljson11 9 | Cflags: -I${includedir} 10 | -------------------------------------------------------------------------------- /src/master/CMakeLists.old.txt: -------------------------------------------------------------------------------- 1 | # 此文件应该放在源文件目录下 2 | cmake_minimum_required (VERSION 3.09) 3 | project (db) 4 | 5 | # include PROTO_HEADER 6 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 7 | 8 | include(FindProtobuf) 9 | # 头文件目录 10 | if (NOT PROTOBUF_PROTOC_EXECUTABLE) 11 | get_filename_component(PROTO_LIB_DIR ${PROTOBUF_LIBRARY} DIRECTORY) 12 | set (PROTOBUF_PROTOC_EXECUTABLE "${PROTO_LIB_DIR}/../bin/protoc") 13 | endif() 14 | protobuf_generate_cpp(MASTER_PROTO_SRC MASTER_PROTO_HEADER "${PROJECT_SOURCE_DIR}/master.proto") 15 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 16 | 17 | include_directories("../include") 18 | 19 | # 源文件目录放进工程 20 | file(GLOB SRCS "./head/*") 21 | 22 | # 库目录（前者本地，后者系统） 23 | link_directories("../libs") 24 | link_directories("/usr/lib") 25 | 26 | # 可执行文件 27 | add_executable(client test_client.cpp ${SRCS}) 28 | add_executable(server master_main.cpp ${SRCS}) 29 | add_executable(test_main test_main.cpp ${SRCS}) 30 | # 连接库(注意顺序) 31 | target_link_libraries(client libbraft.so libbrpc.so libgflags.so libprotobuf.so libleveldb.so libpthread.so libsnappy.so libcrypto.so) 32 | target_link_libraries(server libbraft.so libbrpc.so libgflags.so libprotobuf.so libleveldb.so libpthread.so libsnappy.so libcrypto.so) 33 | target_link_libraries(test_main libbraft.so libbrpc.so libgflags.so libprotobuf.so libleveldb.so libpthread.so libsnappy.so libcrypto.so) 34 | 35 | -------------------------------------------------------------------------------- /src/master/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.09) 2 | project(asynchronous_echo_c++ C CXX) 3 | 4 | option(EXAMPLE_LINK_SO "Whether examples are linked dynamically" ON) 5 | 6 | 7 | set(CMAKE_PREFIX_PATH ${OUTPUT_PATH}) 8 | 9 | include(FindThreads) 10 | include(FindProtobuf) 11 | protobuf_generate_cpp(PROTO_SRC PROTO_HEADER master.proto) 12 | # include PROTO_HEADER 13 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 14 | 15 | # Search for libthrift* by best effort. If it is not found and brpc is 16 | # compiled with thrift protocol enabled, a link error would be reported. 17 | find_library(THRIFT_LIB NAMES thrift) 18 | if (NOT THRIFT_LIB) 19 | set(THRIFT_LIB "") 20 | endif() 21 | 22 | find_library(THRIFTNB_LIB NAMES thriftnb) 23 | if (NOT THRIFTNB_LIB) 24 | set(THRIFTNB_LIB "") 25 | endif() 26 | 27 | find_path(BRPC_INCLUDE_PATH NAMES brpc/server.h) 28 | if(EXAMPLE_LINK_SO) 29 | find_library(BRPC_LIB NAMES brpc) 30 | find_library(BRAFT_LIB NAMES braft) 31 | else() 32 | find_library(BRPC_LIB NAMES libbrpc.a brpc) 33 | find_library(BRAFT_LIB NAMES libbraft.a braft) 34 | endif() 35 | 36 | if((NOT BRPC_INCLUDE_PATH) OR (NOT BRPC_LIB)) 37 | message(FATAL_ERROR "Fail to find brpc") 38 | endif() 39 | include_directories(${BRPC_INCLUDE_PATH}) 40 | 41 | find_path(BRAFT_INCLUDE_PATH NAMES braft/raft.h) 42 | if ((NOT BRAFT_INCLUDE_PATH) OR (NOT BRAFT_LIB)) 43 | message (FATAL_ERROR "Fail to find braft") 44 | endif() 45 | 46 | include_directories(${BRAFT_INCLUDE_PATH}) 47 | 48 | find_path(GFLAGS_INCLUDE_PATH gflags/gflags.h) 49 | find_library(GFLAGS_LIBRARY NAMES gflags libgflags) 50 | if((NOT GFLAGS_INCLUDE_PATH) OR (NOT GFLAGS_LIBRARY)) 51 | message(FATAL_ERROR "Fail to find gflags") 52 | endif() 53 | include_directories(${GFLAGS_INCLUDE_PATH}) 54 | 55 | execute_process( 56 | COMMAND bash -c "grep \"namespace [_A-Za-z0-9]\\+ {\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $2}' | tr -d '\n'" 57 | OUTPUT_VARIABLE GFLAGS_NS 58 | ) 59 | if(${GFLAGS_NS} STREQUAL "GFLAGS_NAMESPACE") 60 | execute_process( 61 | COMMAND bash -c "grep \"#define GFLAGS_NAMESPACE [_A-Za-z0-9]\\+\" ${GFLAGS_INCLUDE_PATH}/gflags/gflags_declare.h | head -1 | awk '{print $3}' | tr -d '\n'" 62 | OUTPUT_VARIABLE GFLAGS_NS 63 | ) 64 | endif() 65 | if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") 66 | include(CheckFunctionExists) 67 | CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME) 68 | if(NOT HAVE_CLOCK_GETTIME) 69 | set(DEFINE_CLOCK_GETTIME "-DNO_CLOCK_GETTIME_IN_MAC") 70 | endif() 71 | endif() 72 | 73 | set(CMAKE_CPP_FLAGS "${DEFINE_CLOCK_GETTIME} -DGFLAGS_NS=${GFLAGS_NS}") 74 | set(CMAKE_CXX_FLAGS "${CMAKE_CPP_FLAGS} -DNDEBUG -O2 -D__const__= -pipe -W -Wall -Wno-unused-parameter -fPIC -fno-omit-frame-pointer") 75 | 76 | if(CMAKE_VERSION VERSION_LESS "3.1.3") 77 | if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") 78 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 79 | endif() 80 | if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") 81 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") 82 | endif() 83 | else() 84 | set(CMAKE_CXX_STANDARD 11) 85 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 86 | endif() 87 | 88 | find_path(LEVELDB_INCLUDE_PATH NAMES leveldb/db.h) 89 | find_library(LEVELDB_LIB NAMES leveldb) 90 | if ((NOT LEVELDB_INCLUDE_PATH) OR (NOT LEVELDB_LIB)) 91 | message(FATAL_ERROR "Fail to find leveldb") 92 | endif() 93 | include_directories(${LEVELDB_INCLUDE_PATH}) 94 | 95 | find_library(SSL_LIB NAMES ssl) 96 | if (NOT SSL_LIB) 97 | message(FATAL_ERROR "Fail to find ssl") 98 | endif() 99 | 100 | find_library(CRYPTO_LIB NAMES crypto) 101 | if (NOT CRYPTO_LIB) 102 | message(FATAL_ERROR "Fail to find crypto") 103 | endif() 104 | 105 | set(DYNAMIC_LIB 106 | ${CMAKE_THREAD_LIBS_INIT} 107 | ${GFLAGS_LIBRARY} 108 | ${PROTOBUF_LIBRARIES} 109 | ${LEVELDB_LIB} 110 | ${SSL_LIB} 111 | ${BRAFT_LIB} 112 | ${BRPC_LIB} 113 | 114 | ${THRIFT_LIB} 115 | ${THRIFTNB_LIB} 116 | dl 117 | ) 118 | 119 | if(CMAKE_SYSTEM_NAME STREQUAL "Darwin") 120 | set(DYNAMIC_LIB ${DYNAMIC_LIB} 121 | pthread 122 | "-framework CoreFoundation" 123 | "-framework CoreGraphics" 124 | "-framework CoreData" 125 | "-framework CoreText" 126 | "-framework Security" 127 | "-framework Foundation" 128 | "-Wl,-U,_MallocExtension_ReleaseFreeMemory" 129 | "-Wl,-U,_ProfilerStart" 130 | "-Wl,-U,_ProfilerStop") 131 | endif() 132 | 133 | # add_executable(asynchronous_echo_client client.cpp ${PROTO_SRC}) 134 | # add_executable(asynchronous_echo_server server.cpp ${PROTO_SRC}) 135 | file(GLOB SRCS "./head/*") 136 | 137 | include_directories(${PROJECT_SOURCE_DIR}/head) 138 | 139 | add_executable(server "${PROJECT_SOURCE_DIR}/master_main.cpp") 140 | target_sources(server PUBLIC 141 | ${PROTO_SRC} 142 | ${SRCS} 143 | ) 144 | 145 | add_executable(test_main "${PROJECT_SOURCE_DIR}/test_main.cpp") 146 | target_sources(test_main PUBLIC 147 | ${PROTO_SRC} 148 | ) 149 | 150 | # target_link_libraries(asynchronous_echo_client ${BRPC_LIB} ${DYNAMIC_LIB}) 151 | # target_link_libraries(asynchronous_echo_server ${BRPC_LIB} ${DYNAMIC_LIB}) 152 | target_link_libraries(test_main 153 | "-Xlinker \"-(\"" 154 | ${DYNAMIC_LIB} 155 | "-Xlinker \"-)\"") 156 | 157 | target_link_libraries(server 158 | "-Xlinker \"-(\"" 159 | ${DYNAMIC_LIB} 160 | "-Xlinker \"-)\"") -------------------------------------------------------------------------------- /src/master/head/guard_dog.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "guard_dog.h" 4 | 5 | namespace pidb{ 6 | GuardDog::GuardDog(int cycle, int interval){ 7 | this->cycle_ = cycle; 8 | this->interval_ = interval; 9 | this->current_time_ = 0; 10 | this->things_.resize(cycle * 1000 / interval); 11 | bthread_mutex_init(&this->mutex_, NULL); 12 | } 13 | 14 | GuardDog::~GuardDog(){ 15 | this->things_.clear(); 16 | } 17 | 18 | void GuardDog::AddThing(bool(*func)(void*), void* arg){ 19 | bthread_mutex_lock(&this->mutex_); 20 | // 一个简单的插入vector操作 21 | // 这里-1的原因是留给下一周期检查，新来的不用检查 22 | int current_time = (this->current_time_+things_.size()-1)%things_.size(); 23 | assert(current_time>=0 && current_timethings_[current_time].push_back({func, arg}); 25 | bthread_mutex_unlock(&this->mutex_); 26 | } 27 | 28 | void GuardDog::HandleThings(){ 29 | bthread_t t; 30 | if (bthread_start_background(&t, NULL, 31 | GuardDog::DoHandleThings, this) != 0) { 32 | LOG(ERROR) << "create bthread guard_dog fail"; 33 | } 34 | else LOG(INFO) << "create bthread guard_dog success"; 35 | 36 | } 37 | 38 | void* GuardDog::DoHandleThings(void* guard_dog){ 39 | GuardDog* dog = (GuardDog*)guard_dog; 40 | int time = dog->cycle_ * 1000 / dog->interval_; 41 | // 一直执行，一个周期接一个周期 42 | while(true){ 43 | // 时间计数器归0（周期初始） 44 | bthread_mutex_lock(&dog->mutex_); 45 | dog->current_time_ = 0; 46 | bthread_mutex_unlock(&dog->mutex_); 47 | // 执行对应节点的事情，若执行“不成功”就删除该事情 48 | for(int i = 0; i < time; ++i){ 49 | usleep(dog->interval_ * 1000); 50 | bthread_mutex_lock(&dog->mutex_); 51 | 52 | for(auto iter = dog->things_[i].begin(); 53 | iter != dog->things_[i].end(); ) { 54 | 55 | if (!iter->first(iter->second)) 56 | iter = dog->things_[i].erase(iter); 57 | else { 58 | iter++; 59 | LOG(INFO) << "ALIVE"; 60 | } 61 | } 62 | bthread_mutex_unlock(&dog->mutex_); 63 | 64 | // 时间计数器加1 65 | bthread_mutex_lock(&dog->mutex_); 66 | dog->current_time_ += 1; 67 | bthread_mutex_unlock(&dog->mutex_); 68 | } 69 | } 70 | } 71 | } -------------------------------------------------------------------------------- /src/master/head/guard_dog.h: -------------------------------------------------------------------------------- 1 | 2 | // 此部分目的是维护一个定时器，到点了执行某件事 3 | // 具体实现是维持一个周期，在此周期范围内每隔一定的时间间隔 4 | // 执行某件事情（若有事）.例如周期3s,时间间隔10ms,那么每 5 | // 个周期就有300次去处理对应时刻可能存在的事情 6 | 7 | #ifndef PIDB_GUARD_DOG_H 8 | #define PIDB_GUARD_DOG_H 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | using std::vector; 15 | using std::pair; 16 | using std::string; 17 | 18 | namespace pidb{ 19 | class GuardDog{ 20 | private: 21 | int current_time_; // 当前时间，用计数器表示 22 | int cycle_; // 周期(s) 23 | int interval_; // 时间间隔(ms) 24 | bthread_mutex_t mutex_; // 一把互斥锁 25 | 26 | // 要定期处理的事情以及参数表 27 | // 函数固定类型bool(*)(void*) 28 | // void×是函数参数，传指针(若有多个，放在一个结构体里) 29 | vector>> things_; 30 | 31 | public: 32 | GuardDog(int cycle, int interval); 33 | ~GuardDog(); 34 | 35 | // 新增一个事情及参数(根据当前时间放入对应的时刻) 36 | void AddThing(bool(*func)(void*), void* arg); 37 | 38 | // 具体定期处理事情（后台线程，循环处理） 39 | void HandleThings(); 40 | static void* DoHandleThings(void* guard_dog); 41 | }; 42 | }// namespace pidb 43 | 44 | #endif //PIDB_GUARD_DOG_H 45 | -------------------------------------------------------------------------------- /src/master/head/master.h: -------------------------------------------------------------------------------- 1 | // Created by blue on 2019/6/3. 2 | 3 | // master主要提供三方面功能 4 | // 和client交互返回路由信息、处理来自server的心跳 5 | // 处理来自region的leader的心跳 6 | 7 | #ifndef PIDB_MASTER_H 8 | #define PIDB_MASTER_H 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include "./route_table.h" 16 | #include "./store_heartbeat.h" 17 | #include "master.pb.h" 18 | #include "./guard_dog.h" 19 | #include "./raft_manage.h" 20 | 21 | using std::string; 22 | 23 | namespace pidb{ 24 | 25 | class ClientClosure; // 跟client交互的rpc闭包 26 | class StoreClosure; // 接收store心跳的rpc闭包 27 | class RegionClosure; // 接收region心跳的rpc闭包 28 | class SplitClosure; // 处理Region分裂的rpc闭包 29 | 30 | class Master; 31 | // 用于raft调度的参数封装，因为创建线程只有一个参数 32 | struct RM{ 33 | Master* master_; 34 | string min_key_; 35 | string max_key_; 36 | bool is_split_; // 区分是初始化新增0还是分裂新增1 37 | void* arg_; // raft调度的回调函数的参数封装 38 | RM(Master* m,const string& n,const string& x,bool i,void* v): 39 | master_(m),min_key_(n),max_key_(x),is_split_(i),arg_(v){}; 40 | }; 41 | // 用于检测分裂的region是否及时选出leader的参数封装 42 | struct ISHL{ 43 | Master* master_; 44 | string old_group_; 45 | string new_group_; 46 | ISHL(Master* m, const string& o, const string& n): 47 | master_(m), old_group_(o), new_group_(n){}; 48 | }; 49 | // 用于初始化Master的参数封装（太多了） 50 | struct MasterArg{ 51 | int store_heart; // store心跳间隔（s） 52 | int check_store_interval; // 看门狗检测store的间隔（ms） 53 | int region_heart; // region心跳间隔（s） 54 | int check_region_interval; // 看门狗检测region的间隔（ms） 55 | int raft_copy_num; // 默认最大raft副本数 56 | const string& rto_path; // ok路由表的相对路径 57 | const string& rtb_path; // bad路由表的相对路径 58 | const string& st_path; // store信息表的相对路径 59 | 60 | MasterArg(int a, int b, int c, int d, int e, int f, int g, 61 | const string& h, const string& i, const string& j): 62 | store_heart(a), check_store_interval(b), region_heart(c), 63 | check_region_interval(d), raft_copy_num(g), 64 | rto_path(h), rtb_path(i), st_path(j){}; 65 | }; 66 | // 把Master实现为一个状态机 67 | class Master : public braft::StateMachine { 68 | private: 69 | // 自己的参数 70 | RouteTable* route_table_ok_; // 正常路由表 71 | RouteTable* route_table_bad_; // 非正常路由表 72 | StoreHeartbeat* store_table_; // stroe信息表 73 | GuardDog* guard_dog_store_; // 检查store存活的看门狗 74 | GuardDog* guard_dog_region_; // 检查region存活的看门狗 75 | RaftManage* rm_; // 用于raft节点调度 76 | int raft_copy_num_; // raft组默认的最大副本数 77 | // 不同的Raft操作区分标志（因为都是在同一个on_apply中执行） 78 | enum RaftOpType{ 79 | DefalutOp_, StoreHeartOp_, RegionHeartOp_, RegionSplitOp_ 80 | }; 81 | 82 | // braft的参数 83 | braft::Node* volatile node_; // raft的一个节点 84 | butil::atomic leader_term_; // 任期号 85 | 86 | public: 87 | // store心跳时间(s),检查store存活的时间间隔(ms) 88 | Master(MasterArg* master_arg); 89 | ~Master(); 90 | void testtest(){ 91 | auto rm = new pidb::RaftManage(); 92 | rm->AddNode("","","","","127.0.1.1:8200", true,nullptr, nullptr); 93 | }; 94 | // 与client的交互（即路由信息查询） 95 | void QueryRoute(const PiDBClientRequest* request, 96 | PiDBClientResponse* response); 97 | 98 | // 与Store的交互 99 | // 新增3个region 100 | void AddRegion(const string& min_key, const string& max_key, 101 | bool is_split, void* arg); 102 | static void* DoAddRegion(void* arg); 103 | // 接收心跳 104 | void HandleStore(const PiDBStoreRequest* request, 105 | PiDBStoreResponse* response, google::protobuf::Closure* done); 106 | 107 | // 与Region的交互 108 | // 处理分裂 109 | void RegionSplit(const PiDBSplitRequest* request, 110 | PiDBSplitResponse* response, google::protobuf::Closure* done); 111 | // 接收心跳并处理（普通新增或删除节点） 112 | void HandleRegion(const PiDBRegionRequest* request, 113 | PiDBRegionResponse* response, google::protobuf::Closure* done); 114 | 115 | // 分裂后的region选出leader后的回调函数 116 | static void IfSplitHasLeader(void* arg, brpc::Controller* cntl); 117 | 118 | // raft节点启动 119 | int Start(int port, string conf, string group); 120 | 121 | // 关闭此raft节点. 122 | void shutdown() {if (node_) node_->shutdown(NULL);} 123 | 124 | // 阻塞线程直到节点完成关闭 125 | void join() {if (node_) node_->join();} 126 | 127 | private: 128 | friend class ClientClosure; 129 | friend class StoreClosure; 130 | friend class RegionClosure; 131 | friend class SplitClosure; 132 | // leader的重定向 133 | void Redirect(PiDBClientResponse* response); 134 | void Redirect(PiDBStoreResponse* response); 135 | void Redirect(PiDBRegionResponse* response); 136 | void Redirect(PiDBSplitResponse* response); 137 | 138 | // 具体处理来自Store的心跳信息 139 | void DoHandleStore(braft::Iterator& iter); 140 | 141 | // 具体处理来自Store的心跳信息 142 | void DoHandleRegion(braft::Iterator& iter); 143 | 144 | // 具体处理Region的分裂 145 | void DoRegionSplit(braft::Iterator& iter); 146 | 147 | // @braft::StateMachine 148 | void on_apply(braft::Iterator& iter);; 149 | 150 | // braft的一堆东西 151 | void on_leader_start(int64_t term) override ; 152 | void on_leader_stop(const butil::Status& status) override ; 153 | void on_snapshot_save(braft::SnapshotWriter* writer,braft::Closure* done) override; 154 | int on_snapshot_load(braft::SnapshotReader* reader) override; 155 | void on_shutdown() override ; 156 | void on_error(const ::braft::Error& e) override ; 157 | void on_configuration_committed(const ::braft::Configuration& conf) override ; 158 | void on_stop_following(const ::braft::LeaderChangeContext& ctx) override ; 159 | void on_start_following(const ::braft::LeaderChangeContext& ctx) override ; 160 | }; 161 | 162 | class ClientClosure : public braft::Closure { 163 | public: 164 | ClientClosure(Master* master, const PiDBClientRequest* request, 165 | PiDBClientResponse* response, google::protobuf::Closure* done) 166 | : master_(master), _request(request) 167 | , _response(response), _done(done) {} 168 | ~ClientClosure() {} 169 | 170 | const PiDBClientRequest* request() const { return _request; } 171 | PiDBClientResponse* response() const { return _response; } 172 | void Run(); 173 | 174 | private: 175 | Master* master_; 176 | const PiDBClientRequest* _request; 177 | PiDBClientResponse* _response; 178 | google::protobuf::Closure* _done; 179 | }; 180 | 181 | class StoreClosure : public braft::Closure { 182 | public: 183 | StoreClosure(Master* master, const PiDBStoreRequest* request, 184 | PiDBStoreResponse* response, google::protobuf::Closure* done) 185 | : master_(master), request_(request) 186 | , response_(response), done_(done) {} 187 | ~StoreClosure() {} 188 | 189 | const PiDBStoreRequest* request() const { return request_; } 190 | PiDBStoreResponse* response() const { return response_; } 191 | void Run(); 192 | 193 | private: 194 | Master* master_; 195 | const PiDBStoreRequest* request_; 196 | PiDBStoreResponse* response_; 197 | google::protobuf::Closure* done_; 198 | }; 199 | 200 | class RegionClosure : public braft::Closure { 201 | public: 202 | RegionClosure(Master* master, const PiDBRegionRequest* request, 203 | PiDBRegionResponse* response, google::protobuf::Closure* done) 204 | : master_(master), request_(request) 205 | , response_(response), done_(done) {} 206 | ~RegionClosure() {} 207 | 208 | const PiDBRegionRequest* request() const { return request_; } 209 | PiDBRegionResponse* response() const { return response_; } 210 | void Run(); 211 | 212 | private: 213 | Master* master_; 214 | const PiDBRegionRequest* request_; 215 | PiDBRegionResponse* response_; 216 | google::protobuf::Closure* done_; 217 | }; 218 | 219 | class SplitClosure : public braft::Closure { 220 | public: 221 | SplitClosure(Master* master, const PiDBSplitRequest* request, 222 | PiDBSplitResponse* response, google::protobuf::Closure* done) 223 | : master_(master), request_(request) 224 | , response_(response), done_(done) {} 225 | ~SplitClosure() {} 226 | 227 | const PiDBSplitRequest* request() const { return request_; } 228 | PiDBSplitResponse* response() const { return response_; } 229 | void Run(); 230 | 231 | private: 232 | Master* master_; 233 | const PiDBSplitRequest* request_; 234 | PiDBSplitResponse* response_; 235 | google::protobuf::Closure* done_; 236 | }; 237 | 238 | // 实现MasterService(brpc) 239 | class MasterServiceImpl : public MasterService { 240 | public: 241 | explicit MasterServiceImpl(Master* master) : master_(master) {} 242 | void QueryRoute(google::protobuf::RpcController* controller, 243 | const PiDBClientRequest* request, PiDBClientResponse* response, 244 | google::protobuf::Closure* done) { 245 | brpc::ClosureGuard done_guard(done); 246 | return master_->QueryRoute(request, response); 247 | } 248 | void StoreHeartbeat(google::protobuf::RpcController* controller, 249 | const PiDBStoreRequest* request, PiDBStoreResponse* response, 250 | google::protobuf::Closure* done) { 251 | return master_->HandleStore(request, response, done); 252 | } 253 | void RegionHeartbeat(google::protobuf::RpcController* controller, 254 | const PiDBRegionRequest* request, PiDBRegionResponse* response, 255 | google::protobuf::Closure* done) { 256 | return master_->HandleRegion(request, response, done); 257 | } 258 | void RegionSplit(google::protobuf::RpcController* controller, 259 | const PiDBSplitRequest* request, PiDBSplitResponse* response, 260 | google::protobuf::Closure* done) { 261 | return master_->RegionSplit(request, response, done); 262 | } 263 | 264 | private: 265 | Master* master_; 266 | }; 267 | 268 | }// namespace pidb 269 | 270 | #endif //PIDB_MASTER_H 271 | -------------------------------------------------------------------------------- /src/master/head/raft_manage.cpp: -------------------------------------------------------------------------------- 1 | #include "raft_manage.h" 2 | 3 | namespace pidb{ 4 | RaftManage::RaftManage(){ 5 | options_.protocol = "baidu_std"; 6 | options_.connection_type = ""; 7 | options_.timeout_ms = 100; // ms; 8 | options_.max_retry = 3; 9 | } 10 | 11 | void RaftManage::InitialStub(pidb::MasterService_Stub** stub, const string& addr, 12 | brpc::Channel &channel){ 13 | if (channel.Init(addr.c_str(), "", &options_) != 0) { 14 | LOG(ERROR) << "Fail to initialize channel"; 15 | return; 16 | } 17 | *stub = new pidb::MasterService_Stub(&channel); 18 | } 19 | 20 | void RaftManage::AddNode(const string& group, const string& conf, 21 | const string& min_key, const string& max_key, 22 | const string& addr,bool is_split, 23 | void(*CallBack)(void*, brpc::Controller*), void* arg){ 24 | 25 | // pidb::MasterService_Stub* stub = NULL; 26 | // LOG(INFO)<<"Add Node"; 27 | brpc::Channel channel; 28 | // InitialStub(&stub, addr,channel); 29 | // brpc::ChannelOptions options; 30 | // if (channel.Init(addr.c_str(), "", &options_) != 0) { 31 | // LOG(ERROR) << "Fail to initialize channel"; 32 | // return; 33 | // } 34 | 35 | //Initialize the channel, NULL means using default options. 36 | brpc::ChannelOptions options; 37 | options.protocol = "baidu_std"; 38 | options.connection_type = ""; 39 | options.timeout_ms = 100/*milliseconds*/; 40 | options.max_retry = 3; 41 | if (channel.Init(addr.c_str(), "", &options) != 0) { 42 | LOG(ERROR) << "Fail to pinitialize channel"; 43 | } 44 | 45 | // Normally, you should not call a Channel directly, but instead construct 46 | // a stub Service wrapping it. stub can be shared by all threads as well. 47 | pidb::MasterService_Stub stub(&channel); 48 | 49 | brpc::Controller *cntl = new brpc::Controller(); 50 | 51 | // google::protobuf::Closure* done = brpc::NewCallback( 52 | // &HandleEchoResponse, cntl); 53 | 54 | // pidb::EchoResponse response1; 55 | 56 | // Notice that you don't have to new request, which can be modified 57 | // or destroyed just after stub.Echo is called. 58 | // pidb::EchoRequest request1; 59 | // request1.set_message("hello world"); 60 | // LOG(INFO)<<"SEND ECHO"; 61 | // stub->Echo(cntl,&request1,&response1,done); 62 | //sleep(1); 63 | 64 | // InitialStub(&stub, addr,channel); 65 | // brpc::Channel channel; 66 | // brpc::ChannelOptions options; 67 | // if (channel.Init(addr.c_str(), "", &options_) != 0) { 68 | // LOG(ERROR) << "Fail to initialize channel"; 69 | // return; 70 | // } 71 | // stub = new pidb::MasterService_Stub(&channel); 72 | 73 | 74 | LOG(INFO)<<"REMOTE RAFT MANAGE ADDR "<RaftManage(cntl, &request, response, done); 102 | } 103 | else{ 104 | LOG(INFO)<<"hehehehehehehehehheheheheheh"; 105 | // 异步调用需要一直保存下列内容直到回调函数完成 106 | pidb::PiDBRaftManageResponse response; 107 | 108 | //assert(CallBack != nullptr); 109 | 110 | pidb::PiDBRaftManageRequest request; 111 | pidb::PiDBRaftManageRequest request1; 112 | 113 | 114 | //delete &request1; 115 | request.set_raft_group(group); 116 | request.set_raft_conf(conf); 117 | request.set_min_key(min_key); 118 | request.set_max_key(max_key); 119 | request.set_is_new(true); 120 | 121 | // 发送请求 122 | google::protobuf::Closure* done = brpc::NewCallback( 123 | &HandleEchoResponse,cntl, &response); 124 | 125 | LOG(INFO)<<"RaftManage send request"; 126 | stub.RaftManage(cntl, &request, &response, done); 127 | 128 | // if(!cntl->Failed()){ 129 | // LOG(INFO)<<"RAFT MANAGER SEND SUCCESS"<ErrorText(); 132 | // } 133 | } 134 | 135 | } 136 | 137 | bool RaftManage::PullData(const string& old_addr, const string& group, 138 | const string& conf, const string& new_addr){ 139 | MasterService_Stub* stub = NULL; 140 | brpc::Channel channel; 141 | InitialStub(&stub, new_addr,channel); 142 | 143 | // 同步调用 144 | PiDBPullResponse response; 145 | pidb::PiDBPullRequest request; 146 | request.set_leader_addr(old_addr); 147 | request.set_raft_group(group); 148 | request.set_raft_conf(conf); 149 | 150 | // 发送请求 151 | brpc::Controller* cntl = new brpc::Controller(); 152 | stub->PullData(cntl, &request, &response, NULL); 153 | LOG(INFO)<<"PullData"; 154 | if(cntl->Failed()){ 155 | delete cntl; 156 | return false; 157 | } 158 | return true; 159 | } 160 | 161 | void RaftManage::RemoveNode(const string& group, 162 | const string& addr){ 163 | pidb::MasterService_Stub* stub = NULL; 164 | brpc::Channel channel; 165 | InitialStub(&stub, addr,channel); 166 | 167 | // 异步调用需要一直保存下列内容直到回调函数完成 168 | pidb::PiDBRaftManageResponse* response = new pidb::PiDBRaftManageResponse(); 169 | brpc::Controller* cntl = new brpc::Controller(); 170 | 171 | pidb::PiDBRaftManageRequest request; 172 | request.set_raft_group(group); 173 | request.set_is_new(false); 174 | 175 | // 发送请求 176 | stub->RaftManage(cntl, &request, response, NULL); 177 | } 178 | } -------------------------------------------------------------------------------- /src/master/head/raft_manage.h: -------------------------------------------------------------------------------- 1 | // 此部分主要目的是CheckAlive 2 | // 具体实现是维持一个周期，在此周期范围内 3 | // 每隔一定的时间间隔执行某件事情（若有事） 4 | // 例如周期3s,时间间隔10ms,那么每个周期就有300 5 | // 次去处理对应时刻可能存在的事情 6 | 7 | #ifndef PIDB_RAFT_MANAGE_H 8 | #define PIDB_RAFT_MANAGE_H 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "master.pb.h" 18 | 19 | using std::vector; 20 | using std::string; 21 | 22 | namespace pidb{ 23 | 24 | class RaftManage{ 25 | private: 26 | brpc::ChannelOptions options_; // brpc的一个参数 27 | 28 | public: 29 | RaftManage(); 30 | ~RaftManage(){}; 31 | 32 | // 初始化Stub配置,主要是server(store)地址不一样 33 | void InitialStub(pidb::MasterService_Stub** stub, const string& addr,brpc::Channel &channel); 34 | 35 | // 新增一个节点,addr是需要新增节点的store的地址 36 | // 若分裂会有回调函数及其参数 37 | void AddNode(const string& group, const string& conf, 38 | const string& min_key, const string& max_key, 39 | const string& addr, bool is_split, 40 | void(*CallBack)(void*, brpc::Controller*), void* arg); 41 | 42 | // 删除一个节点,即删除addr上的组名为group的region 43 | void RemoveNode(const string& group, const string& addr); 44 | 45 | // 通知某个节点开始拉数据 46 | // old_addr和new_addr分别是老region和新region的地址 47 | // 返回值代表是否成功通知到 48 | bool PullData(const string& old_addr, const string& group, 49 | const string& conf, const string& new_addr); 50 | 51 | // shanchu 52 | static void HandleEchoResponse( 53 | brpc::Controller* cntl,pidb::PiDBRaftManageResponse* response) { 54 | // std::unique_ptr makes sure cntl/response will be deleted before returning. 55 | //std::unique_ptr cntl_guard(cntl); 56 | 57 | if (cntl->Failed()) { 58 | LOG(WARNING) << "Fail to send EchoRequest, " << cntl->ErrorText(); 59 | return; 60 | } 61 | LOG(INFO) << "Received response from " << cntl->remote_side() 62 | << ": " << " (attached=" 63 | << ")" 64 | << " latency=" << cntl->latency_us() << "us"; 65 | } 66 | }; 67 | }// namespace pidb 68 | 69 | #endif //PIDB_RAFT_DISPATCH_H 70 | -------------------------------------------------------------------------------- /src/master/head/route_table.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "route_table.h" 5 | 6 | namespace pidb{ 7 | RouteTable::RouteTable() { 8 | bthread_mutex_init(&this->mutex_, NULL); 9 | } 10 | RouteTable::~RouteTable() { 11 | } 12 | 13 | int RouteTable::UpdateRouteInfo(const string& min_key, 14 | const string& max_key, const string& raft_group, 15 | const string& leader_addr, const string& conf, int state) { 16 | bthread_mutex_lock(&this->mutex_); 17 | // 找到第一个min_key_≥min_key的记录位置 18 | auto iter = std::lower_bound(route_table_.begin(), 19 | route_table_.end(), min_key, [](const RouteInfo& r, 20 | const string& key)->bool{return r.min_key_ < key;}); 21 | // 确认找到 22 | if(iter != route_table_.end() && iter->min_key_ == min_key){ 23 | iter->max_key_ = max_key; 24 | iter->raft_group_ = raft_group; 25 | iter->leader_addr_ = leader_addr; 26 | iter->conf_ = conf; 27 | iter->state_ = state; 28 | iter->is_alive = true; 29 | bthread_mutex_unlock(&this->mutex_); 30 | return 1; 31 | } 32 | // 否则新增 33 | else{ 34 | RouteInfo tmp(min_key, max_key, raft_group, 35 | leader_addr, conf, state); 36 | route_table_.insert(iter, tmp); 37 | bthread_mutex_unlock(&this->mutex_); 38 | return 0; 39 | } 40 | 41 | } 42 | 43 | void RouteTable::RemoveRouteInfo(const string& raft_group){ 44 | bthread_mutex_lock(&this->mutex_); 45 | for(auto iter = route_table_.begin(); 46 | iter != route_table_.end(); ++iter) 47 | if(iter->raft_group_ == raft_group){ 48 | route_table_.erase(iter); 49 | break; 50 | } 51 | bthread_mutex_unlock(&this->mutex_); 52 | } 53 | 54 | int RouteTable::GetState(const string& raft_group){ 55 | bthread_mutex_lock(&this->mutex_); 56 | for(auto iter = route_table_.begin(); 57 | iter != route_table_.end(); ++iter) 58 | if(iter->raft_group_ == raft_group) { 59 | bthread_mutex_unlock(&this->mutex_); 60 | return iter->state_; 61 | } 62 | bthread_mutex_unlock(&this->mutex_); 63 | return -4; //找不到就凉凉 64 | } 65 | 66 | bool RouteTable::GetAddr(const string& group, string& addr){ 67 | bthread_mutex_lock(&this->mutex_); 68 | for(auto iter = route_table_.begin(); 69 | iter != route_table_.end(); ++iter) 70 | if(iter->raft_group_ == group) { 71 | addr = iter->leader_addr_; 72 | LOG(INFO)<<"GetAddr"<mutex_); 74 | return true; 75 | } 76 | bthread_mutex_unlock(&this->mutex_); 77 | return false; 78 | } 79 | 80 | bool RouteTable::GetConf(const string& group, string& conf){ 81 | bthread_mutex_lock(&this->mutex_); 82 | for(auto iter = route_table_.begin(); 83 | iter != route_table_.end(); ++iter) 84 | if(iter->raft_group_ == group) { 85 | conf = iter->conf_; 86 | bthread_mutex_unlock(&this->mutex_); 87 | return true; 88 | } 89 | bthread_mutex_unlock(&this->mutex_); 90 | return false; 91 | } 92 | 93 | int RouteTable::GetSize() { 94 | return route_table_.size(); 95 | } 96 | 97 | void RouteTable::Clear() { 98 | this->route_table_.clear(); 99 | } 100 | 101 | bool RouteTable::GetRouteInfo(const string& key, string& raft_group, 102 | string& leader_addr, string& conf, int& state) { 103 | bthread_mutex_lock(&this->mutex_); 104 | // 排除路由表为空的情况 105 | if(route_table_.size() == 0){ 106 | bthread_mutex_unlock(&this->mutex_); 107 | return false; 108 | } 109 | // 找到第一个min_key_>key的记录位置 110 | auto iter = std::upper_bound(route_table_.begin(), 111 | route_table_.end(), key, [](const string& key, 112 | const RouteInfo& r)->bool{return key < r.min_key_;}); 113 | // 没找到但存在（所有min_key都≤key但最后一条max_key>key） 114 | if(iter == route_table_.end()){ 115 | if((iter - 1)->max_key_ > key || (iter - 1)->max_key_ == ""){ 116 | raft_group = iter->raft_group_; 117 | leader_addr = iter->leader_addr_; 118 | conf = iter->conf_; 119 | state = iter->state_; 120 | bthread_mutex_unlock(&this->mutex_); 121 | return true; 122 | } 123 | } 124 | // 找到了且存在（排除路由表断层的情况） 125 | else if(iter != route_table_.begin() && (iter - 1)->max_key_ > key){ 126 | raft_group = iter->raft_group_; 127 | leader_addr = iter->leader_addr_; 128 | conf = iter->conf_; 129 | state = iter->state_; 130 | bthread_mutex_unlock(&this->mutex_); 131 | return true; 132 | } 133 | // 到这的都是不存在的情况 134 | bthread_mutex_unlock(&this->mutex_); 135 | return false; 136 | } 137 | 138 | bool RouteTable::GetRange(const string& group,string& min_key, 139 | string& max_key){ 140 | // 遍历寻找 141 | for(auto& r : route_table_) 142 | if(r.raft_group_ == group){ 143 | min_key = r.min_key_; 144 | max_key = r.max_key_; 145 | return true; 146 | } 147 | return false; 148 | } 149 | 150 | bool RouteTable::CheckIsAlive(void* check_alive){ 151 | RouteTable* r = ((CheckRAlive*)check_alive)->r_; 152 | string group = ((CheckRAlive*)check_alive)->group_; 153 | bthread_mutex_lock(&r->mutex_); 154 | for(auto iter = r->route_table_.begin(); 155 | iter != r->route_table_.end(); ++iter){ 156 | if(iter->raft_group_ == group){ 157 | if(iter->is_alive == true){ 158 | iter->is_alive = false; 159 | bthread_mutex_unlock(&r->mutex_); 160 | return true; 161 | } 162 | else { 163 | r->route_table_.erase(iter); 164 | bthread_mutex_unlock(&r->mutex_); 165 | return false; 166 | } 167 | } 168 | } 169 | } 170 | 171 | bool RouteTable::WriteToFile(const string& file_name) { 172 | std::ofstream out(file_name, std::fstream::out | std::fstream::binary); 173 | if (!out) return false; 174 | 175 | string json_str = json11::Json(this->route_table_).dump(); 176 | out.write(json_str.c_str(), json_str.size()); 177 | return true; 178 | } 179 | 180 | int RouteTable::ReadFromFile(const string& file_name) { 181 | std::ifstream in(file_name, std::fstream::in | std::fstream::binary); 182 | int times = 3; // 尝试3次打开文件（若失败） 183 | while(!in && times != 0){ 184 | sleep(1); 185 | times-- ; 186 | in.clear(); 187 | in.open(file_name, std::fstream::in | std::fstream::binary); 188 | } 189 | if (!in) return -1; 190 | 191 | // 从文件中读取数据并解析成json格式 192 | std::istreambuf_iterator start(in), end; 193 | string data(start, end); 194 | in.close(); 195 | string error; 196 | auto json = json11::Json::parse(data, error); 197 | if (!error.empty()) return -2; 198 | 199 | // 应该是数组（vector）格式 200 | auto array = json.array_items(); 201 | for (auto& object : array) { 202 | RouteInfo route_info(object["min_key"].string_value(), 203 | object["max_key"].string_value(), object["raft_group"].string_value(), 204 | object["leader_addr"].string_value(),object["conf"].string_value(), 205 | object["state"].int_value()); 206 | this->route_table_.push_back(route_info); 207 | } 208 | return 0; 209 | } 210 | }//namespace pidb -------------------------------------------------------------------------------- /src/master/head/route_table.h: -------------------------------------------------------------------------------- 1 | // 路由表即存放region的路由信息，有两种（分成两张表） 2 | // 一种正常的，既含leader又有完整数据 3 | // 一种不正常的，可能没有leader，或者有leader但无完整数据 4 | 5 | #ifndef PIDB_ROUTE_TABLE_H 6 | #define PIDB_ROUTE_TABLE_H 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "./json11.hpp" 13 | 14 | using std::string; 15 | 16 | namespace pidb{ 17 | class RouteTable; 18 | 19 | // 用于测活的参数封装，因为看门狗那里只有一个参数 20 | struct CheckRAlive{ 21 | RouteTable* r_; 22 | string group_; 23 | CheckRAlive(RouteTable* r, const string& group): 24 | r_(r),group_(group){}; 25 | }; 26 | 27 | class RouteTable { 28 | private: 29 | // 路由表存放的内容结构体 30 | struct RouteInfo { 31 | string min_key_; // 最小key值 32 | string max_key_; // 最大key值 33 | string raft_group_; // 所属raft组名 34 | string leader_addr_; // 所属raft的leader地址 35 | string conf_; // 所属raft组配置 36 | // 路由记录状态 37 | // 0正常，-4为彻底凉凉 38 | // -1为无leader(单纯新增region) 39 | // -2为无leader（分裂新增的region） 40 | // -3为有leader但数据不完整（分类新增的leader） 41 | int state_; 42 | bool is_alive; // 某个region是否活着 43 | 44 | RouteInfo(const string& min, const string& max, 45 | const string& r, const string& l, const string& c, int s): 46 | min_key_(min), max_key_(max), raft_group_(r), 47 | leader_addr_(l), conf_(c), state_(s), is_alive(false) {} 48 | ~RouteInfo() {} 49 | 50 | // 自动把此结构体转成json格式（必须以to_json命名） 51 | json11::Json to_json() const { 52 | return json11::Json::object{{"min_key",min_key_}, 53 | {"max_key",max_key_}, 54 | {"raft_group",raft_group_}, 55 | {"leader_addr",leader_addr_}, 56 | {"conf", conf_}, 57 | {"state", state_}}; 58 | } 59 | }; 60 | std::vector route_table_; // 存放整个路由表 61 | bthread_mutex_t mutex_; // 一把互斥锁 62 | 63 | public: 64 | RouteTable(); 65 | ~RouteTable(); 66 | 67 | // 更新路由信息，state意义详见类成员定义 68 | // 若已存在就修改并返回1，若新增返回0 69 | int UpdateRouteInfo(const string& min_key, const string& max_key, 70 | const string& raft_group, const string& leader_addr, 71 | const string& conf, int state); 72 | 73 | // 根据group删除某条路由记录 74 | void RemoveRouteInfo(const string& raft_group); 75 | 76 | // 根据group获取state情况 77 | int GetState(const string& raft_group); 78 | // 根据group获取leader_addr 79 | bool GetAddr(const string& raft_group, string& addr); 80 | // 根据group获取conf 81 | bool GetConf(const string& raft_group, string& conf); 82 | // 返回路由表条目数 83 | int GetSize(); 84 | // 根据group名获取对应region的范围 85 | bool GetRange(const string& raft_group, string& min_key, 86 | string& max_key); 87 | 88 | // 从路由表查询记录，state意义详见类成员定义 89 | bool GetRouteInfo(const string& key, string& raft_group, 90 | string& leader_addr, string& conf, int& state); 91 | 92 | // 根据group检查store是否活着（根据is_alive_标志） 93 | // 若store还活着则设置标志为假（每个心跳会置真）,并返回true 94 | // 若store已死，清理它的信息，并返回false 95 | static bool CheckIsAlive(void* check_alive); 96 | 97 | // 整个路由表写入磁盘持久化 98 | // 传入参数 file_name 写入文件的路径+文件名 99 | bool WriteToFile(const string& file_name); 100 | 101 | // 尝试从磁盘文件读入路由表（一般用于启动时） 102 | // 传入参数 file_name 读取文件的路径+文件名 103 | // 返回值 -1表示打开文件失败，-2表示解析文件失败，0表示成功 104 | int ReadFromFile(const string& file_name); 105 | 106 | // 清空路由 107 | void Clear(); 108 | 109 | // 测试用 110 | void PrintInfo(){ 111 | for(const auto &v:route_table_){ 112 | LOG(INFO)< 2 | #include 3 | #include 4 | #include // sleep.h 5 | #include "store_heartbeat.h" 6 | 7 | namespace pidb{ 8 | 9 | StoreHeartbeat* StoreHeartbeat::s = new StoreHeartbeat; 10 | 11 | StoreHeartbeat::StoreHeartbeat() { 12 | bthread_mutex_init(&this->mutex_, NULL); 13 | bthread_mutex_init(&this->mutex1_, NULL); 14 | } 15 | 16 | void StoreHeartbeat::AddLeast(const string& addr, int region_num){ 17 | bthread_mutex_lock(&this->mutex1_); 18 | for(auto iter = least_region_.begin(); 19 | iter != least_region_.end(); ++iter){ 20 | int tmp = store_table_[*iter].region_num_; 21 | if(region_num < tmp){ 22 | least_region_.insert(iter, addr); 23 | bthread_mutex_lock(&this->mutex1_); 24 | return; 25 | } 26 | } 27 | // 没找到合适的位置 28 | least_region_.insert(least_region_.end(), addr); 29 | bthread_mutex_unlock(&this->mutex1_); 30 | } 31 | void StoreHeartbeat::DeleteLeast(const string& addr){ 32 | bthread_mutex_lock(&this->mutex1_); 33 | auto iter = std::find(least_region_.begin(), 34 | least_region_.end(), addr); 35 | if(iter != least_region_.end()) least_region_.erase(iter); 36 | bthread_mutex_unlock(&this->mutex1_); 37 | } 38 | 39 | int StoreHeartbeat::UpdateStoreInfo(const string& addr, 40 | int region_num, int leader_num) { 41 | int return_value = 1; // 返回值 42 | bthread_mutex_lock(&this->mutex_); 43 | // 新store 44 | if(store_table_.find(addr) == store_table_.end()){ 45 | StoreInfo s(region_num, leader_num); 46 | this->store_table_.insert({addr, s}); 47 | return_value = 0; 48 | // 然后更新least表 49 | AddLeast(addr, region_num); 50 | } 51 | // 旧store 52 | else { 53 | if(region_num != -1) store_table_[addr].region_num_ = region_num; 54 | if(leader_num != -1) store_table_[addr].leader_num_ = leader_num; 55 | store_table_[addr].is_alive_ = true; 56 | // 然后更新least表 57 | if(region_num != -1){ 58 | DeleteLeast(addr); 59 | AddLeast(addr, region_num); 60 | } 61 | } 62 | bthread_mutex_unlock(&this->mutex_); 63 | return return_value; 64 | } 65 | 66 | void StoreHeartbeat::Clear() { 67 | this->store_table_.clear(); 68 | } 69 | 70 | int StoreHeartbeat::GetSize() { 71 | return this->store_table_.size(); 72 | } 73 | 74 | int StoreHeartbeat::GetStoreInfo(int size, const std::vector* vin, 75 | std::vector *vout, bool is_new){ 76 | // 如果新增，要找不在vin里面的store(region最少的) 77 | bthread_mutex_lock(&this->mutex_); 78 | if(is_new){ 79 | if(least_region_.size() < size) return -1; 80 | auto iter = least_region_.begin(); 81 | if(vin == nullptr){ 82 | for(int i = 0; ipush_back(*iter); 85 | } 86 | else{ 87 | for(int i = 0; ibegin(), vin->end(), *iter); 89 | if(iter1 == vin->end()){ 90 | vout->push_back(*iter); 91 | ++i; 92 | } 93 | } 94 | } 95 | bthread_mutex_unlock(&this->mutex_); 96 | if(vout->size() == size) return 0; 97 | else return -1; 98 | } 99 | // 如果删除,要找在vin里面的store(region最多的) 100 | else{ 101 | // 从后往前找，找的就是region最多的 102 | auto iter = least_region_.end()-1; 103 | for(int i = 0; ibegin(), vin->end(), *iter); 105 | if(iter1 != vin->end()){ 106 | vout->push_back(*iter); 107 | ++i; 108 | } 109 | } 110 | bthread_mutex_unlock(&this->mutex_); 111 | if(vout->size() == size) return 0; 112 | else return -1; 113 | } 114 | } 115 | 116 | bool StoreHeartbeat::CheckIsAlive(void* addr){ 117 | StoreHeartbeat* s = StoreHeartbeat::Initial(); 118 | string* addr_ = (string*)addr; 119 | bthread_mutex_lock(&s->mutex_); 120 | if(s->store_table_.find(*addr_) != s->store_table_.end()){ 121 | if(s->store_table_[*addr_].is_alive_ == true){ 122 | s->store_table_[*addr_].is_alive_ = false; 123 | bthread_mutex_unlock(&s->mutex_); 124 | return true; 125 | } 126 | else { 127 | s->store_table_.erase(*addr_); 128 | delete addr_; // 释放内存 129 | bthread_mutex_unlock(&s->mutex_); 130 | return false; 131 | } 132 | } 133 | } 134 | 135 | bool StoreHeartbeat::WriteToFile(const string& file_name) { 136 | std::ofstream out(file_name, std::fstream::out | std::fstream::binary); 137 | if (!out) return false; 138 | 139 | string json_str = json11::Json(this->store_table_).dump(); 140 | out.write(json_str.c_str(), json_str.size()); 141 | return true; 142 | } 143 | 144 | int StoreHeartbeat::ReadFromFile(const string& file_name) { 145 | std::ifstream in(file_name, std::fstream::in | std::fstream::binary); 146 | int times = 3; // 尝试3次打开文件（若失败） 147 | while(!in && times != 0){ 148 | sleep(1); 149 | times-- ; 150 | in.clear(); 151 | in.open(file_name, std::fstream::in | std::fstream::binary); 152 | } 153 | if (!in) return -1; 154 | 155 | // 从文件中读取数据并解析成json格式 156 | std::istreambuf_iterator start(in), end; 157 | string data(start, end); 158 | in.close(); 159 | string error; 160 | auto json = json11::Json::parse(data, error); 161 | if (!error.empty()) return -2; 162 | 163 | // 应该是map格式 164 | auto map_object = json.object_items(); 165 | for (auto& object : map_object) { 166 | StoreInfo store_info( object.second["region_num"].int_value(), 167 | object.second["leader_num"].int_value()); 168 | this->store_table_.insert({object.first, store_info}); 169 | } 170 | return 0; 171 | } 172 | }//namespace pidb -------------------------------------------------------------------------------- /src/master/head/store_heartbeat.h: -------------------------------------------------------------------------------- 1 | #ifndef PIDB_STORE_HEARTBEAT_H 2 | #define PIDB_STORE_HEARTBEAT_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "./json11.hpp" 10 | 11 | using std::string; 12 | 13 | namespace pidb{ 14 | class StoreHeartbeat { 15 | private: 16 | // 一个store心跳的内容结构体 17 | struct StoreInfo{ 18 | int region_num_; // store上的region数量 19 | int leader_num_; // store上的leader数量 20 | bool is_alive_; // 用于master确定store是否活着 21 | StoreInfo(){}; 22 | StoreInfo(int r, int l):region_num_(r), leader_num_(l), 23 | is_alive_(false) {} 24 | ~StoreInfo() {} 25 | 26 | // 自动把此结构体转成json格式（必须以to_json命名） 27 | json11::Json to_json() const { 28 | return json11::Json::object{{"region_num",region_num_}, 29 | {"leader_num",leader_num_}}; 30 | } 31 | }; 32 | 33 | std::map store_table_; // 存放所有store信息，string是store的addr 34 | bthread_mutex_t mutex_; // 一把互斥锁 35 | bthread_mutex_t mutex1_; // 一把互斥锁 36 | 37 | std::vector least_region_; // 维护一个region数目递增的store表 38 | 39 | StoreHeartbeat(); 40 | static StoreHeartbeat* s; 41 | 42 | // 往least_region_里增加、删除一条记录 43 | void AddLeast(const string& addr, int region_num); 44 | void DeleteLeast(const string& addr); 45 | 46 | public: 47 | static StoreHeartbeat* Initial() { 48 | return s; 49 | } 50 | 51 | // 根据addr更新store的相关信息以及least表，(addr存在就修改，否则新增) 52 | // 另外若region_num 或者 leader_num为-1则代表不需要修改。 53 | // 返回值0表示新增，1表示修改 54 | int UpdateStoreInfo(const string& addr, int region_num, 55 | int leader_num); 56 | 57 | // 获取store表条目数 58 | int GetSize(); 59 | 60 | // 获取store信息用于调度(主要是store的addr) 61 | // size是需要的store的数量，is_new区分是新增还是删除 62 | // vin是传入的store地址，vout是传出的store地址 63 | // 返回值表示是否成功获取了完整信息(0成功) 64 | int GetStoreInfo(int size, const std::vector* vin, 65 | std::vector *vout, bool is_new); 66 | 67 | // 根据addr检查store是否活着（根据is_alive_标志） 68 | // 若store还活着则设置标志为假（每个心跳会设置为真） 69 | // 若store已死，清理它的信息 70 | static bool CheckIsAlive(void* addr); 71 | 72 | // Store信息写入磁盘持久化 73 | bool WriteToFile(const string& file_name); 74 | 75 | // 尝试从磁盘文件读取Store信息（一般用于启动时） 76 | int ReadFromFile(const string& file_name); 77 | 78 | // 清空store信息 79 | void Clear(); 80 | }; 81 | }// namespace pidb 82 | 83 | #endif //PIDB_STORE_HEARTBEAT_H 84 | -------------------------------------------------------------------------------- /src/master/head/store_table.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CDDSCLab/PiDB/7cc22fccdf578c9b4dcb8f8bfb76db157358f544/src/master/head/store_table.json -------------------------------------------------------------------------------- /src/master/master.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pidb; 3 | option cc_generic_services = true; 4 | // 注：master默认是一个3副本的raft组，也有leader 5 | // 暂未考虑master的宕机情况 6 | 7 | // client根据key向master请求路由信息 8 | message PiDBClientRequest{ 9 | required string key = 1; // client根据此key查询路由信息 10 | enum Action{ 11 | GET = 1; 12 | PUT = 2; 13 | DELETE = 3; 14 | } 15 | required Action action = 2; // 操作类型（主要区分读写） 16 | } 17 | message PiDBClientResponse{ 18 | required bool success = 1; // client查询路由信息是否成功 19 | optional string redirect = 2; // master的leader更替 20 | optional string leader_addr = 3; // 返回对应region的leader地址 21 | optional string raft_group = 4; // 返回对应region的raft组名 22 | optional string raft_conf =5; // 返回对应region的raft配置 23 | } 24 | 25 | // server端（Store）的心跳 26 | message PiDBStoreRequest{ 27 | required string store_addr = 1; // store的地址 28 | optional int32 region_num = 2; // store上的region数量 29 | optional int32 leader_num = 3; // store上的leader数量 30 | } 31 | message PiDBStoreResponse{ 32 | required bool success = 1; // 一个成功通信的标记 33 | optional string redirect = 2; // master的leader更替 34 | } 35 | 36 | // 主动发rpc给某些机器通知新增raft节点(关于一个region) 37 | message PiDBRaftManageRequest{ 38 | required bool is_new = 1; // 是新增还是删除 39 | required string raft_group = 2; // 组名 40 | optional string raft_conf = 3; // raft组配置信息 41 | optional string min_key = 4; // 组相关region的范围 42 | optional string max_key = 5; // 同上 43 | } 44 | // 空包 45 | message PiDBRaftManageResponse{ 46 | required bool is_leader = 1; // 是否是leader 47 | } 48 | 49 | // 具体region的心跳（只有leader会上报新跳） 50 | message PiDBRegionRequest{ 51 | required string leader_addr = 1; // 发送心跳的leader的地址 52 | required string raft_group = 2; // region所在raft组名 53 | repeated string peer_addr = 3; // regiond的副本地址(不含leader) 54 | } 55 | message PiDBRegionResponse{ 56 | required bool success = 1; // 一个成功通信的标记 57 | optional string redirect = 2; // master的leader更替 58 | optional string conf = 3; // 新raft组配置信息（若有节点增减） 59 | } 60 | 61 | // region的分裂 62 | message PiDBSplitRequest{ 63 | required string leader_addr = 1; // 分裂的leader的地址 64 | required string raft_group = 2; // region所在raft组名 65 | required string split_key = 3; // region根据此key分裂 66 | } 67 | message PiDBSplitResponse{ 68 | required bool success = 1; // 一个成功通信的标记 69 | optional string redirect = 3; // master的leader更替 70 | } 71 | 72 | // 通知新分裂的region的leader开始拉分裂数据 73 | message PiDBPullRequest{ 74 | required string leader_addr = 1; // 老region的leader的地址 75 | required string raft_group = 2; // 老region所在raft组名 76 | required string raft_conf = 3; // 老region的让raft组配置 77 | } 78 | message PiDBPullResponse{ 79 | required bool success = 1; // 一个成功通信的标记 80 | optional string redirect = 2; // master的leader更替 81 | } 82 | 83 | message EchoRequest { 84 | required string message = 1; 85 | }; 86 | 87 | message EchoResponse { 88 | required string message = 1; 89 | }; 90 | 91 | service MasterService { 92 | rpc QueryRoute (PiDBClientRequest) returns (PiDBClientResponse); 93 | rpc StoreHeartbeat (PiDBStoreRequest) returns (PiDBStoreResponse); 94 | rpc RegionHeartbeat (PiDBRegionRequest) returns (PiDBRegionResponse); 95 | rpc RaftManage (PiDBRaftManageRequest) returns (PiDBRaftManageResponse); 96 | rpc RegionSplit (PiDBSplitRequest) returns (PiDBSplitResponse); 97 | rpc PullData (PiDBPullRequest) returns (PiDBPullResponse); 98 | rpc Echo(EchoRequest) returns (EchoResponse); 99 | } 100 | -------------------------------------------------------------------------------- /src/master/master_main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "master.h" 7 | 8 | using std::cout; 9 | using std::endl; 10 | using std::string; 11 | using namespace pidb; 12 | 13 | DEFINE_string(conf, "127.0.1.1:8300:0", "master的raft配置信息（地址）"); 14 | DEFINE_string(group, "master", "master的raft组名"); 15 | DEFINE_int32(port, 8300, "port of server to listen on"); 16 | DEFINE_int32(store_heart, 3, "store心跳时间 s"); 17 | DEFINE_int32(check_store_interval, 100, "检查store存活情况的时间间隔 ms"); 18 | DEFINE_int32(region_heart, 3, "region心跳时间 s"); 19 | DEFINE_int32(check_region_interval, 10, "检查region存活情况的时间间隔 ms"); 20 | DEFINE_int32(split_cycle, 10, "分裂的新region选出leader时间 s"); 21 | DEFINE_int32(check_split_interval, 1000, "检查分裂的新region是否及时选出leader ms"); 22 | DEFINE_int32(raft_copy_num, 1, "默认的最大raft副本数量"); 23 | DEFINE_string(rto_path, "./route_table_ok.json", "正常路由表持久化地址"); 24 | DEFINE_string(rtb_path, "./route_table_bad.json", "不正常路由表持久化地址"); 25 | DEFINE_string(st_path, "./store_table.json", "store信息表持久化地址"); 26 | 27 | 28 | 29 | int main(int argc,char *argv[]){ 30 | google::ParseCommandLineFlags(&argc,&argv,true); 31 | 32 | 33 | butil::AtExitManager exit_manager; 34 | // 创建文件夹 35 | if(!butil::DirectoryExists(butil::FilePath("./master"))){ 36 | butil::CreateDirectory(butil::FilePath("./master")); 37 | } 38 | // 初始化master的rpc服务器 39 | brpc::Server server; 40 | pidb::MasterArg arg(FLAGS_store_heart,FLAGS_check_store_interval, 41 | FLAGS_region_heart, FLAGS_check_region_interval, 42 | FLAGS_split_cycle, FLAGS_check_split_interval, 43 | FLAGS_raft_copy_num, FLAGS_rto_path, 44 | FLAGS_rtb_path, FLAGS_st_path); 45 | auto master = new Master(&arg); 46 | MasterServiceImpl service(master); 47 | // 增加master的rpc服务 48 | if (server.AddService(&service, 49 | brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { 50 | LOG(ERROR) << "Fail to add rpc service"; 51 | return -1; 52 | } 53 | // 增加raft的服务，与master的共享 54 | if (braft::add_service(&server, FLAGS_port) != 0) { 55 | LOG(ERROR) << "Fail to add raft service"; 56 | return -1; 57 | } 58 | // 启动brpc server 59 | if (server.Start(FLAGS_port, NULL) != 0) { 60 | LOG(ERROR) << "Fail to start rpc"; 61 | return -1; 62 | } 63 | 64 | if (master->Start(FLAGS_port, FLAGS_conf, FLAGS_group) != 0) { 65 | LOG(ERROR) << "Fail to start raft"; 66 | return -1; 67 | } 68 | 69 | 70 | while (!brpc::IsAskedToQuit()) { 71 | //LOG(INFO) << "正常工作中"; 72 | sleep(1); 73 | } 74 | 75 | // 先停raft再停rpc 76 | master->shutdown(); 77 | server.Stop(0); 78 | master->join(); 79 | server.Join(); 80 | 81 | return 0; 82 | 83 | } -------------------------------------------------------------------------------- /src/master/run_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | mydir="${BASH_SOURCE%/*}" 4 | if [[ ! -d "$mydir" ]]; then mydir="$PWD"; fi 5 | . $mydir/../shflags 6 | 7 | 8 | # define command-line flags 9 | DEFINE_integer server_port 8300 "Port of the first server" 10 | DEFINE_integer server_num '3' 'Number of servers' 11 | DEFINE_string valgrind 'false' 'Run in valgrind' 12 | 13 | FLAGS "$@" || exit 1 14 | 15 | # hostname prefers ipv6 16 | IP=`hostname -i | awk '{print $NF}'` 17 | 18 | if [ "$FLAGS_valgrind" == "true" ] && [ $(which valgrind) ] ; then 19 | VALGRIND="valgrind --tool=memcheck --leak-check=full" 20 | fi 21 | 22 | raft_peers="" 23 | for ((i=0; i<$FLAGS_server_num; ++i)); do 24 | raft_peers="${raft_peers}${IP}:$((${FLAGS_server_port}+i)):0," 25 | done 26 | 27 | export TCMALLOC_SAMPLE_PARAMETER=524288 28 | 29 | ${VALGRIND} ./client \ 30 | -conf="${raft_peers}" \ 31 | -------------------------------------------------------------------------------- /src/master/run_server.sh: -------------------------------------------------------------------------------- 1 | # created by blue 2019/6/2 2 | 3 | mydir="${BASH_SOURCE%/*}" 4 | if [[ ! -d "$mydir" ]]; then mydir="$PWD"; fi 5 | . $mydir/../shflags 6 | 7 | # define command-line flags 8 | DEFINE_string valgrind 'false' 'Run in valgrind' 9 | DEFINE_integer server_num '1' 'Number of servers' 10 | DEFINE_boolean clean 1 'Remove old "runtime" dir before running' 11 | DEFINE_integer port 8300 "Port of the first server" 12 | 13 | # parse the command-line 14 | FLAGS "$@" || exit 1 15 | eval set -- "${FLAGS_ARGV}" 16 | 17 | # The alias for printing to stderr 18 | alias error=">&2 echo hehe: " 19 | 20 | # hostname prefers ipv6 21 | IP=`hostname -i | awk '{print $NF}'` 22 | 23 | if [ "$FLAGS_valgrind" == "true" ] && [ $(which valgrind) ] ; then 24 | VALGRIND="valgrind --tool=memcheck --leak-check=full" 25 | fi 26 | 27 | raft_peers="" 28 | for ((i=0; i<$FLAGS_server_num; ++i)); do 29 | raft_peers="${raft_peers}${IP}:$((${FLAGS_port}+i)):0," 30 | done 31 | 32 | if [ "$FLAGS_clean" == "1" ]; then 33 | rm -rf runtime 34 | fi 35 | 36 | export TCMALLOC_SAMPLE_PARAMETER=524288 37 | 38 | for ((i=0; i<$FLAGS_server_num; ++i)); do 39 | mkdir -p runtime/$i 40 | cp ./server runtime/$i 41 | cp ../head/route_table.json runtime/$i 42 | cp ../head/store_table.json runtime/$i 43 | cd runtime/$i 44 | ${VALGRIND} ./server \ 45 | -port=$((${FLAGS_port}+i)) -conf="${raft_peers}" > std.log 2>&1 & 46 | 47 | cd ../.. 48 | done 49 | -------------------------------------------------------------------------------- /src/master/test_client.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "master.pb.h" 10 | 11 | DEFINE_int32(timeout_ms, 1000, "请求超时时间"); 12 | DEFINE_string(conf, "", "raft组的配置信息（地址）"); 13 | DEFINE_string(group, "master", "raft组的ID"); 14 | 15 | using namespace pidb; 16 | 17 | bvar::LatencyRecorder g_latency_recorder("block_client"); 18 | 19 | static void* sender(void* arg) { 20 | braft::PeerId leader; 21 | while (!brpc::IsAskedToQuit()) { 22 | // 从路由表选择leader 23 | if (braft::rtb::select_leader(FLAGS_group, &leader) != 0) { 24 | // 如果没有通过发rpc选举leader 25 | butil::Status st = braft::rtb::refresh_leader( 26 | FLAGS_group, FLAGS_timeout_ms); 27 | if (!st.ok()) { 28 | // 若没选出来，等会儿再选 29 | LOG(WARNING) << "选举失败：" << st; 30 | bthread_usleep(FLAGS_timeout_ms * 1000L); 31 | } 32 | continue; 33 | } 34 | } 35 | // 现在有leader了,构建stub然后发rpc 36 | brpc::Channel channel; 37 | if (channel.Init(leader.addr, NULL) != 0) { 38 | LOG(ERROR) << "初始化通道失败：" << leader; 39 | bthread_usleep(FLAGS_timeout_ms * 1000L); 40 | return NULL; 41 | } 42 | MasterService_Stub stub(&channel); 43 | brpc::Controller cntl; 44 | cntl.set_timeout_ms(FLAGS_timeout_ms); 45 | 46 | // 发送请求 47 | PiDBClientResponse response; 48 | PiDBClientRequest request; 49 | request.set_key("bb"); 50 | stub.QueryRoute(&cntl, &request, &response, NULL); 51 | 52 | // 出错处理 53 | if (cntl.Failed()) { 54 | LOG(WARNING) << "Fail to send request to " 55 | << leader << " : " << cntl.ErrorText(); 56 | // 更新leader 57 | // braft::rtb::update_leader(FLAGS_group, braft::PeerId()); 58 | return NULL; 59 | } 60 | if (!response.success()) { 61 | if(response.has_redirect()){ 62 | LOG(WARNING) << "Fail to send request to " 63 | << leader << ", redirecting to " << response.redirect(); 64 | // 更新leader 65 | // braft::rtb::update_leader(FLAGS_group, response.redirect()); 66 | } 67 | else{ 68 | LOG(WARNING) << "send success but hehehehehe"; 69 | } 70 | 71 | return NULL; 72 | } 73 | LOG(INFO) << response.leader_addr(); 74 | 75 | return NULL; 76 | } 77 | 78 | int main(int argc, char* argv[]) { 79 | google::ParseCommandLineFlags(&argc, &argv, true); 80 | butil::AtExitManager exit_manager; 81 | 82 | // Register configuration of target group to RouteTable 83 | if (braft::rtb::update_configuration(FLAGS_group, FLAGS_conf) != 0) { 84 | LOG(ERROR) << "Fail to register configuration " << FLAGS_conf 85 | << " of group " << FLAGS_group; 86 | return -1; 87 | } 88 | 89 | bthread_t t; 90 | if (bthread_start_background(&t, NULL, sender, NULL) != 0) { 91 | LOG(ERROR) << "Fail to create bthread"; 92 | return -1; 93 | } 94 | 95 | while (!brpc::IsAskedToQuit()) { 96 | sleep(3); 97 | LOG(INFO) << "正常工作中……"; 98 | } 99 | 100 | LOG(INFO) << "客户端已退出"; 101 | bthread_join(t, NULL); 102 | 103 | return 0; 104 | } -------------------------------------------------------------------------------- /src/master/test_main.cpp: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2014 Baidu, Inc. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | // A client sending requests to server asynchronously every 1 second. 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "master.pb.h" 22 | 23 | DEFINE_bool(send_attachment, true, "Carry attachment along with requests"); 24 | DEFINE_string(protocol, "baidu_std", "Protocol type. Defined in src/brpc/options.proto"); 25 | DEFINE_string(connection_type, "", "Connection type. Available values: single, pooled, short"); 26 | DEFINE_string(server, "0.0.0.0:8003", "IP Address of server"); 27 | DEFINE_string(load_balancer, "", "The algorithm for load balancing"); 28 | DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds"); 29 | DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 30 | using namespace pidb; 31 | void HandleEchoResponse( 32 | brpc::Controller* cntl,EchoResponse *response) { 33 | // std::unique_ptr makes sure cntl/response will be deleted before returning. 34 | //std::unique_ptr cntl_guard(cntl); 35 | 36 | if (cntl->Failed()) { 37 | LOG(WARNING) << "Fail to send EchoRequest, " << cntl->ErrorText(); 38 | return; 39 | } 40 | LOG(INFO) << "Received response from " << cntl->remote_side() 41 | << ": " << " (attached=" 42 | << ")"<message() 43 | << " latency=" << cntl->latency_us() << "us"; 44 | } 45 | 46 | void Send(){ 47 | brpc::Channel channel; 48 | 49 | // Initialize the channel, NULL means using default options. 50 | brpc::ChannelOptions options; 51 | options.protocol = FLAGS_protocol; 52 | options.connection_type = FLAGS_connection_type; 53 | options.timeout_ms = FLAGS_timeout_ms/*milliseconds*/; 54 | options.max_retry = FLAGS_max_retry; 55 | if (channel.Init(FLAGS_server.c_str(), FLAGS_load_balancer.c_str(), &options) != 0) { 56 | LOG(ERROR) << "Fail to initialize channel"; 57 | } 58 | 59 | // Normally, you should not call a Channel directly, but instead construct 60 | // a stub Service wrapping it. stub can be shared by all threads as well. 61 | MasterService_Stub stub(&channel); 62 | 63 | // Send a request and wait for the response every 1 second. 64 | int log_id = 0; 65 | // while (!brpc::IsAskedToQuit()) { 66 | // Since we are sending asynchronous RPC (`done' is not NULL), 67 | // these objects MUST remain valid until `done' is called. 68 | // As a result, we allocate these objects on heap 69 | EchoResponse response; 70 | brpc::Controller* cntl = new brpc::Controller(); 71 | 72 | // Notice that you don't have to new request, which can be modified 73 | // or destroyed just after stub.Echo is called. 74 | EchoRequest request; 75 | 76 | EchoRequest request1; 77 | 78 | request.set_message("hello world"); 79 | 80 | cntl->set_log_id(log_id ++); // set by user 81 | if (FLAGS_send_attachment) { 82 | // Set attachment which is wired to network directly instead of 83 | // being serialized into protobuf messages. 84 | cntl->request_attachment().append("foo"); 85 | } 86 | 87 | // We use protobuf utility `NewCallback' to create a closure object 88 | // that will call our callback `HandleEchoResponse'. This closure 89 | // will automatically delete itself after being called once 90 | google::protobuf::Closure* done = brpc::NewCallback( 91 | &HandleEchoResponse, cntl,&response); 92 | stub.Echo(cntl, &request, &response, done); 93 | LOG(INFO)<<"dddd"; 94 | //delete response; 95 | 96 | // This is an asynchronous RPC, so we can only fetch the result 97 | // inside the callback 98 | // } 99 | 100 | } 101 | int main(int argc, char* argv[]) { 102 | // Parse gflags. We recommend you to use gflags as well. 103 | google::ParseCommandLineFlags(&argc, &argv, true); 104 | Send(); 105 | sleep(2); 106 | 107 | LOG(INFO) << "EchoClient is going to quit"; 108 | return 0; 109 | } 110 | -------------------------------------------------------------------------------- /src/port/README: -------------------------------------------------------------------------------- 1 | This directory contains interfaces and implementations that isolate the 2 | rest of the package from platform details. 3 | 4 | Code in the rest of the package includes "port.h" from this directory. 5 | "port.h" in turn includes a platform specific "port_.h" file 6 | that provides the platform specific implementation. 7 | 8 | See port_stdcxx.h for an example of what must be provided in a platform 9 | specific header file. 10 | 11 | -------------------------------------------------------------------------------- /src/port/atomic_pointer.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // AtomicPointer provides storage for a lock-free pointer. 6 | // Platform-dependent implementation of AtomicPointer: 7 | // - If the platform provides a cheap barrier, we use it with raw pointers 8 | // - If is present (on newer versions of gcc, it is), we use 9 | // a -based AtomicPointer. However we prefer the memory 10 | // barrier based version, because at least on a gcc 4.4 32-bit build 11 | // on linux, we have encountered a buggy implementation. 12 | // Also, some implementations are much slower than a memory-barrier 13 | // based implementation (~16ns for based acquire-load vs. ~1ns for 14 | // a barrier based acquire-load). 15 | // This code is based on atomicops-internals-* in Google's perftools: 16 | // http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase 17 | 18 | #ifndef PORT_ATOMIC_POINTER_H_ 19 | #define PORT_ATOMIC_POINTER_H_ 20 | 21 | #include 22 | 23 | #include 24 | 25 | #ifdef OS_WIN 26 | #include 27 | #endif 28 | 29 | #if defined(_M_X64) || defined(__x86_64__) 30 | #define ARCH_CPU_X86_FAMILY 1 31 | #elif defined(_M_IX86) || defined(__i386__) || defined(__i386) 32 | #define ARCH_CPU_X86_FAMILY 1 33 | #elif defined(__ARMEL__) 34 | #define ARCH_CPU_ARM_FAMILY 1 35 | #elif defined(__aarch64__) 36 | #define ARCH_CPU_ARM64_FAMILY 1 37 | #elif defined(__ppc__) || defined(__powerpc__) || defined(__powerpc64__) 38 | #define ARCH_CPU_PPC_FAMILY 1 39 | #elif defined(__mips__) 40 | #define ARCH_CPU_MIPS_FAMILY 1 41 | #endif 42 | 43 | namespace leveldb { 44 | namespace port { 45 | 46 | // Define MemoryBarrier() if available 47 | // Windows on x86 48 | #if defined(OS_WIN) && defined(COMPILER_MSVC) && defined(ARCH_CPU_X86_FAMILY) 49 | // windows.h already provides a MemoryBarrier(void) macro 50 | // http://msdn.microsoft.com/en-us/library/ms684208(v=vs.85).aspx 51 | #define LEVELDB_HAVE_MEMORY_BARRIER 52 | 53 | // Mac OS 54 | #elif defined(__APPLE__) 55 | inline void MemoryBarrier() { 56 | std::atomic_thread_fence(std::memory_order_seq_cst); 57 | } 58 | #define LEVELDB_HAVE_MEMORY_BARRIER 59 | 60 | // Gcc on x86 61 | #elif defined(ARCH_CPU_X86_FAMILY) && defined(__GNUC__) 62 | inline void MemoryBarrier() { 63 | // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 64 | // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 65 | __asm__ __volatile__("" : : : "memory"); 66 | } 67 | #define LEVELDB_HAVE_MEMORY_BARRIER 68 | 69 | // Sun Studio 70 | #elif defined(ARCH_CPU_X86_FAMILY) && defined(__SUNPRO_CC) 71 | inline void MemoryBarrier() { 72 | // See http://gcc.gnu.org/ml/gcc/2003-04/msg01180.html for a discussion on 73 | // this idiom. Also see http://en.wikipedia.org/wiki/Memory_ordering. 74 | asm volatile("" : : : "memory"); 75 | } 76 | #define LEVELDB_HAVE_MEMORY_BARRIER 77 | 78 | // ARM Linux 79 | #elif defined(ARCH_CPU_ARM_FAMILY) && defined(__linux__) 80 | typedef void (*LinuxKernelMemoryBarrierFunc)(void); 81 | // The Linux ARM kernel provides a highly optimized device-specific memory 82 | // barrier function at a fixed memory address that is mapped in every 83 | // user-level process. 84 | // 85 | // This beats using CPU-specific instructions which are, on single-core 86 | // devices, un-necessary and very costly (e.g. ARMv7-A "dmb" takes more 87 | // than 180ns on a Cortex-A8 like the one on a Nexus One). Benchmarking 88 | // shows that the extra function call cost is completely negligible on 89 | // multi-core devices. 90 | // 91 | inline void MemoryBarrier() { 92 | (*(LinuxKernelMemoryBarrierFunc)0xffff0fa0)(); 93 | } 94 | #define LEVELDB_HAVE_MEMORY_BARRIER 95 | 96 | // ARM64 97 | #elif defined(ARCH_CPU_ARM64_FAMILY) 98 | inline void MemoryBarrier() { 99 | asm volatile("dmb sy" : : : "memory"); 100 | } 101 | #define LEVELDB_HAVE_MEMORY_BARRIER 102 | 103 | // PPC 104 | #elif defined(ARCH_CPU_PPC_FAMILY) && defined(__GNUC__) 105 | inline void MemoryBarrier() { 106 | // TODO for some powerpc expert: is there a cheaper suitable variant? 107 | // Perhaps by having separate barriers for acquire and release ops. 108 | asm volatile("sync" : : : "memory"); 109 | } 110 | #define LEVELDB_HAVE_MEMORY_BARRIER 111 | 112 | // MIPS 113 | #elif defined(ARCH_CPU_MIPS_FAMILY) && defined(__GNUC__) 114 | inline void MemoryBarrier() { 115 | __asm__ __volatile__("sync" : : : "memory"); 116 | } 117 | #define LEVELDB_HAVE_MEMORY_BARRIER 118 | 119 | #endif 120 | 121 | // AtomicPointer built using platform-specific MemoryBarrier(). 122 | #if defined(LEVELDB_HAVE_MEMORY_BARRIER) 123 | class AtomicPointer { 124 | private: 125 | void* rep_; 126 | public: 127 | AtomicPointer() { } 128 | explicit AtomicPointer(void* p) : rep_(p) {} 129 | inline void* NoBarrier_Load() const { return rep_; } 130 | inline void NoBarrier_Store(void* v) { rep_ = v; } 131 | inline void* Acquire_Load() const { 132 | void* result = rep_; 133 | MemoryBarrier(); 134 | return result; 135 | } 136 | inline void Release_Store(void* v) { 137 | MemoryBarrier(); 138 | rep_ = v; 139 | } 140 | }; 141 | 142 | // AtomicPointer based on C++11 . 143 | #else 144 | class AtomicPointer { 145 | private: 146 | std::atomic rep_; 147 | public: 148 | AtomicPointer() { } 149 | explicit AtomicPointer(void* v) : rep_(v) { } 150 | inline void* Acquire_Load() const { 151 | return rep_.load(std::memory_order_acquire); 152 | } 153 | inline void Release_Store(void* v) { 154 | rep_.store(v, std::memory_order_release); 155 | } 156 | inline void* NoBarrier_Load() const { 157 | return rep_.load(std::memory_order_relaxed); 158 | } 159 | inline void NoBarrier_Store(void* v) { 160 | rep_.store(v, std::memory_order_relaxed); 161 | } 162 | }; 163 | 164 | #endif 165 | 166 | #undef LEVELDB_HAVE_MEMORY_BARRIER 167 | #undef ARCH_CPU_X86_FAMILY 168 | #undef ARCH_CPU_ARM_FAMILY 169 | #undef ARCH_CPU_ARM64_FAMILY 170 | #undef ARCH_CPU_PPC_FAMILY 171 | 172 | } // namespace port 173 | } // namespace leveldb 174 | 175 | #endif // PORT_ATOMIC_POINTER_H_ 176 | -------------------------------------------------------------------------------- /src/port/port.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_H_ 7 | 8 | #include 9 | 10 | // Include the appropriate platform specific file below. If you are 11 | // porting to a new platform, see "port_example.h" for documentation 12 | // of what the new port_.h file must provide. 13 | #if defined(LEVELDB_PLATFORM_POSIX) 14 | # include "port/port_stdcxx.h" 15 | #elif defined(LEVELDB_PLATFORM_CHROMIUM) 16 | # include "port/port_chromium.h" 17 | #endif 18 | 19 | #endif // STORAGE_LEVELDB_PORT_PORT_H_ 20 | -------------------------------------------------------------------------------- /src/port/port_config.h.in: -------------------------------------------------------------------------------- 1 | // Copyright 2017 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ 7 | 8 | // Define to 1 if you have a definition for fdatasync() in . 9 | #if !defined(HAVE_FDATASYNC) 10 | #cmakedefine01 HAVE_FDATASYNC 11 | #endif // !defined(HAVE_FDATASYNC) 12 | 13 | // Define to 1 if you have a definition for F_FULLFSYNC in . 14 | #if !defined(HAVE_FULLFSYNC) 15 | #cmakedefine01 HAVE_FULLFSYNC 16 | #endif // !defined(HAVE_FULLFSYNC) 17 | 18 | // Define to 1 if you have Google CRC32C. 19 | #if !defined(HAVE_CRC32C) 20 | #cmakedefine01 HAVE_CRC32C 21 | #endif // !defined(HAVE_CRC32C) 22 | 23 | // Define to 1 if you have Google Snappy. 24 | #if !defined(HAVE_SNAPPY) 25 | #cmakedefine01 HAVE_SNAPPY 26 | #endif // !defined(HAVE_SNAPPY) 27 | 28 | // Define to 1 if your processor stores words with the most significant byte 29 | // first (like Motorola and SPARC, unlike Intel and VAX). 30 | #if !defined(LEVELDB_IS_BIG_ENDIAN) 31 | #cmakedefine01 LEVELDB_IS_BIG_ENDIAN 32 | #endif // !defined(LEVELDB_IS_BIG_ENDIAN) 33 | 34 | #endif // STORAGE_LEVELDB_PORT_PORT_CONFIG_H_ -------------------------------------------------------------------------------- /src/port/port_example.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // 5 | // This file contains the specification, but not the implementations, 6 | // of the types/operations/etc. that should be defined by a platform 7 | // specific port_.h file. Use this file as a reference for 8 | // how to port this package to a new platform. 9 | 10 | #ifndef STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 11 | #define STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 12 | 13 | #include "port/thread_annotations.h" 14 | 15 | namespace leveldb { 16 | namespace port { 17 | 18 | // TODO(jorlow): Many of these belong more in the environment class rather than 19 | // here. We should try moving them and see if it affects perf. 20 | 21 | // The following boolean constant must be true on a little-endian machine 22 | // and false otherwise. 23 | static const bool kLittleEndian = true /* or some other expression */; 24 | 25 | // ------------------ Threading ------------------- 26 | 27 | // A Mutex represents an exclusive lock. 28 | class LOCKABLE Mutex { 29 | public: 30 | Mutex(); 31 | ~Mutex(); 32 | 33 | // Lock the mutex. Waits until other lockers have exited. 34 | // Will deadlock if the mutex is already locked by this thread. 35 | void Lock() EXCLUSIVE_LOCK_FUNCTION(); 36 | 37 | // Unlock the mutex. 38 | // REQUIRES: This mutex was locked by this thread. 39 | void Unlock() UNLOCK_FUNCTION(); 40 | 41 | // Optionally crash if this thread does not hold this mutex. 42 | // The implementation must be fast, especially if NDEBUG is 43 | // defined. The implementation is allowed to skip all checks. 44 | void AssertHeld() ASSERT_EXCLUSIVE_LOCK(); 45 | }; 46 | 47 | class CondVar { 48 | public: 49 | explicit CondVar(Mutex* mu); 50 | ~CondVar(); 51 | 52 | // Atomically release *mu and block on this condition variable until 53 | // either a call to SignalAll(), or a call to Signal() that picks 54 | // this thread to wakeup. 55 | // REQUIRES: this thread holds *mu 56 | void Wait(); 57 | 58 | // If there are some threads waiting, wake up at least one of them. 59 | void Signal(); 60 | 61 | // Wake up all waiting threads. 62 | void SignallAll(); 63 | }; 64 | 65 | // A type that holds a pointer that can be read or written atomically 66 | // (i.e., without word-tearing.) 67 | class AtomicPointer { 68 | private: 69 | intptr_t rep_; 70 | public: 71 | // Initialize to arbitrary value 72 | AtomicPointer(); 73 | 74 | // Initialize to hold v 75 | explicit AtomicPointer(void* v) : rep_(v) { } 76 | 77 | // Read and return the stored pointer with the guarantee that no 78 | // later memory access (read or write) by this thread can be 79 | // reordered ahead of this read. 80 | void* Acquire_Load() const; 81 | 82 | // Set v as the stored pointer with the guarantee that no earlier 83 | // memory access (read or write) by this thread can be reordered 84 | // after this store. 85 | void Release_Store(void* v); 86 | 87 | // Read the stored pointer with no ordering guarantees. 88 | void* NoBarrier_Load() const; 89 | 90 | // Set va as the stored pointer with no ordering guarantees. 91 | void NoBarrier_Store(void* v); 92 | }; 93 | 94 | // ------------------ Compression ------------------- 95 | 96 | // Store the snappy compression of "input[0,input_length-1]" in *output. 97 | // Returns false if snappy is not supported by this port. 98 | bool Snappy_Compress(const char* input, size_t input_length, 99 | std::string* output); 100 | 101 | // If input[0,input_length-1] looks like a valid snappy compressed 102 | // buffer, store the size of the uncompressed data in *result and 103 | // return true. Else return false. 104 | bool Snappy_GetUncompressedLength(const char* input, size_t length, 105 | size_t* result); 106 | 107 | // Attempt to snappy uncompress input[0,input_length-1] into *output. 108 | // Returns true if successful, false if the input is invalid lightweight 109 | // compressed data. 110 | // 111 | // REQUIRES: at least the first "n" bytes of output[] must be writable 112 | // where "n" is the result of a successful call to 113 | // Snappy_GetUncompressedLength. 114 | bool Snappy_Uncompress(const char* input_data, size_t input_length, 115 | char* output); 116 | 117 | // ------------------ Miscellaneous ------------------- 118 | 119 | // If heap profiling is not supported, returns false. 120 | // Else repeatedly calls (*func)(arg, data, n) and then returns true. 121 | // The concatenation of all "data[0,n-1]" fragments is the heap profile. 122 | bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg); 123 | 124 | // Extend the CRC to include the first n bytes of buf. 125 | // 126 | // Returns zero if the CRC cannot be extended using acceleration, else returns 127 | // the newly extended CRC value (which may also be zero). 128 | uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size); 129 | 130 | } // namespace port 131 | } // namespace leveldb 132 | 133 | #endif // STORAGE_LEVELDB_PORT_PORT_EXAMPLE_H_ 134 | -------------------------------------------------------------------------------- /src/port/port_stdcxx.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ 6 | #define STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ 7 | 8 | // port/port_config.h availability is automatically detected via __has_include 9 | // in newer compilers. If LEVELDB_HAS_PORT_CONFIG_H is defined, it overrides the 10 | // configuration detection. 11 | #if defined(LEVELDB_HAS_PORT_CONFIG_H) 12 | 13 | #if LEVELDB_HAS_PORT_CONFIG_H 14 | #include "port/port_config.h" 15 | #endif // LEVELDB_HAS_PORT_CONFIG_H 16 | 17 | #elif defined(__has_include) 18 | 19 | #if __has_include("port/port_config.h") 20 | #include "port/port_config.h" 21 | #endif // __has_include("port/port_config.h") 22 | 23 | #endif // defined(LEVELDB_HAS_PORT_CONFIG_H) 24 | 25 | #if HAVE_CRC32C 26 | #include 27 | #endif // HAVE_CRC32C 28 | #if HAVE_SNAPPY 29 | #include 30 | #endif // HAVE_SNAPPY 31 | 32 | #include 33 | #include 34 | #include 35 | #include // NOLINT 36 | #include // NOLINT 37 | #include 38 | #include "port/atomic_pointer.h" 39 | #include "port/thread_annotations.h" 40 | 41 | namespace leveldb { 42 | namespace port { 43 | 44 | static const bool kLittleEndian = !LEVELDB_IS_BIG_ENDIAN; 45 | 46 | class CondVar; 47 | 48 | // Thinly wraps std::mutex. 49 | class LOCKABLE Mutex { 50 | public: 51 | Mutex() = default; 52 | ~Mutex() = default; 53 | 54 | Mutex(const Mutex&) = delete; 55 | Mutex& operator=(const Mutex&) = delete; 56 | 57 | void Lock() EXCLUSIVE_LOCK_FUNCTION() { mu_.lock(); } 58 | void Unlock() UNLOCK_FUNCTION() { mu_.unlock(); } 59 | void AssertHeld() ASSERT_EXCLUSIVE_LOCK() { } 60 | 61 | private: 62 | friend class CondVar; 63 | std::mutex mu_; 64 | }; 65 | 66 | // Thinly wraps std::condition_variable. 67 | class CondVar { 68 | public: 69 | explicit CondVar(Mutex* mu) : mu_(mu) { assert(mu != nullptr); } 70 | ~CondVar() = default; 71 | 72 | CondVar(const CondVar&) = delete; 73 | CondVar& operator=(const CondVar&) = delete; 74 | 75 | void Wait() { 76 | std::unique_lock lock(mu_->mu_, std::adopt_lock); 77 | cv_.wait(lock); 78 | lock.release(); 79 | } 80 | void Signal() { cv_.notify_one(); } 81 | void SignalAll() { cv_.notify_all(); } 82 | private: 83 | std::condition_variable cv_; 84 | Mutex* const mu_; 85 | }; 86 | 87 | inline bool Snappy_Compress(const char* input, size_t length, 88 | ::std::string* output) { 89 | #if HAVE_SNAPPY 90 | output->resize(snappy::MaxCompressedLength(length)); 91 | size_t outlen; 92 | snappy::RawCompress(input, length, &(*output)[0], &outlen); 93 | output->resize(outlen); 94 | return true; 95 | #endif // HAVE_SNAPPY 96 | 97 | return false; 98 | } 99 | 100 | inline bool Snappy_GetUncompressedLength(const char* input, size_t length, 101 | size_t* result) { 102 | #if HAVE_SNAPPY 103 | return snappy::GetUncompressedLength(input, length, result); 104 | #else 105 | return false; 106 | #endif // HAVE_SNAPPY 107 | } 108 | 109 | inline bool Snappy_Uncompress(const char* input, size_t length, char* output) { 110 | #if HAVE_SNAPPY 111 | return snappy::RawUncompress(input, length, output); 112 | #else 113 | return false; 114 | #endif // HAVE_SNAPPY 115 | } 116 | 117 | inline bool GetHeapProfile(void (*func)(void*, const char*, int), void* arg) { 118 | return false; 119 | } 120 | 121 | inline uint32_t AcceleratedCRC32C(uint32_t crc, const char* buf, size_t size) { 122 | #if HAVE_CRC32C 123 | return ::crc32c::Extend(crc, reinterpret_cast(buf), size); 124 | #else 125 | return 0; 126 | #endif // HAVE_CRC32C 127 | } 128 | 129 | } // namespace port 130 | } // namespace leveldb 131 | 132 | #endif // STORAGE_LEVELDB_PORT_PORT_STDCXX_H_ 133 | -------------------------------------------------------------------------------- /src/port/thread_annotations.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2012 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 6 | #define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 7 | 8 | // Use Clang's thread safety analysis annotations when available. In other 9 | // environments, the macros receive empty definitions. 10 | // Usage documentation: https://clang.llvm.org/docs/ThreadSafetyAnalysis.html 11 | 12 | #if !defined(THREAD_ANNOTATION_ATTRIBUTE__) 13 | 14 | #if defined(__clang__) 15 | 16 | #define THREAD_ANNOTATION_ATTRIBUTE__(x) __attribute__((x)) 17 | #else 18 | #define THREAD_ANNOTATION_ATTRIBUTE__(x) // no-op 19 | #endif 20 | 21 | #endif // !defined(THREAD_ANNOTATION_ATTRIBUTE__) 22 | 23 | #ifndef GUARDED_BY 24 | #define GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(guarded_by(x)) 25 | #endif 26 | 27 | #ifndef PT_GUARDED_BY 28 | #define PT_GUARDED_BY(x) THREAD_ANNOTATION_ATTRIBUTE__(pt_guarded_by(x)) 29 | #endif 30 | 31 | #ifndef ACQUIRED_AFTER 32 | #define ACQUIRED_AFTER(...) \ 33 | THREAD_ANNOTATION_ATTRIBUTE__(acquired_after(__VA_ARGS__)) 34 | #endif 35 | 36 | #ifndef ACQUIRED_BEFORE 37 | #define ACQUIRED_BEFORE(...) \ 38 | THREAD_ANNOTATION_ATTRIBUTE__(acquired_before(__VA_ARGS__)) 39 | #endif 40 | 41 | #ifndef EXCLUSIVE_LOCKS_REQUIRED 42 | #define EXCLUSIVE_LOCKS_REQUIRED(...) \ 43 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_locks_required(__VA_ARGS__)) 44 | #endif 45 | 46 | #ifndef SHARED_LOCKS_REQUIRED 47 | #define SHARED_LOCKS_REQUIRED(...) \ 48 | THREAD_ANNOTATION_ATTRIBUTE__(shared_locks_required(__VA_ARGS__)) 49 | #endif 50 | 51 | #ifndef LOCKS_EXCLUDED 52 | #define LOCKS_EXCLUDED(...) \ 53 | THREAD_ANNOTATION_ATTRIBUTE__(locks_excluded(__VA_ARGS__)) 54 | #endif 55 | 56 | #ifndef LOCK_RETURNED 57 | #define LOCK_RETURNED(x) \ 58 | THREAD_ANNOTATION_ATTRIBUTE__(lock_returned(x)) 59 | #endif 60 | 61 | #ifndef LOCKABLE 62 | #define LOCKABLE \ 63 | THREAD_ANNOTATION_ATTRIBUTE__(lockable) 64 | #endif 65 | 66 | #ifndef SCOPED_LOCKABLE 67 | #define SCOPED_LOCKABLE \ 68 | THREAD_ANNOTATION_ATTRIBUTE__(scoped_lockable) 69 | #endif 70 | 71 | #ifndef EXCLUSIVE_LOCK_FUNCTION 72 | #define EXCLUSIVE_LOCK_FUNCTION(...) \ 73 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_lock_function(__VA_ARGS__)) 74 | #endif 75 | 76 | #ifndef SHARED_LOCK_FUNCTION 77 | #define SHARED_LOCK_FUNCTION(...) \ 78 | THREAD_ANNOTATION_ATTRIBUTE__(shared_lock_function(__VA_ARGS__)) 79 | #endif 80 | 81 | #ifndef EXCLUSIVE_TRYLOCK_FUNCTION 82 | #define EXCLUSIVE_TRYLOCK_FUNCTION(...) \ 83 | THREAD_ANNOTATION_ATTRIBUTE__(exclusive_trylock_function(__VA_ARGS__)) 84 | #endif 85 | 86 | #ifndef SHARED_TRYLOCK_FUNCTION 87 | #define SHARED_TRYLOCK_FUNCTION(...) \ 88 | THREAD_ANNOTATION_ATTRIBUTE__(shared_trylock_function(__VA_ARGS__)) 89 | #endif 90 | 91 | #ifndef UNLOCK_FUNCTION 92 | #define UNLOCK_FUNCTION(...) \ 93 | THREAD_ANNOTATION_ATTRIBUTE__(unlock_function(__VA_ARGS__)) 94 | #endif 95 | 96 | #ifndef NO_THREAD_SAFETY_ANALYSIS 97 | #define NO_THREAD_SAFETY_ANALYSIS \ 98 | THREAD_ANNOTATION_ATTRIBUTE__(no_thread_safety_analysis) 99 | #endif 100 | 101 | #ifndef ASSERT_EXCLUSIVE_LOCK 102 | #define ASSERT_EXCLUSIVE_LOCK(...) \ 103 | THREAD_ANNOTATION_ATTRIBUTE__(assert_exclusive_lock(__VA_ARGS__)) 104 | #endif 105 | 106 | #ifndef ASSERT_SHARED_LOCK 107 | #define ASSERT_SHARED_LOCK(...) \ 108 | THREAD_ANNOTATION_ATTRIBUTE__(assert_shared_lock(__VA_ARGS__)) 109 | #endif 110 | 111 | #endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_ 112 | -------------------------------------------------------------------------------- /src/port/win/stdint.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | // MSVC didn't ship with this file until the 2010 version. 6 | 7 | #ifndef STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 8 | #define STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 9 | 10 | #if !defined(_MSC_VER) 11 | #error This file should only be included when compiling with MSVC. 12 | #endif 13 | 14 | // Define C99 equivalent types. 15 | typedef signed char int8_t; 16 | typedef signed short int16_t; 17 | typedef signed int int32_t; 18 | typedef signed long long int64_t; 19 | typedef unsigned char uint8_t; 20 | typedef unsigned short uint16_t; 21 | typedef unsigned int uint32_t; 22 | typedef unsigned long long uint64_t; 23 | 24 | #endif // STORAGE_LEVELDB_PORT_WIN_STDINT_H_ 25 | -------------------------------------------------------------------------------- /src/server/RouteTableTest.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | #include "route_table.h" 3 | 4 | TEST(FooTest, Test1){ 5 | EXPECT_EQ(1,1); 6 | EXPECT_TRUE(3>0); 7 | EXPECT_TRUE(3<0); 8 | } 9 | 10 | TEST(RouteTest, AddTest){ 11 | pidb::RouteTable r; 12 | r.AddRecord("","a","group1"); 13 | r.AddRecord("b","d","group2"); 14 | EXPECT_TRUE(r[""]=="group1"); 15 | EXPECT_TRUE(r["b"]=="group2"); 16 | 17 | EXPECT_TRUE(r.FindRegion("1") == "group1"); 18 | EXPECT_TRUE(r.FindRegion("c")=="group2"); 19 | EXPECT_TRUE(r.FindRegion("e")==""); 20 | } 21 | 22 | int main(int argc,char *argv[]){ 23 | ::testing::InitGoogleTest(&argc,argv); 24 | return RUN_ALL_TESTS(); 25 | } -------------------------------------------------------------------------------- /src/server/context_cache.cpp: -------------------------------------------------------------------------------- 1 | #include "context_cache.h" 2 | namespace pidb{ 3 | 4 | }//namespace pidb -------------------------------------------------------------------------------- /src/server/context_cache.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-17. 3 | // 4 | #pragma once 5 | 6 | #ifndef PIDB_CONTEXT_CACHE_H 7 | #define PIDB_CONTEXT_CACHE_H 8 | 9 | #include "leveldb/db.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace pidb { 17 | class Context { 18 | public: 19 | Context() = default; 20 | 21 | Context(const Context &) = default; 22 | 23 | Context &operator=(const Context &) = default; 24 | 25 | virtual ~Context() = default; 26 | 27 | void SetID(int64_t id)noexcept { 28 | id_=id; 29 | } 30 | int64_t GetID() const{ 31 | return id_; 32 | } 33 | private: 34 | int64_t id_; 35 | }; 36 | 37 | 38 | class SnapshotContext : public Context { 39 | public: 40 | SnapshotContext(const leveldb::Snapshot *snapshot){ 41 | snapshot_ = snapshot; 42 | }; 43 | 44 | SnapshotContext(const SnapshotContext &) = delete; 45 | 46 | SnapshotContext &operator=(const SnapshotContext &) = delete; 47 | 48 | ~SnapshotContext() { 49 | //在这里我们需要释放snapshot,但是snapshot是存储在db里面的snapshotlist 50 | //需要调用db的release_snapshot. 51 | //测试 52 | std::cout<<"release snapshot"< iterator,const std::string &group) 65 | :iterator_(std::move(iterator)),group_(group){}; 66 | 67 | IteratorContext(const IteratorContext &) = delete; 68 | 69 | IteratorContext &operator=(const IteratorContext &) = delete; 70 | ~IteratorContext(){ 71 | //shoud we do somthing? 72 | } 73 | leveldb::Iterator * Get() const{ 74 | return iterator_.get(); 75 | } 76 | private: 77 | std::unique_ptr iterator_; 78 | // std::string start_; //当前iterator起始key 79 | // std::string end_; //当前iterator的结束key 80 | const std::string group_; //用于标识当前iterator属于哪个group 81 | 82 | 83 | }; 84 | 85 | //用于缓存类型为T的指针，以ID为name,用完需要调用Erase释放资源 86 | template 87 | class ContextCache { 88 | public: 89 | ContextCache() noexcept{ 90 | context_id_.store(0,std::memory_order_relaxed); 91 | }; 92 | 93 | //no copy 94 | ContextCache(const ContextCache &) = delete; 95 | ContextCache &operator=(const ContextCache &) = delete; 96 | 97 | 98 | virtual ~ContextCache() { map_.clear(); } 99 | //放如一个对象，并且返回一个全局id 100 | uint64_t Put(std::unique_ptr context); 101 | //通过全局ID获得对象 102 | std::shared_ptr Get(int64_t context_id); 103 | //从缓存中删除这个对象 104 | virtual bool Erase(int64_t context_id); 105 | //清空缓存 106 | virtual void Clear(){map_.clear();} 107 | //获得cache的大小 108 | virtual int Size() const noexcept{ return map_.size();} 109 | private: 110 | std::unordered_map> map_; 111 | std::atomic context_id_; 112 | }; 113 | 114 | 115 | template 116 | uint64_t ContextCache::Put(std::unique_ptr context) { 117 | //TODO 测试多线程一致性, s是否是当前+1前的值 118 | //https://stackoverflow.com/questions/56185373/how-to-guarantee-data-dependence-with-atomic 119 | 120 | auto s = std::shared_ptr(reinterpret_cast(context.release())); 121 | auto res = context_id_.fetch_add(1,std::memory_order_relaxed); 122 | map_[res]=s; 123 | return res; 124 | } 125 | 126 | template 127 | std::shared_ptr ContextCache::Get(int64_t context_id) { 128 | if(map_.find(context_id) == map_.end()){ 129 | return nullptr; 130 | } 131 | return map_[context_id]; 132 | } 133 | 134 | template 135 | bool ContextCache::Erase(int64_t context_id) { 136 | if(map_.find(context_id) == map_.end()){ 137 | return false; 138 | } 139 | 140 | map_.erase(context_id); 141 | return true; 142 | } 143 | 144 | 145 | }//namespace pidb 146 | #endif //PIDB_CONTEXT_CACHE_H 147 | -------------------------------------------------------------------------------- /src/server/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "server.h" 4 | #include "context_cache.h" 5 | #include "route_table.h" 6 | #include 7 | #include "raftnode.h" 8 | #include "master_service_impl.h" 9 | 10 | DEFINE_string(data_path,"./data","Path of data stored on"); 11 | DEFINE_int32(port,8100,"port of server to listen on"); 12 | DEFINE_string(master_server,"127.0.1.1:8300","port of server to listen on"); 13 | 14 | int main(int argc,char *argv[]){ 15 | GFLAGS_NS::ParseCommandLineFlags(&argc,&argv,true); 16 | butil::AtExitManager exit_manager; 17 | //创建文件夹 18 | if(!butil::DirectoryExists(butil::FilePath("./data"))){ 19 | butil::CreateDirectory(butil::FilePath("./data")); 20 | } 21 | //初始化server 22 | brpc::Server server; 23 | pidb::ServerOption options(FLAGS_data_path,FLAGS_port); 24 | options.heartbeat_timeout_ms = 2000; 25 | auto s = std::shared_ptr(new pidb::Server(options)); 26 | pidb::PiDBServiceImpl service(s.get()); 27 | pidb::MasterServiceImpl master(s.get()); 28 | 29 | //增加Node Server的服务 30 | if (server.AddService(&service, 31 | brpc::SERVER_DOESNT_OWN_SERVICE) != 0) { 32 | LOG(ERROR) << "Fail to add service"; 33 | return -1; 34 | } 35 | 36 | 37 | if(server.AddService(&master,brpc::SERVER_DOESNT_OWN_SERVICE)!=0){ 38 | LOG(ERROR)<<"Fail to add master service"; 39 | return -1; 40 | } 41 | 42 | //增加raft的服务，与Node Server共享 43 | if (braft::add_service(&server, FLAGS_port) != 0) { 44 | LOG(ERROR) << "Fail to add raft service"; 45 | return -1; 46 | } 47 | //启动brpc server 48 | if (server.Start(FLAGS_port, NULL) != 0) { 49 | LOG(ERROR) << "Fail to start Server"; 50 | return -1; 51 | } 52 | //启动Node server 53 | s->Start(); 54 | 55 | //等待用户结束 56 | // pidb::RaftOption option; 57 | // option.port = 8200; 58 | // option.group = "group1"; 59 | // option.conf = "127.0.1.1:8200:1"; 60 | // option.data_path ="./group1"; 61 | // auto status= s->registerRaftNode(option,pidb::Range("","")); 62 | 63 | // option.conf = "127.0.1.1:8100:0"; 64 | // option.group = "group0"; 65 | // option.data_path ="./group0"; 66 | // status= s->registerRaftNode(option,pidb::Range("","")); 67 | 68 | // LOG(INFO)<<"Register Node"<raft_group(); 14 | std::string group = request->raft_group(); 15 | LOG(INFO)<set_is_leader(true); 18 | server_->HandleRaftManage(request,NULL,NULL); 19 | } 20 | 21 | void MasterServiceImpl::PullData(::google::protobuf::RpcController *controller, 22 | const ::pidb::PiDBPullRequest *request, ::pidb::PiDBPullResponse *response, 23 | ::google::protobuf::Closure *done) { 24 | brpc::ClosureGuard done_guard(done); 25 | response->set_success(true); 26 | LOG(INFO)<raft_conf()<raft_group(); 27 | } 28 | 29 | }//namespace pidb -------------------------------------------------------------------------------- /src/server/master_service_impl.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 7/14/19. 3 | // 4 | 5 | #ifndef PIDB_MASTER_SERVICE_IMPL_H 6 | #define PIDB_MASTER_SERVICE_IMPL_H 7 | 8 | #include "master.pb.h" 9 | namespace pidb { 10 | class Server; 11 | class MasterServiceImpl : public MasterService { 12 | public: 13 | explicit MasterServiceImpl(Server *server) : server_(server) {}; 14 | 15 | virtual void RaftManage(::google::protobuf::RpcController *controller, 16 | const ::pidb::PiDBRaftManageRequest *request, 17 | ::pidb::PiDBRaftManageResponse *response, 18 | ::google::protobuf::Closure *done) override; 19 | 20 | virtual void PullData(::google::protobuf::RpcController* controller, 21 | const ::pidb::PiDBPullRequest* request, 22 | ::pidb::PiDBPullResponse* response, 23 | ::google::protobuf::Closure* done); 24 | 25 | 26 | virtual ~MasterServiceImpl() {}; 27 | 28 | private: 29 | Server *server_; 30 | 31 | }; 32 | } 33 | 34 | #endif //PIDB_MASTER_SERVICE_IMPL_H 35 | -------------------------------------------------------------------------------- /src/server/pidb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | package pidb; 3 | option cc_generic_services = true; 4 | 5 | message Empty{} 6 | 7 | message Success{ 8 | required bool success = 1; 9 | optional string message = 2; 10 | } 11 | message PiDBSnapshot{ 12 | required int32 id = 1; //snapshot id 13 | } 14 | 15 | 16 | message PiDBRequest{ 17 | required string key = 1; 18 | optional string value = 2; 19 | optional PiDBSnapshot snapshot = 3; //读清求的snapshot 20 | } 21 | 22 | message PiDBOperator{ 23 | required int32 op = 1; //0: unknown 0:Put 1:Delete 2:Write 完善 24 | required string key=2; 25 | optional string value = 3; 26 | } 27 | 28 | message PiDBWriteBatch{ 29 | repeated PiDBOperator WriteBatch = 1; 30 | } 31 | 32 | message PiDBResponse{ 33 | required bool success = 1; 34 | optional string old_value = 2; 35 | optional string new_value = 3; 36 | optional string redirect = 4; 37 | } 38 | 39 | //Iterator的相关信息 40 | message PiDBIterator{ 41 | required int32 id = 1; // Iterator的标识 42 | optional string start = 2; // Iterator的起始位置 暂时为 optional 43 | optional string stop = 3; // Iterator的终点位置 44 | } 45 | 46 | message PiDBFileRequest{ 47 | required string filename=1; 48 | } 49 | 50 | message PiDBFileResponse{ 51 | required bool success = 1; 52 | } 53 | 54 | 55 | service PiDBService { 56 | rpc Put(PiDBRequest) returns (PiDBResponse); 57 | rpc Get(PiDBRequest) returns (PiDBResponse); 58 | rpc Write(PiDBWriteBatch) returns (PiDBResponse); 59 | rpc GetSnapshot(Empty) returns (PiDBSnapshot); 60 | rpc ReleaseSnapshot(PiDBSnapshot) returns (Success); 61 | rpc GetIterator(PiDBIterator) returns (PiDBIterator); 62 | rpc Iterate (PiDBIterator) returns (PiDBResponse); //Iterate 暂时默认为只能next 63 | rpc RequestFile (PiDBFileRequest) returns(PiDBFileResponse); 64 | rpc PushFile (PiDBFileRequest) returns (PiDBFileResponse); 65 | } 66 | 67 | -------------------------------------------------------------------------------- /src/server/pidb_service_impl.cpp: -------------------------------------------------------------------------------- 1 | #include "pidb_service_impl.h" 2 | #include "server.h" 3 | #include "brpc/controller.h" 4 | #include 5 | #include 6 | #include 7 | 8 | namespace pidb { 9 | 10 | class StreamReceiver : public brpc::StreamInputHandler { 11 | public: 12 | StreamReceiver(leveldb::WritableFile *file):file_(file){ 13 | LOG(INFO)<<"INIT FILE"; 14 | } 15 | virtual int on_received_messages(brpc::StreamId id, 16 | butil::IOBuf *const messages[], 17 | size_t size) { 18 | //std::ostringstream os; 19 | assert(file!= nullptr); 20 | for (size_t i = 0; i < size; ++i) { 21 | // os << "msg[" << i << "]=" << *messages[i]; 22 | //LOG(INFO)<<*messages[i]; 23 | file_->Append(messages[i]->to_string()); 24 | } 25 | //LOG(INFO) << "Received from Stream=" << id << ": " << os.str(); 26 | return 0; 27 | } 28 | 29 | virtual void on_idle_timeout(brpc::StreamId id) { 30 | LOG(INFO) << "Stream=" << id << " has no data transmission for a while"; 31 | } 32 | 33 | virtual void on_closed(brpc::StreamId id) { 34 | LOG(INFO) << "Stream=" << id << " is closed"; 35 | file_->Flush(); 36 | delete file_; 37 | delete this; 38 | //file_->Close(); 39 | } 40 | 41 | ~StreamReceiver(){ 42 | LOG(INFO)<<"HANDLER IS DONE"; 43 | } 44 | 45 | private: 46 | leveldb::WritableFile *file_; 47 | }; 48 | 49 | 50 | void PiDBServiceImpl::Put(::google::protobuf::RpcController *controller, 51 | const ::pidb::PiDBRequest *request, 52 | ::pidb::PiDBResponse *response, 53 | ::google::protobuf::Closure *done) { 54 | //brpc::Controller *cntl = static_cast(controller); 55 | server_->Put(request, response, done); 56 | 57 | } 58 | 59 | void PiDBServiceImpl::Get(::google::protobuf::RpcController *controller, 60 | const ::pidb::PiDBRequest *request, 61 | ::pidb::PiDBResponse *response, 62 | ::google::protobuf::Closure *done) { 63 | //auto cntl = static_cast(controller); 64 | brpc::ClosureGuard done_guard(done); 65 | server_->Get(request, response, done); 66 | } 67 | 68 | void PiDBServiceImpl::Write(::google::protobuf::RpcController *controller, 69 | const ::pidb::PiDBWriteBatch *request, 70 | ::pidb::PiDBResponse *response, 71 | ::google::protobuf::Closure *done) { 72 | 73 | //brpc是使用static_cast转化，这是不安全的，https://docs.microsoft.com/en-us/cpp/cpp/static-cast-operator?view=vs-2019 74 | //应该使用dynamic_cast 75 | // auto cntl = static_cast(controller); 76 | auto req = request; 77 | auto batch_size = req->writebatch_size(); 78 | if(batch_size<1){ 79 | response->set_success(false); 80 | return; 81 | } 82 | return server_->Write(request,response,done); 83 | 84 | } 85 | 86 | void PiDBServiceImpl::GetSnapshot(::google::protobuf::RpcController *controller, 87 | const ::pidb::Empty *request, 88 | ::pidb::PiDBSnapshot *response, 89 | ::google::protobuf::Closure *done){ 90 | brpc::ClosureGuard done_guard(done); 91 | 92 | auto id = server_->GetSnapshot(); 93 | response->set_id(id); 94 | 95 | } 96 | 97 | void PiDBServiceImpl::ReleaseSnapshot(::google::protobuf::RpcController *controller, 98 | const ::pidb::PiDBSnapshot *request, 99 | ::pidb::Success *response, 100 | ::google::protobuf::Closure *done) { 101 | brpc::ClosureGuard done_guard(done); 102 | auto id = request->id(); 103 | auto s = server_->ReleaseSnapshot(id); 104 | if(s.ok()){ 105 | response->set_success(true); 106 | LOG(INFO)<<"RELEASE SUCCESS"; 107 | }else{ 108 | response->set_success(false); 109 | response->set_message(s.ToString()); 110 | } 111 | 112 | } 113 | 114 | void PiDBServiceImpl::GetIterator(::google::protobuf::RpcController *controller, 115 | const ::pidb::PiDBIterator *request, 116 | ::pidb::PiDBIterator *response, 117 | ::google::protobuf::Closure *done) { 118 | 119 | brpc::ClosureGuard done_guard(done); 120 | auto id = server_->GetIterator(request->start(),request->stop()); 121 | LOG(INFO)<set_id(id); 123 | 124 | } 125 | void PiDBServiceImpl::Iterate(::google::protobuf::RpcController *controller, 126 | const ::pidb::PiDBIterator *request, 127 | ::pidb::PiDBResponse *response, 128 | ::google::protobuf::Closure *done) { 129 | brpc::ClosureGuard self_guard(done); 130 | auto id = request->id(); 131 | std::string value; 132 | auto s = server_->Next(id,&value); 133 | if (s.ok()) { 134 | response->set_success(true); 135 | response->set_new_value(value); 136 | }else{ 137 | LOG(INFO)<set_success(false); 139 | } 140 | } 141 | void PiDBServiceImpl::RequestFile(::google::protobuf::RpcController *controller, 142 | const ::pidb::PiDBFileRequest *request, ::pidb::PiDBFileResponse *response, 143 | ::google::protobuf::Closure *done) { 144 | brpc::ClosureGuard self_gurad(done); 145 | LOG(INFO)<<"Reques file"<filename(); 146 | 147 | //TODO 判断是否有文件 148 | response->set_success(true); 149 | } 150 | 151 | 152 | void PiDBServiceImpl::PushFile(::google::protobuf::RpcController *controller, 153 | const ::pidb::PiDBFileRequest *request, ::pidb::PiDBFileResponse *response, 154 | ::google::protobuf::Closure *done) { 155 | 156 | brpc::ClosureGuard done_guard(done); 157 | 158 | brpc::Controller* cntl = 159 | static_cast(controller); 160 | brpc::StreamOptions stream_options; 161 | leveldb::WritableFile *file; 162 | 163 | std::string path = "./receive/test.txt"; 164 | 165 | leveldb::Env *env = leveldb::Env::Default(); 166 | 167 | env->NewWritableFile(path,&file); 168 | 169 | StreamReceiver* r=new StreamReceiver(file); 170 | stream_options.handler = r; 171 | brpc::StreamId _sd; 172 | if (brpc::StreamAccept(&_sd, *cntl, &stream_options) != 0) { 173 | cntl->SetFailed("Fail to accept stream"); 174 | LOG(INFO)<<"STREAM ID IS DONE"<<_sd; 175 | return; 176 | } 177 | response->set_success(true); 178 | 179 | } 180 | 181 | } 182 | //TO-DO 加入新的功能。 -------------------------------------------------------------------------------- /src/server/pidb_service_impl.h: -------------------------------------------------------------------------------- 1 | #ifndef PIDB_PIDB_SERVICE_IMPL_H_ 2 | #define PIDB_PIDB_SERVICE_IMPL_H_ 3 | #include "pidb.pb.h" 4 | 5 | 6 | 7 | namespace pidb{ 8 | class Server; 9 | 10 | class PiDBServiceImpl:public PiDBService{ 11 | public: 12 | explicit PiDBServiceImpl(Server* server):server_(server){}; 13 | 14 | void Get(::google::protobuf::RpcController* controller, 15 | const ::pidb::PiDBRequest* request, 16 | ::pidb::PiDBResponse* response, 17 | ::google::protobuf::Closure* done) override; 18 | 19 | void Put(::google::protobuf::RpcController* controller, 20 | const ::pidb::PiDBRequest* request, 21 | ::pidb::PiDBResponse* response, 22 | ::google::protobuf::Closure* done) override; 23 | 24 | void Write(::google::protobuf::RpcController* controller, 25 | const ::pidb::PiDBWriteBatch* request, 26 | ::pidb::PiDBResponse* response, 27 | ::google::protobuf::Closure* done) override; 28 | 29 | void GetSnapshot(::google::protobuf::RpcController* controller, 30 | const ::pidb::Empty* request, 31 | ::pidb::PiDBSnapshot* response, 32 | ::google::protobuf::Closure* done) override; 33 | 34 | void ReleaseSnapshot(::google::protobuf::RpcController* controller, 35 | const ::pidb::PiDBSnapshot* request, 36 | ::pidb::Success* response, 37 | ::google::protobuf::Closure* done) override; 38 | 39 | void GetIterator(::google::protobuf::RpcController* controller, 40 | const ::pidb::PiDBIterator* request, 41 | ::pidb::PiDBIterator* response, 42 | ::google::protobuf::Closure* done) override; 43 | 44 | void Iterate(::google::protobuf::RpcController* controller, 45 | const ::pidb::PiDBIterator* request, 46 | ::pidb::PiDBResponse* response, 47 | ::google::protobuf::Closure* done) override ; 48 | void RequestFile(::google::protobuf::RpcController* controller, 49 | const ::pidb::PiDBFileRequest* request, 50 | ::pidb::PiDBFileResponse* response, 51 | ::google::protobuf::Closure* done) override ; 52 | 53 | void PushFile(::google::protobuf::RpcController* controller, 54 | const ::pidb::PiDBFileRequest* request, 55 | ::pidb::PiDBFileResponse* response, 56 | ::google::protobuf::Closure* done) override ; 57 | 58 | private: 59 | Server *server_; 60 | }; 61 | } 62 | #endif -------------------------------------------------------------------------------- /src/server/raftnode.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef PIDB_RAFTNODE_H_ 3 | #define PIDB_RAFTNODE_H_ 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include //braft::AsyncClosureGuard 9 | #include //braf::SnapshotWriter 10 | #include 11 | #include "braft/repeated_timer_task.h" 12 | #include 13 | #include "shareddb.h" 14 | #include 15 | 16 | namespace pidb 17 | { 18 | class RaftNode; 19 | struct RaftOption; 20 | class Status; 21 | 22 | class RaftNodeClosure: public braft::Closure{ 23 | public: 24 | RaftNodeClosure(RaftNode* node, 25 | const PiDBRequest *request, 26 | PiDBResponse* response, 27 | google::protobuf::Closure* done) 28 | :node_(node), 29 | request_(request), 30 | response_(response), 31 | done_(done){} 32 | ~RaftNodeClosure(){} 33 | const PiDBRequest* request() const {return request_;} 34 | PiDBResponse* response() const {return response_;} 35 | void Run(); 36 | 37 | private: 38 | RaftNode* node_; 39 | const PiDBRequest* request_; 40 | PiDBResponse* response_; 41 | google::protobuf::Closure* done_; 42 | }; 43 | 44 | struct Range{ 45 | Slice start; 46 | Slice limit; 47 | Range(){} 48 | 49 | Range(const Slice& s,const Slice& l):start(s),limit(l){} 50 | }; 51 | 52 | using leveldb::Snapshot; 53 | using leveldb::Iterator; 54 | 55 | class RaftTimer:public braft::RepeatedTimerTask{ 56 | public: 57 | RaftTimer():raft_(nullptr){} 58 | int init(RaftNode* raft, int timeout_ms); 59 | virtual void run() = 0; 60 | 61 | protected: 62 | void on_destroy(); 63 | //std::shared_ptr raft_; 64 | RaftNode* raft_; 65 | }; 66 | 67 | class RaftHeartbeatTimer:public RaftTimer{ 68 | protected: 69 | void run(); 70 | }; 71 | 72 | 73 | class RaftNode : public braft::StateMachine{ 74 | public: 75 | 76 | enum RaftOpType { 77 | kUnknownOp = 0, 78 | kGetOp = 1, 79 | kPutOp = 2, 80 | kDeleteOP = 3, 81 | kWriteOp = 4 82 | }; 83 | int count_=0; 84 | 85 | //RaftNode 的Rnage初始化不用option传入，因为range可能在运行中是经常变化的 86 | //我认为不要写入option中可能好一点 87 | //使用option设置node的端口和group(启动之后不会变化) 88 | RaftNode(const RaftOption &option,const Range &range); 89 | ~RaftNode() { 90 | delete node_; 91 | } 92 | Status start(); 93 | 94 | virtual Status Get(const PiDBRequest *request,PiDBResponse* response, 95 | ::google::protobuf::Closure* done) const; 96 | virtual void Put(const PiDBRequest *request,PiDBResponse* response, 97 | ::google::protobuf::Closure* done); 98 | 99 | virtual void Write(const leveldb::WriteOptions& options,std::unique_ptr batch, 100 | braft::Closure *done); 101 | 102 | /*待实现 103 | virtual Status Write(const PiDBRequest *request,PiDBResponse* response, 104 | ::google::protobuf::Closure* done); 105 | virtual Status Delete(const PiDBRequest *request,PiDBResponse* response, 106 | ::google::protobuf::Closure* done); 107 | virtual Iterator * NewIterator(const PiDBRequest *request,PiDBResponse* response, 108 | ::google::protobuf::Closure* done); 109 | virtual Snapshot* GetSnapshot(); 110 | virtual void ReleaseSnapshot(const Snapshot* snapshot); 111 | */ 112 | void on_apply(braft::Iterator &iter) override; 113 | void on_snapshot_save(braft::SnapshotWriter* writer,braft::Closure* done) override; 114 | int on_snapshot_load(braft::SnapshotReader* reader) override; 115 | void on_leader_start(int64_t term) override; 116 | void on_shutdown() override; 117 | void on_error(const ::braft::Error &e) override; 118 | void on_configuration_committed(const ::braft::Configuration& conf) override ; 119 | void on_stop_following(const ::braft::LeaderChangeContext& ctx) override; 120 | void on_start_following(const ::braft::LeaderChangeContext& ctx) override; 121 | void backup_data(); 122 | bool is_leader() const{ return leader_term_.load(std::memory_order_acquire)>0;} 123 | void on_role_change(); 124 | 125 | Status do_put_or_del(uint8_t type,const butil::IOBuf& data, braft::Closure *done); 126 | Status do_write(uint8_t type,const butil::IOBuf& data,braft::Closure *done); 127 | 128 | Status SetRange(const Slice &s,const Slice &l){ 129 | //TO-DO 判断s和l是否合法，例如s<=l 等等 130 | range_ = Range(s,l); 131 | return Status::OK(); 132 | } 133 | // Starts this node 134 | scoped_db GetDB() const{ 135 | BAIDU_SCOPED_LOCK(_db_mutex); 136 | return db_; 137 | } 138 | void SetDB(scoped_db db){ 139 | //也可以使用std::lock_guard 140 | BAIDU_SCOPED_LOCK(_db_mutex); 141 | db_ = db; 142 | 143 | } 144 | bool IsLeader() const{ 145 | return leader_term_.load(std::memory_order_acquire) >0; 146 | } 147 | void redirect(PiDBResponse * response); 148 | 149 | void shutdown(){ 150 | if(node_) node_->shutdown(NULL); 151 | } 152 | 153 | void join(){ 154 | if(node_) node_->join(); 155 | } 156 | 157 | void HandleHeartbeat(); 158 | static void* RequestSplit(void *arg); 159 | 160 | static void *save_snapshot(void* arg); 161 | static int link_overwrite(const char* old_path, const char* new_path); 162 | 163 | private: 164 | friend class RaftNodeClosure; 165 | mutable butil::Mutex _db_mutex; 166 | braft::Node* volatile node_; 167 | 168 | scoped_db db_; 169 | Range range_; 170 | 171 | 172 | //TODO 可以将这些信息直接用option来存储,而不是将他们独立出来 173 | //暂时用string 代表group,暂时让他们不可更改 174 | const std::string group_; 175 | //当前region的端口号 176 | const int32_t port_; 177 | //当前region的集群信息 178 | const std::string conf_; 179 | const std::string data_path_; 180 | //当前的任期号 181 | std::atomic leader_term_; 182 | 183 | //raft timer 184 | RaftHeartbeatTimer raft_timer_; 185 | //用于存储当前region的snapshot 的handle 186 | struct SnapshotHandle{ 187 | scoped_db db; 188 | braft::SnapshotWriter* writer; 189 | braft::Closure* done; 190 | Range * range; 191 | std::string data_path; 192 | 193 | }; 194 | 195 | //当raft成为leader或者follower后需要回调server传过来的方法 196 | ::google::protobuf::Closure* role_change; 197 | }; 198 | 199 | } // pidb 200 | 201 | #endif 202 | -------------------------------------------------------------------------------- /src/server/remote_file_send.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 7/24/19. 3 | // 4 | 5 | #include "remote_file_send.h" 6 | #include "pidb.pb.h" 7 | // 8 | //DEFINE_string(server, "0.0.0.0:8001", "IP Address of server"); 9 | //DEFINE_int32(timeout_ms, 100, "RPC timeout in milliseconds"); 10 | //DEFINE_int32(max_retry, 3, "Max retries(not including the first RPC)"); 11 | // 12 | 13 | namespace pidb{ 14 | 15 | RemoteFileSend::RemoteFileSend(const std::string &addr, const std::string &source_path) 16 | :remote_addr_(addr),source_path_(source_path) 17 | { 18 | brpc::ChannelOptions options; 19 | options.protocol = brpc::PROTOCOL_BAIDU_STD; 20 | options.connection_type = ""; 21 | options.timeout_ms = 100/*milliseconds*/; 22 | options.max_retry = 3; 23 | 24 | //初始化channel_ 25 | if (channel_.Init(remote_addr_.c_str(), NULL) != 0) { 26 | LOG(ERROR) << "Fail to initialize channel"; 27 | } 28 | 29 | //初始化stream 30 | if (brpc::StreamCreate(&stream_, cntl_, NULL) != 0) { 31 | LOG(ERROR) << "Fail to create stream"; 32 | } 33 | env_ = leveldb::Env::Default(); 34 | st_ = leveldb::Status::OK(); 35 | 36 | } 37 | 38 | //开始推送数据 39 | void RemoteFileSend::start() { 40 | 41 | pidb::PiDBService_Stub stub(&channel_); 42 | pidb::PiDBFileRequest request; 43 | pidb::PiDBFileResponse response; 44 | request.set_filename(source_path_); 45 | 46 | stub.PushFile(&cntl_,&request,&response,NULL); 47 | if(cntl_.Failed()){ 48 | LOG(INFO)<<"建立连接失败"<FileExists(source_path_)){ 53 | st_.Corruption(source_path_+" File not exists"); 54 | return; 55 | } 56 | 57 | leveldb::SequentialFile *file; 58 | env_->NewSequentialFile(source_path_,&file); 59 | 60 | if (brpc::StreamCreate(&stream_, cntl_, NULL) != 0) { 61 | LOG(ERROR) << "Fail to create stream"; 62 | } 63 | 64 | //写数据 65 | char *buffer = new char[128*1024]; 66 | leveldb::Slice res; 67 | butil::IOBuf msg; 68 | int retry = 5; 69 | //每次读写128k数据 70 | while ((st_=file->Read(128*1024,&res,buffer)).ok() && res.size()>0){ 71 | msg.append(buffer,res.size()); 72 | //需要重试 73 | auto code = brpc::StreamWrite(stream_, msg); 74 | LOG(INFO)<=5){ 83 | st_ = leveldb::Status::Corruption("Failt to send stream to remote server"); 84 | return; 85 | } 86 | 87 | } 88 | LOG(INFO)< 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | 19 | namespace pidb{ 20 | class RemoteFileSend { 21 | public: 22 | RemoteFileSend(const std::string& addr,const std::string &source_path); 23 | 24 | void start(); 25 | 26 | //待实现 27 | // void cancle(); 28 | const leveldb::Status &status() const {return st_;} 29 | 30 | 31 | private: 32 | //发送数据到指定group的region 33 | std::string group_; 34 | brpc::Controller cntl_; 35 | brpc::Channel channel_; 36 | leveldb::Env *env_; 37 | std::string source_path_; 38 | 39 | //远程braft的conf 40 | std::string conf_; 41 | std::string remote_addr_; 42 | brpc::StreamId stream_; 43 | 44 | //记录在传输过程中的状态 45 | leveldb::Status st_; 46 | 47 | 48 | }; 49 | }//namespace pidb 50 | 51 | #endif //PIDB_REMOTE_FILE_SEND_H 52 | -------------------------------------------------------------------------------- /src/server/route_table.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-4. 3 | 4 | #include "route_table.h" 5 | #include "json11.hpp" 6 | #include 7 | #include 8 | namespace pidb{ 9 | 10 | void RouteTable::BuildRouteTable(const std::string &name) const{ 11 | if(router_.empty()){ 12 | //内容为空 //log 13 | return; 14 | } 15 | leveldb::Env *env = leveldb::Env::Default(); 16 | //先将map中的record转化为ector 17 | std::string jsonstr = json11::Json(router_).dump(); 18 | auto s = leveldb::WriteStringToFile(env,jsonstr,name); 19 | if(!s.ok()){ 20 | //TO-DO log 21 | } 22 | 23 | } 24 | 25 | void RouteTable::AddRecord(const std::string &s, const std::string &l, const std::string group) { 26 | //暂时先这样，后面需要加入一些判断 27 | assert(s.compare(l)<=0); 28 | //还需检查是否有重叠部分 29 | router_[l] = Record(s,l,group); 30 | 31 | } 32 | 33 | void RouteTable::ReadRouter(const std::string &name) { 34 | leveldb::Env *env = leveldb::Env::Default(); 35 | std::string content; 36 | auto s = leveldb::ReadFileToString(env,name,&content); 37 | if(!s.ok()){ 38 | //LOG 39 | return; 40 | } 41 | std::string error; 42 | auto parse = json11::Json::parse(content,error,json11::JsonParse::STANDARD); 43 | if(!error.empty()){ 44 | //LOG ERROR 45 | return; 46 | } 47 | router_.clear(); 48 | for(const auto & v:parse.object_items()){ 49 | router_[v.first] = {v.first,v.second["limit"].string_value(),v.second["group"].string_value()}; 50 | } 51 | 52 | } 53 | 54 | const std::string & RouteTable::FindRegion(const std::string &key) const { 55 | if (router_.size()==0) 56 | return NULL ; 57 | //("“, a),[a,c),[c,e),[e,"") 58 | //因为我们使用limit作为标识，所以直接通过upper_bound来找到和是的region 59 | //假设key为b,查找发现位于第二个region，因为bsecond.group; 64 | 65 | //没有找到合适的上界，检查是否有无穷的limit 66 | if(router_.find("")!=router_.end()) { 67 | //有limit为无穷大的region 68 | assert(key>router_[""].smallest); 69 | return router_.find(key)->second.group; 70 | //需要判断 71 | } 72 | return NULL; 73 | } 74 | const std::string & RouteTable::operator[](const std::string &key) const { 75 | //return router_.find(key) == router_.end()?NULL:router_.find(key)->second.group; 76 | if(router_.find(key)==router_.end()) 77 | return NULL; 78 | return router_.find(key)->second.group; 79 | } 80 | 81 | }//namespace pidb 82 | -------------------------------------------------------------------------------- /src/server/route_table.h: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-4. 3 | // 4 | 5 | #ifndef PIDB_ROUTE_TABLE_H 6 | #define PIDB_ROUTE_TABLE_H 7 | #include 8 | #include 9 | #include 10 | 11 | //用与存储 region的路由信息 12 | // 对于每一个region,由最大的key来标识，对于该region所存储的值必须不小于smallest 13 | //这样的设计方便查找key所处的范围 14 | //如果smallest = “” 代表最小的key ，largest=“” 代表最大的值 15 | 16 | 17 | namespace pidb{ 18 | 19 | class RouteTable { 20 | public: 21 | RouteTable() = default; 22 | 23 | RouteTable(const RouteTable&) = delete; 24 | RouteTable& operator =(const RouteTable &)= delete; 25 | 26 | ~RouteTable() = default; 27 | 28 | void BuildRouteTable(const std::string &name) const; 29 | void AddRecord(const std::string &s,const std::string &l,std::string group); 30 | void SplitRecord(const std::string &a,const std::string &newKey); 31 | void ReadRouter(const std::string &name); 32 | const std::string &FindRegion(const std::string &key) const; 33 | const std::string &operator [](const std::string &key) const; 34 | 35 | 36 | private: 37 | //使用maps 来代表routetable的取值范围 38 | struct Record{ 39 | std::string smallest; 40 | std::string limit; 41 | std::string group; 42 | Record(){} 43 | Record(const std::string &s,const std::string &l,const std::string &g) 44 | :smallest(std::move(s)),limit(std::move(l)),group(std::move(g)){} 45 | json11::Json to_json() const{ 46 | return json11::Json::object{{"smallest",smallest},{"limit",limit},{"group",group}}; 47 | } 48 | 49 | }; 50 | std::map router_; // limitkey -> smallest,regionGrou 51 | std::vector routers_; 52 | }; 53 | }// namespace pidb 54 | 55 | #endif //PIDB_ROUTE_TABLE_H 56 | -------------------------------------------------------------------------------- /src/server/server.h: -------------------------------------------------------------------------------- 1 | //控制raftnode，并且回报情况给master 2 | #ifndef PIDB_SERVER_H 3 | #define PIDB_SERVER_H 4 | #include 5 | #include 6 | #include 7 | #include //srever 8 | #include "context_cache.h" 9 | #include "shareddb.h" 10 | #include "braft/repeated_timer_task.h" 11 | #include "pidb/status.h" 12 | #include "pidb_service_impl.h" 13 | #include "master.pb.h" 14 | #include "pidb/options.h" 15 | 16 | 17 | namespace pidb{ 18 | class RaftNode; 19 | struct Range; 20 | class Server; 21 | class ServerClosure: public braft::Closure{ 22 | public: 23 | ServerClosure(PiDBResponse* response, 24 | google::protobuf::Closure* done,std::vector groups); 25 | 26 | ~ServerClosure(){} 27 | PiDBResponse* response() const {return response_;} 28 | void SetDone(const std::string &group); 29 | bool IsDone(); 30 | 31 | //Run用于判断batchs的操作是否已经全部完成 32 | void Run(); 33 | Status s_; 34 | private: 35 | //TODO 更一般化的形式 36 | std::mutex mutex_; 37 | std::atomic count_; 38 | std::unordered_map batchs; 39 | PiDBResponse* response_; 40 | google::protobuf::Closure* done_; 41 | }; 42 | 43 | 44 | class ServerTimer:public braft::RepeatedTimerTask{ 45 | public: 46 | ServerTimer():server_(nullptr){} 47 | int init(std::shared_ptr server, int timeout_ms); 48 | virtual void run() = 0; 49 | 50 | protected: 51 | void on_destroy(); 52 | std::shared_ptr server_; 53 | }; 54 | 55 | class HeartbeatTimer:public ServerTimer{ 56 | protected: 57 | void run(); 58 | }; 59 | 60 | 61 | class Server:public std::enable_shared_from_this{ 62 | public: 63 | //operation for write 64 | 65 | explicit Server(const ServerOption &serveroption); 66 | //no copy and = 67 | Server(const Server&) = delete; 68 | Server& operator = (const Server&) = delete; 69 | 70 | Status Start(); 71 | Status Stop(); 72 | 73 | //TO-DO 获得server的信息 要先定义一个获得信息的handler 74 | class InfoHandler; 75 | void getServerInfo(InfoHandler * handler) const; 76 | 77 | Status registerRaftNode(const RaftOption &option,const Range &range); 78 | Status registerRaftNode(const RaftOption &option,const Range &range,::pidb::PiDBRaftManageResponse *response,::google::protobuf::Closure *done); 79 | Status removeRaftNode(const RaftOption &option); 80 | 81 | //处理Put请求，需要转发到raft节点 82 | void Put(const ::pidb::PiDBRequest* request, 83 | ::pidb::PiDBResponse* response, 84 | ::google::protobuf::Closure* done); 85 | 86 | 87 | //处理用户的Get请求,server可以直接处理 88 | Status Get(const ::pidb::PiDBRequest* request, 89 | ::pidb::PiDBResponse* response, 90 | ::google::protobuf::Closure* done); 91 | 92 | //处理用户发过来的write的请求 93 | void Write(const ::pidb::PiDBWriteBatch* request, 94 | ::pidb::PiDBResponse* response, 95 | ::google::protobuf::Closure* done); 96 | 97 | 98 | //Get Snapshot 99 | //return unique id 用于标识snapshot的id 100 | int64_t GetSnapshot(); 101 | //当使用完snapshot的时候需要释放snapshot 102 | Status ReleaseSnapshot(int64_t id); 103 | 104 | 105 | int64_t GetIterator(const std::string &start, const std::string &end); 106 | Status ReleaseIterator(int64_t id); 107 | Status Next(int64_t id,std::string* value); 108 | 109 | void HandleHeartbeat(); 110 | 111 | void Recover(); 112 | void DestroyServer(); 113 | 114 | // 给master发送心跳信息 115 | void Heartbeat(); 116 | void HandleRaftManage( const ::pidb::PiDBRaftManageRequest *request, 117 | ::pidb::PiDBRaftManageResponse *response, 118 | ::google::protobuf::Closure *done); 119 | 120 | static void StartRaftCallback(::pidb::PiDBRaftManageResponse *response,::google::protobuf::Closure *done,std::shared_ptr raft); 121 | 122 | ~Server(){} 123 | 124 | private: 125 | //用于存储用户存储的snapshotContext 126 | pidb::ContextCache snapshots_; 127 | pidb::ContextCache iterators_; 128 | ServerOption option_; 129 | int32_t port_; 130 | scoped_db db_; 131 | //可能需要换一种数据结构,暂时用map代替 132 | //TODO 一个Server里面的group只能一个raft节点？？？ 133 | std::map> nodes_; 134 | std::string data_path_; 135 | 136 | HeartbeatTimer hearbeat_timer_; 137 | }; 138 | }//namespace pidb 139 | 140 | #endif // STORAGE_LEVELDB_DB_FILENAME_H_ 141 | -------------------------------------------------------------------------------- /src/server/shareddb.h: -------------------------------------------------------------------------------- 1 | #ifndef PIDB_SHARED_DB_H_ 2 | #define PIDB_SHARED_DB_H_ 3 | #include // braft::SnapshotWriter 4 | #include 5 | //使用自带的引用计数，将其独立成文件，因为其他地方也可能用到 6 | //考虑后面能否直接用shared_ptr代替 7 | namespace pidb 8 | { 9 | class SharedDB : public butil::RefCountedThreadSafe { 10 | public: 11 | explicit SharedDB(leveldb::DB* db) : _db(db) {} 12 | leveldb::DB* db() const { return _db; } 13 | private: 14 | friend class butil::RefCountedThreadSafe; 15 | ~SharedDB() { 16 | if (_db!=nullptr) { 17 | delete _db; 18 | } 19 | } 20 | leveldb::DB* _db; 21 | }; 22 | typedef scoped_refptr scoped_db; 23 | 24 | } // pidb 25 | 26 | #endif -------------------------------------------------------------------------------- /src/util/status.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include 6 | #include "pidb/status.h" 7 | 8 | namespace pidb { 9 | 10 | const char* Status::CopyState(const char* state) { 11 | uint32_t size; 12 | memcpy(&size, state, sizeof(size)); 13 | char* result = new char[size + 5]; 14 | memcpy(result, state, size + 5); 15 | return result; 16 | } 17 | 18 | Status::Status(Code code, const Slice& msg, const Slice& msg2) { 19 | assert(code != kOk); 20 | const uint32_t len1 = msg.size(); 21 | const uint32_t len2 = msg2.size(); 22 | const uint32_t size = len1 + (len2 ? (2 + len2) : 0); 23 | char* result = new char[size + 5]; 24 | memcpy(result, &size, sizeof(size)); 25 | result[4] = static_cast(code); 26 | memcpy(result + 5, msg.data(), len1); 27 | if (len2) { 28 | result[5 + len1] = ':'; 29 | result[6 + len1] = ' '; 30 | memcpy(result + 7 + len1, msg2.data(), len2); 31 | } 32 | state_ = result; 33 | } 34 | 35 | std::string Status::ToString() const { 36 | if (state_ == nullptr) { 37 | return "OK"; 38 | } else { 39 | char tmp[30]; 40 | const char* type; 41 | switch (code()) { 42 | case kOk: 43 | type = "OK"; 44 | break; 45 | case kNotFound: 46 | type = "NotFound: "; 47 | break; 48 | case kCorruption: 49 | type = "Corruption: "; 50 | break; 51 | case kNotSupported: 52 | type = "Not implemented: "; 53 | break; 54 | case kInvalidArgument: 55 | type = "Invalid argument: "; 56 | break; 57 | case kIOError: 58 | type = "IO error: "; 59 | break; 60 | default: 61 | snprintf(tmp, sizeof(tmp), "Unknown code(%d): ", 62 | static_cast(code())); 63 | type = tmp; 64 | break; 65 | } 66 | std::string result(type); 67 | uint32_t length; 68 | memcpy(&length, state_, sizeof(length)); 69 | result.append(state_ + 5, length); 70 | return result; 71 | } 72 | } 73 | 74 | } // namespace leveldb 75 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | include_directories(${CMAKE_CURRENT_BINARY_DIR}) 3 | include_directories(${CMAKE_SOURCE_DIR}/test) 4 | 5 | find_path(GTEST_HEADER NAMES gtest/gtest.h) 6 | find_library(GTEST_LIB NAMES gtest) 7 | find_library(GTEST_MAIN_LIB NAMES gtest_main) 8 | 9 | file(GLOB TEST_PIDB_SRCS "test_*.cpp") 10 | 11 | foreach(PIDB_UT ${TEST_PIDB_SRCS}) 12 | get_filename_component(PIDB_UT_WE ${PIDB_UT} NAME_WE) 13 | add_executable(${PIDB_UT_WE} ${PIDB_UT}) 14 | target_link_libraries( ${PIDB_UT_WE} 15 | 16 | "-Xlinker \"-(\"" 17 | ${GTEST_MAIN_LIB} 18 | ${GTEST_LIB} 19 | pidb 20 | dl 21 | "-Xlinker \"-)\"") 22 | endforeach() 23 | 24 | -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # test 2 | 3 | # 关于测试 4 | 1. 请将所有的单元测试写在根目录的test文件夹下 5 | 2. 使用gtest的测试框架，具体请参考gtest的用法 6 | 3. 注意一些包的依赖，比如我要测试pidb server的一些功能， 7 | 要链接pidb的库文件 8 | 9 | 10 | -------------------------------------------------------------------------------- /test/test_context_cache.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-17. 3 | // 4 | #include "gtest/gtest.h" 5 | #include "leveldb/db.h" 6 | #include "context_cache.h" 7 | 8 | TEST(ContextTest,put_and_get_test){ 9 | leveldb::DB* db; 10 | leveldb::Options options; 11 | options.create_if_missing = true; 12 | leveldb::Status status = leveldb::DB::Open(options, "/tmp/testdb", &db); 13 | pidb::ContextCache a; 14 | 15 | std::unique_ptr s (new pidb::SnapshotContext (db->GetSnapshot())); 16 | auto id = a.Put(std::move(s)); 17 | EXPECT_TRUE(id==0); 18 | auto snapshot = a.Get(id); 19 | EXPECT_TRUE(snapshot!= nullptr); 20 | db->ReleaseSnapshot(snapshot.get()->Get()); 21 | a.Erase(id); 22 | EXPECT_TRUE(a.Get(id)== nullptr); 23 | } 24 | -------------------------------------------------------------------------------- /test/test_push_file.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 19-5-17. 3 | // 4 | #include "gtest/gtest.h" 5 | #include "leveldb/db.h" 6 | #include "remote_file_send.h" 7 | 8 | TEST(RemoteFile,push_file){ 9 | ASSERT_TRUE(1); 10 | } 11 | -------------------------------------------------------------------------------- /test/test_route_table.cpp: -------------------------------------------------------------------------------- 1 | #include "gtest/gtest.h" 2 | #include "route_table.h" 3 | 4 | //简单测试 5 | TEST(FooTest, Test1){ 6 | pidb::RouteTable r; 7 | r.AddRecord("","a","group1"); 8 | r.AddRecord("c","d","group2"); 9 | r.AddRecord("d","f","group3"); 10 | std::string a = r.FindRegion("abc"); 11 | EXPECT_TRUE(a=="group2"); 12 | auto b = r.FindRegion("1"); 13 | EXPECT_TRUE(b=="group1"); 14 | } 15 | -------------------------------------------------------------------------------- /test/test_timer.cpp: -------------------------------------------------------------------------------- 1 | // 2 | // Created by ehds on 5/31/19. 3 | // 4 | 5 | -------------------------------------------------------------------------------- /utils/README.md: -------------------------------------------------------------------------------- 1 | # util 2 | 3 | 4 | --------------------------------------------------------------------------------