├── .gitignore ├── CMakeLists.txt ├── LICENSE ├── NOTICE ├── README.md ├── cmake ├── FindFolly.cmake └── FindZooKeeper.cmake └── src ├── Bookie.cpp ├── Bookie.h ├── BookieCodecV2.cpp ├── BookieCodecV2.h ├── BookieConfig.cpp ├── BookieConfig.h ├── BookieHandler.cpp ├── BookieHandler.h ├── BookiePipeline.cpp ├── BookiePipeline.h ├── BookieProtocol.cpp ├── BookieProtocol.h ├── BookieRegistration.cpp ├── BookieRegistration.h ├── Logging.cpp ├── Logging.h ├── Metrics-inl.h ├── Metrics.cpp ├── Metrics.h ├── RateLimiter.h ├── Storage.cpp ├── Storage.h ├── ZooKeeper.cpp ├── ZooKeeper.h ├── main.cpp └── perfClient.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | .project 3 | .cproject 4 | bookie 5 | perfClient 6 | .settings 7 | *.o 8 | CMakeCache.txt 9 | CMakeFiles 10 | Makefile 11 | cmake_install.cmake 12 | data/ 13 | wal/ 14 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project (bookie-cpp) 3 | 4 | set(CMAKE_CXX_FLAGS "-std=c++14 -g0 -O3 -march=core2 -pthread") 5 | set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/) 6 | 7 | find_package(Folly REQUIRED) 8 | find_package(Boost REQUIRED COMPONENTS program_options filesystem regex) 9 | find_package(OpenSSL REQUIRED) 10 | find_package(Threads REQUIRED) 11 | find_library(GLOG_LIBRARY_PATH glog) 12 | find_library(GFLAGS_LIBRARY_PATH gflags) 13 | 14 | find_package(ZooKeeper REQUIRED) 15 | find_library(LOG4CXX_LIBRARY_PATH log4cxx) 16 | find_library(ROCKSDB_LIBRARY_PATH rocksdb) 17 | find_library(WANGLE_LIBRARY_PATH wangle) 18 | find_library(JEMALLOC_LIBRARY_PATH jemalloc) 19 | find_library(Z_LIBRARY_PATH z) 20 | find_library(LZ4_LIBRARY_PATH lz4) 21 | find_library(BZ2_LIBRARY_PATH bz2) 22 | 23 | include_directories( 24 | ${CMAKE_SOURCE_DIR}/.. 25 | ${FOLLY_INCLUDE_DIR} 26 | ${OPENSSL_INCLUDE_DIR} 27 | ${INCLUDE_DIR} 28 | ) 29 | 30 | set(BOOKIE_SOURCES 31 | src/Bookie.cpp 32 | src/BookieCodecV2.cpp 33 | src/BookieConfig.cpp 34 | src/BookieHandler.cpp 35 | src/BookiePipeline.cpp 36 | src/BookieProtocol.cpp 37 | src/BookieRegistration.cpp 38 | src/Logging.cpp 39 | src/Storage.cpp 40 | src/ZooKeeper.cpp 41 | src/Metrics.cpp 42 | src/main.cpp 43 | ) 44 | 45 | add_executable(bookie ${BOOKIE_SOURCES}) 46 | 47 | set(COMMON_LIBS 48 | ${FOLLY_LIBRARIES} 49 | ${LOG4CXX_LIBRARIES} 50 | ${Boost_LIBRARIES} 51 | ${OPENSSL_LIBRARIES} 52 | ${GLOG_LIBRARY_PATH} 53 | ${GFLAGS_LIBRARY_PATH} 54 | ${LOG4CXX_LIBRARY_PATH} 55 | ${WANGLE_LIBRARY_PATH} 56 | ${JEMALLOC_LIBRARY_PATH} 57 | ${Z_LIBRARY_PATH} 58 | ${LZ4_LIBRARY_PATH} 59 | ${BZ2_LIBRARY_PATH} 60 | ) 61 | 62 | if (NOT APPLE) 63 | set(COMMON_LIBS ${COMMON_LIBS} rt atomic) 64 | endif() 65 | 66 | target_link_libraries(bookie 67 | ${COMMON_LIBS} 68 | ${ROCKSDB_LIBRARY_PATH} 69 | ${Zookeeper_LIBRARY} 70 | ) 71 | 72 | # Test tool 73 | 74 | set(PERF_CLIENT_SOURCES 75 | src/perfClient.cpp 76 | src/Logging.cpp 77 | src/Metrics.cpp 78 | src/BookieCodecV2.cpp 79 | src/BookieProtocol.cpp 80 | ) 81 | 82 | add_executable(perfClient ${PERF_CLIENT_SOURCES}) 83 | target_link_libraries(perfClient ${COMMON_LIBS}) 84 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Apache BookKeeper 2 | Copyright 2011-2016 The Apache Software Foundation 3 | 4 | This product includes software developed at 5 | The Apache Software Foundation (http://www.apache.org/). 6 | 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bookie-cpp 2 | 3 | Proof of concept C++ server implementing Apache BookKeeper server functionality 4 | 5 | ### Dependencies 6 | * C++ 14 compiler 7 | * [Folly](https://github.com/facebook/folly/) 8 | * [Wangle](https://github.com/facebook/wangle/) 9 | * [ZooKeeper C client](https://zookeeper.apache.org/) 10 | * [Log4cxx](https://logging.apache.org/log4cxx/) 11 | 12 | ### Compile 13 | ```shell 14 | cmake . 15 | make 16 | ``` 17 | 18 | ### Run the bookie 19 | 20 | ``` 21 | ./bookie -h 22 | Allowed options: 23 | -h [ --help ] This help message 24 | -z [ --zkServers ] arg (=localhost:2181) List of ZooKeeper servers 25 | --zkSessionTimeout arg (=30000) ZooKeeper session timeout 26 | --bookieHost arg (=localhost) Boookie hostname 27 | -p [ --bookiePort ] arg (=3181) Bookie TCP port 28 | -d [ --dataDir ] arg (=./data) Location where to store data 29 | -w [ --walDir ] arg (=./wal) Location where to put RocksDB Write-ahead-log 30 | -s [ --fsyncWal ] arg (=1) Fsync the WAL before acking the entry 31 | -r [ --statsReportingIntervalSeconds ] arg (=60) Interval for stats reporting 32 | ``` 33 | 34 | Test client 35 | 36 | ``` 37 | ./perfClient -h 38 | -h [ --help ] This help message 39 | -a [ --bookieAddress ] arg (=localhost:3181) 40 | Boookie hostname and port 41 | -r [ --rate ] arg (=100) Add entry rate 42 | -s [ --msg-size ] arg (=1024) Message size 43 | -c [ --num-connections ] arg (=16) Number of connections 44 | --format-stats arg (=1) Format stats JSON output 45 | --stats-reporting arg (=10) Interval to report latency stats in 46 | seconds 47 | ``` 48 | -------------------------------------------------------------------------------- /cmake/FindFolly.cmake: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2014, Facebook, Inc. 2 | # All rights reserved. 3 | # 4 | # This source code is licensed under the BSD-style license found in the 5 | # LICENSE file in the root directory of this source tree. An additional grant 6 | # of patent rights can be found in the PATENTS file in the same directory. 7 | # 8 | # - Try to find folly 9 | # This will define 10 | # FOLLY_FOUND 11 | # FOLLY_INCLUDE_DIR 12 | # FOLLY_LIBRARIES 13 | 14 | CMAKE_MINIMUM_REQUIRED(VERSION 2.8.7 FATAL_ERROR) 15 | 16 | INCLUDE(FindPackageHandleStandardArgs) 17 | 18 | FIND_LIBRARY(FOLLY_LIBRARY folly PATHS ${FOLLY_LIBRARYDIR}) 19 | FIND_PATH(FOLLY_INCLUDE_DIR "folly/String.h" PATHS ${FOLLY_INCLUDEDIR}) 20 | 21 | SET(FOLLY_LIBRARIES ${FOLLY_LIBRARY}) 22 | 23 | FIND_PACKAGE_HANDLE_STANDARD_ARGS(Folly 24 | REQUIRED_ARGS FOLLY_INCLUDE_DIR FOLLY_LIBRARIES) 25 | -------------------------------------------------------------------------------- /cmake/FindZooKeeper.cmake: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one 3 | # or more contributor license agreements. See the NOTICE file 4 | # distributed with this work for additional information 5 | # regarding copyright ownership. The ASF licenses this file 6 | # to you under the Apache License, Version 2.0 (the 7 | # "License"); you may not use this file except in compliance 8 | # with the License. You may obtain a copy of the License at 9 | # 10 | # http://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | # - Try to find Zookeeper 20 | # Defines 21 | # Zookeeper_FOUND - System has Zookeeper 22 | # Zookeeper_INCLUDE_DIRS - The Zookeeper include directories 23 | # Zookeeper_LIBRARIES - The libraries needed to use Zookeeper 24 | # Zookeeper_DEFINITIONS - Compiler switches required for using LibZookeeper 25 | 26 | #find_package(PkgConfig) 27 | #pkg_check_modules(PC_LIBXML QUIET libxml-2.0) 28 | #set(Zookeeper_DEFINITIONS ${PC_LIBXML_CFLAGS_OTHER}) 29 | 30 | if (MSVC) 31 | if(${CMAKE_BUILD_TYPE} MATCHES "Debug") 32 | set(ZK_BuildOutputDir "Debug") 33 | else() 34 | set(ZK_BuildOutputDir "Release") 35 | endif() 36 | if("${ZOOKEEPER_HOME}_" MATCHES "^_$") 37 | message(" ") 38 | message("- Please set the cache variable ZOOKEEPER_HOME to point to the directory with the zookeeper source.") 39 | message("- CMAKE will look for zookeeper include files in $ZOOKEEPER_HOME/src/c/include.") 40 | message("- CMAKE will look for zookeeper library files in $ZOOKEEPER_HOME/src/c/Debug or $ZOOKEEPER_HOME/src/c/Release.") 41 | else() 42 | FILE(TO_CMAKE_PATH ${ZOOKEEPER_HOME} Zookeeper_HomePath) 43 | set(Zookeeper_LIB_PATHS ${Zookeeper_HomePath}/src/c/${ZK_BuildOutputDir} ${Zookeeper_HomePath}/src/c/x64/${ZK_BuildOutputDir} ) 44 | 45 | find_path(ZK_INCLUDE_DIR zookeeper.h ${Zookeeper_HomePath}/src/c/include) 46 | find_path(ZK_INCLUDE_DIR_GEN zookeeper.jute.h ${Zookeeper_HomePath}/src/c/generated) 47 | set(Zookeeper_INCLUDE_DIR zookeeper.h ${ZK_INCLUDE_DIR} ${ZK_INCLUDE_DIR_GEN} ) 48 | find_library(Zookeeper_LIBRARY NAMES zookeeper PATHS ${Zookeeper_LIB_PATHS}) 49 | endif() 50 | else() 51 | set(Zookeeper_LIB_PATHS /usr/local/lib /opt/local/lib) 52 | find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include) 53 | find_library(Zookeeper_LIBRARY NAMES zookeeper_mt PATHS ${Zookeeper_LIB_PATHS}) 54 | endif() 55 | 56 | 57 | set(Zookeeper_LIBRARIES ${Zookeeper_LIBRARY} ) 58 | set(Zookeeper_INCLUDE_DIRS ${Zookeeper_INCLUDE_DIR} ) 59 | 60 | include(FindPackageHandleStandardArgs) 61 | # handle the QUIETLY and REQUIRED arguments and set Zookeeper_FOUND to TRUE 62 | # if all listed variables are TRUE 63 | find_package_handle_standard_args(Zookeeper DEFAULT_MSG 64 | Zookeeper_LIBRARY Zookeeper_INCLUDE_DIR) 65 | 66 | mark_as_advanced(Zookeeper_INCLUDE_DIR Zookeeper_LIBRARY ) -------------------------------------------------------------------------------- /src/Bookie.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Bookie.h" 22 | #include "Logging.h" 23 | 24 | #include 25 | #include 26 | #include 27 | 28 | DECLARE_LOG_OBJECT(); 29 | 30 | Bookie::Bookie(const BookieConfig& conf) : 31 | conf_(conf), 32 | metricsManager_(conf.statsReportingInterval()), 33 | zk_(conf.zkServers(), milliseconds(conf.zkSessionTimeout())), 34 | bookieRegistration_(&zk_, conf), 35 | storage_(conf, metricsManager_) { 36 | server_.childPipeline(std::make_shared(*this)); 37 | } 38 | 39 | void Bookie::start() { 40 | SocketAddress bookieAddress("0.0.0.0", conf_.bookiePort()); 41 | LOG_INFO("Starting bookie on " << bookieAddress); 42 | server_.bind(bookieAddress); 43 | 44 | zk_.startSession(); 45 | LOG_INFO("Started bookie"); 46 | } 47 | 48 | void Bookie::stop() { 49 | server_.stop(); 50 | } 51 | 52 | void Bookie::waitForStop() { 53 | server_.waitForStop(); 54 | } 55 | 56 | BookieHandler Bookie::newHandler() { 57 | return BookieHandler(*this, metricsManager_); 58 | } 59 | 60 | Future Bookie::addEntry(int64_t ledgerId, int64_t entryId, IOBufPtr data) { 61 | return storage_.put(ledgerId, entryId, std::move(data)); 62 | } 63 | 64 | Future Bookie::getLastEntry(int64_t ledgerId) { 65 | } 66 | 67 | Future Bookie::readEntry(int64_t ledgerId, int64_t entryId) { 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/Bookie.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | 26 | #include "BookiePipeline.h" 27 | #include "BookieRegistration.h" 28 | #include "ZooKeeper.h" 29 | #include "BookieHandler.h" 30 | #include "BookieConfig.h" 31 | #include "Metrics.h" 32 | #include "Storage.h" 33 | 34 | using namespace wangle; 35 | 36 | class Bookie { 37 | public: 38 | explicit Bookie(const BookieConfig& conf); 39 | 40 | void start(); 41 | 42 | void stop(); 43 | 44 | void waitForStop(); 45 | 46 | BookieHandler newHandler(); 47 | 48 | Future addEntry(int64_t ledgerId, int64_t entryId, IOBufPtr data); 49 | 50 | Future getLastEntry(int64_t ledgerId); 51 | 52 | Future readEntry(int64_t ledgerId, int64_t entryId); 53 | 54 | private: 55 | const BookieConfig& conf_; 56 | MetricsManager metricsManager_; 57 | ServerBootstrap server_; 58 | 59 | ZooKeeper zk_; 60 | BookieRegistration bookieRegistration_; 61 | Storage storage_; 62 | }; 63 | 64 | -------------------------------------------------------------------------------- /src/BookieCodecV2.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "BookieCodecV2.h" 22 | 23 | #include "Logging.h" 24 | 25 | DECLARE_LOG_OBJECT(); 26 | 27 | struct PacketHeader { 28 | int8_t version; 29 | BookieOperation opCode; 30 | int16_t flags; 31 | 32 | static PacketHeader fromInt(int value) { 33 | PacketHeader hdr; 34 | hdr.version = value >> 24; 35 | hdr.opCode = (BookieOperation) ((value >> 16) & 0xFF); 36 | hdr.flags = value & 0xFF; 37 | return hdr; 38 | } 39 | 40 | int toInt() const { 41 | return ((version & 0xFF) << 24) | (((int8_t) opCode & 0xFF) << 16) | ((int16_t) flags & 0xFFFF); 42 | } 43 | }; 44 | 45 | void BookieServerCodecV2::read(Context* ctx, IOBufPtr buf) { 46 | if (!buf) { 47 | return; 48 | } 49 | 50 | io::Cursor reader { buf.get() }; 51 | if (reader.totalLength() < sizeof(int32_t)) { 52 | // Short request 53 | ctx->fireClose(); 54 | return; 55 | } 56 | 57 | Request request; 58 | PacketHeader hdr = PacketHeader::fromInt(reader.readBE()); 59 | request.protocolVersion = hdr.version; 60 | request.opCode = hdr.opCode; 61 | request.flags = hdr.flags; 62 | 63 | switch (request.opCode) { 64 | case BookieOperation::AddEntry: 65 | static const int32_t addRequestSize = BookieConstant::MasterKeyLength + 2 * sizeof(int64_t); 66 | if (reader.totalLength() < addRequestSize) { 67 | LOG_WARN( 68 | "Invalid add entry request size: " << reader.totalLength() << " -- expecting at least: " << addRequestSize); 69 | ctx->fireClose(); 70 | return; 71 | } 72 | reader.skip(BookieConstant::MasterKeyLength); 73 | request.ledgerId = reader.readBE(); 74 | request.entryId = reader.readBE(); 75 | 76 | reader.clone(request.data, reader.totalLength()); 77 | break; 78 | 79 | case BookieOperation::ReadEntry: { 80 | const int32_t readRequestSize = 2 * sizeof(int64_t) 81 | + (request.isFencing() ? BookieConstant::MasterKeyLength : 0); 82 | if (reader.totalLength() < readRequestSize) { 83 | LOG_WARN( 84 | "Invalid read entry request size: " << reader.totalLength() << " -- expecting: " << readRequestSize); 85 | ctx->fireClose(); 86 | return; 87 | } 88 | 89 | request.ledgerId = reader.readBE(); 90 | request.entryId = reader.readBE(); 91 | 92 | if (request.isFencing()) { 93 | // Fencing reads will provide the master key which we'll ignore 94 | reader.skip(BookieConstant::MasterKeyLength); 95 | } 96 | break; 97 | } 98 | case BookieOperation::Auth: 99 | break; 100 | } 101 | 102 | LOG_DEBUG("Deserialized request: " << request); 103 | ctx->fireRead(std::move(request)); 104 | } 105 | 106 | Future BookieServerCodecV2::write(Context* ctx, Response response) { 107 | LOG_DEBUG("Serializing response: " << response); 108 | 109 | const int headerSize = 4 + 24; 110 | const int frameSize = headerSize /* + sizeof(data) */; 111 | const int bufferSize = frameSize + 4; 112 | IOBufPtr buf = IOBuf::create(bufferSize); 113 | buf->append(bufferSize); 114 | 115 | PacketHeader pktHeader { response.protocolVersion, response.opCode, 0 }; 116 | io::RWPrivateCursor writer(buf.get()); 117 | 118 | writer.writeBE(frameSize); 119 | writer.writeBE(pktHeader.toInt()); 120 | 121 | switch (response.opCode) { 122 | case BookieOperation::AddEntry: 123 | writer.writeBE((int32_t) response.errorCode); 124 | writer.writeBE(response.ledgerId); 125 | writer.writeBE(response.entryId); 126 | break; 127 | 128 | case BookieOperation::ReadEntry: 129 | writer.writeBE((int32_t) response.errorCode); 130 | writer.writeBE(response.ledgerId); 131 | writer.writeBE(response.entryId); 132 | 133 | if (response.data) { 134 | // Also write data 135 | // buf-> 136 | } 137 | 138 | break; 139 | 140 | case BookieOperation::Auth: 141 | break; 142 | } 143 | 144 | return ctx->fireWrite(std::move(buf)); 145 | } 146 | 147 | void BookieClientCodecV2::read(Context* ctx, IOBufPtr buf) { 148 | if (!buf) { 149 | return; 150 | } 151 | 152 | io::Cursor reader { buf.get() }; 153 | 154 | LOG_DEBUG("Received response: len=" << reader.totalLength()); 155 | if (reader.totalLength() < sizeof(int32_t)) { 156 | // Short request 157 | ctx->fireClose(); 158 | return; 159 | } 160 | 161 | Response response; 162 | PacketHeader hdr = PacketHeader::fromInt(reader.readBE()); 163 | response.protocolVersion = hdr.version; 164 | response.opCode = hdr.opCode; 165 | 166 | switch (response.opCode) { 167 | case BookieOperation::AddEntry: 168 | response.errorCode = (BookieError) reader.readBE(); 169 | response.ledgerId = reader.readBE(); 170 | response.entryId = reader.readBE(); 171 | break; 172 | 173 | case BookieOperation::ReadEntry: { 174 | response.errorCode = (BookieError) reader.readBE(); 175 | response.ledgerId = reader.readBE(); 176 | response.entryId = reader.readBE(); 177 | // TODO 178 | break; 179 | } 180 | case BookieOperation::Auth: 181 | // TODO 182 | break; 183 | } 184 | 185 | LOG_DEBUG("Deserialized response: " << response); 186 | ctx->fireRead(std::move(response)); 187 | } 188 | 189 | Future BookieClientCodecV2::write(Context* ctx, Request request) { 190 | LOG_DEBUG("Serializing request: " << request); 191 | 192 | constexpr int headerSize = sizeof(int32_t) + BookieConstant::MasterKeyLength + 2 * sizeof(int64_t); 193 | const int frameSize = headerSize + request.data->length(); 194 | const int bufferSize = headerSize + 4; 195 | 196 | IOBufPtr buffer = IOBuf::create(bufferSize); 197 | buffer->append(bufferSize); 198 | 199 | PacketHeader pktHeader { request.protocolVersion, request.opCode, request.flags }; 200 | io::RWPrivateCursor writer(buffer.get()); 201 | 202 | writer.writeBE(frameSize); 203 | writer.writeBE(pktHeader.toInt()); 204 | 205 | switch (request.opCode) { 206 | case BookieOperation::AddEntry: 207 | writer.skip(BookieConstant::MasterKeyLength); 208 | writer.writeBE(request.ledgerId); 209 | writer.writeBE(request.entryId); 210 | writer.insert(std::move(request.data)); 211 | break; 212 | 213 | case BookieOperation::ReadEntry: 214 | // TODO 215 | break; 216 | 217 | case BookieOperation::Auth: 218 | // TODO 219 | break; 220 | } 221 | 222 | return ctx->fireWrite(std::move(buffer)); 223 | } 224 | 225 | -------------------------------------------------------------------------------- /src/BookieCodecV2.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | 25 | #include "BookieProtocol.h" 26 | 27 | using namespace wangle; 28 | using namespace folly; 29 | 30 | /** 31 | * Codec for BookKeeper V2 wire format 32 | */ 33 | class BookieServerCodecV2: public Handler { 34 | public: 35 | void read(Context* ctx, IOBufPtr buf) override; 36 | 37 | Future write(Context* ctx, Response response) override; 38 | }; 39 | 40 | /** 41 | * Codec for BookKeeper V2 wire format 42 | */ 43 | class BookieClientCodecV2: public Handler { 44 | public: 45 | void read(Context* ctx, IOBufPtr buf) override; 46 | 47 | Future write(Context* ctx, Request response) override; 48 | }; 49 | -------------------------------------------------------------------------------- /src/BookieConfig.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "BookieConfig.h" 22 | #include 23 | 24 | #include 25 | 26 | BookieConfig::BookieConfig() : 27 | zkServers_(), 28 | zkSessionTimeout_(0), 29 | bookiePort_(), 30 | dataDirectory_(), 31 | walDirectory_(), 32 | options_("Allowed options", 100) { 33 | 34 | char defaultHostname[256]; 35 | if (gethostname(defaultHostname, sizeof(defaultHostname)) != 0) { 36 | throw std::runtime_error("Failed to get default hostname"); 37 | } 38 | 39 | options_.add_options() // 40 | ("help,h", "This help message") // 41 | ("zkServers,z", po::value(&zkServers_)->default_value("localhost:2181"), "List of ZooKeeper servers") // 42 | ("zkSessionTimeout", po::value(&zkSessionTimeout_)->default_value(30000), "ZooKeeper session timeout") // 43 | ("bookieHost", po::value(&bookieHost_)->default_value(defaultHostname), "Boookie hostname") // 44 | ("bookiePort,p", po::value(&bookiePort_)->default_value(3181), "Bookie TCP port") // 45 | ("dataDir,d", po::value(&dataDirectory_)->default_value("./data"), "Location where to store data") // 46 | ("walDir,w", po::value(&walDirectory_)->default_value("./wal"), 47 | "Location where to put RocksDB Write-ahead-log") // 48 | ("fsyncWal,s", po::value(&fsyncWal_)->default_value(true), "Fsync the WAL before acking the entry") // 49 | 50 | ("statsReportingIntervalSeconds,r", po::value(&statsReportingIntervalSeconds_)->default_value(60), 51 | "Interval for stats reporting") // 52 | // 53 | ; 54 | } 55 | 56 | bool BookieConfig::parse(int argc, char** argv) { 57 | po::variables_map map; 58 | try { 59 | po::store(po::command_line_parser(argc, argv).options(options_).run(), map); 60 | po::notify(map); 61 | 62 | if (map.count("help")) { 63 | std::cerr << options_ << std::endl; 64 | exit(1); 65 | } 66 | 67 | return true; 68 | } 69 | catch (const std::exception& e) { 70 | std::cerr << "Error parsing parameters -- " << e.what() << std::endl << std::endl; 71 | std::cerr << options_ << std::endl; 72 | return false; 73 | } 74 | } 75 | 76 | -------------------------------------------------------------------------------- /src/BookieConfig.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | namespace po = boost::program_options; 28 | 29 | using namespace std::chrono; 30 | 31 | class BookieConfig { 32 | public: 33 | BookieConfig(); 34 | 35 | bool parse(int argc, char** argv); 36 | 37 | const std::string& zkServers() const { 38 | return zkServers_; 39 | } 40 | 41 | int zkSessionTimeout() const { 42 | return zkSessionTimeout_; 43 | } 44 | 45 | const std::string bookieHost() const { 46 | return bookieHost_; 47 | } 48 | 49 | int bookiePort() const { 50 | return bookiePort_; 51 | } 52 | 53 | const std::string& dataDirectory() const { 54 | return dataDirectory_; 55 | } 56 | 57 | const std::string& walDirectory() const { 58 | return walDirectory_; 59 | } 60 | 61 | bool fsyncWal() const { 62 | return fsyncWal_; 63 | } 64 | 65 | seconds statsReportingInterval() const { 66 | return seconds(statsReportingIntervalSeconds_); 67 | } 68 | 69 | private: 70 | std::string zkServers_; 71 | int zkSessionTimeout_; 72 | 73 | std::string bookieHost_; 74 | int bookiePort_; 75 | 76 | std::string dataDirectory_; 77 | std::string walDirectory_; 78 | bool fsyncWal_; 79 | 80 | int statsReportingIntervalSeconds_; 81 | 82 | po::options_description options_; 83 | }; 84 | -------------------------------------------------------------------------------- /src/BookieHandler.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Logging.h" 22 | #include "BookieHandler.h" 23 | #include "Bookie.h" 24 | 25 | DECLARE_LOG_OBJECT(); 26 | 27 | BookieHandler::BookieHandler(Bookie& bookie, MetricsManager& metricsManager) : 28 | bookie_(bookie), 29 | addEntryLatency_(metricsManager.createMetric("addEntry")) { 30 | } 31 | 32 | void BookieHandler::transportActive(Context* ctx) { 33 | ctx->getTransport()->getPeerAddress(&peerAddress_); 34 | LOG_INFO("New connection from " << peerAddress_); 35 | ctx->fireTransportActive(); 36 | } 37 | 38 | void BookieHandler::readEOF(Context* ctx) { 39 | LOG_INFO("Closed connection from " << peerAddress_); 40 | ctx->fireReadEOF(); 41 | } 42 | 43 | void BookieHandler::read(Context* ctx, Request request) { 44 | switch (request.opCode) { 45 | case BookieOperation::AddEntry: 46 | handleAddEntry(ctx, std::move(request)); 47 | break; 48 | 49 | case BookieOperation::ReadEntry: 50 | handleReadEntry(ctx, std::move(request)); 51 | break; 52 | 53 | } 54 | } 55 | 56 | void BookieHandler::handleAddEntry(Context* ctx, Request request) { 57 | int64_t ledgerId = request.ledgerId; 58 | int64_t entryId = request.entryId; 59 | uint64_t entryLength = request.data->length(); 60 | 61 | Clock::time_point start = Clock::now(); 62 | 63 | Future future = bookie_.addEntry(request.ledgerId, request.entryId, std::move(request.data)); // 64 | future.then(ctx->getTransport()->getEventBase(), [=](Unit u) { 65 | LOG_DEBUG("Entry persisted at " << ledgerId << ":" << entryId << " -- size: " << entryLength); 66 | Response response {2, BookieOperation::AddEntry, BookieError::OK, ledgerId, entryId}; 67 | 68 | write(ctx, std::move(response)); 69 | 70 | addEntryLatency_->addLatencySample(Clock::now() - start); 71 | }) // 72 | .onError([=](const std::exception& e) { 73 | LOG_WARN("Failed to persist entry at " << ledgerId << ":" << entryId << " : " << e.what()); 74 | Response response {2, BookieOperation::AddEntry, BookieError::IOError, ledgerId, entryId}; 75 | 76 | write(ctx, std::move(response)); 77 | }); 78 | } 79 | 80 | void BookieHandler::handleReadEntry(Context* ctx, Request request) { 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/BookieHandler.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include "BookieProtocol.h" 24 | #include "Metrics.h" 25 | 26 | #include 27 | #include 28 | 29 | using namespace wangle; 30 | using namespace folly; 31 | 32 | class Bookie; 33 | 34 | class BookieHandler: public HandlerAdapter { 35 | public: 36 | BookieHandler(Bookie& bookie, MetricsManager& metricsManager); 37 | 38 | virtual void transportActive(Context* ctx) override; 39 | 40 | virtual void readEOF(Context* ctx) override; 41 | 42 | virtual void read(Context* ctx, Request request) override; 43 | 44 | private: 45 | void handleAddEntry(Context* ctx, Request request); 46 | void handleReadEntry(Context* ctx, Request request); 47 | 48 | Bookie& bookie_; 49 | SocketAddress peerAddress_; 50 | 51 | MetricPtr addEntryLatency_; 52 | }; 53 | -------------------------------------------------------------------------------- /src/BookiePipeline.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "BookiePipeline.h" 22 | #include "BookieCodecV2.h" 23 | #include "Bookie.h" 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | BookiePipelineFactory::BookiePipelineFactory(Bookie& bookie) : 30 | bookie_(bookie) { 31 | } 32 | 33 | BookiePipeline::Ptr BookiePipelineFactory::newPipeline(std::shared_ptr sock) { 34 | auto pipeline = BookiePipeline::create(); 35 | pipeline->addBack(AsyncSocketHandler(sock)); 36 | pipeline->addBack(LengthFieldBasedFrameDecoder(4, BookieConstant::MaxFrameSize)); 37 | pipeline->addBack(BookieServerCodecV2()); 38 | pipeline->addBack(bookie_.newHandler()); 39 | pipeline->finalize(); 40 | return pipeline; 41 | } 42 | -------------------------------------------------------------------------------- /src/BookiePipeline.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | 25 | #include "BookieProtocol.h" 26 | 27 | using namespace wangle; 28 | using namespace folly; 29 | 30 | typedef Pipeline BookiePipeline; 31 | 32 | class Bookie; 33 | 34 | /** 35 | * Define the processing pipeline for serialize/deserialize bookie commands 36 | */ 37 | class BookiePipelineFactory: public PipelineFactory { 38 | public: 39 | BookiePipelineFactory(Bookie& bookie); 40 | 41 | BookiePipeline::Ptr newPipeline(std::shared_ptr sock) override; 42 | 43 | private: 44 | Bookie& bookie_; 45 | }; 46 | -------------------------------------------------------------------------------- /src/BookieProtocol.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "BookieProtocol.h" 22 | #include 23 | #include 24 | 25 | using namespace folly; 26 | 27 | std::ostream& operator<<(std::ostream& s, BookieOperation op) { 28 | switch (op) { 29 | case BookieOperation::AddEntry: 30 | s << "AddEntry"; 31 | break; 32 | case BookieOperation::ReadEntry: 33 | s << "ReadEntry"; 34 | break; 35 | case BookieOperation::Auth: 36 | s << "Auth"; 37 | break; 38 | default: 39 | s << "Unknown bookie op (" << (int) op << ")"; 40 | break; 41 | } 42 | 43 | return s; 44 | } 45 | 46 | std::ostream& operator<<(std::ostream& s, BookieError error) { 47 | switch (error) { 48 | case BookieError::OK: 49 | s << "OK"; 50 | break; 51 | case BookieError::NoLedger: 52 | s << "NoLedger"; 53 | break; 54 | case BookieError::NoEntry: 55 | s << "NoEntry"; 56 | break; 57 | case BookieError::BadRequest: 58 | s << "BadRequest"; 59 | break; 60 | case BookieError::IOError: 61 | s << "IOError"; 62 | break; 63 | case BookieError::UnauthorizedAccesss: 64 | s << "UnauthorizedAccess"; 65 | break; 66 | case BookieError::BadVersion: 67 | s << "BadVersion"; 68 | break; 69 | case BookieError::Fenced: 70 | s << "Fenced"; 71 | break; 72 | case BookieError::ReadOnly: 73 | s << "ReadOnly"; 74 | break; 75 | case BookieError::TooManyRequests: 76 | s << "TooManyRequests"; 77 | break; 78 | } 79 | 80 | return s; 81 | } 82 | 83 | std::ostream& operator<<(std::ostream& s, const Request& r) { 84 | s << "Request(" // 85 | << "version:" << (int) r.protocolVersion // 86 | << " opCode:" << r.opCode // 87 | << " ledgerId:" << r.ledgerId // 88 | << " entryId:" << r.entryId // 89 | << " flags:0x" << format("{0:04x}", r.flags) // 90 | << " data-len:" << (r.data ? r.data->length() : 0) // 91 | << ")"; 92 | 93 | return s; 94 | } 95 | 96 | std::ostream& operator<<(std::ostream& s, const Response& r) { 97 | s << "Response(" // 98 | << "version:" << (int) r.protocolVersion // 99 | << " opCode:" << r.opCode // 100 | << " error:" << r.errorCode // 101 | << " ledgerId:" << r.ledgerId // 102 | << " entryId:" << r.entryId // 103 | << " data-len:" << (r.data ? r.data->length() : 0) // 104 | << ")"; 105 | 106 | return s; 107 | } 108 | 109 | -------------------------------------------------------------------------------- /src/BookieProtocol.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | 25 | #include 26 | 27 | using folly::IOBuf; 28 | 29 | enum class BookieOperation 30 | : int8_t { 31 | /** 32 | * The Add entry request payload will be a ledger entry exactly as it should 33 | * be logged. The response payload will be a 4-byte integer that has the 34 | * error code followed by the 8-byte ledger number and 8-byte entry number 35 | * of the entry written. 36 | */ 37 | AddEntry = 1, 38 | 39 | /** 40 | * The Read entry request payload will be the ledger number and entry number 41 | * to read. (The ledger number is an 8-byte integer and the entry number is 42 | * a 8-byte integer.) The response payload will be a 4-byte integer 43 | * representing an error code and a ledger entry if the error code is EOK, 44 | * otherwise it will be the 8-byte ledger number and the 4-byte entry number 45 | * requested. (Note that the first sixteen bytes of the entry happen to be 46 | * the ledger number and entry number as well.) 47 | */ 48 | ReadEntry = 2, 49 | 50 | /** 51 | * Auth message. This code is for passing auth messages between the auth 52 | * providers on the client and bookie. The message payload is determined 53 | * by the auth providers themselves. 54 | */ 55 | Auth = 3, 56 | }; 57 | 58 | std::ostream& operator<<(std::ostream& s, BookieOperation op); 59 | 60 | enum class BookieError 61 | : int8_t { 62 | /** 63 | * The error code that indicates success 64 | */ 65 | OK = 0, 66 | 67 | /** 68 | * The error code that indicates that the ledger does not exist 69 | */ 70 | NoLedger = 1, 71 | 72 | /** 73 | * The error code that indicates that the requested entry does not exist 74 | */ 75 | NoEntry = 2, 76 | 77 | /** 78 | * The error code that indicates an invalid request type 79 | */ 80 | BadRequest = 100, 81 | 82 | /** 83 | * General error occurred at the server 84 | */ 85 | IOError = 101, 86 | 87 | /** 88 | * Unauthorized access to ledger 89 | */ 90 | UnauthorizedAccesss = 102, 91 | 92 | /** 93 | * The server version is incompatible with the client 94 | */ 95 | BadVersion = 103, 96 | 97 | /** 98 | * Attempt to write to fenced ledger 99 | */ 100 | Fenced = 104, 101 | 102 | /** 103 | * The server is running as read-only mode 104 | */ 105 | ReadOnly = 105, 106 | 107 | /** 108 | * Too many concurrent requests 109 | */ 110 | TooManyRequests = 106, 111 | }; 112 | 113 | std::ostream& operator<<(std::ostream& s, BookieError error); 114 | 115 | enum class BookieFlag 116 | : int16_t { 117 | None = 0x0, DoFencing = 0x0001, Recovery = 0x0002, 118 | }; 119 | 120 | struct BookieConstant { 121 | static const int64_t InvalidLedgerId = -1L; 122 | static const int64_t InvalidEntryId = -1L; 123 | static const uint32_t MasterKeyLength = 20; 124 | 125 | static constexpr uint32_t MaxFrameSize = 5 * 1024 * 1024; 126 | }; 127 | 128 | typedef std::unique_ptr IOBufPtr; 129 | 130 | struct Request { 131 | int8_t protocolVersion; 132 | BookieOperation opCode; 133 | int64_t ledgerId; 134 | int64_t entryId; 135 | int16_t flags; 136 | 137 | IOBufPtr data; 138 | 139 | // Master key not supported 140 | // int8_t[] masterKey; 141 | 142 | bool isRecovery() const { 143 | return flags & (int16_t) BookieFlag::Recovery; 144 | } 145 | 146 | bool isFencing() const { 147 | return flags & (int16_t) BookieFlag::DoFencing; 148 | } 149 | }; 150 | 151 | std::ostream& operator<<(std::ostream& s, const Request& request); 152 | 153 | /////// Responses 154 | 155 | struct Response { 156 | int8_t protocolVersion; 157 | BookieOperation opCode; 158 | BookieError errorCode; 159 | int64_t ledgerId; 160 | int64_t entryId; 161 | 162 | IOBufPtr data; 163 | }; 164 | 165 | std::ostream& operator<<(std::ostream& s, const Response& response); 166 | -------------------------------------------------------------------------------- /src/BookieRegistration.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Bookie.h" 22 | #include "BookieRegistration.h" 23 | #include "Logging.h" 24 | #include "ZooKeeper.h" 25 | 26 | #include 27 | using folly::EventBaseManager; 28 | 29 | DECLARE_LOG_OBJECT(); 30 | 31 | BookieRegistration::BookieRegistration(ZooKeeper* zk, const BookieConfig& conf) : 32 | zk_(zk) { 33 | registrationPath_ = format("/ledgers/available/{}:{}", conf.bookieHost(), conf.bookiePort()).str(); 34 | zk_->registerSessionListener(std::bind(&BookieRegistration::handleNewZooKeeperSession, this)); 35 | } 36 | 37 | void BookieRegistration::handleNewZooKeeperSession() { 38 | LOG_INFO("Registering bookie on new ZK session"); 39 | registerBookie(); 40 | } 41 | 42 | void BookieRegistration::registerBookie() { 43 | zk_->create(registrationPath_, "", { ZooKeeper::CreateFlag::Ephemeral }) // 44 | .onError([this](const ZooKeeperException& e) { 45 | // TODO: deferred task is not working 46 | LOG_FATAL("Error registering bookie: " << e.what() << " -- Exiting"); 47 | std::exit(1); 48 | // LOG_WARN("Error registering bookie: " << e.what() << " -- Retrying later"); 49 | // 50 | // EventBaseManager::get()->getEventBase()->runAfterDelay([this]() { 51 | // LOG_INFO("Retrying bookie registration after error"); 52 | // this->registerBookie(); 53 | // }, std::chrono::milliseconds(10000).count()); 54 | return std::string(""); 55 | }) // 56 | .then([](std::string path) { 57 | if (!path.empty()) { 58 | LOG_INFO("Registered bookie at " << path); 59 | } 60 | }); 61 | } 62 | -------------------------------------------------------------------------------- /src/BookieRegistration.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | class ZooKeeper; 24 | class BookieConfig; 25 | 26 | #include 27 | 28 | /** 29 | * Register the bookie under /ledgers/available and makes sure the z-node is re-created after the session expires 30 | */ 31 | class BookieRegistration { 32 | public: 33 | BookieRegistration(ZooKeeper* zk_, const BookieConfig& conf); 34 | 35 | private: 36 | void handleNewZooKeeperSession(); 37 | 38 | void registerBookie(); 39 | 40 | ZooKeeper* zk_; 41 | std::string registrationPath_; 42 | }; 43 | -------------------------------------------------------------------------------- /src/Logging.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Logging.h" 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | using namespace log4cxx; 31 | 32 | void Logging::init() { 33 | Logging::init(""); 34 | } 35 | 36 | void Logging::init(const std::string& logfilePath) { 37 | try { 38 | if (logfilePath.empty()) { 39 | if (!LogManager::getLoggerRepository()->isConfigured()) { 40 | LogManager::getLoggerRepository()->setConfigured(true); 41 | LoggerPtr root = Logger::getRootLogger(); 42 | static const LogString TTCC_CONVERSION_PATTERN(LOG4CXX_STR("%d{HH:mm:ss.SSS} [%t] %-5p %l - %m%n")); 43 | LayoutPtr layout(new PatternLayout(TTCC_CONVERSION_PATTERN)); 44 | AppenderPtr appender(new ConsoleAppender(layout)); 45 | root->setLevel(log4cxx::Level::getInfo()); 46 | root->addAppender(appender); 47 | } 48 | } else { 49 | log4cxx::PropertyConfigurator::configure(logfilePath); 50 | } 51 | } 52 | catch (const std::exception& e) { 53 | std::cerr << "exception caught while configuring log4cpp via '" << logfilePath << "': " << e.what() 54 | << std::endl; 55 | } 56 | catch (...) { 57 | std::cerr << "unknown exception while configuring log4cpp via '" << logfilePath << "'." << std::endl; 58 | } 59 | } 60 | 61 | -------------------------------------------------------------------------------- /src/Logging.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | 26 | #define DECLARE_LOG_OBJECT() \ 27 | static log4cxx::LoggerPtr& logger() \ 28 | { \ 29 | static log4cxx::LoggerPtr result = log4cxx::Logger::getLogger("bookie." __FILE__); \ 30 | return result; \ 31 | } \ 32 | 33 | 34 | #define LOG_DEBUG(message) { \ 35 | if (LOG4CXX_UNLIKELY(logger()->isDebugEnabled())) {\ 36 | ::log4cxx::helpers::MessageBuffer oss_; \ 37 | logger()->forcedLog(::log4cxx::Level::getDebug(), oss_.str(((std::ostream&)oss_) << message), LOG4CXX_LOCATION); }} 38 | 39 | #define LOG_INFO(message) { \ 40 | if (logger()->isInfoEnabled()) {\ 41 | ::log4cxx::helpers::MessageBuffer oss_; \ 42 | logger()->forcedLog(::log4cxx::Level::getInfo(), oss_.str(((std::ostream&)oss_) << message), LOG4CXX_LOCATION); }} 43 | 44 | #define LOG_WARN(message) { \ 45 | if (LOG4CXX_UNLIKELY(logger()->isWarnEnabled())) {\ 46 | ::log4cxx::helpers::MessageBuffer oss_; \ 47 | logger()->forcedLog(::log4cxx::Level::getWarn(), oss_.str(((std::ostream&)oss_) << message), LOG4CXX_LOCATION); }} 48 | 49 | #define LOG_ERROR(message) { \ 50 | if (LOG4CXX_UNLIKELY(logger()->isErrorEnabled())) {\ 51 | ::log4cxx::helpers::MessageBuffer oss_; \ 52 | logger()->forcedLog(::log4cxx::Level::getError(), oss_.str(((std::ostream&)oss_) << message), LOG4CXX_LOCATION); }} 53 | 54 | #define LOG_FATAL(message) { \ 55 | if (LOG4CXX_UNLIKELY(logger()->isFatalEnabled())) {\ 56 | ::log4cxx::helpers::MessageBuffer oss_; \ 57 | logger()->forcedLog(::log4cxx::Level::getFatal(), oss_.str(((std::ostream&)oss_) << message), LOG4CXX_LOCATION); }} 58 | 59 | class Logging { 60 | public: 61 | static void init(); 62 | static void init(const std::string& logConfFilePath); 63 | }; 64 | 65 | -------------------------------------------------------------------------------- /src/Metrics-inl.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | inline Timer::Timer() : 24 | metric_(nullptr), 25 | startTime_() { 26 | } 27 | 28 | inline Timer::Timer(Metric* metric) : 29 | metric_(metric), 30 | startTime_(Clock::now()) { 31 | } 32 | 33 | inline void Timer::completed() { 34 | metric_->addLatencySample(Clock::now() - startTime_); 35 | } 36 | 37 | inline const std::string& Metric::name() const { 38 | return name_; 39 | } 40 | 41 | inline Timer Metric::startTimer() { 42 | return Timer(this); 43 | } 44 | 45 | inline void Metric::addLatencySample(Clock::duration latency) { 46 | histogram_->addValue(duration_cast(latency).count()); 47 | } 48 | 49 | inline void Metric::addValueSample(uint64_t value) { 50 | // Multiply value since it's expecting to get a "micros" latency 51 | // that will be later presented in millis 52 | histogram_->addValue(value * 1000); 53 | } 54 | -------------------------------------------------------------------------------- /src/Metrics.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | 22 | #include "Logging.h" 23 | #include "Metrics.h" 24 | 25 | #include 26 | #include 27 | 28 | DECLARE_LOG_OBJECT(); 29 | 30 | static const int64_t BucketSize = 100; 31 | static const int64_t MinValue = 0; 32 | static const int64_t MaxValue = microseconds(seconds(1)).count(); 33 | 34 | Metric::Metric(const std::string& name) : 35 | name_(name), 36 | histogram_([]() { 37 | return new LatencyHistogram(BucketSize, MinValue, MaxValue); 38 | }), 39 | stats_(dynamic::object()) { 40 | } 41 | 42 | typedef duration double_millis; 43 | 44 | inline double toMillis(int64_t micros) { 45 | return double_millis(microseconds(micros)).count(); 46 | } 47 | 48 | dynamic Metric::getStats() { 49 | return stats_; 50 | } 51 | 52 | void Metric::updateStats(seconds statsPeriod) { 53 | LatencyHistogram aggregated(BucketSize, MinValue, MaxValue); 54 | for (LatencyHistogram& hist : histogram_.accessAllThreads()) { 55 | aggregated.merge(hist); 56 | hist.clear(); 57 | } 58 | 59 | uint64_t count = 0; 60 | for (int i = 0; i < aggregated.getNumBuckets(); i++) { 61 | count += aggregated.getBucketByIndex(i).count; 62 | } 63 | 64 | double rate = count / (double) statsPeriod.count(); 65 | 66 | stats_["min"] = toMillis(aggregated.getPercentileEstimate(0.0000)); 67 | stats_["pct50"] = toMillis(aggregated.getPercentileEstimate(0.5000)); 68 | stats_["pct75"] = toMillis(aggregated.getPercentileEstimate(0.7500)); 69 | stats_["pct90"] = toMillis(aggregated.getPercentileEstimate(0.9000)); 70 | stats_["pct95"] = toMillis(aggregated.getPercentileEstimate(0.9500)); 71 | stats_["pct99"] = toMillis(aggregated.getPercentileEstimate(0.9900)); 72 | stats_["pct999"] = toMillis(aggregated.getPercentileEstimate(0.9990)); 73 | stats_["pct9999"] = toMillis(aggregated.getPercentileEstimate(0.9999)); 74 | stats_["max"] = toMillis(aggregated.getPercentileEstimate(1.0000)); 75 | stats_["count"] = count; 76 | stats_["rate"] = rate; 77 | } 78 | 79 | MetricsManager::MetricsManager(seconds statsPeriod) : 80 | statsPeriod_(statsPeriod), 81 | eventBase_(), 82 | statsUpdateThread_([=] { 83 | setThreadName("bookie-stats-updater"); 84 | eventBase_.runAfterDelay(std::bind(&MetricsManager::updateStats, this), milliseconds(statsPeriod_).count()); 85 | eventBase_.loopForever(); 86 | }) { 87 | } 88 | 89 | MetricsManager::~MetricsManager() { 90 | eventBase_.terminateLoopSoon(); 91 | statsUpdateThread_.join(); 92 | } 93 | 94 | MetricPtr MetricsManager::createMetric(const std::string& name) { 95 | std::lock_guard lock(mutex_); 96 | auto it = metrics_.find(name); 97 | if (it != metrics_.end()) { 98 | return it->second; 99 | } 100 | 101 | // Insert new metric 102 | MetricPtr metric = std::make_shared(name); 103 | metrics_[name] = metric; 104 | return metric; 105 | } 106 | 107 | void MetricsManager::updateStats() { 108 | json::serialization_opts opts; 109 | opts.pretty_formatting = false; 110 | opts.sort_keys = true; 111 | 112 | 113 | std::lock_guard lock(mutex_); 114 | 115 | LOG_INFO("--- Stats ---"); 116 | for (auto& metric : metrics_) { 117 | metric.second->updateStats(statsPeriod_); 118 | 119 | LOG_INFO(metric.first << " : " << json::serialize(metric.second->getStats(), opts)); 120 | } 121 | 122 | // Schedule next stats update 123 | eventBase_.runAfterDelay(std::bind(&MetricsManager::updateStats, this), milliseconds(statsPeriod_).count()); 124 | } 125 | 126 | std::string MetricsManager::getJsonStats(bool formatJson) { 127 | std::lock_guard lock(mutex_); 128 | return getJsonStatsNoLock(formatJson); 129 | } 130 | 131 | std::string MetricsManager::getJsonStatsNoLock(bool formatJson) { 132 | dynamic stats = dynamic::object(); 133 | for (auto& metric : metrics_) { 134 | stats[metric.first] = metric.second->getStats(); 135 | } 136 | 137 | json::serialization_opts opts; 138 | opts.pretty_formatting = formatJson; 139 | opts.sort_keys = true; 140 | return json::serialize(stats, opts); 141 | } 142 | -------------------------------------------------------------------------------- /src/Metrics.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | using namespace std::chrono; 33 | using namespace folly; 34 | 35 | typedef system_clock Clock; 36 | typedef Clock::time_point TimePoint; 37 | 38 | typedef Histogram LatencyHistogram; 39 | 40 | class Metric; 41 | 42 | /** 43 | * A one-time use timer to track the time taken for a certain operation 44 | */ 45 | class Timer { 46 | public: 47 | Timer(); 48 | void completed(); 49 | 50 | private: 51 | Timer(Metric* metric); 52 | 53 | Metric* metric_; 54 | TimePoint startTime_; 55 | 56 | friend class Metric; 57 | }; 58 | 59 | typedef std::shared_ptr MetricPtr; 60 | 61 | class Metric { 62 | public: 63 | Metric(const std::string& name); 64 | 65 | Timer startTimer(); 66 | 67 | void addLatencySample(Clock::duration latency); 68 | void addValueSample(uint64_t value); 69 | 70 | const std::string& name() const; 71 | 72 | private: 73 | dynamic getStats(); 74 | 75 | 76 | void updateStats(seconds statsPeriod); 77 | 78 | const std::string name_; 79 | 80 | class HistogramTag; 81 | ThreadLocal histogram_; 82 | 83 | dynamic stats_; 84 | 85 | friend class Timer; 86 | friend class MetricsManager; 87 | }; 88 | 89 | typedef std::shared_ptr MetricPtr; 90 | 91 | class MetricsManager { 92 | public: 93 | MetricsManager(seconds statsPeriod); 94 | ~MetricsManager(); 95 | 96 | MetricPtr createMetric(const std::string& name); 97 | 98 | std::string getJsonStats(bool formatJson = true); 99 | 100 | private: 101 | void updateStats(); 102 | std::string getJsonStatsNoLock(bool formatJson); 103 | 104 | std::map metrics_; 105 | seconds statsPeriod_; 106 | EventBase eventBase_; 107 | std::thread statsUpdateThread_; 108 | std::mutex mutex_; 109 | }; 110 | 111 | #include "Metrics-inl.h" 112 | -------------------------------------------------------------------------------- /src/RateLimiter.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | class RateLimiter { 29 | public: 30 | RateLimiter(double rate); 31 | 32 | void aquire(); 33 | 34 | void aquire(int permits); 35 | 36 | private: 37 | RateLimiter(const RateLimiter&); 38 | RateLimiter& operator=(const RateLimiter&); 39 | typedef std::chrono::high_resolution_clock Clock; 40 | Clock::duration interval_; 41 | 42 | long storedPermits_; 43 | double maxPermits_; 44 | Clock::time_point nextFree_; 45 | }; 46 | 47 | RateLimiter::RateLimiter(double rate) 48 | : interval_(std::chrono::microseconds((long)(1e6 / rate))), 49 | storedPermits_(0.0), 50 | maxPermits_(rate), 51 | nextFree_() { 52 | assert(rate < 1e6 && "Exceeded maximum rate"); 53 | } 54 | 55 | void RateLimiter::aquire() { 56 | aquire(1); 57 | } 58 | 59 | void RateLimiter::aquire(int permits) { 60 | Clock::time_point now = Clock::now(); 61 | 62 | if (now > nextFree_) { 63 | storedPermits_ = std::min(maxPermits_, 64 | storedPermits_ + (now - nextFree_) / interval_); 65 | nextFree_ = now; 66 | } 67 | 68 | Clock::duration wait = nextFree_ - now; 69 | 70 | // Determine how many stored and fresh permits to consume 71 | long stored = std::min(permits, storedPermits_); 72 | long fresh = permits - stored; 73 | 74 | // In the general RateLimiter, stored permits have no wait time, 75 | // and thus we only have to wait for however many fresh permits we consume 76 | Clock::duration next = fresh * interval_; 77 | nextFree_ += next; 78 | storedPermits_ -= stored; 79 | 80 | if (wait != Clock::duration::zero()) { 81 | std::this_thread::sleep_for(wait); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/Storage.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Logging.h" 22 | #include "RateLimiter.h" 23 | #include "Storage.h" 24 | 25 | #include 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | using namespace rocksdb; 33 | using namespace std::chrono; 34 | 35 | DECLARE_LOG_OBJECT(); 36 | 37 | constexpr unsigned long long int operator ""_KB(unsigned long long int kilobytes) { 38 | return kilobytes * 1024; 39 | } 40 | 41 | constexpr unsigned long long int operator ""_MB(unsigned long long int megabytes) { 42 | return megabytes * 1024 * 1024; 43 | } 44 | 45 | constexpr unsigned long long int operator ""_GB(unsigned long long int gigabytes) { 46 | return gigabytes * 1024 * 1024 * 1024; 47 | } 48 | 49 | Storage::Storage(const BookieConfig& conf, MetricsManager& metricsManager) : 50 | db_(nullptr), 51 | writeOptions_(), 52 | journalQueue_(10000), 53 | fsyncWal_(conf.fsyncWal()), 54 | journalThread_(std::bind(&Storage::runJournal, this)), 55 | rocksDbPutLatency_(metricsManager.createMetric("rocksDbPut")), 56 | addEntryEnqueueLatency_(metricsManager.createMetric("addEntryEnqueueLatency")), 57 | walSyncLatency_(metricsManager.createMetric("walSync")), 58 | walQueueLatency_(metricsManager.createMetric("walQueueLatency")) { 59 | Options options; 60 | options.create_if_missing = true; 61 | options.write_buffer_size = 1_GB; 62 | options.max_write_buffer_number = 4; 63 | options.max_background_compactions = 16; 64 | options.max_background_flushes = 4; 65 | options.IncreaseParallelism(std::thread::hardware_concurrency()); 66 | options.max_open_files = -1; 67 | options.max_file_opening_threads = 16; 68 | options.target_file_size_base = 1_GB; 69 | options.max_bytes_for_level_base = 10_GB; 70 | options.delete_obsolete_files_period_micros = duration_cast(hours(1)).count(); 71 | options.compaction_readahead_size = 8_MB; 72 | options.allow_concurrent_memtable_write = true; 73 | 74 | // Keys are always 16 bytes (ledgerId, entryId) 75 | options.prefix_extractor.reset(NewFixedPrefixTransform(8)); 76 | 77 | options.log_file_time_to_roll = duration_cast(hours(24)).count(); 78 | options.keep_log_file_num = 30; 79 | options.stats_dump_period_sec = 60; 80 | 81 | options.wal_dir = conf.walDirectory(); 82 | 83 | BlockBasedTableOptions table_options; 84 | table_options.block_size = 256_KB; 85 | table_options.format_version = 2; 86 | table_options.checksum = kxxHash; 87 | table_options.block_cache = NewLRUCache(8_GB, 8); 88 | table_options.cache_index_and_filter_blocks = true; 89 | table_options.filter_policy.reset(NewBloomFilterPolicy(10, false)); 90 | options.table_factory.reset(NewBlockBasedTableFactory(table_options)); 91 | 92 | LOG_INFO("Opening database at " << conf.dataDirectory()); 93 | 94 | Status res = DB::Open(options, conf.dataDirectory(), &db_); 95 | if (!res.ok()) { 96 | LOG_FATAL("Failed to open database: " << res.code()); 97 | std::exit(1); 98 | } 99 | 100 | LOG_INFO("Database opened successfully"); 101 | } 102 | 103 | Storage::~Storage() { 104 | // Write a null promise to make the journal thread to exit 105 | JournalEntry entry { { }, { }, nullptr, walQueueLatency_->startTimer() }; 106 | journalQueue_.blockingWrite(std::move(entry)); 107 | journalThread_.join(); 108 | delete db_; 109 | } 110 | 111 | Future Storage::put(int64_t ledgerId, int64_t entryId, IOBufPtr data) { 112 | PromisePtr promise = make_unique>(); 113 | Future future = promise->getFuture(); 114 | 115 | union Key { 116 | struct { 117 | int64_t ledgerId; 118 | int64_t entryId; 119 | }; 120 | char data[0]; 121 | }; 122 | 123 | Key key; 124 | key.ledgerId = Endian::big(ledgerId); 125 | key.entryId = Endian::big(entryId); 126 | 127 | JournalEntry entry { Slice(key.data, sizeof(key)), std::move(data), std::move(promise), 128 | walQueueLatency_->startTimer() }; 129 | 130 | Slice valueSlice((const char*) data->data(), data->length()); 131 | 132 | Timer addEntryEnqueueTimer = addEntryEnqueueLatency_->startTimer(); 133 | journalQueue_.blockingWrite(std::move(entry)); 134 | addEntryEnqueueTimer.completed(); 135 | 136 | return future; 137 | } 138 | 139 | void Storage::runJournal() { 140 | setThreadName("bookie-journal"); 141 | 142 | std::vector entriesToSync; 143 | Unit unit; 144 | Metric* journalSyncLatency = walSyncLatency_.get(); 145 | WriteOptions syncOptions; 146 | syncOptions.sync = fsyncWal_; 147 | WriteBatch writeBatch; 148 | 149 | JournalEntry entry; 150 | bool blockForNextEntry = false; 151 | 152 | while (true) { 153 | // Collect all items from queue 154 | int toSyncCount = 0; 155 | 156 | while (true) { 157 | if (blockForNextEntry) { 158 | journalQueue_.blockingRead(entry); 159 | blockForNextEntry = false; 160 | } else { 161 | if (!journalQueue_.read(entry)) { 162 | blockForNextEntry = true; 163 | if (toSyncCount == 0) { 164 | // Block until new entry is available 165 | continue; 166 | } else { 167 | // Queue is drained, write entries to db 168 | break; 169 | } 170 | } 171 | } 172 | 173 | if (entry.promise.get() == nullptr) { 174 | // Journal is exiting 175 | return; 176 | } 177 | 178 | entry.walTimeSpentInQueue.completed(); 179 | entriesToSync.emplace_back(std::move(entry.promise)); 180 | writeBatch.Put(entry.key, Slice((const char*) entry.data->data(), entry.data->length())); 181 | 182 | if (toSyncCount++ == 1000) { 183 | break; 184 | } 185 | } 186 | 187 | if (entriesToSync.empty()) { 188 | continue; 189 | } 190 | 191 | Timer syncLatencyTimer = journalSyncLatency->startTimer(); 192 | db_->Write(syncOptions, &writeBatch); 193 | syncLatencyTimer.completed(); 194 | 195 | for (auto& pr : entriesToSync) { 196 | pr->setValue(unit); 197 | } 198 | 199 | entriesToSync.clear(); 200 | writeBatch.Clear(); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /src/Storage.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #include 29 | #include 30 | 31 | #include "BookieConfig.h" 32 | #include "Metrics.h" 33 | 34 | using namespace folly; 35 | using rocksdb::Slice; 36 | typedef std::unique_ptr IOBufPtr; 37 | 38 | class Storage { 39 | public: 40 | Storage(const BookieConfig& conf, MetricsManager& metricsManager); 41 | ~Storage(); 42 | 43 | Future put(int64_t ledgerId, int64_t entryId, IOBufPtr data); 44 | 45 | private: 46 | void runJournal(); 47 | 48 | rocksdb::DB* db_; 49 | const rocksdb::WriteOptions writeOptions_; 50 | 51 | typedef std::unique_ptr> PromisePtr; 52 | 53 | struct JournalEntry { 54 | Slice key; 55 | IOBufPtr data; 56 | PromisePtr promise; 57 | Timer walTimeSpentInQueue; 58 | }; 59 | 60 | MPMCQueue journalQueue_; 61 | 62 | const bool fsyncWal_; 63 | std::thread journalThread_; 64 | 65 | MetricPtr rocksDbPutLatency_; 66 | MetricPtr addEntryEnqueueLatency_; 67 | MetricPtr walSyncLatency_; 68 | MetricPtr walQueueLatency_; 69 | }; 70 | 71 | -------------------------------------------------------------------------------- /src/ZooKeeper.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "ZooKeeper.h" 22 | 23 | #include "Logging.h" 24 | #include 25 | #include 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | 33 | DECLARE_LOG_OBJECT(); 34 | 35 | template 36 | struct Context { 37 | ZooKeeper* client; 38 | std::string path; 39 | Promise promise; 40 | }; 41 | 42 | ZooKeeper::ZooKeeper(const std::string& zkServers, std::chrono::milliseconds sessionTimeout) : 43 | zkServers_(zkServers), 44 | sessionTimeout_(sessionTimeout), 45 | zk_(nullptr) { 46 | } 47 | 48 | ZooKeeper::~ZooKeeper() { 49 | if (zk_) { 50 | LOG_INFO("Closing zk session " << format("0x{0:x}", zoo_client_id(zk_)->client_id)); 51 | int rc = zookeeper_close(zk_); 52 | if (rc != ZOK) { 53 | ZooKeeperError err = ZooKeeperException::getError(rc); 54 | LOG_WARN("Failed to close zk session: " << ZooKeeperException::getErrorMsg(err)); 55 | } 56 | } 57 | } 58 | 59 | Future ZooKeeper::startSession() { 60 | std::lock_guard lock { mutex_ }; 61 | 62 | LOG_INFO("Creating ZooKeeper session"); 63 | zk_ = zookeeper_init(zkServers_.c_str(), &ZooKeeper::handleSessionEvent, sessionTimeout_.count(), nullptr, this, 0); 64 | return sessionPromise_.getFuture(); 65 | } 66 | 67 | void ZooKeeper::handleSessionEvent(zhandle_t* zh, int type, int state, const char* path, void* watcherCtx) { 68 | ZooKeeper* client = reinterpret_cast(watcherCtx); 69 | std::unique_lock lock { client->mutex_ }; 70 | 71 | LOG_INFO("Received ZK watch event. Type: " << getEventTypeStr(type) // 72 | << " -- State: " << getSessionStateStr(state)// 73 | << " -- path: '" << path << "'"); 74 | 75 | if (state == ZOO_CONNECTED_STATE) { 76 | client->zk_ = zh; 77 | if (!client->sessionPromise_.isFulfilled()) { 78 | int64_t zkClientId = zoo_client_id(zh)->client_id; 79 | LOG_INFO("Established new ZooKeeper session " << format("0x{0:x}", zkClientId)); 80 | client->sessionPromise_.setValue(zkClientId); 81 | 82 | // Notify listeners that a new session is ready 83 | auto toNotify { client->sessionListeners_ }; 84 | lock.unlock(); 85 | 86 | for (auto& listener : toNotify) { 87 | listener(); 88 | } 89 | } 90 | } else if (state == ZOO_EXPIRED_SESSION_STATE) { 91 | LOG_WARN("ZooKeeper session expired"); 92 | client->sessionPromise_ = {}; 93 | 94 | lock.unlock(); 95 | client->startSession(); 96 | } 97 | } 98 | 99 | void ZooKeeper::registerSessionListener(SessionListener listener) { 100 | std::unique_lock lock { mutex_ }; 101 | 102 | sessionListeners_.push_back(listener); 103 | if (sessionPromise_.isFulfilled()) { 104 | // Session is already ready, notify immediately 105 | lock.unlock(); 106 | 107 | listener(); 108 | } 109 | } 110 | 111 | Future ZooKeeper::create(const std::string& path, const std::string& value, 112 | std::initializer_list createFlags) { 113 | std::lock_guard lock { mutex_ }; 114 | 115 | Context* ctx = new Context { this, path }; 116 | int flags = 0; 117 | for (auto flag : createFlags) { 118 | flags |= flag; 119 | } 120 | 121 | int rc = zoo_acreate(zk_, path.c_str(), value.c_str(), value.length(), &ZOO_OPEN_ACL_UNSAFE, flags, 122 | [](int rc, const char* path, const void* zkCtx) { 123 | Context* ctx = (Context*)zkCtx; 124 | 125 | if (rc == ZOK) { 126 | LOG_DEBUG("Successfully created z-node at " << path); 127 | ctx->promise.setValue(path); 128 | } else { 129 | ctx->promise.setException(make_exception_wrapper(rc, // 130 | to("Failed to create z-node at ", ctx->path) )); 131 | } 132 | 133 | delete ctx; 134 | }, ctx); 135 | 136 | if (rc != ZOK) { 137 | return makeFuture( 138 | make_exception_wrapper(rc, to("Failed to create z-node at ", path))); 139 | } 140 | 141 | return ctx->promise.getFuture(); 142 | } 143 | 144 | std::string ZooKeeper::getEventTypeStr(int type) { 145 | switch (type) { 146 | case -1: 147 | return "None"; 148 | case 1: 149 | return "NodeCreated"; 150 | case 2: 151 | return "NodeDeleted"; 152 | case 3: 153 | return "NodeDataChanged"; 154 | case 4: 155 | return "NodeChildrenChanged"; 156 | default: 157 | return to(type); 158 | } 159 | } 160 | 161 | std::string ZooKeeper::getSessionStateStr(int state) { 162 | switch (state) { 163 | case 0: 164 | return "Disconnected"; 165 | case 1: 166 | return "NoSyncConnected"; 167 | case 3: 168 | return "Connected"; 169 | case 5: 170 | return "ConnectedReadOnly"; 171 | case 6: 172 | return "SaslAuthenticated"; 173 | case -112: 174 | return "Expired"; 175 | default: 176 | return to(state); 177 | } 178 | } 179 | 180 | ZooKeeperException::ZooKeeperException(int rc) : 181 | zkError_(getError(rc)), 182 | msg_(getErrorMsg(zkError_)) { 183 | } 184 | 185 | ZooKeeperException::ZooKeeperException(int rc, const std::string& msg) : 186 | zkError_(getError(rc)), 187 | msg_(to(msg, " - ", getErrorMsg(zkError_))) { 188 | } 189 | 190 | const char* ZooKeeperException::what() const noexcept { 191 | return msg_.c_str(); 192 | } 193 | 194 | ZooKeeperError ZooKeeperException::error() const { 195 | return zkError_; 196 | } 197 | 198 | ZooKeeperError ZooKeeperException::getError(int rc) { 199 | return static_cast(rc); 200 | } 201 | 202 | std::string ZooKeeperException::getErrorMsg(ZooKeeperError zkError) { 203 | switch (zkError) { 204 | case ZooKeeperError::OK: 205 | return "OK"; 206 | case ZooKeeperError::SystemError: 207 | return "System Error"; 208 | case ZooKeeperError::RuntimeInconsistency: 209 | return "Runtime inconsistency"; 210 | case ZooKeeperError::DataIinconsistency: 211 | return "Data inconsistency"; 212 | case ZooKeeperError::ConnectionLoss: 213 | return "Connection loss"; 214 | case ZooKeeperError::MarshallingError: 215 | return "Marshalling error"; 216 | case ZooKeeperError::Unimplemented: 217 | return "Unimplemented"; 218 | case ZooKeeperError::OperationTimeout: 219 | return "Operation timeout"; 220 | case ZooKeeperError::BadArguments: 221 | return "Bad arguments"; 222 | case ZooKeeperError::InvalidState: 223 | return "Invalid state"; 224 | case ZooKeeperError::ApiError: 225 | return "API error"; 226 | case ZooKeeperError::NoNode: 227 | return "No node"; 228 | case ZooKeeperError::NoAuth: 229 | return "No Auth"; 230 | case ZooKeeperError::BadVersion: 231 | return "Bad Version"; 232 | case ZooKeeperError::NoChildrenForEphemerals: 233 | return "No children for ephemerals"; 234 | case ZooKeeperError::NodeExists: 235 | return "Node exists"; 236 | case ZooKeeperError::NotEmpty: 237 | return "Not empty"; 238 | case ZooKeeperError::SessionExpired: 239 | return "Session expired"; 240 | case ZooKeeperError::InvalidCallback: 241 | return "Invalid callback"; 242 | case ZooKeeperError::InvalidACL: 243 | return "Invalid ACL"; 244 | case ZooKeeperError::AuthFailed: 245 | return "Auth failed"; 246 | case ZooKeeperError::Closing: 247 | return "Closing"; 248 | case ZooKeeperError::Nothing: 249 | return "Nothing"; 250 | case ZooKeeperError::SessionMoved: 251 | return "Session moved"; 252 | default: 253 | return to("Unknown error (", zkError, ")"); 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/ZooKeeper.h: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #pragma once 22 | 23 | #include 24 | #include 25 | 26 | #include 27 | 28 | struct _zhandle; 29 | 30 | using namespace folly; 31 | 32 | enum class ZooKeeperError 33 | : int { 34 | OK = 0, /*!< Everything is OK */ 35 | 36 | /** System and server-side errors. 37 | * This is never thrown by the server, it shouldn't be used other than 38 | * to indicate a range. Specifically error codes greater than this 39 | * value, but lesser than {@link #ZAPIERROR}, are system errors. */ 40 | SystemError = -1, // 41 | RuntimeInconsistency = -2, /*!< A runtime inconsistency was found */ 42 | DataIinconsistency = -3, /*!< A data inconsistency was found */ 43 | ConnectionLoss = -4, /*!< Connection to the server has been lost */ 44 | MarshallingError = -5, /*!< Error while marshalling or unmarshalling data */ 45 | Unimplemented = -6, /*!< Operation is unimplemented */ 46 | OperationTimeout = -7, /*!< Operation timeout */ 47 | BadArguments = -8, /*!< Invalid arguments */ 48 | InvalidState = -9, /*!< Invliad zhandle state */ 49 | 50 | /** API errors. 51 | * This is never thrown by the server, it shouldn't be used other than 52 | * to indicate a range. Specifically error codes greater than this 53 | * value are API errors (while values less than this indicate a 54 | * {@link #ZSYSTEMERROR}). 55 | */ 56 | ApiError = -100, 57 | NoNode = -101, /*!< Node does not exist */ 58 | NoAuth = -102, /*!< Not authenticated */ 59 | BadVersion = -103, /*!< Version conflict */ 60 | NoChildrenForEphemerals = -108, /*!< Ephemeral nodes may not have children */ 61 | NodeExists = -110, /*!< The node already exists */ 62 | NotEmpty = -111, /*!< The node has children */ 63 | SessionExpired = -112, /*!< The session has been expired by the server */ 64 | InvalidCallback = -113, /*!< Invalid callback specified */ 65 | InvalidACL = -114, /*!< Invalid ACL specified */ 66 | AuthFailed = -115, /*!< Client authentication failed */ 67 | Closing = -116, /*!< ZooKeeper is closing */ 68 | Nothing = -117, /*!< (not error) no server responses to process */ 69 | SessionMoved = -118 /*! SessionListener; 91 | 92 | enum CreateFlag { 93 | Ephemeral = 0x01, // 94 | Sequence = 0x02, 95 | }; 96 | 97 | ZooKeeper(const std::string& zkServers, std::chrono::milliseconds sessionTimeout); 98 | ZooKeeper(const ZooKeeper& x) = delete; 99 | ZooKeeper& operator =(const ZooKeeper& x) = delete; 100 | ~ZooKeeper(); 101 | 102 | Future startSession(); 103 | 104 | void registerSessionListener(SessionListener listener); 105 | 106 | /** 107 | * Create a z-node 108 | * 109 | * @param path 110 | * @param value 111 | * @param createFlags 112 | * @return a future yielding the effective path of the created z-node 113 | */ 114 | Future create(const std::string& path, const std::string& value, 115 | std::initializer_list createFlags); 116 | 117 | private: 118 | static void handleSessionEvent(_zhandle* zh, int type, int state, const char* path, void* watcherCtx); 119 | 120 | static std::string getEventTypeStr(int type); 121 | static std::string getSessionStateStr(int state); 122 | 123 | std::mutex mutex_; 124 | std::string zkServers_; 125 | std::chrono::milliseconds sessionTimeout_; 126 | 127 | struct _zhandle* zk_; 128 | 129 | Promise sessionPromise_; 130 | std::vector sessionListeners_; 131 | }; 132 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include "Bookie.h" 22 | #include "BookieConfig.h" 23 | #include "Logging.h" 24 | 25 | #include 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | 33 | DECLARE_LOG_OBJECT(); 34 | 35 | std::unique_ptr bookie; 36 | 37 | static void signalHandler(int signal) { 38 | LOG_INFO("Received signal " << signal << " - Shutting down"); 39 | if (bookie) { 40 | bookie->stop(); 41 | } 42 | } 43 | 44 | int main(int argc, char** argv) { 45 | std::signal(SIGINT, signalHandler); 46 | std::signal(SIGTERM, signalHandler); 47 | std::signal(SIGQUIT, signalHandler); 48 | 49 | Logging::init(); 50 | google::InitGoogleLogging(argv[0]); 51 | 52 | BookieConfig config; 53 | if (!config.parse(argc, argv)) { 54 | return -1; 55 | } 56 | 57 | bookie = make_unique(config); 58 | bookie->start(); 59 | bookie->waitForStop(); 60 | 61 | // Trigger bookie destructor 62 | bookie.reset(nullptr); 63 | 64 | return 0; 65 | } 66 | -------------------------------------------------------------------------------- /src/perfClient.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * Licensed to the Apache Software Foundation (ASF) under one 4 | * or more contributor license agreements. See the NOTICE file 5 | * distributed with this work for additional information 6 | * regarding copyright ownership. The ASF licenses this file 7 | * to you under the Apache License, Version 2.0 (the 8 | * "License"); you may not use this file except in compliance 9 | * with the License. You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, 14 | * software distributed under the License is distributed on an 15 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | * KIND, either express or implied. See the License for the 17 | * specific language governing permissions and limitations 18 | * under the License. 19 | * 20 | */ 21 | #include 22 | 23 | #include "Logging.h" 24 | #include "Metrics.h" 25 | #include "BookieCodecV2.h" 26 | #include "RateLimiter.h" 27 | 28 | #include 29 | #include 30 | 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include 38 | namespace po = boost::program_options; 39 | 40 | DECLARE_LOG_OBJECT(); 41 | 42 | struct Arguments { 43 | std::string bookieAddress; 44 | double rate; 45 | int msgSize; 46 | int numberOfConnections; 47 | int statsReportingRateSeconds; 48 | bool formatStatsJson; 49 | }; 50 | 51 | typedef Pipeline BookieClientPipeline; 52 | typedef ClientBootstrap Client; 53 | 54 | class AddEntryTask: public HandlerAdapter { 55 | public: 56 | AddEntryTask(BookieClientPipeline::Ptr pipeline, double rate, int msgSize, MetricPtr addEntryMetric) : 57 | pipeline_(pipeline), 58 | rateLimiter_(rate), 59 | msgSize_(msgSize), 60 | addEntryMetric_(addEntryMetric), 61 | thread_() { 62 | } 63 | 64 | void start() { 65 | LOG_INFO("Started add entry task " << bookieAddress_); 66 | int64_t ledgerId = ledgerIdGenerator_++; 67 | int64_t entryIdGenerator = 0; 68 | 69 | std::string payload; 70 | payload.resize(msgSize_, 'X'); 71 | 72 | auto pipeline = pipeline_.get(); 73 | EventBase* eventBase = pipeline_->getTransport()->getEventBase(); 74 | 75 | while (true) { 76 | rateLimiter_.aquire(); 77 | 78 | int64_t entryId = entryIdGenerator++; 79 | 80 | eventBase->runInEventBaseThread([entryId, &ledgerId, &payload, &pipeline, this]() { 81 | Request request {2, BookieOperation::AddEntry, ledgerId, entryId, 0, IOBuf::wrapBuffer(payload.c_str(), 82 | payload.length())}; 83 | LOG_DEBUG("Sending request " << request); 84 | pipeline->write(std::move(request)); 85 | 86 | pendingRequests_.insert( {entryId, std::move(addEntryMetric_->startTimer())}); 87 | }); 88 | } 89 | } 90 | 91 | virtual void transportActive(Context* ctx) override { 92 | ctx->fireTransportActive(); 93 | ctx->getTransport()->getPeerAddress(&bookieAddress_); 94 | thread_ = std::make_unique(std::bind(&AddEntryTask::start, this)); 95 | } 96 | 97 | virtual void read(Context* ctx, Response response) override { 98 | LOG_DEBUG("Received response: " << response); 99 | if (UNLIKELY(response.errorCode != BookieError::OK)) { 100 | LOG_ERROR("Received error response: " << response.errorCode); 101 | std::exit(-1); 102 | } 103 | 104 | auto it = pendingRequests_.find(response.entryId); 105 | it->second.completed(); 106 | pendingRequests_.erase(it); 107 | } 108 | 109 | virtual void readEOF(Context* ctx) override { 110 | std::cout << "EOF received" << std::endl; 111 | close(ctx); 112 | } 113 | 114 | private: 115 | BookieClientPipeline::Ptr pipeline_; 116 | SocketAddress bookieAddress_; 117 | RateLimiter rateLimiter_; 118 | int msgSize_; 119 | MetricPtr addEntryMetric_; 120 | std::unique_ptr thread_; 121 | 122 | std::unordered_map pendingRequests_; 123 | 124 | static std::atomic ledgerIdGenerator_; 125 | }; 126 | 127 | std::atomic AddEntryTask::ledgerIdGenerator_; 128 | 129 | class BookieClientPipelineFactory: public PipelineFactory { 130 | double perConnectionRate_; 131 | int msgSize_; 132 | MetricPtr addEntryMetric_; 133 | 134 | public: 135 | 136 | BookieClientPipelineFactory(double rate, int msgSize, MetricPtr addEntryMetric) : 137 | perConnectionRate_(rate), 138 | msgSize_(msgSize), 139 | addEntryMetric_(addEntryMetric) { 140 | } 141 | 142 | BookieClientPipeline::Ptr newPipeline(std::shared_ptr sock) { 143 | auto pipeline = BookieClientPipeline::create(); 144 | pipeline->addBack(AsyncSocketHandler(sock)); 145 | pipeline->addBack(LengthFieldBasedFrameDecoder(4, BookieConstant::MaxFrameSize)); 146 | pipeline->addBack(BookieClientCodecV2()); 147 | pipeline->addBack(std::make_shared(pipeline, perConnectionRate_, msgSize_, addEntryMetric_)); 148 | pipeline->finalize(); 149 | return pipeline; 150 | } 151 | }; 152 | 153 | int main(int argc, char** argv) { 154 | Logging::init(); 155 | 156 | Arguments args; 157 | 158 | po::options_description options; 159 | options.add_options() // 160 | ("help,h", "This help message") // 161 | ("bookieAddress,a", po::value(&args.bookieAddress)->default_value("localhost:3181"), 162 | "Boookie hostname and port") // 163 | ("rate,r", po::value(&args.rate)->default_value(100), "Add entry rate") // 164 | ("msg-size,s", po::value(&args.msgSize)->default_value(1024), "Message size") // 165 | ("num-connections,c", po::value(&args.numberOfConnections)->default_value(16), "Number of connections") // 166 | ("format-stats", po::value(&args.formatStatsJson)->default_value(true), "Format stats JSON output") // 167 | ("stats-reporting", po::value(&args.statsReportingRateSeconds)->default_value(10), 168 | "Interval to report latency stats in seconds") // 169 | ; 170 | 171 | po::variables_map map; 172 | try { 173 | po::store(po::command_line_parser(argc, argv).options(options).run(), map); 174 | po::notify(map); 175 | 176 | if (map.count("help")) { 177 | std::cerr << options << std::endl; 178 | exit(1); 179 | } 180 | } 181 | catch (const std::exception& e) { 182 | std::cerr << "Error parsing parameters -- " << e.what() << std::endl << std::endl; 183 | std::cerr << options << std::endl; 184 | return -1; 185 | } 186 | 187 | seconds statsReportingPeriod(args.statsReportingRateSeconds); 188 | 189 | MetricsManager metricsManager(statsReportingPeriod); 190 | MetricPtr addEntryMetric = metricsManager.createMetric("add-entry-metric"); 191 | 192 | SocketAddress bookieAddress; 193 | bookieAddress.setFromHostPort(args.bookieAddress); 194 | LOG_INFO("Bookie address: " << bookieAddress); 195 | 196 | double perConnectionRate = args.rate / args.numberOfConnections; 197 | 198 | ClientBootstrap client; 199 | client.group(std::make_shared(std::thread::hardware_concurrency())); 200 | client.pipelineFactory( 201 | std::make_shared(perConnectionRate, args.msgSize, addEntryMetric)); 202 | 203 | std::vector> connectFutures; 204 | for (int i = 0; i < args.numberOfConnections; i++) { 205 | auto future = client.connect(bookieAddress); 206 | connectFutures.push_back(std::move(future)); 207 | } 208 | 209 | for (auto& future : connectFutures) { 210 | future.get(); 211 | } 212 | 213 | while (true) { 214 | std::this_thread::sleep_for(statsReportingPeriod); 215 | // LOG_INFO("Stats : " << metricsManager.getJsonStats(args.formatStatsJson)); 216 | } 217 | } 218 | --------------------------------------------------------------------------------