├── test_package ├── grpc_test.cpp ├── example_decl.cpp ├── conanfile.py ├── CMakeLists.txt └── test_package.cpp ├── src ├── flip │ ├── client │ │ └── python │ │ │ ├── setup_python_client.sh │ │ │ ├── flip_client_example.py │ │ │ └── flip_rpc_client.py │ ├── proto │ │ ├── flip_server.proto │ │ ├── CMakeLists.txt │ │ └── flip_spec.proto │ ├── CMakeLists.txt │ └── lib │ │ ├── test_flip_server.cpp │ │ └── flip_rpc_server.cpp ├── grpc │ ├── tests │ │ ├── CMakeLists.txt │ │ ├── proto │ │ │ ├── CMakeLists.txt │ │ │ └── grpc_helper_test.proto │ │ ├── unit │ │ │ └── CMakeLists.txt │ │ └── function │ │ │ ├── CMakeLists.txt │ │ │ └── echo_sync_client.cpp │ ├── CMakeLists.txt │ └── utils.hpp ├── options │ ├── CMakeLists.txt │ ├── options.cpp │ └── tests │ │ └── basic.cpp ├── sobject │ └── CMakeLists.txt ├── version │ ├── CMakeLists.txt │ ├── tests │ │ └── test_version.cpp │ └── version.cpp ├── file_watcher │ └── CMakeLists.txt ├── logging │ ├── CMakeLists.txt │ ├── stacktrace_release.h │ └── test │ │ └── example.cpp ├── settings │ ├── CMakeLists.txt │ └── tests │ │ └── test_app_schema.fbs ├── metrics │ ├── tests │ │ ├── tree.cpp │ │ ├── cache.cpp │ │ ├── tree.hpp │ │ ├── wrapper_test2.cpp │ │ └── cache.hpp │ ├── CMakeLists.txt │ ├── metrics_atomic.cpp │ ├── metrics_rcu.cpp │ ├── metrics.cpp │ └── README.md ├── CMakeLists.txt ├── wisr │ ├── CMakeLists.txt │ └── tests │ │ ├── wisr_list_benchmark.cpp │ │ ├── wisr_deque_benchmark.cpp │ │ ├── wisr_vector_benchmark.cpp │ │ └── wisr_intrusive_slist_benchmark.cpp ├── cache │ └── CMakeLists.txt ├── utility │ ├── CMakeLists.txt │ ├── demangler.hpp │ └── tests │ │ └── test_objlife_counter.cpp └── fds │ ├── tests │ ├── test_obj_allocator.cpp │ ├── concurrent_insert_vector_bench.cpp │ ├── test_atomic_status_counter.cpp │ ├── test_tcmalloc_helper.cpp │ ├── test_idreserver.cpp │ └── obj_allocator_benchmark.cpp │ ├── buffer.cpp │ └── CMakeLists.txt ├── 3rd_party ├── userspace-rcu │ ├── conandata.yml │ └── conanfile.py └── folly │ ├── patches │ ├── 2023-001-compiler-flags.patch │ ├── 2022-001-compiler-flags.patch │ ├── 2023-002-timespec.patch │ └── 2024-001-timespec.patch │ ├── conandata.yml │ └── conan_deps.cmake ├── prepare_v2.sh ├── gcovr.cfg ├── cmake ├── mem_sanitizer.cmake └── settings_gen.cmake ├── .travis.yml.bak ├── CHANGELOG.md ├── .github ├── actions │ ├── store_conan2 │ │ └── action.yml │ ├── setup_conan │ │ └── action.yml │ ├── setup_conan2 │ │ └── action.yml │ ├── store_conan │ │ └── action.yml │ ├── load_conan │ │ └── action.yml │ └── load_conan2 │ │ └── action.yml └── workflows │ ├── version_change_check.yml │ ├── pr_build.yml │ └── merge_build.yml ├── include └── sisl │ ├── auth_manager │ ├── token_client.hpp │ └── token_verifier.hpp │ ├── grpc │ ├── rpc_common.hpp │ └── generic_service.hpp │ ├── flip │ ├── flip_rpc_server.hpp │ └── flip_client.hpp │ ├── version.hpp │ ├── file_watcher │ └── file_watcher.hpp │ ├── utility │ ├── status_factory.hpp │ ├── thread_factory.hpp │ └── non_null_ptr.hpp │ ├── fds │ ├── compress.hpp │ ├── vector_pool.hpp │ ├── sparse_vector.hpp │ ├── obj_allocator.hpp │ ├── id_reserver.hpp │ └── thread_vector.hpp │ ├── metrics │ ├── metrics_rcu.hpp │ └── reporter.hpp │ └── cache │ ├── hash_entry_base.hpp │ └── evictor.hpp ├── .gitignore ├── .jenkins └── Jenkinsfile └── CMakeLists.txt /test_package/grpc_test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() { auto server = sisl::GrpcServer::make("127.0.0.1", nullptr, 1, "", ""); } 4 | -------------------------------------------------------------------------------- /test_package/example_decl.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | SISL_LOGGING_DEF(my_module) 4 | 5 | void example_decl() { 6 | LOGINFOMOD(my_module, "Example def!"); 7 | } 8 | -------------------------------------------------------------------------------- /src/flip/client/python/setup_python_client.sh: -------------------------------------------------------------------------------- 1 | pip install grpcio-tools 2 | python3 -m grpc_tools.protoc -I proto/ --python_out=src/client/python/gen_src --grpc_python_out=src/client/python/gen_src/ proto/*.proto 3 | -------------------------------------------------------------------------------- /3rd_party/userspace-rcu/conandata.yml: -------------------------------------------------------------------------------- 1 | sources: 2 | "nu2.0.14.0": 3 | url: "https://github.com/urcu/userspace-rcu/archive/refs/tags/v0.14.0.tar.gz" 4 | sha256: "42fb5129a3fffe5a4b790dfe1ea3a734c69ee095fefbf649326269bba94c262d" 5 | -------------------------------------------------------------------------------- /src/grpc/tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_subdirectory(proto) 4 | 5 | include_directories(BEFORE "${CMAKE_CURRENT_BINARY_DIR}/proto") 6 | 7 | add_subdirectory(function) 8 | add_subdirectory(unit) 9 | -------------------------------------------------------------------------------- /prepare_v2.sh: -------------------------------------------------------------------------------- 1 | set -eu 2 | 3 | echo -n "Exporting custom recipes..." 4 | echo -n "folly." 5 | conan export 3rd_party/folly --name folly --version nu2.2023.12.18.00 >/dev/null 6 | echo -n "userspace rcu." 7 | conan export 3rd_party/userspace-rcu --name userspace-rcu --version nu2.0.14.0 >/dev/null 8 | echo "done." 9 | -------------------------------------------------------------------------------- /src/grpc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}/../auth_manager) 4 | 5 | add_library(sisl_grpc) 6 | target_sources(sisl_grpc PRIVATE 7 | rpc_server.cpp 8 | rpc_client.cpp 9 | generic_service.cpp 10 | ) 11 | target_link_libraries(sisl_grpc PUBLIC 12 | sisl_buffer 13 | gRPC::grpc++ 14 | ) 15 | 16 | add_subdirectory(tests) 17 | -------------------------------------------------------------------------------- /src/options/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_options) 4 | target_sources(sisl_options PRIVATE 5 | options.cpp 6 | ) 7 | target_link_libraries(sisl_options PUBLIC 8 | boost::boost 9 | cxxopts::cxxopts 10 | ) 11 | 12 | add_executable(basic_test) 13 | target_sources(basic_test PRIVATE 14 | tests/basic.cpp 15 | ) 16 | target_link_libraries(basic_test sisl_options GTest::gtest) 17 | add_test(NAME OptionsBasics COMMAND basic_test) 18 | -------------------------------------------------------------------------------- /gcovr.cfg: -------------------------------------------------------------------------------- 1 | high-threshold = 80 2 | medium-threshold = 65 3 | 4 | merge-lines = yes 5 | exclude-unreachable-branches = yes 6 | 7 | exclude = .*build.* 8 | exclude = .*flip.* 9 | exclude = .*generate.* 10 | exclude = .*test.* 11 | exclude = .*backtrace.cpp 12 | exclude = .*stacktrace.* 13 | exclude = .*callback_mutex.hpp 14 | 15 | exclude-lines-by-pattern = .*DEBUG.* 16 | exclude-lines-by-pattern = .*DBG.* 17 | exclude-lines-by-pattern = .*LOG.* 18 | 19 | gcov-ignore-parse-errors = all 20 | -------------------------------------------------------------------------------- /src/sobject/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_sobject) 4 | target_sources(sisl_sobject PRIVATE 5 | sobject.cpp 6 | ) 7 | target_link_libraries(sisl_sobject PUBLIC 8 | sisl_logging 9 | nlohmann_json::nlohmann_json 10 | ) 11 | 12 | add_executable(test_sobject) 13 | target_sources(test_sobject PRIVATE 14 | tests/test_sobject.cpp 15 | ) 16 | target_link_libraries(test_sobject sisl_sobject GTest::gtest) 17 | add_test(NAME Sobject COMMAND test_sobject) 18 | -------------------------------------------------------------------------------- /src/grpc/tests/proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | add_library(test_proto OBJECT) 4 | target_sources(test_proto PRIVATE 5 | grpc_helper_test.proto 6 | ) 7 | target_link_libraries(test_proto PUBLIC 8 | gRPC::grpc++ 9 | ) 10 | 11 | protobuf_generate(LANGUAGE cpp TARGET test_proto PROTOS) 12 | protobuf_generate( 13 | TARGET test_proto 14 | LANGUAGE grpc 15 | GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc 16 | PLUGIN protoc-gen-grpc=$ 17 | ) 18 | -------------------------------------------------------------------------------- /src/version/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_version) 4 | target_sources(sisl_version PRIVATE 5 | version.cpp 6 | ) 7 | target_link_libraries(sisl_version PUBLIC 8 | sisl_logging 9 | zmarok-semver::zmarok-semver 10 | ) 11 | 12 | add_executable(test_version) 13 | target_sources(test_version PRIVATE 14 | tests/test_version.cpp 15 | ) 16 | target_link_libraries(test_version sisl_version GTest::gtest) 17 | add_test(NAME Version COMMAND test_version) 18 | -------------------------------------------------------------------------------- /cmake/mem_sanitizer.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address -fsanitize=undefined -fsanitize-address-use-after-scope -fno-sanitize=alignment -DCDS_ADDRESS_SANITIZER_ENABLED -fno-omit-frame-pointer -fno-optimize-sibling-calls") 2 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fsanitize=address,undefined -fsanitize-address-use-after-scope -DCDS_ADDRESS_SANITIZER_ENABLED -fno-omit-frame-pointer -fno-optimize-sibling-calls") 3 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address -fsanitize=undefined") 4 | -------------------------------------------------------------------------------- /src/file_watcher/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_file_watcher) 4 | target_sources(sisl_file_watcher PRIVATE 5 | file_watcher.cpp 6 | ) 7 | target_link_libraries(sisl_file_watcher PUBLIC 8 | sisl_logging) 9 | 10 | add_executable(test_file_watcher) 11 | target_sources(test_file_watcher PRIVATE 12 | file_watcher_test.cpp 13 | ) 14 | target_link_libraries(test_file_watcher sisl_file_watcher GTest::gtest GTest::gmock) 15 | add_test(NAME FileWatcher COMMAND test_file_watcher) 16 | -------------------------------------------------------------------------------- /src/grpc/tests/unit/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.12) 2 | 3 | add_executable(auth_test 4 | auth_test.cpp 5 | $ 6 | ) 7 | target_link_libraries(auth_test 8 | sisl_grpc 9 | GTest::gmock 10 | ) 11 | add_test(NAME Auth_Test COMMAND auth_test) 12 | 13 | add_executable(client_test 14 | client_test.cpp 15 | ) 16 | target_link_libraries(client_test 17 | sisl_grpc 18 | GTest::gmock 19 | ${COMMON_DEPS} 20 | ) 21 | add_test(NAME Client_Test COMMAND client_test) 22 | -------------------------------------------------------------------------------- /.travis.yml.bak: -------------------------------------------------------------------------------- 1 | os: linux 2 | language: cpp 3 | dist: focal 4 | compiler: 5 | - gcc 6 | before_install: 7 | - sudo apt-get update 8 | - sudo apt-get upgrade -y 9 | - sudo apt-get install -y --no-install-recommends binutils ca-certificates libaio1 libpython2.7 libstdc++6 libunwind8 netbase python3 tzdata 10 | 11 | install: 12 | # Install conan 13 | - pip install conan 14 | 15 | # Automatic detection of your arch, compiler, etc. 16 | - conan user 17 | 18 | script: 19 | # Download dependencies and build project 20 | - conan install --build missing . 21 | - conan build . 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [Unreleased] 8 | 9 | ### Dependency Changes 10 | 11 | - Updated: cpr/1.9.3 12 | - Updated: pistache/0.0.5 13 | - Updated: prometheus-cpp/1.1.0 14 | 15 | ## [8.x] 16 | 17 | ### Changed 18 | 19 | - Moved SISL code to github.com; start Changelog 20 | 21 | [Unreleased]: https://github.com/eBay/sisl/compare/stable/v8.x...HEAD 22 | [8.x]: https://github.com/eBay/sisl/compare/v5.0.10...stable/v8.x 23 | -------------------------------------------------------------------------------- /.github/actions/store_conan2/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Store Conan Cache' 2 | description: 'Cleans Local Conan Cache and Persists Dirty Packages' 3 | inputs: 4 | key_prefix: 5 | description: 'Cache prefix' 6 | required: true 7 | default: 'Deps' 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Setup Conan and Export Recipes 12 | shell: bash 13 | run: | 14 | conan cache save --no-source --file ~/conan_cache_save.tgz -l ~/pkglist.json 15 | 16 | - name: Save Cache 17 | uses: actions/cache/save@v4 18 | with: 19 | path: | 20 | ~/conan_cache_save.tgz 21 | key: ${{ inputs.key_prefix }}-${{ hashFiles('conanfile.py', '3rd_party/**/conanfile.py') }} 22 | 23 | -------------------------------------------------------------------------------- /.github/workflows/version_change_check.yml: -------------------------------------------------------------------------------- 1 | name: Check Modified File 2 | 3 | on: 4 | pull_request: 5 | types: [opened, edited, synchronize] 6 | 7 | jobs: 8 | check-file: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - name: Checkout code 13 | uses: actions/checkout@main 14 | with: 15 | fetch-depth: 2 16 | - name: Check if file is modified 17 | run: | 18 | if git diff -r HEAD^1 HEAD -- conanfile.py | egrep "[ ]+version = "; then 19 | echo "Version is updated with this PR, OK" 20 | else 21 | echo "Conan version is not updated with this PR. Please update that to allow PR merge" 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /3rd_party/folly/patches/2023-001-compiler-flags.patch: -------------------------------------------------------------------------------- 1 | --- CMake/FollyCompilerUnix.cmake 2 | +++ CMake/FollyCompilerUnix.cmake 3 | @@ -12,7 +12,7 @@ 4 | # See the License for the specific language governing permissions and 5 | # limitations under the License. 6 | 7 | -set(CMAKE_CXX_FLAGS_COMMON "-g -Wall -Wextra") 8 | +set(CMAKE_CXX_FLAGS_COMMON "-Wall -Wextra") 9 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_CXX_FLAGS_COMMON}") 10 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_COMMON} -O3") 11 | 12 | @@ -25,7 +25,6 @@ 13 | ) 14 | target_compile_options(${THETARGET} 15 | PRIVATE 16 | - -g 17 | -finput-charset=UTF-8 18 | -fsigned-char 19 | -Wall 20 | -------------------------------------------------------------------------------- /src/grpc/tests/function/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | # build echo_server 4 | add_executable(echo_server) 5 | target_sources(echo_server PRIVATE 6 | echo_server.cpp 7 | $ 8 | ) 9 | target_link_libraries(echo_server 10 | sisl_grpc 11 | GTest::gtest 12 | ) 13 | add_test(NAME Echo_Ping_Server COMMAND echo_server) 14 | 15 | # build echo_async_client 16 | add_executable(echo_async_client) 17 | target_sources(echo_async_client PRIVATE 18 | echo_async_client.cpp 19 | $ 20 | ) 21 | target_link_libraries(echo_async_client 22 | sisl_grpc 23 | GTest::gtest 24 | ) 25 | add_test(NAME Echo_Ping_Async_Client_Server COMMAND echo_async_client) 26 | -------------------------------------------------------------------------------- /src/flip/proto/flip_server.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package flip; 4 | 5 | import "flip_spec.proto"; 6 | 7 | message FlipListResponse { 8 | message FlipInfo { string info = 1; } 9 | 10 | repeated FlipInfo infos = 1; 11 | } 12 | 13 | message FlipNameRequest { string name = 1; } 14 | 15 | message FlipRemoveRequest { string name = 1; } 16 | 17 | message FlipRemoveResponse { uint32 num_removed = 1; } 18 | 19 | service FlipServer { 20 | // Inject a fault rpc 21 | rpc InjectFault(flip.FlipSpec) returns (flip.FlipResponse); 22 | 23 | // Get details about one or all faults 24 | rpc GetFaults(FlipNameRequest) returns (FlipListResponse); 25 | 26 | // Remove a fault added earlier 27 | rpc RemoveFault(FlipRemoveRequest) returns (FlipRemoveResponse); 28 | } -------------------------------------------------------------------------------- /include/sisl/auth_manager/token_client.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace sisl { 5 | 6 | // Interface to get a token for authorization 7 | 8 | class TokenClient { 9 | public: 10 | virtual ~TokenClient() = default; 11 | 12 | virtual std::string get_token() = 0; 13 | }; 14 | 15 | // the key value pairs (m_auth_header_key, get_token()) are sent as metadata in the grpc client context 16 | 17 | class GrpcTokenClient : public TokenClient { 18 | public: 19 | explicit GrpcTokenClient(std::string const& auth_header_key) : m_auth_header_key(auth_header_key) {} 20 | virtual ~GrpcTokenClient() = default; 21 | 22 | std::string get_auth_header_key() const { return m_auth_header_key; } 23 | 24 | private: 25 | std::string m_auth_header_key; 26 | }; 27 | } // namespace sisl -------------------------------------------------------------------------------- /src/logging/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | list(APPEND LOGGING_DEPS 4 | sisl_options 5 | spdlog::spdlog 6 | nlohmann_json::nlohmann_json 7 | ) 8 | if(${breakpad_FOUND}) 9 | list(APPEND LOGGING_DEPS breakpad::breakpad) 10 | endif() 11 | 12 | list(APPEND LOGGING_SOURCE_FILES 13 | logging.cpp 14 | stacktrace.cpp 15 | ) 16 | if (${CMAKE_BUILD_TYPE} STREQUAL Debug) 17 | list(APPEND LOGGING_SOURCE_FILES backtrace.cpp) 18 | endif() 19 | 20 | add_library(sisl_logging) 21 | target_sources(sisl_logging PRIVATE 22 | ${LOGGING_SOURCE_FILES} 23 | ) 24 | target_link_libraries(sisl_logging PUBLIC ${LOGGING_DEPS} -rdynamic) 25 | 26 | add_executable(logging_example) 27 | target_sources(logging_example PRIVATE 28 | test/example.cpp 29 | ) 30 | target_link_libraries(logging_example sisl_logging) 31 | -------------------------------------------------------------------------------- /test_package/conanfile.py: -------------------------------------------------------------------------------- 1 | from conan import ConanFile 2 | from conan.tools.build import can_run 3 | from conan.tools.cmake import cmake_layout, CMake 4 | import os 5 | 6 | 7 | class TestPackageConan(ConanFile): 8 | settings = "os", "compiler", "build_type", "arch" 9 | generators = "CMakeDeps", "CMakeToolchain", "VirtualRunEnv" 10 | test_type = "explicit" 11 | 12 | def requirements(self): 13 | self.requires(self.tested_reference_str) 14 | 15 | def layout(self): 16 | cmake_layout(self) 17 | 18 | def build(self): 19 | cmake = CMake(self) 20 | cmake.configure() 21 | cmake.build() 22 | 23 | def test(self): 24 | if can_run(self): 25 | bin_path = os.path.join(self.cpp.build.bindir, "test_package") 26 | self.run(bin_path, env="conanrun") 27 | -------------------------------------------------------------------------------- /3rd_party/folly/patches/2022-001-compiler-flags.patch: -------------------------------------------------------------------------------- 1 | --- CMake/FollyCompilerUnix.cmake 2 | +++ CMake/FollyCompilerUnix.cmake 3 | @@ -28,9 +28,9 @@ set( 4 | ) 5 | mark_as_advanced(CXX_STD) 6 | 7 | -set(CMAKE_CXX_FLAGS_COMMON "-g -Wall -Wextra") 8 | +set(CMAKE_CXX_FLAGS_COMMON "-Wall -Wextra") 9 | set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} ${CMAKE_CXX_FLAGS_COMMON}") 10 | set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} ${CMAKE_CXX_FLAGS_COMMON} -O3") 11 | 12 | # Note that CMAKE_REQUIRED_FLAGS must be a string, not a list 13 | set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=${CXX_STD}") 14 | @@ -43,7 +43,6 @@ function(apply_folly_compile_options_to_target THETARGET) 15 | ) 16 | target_compile_options(${THETARGET} 17 | PRIVATE 18 | - -g 19 | -std=${CXX_STD} 20 | -finput-charset=UTF-8 21 | -fsigned-char 22 | -------------------------------------------------------------------------------- /src/settings/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_settings) 4 | target_sources(sisl_settings PRIVATE 5 | settings.cpp 6 | ) 7 | target_link_libraries(sisl_settings PUBLIC 8 | sisl_logging 9 | flatbuffers::flatbuffers 10 | userspace-rcu::userspace-rcu 11 | ) 12 | 13 | add_executable(test_settings) 14 | target_sources(test_settings PRIVATE 15 | tests/test_settings.cpp 16 | ) 17 | settings_gen_cpp( 18 | ${FLATBUFFERS_FLATC_EXECUTABLE} 19 | ${CMAKE_CURRENT_BINARY_DIR}/generated/ 20 | test_settings 21 | tests/test_app_schema.fbs 22 | ) 23 | if(NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) 24 | target_include_directories(test_settings BEFORE PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) 25 | endif() 26 | target_include_directories(test_settings BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 27 | target_link_libraries(test_settings sisl_settings GTest::gtest) 28 | add_test(NAME Settings COMMAND test_settings) 29 | -------------------------------------------------------------------------------- /.github/actions/setup_conan/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Setup Conan' 2 | description: 'Sets up Conan for Sisl Builds' 3 | inputs: 4 | platform: 5 | description: 'Platform conan will be building on' 6 | required: true 7 | default: 'ubuntu-22.04' 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Setup Python 12 | uses: actions/setup-python@v5 13 | with: 14 | python-version: "3.10" 15 | 16 | - name: Setup Conan and Export Recipes 17 | shell: bash 18 | run: | 19 | python -m pip install --upgrade pip 20 | python -m pip install conan~=1.0 21 | python -m pip install gcovr 22 | conan user 23 | conan profile new --detect default 24 | 25 | - name: Fixup libstdc++ 26 | shell: bash 27 | run: | 28 | # Set std::string to non-CoW C++11 version 29 | sed -i 's,compiler.libcxx=libstdc++$,compiler.libcxx=libstdc++11,g' ~/.conan/profiles/default 30 | if: ${{ inputs.platform == 'ubuntu-22.04' }} 31 | 32 | -------------------------------------------------------------------------------- /src/options/options.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Brian Szmyd 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | 19 | SISL_OPTION_GROUP(main, (help, "h", "help", "Help message", ::cxxopts::value< bool >(), "")) 20 | -------------------------------------------------------------------------------- /src/flip/proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | add_library(flip_proto OBJECT 4 | flip_server.proto 5 | flip_spec.proto 6 | ) 7 | protobuf_generate(LANGUAGE cpp TARGET flip_proto PROTOS flip_spec.proto) 8 | protobuf_generate(LANGUAGE cpp TARGET flip_proto PROTOS flip_server.proto) 9 | protobuf_generate( 10 | TARGET flip_proto 11 | LANGUAGE grpc 12 | GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc 13 | PLUGIN protoc-gen-grpc=$ 14 | ) 15 | target_link_libraries(flip_proto 16 | protobuf::libprotobuf 17 | gRPC::grpc++ 18 | ) 19 | 20 | add_custom_target(flip_py_proto ALL) 21 | protobuf_generate(LANGUAGE python TARGET flip_py_proto PROTOS flip_server.proto) 22 | protobuf_generate(LANGUAGE python TARGET flip_py_proto PROTOS flip_spec.proto) 23 | protobuf_generate( 24 | TARGET flip_py_proto 25 | PROTOS flip_server.proto flip_spec.proto 26 | LANGUAGE grpc 27 | GENERATE_EXTENSIONS _pb2_grpc.py 28 | PLUGIN protoc-gen-grpc=$) 29 | -------------------------------------------------------------------------------- /test_package/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | project(test_package) 3 | 4 | set(CMAKE_CXX_STANDARD 20) 5 | 6 | find_package(sisl QUIET REQUIRED) 7 | 8 | add_executable(${PROJECT_NAME} test_package.cpp example_decl.cpp) 9 | target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_20) 10 | target_link_libraries(${PROJECT_NAME} sisl::sisl) 11 | 12 | # Test linking just sisl::settings 13 | add_executable(${PROJECT_NAME}_log_only test_package.cpp example_decl.cpp) 14 | target_compile_features(${PROJECT_NAME}_log_only PUBLIC cxx_std_20) 15 | target_link_libraries(${PROJECT_NAME}_log_only sisl::logging) 16 | 17 | if (${gRPC_FOUND}) 18 | add_executable(${PROJECT_NAME}_grpc grpc_test.cpp) 19 | target_compile_features(${PROJECT_NAME}_grpc PUBLIC cxx_std_20) 20 | target_link_libraries(${PROJECT_NAME}_grpc sisl::grpc) 21 | 22 | # Test linking just sisl::grpc 23 | add_executable(${PROJECT_NAME}_grpc_only grpc_test.cpp) 24 | target_compile_features(${PROJECT_NAME}_grpc_only PUBLIC cxx_std_20) 25 | target_link_libraries(${PROJECT_NAME}_grpc_only sisl::grpc) 26 | endif() 27 | -------------------------------------------------------------------------------- /.github/actions/setup_conan2/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Setup Conan v2' 2 | description: 'Sets up Conan v2 for Sisl v13+ Builds' 3 | inputs: 4 | platform: 5 | description: 'Platform conan will be building on' 6 | required: true 7 | default: 'ubuntu-24.04' 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Setup Python 12 | uses: actions/setup-python@v5 13 | with: 14 | python-version: "3.10" 15 | 16 | - name: Setup Conan and Export Recipes 17 | shell: bash 18 | run: | 19 | python -m pip install --upgrade pip 20 | python -m pip install conan~=2.0 21 | python -m pip install gcovr 22 | conan profile detect --name default 23 | 24 | - name: Fixup libstdc++ 25 | shell: bash 26 | run: | 27 | # Set std::string to non-CoW C++11 version 28 | sed -i 's,compiler.libcxx=libstdc++$,compiler.libcxx=libstdc++11,g' ~/.conan2/profiles/default 29 | # Set C++ version to C++20 30 | sed -i 's,compiler.cppstd=.*$,compiler.cppstd=20,g' ~/.conan2/profiles/default 31 | if: ${{ inputs.platform == 'ubuntu-24.04' }} 32 | 33 | -------------------------------------------------------------------------------- /.github/workflows/pr_build.yml: -------------------------------------------------------------------------------- 1 | name: Sisl v13 Build 2 | 3 | on: 4 | workflow_dispatch: 5 | pull_request: 6 | branches: 7 | - stable/v12.x 8 | - master 9 | 10 | jobs: 11 | Build: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | platform: ["ubuntu-24.04"] 16 | build-type: ["Debug", "Release"] 17 | malloc-impl: ["libc", "tcmalloc"] 18 | tooling: ["Sanitize", "Coverage", "None"] 19 | exclude: 20 | - build-type: Debug 21 | tooling: None 22 | - build-type: Debug 23 | malloc-impl: tcmalloc 24 | - build-type: Release 25 | malloc-impl: libc 26 | - build-type: Release 27 | tooling: Sanitize 28 | - build-type: Release 29 | tooling: Coverage 30 | uses: ./.github/workflows/build_dependencies.yml 31 | with: 32 | platform: ${{ matrix.platform }} 33 | branch: ${{ github.ref }} 34 | build-type: ${{ matrix.build-type }} 35 | malloc-impl: ${{ matrix.malloc-impl }} 36 | tooling: ${{ matrix.tooling }} 37 | testing: 'True' 38 | -------------------------------------------------------------------------------- /src/metrics/tests/tree.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "tree.hpp" 18 | 19 | Tree::Tree(const char* grp_name) : m_metrics(grp_name) {} 20 | void Tree::update() { 21 | COUNTER_INCREMENT(m_metrics, tree_node_count, 1); 22 | COUNTER_INCREMENT(m_metrics, tree_obj_count, 4); 23 | COUNTER_INCREMENT(m_metrics, tree_obj_count, 8); 24 | COUNTER_INCREMENT(m_metrics, tree_txns, 2); 25 | } 26 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | if(${MALLOC_IMPL} STREQUAL "tcmalloc") 4 | find_package(gperftools REQUIRED) 5 | list(APPEND CORE_DEPS gperftools::gperftools) 6 | endif() 7 | 8 | if(${MALLOC_IMPL} STREQUAL "jemalloc") 9 | find_package(jemalloc REQUIRED) 10 | list(APPEND CORE_DEPS jemalloc::jemalloc) 11 | endif() 12 | 13 | find_package(Boost REQUIRED) 14 | find_package(cxxopts REQUIRED) 15 | find_package(nlohmann_json REQUIRED) 16 | find_package(spdlog REQUIRED) 17 | find_package(zmarok-semver REQUIRED) 18 | find_package(breakpad) 19 | 20 | add_subdirectory(logging) 21 | add_subdirectory(options) 22 | add_subdirectory(sobject) 23 | add_subdirectory(file_watcher) 24 | add_subdirectory(version) 25 | 26 | find_package(folly QUIET) 27 | if (${folly_FOUND}) 28 | find_package(benchmark REQUIRED) 29 | find_package(flatbuffers REQUIRED) 30 | find_package(prometheus-cpp REQUIRED) 31 | find_package(userspace-rcu REQUIRED) 32 | add_subdirectory(metrics) 33 | add_subdirectory(cache) 34 | add_subdirectory(fds) 35 | add_subdirectory(settings) 36 | add_subdirectory(utility) 37 | add_subdirectory(wisr) 38 | 39 | find_package(gRPC QUIET) 40 | if(${gRPC_FOUND}) 41 | add_subdirectory(flip) 42 | add_subdirectory(grpc) 43 | endif() 44 | endif() 45 | -------------------------------------------------------------------------------- /src/settings/tests/test_app_schema.fbs: -------------------------------------------------------------------------------- 1 | native_include "sisl/utility/non_null_ptr.hpp"; 2 | 3 | namespace testapp; 4 | 5 | attribute "hotswap"; 6 | attribute "deprecated"; 7 | 8 | table DatabaseSettings { 9 | databaseHost: string; 10 | databasePort: uint32 = 27017; 11 | numThreads: uint32 = 8; 12 | } 13 | 14 | table DBConnection { 15 | minDBConnections: uint32 = 2; 16 | maxDBConnections: uint32 = 10; 17 | dbConnectionOptimalLoad: uint64 = 100 (hotswap); 18 | dbConnectionMaxLoad: uint64 (deprecated); 19 | } 20 | 21 | table Glog { 22 | FLAGS_v: uint32 = 1; 23 | FLAGS_max_log_size: uint32 = 50; 24 | FLAGS_logtostderr: uint32 = 0; 25 | //Buffer log messages logged at this level or lower";0-INFO, 1-WARNING, 2-ERROR, 3-FATAL 26 | FLAGS_logbuflevel: int32 = 0; 27 | //copy GLOG levels at above this level to stderr; 28 | FLAGS_stderrthreshold: int32 = 3; 29 | FLAGS_alsologtostderr: uint32 = 0; 30 | FLAGS_vmodule: string (hotswap); 31 | } 32 | 33 | table Config { 34 | database: DatabaseSettings; 35 | dbconnection: DBConnection; 36 | glog: Glog; 37 | } 38 | 39 | table TestAppSettings { 40 | version: uint32; 41 | config: Config; 42 | } 43 | 44 | root_type TestAppSettings; 45 | 46 | -------------------------------------------------------------------------------- /src/metrics/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_metrics) 4 | target_sources(sisl_metrics PRIVATE 5 | metrics.cpp 6 | metrics_atomic.cpp 7 | metrics_group_impl.cpp 8 | metrics_rcu.cpp 9 | metrics_tlocal.cpp 10 | ) 11 | target_link_libraries(sisl_metrics PUBLIC 12 | sisl_logging 13 | folly::folly 14 | prometheus-cpp::prometheus-cpp 15 | userspace-rcu::userspace-rcu 16 | ) 17 | 18 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 19 | 20 | add_executable(metrics_farm_test) 21 | target_sources(metrics_farm_test PRIVATE 22 | tests/farm_test.cpp 23 | ) 24 | target_link_libraries(metrics_farm_test sisl_metrics GTest::gtest) 25 | add_test(NAME MetricsFarm COMMAND metrics_farm_test) 26 | 27 | add_executable(metrics_wrapper_test) 28 | target_sources(metrics_wrapper_test PRIVATE 29 | tests/wrapper_test.cpp 30 | ) 31 | target_link_libraries(metrics_wrapper_test sisl_metrics GTest::gtest) 32 | add_test(NAME MetricsWrapper COMMAND metrics_wrapper_test) 33 | 34 | add_executable(metrics_benchmark) 35 | target_sources(metrics_benchmark PRIVATE 36 | tests/metrics_benchmark.cpp 37 | ) 38 | target_link_libraries(metrics_benchmark sisl_metrics benchmark::benchmark) 39 | add_test(NAME MetricsBenchmark COMMAND metrics_benchmark) 40 | -------------------------------------------------------------------------------- /src/wisr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 4 | add_executable(wisr_vector_test) 5 | target_sources(wisr_vector_test PRIVATE 6 | tests/test_wisr_vector.cpp 7 | ) 8 | target_link_libraries(wisr_vector_test sisl_metrics benchmark::benchmark GTest::gtest) 9 | add_test(NAME WisrVector COMMAND wisr_vector_test) 10 | 11 | add_executable(wisr_vector_benchmark) 12 | target_sources(wisr_vector_benchmark PRIVATE 13 | tests/wisr_vector_benchmark.cpp 14 | ) 15 | target_link_libraries(wisr_vector_benchmark sisl_metrics benchmark::benchmark) 16 | 17 | add_executable(wisr_list_benchmark) 18 | target_sources(wisr_list_benchmark PRIVATE 19 | tests/wisr_list_benchmark.cpp 20 | ) 21 | target_link_libraries(wisr_list_benchmark sisl_metrics benchmark::benchmark) 22 | 23 | add_executable(wisr_deque_benchmark) 24 | target_sources(wisr_deque_benchmark PRIVATE 25 | tests/wisr_deque_benchmark.cpp 26 | ) 27 | target_link_libraries(wisr_deque_benchmark sisl_metrics benchmark::benchmark) 28 | 29 | add_executable(wisr_intrusive_slist_benchmark) 30 | target_sources(wisr_intrusive_slist_benchmark PRIVATE 31 | tests/wisr_intrusive_slist_benchmark.cpp 32 | ) 33 | target_link_libraries(wisr_intrusive_slist_benchmark sisl_metrics benchmark::benchmark) 34 | -------------------------------------------------------------------------------- /src/version/tests/test_version.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace sisl; 11 | 12 | SISL_OPTIONS_ENABLE(logging) 13 | SISL_LOGGING_INIT() 14 | 15 | void entry() { 16 | auto ver{version::Semver200_version(BOOST_PP_STRINGIZE(PACKAGE_VERSION))}; 17 | sisl::VersionMgr::addVersion("dummy", ver); 18 | } 19 | 20 | TEST(entryTest, entry) { 21 | entry(); 22 | 23 | std::stringstream dummy_ver; 24 | dummy_ver << sisl::VersionMgr::getVersion("dummy"); 25 | LOGINFO("Dummy ver. {}", dummy_ver.str()); 26 | 27 | std::stringstream sisl_ver; 28 | sisl_ver << sisl::VersionMgr::getVersion("sisl"); 29 | LOGINFO("SISL ver. {}", sisl_ver.str()); 30 | 31 | EXPECT_EQ(dummy_ver.str(), sisl_ver.str()); 32 | 33 | auto versions{sisl::VersionMgr::getVersions()}; 34 | EXPECT_EQ((int)versions.size(), 2); 35 | } 36 | 37 | int main(int argc, char* argv[]) { 38 | ::testing::InitGoogleTest(&argc, argv); 39 | SISL_OPTIONS_LOAD(argc, argv, logging); 40 | sisl::logging::SetLogger("test_version"); 41 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 42 | return RUN_ALL_TESTS(); 43 | } 44 | -------------------------------------------------------------------------------- /src/cache/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_cache) 4 | target_sources(sisl_cache PRIVATE 5 | lru_evictor.cpp 6 | ) 7 | target_link_libraries(sisl_cache PUBLIC 8 | sisl_buffer 9 | ) 10 | 11 | add_executable(test_range_hashmap) 12 | target_sources(test_range_hashmap PRIVATE 13 | tests/test_range_hashmap.cpp 14 | ) 15 | target_include_directories(test_range_hashmap BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 16 | target_link_libraries(test_range_hashmap sisl_cache GTest::gtest) 17 | #add_test(NAME RangeHashMap COMMAND test_range_hashmap --num_iters 10000) 18 | 19 | add_executable(test_range_cache) 20 | target_sources(test_range_cache PRIVATE 21 | tests/test_range_cache.cpp 22 | ) 23 | target_include_directories(test_range_cache BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 24 | target_link_libraries(test_range_cache sisl_cache GTest::gtest) 25 | #add_test(NAME RangeCache COMMAND test_range_cache --num_iters 1000) 26 | 27 | add_executable(test_simple_cache) 28 | target_sources(test_simple_cache PRIVATE 29 | tests/test_simple_cache.cpp 30 | ) 31 | target_include_directories(test_simple_cache BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) 32 | target_link_libraries(test_simple_cache sisl_cache GTest::gtest) 33 | add_test(NAME SimpleCache COMMAND test_simple_cache --num_iters 1000) 34 | -------------------------------------------------------------------------------- /src/flip/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.11) 2 | 3 | add_subdirectory (proto) 4 | 5 | if(NOT ${CMAKE_CURRENT_BINARY_DIR} STREQUAL ${CMAKE_CURRENT_SOURCE_DIR}) 6 | include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) 7 | endif() 8 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 9 | 10 | add_library(flip) 11 | target_sources(flip PRIVATE 12 | lib/flip_rpc_server.cpp 13 | $ 14 | ) 15 | target_link_libraries(flip PUBLIC 16 | sisl_logging 17 | gRPC::grpc++ 18 | spdlog::spdlog 19 | nlohmann_json::nlohmann_json 20 | ) 21 | 22 | add_executable(test_flip) 23 | target_sources(test_flip PRIVATE 24 | lib/test_flip.cpp 25 | ) 26 | target_link_libraries(test_flip flip cxxopts::cxxopts) 27 | add_test(NAME Flip COMMAND test_flip) 28 | 29 | add_executable(test_flip_server) 30 | target_sources(test_flip_server PRIVATE 31 | lib/test_flip_server.cpp 32 | ) 33 | target_link_libraries(test_flip_server flip cxxopts::cxxopts) 34 | 35 | add_executable(test_flip_local_client) 36 | target_sources(test_flip_local_client PRIVATE 37 | client/local/test_flip_local_client.cpp 38 | ) 39 | target_link_libraries(test_flip_local_client flip cxxopts::cxxopts) 40 | add_test(NAME FlipLocalClient COMMAND test_flip_local_client) 41 | -------------------------------------------------------------------------------- /src/flip/lib/test_flip_server.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "sisl/flip/flip.hpp" 18 | 19 | #include 20 | 21 | SISL_LOGGING_INIT(flip) 22 | 23 | SISL_OPTIONS_ENABLE(logging) 24 | 25 | int main(int argc, char* argv[]) { 26 | SISL_OPTIONS_LOAD(argc, argv, logging) 27 | sisl::logging::SetLogger(std::string(argv[0])); 28 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 29 | 30 | flip::Flip f; 31 | f.start_rpc_server(); 32 | 33 | sleep(1000); 34 | return 0; 35 | } 36 | -------------------------------------------------------------------------------- /3rd_party/folly/patches/2023-002-timespec.patch: -------------------------------------------------------------------------------- 1 | diff -Naur a/folly/io/async/AsyncSocket.cpp b/folly/io/async/AsyncSocket.cpp 2 | --- a/folly/io/async/AsyncSocket.cpp 2023-12-08 20:38:13.000000000 -0700 3 | +++ b/folly/io/async/AsyncSocket.cpp 2023-12-12 10:15:06.023030521 -0700 4 | @@ -18,6 +18,9 @@ 5 | 6 | #include 7 | 8 | +/* for struct sock_extended_err*/ 9 | +#include 10 | + 11 | #include 12 | #include 13 | #include 14 | diff -Naur a/folly/io/async/AsyncUDPSocket.cpp b/folly/io/async/AsyncUDPSocket.cpp 15 | --- a/folly/io/async/AsyncUDPSocket.cpp 2023-12-08 20:38:13.000000000 -0700 16 | +++ b/folly/io/async/AsyncUDPSocket.cpp 2023-12-12 10:19:06.419424565 -0700 17 | @@ -17,6 +17,9 @@ 18 | #include 19 | #include 20 | 21 | +/* for struct sock_extended_err*/ 22 | +#include 23 | + 24 | #include 25 | 26 | #include 27 | diff -Naur a/folly/net/NetOps.h b/folly/net/NetOps.h 28 | --- a/folly/net/NetOps.h 2023-12-12 10:16:10.675139766 -0700 29 | +++ b/folly/net/NetOps.h 2023-12-12 10:15:55.087113425 -0700 30 | @@ -114,7 +114,7 @@ 31 | #endif 32 | #endif 33 | /* for struct sock_extended_err*/ 34 | -#include 35 | +#include 36 | #endif 37 | #endif 38 | 39 | -------------------------------------------------------------------------------- /src/utility/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 4 | add_executable(test_atomic_counter) 5 | target_sources(test_atomic_counter PRIVATE 6 | tests/test_atomic_counter.cpp 7 | ) 8 | target_link_libraries(test_atomic_counter sisl_logging GTest::gtest) 9 | add_test(NAME AtomicCounter COMMAND test_atomic_counter) 10 | 11 | add_executable(test_thread_buffer) 12 | target_sources(test_thread_buffer PRIVATE 13 | tests/test_thread_buffer.cpp 14 | ) 15 | target_link_libraries(test_thread_buffer sisl_metrics GTest::gtest) 16 | add_test(NAME ThreadBuffer COMMAND test_thread_buffer) 17 | 18 | add_executable(test_status_factory) 19 | target_sources(test_status_factory PRIVATE 20 | tests/test_status_factory.cpp 21 | ) 22 | target_link_libraries(test_status_factory sisl_metrics benchmark::benchmark) 23 | add_test(NAME StatusFactory COMMAND test_status_factory) 24 | 25 | add_executable(test_enum) 26 | target_sources(test_enum PRIVATE 27 | tests/test_enum.cpp 28 | ) 29 | target_link_libraries(test_enum sisl_metrics GTest::gtest) 30 | add_test(NAME Enum COMMAND test_enum) 31 | 32 | add_executable(test_objlife) 33 | target_sources(test_objlife PRIVATE 34 | tests/test_objlife_counter.cpp 35 | ) 36 | target_link_libraries(test_objlife sisl_metrics GTest::gtest) 37 | add_test(NAME ObjLife COMMAND test_objlife) 38 | -------------------------------------------------------------------------------- /3rd_party/folly/patches/2024-001-timespec.patch: -------------------------------------------------------------------------------- 1 | diff -Naur a/folly/io/async/AsyncSocket.cpp b/folly/io/async/AsyncSocket.cpp 2 | --- a/folly/io/async/AsyncSocket.cpp 2024-08-12 09:28:44.000000000 +0000 3 | +++ b/folly/io/async/AsyncSocket.cpp 2025-10-22 16:13:34.439374937 +0000 4 | @@ -17,6 +17,8 @@ 5 | #include 6 | 7 | #include 8 | +/* for struct sock_extended_err*/ 9 | +#include 10 | 11 | #include 12 | #include 13 | diff -Naur a/folly/io/async/AsyncUDPSocket.cpp b/folly/io/async/AsyncUDPSocket.cpp 14 | --- a/folly/io/async/AsyncUDPSocket.cpp 2024-08-12 09:28:44.000000000 +0000 15 | +++ b/folly/io/async/AsyncUDPSocket.cpp 2025-10-22 16:13:34.440374956 +0000 16 | @@ -17,6 +17,8 @@ 17 | #include 18 | #include 19 | 20 | +/* for struct sock_extended_err*/ 21 | +#include 22 | #include 23 | 24 | #include 25 | diff -Naur a/folly/net/NetOps.h b/folly/net/NetOps.h 26 | --- a/folly/net/NetOps.h 2024-08-12 09:28:44.000000000 +0000 27 | +++ b/folly/net/NetOps.h 2025-10-22 16:14:31.665479235 +0000 28 | @@ -114,7 +114,7 @@ 29 | #endif 30 | #endif 31 | /* for struct sock_extended_err*/ 32 | -#include 33 | +#include 34 | #endif 35 | #endif 36 | 37 | -------------------------------------------------------------------------------- /src/logging/stacktrace_release.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #if defined(__linux__) 5 | #include 6 | #endif 7 | 8 | namespace sisl { 9 | namespace logging { 10 | 11 | static bool g_custom_signal_handler_installed{false}; 12 | static size_t g_custom_signal_handlers{0}; 13 | static bool g_crash_handle_all_threads{true}; 14 | static std::mutex g_hdlr_mutex; 15 | 16 | static void flush_logs() { // flush all logs 17 | spdlog::apply_all([&](std::shared_ptr< spdlog::logger > l) { 18 | if (l) l->flush(); 19 | }); 20 | std::this_thread::sleep_for(std::chrono::milliseconds{250}); 21 | } 22 | 23 | #if defined(__linux__) 24 | static bool dumpCallback(const google_breakpad::MinidumpDescriptor& descriptor, [[maybe_unused]] void*, 25 | bool succeeded) { 26 | std::cerr << std::endl << "Minidump path: " << descriptor.path() << std::endl; 27 | return succeeded; 28 | } 29 | #endif 30 | 31 | static void bt_dumper([[maybe_unused]] const SignalType signal_number) { 32 | #if defined(__linux__) 33 | google_breakpad::ExceptionHandler::WriteMinidump(get_base_dir().string(), dumpCallback, nullptr); 34 | #endif 35 | } 36 | 37 | static void log_stack_trace(const bool, const SignalType signal_number) { bt_dumper(signal_number); } 38 | 39 | } // namespace logging 40 | } // namespace sisl -------------------------------------------------------------------------------- /.github/actions/store_conan/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Store Conan Cache' 2 | description: 'Cleans Local Conan Cache and Persists Dirty Packages' 3 | inputs: 4 | key_prefix: 5 | description: 'Cache prefix' 6 | required: true 7 | default: 'Deps' 8 | runs: 9 | using: "composite" 10 | steps: 11 | - name: Setup Conan and Export Recipes 12 | shell: bash 13 | run: | 14 | if [ -d 3rd_party ]; then 15 | dep_pkgs=$(ls -1d 3rd_party/* 2>/dev/null | cut -d'/' -f2 | paste -sd'|' - -) 16 | fi 17 | if [ -z "${dep_pkgs}" ]; then 18 | dep_pkgs="no_3rd_party" 19 | fi 20 | dirty_pkgs=$(ls -1d ~/.conan/data/*/*/*/*/build 2>/dev/null | sed 's,.*data/,,') 21 | if [ -z "${dirty_pkgs}" ]; then 22 | dirty_pkgs="no_public/0" 23 | fi 24 | dirty_pkgs_d=$(echo "${dirty_pkgs}" | cut -d'/' -f1 | paste -sd'|' - -) 25 | echo "::info:: Caching: ${dirty_pkgs_d}|${dep_pkgs}" 26 | ls -1d ~/.conan/data/* | grep -Ev "(${dirty_pkgs_d}|${dep_pkgs})" | xargs rm -rf 27 | rm -rf ~/.conan/data/*/*/*/*/build 28 | rm -rf ~/.conan/data/*/*/*/*/source 29 | 30 | - name: Save Cache 31 | uses: actions/cache/save@v4 32 | with: 33 | path: | 34 | ~/.conan/data 35 | key: ${{ inputs.key_prefix }}-${{ hashFiles('conanfile.py', '3rd_party/**/conanfile.py') }} 36 | 37 | -------------------------------------------------------------------------------- /3rd_party/folly/conandata.yml: -------------------------------------------------------------------------------- 1 | sources: 2 | "nu2.2024.08.12.00": 3 | url: "https://github.com/facebook/folly/releases/download/v2024.08.12.00/folly-v2024.08.12.00.tar.gz" 4 | sha256: "18d7be721721db547cb9c5dd5cc50df05cd88b0a8e683e3126ec6f9ce2c41c4d" 5 | "nu2.2023.12.18.00": 6 | url: "https://github.com/facebook/folly/releases/download/v2023.12.18.00/folly-v2023.12.18.00.tar.gz" 7 | sha256: "57ce880e3ae7b4d4fe0980be64da9e6ca7dd09e2de477670bf984e11cf7739f2" 8 | "2022.10.31.00": 9 | url: "https://github.com/facebook/folly/releases/download/v2022.10.31.00/folly-v2022.10.31.00.tar.gz" 10 | sha256: "d7749f78eee2a327c1fa6b4a290e4bcd7115cdd7f7ef59f9e043ed59e597ab30" 11 | patches: 12 | "nu2.2024.08.12.00": 13 | - patch_file: "patches/2024-001-timespec.patch" 14 | patch_description: "Fix liburing inclusion of timespec" 15 | patch_type: "conan" 16 | "nu2.2023.12.18.00": 17 | - patch_file: "patches/2023-001-compiler-flags.patch" 18 | patch_description: "Do not hard-code debug flag for all build types" 19 | patch_type: "conan" 20 | - patch_file: "patches/2023-002-timespec.patch" 21 | patch_description: "Fix liburing inclusion of timespec" 22 | patch_type: "conan" 23 | "2022.10.31.00": 24 | - patch_file: "patches/2022-001-compiler-flags.patch" 25 | patch_description: "Do not hard-code debug flag for all build types" 26 | patch_type: "conan" 27 | -------------------------------------------------------------------------------- /src/metrics/tests/cache.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "cache.hpp" 18 | 19 | Cache::Cache(const char* grp_name) : m_metrics(grp_name) {} 20 | 21 | void Cache::update() { 22 | GAUGE_UPDATE(m_metrics, cache_size, 1); 23 | GAUGE_UPDATE(m_metrics, cache_size, 4); 24 | GAUGE_UPDATE(m_metrics, cache_eviction_pct, 8); 25 | GAUGE_UPDATE(m_metrics, cache_writes_rate, 2); 26 | 27 | HISTOGRAM_OBSERVE(m_metrics, cache_write_latency, 100); 28 | HISTOGRAM_OBSERVE(m_metrics, cache_write_latency, 150); 29 | HISTOGRAM_OBSERVE(m_metrics, cache_read_latency, 150); 30 | HISTOGRAM_OBSERVE(m_metrics, cache_delete_latency, 200); 31 | } 32 | -------------------------------------------------------------------------------- /src/utility/demangler.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | namespace sisl { 26 | template < typename T > 27 | struct DeMangler { 28 | static std::string name() { 29 | int status; 30 | const char* const realname{abi::__cxa_demangle(typeid(T).name(), 0, 0, &status)}; 31 | if (realname) { 32 | const std::string str{realname}; 33 | std::free(realname); 34 | return str; 35 | } else 36 | return std::string{}; 37 | } 38 | }; 39 | 40 | } // namespace sisl 41 | -------------------------------------------------------------------------------- /test_package/test_package.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | SISL_LOGGING_DECL(my_module) 6 | SISL_LOGGING_INIT(my_module) 7 | 8 | SISL_OPTIONS_ENABLE(logging) 9 | 10 | extern void example_decl(); 11 | 12 | using namespace std::chrono_literals; 13 | 14 | [[ maybe_unused ]] 15 | void crash() { volatile int* a = (int*)(NULL); *a = 1; } 16 | 17 | int main(int argc, char** argv) { 18 | SISL_OPTIONS_LOAD(argc, argv, logging) 19 | sisl::logging::SetLogger(std::string(argv[0])); 20 | sisl::logging::install_crash_handler(); 21 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 22 | 23 | LOGTRACE("Trace"); 24 | LOGDEBUG("Debug"); 25 | LOGINFO("Info"); 26 | LOGWARN("Warning"); 27 | LOGERROR("Error"); 28 | LOGCRITICAL("Critical"); 29 | 30 | auto _thread = std::thread([]() { 31 | LOGWARNMOD(my_module, "Sleeping..."); 32 | std::this_thread::sleep_for(1500ms); 33 | LOGINFOMOD(my_module, "Waking..."); 34 | std::this_thread::sleep_for(1500ms); 35 | }); 36 | sisl::name_thread(_thread, "example_decl"); 37 | std::this_thread::sleep_for(300ms); 38 | 39 | auto custom_logger = 40 | sisl::logging::CreateCustomLogger("test_package", "_custom", false /*stdout*/, true /*stderr*/); 41 | LOGINFOMOD_USING_LOGGER(my_module, custom_logger, "hello world"); 42 | DEBUG_ASSERT(true, "Always True"); 43 | _thread.join(); 44 | // crash(); 45 | return 0; 46 | } 47 | -------------------------------------------------------------------------------- /include/sisl/grpc/rpc_common.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed 10 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | * specific language governing permissions and limitations under the License. 13 | * 14 | *********************************************************************************/ 15 | #pragma once 16 | 17 | namespace sisl { 18 | class GrpcServer; 19 | class GenericRpcData; 20 | 21 | using generic_rpc_handler_cb_t = std::function< bool(boost::intrusive_ptr< GenericRpcData >&) >; 22 | using generic_rpc_completed_cb_t = std::function< void(boost::intrusive_ptr< GenericRpcData >&) >; 23 | 24 | struct RPCHelper { 25 | static bool has_server_shutdown(const GrpcServer* server); 26 | static bool run_generic_handler_cb(GrpcServer* server, const std::string& method, 27 | boost::intrusive_ptr< GenericRpcData >& rpc_data); 28 | static grpc::Status do_authorization(const GrpcServer* server, const grpc::ServerContext* srv_ctx); 29 | }; 30 | } // namespace sisl::grpc 31 | -------------------------------------------------------------------------------- /src/grpc/tests/proto/grpc_helper_test.proto: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed 10 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | * specific language governing permissions and limitations under the License. 13 | * 14 | *********************************************************************************/ 15 | syntax = "proto3"; 16 | 17 | package grpc_helper_test; 18 | 19 | service EchoService { 20 | rpc Echo(EchoRequest) returns (EchoReply) {} 21 | 22 | rpc EchoMetadata(EchoRequest) returns (EchoReply) {} 23 | 24 | rpc EchoLongReply(EchoRequest) returns (stream EchoReply) {} 25 | 26 | rpc LongEcho(stream EchoRequest) returns (EchoReply) {} 27 | 28 | rpc LongEchoLongReply(stream EchoRequest) returns (stream EchoReply) {} 29 | } 30 | 31 | message EchoRequest { string message = 1; } 32 | 33 | message EchoReply { string message = 1; } 34 | 35 | service PingService { 36 | rpc Ping(PingRequest) returns (PingReply) {} 37 | } 38 | 39 | message PingRequest { uint32 seqno = 1; } 40 | 41 | message PingReply { uint32 seqno = 1; } 42 | -------------------------------------------------------------------------------- /src/metrics/tests/tree.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "metrics.hpp" 18 | 19 | #ifndef ASYNC_HTTP_TREE_HPP 20 | #define ASYNC_HTTP_TREE_HPP 21 | 22 | using namespace sisl; 23 | 24 | class TreeMetrics : public MetricsGroupWrapper { 25 | public: 26 | explicit TreeMetrics(const char* grp_name) : MetricsGroupWrapper(grp_name) { 27 | REGISTER_COUNTER(tree_node_count, "tree_node_count", ""); 28 | REGISTER_COUNTER(tree_obj_count, "tree_obj_count", ""); 29 | REGISTER_COUNTER(tree_txns, "tree_txns", ""); 30 | 31 | register_me_to_farm(); 32 | } 33 | }; 34 | 35 | class Tree { 36 | private: 37 | TreeMetrics m_metrics; 38 | 39 | public: 40 | Tree(const char* grp_name); 41 | void update(); 42 | }; 43 | 44 | #endif // ASYNC_HTTP_TREE_HPP 45 | -------------------------------------------------------------------------------- /include/sisl/flip/flip_rpc_server.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | #include 19 | 20 | #include "proto/flip_spec.pb.h" 21 | #include "proto/flip_server.grpc.pb.h" 22 | 23 | namespace flip { 24 | class FlipRPCServer final : public FlipServer::Service { 25 | public: 26 | FlipRPCServer() = default; 27 | grpc::Status InjectFault(grpc::ServerContext* context, const FlipSpec* request, FlipResponse* response) override; 28 | grpc::Status GetFaults(grpc::ServerContext* context, const FlipNameRequest* request, 29 | FlipListResponse* response) override; 30 | grpc::Status RemoveFault(grpc::ServerContext*, const FlipRemoveRequest* request, 31 | FlipRemoveResponse* response) override; 32 | }; 33 | } // namespace flip 34 | -------------------------------------------------------------------------------- /src/metrics/tests/wrapper_test2.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include "cache.hpp" 22 | #include "tree.hpp" 23 | #include 24 | 25 | #define ITERATIONS 4 26 | 27 | RCU_REGISTER_INIT 28 | 29 | using namespace sisl; 30 | 31 | TEST(counterTest, wrapperTest) { 32 | Tree tree1("tree1"), tree2("tree2"); 33 | tree1.update(); 34 | tree2.update(); 35 | 36 | Cache cache1("cache1"), cache2("cache2"); 37 | cache1.update(); 38 | cache2.update(); 39 | 40 | auto output = MetricsFarm::getInstance().get_result_in_json_string(); 41 | std::cout << "Output of gather = " << output << "\n"; 42 | } 43 | 44 | int main(int argc, char* argv[]) { 45 | ::testing::InitGoogleTest(&argc, argv); 46 | return RUN_ALL_TESTS(); 47 | } 48 | -------------------------------------------------------------------------------- /include/sisl/auth_manager/token_verifier.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | 8 | namespace grpc { 9 | class Status; 10 | class ServerContext; 11 | } // namespace grpc 12 | 13 | namespace sisl { 14 | 15 | // An interface for verifing a token for authorization. This can be used in conjunction with the TokenClient which is an 16 | // interface to get a token. The implementation is deployment specific, one example is jwt based tokens provided by 17 | // ebay/TrustFabric 18 | 19 | ENUM(VerifyCode, uint8_t, OK, UNAUTH, FORBIDDEN) 20 | 21 | // This class represents the return value to the token verify call. 22 | // Derive from this class if the return value needs to contain some information from the decoded token. 23 | class TokenVerifyState { 24 | public: 25 | TokenVerifyState() = default; 26 | TokenVerifyState(VerifyCode const c, std::string const& m) : code(c), msg(m) {} 27 | virtual ~TokenVerifyState() {} 28 | VerifyCode code; 29 | std::string msg; 30 | }; 31 | 32 | using token_state_ptr = std::shared_ptr< TokenVerifyState >; 33 | 34 | class TokenVerifier { 35 | public: 36 | virtual ~TokenVerifier() = default; 37 | virtual token_state_ptr verify(std::string const& token) const = 0; 38 | }; 39 | 40 | // extracts the key value pairs (m_auth_header_key, get_token()) from grpc client context and verifies the token 41 | class GrpcTokenVerifier : public TokenVerifier { 42 | public: 43 | explicit GrpcTokenVerifier(std::string const& auth_header_key) : m_auth_header_key(auth_header_key) {} 44 | virtual ~GrpcTokenVerifier() = default; 45 | 46 | virtual grpc::Status verify_ctx(grpc::ServerContext const* srv_ctx) const = 0; 47 | 48 | protected: 49 | std::string m_auth_header_key; 50 | }; 51 | 52 | } // namespace sisl 53 | -------------------------------------------------------------------------------- /src/fds/tests/test_obj_allocator.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | 20 | #include "sisl/logging/logging.h" 21 | #include "sisl/options/options.h" 22 | 23 | #include "sisl/fds/obj_allocator.hpp" 24 | 25 | SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) 26 | 27 | using namespace sisl; 28 | using namespace std; 29 | 30 | namespace { 31 | template < typename T > 32 | class Node { 33 | public: 34 | Node(const T& id) { m_id = id; } 35 | 36 | const T& get_id() const { return m_id; } 37 | 38 | ~Node() { std::cout << "Destructor of Node " << m_id << " called\n"; } 39 | 40 | private: 41 | T m_id; 42 | }; 43 | } // namespace 44 | 45 | int main() { 46 | Node< uint64_t >* const ptr1{sisl::ObjectAllocator< Node< uint64_t > >::make_object(~static_cast< uint64_t >(0))}; 47 | std::cout << "ptr1 = " << static_cast< const void* >(ptr1) << " Id = " << ptr1->get_id() << std::endl; 48 | sisl::ObjectAllocator< Node< uint64_t > >::deallocate(ptr1); 49 | } 50 | -------------------------------------------------------------------------------- /include/sisl/version.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Sounak Gupta 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace sisl { 24 | 25 | typedef std::pair< std::string, version::Semver200_version > modinfo; 26 | 27 | class VersionMgr { 28 | private: 29 | mutable std::mutex m_mutex; 30 | std::unordered_map< std::string, version::Semver200_version > m_version_map; 31 | 32 | VersionMgr() = default; 33 | 34 | static VersionMgr* m_instance; 35 | static std::once_flag m_init_flag; 36 | static void createAndInit(); 37 | 38 | public: 39 | VersionMgr(VersionMgr const&) = delete; 40 | void operator=(VersionMgr const&) = delete; 41 | 42 | static VersionMgr* getInstance(); 43 | static version::Semver200_version getVersion(const std::string& name); 44 | static std::vector< modinfo > getVersions(); 45 | static void addVersion(const std::string& name, const version::Semver200_version& ver); 46 | }; 47 | 48 | } // namespace sisl 49 | -------------------------------------------------------------------------------- /src/metrics/tests/cache.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "metrics.hpp" 18 | 19 | #ifndef ASYNC_HTTP_CACHE_HPP 20 | #define ASYNC_HTTP_CACHE_HPP 21 | 22 | using namespace sisl; 23 | 24 | class CacheMetrics : public MetricsGroupWrapper { 25 | public: 26 | explicit CacheMetrics(const char* grp_name) : MetricsGroupWrapper(grp_name) { 27 | REGISTER_GAUGE(cache_size, "cache_size", ""); 28 | REGISTER_GAUGE(cache_eviction_pct, "cache_eviction_pct", ""); 29 | REGISTER_GAUGE(cache_writes_rate, "cache_writes_rate", ""); 30 | 31 | REGISTER_HISTOGRAM(cache_write_latency, "cache_write_latency", ""); 32 | REGISTER_HISTOGRAM(cache_read_latency, "cache_read_latency", ""); 33 | REGISTER_HISTOGRAM(cache_delete_latency, "cache_delete_latency", ""); 34 | 35 | register_me_to_farm(); 36 | } 37 | }; 38 | 39 | class Cache { 40 | private: 41 | CacheMetrics m_metrics; 42 | 43 | public: 44 | Cache(const char* grp_name); 45 | void update(); 46 | }; 47 | #endif // ASYNC_HTTP_CACHE_HPP 48 | -------------------------------------------------------------------------------- /include/sisl/file_watcher/file_watcher.hpp: -------------------------------------------------------------------------------- 1 | // This implementation of file watcher only works on Linux machines. 2 | #pragma once 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | namespace sisl { 9 | 10 | using file_event_cb_t = std::function< void(const std::string, const bool) >; 11 | 12 | // structure to hold file contents and closures to be run 13 | struct FileInfo { 14 | FileInfo() {} 15 | std::string m_filepath; 16 | // file contents 17 | std::string m_filecontents; 18 | // closures to be called when file modification is detected, one per listener 19 | std::map< std::string, file_event_cb_t > m_handlers; 20 | int m_wd; 21 | }; 22 | 23 | class FileWatcher { 24 | public: 25 | FileWatcher() = default; 26 | 27 | bool start(); 28 | bool register_listener(const std::string& file_path, const std::string& listener_id, 29 | const file_event_cb_t& file_event_handler); 30 | bool unregister_listener(const std::string& file_path, const std::string& listener_id); 31 | bool stop(); 32 | 33 | private: 34 | void run(); 35 | void handle_events(); 36 | void get_fileinfo(const int wd, FileInfo& file_info) const; 37 | void set_fileinfo_content(const int wd, const std::string&); 38 | void on_modified_event(const int wd, const bool is_deleted); 39 | bool remove_watcher(FileInfo& file_info); 40 | static bool get_file_contents(const std::string& file_name, std::string& contents); 41 | static bool check_file_size(const std::string& file_path); 42 | 43 | private: 44 | int m_inotify_fd; 45 | std::map< std::string, FileInfo > m_files; 46 | mutable std::mutex m_files_lock; 47 | std::unique_ptr< std::thread > m_fw_thread; 48 | // This is used to notify poll loop to exit 49 | int m_pipefd[2] = {-1, -1}; 50 | }; 51 | 52 | } // namespace sisl 53 | -------------------------------------------------------------------------------- /.github/actions/load_conan/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Load Conan Cache' 2 | description: 'Loads Local Conan Cache' 3 | inputs: 4 | testing: 5 | description: 'Support building tests' 6 | required: true 7 | key_prefix: 8 | description: 'Cache prefix' 9 | required: true 10 | default: 'Deps' 11 | fail_on_cache_miss: 12 | description: 'Fail if key missing' 13 | required: false 14 | default: false 15 | path: 16 | description: 'Recipe path' 17 | required: false 18 | default: '.' 19 | load_any: 20 | description: 'Load cache miss' 21 | required: false 22 | default: 'False' 23 | outputs: 24 | cache-hit: 25 | description: 'Cache match found' 26 | value: ${{ steps.restore-cache.outputs.cache-hit }} 27 | runs: 28 | using: "composite" 29 | steps: 30 | - id: hash-key-primary 31 | shell: bash 32 | run: | 33 | echo "key=${{ inputs.path }}/conanfile.py" >> $GITHUB_OUTPUT 34 | 35 | - id: hash-key-3rd 36 | shell: bash 37 | run: | 38 | echo "keys=${{ inputs.path }}/3rd_party/**/conanfile.py" >> $GITHUB_OUTPUT 39 | 40 | - name: Restore Cache 41 | id: restore-cache 42 | uses: actions/cache/restore@v4 43 | with: 44 | path: | 45 | ~/.conan/data 46 | key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} 47 | fail-on-cache-miss: ${{ inputs.fail_on_cache_miss }} 48 | 49 | - name: Restore Testing Cache 50 | uses: actions/cache/restore@v4 51 | with: 52 | path: | 53 | ~/.conan/data 54 | key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} 55 | restore-keys: ${{ inputs.key_prefix }}- 56 | if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && (( github.event_name == 'pull_request' && inputs.testing == 'True' ) || ( inputs.load_any == 'True' )) }} 57 | 58 | -------------------------------------------------------------------------------- /3rd_party/folly/conan_deps.cmake: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.17) 2 | # Set the dependency flags expected by https://github.com/facebook/folly/blob/v2023.12.18.00/CMake/folly-deps.cmake 3 | 4 | macro(custom_find_package package_name variable_prefix) 5 | find_package(${package_name} REQUIRED CONFIG ${ARGN}) 6 | list(APPEND FROM FOUND VERSION VERSION_STRING INCLUDE_DIRS INCLUDE_DIR INCLUDE_DIR LIBRARIES LIBRARIES LIBRARIES DEFINITIONS) 7 | list(APPEND TO FOUND VERSION VERSION_STRING INCLUDE_DIRS INCLUDE_DIR INCLUDE LIB LIBRARY LIBRARIES DEFINITIONS) 8 | 9 | foreach (from_substr to_substr IN ZIP_LISTS FROM TO) 10 | set(src_var ${package_name}_${from_substr}) 11 | set(dst_var ${variable_prefix}_${to_substr}) 12 | if (NOT DEFINED ${src_var}) 13 | continue() 14 | endif() 15 | if ((DEFINED ${dst_var}) AND ("${${dst_var}}" STREQUAL "${${src_var}}")) 16 | # if they're equal, skip 17 | continue() 18 | endif() 19 | message(DEBUG "custom_find_package definining ${dst_var} with ${src_var} contents: ${${src_var}}") 20 | set(${dst_var} ${${src_var}}) 21 | endforeach() 22 | endmacro() 23 | 24 | custom_find_package(BZip2 BZIP2) 25 | custom_find_package(DoubleConversion DOUBLE_CONVERSION REQUIRED) 26 | custom_find_package(Gflags LIBGFLAGS) 27 | custom_find_package(Glog GLOG) 28 | custom_find_package(LZ4 LZ4) 29 | custom_find_package(LibEvent LIBEVENT REQUIRED) 30 | custom_find_package(LibLZMA LIBLZMA) 31 | custom_find_package(Libsodium LIBSODIUM) 32 | custom_find_package(OpenSSL OPENSSL REQUIRED) 33 | custom_find_package(Snappy SNAPPY) 34 | custom_find_package(ZLIB ZLIB) 35 | custom_find_package(Zstd ZSTD) 36 | custom_find_package(fmt FMT REQUIRED) 37 | 38 | if (NOT MSVC) 39 | custom_find_package(LibDwarf LIBDWARF) 40 | endif() 41 | if (UNIX AND NOT APPLE) 42 | custom_find_package(LibUring LIBURING) 43 | custom_find_package(LibUnwind LIBUNWIND) 44 | custom_find_package(Libiberty LIBIBERTY) 45 | endif() -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled directories 2 | bin/ 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | ### Vi Swap Files 35 | *.swp 36 | 37 | ### JetBrains template 38 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 39 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 40 | 41 | # User-specific stuff: 42 | .idea/workspace.xml 43 | .idea/tasks.xml 44 | .idea/dictionaries 45 | .idea/vcs.xml 46 | .idea/jsLibraryMappings.xml 47 | 48 | # Sensitive or high-churn files: 49 | .idea/dataSources.ids 50 | .idea/dataSources.xml 51 | .idea/dataSources.local.xml 52 | .idea/sqlDataSources.xml 53 | .idea/dynamic.xml 54 | .idea/uiDesigner.xml 55 | 56 | # Gradle: 57 | .idea/gradle.xml 58 | .idea/libraries 59 | 60 | # Mongo Explorer plugin: 61 | .idea/mongoSettings.xml 62 | 63 | ## File-based project format: 64 | *.iws 65 | 66 | ## Plugin-specific files: 67 | 68 | # IntelliJ 69 | /out/ 70 | 71 | # mpeltonen/sbt-idea plugin 72 | .idea_modules/ 73 | 74 | # JIRA plugin 75 | atlassian-ide-plugin.xml 76 | 77 | # Runtime data 78 | pids 79 | *.pid 80 | *.seed 81 | Testing/Temporary 82 | 83 | *.cbp 84 | conanbuildinfo.txt 85 | conaninfo.txt 86 | .idea/** 87 | 88 | # ctags 89 | tags* 90 | .tags* 91 | src/tags* 92 | src/.tags* 93 | 94 | # build generated products 95 | build/** 96 | test_package/build 97 | debug/** 98 | release/** 99 | cmake-*/** 100 | 101 | # vscode 102 | .vscode/** 103 | 104 | # Visual Studio 105 | CMakeUserPresets.json 106 | CMakeSettings.json 107 | .vs/** 108 | 109 | # Clangd 110 | .cache/clangd 111 | 112 | CMakeUserPresets.json 113 | logs/ 114 | conan.lock 115 | graph_info.json 116 | -------------------------------------------------------------------------------- /.github/workflows/merge_build.yml: -------------------------------------------------------------------------------- 1 | name: Sisl v13 Build 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | branches: 7 | - stable/v12.x 8 | - master 9 | 10 | jobs: 11 | Build: 12 | strategy: 13 | fail-fast: false 14 | matrix: 15 | platform: ["ubuntu-24.04"] 16 | build-type: ["Debug", "Release"] 17 | malloc-impl: ["libc", "tcmalloc"] 18 | tooling: ["Sanitize", "Coverage", "None"] 19 | exclude: 20 | - build-type: Debug 21 | tooling: None 22 | - build-type: Debug 23 | malloc-impl: tcmalloc 24 | - build-type: Release 25 | malloc-impl: libc 26 | - build-type: Release 27 | tooling: Sanitize 28 | - build-type: Release 29 | tooling: Coverage 30 | uses: ./.github/workflows/build_dependencies.yml 31 | with: 32 | platform: ${{ matrix.platform }} 33 | branch: ${{ github.ref }} 34 | build-type: ${{ matrix.build-type }} 35 | malloc-impl: ${{ matrix.malloc-impl }} 36 | tooling: ${{ matrix.tooling }} 37 | testing: 'True' 38 | ChainBuild: 39 | runs-on: "ubuntu-24.04" 40 | steps: 41 | - name: Start IOManager Build 42 | run: | 43 | curl -L \ 44 | -X POST \ 45 | -H "Accept: application/vnd.github+json" \ 46 | -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ 47 | -H "X-GitHub-Api-Version: 2022-11-28" \ 48 | https://api.github.com/repos/eBay/iomanager/actions/workflows/merge_build.yml/dispatches \ 49 | -d '{"ref":"master","inputs":{}}' 50 | if: ${{ github.ref == 'refs/heads/master' }} 51 | - name: Start NuraftMesg Build 52 | run: | 53 | curl -L \ 54 | -X POST \ 55 | -H "Accept: application/vnd.github+json" \ 56 | -H "Authorization: Bearer ${{ secrets.CHAIN_BUILD_TOKEN }}"\ 57 | -H "X-GitHub-Api-Version: 2022-11-28" \ 58 | https://api.github.com/repos/eBay/nuraft_mesg/actions/workflows/conan_build.yml/dispatches \ 59 | -d '{"ref":"main","inputs":{}}' 60 | if: ${{ github.ref == 'refs/heads/master' }} 61 | -------------------------------------------------------------------------------- /.github/actions/load_conan2/action.yml: -------------------------------------------------------------------------------- 1 | name: 'Load Conan Cache' 2 | description: 'Loads Local Conan Cache' 3 | inputs: 4 | testing: 5 | description: 'Support building tests' 6 | required: true 7 | key_prefix: 8 | description: 'Cache prefix' 9 | required: true 10 | default: 'Deps' 11 | fail_on_cache_miss: 12 | description: 'Fail if key missing' 13 | required: false 14 | default: false 15 | path: 16 | description: 'Recipe path' 17 | required: false 18 | default: '.' 19 | load_any: 20 | description: 'Load cache miss' 21 | required: false 22 | default: 'False' 23 | outputs: 24 | cache-hit: 25 | description: 'Cache match found' 26 | value: ${{ steps.restore-cache.outputs.cache-hit }} 27 | runs: 28 | using: "composite" 29 | steps: 30 | - id: hash-key-primary 31 | shell: bash 32 | run: | 33 | echo "key=${{ inputs.path }}/conanfile.py" >> $GITHUB_OUTPUT 34 | 35 | - id: hash-key-3rd 36 | shell: bash 37 | run: | 38 | echo "keys=${{ inputs.path }}/3rd_party/**/conanfile.py" >> $GITHUB_OUTPUT 39 | 40 | - name: Restore Cache 41 | id: restore-cache 42 | uses: actions/cache/restore@v4 43 | with: 44 | path: | 45 | ~/conan_cache_save.tgz 46 | key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} 47 | fail-on-cache-miss: ${{ inputs.fail_on_cache_miss }} 48 | 49 | - name: Restore Testing Cache 50 | id: restore-test-cache 51 | uses: actions/cache/restore@v4 52 | with: 53 | path: | 54 | ~/conan_cache_save.tgz 55 | key: ${{ inputs.key_prefix }}-${{ hashFiles(steps.hash-key-primary.outputs.key, steps.hash-key-3rd.outputs.keys) }} 56 | restore-keys: ${{ inputs.key_prefix }}- 57 | if: ${{ steps.restore-cache.outputs.cache-hit != 'true' && (( github.event_name == 'pull_request' && inputs.testing == 'True' ) || ( inputs.load_any == 'True' )) }} 58 | 59 | - name: Restore Tarball 60 | shell: bash 61 | run: | 62 | if [ -r ~/conan_cache_save.tgz ]; then conan cache restore ~/conan_cache_save.tgz; fi 63 | -------------------------------------------------------------------------------- /include/sisl/utility/status_factory.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | #include "urcu_helper.hpp" 19 | #include 20 | 21 | namespace sisl { 22 | template < typename StatusT > 23 | class StatusFactory { 24 | public: 25 | template < class... Args1 > 26 | StatusFactory(Args1&&... args) { 27 | m_cur_status = new StatusT(std::forward< Args1 >(args)...); 28 | } 29 | 30 | void readable(const auto& cb) const { 31 | rcu_read_lock(); 32 | auto s = rcu_dereference(m_cur_status); 33 | cb((const StatusT*)s); 34 | rcu_read_unlock(); 35 | } 36 | 37 | urcu_scoped_ptr< StatusT > access() const { return urcu_scoped_ptr< StatusT >(m_cur_status); } 38 | 39 | void updateable(const auto& edit_cb) { 40 | std::scoped_lock l(m_updater_mutex); 41 | StatusT* new_ptr = new StatusT((const StatusT&)*(access().get())); // Create new status from old status 42 | edit_cb(new_ptr); 43 | 44 | auto old_ptr = rcu_xchg_pointer(&m_cur_status, new_ptr); 45 | synchronize_rcu(); 46 | delete old_ptr; 47 | } 48 | 49 | private: 50 | /* RCU protected status data */ 51 | StatusT* m_cur_status = nullptr; 52 | 53 | // Mutex to protect multiple updaters to run the copy step in parallel. 54 | std::mutex m_updater_mutex; 55 | }; 56 | } // namespace sisl 57 | -------------------------------------------------------------------------------- /src/options/tests/basic.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Brian Szmyd 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | SISL_OPTION_GROUP(logging, 26 | (verbosity, "v", "verbosity", "Verbosity level (0-5)", 27 | ::cxxopts::value< uint32_t >()->default_value("2"), "level"), 28 | (synclog, "s", "synclog", "Synchronized logging", ::cxxopts::value< bool >(), "")) 29 | 30 | SISL_OPTIONS_ENABLE(logging) 31 | 32 | TEST(Options, Loaded) { EXPECT_EQ(2u, SISL_OPTIONS["verbosity"].as< uint32_t >()); } 33 | 34 | TEST(Options, AllTrue) { 35 | const bool result1{sisl::options::all_true< true, true, true >::value}; 36 | EXPECT_TRUE(result1); 37 | const bool result2{sisl::options::all_true< true, true, false >::value}; 38 | const bool result3{sisl::options::all_true< true, false, true >::value}; 39 | const bool result4{sisl::options::all_true< true, false, true >::value}; 40 | EXPECT_FALSE(result2); 41 | EXPECT_FALSE(result3); 42 | EXPECT_FALSE(result4); 43 | } 44 | 45 | int main(int argc, char* argv[]) { 46 | ::testing::InitGoogleTest(&argc, argv); 47 | argc = 1; 48 | SISL_OPTIONS_LOAD(argc, argv, logging); 49 | if (SISL_OPTIONS.count("synclog")) { std::cout << "Sync log enabled!" << std::endl; } 50 | return RUN_ALL_TESTS(); 51 | } 52 | -------------------------------------------------------------------------------- /src/fds/buffer.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include "sisl/fds/buffer.hpp" 19 | 20 | namespace sisl { 21 | uint8_t* AlignedAllocatorImpl::aligned_alloc(const size_t align, const size_t sz, const sisl::buftag tag) { 22 | auto* buf{static_cast< uint8_t* >(std::aligned_alloc(align, sisl::round_up(sz, align)))}; 23 | AlignedAllocator::metrics().increment(tag, buf_size(buf)); 24 | return buf; 25 | } 26 | 27 | void AlignedAllocatorImpl::aligned_free(uint8_t* const b, const sisl::buftag tag) { 28 | AlignedAllocator::metrics().decrement(tag, buf_size(b)); 29 | return std::free(b); 30 | } 31 | 32 | uint8_t* AlignedAllocatorImpl::aligned_realloc(uint8_t* const old_buf, const size_t align, const size_t new_sz, 33 | const size_t old_sz) { 34 | // Glibc does not have an implementation of efficient realloc and hence we are using alloc/copy method here 35 | const size_t old_real_size{(old_sz == 0) ? ::malloc_usable_size(static_cast< void* >(old_buf)) : old_sz}; 36 | if (old_real_size >= new_sz) return old_buf; 37 | 38 | uint8_t* const new_buf{this->aligned_alloc(align, sisl::round_up(new_sz, align), buftag::common)}; 39 | std::memcpy(static_cast< void* >(new_buf), static_cast< const void* >(old_buf), old_real_size); 40 | 41 | aligned_free(old_buf, buftag::common); 42 | return new_buf; 43 | } 44 | } // namespace sisl 45 | -------------------------------------------------------------------------------- /include/sisl/fds/compress.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Yaming Kuang 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | #include 19 | // #include 20 | 21 | namespace sisl { 22 | #if 0 23 | static inline int compress_bound(size_t size) { return LZ4_compressBound(size); } 24 | 25 | static inline int compress(const char* src, char* dst, int src_size, int dst_capacity) { 26 | return LZ4_compress_default(src, dst, src_size, dst_capacity); 27 | } 28 | 29 | static inline int decompress(const char* src, char* dst, int compressed_size, int dst_capacity) { 30 | return LZ4_decompress_safe(src, dst, compressed_size, dst_capacity); 31 | } 32 | #endif 33 | 34 | class Compress { 35 | public: 36 | static size_t max_compress_len(size_t size) { return snappy_max_compressed_length(size); } 37 | 38 | static int compress(const char* src, char* dst, size_t src_size, size_t* dst_capacity) { 39 | auto ret = snappy_compress(src, src_size, dst, dst_capacity); 40 | if (ret == SNAPPY_OK) { 41 | return 0; 42 | } else { 43 | return ret; 44 | } 45 | } 46 | 47 | static int decompress(const char* src, char* dst, size_t compressed_size, size_t* dst_capacity) { 48 | auto ret = snappy_uncompress(src, compressed_size, dst, dst_capacity); 49 | if (ret == SNAPPY_OK) { 50 | return 0; 51 | } else { 52 | return ret; 53 | } 54 | } 55 | 56 | private: 57 | }; 58 | 59 | } // namespace sisl 60 | -------------------------------------------------------------------------------- /src/flip/client/python/flip_client_example.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import random 4 | import logging 5 | from flip_rpc_client import * 6 | 7 | if __name__ == '__main__': 8 | logging.basicConfig() 9 | fclient = FlipRPCClient('localhost:50051') 10 | 11 | fclient.inject_test_flip("flip1", 12 | fspec.FlipFrequency(count=4, percent=100), 13 | [ 14 | fspec.FlipCondition(oper=fspec.Operator.NOT_EQUAL, value=fspec.ParamValue(int_value=5)), 15 | fspec.FlipCondition(oper=fspec.Operator.DONT_CARE) 16 | ]) 17 | fclient.inject_test_flip("flip2", 18 | fspec.FlipFrequency(count=2, every_nth=5), 19 | []) 20 | 21 | fclient.inject_ret_flip("flip3", 22 | fspec.FlipFrequency(count=2, percent=100), 23 | [ 24 | fspec.FlipCondition(oper=fspec.Operator.NOT_EQUAL, value=fspec.ParamValue(int_value=5)), 25 | ], 26 | fspec.ParamValue(string_value="Simulated corruption") 27 | ) 28 | 29 | fclient.inject_delay_flip("flip4", 30 | fspec.FlipFrequency(count=10000, percent=100), 31 | [ 32 | fspec.FlipCondition(oper=fspec.Operator.GREATER_THAN_OR_EQUAL, 33 | value=fspec.ParamValue(long_value=50000)), 34 | ], 35 | 1000 36 | ) 37 | 38 | fclient.inject_delay_ret_flip("flip5", 39 | fspec.FlipFrequency(count=1, percent=50), 40 | [ 41 | fspec.FlipCondition(oper=fspec.Operator.LESS_THAN_OR_EQUAL, 42 | value=fspec.ParamValue(double_value=800.15)), 43 | ], 44 | 1000, 45 | fspec.ParamValue(bool_value=False) 46 | ) 47 | 48 | fclient.flip_details("flip2") 49 | fclient.all_flip_details() -------------------------------------------------------------------------------- /include/sisl/metrics/metrics_rcu.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | 26 | #include "histogram_buckets.hpp" 27 | #include "metrics_tlocal.hpp" 28 | 29 | namespace sisl { 30 | using WisrBufferMetrics = 31 | sisl::wisr_framework< PerThreadMetrics, const std::vector< HistogramStaticInfo >&, uint32_t, uint32_t >; 32 | 33 | class WisrBufferMetricsGroup : public MetricsGroupImpl { 34 | public: 35 | WisrBufferMetricsGroup(const char* grp_name, const char* inst_name) : MetricsGroupImpl(grp_name, inst_name) {} 36 | WisrBufferMetricsGroup(const std::string& grp_name, const std::string& inst_name) : 37 | MetricsGroupImpl(grp_name, inst_name) {} 38 | 39 | void counter_increment(uint64_t index, int64_t val = 1) override; 40 | void counter_decrement(uint64_t index, int64_t val = 1) override; 41 | void histogram_observe(uint64_t index, int64_t val) override; 42 | void histogram_observe(uint64_t index, int64_t val, uint64_t count) override; 43 | 44 | group_impl_type_t impl_type() const { return group_impl_type_t::rcu; } 45 | 46 | private: 47 | void on_register(); 48 | void gather_result(bool need_latest, const counter_gather_cb_t& counter_cb, const gauge_gather_cb_t& gauge_cb, 49 | const histogram_gather_cb_t& histogram_cb) override; 50 | 51 | private: 52 | std::unique_ptr< WisrBufferMetrics > m_metrics; 53 | }; 54 | } // namespace sisl 55 | -------------------------------------------------------------------------------- /src/flip/client/python/flip_rpc_client.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | 3 | import random 4 | import logging 5 | import sys 6 | 7 | import grpc 8 | import flip_spec_pb2 as fspec 9 | import flip_spec_pb2_grpc 10 | import flip_server_pb2 11 | import flip_server_pb2_grpc 12 | 13 | class FlipRPCClient: 14 | def __init__(self, server_address): 15 | self.channel = grpc.insecure_channel(server_address) 16 | self.stub = flip_server_pb2_grpc.FlipServerStub(self.channel) 17 | 18 | def inject_fault(self, name, freq, conds, action): 19 | self.stub.InjectFault(fspec.FlipSpec(flip_name=name, conditions=conds, flip_action=action, flip_frequency=freq)) 20 | 21 | def inject_test_flip(self, name, freq, conds): 22 | print("------ Inject test flip", name, "-------------") 23 | self.inject_fault(name, freq, conds, fspec.FlipAction(no_action=1)) 24 | 25 | def inject_ret_flip(self, name, freq, conds, retval): 26 | print("------ Inject ret flip", name, "-------------") 27 | self.inject_fault(name, freq, conds, fspec.FlipAction(returns=fspec.FlipAction.ActionReturns(retval=retval))) 28 | 29 | def inject_delay_flip(self, name, freq, conds, delay_usec): 30 | print("------ Inject delay flip", name, "-------------") 31 | self.inject_fault(name, freq, conds, 32 | fspec.FlipAction(delays=fspec.FlipAction.ActionDelays(delay_in_usec=delay_usec))) 33 | 34 | def inject_delay_ret_flip(self, name, freq, conds, delay_usec, retval): 35 | print("------ Inject delay and then ret flip", name, "-------------") 36 | self.inject_fault(name, freq, conds, 37 | fspec.FlipAction(delay_returns=fspec.FlipAction.ActionDelayedReturns( 38 | delay_in_usec=delay_usec, 39 | retval=retval))) 40 | 41 | def flip_details(self, name): 42 | list_response = self.stub.GetFaults(flip_server_pb2.FlipNameRequest(name=name)) 43 | for finfo in list_response.infos: 44 | print(finfo.info) 45 | 46 | def all_flip_details(self): 47 | list_response = self.stub.GetFaults(flip_server_pb2.FlipNameRequest(name=None)) 48 | for finfo in list_response.infos: 49 | print(finfo.info) 50 | -------------------------------------------------------------------------------- /src/version/version.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Brian Szmyd 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | 19 | #include 20 | #include 21 | 22 | namespace sisl { 23 | 24 | VersionMgr* VersionMgr::m_instance = nullptr; 25 | std::once_flag VersionMgr::m_init_flag; 26 | 27 | void VersionMgr::createAndInit() { 28 | m_instance = new VersionMgr(); 29 | auto ver{version::Semver200_version(BOOST_PP_STRINGIZE(PACKAGE_VERSION))}; 30 | m_instance->m_version_map["sisl"] = ver; 31 | } 32 | 33 | VersionMgr* VersionMgr::getInstance() { 34 | std::call_once(m_init_flag, &VersionMgr::createAndInit); 35 | return m_instance; 36 | } 37 | 38 | version::Semver200_version VersionMgr::getVersion(const std::string& name) { 39 | auto ver_info{VersionMgr::getInstance()}; 40 | std::unique_lock l{ver_info->m_mutex}; 41 | auto it{ver_info->m_version_map.find(name)}; 42 | assert(it != ver_info->m_version_map.end()); 43 | return it->second; 44 | } 45 | 46 | std::vector< modinfo > VersionMgr::getVersions() { 47 | std::vector< modinfo > res; 48 | auto ver_info{VersionMgr::getInstance()}; 49 | std::unique_lock l{ver_info->m_mutex}; 50 | res.insert(res.end(), ver_info->m_version_map.begin(), ver_info->m_version_map.end()); 51 | return res; 52 | } 53 | 54 | void VersionMgr::addVersion(const std::string& name, const version::Semver200_version& ver) { 55 | auto ver_info{VersionMgr::getInstance()}; 56 | std::unique_lock l{ver_info->m_mutex}; 57 | auto it{ver_info->m_version_map.find(name)}; 58 | if (it == ver_info->m_version_map.end()) { ver_info->m_version_map[name] = ver; } 59 | } 60 | 61 | } // namespace sisl 62 | -------------------------------------------------------------------------------- /include/sisl/fds/vector_pool.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | // Copyright © 2016 Kadayam, Hari. All rights reserved. 18 | #pragma once 19 | 20 | #include 21 | #include 22 | #include 23 | 24 | namespace sisl { 25 | 26 | #define VECTOR_POOL_CACHE_COUNT 100 27 | 28 | template < typename T, size_t CacheCount = VECTOR_POOL_CACHE_COUNT > 29 | class VectorPoolImpl { 30 | public: 31 | VectorPoolImpl() { 32 | for (auto i = 0u; i < CacheCount; ++i) { 33 | m_pool[i] = new std::vector< T >(); 34 | } 35 | m_last = CacheCount; 36 | } 37 | 38 | ~VectorPoolImpl() { 39 | for (auto i = 0u; i < m_last; ++i) { 40 | delete (m_pool[i]); 41 | } 42 | } 43 | 44 | std::vector< T >* allocate() { return (m_last == 0) ? new std::vector< T >() : m_pool[--m_last]; } 45 | void deallocate(std::vector< T >* v) { 46 | if (m_last == CacheCount) { 47 | delete (v); 48 | } else { 49 | v->clear(); 50 | m_pool[m_last++] = v; 51 | } 52 | } 53 | 54 | private: 55 | std::array< std::vector< T >*, CacheCount > m_pool; 56 | size_t m_last = 0; 57 | }; 58 | 59 | template < typename T, size_t CacheCount = VECTOR_POOL_CACHE_COUNT > 60 | class VectorPool { 61 | public: 62 | static std::vector< T >* alloc() { return impl().allocate(); } 63 | static void free(std::vector< T >* v, bool no_cache = false) { no_cache ? delete (v) : impl().deallocate(v); } 64 | 65 | private: 66 | static VectorPoolImpl< T, CacheCount >& impl() { 67 | static thread_local VectorPoolImpl< T, CacheCount > _impl; 68 | return _impl; 69 | } 70 | }; 71 | 72 | } // namespace sisl 73 | -------------------------------------------------------------------------------- /include/sisl/fds/sparse_vector.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace sisl { 24 | /* 25 | * This class provides additional functionality to std::vector where the entries can be sparse. So it can be 26 | * inserted in any order and can be looked up with index, like regular array. 27 | * 28 | * One additional expectation to std::vector is that the type it carries should support default no argument constructor 29 | */ 30 | template < typename T > 31 | class sparse_vector : public std::vector< T > { 32 | public: 33 | template < typename... Args > 34 | sparse_vector(Args&&... args) : std::vector< T >(std::forward< Args >(args)...) {} 35 | sparse_vector(const sparse_vector&) = delete; 36 | sparse_vector(sparse_vector&&) noexcept = delete; 37 | sparse_vector& operator=(const sparse_vector&) = delete; 38 | sparse_vector& operator=(sparse_vector&&) noexcept = delete; 39 | 40 | ~sparse_vector() = default; 41 | 42 | T& operator[](const size_t index) { 43 | fill_void(index); 44 | return std::vector< T >::operator[](index); 45 | } 46 | 47 | bool index_exists(const size_t index) const { return (index < std::vector< T >::size()); } 48 | 49 | T& at(const size_t index) { 50 | fill_void(index); 51 | return std::vector< T >::at(index); 52 | } 53 | 54 | const T& operator[](const size_t index) const { 55 | assert(index < std::vector< T >::size()); 56 | return std::vector< T >::operator[](index); 57 | } 58 | 59 | const T& at(const size_t index) const { return std::vector< T >::at(index); } 60 | 61 | private: 62 | void fill_void(const size_t index) { 63 | for (size_t i{std::vector< T >::size()}; i <= index; ++i) { 64 | std::vector< T >::emplace_back(); 65 | } 66 | } 67 | }; 68 | } // namespace sisl 69 | -------------------------------------------------------------------------------- /src/fds/tests/concurrent_insert_vector_bench.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | using namespace sisl; 27 | 28 | static constexpr uint32_t NUM_THREADS = 1; 29 | std::unique_ptr< std::vector< uint64_t > > glob_lock_vector; 30 | std::mutex glob_vector_mutex; 31 | 32 | std::unique_ptr< sisl::ConcurrentInsertVector< uint64_t > > glob_cvec; 33 | 34 | void test_locked_vector_insert(benchmark::State& state) { 35 | // auto const per_thread_count = nentries / state.threads(); 36 | 37 | LOGINFO("Running on {} iterations in {} threads", state.iterations(), state.threads()); 38 | std::cout << "Running on iterations=" << state.iterations() << " in threads=" << state.threads() << "\n"; 39 | glob_lock_vector = std::make_unique< std::vector< uint64_t > >(); 40 | 41 | uint64_t i{0}; 42 | for (auto _ : state) { // Loops upto iteration count 43 | std::lock_guard< std::mutex > lg(glob_vector_mutex); 44 | glob_lock_vector->emplace_back(++i); 45 | } 46 | } 47 | 48 | void test_concurrent_vector_insert(benchmark::State& state) { 49 | std::cout << "Running on iterations=" << state.iterations() << " in threads=" << state.threads() << "\n"; 50 | glob_cvec = std::make_unique< sisl::ConcurrentInsertVector< uint64_t > >(); 51 | 52 | uint64_t i{0}; 53 | for (auto _ : state) { // Loops upto iteration count 54 | glob_cvec->emplace_back(++i); 55 | } 56 | } 57 | 58 | BENCHMARK(test_locked_vector_insert)->Threads(NUM_THREADS); 59 | BENCHMARK(test_concurrent_vector_insert)->Threads(NUM_THREADS); 60 | 61 | int main(int argc, char** argv) { 62 | int parsed_argc{argc}; 63 | ::benchmark::Initialize(&parsed_argc, argv); 64 | 65 | // setup(); 66 | ::benchmark::RunSpecifiedBenchmarks(); 67 | } 68 | -------------------------------------------------------------------------------- /include/sisl/utility/thread_factory.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Aditya Marella 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #ifdef _POSIX_THREADS 27 | #include 28 | #endif 29 | 30 | namespace sisl { 31 | 32 | template < class F, class... Args > 33 | std::thread thread_factory(const std::string name, F&& f, Args&&... args) { 34 | return std::thread([=] { 35 | #ifdef _POSIX_THREADS 36 | #ifdef __APPLE__ 37 | pthread_setname_np(name.c_str()); 38 | #else 39 | pthread_setname_np(pthread_self(), name.c_str()); 40 | #endif /* __APPLE__ */ 41 | #endif /* _POSIX_THREADS */ 42 | auto fun = std::mem_fn(f); 43 | fun(args...); 44 | }); 45 | } 46 | 47 | template < class F, class... Args > 48 | std::unique_ptr< std::thread > make_unique_thread(const std::string name, F&& f, Args&&... args) { 49 | return std::make_unique< std::thread >([=] { 50 | #ifdef _POSIX_THREADS 51 | #ifdef __APPLE__ 52 | pthread_setname_np(name.c_str()); 53 | #else 54 | pthread_setname_np(pthread_self(), name.c_str()); 55 | #endif /* __APPLE__ */ 56 | #endif /* _POSIX_THREADS */ 57 | auto fun = std::mem_fn(f); 58 | fun(args...); 59 | }); 60 | } 61 | 62 | template < class T > 63 | void name_thread([[maybe_unused]] T& t, std::string const& name) { 64 | #if defined(_POSIX_THREADS) && !defined(__APPLE__) 65 | if (auto ret = pthread_setname_np(t.native_handle(), name.substr(0, 15).c_str()); ret != 0) 66 | LOGERROR("Set name of thread to {} failed ret={}", name, ret); 67 | #else 68 | LOGINFO("No ability to set thread name: {}", name); 69 | #endif /* _POSIX_THREADS */ 70 | } 71 | 72 | template < class... Args > 73 | auto named_thread(const std::string name, Args&&... args) { 74 | auto t = std::thread(std::forward< Args >(args)...); 75 | name_thread(t, name); 76 | return t; 77 | } 78 | 79 | } // namespace sisl 80 | -------------------------------------------------------------------------------- /.jenkins/Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent { label 'sds-builder-v8' } 3 | 4 | environment { 5 | ARTIFACTORY_PASS = credentials('ARTIFACTORY_PASS') 6 | CONAN_USER = 'oss' 7 | TARGET_BRANCH = 'master' 8 | STABLE_BRANCH = 'stable/v*' 9 | BUILD_MISSING = "--build missing" 10 | } 11 | 12 | stages { 13 | stage('Get Version') { 14 | steps { 15 | script { 16 | PROJECT = sh(script: "grep -m 1 'name =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true) 17 | VER = sh(script: "grep -m 1 ' version =' conanfile.py | awk '{print \$3}' | tr -d '\n' | tr -d '\"'", returnStdout: true) 18 | CONAN_CHANNEL = sh(script: "echo ${BRANCH_NAME} | sed -E 's,(\\w+-?\\d*)/.*,\\1,' | sed -E 's,-,_,' | tr -d '\n'", returnStdout: true) 19 | TAG = "${VER}@${CONAN_USER}/${CONAN_CHANNEL}" 20 | CONAN_FLAGS="--name ${PROJECT} --user ${CONAN_USER} --channel ${CONAN_CHANNEL}" 21 | slackSend color: '#0063D1', channel: '#sds-ci', message: "*${PROJECT}/${TAG}* is building." 22 | } 23 | } 24 | } 25 | 26 | stage('Tag Stable Build') { 27 | when { branch "${STABLE_BRANCH}" } 28 | steps { 29 | script { 30 | TAG = "${VER}@" 31 | } 32 | } 33 | } 34 | 35 | stage("Compile") { 36 | steps { 37 | sh "conan create ${BUILD_MISSING} -s:h build_type=Debug -o ${PROJECT}/*:sanitize=True ${CONAN_FLAGS} . ; \ 38 | conan create ${BUILD_MISSING} -s:h build_type=Debug ${CONAN_FLAGS} . ; \ 39 | conan create ${BUILD_MISSING} -s:h build_type=RelWithDebInfo -o ${PROJECT}/*:malloc_impl=tcmalloc ${CONAN_FLAGS} . ; \ 40 | " 41 | } 42 | } 43 | 44 | stage("Deploy") { 45 | when { 46 | expression { !(env.BRANCH_NAME =~ /PR-/) } 47 | } 48 | steps { 49 | sh "conan remote login -p ${ARTIFACTORY_PASS} ebay-local _service_sds" 50 | sh "conan graph info ./ | grep 'ref: ' | awk '{print \$2}' | sort | uniq | grep -v ${PROJECT} | grep -v '#' | while read pkg; do conan upload -r ebay-local -c \"\${pkg}\"; done" 51 | sh "conan upload ${PROJECT}/${TAG} -c -r ebay-local" 52 | } 53 | } 54 | } 55 | 56 | post { 57 | failure { 58 | slackSend color: '#E43237', channel: '#sds-ci', message: "*${PROJECT}/${TAG}* has had a failure : ${BUILD_URL}" 59 | } 60 | success { 61 | slackSend color: '#85B717', channel: '#sds-ci', message: "*${PROJECT}/${TAG}* has completed." 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /include/sisl/fds/obj_allocator.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "freelist_allocator.hpp" 23 | 24 | namespace sisl { 25 | 26 | static constexpr size_t FREELIST_CACHE_COUNT{200}; 27 | 28 | /** 29 | * @brief Object Allocator is an object wrapper on top of freelist allocator. It provides convenient method to create 30 | * a C++ object and destruct them. 31 | */ 32 | template < typename T, const size_t CacheCount = FREELIST_CACHE_COUNT > 33 | class ObjectAllocator { 34 | public: 35 | ObjectAllocator() { 36 | m_allocator = std::make_unique< sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) > >(); 37 | } 38 | ObjectAllocator(const ObjectAllocator&) = delete; 39 | ObjectAllocator(ObjectAllocator&&) noexcept = delete; 40 | ObjectAllocator& operator=(const ObjectAllocator&) = delete; 41 | ObjectAllocator& operator=(ObjectAllocator&&) noexcept = delete; 42 | 43 | template < class... Args > 44 | static T* make_object(Args&&... args) { 45 | uint8_t* const mem{get_obj_allocator()->m_allocator->allocate(sizeof(T))}; 46 | T* const ptr{new (mem) T(std::forward< Args >(args)...)}; 47 | return ptr; 48 | } 49 | 50 | static void deallocate(T* const mem, const size_t obj_size = sizeof(T)) { 51 | mem->~T(); 52 | get_obj_allocator()->m_allocator->deallocate(reinterpret_cast< uint8_t* >(mem), obj_size); 53 | } 54 | 55 | static std::unique_ptr< ObjectAllocator< T, CacheCount > > obj_allocator; 56 | 57 | private: 58 | sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) >* get_freelist_allocator() { return m_allocator.get(); } 59 | 60 | private: 61 | std::unique_ptr< sisl::FreeListAllocator< FREELIST_CACHE_COUNT, sizeof(T) > > m_allocator; 62 | 63 | static ObjectAllocator< T, CacheCount >* get_obj_allocator() { 64 | static ObjectAllocator< T, CacheCount > obj_allocator{}; 65 | return &obj_allocator; 66 | } 67 | }; 68 | 69 | } // namespace sisl 70 | -------------------------------------------------------------------------------- /src/fds/tests/test_atomic_status_counter.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Bryan Zimmerman 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | 20 | #include 21 | 22 | #include "sisl/fds/atomic_status_counter.hpp" 23 | 24 | using namespace sisl; 25 | 26 | SISL_LOGGING_INIT(test_atomic_status_counter) 27 | SISL_OPTIONS_ENABLE(logging) 28 | 29 | namespace { 30 | class AtomicStatusCounterTest : public testing::Test { 31 | public: 32 | AtomicStatusCounterTest() : testing::Test(){}; 33 | AtomicStatusCounterTest(const AtomicStatusCounterTest&) = delete; 34 | AtomicStatusCounterTest(AtomicStatusCounterTest&&) noexcept = delete; 35 | AtomicStatusCounterTest& operator=(const AtomicStatusCounterTest&) = delete; 36 | AtomicStatusCounterTest& operator=(AtomicStatusCounterTest&&) noexcept = delete; 37 | virtual ~AtomicStatusCounterTest() override = default; 38 | 39 | protected: 40 | void SetUp() override {} 41 | void TearDown() override {} 42 | }; 43 | } // namespace 44 | 45 | TEST_F(AtomicStatusCounterTest, BasicStatusCounter) { 46 | _status_counter< uint8_t, 0 > sc1{1}; 47 | EXPECT_EQ(sc1.to_integer(), static_cast< uint64_t >(1)); 48 | _status_counter< uint8_t, 0 > sc2{1, 2}; 49 | const uint8_t counter_size_bits{sizeof(typename decltype(sc2)::counter_type) * 8}; 50 | EXPECT_EQ(sc2.to_integer(), static_cast< uint64_t >(1) | (static_cast< uint64_t >(2) << counter_size_bits)); 51 | } 52 | 53 | TEST_F(AtomicStatusCounterTest, BasicTest) { 54 | atomic_status_counter< uint8_t, 0 > asc{1, 2}; 55 | EXPECT_EQ(asc.count(), static_cast< typename decltype(asc)::counter_type >(1)); 56 | EXPECT_EQ(asc.get_status(), static_cast< uint8_t >(2)); 57 | } 58 | 59 | int main(int argc, char* argv[]) { 60 | SISL_OPTIONS_LOAD(argc, argv, logging) 61 | ::testing::InitGoogleTest(&argc, argv); 62 | sisl::logging::SetLogger("test_atomic_status_counter"); 63 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 64 | 65 | const auto result{RUN_ALL_TESTS()}; 66 | return result; 67 | } 68 | -------------------------------------------------------------------------------- /src/flip/lib/flip_rpc_server.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #include "sisl/flip/flip_rpc_server.hpp" 26 | #include "sisl/flip/flip.hpp" 27 | 28 | SISL_LOGGING_DEF(flip) 29 | 30 | namespace flip { 31 | grpc::Status FlipRPCServer::InjectFault(grpc::ServerContext*, const FlipSpec* request, FlipResponse* response) { 32 | LOGTRACEMOD(flip, "InjectFault request = {}", request->DebugString()); 33 | flip::Flip::instance().add(*request); 34 | response->set_success(true); 35 | return grpc::Status::OK; 36 | } 37 | 38 | grpc::Status FlipRPCServer::GetFaults(grpc::ServerContext*, const FlipNameRequest* request, 39 | FlipListResponse* response) { 40 | LOGTRACEMOD(flip, "GetFaults request = {}", request->DebugString()); 41 | auto resp = request->name().size() ? flip::Flip::instance().get(request->name()) : flip::Flip::instance().get_all(); 42 | for (const auto& r : resp) { 43 | response->add_infos()->set_info(r); 44 | } 45 | LOGTRACEMOD(flip, "GetFaults response = {}", response->DebugString()); 46 | return grpc::Status::OK; 47 | } 48 | 49 | grpc::Status FlipRPCServer::RemoveFault(grpc::ServerContext*, const FlipRemoveRequest* request, 50 | FlipRemoveResponse* response) { 51 | LOGTRACEMOD(flip, "RemoveFault request = {}", request->DebugString()); 52 | response->set_num_removed(flip::Flip::instance().remove(request->name())); 53 | return grpc::Status::OK; 54 | } 55 | 56 | class FlipRPCServiceWrapper : public FlipRPCServer::Service { 57 | public: 58 | void print_method_names() { 59 | for (auto i = 0; i < 2; ++i) { 60 | auto method = (::grpc::internal::RpcServiceMethod*)GetHandler(i); 61 | if (method) { LOGINFOMOD(flip, "Method name = {}", method->name()); } 62 | } 63 | } 64 | }; 65 | 66 | } // namespace flip 67 | -------------------------------------------------------------------------------- /src/flip/proto/flip_spec.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package flip; 4 | 5 | option cc_enable_arenas = true; 6 | enum Operator { 7 | // Equal. 8 | EQUAL = 0; 9 | 10 | // Inequality. 11 | NOT_EQUAL = 1; 12 | 13 | // Less than. 14 | LESS_THAN = 2; 15 | 16 | // Less than or equal. 17 | LESS_THAN_OR_EQUAL = 3; 18 | 19 | // Greater than. 20 | GREATER_THAN = 4; 21 | 22 | // Greater than or equal. 23 | GREATER_THAN_OR_EQUAL = 5; 24 | 25 | // Don't care about 26 | DONT_CARE = 6; 27 | 28 | // RegEx Pattern 29 | REG_EX = 7; 30 | } 31 | 32 | enum Frequency { 33 | // Every time generate a fault 34 | ALWAYS = 0; 35 | 36 | // Generate fault alternate attempts 37 | ALTERNATE = 1; 38 | 39 | // Fault on uniform random basis 40 | UNI_RANDOM = 2; 41 | } 42 | 43 | message ParamValue { 44 | // The kind of value. 45 | oneof kind { 46 | bool null_value = 1; 47 | 48 | int32 int_value = 2; 49 | 50 | int64 long_value = 3; 51 | 52 | // Represents a double value. 53 | double double_value = 4; 54 | 55 | // Represents a string value. 56 | string string_value = 6; 57 | 58 | // Represents a boolean value. 59 | bool bool_value = 7; 60 | 61 | bytes binary_value = 10; 62 | 63 | //google.protobuf.Any struct_encoded = 22; 64 | 65 | //This is a wrapper object that contains a single attribute with reserved name `list` 66 | //google.protobuf.Any list_encoded = 23; 67 | } 68 | } 69 | 70 | message FlipCondition { 71 | string name = 1; 72 | Operator oper = 2; 73 | ParamValue value = 3; 74 | } 75 | 76 | message FlipAction { 77 | message ActionReturns { 78 | ParamValue retval = 1; 79 | } 80 | 81 | message ActionDelays { 82 | uint64 delay_in_usec = 1; 83 | } 84 | 85 | message ActionDelayedReturns { 86 | uint64 delay_in_usec = 1; 87 | ParamValue retval = 2; 88 | } 89 | 90 | oneof action { 91 | bool no_action = 1; 92 | ActionReturns returns = 2; 93 | ActionDelays delays = 3; 94 | ActionDelayedReturns delay_returns = 4; 95 | } 96 | } 97 | 98 | message FlipFrequency { 99 | uint32 count = 1; // How many faults to generate, Default no limit 100 | 101 | oneof frequency { 102 | uint32 percent = 2; // Percentage of requests that matches the condition to generate fault 103 | uint32 every_nth = 3; // Generate fault for every nth request 104 | } 105 | } 106 | 107 | message FlipSpec { 108 | string flip_name = 1; 109 | repeated FlipCondition conditions = 2; 110 | FlipAction flip_action = 3; 111 | FlipFrequency flip_frequency = 4; 112 | } 113 | 114 | message FlipResponse { 115 | bool success = 1; 116 | 117 | map metadata = 200; 118 | } 119 | -------------------------------------------------------------------------------- /src/metrics/metrics_atomic.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "sisl/metrics/metrics_atomic.hpp" 24 | 25 | namespace sisl { 26 | 27 | void AtomicMetricsGroup::on_register() { 28 | // std::make_unique<[]> will allocate and construct 29 | m_counter_values = std::make_unique< AtomicCounterValue[] >(num_counters()); 30 | m_histogram_values = std::make_unique< AtomicHistogramValue[] >(num_histograms()); 31 | } 32 | 33 | void AtomicMetricsGroup::gather_result([[maybe_unused]] bool need_latest, const counter_gather_cb_t& counter_cb, 34 | const gauge_gather_cb_t& gauge_cb, const histogram_gather_cb_t& histogram_cb) { 35 | for (size_t i{0}; i < num_counters(); ++i) { 36 | counter_cb(i, m_counter_values[i].to_counter_value()); 37 | } 38 | 39 | for (size_t i{0}; i < num_gauges(); ++i) { 40 | gauge_cb(i, m_gauge_values[i]); 41 | } 42 | 43 | for (size_t i{0}; i < num_histograms(); ++i) { 44 | histogram_cb(i, m_histogram_values[i].to_histogram_value()); 45 | } 46 | } 47 | 48 | void AtomicMetricsGroup::counter_increment(uint64_t index, int64_t val) { m_counter_values[index].increment(val); } 49 | 50 | void AtomicMetricsGroup::counter_decrement(uint64_t index, int64_t val) { m_counter_values[index].decrement(val); } 51 | 52 | // If we were to call the method with count parameter and compiler inlines them, binaries linked with libsisl gets 53 | // linker errors. At the same time we also don't want to non-inline this method, since its the most obvious call 54 | // everyone makes and wanted to avoid additional function call in the stack. Hence we are duplicating the function 55 | // one with count and one without count. In any case this is a single line method. 56 | void AtomicMetricsGroup::histogram_observe(uint64_t index, int64_t val) { 57 | m_histogram_values[index].observe(val, hist_static_info(index).get_boundaries(), 1); 58 | } 59 | 60 | void AtomicMetricsGroup::histogram_observe(uint64_t index, int64_t val, uint64_t count) { 61 | m_histogram_values[index].observe(val, hist_static_info(index).get_boundaries(), count); 62 | } 63 | } // namespace sisl 64 | -------------------------------------------------------------------------------- /include/sisl/metrics/reporter.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include "histogram_buckets.hpp" 23 | 24 | namespace sisl { 25 | typedef std::pair< std::string, std::string > metric_label; 26 | 27 | enum ReportFormat { kUnknownFormat, kTextFormat, kJsonFormat, kProtoBufferFormat }; 28 | 29 | class ReportCounter { 30 | public: 31 | virtual void set_value(double value) = 0; 32 | }; 33 | 34 | class ReportGauge { 35 | public: 36 | virtual void set_value(double value) = 0; 37 | }; 38 | 39 | class ReportHistogram { 40 | public: 41 | virtual void set_value(std::vector< double >& bucket_values, double sum) = 0; 42 | }; 43 | 44 | class Reporter { 45 | public: 46 | virtual ~Reporter() = default; 47 | virtual std::shared_ptr< ReportCounter > add_counter(const std::string& name, const std::string& desc, 48 | const std::string& instance_name, 49 | const metric_label& label_pair = {"", ""}) = 0; 50 | virtual std::shared_ptr< ReportGauge > add_gauge(const std::string& name, const std::string& desc, 51 | const std::string& instance_name, 52 | const metric_label& label_pair = {"", ""}) = 0; 53 | virtual std::shared_ptr< ReportHistogram > add_histogram(const std::string& name, const std::string& desc, 54 | const std::string& instance_name, 55 | const hist_bucket_boundaries_t& bkt_boundaries, 56 | const metric_label& label_pair = {"", ""}) = 0; 57 | 58 | virtual void remove_counter(const std::string& name, const std::shared_ptr< ReportCounter >& hist) = 0; 59 | virtual void remove_gauge(const std::string& name, const std::shared_ptr< ReportGauge >& hist) = 0; 60 | virtual void remove_histogram(const std::string& name, const std::shared_ptr< ReportHistogram >& hist) = 0; 61 | 62 | virtual std::string serialize(ReportFormat format) = 0; 63 | }; 64 | } // namespace sisl 65 | -------------------------------------------------------------------------------- /src/grpc/utils.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed 10 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | * specific language governing permissions and limitations under the License. 13 | * 14 | *********************************************************************************/ 15 | #pragma once 16 | 17 | #include 18 | #include 19 | 20 | namespace sisl { 21 | 22 | [[maybe_unused]] static bool get_file_contents(const std::string& file_name, std::string& contents) { 23 | try { 24 | std::ifstream f(file_name); 25 | std::string buffer(std::istreambuf_iterator< char >{f}, std::istreambuf_iterator< char >{}); 26 | contents = buffer; 27 | return !contents.empty(); 28 | } catch (...) {} 29 | return false; 30 | } 31 | 32 | [[maybe_unused]] static void serialize_to_byte_buffer(io_blob_list_t const& cli_buf, grpc::ByteBuffer& cli_byte_buf) { 33 | folly::small_vector< grpc::Slice, 4 > slices; 34 | for (auto const& blob : cli_buf) { 35 | slices.emplace_back(blob.cbytes(), blob.size(), grpc::Slice::STATIC_SLICE); 36 | } 37 | cli_byte_buf.Clear(); 38 | grpc::ByteBuffer tmp(slices.data(), cli_buf.size()); 39 | cli_byte_buf.Swap(&tmp); 40 | } 41 | 42 | [[maybe_unused]] static grpc::Status try_deserialize_from_byte_buffer(grpc::ByteBuffer const& cli_byte_buf, 43 | io_blob& cli_buf) { 44 | grpc::Slice slice; 45 | auto status = cli_byte_buf.TrySingleSlice(&slice); 46 | if (status.ok()) { 47 | // if slice is inlined, we can not use the reference to the slice 48 | auto raw_slice = slice.c_slice(); 49 | if ((raw_slice).refcount) { 50 | cli_buf.set_bytes(slice.begin()); 51 | cli_buf.set_size(slice.size()); 52 | } else { 53 | status = grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, 54 | "Inlined slice, can not use the reference to the slice."); 55 | } 56 | grpc_slice_unref(raw_slice); 57 | } 58 | return status; 59 | } 60 | 61 | [[maybe_unused]] static grpc::Status deserialize_from_byte_buffer(grpc::ByteBuffer const& cli_byte_buf, 62 | io_blob& cli_buf) { 63 | grpc::Slice slice; 64 | auto status = cli_byte_buf.DumpToSingleSlice(&slice); 65 | if (status.ok()) { 66 | cli_buf.buf_alloc(slice.size()); 67 | std::memcpy(voidptr_cast(cli_buf.bytes()), c_voidptr_cast(slice.begin()), slice.size()); 68 | } 69 | return status; 70 | } 71 | 72 | } // namespace sisl 73 | -------------------------------------------------------------------------------- /include/sisl/fds/id_reserver.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "bitset.hpp" 25 | #include "utils.hpp" 26 | 27 | namespace sisl { 28 | class IDReserver { 29 | public: 30 | IDReserver(uint32_t estimated_ids = 1024) : m_reserved_bits(estimated_ids) { assert(estimated_ids != 0); } 31 | 32 | IDReserver(const sisl::byte_array& b) : m_reserved_bits(b) {} 33 | 34 | uint32_t reserve() { 35 | std::unique_lock lg(m_mutex); 36 | size_t nbit = m_reserved_bits.get_next_reset_bit(0); 37 | if (nbit == Bitset::npos) { 38 | // We ran out of room to allocate bits, resize and allocate more 39 | const auto cur_size = m_reserved_bits.size(); 40 | assert(cur_size != 0); 41 | m_reserved_bits.resize(cur_size * 2); 42 | nbit = cur_size; 43 | } 44 | m_reserved_bits.set_bit(nbit); 45 | return nbit; 46 | } 47 | 48 | void reserve(uint32_t id) { 49 | std::unique_lock lg(m_mutex); 50 | assert(!(m_reserved_bits.get_bitval(id))); 51 | assert(id < m_reserved_bits.size()); 52 | m_reserved_bits.set_bit(id); 53 | } 54 | 55 | void unreserve(uint32_t id) { 56 | std::unique_lock lg(m_mutex); 57 | assert(id < m_reserved_bits.size()); 58 | m_reserved_bits.reset_bit(id); 59 | } 60 | 61 | bool is_reserved(uint32_t id) { 62 | std::unique_lock lg(m_mutex); 63 | return m_reserved_bits.get_bitval(id); 64 | } 65 | 66 | sisl::byte_array serialize() { 67 | std::unique_lock lg(m_mutex); 68 | return m_reserved_bits.serialize(); 69 | } 70 | 71 | bool first_reserved_id(uint32_t& found_id) { return find_next_reserved_id(true, found_id); } 72 | bool next_reserved_id(uint32_t& last_found_id) { return find_next_reserved_id(false, last_found_id); } 73 | 74 | private: 75 | bool find_next_reserved_id(bool first, uint32_t& last_found_id) { 76 | std::unique_lock lg(m_mutex); 77 | size_t nbit = m_reserved_bits.get_next_set_bit(first ? 0 : last_found_id + 1); 78 | if (nbit == Bitset::npos) return false; 79 | last_found_id = (uint32_t)nbit; 80 | return true; 81 | } 82 | 83 | private: 84 | std::mutex m_mutex; 85 | sisl::Bitset m_reserved_bits; 86 | }; 87 | } // namespace sisl 88 | -------------------------------------------------------------------------------- /src/metrics/metrics_rcu.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include "sisl/metrics/metrics_rcu.hpp" 18 | #include 19 | 20 | namespace sisl { 21 | 22 | void WisrBufferMetricsGroup::on_register() { 23 | m_metrics = std::make_unique< WisrBufferMetrics >(m_static_info->m_histograms, num_counters(), num_histograms()); 24 | } 25 | 26 | void WisrBufferMetricsGroup::counter_increment(uint64_t index, int64_t val) { 27 | auto m{m_metrics->insert_access()}; 28 | m->get_counter(index).increment(val); 29 | } 30 | 31 | void WisrBufferMetricsGroup::counter_decrement(uint64_t index, int64_t val) { 32 | auto m{m_metrics->insert_access()}; 33 | m->get_counter(index).decrement(val); 34 | } 35 | 36 | // If we were to call the method with count parameter and compiler inlines them, binaries linked with libsisl gets 37 | // linker errors. At the same time we also don't want to non-inline this method, since its the most obvious call 38 | // everyone makes and wanted to avoid additional function call in the stack. Hence we are duplicating the function 39 | // one with count and one without count. In any case this is a single line method. 40 | void WisrBufferMetricsGroup::histogram_observe(uint64_t index, int64_t val) { 41 | auto m{m_metrics->insert_access()}; 42 | m->get_histogram(index).observe(val, hist_static_info(index).get_boundaries(), 1); 43 | } 44 | 45 | void WisrBufferMetricsGroup::histogram_observe(uint64_t index, int64_t val, uint64_t count) { 46 | auto m{m_metrics->insert_access()}; 47 | m->get_histogram(index).observe(val, hist_static_info(index).get_boundaries(), count); 48 | } 49 | 50 | void WisrBufferMetricsGroup::gather_result(bool need_latest, const counter_gather_cb_t& counter_cb, 51 | const gauge_gather_cb_t& gauge_cb, 52 | const histogram_gather_cb_t& histogram_cb) { 53 | PerThreadMetrics* tmetrics; 54 | if (need_latest) { 55 | tmetrics = m_metrics->now(); 56 | } else { 57 | tmetrics = m_metrics->delayed(); 58 | } 59 | 60 | for (size_t i{0}; i < num_counters(); ++i) { 61 | counter_cb(i, tmetrics->get_counter(i)); 62 | } 63 | 64 | for (size_t i{0}; i < num_gauges(); ++i) { 65 | gauge_cb(i, m_gauge_values[i]); 66 | } 67 | 68 | for (size_t i{0}; i < num_histograms(); ++i) { 69 | histogram_cb(i, tmetrics->get_histogram(i)); 70 | } 71 | } 72 | } // namespace sisl 73 | -------------------------------------------------------------------------------- /3rd_party/userspace-rcu/conanfile.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from conan import ConanFile 4 | from conan.errors import ConanInvalidConfiguration 5 | from conan.tools.env import VirtualBuildEnv 6 | from conan.tools.files import chdir, copy, get, rm, rmdir 7 | from conan.tools.gnu import Autotools, AutotoolsToolchain 8 | from conan.tools.layout import basic_layout 9 | 10 | required_conan_version = ">=1.53.0" 11 | 12 | 13 | class UserspaceRCUConan(ConanFile): 14 | name = "userspace-rcu" 15 | description = "Userspace RCU (read-copy-update) library" 16 | license = "LGPL-2.1" 17 | url = "https://github.com/conan-io/conan-center-index" 18 | homepage = "https://liburcu.org/" 19 | topics = "urcu" 20 | 21 | package_type = "library" 22 | settings = "os", "arch", "compiler", "build_type" 23 | options = { 24 | "shared": [True, False], 25 | "fPIC": [True, False], 26 | } 27 | default_options = { 28 | "shared": False, 29 | "fPIC": True, 30 | } 31 | 32 | def configure(self): 33 | self.settings.rm_safe("compiler.libcxx") 34 | self.settings.rm_safe("compiler.cppstd") 35 | if self.options.shared: 36 | self.options.rm_safe("fPIC") 37 | 38 | def layout(self): 39 | basic_layout(self, src_folder="src") 40 | 41 | def validate(self): 42 | if self.settings.os not in ["Linux", "FreeBSD", "Macos"]: 43 | raise ConanInvalidConfiguration(f"Building for {self.settings.os} unsupported") 44 | if self.version == "0.11.4" and self.settings.compiler == "apple-clang": 45 | # Fails with "cds_hlist_add_head_rcu.c:19:10: fatal error: 'urcu/urcu-memb.h' file not found" 46 | raise ConanInvalidConfiguration(f"{self.ref} is not compatible with apple-clang") 47 | 48 | def build_requirements(self): 49 | self.tool_requires("libtool/2.4.7") 50 | 51 | def source(self): 52 | get(self, **self.conan_data["sources"][self.version], strip_root=True) 53 | 54 | def generate(self): 55 | env = VirtualBuildEnv(self) 56 | env.generate() 57 | tc = AutotoolsToolchain(self) 58 | tc.generate() 59 | 60 | def build(self): 61 | autotools = Autotools(self) 62 | autotools.autoreconf() 63 | autotools.configure() 64 | autotools.make() 65 | 66 | def package(self): 67 | copy(self, "LICENSE*", 68 | src=self.source_folder, 69 | dst=os.path.join(self.package_folder, "licenses")) 70 | autotools = Autotools(self) 71 | autotools.install() 72 | 73 | rm(self, "*.la", self.package_folder, recursive=True) 74 | rmdir(self, os.path.join(self.package_folder, "lib", "pkgconfig")) 75 | rmdir(self, os.path.join(self.package_folder, "share")) 76 | 77 | def package_info(self): 78 | for lib_type in ["","-memb"]: 79 | component_name = f"urcu{lib_type}" 80 | self.cpp_info.components[component_name].libs = ["urcu-common", component_name] 81 | self.cpp_info.components[component_name].set_property("pkg_config_name", component_name) 82 | if self.settings.os in ["Linux", "FreeBSD"]: 83 | self.cpp_info.components[component_name].system_libs = ["pthread"] 84 | 85 | # Some definitions needed for MB and Signal variants 86 | self.cpp_info.components["urcu-memb"].defines = ["RCU_MEMB"] -------------------------------------------------------------------------------- /src/fds/tests/test_tcmalloc_helper.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Bryan Zimmerman 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #ifdef USING_TCMALLOC 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "sisl/logging/logging.h" 25 | #include "sisl/options/options.h" 26 | #include "sisl/utility/thread_buffer.hpp" 27 | 28 | #include 29 | 30 | #include "sisl/fds/malloc_helper.hpp" 31 | 32 | using namespace sisl; 33 | 34 | SISL_LOGGING_INIT(test_jemalloc) 35 | 36 | namespace { 37 | uint32_t g_num_threads; 38 | 39 | struct TcmallocTest : public testing::Test { 40 | public: 41 | TcmallocTest() : testing::Test{} { LOGINFO("Initializing new TcmallocTest class"); } 42 | TcmallocTest(const TcmallocTest&) = delete; 43 | TcmallocTest(TcmallocTest&&) noexcept = delete; 44 | TcmallocTest& operator=(const TcmallocTest&) = delete; 45 | TcmallocTest& operator=(TcmallocTest&&) noexcept = delete; 46 | virtual ~TcmallocTest() override = default; 47 | 48 | protected: 49 | void SetUp() override {} 50 | void TearDown() override {} 51 | 52 | void MultiThreadedAllocDealloc(const size_t iterations, const size_t mem_count = 1000000) const { 53 | const auto thread_lambda{[&iterations, &mem_count]() { 54 | // allocated/deallocate memory 55 | for (size_t iteration{0}; iteration < iterations; ++iteration) { 56 | std::unique_ptr< uint64_t[] > mem{new uint64_t[mem_count]}; 57 | } 58 | }}; 59 | 60 | std::vector< std::thread > threads; 61 | for (uint32_t thread_num{0}; thread_num < g_num_threads; ++thread_num) { 62 | threads.emplace_back(thread_lambda); 63 | } 64 | 65 | for (auto& alloc_dealloc_thread : threads) { 66 | if (alloc_dealloc_thread.joinable()) alloc_dealloc_thread.join(); 67 | }; 68 | } 69 | }; 70 | } // namespace 71 | 72 | TEST_F(TcmallocTest, GetDirtyPageCount) { MultiThreadedAllocDealloc(100); } 73 | 74 | SISL_OPTIONS_ENABLE(logging, test_tcmalloc) 75 | 76 | SISL_OPTION_GROUP(test_tcmalloc, 77 | (num_threads, "", "num_threads", "number of threads", 78 | ::cxxopts::value< uint32_t >()->default_value("8"), "number")) 79 | 80 | int main(int argc, char* argv[]) { 81 | SISL_OPTIONS_LOAD(argc, argv, logging, test_tcmalloc); 82 | ::testing::InitGoogleTest(&argc, argv); 83 | sisl::logging::SetLogger("test_bitset"); 84 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 85 | 86 | g_num_threads = SISL_OPTIONS["num_threads"].as< uint32_t >(); 87 | 88 | const auto ret{RUN_ALL_TESTS()}; 89 | return ret; 90 | } 91 | 92 | #endif 93 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | project (sisl CXX) 3 | 4 | # Set Global CMake Options 5 | set_property(GLOBAL PROPERTY USE_FOLDERS ON) # turn on folder hierarchies 6 | 7 | include (cmake/Flags.cmake) 8 | 9 | set(CMAKE_CXX_STANDARD 20) 10 | 11 | enable_testing() 12 | find_package(GTest QUIET REQUIRED) 13 | 14 | if (DEFINED BUILD_COVERAGE) 15 | if (${BUILD_COVERAGE}) 16 | include (cmake/CodeCoverage.cmake) 17 | APPEND_COVERAGE_COMPILER_FLAGS() 18 | SETUP_TARGET_FOR_COVERAGE_GCOVR_XML(NAME coverage EXECUTABLE ctest DEPENDENCIES ) 19 | endif() 20 | endif() 21 | 22 | if(${CMAKE_BUILD_TYPE} STREQUAL "Debug") 23 | message(STATUS "Debug build") 24 | add_flags("-D_DEBUG") 25 | endif() 26 | 27 | if (DEFINED MALLOC_IMPL) 28 | if (${MALLOC_IMPL} STREQUAL "jemalloc") 29 | add_flags("-DUSE_JEMALLOC=1") 30 | # turn off memory sanitizer if using jemalloc otherwise folly will cause double definition errors 31 | if (DEFINED MEMORY_SANITIZER_ON) 32 | unset(MEMORY_SANITIZER_ON CACHE) 33 | endif() 34 | elseif (${MALLOC_IMPL} STREQUAL "tcmalloc") 35 | add_flags("-DUSING_TCMALLOC=1") 36 | endif() 37 | endif() 38 | 39 | if (DEFINED MEMORY_SANITIZER_ON) 40 | if (${MEMORY_SANITIZER_ON}) 41 | include (cmake/mem_sanitizer.cmake) 42 | message(STATUS "********* WARNING: Running with Memory Sanitizer ON *********") 43 | endif() 44 | else () 45 | message(STATUS "********* WARNING: Running with Memory Sanitizer OFF *********") 46 | endif() 47 | 48 | include(cmake/settings_gen.cmake) 49 | 50 | if (NOT ${CMAKE_SYSTEM_NAME} STREQUAL "Windows") 51 | set(CMAKE_THREAD_PREFER_PTHREAD TRUE) 52 | endif() 53 | 54 | find_program(CCACHE_FOUND ccache) 55 | if (CCACHE_FOUND) 56 | set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ccache) 57 | set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ccache) 58 | endif () 59 | 60 | if((${CMAKE_CXX_COMPILER_ID} STREQUAL "GNU") OR (${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")) 61 | # add compiler options 62 | add_flags("-fconcepts" Languages "CXX") 63 | 64 | # add warning options 65 | add_flags("-Wall -Wextra -Werror -Wpedantic") 66 | endif() 67 | 68 | if(UNIX) 69 | # enable proper GNU extensions 70 | add_flags("-D_POSIX_C_SOURCE=200809L -D_FILE_OFFSET_BITS=64 -D_LARGEFILE64_SOURCE") 71 | endif() 72 | 73 | include_directories(BEFORE 74 | ${CMAKE_CURRENT_SOURCE_DIR}/include 75 | ) 76 | 77 | add_subdirectory(src) 78 | 79 | # build info 80 | string(TOUPPER "${CMAKE_BUILD_TYPE}" UC_CMAKE_BUILD_TYPE) 81 | message(STATUS "Build configuration: ${CMAKE_BUILD_TYPE}") 82 | message(STATUS "C compiler info: ${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION} ${CMAKE_C_COMPILER_EXTERNAL_TOOLCHAIN}") 83 | message(STATUS "C++ compiler info: ${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION} ${CMAKE_CXX_COMPILER_EXTERNAL_TOOLCHAIN}") 84 | message(STATUS "C flags: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 85 | message(STATUS "C++ flags: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 86 | message(STATUS "Linker flags (executable): ${CMAKE_EXE_LINKER_FLAGS} ${CMAKE_EXE_LINKER_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 87 | message(STATUS "Linker flags (shared): ${CMAKE_SHARED_LINKER_FLAGS} ${CMAKE_SHARED_LINKER_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 88 | message(STATUS "Linker flags (module): ${CMAKE_MODULE_LINKER_FLAGS} ${CMAKE_MODULE_LINKER_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 89 | message(STATUS "Linker flags (static): ${CMAKE_STATIC_LINKER_FLAGS} ${CMAKE_STATIC_LINKER_FLAGS_${UC_CMAKE_BUILD_TYPE}}") 90 | -------------------------------------------------------------------------------- /src/fds/tests/test_idreserver.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | #include "sisl/fds/id_reserver.hpp" 25 | 26 | #include 27 | 28 | using namespace sisl; 29 | 30 | SISL_LOGGING_INIT(test_bitset); 31 | 32 | SISL_OPTIONS_ENABLE(logging, test_id_reserver) 33 | 34 | SISL_OPTION_GROUP(test_id_reserver, 35 | (num_threads, "", "num_threads", "number of threads", 36 | ::cxxopts::value< uint32_t >()->default_value("8"), "number"), 37 | (max_ids, "", "max_ids", "maximum number of ids", 38 | ::cxxopts::value< uint32_t >()->default_value("1000"), "number"), 39 | 40 | namespace { 41 | uint32_t g_max_ids; 42 | uint32_t g_num_threads; 43 | 44 | void run_parallel(uint32_t nthreads, const std::function< void(uint32_t) >& thr_fn) { 45 | std::vector< std::thread > threads; 46 | auto n_per_thread = std::ceil((double)g_max_ids / nthreads); 47 | int32_t remain_ids = (int32_t)g_max_ids; 48 | 49 | while (remain_ids > 0) { 50 | threads.emplace_back(thr_fn, std::min(remain_ids, (int32_t)n_per_thread)); 51 | remain_ids -= n_per_thread; 52 | } 53 | for (auto t : threads) { 54 | if (t.joinable()) t.join(); 55 | } 56 | } 57 | 58 | struct IDReserverTest : public testing::Test { 59 | public: 60 | IDReserverTest(const IDReserverTest&) = delete; 61 | IDReserverTest(IDReserverTest&&) noexcept = delete; 62 | IDReserverTest& operator=(const IDReserverTest&) = delete; 63 | IDReserverTest& operator=(IDReserverTest&&) noexcept = delete; 64 | virtual ~IDReserverTest() override = default; 65 | 66 | protected: 67 | IDReserver m_reserver; 68 | 69 | void SetUp() override {} 70 | void TearDown() override {} 71 | }; 72 | } 73 | 74 | TEST_F(IDReserverTest, RandomIDSet) { 75 | run_parallel(g_num_threads, [&](int32_t n_ids_this_thread) { 76 | LOGINFO("INFO: Setting alternate bits (set even and reset odd) in range[{} - {}]", start, start + count - 1); 77 | for (auto i = 0; i < n_ids_this_thread; ++i) { 78 | if (i % 2 == 0) { reserve(); } 79 | } 80 | }); 81 | } 82 | 83 | int main(int argc, char* argv[]) { 84 | SISL_OPTIONS_LOAD(argc, argv, logging, test_id_reserver); 85 | ::testing::InitGoogleTest(&argc, argv); 86 | sisl::logging::SetLogger("test_id_reserver"); 87 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 88 | 89 | g_max_ids = SISL_OPTIONS["max_ids"].as< uint32_t >(); 90 | g_num_threads = SISL_OPTIONS["num_threads"].as< uint32_t >(); 91 | 92 | auto ret = RUN_ALL_TESTS(); 93 | return ret; 94 | } 95 | -------------------------------------------------------------------------------- /src/logging/test/example.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Brian Szmyd 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | SISL_LOGGING_DECL(my_module) 28 | SISL_LOGGING_DEF(my_module) 29 | SISL_LOGGING_INIT(my_module) 30 | 31 | void func() { 32 | LOGINFO("Thread func started"); 33 | size_t i{0}; 34 | while (i < 3) { 35 | LOGINFO("Thread func {}th iteration", i + 1); 36 | std::this_thread::sleep_for(std::chrono::seconds{3}); 37 | ++i; 38 | } 39 | } 40 | 41 | // clang-format off 42 | SISL_OPTION_GROUP(test_logging, 43 | (signal, "si", "signal option", "signal option", ::cxxopts::value(), "1-6")) 44 | // clang-format on 45 | 46 | #define ENABLED_OPTIONS test_logging, logging 47 | 48 | SISL_OPTIONS_ENABLE(ENABLED_OPTIONS) 49 | 50 | int main(int argc, char** argv) { 51 | SISL_OPTIONS_LOAD(argc, argv, ENABLED_OPTIONS) 52 | sisl::logging::SetLogger(std::string{argv[0]}); 53 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 54 | 55 | SISL_LOG_LEVEL(my_module, spdlog::level::level_enum::trace); 56 | sisl::logging::install_crash_handler(); 57 | 58 | std::thread t{func}; 59 | std::this_thread::sleep_for(std::chrono::seconds{1}); 60 | std::cout << "spdlog level base = " << module_level_base << "\n"; 61 | LOGTRACE("Trace"); 62 | LOGDEBUG("Debug"); 63 | LOGINFO("Info"); 64 | LOGWARN("Warning"); 65 | LOGERROR("Error"); 66 | LOGCRITICAL("Critical"); 67 | 68 | SISL_LOG_LEVEL(my_module, spdlog::level::level_enum::info); 69 | LOGINFOMOD(my_module, "Enabled Module Logger"); 70 | LOGTRACEMOD(my_module, "Trace Module"); 71 | 72 | // RELEASE_ASSERT_EQ(0, 1, "test"); 73 | 74 | // sisl::logging::log_stack_trace(true); 75 | #if 0 76 | sisl::logging::log_stack_trace(); 77 | #else 78 | /* 79 | // NOTE: Some reason signal not being recognized as option 80 | switch (SISL_OPTIONS["signal"].as< uint32_t >()) { 81 | case 1: 82 | std::raise(SIGABRT); 83 | case 2: 84 | std::raise(SIGFPE); 85 | case 3: 86 | std::raise(SIGSEGV); 87 | case 4: 88 | std::raise(SIGILL); 89 | case 5: 90 | std::raise(SIGTERM); 91 | case 6: 92 | std::raise(SIGINT); 93 | default: break; 94 | } 95 | */ 96 | 97 | std::raise(SIGABRT); 98 | // std::raise(SIGFPE); 99 | // std::raise(SIGSEGV); 100 | // std::raise(SIGILL); 101 | // std::raise(SIGTERM); 102 | // std::raise(SIGINT); 103 | 104 | /* 105 | int* ptr{nullptr}; 106 | [[maybe_unused]] int i{*ptr}; 107 | std::cout << i << std::endl; 108 | */ 109 | 110 | #endif 111 | 112 | if (t.joinable()) t.join(); 113 | return 0; 114 | } 115 | -------------------------------------------------------------------------------- /src/fds/tests/obj_allocator_benchmark.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include 25 | #include "sisl/logging/logging.h" 26 | #include "sisl/options/options.h" 27 | 28 | #include "sisl/metrics/metrics.hpp" 29 | #include "sisl/fds/obj_allocator.hpp" 30 | 31 | SISL_LOGGING_INIT(HOMESTORE_LOG_MODS) 32 | RCU_REGISTER_INIT 33 | 34 | namespace { 35 | std::mutex s_print_mutex; 36 | constexpr size_t ITERATIONS{10000000}; 37 | constexpr size_t THREADS{8}; 38 | 39 | struct my_request { 40 | int m_a; 41 | int m_b[10]; 42 | std::string m_c; 43 | uint64_t m_d; 44 | }; 45 | 46 | void setup() {} 47 | void test_malloc(benchmark::State& state) { 48 | uint64_t counter{0}; 49 | static thread_local std::random_device rd{}; 50 | static thread_local std::default_random_engine engine{rd()}; 51 | 52 | for ([[maybe_unused]] auto si : state) { // Loops up to iteration count 53 | my_request* req; 54 | benchmark::DoNotOptimize(req = new my_request()); 55 | req->m_a = 10; 56 | req->m_b[0] = 100; 57 | std::uniform_int_distribution< uint64_t > dist{0, RAND_MAX}; 58 | req->m_d = req->m_a * dist(engine); 59 | counter += req->m_d; 60 | delete (req); 61 | } 62 | { 63 | std::scoped_lock< std::mutex > lock{s_print_mutex}; 64 | std::cout << "Counter = " << counter << std::endl; 65 | } 66 | } 67 | 68 | void test_obj_alloc(benchmark::State& state) { 69 | uint64_t counter{0}; 70 | static thread_local std::random_device rd{}; 71 | static thread_local std::default_random_engine engine{rd()}; 72 | for ([[maybe_unused]] auto si : state) { // Loops up to iteration count 73 | my_request* req; 74 | benchmark::DoNotOptimize(req = sisl::ObjectAllocator< my_request >::make_object()); 75 | req->m_a = 10; 76 | req->m_b[0] = 100; 77 | std::uniform_int_distribution< uint64_t > dist{0, RAND_MAX}; 78 | req->m_d = req->m_a * dist(engine); 79 | counter += req->m_d; 80 | sisl::ObjectAllocator< my_request >::deallocate(req); 81 | } 82 | 83 | { 84 | std::scoped_lock< std::mutex > lock{s_print_mutex}; 85 | std::cout << "Counter = " << counter << std::endl; 86 | } 87 | } 88 | } // namespace 89 | 90 | BENCHMARK(test_malloc)->Iterations(ITERATIONS)->Threads(THREADS); 91 | BENCHMARK(test_obj_alloc)->Iterations(ITERATIONS)->Threads(THREADS); 92 | 93 | SISL_OPTIONS_ENABLE(logging) 94 | int main(int argc, char** argv) { 95 | SISL_OPTIONS_LOAD(argc, argv, logging) 96 | setup(); 97 | ::benchmark::Initialize(&argc, argv); 98 | ::benchmark::RunSpecifiedBenchmarks(); 99 | // std::cout << "Metrics: " << sisl::MetricsFarm::getInstance().get_result_in_json().dump(4) << "\n"; 100 | } 101 | -------------------------------------------------------------------------------- /src/fds/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 3.11) 2 | 3 | add_library(sisl_buffer) 4 | target_sources(sisl_buffer PRIVATE 5 | buffer.cpp 6 | ) 7 | target_link_libraries(sisl_buffer PUBLIC 8 | sisl_metrics 9 | folly::folly 10 | ) 11 | 12 | include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}) 13 | 14 | add_executable(test_stream_tracker) 15 | target_sources(test_stream_tracker PRIVATE 16 | tests/test_stream_tracker.cpp 17 | ) 18 | target_link_libraries(test_stream_tracker sisl_buffer GTest::gtest) 19 | add_test(NAME StreamTracker COMMAND test_stream_tracker) 20 | 21 | add_executable(test_atomic_status_counter) 22 | target_sources(test_atomic_status_counter PRIVATE 23 | tests/test_atomic_status_counter.cpp 24 | ) 25 | target_link_libraries(test_atomic_status_counter sisl_logging GTest::gtest atomic) 26 | add_test(NAME AtomicStatusCounter COMMAND test_atomic_status_counter) 27 | 28 | add_executable(test_bitset) 29 | target_sources(test_bitset PRIVATE 30 | tests/test_bitset.cpp 31 | ) 32 | target_link_libraries(test_bitset sisl_buffer GTest::gtest) 33 | add_test(NAME Bitset COMMAND test_bitset) 34 | 35 | add_executable(test_bitword) 36 | target_sources(test_bitword PRIVATE 37 | tests/test_bitword.cpp 38 | ) 39 | target_link_libraries(test_bitword sisl_logging GTest::gtest) 40 | add_test(NAME Bitword COMMAND test_bitset) 41 | 42 | add_executable(test_compact_bitset) 43 | target_sources(test_compact_bitset PRIVATE 44 | tests/test_compact_bitset.cpp 45 | ) 46 | target_link_libraries(test_compact_bitset sisl_buffer GTest::gtest) 47 | add_test(NAME CompactBitset COMMAND test_compact_bitset) 48 | 49 | add_executable(test_concurrent_insert_vector) 50 | target_sources(test_concurrent_insert_vector PRIVATE 51 | tests/test_concurrent_insert_vector.cpp 52 | ) 53 | target_link_libraries(test_concurrent_insert_vector sisl_buffer GTest::gtest) 54 | add_test(NAME ConcurrentInsertVector COMMAND test_concurrent_insert_vector) 55 | 56 | add_executable(concurrent_insert_vector_bench) 57 | target_sources(concurrent_insert_vector_bench PRIVATE 58 | tests/concurrent_insert_vector_bench.cpp 59 | ) 60 | target_link_libraries(concurrent_insert_vector_bench sisl_buffer benchmark::benchmark) 61 | add_test(NAME ConcurrentVectorBench COMMAND concurrent_insert_vector_bench) 62 | 63 | add_executable(obj_allocator_benchmark) 64 | target_sources(obj_allocator_benchmark PRIVATE 65 | tests/obj_allocator_benchmark.cpp 66 | ) 67 | target_link_libraries(obj_allocator_benchmark sisl_buffer benchmark::benchmark) 68 | add_test(NAME ObjAllocatorBenchmark COMMAND obj_allocator_benchmark) 69 | 70 | add_executable(test_obj_allocator) 71 | target_sources(test_obj_allocator PRIVATE 72 | tests/test_obj_allocator.cpp 73 | ) 74 | target_link_libraries(test_obj_allocator sisl_buffer) 75 | add_test(NAME ObjAlloc COMMAND test_obj_allocator) 76 | 77 | add_executable(test_sg_list) 78 | target_sources(test_sg_list PRIVATE 79 | tests/test_sg_list.cpp 80 | ) 81 | target_link_libraries(test_sg_list sisl_buffer folly::folly GTest::gtest) 82 | add_test(NAME SgList COMMAND test_sg_list) 83 | 84 | 85 | if (DEFINED MALLOC_IMPL) 86 | if (${MALLOC_IMPL} STREQUAL "jemalloc") 87 | add_executable(test_jemalloc) 88 | target_sources(test_jemalloc PRIVATE 89 | tests/test_jemalloc_helper.cpp 90 | ) 91 | target_link_libraries(test_jemalloc sisl_buffer GTest::gtest jemalloc::jemalloc) 92 | add_test(NAME TestJemalloc COMMAND test_jemalloc) 93 | elseif (${MALLOC_IMPL} STREQUAL "tcmalloc") 94 | add_executable(test_tcmalloc) 95 | target_sources(test_tcmalloc PRIVATE 96 | tests/test_tcmalloc_helper.cpp 97 | ) 98 | target_link_libraries(test_tcmalloc sisl_buffer GTest::gtest gperftools::gperftools) 99 | add_test(NAME TestTcmalloc COMMAND test_tcmalloc) 100 | endif() 101 | endif() 102 | -------------------------------------------------------------------------------- /src/wisr/tests/wisr_list_benchmark.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | #include "sisl/utility/thread_buffer.hpp" 24 | #include "sisl/wisr/wisr_ds.hpp" 25 | 26 | THREAD_BUFFER_INIT 27 | RCU_REGISTER_INIT 28 | 29 | //#define ITERATIONS 100000 30 | static constexpr size_t ITERATIONS{100}; 31 | static constexpr size_t THREADS{8}; 32 | 33 | using namespace sisl; 34 | 35 | std::unique_ptr< std::list< uint64_t > > glob_lock_list; 36 | std::mutex glob_list_mutex; 37 | 38 | std::unique_ptr< sisl::wisr_list< uint64_t > > glob_wisr_list; 39 | 40 | #define NENTRIES_PER_THREAD 20000 41 | 42 | void setup() { 43 | glob_lock_list = std::make_unique< std::list< uint64_t > >(); 44 | glob_wisr_list = std::make_unique< sisl::wisr_list< uint64_t > >(); 45 | } 46 | 47 | void test_locked_list_insert(benchmark::State& state) { 48 | for (auto s : state) { // Loops upto iteration count 49 | // state.PauseTiming(); 50 | // glob_lock_list.reserve(NENTRIES_PER_THREAD * THREADS); 51 | // state.ResumeTiming(); 52 | 53 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 54 | std::lock_guard< std::mutex > lg(glob_list_mutex); 55 | glob_lock_list->emplace_back(i); 56 | } 57 | 58 | // state.PauseTiming(); 59 | // glob_lock_list.clear(); 60 | // state.ResumeTiming(); 61 | } 62 | } 63 | 64 | void test_wisr_list_insert(benchmark::State& state) { 65 | for (auto s : state) { 66 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 67 | glob_wisr_list->emplace_back(i); 68 | } 69 | } 70 | } 71 | 72 | void test_locked_list_read(benchmark::State& state) { 73 | uint64_t ret; 74 | for (auto s : state) { // Loops upto iteration count 75 | std::lock_guard< std::mutex > lg(glob_list_mutex); 76 | for (auto& v : *glob_lock_list) { 77 | benchmark::DoNotOptimize(ret = v * 2); 78 | } 79 | } 80 | } 81 | 82 | void test_wisr_list_read(benchmark::State& state) { 83 | uint64_t ret; 84 | for (auto s : state) { // Loops upto iteration count 85 | auto vec = glob_wisr_list->get_copy_and_reset(); 86 | for (auto& v : *vec) { 87 | benchmark::DoNotOptimize(ret = v * 2); 88 | } 89 | } 90 | } 91 | 92 | BENCHMARK(test_locked_list_insert)->Iterations(ITERATIONS)->Threads(THREADS); 93 | BENCHMARK(test_wisr_list_insert)->Iterations(ITERATIONS)->Threads(THREADS); 94 | BENCHMARK(test_locked_list_insert)->Iterations(ITERATIONS)->Threads(1); 95 | BENCHMARK(test_wisr_list_insert)->Iterations(ITERATIONS)->Threads(1); 96 | BENCHMARK(test_locked_list_read)->Iterations(ITERATIONS)->Threads(1); 97 | BENCHMARK(test_wisr_list_read)->Iterations(ITERATIONS)->Threads(1); 98 | 99 | int main(int argc, char** argv) { 100 | setup(); 101 | ::benchmark::Initialize(&argc, argv); 102 | ::benchmark::RunSpecifiedBenchmarks(); 103 | } 104 | -------------------------------------------------------------------------------- /include/sisl/cache/hash_entry_base.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | using namespace boost::intrusive; 23 | 24 | namespace sisl { 25 | #pragma pack(1) 26 | class ValueEntryBase { 27 | static constexpr size_t SIZE_BITS = 29; 28 | static constexpr size_t PINNED_BITS = 1; 29 | static constexpr size_t RECORD_FAMILY_ID_BITS = 2; 30 | 31 | struct cache_info { 32 | uint32_t size : SIZE_BITS; 33 | uint32_t pinned : PINNED_BITS; 34 | uint32_t record_family_id : RECORD_FAMILY_ID_BITS; 35 | 36 | cache_info() : size{0}, pinned{0}, record_family_id{0} {} 37 | void set_pinned(bool is_pinned) { pinned = is_pinned ? 1 : 0; } 38 | void set_size(uint32_t sz) { size = sz; } 39 | void set_family_id(uint32_t fid) { record_family_id = fid; } 40 | }; 41 | 42 | public: 43 | mutable list_member_hook< link_mode< auto_unlink > > m_member_hook; 44 | mutable cache_info m_u; 45 | 46 | public: 47 | ValueEntryBase() = default; 48 | ValueEntryBase(const ValueEntryBase&) = delete; 49 | ValueEntryBase& operator=(const ValueEntryBase&) = delete; 50 | ValueEntryBase(ValueEntryBase&& other) { 51 | m_u = std::move(other.m_u); 52 | m_member_hook.swap_nodes(other.m_member_hook); 53 | } 54 | ValueEntryBase& operator=(ValueEntryBase&& other) { 55 | m_u = std::move(other.m_u); 56 | m_member_hook.swap_nodes(other.m_member_hook); 57 | return *this; 58 | } 59 | 60 | void set_size(const uint32_t size) { m_u.size = size; } 61 | void set_pinned() { m_u.set_pinned(true); } 62 | void set_unpinned() { m_u.set_pinned(false); } 63 | void set_record_family(const uint32_t record_fid) { m_u.record_family_id = record_fid; } 64 | 65 | uint32_t size() const { return m_u.size; } 66 | bool is_pinned() const { return (m_u.pinned == 1); } 67 | uint32_t record_family_id() const { return m_u.record_family_id; } 68 | 69 | static constexpr size_t max_record_families() { return (1 << RECORD_FAMILY_ID_BITS); } 70 | }; 71 | #pragma pack() 72 | 73 | class CacheMetrics : public sisl::MetricsGroupWrapper { 74 | public: 75 | explicit CacheMetrics() : sisl::MetricsGroupWrapper("Cache") { 76 | REGISTER_COUNTER(cache_object_count, "Total number of cache entries", sisl::_publish_as::publish_as_gauge); 77 | REGISTER_COUNTER(cache_size, "Total size of cache", sisl::_publish_as::publish_as_gauge); 78 | REGISTER_COUNTER(cache_num_evictions, "Total number of cache evictions"); 79 | REGISTER_COUNTER(cache_num_evictions_punt, "Total number of cache evictions punted because of busy"); 80 | 81 | register_me_to_farm(); 82 | } 83 | 84 | CacheMetrics(const CacheMetrics&) = delete; 85 | CacheMetrics(CacheMetrics&&) noexcept = delete; 86 | CacheMetrics& operator=(const CacheMetrics&) = delete; 87 | CacheMetrics& operator=(CacheMetrics&&) noexcept = delete; 88 | 89 | ~CacheMetrics() { deregister_me_from_farm(); } 90 | }; 91 | 92 | } // namespace sisl 93 | -------------------------------------------------------------------------------- /src/wisr/tests/wisr_deque_benchmark.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "sisl/utility/thread_buffer.hpp" 25 | #include "sisl/wisr/wisr_ds.hpp" 26 | 27 | THREAD_BUFFER_INIT 28 | RCU_REGISTER_INIT 29 | 30 | using namespace sisl; 31 | 32 | std::unique_ptr< std::deque< uint64_t > > glob_lock_deque; 33 | std::mutex glob_deque_mutex; 34 | 35 | std::unique_ptr< sisl::wisr_deque< uint64_t > > glob_wisr_deque; 36 | 37 | //#define ITERATIONS 100000 38 | static constexpr size_t ITERATIONS{100}; 39 | static constexpr size_t THREADS{8}; 40 | static constexpr size_t NENTRIES_PER_THREAD{20000}; 41 | 42 | void setup() { 43 | glob_lock_deque = std::make_unique< std::deque< uint64_t > >(); 44 | glob_wisr_deque = std::make_unique< sisl::wisr_deque< uint64_t > >(); 45 | } 46 | 47 | void test_locked_deque_insert(benchmark::State& state) { 48 | for (auto s : state) { // Loops upto iteration count 49 | // state.PauseTiming(); 50 | // glob_lock_deque.reserve(NENTRIES_PER_THREAD * THREADS); 51 | // state.ResumeTiming(); 52 | 53 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 54 | std::lock_guard< std::mutex > lg(glob_deque_mutex); 55 | glob_lock_deque->emplace_back(i); 56 | } 57 | 58 | // state.PauseTiming(); 59 | // glob_lock_deque.clear(); 60 | // state.ResumeTiming(); 61 | } 62 | } 63 | 64 | void test_wisr_deque_insert(benchmark::State& state) { 65 | for (auto s : state) { 66 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 67 | glob_wisr_deque->emplace_back(i); 68 | } 69 | } 70 | } 71 | 72 | void test_locked_deque_read(benchmark::State& state) { 73 | uint64_t ret; 74 | for (auto s : state) { // Loops upto iteration count 75 | std::lock_guard< std::mutex > lg(glob_deque_mutex); 76 | for (const auto& v : *glob_lock_deque) { 77 | benchmark::DoNotOptimize(ret = v * 2); 78 | } 79 | } 80 | } 81 | 82 | void test_wisr_deque_read(benchmark::State& state) { 83 | uint64_t ret; 84 | for (auto s : state) { // Loops upto iteration count 85 | auto vec = glob_wisr_deque->get_copy_and_reset(); 86 | for (const auto& v : *vec) { 87 | benchmark::DoNotOptimize(ret = v * 2); 88 | } 89 | } 90 | } 91 | 92 | BENCHMARK(test_locked_deque_insert)->Iterations(ITERATIONS)->Threads(THREADS); 93 | BENCHMARK(test_wisr_deque_insert)->Iterations(ITERATIONS)->Threads(THREADS); 94 | BENCHMARK(test_locked_deque_insert)->Iterations(ITERATIONS)->Threads(1); 95 | BENCHMARK(test_wisr_deque_insert)->Iterations(ITERATIONS)->Threads(1); 96 | BENCHMARK(test_locked_deque_read)->Iterations(ITERATIONS)->Threads(1); 97 | BENCHMARK(test_wisr_deque_read)->Iterations(ITERATIONS)->Threads(1); 98 | 99 | int main(int argc, char** argv) { 100 | setup(); 101 | ::benchmark::Initialize(&argc, argv); 102 | ::benchmark::RunSpecifiedBenchmarks(); 103 | } 104 | -------------------------------------------------------------------------------- /src/utility/tests/test_objlife_counter.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "sisl/fds/buffer.hpp" 14 | #include "sisl/utility/obj_life_counter.hpp" 15 | 16 | SISL_LOGGING_INIT(test_objlife) 17 | 18 | template < typename T1, typename T2 > 19 | struct TestClass : sisl::ObjLifeCounter< TestClass< T1, T2 > > { 20 | TestClass() : m_x{rand()} {} 21 | 22 | private: 23 | int m_x; 24 | }; 25 | 26 | struct ObjLifeTest : public testing::Test { 27 | public: 28 | ObjLifeTest() : testing::Test{} {} 29 | ObjLifeTest(const ObjLifeTest&) = delete; 30 | ObjLifeTest(ObjLifeTest&&) noexcept = delete; 31 | ObjLifeTest& operator=(const ObjLifeTest&) = delete; 32 | ObjLifeTest& operator=(ObjLifeTest&&) noexcept = delete; 33 | virtual ~ObjLifeTest() override = default; 34 | }; 35 | 36 | TEST_F(ObjLifeTest, BasicCount) { 37 | TestClass< char*, unsigned int > i1; 38 | TestClass< double, sisl::blob > d1; 39 | 40 | sisl::ObjCounterRegistry::enable_metrics_reporting(); 41 | { 42 | auto ip2 = std::make_unique< TestClass< char*, unsigned int > >(); 43 | sisl::ObjCounterRegistry::foreach ([this](const std::string& name, int64_t created, int64_t alive) { 44 | if (name == "TestClass") { 45 | ASSERT_EQ(created, 2); 46 | ASSERT_EQ(alive, 2); 47 | } else if (name == "TestClass") { 48 | ASSERT_EQ(created, 1); 49 | ASSERT_EQ(created, 1); 50 | } else { 51 | ASSERT_TRUE(false); 52 | } 53 | }); 54 | } 55 | 56 | sisl::ObjCounterRegistry::foreach ([this](const std::string& name, int64_t created, int64_t alive) { 57 | if (name == "TestClass") { 58 | ASSERT_EQ(created, 2); 59 | ASSERT_EQ(alive, 1); 60 | } else if (name == "TestClass") { 61 | ASSERT_EQ(created, 1); 62 | ASSERT_EQ(created, 1); 63 | } else { 64 | ASSERT_TRUE(false); 65 | } 66 | }); 67 | 68 | const nlohmann::json j{sisl::MetricsFarm::getInstance().get_result_in_json()}; 69 | std::cout << "Json output = " << j.dump(2); 70 | 71 | const auto prom_format{sisl::MetricsFarm::getInstance().report(sisl::ReportFormat::kTextFormat)}; 72 | std::cout << "Prometheus Output = " << prom_format; 73 | EXPECT_TRUE(prom_format.find(R"(TestClass_double__sisl::blob_{entity="Singleton",type="alive"} 1)") != 74 | std::string::npos); 75 | EXPECT_TRUE(prom_format.find(R"(TestClass_double__sisl::blob_{entity="Singleton",type="created"} 1)") != 76 | std::string::npos); 77 | EXPECT_TRUE(prom_format.find(R"(TestClass_charP__unsigned_int_{entity="Singleton",type="alive"} 1)") != 78 | std::string::npos); 79 | EXPECT_TRUE(prom_format.find(R"(TestClass_charP__unsigned_int_{entity="Singleton",type="created"} 2)") != 80 | std::string::npos); 81 | } 82 | 83 | uint32_t g_num_threads; 84 | SISL_OPTIONS_ENABLE(logging, test_objlife) 85 | SISL_OPTION_GROUP(test_objlife, 86 | (num_threads, "", "num_threads", "number of threads", 87 | ::cxxopts::value< uint32_t >()->default_value("8"), "number")) 88 | 89 | int main(int argc, char* argv[]) { 90 | ::testing::InitGoogleTest(&argc, argv); 91 | SISL_OPTIONS_LOAD(argc, argv, logging, test_objlife); 92 | sisl::logging::SetLogger("test_objlife"); 93 | spdlog::set_pattern("[%D %T%z] [%^%l%$] [%n] [%t] %v"); 94 | 95 | g_num_threads = SISL_OPTIONS["num_threads"].as< uint32_t >(); 96 | 97 | const auto ret{RUN_ALL_TESTS()}; 98 | return ret; 99 | } 100 | -------------------------------------------------------------------------------- /include/sisl/utility/non_null_ptr.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | 21 | namespace sisl { 22 | template < typename T > 23 | class non_null_unique_ptr : public std::unique_ptr< T > { 24 | public: 25 | non_null_unique_ptr() noexcept : std::unique_ptr< T >{new T} {}; 26 | non_null_unique_ptr(T* ptr) : std::unique_ptr< T >{ptr} { 27 | if (!*this) { std::unique_ptr< T >::reset(new T); } 28 | } 29 | non_null_unique_ptr(const non_null_unique_ptr& other) = delete; 30 | non_null_unique_ptr& operator=(const non_null_unique_ptr< T >& other) = delete; 31 | 32 | ~non_null_unique_ptr() { std::unique_ptr< T >::reset(); } 33 | 34 | non_null_unique_ptr(non_null_unique_ptr&& other) : std::unique_ptr< T >{std::move(other)} { assert(*this); } 35 | 36 | non_null_unique_ptr& operator=(non_null_unique_ptr&& other) { 37 | assert(other); 38 | std::unique_ptr< T >::operator=(std::move(other)); 39 | return *this; 40 | }; 41 | 42 | non_null_unique_ptr& operator=(std::unique_ptr< T >&& other) { 43 | assert(other); 44 | std::unique_ptr< T >::operator=(std::move(other)); 45 | return *this; 46 | }; 47 | 48 | T& operator*() const { 49 | assert(*this); 50 | return std::unique_ptr< T >::operator*(); 51 | } 52 | 53 | T* operator->() const noexcept { 54 | assert(*this); 55 | return std::unique_ptr< T >::operator->(); 56 | } 57 | 58 | T* release() noexcept { 59 | assert(*this); 60 | T* ret = std::unique_ptr< T >::release(); 61 | reset(); 62 | return ret; 63 | } 64 | 65 | void reset(T* t = nullptr) noexcept { 66 | if (t == nullptr) t = new T; 67 | std::unique_ptr< T >::reset(t); 68 | } 69 | 70 | void swap(non_null_unique_ptr& other) noexcept { 71 | if (!other) other.reset(new T); 72 | std::unique_ptr< T >::swap(other); 73 | } 74 | }; 75 | 76 | template < typename T > 77 | struct embedded_t : public T { 78 | embedded_t(T* t) { 79 | if (t != nullptr) { 80 | *static_cast< T* >(this) = std::move(*t); 81 | delete t; 82 | } 83 | } 84 | embedded_t() = default; 85 | 86 | const T* get() const noexcept { return this; } 87 | T* get() noexcept { return this; } 88 | const T* operator->() const noexcept { return this; } 89 | T* operator->() noexcept { return this; } 90 | 91 | const T& operator*() const noexcept { return *this; } 92 | T& operator*() noexcept { return *this; } 93 | 94 | explicit operator bool() const noexcept { return true; } 95 | 96 | T* release() noexcept { 97 | embedded_t* ret = new embedded_t(); 98 | *ret = *this; 99 | return static_cast< T* >(ret); 100 | } 101 | 102 | void reset() noexcept { *this = embedded_t{nullptr}; } 103 | }; 104 | 105 | template < class T > 106 | constexpr bool operator==(const embedded_t< T >& x, std::nullptr_t) noexcept { 107 | return false; 108 | } 109 | 110 | template < class T > 111 | constexpr bool operator!=(const embedded_t< T >& x, std::nullptr_t) noexcept { 112 | return true; 113 | } 114 | } // namespace sisl 115 | -------------------------------------------------------------------------------- /include/sisl/fds/thread_vector.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | namespace sisl { 27 | 28 | struct thread_vector_iterator { 29 | size_t next_thread{0}; 30 | size_t next_idx_in_thread{0}; 31 | }; 32 | 33 | /* 34 | * This data structure inserts elements into per thread buffer and provide apis to access the elements. 35 | */ 36 | template < typename T > 37 | class ThreadVector { 38 | public: 39 | ThreadVector() {} 40 | ThreadVector(const uint64_t size) : m_wvec{size} {} 41 | ThreadVector(const ThreadVector&) = delete; 42 | ThreadVector(ThreadVector&&) noexcept = delete; 43 | ThreadVector& operator=(const ThreadVector&) = delete; 44 | ThreadVector& operator=(ThreadVector&&) noexcept = delete; 45 | ~ThreadVector() { clear_old_version(); } 46 | 47 | template < typename InputType, 48 | typename = typename std::enable_if< 49 | std::is_convertible< typename std::decay< InputType >::type, T >::value >::type > 50 | void push_back(InputType&& ele) { 51 | m_wvec.insertable([&ele](std::vector< T >* const tvec) { tvec->push_back(std::forward< InputType >(ele)); }); 52 | } 53 | 54 | thread_vector_iterator begin(bool latest) { 55 | if (latest) { 56 | const auto new_vec{m_wvec.get_unmerged_and_reset()}; 57 | m_per_thread_vec_ptrs.insert(std::end(m_per_thread_vec_ptrs), std::cbegin(new_vec), std::cend(new_vec)); 58 | } 59 | return thread_vector_iterator{}; 60 | } 61 | 62 | T* next(thread_vector_iterator& it) { 63 | while (it.next_thread < m_per_thread_vec_ptrs.size()) { 64 | const auto& tvec = m_per_thread_vec_ptrs[it.next_thread]; 65 | if (it.next_idx_in_thread < tvec->size()) { 66 | return &tvec->at(it.next_idx_in_thread++); 67 | } else { 68 | ++it.next_thread; 69 | it.next_idx_in_thread = 0; 70 | } 71 | } 72 | return nullptr; 73 | } 74 | 75 | void clear() { 76 | m_wvec.reset(); // Erase all newer version ptrs 77 | clear_old_version(); // Erase all older version ptrs 78 | } 79 | 80 | size_t size() { 81 | size_t sz{0}; 82 | // Get the size from older version 83 | for (const auto& tvec : m_per_thread_vec_ptrs) { 84 | sz += tvec->size(); 85 | } 86 | 87 | // Get the size from current running version 88 | m_wvec.foreach_thread_member([&sz](const sisl::vector_wrapper< T >* tvec) { 89 | if (tvec) { sz += tvec->size(); } 90 | }); 91 | return sz; 92 | } 93 | 94 | private: 95 | void clear_old_version() { 96 | for (auto& tvec : m_per_thread_vec_ptrs) { 97 | delete tvec; 98 | } 99 | m_per_thread_vec_ptrs.clear(); 100 | } 101 | 102 | private: 103 | sisl::wisr_framework< sisl::vector_wrapper< T >, size_t > m_wvec; 104 | std::vector< sisl::vector_wrapper< T >* > m_per_thread_vec_ptrs; 105 | }; 106 | 107 | } // namespace sisl 108 | -------------------------------------------------------------------------------- /src/wisr/tests/wisr_vector_benchmark.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | 24 | #include "sisl/utility/thread_buffer.hpp" 25 | #include "sisl/wisr/wisr_ds.hpp" 26 | 27 | THREAD_BUFFER_INIT 28 | RCU_REGISTER_INIT 29 | 30 | using namespace sisl; 31 | 32 | std::unique_ptr< std::vector< uint64_t > > glob_lock_vector; 33 | std::mutex glob_vector_mutex; 34 | 35 | std::unique_ptr< sisl::wisr_vector< uint64_t > > glob_wisr_vector; 36 | 37 | //#define ITERATIONS 100000 38 | static constexpr size_t ITERATIONS{100}; 39 | static constexpr size_t THREADS{8}; 40 | static constexpr size_t NENTRIES_PER_THREAD{200}; 41 | 42 | void setup() { 43 | glob_lock_vector = std::make_unique< std::vector< uint64_t > >(); 44 | glob_lock_vector->reserve(NENTRIES_PER_THREAD * THREADS * ITERATIONS); 45 | glob_wisr_vector = 46 | std::make_unique< sisl::wisr_vector< uint64_t > >((size_t)NENTRIES_PER_THREAD * THREADS * ITERATIONS); 47 | } 48 | 49 | void test_locked_vector_insert(benchmark::State& state) { 50 | for (auto s : state) { // Loops upto iteration count 51 | // state.PauseTiming(); 52 | // glob_lock_vector.reserve(NENTRIES_PER_THREAD * THREADS); 53 | // state.ResumeTiming(); 54 | 55 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 56 | std::lock_guard< std::mutex > lg(glob_vector_mutex); 57 | glob_lock_vector->emplace_back(i); 58 | } 59 | 60 | // state.PauseTiming(); 61 | // glob_lock_vector.clear(); 62 | // state.ResumeTiming(); 63 | } 64 | } 65 | 66 | void test_wisr_vector_insert(benchmark::State& state) { 67 | for (auto s : state) { 68 | for (size_t i{0}; i < NENTRIES_PER_THREAD; ++i) { 69 | glob_wisr_vector->emplace_back(i); 70 | } 71 | } 72 | } 73 | 74 | void test_locked_vector_read(benchmark::State& state) { 75 | uint64_t ret; 76 | for (auto s : state) { // Loops upto iteration count 77 | std::lock_guard< std::mutex > lg(glob_vector_mutex); 78 | for (auto& v : *glob_lock_vector) { 79 | benchmark::DoNotOptimize(ret = v * 2); 80 | } 81 | } 82 | } 83 | 84 | void test_wisr_vector_read(benchmark::State& state) { 85 | uint64_t ret; 86 | for (auto s : state) { // Loops upto iteration count 87 | auto vec = glob_wisr_vector->get_copy_and_reset(); 88 | for (auto& v : *vec) { 89 | benchmark::DoNotOptimize(ret = v * 2); 90 | } 91 | } 92 | } 93 | 94 | BENCHMARK(test_locked_vector_insert)->Iterations(ITERATIONS)->Threads(THREADS); 95 | BENCHMARK(test_wisr_vector_insert)->Iterations(ITERATIONS)->Threads(THREADS); 96 | BENCHMARK(test_locked_vector_insert)->Iterations(ITERATIONS)->Threads(1); 97 | BENCHMARK(test_wisr_vector_insert)->Iterations(ITERATIONS)->Threads(1); 98 | BENCHMARK(test_locked_vector_read)->Iterations(ITERATIONS)->Threads(1); 99 | BENCHMARK(test_wisr_vector_read)->Iterations(ITERATIONS)->Threads(1); 100 | 101 | int main(int argc, char** argv) { 102 | setup(); 103 | ::benchmark::Initialize(&argc, argv); 104 | ::benchmark::RunSpecifiedBenchmarks(); 105 | } 106 | -------------------------------------------------------------------------------- /include/sisl/cache/evictor.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | namespace sisl { 27 | typedef ValueEntryBase CacheRecord; 28 | 29 | class Evictor { 30 | public: 31 | typedef std::function< bool(const CacheRecord&) > eviction_cb_t; 32 | using can_evict_cb_t = eviction_cb_t; 33 | 34 | // struct to hold the eviction callbacks for each record family 35 | // can_evict_cb: called before eviction to check if the record can be evicted. 36 | // post_eviction_cb: called after eviction to do any cleanup. If this returns false, the record is reinserted. 37 | // and we try to evict the next record. 38 | struct RecordFamily { 39 | Evictor::eviction_cb_t can_evict_cb{nullptr}; 40 | Evictor::eviction_cb_t post_eviction_cb{nullptr}; 41 | }; 42 | 43 | Evictor(const int64_t max_size, const uint32_t num_partitions) : 44 | m_max_size{max_size}, m_num_partitions{num_partitions} {} 45 | Evictor(const Evictor&) = delete; 46 | Evictor(Evictor&&) noexcept = delete; 47 | Evictor& operator=(const Evictor&) = delete; 48 | Evictor& operator=(Evictor&&) noexcept = delete; 49 | virtual ~Evictor() = default; 50 | 51 | uint32_t register_record_family(RecordFamily record_family) { 52 | uint32_t id{0}; 53 | std::unique_lock lk(m_reg_mtx); 54 | while (id < m_eviction_cbs.size()) { 55 | if (m_eviction_cbs[id].first == false) { 56 | m_eviction_cbs[id] = std::make_pair(true, record_family); 57 | return id; 58 | } 59 | ++id; 60 | } 61 | RELEASE_ASSERT(false, "More than {} record types registered", CacheRecord::max_record_families()); 62 | return 0; 63 | } 64 | 65 | void unregister_record_family(const uint32_t record_type_id) { 66 | std::unique_lock lk(m_reg_mtx); 67 | m_eviction_cbs[record_type_id] = std::make_pair(false, RecordFamily{}); 68 | } 69 | 70 | void add_metrics(CacheMetrics* metrics) { 71 | m_metrics = metrics; 72 | } 73 | CacheMetrics* metrics_ptr() { 74 | return m_metrics; 75 | } 76 | 77 | virtual bool add_record(uint64_t hash_code, CacheRecord& record) = 0; 78 | virtual void remove_record(uint64_t hash_code, CacheRecord& record) = 0; 79 | virtual void record_accessed(uint64_t hash_code, CacheRecord& record) = 0; 80 | virtual void record_resized(uint64_t hash_code, const CacheRecord& record, uint32_t old_size) = 0; 81 | 82 | int64_t max_size() const { return m_max_size; } 83 | uint32_t num_partitions() const { return m_num_partitions; } 84 | const eviction_cb_t& can_evict_cb(const uint32_t record_id) const { return m_eviction_cbs[record_id].second.can_evict_cb; } 85 | const eviction_cb_t& post_eviction_cb(const uint32_t record_id) const { return m_eviction_cbs[record_id].second.post_eviction_cb; } 86 | 87 | private: 88 | int64_t m_max_size; 89 | uint32_t m_num_partitions; 90 | 91 | std::mutex m_reg_mtx; 92 | std::array< std::pair< bool /*registered*/, RecordFamily >, CacheRecord::max_record_families() > m_eviction_cbs; 93 | // metrics raw ptr, we do not own it 94 | CacheMetrics* m_metrics{nullptr}; 95 | }; 96 | } // namespace sisl 97 | -------------------------------------------------------------------------------- /src/grpc/tests/function/echo_sync_client.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed 10 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | * specific language governing permissions and limitations under the License. 13 | * 14 | *********************************************************************************/ 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | 26 | #include "grpc_helper/rpc_client.hpp" 27 | #include "sds_grpc_test.grpc.pb.h" 28 | 29 | using namespace ::grpc; 30 | using namespace ::sisl; 31 | using namespace ::sds_grpc_test; 32 | using namespace std::placeholders; 33 | 34 | class EchoAndPingClient : public GrpcSyncClient { 35 | 36 | public: 37 | using GrpcSyncClient::GrpcSyncClient; 38 | 39 | virtual void init() { 40 | GrpcSyncClient::init(); 41 | 42 | echo_stub_ = MakeStub< EchoService >(); 43 | ping_stub_ = MakeStub< PingService >(); 44 | } 45 | 46 | const std::unique_ptr< EchoService::StubInterface >& echo_stub() { return echo_stub_; } 47 | 48 | const std::unique_ptr< PingService::StubInterface >& ping_stub() { return ping_stub_; } 49 | 50 | private: 51 | std::unique_ptr< EchoService::StubInterface > echo_stub_; 52 | std::unique_ptr< PingService::StubInterface > ping_stub_; 53 | }; 54 | 55 | #define GRPC_CALL_COUNT 10 56 | 57 | int RunClient(const std::string& server_address) { 58 | 59 | auto client = std::make_unique< EchoAndPingClient >(server_address, "", ""); 60 | if (!client) { 61 | LOGERROR("Create grpc sync client failed."); 62 | return -1; 63 | } 64 | client->init(); 65 | 66 | int ret = 0; 67 | for (int i = 0; i < GRPC_CALL_COUNT; i++) { 68 | ClientContext context; 69 | 70 | if (i % 2 == 0) { 71 | EchoRequest request; 72 | EchoReply reply; 73 | 74 | request.set_message(std::to_string(i)); 75 | Status status = client->echo_stub()->Echo(&context, request, &reply); 76 | if (!status.ok()) { 77 | LOGERROR("echo request {} failed, status {}: {}", request.message(), status.error_code(), 78 | status.error_message()); 79 | continue; 80 | } 81 | 82 | LOGINFO("echo request {} reply {}", request.message(), reply.message()); 83 | 84 | if (request.message() == reply.message()) { ret++; } 85 | } else { 86 | PingRequest request; 87 | PingReply reply; 88 | 89 | request.set_seqno(i); 90 | Status status = client->ping_stub()->Ping(&context, request, &reply); 91 | if (!status.ok()) { 92 | LOGERROR("ping request {} failed, status {}: {}", request.seqno(), status.error_code(), 93 | status.error_message()); 94 | continue; 95 | } 96 | 97 | LOGINFO("ping request {} reply {}", request.seqno(), reply.seqno()); 98 | 99 | if (request.seqno() == reply.seqno()) { ret++; } 100 | } 101 | } 102 | 103 | return ret; 104 | } 105 | 106 | SISL_LOGGING_INIT() 107 | SISL_OPTIONS_ENABLE(logging) 108 | 109 | int main(int argc, char** argv) { 110 | SISL_OPTIONS_LOAD(argc, argv, logging) 111 | sisl::logging::SetLogger("sync_client"); 112 | 113 | std::string server_address("0.0.0.0:50051"); 114 | 115 | if (RunClient(server_address) != GRPC_CALL_COUNT) { 116 | LOGERROR("Only {} calls are successful", GRPC_CALL_COUNT); 117 | return 1; 118 | } 119 | 120 | return 0; 121 | } 122 | -------------------------------------------------------------------------------- /include/sisl/flip/flip_client.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #pragma once 18 | #include "flip.hpp" 19 | 20 | namespace flip { 21 | class FlipClient { 22 | public: 23 | explicit FlipClient(Flip* f) : m_flip(f) {} 24 | 25 | template < typename T > 26 | void create_condition(const std::string& param_name, flip::Operator oper, const T& value, 27 | FlipCondition* out_condition) { 28 | *(out_condition->mutable_name()) = param_name; 29 | out_condition->set_oper(oper); 30 | to_proto_converter< T >()(value, out_condition->mutable_value()); 31 | } 32 | 33 | template < typename T > 34 | FlipCondition create_condition(const std::string& param_name, flip::Operator oper, const T& value) { 35 | FlipCondition fcond; 36 | create_condition(param_name, oper, value, &fcond); 37 | return fcond; 38 | } 39 | 40 | bool inject_noreturn_flip(std::string flip_name, const std::vector< FlipCondition >& conditions, 41 | const FlipFrequency& freq) { 42 | FlipSpec fspec; 43 | 44 | _create_flip_spec(flip_name, conditions, freq, fspec); 45 | fspec.mutable_flip_action()->set_no_action(true); 46 | 47 | m_flip->add(fspec); 48 | return true; 49 | } 50 | 51 | template < typename T > 52 | bool inject_retval_flip(std::string flip_name, const std::vector< FlipCondition >& conditions, 53 | const FlipFrequency& freq, const T& retval) { 54 | FlipSpec fspec; 55 | 56 | _create_flip_spec(flip_name, conditions, freq, fspec); 57 | to_proto_converter< T >()(retval, fspec.mutable_flip_action()->mutable_returns()->mutable_retval()); 58 | 59 | m_flip->add(fspec); 60 | return true; 61 | } 62 | 63 | bool inject_delay_flip(std::string flip_name, const std::vector< FlipCondition >& conditions, 64 | const FlipFrequency& freq, uint64_t delay_usec) { 65 | FlipSpec fspec; 66 | 67 | _create_flip_spec(flip_name, conditions, freq, fspec); 68 | fspec.mutable_flip_action()->mutable_delays()->set_delay_in_usec(delay_usec); 69 | 70 | m_flip->add(fspec); 71 | return true; 72 | } 73 | 74 | template < typename T > 75 | bool inject_delay_and_retval_flip(std::string flip_name, const std::vector< FlipCondition >& conditions, 76 | const FlipFrequency& freq, uint64_t delay_usec, const T& retval) { 77 | FlipSpec fspec; 78 | 79 | _create_flip_spec(flip_name, conditions, freq, fspec); 80 | fspec.mutable_flip_action()->mutable_delays()->set_delay_in_usec(delay_usec); 81 | to_proto_converter< T >()(retval, fspec.mutable_flip_action()->mutable_delay_returns()->mutable_retval()); 82 | 83 | m_flip->add(fspec); 84 | return true; 85 | } 86 | 87 | uint32_t remove_flip(const std::string& flip_name) { return m_flip->remove(flip_name); } 88 | 89 | private: 90 | void _create_flip_spec(std::string flip_name, const std::vector< FlipCondition >& conditions, 91 | const FlipFrequency& freq, FlipSpec& out_fspec) { 92 | *(out_fspec.mutable_flip_name()) = flip_name; 93 | for (auto& c : conditions) { 94 | *(out_fspec.mutable_conditions()->Add()) = c; 95 | } 96 | *(out_fspec.mutable_flip_frequency()) = freq; 97 | } 98 | 99 | private: 100 | Flip* m_flip; 101 | }; 102 | } // namespace flip -------------------------------------------------------------------------------- /src/wisr/tests/wisr_intrusive_slist_benchmark.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | 25 | #include 26 | 27 | #include "sisl/utility/thread_buffer.hpp" 28 | #include "sisl/wisr/wisr_ds.hpp" 29 | 30 | THREAD_BUFFER_INIT 31 | RCU_REGISTER_INIT 32 | 33 | static constexpr size_t ITERATIONS{1000000}; 34 | static constexpr size_t THREADS{8}; 35 | 36 | using namespace sisl; 37 | 38 | struct Entry : public boost::intrusive::slist_base_hook<> { 39 | explicit Entry(const uint64_t n) : m_n(n) {} 40 | uint64_t m_n; 41 | }; 42 | 43 | std::unique_ptr< boost::intrusive::slist< Entry > > glob_lock_list; 44 | std::mutex glob_list_mutex; 45 | 46 | std::unique_ptr< sisl::wisr_intrusive_slist< Entry > > glob_wisr_list; 47 | std::array< std::vector< Entry >, THREADS > glob_entries; 48 | 49 | void setup() { 50 | glob_wisr_list = std::make_unique< sisl::wisr_intrusive_slist< Entry > >(); 51 | glob_lock_list = std::make_unique< boost::intrusive::slist< Entry > >(); 52 | 53 | for (size_t i{0}; i < THREADS; ++i) { 54 | auto v = &glob_entries[i]; 55 | for (size_t j{0}; j < ITERATIONS; ++j) { 56 | v->emplace_back(Entry((i * ITERATIONS) + j)); 57 | } 58 | } 59 | } 60 | 61 | void test_locked_list_insert(benchmark::State& state) { 62 | auto it = glob_entries[state.thread_index()].begin(); 63 | for (auto s : state) { // Loops upto iteration count 64 | std::lock_guard< std::mutex > lg(glob_list_mutex); 65 | glob_lock_list->push_front(*it); 66 | ++it; 67 | } 68 | 69 | if (state.thread_index() == 0) { glob_lock_list->clear(); } 70 | } 71 | 72 | void test_wisr_list_insert(benchmark::State& state) { 73 | auto it = glob_entries[state.thread_index()].begin(); 74 | for (auto s : state) { // Loops upto iteration count 75 | glob_wisr_list->push_front(*it); 76 | ++it; 77 | } 78 | 79 | if (state.thread_index() == 0) { 80 | auto l = glob_wisr_list->get_copy_and_reset(); 81 | l->clear(); 82 | } 83 | } 84 | 85 | #if 0 86 | void test_locked_list_read(benchmark::State& state) { 87 | uint64_t ret; 88 | for (auto _ : state) { // Loops upto iteration count 89 | std::lock_guard lg(glob_list_mutex); 90 | for (auto v : *glob_lock_list) { 91 | benchmark::DoNotOptimize(ret = v * 2); 92 | } 93 | } 94 | } 95 | 96 | void test_wisr_list_read(benchmark::State &state) { 97 | uint64_t ret; 98 | for (auto _ : state) { // Loops upto iteration count 99 | auto vec = glob_wisr_list->get_copy(); 100 | for (auto v : *vec) { 101 | benchmark::DoNotOptimize(ret = v * 2); 102 | } 103 | } 104 | } 105 | #endif 106 | 107 | BENCHMARK(test_wisr_list_insert)->Iterations(ITERATIONS)->Threads(THREADS); 108 | BENCHMARK(test_locked_list_insert)->Iterations(ITERATIONS)->Threads(THREADS); 109 | #if 0 110 | BENCHMARK(test_locked_list_insert)->Iterations(ITERATIONS)->Threads(1); 111 | BENCHMARK(test_wisr_list_insert)->Iterations(ITERATIONS)->Threads(1); 112 | BENCHMARK(test_locked_list_read)->Iterations(ITERATIONS)->Threads(1); 113 | BENCHMARK(test_wisr_list_read)->Iterations(ITERATIONS)->Threads(1); 114 | #endif 115 | 116 | int main(int argc, char** argv) { 117 | setup(); 118 | ::benchmark::Initialize(&argc, argv); 119 | ::benchmark::RunSpecifiedBenchmarks(); 120 | } 121 | -------------------------------------------------------------------------------- /cmake/settings_gen.cmake: -------------------------------------------------------------------------------- 1 | 2 | # settings_gen_cpp 3 | # -------------------------- 4 | # 5 | # Add custom commands to process ``.fbs`` files to C++ using flatc compiler 6 | # 7 | # settings_gen_cpp ( <*.proto files> ) 8 | # 9 | # ``ARGN`` 10 | # ``.fbs`` files 11 | # 12 | macro(settings_gen_cpp flatbuffer_bin_path gen_out_path _target) 13 | # if(NOT TARGET ${_target}) 14 | # message(SEND_ERROR "settings_gen_cpp requires target as third argument") 15 | # return() 16 | # endif() 17 | if(NOT ${ARGC} GREATER 1) 18 | message(SEND_ERROR "Error: settings_gen_cpp() called without any files as arguments") 19 | return() 20 | endif() 21 | if(NOT _generated_headers) 22 | set(_generated_headers) 23 | endif() 24 | 25 | foreach(FIL ${ARGN}) 26 | get_filename_component(ABS_FIL ${FIL} ABSOLUTE) 27 | message(STATUS "settings_gen_cpp: processing ${ABS_FIL}") 28 | get_filename_component(FIL_WE ${FIL} NAME_WE) 29 | get_filename_component(NAM ${FIL} NAME) 30 | 31 | file(RELATIVE_PATH REL_FIL ${CMAKE_CURRENT_SOURCE_DIR}/proto ${ABS_FIL}) 32 | get_filename_component(REL_DIR ${REL_FIL} DIRECTORY) 33 | get_filename_component(ABS_DIR ${ABS_FIL} DIRECTORY) 34 | 35 | if (REL_DIR) 36 | set(RELFIL_WE "${REL_DIR}/${FIL_WE}") 37 | else() 38 | set(RELFIL_WE ${FIL_WE}) 39 | endif () 40 | 41 | #message(STATUS "settings_gen_cpp: FIL_WE = ${FIL_WE}") 42 | #message(STATUS "settings_gen_cpp: REL_FIL = ${REL_FIL}") 43 | #message(STATUS "settings_gen_cpp: REL_DIR = ${REL_DIR}") 44 | #message(STATUS "settings_gen_cpp: ABS_DIR = ${ABS_DIR}") 45 | #message(STATUS "settings_gen_cpp: RELFIL_WE = ${RELFIL_WE}") 46 | #message(STATUS "settings_gen_cpp: gen_out_path = ${gen_out_path}") 47 | #message(STATUS "settings_gen_cpp: CMAKE_CURRENT_BUILD_DIR = ${CMAKE_CURRENT_BINARY_DIR}") 48 | 49 | set(_BIN_DUMP_SOURCE 50 | ${gen_out_path}/${FIL_WE}_bindump.cpp) 51 | set(_GEN_HEADERS 52 | ${gen_out_path}/${FIL_WE}_generated.h 53 | ) 54 | list(APPEND _generated_headers "${_GEN_HEADERS}") 55 | target_sources(${_target} PRIVATE ${_BIN_DUMP_SOURCE} ${_GEN_HEADERS}) 56 | set_source_files_properties(${_BIN_DUMP_SOURCE} ${_GEN_HEADERS} PROPERTIES GENERATED TRUE) 57 | 58 | add_custom_command( 59 | OUTPUT ${_GEN_HEADERS} 60 | COMMAND ${flatbuffer_bin_path} 61 | ARGS -c -o ${gen_out_path} 62 | --no-prefix 63 | --scoped-enums 64 | --gen-mutable 65 | --gen-object-api 66 | --gen-name-strings 67 | --grpc 68 | --bfbs-comments 69 | --reflect-types 70 | --reflect-names 71 | --cpp-ptr-type sisl::embedded_t 72 | ${FIL} 73 | DEPENDS ${FIL} 74 | WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} 75 | COMMENT "Running flatc compiler on ${CMAKE_CURRENT_SOURCE_DIR} ${FIL} to generate ${_GEN_HEADERS}" 76 | VERBATIM 77 | ) 78 | 79 | add_custom_command( 80 | OUTPUT ${_BIN_DUMP_SOURCE} 81 | WORKING_DIRECTORY ${ABS_DIR} 82 | COMMAND mkdir -p ${gen_out_path} && xxd -i ${NAM} ${_BIN_DUMP_SOURCE} 83 | DEPENDS ${ABS_FIL} 84 | COMMENT "invoking xxd to generate object file for schema ${ABS_FIL} to ${_BIN_DUMP_SOURCE}" 85 | VERBATIM 86 | ) 87 | 88 | #message(STATUS "settings_gen_cpp: _BIN_DUMP_HEADER = ${_BIN_DUMP_HEADER}") 89 | #message(STATUS "settings_gen_cpp: _GEN_HEADERS = ${_GEN_HEADERS}") 90 | 91 | #since some of the headers generated by this command are also included in hand-written sources make this command run before the _target 92 | get_property(_sources TARGET ${_target} PROPERTY SOURCES) 93 | #message(STATUS "settings_gen_cpp: target ${_target}: _sources property = ${_sources} depends on ${_GEN_HEADERS} ${_BIN_DUMP_HEADER}") 94 | set_source_files_properties(${_sources} PROPERTIES OBJECT_DEPENDS ${_GEN_HEADERS}) 95 | set_source_files_properties(${_sources} PROPERTIES OBJECT_DEPENDS ${_BIN_DUMP_SOURCE}) 96 | 97 | endforeach() 98 | endmacro() 99 | -------------------------------------------------------------------------------- /include/sisl/grpc/generic_service.hpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed 10 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | * specific language governing permissions and limitations under the License. 13 | * 14 | *********************************************************************************/ 15 | #pragma once 16 | 17 | #include 18 | #include "sisl/fds/buffer.hpp" 19 | #include "rpc_call.hpp" 20 | 21 | namespace sisl { 22 | 23 | /** 24 | * Callbacks are registered by a name. The client generic stub uses the method name to call the RPC 25 | * We assume the Request and Response types are grpc::ByteBuffer 26 | * The user is responsible to serialize / deserialize their messages to and from grpc::ByteBuffer 27 | */ 28 | 29 | class GenericRpcStaticInfo : public RpcStaticInfoBase { 30 | public: 31 | GenericRpcStaticInfo(GrpcServer* server, grpc::AsyncGenericService* service); 32 | 33 | GrpcServer* m_server; 34 | grpc::AsyncGenericService* m_generic_service; 35 | }; 36 | 37 | class GenericRpcContextBase { 38 | public: 39 | virtual ~GenericRpcContextBase() = default; 40 | }; 41 | using generic_rpc_ctx_ptr = std::unique_ptr< GenericRpcContextBase >; 42 | 43 | class GenericRpcData : public RpcDataAbstract, sisl::ObjLifeCounter< GenericRpcData > { 44 | public: 45 | static RpcDataAbstract* make(GenericRpcStaticInfo* rpc_info, size_t queue_idx); 46 | 47 | RpcDataAbstract* create_new() override; 48 | void set_status(grpc::Status& status); 49 | 50 | ~GenericRpcData() override; 51 | 52 | // There is only one generic static rpc data for all rpcs. 53 | size_t get_rpc_idx() const override; 54 | 55 | const grpc::ByteBuffer& request() const; 56 | sisl::io_blob& request_blob(); 57 | 58 | grpc::ByteBuffer& response(); 59 | 60 | void enqueue_call_request(::grpc::ServerCompletionQueue& cq) override; 61 | 62 | void send_response(); 63 | void send_response(io_blob_list_t const& response_blob_list); 64 | 65 | void set_context(generic_rpc_ctx_ptr ctx); 66 | GenericRpcContextBase* get_context(); 67 | 68 | void set_comp_cb(generic_rpc_completed_cb_t const& comp_cb); 69 | 70 | GenericRpcData(GenericRpcStaticInfo* rpc_info, size_t queue_idx); 71 | 72 | private: 73 | GenericRpcStaticInfo* m_rpc_info; 74 | grpc::GenericServerAsyncReaderWriter m_stream; 75 | grpc::GenericServerContext m_ctx; 76 | grpc::ByteBuffer m_request; 77 | grpc::ByteBuffer m_response; 78 | sisl::io_blob m_request_blob; 79 | bool m_request_blob_allocated{false}; 80 | grpc::Status m_retstatus{grpc::Status::OK}; 81 | // user can set and retrieve the context. Its life cycle is tied to that of rpc data. 82 | generic_rpc_ctx_ptr m_rpc_context; 83 | // the handler cb can fill in the completion cb if it needs one 84 | generic_rpc_completed_cb_t m_comp_cb{nullptr}; 85 | 86 | private: 87 | bool do_authorization(); 88 | 89 | RpcDataAbstract* on_request_received(bool ok); 90 | RpcDataAbstract* on_buf_read(bool); 91 | RpcDataAbstract* on_buf_write(bool); 92 | RpcDataAbstract* on_request_completed(bool); 93 | 94 | struct RpcTagImpl : public RpcTag { 95 | using callback_type = RpcDataAbstract* (GenericRpcData::*)(bool); 96 | RpcTagImpl(GenericRpcData* rpc, callback_type cb); 97 | 98 | RpcDataAbstract* do_process(bool ok) override; 99 | 100 | callback_type m_callback; 101 | }; 102 | 103 | // Used as void* completion markers from grpc to indicate different events of interest for a 104 | // Call. 105 | RpcTagImpl m_request_received_tag{this, &GenericRpcData::on_request_received}; 106 | RpcTagImpl m_buf_read_tag{this, &GenericRpcData::on_buf_read}; 107 | RpcTagImpl m_buf_write_tag{this, &GenericRpcData::on_buf_write}; 108 | RpcTagImpl m_request_completed_tag{this, &GenericRpcData::on_request_completed}; 109 | }; 110 | 111 | } // namespace sisl 112 | -------------------------------------------------------------------------------- /src/metrics/metrics.cpp: -------------------------------------------------------------------------------- 1 | /********************************************************************************* 2 | * Modifications Copyright 2017-2019 eBay Inc. 3 | * 4 | * Author/Developer(s): Harihara Kadayam 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * https://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software distributed 12 | * under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 13 | * CONDITIONS OF ANY KIND, either express or implied. See the License for the 14 | * specific language governing permissions and limitations under the License. 15 | * 16 | *********************************************************************************/ 17 | #include 18 | 19 | #include "sisl/metrics/metrics.hpp" 20 | 21 | THREAD_BUFFER_INIT 22 | 23 | namespace sisl { 24 | 25 | #define PROMETHEUS_METRICS_REPORTER 26 | 27 | static std::atomic< bool > metrics_farm_initialized{false}; 28 | 29 | /**************************** MetricsFarm ***********************************/ 30 | Reporter& MetricsFarm::get_reporter() { return *getInstance().m_reporter; } 31 | 32 | MetricsFarm::MetricsFarm() { 33 | metrics_farm_initialized = true; 34 | m_treg = ThreadRegistry::get_instance_ptr(); 35 | 36 | #ifdef PROMETHEUS_METRICS_REPORTER 37 | // m_reporter = std::dynamic_pointer_cast< Reporter >(std::make_unique< PrometheusReporter >()); 38 | m_reporter = std::make_unique< PrometheusReporter >(); 39 | #endif 40 | } 41 | 42 | MetricsFarm::~MetricsFarm() { metrics_farm_initialized = false; } 43 | 44 | bool MetricsFarm::is_initialized() { return metrics_farm_initialized.load(); } 45 | 46 | void MetricsFarm::register_metrics_group(MetricsGroupImplPtr mgrp_impl, const bool add_to_farm_list) { 47 | assert(mgrp_impl != nullptr); 48 | auto locked{lock()}; 49 | mgrp_impl->on_register(); 50 | if (add_to_farm_list) { m_mgroups.insert(mgrp_impl); } 51 | mgrp_impl->registration_completed(); 52 | } 53 | 54 | void MetricsFarm::deregister_metrics_group(MetricsGroupImplPtr mgrp_impl) { 55 | assert(mgrp_impl != nullptr); 56 | auto locked{lock()}; 57 | m_mgroups.erase(mgrp_impl); 58 | } 59 | 60 | nlohmann::json MetricsFarm::get_result_in_json(bool need_latest) { 61 | nlohmann::json json; 62 | auto locked{lock()}; 63 | for (auto& mgroup : m_mgroups) { 64 | json[mgroup->get_group_name()][mgroup->get_instance_name()] = mgroup->get_result_in_json(need_latest); 65 | } 66 | 67 | return json; 68 | } 69 | 70 | std::string MetricsFarm::get_result_in_json_string(bool need_latest) { return get_result_in_json(need_latest).dump(); } 71 | 72 | std::string MetricsFarm::report(ReportFormat format) { 73 | auto locked{lock()}; 74 | for (auto& mgroup : m_mgroups) { 75 | mgroup->publish_result(); 76 | } 77 | 78 | // Now everything is published to reporter, serialize them 79 | return m_reporter->serialize(format); 80 | } 81 | 82 | void MetricsFarm::gather() { 83 | nlohmann::json json; 84 | auto locked{lock()}; 85 | bool flushed{false}; 86 | for (auto& mgroup : m_mgroups) { 87 | if (!flushed && (mgroup->impl_type() == group_impl_type_t::thread_buf_signal)) { 88 | ThreadBufferMetricsGroup::flush_core_cache(); 89 | } 90 | flushed = true; 91 | mgroup->gather(); 92 | } 93 | } 94 | 95 | ////////////////////////////////////////// Helper Routine section //////////////////////////////////////////////// 96 | /// 97 | std::string MetricsFarm::ensure_unique(const std::string& grp_name, const std::string& inst_name) { 98 | auto locked{lock()}; 99 | 100 | // If 2 instances are provided with same name (unknowingly), prometheus with same label pairs, return the same 101 | // prometheus::Counter pointer, which means if one of them freed, other could access it. To prevent that, we 102 | // are creating a unique instance name, so that we have one per registration. 103 | const auto it{m_uniq_inst_maintainer.find(grp_name + inst_name)}; 104 | if (it == std::end(m_uniq_inst_maintainer)) { 105 | m_uniq_inst_maintainer.insert(std::make_pair<>(grp_name + inst_name, 1)); 106 | return inst_name; 107 | } else { 108 | ++(it->second); 109 | return inst_name + "_" + std::to_string(it->second); 110 | } 111 | } 112 | } // namespace sisl 113 | -------------------------------------------------------------------------------- /src/metrics/README.md: -------------------------------------------------------------------------------- 1 | # High Performing Metrics Captures 2 | On any high performing applications, collecting and reporting metrics is one of the key requirements. It provides clear 3 | insight to the application and some times help find bugs. However, for very high performing applications, collecting 4 | metrics itself might be hindering performance and developers in those cases could use conservative approach to adding 5 | metrics. This library aims to provide very high performing metrics collection and reporting among other features. 6 | 7 | At high level this library provides following key features 8 | * Various choices for very high performance metrics collection 9 | * Different reporting of metrics 10 | * Framework to organize multiple dimensions of metrics and do compile time look of them. 11 | 12 | While this README takes prometheus as an example, it can be extended or used for any metrics server. 13 | 14 | ## Performance of Metrics collection. 15 | Prometheus C++ is a well known open source product for collection, reporting, maintaining of metrics. It provides a client 16 | library for application to capture individual metrics and then provide a method to serialize into prometheus format. It 17 | provides a multi-threaded safe way to capture the metrics, by using std::atomic variables. The key issue is having 18 | lots of atomic fetch_and_add in a system increases the CPU impact. Having few of the metrics does not matter, however, 19 | if at any given point the application updates lots of metrics, it comes out of its negligibility. 20 | 21 | sisl::metrics provide few alternatives to this approach 22 | 23 | ### ThreadBufferWithSignal 24 | In this method, it avoids atomic variables, by replacing it with a *sisl::ThreadBuffer*. 25 | 26 | **About ThreadBuffer**: ThreadBuffer is simply an header only implementation which keeps track of the threads that 27 | uses the facility and as soon as thread comes in, it calls all registered callers to initialize a specific version of 28 | its instance for this thread. Thus each thread can access these objects in lock-free manner. This is different than a 29 | thread-local variable that OS/language provides, in that this thread buffer can be a member in a class/struct and can be 30 | non-static. Apart from that it provides method to access a specific threads buffer or all the thread buffers from a 31 | non-participating thread as well. So far it is very similar to *folly::ThreadLocalPtr*. However, sisl::ThreadBuffer 32 | provides one more critical functionality which is exit safety. Once thread has exited, typically that threads version of 33 | buffer is also gone. This might be ok for several scenarios, but on some occasions it is important to retain the thread 34 | buffer even after thread exits. One such requirement is for wait free metrics collection. 35 | 36 | **ThreadBufferWithSignal**: Using this threadbuffer, sisl::metrics library collects all the metrics in its own thread and access, counters, 37 | histograms in a wait free manner. Even if the thread exits, the ThreadBuffer class should hold onto the metrics 38 | its collected, long enough for next scrapping, before it discards them. Each time a metric, say counter is update, it 39 | is simply updating its thread version of it. As itself, this is not correct value. However, when the metrics are 40 | scrapped or viewed - which is probably typically once a minute or some cases on-demand - it gathers metrics from all 41 | the thread buffers and then merges them and reports them. 42 | 43 | It is important to note that when scrapping happens it will be accessing other threads metrics. However, when scrapping 44 | and collection are running in multi-threaded fashion, there is no absolute point-in-time metrics unless entire system 45 | is locked, which is highly undesirable. However, while access other threads metrics without locks or atomics, we 46 | suffer from L1/L2/L3 caches in-consistency across cores. To overcome that, this style of metrics gathering, send a 47 | signal to all participating threads to do a atomic fencing and then upon confirmation, it access other thread's data. 48 | Once all thread data is collected, they are merged and then reported using configured reporter. 49 | 50 | Please note that this method still does not guarantee atomicity of all metrics. Histograms are typically represented 51 | as + . This cannot be possibily atomically updated. However, key point is 52 | most of the metrics collection - including promethus library does not support that atomicity. Given that its a metric, 53 | inconsistency on sample, shouldn't skew the results and in general its an acceptable approach. 54 | 55 | ### WISR RCU Framework 56 | 57 | 58 | --------------------------------------------------------------------------------