├── .clang-format ├── .clang_complete ├── .dockerignore ├── .gitignore ├── .gitmodules ├── .travis.yml ├── .vscode ├── c_cpp_properties.json ├── launch.json └── settings.json ├── CMakeLists.txt ├── CPPLINT.cfg ├── LICENSE ├── README.md ├── base ├── CMakeLists.txt ├── ProducerConsumerQueue.h ├── RWSpinLock.h ├── RWSpinLock_test.cc ├── arena.cc ├── arena.h ├── arena_test.cc ├── array.h ├── array_test.cc ├── atomic_wrapper.h ├── bits.cc ├── bits.h ├── bits_test.cc ├── casts.h ├── chunked_array.h ├── coder.cc ├── coder.h ├── coder_test.cc ├── counting_allocator.h ├── crc32c.cc ├── crc32c.h ├── crc32c_test.cc ├── cxx_test.cc ├── endian.h ├── event_count.h ├── event_count_test.cc ├── expected.hpp ├── fixed.h ├── flags.h ├── flat_arrays_vector.h ├── flit.h ├── flit_test.cc ├── function2.hpp ├── gtest.h ├── gtest_main.cc ├── hash.cc ├── hash.h ├── hash_test.cc ├── histogram.cc ├── histogram.h ├── histogram_test.cc ├── init.cc ├── init.h ├── int128.cc ├── int128.h ├── integral_types.h ├── lambda_test.cc ├── logging.cc ├── logging.h ├── macros.h ├── map-util.h ├── memory.h ├── mpmc_bounded_queue.h ├── mpmc_bounded_queue_test.cc ├── object_pool.h ├── pmr.h ├── pmr_test.cc ├── pod_array.h ├── pod_array_test.cc ├── port.h ├── posix_call.h ├── pthread_utils.cc ├── pthread_utils.h ├── simd.cc ├── simd.h ├── simd_test.cc ├── stl_util.h ├── testdata │ └── ids.txt.gz ├── type_traits.h ├── varint.cc ├── varint.h ├── walltime.cc ├── walltime.h ├── walltime_test.cc ├── wheel_timer.h ├── wheel_timer_test.cc └── xml_test.cc ├── blaze.sh ├── cmake ├── FindBoost.cmake ├── FindPackageHandleStandardArgs.cmake ├── FindPackageMessage.cmake ├── internal.cmake └── third_party.cmake ├── doc ├── CMakeLists.txt ├── Doxyfile.in ├── README.md ├── async_model.md ├── docker_build.md ├── doxygen.css ├── header.html.in ├── hello_world.md ├── mainpage.md ├── mr3.md └── qsbr.md ├── docker ├── alpine.Dockerfile └── bin_build.Dockerfile ├── examples ├── CMakeLists.txt ├── asio_fibers.cc ├── file_read.cc ├── gcs_demo.cc ├── gsod_group.cc ├── http_client_tool.cc ├── movies_join.cc ├── mr3.cc ├── mr_read_test.cc ├── mrgrep.cc ├── osmium_road_length.cc ├── pingserver │ ├── CMakeLists.txt │ ├── ping_command.cc │ ├── ping_command.h │ ├── ping_epoll_server.cc │ ├── ping_iouring_server.cc │ ├── ping_server.cc │ ├── ping_sync_server.cc │ ├── resp_parser.cc │ └── resp_parser.h ├── redis │ ├── CMakeLists.txt │ ├── redis_command.cc │ ├── redis_command.h │ ├── redis_toy_server.cc │ ├── resp_connection_handler.cc │ ├── resp_connection_handler.h │ ├── resp_parser.cc │ └── resp_parser.h ├── s3_demo.cc └── wordcount │ ├── CMakeLists.txt │ ├── WordCount.java │ ├── warc_parse.cc │ └── word_count.cc ├── file ├── CMakeLists.txt ├── compressors.cc ├── compressors.h ├── fiber_file.cc ├── fiber_file.h ├── file.cc ├── file.h ├── file_test.cc ├── file_util.cc ├── file_util.h ├── filesource.cc ├── filesource.h ├── gzip_file.cc ├── gzip_file.h ├── list_file.cc ├── list_file.h ├── list_file_format.h ├── list_file_format2.h ├── list_file_py.cc ├── list_file_reader.cc ├── list_file_reader.h ├── list_file_test.cc ├── lst2_impl.cc ├── lst2_impl.h ├── lz4_compressor.cc ├── lz4_file.cc ├── lz4_file.h ├── meta_map_block.cc ├── meta_map_block.h ├── proto_writer.cc ├── proto_writer.h ├── proto_writer_test.cc ├── proto_writer_test.proto ├── s3_file.cc ├── s3_file.h ├── test_util.cc └── test_util.h ├── install-dependencies.sh ├── mr ├── CMakeLists.txt ├── do_context.h ├── impl │ ├── CMakeLists.txt │ ├── dest_file_set.cc │ ├── dest_file_set.h │ ├── freq_map_wrapper.cc │ ├── freq_map_wrapper.h │ ├── local_context.cc │ ├── local_context.h │ └── table_impl.h ├── joiner_executor.cc ├── joiner_executor.h ├── local_runner.cc ├── local_runner.h ├── local_runner_test.cc ├── mapper_executor.cc ├── mapper_executor.h ├── mr.cc ├── mr3.proto ├── mr_main.cc ├── mr_main.h ├── mr_pb.cc ├── mr_pb.h ├── mr_test.cc ├── mr_types.h ├── operator_executor.cc ├── operator_executor.h ├── output.h ├── pipeline.cc ├── pipeline.h ├── ptable.h ├── readme.md ├── runner.h ├── test_utils.cc └── test_utils.h ├── scripts ├── run_asio.sh └── run_test.sh ├── strings ├── CMakeLists.txt ├── charset.cc ├── charset.h ├── escaping.cc ├── escaping.h ├── hash.h ├── human_readable.cc ├── human_readable.h ├── join.h ├── numbers.cc ├── numbers.h ├── numbers_test.cc ├── range.cc ├── range.h ├── range_test.cc ├── slice.h ├── split.cc ├── split.h ├── strcat.cc ├── strcat.h ├── strcat_test.cc ├── stringpiece.cc ├── stringpiece.h ├── stringprintf.cc ├── stringprintf.h ├── strip.h ├── strpmr.cc ├── strpmr.h ├── strpmr_test.cc ├── strtoint.cc ├── strtoint.h ├── unique_strings.cc ├── unique_strings.h ├── unique_strings_test.cc ├── utf8 │ ├── rune.c │ ├── utf.h │ └── utfdef.h └── util.h └── util ├── CMakeLists.txt ├── asio ├── CMakeLists.txt ├── accept_server.cc ├── accept_server.h ├── asio_utils.h ├── connection_handler.cc ├── connection_handler.h ├── detail │ ├── fiber_socket_impl.h │ └── yield.hpp ├── error.cc ├── error.h ├── fiber_socket.cc ├── fiber_socket.h ├── fiber_socket_test.cc ├── glog_asio_sink.cc ├── glog_asio_sink.h ├── io_context.cc ├── io_context.h ├── io_context_pool.cc ├── io_context_pool.h ├── io_context_test.cc ├── periodic_task.cc ├── periodic_task.h ├── periodic_task_test.cc ├── prebuilt_asio.cc ├── yield.cc └── yield.h ├── asio_stream_adapter.h ├── aws ├── CMakeLists.txt ├── aws.cc ├── aws.h ├── s3.cc ├── s3.h └── s3_test.cc ├── bzip_source.cc ├── bzip_source.h ├── coding ├── CMakeLists.txt ├── block_compressor.cc ├── block_compressor.h ├── block_compressor_test.cc ├── double_compressor.cc ├── double_compressor.h ├── double_compressor_test.cc ├── sequence_array.cc ├── sequence_array.h ├── set_encoder.cc ├── set_encoder.h └── set_encoder_test.cc ├── fibers ├── CMakeLists.txt ├── event_count.h ├── fiberqueue_threadpool.cc ├── fiberqueue_threadpool.h ├── fibers_ext.cc ├── fibers_ext.h ├── fibers_ext_test.cc └── simple_channel.h ├── gce ├── CMakeLists.txt ├── detail │ ├── gcs_utils.cc │ └── gcs_utils.h ├── gce.cc ├── gce.h ├── gcs.cc ├── gcs.h ├── gcs_read_file.cc └── gcs_write_file.cc ├── html ├── CMakeLists.txt ├── main.js ├── sorted_table.cc ├── sorted_table.h └── style.css ├── http ├── CMakeLists.txt ├── beast_rj_utils.h ├── captain.gif ├── favicon-32x32.png ├── http_client.cc ├── http_client.h ├── http_common.cc ├── http_common.h ├── http_conn_handler.cc ├── http_conn_handler.h ├── http_main.cc ├── http_status_code.cc ├── http_status_code.h ├── http_test.cc ├── http_testing.cc ├── http_testing.h ├── https_client.cc ├── https_client.h ├── https_client_pool.cc ├── https_client_pool.h ├── logo.png ├── prebuilt_beast.cc ├── profilez_handler.cc ├── ssl_stream.cc ├── ssl_stream.h ├── ssl_stream_test.cc ├── status_page.cc ├── status_page.css ├── status_page.h └── status_page.js ├── math ├── CMakeLists.txt ├── exactfloat │ ├── exactfloat.cc │ └── exactfloat.h ├── float2decimal.cc ├── float2decimal.h ├── float2decimal_test.cc ├── ieeefloat.h ├── mathlimits.cc ├── mathlimits.h ├── mathutil.cc ├── mathutil.h ├── matrix3x3-inl.h ├── matrix3x3.h ├── vector2-inl.h ├── vector2.h ├── vector3-inl.h ├── vector3.h ├── vector4-inl.h └── vector4.h ├── mimalloc_test.cc ├── pb └── refl.h ├── pb2json.cc ├── pb2json.h ├── pb2json_test.cc ├── plang ├── CMakeLists.txt ├── addressbook.proto ├── plang.cc ├── plang.h ├── plang_parser.y ├── plang_scanner.h ├── plang_scanner.lex ├── plang_test.cc └── proto_test.cc ├── pprint ├── CMakeLists.txt ├── file_printer.cc ├── file_printer.h ├── lst_print_example.cc ├── pprint_utils.cc ├── pprint_utils.h ├── pprint_utils_test.cc └── pprint_utils_test.proto ├── proc_stats.cc ├── proc_stats.h ├── rpc ├── CMakeLists.txt ├── async_client_test.cc ├── channel.cc ├── channel.h ├── frame_format.cc ├── frame_format.h ├── impl │ ├── rpc_conn_handler.cc │ └── rpc_conn_handler.h ├── rpc_connection.cc ├── rpc_connection.h ├── rpc_envelope.h ├── rpc_test.cc ├── rpc_test_utils.cc ├── rpc_test_utils.h ├── service_descriptor.cc └── service_descriptor.h ├── sentry ├── CMakeLists.txt ├── sentry.cc ├── sentry.h └── sentry_test.cc ├── sinksource.cc ├── sinksource.h ├── sinksource_test.cc ├── sp_task_pool.cc ├── sp_task_pool.h ├── sp_task_pool_test.cc ├── spawn.cc ├── spawn.h ├── stats ├── CMakeLists.txt ├── sliding_counter.cc ├── sliding_counter.h ├── sliding_counter_test.cc ├── varz_node.cc ├── varz_node.h ├── varz_stats.cc ├── varz_stats.h └── varz_value.h ├── status.cc ├── status.h ├── status.proto ├── sync_stream_interface.h ├── uring ├── CMakeLists.txt ├── accept_server.cc ├── accept_server.h ├── accept_server_test.cc ├── connection.h ├── fiber_socket.cc ├── fiber_socket.h ├── http_handler.cc ├── http_handler.h ├── prebuilt_asio.cc ├── proactor.cc ├── proactor.h ├── proactor_pool.cc ├── proactor_pool.h ├── proactor_test.cc ├── sliding_counter.cc ├── sliding_counter.h ├── submit_entry.h ├── uring_fiber_algo.cc ├── uring_fiber_algo.h ├── varz.cc └── varz.h ├── zlib_source.cc ├── zlib_source.h ├── zstd_sinksource.cc └── zstd_sinksource.h /.clang-format: -------------------------------------------------------------------------------- 1 | # --- 2 | # We'll use defaults from the Google style, but with 2 columns indentation. 3 | BasedOnStyle: Google 4 | IndentWidth: 2 5 | ColumnLimit: 100 6 | --- 7 | Language: Cpp 8 | AllowShortLoopsOnASingleLine: false 9 | AllowShortFunctionsOnASingleLine: false 10 | AllowShortIfStatementsOnASingleLine: false 11 | AlwaysBreakTemplateDeclarations: false 12 | ConstructorInitializerAllOnOneLineOrOnePerLine: false 13 | DerivePointerAlignment: false 14 | PointerAlignment: Left 15 | BasedOnStyle: Google 16 | ColumnLimit: 100 17 | --- 18 | Language: Proto 19 | BasedOnStyle: Google 20 | -------------------------------------------------------------------------------- /.clang_complete: -------------------------------------------------------------------------------- 1 | -isystem/opt/boost/include/ 2 | -I. 3 | -Igenfiles 4 | -Iabseil-cpp 5 | -Ithird_party/libs/benchmark/include 6 | -Ithird_party/libs/gtest/include 7 | -Ithird_party/libs/gflags/include 8 | -Ithird_party/libs/glog/include 9 | -Ithird_party/libs/dconv/include 10 | -Ithird_party/libs/folly/include 11 | -Ithird_party/libs/zstd/include 12 | -Ithird_party/libs/protobuf/include 13 | -Ithird_party/libs/pmr/include 14 | -Ithird_party/libs/sparsehash/include 15 | -Ithird_party/libs/xxhash/include 16 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | third_party/* 2 | boost_1_67_0/* 3 | docker/* 4 | build-* 5 | doc/* 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/* 2 | build-* 3 | .vscode/*.db 4 | third_party/* 5 | genfiles/* 6 | *.sublime-* 7 | .tags 8 | !third_party/include/* 9 | *.pyc 10 | /CMakeLists.txt.user 11 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "abseil-cpp"] 2 | path = abseil-cpp 3 | url = https://github.com/abseil/abseil-cpp 4 | branch = master 5 | -------------------------------------------------------------------------------- /.vscode/c_cpp_properties.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Linux", 5 | "defines": [ 6 | "_TEST_BASE_FILE_=\"mytest.cc\"", 7 | "OPENSSL_VERSION_NUMBER=0x1010100fL", 8 | "ABSL_INTERNAL_ENABLE_FORMAT_CHECKER=0" 9 | ], 10 | "cppStandard": "c++14", 11 | "includePath": [ 12 | "${workspaceFolder}/abseil-cpp", 13 | "${workspaceFolder}/third_party/libs/protobuf/include", 14 | "${workspaceFolder}/third_party/libs/glog/include", 15 | "${workspaceFolder}/third_party/libs/gflags/include", 16 | "${workspaceFolder}/third_party/libs/pmr/include", 17 | "${workspaceFolder}/third_party/libs/sparsehash/include", 18 | "${workspaceFolder}/third_party/libs/zstd/include", 19 | "${workspaceFolder}/third_party/libs/benchmark/include", 20 | "${workspaceFolder}/third_party/libs/gtest/include", 21 | "${workspaceFolder}/third_party/libs/gperf/include", 22 | "${workspaceFolder}/third_party/libs/rapidjson/include", 23 | "${workspaceFolder}/third_party/libs/crc32c/include", 24 | "${workspaceFolder}/third_party/libs/xxhash/include", 25 | "${workspaceFolder}/third_party/libs/lz4/include", 26 | "${workspaceFolder}/third_party/libs/re2/include", 27 | "${workspaceFolder}/third_party/libs/mimalloc/include", 28 | "${workspaceFolder}", 29 | "${workspaceFolder}/genfiles", 30 | "${default}" 31 | ] 32 | } 33 | ], 34 | "version": 4 35 | } 36 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Launch Program", 11 | "program": "${file}" 12 | } 13 | ] 14 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | } 3 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.4) 2 | 3 | # package information 4 | set(PACKAGE_NAME "gaia") 5 | set(PACKAGE_VERSION "") 6 | set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") 7 | set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") 8 | set(PACKAGE_BUGREPORT "TBD") 9 | set(PROJECT_CONTACT romange@gmail.com) 10 | 11 | project(${PACKAGE_NAME}) 12 | 13 | set(VERSION_MAJOR 0) 14 | set(VERSION_MINOR 1) 15 | set(VERSION_PATCH 0) 16 | 17 | option (ONLY_THIRD_PARTY "Build third party only" OFF) 18 | option (BUILD_DOCS "Generate documentation " ON) 19 | 20 | # Check target architecture 21 | if (NOT CMAKE_SIZEOF_VOID_P EQUAL 8) 22 | message(FATAL_ERROR "Gaia requires a 64bit target architecture.") 23 | endif() 24 | 25 | if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") 26 | message(FATAL_ERROR "Requires running on linux, found ${CMAKE_SYSTEM_NAME} instead") 27 | endif() 28 | 29 | Message("PROJECT_BINARY_DIR ${PROJECT_BINARY_DIR} GENERATOR ${CMAKE_GENERATOR}") 30 | 31 | get_directory_property(HAS_PARENT PARENT_DIRECTORY) 32 | 33 | if (NOT HAS_PARENT) 34 | set(CMAKE_CXX_STANDARD 14) 35 | list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake) 36 | 37 | include(third_party) 38 | include(internal) 39 | endif () 40 | 41 | 42 | set(ABSL_CCTZ_TARGET TRDP::cctz) 43 | set(BUILD_TESTING OFF) 44 | set(CCTZ_INCLUDE_DIRS ${CCTZ_INCLUDE_DIR}) 45 | set(GOLD_CXX_FLAGS ${CMAKE_CXX_FLAGS}) 46 | 47 | include_directories(${CMAKE_CURRENT_SOURCE_DIR}) 48 | 49 | if (NOT ONLY_THIRD_PARTY) 50 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-sign-compare -Wno-missing-field-initializers \ 51 | -Wno-shadow-uncaptured-local -Wno-tautological-type-limit-compare") 52 | add_subdirectory(abseil-cpp) 53 | set(CMAKE_CXX_FLAGS "${GOLD_CXX_FLAGS}") 54 | 55 | add_subdirectory(base) 56 | add_subdirectory(strings) 57 | add_subdirectory(util) 58 | add_subdirectory(file) 59 | add_subdirectory(mr) 60 | add_subdirectory(examples) 61 | 62 | if (BUILD_DOCS) 63 | add_subdirectory(doc) 64 | endif() 65 | endif() 66 | -------------------------------------------------------------------------------- /CPPLINT.cfg: -------------------------------------------------------------------------------- 1 | filter=-build/namespace,-build/c++11,-build/include_what_you_use 2 | linelength=100 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013-present by Roman Gershman 2 | 3 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 4 | 5 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 6 | 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 8 | 9 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 10 | -------------------------------------------------------------------------------- /base/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(base arena.cc bits.cc crc32c.cc coder.cc hash.cc histogram.cc 2 | init.cc logging.cc simd.cc varint.cc walltime.cc pthread_utils.cc) 3 | cxx_link(base TRDP::glog TRDP::gflags TRDP::pmr TRDP::xxhash atomic rt 4 | absl_symbolize absl_failure_signal_handler) # rt for timer_create etc. 5 | add_dependencies(base sparsehash_project) 6 | 7 | cxx_test(array_test base LABELS CI) 8 | cxx_test(bits_test LABELS CI) 9 | cxx_test(pod_array_test base LABELS CI) 10 | cxx_test(arena_test base strings LABELS CI) 11 | cxx_test(pmr_test base TRDP::pmr LABELS CI) 12 | cxx_test(simd_test base LABELS CI) 13 | cxx_test(crc32c_test base strings LABELS CI) 14 | cxx_test(walltime_test base LABELS CI) 15 | cxx_test(flit_test base strings LABELS CI) 16 | cxx_test(cxx_test base LABELS CI) 17 | cxx_test(hash_test base file DATA testdata/ids.txt.gz LABELS CI) 18 | cxx_test(RWSpinLock_test base LABELS CI) 19 | cxx_test(event_count_test base LABELS CI) 20 | cxx_test(coder_test base LABELS CI) 21 | cxx_test(wheel_timer_test base LABELS CI) 22 | cxx_test(lambda_test base LABELS CI) 23 | cxx_test(mpmc_bounded_queue_test base LABELS CI) 24 | 25 | 26 | 27 | find_package(LibXml2) # libxml2 package 28 | include_directories(${LIBXML2_INCLUDE_DIR}) 29 | cxx_test(xml_test base absl_str_format ${LIBXML2_LIBRARIES}) 30 | 31 | # Define default gtest_main for tests. 32 | add_library(gaia_gtest_main gtest_main.cc) 33 | target_link_libraries(gaia_gtest_main TRDP::glog TRDP::gflags TRDP::gtest base TRDP::benchmark) 34 | 35 | -------------------------------------------------------------------------------- /base/arena.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | // integrated here by Roman Gershman (romange@gmail.com) 5 | 6 | #ifndef _BASE_UTIL_ARENA_H_ 7 | #define _BASE_UTIL_ARENA_H_ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace base { 15 | 16 | class Arena { 17 | public: 18 | Arena(); 19 | ~Arena(); 20 | 21 | // Return a pointer to a newly allocated memory block of "bytes" bytes. 22 | char* Allocate(size_t bytes); 23 | 24 | // Allocate memory with the normal alignment guarantees provided by malloc 25 | char* AllocateAligned(size_t bytes); 26 | 27 | // Returns an estimate of the total memory usage of data allocated 28 | // by the arena (including space allocated but not yet used for user 29 | // allocations). 30 | size_t MemoryUsage() const { 31 | return blocks_memory_ + blocks_.capacity() * sizeof(char*); 32 | } 33 | 34 | void Swap(Arena& other); 35 | 36 | private: 37 | char* AllocateFallback(size_t bytes); 38 | char* AllocateNewBlock(size_t block_bytes); 39 | 40 | // Allocation state 41 | char* alloc_ptr_; 42 | size_t alloc_bytes_remaining_; 43 | 44 | // Array of new[] allocated memory blocks 45 | std::vector blocks_; 46 | 47 | // Bytes of memory in blocks allocated so far 48 | size_t blocks_memory_; 49 | 50 | // No copying allowed 51 | Arena(const Arena&); 52 | void operator=(const Arena&); 53 | }; 54 | 55 | inline char* Arena::Allocate(size_t bytes) { 56 | // The semantics of what to return are a bit messy if we allow 57 | // 0-byte allocations, so we disallow them here (we don't need 58 | // them for our internal use). 59 | assert(bytes > 0); 60 | if (bytes <= alloc_bytes_remaining_) { 61 | char* result = alloc_ptr_; 62 | alloc_ptr_ += bytes; 63 | alloc_bytes_remaining_ -= bytes; 64 | return result; 65 | } 66 | return AllocateFallback(bytes); 67 | } 68 | 69 | } // namespace base 70 | 71 | #endif // _BASE_UTIL_ARENA_H_ 72 | -------------------------------------------------------------------------------- /base/arena_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "base/arena.h" 6 | 7 | #include 8 | #include 9 | 10 | #include "base/gtest.h" 11 | #include "base/logging.h" 12 | #include "base/walltime.h" 13 | #include "strings/human_readable.h" 14 | 15 | namespace base { 16 | 17 | class ArenaTest { }; 18 | 19 | TEST(ArenaTest, Empty) { 20 | Arena arena; 21 | } 22 | 23 | TEST(ArenaTest, Simple) { 24 | std::default_random_engine rand(301); 25 | std::vector > allocated; 26 | Arena arena; 27 | const int N = 100000; 28 | size_t bytes = 0; 29 | for (int i = 0; i < N; i++) { 30 | size_t s; 31 | if (i % (N / 10) == 0) { 32 | s = i; 33 | } else { 34 | s = (rand() % 4000 == 1) ? rand() % 6000 : 35 | (rand() % 10 == 1) ? rand() % 100 : rand() % 20; 36 | } 37 | if (s == 0) { 38 | // Our arena disallows size 0 allocations. 39 | s = 1; 40 | } 41 | char* r; 42 | if (rand() % 10 == 0) { 43 | r = arena.AllocateAligned(s); 44 | } else { 45 | r = arena.Allocate(s); 46 | } 47 | 48 | for (size_t b = 0; b < s; b++) { 49 | // Fill the "i"th allocation with a known bit pattern 50 | r[b] = i % 256; 51 | } 52 | bytes += s; 53 | allocated.push_back(std::make_pair(s, r)); 54 | ASSERT_GE(arena.MemoryUsage(), bytes); 55 | if (i > N/10) { 56 | ASSERT_LE(arena.MemoryUsage(), bytes * 1.10); 57 | } 58 | } 59 | for (size_t i = 0; i < allocated.size(); i++) { 60 | size_t num_bytes = allocated[i].first; 61 | const char* p = allocated[i].second; 62 | 63 | for (size_t b = 0; b < num_bytes; b++) { 64 | // Check the "i"th allocation for the known bit pattern 65 | ASSERT_EQ(int(p[b]) & 0xff, i % 256); 66 | } 67 | } 68 | } 69 | 70 | } // namespace base 71 | -------------------------------------------------------------------------------- /base/array_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/gtest.h" 6 | #include "base/array.h" 7 | 8 | namespace base { 9 | 10 | using namespace std; 11 | 12 | class NonTrivialCtor { 13 | public: 14 | static int count; 15 | 16 | NonTrivialCtor(int i, int j) { 17 | ++count; 18 | } 19 | ~NonTrivialCtor() { 20 | --count; 21 | } 22 | }; 23 | 24 | int NonTrivialCtor::count = 0; 25 | class ArrayTest { }; 26 | 27 | TEST(ArrayTest, Basic) { 28 | const array a(42); 29 | EXPECT_EQ(42, a[0]); 30 | EXPECT_EQ(42, a[1]); 31 | static_assert(sizeof(int) * a.size() == sizeof(a), ""); 32 | 33 | const array b(int(43)); 34 | for (auto i : b) { 35 | EXPECT_EQ(43, i); 36 | } 37 | } 38 | 39 | TEST(ArrayTest, String) { 40 | const array a("Roman"); 41 | EXPECT_EQ("Roman", a[0]); 42 | EXPECT_EQ("Roman", a[1]); 43 | 44 | const array c(5, 'c'); 45 | for (const auto& s : c) { 46 | EXPECT_EQ("ccccc", s); 47 | } 48 | auto d = c; 49 | for (const auto& s : d) { 50 | EXPECT_EQ("ccccc", s); 51 | } 52 | d = a; 53 | for (const auto& s : d) { 54 | EXPECT_EQ("Roman", s); 55 | } 56 | } 57 | 58 | TEST(ArrayTest, Destruct) { 59 | { 60 | array a(5, 6); 61 | EXPECT_EQ(5, NonTrivialCtor::count); 62 | } 63 | EXPECT_EQ(0, NonTrivialCtor::count); 64 | } 65 | 66 | } // namespace base 67 | -------------------------------------------------------------------------------- /base/bits.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/bits.h" 5 | 6 | namespace base { 7 | 8 | const uint64_t powers_of_10_internal[] = { 9 | 0, 10 | 10, 11 | 100, 12 | 1000, 13 | 10000, 14 | 100000, 15 | 1000000, 16 | 10000000, 17 | 100000000, 18 | 1000000000, 19 | 10000000000, 20 | 100000000000, 21 | 1000000000000, 22 | 10000000000000, 23 | 100000000000000, 24 | 1000000000000000, 25 | 10000000000000000, 26 | 100000000000000000, 27 | 1000000000000000000, 28 | 10000000000000000000U 29 | }; 30 | 31 | } // namespace base 32 | -------------------------------------------------------------------------------- /base/crc32c.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | namespace crc32c { 11 | 12 | // Return the crc32c of concat(A, data[0,n-1]) where init_crc is the 13 | // crc32c of some string A. Extend() is often used to maintain the 14 | // crc32c of a stream of data. 15 | extern uint32_t Extend(uint32_t init_crc, const uint8_t* data, size_t n); 16 | 17 | // Return the crc32c of data[0,n-1] 18 | inline uint32_t Value(const uint8_t* data, size_t n) { 19 | return Extend(0, data, n); 20 | } 21 | 22 | static const uint32_t kMaskDelta = 0xa282ead8ul; 23 | 24 | // Return a masked representation of crc. 25 | // 26 | // Motivation: it is problematic to compute the CRC of a string that 27 | // contains embedded CRCs. Therefore we recommend that CRCs stored 28 | // somewhere (e.g., in files) should be masked before being stored. 29 | inline uint32_t Mask(uint32_t crc) { 30 | // Rotate right by 15 bits and add a constant. 31 | return ((crc >> 15) | (crc << 17)) + kMaskDelta; 32 | } 33 | 34 | // Return the crc whose masked representation is masked_crc. 35 | inline uint32_t Unmask(uint32_t masked_crc) { 36 | uint32_t rot = masked_crc - kMaskDelta; 37 | return ((rot >> 17) | (rot << 15)); 38 | } 39 | 40 | } // namespace crc32c 41 | -------------------------------------------------------------------------------- /base/crc32c_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 The LevelDB Authors. All rights reserved. 2 | // Use of this source code is governed by a BSD-style license that can be 3 | // found in the LICENSE file. See the AUTHORS file for names of contributors. 4 | 5 | #include "base/crc32c.h" 6 | 7 | #include "base/integral_types.h" 8 | #include "strings/stringpiece.h" 9 | #include 10 | 11 | namespace crc32c { 12 | 13 | class CRC { }; 14 | 15 | TEST(CRC, StandardResults) { 16 | // From rfc3720 section B.4. 17 | uint8 buf[32]; 18 | 19 | memset(buf, 0, sizeof(buf)); 20 | ASSERT_EQ(0x8a9136aa, Value(buf, sizeof(buf))); 21 | 22 | memset(buf, 0xff, sizeof(buf)); 23 | ASSERT_EQ(0x62a8ab43, Value(buf, sizeof(buf))); 24 | 25 | for (int i = 0; i < 32; i++) { 26 | buf[i] = i; 27 | } 28 | ASSERT_EQ(0x46dd794e, Value(buf, sizeof(buf))); 29 | 30 | for (int i = 0; i < 32; i++) { 31 | buf[i] = 31 - i; 32 | } 33 | ASSERT_EQ(0x113fdb5c, Value(buf, sizeof(buf))); 34 | 35 | unsigned char data[48] = { 36 | 0x01, 0xc0, 0x00, 0x00, 37 | 0x00, 0x00, 0x00, 0x00, 38 | 0x00, 0x00, 0x00, 0x00, 39 | 0x00, 0x00, 0x00, 0x00, 40 | 0x14, 0x00, 0x00, 0x00, 41 | 0x00, 0x00, 0x04, 0x00, 42 | 0x00, 0x00, 0x00, 0x14, 43 | 0x00, 0x00, 0x00, 0x18, 44 | 0x28, 0x00, 0x00, 0x00, 45 | 0x00, 0x00, 0x00, 0x00, 46 | 0x02, 0x00, 0x00, 0x00, 47 | 0x00, 0x00, 0x00, 0x00, 48 | }; 49 | ASSERT_EQ(0xd9963a56, Value(reinterpret_cast(data), sizeof(data))); 50 | } 51 | 52 | static uint32_t Value(StringPiece pc) { 53 | return Value(reinterpret_cast(pc.data()), pc.size()); 54 | } 55 | 56 | TEST(CRC, Values) { 57 | ASSERT_NE(Value("a"), Value("foo")); 58 | } 59 | 60 | TEST(CRC, Extend) { 61 | ASSERT_EQ(Value("hello world"), 62 | Extend(Value("hello "), reinterpret_cast("world"), 5)); 63 | } 64 | 65 | TEST(CRC, Mask) { 66 | uint32_t crc = Value("foo"); 67 | ASSERT_NE(crc, Mask(crc)); 68 | ASSERT_NE(crc, Mask(Mask(crc))); 69 | ASSERT_EQ(crc, Unmask(Mask(crc))); 70 | ASSERT_EQ(crc, Unmask(Unmask(Mask(Mask(crc))))); 71 | } 72 | 73 | } // namespace crc32c -------------------------------------------------------------------------------- /base/fixed.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef CODING_H 5 | #define CODING_H 6 | 7 | #include 8 | #include "base/endian.h" 9 | 10 | namespace coding { 11 | 12 | // const uint8 kMaxVarintBytes = 10; 13 | // const uint8 kMaxVarint32Bytes = 5; 14 | const uint8 kFixed32Bytes = 4; 15 | const uint8 kFixed64Bytes = 8; 16 | 17 | inline uint8* EncodeFixed32(uint32 value, uint8* buf) { 18 | LittleEndian::Store32(buf, value); 19 | return buf + kFixed32Bytes; 20 | } 21 | 22 | inline uint8* EncodeFixed64(uint64 value, uint8* buf) { 23 | LittleEndian::Store64(buf, value); 24 | return buf + kFixed64Bytes; 25 | } 26 | 27 | inline void AppendFixed32(uint32 value, std::string* dest) { 28 | uint8 buf[kFixed32Bytes]; 29 | EncodeFixed32(value, buf); 30 | dest->append(reinterpret_cast(buf), kFixed32Bytes); 31 | } 32 | 33 | inline void AppendFixed64(uint64 value, std::string* dest) { 34 | uint8 buf[kFixed64Bytes]; 35 | EncodeFixed64(value, buf); 36 | dest->append(reinterpret_cast(buf), kFixed64Bytes); 37 | } 38 | 39 | inline uint32 DecodeFixed32(const uint8* buf) { 40 | return LittleEndian::Load32(reinterpret_cast(buf)); 41 | } 42 | 43 | inline const uint8* DecodeFixed32(const uint8* buf, uint32* val) { 44 | *val = LittleEndian::Load32(reinterpret_cast(buf)); 45 | return buf + sizeof(*val); 46 | } 47 | 48 | inline const uint8* DecodeFixed64(const uint8* buf, uint64* val) { 49 | *val = LittleEndian::Load64(reinterpret_cast(buf)); 50 | return buf + kFixed64Bytes; 51 | } 52 | 53 | 54 | } // namespace coding 55 | 56 | #endif // CODING_H 57 | -------------------------------------------------------------------------------- /base/flags.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | -------------------------------------------------------------------------------- /base/flat_arrays_vector.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "base/integral_types.h" 9 | 10 | namespace base { 11 | 12 | // Represents vector of vectors using flat memory block. 13 | // Useful when you build vector of vectors once and continously use them afterwards. 14 | // The data structure allows to map from dense index to vector items of type T. 15 | template class FlatArraysVec { 16 | std::vector offsets_; 17 | std::vector data_; 18 | 19 | public: 20 | // The auxillary class that allows iterating over elements belonging to data_. 21 | // This is allowed by implementing begin(), end() methods. 22 | class RangeWrapper { 23 | const T* t_; 24 | uint32 sz_; 25 | public: 26 | RangeWrapper(const T* t, uint32 sz) : t_(t), sz_(sz) {} 27 | 28 | const T* begin() const { return t_; } 29 | const T* end() const { return t_ + sz_; } 30 | }; 31 | 32 | void Add(const std::vector& items) { 33 | offsets_.push_back(data_.size()); // adds start offset. 34 | data_.insert(data_.end(), items.begin(), items.end()); // copied the data. 35 | } 36 | 37 | void Finalize() { 38 | offsets_.shrink_to_fit(); 39 | data_.shrink_to_fit(); 40 | } 41 | 42 | RangeWrapper range(uint32 index) const { 43 | assert(index < offsets_.size()); 44 | uint32 start = offsets_[index]; 45 | uint32 end = index+1 < offsets_.size() ? offsets_[index + 1] : data_.size(); 46 | return RangeWrapper(data_.data() + start, end - start); 47 | } 48 | 49 | // Returns number of arrays in flat array. 50 | uint32 size() const { return offsets_.size(); } 51 | }; 52 | 53 | } // namespace base 54 | -------------------------------------------------------------------------------- /base/gtest.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace base { 10 | 11 | // Used to avoid compiler optimizations for these benchmarks. 12 | // Just call it with the return value of the function. 13 | template void sink_result(const T& t0) { 14 | volatile T t = t0; 15 | (void)t; 16 | } 17 | 18 | // Returns unique test dir - the same for the run of the process. 19 | // The directory is cleaned automatically if all the tests finish succesfully. 20 | std::string GetTestTempDir(); 21 | std::string GetTestTempPath(const std::string& base_name); 22 | 23 | std::string RandStr(const unsigned len); 24 | 25 | std::string ProgramRunfile(const std::string& relative_path); // relative to runtime dir. 26 | 27 | } // namespace base 28 | -------------------------------------------------------------------------------- /base/hash.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | // based on MurmurHash code. 5 | // 6 | #include "base/hash.h" 7 | #include "absl/base/macros.h" 8 | #include "absl/base/optimization.h" 9 | #include 10 | #include 11 | 12 | namespace { 13 | 14 | inline uint32_t fmix(uint32_t h) { 15 | h ^= h >> 16; 16 | h *= 0x85ebca6b; 17 | h ^= h >> 13; 18 | h *= 0xc2b2ae35; 19 | h ^= h >> 16; 20 | 21 | return h; 22 | } 23 | 24 | inline uint32_t rotl32(uint32_t x, int8_t r) { 25 | return (x << r) | (x >> (32 - r)); 26 | } 27 | 28 | } // namespace 29 | 30 | 31 | namespace base { 32 | 33 | uint32_t MurmurHash3_x86_32(const uint8_t* data, uint32_t len, uint32_t seed) { 34 | const uint32_t nblocks = len / 4; 35 | 36 | uint32_t h1 = seed; 37 | 38 | uint32_t c1 = 0xcc9e2d51; 39 | uint32_t c2 = 0x1b873593; 40 | 41 | //---------- 42 | // body 43 | 44 | const uint32_t * blocks = (const uint32_t*) (data + nblocks * 4); 45 | 46 | int i; 47 | for(i = -nblocks; i; i++) 48 | { 49 | uint32_t k1; 50 | memcpy(&k1, blocks + i, sizeof(uint32_t)); 51 | 52 | k1 *= c1; 53 | k1 = rotl32(k1, 15); 54 | k1 *= c2; 55 | 56 | h1 ^= k1; 57 | h1 = rotl32(h1, 13); 58 | h1 = h1*5+0xe6546b64; 59 | } 60 | 61 | //---------- 62 | // tail 63 | 64 | const uint8_t * tail = data + nblocks*4; 65 | 66 | uint32_t k1 = 0; 67 | 68 | switch(len & 3) 69 | { 70 | case 3: k1 ^= tail[2] << 16;ABSL_FALLTHROUGH_INTENDED; 71 | case 2: k1 ^= tail[1] << 8;ABSL_FALLTHROUGH_INTENDED; 72 | case 1: k1 ^= tail[0]; 73 | k1 *= c1; k1 = rotl32(k1,15); k1 *= c2; h1 ^= k1; 74 | } 75 | 76 | //---------- 77 | // finalization 78 | 79 | h1 ^= len; 80 | 81 | h1 = fmix(h1); 82 | 83 | return h1; 84 | } 85 | 86 | uint64_t Fingerprint(const char* str, uint32_t len) { 87 | uint64_t res = XXH64(str, len, 24061983); 88 | if (ABSL_PREDICT_TRUE(res > 1)) 89 | return res; 90 | return 2; 91 | } 92 | 93 | } // namespace base 94 | -------------------------------------------------------------------------------- /base/hash_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/hash.h" 5 | 6 | #include "base/gtest.h" 7 | #include "base/logging.h" 8 | #include "file/filesource.h" 9 | #include "strings/strip.h" 10 | 11 | using namespace std; 12 | 13 | namespace base { 14 | 15 | static std::vector ReadIds() { 16 | file::LineReader line_reader(base::ProgramRunfile("testdata/ids.txt.gz")); 17 | decltype(ReadIds()) res; 18 | 19 | StringPiece line; 20 | while (line_reader.Next(&line)) { 21 | line = absl::StripAsciiWhitespace(line); 22 | res.push_back(strings::AsString(line)); 23 | CHECK(!line.empty()); 24 | } 25 | return res; 26 | } 27 | 28 | class HashTest : public testing::Test { 29 | protected: 30 | }; 31 | 32 | TEST_F(HashTest, Basic) { 33 | auto ids = ReadIds(); 34 | ASSERT_GT(ids.size(), 10); 35 | } 36 | 37 | static void BM_MurMur(benchmark::State& state) { 38 | auto ids = ReadIds(); 39 | uint32 i = 0; 40 | while (state.KeepRunning()) { 41 | int j = i++ % ids.size(); 42 | const auto* val = reinterpret_cast(ids[j].data()); 43 | sink_result(base::MurmurHash3_x86_32(val, ids[j].size(), i)); 44 | } 45 | } 46 | BENCHMARK(BM_MurMur); 47 | 48 | } // namespace base 49 | -------------------------------------------------------------------------------- /base/init.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "base/flags.h" 7 | #include "base/logging.h" 8 | 9 | namespace __internal__ { 10 | 11 | class ModuleInitializer { 12 | public: 13 | typedef void (*VoidFunction)(void); 14 | 15 | ModuleInitializer(VoidFunction ctor, bool is_ctor); 16 | 17 | static void RunFtors(bool is_ctor); 18 | 19 | private: 20 | struct CtorNode { 21 | VoidFunction func; 22 | CtorNode* next; 23 | 24 | bool is_ctor; 25 | } __attribute__((packed)); 26 | 27 | CtorNode node_; 28 | 29 | static CtorNode* & global_list(); 30 | 31 | ModuleInitializer(const ModuleInitializer&) = delete; 32 | void operator=(const ModuleInitializer&) = delete; 33 | }; 34 | 35 | } // __internal__ 36 | 37 | #define REGISTER_MODULE_INITIALIZER(name, body) \ 38 | namespace { \ 39 | static void google_init_module_##name () { body; } \ 40 | __internal__::ModuleInitializer google_initializer_module_##name( \ 41 | google_init_module_##name, true); \ 42 | } 43 | 44 | #define REGISTER_MODULE_DESTRUCTOR(name, body) \ 45 | namespace { \ 46 | static void google_destruct_module_##name () { body; } \ 47 | __internal__::ModuleInitializer google_destructor_module_##name( \ 48 | google_destruct_module_##name, false); \ 49 | } 50 | 51 | class MainInitGuard { 52 | public: 53 | MainInitGuard(int* argc, char*** argv, uint32_t flags = 0); 54 | 55 | ~MainInitGuard(); 56 | }; 57 | 58 | constexpr uint32_t DISABLE_JIFFIES_THREAD = 1; 59 | 60 | #define MainInitGuard(x, y) static_assert(false, "Forgot variable name") 61 | -------------------------------------------------------------------------------- /base/int128.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2004 Google Inc. 2 | // All Rights Reserved. 3 | // 4 | // 5 | 6 | #include 7 | using std::cout; 8 | using std::endl; 9 | #include "base/int128.h" 10 | #include "base/integral_types.h" 11 | 12 | const uint128_pod kuint128max = { 13 | static_cast(GG_LONGLONG(0xFFFFFFFFFFFFFFFF)), 14 | static_cast(GG_LONGLONG(0xFFFFFFFFFFFFFFFF)) 15 | }; 16 | 17 | std::ostream& operator<<(std::ostream& o, const uint128& b) { 18 | return (o << b.hi_ << "::" << b.lo_); 19 | } 20 | -------------------------------------------------------------------------------- /base/lambda_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | #include 6 | 7 | #include "base/gtest.h" 8 | #include "base/logging.h" 9 | 10 | using namespace std; 11 | 12 | static bool log_new = false; 13 | void* operator new(std::size_t n) { 14 | if (log_new) 15 | cerr << "Allocating " << n << " bytes" << endl; 16 | return malloc(n); 17 | } 18 | void operator delete(void* p, size_t sz) { 19 | free(p); 20 | } 21 | 22 | void operator delete(void* p) noexcept { 23 | free(p); 24 | } 25 | 26 | namespace base { 27 | 28 | class LambdaTest : public testing::Test { 29 | }; 30 | 31 | #pragma clang diagnostic ignored "-Wunused-lambda-capture" 32 | 33 | TEST_F(LambdaTest, Cb) { 34 | std::array arr; 35 | arr.fill(5); 36 | log_new = true; 37 | 38 | auto cb = [arr] {}; 39 | static_assert(sizeof(cb) > 4000, ""); 40 | cb(); 41 | std::function f1 = cb; 42 | static_assert(sizeof(f1) == 32, ""); 43 | 44 | std::array arr2; 45 | arr2.fill(5); 46 | 47 | std::function f2 = [arr2] {}; 48 | 49 | log_new = false; 50 | } 51 | 52 | 53 | 54 | } // namespace base 55 | -------------------------------------------------------------------------------- /base/logging.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/logging.h" 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | namespace base { 12 | 13 | using std::string; 14 | 15 | static constexpr char kProcSelf[] = "/proc/self/exe"; 16 | static constexpr char kDeletedSuffix[] = " (deleted)"; 17 | 18 | constexpr size_t kDeletedSuffixLen = sizeof(kDeletedSuffix) - 1; 19 | 20 | string ProgramAbsoluteFileName() { 21 | string res(2048, '\0'); 22 | size_t sz = readlink(kProcSelf, &res.front(), res.size()); 23 | CHECK_GT(sz, 0); 24 | if (sz > kDeletedSuffixLen) { 25 | // When binary was deleted, linux link contains kDeletedSuffix at the end. 26 | // Lets strip it. 27 | if (res.compare(sz - kDeletedSuffixLen, kDeletedSuffixLen, kDeletedSuffix) == 0) { 28 | sz -= kDeletedSuffixLen; 29 | res[sz] = '\0'; 30 | } 31 | } 32 | res.resize(sz); 33 | return res; 34 | } 35 | 36 | string ProgramBaseName() { 37 | string res = ProgramAbsoluteFileName(); 38 | size_t pos = res.rfind("/"); 39 | if (pos == string::npos) 40 | return res; 41 | return res.substr(pos + 1); 42 | } 43 | 44 | string MyUserName() { 45 | const char* str = std::getenv("USER"); 46 | return str ? str : string("unknown-user"); 47 | } 48 | 49 | void ConsoleLogSink::send(google::LogSeverity severity, const char* full_filename, 50 | const char* base_filename, int line, 51 | const struct ::tm* tm_time, 52 | const char* message, size_t message_len) { 53 | std::cout.write(message, message_len); 54 | std::cout << std::endl; 55 | } 56 | 57 | ConsoleLogSink* ConsoleLogSink::instance() { 58 | static ConsoleLogSink sink; 59 | return &sink; 60 | } 61 | 62 | const char* kProgramName = ""; 63 | 64 | } // namespace base 65 | -------------------------------------------------------------------------------- /base/logging.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | namespace base { 11 | std::string ProgramAbsoluteFileName(); 12 | 13 | std::string ProgramBaseName(); 14 | 15 | std::string MyUserName(); 16 | 17 | 18 | class ConsoleLogSink : public google::LogSink { 19 | public: 20 | virtual void send(google::LogSeverity severity, const char* full_filename, 21 | const char* base_filename, int line, 22 | const struct ::tm* tm_time, 23 | const char* message, size_t message_len) override; 24 | 25 | static ConsoleLogSink* instance(); 26 | }; 27 | 28 | extern const char* kProgramName; 29 | 30 | } // namespace base 31 | 32 | #define CONSOLE_INFO LOG_TO_SINK(base::ConsoleLogSink::instance(), INFO) 33 | -------------------------------------------------------------------------------- /base/macros.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #define COMPILE_ASSERT(expr, msg) \ 10 | static_assert(bool(expr), #msg) 11 | 12 | 13 | #ifndef DISALLOW_COPY_AND_ASSIGN 14 | #define DISALLOW_COPY_AND_ASSIGN(TypeName) \ 15 | TypeName(const TypeName&) = delete; \ 16 | void operator=(const TypeName&) = delete 17 | #endif 18 | 19 | // An older, politically incorrect name for the above. 20 | // Prefer DISALLOW_COPY_AND_ASSIGN for new code. 21 | #define DISALLOW_EVIL_CONSTRUCTORS(TypeName) DISALLOW_COPY_AND_ASSIGN(TypeName) 22 | 23 | // A macro to disallow all the implicit constructors, namely the 24 | // default constructor, copy constructor and operator= functions. 25 | // 26 | // This should be used in the private: declarations for a class 27 | // that wants to prevent anyone from instantiating it. This is 28 | // especially useful for classes containing only static methods. 29 | #define DISALLOW_IMPLICIT_CONSTRUCTORS(TypeName) \ 30 | TypeName(); \ 31 | DISALLOW_COPY_AND_ASSIGN(TypeName) 32 | 33 | template 34 | char (&ArraySizeHelper(T (&array)[N]))[N]; 35 | 36 | template 37 | char (&ArraySizeHelper(const T (&array)[N]))[N]; 38 | 39 | #define arraysize(array) (sizeof(ArraySizeHelper(array))) 40 | 41 | // A macro to turn a symbol into a string 42 | #define AS_STRING(x) AS_STRING_INTERNAL(x) 43 | #define AS_STRING_INTERNAL(x) #x 44 | 45 | #define LIKELY(x) (__builtin_expect(!!(x), 1)) 46 | #define UNLIKELY(x) (__builtin_expect(!!(x), 0)) 47 | -------------------------------------------------------------------------------- /base/memory.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | namespace base { 10 | 11 | // C++ new/delete can not allocate arrays initialized with c'tor. 12 | // For that we use allocate_unique_array: 13 | // my_unique_arr arr = allocate_unique_array(count, alloc) 14 | 15 | template class AllocArrayDeleter { 16 | Alloc alloc_; 17 | size_t sz_; 18 | public: 19 | AllocArrayDeleter(Alloc alloc, size_t s) : alloc_(alloc), sz_(s) {} 20 | 21 | void operator()(typename std::allocator_traits::pointer p) { 22 | for (size_t i = 0; i < sz_; ++i) { 23 | std::allocator_traits::destroy(alloc_, p + i); 24 | } 25 | std::allocator_traits::deallocate(alloc_, p, sz_); 26 | } 27 | }; 28 | 29 | template 30 | std::unique_ptr> 31 | allocate_unique_array(size_t count, Alloc alloc, Args&&... args) { 32 | std::unique_ptr> res{ 33 | alloc.allocate(count), AllocArrayDeleter(alloc, count)}; 34 | for (size_t i = 0; i < count; ++i) { 35 | std::allocator_traits::construct(alloc, res.get() + i, std::forward(args)...); 36 | } 37 | return res; 38 | } 39 | 40 | 41 | } // namespace base 42 | -------------------------------------------------------------------------------- /base/mpmc_bounded_queue_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/mpmc_bounded_queue.h" 5 | 6 | #include 7 | #include "base/gtest.h" 8 | #include "base/logging.h" 9 | 10 | using namespace std; 11 | 12 | namespace base { 13 | 14 | class MPMCTest : public testing::Test {}; 15 | 16 | struct A { 17 | static int ref; 18 | 19 | A() { ++ref; } 20 | A(const A&) { ++ref; } 21 | A(A&&) { ++ref; } 22 | 23 | ~A() { --ref; } 24 | }; 25 | 26 | int A::ref = 0; 27 | 28 | TEST_F(MPMCTest, Enqueue) { 29 | mpmc_bounded_queue q(2); 30 | ASSERT_TRUE(q.try_enqueue(5)); 31 | const int val = 6; 32 | ASSERT_TRUE(q.try_enqueue(val)); 33 | ASSERT_FALSE(q.try_enqueue(val)); 34 | 35 | int tmp = 0; 36 | ASSERT_TRUE(q.try_dequeue(tmp)); 37 | EXPECT_EQ(5, tmp); 38 | ASSERT_TRUE(q.try_dequeue(tmp)); 39 | EXPECT_EQ(6, tmp); 40 | ASSERT_FALSE(q.try_dequeue(tmp)); 41 | 42 | mpmc_bounded_queue> sh_q(2); 43 | int* const ptr = new int(5); 44 | ASSERT_TRUE(sh_q.try_enqueue(ptr)); 45 | 46 | auto ptr2 = std::make_unique(3); 47 | ASSERT_TRUE(sh_q.try_enqueue(std::move(ptr2))); 48 | ASSERT_FALSE(ptr2); 49 | ptr2 = std::make_unique(3); 50 | 51 | ASSERT_FALSE(sh_q.try_enqueue(std::move(ptr2))); 52 | ASSERT_TRUE(ptr2); 53 | } 54 | 55 | TEST_F(MPMCTest, Dtor) { 56 | for (unsigned i = 1; i <= 8; ++i) { 57 | mpmc_bounded_queue tst(8); 58 | EXPECT_EQ(0, A::ref); 59 | 60 | for (unsigned j = 0; j < i; ++j) { 61 | EXPECT_TRUE(tst.try_enqueue(A{})); 62 | EXPECT_EQ(j + 1, A::ref); 63 | } 64 | } 65 | } 66 | 67 | } // namespace base 68 | -------------------------------------------------------------------------------- /base/pod_array_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/pod_array.h" 6 | 7 | #include "base/gtest.h" 8 | 9 | namespace base { 10 | class PodArrayTest { 11 | }; 12 | 13 | 14 | TEST(BitsTest, Padded) { 15 | PODArray arr; 16 | typedef decltype(arr)::value_type value_t; 17 | arr.push_back(0); 18 | EXPECT_EQ(0, (ptrdiff_t)arr.data() % arr.alignment_v ) << arr.data(); 19 | EXPECT_EQ(arr.alignment_v / sizeof(value_t), arr.capacity()); 20 | 21 | for (unsigned i = 1; i < 1024; ++i) 22 | arr.push_back(i); 23 | for (unsigned i = 0; i < 1024; ++i) { 24 | ASSERT_EQ(i, arr[i]); 25 | } 26 | EXPECT_EQ(1024, arr.allocated_size() / sizeof(value_t)); 27 | 28 | arr.emplace_back(0); 29 | 30 | EXPECT_EQ(0, arr.back()); 31 | EXPECT_EQ(1025, arr.size()); 32 | EXPECT_EQ(2048, arr.capacity()); 33 | } 34 | 35 | } // namespace base 36 | -------------------------------------------------------------------------------- /base/posix_call.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef _POSIX_CALL_H 5 | #define _POSIX_CALL_H 6 | 7 | #include 8 | #include 9 | #include "base/logging.h" 10 | 11 | namespace base { 12 | 13 | inline std::string PosixStrError(int e) { 14 | char buf[1024]; 15 | return strerror_r(e, buf, sizeof buf); 16 | } 17 | 18 | inline std::string PosixStrError() { 19 | return PosixStrError(errno); 20 | } 21 | 22 | } // namespace base 23 | 24 | #define POSIX_CALL(x) do { \ 25 | int r = x; \ 26 | if (r != 0) \ 27 | LOG(ERROR) << "Error calling " #x << ", msg: " << base::PosixStrError(); \ 28 | } while(false); 29 | 30 | #endif // _POSIX_CALL_H 31 | -------------------------------------------------------------------------------- /base/pthread_utils.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/logging.h" 6 | #include "base/pthread_utils.h" 7 | 8 | namespace base { 9 | 10 | static void* start_cpp_function(void *arg) { 11 | std::function* fp = (std::function*)arg; 12 | CHECK(*fp); 13 | 14 | (*fp)(); 15 | delete fp; 16 | 17 | return nullptr; 18 | } 19 | 20 | void InitCondVarWithClock(clockid_t clock_id, pthread_cond_t* var) { 21 | pthread_condattr_t attr; 22 | PTHREAD_CHECK(condattr_init(&attr)); 23 | PTHREAD_CHECK(condattr_setclock(&attr, clock_id)); 24 | 25 | PTHREAD_CHECK(cond_init(var, &attr)); 26 | PTHREAD_CHECK(condattr_destroy(&attr)); 27 | } 28 | 29 | 30 | pthread_t StartThread(const char* name, void *(*start_routine) (void *), void *arg) { 31 | CHECK_LT(strlen(name), 16); 32 | 33 | pthread_attr_t attrs; 34 | PTHREAD_CHECK(attr_init(&attrs)); 35 | PTHREAD_CHECK(attr_setstacksize(&attrs, kThreadStackSize)); 36 | 37 | pthread_t result; 38 | VLOG(1) << "Starting thread " << name; 39 | 40 | PTHREAD_CHECK(create(&result, &attrs, start_routine, arg)); 41 | int my_err = pthread_setname_np(result, name); 42 | if (my_err != 0) { 43 | LOG(WARNING) << "Could not set name on thread " << result << " : " << strerror(my_err); 44 | } 45 | PTHREAD_CHECK(attr_destroy(&attrs)); 46 | return result; 47 | } 48 | 49 | pthread_t StartThread(const char* name, std::function f) { 50 | return StartThread(name, start_cpp_function, new std::function(std::move(f))); 51 | } 52 | 53 | } // namespace base -------------------------------------------------------------------------------- /base/pthread_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include "base/logging.h" 9 | 10 | #define PTHREAD_CHECK(x) \ 11 | do { \ 12 | int my_err = pthread_ ## x; \ 13 | CHECK_EQ(0, my_err) << #x << ", error: " << strerror(my_err); \ 14 | } while(false) 15 | 16 | namespace base { 17 | 18 | constexpr int kThreadStackSize = 65536; 19 | 20 | void InitCondVarWithClock(clockid_t clock_id, pthread_cond_t* var); 21 | 22 | 23 | pthread_t StartThread(const char* name, void *(*start_routine) (void *), void *arg); 24 | pthread_t StartThread(const char* name, std::function f); 25 | 26 | } // namespace base 27 | -------------------------------------------------------------------------------- /base/simd.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace base { 10 | 11 | // [ptr, ptr+len) should be inside memory range with alignment at least 16, i.e. in some cases 12 | // it will access memory before ptr and after ptr + len. 13 | // Returns how many times val appeared in the range. 14 | size_t CountVal8(const uint8_t* ptr, size_t len, char val); 15 | 16 | // Writes to buffer the successive differences of buffer 17 | // (buffer[0]-starting_point, buffer[1]-buffer[2], ...) 18 | void ComputeDeltasInplace(uint32_t * buffer, size_t length, uint32_t starting_point); 19 | void ComputeDeltasInplace(uint16_t * buffer, size_t length, uint16_t starting_point); 20 | 21 | } // namespace base 22 | -------------------------------------------------------------------------------- /base/stl_util.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include "base/integral_types.h" 11 | 12 | 13 | template std::ostream& operator<<(std::ostream& o, const std::vector& vec) { 14 | o << "["; 15 | for (size_t i = 0; i < vec.size(); ++i) { 16 | o << vec[i]; 17 | if (i + 1 < vec.size()) o << ","; 18 | } 19 | o << "]"; 20 | return o; 21 | } 22 | 23 | namespace base { 24 | 25 | template bool _in(const T& t, 26 | std::initializer_list l) { 27 | return std::find(l.begin(), l.end(), t) != l.end(); 28 | } 29 | 30 | 31 | template struct MinMax { 32 | 33 | T min_val = std::numeric_limits::max(), max_val = std::numeric_limits::min(); 34 | 35 | void Update(T val) { 36 | min_val = std::min(min_val, val); 37 | max_val = std::max(max_val, val); 38 | } 39 | }; 40 | 41 | } // namespace base 42 | -------------------------------------------------------------------------------- /base/testdata/ids.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/romange/gaia/8ef14627a4bf42eba83bb6df4d180beca305b307/base/testdata/ids.txt.gz -------------------------------------------------------------------------------- /blaze.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | TARGET_BUILD_TYPE=Debug 4 | BUILD_DIR=build-dbg 5 | COMPILER=`which g++` 6 | GENERATOR='' 7 | LAUNCHER=$(command -v ccache) 8 | if [ -x $LAUNCHER ]; then 9 | echo "Using launcher $LAUNCHER" 10 | LAUNCHER="-DCMAKE_CXX_COMPILER_LAUNCHER=$LAUNCHER" 11 | else 12 | LAUNCHER='' 13 | fi 14 | 15 | for ARG in $* 16 | do 17 | case "$ARG" in 18 | -release) 19 | TARGET_BUILD_TYPE=Release 20 | BUILD_DIR=build-opt 21 | shift 22 | ;; 23 | -clang) 24 | COMPILER=`which clang++` 25 | shift 26 | ;; 27 | -ninja) 28 | GENERATOR='-GNinja' 29 | shift 30 | ;; 31 | *) 32 | echo bad option "$ARG" 33 | exit 1 34 | ;; 35 | esac 36 | shift 37 | done 38 | 39 | mkdir -p $BUILD_DIR && cd $BUILD_DIR 40 | set -x 41 | 42 | cmake -L -DCMAKE_BUILD_TYPE=$TARGET_BUILD_TYPE -DCMAKE_CXX_COMPILER=$COMPILER $GENERATOR $LAUNCHER .. 43 | 44 | 45 | -------------------------------------------------------------------------------- /cmake/FindPackageMessage.cmake: -------------------------------------------------------------------------------- 1 | # Distributed under the OSI-approved BSD 3-Clause License. See accompanying 2 | # file Copyright.txt or https://cmake.org/licensing for details. 3 | 4 | #[=======================================================================[.rst: 5 | FindPackageMessage 6 | ------------------ 7 | 8 | .. code-block:: cmake 9 | 10 | find_package_message( "message for user" "find result details") 11 | 12 | This function is intended to be used in FindXXX.cmake modules files. 13 | It will print a message once for each unique find result. This is 14 | useful for telling the user where a package was found. The first 15 | argument specifies the name (XXX) of the package. The second argument 16 | specifies the message to display. The third argument lists details 17 | about the find result so that if they change the message will be 18 | displayed again. The macro also obeys the QUIET argument to the 19 | find_package command. 20 | 21 | Example: 22 | 23 | .. code-block:: cmake 24 | 25 | if(X11_FOUND) 26 | find_package_message(X11 "Found X11: ${X11_X11_LIB}" 27 | "[${X11_X11_LIB}][${X11_INCLUDE_DIR}]") 28 | else() 29 | ... 30 | endif() 31 | #]=======================================================================] 32 | 33 | function(find_package_message pkg msg details) 34 | # Avoid printing a message repeatedly for the same find result. 35 | if(NOT ${pkg}_FIND_QUIETLY) 36 | string(REPLACE "\n" "" details "${details}") 37 | set(DETAILS_VAR FIND_PACKAGE_MESSAGE_DETAILS_${pkg}) 38 | if(NOT "${details}" STREQUAL "${${DETAILS_VAR}}") 39 | # The message has not yet been printed. 40 | message(STATUS "${msg}") 41 | 42 | # Save the find details in the cache to avoid printing the same 43 | # message again. 44 | set("${DETAILS_VAR}" "${details}" 45 | CACHE INTERNAL "Details about finding ${pkg}") 46 | endif() 47 | endif() 48 | endfunction() 49 | -------------------------------------------------------------------------------- /doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(Doxygen) 2 | 3 | IF(NOT DOXYGEN_FOUND) 4 | MESSAGE(STATUS "No Doxygen found. Documentation won't be built") 5 | ELSE() 6 | file(GLOB SOURCES ${CMAKE_CURRENT_LIST_DIR}/../mr/*) 7 | file(GLOB MARKDOWN_DOC ${CMAKE_CURRENT_LIST_DIR}/*.md) 8 | list(APPEND MARKDOWN_DOC ${CMAKE_CURRENT_LIST_DIR}/../README.md) 9 | 10 | CONFIGURE_FILE(Doxyfile.in Doxyfile @ONLY) 11 | CONFIGURE_FILE(header.html.in header.html @ONLY) 12 | CONFIGURE_FILE(doxygen.css html/doxygen.css COPYONLY) 13 | 14 | file(GLOB DOXYFILES ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile*) 15 | 16 | set(DOXYGEN_INPUT_DIR ${PROJECT_SOURCE_DIR}) 17 | Message("Doxygen input dir ${DOXYGEN_INPUT_DIR}") 18 | add_custom_command(OUTPUT html 19 | COMMAND ${DOXYGEN_EXECUTABLE} ${CMAKE_CURRENT_BINARY_DIR}/Doxyfile 20 | COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/html 21 | DEPENDS ${MARKDOWN_DOC} ${SOURCES} ${DOXYFILES} 22 | WORKING_DIRECTORY ${DOXYGEN_INPUT_DIR} 23 | ) 24 | 25 | add_custom_target(doc ALL DEPENDS html) 26 | #install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/html 27 | # DESTINATION ${DOC_INSTALL_DIR} 28 | # COMPONENT doc) 29 | ENDIF() 30 | -------------------------------------------------------------------------------- /doc/README.md: -------------------------------------------------------------------------------- 1 | [Mastering Doxygen](https://exhale.readthedocs.io/en/latest/mastering_doxygen.html#) 2 | 3 | [Link MD pages](https://stackoverflow.com/questions/19275887/how-do-i-link-between-markdown-documents-in-doxygen) 4 | 5 | Tutorials: 6 | https://flcwiki.desy.de/How%20to%20document%20your%20code%20using%20doxygen 7 | 8 | -------------------------------------------------------------------------------- /doc/docker_build.md: -------------------------------------------------------------------------------- 1 | # How to build docker image with a gaia binary 2 | 3 | I've prepared development and production docker images based on Ubuntu that allow building self 4 | contained GAIA binaries. The build process is as followse: 5 | 6 | 1. [bin_build.Dockerfile](../docker/bin_build.Dockerfile) orchestrates the build. It requires few build arguments to pass: 7 | * the binary target to build 8 | * optional ubuntu and boost versions 9 | * image tag name for the newly created image. 10 | 2. The script pulls the development image with specified ubuntu version and boost development libraries installed (by default it's Ubuntu 18 with Boost 1.71) and builds the specified target. Thus no meddling with host system is needed. 11 | 3. Then the script pulls the appropriate run-time image of the same version but with runtime boost libraries installed and copies there the built binary and its shared lib dependencies. 12 | Therefore, the final image is as slim as possible and much smaller than the development image. 13 | 4. Finally it tags the image locally with the specified tag name. 14 | 15 | The `docker build` command should be run from gaia root dir and it looks as follows: 16 | 17 | ```bash 18 | > docker build -f docker/bin_build.Dockerfile --build-arg TARGET= . -t 19 | ``` 20 | 21 | for example, 22 | 23 | ```bash 24 | > docker build -t asio_fibers:mytag -f docker/bin_build.Dockerfile --build-arg TARGET=asio_fibers . 25 | 26 | > docker run asio_fibers:mytag --logtostderr 27 | ``` 28 | 29 | Optionally, it's possible to pass build argument `IMAGE_TAG=...` . Currently, supported values are: ` 16_1_71_0`, ` 16_1_70_0`, ` 18_1_69_0`, ` 18_1_70_0`, ` 18_1_71_0`. 30 | -------------------------------------------------------------------------------- /doc/header.html.in: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | GAIA 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 18 |
19 | -------------------------------------------------------------------------------- /doc/mainpage.md: -------------------------------------------------------------------------------- 1 | Welcome to GAIA-MR 2 | ============ 3 | 4 | Multi-threaded, efficient mapreduce framework for data processing. 5 | 6 | ## Parallel cloud GREP 7 | Suppose we want a simple tool that can read and grep text files, maybe compressed, or maybe stored 8 | on cloud storage.Here is how [we can do it](https://github.com/romange/gaia/blob/master/examples/mrgrep.cc). 9 | 10 | ```cpp 11 | class Grepper { 12 | RE2 re_; 13 | public: 14 | Grepper(string reg_exp) : re_(reg_exp) {} 15 | 16 | void Do(string val, mr3::DoContext* context) { 17 | if (RE2::PartialMatch(val, re_)) { 18 | auto* raw = context->raw(); 19 | cout << raw->input_file_name() << ":" << raw->input_pos() << " " << val << endl; 20 | } 21 | } 22 | }; 23 | 24 | int main(int argc, char** argv) { 25 | PipelineMain pm(&argc, &argv); 26 | 27 | vector inputs; 28 | for (int i = 1; i < argc; ++i) { 29 | inputs.push_back(argv[i]); 30 | } 31 | CHECK(!FLAGS_e.empty()); 32 | 33 | Pipeline* pipeline = pm.pipeline(); 34 | StringTable st = pipeline->ReadText("read_input", inputs); 35 | StringTable null_output = st.Map("grep", FLAGS_e); 36 | null_output.Write("null", pb::WireFormat::TXT); // To force this mapper to run. 37 | 38 | LocalRunner* runner = pm.StartLocalRunner("/tmp/"); 39 | pipeline->Run(runner); 40 | return 0; 41 | } 42 | ``` 43 | -------------------------------------------------------------------------------- /docker/alpine.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM frolvlad/alpine-gxx 2 | 3 | RUN apk update && \ 4 | apk upgrade && \ 5 | apk add git autoconf automake libtool make linux-headers 6 | 7 | -------------------------------------------------------------------------------- /docker/bin_build.Dockerfile: -------------------------------------------------------------------------------- 1 | # To build a binary using this docker file run, for example: 2 | # docker build -f docker/bin_build.Dockerfile --build-arg IMAGE_TAG=18_1_71_0 --build-arg TARGET=mr3 . 3 | ARG IMAGE_TAG=18_1_71_0 4 | 5 | FROM romange/boost-dev:${IMAGE_TAG} as third_party 6 | ARG IMAGE_TAG 7 | 8 | # To allow caching third_party libs via "minimal" build. 9 | # We do not use ARG here, not even reference it here because otherwise it would cause cache misses 10 | # during this STEP for different ARGs. 11 | COPY ./CMakeLists.txt /src/ 12 | COPY ./cmake /src/cmake 13 | WORKDIR /build 14 | RUN echo "PATH IS: $PATH" && cmake --version 15 | RUN echo "IMAGE_TAG ${IMAGE_TAG}" && cmake -L -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=g++ \ 16 | -DONLY_THIRD_PARTY=ON -GNinja /src 17 | 18 | RUN ninja -j4 protobuf_project glog_project sparsehash_project gperf_project zstd_project \ 19 | lz4_project xxhash_project gtest_project pmr_project 20 | 21 | FROM third_party as src 22 | COPY ./ /src/ 23 | RUN mkdir /pkg && cmake -L -DONLY_THIRD_PARTY=OFF -DBUILD_DOCS=OFF -GNinja /src 24 | 25 | FROM src as bin 26 | # Now really building the target. Previous containers will be reused between the builds. 27 | ARG TARGET 28 | ARG IMAGE_TAG 29 | RUN echo "Building ${TARGET}" && ninja -j4 $TARGET && strip $TARGET && mv $TARGET /pkg/ 30 | RUN ldd -r /pkg/$TARGET | grep "/build/third_party" | awk '{print $3}' | \ 31 | while read n; do cp -nH $n /pkg/; done 32 | 33 | FROM romange/boost-prod:${IMAGE_TAG} 34 | ARG TARGET 35 | COPY --from=bin /pkg/ /app/ 36 | RUN ldconfig /app && ln -s $TARGET start_app 37 | ENV PATH /app:$PATH 38 | 39 | ENTRYPOINT ["start_app"] 40 | -------------------------------------------------------------------------------- /examples/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(file_read file_read.cc) 2 | cxx_link(file_read base TRDP::folly dl) 3 | 4 | add_executable(asio_fibers asio_fibers.cc) 5 | cxx_link(asio_fibers base http_v2 rpc sentry strings) 6 | 7 | add_executable(http_client_tool http_client_tool.cc) 8 | cxx_link(http_client_tool http_client_lib asio_fiber_lib) 9 | 10 | add_executable(mr3 mr3.cc) 11 | cxx_link(mr3 fiber_file asio_fiber_lib mr3_lib absl_hash absl_str_format http_v2) 12 | 13 | add_executable(gcs_demo gcs_demo.cc) 14 | cxx_link(gcs_demo gce_lib) 15 | 16 | add_executable(s3_demo s3_demo.cc) 17 | cxx_link(s3_demo file https_client_lib aws_lib) 18 | 19 | add_executable(gsod_group gsod_group.cc) 20 | cxx_link(gsod_group mr3_lib absl_hash absl_str_format http_v2) 21 | 22 | add_executable(movies_join movies_join.cc) 23 | cxx_link(movies_join mr3_lib absl_hash absl_str_format http_v2 TRDP::re2) 24 | 25 | add_executable(mr_read_test mr_read_test.cc) 26 | cxx_link(mr_read_test mr3_lib http_v2) 27 | 28 | add_executable(mrgrep mrgrep.cc) 29 | cxx_link(mrgrep mr3_lib http_v2 TRDP::re2) 30 | 31 | add_executable(osmium_road_length osmium_road_length.cc) 32 | cxx_link(osmium_road_length base TRDP::libosmium) 33 | 34 | add_subdirectory(wordcount) 35 | add_subdirectory(pingserver) 36 | add_subdirectory(redis) 37 | -------------------------------------------------------------------------------- /examples/http_client_tool.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/init.h" 6 | #include "base/logging.h" 7 | #include "base/walltime.h" 8 | 9 | #include "absl/strings/str_split.h" 10 | #include "strings/stringpiece.h" 11 | #include "util/asio/io_context_pool.h" 12 | #include "util/http/http_client.h" 13 | 14 | using namespace util; 15 | using namespace std; 16 | 17 | DEFINE_string(connect, "localhost:8080", ""); 18 | 19 | class HttpClientFiber : public IoContext::Cancellable { 20 | IoContext& cntx_; 21 | http::Client client_; 22 | bool cancelled_ = false; 23 | 24 | public: 25 | HttpClientFiber(IoContext& cntx) : cntx_(cntx), client_(cntx_) {} 26 | 27 | void Run() final { 28 | auto ec = client_.Connect("www.walla.com", "http"); 29 | if (ec) { 30 | LOG(INFO) << "Did not connect " << ec.message(); 31 | return; 32 | } 33 | 34 | while (!cancelled_) { 35 | http::Client::Response resp; 36 | ec = client_.Get("/", &resp); 37 | LOG_IF(WARNING, ec) << ec.message(); 38 | if (ec) 39 | break; 40 | } 41 | LOG(INFO) << "Finished the run"; 42 | } 43 | 44 | void Cancel() final { 45 | cancelled_ = true; 46 | client_.Cancel(); 47 | }; 48 | }; 49 | 50 | int main(int argc, char** argv) { 51 | MainInitGuard guard(&argc, &argv); 52 | 53 | IoContextPool pool; 54 | pool.Run(); 55 | vector parts = absl::StrSplit(FLAGS_connect, ":"); 56 | CHECK_EQ(2, parts.size()); 57 | 58 | IoContext& cntx = pool.GetNextContext(); 59 | cntx.AttachCancellable(new HttpClientFiber(cntx)); 60 | 61 | SleepForMilliseconds(500); 62 | LOG(INFO) << "Stopping pool"; 63 | pool.Stop(); 64 | 65 | return 0; 66 | } 67 | -------------------------------------------------------------------------------- /examples/mr3.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #define XXH_STATIC_LINKING_ONLY 6 | #include 7 | 8 | #include "absl/strings/str_cat.h" 9 | #include "absl/strings/str_format.h" 10 | #include "base/init.h" 11 | #include "base/logging.h" 12 | 13 | #include "file/file_util.h" 14 | #include "file/filesource.h" 15 | #include "mr/local_runner.h" 16 | #include "mr/mr_main.h" 17 | #include "mr/pipeline.h" 18 | 19 | #include "util/asio/accept_server.h" 20 | #include "util/asio/io_context_pool.h" 21 | 22 | using namespace std; 23 | using namespace boost; 24 | using namespace util; 25 | 26 | DEFINE_string(compress, "gzip", "can be '', 'gzip' or 'zstd'"); 27 | DEFINE_string(dest_dir, "~/mr_output", ""); 28 | DEFINE_uint32(num_shards, 10, ""); 29 | DEFINE_int32(compress_level, 1, ""); 30 | 31 | using namespace mr3; 32 | using namespace util; 33 | 34 | string ShardNameFunc(const std::string& line) { 35 | absl::Dec dec(XXH3_64bits(line.data(), line.size()) % FLAGS_num_shards, absl::kZeroPad4); 36 | return absl::StrCat("shard-", dec); 37 | } 38 | 39 | int main(int argc, char** argv) { 40 | PipelineMain pm(&argc, &argv); 41 | 42 | std::vector inputs; 43 | for (int i = 1; i < argc; ++i) { 44 | inputs.push_back(argv[i]); 45 | } 46 | CHECK(!inputs.empty()); 47 | 48 | Pipeline* pipeline = pm.pipeline(); 49 | 50 | StringTable ss = pipeline->ReadText("inp1", inputs); 51 | auto& outp = ss.Write("outp1", pb::WireFormat::TXT).WithCustomSharding(ShardNameFunc); 52 | if (!FLAGS_compress.empty()) { 53 | if (FLAGS_compress == "gzip") 54 | outp.AndCompress(pb::Output::GZIP); 55 | else if (FLAGS_compress == "zstd") { 56 | outp.AndCompress(pb::Output::ZSTD, FLAGS_compress_level); 57 | } else { 58 | LOG(FATAL) << "Unknown compress argument " << FLAGS_compress; 59 | } 60 | } 61 | 62 | LocalRunner* runner = pm.StartLocalRunner(FLAGS_dest_dir); 63 | 64 | pipeline->Run(runner); 65 | LOG(INFO) << "After pipeline run"; 66 | 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /examples/mr_read_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/init.h" 5 | #include "base/logging.h" 6 | 7 | #include "mr/local_runner.h" 8 | #include "mr/mr_main.h" 9 | #include "mr/pipeline.h" 10 | 11 | using namespace std; 12 | using namespace boost; 13 | using namespace util; 14 | 15 | DEFINE_string(dest_dir, "~/mr_output", ""); 16 | 17 | using namespace mr3; 18 | using namespace util; 19 | 20 | class NoOpMapper { 21 | public: 22 | void Do(string val, mr3::DoContext* context) {} 23 | }; 24 | 25 | int main(int argc, char** argv) { 26 | PipelineMain pm(&argc, &argv); 27 | 28 | std::vector inputs; 29 | for (int i = 1; i < argc; ++i) { 30 | inputs.push_back(argv[i]); 31 | } 32 | CHECK(!inputs.empty()); 33 | 34 | Pipeline* pipeline = pm.pipeline(); 35 | 36 | StringTable ss = pipeline->ReadText("inp1", inputs); 37 | StringTable out_table = ss.Map("map"); 38 | out_table.Write("outp1", pb::WireFormat::TXT) 39 | .WithModNSharding(10, [](const string& s) { return 0; }) 40 | .AndCompress(pb::Output::ZSTD, 1); 41 | 42 | LocalRunner* runner = pm.StartLocalRunner(FLAGS_dest_dir); 43 | 44 | pipeline->Run(runner); 45 | LOG(INFO) << "After pipeline run"; 46 | 47 | return 0; 48 | } 49 | -------------------------------------------------------------------------------- /examples/mrgrep.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "base/logging.h" 4 | #include "file/file_util.h" 5 | 6 | #include "mr/local_runner.h" 7 | #include "mr/mr_main.h" 8 | 9 | using namespace mr3; 10 | using namespace util; 11 | using namespace std; 12 | using re2::RE2; 13 | 14 | DEFINE_string(e, "", 15 | "Regular expression with perl-like syntax.\n" 16 | "See https://github.com/google/re2/wiki/Syntax for details."); 17 | 18 | class Grepper { 19 | public: 20 | Grepper(string reg_exp) : re_(reg_exp) { 21 | CHECK(re_.ok()) << "Error parsing 'e': " << re_.error_code() << " offending part " 22 | << re_.error_arg(); 23 | } 24 | 25 | void Do(string val, DoContext* context) { 26 | if (RE2::PartialMatch(val, re_)) { 27 | auto* raw = context->raw(); 28 | cout << raw->input_file_name() << ":" << raw->input_pos() << " " << val << endl; 29 | } 30 | } 31 | 32 | private: 33 | RE2 re_; // Even though RE2 is thread-safe we allocate separate instances to reduce contention. 34 | }; 35 | 36 | int main(int argc, char** argv) { 37 | PipelineMain pm(&argc, &argv); 38 | if (argc < 2) { 39 | return 0; 40 | } 41 | 42 | vector inputs; 43 | for (int i = 1; i < argc; ++i) { 44 | inputs.push_back(argv[i]); 45 | } 46 | CHECK(!FLAGS_e.empty()); 47 | 48 | Pipeline* pipeline = pm.pipeline(); 49 | StringTable st = pipeline->ReadText("read_input", inputs); 50 | StringTable no_output = st.Map("grep", FLAGS_e); 51 | no_output.Write("null", pb::WireFormat::TXT); 52 | 53 | LocalRunner* runner = pm.StartLocalRunner("/tmp/"); 54 | pipeline->Run(runner); 55 | 56 | LOG(INFO) << "After pipeline run"; 57 | 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /examples/pingserver/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(ping_server ping_server.cc resp_parser.cc ping_command.cc) 2 | cxx_link(ping_server base asio_fiber_lib http_v2) 3 | 4 | add_executable(ping_sync_server ping_sync_server.cc resp_parser.cc ping_command.cc) 5 | cxx_link(ping_sync_server base asio_fiber_lib http_v2) 6 | 7 | add_executable(ping_epoll_server ping_epoll_server.cc resp_parser.cc ping_command.cc) 8 | cxx_link(ping_epoll_server base asio_fiber_lib http_v2) 9 | 10 | add_executable(ping_iouring_server ping_iouring_server.cc resp_parser.cc ping_command.cc) 11 | cxx_link(ping_iouring_server base stats_lib uring_fiber_lib) 12 | -------------------------------------------------------------------------------- /examples/pingserver/ping_command.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "examples/pingserver/resp_parser.h" 11 | 12 | class PingCommand { 13 | redis::RespParser resp_parser_; 14 | enum State { READ_NUM_TOKS, STR_LEN, STR_DATA } state_ = READ_NUM_TOKS; 15 | 16 | static const char kReply[]; 17 | 18 | public: 19 | PingCommand() { 20 | } 21 | 22 | bool Decode(size_t len); 23 | 24 | boost::asio::mutable_buffer read_buffer() { 25 | auto span = resp_parser_.GetDestBuf(); 26 | return boost::asio::buffer(span.data(), span.size()); 27 | } 28 | 29 | boost::asio::const_buffer reply() const; 30 | 31 | private: 32 | bool HandleLine(absl::string_view line); 33 | }; 34 | 35 | void ConfigureSocket(boost::asio::ip::tcp::socket* sock); 36 | -------------------------------------------------------------------------------- /examples/pingserver/ping_sync_server.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | #include "base/init.h" 14 | #include "examples/pingserver/ping_command.h" 15 | 16 | DEFINE_int32(http_port, 8080, "Http port."); 17 | DEFINE_int32(port, 6380, "Redis port"); 18 | 19 | using namespace boost; 20 | using asio::ip::tcp; 21 | using std::string; 22 | 23 | inline bool IsExpectedFinish(system::error_code ec) { 24 | using namespace boost::asio; 25 | return ec == error::eof || ec == error::operation_aborted || ec == error::connection_reset || 26 | ec == error::not_connected; 27 | } 28 | 29 | void RedisConnection(tcp::socket sock) { 30 | PingCommand cmd; 31 | 32 | while (true) { 33 | system::error_code ec; 34 | size_t length = sock.read_some(cmd.read_buffer(), ec); 35 | if (ec) { 36 | LOG_IF(ERROR, !IsExpectedFinish(ec)) << ec; 37 | break; 38 | } 39 | 40 | if (cmd.Decode(length)) { 41 | asio::write(sock, cmd.reply()); 42 | } 43 | } 44 | } 45 | 46 | void ServePing(boost::asio::io_context& io_context, const tcp::endpoint& endpoint) { 47 | tcp::acceptor a(io_context, endpoint); 48 | 49 | for (;;) { 50 | tcp::socket sock(io_context); 51 | a.accept(sock); 52 | 53 | LOG(INFO) << "Accepted socket from " << sock.remote_endpoint(); 54 | 55 | std::thread{&RedisConnection, std::move(sock)}.detach(); 56 | } 57 | } 58 | 59 | int main(int argc, char* argv[]) { 60 | MainInitGuard guard(&argc, &argv); 61 | 62 | tcp::endpoint endpoint(tcp::v4(), FLAGS_port); 63 | boost::asio::io_context io_context; 64 | 65 | try { 66 | ServePing(io_context, endpoint); 67 | } catch (std::exception& e) { 68 | std::cerr << "Exception: " << e.what() << "\n"; 69 | } 70 | 71 | return 0; 72 | } 73 | -------------------------------------------------------------------------------- /examples/pingserver/resp_parser.cc: -------------------------------------------------------------------------------- 1 | ../redis/resp_parser.cc -------------------------------------------------------------------------------- /examples/pingserver/resp_parser.h: -------------------------------------------------------------------------------- 1 | ../redis/resp_parser.h -------------------------------------------------------------------------------- /examples/redis/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(redis_toy_server redis_toy_server.cc redis_command.cc 2 | resp_parser.cc resp_connection_handler.cc) 3 | cxx_link(redis_toy_server base http_v2 absl_variant) 4 | -------------------------------------------------------------------------------- /examples/redis/redis_command.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "examples/redis/redis_command.h" 5 | 6 | #include "absl/strings/str_cat.h" 7 | #include "base/bits.h" 8 | #include "base/logging.h" 9 | 10 | namespace redis { 11 | 12 | const char* Command::FlagName(CommandFlag fl) { 13 | switch (fl) { 14 | case FL_WRITE: 15 | return "write"; 16 | case FL_READONLY: 17 | return "readonly"; 18 | case FL_DENYOOM: 19 | return "denyoom"; 20 | case FL_FAST: 21 | return "fast"; 22 | case FL_STALE: 23 | return "stale"; 24 | case FL_LOADING: 25 | return "loading"; 26 | case FL_RANDOM: 27 | return "random"; 28 | default: 29 | LOG(FATAL) << "Unknown flag " << fl; 30 | } 31 | } 32 | 33 | uint32_t Command::FlagsCount(uint32_t flags) { 34 | return Bits::CountOnes(flags); 35 | } 36 | 37 | void Command::Call(const Args& args, std::string* dest) const { 38 | CHECK_NE(0, arity_); 39 | 40 | if ((arity_ > 0 && args.size() != size_t(arity_)) || 41 | (arity_ < 0 && args.size() < size_t(-arity_))) { 42 | *dest = absl::StrCat("-ERR wrong number of arguments for '", name_, "' command\r\n"); 43 | return; 44 | } 45 | 46 | fun_(args, dest); 47 | } 48 | 49 | } // namespace redis 50 | -------------------------------------------------------------------------------- /examples/redis/redis_toy_server.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "base/init.h" 6 | #include "examples/redis/resp_connection_handler.h" 7 | #include "util/asio/accept_server.h" 8 | #include "util/asio/io_context_pool.h" 9 | #include "util/http/http_conn_handler.h" 10 | 11 | using namespace util; 12 | using namespace redis; 13 | 14 | DEFINE_int32(http_port, 8080, "Http port."); 15 | DEFINE_int32(port, 6380, "Redis port"); 16 | 17 | int main(int argc, char** argv) { 18 | MainInitGuard guard(&argc, &argv); 19 | IoContextPool pool; 20 | pool.Run(); 21 | 22 | std::unique_ptr server(new AcceptServer(&pool)); 23 | http::Listener<> http_listener; 24 | uint16_t port = server->AddListener(FLAGS_http_port, &http_listener); 25 | 26 | LOG(INFO) << "Started http server on port " << port; 27 | 28 | RespListener resp_listener; 29 | resp_listener.Init(); 30 | 31 | port = server->AddListener(FLAGS_port, &resp_listener); 32 | LOG(INFO) << "Started redis server on port " << port; 33 | server->Run(); 34 | server->Wait(); 35 | 36 | return 0; 37 | } 38 | -------------------------------------------------------------------------------- /examples/redis/resp_parser.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | 6 | #include "absl/strings/string_view.h" 7 | #include "absl/types/span.h" 8 | 9 | namespace redis { 10 | 11 | class RespParser { 12 | static constexpr uint16_t kBufSz = 256; 13 | 14 | public: 15 | using Buffer = absl::Span; 16 | enum ParseStatus { LINE_FINISHED = 1, MORE_DATA = 2 }; 17 | 18 | RespParser(); 19 | 20 | //! 0-copy interface 21 | Buffer GetDestBuf() const { 22 | return Buffer(write_start_, buf_.data() + kBufSz - write_start_); 23 | } 24 | 25 | void WriteCommit(size_t write_sz) { write_start_ += write_sz; } 26 | 27 | //! Tries to parse the next line in the buffer ending with \r\n. 28 | //! If succeeds, sets the 'line' argument to point to the buffer representing the line and 29 | //! returns LINE_FINISHED. Otherwise, returns MORE_DATA. 30 | //! In case of LINE_FINISHED, the returned buffer is "consumed" but still valid until the next 31 | // operation to RespParser. 32 | ParseStatus ParseNext(absl::string_view* line); 33 | 34 | Buffer ReadBuf() const { 35 | return Buffer(next_read_, write_start_ - next_read_); 36 | } 37 | 38 | bool IsReadEof() const { return next_read_ == write_start_; } 39 | 40 | void Reset() { 41 | write_start_ = buf_.data(); 42 | next_parse_ = next_read_ = write_start_; 43 | } 44 | 45 | // Consumes and discards 'sz' characters. 46 | void Consume(size_t sz) { 47 | next_read_ += sz; 48 | Realign(); 49 | } 50 | 51 | void ConsumeLine() { 52 | Realign(); 53 | } 54 | 55 | void Realign(); 56 | 57 | void ConsumeWs(); 58 | private: 59 | uint8_t* WriteEnd() { 60 | return buf_.data() + kBufSz; 61 | } 62 | 63 | std::array buf_; 64 | 65 | // The structure is: |buf_.data()____next_read_____next_parse____write_start____buf_.end()| 66 | uint8_t *write_start_, *next_read_ = nullptr, *next_parse_ = nullptr; 67 | }; 68 | 69 | } // namespace redis 70 | -------------------------------------------------------------------------------- /examples/wordcount/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(warc_parse warc_parse.cc) 2 | cxx_link(warc_parse mr3_lib http_v2 TRDP::re2) 3 | 4 | add_executable(word_count word_count.cc) 5 | cxx_link(word_count mr3_lib http_v2 TRDP::re2 TRDP::chimera TRDP::hs TRDP::pcre) 6 | -------------------------------------------------------------------------------- /file/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(file file.cc file_util.cc filesource.cc gzip_file.cc list_file.cc list_file_reader.cc 2 | meta_map_block.cc compressors.cc lst2_impl.cc) 3 | cxx_link(file base strings util TRDP::lz4 TRDP::crc32c) 4 | 5 | add_library(file_test_util test_util.cc) 6 | target_link_libraries(file_test_util base file gaia_gtest_main) 7 | 8 | cxx_proto_lib(proto_writer_test DEPENDS addressbook_proto) 9 | 10 | add_library(proto_writer proto_writer.cc) 11 | cxx_link(proto_writer file TRDP::protobuf util) 12 | 13 | add_library(lz4_file lz4_file.cc) 14 | cxx_link(lz4_file file TRDP::lz4) 15 | 16 | add_library(list_file_py SHARED list_file_py.cc) 17 | set_target_properties(list_file_py PROPERTIES PREFIX "") 18 | 19 | target_link_libraries(list_file_py python2.7 file) 20 | add_include(list_file_py "/usr/include/python2.7/") 21 | 22 | add_library(fiber_file fiber_file.cc) 23 | cxx_link(fiber_file file fibers_ext) 24 | 25 | 26 | cxx_test(file_test file lz4_file file_test_util LABELS CI) 27 | cxx_test(list_file_test file file_test_util LABELS CI) 28 | cxx_test(proto_writer_test proto_writer proto_writer_test_proto LABELS CI) 29 | 30 | -------------------------------------------------------------------------------- /file/compressors.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015, .com . All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "util/status.h" 7 | #include "file/list_file_format.h" 8 | #include 9 | 10 | namespace file { 11 | 12 | typedef std::function UncompressFunction; 14 | 15 | typedef std::function CompressFunction; 17 | typedef std::function CompressBoundFunction; 18 | 19 | 20 | UncompressFunction GetUncompress(list_file::CompressMethod m); 21 | CompressFunction GetCompress(list_file::CompressMethod m); 22 | 23 | CompressBoundFunction GetCompressBound(list_file::CompressMethod method); 24 | 25 | } // namespace file 26 | -------------------------------------------------------------------------------- /file/fiber_file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "file/file.h" 6 | #include "util/fibers/fiberqueue_threadpool.h" 7 | 8 | namespace file { 9 | 10 | // Fiber-friendly file handler. Returns ReadonlyFile* instance that does not block the current 11 | // thread unlike the regular posix implementation. All the read opearations will run 12 | // in FiberQueueThreadPool. 13 | struct FiberReadOptions : public ReadonlyFile::Options { 14 | struct Stats { 15 | size_t cache_bytes = 0; // read because of prefetch logic. 16 | size_t disk_bytes = 0; // read via ThreadPool calls. 17 | size_t read_prefetch_cnt = 0; 18 | size_t preempt_cnt = 0; 19 | }; 20 | 21 | size_t prefetch_size = 0; 22 | Stats* stats = nullptr; 23 | }; 24 | 25 | util::StatusObject OpenFiberReadFile( 26 | StringPiece name, util::fibers_ext::FiberQueueThreadPool* tp, 27 | const FiberReadOptions& opts = FiberReadOptions{}) MUST_USE_RESULT; 28 | 29 | struct FiberWriteOptions : public OpenOptions { 30 | bool consistent_thread = true; // whether to send the write request to the same pool-thread. 31 | }; 32 | 33 | util::StatusObject 34 | OpenFiberWriteFile(StringPiece name, util::fibers_ext::FiberQueueThreadPool* tp, 35 | const FiberWriteOptions& opts = FiberWriteOptions()) MUST_USE_RESULT; 36 | 37 | } // namespace file 38 | -------------------------------------------------------------------------------- /file/gzip_file.cc: -------------------------------------------------------------------------------- 1 | 2 | // 3 | #include "file/gzip_file.h" 4 | #include 5 | 6 | #include "absl/strings/str_cat.h" 7 | #include "base/logging.h" 8 | 9 | using util::Status; 10 | using util::StatusCode; 11 | 12 | namespace file { 13 | 14 | GzipFile::GzipFile(StringPiece file_name, unsigned level) : WriteFile(file_name), level_(level) { 15 | } 16 | 17 | bool GzipFile::Open() { 18 | gz_file_ = gzopen(create_file_name_.c_str(), "wb"); 19 | if (gz_file_ == nullptr) { 20 | LOG(WARNING) << "Can't open " << create_file_name_ 21 | << " (errno = " << StatusFileError() << ")."; 22 | return false; 23 | } 24 | gzbuffer(gz_file_, 1 << 16); 25 | gzsetparams(gz_file_, level_, Z_DEFAULT_STRATEGY); 26 | 27 | return true; 28 | } 29 | 30 | bool GzipFile::Close() { 31 | gzclose_w(gz_file_); 32 | 33 | delete this; 34 | 35 | return true; 36 | } 37 | 38 | Status GzipFile::Write(const uint8* buffer, uint64 length) { 39 | CHECK_GT(length, 0); 40 | 41 | unsigned bytes = gzwrite(gz_file_, buffer, length); 42 | if (bytes == 0) { 43 | int err; 44 | const char* str = gzerror(gz_file_, &err); 45 | return Status(StatusCode::IO_ERROR, absl::StrCat(str, " ", err)); 46 | } 47 | CHECK_EQ(bytes, length); 48 | 49 | return Status::OK; 50 | } 51 | 52 | } // namespace file 53 | -------------------------------------------------------------------------------- /file/gzip_file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "file/file.h" 7 | 8 | extern "C" struct gzFile_s; 9 | 10 | namespace file { 11 | 12 | // Creates a gzipped file for writing. 13 | class GzipFile : public WriteFile { 14 | gzFile_s* gz_file_ = nullptr; 15 | uint8 level_; 16 | GzipFile(StringPiece file_name, unsigned level); 17 | public: 18 | static GzipFile* Create(StringPiece file_name, unsigned level = 2) { 19 | return new GzipFile(file_name, std::max(1u, level)); 20 | } 21 | 22 | bool Open() override; 23 | bool Close() override; 24 | 25 | util::Status Write(const uint8* buffer, uint64 length) override; 26 | }; 27 | 28 | } // namespace file 29 | -------------------------------------------------------------------------------- /file/lst2_impl.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "file/list_file.h" 7 | #include "file/list_file_format2.h" 8 | 9 | namespace file { 10 | namespace lst2 { 11 | 12 | class Lst2Impl : public ListWriter::WriterImpl { 13 | public: 14 | Lst2Impl(util::Sink* sink, const ListWriter::Options& opts); 15 | ~Lst2Impl(); 16 | 17 | util::Status Init(const std::map& meta) final; 18 | util::Status AddRecord(StringPiece slice) final; 19 | util::Status Flush() final; 20 | 21 | private: 22 | util::Status EmitPhysicalRecord(strings::ByteRange header, strings::ByteRange record); 23 | util::Status EmitSingleRecord(RecordType type, StringPiece record); 24 | util::Status WriteFragmented(StringPiece record); 25 | 26 | void AddRecordToArray(StringPiece size_enc, StringPiece record); 27 | util::Status FlushArray(); 28 | 29 | uint32_t block_size() const { return block_size_; } 30 | 31 | std::unique_ptr array_store_; 32 | std::unique_ptr compress_buf_; 33 | 34 | uint8 *array_next_ = nullptr, *array_end_ = nullptr; // wraps array_store_ 35 | bool init_called_ = false; 36 | 37 | uint32_t block_offset_ = 0; // Current offset in block 38 | uint32_t array_records_ = 0; 39 | uint32_t block_size_ = 0; 40 | }; 41 | 42 | 43 | class ReaderImpl : public ListReader::FormatImpl { 44 | public: 45 | using FormatImpl::FormatImpl; 46 | 47 | bool ReadHeader(std::map* dest) final; 48 | 49 | bool ReadRecord(StringPiece* record, std::string* scratch) final; 50 | 51 | private: 52 | enum { 53 | kEof = kMaxRecordVal + 1, 54 | 55 | // Returned whenever we find an invalid physical record. 56 | // Currently there are these situations in which this happens: 57 | // * The record has an invalid CRC (ReadPhysicalRecord reports a drop) 58 | kBadRecord = kMaxRecordVal + 2 59 | }; 60 | unsigned ReadPhysicalRecord(StringPiece* dest); 61 | }; 62 | 63 | } // namespace lst2 64 | } // namespace file 65 | -------------------------------------------------------------------------------- /file/lz4_compressor.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2015, .com . All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include 6 | 7 | #include "util/compressors.h" 8 | 9 | #include "base/init.h" 10 | 11 | #include "strings/stringpiece.h" 12 | 13 | namespace { 14 | 15 | using base::Status; 16 | using base::StatusCode; 17 | using strings::charptr; 18 | 19 | Status UncompressZ4(const void* src, size_t len, void* dest, size_t* uncompress_size) { 20 | int res = LZ4_decompress_safe((const char*)src, (char*)dest, len, *uncompress_size); 21 | if (res <= 0) { 22 | return Status(StatusCode::INTERNAL_ERROR); 23 | } 24 | *uncompress_size = res; 25 | return Status::OK; 26 | } 27 | 28 | size_t BoundFunctionLZ4(size_t len) { 29 | return LZ4_compressBound(len); 30 | } 31 | 32 | Status CompressLZ4(int level, const void* src, size_t len, void* dest, 33 | size_t* compress_size) { 34 | int res = LZ4_compress_fast((const char*)src, (char*)dest, len, *compress_size, 1); 35 | if (res <= 0) { 36 | return Status(StatusCode::INTERNAL_ERROR); 37 | } 38 | *compress_size = res; 39 | return Status::OK; 40 | } 41 | 42 | } // namespace 43 | 44 | using namespace ::util::compressors; 45 | 46 | REGISTER_MODULE_INITIALIZER(lz4codec, { 47 | internal::Register(LZ4_METHOD, &BoundFunctionLZ4, &CompressLZ4, &UncompressZ4); 48 | }); 49 | 50 | namespace util { 51 | namespace compressors { 52 | 53 | int dummy_lz4() { return 0; } 54 | 55 | } 56 | } 57 | 58 | // REGISTER_COMPRESS( 59 | -------------------------------------------------------------------------------- /file/lz4_file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2015, .com . All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "file/file.h" 7 | 8 | struct LZ4F_cctx_s; 9 | 10 | namespace file { 11 | 12 | // Creates a lz4-compressed file for writing. 13 | class LZ4File : public WriteFile { 14 | LZ4F_cctx_s* context_ = nullptr; 15 | uint8* buf_ = nullptr; 16 | size_t buf_size_ = 0; 17 | 18 | uint8 level_ = 0; 19 | int fd_ = 0; 20 | 21 | LZ4File(StringPiece file_name, unsigned level); 22 | ~LZ4File(); 23 | public: 24 | static LZ4File* Create(StringPiece file_name, unsigned level = 2) { 25 | return new LZ4File(file_name, level); 26 | } 27 | 28 | bool Open() override; 29 | bool Close() override; 30 | 31 | util::Status Write(const uint8* buffer, uint64 length) override; 32 | }; 33 | 34 | 35 | } // namespace file 36 | -------------------------------------------------------------------------------- /file/meta_map_block.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef _META_MAP_BLOCK_CC 5 | #define _META_MAP_BLOCK_CC 6 | 7 | #include "file/meta_map_block.h" 8 | #include "base/varint.h" 9 | 10 | using util::Status; 11 | using util::StatusCode; 12 | using std::string; 13 | using strings::u8ptr; 14 | 15 | namespace file { 16 | 17 | void MetaMapBlock::EncodeTo(std::string* dest) const { 18 | // format: varint32 map size, 19 | // (varint string size, string data)+ 20 | Varint::Append32(dest, meta_.size()); 21 | for (const auto& k_v : meta_) { 22 | Varint::EncodeTwo32Values(dest, k_v.first.size(), k_v.second.size()); 23 | dest->append(k_v.first).append(k_v.second); 24 | } 25 | } 26 | 27 | Status MetaMapBlock::DecodeFrom(StringPiece input) { 28 | const uint8* ptr = u8ptr(input), *limit = u8ptr(input) + input.size(); 29 | uint32 sz = 0; 30 | if ((ptr = Varint::Parse32WithLimit(ptr, limit, &sz)) == nullptr) { 31 | return Status(StatusCode::IO_ERROR); 32 | } 33 | uint32 ksz = 0, vsz = 0; 34 | for (uint32 i = 0; i < sz; ++i) { 35 | if (ptr >= limit) { 36 | return Status(StatusCode::IO_ERROR); 37 | } 38 | ptr = Varint::DecodeTwo32Values(ptr, &ksz, &vsz); 39 | if (ptr + ksz + vsz > limit) { 40 | return Status(StatusCode::IO_ERROR); 41 | } 42 | string key; 43 | key.append(strings::charptr(ptr), ksz); 44 | ptr += ksz; 45 | meta_[key] = string(strings::charptr(ptr), vsz); 46 | ptr += vsz; 47 | } 48 | return Status::OK; 49 | } 50 | 51 | } // namespace file 52 | 53 | #endif // _META_MAP_BLOCK_CC 54 | -------------------------------------------------------------------------------- /file/meta_map_block.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef _META_MAP_BLOCK_H 5 | #define _META_MAP_BLOCK_H 6 | 7 | #include 8 | #include 9 | #include "util/status.h" 10 | #include "strings/stringpiece.h" 11 | 12 | namespace file { 13 | 14 | class MetaMapBlock { 15 | std::map meta_; 16 | 17 | public: 18 | void Add(const std::string& k, const std::string& v) { 19 | meta_[k] = v; 20 | } 21 | 22 | const std::map& meta() const { return meta_; } 23 | 24 | // Encodes the MetaMapBlock and appends its serialized contents to dest. 25 | void EncodeTo(std::string* dest) const; 26 | 27 | util::Status DecodeFrom(StringPiece input); 28 | 29 | bool empty() const { return meta_.empty(); } 30 | }; 31 | 32 | } // namespace file 33 | 34 | #endif // _META_MAP_BLOCK_H -------------------------------------------------------------------------------- /file/proto_writer_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "file/proto_writer.h" 5 | #include "file/proto_writer_test.pb.h" 6 | #include "base/gtest.h" 7 | 8 | namespace file { 9 | 10 | class ProtoWriterTest : public ::testing::Test { 11 | protected: 12 | }; 13 | 14 | TEST_F(ProtoWriterTest, Basic) { 15 | ListProtoWriter writer("foo.lst", test::Container::descriptor()); 16 | test::Container boo; 17 | boo.mutable_person()->set_name("Roman"); 18 | boo.mutable_person()->set_id(5); 19 | ASSERT_TRUE(writer.Add(boo).ok()); 20 | } 21 | 22 | TEST_F(ProtoWriterTest, Empty) { 23 | ListProtoWriter writer("foo.lst", test::Container::descriptor()); 24 | } 25 | 26 | } // namespace file 27 | -------------------------------------------------------------------------------- /file/proto_writer_test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | package test; 4 | import "util/plang/addressbook.proto"; 5 | 6 | message Container { 7 | optional tutorial.Person person = 1; 8 | optional tutorial.Person.PhoneType val = 2 [ default = HOME ]; 9 | } -------------------------------------------------------------------------------- /file/s3_file.h: -------------------------------------------------------------------------------- 1 | // Copyright 2014, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | // Internal - should not be used directly. 5 | #ifndef _S3_FILE_H 6 | #define _S3_FILE_H 7 | 8 | #include "strings/stringpiece.h" 9 | #include "base/status.h" 10 | 11 | namespace file { 12 | class ReadonlyFile; 13 | 14 | base::StatusObject OpenS3File(StringPiece name); 15 | bool ExistsS3File(StringPiece name); 16 | 17 | // Save local file to S3. This overwrites any existing object at that key. 18 | base::Status CopyFileToS3(StringPiece file_path, StringPiece s3_dir); 19 | 20 | 21 | struct StatShort; 22 | 23 | base::Status ListFiles(StringPiece path, unsigned max_items, std::vector* file_names); 24 | base::Status ListFiles(StringPiece path, unsigned max_items, std::vector* file_names); 25 | 26 | } // namespace file 27 | 28 | #endif // _S3_FILE_H -------------------------------------------------------------------------------- /file/test_util.h: -------------------------------------------------------------------------------- 1 | #ifndef TEST_UTIL_H 2 | #define TEST_UTIL_H 3 | 4 | #include 5 | #include "file/file.h" 6 | #include "util/sinksource.h" 7 | 8 | // When running unittests, get the directory containing the source code. 9 | std::string TestSourceDir(); 10 | 11 | // When running unittests, get a directory where temporary files may be 12 | // placed. 13 | std::string TestTempDir(); 14 | 15 | namespace file { 16 | 17 | 18 | class NullFile : public WriteFile { 19 | public: 20 | NullFile() : WriteFile("NullFile") {} 21 | virtual bool Close() override { return true; } 22 | virtual bool Open() override { return true; } 23 | 24 | util::Status Write(const uint8* ,uint64) override { return util::Status::OK; } 25 | }; 26 | 27 | class ReadonlyStringFile : public ReadonlyFile { 28 | std::string contents_; 29 | public: 30 | ReadonlyStringFile(std::string str = std::string()) : contents_(std::move(str)) {} 31 | 32 | // Reads upto length bytes and updates the result to point to the data. 33 | // May use buffer for storing data. 34 | util::StatusObject Read(size_t offset, const strings::MutableByteRange& range) override; 35 | 36 | // releases the system handle for this file. 37 | util::Status Close() override { return util::Status::OK; } 38 | 39 | size_t Size() const override { return contents_.size(); } 40 | 41 | bool force_error = false; 42 | 43 | void set_contents(const std::string& c) { contents_ = c; } 44 | 45 | int Handle() const { return 0; } 46 | private: 47 | bool returned_partial_ = false; 48 | }; 49 | 50 | /** 51 | * @brief Infinite source that wraps itself around the provided buffer. 52 | * 53 | */ 54 | class RingSource : public util::Source { 55 | public: 56 | RingSource(size_t sz, const std::string& buf); 57 | 58 | protected: 59 | util::StatusObject ReadInternal(const strings::MutableByteRange& range) final; 60 | 61 | size_t index_ = 0; 62 | size_t read_size_; 63 | const std::string& buf_; 64 | }; 65 | 66 | } // namespace file 67 | 68 | #endif // TEST_UTIL_H 69 | -------------------------------------------------------------------------------- /install-dependencies.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | apt install -y cmake libunwind-dev zip libfl-dev bison ninja-build autoconf-archive libtool 6 | apt install -y curl libxml2-dev 7 | g++ --version 8 | 9 | BVER=1.71.0 10 | BOOST=boost_${BVER//./_} # replace all . with _ 11 | 12 | # For sake of boost install we always use g++. 13 | export CXX=g++ 14 | 15 | install_boost() { 16 | mkdir -p /tmp/boost && pushd /tmp/boost 17 | if ! [ -d $BOOST ]; then 18 | url="https://boostorg.jfrog.io/artifactory/main/release/${BVER}/source/$BOOST.tar.bz2" 19 | echo "Downloading from $url" 20 | if ! [ -e $BOOST.tar.bz2 ]; then wget -nv ${url} -O $BOOST.tar.bz2; fi 21 | 22 | tar -xjf $BOOST.tar.bz2 && chown ${SUDO_USER}:${SUDO_USER} -R $BOOST.tar.bz2 $BOOST 23 | fi 24 | 25 | booststap_arg="--prefix=/opt/${BOOST} --without-libraries=graph_parallel,graph,wave,test,mpi,python" 26 | cd $BOOST 27 | boostrap_cmd=`readlink -f bootstrap.sh` 28 | 29 | echo "CXX compiler ${CXX}" 30 | echo "Running ${boostrap_cmd} ${booststap_arg}" 31 | ${boostrap_cmd} ${booststap_arg} || { cat bootstrap.log; return 1; } 32 | b2_args=(define=BOOST_COROUTINES_NO_DEPRECATION_WARNING=1 link=shared variant=release debug-symbols=on 33 | threading=multi --without-test --without-math --without-log --without-locale --without-wave 34 | --without-regex --without-python -j4) 35 | 36 | echo "Building targets with ${b2_args[@]}" 37 | ./b2 "${b2_args[@]}" cxxflags='-std=c++14 -Wno-deprecated-declarations' 38 | ./b2 install "${b2_args[@]}" -d0 39 | chown ${SUDO_USER}:${SUDO_USER} -R ./ 40 | popd 41 | } 42 | 43 | if ! [ -d /opt/${BOOST}/lib ]; then 44 | install_boost 45 | else 46 | echo "Skipping installing ${BOOST}" 47 | fi 48 | 49 | if ! [ -d /opt/boost ]; then 50 | ln -sf /opt/${BOOST} /opt/boost 51 | fi 52 | -------------------------------------------------------------------------------- /mr/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cxx_proto_lib(mr3) 2 | 3 | add_library(mr3_lib mr.cc operator_executor.cc pipeline.cc joiner_executor.cc local_runner.cc 4 | mapper_executor.cc mr_pb.cc mr_main.cc) 5 | cxx_link(mr3_lib absl_flat_hash_map absl_variant absl_str_format base mr3_impl_lib 6 | fiber_file asio_fiber_lib gce_lib aws_lib pb2json sentry TRDP::rapidjson) 7 | add_subdirectory(impl) 8 | 9 | add_library(mr_test_lib test_utils.cc) 10 | cxx_link(mr_test_lib mr3_lib absl_flat_hash_map gaia_gtest_main) 11 | 12 | cxx_test(mr_test mr_test_lib addressbook_proto LABELS CI) 13 | cxx_test(local_runner_test mr_test_lib addressbook_proto file_test_util LABELS CI) 14 | -------------------------------------------------------------------------------- /mr/impl/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(mr3_impl_lib local_context.cc dest_file_set.cc freq_map_wrapper.cc) 2 | cxx_link(mr3_impl_lib strings fiber_file proto_writer mr3_proto) 3 | -------------------------------------------------------------------------------- /mr/impl/freq_map_wrapper.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Hagana 159. All rights reserved. 2 | // Author: Ori Brostovski (oribrost@gmail.com) 3 | // 4 | #include "mr/impl/freq_map_wrapper.h" 5 | 6 | #include "base/logging.h" 7 | 8 | namespace mr3 { 9 | namespace detail { 10 | 11 | void FreqMapWrapper::CheckType(const std::type_info& t) const { 12 | CHECK(t == any_.type()) << "Mismatch between " << t.name() << " and " << any_.type().name(); 13 | } 14 | 15 | void FreqMapWrapper::Add(const FreqMapWrapper& other) { 16 | ExtraFunctions *ef = ExtraFuncsFromNullablePair(this, &other); 17 | ef->Add(other, this); 18 | } 19 | 20 | FreqMapWrapper::ExtraFunctions* 21 | FreqMapWrapper::ExtraFuncsFromNullablePair(const FreqMapWrapper *f1, const FreqMapWrapper *f2) { 22 | ExtraFunctions *ef1 = f1->extra_functions_.get(), *ef2 = f2->extra_functions_.get(); 23 | CHECK(ef1 || ef2) << "Can't run on two empty maps"; 24 | CHECK(!ef1 || typeid(*ef1) == typeid(*ef2)) << "Can't run on maps of different types"; 25 | return ef1 ? ef1 : ef2; 26 | } 27 | 28 | } // detail 29 | } // mr3 30 | -------------------------------------------------------------------------------- /mr/impl/freq_map_wrapper.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Hagana 159. All rights reserved. 2 | // Author: Ori Brostovski (oribrost@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/container/flat_hash_map.h" 7 | #include "absl/types/any.h" 8 | 9 | template using FrequencyMap = absl::flat_hash_map; 10 | 11 | namespace mr3 { 12 | namespace detail { 13 | 14 | class FreqMapWrapper { 15 | public: 16 | template 17 | FreqMapWrapper(FrequencyMap&& m) : any_(std::move(m)), 18 | extra_functions_(new ExtraFunctionsImpl) {} 19 | 20 | FreqMapWrapper() {} 21 | 22 | bool has_value() const { return any_.has_value(); } 23 | std::type_info type() const; 24 | void Add(const FreqMapWrapper& other); 25 | 26 | template FrequencyMap& Cast() { 27 | CheckType(typeid(FrequencyMap)); 28 | return *absl::any_cast>(&any_); 29 | } 30 | template const FrequencyMap& Cast() const { 31 | CheckType(typeid(FrequencyMap)); 32 | return *absl::any_cast>(&any_); 33 | } 34 | 35 | private: 36 | class ExtraFunctions { 37 | public: 38 | virtual void Add(const FreqMapWrapper& other, FreqMapWrapper *that) = 0; 39 | virtual ~ExtraFunctions() {} 40 | }; 41 | 42 | template 43 | class ExtraFunctionsImpl : public ExtraFunctions { 44 | public: 45 | void Add(const FreqMapWrapper& other, FreqMapWrapper *that) override { 46 | if (!other.has_value()) // other is empty, so we don't need to add anything. 47 | return; 48 | if (!that->has_value()) // that is empty, so before we fill it, we create it. 49 | that->any_ = FrequencyMap(); 50 | auto& casted_that = that->Cast(); 51 | for (auto& map_value : other.Cast()) 52 | casted_that[map_value.first] += map_value.second; 53 | } 54 | }; 55 | 56 | static ExtraFunctions *ExtraFuncsFromNullablePair(const FreqMapWrapper*, const FreqMapWrapper*); 57 | void CheckType(const std::type_info& t) const; 58 | 59 | absl::any any_; 60 | std::shared_ptr extra_functions_; 61 | }; 62 | 63 | } // detail 64 | } // mr3 65 | -------------------------------------------------------------------------------- /mr/impl/local_context.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "mr/do_context.h" 7 | #include "mr/impl/dest_file_set.h" 8 | 9 | namespace mr3 { 10 | 11 | namespace detail { 12 | 13 | class BufferedWriter; 14 | 15 | class LocalContext : public RawContext { 16 | public: 17 | explicit LocalContext(DestFileSet* mgr); 18 | ~LocalContext(); 19 | 20 | void Flush() final; 21 | 22 | void CloseShard(const ShardId& sid) final; 23 | 24 | private: 25 | void WriteInternal(const ShardId& shard_id, std::string&& record) final; 26 | 27 | absl::flat_hash_map custom_shard_files_; 28 | 29 | DestFileSet* mgr_; 30 | }; 31 | 32 | } // namespace detail 33 | } // namespace mr3 34 | -------------------------------------------------------------------------------- /mr/joiner_executor.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #include "mr/operator_executor.h" 10 | 11 | namespace mr3 { 12 | 13 | class JoinerExecutor : public OperatorExecutor { 14 | struct IndexedInput { 15 | uint32_t index; 16 | const pb::Input::FileSpec* fspec; 17 | const pb::WireFormat* wf; 18 | }; 19 | 20 | using ShardInput = std::pair>; 21 | public: 22 | JoinerExecutor(util::IoContextPool* pool, Runner* runner); 23 | ~JoinerExecutor(); 24 | 25 | void Run(const std::vector& inputs, detail::TableBase* tb, 26 | ShardFileMap* out_files) final; 27 | 28 | // Stops the executor in the middle. 29 | void Stop() final; 30 | 31 | private: 32 | void InitInternal() final; 33 | void CheckInputs(const std::vector& inputs); 34 | 35 | void ProcessInputQ(detail::TableBase* tb); 36 | 37 | void JoinerFiber(); 38 | 39 | ::boost::fibers::unbuffered_channel input_q_; 40 | 41 | std::atomic finish_shard_latency_sum_{0}, finish_shard_latency_cnt_{0}; 42 | }; 43 | 44 | } // namespace mr3 45 | -------------------------------------------------------------------------------- /mr/local_runner.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "mr/runner.h" 7 | 8 | namespace util { 9 | class IoContextPool; 10 | } // namespace util 11 | 12 | namespace file { 13 | class ReadonlyFile; 14 | } // namespace file 15 | 16 | namespace mr3 { 17 | 18 | class LocalRunner : public Runner { 19 | public: 20 | LocalRunner(util::IoContextPool* pool, const std::string& data_dir); 21 | ~LocalRunner(); 22 | 23 | void Init() final; 24 | 25 | void Shutdown() final; 26 | 27 | void OperatorStart(const pb::Operator* op) final; 28 | 29 | // Must be thread-safe. Called from multiple threads in pipeline_executor. 30 | RawContext* CreateContext() final; 31 | 32 | void OperatorEnd(ShardFileMap* out_files) final; 33 | 34 | // For GCS, if glob ends with "**", expands it recursively. 35 | void ExpandGlob(const std::string& glob, ExpandCb cb) final; 36 | 37 | // Read file and fill queue. This function must be fiber-friendly. 38 | size_t ProcessInputFile(const std::string& filename, pb::WireFormat::Type type, 39 | RawSinkCb cb) final; 40 | 41 | void SaveFile(absl::string_view fn, absl::string_view data); 42 | 43 | void Stop(); 44 | 45 | private: 46 | struct Impl; 47 | std::unique_ptr impl_; 48 | }; 49 | 50 | } // namespace mr3 51 | -------------------------------------------------------------------------------- /mr/mapper_executor.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include "mr/operator_executor.h" 10 | #include "util/fibers/simple_channel.h" 11 | 12 | namespace mr3 { 13 | 14 | class MapperExecutor : public OperatorExecutor { 15 | struct FileInput { 16 | const pb::Input* input; 17 | size_t spec_index; 18 | size_t file_size; 19 | ::std::string file_name; 20 | }; 21 | using FileNameQueue = ::boost::fibers::buffered_channel; 22 | 23 | struct Record { 24 | enum Operand { UNDEFINED, BINARY_FORMAT, TEXT_FORMAT, METADATA, RECORD} op = UNDEFINED; 25 | 26 | // either file spec or pair. 27 | absl::variant> payload; 28 | 29 | Record() = default; 30 | 31 | Record(Operand op2, size_t pos, ::std::string val) 32 | : op(op2), payload(::std::pair{pos, ::std::move(val)}) {} 33 | 34 | Record(Operand op2, const pb::Input::FileSpec* fspec) 35 | : op(op2), payload(fspec) {} 36 | }; 37 | 38 | using RecordQueue = util::fibers_ext::SimpleChannel; 39 | 40 | public: 41 | MapperExecutor(util::IoContextPool* pool, Runner* runner); 42 | ~MapperExecutor(); 43 | 44 | void Run(const std::vector& inputs, detail::TableBase* ss, 45 | ShardFileMap* out_files) final; 46 | 47 | // Stops the executor in the middle. 48 | void Stop() final; 49 | 50 | private: 51 | void InitInternal() final; 52 | 53 | void PushInput(const InputBase*); 54 | 55 | // Input managing fiber that reads files from disk and pumps data into record_q. 56 | // One per IO thread. 57 | void IOReadFiber(detail::TableBase* tb); 58 | 59 | // index - io thread index. 60 | void SetupPerIoThread(unsigned index, detail::TableBase* tb); 61 | 62 | static void MapFiber(RecordQueue* record_q, detail::TableBase* tb); 63 | 64 | std::unique_ptr file_name_q_; 65 | }; 66 | 67 | } // namespace mr3 68 | -------------------------------------------------------------------------------- /mr/mr3.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | syntax = "proto2"; 5 | 6 | package mr3.pb; 7 | 8 | message WireFormat { 9 | enum Type { 10 | LST = 2; 11 | TXT = 3; 12 | } 13 | required Type type = 1; 14 | } 15 | 16 | message ShardSpec { 17 | enum Type { 18 | MODN = 1; 19 | USER_DEFINED = 2; 20 | } 21 | 22 | required Type type = 1; 23 | optional uint32 modn = 2; 24 | 25 | // An approximate limit on raw shard size before file format transformations or compressions. 26 | // Can be passed slightly due to internal buffering in the system. In megabytes. 27 | optional uint32 max_raw_size_mb = 5; 28 | } 29 | 30 | message Input { 31 | required string name = 1; 32 | 33 | required WireFormat format = 2; 34 | 35 | message FileSpec { 36 | required string url_glob = 1; 37 | 38 | oneof shard_id_ref { 39 | string custom_shard_id = 2; 40 | uint32 shard_id = 3; 41 | } 42 | 43 | oneof metadata { 44 | int64 i64val = 4; 45 | string strval = 5; 46 | } 47 | }; 48 | 49 | // In case of sharded input, each file_spec corresponds to a shard. 50 | repeated FileSpec file_spec = 4; 51 | optional uint32 skip_header = 5; 52 | } 53 | 54 | message Output { 55 | required string name = 1; 56 | required WireFormat format = 2; 57 | 58 | enum CompressType { 59 | NONE = 0; 60 | GZIP = 1; 61 | ZSTD = 2; 62 | } 63 | 64 | message Compress { 65 | required CompressType type = 1; 66 | optional int32 level = 2 [default = 1]; 67 | } 68 | 69 | optional Compress compress = 3; 70 | optional ShardSpec shard_spec = 4; 71 | 72 | optional string type_name = 5; // The type name of the record serialized, when applicable. 73 | } 74 | 75 | 76 | // Can be mapper or joiner. 77 | message Operator { 78 | repeated string input_name = 1; // corresponds to the name in Input.name. 79 | 80 | optional string op_name = 2; 81 | 82 | required Output output = 3; 83 | 84 | enum Type { 85 | MAP = 1; 86 | GROUP = 2; 87 | } 88 | optional Type type = 4; 89 | } 90 | -------------------------------------------------------------------------------- /mr/mr_main.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "mr/mr_main.h" 6 | 7 | #include "base/init.h" 8 | 9 | #include "file/file_util.h" 10 | 11 | #include "mr/local_runner.h" 12 | #include "util/asio/accept_server.h" 13 | #include "util/asio/io_context_pool.h" 14 | #include "util/sentry/sentry.h" 15 | 16 | namespace mr3 { 17 | 18 | DEFINE_int32(http_port, 8080, "Port number."); 19 | 20 | using namespace util; 21 | 22 | PipelineMain::PipelineMain() { 23 | Init(); 24 | } 25 | 26 | PipelineMain::PipelineMain(int* argc, char*** argv) : guard_(new MainInitGuard{argc, argv}) { 27 | Init(); 28 | } 29 | 30 | void PipelineMain::ResetPipeline() { 31 | pipeline_.reset(new Pipeline(pool_.get())); 32 | runner_.reset(); 33 | } 34 | 35 | void PipelineMain::Init() { 36 | pool_.reset(new IoContextPool); 37 | pool_->Run(); 38 | util::EnableSentry(&pool_->GetNextContext()); 39 | ResetPipeline(); 40 | 41 | acc_server_.reset(new AcceptServer(pool_.get())); 42 | if (FLAGS_http_port >= 0) { 43 | uint16_t port = acc_server_->AddListener(FLAGS_http_port, &http_listener_); 44 | LOG(INFO) << "Started http server on port " << port; 45 | } 46 | acc_server_->Run(); 47 | } 48 | 49 | PipelineMain::~PipelineMain() { 50 | acc_server_->Stop(true); 51 | pool_->Stop(); 52 | } 53 | 54 | LocalRunner* PipelineMain::StartLocalRunner(const std::string& root_dir, bool stop_on_break) { 55 | CHECK(!runner_); 56 | runner_.reset(new LocalRunner(pool_.get(), file_util::ExpandPath(root_dir))); 57 | if (stop_on_break) { 58 | acc_server_->TriggerOnBreakSignal([this] { 59 | pipeline_->Stop(); 60 | runner_->Stop(); 61 | }); 62 | } 63 | return runner_.get(); 64 | } 65 | 66 | } // namespace mr3 67 | -------------------------------------------------------------------------------- /mr/mr_main.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "mr/pipeline.h" 7 | #include "util/http/http_conn_handler.h" 8 | 9 | class MainInitGuard; 10 | 11 | namespace util { 12 | class AcceptServer; 13 | } // util 14 | 15 | namespace mr3 { 16 | 17 | class LocalRunner; 18 | 19 | class PipelineMain { 20 | public: 21 | //! This sets up MR pipeline object and all its dependencies like IO pool. 22 | PipelineMain(); 23 | 24 | //! This c'tor runs MainInitGuard as part of its initialization flow and then 25 | // everything that PipelineMain() does. 26 | PipelineMain(int* argc, char*** argv); 27 | 28 | ~PipelineMain(); 29 | 30 | util::IoContextPool* pool() { return pool_.get(); } 31 | Pipeline* pipeline() { return pipeline_.get(); } 32 | void ResetPipeline(); 33 | util::AcceptServer* accept_server() { return acc_server_.get(); } 34 | 35 | LocalRunner* StartLocalRunner(const std::string& root_dir, bool stop_on_break = true); 36 | 37 | private: 38 | void Init(); 39 | 40 | std::unique_ptr guard_; // Must be first to be destructed last. 41 | 42 | std::unique_ptr pool_; 43 | std::unique_ptr pipeline_; 44 | std::unique_ptr acc_server_; 45 | util::http::Listener<> http_listener_; 46 | std::unique_ptr runner_; 47 | }; 48 | 49 | } // namespace mr3 50 | -------------------------------------------------------------------------------- /mr/mr_pb.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/logging.h" 5 | #include "mr/mr_pb.h" 6 | #include "util/pb2json.h" 7 | 8 | namespace mr3 { 9 | 10 | std::string PB_Serializer::To(bool is_binary, const Message* msg) { 11 | if (is_binary) 12 | return msg->SerializeAsString(); 13 | return util::Pb2Json(*msg); 14 | } 15 | 16 | bool PB_Serializer::From(bool is_binary, std::string tmp, Message* res) { 17 | if (is_binary) { 18 | return res->ParseFromString(tmp); 19 | } 20 | 21 | util::Status status = util::Json2Pb(std::move(tmp), res); 22 | DVLOG(1) << "Status: " << status; 23 | 24 | return status.ok(); 25 | } 26 | 27 | } // namespace mr3 28 | -------------------------------------------------------------------------------- /mr/mr_pb.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #include "mr/do_context.h" 10 | 11 | namespace mr3 { 12 | 13 | struct PB_Serializer { 14 | using Message = google::protobuf::Message; 15 | 16 | static std::string To(bool is_binary, const Message* msg); 17 | 18 | // Need std::string on stack because of json2pb which requires mutable string for insitu parsing. 19 | static bool From(bool is_binary, std::string tmp, Message* res); 20 | }; 21 | 22 | template 23 | class RecordTraits::value>> 24 | : public PB_Serializer { 25 | public: 26 | static std::string Serialize(bool is_binary, const PB& doc) { 27 | return PB_Serializer::To(is_binary, &doc); 28 | } 29 | 30 | static bool Parse(bool is_binary, std::string tmp, PB* res) { 31 | return PB_Serializer::From(is_binary, std::move(tmp), res); 32 | } 33 | 34 | static std::string TypeName() { 35 | PB msg; 36 | return msg.GetTypeName(); 37 | } 38 | }; 39 | } // namespace mr3 40 | -------------------------------------------------------------------------------- /mr/mr_types.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include "absl/strings/string_view.h" 10 | #include "absl/types/variant.h" 11 | 12 | namespace mr3 { 13 | 14 | template class DoContext; 15 | 16 | template 17 | using EmitMemberFn = void (Class::*)(FromType, DoContext*); 18 | 19 | using RawRecord = ::std::string; 20 | 21 | typedef std::function RawSinkCb; 22 | 23 | template 24 | using RawSinkMethodFactory = std::function* context)>; 25 | 26 | struct ShardId : public absl::variant { 27 | using Parent = absl::variant; 28 | 29 | using Parent::Parent; 30 | 31 | ShardId() = default; 32 | 33 | std::string ToString(absl::string_view basename) const; 34 | 35 | bool is_defined() const { return !absl::holds_alternative(*this); } 36 | }; 37 | 38 | using MetricMap = std::map; 39 | 40 | } // namespace mr3 41 | 42 | std::ostream& operator<<(std::ostream& os, const mr3::ShardId& sid); 43 | 44 | namespace std { 45 | 46 | template <> struct hash { 47 | size_t operator()(const mr3::ShardId& sid) const { return hash{}(sid); } 48 | }; 49 | 50 | } // namespace std 51 | -------------------------------------------------------------------------------- /mr/runner.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/container/flat_hash_map.h" 7 | #include "absl/strings/string_view.h" 8 | #include "mr/mr3.pb.h" 9 | #include "mr/mr_types.h" 10 | 11 | namespace mr3 { 12 | 13 | // value.second (string) - is a file glob that corresponds to 1 or more files comprising the shard. 14 | // i.e can be "shard-0000-*.txt.gz" but can be a single file as well. 15 | // To get the exact list, call ExpandGlob() on each value. 16 | using ShardFileMap = absl::flat_hash_map; 17 | 18 | class RawContext; 19 | 20 | class Runner { 21 | public: 22 | virtual ~Runner(); 23 | 24 | virtual void Init() = 0; 25 | 26 | virtual void Shutdown() = 0; 27 | 28 | // It's guaranteed that op will live until OperatorEnd is called. 29 | virtual void OperatorStart(const pb::Operator* op) = 0; 30 | 31 | // Must be thread-safe. Called from multiple threads in operator_executors. 32 | virtual RawContext* CreateContext() = 0; 33 | 34 | virtual void OperatorEnd(ShardFileMap* out_files) = 0; 35 | 36 | using ExpandCb = std::function; 37 | 38 | virtual void ExpandGlob(const std::string& glob, ExpandCb cb) = 0; 39 | 40 | // Read file and fill queue. This function must be fiber-friendly. 41 | // Returns number of records processed. 42 | virtual size_t ProcessInputFile(const std::string& filename, pb::WireFormat::Type type, 43 | RawSinkCb cb) = 0; 44 | 45 | virtual void SaveFile(absl::string_view fn, absl::string_view data) = 0; 46 | }; 47 | 48 | } // namespace mr3 49 | -------------------------------------------------------------------------------- /scripts/run_asio.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | run_asio() { 4 | ./asio_fibers --logtostderr --vmodule=rpc_connection=1,connection_handler=1 & 5 | SRV_PID=$! 6 | 7 | ./asio_fibers --connect=localhost --count 100000 --num_connections=1 & 8 | CL_PID=$! 9 | echo "Run client/server, iteration $1" 10 | 11 | sleep 0.5 12 | 13 | kill $SRV_PID 14 | wait $CL_PID 15 | wait $SRV_PID 16 | } 17 | 18 | COUNTER=0 19 | NUM_ITERS=${1:-10} 20 | echo $NUM_ITERS 21 | 22 | while [ $COUNTER -lt $NUM_ITERS ]; do 23 | run_asio $COUNTER 24 | let COUNTER=COUNTER+1 25 | done 26 | -------------------------------------------------------------------------------- /scripts/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | usage() { 4 | echo "Usage: $0 -l --name -t " 5 | } 6 | 7 | 8 | while [[ $# -gt 1 ]] 9 | do 10 | key="$1" 11 | TIMEOUT=10 12 | LOG_DIR=`pwd` 13 | 14 | case $key in 15 | -l) 16 | LOG_DIR="$2" 17 | shift # past argument 18 | ;; 19 | --name) 20 | TEST_NAME="$2" 21 | shift # past argument 22 | ;; 23 | -t) 24 | TIMEOUT="$2" 25 | shift # past argument 26 | ;; 27 | -h|--help) 28 | usage; shift; exit 0 29 | ;; 30 | *) usage; exit 1 31 | ;; 32 | esac 33 | shift # past argument or value 34 | done 35 | 36 | if [[ -z $TEST_NAME ]]; then 37 | usage; exit 1; 38 | fi 39 | 40 | set -o pipefail 41 | 42 | gdbbt() { 43 | tmp=$(tempfile) 44 | echo thread apply all bt >$tmp 45 | sudo gdb -batch -nx -q -x $tmp -p $1 46 | kill -9 $1 47 | rm -f $tmp 48 | } 49 | 50 | run_timeout() { 51 | timeout=$1 52 | (./"${TEST_NAME}" --log_dir="${LOG_DIR}" --gtest_output=xml:"${LOG_DIR}"/"${TEST_NAME}".xml \ 53 | --gtest_color=yes | tee ${TEST_NAME}.test_result.log) & pid=$! 54 | 55 | while kill -0 $pid &>/dev/null && [ "${timeout}" -ne 0 ]; do 56 | sleep 1 57 | timeout=$((timeout - 1)) 58 | done 59 | 60 | if kill -0 $pid &>/dev/null; then 61 | gdbbt $pid 62 | exit 124 63 | fi 64 | 65 | wait $pid 66 | exit $? 67 | } 68 | 69 | run_timeout "${TIMEOUT}" 70 | -------------------------------------------------------------------------------- /strings/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(strings escaping.cc human_readable.cc 2 | stringpiece.cc range.cc split.cc strcat.cc stringprintf.cc numbers.cc 3 | unique_strings.cc) 4 | target_link_libraries(strings base absl_strings) 5 | add_dependencies(strings sparsehash_project) 6 | set_property(TARGET strings APPEND PROPERTY COMPILE_OPTIONS "-Wno-implicit-fallthrough") 7 | 8 | add_library(strpmr strpmr.cc) 9 | target_link_libraries(strpmr TRDP::pmr strings) 10 | 11 | 12 | cxx_test(range_test strings LABELS CI) 13 | cxx_test(unique_strings_test strings LABELS CI) 14 | cxx_test(strcat_test strings LABELS CI) 15 | cxx_test(strpmr_test strings LABELS CI) 16 | cxx_test(numbers_test strings LABELS CI) 17 | -------------------------------------------------------------------------------- /strings/charset.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 Google Inc. All Rights Reserved. 2 | 3 | #include "strings/charset.h" 4 | 5 | #include 6 | #include "strings/stringpiece.h" 7 | 8 | namespace strings { 9 | 10 | CharSet::CharSet() { 11 | memset(bits_, 0, sizeof(bits_)); 12 | } 13 | 14 | CharSet::CharSet(StringPiece characters) { 15 | memset(bits_, 0, sizeof(bits_)); 16 | for (size_t i = 0; i < characters.size(); ++i) { 17 | Add(characters[i]); 18 | } 19 | } 20 | 21 | } // namespace strings 22 | -------------------------------------------------------------------------------- /strings/escaping.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "strings/escaping.h" 5 | 6 | namespace strings { 7 | 8 | using namespace std; 9 | 10 | template bool _in(const T& t, const T (&arr) [N]) { 11 | return std::find(arr, arr + N, t) != arr + N; 12 | } 13 | 14 | inline bool IsValidUrlChar(char ch) { 15 | static constexpr char kArr[] = "-_.!~*'()"; 16 | return absl::ascii_isalnum(ch) || _in(ch, kArr); 17 | } 18 | 19 | static size_t InternalUrlEncode(absl::string_view src, char* dest) { 20 | static const char digits[] = "0123456789ABCDEF"; 21 | 22 | char* start = dest; 23 | for (char ch_c : src) { 24 | unsigned char ch = static_cast(ch_c); 25 | if (IsValidUrlChar(ch)) { 26 | *dest++ = ch_c; 27 | } else { 28 | *dest++ = '%'; 29 | *dest++ = digits[(ch >> 4) & 0x0F]; 30 | *dest++ = digits[ch & 0x0F]; 31 | } 32 | } 33 | *dest = 0; 34 | 35 | return static_cast(dest - start); 36 | } 37 | 38 | void AppendEncodedUrl(const absl::string_view src, string* dest) { 39 | size_t sz = dest->size(); 40 | dest->resize(dest->size() + src.size() * 3 + 1); 41 | char* next = &dest->front() + sz; 42 | size_t written = InternalUrlEncode(src, next); 43 | dest->resize(sz + written); 44 | } 45 | 46 | } // namespace strings 47 | -------------------------------------------------------------------------------- /strings/escaping.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/strings/escaping.h" 7 | 8 | namespace strings { 9 | 10 | void AppendEncodedUrl(const absl::string_view src, std::string* dest); 11 | 12 | } // namespace strings 13 | -------------------------------------------------------------------------------- /strings/hash.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef _STRINGS_HASH_H 5 | #define _STRINGS_HASH_H 6 | 7 | #include 8 | #include "base/hash.h" 9 | #include "strings/stringpiece.h" 10 | 11 | namespace std { 12 | 13 | #if __cplusplus < 201703L 14 | template<> struct hash { 15 | size_t operator()(StringPiece slice) const { 16 | return base::MurmurHash3_x86_32( 17 | reinterpret_cast(slice.data()), 18 | slice.size() * sizeof(typename StringPiece::value_type), 16785407UL); 19 | } 20 | }; 21 | #endif 22 | 23 | template struct hash> { 24 | size_t operator()(strings::Range slice) const { 25 | return base::MurmurHash3_x86_32( 26 | reinterpret_cast(slice.data()), 27 | slice.size() * sizeof(typename strings::Range::value_type), 16785407UL); 28 | } 29 | }; 30 | 31 | } // namespace std 32 | 33 | 34 | #endif // _STRINGS_HASH_H 35 | 36 | -------------------------------------------------------------------------------- /strings/join.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/strings/str_join.h" 7 | -------------------------------------------------------------------------------- /strings/numbers_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All right reserved. 2 | // Author: Ori Brostovski (ori@ubimo.com) 3 | 4 | #include "strings/numbers.h" 5 | #include 6 | 7 | class NumberTest : public testing::Test {}; 8 | 9 | TEST_F(NumberTest, u64tostr) { 10 | char buf[65]; 11 | char buf2[2] = {'x', 'x'}; 12 | 13 | u64tostr(8765, sizeof(buf2), buf2, 10); 14 | EXPECT_EQ('8', buf2[0]); 15 | EXPECT_EQ('\0', buf2[1]); 16 | 17 | u64tostr(0, sizeof(buf), buf, 36); 18 | EXPECT_EQ(std::string("0"), buf); 19 | 20 | u64tostr((('O' - 'A' + 10) * (36 * 36) + 21 | ('R' - 'A' + 10) * 36 + 22 | ('I' - 'A' + 10)), 23 | sizeof(buf), buf, 36); 24 | EXPECT_EQ(std::string("ORI"), buf); 25 | 26 | u64tostr(0xFFFFFFFFFFFFFFFF, sizeof(buf), buf, 2); 27 | EXPECT_EQ(std::string(64, '1'), buf); 28 | 29 | 30 | } 31 | -------------------------------------------------------------------------------- /strings/range.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Facebook, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | // 18 | // Modified by Roman Gershman (romange@gmail.com) 19 | 20 | #include "strings/range.h" 21 | 22 | #include "strings/charset.h" 23 | 24 | namespace strings { 25 | namespace detail { 26 | 27 | size_t qfind_first_byte_of_charset(const StringPiece haystack, 28 | const StringPiece needles) { 29 | CharSet s(needles); 30 | 31 | for (size_t index = 0; index < haystack.size(); ++index) { 32 | if (s.Test(haystack[index])) { 33 | return index; 34 | } 35 | } 36 | return std::string::npos; 37 | } 38 | 39 | 40 | size_t qfind_first_not_of_charset(const StringPiece haystack, 41 | const StringPiece needles) { 42 | CharSet s(needles); 43 | 44 | for (size_t index = 0; index < haystack.size(); ++index) { 45 | if (!s.Test(haystack[index])) { 46 | return index; 47 | } 48 | } 49 | return std::string::npos; 50 | } 51 | 52 | } // namespace detail 53 | } // namespace strings 54 | -------------------------------------------------------------------------------- /strings/range_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "base/integral_types.h" 5 | #include "base/logging.h" 6 | 7 | #include "strings/stringpiece.h" 8 | #include "strings/split.h" 9 | 10 | using namespace std; 11 | 12 | namespace strings { 13 | 14 | class StringPieceTest : public testing::Test { 15 | }; 16 | 17 | TEST_F(StringPieceTest, Length) { 18 | StringPiece pc("Fooo"); 19 | EXPECT_EQ(4, pc.size()); 20 | 21 | std::array array{{1, 3, 4, 8}}; 22 | strings::ByteRange slice(array); 23 | EXPECT_EQ(4, slice.size()); 24 | EXPECT_EQ(3, slice[1]); 25 | EXPECT_EQ(3, pc.rfind('o')); 26 | } 27 | 28 | TEST_F(StringPieceTest, Range) { 29 | std::array arr{{1, 2, 3, 4}}; 30 | std::vector v(5); 31 | Range r1(arr); 32 | EXPECT_EQ(arr.size(), r1.size()); 33 | 34 | Range r2(v); 35 | EXPECT_EQ(Range(v), r2); 36 | } 37 | 38 | } // namespace strings 39 | -------------------------------------------------------------------------------- /strings/split.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/strings/str_split.h" 7 | 8 | 9 | void SplitCSVLineWithDelimiter(char* line, char delimiter, 10 | std::vector* cols); 11 | -------------------------------------------------------------------------------- /strings/strcat.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 and onwards Google Inc. All rights reserved. 2 | 3 | #include "strings/strcat.h" 4 | 5 | #include "base/logging.h" 6 | 7 | char* StrAppend(char* dest, unsigned n, std::initializer_list list) { 8 | for (const auto& val : list) { 9 | if (val.size() >= n) 10 | break; 11 | memcpy(dest, val.data(), val.size()); 12 | dest += val.size(); 13 | n -= val.size(); 14 | } 15 | *dest = '\0'; 16 | return dest; 17 | } 18 | -------------------------------------------------------------------------------- /strings/strcat.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/strings/str_cat.h" 7 | 8 | /** 9 | * @brief Appends a list of string arguments to the destination buffer. Does not cross the 10 | * specified max capacity. After the append operation is completed sets the null character 11 | * at the end of the string and returns the pointer to it. 12 | * 13 | * @param dest - the destination buffer 14 | * @param n - its maximum capacity. 15 | * @param list - list of string arguments. 16 | * @return char* - a pointer to the end of the string (\0). 17 | */ 18 | char* StrAppend(char* dest, unsigned n, std::initializer_list list); 19 | 20 | template char* StrAppend(char* dest, unsigned n, const absl::AlphaNum& a1, 21 | const Args... args) { 22 | return StrAppend(dest, n, {a1.Piece(), static_cast(args).Piece()...}); 23 | } 24 | -------------------------------------------------------------------------------- /strings/strcat_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "strings/strcat.h" 5 | #include 6 | 7 | class StrCatTest : public testing::Test { 8 | }; 9 | 10 | TEST_F(StrCatTest, Base) { 11 | char buf[200] = {0}; 12 | StrAppend(buf, sizeof(buf), "a", "b", 5, "cdef"); 13 | EXPECT_STREQ("ab5cdef", buf); 14 | memset(buf, 'a', 200); 15 | StrAppend(buf, sizeof(buf), "foo", 5, "bar"); 16 | EXPECT_STREQ("foo5bar", buf); 17 | } 18 | -------------------------------------------------------------------------------- /strings/stringpiece.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "strings/stringpiece.h" -------------------------------------------------------------------------------- /strings/stringpiece.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "strings/range.h" 7 | #include "absl/strings/string_view.h" 8 | 9 | typedef absl::string_view StringPiece; 10 | 11 | namespace strings { 12 | 13 | inline const char* charptr(const unsigned char* ptr) { 14 | return reinterpret_cast(ptr); 15 | } 16 | 17 | inline char* charptr(unsigned char* ptr) { 18 | return reinterpret_cast(ptr); 19 | } 20 | 21 | inline const uint8_t* u8ptr(const char* ptr) { 22 | return reinterpret_cast(ptr); 23 | } 24 | 25 | inline const uint8_t* u8ptr(StringPiece s) { 26 | return reinterpret_cast(s.data()); 27 | } 28 | 29 | 30 | inline uint8_t* u8ptr(char* ptr) { 31 | return reinterpret_cast(ptr); 32 | } 33 | 34 | inline ByteRange ToByteRange(StringPiece s) { 35 | return ByteRange(reinterpret_cast(s.data()), s.size()); 36 | } 37 | 38 | inline MutableByteRange AsMutableByteRange(std::string& s) { 39 | return MutableByteRange(reinterpret_cast(&s.front()), s.size()); 40 | } 41 | 42 | inline std::string AsString(StringPiece piece) { return std::string(piece.data(), piece.size()); } 43 | 44 | inline StringPiece FromBuf(const uint8_t* ptr, size_t len) { 45 | return StringPiece(reinterpret_cast(ptr), len); 46 | } 47 | 48 | } // namespace strings 49 | 50 | -------------------------------------------------------------------------------- /strings/stringprintf.h: -------------------------------------------------------------------------------- 1 | // Copyright 2002 and onwards Google Inc. 2 | // 3 | // Printf variants that place their output in a C++ string. 4 | // 5 | // Usage: 6 | // string result = StringPrintf("%d %s\n", 10, "hello"); 7 | // SStringPrintf(&result, "%d %s\n", 10, "hello"); 8 | // StringAppendF(&result, "%d %s\n", 20, "there"); 9 | 10 | #ifndef _BASE_STRINGPRINTF_H 11 | #define _BASE_STRINGPRINTF_H 12 | 13 | #include 14 | #include 15 | using std::string; 16 | #include 17 | using std::vector; 18 | 19 | #include "base/port.h" 20 | 21 | // Return a C++ string 22 | extern string StringPrintf(const char* format, ...) 23 | // Tell the compiler to do printf format string checking. 24 | PRINTF_ATTRIBUTE(1,2); 25 | 26 | // Store result into a supplied string and return it 27 | extern const string& SStringPrintf(string* dst, const char* format, ...) 28 | // Tell the compiler to do printf format string checking. 29 | PRINTF_ATTRIBUTE(2,3); 30 | 31 | // Append result to a supplied string 32 | extern void StringAppendF(string* dst, const char* format, ...) 33 | // Tell the compiler to do printf format string checking. 34 | PRINTF_ATTRIBUTE(2,3); 35 | 36 | // Lower-level routine that takes a va_list and appends to a specified 37 | // string. All other routines are just convenience wrappers around it. 38 | extern void StringAppendV(string* dst, const char* format, va_list ap); 39 | 40 | // The max arguments supported by StringPrintfVector 41 | extern const uint32 kStringPrintfVectorMaxArgs; 42 | 43 | // You can use this version when all your arguments are strings, but 44 | // you don't know how many arguments you'll have at compile time. 45 | // StringPrintfVector will LOG(FATAL) if v.size() > kStringPrintfVectorMaxArgs 46 | extern string StringPrintfVector(const char* format, const vector& v); 47 | 48 | #endif /* _BASE_STRINGPRINTF_H */ 49 | -------------------------------------------------------------------------------- /strings/strip.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/strings/strip.h" 7 | -------------------------------------------------------------------------------- /strings/strpmr.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2016, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "strings/strpmr.h" 6 | #include "base/logging.h" 7 | 8 | namespace strings { 9 | 10 | StringPiece DeepCopy(StringPiece src, pmr::memory_resource* mr) { 11 | DCHECK(mr); 12 | if (src.empty()) 13 | return src; 14 | char* dest = (char*)mr->allocate(src.size()); 15 | memcpy(dest, src.data(), src.size()); 16 | return StringPiece(dest, src.size()); 17 | } 18 | 19 | std::pair StringPieceSet::Insert(StringPiece source) { 20 | assert(!source.empty()); 21 | 22 | auto it = db_.find(source); 23 | if (it != db_.end()) 24 | return std::make_pair(*it, false); 25 | char* str = (char*)arena_.allocate(source.size()); 26 | memcpy(str, source.data(), source.size()); 27 | StringPiece val(str, source.size()); 28 | db_.insert(val); 29 | return std::make_pair(val, true); 30 | } 31 | 32 | } // namespace strings 33 | -------------------------------------------------------------------------------- /strings/strpmr_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "strings/strpmr.h" 6 | 7 | #include 8 | 9 | #include "base/gtest.h" 10 | #include "base/integral_types.h" 11 | #include "base/macros.h" 12 | 13 | using testing::ContainerEq; 14 | 15 | namespace strings { 16 | 17 | 18 | class StrPmrTest : public testing::Test { 19 | public: 20 | StrPmrTest() { 21 | } 22 | protected: 23 | }; 24 | 25 | 26 | TEST_F(StrPmrTest, Basic) { 27 | ArraysMap am(pmr::get_default_resource()); 28 | 29 | std::vector v(3); 30 | 31 | Range r1(v.data(), v.size()); 32 | 33 | am.emplace(r1, 5); 34 | auto it = am.begin(); 35 | EXPECT_EQ(it->first, r1); 36 | } 37 | 38 | 39 | } // namespace strings 40 | 41 | -------------------------------------------------------------------------------- /strings/strtoint.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 Google Inc. All Rights Reserved. 2 | // 3 | // Architecture-neutral plug compatible replacements for strtol() friends. 4 | // See strtoint.h for details on how to use this component. 5 | // 6 | 7 | #include 8 | #include "base/port.h" 9 | #include "strings/strtoint.h" 10 | 11 | // Replacement strto[u]l functions that have identical overflow and underflow 12 | // characteristics for both ILP-32 and LP-64 platforms, including errno 13 | // preservation for error-free calls. 14 | int32 strto32_adapter(const char *nptr, char **endptr, int base) { 15 | const int saved_errno = errno; 16 | errno = 0; 17 | const long result = strtol(nptr, endptr, base); 18 | if (errno == ERANGE && result == LONG_MIN) { 19 | return kint32min; 20 | } else if (errno == ERANGE && result == LONG_MAX) { 21 | return kint32max; 22 | } else if (errno == 0 && result < kint32min) { 23 | errno = ERANGE; 24 | return kint32min; 25 | } else if (errno == 0 && result > kint32max) { 26 | errno = ERANGE; 27 | return kint32max; 28 | } 29 | if (errno == 0) 30 | errno = saved_errno; 31 | return static_cast(result); 32 | } 33 | 34 | uint32 strtou32_adapter(const char *nptr, char **endptr, int base) { 35 | const int saved_errno = errno; 36 | errno = 0; 37 | const unsigned long result = strtoul(nptr, endptr, base); 38 | if (errno == ERANGE && result == ULONG_MAX) { 39 | return kuint32max; 40 | } else if (errno == 0 && result > kuint32max) { 41 | errno = ERANGE; 42 | return kuint32max; 43 | } 44 | if (errno == 0) 45 | errno = saved_errno; 46 | return static_cast(result); 47 | } 48 | -------------------------------------------------------------------------------- /strings/unique_strings.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014, .com . All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "strings/unique_strings.h" 5 | 6 | std::pair UniqueStrings::Insert(StringPiece source) { 7 | auto it = db_.find(source); 8 | if (it != db_.end()) 9 | return std::make_pair(*it, false); 10 | if (source.empty()) { 11 | auto res = db_.insert(StringPiece()); 12 | return std::make_pair(*res.first, res.second); 13 | } 14 | char* str = arena_.Allocate(source.size()); 15 | memcpy(str, source.data(), source.size()); 16 | StringPiece val(str, source.size()); 17 | db_.insert(val); 18 | return std::make_pair(val, true); 19 | } -------------------------------------------------------------------------------- /strings/unique_strings_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "strings/unique_strings.h" 5 | #include 6 | 7 | using std::string; 8 | 9 | class UniqueStringsTest : public testing::Test { 10 | }; 11 | 12 | TEST_F(UniqueStringsTest, Base) { 13 | UniqueStrings unique; 14 | StringPiece foo1 = unique.Get("foo"); 15 | string str("foo"); 16 | StringPiece foo2 = unique.Get(str); 17 | 18 | EXPECT_EQ(foo1, foo2); 19 | EXPECT_EQ(foo1.data(), foo2.data()); 20 | StringPiece bar = unique.Get("bar"); 21 | EXPECT_NE(bar, foo2); 22 | } 23 | 24 | TEST_F(UniqueStringsTest, DenseMap) { 25 | StringPieceDenseMap unique; 26 | unique.set_empty_key(StringPiece()); 27 | unique["r1"] = 1; 28 | unique["r2"] = 2; 29 | unique.insert(StringPieceDenseMap::value_type("r3", 3)); 30 | auto it = unique.find("r1"); 31 | EXPECT_TRUE(it != unique.end()); 32 | EXPECT_EQ(1, it->second); 33 | EXPECT_EQ("r1", it->first); 34 | 35 | unique["r1"]++; 36 | it = unique.find("r1"); 37 | EXPECT_TRUE(it != unique.end()); 38 | EXPECT_EQ(2, it->second); 39 | EXPECT_EQ(2, unique["r1"]); 40 | 41 | EXPECT_EQ(3, unique["r3"]); 42 | } 43 | -------------------------------------------------------------------------------- /strings/utf8/utfdef.h: -------------------------------------------------------------------------------- 1 | #define uchar _utfuchar 2 | #define ushort _utfushort 3 | #define uint _utfuint 4 | #define ulong _utfulong 5 | #define vlong _utfvlong 6 | #define uvlong _utfuvlong 7 | 8 | typedef unsigned char uchar; 9 | typedef unsigned short ushort; 10 | typedef unsigned int uint; 11 | typedef unsigned long ulong; 12 | 13 | #define nelem(x) (sizeof(x)/sizeof((x)[0])) 14 | #define nil ((void*)0) 15 | -------------------------------------------------------------------------------- /util/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cxx_proto_lib(status PY) 2 | 3 | add_library(status status.cc) 4 | cxx_link(status base status_proto) 5 | 6 | add_library(util zlib_source.cc bzip_source.cc 7 | sinksource.cc zstd_sinksource.cc) 8 | cxx_link(util strings status TRDP::lz4 TRDP::zstd bz2 TRDP::intel_z) 9 | 10 | add_library(pb2json pb2json.cc) 11 | cxx_link(pb2json strings status TRDP::protobuf TRDP::rapidjson absl_variant absl_str_format) 12 | add_dependencies(pb2json rapidjson_project) 13 | 14 | add_library(sp_task_pool sp_task_pool.cc) 15 | cxx_link(sp_task_pool base) 16 | 17 | cxx_test(sp_task_pool_test base sp_task_pool util LABELS CI) 18 | 19 | add_library(proc_stats proc_stats.cc spawn.cc) 20 | cxx_link(proc_stats strings) 21 | 22 | cxx_test(sinksource_test strings util LABELS CI) 23 | cxx_test(pb2json_test pb2json addressbook_proto LABELS CI) 24 | cxx_test(mimalloc_test TRDP::mimalloc absl_strings) 25 | 26 | add_subdirectory(asio) 27 | add_subdirectory(fibers) 28 | add_subdirectory(http) 29 | add_subdirectory(html) 30 | add_subdirectory(math) 31 | add_subdirectory(coding) 32 | add_subdirectory(plang) 33 | add_subdirectory(pprint) 34 | add_subdirectory(rpc) 35 | add_subdirectory(sentry) 36 | add_subdirectory(stats) 37 | add_subdirectory(aws) 38 | add_subdirectory(gce) 39 | add_subdirectory(uring) 40 | -------------------------------------------------------------------------------- /util/asio/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(asio_fiber_lib io_context.cc io_context_pool.cc error.cc 2 | connection_handler.cc yield.cc accept_server.cc periodic_task.cc 3 | glog_asio_sink.cc fiber_socket.cc prebuilt_asio.cc) 4 | cxx_link(asio_fiber_lib base stats_lib fibers_ext absl_optional) 5 | 6 | add_definitions(-DBOOST_ASIO_NO_DEPRECATED) 7 | 8 | cxx_test(periodic_task_test asio_fiber_lib LABELS CI) 9 | cxx_test(io_context_test asio_fiber_lib LABELS CI) 10 | cxx_test(fiber_socket_test http_test_lib LABELS CI) 11 | -------------------------------------------------------------------------------- /util/asio/error.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "util/asio/error.h" 6 | #include "absl/strings/str_cat.h" 7 | 8 | namespace util { 9 | 10 | using namespace boost; 11 | using namespace std; 12 | 13 | namespace { 14 | 15 | class gaia_error_category : public system::error_category { 16 | public: 17 | const char* name() const noexcept final { return "gaia.util"; } 18 | 19 | std::string message(int ev) const override { 20 | switch (static_cast(ev)) { 21 | case gaia_error::bad_header: 22 | return "bad header"; 23 | case gaia_error::invalid_version: 24 | return "invalid header version"; 25 | default: 26 | return absl::StrCat("gaia.util error(", ev, ")"); 27 | } 28 | } 29 | 30 | system::error_condition default_error_condition(int ev) const noexcept override { 31 | return system::error_condition{ev, *this}; 32 | } 33 | 34 | bool equivalent(int ev, system::error_condition const& condition) const noexcept override { 35 | return condition.value() == ev && &condition.category() == this; 36 | } 37 | 38 | bool equivalent(system::error_code const& error, int ev) const noexcept override { 39 | return error.value() == ev && &error.category() == this; 40 | } 41 | }; 42 | 43 | } // namespace 44 | 45 | system::error_code make_error_code(gaia_error ev) { 46 | static const gaia_error_category cat; 47 | return system::error_code{static_cast(ev), cat}; 48 | } 49 | 50 | } // namespace util 51 | -------------------------------------------------------------------------------- /util/asio/error.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | 8 | namespace util { 9 | 10 | enum class gaia_error { 11 | bad_header = 1, 12 | invalid_version = 2, 13 | }; 14 | 15 | ::boost::system::error_code 16 | make_error_code(gaia_error ev); 17 | 18 | } // namespace util 19 | 20 | namespace boost { 21 | namespace system { 22 | template<> 23 | struct is_error_code_enum<::util::gaia_error> 24 | { 25 | static bool const value = true; 26 | }; 27 | 28 | } // system 29 | } // boost 30 | -------------------------------------------------------------------------------- /util/asio/glog_asio_sink.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/asio/glog_asio_sink.h" 5 | #include 6 | 7 | namespace util { 8 | using namespace ::boost; 9 | using namespace ::std; 10 | 11 | using namespace fibers; 12 | 13 | GlogAsioSink::GlogAsioSink() : msg_q_(64) { 14 | } 15 | 16 | GlogAsioSink::~GlogAsioSink() noexcept { 17 | } 18 | 19 | void GlogAsioSink::Run() { 20 | google::AddLogSink(this); 21 | RAW_DLOG(INFO, "Started running"); 22 | run_started_.store(true, std::memory_order_seq_cst); 23 | ec_.notifyAll(); 24 | 25 | Item item; 26 | while (true) { 27 | channel_op_status st = msg_q_.pop(item); 28 | if (st == channel_op_status::closed) 29 | break; 30 | 31 | CHECK_EQ(channel_op_status::success, st); 32 | // RAW_VLOG(1, "HandleItem : %s:%d", item.base_filename, item.line); 33 | HandleItem(item); 34 | } 35 | 36 | LOG_IF(INFO, lost_messages_ > 0) << "GlogAsioSink lost " << lost_messages_ << " lost messages "; 37 | } 38 | 39 | void GlogAsioSink::Cancel() { 40 | google::RemoveLogSink(this); 41 | msg_q_.close(); 42 | } 43 | 44 | void GlogAsioSink::WaitTillRun() { 45 | ec_.await([this] { return run_started_.load(std::memory_order_acquire); }); 46 | } 47 | 48 | void GlogAsioSink::send(google::LogSeverity severity, const char* full_filename, 49 | const char* base_filename, int line, const struct ::tm* tm_time, 50 | const char* message, size_t message_len) { 51 | if (ShouldIgnore(severity, full_filename, line)) 52 | return; 53 | 54 | // string creation might have potential performance impact. 55 | channel_op_status st = msg_q_.push_wait_for( 56 | Item{full_filename, base_filename, severity, line, *tm_time, string{message, message_len}}, 57 | 100us); 58 | 59 | if (st != channel_op_status::success) { 60 | ++lost_messages_; 61 | } 62 | RAW_VLOG(1, "GlogAsioSink::SendExit : %d", int(st)); 63 | } 64 | 65 | void GlogAsioSink::WaitTillSent() { 66 | /* Noop to reduce send latency */ 67 | } 68 | 69 | } // namespace util 70 | -------------------------------------------------------------------------------- /util/asio/glog_asio_sink.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "util/asio/io_context.h" 11 | #include "util/fibers/event_count.h" 12 | 13 | namespace util { 14 | 15 | class GlogAsioSink : public IoContext::Cancellable, ::google::LogSink { 16 | public: 17 | GlogAsioSink(); 18 | ~GlogAsioSink() noexcept; 19 | 20 | void Run() override; 21 | void Cancel() override; 22 | 23 | void WaitTillRun(); 24 | 25 | protected: 26 | struct Item { 27 | const char* full_filename; 28 | const char* base_filename; 29 | google::LogSeverity severity; 30 | int line; 31 | struct ::tm tm_time; 32 | std::string message; // Can cause performance penalty. 33 | }; 34 | ::boost::fibers::buffered_channel msg_q_; 35 | 36 | unsigned lost_messages_ = 0; 37 | 38 | virtual bool ShouldIgnore(google::LogSeverity severity, const char* full_filename, int line) { 39 | return false; 40 | } 41 | 42 | // Is called from Run loop. Should not block the thread. 43 | virtual void HandleItem(const Item& item) = 0; 44 | 45 | private: 46 | //! Derived from LogSink. 47 | void send(google::LogSeverity severity, const char* full_filename, const char* base_filename, 48 | int line, const struct ::tm* tm_time, const char* message, size_t message_len) override; 49 | 50 | void WaitTillSent() override; 51 | 52 | std::atomic_bool run_started_{false}; 53 | fibers_ext::EventCount ec_; 54 | }; 55 | 56 | } // namespace util 57 | -------------------------------------------------------------------------------- /util/asio/prebuilt_asio.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #ifdef BOOST_ASIO_SEPARATE_COMPILATION 6 | #include 7 | #endif 8 | 9 | -------------------------------------------------------------------------------- /util/asio/yield.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "util/asio/yield.h" 6 | 7 | namespace util { 8 | namespace fibers_ext { 9 | 10 | /// canonical instance 11 | thread_local yield_t yield{}; 12 | 13 | } // namespace fibers 14 | } // namespace util 15 | -------------------------------------------------------------------------------- /util/asio/yield.h: -------------------------------------------------------------------------------- 1 | // Copyright 2003-2013 Christopher M. Kohlhoff 2 | // Copyright Oliver Kowalke, Nat Goodspeed 2015. 3 | // Distributed under the Boost Software License, Version 1.0. 4 | // (See accompanying file LICENSE_1_0.txt or copy at 5 | // http://www.boost.org/LICENSE_1_0.txt) 6 | 7 | #pragma once 8 | #include 9 | 10 | namespace util { 11 | namespace fibers_ext { 12 | 13 | //[fibers_asio_yield_t 14 | class yield_t { 15 | public: 16 | yield_t() = default; 17 | 18 | /** 19 | * @code 20 | * static yield_t yield; 21 | * boost::system::error_code myec; 22 | * func(yield[myec]); 23 | * @endcode 24 | * @c yield[myec] returns an instance of @c yield_t whose @c ec_ points 25 | * to @c myec. The expression @c yield[myec] "binds" @c myec to that 26 | * (anonymous) @c yield_t instance, instructing @c func() to store any 27 | * @c error_code it might produce into @c myec rather than throwing @c 28 | * boost::system::system_error. 29 | */ 30 | yield_t operator[]( boost::system::error_code & ec) const { 31 | yield_t tmp; 32 | tmp.ec_ = & ec; 33 | return tmp; 34 | } 35 | 36 | //private: 37 | // ptr to bound error_code instance if any 38 | boost::system::error_code * ec_{ nullptr }; 39 | }; 40 | 41 | /// canonical instance 42 | extern thread_local yield_t yield; 43 | //] 44 | 45 | } // namespace fibers_ext 46 | } // namespace util 47 | 48 | #include "detail/yield.hpp" 49 | -------------------------------------------------------------------------------- /util/aws/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(LibXml2) 2 | 3 | include_directories(${LIBXML2_INCLUDE_DIR}) 4 | 5 | add_library(aws_lib aws.cc s3.cc) 6 | cxx_link(aws_lib asio_fiber_lib file status http_common https_client_lib ${LIBXML2_LIBRARIES}) 7 | 8 | cxx_test(s3_test aws_lib LABELS CI) 9 | -------------------------------------------------------------------------------- /util/aws/aws.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "absl/strings/string_view.h" 14 | #include "util/status.h" 15 | 16 | namespace util { 17 | 18 | class AWS { 19 | public: 20 | AWS(const std::string& region_id, const std::string& service) 21 | : region_id_(region_id), service_(service) { 22 | } 23 | 24 | Status Init(); 25 | 26 | // TODO: we should remove domain argument in favor to subdomain (bucket). 27 | // and build the whole domain it from service and region 28 | // for example, ".s3.eu-west-1.amazonaws.com" 29 | // See: https://docs.aws.amazon.com/general/latest/gr/s3.html 30 | // 31 | void Sign(absl::string_view domain, absl::string_view body_hash256, 32 | ::boost::beast::http::header* header) const; 33 | 34 | void SignEmpty(absl::string_view domain, 35 | ::boost::beast::http::header* header) const { 36 | return Sign(domain, absl::string_view{kHashEmpty, 64}, header); 37 | } 38 | 39 | static ::boost::asio::ssl::context CheckedSslContext(); 40 | 41 | private: 42 | 43 | static const char kHashEmpty[]; 44 | 45 | std::string AuthHeader(absl::string_view method, absl::string_view headers, 46 | absl::string_view target, absl::string_view content_sha256, 47 | absl::string_view amz_date) const; 48 | 49 | std::string region_id_, service_, secret_, access_key_; 50 | 51 | mutable ::boost::fibers::mutex mu_; 52 | mutable std::string sign_key_; 53 | 54 | std::string credential_scope_; 55 | char date_str_[32]; 56 | }; 57 | 58 | namespace detail { 59 | 60 | void Sha256String(absl::string_view str, char out[65]); 61 | void Sha256String(const ::boost::beast::multi_buffer& mb, char out[65]); 62 | 63 | } // namespace detail 64 | 65 | } // namespace util 66 | -------------------------------------------------------------------------------- /util/bzip_source.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef BZIP_SOURCE_H 5 | #define BZIP_SOURCE_H 6 | 7 | #include 8 | #include "util/sinksource.h" 9 | 10 | namespace util { 11 | 12 | class BzipSource : public Source { 13 | public: 14 | // Takes ownership over sub_source 15 | BzipSource(Source* sub_source); 16 | ~BzipSource() override; 17 | 18 | static bool IsBzipSource(Source* source); 19 | private: 20 | StatusObject ReadInternal(const strings::MutableByteRange& range) override; 21 | 22 | struct Rep; 23 | 24 | std::unique_ptr sub_stream_; 25 | std::unique_ptr rep_; 26 | }; 27 | 28 | } // namespace util 29 | 30 | #endif // BZIP_SOURCE_H 31 | -------------------------------------------------------------------------------- /util/coding/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(coding double_compressor.cc block_compressor.cc) 2 | cxx_link(coding base math TRDP::lz4 TRDP::blosc TRDP::zstd) 3 | 4 | add_library(set_encoder_lib set_encoder.cc sequence_array.cc) 5 | cxx_link(set_encoder_lib strings coding) 6 | 7 | cxx_test(double_compressor_test coding LABELS CI) 8 | cxx_test(block_compressor_test coding LABELS CI) 9 | 10 | cxx_test(set_encoder_test LABELS CI) 11 | cxx_link(set_encoder_test set_encoder_lib) 12 | 13 | -------------------------------------------------------------------------------- /util/coding/block_compressor_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "util/coding/block_compressor.h" 6 | 7 | #include "base/gtest.h" 8 | #include "base/logging.h" 9 | #include "strings/stringpiece.h" 10 | 11 | namespace util { 12 | 13 | using namespace std; 14 | using strings::ByteRange; 15 | using strings::ToByteRange; 16 | 17 | class BlockCompressorTest : public testing::Test { 18 | public: 19 | BlockCompressorTest() {} 20 | 21 | protected: 22 | BlockCompressor bc_; 23 | BlockDecompressor bdc_; 24 | }; 25 | 26 | 27 | TEST_F(BlockCompressorTest, Basic) { 28 | string inp[2]; 29 | inp[0] = base::RandStr(BlockCompressor::BLOCK_SIZE); 30 | inp[1] = inp[0]; 31 | 32 | 33 | bc_.Add(ToByteRange(inp[0])); 34 | bc_.Add(ToByteRange(inp[1])); 35 | bc_.Finalize(); 36 | 37 | const auto& cb = bc_.compressed_blocks(); 38 | 39 | 40 | unsigned index = 0; 41 | 42 | uint32_t consumed; 43 | for (const auto& range : cb) { 44 | int32_t res = bdc_.Decompress(range, &consumed); 45 | if (res == 0) 46 | break; 47 | ASSERT_GT(res, 0) << index << "/" << range.size(); 48 | ASSERT_LT(index, 2); 49 | ASSERT_EQ(range.size(), consumed); 50 | EXPECT_EQ(inp[index].size(), bdc_.GetDecompressedBlock().size()); 51 | ++index; 52 | } 53 | bc_.ClearCompressedData(); 54 | 55 | bc_.Add(ToByteRange(inp[0])); 56 | } 57 | 58 | 59 | TEST_F(BlockCompressorTest, Unaligned) { 60 | string inp = base::RandStr(16); 61 | bc_.Add(ToByteRange(inp)); 62 | bc_.Finalize(); 63 | uint32_t consumed; 64 | ASSERT_EQ(1, bc_.compressed_blocks().size()); 65 | 66 | EXPECT_EQ(0, bdc_.Decompress(bc_.compressed_blocks().front(), &consumed)); 67 | } 68 | 69 | } // namespace util 70 | 71 | -------------------------------------------------------------------------------- /util/coding/sequence_array.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "base/pod_array.h" 7 | #include "strings/range.h" 8 | 9 | namespace util { 10 | 11 | class SequenceArray { 12 | base::PODArray data_; 13 | std::vector len_; 14 | 15 | public: 16 | class Iterator { 17 | const uint8* pval_; 18 | const uint32* plen_; 19 | 20 | public: 21 | Iterator(const uint8* pv, const uint32_t* pl) : pval_(pv), plen_(pl) { 22 | } 23 | 24 | strings::ByteRange operator*() const { return strings::ByteRange(pval_, *plen_); } 25 | 26 | Iterator& operator++() { 27 | pval_ += *plen_; 28 | ++plen_; 29 | return *this; 30 | } 31 | 32 | bool operator==(const Iterator& o) const { 33 | return pval_ == o.pval_; 34 | } 35 | 36 | bool operator!=(const Iterator& o) const { 37 | return !(*this == o); 38 | } 39 | }; 40 | 41 | typedef Iterator const_iterator; 42 | 43 | template uint32 Add(const U* s, const U* e) { 44 | static_assert(sizeof(U) == 1, ""); 45 | uint32 res = len_.size(); 46 | data_.insert(s, e); 47 | len_.push_back(e - s); 48 | return res; 49 | } 50 | 51 | const base::PODArray& data() const { return data_; } 52 | 53 | bool empty() const { return len_.empty(); } 54 | void reserve(size_t sz) { data_.reserve(sz); } 55 | 56 | size_t data_size() const { return data_.size(); } 57 | const std::vector& len_array() const { return len_; } 58 | 59 | void clear() { 60 | data_.clear(); 61 | len_.clear(); 62 | } 63 | 64 | const_iterator begin() const { return Iterator(data_.begin(), len_.data()); } 65 | const_iterator end() const { return Iterator(data_.end(), nullptr); } 66 | 67 | size_t GetMaxSerializedSize() const; 68 | size_t SerializeTo(uint8* dest) const; 69 | void SerializeFrom(const uint8_t* src, uint32_t count); 70 | }; 71 | 72 | 73 | } // namespace util 74 | -------------------------------------------------------------------------------- /util/fibers/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(fibers_ext fibers_ext.cc fiberqueue_threadpool.cc) 2 | cxx_link(fibers_ext base Boost::fiber absl_strings) 3 | 4 | cxx_test(fibers_ext_test fibers_ext asio_fiber_lib LABELS CI) 5 | -------------------------------------------------------------------------------- /util/fibers/fibers_ext.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/fibers/fibers_ext.h" 5 | 6 | namespace std { 7 | 8 | ostream& operator<<(ostream& o, const ::boost::fibers::channel_op_status op) { 9 | using ::boost::fibers::channel_op_status; 10 | if (op == channel_op_status::success) { 11 | o << "success"; 12 | } else if (op == channel_op_status::closed) { 13 | o << "closed"; 14 | } else if (op == channel_op_status::full) { 15 | o << "full"; 16 | } else if (op == channel_op_status::empty) { 17 | o << "empty"; 18 | } else if (op == channel_op_status::timeout) { 19 | o << "timeout"; 20 | } 21 | return o; 22 | } 23 | 24 | } // namespace std 25 | 26 | namespace util { 27 | namespace fibers_ext { 28 | 29 | 30 | } // namespace fibers_ext 31 | } // namespace util 32 | -------------------------------------------------------------------------------- /util/gce/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(gce_lib gce.cc gcs.cc gcs_read_file.cc gcs_write_file.cc detail/gcs_utils.cc) 2 | cxx_link(gce_lib asio_fiber_lib file status https_client_lib http_common TRDP::rapidjson) 3 | 4 | -------------------------------------------------------------------------------- /util/gce/gce.h: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "util/status.h" 12 | 13 | namespace util { 14 | class IoContext; 15 | 16 | class GCE { 17 | public: 18 | using SslContext = ::boost::asio::ssl::context; 19 | using error_code = ::boost::system::error_code; 20 | 21 | GCE() = default; 22 | 23 | Status Init(); 24 | 25 | const std::string& project_id() const { return project_id_; } 26 | const std::string& client_id() const { return client_id_; } 27 | const std::string& client_secret() const { return client_secret_; } 28 | const std::string& account_id() const { return account_id_; } 29 | 30 | // refresh_token is used for refreshing an access token. 31 | const std::string& refresh_token() const { return refresh_token_; } 32 | 33 | static const char* GoogleCert(); 34 | static const char* kApiDomain; 35 | 36 | static ::boost::asio::ssl::context CheckedSslContext(); 37 | 38 | //! Returns cached access_token. 39 | //! Must be called after RefreshAccessToken has been called. 40 | std::string access_token() const; 41 | 42 | StatusObject RefreshAccessToken(IoContext* context) const; 43 | bool is_prod_env() const { return is_prod_env_; } 44 | 45 | void Test_InjectAcessToken(std::string access_token); 46 | 47 | private: 48 | util::Status ParseDefaultConfig(); 49 | util::Status ReadDevCreds(const std::string& root_path); 50 | util::StatusObject ParseTokenResponse(std::string&& response) const; 51 | 52 | std::string project_id_, client_id_, client_secret_, account_id_, refresh_token_; 53 | 54 | mutable ::boost::fibers::mutex mu_; 55 | mutable std::string access_token_; 56 | 57 | std::unique_ptr ssl_ctx_; 58 | bool is_prod_env_ = false; 59 | }; 60 | 61 | } // namespace util 62 | -------------------------------------------------------------------------------- /util/html/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(html_lib sorted_table.cc) 2 | cxx_link(html_lib strings) 3 | -------------------------------------------------------------------------------- /util/html/main.js: -------------------------------------------------------------------------------- 1 | $(function() { 2 | 3 | $("table").tablesorter({ 4 | theme : "blue", 5 | 6 | widthFixed: true, 7 | 8 | // widget code contained in the jquery.tablesorter.widgets.js file 9 | // use the zebra stripe widget if you plan on hiding any rows (filter widget) 10 | // the uitheme widget is NOT REQUIRED! 11 | widgets : [ "filter", "columns", "zebra" ], 12 | 13 | widgetOptions : { 14 | // using the default zebra striping class name, so it actually isn't included in the theme variable above 15 | // this is ONLY needed for bootstrap theming if you are using the filter widget, because rows are hidden 16 | zebra : ["even", "odd"], 17 | 18 | // class names added to columns when sorted 19 | columns: [ "primary", "secondary", "tertiary" ], 20 | filter_columnFilters: true, 21 | 22 | // reset filters button 23 | filter_reset : ".reset", 24 | filter_placeholder: { search : 'Search...' }, 25 | } 26 | }) 27 | .tablesorterPager({ 28 | // target the pager markup - see the HTML block below 29 | container: $(".ts-pager"), 30 | 31 | // target the pager page select dropdown - choose a page 32 | cssGoto : ".pagenum", 33 | 34 | // remove rows from the table to speed up the sort of large tables. 35 | // setting this to false, only hides the non-visible rows; needed if you plan to add/remove rows with the pager enabled. 36 | removeRows: false, 37 | 38 | // output string - default is '{page}/{totalPages}'; 39 | // possible variables: {page}, {totalPages}, {filteredPages}, {startRow}, {endRow}, {filteredRows} and {totalRows} 40 | output: '{startRow} - {endRow} / {filteredRows} ({totalRows})', 41 | size: 10, 42 | }); 43 | 44 | }); 45 | -------------------------------------------------------------------------------- /util/html/sorted_table.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | #include "strings/stringpiece.h" 6 | 7 | namespace util { 8 | namespace html { 9 | 10 | class SortedTable { 11 | public: 12 | static std::string HtmlStart(); 13 | 14 | static void StartTable(const std::vector& header, std::string* dest); 15 | 16 | static void Row(const std::vector& row, std::string* dest); 17 | 18 | static void EndTable(std::string* dest); 19 | }; 20 | 21 | /* 22 | std::string SortedTable::Start(const Container& header) { 23 | std::string res( 24 | absl::StrCat(" \n")); 25 | for (auto v : header) { 26 | res.append(absl::StrCat("")); 27 | } 28 | res.append("\n \n"); 29 | return res; 30 | } 31 | 32 | template 33 | std::string SortedTable::Row(StringPiece style, const Container& row) { 34 | std::string res(""); 35 | std::string td_tag; 36 | if (style.empty()) { 37 | td_tag = ""); 43 | } 44 | res.append("\n"); 45 | return res; 46 | } 47 | */ 48 | 49 | } // namespace html 50 | } // namespace util 51 | -------------------------------------------------------------------------------- /util/html/style.css: -------------------------------------------------------------------------------- 1 | /* apply additional customization here */ 2 | 3 | .form-inline { 4 | display: flex; 5 | margin: 10px 0; 6 | } -------------------------------------------------------------------------------- /util/http/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(http_beast_prebuilt prebuilt_beast.cc) 2 | 3 | add_library(http_common http_common.cc status_page.cc profilez_handler.cc) 4 | cxx_link(http_common absl_strings base http_beast_prebuilt proc_stats stats_lib fast_malloc) 5 | 6 | add_library(http_v2 http_conn_handler.cc ) 7 | cxx_link(http_v2 asio_fiber_lib strings stats_lib http_common) 8 | 9 | add_executable(http_main http_main.cc) 10 | cxx_link(http_main http_v2 html_lib) 11 | 12 | add_library(http_client_lib http_client.cc) 13 | cxx_link(http_client_lib strings asio_fiber_lib) 14 | 15 | add_library(https_client_lib https_client.cc https_client_pool.cc ssl_stream.cc) 16 | cxx_link(https_client_lib strings asio_fiber_lib absl_variant http_beast_prebuilt ssl crypto) 17 | cxx_test(ssl_stream_test https_client_lib LABELS CI) 18 | 19 | 20 | add_library(http_test_lib http_testing.cc) 21 | cxx_link(http_test_lib http_v2 gaia_gtest_main TRDP::rapidjson) 22 | 23 | cxx_test(http_test http_v2 http_client_lib http_test_lib LABELS CI) 24 | -------------------------------------------------------------------------------- /util/http/captain.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/romange/gaia/8ef14627a4bf42eba83bb6df4d180beca305b307/util/http/captain.gif -------------------------------------------------------------------------------- /util/http/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/romange/gaia/8ef14627a4bf42eba83bb6df4d180beca305b307/util/http/favicon-32x32.png -------------------------------------------------------------------------------- /util/http/http_client.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include 9 | 10 | #include "strings/stringpiece.h" 11 | 12 | namespace util { 13 | class IoContext; 14 | class FiberSyncSocket; 15 | 16 | namespace http { 17 | 18 | /* 19 | Single threaded, fiber-friendly synchronous client: Upon IO block, the calling fiber blocks 20 | but the thread can switch to other active fibers. 21 | */ 22 | class Client { 23 | public: 24 | using Response = boost::beast::http::response; 25 | using Verb = boost::beast::http::verb; 26 | 27 | explicit Client(IoContext* io_context); 28 | ~Client(); 29 | 30 | boost::system::error_code Connect(StringPiece host, StringPiece service); 31 | 32 | boost::system::error_code Send(Verb verb, StringPiece url, StringPiece body, Response* response); 33 | boost::system::error_code Send(Verb verb, StringPiece url, Response* response) { 34 | return Send(verb, url, StringPiece{}, response); 35 | } 36 | 37 | void Shutdown(); 38 | 39 | bool IsConnected() const; 40 | 41 | void set_connect_timeout_ms(uint32_t ms) { connect_timeout_ms_ = ms; } 42 | 43 | // Adds header to all future requests. 44 | void AddHeader(std::string name, std::string value) { 45 | headers_.emplace_back(std::move(name), std::move(value)); 46 | } 47 | 48 | private: 49 | IoContext& io_context_; 50 | uint32_t connect_timeout_ms_ = 2000; 51 | 52 | using HeaderPair = std::pair; 53 | 54 | std::vector headers_; 55 | std::unique_ptr socket_; 56 | }; 57 | 58 | } // namespace http 59 | } // namespace util 60 | 61 | -------------------------------------------------------------------------------- /util/http/http_common.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | #include 6 | #include 7 | 8 | #include "absl/strings/string_view.h" 9 | #include "util/asio_stream_adapter.h" 10 | 11 | namespace util { 12 | namespace http { 13 | 14 | // URL consists of path and query delimited by '?'. 15 | // query can be broken into query args delimited by '&'. 16 | // Each query arg can be a pair of "key=value" values. 17 | // In case there is not '=' delimiter, only the first field is filled. 18 | using QueryParam = std::pair; 19 | typedef std::vector QueryArgs; 20 | 21 | typedef ::boost::beast::http::response<::boost::beast::http::string_body> 22 | StringResponse; 23 | 24 | inline StringResponse MakeStringResponse( 25 | ::boost::beast::http::status st = ::boost::beast::http::status::ok) { 26 | return StringResponse(st, 11); 27 | } 28 | 29 | inline void SetMime(const char* mime, ::boost::beast::http::fields* dest) { 30 | dest->set(::boost::beast::http::field::content_type, mime); 31 | } 32 | 33 | inline absl::string_view as_absl(::boost::string_view s) { 34 | return absl::string_view(s.data(), s.size()); 35 | } 36 | 37 | extern const char kHtmlMime[]; 38 | extern const char kJsonMime[]; 39 | extern const char kSvgMime[]; 40 | extern const char kTextMime[]; 41 | extern const char kXmlMime[]; 42 | extern const char kBinMime[]; 43 | 44 | QueryParam ParseQuery(absl::string_view str); 45 | QueryArgs SplitQuery(absl::string_view query); 46 | StringResponse ParseFlagz(const QueryArgs& args); 47 | 48 | StringResponse BuildStatusPage(const QueryArgs& args, const char* resource_prefix); 49 | StringResponse ProfilezHandler(const QueryArgs& args); 50 | 51 | using FileResponse = ::boost::beast::http::response<::boost::beast::http::file_body>; 52 | ::boost::system::error_code LoadFileResponse(absl::string_view fname, FileResponse* resp); 53 | 54 | } // namespace http 55 | } // namespace util 56 | -------------------------------------------------------------------------------- /util/http/http_status_code.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "http_status_code.h" 5 | #include "base/logging.h" 6 | 7 | namespace http { 8 | 9 | const char* StatusStringFromCode(HttpStatusCode code) { 10 | switch (code) { 11 | case HTTP_OK: return "200 OK"; 12 | case HTTP_ACCEPTED: return"202 Accepted"; 13 | case HTTP_NO_CONTENT: return"204 No Content"; 14 | case HTTP_BAD_REQUEST: return "400 Bad Request"; 15 | case HTTP_UNAUTHORIZED: return "401 Unauthorized"; 16 | case HTTP_FORBIDDEN: return "403 Forbidden"; 17 | case HTTP_NOT_FOUND: return "404 Not Found"; 18 | default: 19 | LOG(FATAL) << "Not implemented " << code; 20 | } 21 | return nullptr; 22 | } 23 | } // namespace http -------------------------------------------------------------------------------- /util/http/http_status_code.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #ifndef HTTP_STATUS_CODE_H 6 | #define HTTP_STATUS_CODE_H 7 | 8 | namespace http { 9 | 10 | // HTTP status codes. 11 | // Taken from RFC 2616 Section 10. 12 | enum HttpStatusCode { 13 | // Informational 1xx 14 | HTTP_CONTINUE = 100, 15 | HTTP_SWITCHING_PROTOCOLS = 101, 16 | 17 | // Successful 2xx 18 | HTTP_OK = 200, 19 | HTTP_CREATED = 201, 20 | HTTP_ACCEPTED = 202, 21 | HTTP_NON_AUTHORITATIVE_INFORMATION = 203, 22 | HTTP_NO_CONTENT = 204, 23 | HTTP_RESET_CONTENT = 205, 24 | HTTP_PARTIAL_CONTENT = 206, 25 | 26 | // Redirection 3xx 27 | HTTP_MULTIPLE_CHOICES = 300, 28 | HTTP_MOVED_PERMANENTLY = 301, 29 | HTTP_FOUND = 302, 30 | HTTP_SEE_OTHER = 303, 31 | HTTP_NOT_MODIFIED = 304, 32 | HTTP_USE_PROXY = 305, 33 | // 306 is no longer used. 34 | HTTP_TEMPORARY_REDIRECT = 307, 35 | 36 | // Client error 4xx 37 | HTTP_BAD_REQUEST = 400, 38 | HTTP_UNAUTHORIZED = 401, 39 | HTTP_PAYMENT_REQUIRED = 402, 40 | HTTP_FORBIDDEN = 403, 41 | HTTP_NOT_FOUND = 404, 42 | HTTP_METHOD_NOT_ALLOWED = 405, 43 | HTTP_NOT_ACCEPTABLE = 406, 44 | HTTP_PROXY_AUTHENTICATION_REQUIRED = 407, 45 | HTTP_REQUEST_TIMEOUT = 408, 46 | HTTP_CONFLICT = 409, 47 | HTTP_GONE = 410, 48 | HTTP_LENGTH_REQUIRED = 411, 49 | HTTP_PRECONDITION_FAILED = 412, 50 | HTTP_REQUEST_ENTITY_TOO_LARGE = 413, 51 | HTTP_REQUEST_URI_TOO_LONG = 414, 52 | HTTP_UNSUPPORTED_MEDIA_TYPE = 415, 53 | HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416, 54 | HTTP_EXPECTATION_FAILED = 417, 55 | 56 | // Server error 5xx 57 | HTTP_INTERNAL_SERVER_ERROR = 500, 58 | HTTP_NOT_IMPLEMENTED = 501, 59 | HTTP_BAD_GATEWAY = 502, 60 | HTTP_SERVICE_UNAVAILABLE = 503, 61 | HTTP_GATEWAY_TIMEOUT = 504, 62 | HTTP_VERSION_NOT_SUPPORTED = 505, 63 | }; 64 | 65 | 66 | // '200 OK' from HTTP_OK etc. 67 | const char* StatusStringFromCode(HttpStatusCode code); 68 | 69 | } // namespace http 70 | 71 | #endif // HTTP_STATUS_CODE_H -------------------------------------------------------------------------------- /util/http/http_testing.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "util/http/http_testing.h" 6 | #include "base/logging.h" 7 | 8 | namespace util { 9 | 10 | using namespace boost; 11 | using namespace std; 12 | 13 | 14 | void HttpBaseTest::SetUp() { 15 | pool_.reset(new IoContextPool); 16 | pool_->Run(); 17 | 18 | server_.reset(new AcceptServer(pool_.get())); 19 | port_ = server_->AddListener(0, &listener_); 20 | server_->Run(); 21 | } 22 | 23 | void HttpBaseTest::TearDown() { 24 | LOG(INFO) << "HttpBaseTest::TearDown"; 25 | 26 | server_.reset(); 27 | VLOG(1) << "After server reset"; 28 | pool_->Stop(); 29 | } 30 | 31 | } // namespace util 32 | -------------------------------------------------------------------------------- /util/http/http_testing.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "base/gtest.h" 5 | 6 | #include "util/asio/accept_server.h" 7 | #include "util/asio/io_context_pool.h" 8 | #include "util/http/http_conn_handler.h" 9 | 10 | namespace util { 11 | 12 | class HttpBaseTest : public testing::Test { 13 | protected: 14 | void SetUp() override; 15 | 16 | void TearDown() override; 17 | 18 | std::unique_ptr server_; 19 | std::unique_ptr pool_; 20 | http::Listener<> listener_; 21 | uint16_t port_ = 0; 22 | }; 23 | 24 | } // namespace util 25 | 26 | -------------------------------------------------------------------------------- /util/http/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/romange/gaia/8ef14627a4bf42eba83bb6df4d180beca305b307/util/http/logo.png -------------------------------------------------------------------------------- /util/http/prebuilt_beast.cc: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if BOOST_VERSION >= 107000 4 | # include 5 | #endif 6 | -------------------------------------------------------------------------------- /util/http/ssl_stream_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2019, Ubimo.com . All rights reserved. 2 | // Author: Roman Gershman (roman@ubimo.com) 3 | // 4 | #include 5 | 6 | #include "base/gtest.h" 7 | #include "base/logging.h" 8 | #include "util/http/ssl_stream.h" 9 | 10 | namespace util { 11 | namespace http { 12 | 13 | class SslStreamTest : public testing::Test { 14 | protected: 15 | void SetUp() override { 16 | } 17 | 18 | void TearDown() { 19 | } 20 | }; 21 | 22 | TEST_F(SslStreamTest, BIO_s_bio_err) { 23 | BIO* bio1 = BIO_new(BIO_s_bio()); 24 | constexpr char kData[] = "ROMAN"; 25 | ASSERT_EQ(0, ERR_get_error()); 26 | EXPECT_EQ(-2, BIO_write(bio1, kData, sizeof(kData))); 27 | int e = ERR_get_error(); 28 | EXPECT_NE(0, e); 29 | EXPECT_EQ(BIO_F_BIO_WRITE_INTERN, ERR_GET_FUNC(e)); 30 | EXPECT_EQ(BIO_R_UNINITIALIZED, ERR_GET_REASON(e)); 31 | BIO_free(bio1); 32 | } 33 | 34 | TEST_F(SslStreamTest, BIO_s_bio_ZeroCopy) { 35 | BIO *bio1, *bio2; 36 | EXPECT_EQ(1, BIO_new_bio_pair(&bio1, 0, &bio2, 0)); 37 | 38 | char *buf1 = nullptr, *buf2 = nullptr; 39 | ssize_t write_size = BIO_nwrite0(bio1, &buf1); 40 | EXPECT_EQ(17 * 1024, write_size); 41 | memset(buf1, 'a', write_size); 42 | 43 | EXPECT_EQ(write_size, BIO_nwrite(bio1, nullptr, write_size)); // commit. 44 | EXPECT_EQ(-1, BIO_nwrite(bio1, nullptr, 1)); // No space to commit. 45 | 46 | EXPECT_EQ(0, BIO_ctrl_get_write_guarantee(bio1)); 47 | 48 | ASSERT_EQ(write_size, BIO_nread0(bio2, &buf2)); 49 | EXPECT_EQ(0, memcmp(buf1, buf2, write_size)); 50 | EXPECT_EQ(write_size, BIO_nread(bio2, nullptr, write_size)); 51 | 52 | // bio1 is empty again. 53 | EXPECT_EQ(write_size, BIO_ctrl_get_write_guarantee(bio1)); 54 | 55 | BIO_free(bio1); 56 | BIO_free(bio2); 57 | } 58 | 59 | } // namespace http 60 | 61 | } // namespace util 62 | -------------------------------------------------------------------------------- /util/http/status_page.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Roboto; 3 | font-size:16px; 4 | line-height: 30px; 5 | font-weight: normal; 6 | color: #4e534b; 7 | margin-left: 30px; 8 | margin-top: 0px; 9 | margin-right: 0px; 10 | margin-bottom: 0px; 11 | /*background-color: #ebebeb;*/ 12 | background-repeat: repeat-x; 13 | } 14 | 15 | .title_text { 16 | color: #09bc8d; 17 | margin-right: 20px; 18 | } 19 | 20 | .key_text, .key_text_bold { 21 | font-style:italic; 22 | margin-right: 7px; 23 | } 24 | 25 | .key_text_bold { 26 | font-weight: bold; 27 | } 28 | 29 | .value_text { 30 | font-weight: bold; 31 | color: #d60f96; 32 | margin-right: 10px; 33 | } 34 | 35 | .separator { 36 | border-top: 1px solid rgba(211, 211, 211, 1); 37 | border-bottom: 1px solid rgba(255, 255, 255, 1); 38 | margin-top: 10px; 39 | margin-bottom: 5px; 40 | } 41 | 42 | .styled_border{ 43 | font-size:14px; 44 | border-style: groove; 45 | border-width: 2px; 46 | border-color: #FFF; 47 | margin-left:80px; 48 | margin-top:30px; 49 | float:left; 50 | padding:15px; 51 | } 52 | 53 | .left_panel { 54 | width:700px; 55 | float:left; 56 | } 57 | -------------------------------------------------------------------------------- /util/http/status_page.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/http/http_common.h" 5 | 6 | namespace util { 7 | namespace http { 8 | 9 | } // namespace http 10 | } // namespace util 11 | 12 | -------------------------------------------------------------------------------- /util/http/status_page.js: -------------------------------------------------------------------------------- 1 | function JsonToHTML(json_obj) { 2 | var str = ''; 3 | Object.keys(json_obj).forEach(function (key) { 4 | value = json_obj[key]; 5 | str += "
" + span(key, 'title_text'); 6 | if (!isObject(value)) { 7 | str += value_text(key, value); 8 | } else { 9 | str += objectObjectToHTML(value); 10 | } 11 | str += "
"; 12 | }); 13 | 14 | return str; 15 | 16 | function objectObjectToHTML(objmap) { 17 | var s = ''; 18 | Object.keys(objmap).forEach(function (key) { 19 | value = objmap[key]; 20 | if (isObject(value)) { 21 | s += span(key + ':', 'key_text_bold') 22 | s += objectObjectToHTML(value); 23 | } else { 24 | s += key_text(key); 25 | s += value_text(key, value); 26 | } 27 | }); 28 | return s; 29 | } 30 | 31 | function isObject(o) { 32 | return Object.prototype.toString.call(o) === '[object Object]'; 33 | } 34 | 35 | function span(t, s) { return "" + t + "";} 36 | 37 | function key_text(t) { 38 | return span(t + ':', 'key_text'); 39 | } 40 | 41 | function value_text(k, t) { 42 | if (k.endsWith('time') && Number.isInteger(t)) { 43 | var date = new Date(t*1000); 44 | t = date.toISOString().slice(0, 19); 45 | } 46 | 47 | return span(t + ' ', 'value_text'); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /util/math/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(math mathlimits.cc mathutil.cc float2decimal.cc exactfloat/exactfloat.cc) 2 | cxx_link(math base strings crypto) 3 | 4 | cxx_test(float2decimal_test math strings TRDP::dconv LABELS CI) 5 | -------------------------------------------------------------------------------- /util/math/mathutil.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 Google Inc. All Rights Reserved. 2 | 3 | #include "util/math/mathutil.h" 4 | #include 5 | using std::vector; 6 | 7 | #include "base/integral_types.h" 8 | #include "base/logging.h" 9 | 10 | double MathUtil::Harmonic(int64 const n, double *const e) { 11 | CHECK_GT(n, 0); 12 | 13 | // Hn ~ ln(n) + 0.5772156649 + 14 | // + 1/(2n) - 1/(12n^2) + 1/(120n^4) - error, 15 | // with 0 < error < 1/(256*n^4). 16 | 17 | double const 18 | d = static_cast(n), 19 | d2 = d * d, 20 | d4 = d2 * d2; 21 | 22 | return (log(d) + 0.5772156649) // ln + Gamma constant 23 | + 1 / (2 * d) - 1 / (12 * d2) + 1 / (120 * d4) 24 | - (*e = 1 / (256 * d4)); 25 | } 26 | -------------------------------------------------------------------------------- /util/math/matrix3x3.h: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright 2003 Google, Inc. 3 | // 4 | // 5 | // A simple class to handle 3x3 matrices 6 | // The aim of this class is to be able to manipulate 3x3 matrices 7 | // and 3D vectors as naturally as possible and make calculations 8 | // readable. 9 | // For that reason, the operators +, -, * are overloaded. 10 | // (Reading a = a + b*2 - c is much easier to read than 11 | // a = Sub(Add(a, Mul(b,2)),c) ) 12 | // This file only define the typenames, for API details, look into 13 | // matrix3x3-inl.h 14 | // 15 | 16 | #ifndef UTIL_MATH_MATRIX3X3_H__ 17 | #define UTIL_MATH_MATRIX3X3_H__ 18 | 19 | template 20 | class Matrix3x3; 21 | 22 | typedef Matrix3x3 Matrix3x3_i; 23 | typedef Matrix3x3 Matrix3x3_f; 24 | typedef Matrix3x3 Matrix3x3_d; 25 | 26 | #endif // UTIL_MATH_MATRIX3X3_H__ 27 | -------------------------------------------------------------------------------- /util/pb2json.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include "util/status.h" 12 | 13 | namespace util { 14 | 15 | struct Pb2JsonOptions { 16 | typedef std::function FieldNameCb; 17 | 18 | typedef std::function BoolAsIntegerPred; 19 | 20 | bool enum_as_ints = false; 21 | 22 | FieldNameCb field_name_cb; 23 | BoolAsIntegerPred bool_as_int; 24 | }; 25 | 26 | std::string Pb2Json(const ::google::protobuf::Message& msg, 27 | const Pb2JsonOptions& options = Pb2JsonOptions()); 28 | 29 | struct Json2PbOptions { 30 | bool skip_unknown_fields; 31 | 32 | Json2PbOptions(bool sk = true) : skip_unknown_fields(sk) {} 33 | }; 34 | 35 | Status Json2Pb(std::string json, ::google::protobuf::Message* msg, const Json2PbOptions& options); 36 | 37 | inline Status Json2Pb(std::string json, ::google::protobuf::Message* msg, 38 | bool skip_unknown_fields = true) { 39 | return Json2Pb(std::move(json), msg, Json2PbOptions(skip_unknown_fields)); 40 | } 41 | 42 | } // namespace util 43 | -------------------------------------------------------------------------------- /util/plang/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cxx_proto_lib(addressbook) 2 | cxx_test(proto_test addressbook_proto LABELS CI) 3 | 4 | add_library(plang plang.cc) 5 | cxx_link(plang TRDP::protobuf strings math plang_parser_bison) 6 | 7 | flex_lib(plang_scanner) 8 | 9 | bison_lib(plang_parser) 10 | cxx_link(plang_parser_bison base strings plang plang_scanner_flex) 11 | 12 | cxx_test(plang_test plang plang_parser_bison addressbook_proto LABELS CI) 13 | -------------------------------------------------------------------------------- /util/plang/addressbook.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | syntax = "proto2"; 5 | 6 | import "google/protobuf/descriptor.proto"; 7 | 8 | package tutorial; 9 | 10 | option cc_generic_services = true; 11 | option java_package = "com.example.tutorial"; 12 | option java_outer_classname = "AddressBookProtos"; 13 | option cc_enable_arenas = true; 14 | 15 | extend google.protobuf.FieldOptions { 16 | optional string fd_type = 10101; 17 | optional string fd_name = 10102; 18 | } 19 | 20 | message Address { 21 | optional string street = 1; 22 | } 23 | 24 | message BankAccount { 25 | repeated int32 activity_id = 3; 26 | optional string bank_name = 1; 27 | optional Address address = 2; 28 | } 29 | 30 | message Person { 31 | required string name = 1; 32 | required int64 id = 2; // Unique ID number for this person. 33 | optional string email = 3; 34 | 35 | enum PhoneType { 36 | MOBILE = 0; 37 | HOME = 1; 38 | WORK = 2; 39 | } 40 | 41 | message PhoneNumber { 42 | required string number = 1; 43 | optional PhoneType type = 2 [default = HOME]; 44 | } 45 | 46 | repeated PhoneNumber phone = 4; 47 | repeated PhoneNumber phone2 = 5; 48 | optional BankAccount account = 6; 49 | repeated string tag = 7; 50 | required double dval = 8; 51 | optional float fval = 9; 52 | } 53 | 54 | // Our address book file is just one of these. 55 | message AddressBook { 56 | repeated Person person = 1; 57 | repeated int64 ts = 2; 58 | repeated uint64 tmp = 3; 59 | map ids = 4; 60 | optional int64 fd1 = 5 [ (fd_name) = "another_name"]; 61 | } 62 | 63 | message JsonParse { 64 | optional bool bval = 1; 65 | } 66 | 67 | service TestService { 68 | rpc Test(Person) returns (AddressBook); 69 | } 70 | -------------------------------------------------------------------------------- /util/plang/plang_scanner.h: -------------------------------------------------------------------------------- 1 | // This code is based on http://www.jonathanbeard.io/tutorials/FlexBisonC++ 2 | 3 | #ifndef _PLANG_SCANNER_H 4 | #define _PLANG_SCANNER_H 5 | 6 | #if ! defined(yyFlexLexerOnce) 7 | #include 8 | #endif 9 | 10 | namespace plang { 11 | class Scanner; 12 | class Driver; 13 | } 14 | #include "util/plang/plang.h" 15 | #include "util/plang/plang_parser.hh" 16 | 17 | namespace plang { 18 | class Scanner : public yyFlexLexer { 19 | public: 20 | Scanner(std::istream *in) : yyFlexLexer(in) { 21 | loc_.reset(new plang::Parser::location_type()); 22 | } 23 | 24 | int lex() { 25 | Parser::semantic_type lval; 26 | Parser::location_type location; 27 | return parser_lex(&lval, &location); 28 | } 29 | 30 | int parser_lex( Parser::semantic_type * const lval, 31 | Parser::location_type *location); 32 | 33 | std::string matched() { return yytext; } 34 | 35 | Parser::location_type *loc() { return &*loc_; } 36 | 37 | private: 38 | std::unique_ptr loc_; 39 | }; 40 | }; 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /util/plang/proto_test.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "util/plang/addressbook.pb.h" 3 | 4 | using namespace google::protobuf; 5 | using tutorial::Person; 6 | 7 | class TestServiceImpl : public tutorial::TestService { 8 | public: 9 | 10 | }; 11 | 12 | class ProtoTest : public testing::Test { 13 | public: 14 | void TestCallback() { 15 | } 16 | }; 17 | 18 | TEST_F(ProtoTest, Basic) { 19 | tutorial::Person person; 20 | person.set_name("Roman"); 21 | } 22 | 23 | TEST_F(ProtoTest, Arena) { 24 | char buf[1024]; 25 | ArenaOptions opts; 26 | opts.initial_block_size = sizeof(buf); 27 | opts.initial_block = buf; 28 | 29 | Arena arena(opts); 30 | EXPECT_EQ(sizeof(buf), arena.SpaceAllocated()); 31 | 32 | Person* person = Arena::CreateMessage(&arena); 33 | person->mutable_account()->set_bank_name("Foo"); 34 | EXPECT_EQ(&arena, person->GetArena()); 35 | 36 | Person::PhoneNumber* pn = new Person::PhoneNumber; 37 | person->mutable_phone()->AddAllocated(pn); 38 | EXPECT_EQ("Foo", person->account().bank_name()); 39 | person->Clear(); 40 | EXPECT_GT(arena.SpaceUsed(), 0); 41 | arena.Reset(); 42 | EXPECT_EQ(sizeof(buf), arena.SpaceAllocated()); 43 | } 44 | -------------------------------------------------------------------------------- /util/pprint/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(pprint_utils pprint_utils.cc file_printer.cc) 2 | target_link_libraries(pprint_utils sp_task_pool strings pb2json proto_writer plang_parser_bison 3 | TRDP::protobuf) 4 | 5 | add_executable(lst_print_example lst_print_example.cc) 6 | target_link_libraries(lst_print_example pprint_utils) 7 | 8 | 9 | cxx_test(pprint_utils_test pprint_utils pprint_utils_test_proto base LABELS CI) 10 | 11 | cxx_proto_lib(pprint_utils_test) 12 | -------------------------------------------------------------------------------- /util/pprint/lst_print_example.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, .com . All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | 11 | #include "absl/strings/match.h" 12 | 13 | #include "base/init.h" 14 | #include "file/list_file.h" 15 | #include "strings/escaping.h" 16 | #include "util/pprint/file_printer.h" 17 | #include "util/pprint/pprint_utils.h" 18 | 19 | DECLARE_string(csv); 20 | 21 | DECLARE_bool(sizes); 22 | DECLARE_bool(raw); 23 | DECLARE_bool(count); 24 | 25 | using namespace util; 26 | using std::string; 27 | using util::Status; 28 | 29 | using std::cout; 30 | 31 | using namespace file; 32 | 33 | void sigpipe_handler(int signal) { 34 | exit(1); 35 | } 36 | 37 | int main(int argc, char** argv) { 38 | MainInitGuard guard(&argc, &argv); 39 | 40 | signal(SIGPIPE, sigpipe_handler); 41 | 42 | size_t count = 0; 43 | 44 | // const Reflection* reflection = msg->GetReflection(); 45 | for (int i = 1; i < argc; ++i) { 46 | StringPiece path(argv[i]); 47 | LOG(INFO) << "Opening " << path; 48 | 49 | pprint::ListReaderPrinter printer; 50 | printer.Init(argv[i]); 51 | auto st = printer.Run(); 52 | CHECK_STATUS(st); 53 | count += printer.count(); 54 | } 55 | if (FLAGS_count) 56 | std::cout << "Count: " << count << std::endl; 57 | 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /util/pprint/pprint_utils_test.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto2"; 2 | 3 | message SimpleString { 4 | required string simple = 1; 5 | } 6 | 7 | message RepeatingString { 8 | repeated string repeating = 1; 9 | } 10 | 11 | message NestedString { 12 | message NestedA { 13 | required string s1 = 1; 14 | repeated string s2 = 2; 15 | } 16 | 17 | message NestedB { 18 | required string s1 = 1; 19 | repeated string s2 = 2; 20 | required string s3 = 3; 21 | } 22 | 23 | 24 | message NestedC { 25 | message NestedD { 26 | required string s1 = 1; 27 | required string s2 = 2; 28 | } 29 | required string s1 = 1; 30 | required string s2 = 2; 31 | required NestedD d1 = 3; 32 | } 33 | 34 | required NestedA a1 = 1; 35 | repeated NestedA a2 = 2; 36 | repeated NestedB b1 = 3; 37 | required NestedC c1 = 4; 38 | } 39 | -------------------------------------------------------------------------------- /util/proc_stats.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #ifndef PROC_STATUS_H 5 | #define PROC_STATUS_H 6 | 7 | #include 8 | 9 | #include "base/integral_types.h" 10 | 11 | namespace util { 12 | 13 | struct ProcessStats { 14 | uint32 vm_peak = 0; 15 | uint32 vm_rss = 0; 16 | uint32 vm_size = 0; 17 | 18 | // Start time of the process in seconds since epoch. 19 | uint64 start_time_seconds = 0; 20 | 21 | static ProcessStats Read(); 22 | }; 23 | 24 | namespace sys { 25 | unsigned int NumCPUs(); 26 | } // namespace sys 27 | 28 | } // namespace util 29 | 30 | std::ostream& operator<<(std::ostream& os, const util::ProcessStats& stats); 31 | 32 | #endif // PROC_STATUS_H 33 | -------------------------------------------------------------------------------- /util/rpc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(rpc frame_format.cc rpc_connection.cc channel.cc service_descriptor.cc 2 | impl/rpc_conn_handler.cc) 3 | cxx_link(rpc base asio_fiber_lib strings absl_hash absl_flat_hash_map 4 | status TRDP::protobuf) 5 | 6 | add_library(rpc_test_lib rpc_test_utils.cc) 7 | cxx_link(rpc_test_lib rpc gaia_gtest_main) 8 | 9 | cxx_test(rpc_test rpc_test_lib LABELS CI) 10 | -------------------------------------------------------------------------------- /util/rpc/rpc_connection.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/rpc/rpc_connection.h" 5 | 6 | #include "base/logging.h" 7 | 8 | #include "util/rpc/impl/rpc_conn_handler.h" 9 | 10 | namespace util { 11 | namespace rpc { 12 | 13 | ConnectionHandler* ServiceInterface::NewConnection(IoContext& context) { 14 | ConnectionBridge* bridge = CreateConnectionBridge(); 15 | return new RpcConnectionHandler(bridge, &context); 16 | } 17 | 18 | } // namespace rpc 19 | } // namespace util 20 | -------------------------------------------------------------------------------- /util/rpc/rpc_envelope.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "base/pod_array.h" 7 | #include "util/asio/asio_utils.h" 8 | 9 | namespace util { 10 | namespace rpc { 11 | 12 | typedef base::PODArray BufferType; 13 | 14 | class Envelope { 15 | public: 16 | BufferType header, letter; 17 | 18 | Envelope() = default; 19 | 20 | Envelope(Envelope&& other) noexcept { 21 | Swap(&other); 22 | } 23 | 24 | Envelope(size_t hsz, size_t lsz) { 25 | Resize(hsz, lsz); 26 | } 27 | 28 | void Clear() { 29 | header.clear(); 30 | letter.clear(); 31 | } 32 | 33 | void Resize(size_t hsz, size_t lsz) { 34 | header.resize(hsz); 35 | letter.resize(lsz); 36 | } 37 | 38 | auto buf_seq() { return make_buffer_seq(header, letter); } 39 | 40 | void Swap(Envelope* other) { 41 | if (other != this) { 42 | other->header.swap(header); 43 | other->letter.swap(letter); 44 | } 45 | } 46 | 47 | Envelope& operator=(Envelope&& other) noexcept { 48 | Swap(&other); 49 | return *this; 50 | } 51 | }; 52 | 53 | } // namespace rpc 54 | } // namespace util 55 | 56 | -------------------------------------------------------------------------------- /util/rpc/rpc_test_utils.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "base/gtest.h" 7 | #include "util/asio/io_context_pool.h" 8 | #include "util/rpc/rpc_connection.h" 9 | 10 | namespace util { 11 | class ReconnectableSocket; 12 | class AcceptServer; 13 | class FiberSyncSocket; 14 | 15 | namespace rpc { 16 | 17 | template 18 | void Copy(const Src& src, Dest* dest) { 19 | dest->resize(src.size()); 20 | std::copy(src.begin(), src.end(), dest->begin()); 21 | } 22 | 23 | class TestBridge final : public ConnectionBridge { 24 | bool clear_; 25 | 26 | public: 27 | TestBridge(bool clear) : clear_(clear) { 28 | } 29 | 30 | ~TestBridge(); 31 | 32 | void Join() final; 33 | 34 | // header and letter are input/output parameters. 35 | // HandleEnvelope reads first the input and if everything is parsed fine, it sends 36 | // back another header, letter pair. 37 | void HandleEnvelope(uint64_t rpc_id, Envelope* envelope, EnvelopeWriter writer) final; 38 | }; 39 | 40 | class TestInterface final : public ServiceInterface { 41 | bool clear_ = false; 42 | 43 | public: 44 | void set_clear(bool c) { 45 | clear_ = c; 46 | } 47 | ConnectionBridge* CreateConnectionBridge() override { 48 | return new TestBridge{clear_}; 49 | } 50 | }; 51 | 52 | class ServerTest : public testing::Test { 53 | public: 54 | ServerTest(); 55 | 56 | protected: 57 | static void SetUpTestCase() { 58 | } 59 | 60 | static void TearDownTestCase() { 61 | } 62 | 63 | void SetUp() override; 64 | 65 | void TearDown() override; 66 | 67 | std::unique_ptr service_; 68 | std::unique_ptr server_; 69 | std::unique_ptr pool_; 70 | std::unique_ptr sock2_; 71 | 72 | ::boost::system::error_code ec_; 73 | uint16_t port_ = 0; 74 | }; 75 | 76 | } // namespace rpc 77 | } // namespace util 78 | -------------------------------------------------------------------------------- /util/rpc/service_descriptor.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (roman@ubimo.com) 3 | // 4 | #include "util/rpc/service_descriptor.h" 5 | 6 | namespace util { 7 | 8 | // RPC-Server side part. 9 | 10 | namespace rpc { 11 | 12 | ServiceDescriptor::ServiceDescriptor() { 13 | } 14 | 15 | ServiceDescriptor::~ServiceDescriptor() { 16 | } 17 | 18 | void ServiceDescriptor::SetOptions(size_t index, const MethodOptions& opts) { 19 | methods_[index].options = opts; 20 | } 21 | 22 | 23 | } // namespace rpc 24 | } // namespace util 25 | -------------------------------------------------------------------------------- /util/sentry/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(sentry sentry.cc) 2 | cxx_link(sentry http_client_lib) 3 | 4 | cxx_test(sentry_test http_v2 sentry http_test_lib LABELS CI) 5 | -------------------------------------------------------------------------------- /util/sentry/sentry.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include "util/asio/io_context.h" 8 | 9 | namespace util { 10 | 11 | void EnableSentry(IoContext* context); 12 | 13 | } // namespace util 14 | -------------------------------------------------------------------------------- /util/sentry/sentry_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/sentry/sentry.h" 5 | 6 | #include "base/logging.h" 7 | #include "strings/strcat.h" 8 | #include "util/fibers/fibers_ext.h" 9 | #include "util/http/http_testing.h" 10 | 11 | DECLARE_string(sentry_dsn); 12 | 13 | namespace util { 14 | 15 | using namespace boost; 16 | using namespace std; 17 | using namespace chrono_literals; 18 | namespace h2 = beast::http; 19 | 20 | namespace { 21 | 22 | } // namespace 23 | 24 | class SentryTest : public util::HttpBaseTest { 25 | protected: 26 | unsigned req_ = 0; 27 | }; 28 | 29 | 30 | TEST_F(SentryTest, Basic) { 31 | fibers_ext::Done done; 32 | 33 | listener_.RegisterCb("/api/id/store/", false, 34 | [this, done](const http::QueryArgs& args, http::HttpHandler::SendFunction* send) mutable { 35 | this->req_++; 36 | http::StringResponse resp = http::MakeStringResponse(h2::status::ok); 37 | done.Notify(); 38 | 39 | return send->Invoke(std::move(resp)); 40 | }); 41 | 42 | FLAGS_sentry_dsn = absl::StrCat("foo@localhost:", port_, "/id"); 43 | EnableSentry(&pool_->GetNextContext()); 44 | // this_fiber::sleep_for(10ms); 45 | 46 | LOG(ERROR) << "Try"; 47 | done.Wait(); 48 | 49 | EXPECT_EQ(1, req_); 50 | } 51 | 52 | 53 | } // namespace util 54 | -------------------------------------------------------------------------------- /util/sinksource.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #include "util/sinksource.h" 6 | #include "base/logging.h" 7 | #include "base/port.h" 8 | 9 | namespace util { 10 | 11 | Sink::WritableBuffer Sink::GetAppendBuffer(size_t min_capacity, WritableBuffer scratch, 12 | size_t /*desired_capacity_hint*/) { 13 | CHECK_GE(scratch.size(), min_capacity); 14 | return scratch; 15 | } 16 | 17 | Status Sink::Flush() { return Status::OK; } 18 | 19 | StatusObject Source::Read(const strings::MutableByteRange& range) { 20 | CHECK(!range.empty()); 21 | 22 | size_t read = 0; 23 | if (!prepend_buf_.empty()) { 24 | if (prepend_buf_.size() >= range.size()) { 25 | memcpy(range.begin(), prepend_buf_.begin(), range.size()); 26 | auto src = prepend_buf_.begin() + range.size(); 27 | size_t new_size = prepend_buf_.size() - range.size(); 28 | memmove(prepend_buf_.begin(), src, new_size); 29 | prepend_buf_.resize(new_size); 30 | 31 | return range.size(); 32 | } 33 | 34 | memcpy(range.begin(), prepend_buf_.begin(), prepend_buf_.size()); 35 | 36 | read = prepend_buf_.size(); 37 | prepend_buf_.clear(); 38 | DCHECK_LT(read, range.size()); 39 | 40 | } 41 | if (eof_) 42 | return read; 43 | 44 | auto lrange = range.subpiece(read); 45 | while (true) { 46 | auto res = ReadInternal(lrange); 47 | if (!res.ok()) 48 | return res; 49 | 50 | read += res.obj; 51 | eof_ = res.obj == 0; 52 | if (eof_ || res.obj == lrange.size()) 53 | break; 54 | lrange = lrange.subpiece(res.obj); 55 | } 56 | return read; 57 | } 58 | 59 | StatusObject StringSource::ReadInternal(const strings::MutableByteRange& range) { 60 | size_t to_fill = std::min({range.size(), block_size_, input_.size()}); 61 | memcpy(range.begin(), input_.begin(), to_fill); 62 | 63 | input_.remove_prefix(to_fill); 64 | return to_fill; 65 | } 66 | 67 | } // namespace util 68 | -------------------------------------------------------------------------------- /util/spawn.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include "util/spawn.h" 12 | 13 | extern "C" char **environ; 14 | 15 | namespace util { 16 | 17 | // Runs a child process efficiently. 18 | // argv must be null terminated array of arguments where the first item in the array is the base 19 | // name of the executable passed by path. 20 | // Returns 0 if child process run successfully and updates child_status with its return status. 21 | static int spawn(const char* path, char* argv[], int* child_status) { 22 | pid_t pid; 23 | // posix_spawn in this configuration calls vfork without duplicating parents 24 | // virtual memory for the child. That's all we want. 25 | int status = posix_spawn(&pid, path, NULL, NULL, argv, environ); 26 | if (status != 0) 27 | return status; 28 | 29 | if (waitpid(pid, child_status, 0) == -1) 30 | return errno; 31 | return 0; 32 | } 33 | 34 | int sh_exec(const char* cmd) { 35 | char sh_bin[] = "sh"; 36 | char arg1[] = "-c"; 37 | 38 | std::string cmd2(cmd); 39 | 40 | char* argv[] = {sh_bin, arg1, &cmd2.front(), NULL}; 41 | int child_status = 0; 42 | return spawn("/bin/sh", argv, &child_status); 43 | } 44 | 45 | } // namespace util 46 | -------------------------------------------------------------------------------- /util/spawn.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | namespace util { 7 | 8 | 9 | // Runs sh with the command. Returns 0 if succeeded. Child status is ignored. 10 | int sh_exec(const char* cmd); 11 | 12 | } // namespace util 13 | -------------------------------------------------------------------------------- /util/stats/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(stats_lib sliding_counter.cc varz_node.cc varz_stats.cc) 2 | cxx_link(stats_lib strings) 3 | cxx_test(sliding_counter_test stats_lib) 4 | -------------------------------------------------------------------------------- /util/stats/sliding_counter.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/stats/sliding_counter.h" 5 | 6 | #include "base/walltime.h" 7 | #include "base/pthread_utils.h" 8 | 9 | #include 10 | #include "base/init.h" 11 | #include "base/port.h" 12 | 13 | namespace util { 14 | 15 | static uint32 g_test_used = kuint32max; 16 | 17 | SlidingSecondBase::SlidingSecondBase() { 18 | } 19 | 20 | uint32 SlidingSecondBase::CurrentTime() { 21 | if (PREDICT_TRUE(g_test_used == kuint32max)) { 22 | return base::GetMonotonicMicrosFast() / base::kNumMicrosPerSecond; 23 | } 24 | return g_test_used; 25 | } 26 | 27 | 28 | void SlidingSecondBase::SetCurrentTime_Test(uint32 time_val) { 29 | g_test_used = time_val; 30 | } 31 | 32 | uint32 QPSCount::Get() const { 33 | constexpr unsigned kWinSize = decltype(window_)::SIZE - 1; 34 | 35 | return window_.SumLast(1, kWinSize) / kWinSize; // Average over kWinSize values. 36 | } 37 | 38 | 39 | } // namespace util 40 | -------------------------------------------------------------------------------- /util/stats/sliding_counter_test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/stats/sliding_counter.h" 5 | 6 | #include "base/gtest.h" 7 | #include "base/logging.h" 8 | 9 | namespace util { 10 | 11 | class SlidingCounterTest : public testing::Test { 12 | }; 13 | 14 | 15 | class SlidingSecondCounterTest : public testing::Test { 16 | }; 17 | 18 | TEST_F(SlidingSecondCounterTest, Basic) { 19 | SlidingSecondBase::SetCurrentTime_Test(1); 20 | SlidingSecondCounter<10,1> second_counter; 21 | second_counter.Inc(); 22 | second_counter.Inc(); 23 | EXPECT_EQ(2, second_counter.Sum()); 24 | SlidingSecondBase::SetCurrentTime_Test(2); 25 | EXPECT_EQ(2, second_counter.Sum()); 26 | EXPECT_EQ(0, second_counter.SumLast(0, 1)); 27 | EXPECT_EQ(2, second_counter.SumLast(1, 1)); 28 | EXPECT_EQ(0, second_counter.SumLast(2, 1)); 29 | second_counter.Inc(); 30 | EXPECT_EQ(1, second_counter.SumLast(0, 1)); 31 | EXPECT_EQ(3, second_counter.Sum()); 32 | 33 | SlidingSecondBase::SetCurrentTime_Test(11); 34 | EXPECT_EQ(1, second_counter.Sum()); 35 | SlidingSecondBase::SetCurrentTime_Test(12); 36 | EXPECT_EQ(0, second_counter.Sum()); 37 | EXPECT_EQ(0, second_counter.DecIfNotLess(1)); 38 | 39 | EXPECT_LE(sizeof(SlidingSecondCounter<10,1>), 44); 40 | } 41 | 42 | 43 | } // namespace util 44 | -------------------------------------------------------------------------------- /util/stats/varz_node.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | #include "base/RWSpinLock.h" 10 | #include "util/stats/varz_value.h" 11 | 12 | namespace util { 13 | 14 | class VarzListNode { 15 | public: 16 | typedef util::VarzValue AnyValue; 17 | 18 | explicit VarzListNode(const char* name); 19 | virtual ~VarzListNode(); 20 | 21 | // New interface. Func is a function accepting 'const char*' and VarzValue&&. 22 | static void Iterate(std::function f); 23 | 24 | // Old interface. Appends string representations of each active node in the list to res. 25 | // Used for outputting the current state. 26 | static void IterateValues(std::function cb) { 27 | Iterate([&](const char* name, AnyValue&& av) { cb(name, Format(av)); }); 28 | } 29 | 30 | protected: 31 | virtual AnyValue GetData() const = 0; 32 | 33 | const char* name_; 34 | 35 | static std::string Format(const AnyValue& av); 36 | 37 | private: 38 | // Returns the head to varz linked list. Note that the list becomes invalid after at least one 39 | // linked list node was destroyed. 40 | static VarzListNode*& global_list(); 41 | static folly::RWSpinLock g_varz_lock; 42 | 43 | VarzListNode* next_; 44 | VarzListNode* prev_; 45 | }; 46 | 47 | } // namespace util 48 | -------------------------------------------------------------------------------- /util/stats/varz_value.h: -------------------------------------------------------------------------------- 1 | // Copyright 2018, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | 9 | namespace util { 10 | 11 | class VarzValue { 12 | public: 13 | typedef std::vector> Map; 14 | 15 | 16 | // Should be union but too complicated to code it in c++11. 17 | int64_t num; 18 | double dbl; 19 | Map key_value_array; 20 | std::string str; 21 | 22 | enum Type { NUM, STRING, MAP, DOUBLE, TIME } type; 23 | 24 | VarzValue(std::string s) : str(std::move(s)), type(STRING) { 25 | } 26 | 27 | VarzValue(Map s) : key_value_array(std::move(s)), type(MAP) { 28 | } 29 | 30 | VarzValue(const VarzValue&) = default; 31 | 32 | VarzValue& operator=(const VarzValue&) = default; 33 | 34 | static VarzValue FromTime(time_t t) { 35 | return VarzValue{t, TIME}; 36 | } 37 | 38 | static VarzValue FromDouble(double d) { 39 | return VarzValue{d}; 40 | } 41 | 42 | static VarzValue FromInt(int64_t n) { 43 | return VarzValue{n, NUM}; 44 | } 45 | 46 | private: 47 | VarzValue(int64_t n, Type t) : num(n), type(t) {} 48 | VarzValue(double d) : dbl(d), type(DOUBLE) {} 49 | }; 50 | 51 | } // namespace util 52 | -------------------------------------------------------------------------------- /util/status.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | syntax = "proto2"; 5 | 6 | package util; 7 | 8 | message StatusCode { 9 | enum Code { 10 | OK = 0; 11 | CANCELLED = 1; 12 | NOT_IMPLEMENTED_ERROR = 2; 13 | RUNTIME_ERROR = 3; 14 | INTERNAL_ERROR = 4; 15 | INVALID_ARGUMENT = 5; 16 | IO_ERROR = 20; 17 | IO_TIMEOUT = 21; 18 | 19 | PARSE_ERROR = 31; 20 | 21 | RPC_INVALID_METHOD = 43; 22 | RPC_DEADLINE_EXCEEDED = 44; 23 | RPC_CONNECTION_REFUSED = 45; 24 | RPC_PARSE_ERROR = 46; 25 | // 47 26 | RPC_PROTOCOL_ERROR = 48; 27 | // 49 28 | RPC_APPLICATION_ERROR = 50; 29 | } 30 | } 31 | 32 | message StatusProto { 33 | required StatusCode.Code status_code = 1; 34 | repeated string error_msg = 2; 35 | } 36 | 37 | -------------------------------------------------------------------------------- /util/sync_stream_interface.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | 5 | #pragma once 6 | 7 | #include 8 | #include "base/expected.hpp" 9 | 10 | namespace util { 11 | 12 | class SyncStreamInterface { 13 | public: 14 | using expected_size_t = nonstd::expected; 15 | 16 | virtual ~SyncStreamInterface() {} 17 | virtual expected_size_t Send(const iovec* ptr, size_t len) = 0; 18 | virtual expected_size_t Recv(iovec* ptr, size_t len) = 0; 19 | }; 20 | 21 | } // namespace util 22 | -------------------------------------------------------------------------------- /util/uring/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(uring_fiber_lib accept_server.cc fiber_socket.cc http_handler.cc 2 | #we need prebuilt_asio for errrors support, consider using our own # 3 | prebuilt_asio.cc proactor.cc proactor_pool.cc sliding_counter.cc 4 | uring_fiber_algo.cc varz.cc) 5 | cxx_link(uring_fiber_lib base http_common absl::flat_hash_map Boost::fiber -luring) 6 | 7 | cxx_test(proactor_test uring_fiber_lib) 8 | cxx_test(accept_server_test uring_fiber_lib http_beast_prebuilt) 9 | -------------------------------------------------------------------------------- /util/uring/connection.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include 8 | #include "util/uring/fiber_socket.h" 9 | 10 | namespace util { 11 | namespace uring { 12 | 13 | class Proactor; 14 | class ListenerInterface; 15 | 16 | class Connection { 17 | using connection_hook_t = ::boost::intrusive::slist_member_hook< 18 | ::boost::intrusive::link_mode<::boost::intrusive::normal_link>>; 19 | connection_hook_t hook_; 20 | 21 | 22 | void SetSocket(FiberSocket&& s) { socket_ = std::move(s); } 23 | 24 | auto native_handle() const { return socket_.native_handle(); } 25 | 26 | public: 27 | using member_hook_t = 28 | ::boost::intrusive::member_hook; 29 | 30 | virtual ~Connection() {} 31 | 32 | protected: 33 | 34 | // The main loop for a connection. Runs in the same proactor thread as of socket_. 35 | virtual void HandleRequests() = 0; 36 | 37 | FiberSocket socket_; 38 | friend class ListenerInterface; 39 | }; 40 | 41 | } // namespace uring 42 | } // namespace util 43 | -------------------------------------------------------------------------------- /util/uring/prebuilt_asio.cc: -------------------------------------------------------------------------------- 1 | ../asio/prebuilt_asio.cc -------------------------------------------------------------------------------- /util/uring/sliding_counter.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include "util/uring/sliding_counter.h" 5 | 6 | #include "base/logging.h" 7 | #include "base/walltime.h" 8 | 9 | using namespace std; 10 | 11 | namespace util { 12 | namespace uring { 13 | namespace detail { 14 | 15 | uint32_t SlidingCounterTLBase::MoveTsIfNeeded(size_t size, int32_t* dest) const { 16 | uint32_t current_sec = base::GetClockMicros() / 1000000UL; 17 | if (last_ts_ + size <= current_sec) { 18 | std::fill(dest, dest + size, 0); 19 | } else { 20 | // Reset delta upto current_time including. 21 | for (uint32_t i = last_ts_ + 1; i <= current_sec; ++i) { 22 | dest[i % size] = 0; 23 | } 24 | } 25 | last_ts_ = current_sec; 26 | 27 | return current_sec % size; 28 | } 29 | 30 | void SlidingCounterBase::InitInternal(ProactorPool* pp) { 31 | CHECK(pp_ == nullptr); 32 | pp_ = CHECK_NOTNULL(pp); 33 | } 34 | 35 | void SlidingCounterBase::CheckInit() const { 36 | CHECK_NOTNULL(pp_); 37 | } 38 | 39 | unsigned SlidingCounterBase::ProactorThreadIndex() const { 40 | unsigned tnum = CHECK_NOTNULL(pp_)->size(); 41 | 42 | int32_t indx = Proactor::GetIndex(); 43 | CHECK_GE(indx, 0) << "Must be called from proactor thread!"; 44 | CHECK_LT(indx, tnum) << "Invalid thread index " << indx; 45 | 46 | return unsigned(indx); 47 | } 48 | 49 | } // namespace detail 50 | } // namespace uring 51 | } // namespace util 52 | -------------------------------------------------------------------------------- /util/uring/uring_fiber_algo.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #include 5 | #include 6 | 7 | namespace util { 8 | namespace uring { 9 | class Proactor; 10 | 11 | class UringFiberProps : public ::boost::fibers::fiber_properties { 12 | public: 13 | UringFiberProps(::boost::fibers::context* ctx) : fiber_properties(ctx) { 14 | } 15 | 16 | void set_name(std::string nm) { 17 | name_ = std::move(nm); 18 | } 19 | 20 | const std::string& name() const { 21 | return name_; 22 | } 23 | 24 | private: 25 | std::string name_; 26 | }; 27 | 28 | class UringFiberAlgo : public ::boost::fibers::algo::algorithm_with_properties { 29 | using ready_queue_type = ::boost::fibers::scheduler::ready_queue_type; 30 | 31 | public: 32 | using FiberContext = ::boost::fibers::context; 33 | using time_point = std::chrono::steady_clock::time_point; 34 | 35 | explicit UringFiberAlgo(Proactor* proactor); 36 | ~UringFiberAlgo(); 37 | 38 | void awakened(FiberContext* ctx, UringFiberProps& props) noexcept override; 39 | 40 | FiberContext* pick_next() noexcept override; 41 | 42 | void property_change(FiberContext* ctx, UringFiberProps& props) noexcept final; 43 | 44 | bool has_ready_fibers() const noexcept final; 45 | 46 | // suspend_until halts the thread in case there are no active fibers to run on it. 47 | // This is done by dispatcher fiber. 48 | void suspend_until(time_point const& abs_time) noexcept final; 49 | //] 50 | 51 | // This function is called from remote threads, to wake this thread in case it's sleeping. 52 | // In our case, "sleeping" means - might stuck the wait function waiting for completion events. 53 | void notify() noexcept final; 54 | 55 | private: 56 | ready_queue_type rqueue_; 57 | Proactor* proactor_; 58 | FiberContext* main_cntx_; 59 | timespec ts_; 60 | uint32_t ready_cnt_ = 0; 61 | }; 62 | 63 | } // namespace uring 64 | } // namespace util 65 | -------------------------------------------------------------------------------- /util/uring/varz.h: -------------------------------------------------------------------------------- 1 | // Copyright 2020, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include "absl/container/flat_hash_map.h" 7 | #include "absl/strings/string_view.h" 8 | #include "base/arena.h" 9 | #include "util/stats/varz_node.h" 10 | #include "util/uring/sliding_counter.h" 11 | 12 | #define DEFINE_VARZ(type, name) ::util::uring::type name(#name) 13 | 14 | namespace util { 15 | namespace uring { 16 | 17 | class VarzQps : public VarzListNode { 18 | public: 19 | explicit VarzQps(const char* varname) : VarzListNode(varname) { 20 | } 21 | 22 | void Init(ProactorPool* pp) { 23 | val_.Init(pp); 24 | } 25 | 26 | void Inc() { 27 | val_.Inc(); 28 | } 29 | 30 | private: 31 | virtual AnyValue GetData() const override; 32 | 33 | using Counter = SlidingCounter<7>; 34 | 35 | // 7-seconds window. We gather data based on the fully filled 6. 36 | Counter val_; 37 | }; 38 | 39 | class VarzMapAverage : public VarzListNode { 40 | using Counter = SlidingCounterTL<7>; 41 | using SumCnt = std::pair; 42 | using Map = absl::flat_hash_map; 43 | 44 | public: 45 | explicit VarzMapAverage(const char* varname) : VarzListNode(varname) { 46 | } 47 | ~VarzMapAverage(); 48 | 49 | void Init(ProactorPool* pp); 50 | 51 | void IncBy(absl::string_view key, int32_t delta) { 52 | auto& map = avg_map_[ProactorThreadIndex()]; 53 | auto it = map.find(key); 54 | if (it == map.end()) { 55 | it = FindSlow(key); 56 | } 57 | Inc(delta, &it->second); 58 | } 59 | 60 | private: 61 | void Inc(int32_t delta, SumCnt* dest) { 62 | dest->first.IncBy(delta); 63 | dest->second.Inc(); 64 | } 65 | 66 | virtual AnyValue GetData() const override; 67 | unsigned ProactorThreadIndex() const; 68 | Map::iterator FindSlow(absl::string_view key); 69 | 70 | ProactorPool* pp_ = nullptr; 71 | std::unique_ptr avg_map_; 72 | }; 73 | 74 | } // namespace uring 75 | } // namespace util 76 | -------------------------------------------------------------------------------- /util/zlib_source.h: -------------------------------------------------------------------------------- 1 | // Copyright 2013, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | 7 | #include 8 | 9 | #include "base/macros.h" 10 | #include "util/sinksource.h" 11 | 12 | namespace util { 13 | 14 | class ZlibSource : public Source { 15 | public: 16 | // Format key for constructor 17 | enum Format { 18 | // zlib will autodetect gzip header or deflate stream 19 | AUTO = 0, 20 | 21 | // GZIP streams have some extra header data for file attributes. 22 | GZIP = 1, 23 | 24 | // Simpler zlib stream format. 25 | ZLIB = 2, 26 | }; 27 | 28 | 29 | // buffer_size and format may be -1 for default of 64kB and GZIP format. 30 | explicit ZlibSource(Source* sub_source, Format format = AUTO); 31 | ~ZlibSource(); 32 | 33 | static bool IsZlibSource(Source* source); 34 | 35 | private: 36 | 37 | StatusObject ReadInternal(const strings::MutableByteRange& range) override; 38 | 39 | Source* sub_stream_; 40 | 41 | Format format_; 42 | z_stream zcontext_; 43 | std::unique_ptr buf_; 44 | 45 | int Inflate(); 46 | 47 | bool RefillInternal(); 48 | 49 | DISALLOW_EVIL_CONSTRUCTORS(ZlibSource); 50 | }; 51 | 52 | class ZlibSink : public Sink { 53 | public: 54 | // Takes ownership over sub-sink. 55 | explicit ZlibSink(Sink* sub, unsigned level = 0, size_t buf_size = 1 << 16); 56 | ~ZlibSink() final; 57 | 58 | Status Append(const strings::ByteRange& slice) final; 59 | Status Flush() final; 60 | 61 | private: 62 | std::unique_ptr sub_; 63 | std::unique_ptr buf_; 64 | size_t buf_size_; 65 | z_stream zcontext_; 66 | }; 67 | 68 | } // namespace util 69 | 70 | -------------------------------------------------------------------------------- /util/zstd_sinksource.h: -------------------------------------------------------------------------------- 1 | // Copyright 2017, Beeri 15. All rights reserved. 2 | // Author: Roman Gershman (romange@gmail.com) 3 | // 4 | #pragma once 5 | 6 | #include 7 | #include "util/sinksource.h" 8 | 9 | namespace util { 10 | 11 | class ZStdSink : public Sink { 12 | public: 13 | // Takes ownership over upstream. 14 | ZStdSink(Sink* upstream); 15 | ~ZStdSink(); 16 | 17 | Status Init(int level); 18 | Status Append(const strings::ByteRange& slice) override; 19 | Status Flush() override; 20 | static size_t CompressBound(size_t src_size); 21 | 22 | private: 23 | size_t buf_sz_; 24 | std::unique_ptr buf_; 25 | std::unique_ptr upstream_; 26 | void* zstd_handle_; 27 | }; 28 | 29 | class ZStdSource : public Source { 30 | public: 31 | explicit ZStdSource(Source* upstream); 32 | ~ZStdSource(); 33 | static bool HasValidHeader(Source* upstream); 34 | 35 | private: 36 | StatusObject ReadInternal(const strings::MutableByteRange& range) override; 37 | 38 | std::unique_ptr sub_stream_; 39 | void* zstd_handle_; 40 | std::unique_ptr buf_; 41 | strings::MutableByteRange buf_range_; 42 | }; 43 | 44 | 45 | } // namespace util 46 | --------------------------------------------------------------------------------
", v, "
"; 38 | } else { 39 | td_tag = absl::StrCat(""); 40 | } 41 | for (const auto& cell : row) { 42 | absl::StrAppend(&res, td_tag, cell, "