├── docs ├── api.rst ├── designimg │ ├── fig1.png │ ├── fig2.png │ ├── fig3.png │ ├── fig4.png │ ├── fig5.png │ └── fig6.png ├── index.rst └── introduction.rst ├── raster ├── 3rd │ ├── http_parser │ │ ├── http_parser.c │ │ ├── CONTRIBUTIONS │ │ ├── CMakeLists.txt │ │ └── LICENSE-MIT │ └── thrift │ │ ├── CMakeLists.txt │ │ ├── protocol │ │ ├── TProtocol.cpp │ │ └── TBase64Utils.h │ │ ├── cxxfunctional.h │ │ └── transport │ │ └── TTransportException.cpp ├── serializer │ ├── CMakeLists.txt │ ├── test │ │ └── CMakeLists.txt │ └── Serializer.h ├── CMakeLists.txt ├── net │ ├── CMakeLists.txt │ ├── NetUtil.h │ ├── Processor.h │ ├── ErrorEnum.cpp │ ├── EventTask.h │ ├── Service.h │ ├── Group.h │ ├── Acceptor.h │ ├── EventHandler.h │ ├── EventPool.h │ ├── NetHub.h │ ├── NetUtil.cpp │ ├── EventPool.cpp │ ├── Exception.h │ ├── Group.cpp │ ├── Channel.h │ ├── Transport.h │ ├── Acceptor.cpp │ ├── Transport.cpp │ └── NetHub.cpp ├── coroutine │ ├── CMakeLists.txt │ ├── FiberHub.h │ ├── FiberManager.h │ ├── Context.h │ ├── FiberHub.cpp │ ├── FiberManager.cpp │ ├── Fiber.h │ └── Fiber.cpp ├── protocol │ ├── binary │ │ ├── CMakeLists.txt │ │ ├── Processor.cpp │ │ ├── AsyncServer.h │ │ ├── SyncTransport.h │ │ ├── Processor.h │ │ ├── AsyncClient.h │ │ ├── SyncClient.h │ │ ├── SyncTransport.cpp │ │ ├── Transport.h │ │ ├── SyncClient.cpp │ │ └── AsyncClient.cpp │ ├── thrift │ │ ├── CMakeLists.txt │ │ ├── Util.h │ │ ├── Util.cpp │ │ ├── AsyncServer.h │ │ ├── SyncClient.h │ │ ├── Processor.h │ │ ├── AsyncClient.h │ │ └── Processor.cpp │ ├── http │ │ ├── CMakeLists.txt │ │ ├── test │ │ │ ├── CMakeLists.txt │ │ │ ├── HTTPHeadersTest.cpp │ │ │ └── CookieTest.cpp │ │ ├── HTTPException.cpp │ │ ├── SyncTransport.h │ │ ├── SyncTransport.cpp │ │ ├── gen_HTTPCommonHeaders.sh │ │ ├── AsyncServer.h │ │ ├── HTTPCommonHeaders.txt │ │ ├── AsyncServer.cpp │ │ ├── HTTPMethod.h │ │ ├── Processor.h │ │ ├── SyncClient.h │ │ ├── AsyncClient.h │ │ ├── RequestHandler.h │ │ ├── HTTPMethod.cpp │ │ ├── HTTPCommonHeaders.template.h │ │ ├── HTTPCommonHeaders.template.gperf │ │ ├── Processor.cpp │ │ ├── Transport.h │ │ └── ParseURL.h │ └── proto │ │ ├── CMakeLists.txt │ │ ├── plugin │ │ └── Config.h │ │ ├── IOBufInputStream.h │ │ ├── IOBufInputStream.cpp │ │ ├── AsyncServer.cpp │ │ ├── RpcController.h │ │ ├── Processor.h │ │ ├── SyncClient.h │ │ ├── AsyncClient.h │ │ └── AsyncServer.h ├── plugins │ └── flume │ │ ├── gen-cpp │ │ ├── scribe_constants.cpp │ │ ├── scribe_constants.h │ │ ├── Scribe_server.skeleton.cpp │ │ └── scribe_types.h │ │ ├── CMakeLists.txt │ │ ├── scribe.thrift │ │ ├── FlumeClient.h │ │ └── FlumeClient.cpp ├── raster-config.h.in ├── framework │ ├── CMakeLists.txt │ ├── test │ │ ├── CMakeLists.txt │ │ └── AutoTaskTest.cpp │ ├── FalconSender.h │ ├── Sampler.cpp │ ├── ConfigUtil.cpp │ ├── Signal.h │ ├── HubAdaptor.h │ ├── Config.h │ ├── Sampler.h │ ├── Degrader.cpp │ ├── HubAdaptor.cpp │ └── Degrader.h └── Portability.h ├── cmake-scripts ├── clang-toolchain.cmake ├── FindRaster.cmake ├── FindAccelerator.cmake ├── FindFlatbuffers.cmake └── FindGFlags.cmake ├── .bunder.yml ├── examples ├── http │ ├── CMakeLists.txt │ ├── server.json │ └── Server.cpp ├── flatbuffers │ ├── CMakeLists.txt │ ├── table.fbs │ ├── Helper.h │ └── server.json ├── empty │ ├── gen-cpp │ │ ├── Empty_constants.cpp │ │ ├── Empty_constants.h │ │ └── Empty_server.skeleton.cpp │ ├── CMakeLists.txt │ ├── Empty.thrift │ ├── server.json │ └── Server.cpp ├── proxy │ ├── gen-cpp │ │ ├── Proxy_constants.cpp │ │ ├── Proxy_constants.h │ │ └── Proxy_server.skeleton.cpp │ ├── CMakeLists.txt │ ├── Proxy.thrift │ └── server.json ├── parallel │ ├── gen-cpp │ │ ├── Parallel_constants.cpp │ │ ├── Parallel_constants.h │ │ └── Parallel_server.skeleton.cpp │ ├── CMakeLists.txt │ ├── Parallel.thrift │ └── server.json ├── pbrpc │ ├── Proxy.proto │ ├── server.json │ └── CMakeLists.txt └── gen.sh ├── .clang-format ├── .gitignore ├── README.md └── .travis.yml /docs/api.rst: -------------------------------------------------------------------------------- 1 | .. _api: 2 | 3 | API参考 4 | ======= 5 | -------------------------------------------------------------------------------- /raster/3rd/http_parser/http_parser.c: -------------------------------------------------------------------------------- 1 | http_parser_cpp.cpp -------------------------------------------------------------------------------- /docs/designimg/fig1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig1.png -------------------------------------------------------------------------------- /docs/designimg/fig2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig2.png -------------------------------------------------------------------------------- /docs/designimg/fig3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig3.png -------------------------------------------------------------------------------- /docs/designimg/fig4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig4.png -------------------------------------------------------------------------------- /docs/designimg/fig5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig5.png -------------------------------------------------------------------------------- /docs/designimg/fig6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Yeolar/raster/HEAD/docs/designimg/fig6.png -------------------------------------------------------------------------------- /cmake-scripts/clang-toolchain.cmake: -------------------------------------------------------------------------------- 1 | set(CMAKE_C_COMPILER clang) 2 | set(CMAKE_CXX_COMPILER clang++) 3 | -------------------------------------------------------------------------------- /.bunder.yml: -------------------------------------------------------------------------------- 1 | # bunder config 2 | 3 | project: 4 | raster 5 | 6 | build: 7 | path: './build' 8 | 9 | depend: 10 | - accelerator-1.2.4 11 | - flatbuffers-1.9.0 12 | -------------------------------------------------------------------------------- /raster/serializer/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_SERIALIZER_HDRS *.h) 4 | 5 | install(FILES ${RASTER_SERIALIZER_HDRS} DESTINATION include/raster/serializer) 6 | -------------------------------------------------------------------------------- /raster/3rd/http_parser/CONTRIBUTIONS: -------------------------------------------------------------------------------- 1 | Contributors must agree to the Contributor License Agreement before patches 2 | can be accepted. 3 | 4 | http://spreadsheets2.google.com/viewform?hl=en&formkey=dDJXOGUwbzlYaWM4cHN1MERwQS1CSnc6MQ 5 | -------------------------------------------------------------------------------- /raster/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Yeolar 2 | 3 | #file(GLOB RASTER_BASE_SRCS *.cpp) 4 | file(GLOB RASTER_BASE_HDRS *.h) 5 | 6 | #add_library(raster_base OBJECT ${RASTER_BASE_SRCS}) 7 | 8 | install(FILES ${RASTER_BASE_HDRS} DESTINATION include/raster) 9 | -------------------------------------------------------------------------------- /raster/net/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_NET_SRCS *.cpp) 4 | file(GLOB RASTER_NET_HDRS *.h) 5 | 6 | add_library(raster_net OBJECT ${RASTER_NET_SRCS}) 7 | 8 | install(FILES ${RASTER_NET_HDRS} DESTINATION include/raster/net) 9 | -------------------------------------------------------------------------------- /raster/coroutine/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_COROUTINE_SRCS *.cpp) 4 | file(GLOB RASTER_COROUTINE_HDRS *.h) 5 | 6 | add_library(raster_coroutine OBJECT ${RASTER_COROUTINE_SRCS}) 7 | 8 | install(FILES ${RASTER_COROUTINE_HDRS} DESTINATION include/raster/coroutine) 9 | -------------------------------------------------------------------------------- /examples/http/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | add_executable(httpserver 4 | Server.cpp 5 | ) 6 | target_link_libraries(httpserver raster_static) 7 | 8 | add_executable(httpserver-bench 9 | Bench.cpp 10 | ) 11 | target_link_libraries(httpserver-bench raster_static) 12 | 13 | -------------------------------------------------------------------------------- /examples/flatbuffers/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | add_executable(flatbuffers 4 | Server.cpp 5 | ) 6 | target_link_libraries(flatbuffers raster_static) 7 | 8 | add_executable(flatbuffers-bench 9 | Bench.cpp 10 | ) 11 | target_link_libraries(flatbuffers-bench raster_static) 12 | 13 | -------------------------------------------------------------------------------- /raster/protocol/binary/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_PROTOCOL_BINARY_SRCS *.cpp) 4 | file(GLOB RASTER_PROTOCOL_BINARY_HDRS *.h) 5 | 6 | add_library(raster_protocol_binary OBJECT ${RASTER_PROTOCOL_BINARY_SRCS}) 7 | 8 | install(FILES ${RASTER_PROTOCOL_BINARY_HDRS} DESTINATION include/raster/protocol/binary) 9 | -------------------------------------------------------------------------------- /raster/protocol/thrift/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_PROTOCOL_THRIFT_SRCS *.cpp) 4 | file(GLOB RASTER_PROTOCOL_THRIFT_HDRS *.h) 5 | 6 | add_library(raster_protocol_thrift OBJECT ${RASTER_PROTOCOL_THRIFT_SRCS}) 7 | 8 | install(FILES ${RASTER_PROTOCOL_THRIFT_HDRS} DESTINATION include/raster/protocol/thrift) 9 | -------------------------------------------------------------------------------- /raster/3rd/http_parser/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Yeolar 3 | # 4 | 5 | set(HTTPPARSER_SRCS 6 | http_parser_cpp.cpp 7 | ) 8 | add_library(httpparser OBJECT ${HTTPPARSER_SRCS}) 9 | 10 | add_executable(httpparser_test 11 | test.c 12 | http_parser.c 13 | ) 14 | add_test(httpparser_test httpparser_test CONFIGURATIONS ${CMAKE_BUILD_TYPE}) 15 | 16 | -------------------------------------------------------------------------------- /.clang-format: -------------------------------------------------------------------------------- 1 | # http://clang.llvm.org/docs/ClangFormatStyleOptions.html 2 | 3 | BasedOnStyle: Google 4 | IndentWidth: 2 5 | UseTab: Never 6 | AccessModifierOffset: -2 7 | IndentCaseLabels: false 8 | AllowShortBlocksOnASingleLine: false 9 | #AllowShortFunctionsOnASingleLine: Empty 10 | AllowShortIfStatementsOnASingleLine: false 11 | AllowShortLoopsOnASingleLine: false 12 | -------------------------------------------------------------------------------- /raster/plugins/flume/gen-cpp/scribe_constants.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "scribe_constants.h" 8 | 9 | namespace rdd { 10 | 11 | const scribeConstants g_scribe_constants; 12 | 13 | scribeConstants::scribeConstants() { 14 | } 15 | 16 | } // namespace 17 | 18 | -------------------------------------------------------------------------------- /examples/empty/gen-cpp/Empty_constants.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "Empty_constants.h" 8 | 9 | namespace rdd { namespace empty { 10 | 11 | const EmptyConstants g_Empty_constants; 12 | 13 | EmptyConstants::EmptyConstants() { 14 | } 15 | 16 | }} // namespace 17 | 18 | -------------------------------------------------------------------------------- /examples/proxy/gen-cpp/Proxy_constants.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "Proxy_constants.h" 8 | 9 | namespace rdd { namespace proxy { 10 | 11 | const ProxyConstants g_Proxy_constants; 12 | 13 | ProxyConstants::ProxyConstants() { 14 | } 15 | 16 | }} // namespace 17 | 18 | -------------------------------------------------------------------------------- /raster/raster-config.h.in: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | /* meta */ 4 | 5 | #define RDD_PACKAGE "@PROJECT_NAME@" 6 | #define RDD_VERSION "@VERSION_MAJOR@.@VERSION_MINOR@.@VERSION_PATCH@" 7 | 8 | /* compile tests for keyword, lib, function, ... */ 9 | 10 | #cmakedefine RDD_HAVE_XSI_STRERROR_R 1 11 | 12 | /* gflags */ 13 | 14 | #cmakedefine RDD_GFLAGS_NAMESPACE @RDD_GFLAGS_NAMESPACE@ 15 | #cmakedefine RDD_UNUSUAL_GFLAGS_NAMESPACE 1 16 | -------------------------------------------------------------------------------- /raster/framework/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_FRAMEWORK_SRCS *.cpp) 4 | file(GLOB RASTER_FRAMEWORK_HDRS *.h) 5 | 6 | add_library(raster_framework OBJECT ${RASTER_FRAMEWORK_SRCS}) 7 | 8 | install(FILES ${RASTER_FRAMEWORK_HDRS} DESTINATION include/raster/framework) 9 | 10 | add_executable(configutil 11 | ConfigUtil.cpp 12 | ) 13 | target_link_libraries(configutil 14 | raster_static) 15 | -------------------------------------------------------------------------------- /examples/parallel/gen-cpp/Parallel_constants.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #include "Parallel_constants.h" 8 | 9 | namespace rdd { namespace parallel { 10 | 11 | const ParallelConstants g_Parallel_constants; 12 | 13 | ParallelConstants::ParallelConstants() { 14 | } 15 | 16 | }} // namespace 17 | 18 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. Raster documentation master file, created by 2 | sphinx-quickstart on Thu Sep 7 13:34:40 2017. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Raster文档 7 | ========== 8 | 9 | .. _contents: 10 | 11 | 目录 12 | ==== 13 | 14 | .. toctree:: 15 | :maxdepth: 2 16 | 17 | introduction 18 | tutorial 19 | moreapps 20 | design 21 | api 22 | 23 | -------------------------------------------------------------------------------- /examples/empty/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | add_executable(empty 4 | gen-cpp/Empty_constants.cpp 5 | gen-cpp/Empty.cpp 6 | gen-cpp/Empty_types.cpp 7 | Server.cpp 8 | ) 9 | target_link_libraries(empty raster_static) 10 | 11 | add_executable(empty-bench 12 | gen-cpp/Empty_constants.cpp 13 | gen-cpp/Empty.cpp 14 | gen-cpp/Empty_types.cpp 15 | Bench.cpp 16 | ) 17 | target_link_libraries(empty-bench raster_static) 18 | 19 | -------------------------------------------------------------------------------- /examples/proxy/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | add_executable(proxy 4 | gen-cpp/Proxy_constants.cpp 5 | gen-cpp/Proxy.cpp 6 | gen-cpp/Proxy_types.cpp 7 | Server.cpp 8 | ) 9 | target_link_libraries(proxy raster_static) 10 | 11 | add_executable(proxy-bench 12 | gen-cpp/Proxy_constants.cpp 13 | gen-cpp/Proxy.cpp 14 | gen-cpp/Proxy_types.cpp 15 | Bench.cpp 16 | ) 17 | target_link_libraries(proxy-bench raster_static) 18 | 19 | -------------------------------------------------------------------------------- /raster/protocol/http/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_PROTOCOL_HTTP_SRCS *.cpp) 4 | file(GLOB RASTER_PROTOCOL_HTTP_HDRS *.h) 5 | file(GLOB RASTER_PROTOCOL_HTTP_HDRS_EX HTTPCommonHeaders.template.h) 6 | list(REMOVE_ITEM RASTER_PROTOCOL_HTTP_HDRS ${RASTER_PROTOCOL_HTTP_HDRS_EX}) 7 | 8 | add_library(raster_protocol_http OBJECT ${RASTER_PROTOCOL_HTTP_SRCS}) 9 | 10 | install(FILES ${RASTER_PROTOCOL_HTTP_HDRS} DESTINATION include/raster/protocol/http) 11 | -------------------------------------------------------------------------------- /examples/flatbuffers/table.fbs: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2017, Yeolar 2 | 3 | namespace rdd.fbs; 4 | 5 | table Query { 6 | traceid: string; 7 | query: string; 8 | forward: string; 9 | } 10 | 11 | enum ResultCode:int { 12 | OK = 0, 13 | 14 | // WARNING <1000 15 | 16 | // ERROR >1000 17 | E_SOURCE__UNTRUSTED = 1001, 18 | E_BACKEND_FAILURE = 1002, 19 | } 20 | 21 | table Result { 22 | traceid: string; 23 | code: ResultCode; 24 | } 25 | 26 | -------------------------------------------------------------------------------- /examples/flatbuffers/Helper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017, Yeolar 3 | */ 4 | 5 | #pragma once 6 | 7 | #include "flatbuffers/flatbuffers.h" 8 | #include "accelerator/Range.h" 9 | 10 | namespace rdd { 11 | namespace fbs { 12 | 13 | template 14 | bool verifyFlatbuffer(T* object, const acc::ByteRange& range) { 15 | flatbuffers::Verifier verifier(range.data(), range.size()); 16 | return object->Verify(verifier); 17 | } 18 | 19 | } // namespace fbs 20 | } // namespace rdd 21 | -------------------------------------------------------------------------------- /raster/plugins/flume/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB_RECURSE RASTER_PLUGINS_FLUME_SRCS *.cpp) 4 | file(GLOB_RECURSE RASTER_PLUGINS_FLUME_SRCS_EX *.skeleton.cpp) 5 | file(GLOB_RECURSE RASTER_PLUGINS_FLUME_HDRS *.h) 6 | list(REMOVE_ITEM RASTER_PLUGINS_FLUME_SRCS ${RASTER_PLUGINS_FLUME_SRCS_EX}) 7 | 8 | add_library(raster_plugins_flume OBJECT ${RASTER_PLUGINS_FLUME_SRCS}) 9 | 10 | install(FILES ${RASTER_PLUGINS_FLUME_HDRS} DESTINATION include/raster/plugins/flume) 11 | -------------------------------------------------------------------------------- /examples/parallel/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | add_executable(parallel 4 | gen-cpp/Parallel_constants.cpp 5 | gen-cpp/Parallel.cpp 6 | gen-cpp/Parallel_types.cpp 7 | Server.cpp 8 | ) 9 | target_link_libraries(parallel raster_static) 10 | 11 | add_executable(parallel-bench 12 | gen-cpp/Parallel_constants.cpp 13 | gen-cpp/Parallel.cpp 14 | gen-cpp/Parallel_types.cpp 15 | Bench.cpp 16 | ) 17 | target_link_libraries(parallel-bench raster_static) 18 | 19 | -------------------------------------------------------------------------------- /raster/framework/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | set(RASTER_FRAMEWORK_TEST_SRCS 4 | AutoTaskTest.cpp 5 | ) 6 | 7 | foreach(test_src ${RASTER_FRAMEWORK_TEST_SRCS}) 8 | get_filename_component(test_name ${test_src} NAME_WE) 9 | set(test raster_framework_${test_name}) 10 | add_executable(${test} ${test_src}) 11 | target_link_libraries(${test} ${GTEST_BOTH_LIBRARIES} raster_static) 12 | add_test(${test} ${test} CONFIGURATIONS ${CMAKE_BUILD_TYPE}) 13 | endforeach() 14 | -------------------------------------------------------------------------------- /raster/serializer/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | set(RASTER_SERIALIZER_TEST_SRCS 4 | SerializerTest.cpp 5 | ) 6 | 7 | foreach(test_src ${RASTER_SERIALIZER_TEST_SRCS}) 8 | get_filename_component(test_name ${test_src} NAME_WE) 9 | set(test raster_serializer_${test_name}) 10 | add_executable(${test} ${test_src}) 11 | target_link_libraries(${test} ${GTEST_BOTH_LIBRARIES} raster_static) 12 | add_test(${test} ${test} CONFIGURATIONS ${CMAKE_BUILD_TYPE}) 13 | endforeach() 14 | -------------------------------------------------------------------------------- /raster/plugins/flume/gen-cpp/scribe_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef scribe_CONSTANTS_H 8 | #define scribe_CONSTANTS_H 9 | 10 | #include "scribe_types.h" 11 | 12 | namespace rdd { 13 | 14 | class scribeConstants { 15 | public: 16 | scribeConstants(); 17 | 18 | }; 19 | 20 | extern const scribeConstants g_scribe_constants; 21 | 22 | } // namespace 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /examples/empty/gen-cpp/Empty_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef Empty_CONSTANTS_H 8 | #define Empty_CONSTANTS_H 9 | 10 | #include "Empty_types.h" 11 | 12 | namespace rdd { namespace empty { 13 | 14 | class EmptyConstants { 15 | public: 16 | EmptyConstants(); 17 | 18 | }; 19 | 20 | extern const EmptyConstants g_Empty_constants; 21 | 22 | }} // namespace 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /examples/proxy/gen-cpp/Proxy_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef Proxy_CONSTANTS_H 8 | #define Proxy_CONSTANTS_H 9 | 10 | #include "Proxy_types.h" 11 | 12 | namespace rdd { namespace proxy { 13 | 14 | class ProxyConstants { 15 | public: 16 | ProxyConstants(); 17 | 18 | }; 19 | 20 | extern const ProxyConstants g_Proxy_constants; 21 | 22 | }} // namespace 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /raster/protocol/proto/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB RASTER_PROTOCOL_PROTO_SRCS *.cpp) 4 | file(GLOB RASTER_PROTOCOL_PROTO_HDRS *.h) 5 | 6 | add_library(raster_protocol_proto OBJECT ${RASTER_PROTOCOL_PROTO_SRCS}) 7 | 8 | install(FILES ${RASTER_PROTOCOL_PROTO_HDRS} DESTINATION include/raster/protocol/proto) 9 | 10 | add_executable(pbrpc_plugin 11 | plugin/RpcGenerator.cpp 12 | plugin/RpcPlugin.cpp 13 | ) 14 | target_link_libraries(pbrpc_plugin 15 | ${PROTOBUF_PROTOC_LIBRARIES} 16 | ) 17 | -------------------------------------------------------------------------------- /examples/parallel/gen-cpp/Parallel_constants.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef Parallel_CONSTANTS_H 8 | #define Parallel_CONSTANTS_H 9 | 10 | #include "Parallel_types.h" 11 | 12 | namespace rdd { namespace parallel { 13 | 14 | class ParallelConstants { 15 | public: 16 | ParallelConstants(); 17 | 18 | }; 19 | 20 | extern const ParallelConstants g_Parallel_constants; 21 | 22 | }} // namespace 23 | 24 | #endif 25 | -------------------------------------------------------------------------------- /raster/protocol/http/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | set(RASTER_PROTOCOL_HTTP_TEST_SRCS 4 | HTTPHeadersTest.cpp 5 | UtilTest.cpp 6 | ) 7 | 8 | foreach(test_src ${RASTER_PROTOCOL_HTTP_TEST_SRCS}) 9 | get_filename_component(test_name ${test_src} NAME_WE) 10 | set(test raster_protocol_http_${test_name}) 11 | add_executable(${test} ${test_src}) 12 | target_link_libraries(${test} ${GTEST_BOTH_LIBRARIES} raster_static) 13 | add_test(${test} ${test} CONFIGURATIONS ${CMAKE_BUILD_TYPE}) 14 | endforeach() 15 | 16 | -------------------------------------------------------------------------------- /raster/3rd/thrift/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2017 Yeolar 2 | 3 | file(GLOB_RECURSE THRIFT_SRCS *.cpp) 4 | file(GLOB THRIFT_HDRS *.h) 5 | file(GLOB THRIFT_PROTOCOL_HDRS protocol/*.h protocol/*.tcc) 6 | file(GLOB THRIFT_TRANSPORT_HDRS transport/*.h) 7 | 8 | add_library(thrift OBJECT ${THRIFT_SRCS}) 9 | 10 | install(FILES ${THRIFT_HDRS} DESTINATION include/raster/3rd/thrift) 11 | install(FILES ${THRIFT_PROTOCOL_HDRS} DESTINATION include/raster/3rd/thrift/protocol) 12 | install(FILES ${THRIFT_TRANSPORT_HDRS} DESTINATION include/raster/3rd/thrift/transport) 13 | -------------------------------------------------------------------------------- /examples/empty/Empty.thrift: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2017, Yeolar 2 | 3 | namespace cpp rdd.empty 4 | 5 | struct Query { 6 | 1: required string traceid; 7 | 2: optional string query; 8 | } 9 | 10 | enum ResultCode { 11 | OK, // 0 12 | 13 | // WARNING <1000 14 | 15 | // ERROR >1000 16 | E_SOURCE__UNTRUSTED = 1001, 17 | E_BACKEND_FAILURE = 1002, 18 | } 19 | 20 | struct Result { 21 | 1: required string traceid; 22 | 2: optional ResultCode code; 23 | } 24 | 25 | service Empty { 26 | Result run(1: Query query); 27 | } 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | *.obj 6 | 7 | # Precompiled Headers 8 | *.gch 9 | *.pch 10 | 11 | # Compiled Dynamic libraries 12 | *.so 13 | *.dylib 14 | *.dll 15 | 16 | # Fortran module files 17 | *.mod 18 | *.smod 19 | 20 | # Python compiled files 21 | *.py[cod] 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | 34 | ### 35 | 36 | *.pb.h 37 | *.pb.cc 38 | 39 | /build/ 40 | /build-deps/ 41 | /docs/_build/ 42 | /raster/raster-config.h 43 | -------------------------------------------------------------------------------- /examples/parallel/Parallel.thrift: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2017, Yeolar 2 | 3 | namespace cpp rdd.parallel 4 | 5 | struct Query { 6 | 1: required string traceid; 7 | 2: optional string query; 8 | } 9 | 10 | enum ResultCode { 11 | OK, // 0 12 | 13 | // WARNING <1000 14 | 15 | // ERROR >1000 16 | E_SOURCE__UNTRUSTED = 1001, 17 | E_BACKEND_FAILURE = 1002, 18 | } 19 | 20 | struct Result { 21 | 1: required string traceid; 22 | 2: optional ResultCode code; 23 | } 24 | 25 | service Parallel { 26 | Result run(1: Query query); 27 | } 28 | -------------------------------------------------------------------------------- /examples/proxy/Proxy.thrift: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2017, Yeolar 2 | 3 | namespace cpp rdd.proxy 4 | 5 | struct Query { 6 | 1: required string traceid; 7 | 2: optional string query; 8 | 3: optional string forward; 9 | } 10 | 11 | enum ResultCode { 12 | OK, // 0 13 | 14 | // WARNING <1000 15 | 16 | // ERROR >1000 17 | E_SOURCE__UNTRUSTED = 1001, 18 | E_BACKEND_FAILURE = 1002, 19 | } 20 | 21 | struct Result { 22 | 1: required string traceid; 23 | 2: optional ResultCode code; 24 | } 25 | 26 | service Proxy { 27 | Result run(1: Query query); 28 | } 29 | -------------------------------------------------------------------------------- /examples/pbrpc/Proxy.proto: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2017, Yeolar 2 | 3 | syntax = "proto2"; 4 | option cc_generic_services = true; 5 | 6 | package rdd.pbrpc; 7 | 8 | message Query { 9 | optional string traceid = 1; 10 | optional string query = 2; 11 | optional string forward = 3; 12 | } 13 | 14 | enum ResultCode { 15 | OK = 0; 16 | 17 | // WARNING <1000 18 | 19 | // ERROR >1000 20 | E_SOURCE__UNTRUSTED = 1001; 21 | E_BACKEND_FAILURE = 1002; 22 | } 23 | 24 | message Result { 25 | optional string traceid = 1; 26 | optional ResultCode code = 2; 27 | } 28 | 29 | service ProxyService { 30 | rpc run(Query) returns(Result); 31 | } 32 | -------------------------------------------------------------------------------- /examples/gen.sh: -------------------------------------------------------------------------------- 1 | ROOT=`pwd` 2 | 3 | # empty 4 | echo "Generate files for Empty.thrift" 5 | cd $ROOT/empty && thrift --gen cpp -out gen-cpp Empty.thrift 6 | 7 | # proxy 8 | echo "Generate files for Proxy.thrift" 9 | cd $ROOT/proxy && thrift --gen cpp -out gen-cpp Proxy.thrift 10 | 11 | # parallel 12 | echo "Generate files for Parallel.thrift" 13 | cd $ROOT/parallel && thrift --gen cpp -out gen-cpp Parallel.thrift 14 | 15 | # pbrpc 16 | echo "Generate files for Proxy.proto" 17 | cd $ROOT/pbrpc && protoc --cpp_out=. Proxy.proto 18 | 19 | # flatbuffers 20 | if [ `command -v flatc` ]; then 21 | echo "Generate files for table.fbs" 22 | cd $ROOT/flatbuffers && flatc --cpp table.fbs 23 | fi 24 | -------------------------------------------------------------------------------- /examples/http/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/http.log", 4 | "level": -5, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Http", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "http" 28 | }, 29 | "job": { 30 | "graph": { 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/empty/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/empty.log", 4 | "level": 1, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Empty", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "empty" 28 | }, 29 | "job": { 30 | "graph": { 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/pbrpc/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/pbrpc.log", 4 | "level": 1, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Proxy", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "pbrpc" 28 | }, 29 | "job": { 30 | "graph": { 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/proxy/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/proxy.log", 4 | "level": 1, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Proxy", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "proxy" 28 | }, 29 | "job": { 30 | "graph": { 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /examples/flatbuffers/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/binary.log", 4 | "level": -5, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Proxy", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "binary" 28 | }, 29 | "job": { 30 | "graph": { 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.org/Yeolar/raster.svg?branch=master)](https://travis-ci.org/Yeolar/raster) 2 | 3 | raster 4 | ====== 5 | 6 | raster是一个完整的高性能C++协程服务框架，项目初衷是可以快速完成C++服务的开发。 7 | 8 | 该框架注重于轻量、易用、扩展性，同时，已经支持了监控统计、存储、数据库等实用需求。 9 | 10 | 框架主要特性： 11 | 12 | - 支持异步，协程 13 | - 支持多个后端并发请求 14 | - 支持binary（flatbuffers）、thrift、pbrpc、http协议 15 | - 支持并行计算 16 | - 支持监控统计、存储、数据库等扩展功能 17 | 18 | 依赖包括 Boost CURL GFlags OpenSSL Protobuf ZLIB。 19 | 20 | 编译安装 21 | 22 | $ mkdir build && cd build 23 | $ make -j8 24 | $ sudo make install 25 | 26 | 运行demo 27 | 28 | $ ./examples/empty/empty -conf ../examples/empty/server.json 29 | $ ./examples/empty/empty-bench -count 1000 30 | 31 | 详细的介绍和使用方法请参考 [Raster](https://www.rddoc.com/doc/Raster/1.0.0/zh/) 文档。 32 | -------------------------------------------------------------------------------- /raster/serializer/Serializer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/serializer/BinarySerializer.h" 20 | #include "raster/serializer/ThriftSerializer.h" 21 | -------------------------------------------------------------------------------- /raster/Portability.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/raster-config.h" 22 | 23 | #if RDD_UNUSUAL_GFLAGS_NAMESPACE 24 | namespace RDD_GFLAGS_NAMESPACE { } 25 | namespace gflags { 26 | using namespace RDD_GFLAGS_NAMESPACE; 27 | } // namespace gflags 28 | #endif 29 | 30 | -------------------------------------------------------------------------------- /examples/pbrpc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # Copyright (C) 2017, Yeolar 2 | 3 | set(EXAMPLES_PBRPC_PROTOS 4 | Proxy.proto 5 | ) 6 | 7 | foreach(proto ${EXAMPLES_PBRPC_PROTOS}) 8 | execute_process(COMMAND 9 | protoc --cpp_out=${PROJECT_SOURCE_DIR} 10 | --proto_path=${PROJECT_SOURCE_DIR} 11 | ${PROJECT_SOURCE_DIR}/examples/pbrpc/${proto}) 12 | # execute_process(COMMAND 13 | # protoc --plugin=protoc-gen-pbrpc_plugin=${PROJECT_BINARY_DIR}/raster/protocol/proto/pbrpc_plugin 14 | # --pbrpc_plugin_out=${PROJECT_SOURCE_DIR} 15 | # --proto_path=${PROJECT_SOURCE_DIR} 16 | # ${PROJECT_SOURCE_DIR}/examples/pbrpc/${proto}) 17 | endforeach() 18 | 19 | add_executable(pbrpc 20 | Proxy.pb.cc 21 | Server.cpp 22 | ) 23 | target_link_libraries(pbrpc raster_static) 24 | 25 | add_executable(pbrpc-bench 26 | Proxy.pb.cc 27 | Bench.cpp 28 | ) 29 | target_link_libraries(pbrpc-bench raster_static) 30 | 31 | -------------------------------------------------------------------------------- /raster/protocol/thrift/Util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/3rd/thrift/transport/TBufferTransports.h" 20 | 21 | namespace rdd { 22 | namespace thrift { 23 | 24 | void setSeqId(::apache::thrift::transport::TMemoryBuffer* buf, int32_t seqid); 25 | 26 | int32_t getSeqId(::apache::thrift::transport::TMemoryBuffer* buf); 27 | 28 | } // namespace thrift 29 | } // namespace rdd 30 | -------------------------------------------------------------------------------- /cmake-scripts/FindRaster.cmake: -------------------------------------------------------------------------------- 1 | # Find raster 2 | # 3 | # RASTER_INCLUDE_DIR - where to find raster/*. 4 | # RASTER_LIBRARY - List of libraries when using raster. 5 | # RASTER_FOUND - True if raster found. 6 | 7 | find_package(PkgConfig) 8 | pkg_check_modules(RASTER QUIET raster) 9 | 10 | if(RASTER_INCLUDE_DIR) 11 | # Already in cache, be silent 12 | set(RASTER_FIND_QUIETLY TRUE) 13 | endif() 14 | 15 | find_path(RASTER_INCLUDE_DIR raster/raster-config.h PATHS 16 | ${PROJECT_BINARY_DIR}/raster/raster/include 17 | ${PROJECT_BINARY_DIR}-deps/raster/raster/include) 18 | find_library(RASTER_LIBRARY raster PATHS 19 | ${PROJECT_BINARY_DIR}/raster/raster/lib 20 | ${PROJECT_BINARY_DIR}-deps/raster/raster/lib) 21 | 22 | # handle the QUIETLY and REQUIRED arguments and set RASTER_FOUND to TRUE 23 | # if all listed variables are TRUE 24 | include(FindPackageHandleStandardArgs) 25 | find_package_handle_standard_args(RASTER DEFAULT_MSG RASTER_LIBRARY RASTER_INCLUDE_DIR) 26 | 27 | mark_as_advanced(RASTER_LIBRARY RASTER_INCLUDE_DIR) 28 | -------------------------------------------------------------------------------- /raster/net/NetUtil.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "accelerator/event/EventUtil.h" 22 | #include "raster/net/Peer.h" 23 | 24 | namespace rdd { 25 | 26 | typedef acc::TimeoutOption TimeoutOption; 27 | 28 | struct ClientOption { 29 | Peer peer; 30 | TimeoutOption timeout; 31 | }; 32 | 33 | std::string getNodeName(); 34 | 35 | std::string getNodeIp(); 36 | 37 | } // namespace rdd 38 | -------------------------------------------------------------------------------- /examples/parallel/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "logging": { 3 | "logfile": "log/parallel.log", 4 | "level": 1, 5 | "async": true 6 | }, 7 | "service": { 8 | "8000": { 9 | "service": "Parallel", 10 | "conn_timeout": 100000, 11 | "recv_timeout": 300000, 12 | "send_timeout": 1000000 13 | } 14 | }, 15 | "thread": { 16 | "io": { "thread_count": 4, "bindcpu": false }, 17 | "0": { "thread_count": 4, "bindcpu": false } 18 | }, 19 | "net": { 20 | "forwarding": false, 21 | "copy": [ 22 | //{"port": 9090, "fhost": "", "fport": 9091, "flow": 100} 23 | ] 24 | }, 25 | "monitor": { 26 | "open": false, 27 | "prefix": "parallel" 28 | }, 29 | "job": { 30 | "graph": { 31 | "graph1": [ 32 | {"name": "ParallelJobExecutor:1", "next": ["ParallelJobExecutor:3"]}, 33 | {"name": "ParallelJobExecutor:2", "next": ["ParallelJobExecutor:3"]}, 34 | {"name": "ParallelJobExecutor:3", "next": []}, 35 | {"name": "ParallelJobExecutor:4", "next": []} 36 | ] 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /raster/coroutine/FiberHub.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/concurrency/CPUThreadPoolExecutor.h" 20 | #include "raster/coroutine/Fiber.h" 21 | 22 | namespace rdd { 23 | 24 | class FiberHub { 25 | public: 26 | virtual acc::CPUThreadPoolExecutor* getCPUThreadPoolExecutor(int poolId) = 0; 27 | 28 | void execute(Fiber* fiber, int poolId); 29 | void execute(std::unique_ptr task, int poolId); 30 | }; 31 | 32 | } // namespace rdd 33 | -------------------------------------------------------------------------------- /raster/plugins/flume/scribe.thrift: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/thrift --gen cpp:pure_enums --gen php 2 | 3 | ## Copyright (c) 2007-2008 Facebook 4 | ## 5 | ## Licensed under the Apache License, Version 2.0 (the "License"); 6 | ## you may not use this file except in compliance with the License. 7 | ## You may obtain a copy of the License at 8 | ## 9 | ## http://www.apache.org/licenses/LICENSE-2.0 10 | ## 11 | ## Unless required by applicable law or agreed to in writing, software 12 | ## distributed under the License is distributed on an "AS IS" BASIS, 13 | ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | ## See the License for the specific language governing permissions and 15 | ## limitations under the License. 16 | ## 17 | ## See accompanying file LICENSE or visit the Scribe site at: 18 | ## http://developers.facebook.com/scribe/ 19 | 20 | namespace cpp rdd 21 | 22 | enum ResultCode 23 | { 24 | OK, 25 | TRY_LATER 26 | } 27 | 28 | struct LogEntry 29 | { 30 | 1: string category, 31 | 2: string message 32 | } 33 | 34 | service Scribe 35 | { 36 | ResultCode Log(1: list messages); 37 | } 38 | -------------------------------------------------------------------------------- /docs/introduction.rst: -------------------------------------------------------------------------------- 1 | .. _introduction: 2 | 3 | 介绍 4 | ==== 5 | 6 | raster是什么？ 7 | -------------- 8 | 9 | RPC框架是后端架构的基石。也许这么说对于使用HTTP的Web服务不太公平，但就现在而言，由于分布式架构的大量需求，无论是搜索、存储，还是各类计算引擎，基本都建立在RPC之上。 10 | 11 | raster是一个完整的C++ RPC框架。 12 | 13 | 也许您在想，RPC框架已经有成熟的gRPC、thrift等等了，raster有什么特别之处呢？ 14 | 15 | 开发raster的想法开始于2015年初。它是作为后端服务的基础组件来开发的，我需要它可以满足基础库的需求；其次我可能会使用多种RPC协议，它要具有多协议支持的能力。当时能够满足这两点的比较出色的框架，就是Facebook的 `folly `_ + `wangle `_ 组合了。但是folly相对比较重，如果想要做定制的话，有些复杂。因此，我借鉴了它的一些思想和内容，但更加注重于轻量、易用、扩展性，同时，增加了并行计算、监控统计、存储、数据库等实用需求。 16 | 17 | raster的主要特性包括： 18 | 19 | - 包含丰富的基础库（有一部分源自 `folly `_ ） 20 | - 支持异步、协程 21 | - 支持多个后端并发请求 22 | - 支持binary、thrift、pbrpc协议 23 | - 支持并行计算 24 | - 支持监控统计、存储、数据库等扩展功能 25 | 26 | raster目前用在哪里？ 27 | -------------------- 28 | 29 | raster的开发初衷是可以用来快速完成C++服务的开发，目前用于 `rddoc.com `_ 的搜索、持久化KV存储、代理服务等。 30 | 31 | raster未来会如何发展？ 32 | ---------------------- 33 | 34 | 分布式系统是目前的热点领域，围绕着存储、计算、机器学习等等，业界做了很多框架，但多数都是依自身需要从头做起，通用性上存在不足。 35 | 36 | 我打算在raster的基础上，开发服务治理、函数计算引擎。以期能够在分布式计算领域实现一整套的解决方案。raster本身作为基础组件提供支持，在未来会在性能、协议、功能等方面进一步优化和扩展。 37 | -------------------------------------------------------------------------------- /raster/3rd/thrift/protocol/TProtocol.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "TProtocol.h" 21 | 22 | namespace apache { 23 | namespace thrift { 24 | namespace protocol { 25 | 26 | uint32_t TProtocol::skip_virt(TType type) { 27 | return ::apache::thrift::protocol::skip(*this, type); 28 | } 29 | 30 | }}} // apache::thrift::protocol 31 | -------------------------------------------------------------------------------- /cmake-scripts/FindAccelerator.cmake: -------------------------------------------------------------------------------- 1 | # Find accelerator 2 | # 3 | # ACCELERATOR_INCLUDE_DIR - where to find accelerator/*. 4 | # ACCELERATOR_LIBRARY - List of libraries when using accelerator. 5 | # ACCELERATOR_FOUND - True if accelerator found. 6 | 7 | find_package(PkgConfig) 8 | pkg_check_modules(ACCELERATOR QUIET accelerator) 9 | 10 | if(ACCELERATOR_INCLUDE_DIR) 11 | # Already in cache, be silent 12 | set(ACCELERATOR_FIND_QUIETLY TRUE) 13 | endif() 14 | 15 | find_path(ACCELERATOR_INCLUDE_DIR accelerator/accelerator-config.h PATHS 16 | ${PROJECT_BINARY_DIR}/accelerator/accelerator/include 17 | ${PROJECT_BINARY_DIR}-deps/accelerator/accelerator/include) 18 | find_library(ACCELERATOR_LIBRARY accelerator PATHS 19 | ${PROJECT_BINARY_DIR}/accelerator/accelerator/lib 20 | ${PROJECT_BINARY_DIR}-deps/accelerator/accelerator/lib) 21 | 22 | # handle the QUIETLY and REQUIRED arguments and set ACCELERATOR_FOUND to TRUE 23 | # if all listed variables are TRUE 24 | include(FindPackageHandleStandardArgs) 25 | find_package_handle_standard_args(ACCELERATOR DEFAULT_MSG ACCELERATOR_LIBRARY ACCELERATOR_INCLUDE_DIR) 26 | 27 | mark_as_advanced(ACCELERATOR_LIBRARY ACCELERATOR_INCLUDE_DIR) 28 | -------------------------------------------------------------------------------- /raster/net/Processor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/net/Event.h" 22 | 23 | namespace rdd { 24 | 25 | class Event; 26 | 27 | class Processor { 28 | public: 29 | Processor(Event* event) : event_(event) {} 30 | virtual ~Processor() {} 31 | 32 | virtual void run() = 0; 33 | 34 | protected: 35 | Event* event_; 36 | }; 37 | 38 | class ProcessorFactory { 39 | public: 40 | virtual ~ProcessorFactory() {} 41 | virtual std::unique_ptr create(Event* event) = 0; 42 | }; 43 | 44 | } // namespace rdd 45 | -------------------------------------------------------------------------------- /raster/framework/FalconSender.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "accelerator/stats/Monitor.h" 23 | 24 | namespace rdd { 25 | 26 | class FalconSender : public acc::Monitor::Sender { 27 | public: 28 | static const char* URL; 29 | 30 | explicit FalconSender(const std::string& url = URL); 31 | 32 | ~FalconSender(); 33 | 34 | bool send(const acc::Monitor::MonMap& value) override; 35 | 36 | private: 37 | bool post(const std::string& data); 38 | 39 | std::string url_; 40 | }; 41 | 42 | } // namespace rdd 43 | -------------------------------------------------------------------------------- /raster/coroutine/FiberManager.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/coroutine/Fiber.h" 20 | 21 | namespace rdd { 22 | 23 | class FiberManager { 24 | public: 25 | static void update(Fiber* fiber); 26 | static Fiber* get(); 27 | 28 | static void run(Fiber* fiber); 29 | static bool yield(); 30 | static bool exit(); 31 | 32 | private: 33 | FiberManager() {} 34 | 35 | FiberManager(const FiberManager&) = delete; 36 | FiberManager& operator=(const FiberManager&) = delete; 37 | 38 | static __thread Fiber* fiber_; 39 | }; 40 | 41 | Fiber::Task* getCurrentFiberTask(); 42 | 43 | } // namespace rdd 44 | -------------------------------------------------------------------------------- /raster/protocol/binary/Processor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/binary/Processor.h" 18 | 19 | namespace rdd { 20 | 21 | void BinaryProcessor::run() { 22 | auto transport = event_->transport(); 23 | try { 24 | ibuf_ = transport->body->coalesce(); 25 | process(obuf_, ibuf_); 26 | transport->sendHeader(obuf_.size()); 27 | transport->sendBody(acc::IOBuf::copyBuffer(obuf_)); 28 | } catch (std::exception& e) { 29 | ACCLOG(WARN) << "catch exception: " << e.what(); 30 | } catch (...) { 31 | ACCLOG(WARN) << "catch unknown exception"; 32 | } 33 | } 34 | 35 | } // namespace rdd 36 | -------------------------------------------------------------------------------- /raster/net/ErrorEnum.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/ErrorEnum.h" 18 | 19 | #define RDD_NET_ERROR_STR(error) #error 20 | 21 | namespace { 22 | static const char* netErrorStrings[] = { 23 | RDD_NET_ERROR_GEN(RDD_NET_ERROR_STR) 24 | }; 25 | } 26 | 27 | namespace rdd { 28 | 29 | const char* getNetErrorString(NetError error) { 30 | if (error < kErrorNone || error >= kErrorMax) { 31 | return netErrorStrings[kErrorMax]; 32 | } else { 33 | return netErrorStrings[error]; 34 | } 35 | } 36 | 37 | const char* getNetErrorStringByIndex(int i) { 38 | return netErrorStrings[i]; 39 | } 40 | 41 | } // namespace rdd 42 | -------------------------------------------------------------------------------- /raster/net/EventTask.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/coroutine/Fiber.h" 20 | #include "raster/net/Event.h" 21 | #include "raster/net/Processor.h" 22 | 23 | namespace rdd { 24 | 25 | class EventTask : public Fiber::Task { 26 | public: 27 | EventTask(Event* event) : event_(event) { 28 | event_->setTask(this); 29 | } 30 | 31 | ~EventTask() override {} 32 | 33 | void handle() override { 34 | event_->processor()->run(); 35 | event_->setState(Event::kToWrite); 36 | } 37 | 38 | Event* event() const { return event_; } 39 | 40 | private: 41 | Event* event_; 42 | }; 43 | 44 | } // namespace rdd 45 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPException.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/HTTPException.h" 18 | 19 | #include 20 | 21 | namespace rdd { 22 | 23 | std::string HTTPException::describe() const { 24 | std::stringstream ss; 25 | ss << *this; 26 | return ss.str(); 27 | } 28 | 29 | std::ostream& operator<<(std::ostream& os, const HTTPException& ex) { 30 | os << "what=\"" << ex.what() 31 | << "\", direction=" << static_cast(ex.getDirection()) 32 | << ", netError=" << getNetErrorString(ex.getNetError()) 33 | << ", statusCode=" << ex.getStatusCode(); 34 | return os; 35 | } 36 | 37 | } // namespace rdd 38 | -------------------------------------------------------------------------------- /raster/protocol/proto/plugin/Config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | #include 28 | #include 29 | 30 | namespace rpc_generator { 31 | 32 | static const char* const kCppGeneratorMessageHeaderExt = ".pb.h"; 33 | static const char* const kCppGeneratorServiceHeaderExt = ".rpc.pb.h"; 34 | 35 | } // namespace rpc_generator 36 | -------------------------------------------------------------------------------- /raster/protocol/http/SyncTransport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/protocol/http/Transport.h" 20 | 21 | namespace rdd { 22 | 23 | class HTTPSyncTransport : public HTTPTransport { 24 | public: 25 | HTTPSyncTransport(const Peer& peer, const TimeoutOption& timeout) 26 | : HTTPTransport(TransportDirection::UPSTREAM), 27 | peer_(peer), 28 | timeout_(timeout) {} 29 | 30 | ~HTTPSyncTransport() override {} 31 | 32 | void open(); 33 | 34 | bool isOpen(); 35 | 36 | void close(); 37 | 38 | void send(); 39 | 40 | void recv(); 41 | 42 | private: 43 | Peer peer_; 44 | TimeoutOption timeout_; 45 | std::unique_ptr socket_; 46 | }; 47 | 48 | } // namespace rdd 49 | -------------------------------------------------------------------------------- /raster/3rd/http_parser/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | http_parser.c is based on src/http/ngx_http_parse.c from NGINX copyright 2 | Igor Sysoev. 3 | 4 | Additional changes are licensed under the same terms as NGINX and 5 | copyright Joyent, Inc. and other Node contributors. All rights reserved. 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to 9 | deal in the Software without restriction, including without limitation the 10 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 11 | sell copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 22 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 23 | IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /raster/net/Service.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Channel.h" 20 | #include "raster/net/NetUtil.h" 21 | 22 | namespace rdd { 23 | 24 | class Service { 25 | public: 26 | Service(acc::StringPiece name) : name_(name.str()) {} 27 | 28 | virtual ~Service() {} 29 | 30 | std::string name() const { return name_; } 31 | 32 | std::shared_ptr channel() const { 33 | if (!channel_) { 34 | throw std::runtime_error("call makeChannel() first"); 35 | } 36 | return channel_; 37 | } 38 | 39 | virtual void makeChannel(int port, const TimeoutOption& timeout) = 0; 40 | 41 | protected: 42 | std::string name_; 43 | std::shared_ptr channel_; 44 | }; 45 | 46 | } // namespace rdd 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | 3 | os: 4 | - linux 5 | 6 | compiler: 7 | - gcc 8 | 9 | env: 10 | global: 11 | - MATRIX_EVAL="CC=gcc-4.9 && CXX=g++-4.9" 12 | matrix: 13 | - BUILD_TYPE=Debug 14 | - BUILD_TYPE=Release 15 | 16 | addons: 17 | apt: 18 | sources: 19 | - ubuntu-toolchain-r-test 20 | - sourceline: 'deb http://package.rddoc.com/debian ./' 21 | packages: 22 | - g++-4.9 23 | - cmake 24 | - libboost-all-dev 25 | - libdouble-conversion-dev 26 | - libgflags-dev 27 | - libgtest-dev 28 | - libicu-dev 29 | - libprotobuf-dev 30 | - zlib1g-dev 31 | - libcurl3 32 | - openssl 33 | - protobuf-compiler 34 | - thrift-compiler 35 | - accelerator 36 | - flatbuffers 37 | 38 | before_install: 39 | - sudo wget https://github.com/google/googletest/archive/release-1.7.0.tar.gz 40 | - sudo tar xf release-1.7.0.tar.gz 41 | - cd googletest-release-1.7.0 42 | - sudo cmake . 43 | - sudo make -j4 44 | - sudo cp -a include/gtest /usr/include 45 | - sudo cp -a libgtest_main.a libgtest.a /usr/lib/ 46 | - cd .. 47 | - eval "${MATRIX_EVAL}" 48 | 49 | script: 50 | - cd examples 51 | - sh gen.sh 52 | - cd .. 53 | - mkdir build 54 | - cd build 55 | - cmake -DCMAKE_BUILD_TYPE=$BUILD_TYPE .. 56 | - make -j4 57 | - make test 58 | - make package 59 | 60 | -------------------------------------------------------------------------------- /raster/protocol/proto/IOBufInputStream.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "accelerator/io/Cursor.h" 22 | 23 | namespace rdd { 24 | 25 | class IOBufInputStream : public google::protobuf::io::ZeroCopyInputStream { 26 | public: 27 | IOBufInputStream(acc::io::Cursor* cursor); 28 | ~IOBufInputStream() override {} 29 | 30 | // implements ZeroCopyInputStream 31 | bool Next(const void** data, int* size) override; 32 | void BackUp(int count) override; 33 | bool Skip(int count) override; 34 | google::protobuf::int64 ByteCount() const override; 35 | 36 | private: 37 | acc::io::Cursor* cursor_; 38 | const int size_; 39 | int position_; 40 | }; 41 | 42 | } // namespace rdd 43 | -------------------------------------------------------------------------------- /raster/protocol/binary/AsyncServer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Service.h" 20 | #include "raster/protocol/binary/Processor.h" 21 | 22 | namespace rdd { 23 | 24 | template 25 | class BinaryAsyncServer : public Service { 26 | public: 27 | BinaryAsyncServer(acc::StringPiece name) : Service(name) {} 28 | ~BinaryAsyncServer() override {} 29 | 30 | void makeChannel(int port, const TimeoutOption& timeout) override { 31 | Peer peer; 32 | peer.setFromLocalPort(port); 33 | channel_ = std::make_shared( 34 | peer, 35 | timeout, 36 | acc::make_unique(), 37 | acc::make_unique>()); 38 | } 39 | }; 40 | 41 | } // namespace rdd 42 | -------------------------------------------------------------------------------- /raster/protocol/http/SyncTransport.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/SyncTransport.h" 18 | 19 | namespace rdd { 20 | 21 | void HTTPSyncTransport::open() { 22 | socket_ = Socket::createSyncSocket(); 23 | socket_->setConnTimeout(timeout_.ctimeout); 24 | socket_->setRecvTimeout(timeout_.rtimeout); 25 | socket_->setSendTimeout(timeout_.wtimeout); 26 | socket_->connect(peer_); 27 | } 28 | 29 | bool HTTPSyncTransport::isOpen() { 30 | return socket_->isConnected(); 31 | } 32 | 33 | void HTTPSyncTransport::close() { 34 | socket_->close(); 35 | } 36 | 37 | void HTTPSyncTransport::send() { 38 | writeData(socket_.get()); 39 | } 40 | 41 | void HTTPSyncTransport::recv() { 42 | readData(socket_.get()); 43 | } 44 | 45 | } // namespace rdd 46 | -------------------------------------------------------------------------------- /raster/net/Group.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "accelerator/thread/SharedMutex.h" 25 | 26 | namespace rdd { 27 | 28 | class Group { 29 | public: 30 | typedef size_t Key; 31 | 32 | Group(size_t capacity = kMaxCapacity); 33 | 34 | Key create(size_t groupSize); 35 | 36 | bool finish(Key group); 37 | 38 | size_t count() const; 39 | 40 | static constexpr size_t kMaxCapacity = 1 << 14; 41 | 42 | private: 43 | void increase(size_t bound); 44 | 45 | size_t capacity_; 46 | std::vector groupCounts_; 47 | std::stack groupKeys_; // available group ids 48 | acc::SharedMutex lock_; 49 | }; 50 | 51 | } // namespace rdd 52 | -------------------------------------------------------------------------------- /raster/net/Acceptor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/event/EventLoop.h" 20 | #include "raster/net/NetHub.h" 21 | #include "raster/net/Service.h" 22 | 23 | namespace rdd { 24 | 25 | class Acceptor { 26 | public: 27 | Acceptor(std::shared_ptr hub); 28 | 29 | void addService(std::unique_ptr service); 30 | 31 | void configService( 32 | const std::string& name, 33 | int port, 34 | const TimeoutOption& timeout); 35 | 36 | void start(); 37 | void stop(); 38 | 39 | private: 40 | void listen(Service* service, int backlog = 64); 41 | 42 | std::shared_ptr hub_; 43 | std::unique_ptr loop_; 44 | std::map> services_; 45 | }; 46 | 47 | } // namespace rdd 48 | -------------------------------------------------------------------------------- /cmake-scripts/FindFlatbuffers.cmake: -------------------------------------------------------------------------------- 1 | # Find flatbuffers 2 | # 3 | # FLATBUFFERS_FLATC - flatc. 4 | # FLATBUFFERS_INCLUDE_DIR - where to find flatbuffers/*. 5 | # FLATBUFFERS_LIBRARY - List of libraries when using flatbuffers. 6 | # FLATBUFFERS_FOUND - True if flatbuffers found. 7 | 8 | find_package(PkgConfig) 9 | pkg_check_modules(FLATBUFFERS QUIET flatbuffers) 10 | 11 | if(FLATBUFFERS_INCLUDE_DIR) 12 | # Already in cache, be silent 13 | set(FLATBUFFERS_FIND_QUIETLY TRUE) 14 | endif() 15 | 16 | find_program(FLATBUFFERS_FLATC flatc PATHS 17 | ${PROJECT_BINARY_DIR}/flatbuffers/flatbuffers/bin 18 | ${PROJECT_BINARY_DIR}-deps/flatbuffers/flatbuffers/bin) 19 | find_path(FLATBUFFERS_INCLUDE_DIR flatbuffers/flatbuffers.h PATHS 20 | ${PROJECT_BINARY_DIR}/flatbuffers/flatbuffers/include 21 | ${PROJECT_BINARY_DIR}-deps/flatbuffers/flatbuffers/include) 22 | find_library(FLATBUFFERS_LIBRARY flatbuffers PATHS 23 | ${PROJECT_BINARY_DIR}/flatbuffers/flatbuffers/lib 24 | ${PROJECT_BINARY_DIR}-deps/flatbuffers/flatbuffers/lib) 25 | 26 | # handle the QUIETLY and REQUIRED arguments and set FLATBUFFERS_FOUND to TRUE 27 | # if all listed variables are TRUE 28 | include(FindPackageHandleStandardArgs) 29 | find_package_handle_standard_args(FLATBUFFERS DEFAULT_MSG FLATBUFFERS_LIBRARY FLATBUFFERS_INCLUDE_DIR FLATBUFFERS_FLATC) 30 | 31 | mark_as_advanced(FLATBUFFERS_LIBRARY FLATBUFFERS_INCLUDE_DIR FLATBUFFERS_FLATC) 32 | -------------------------------------------------------------------------------- /raster/protocol/binary/SyncTransport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/NetUtil.h" 20 | #include "raster/net/Socket.h" 21 | #include "raster/protocol/binary/Transport.h" 22 | 23 | namespace rdd { 24 | 25 | class BinarySyncTransport : public BinaryTransport { 26 | public: 27 | BinarySyncTransport(const Peer& peer, const TimeoutOption& timeout) 28 | : BinaryTransport(), 29 | peer_(peer), 30 | timeout_(timeout) {} 31 | 32 | ~BinarySyncTransport() override {} 33 | 34 | void open(); 35 | 36 | bool isOpen(); 37 | 38 | void close(); 39 | 40 | void send(const acc::ByteRange& request); 41 | 42 | void recv(acc::ByteRange& response); 43 | 44 | private: 45 | Peer peer_; 46 | TimeoutOption timeout_; 47 | std::unique_ptr socket_; 48 | }; 49 | 50 | } // namespace rdd 51 | -------------------------------------------------------------------------------- /raster/framework/Sampler.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/framework/Sampler.h" 18 | 19 | #include "accelerator/Logging.h" 20 | #include "accelerator/MapUtil.h" 21 | #include "accelerator/Random.h" 22 | 23 | namespace rdd { 24 | 25 | void PercentSampler::setup(bool open, double percent) { 26 | ACCLOG(INFO) << "Sampler: setup " 27 | << "open=" << open << ", percent=" << percent; 28 | open_ = open; 29 | percent_ = percent; 30 | } 31 | 32 | bool PercentSampler::hit() { 33 | bool open = open_; 34 | double percent = percent_; 35 | return open && percent > acc::Random::randDouble01(); 36 | } 37 | 38 | bool SamplerManager::hit(const std::string& name) { 39 | std::lock_guard guard(lock_); 40 | auto sampler = acc::get_ptr(samplers_, name); 41 | return sampler ? sampler->get()->hit() : false; 42 | } 43 | 44 | } // namespace rdd 45 | -------------------------------------------------------------------------------- /raster/net/EventHandler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/event/EventHandlerBase.h" 20 | #include "accelerator/event/EventLoop.h" 21 | 22 | namespace rdd { 23 | 24 | class EventHandler : public acc::EventHandlerBase { 25 | public: 26 | EventHandler(acc::EventLoop* loop) : loop_(loop) {} 27 | virtual ~EventHandler() {} 28 | 29 | void onConnect(acc::EventBase* event) override; 30 | void onListen(acc::EventBase* event) override; 31 | void onRead(acc::EventBase* event) override; 32 | void onWrite(acc::EventBase* event) override; 33 | void onTimeout(acc::EventBase* event) override; 34 | void close(acc::EventBase* event) override; 35 | 36 | private: 37 | void onComplete(acc::EventBase* event); 38 | void onError(acc::EventBase* event); 39 | 40 | acc::EventLoop* loop_; 41 | }; 42 | 43 | } // namespace rdd 44 | -------------------------------------------------------------------------------- /raster/net/EventPool.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "raster/net/Event.h" 25 | #include "raster/net/NetUtil.h" 26 | 27 | namespace rdd { 28 | 29 | class EventPool { 30 | public: 31 | EventPool() {} 32 | 33 | std::unique_ptr get(const Peer& peer); 34 | 35 | void giveBack(std::unique_ptr event); 36 | 37 | size_t count() const; 38 | 39 | private: 40 | std::unordered_map>> pool_; 41 | mutable std::mutex lock_; 42 | }; 43 | 44 | class EventPoolManager { 45 | public: 46 | EventPoolManager() {} 47 | 48 | EventPool* getPool(int id); 49 | 50 | private: 51 | std::unordered_map> pool_; 52 | mutable std::mutex lock_; 53 | }; 54 | 55 | } // namespace rdd 56 | -------------------------------------------------------------------------------- /raster/framework/ConfigUtil.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include 18 | 19 | #include "raster/Portability.h" 20 | #include "raster/framework/Config.h" 21 | 22 | DEFINE_bool(gen, false, "Generate default config"); 23 | 24 | using namespace rdd; 25 | 26 | namespace { 27 | 28 | void genDefault() { 29 | std::cout << generateDefault() << "\n"; 30 | } 31 | 32 | } 33 | 34 | int main(int argc, char* argv[]) { 35 | std::string usage = 36 | "Usage:\n" 37 | " configutil\n" 38 | " operate config\n" 39 | " configutil -gen\n" 40 | " generate default config\n"; 41 | 42 | gflags::SetUsageMessage(usage); 43 | gflags::ParseCommandLineFlags(&argc, &argv, true); 44 | 45 | if (FLAGS_gen) { 46 | genDefault(); 47 | } else { 48 | std::cerr << usage; 49 | exit(1); 50 | } 51 | 52 | gflags::ShutDownCommandLineFlags(); 53 | 54 | return 0; 55 | } 56 | -------------------------------------------------------------------------------- /raster/protocol/http/gen_HTTPCommonHeaders.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The `awk` script isn't nearly as hairy as it seems. The only real trick is 4 | # the first line. We're processing two files -- the result of the `cat` pipeline 5 | # above, plus the gperf template. The "NR == FNR" compares the current file's 6 | # line number with the total number of lines we've processed -- i.e., that test 7 | # means "am I in the first file?" So we suck those lines aside. Then we process 8 | # the second file, replacing "%%%%%" with some munging of the lines we sucked 9 | # aside from the `cat` pipeline. 10 | 11 | HEADERS_LIST=HTTPCommonHeaders.txt 12 | 13 | cat ${HEADERS_LIST} | sort | uniq \ 14 | | awk ' 15 | NR == FNR { 16 | n[FNR] = $1; 17 | next 18 | } 19 | $1 == "%%%%%" { 20 | for (i in n) { 21 | h = n[i]; 22 | gsub("-", "_", h); 23 | print " HTTP_HEADER_" toupper(h) " = " i+1 "," 24 | }; 25 | next 26 | } 27 | { 28 | print 29 | } 30 | ' - "HTTPCommonHeaders.template.h" > "HTTPCommonHeaders.h" 31 | 32 | cat ${HEADERS_LIST} | sort | uniq \ 33 | | awk ' 34 | NR == FNR { 35 | n[FNR] = $1; 36 | next 37 | } 38 | $1 == "%%%%%" { 39 | print "%%"; 40 | for (i in n) { 41 | h = n[i]; 42 | gsub("-", "_", h); 43 | print n[i] ", HTTP_HEADER_" toupper(h) 44 | }; 45 | print "%%"; 46 | next 47 | } 48 | { 49 | print 50 | } 51 | ' - "HTTPCommonHeaders.template.gperf" \ 52 | | ${GPERF:-gperf} -m5 --output-file="HTTPCommonHeaders.cpp" 53 | -------------------------------------------------------------------------------- /raster/protocol/http/test/HTTPHeadersTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/HTTPHeaders.h" 18 | #include 19 | 20 | using namespace rdd; 21 | 22 | TEST(headers, all) { 23 | const char* data = 24 | "Foo: bar\r\n" 25 | " baz\r\n" 26 | "Asdf: qwer\r\n" 27 | "\tzxcv\r\n" 28 | "Foo: even\r\n" 29 | " more\r\n" 30 | " lines\r\n"; 31 | 32 | HTTPHeaders headers; 33 | headers.parse(data); 34 | EXPECT_EQ(3, headers.size()); 35 | EXPECT_EQ(1, headers.getNumberOfValues("asdf")); 36 | EXPECT_STREQ("qwer zxcv", headers.combine("asdf").c_str()); 37 | EXPECT_STREQ("qwer zxcv", headers.getSingleOrEmpty("asdf").c_str()); 38 | EXPECT_EQ(2, headers.getNumberOfValues("Foo")); 39 | EXPECT_STREQ("bar baz, even more lines", headers.combine("Foo").c_str()); 40 | EXPECT_TRUE(headers.getSingleOrEmpty("Foo").empty()); 41 | } 42 | 43 | -------------------------------------------------------------------------------- /raster/protocol/binary/Processor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Processor.h" 20 | #include "raster/protocol/binary/Transport.h" 21 | 22 | namespace rdd { 23 | 24 | class BinaryProcessor : public Processor { 25 | public: 26 | BinaryProcessor(Event* event) : Processor(event) {} 27 | ~BinaryProcessor() override {} 28 | 29 | virtual void process(acc::ByteRange& response, const acc::ByteRange& request) = 0; 30 | 31 | void run() override; 32 | 33 | private: 34 | acc::ByteRange ibuf_; 35 | acc::ByteRange obuf_; 36 | }; 37 | 38 | template 39 | class BinaryProcessorFactory : public ProcessorFactory { 40 | public: 41 | BinaryProcessorFactory() {} 42 | ~BinaryProcessorFactory() override {} 43 | 44 | std::unique_ptr create(Event* event) override { 45 | return acc::make_unique

(event); 46 | } 47 | }; 48 | 49 | } // namespace rdd 50 | -------------------------------------------------------------------------------- /cmake-scripts/FindGFlags.cmake: -------------------------------------------------------------------------------- 1 | # - Try to find GFLAGS 2 | # 3 | # The following variables are optionally searched for defaults 4 | # GFLAGS_ROOT_DIR: Base directory where all GFLAGS components are found 5 | # 6 | # The following are set after configuration is done: 7 | # GFLAGS_FOUND 8 | # GFLAGS_INCLUDE_DIRS 9 | # GFLAGS_LIBRARIES 10 | # GFLAGS_LIBRARYRARY_DIRS 11 | 12 | include(FindPackageHandleStandardArgs) 13 | 14 | set(GFLAGS_ROOT_DIR "" CACHE PATH "Folder contains Gflags") 15 | 16 | # We are testing only a couple of files in the include directories 17 | if(WIN32) 18 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 19 | PATHS ${GFLAGS_ROOT_DIR}/src/windows) 20 | else() 21 | find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h 22 | PATHS ${GFLAGS_ROOT_DIR}) 23 | endif() 24 | 25 | if(MSVC) 26 | find_library(GFLAGS_LIBRARY_RELEASE 27 | NAMES libgflags 28 | PATHS ${GFLAGS_ROOT_DIR} 29 | PATH_SUFFIXES Release) 30 | 31 | find_library(GFLAGS_LIBRARY_DEBUG 32 | NAMES libgflags-debug 33 | PATHS ${GFLAGS_ROOT_DIR} 34 | PATH_SUFFIXES Debug) 35 | 36 | set(GFLAGS_LIBRARY optimized ${GFLAGS_LIBRARY_RELEASE} debug ${GFLAGS_LIBRARY_DEBUG}) 37 | else() 38 | find_library(GFLAGS_LIBRARY gflags) 39 | endif() 40 | 41 | find_package_handle_standard_args(GFLAGS DEFAULT_MSG 42 | GFLAGS_INCLUDE_DIR GFLAGS_LIBRARY) 43 | 44 | 45 | if(GFLAGS_FOUND) 46 | set(GFLAGS_INCLUDE_DIRS ${GFLAGS_INCLUDE_DIR}) 47 | set(GFLAGS_LIBRARIES ${GFLAGS_LIBRARY}) 48 | endif() 49 | -------------------------------------------------------------------------------- /examples/empty/gen-cpp/Empty_server.skeleton.cpp: -------------------------------------------------------------------------------- 1 | // This autogenerated skeleton file illustrates how to build a server. 2 | // You should copy it to another filename to avoid overwriting it. 3 | 4 | #include "Empty.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace ::apache::thrift; 11 | using namespace ::apache::thrift::protocol; 12 | using namespace ::apache::thrift::transport; 13 | using namespace ::apache::thrift::server; 14 | 15 | using boost::shared_ptr; 16 | 17 | using namespace ::rdd::empty; 18 | 19 | class EmptyHandler : virtual public EmptyIf { 20 | public: 21 | EmptyHandler() { 22 | // Your initialization goes here 23 | } 24 | 25 | void run(Result& _return, const Query& query) { 26 | // Your implementation goes here 27 | printf("run\n"); 28 | } 29 | 30 | }; 31 | 32 | int main(int argc, char **argv) { 33 | int port = 9090; 34 | shared_ptr handler(new EmptyHandler()); 35 | shared_ptr processor(new EmptyProcessor(handler)); 36 | shared_ptr serverTransport(new TServerSocket(port)); 37 | shared_ptr transportFactory(new TBufferedTransportFactory()); 38 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 39 | 40 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 41 | server.serve(); 42 | return 0; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /examples/proxy/gen-cpp/Proxy_server.skeleton.cpp: -------------------------------------------------------------------------------- 1 | // This autogenerated skeleton file illustrates how to build a server. 2 | // You should copy it to another filename to avoid overwriting it. 3 | 4 | #include "Proxy.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace ::apache::thrift; 11 | using namespace ::apache::thrift::protocol; 12 | using namespace ::apache::thrift::transport; 13 | using namespace ::apache::thrift::server; 14 | 15 | using boost::shared_ptr; 16 | 17 | using namespace ::rdd::proxy; 18 | 19 | class ProxyHandler : virtual public ProxyIf { 20 | public: 21 | ProxyHandler() { 22 | // Your initialization goes here 23 | } 24 | 25 | void run(Result& _return, const Query& query) { 26 | // Your implementation goes here 27 | printf("run\n"); 28 | } 29 | 30 | }; 31 | 32 | int main(int argc, char **argv) { 33 | int port = 9090; 34 | shared_ptr handler(new ProxyHandler()); 35 | shared_ptr processor(new ProxyProcessor(handler)); 36 | shared_ptr serverTransport(new TServerSocket(port)); 37 | shared_ptr transportFactory(new TBufferedTransportFactory()); 38 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 39 | 40 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 41 | server.serve(); 42 | return 0; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /raster/protocol/binary/AsyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/AsyncClient.h" 20 | 21 | namespace rdd { 22 | 23 | class BinaryAsyncClient : public AsyncClient { 24 | public: 25 | BinaryAsyncClient(const ClientOption& option); 26 | 27 | BinaryAsyncClient(const Peer& peer, 28 | const TimeoutOption& timeout); 29 | 30 | BinaryAsyncClient(const Peer& peer, 31 | uint64_t ctimeout = 100000, 32 | uint64_t rtimeout = 1000000, 33 | uint64_t wtimeout = 300000); 34 | 35 | ~BinaryAsyncClient() override {} 36 | 37 | bool recv(acc::ByteRange& response); 38 | 39 | bool send(const acc::ByteRange& request); 40 | 41 | bool fetch(acc::ByteRange& response, const acc::ByteRange& request); 42 | 43 | protected: 44 | std::shared_ptr makeChannel() override; 45 | }; 46 | 47 | } // namespace rdd 48 | -------------------------------------------------------------------------------- /raster/net/NetHub.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/event/EventLoop.h" 20 | #include "raster/coroutine/FiberHub.h" 21 | #include "raster/net/Event.h" 22 | #include "raster/net/Group.h" 23 | #include "raster/net/NetUtil.h" 24 | 25 | namespace rdd { 26 | 27 | struct ForwardTarget { 28 | int port{0}; 29 | Peer fpeer; 30 | int flow{0}; 31 | }; 32 | 33 | class NetHub : public FiberHub { 34 | public: 35 | virtual acc::EventLoop* getEventLoop() = 0; 36 | 37 | void execute(Event* event); 38 | 39 | void addEvent(Event* event); 40 | void forwardEvent(Event* event, const Peer& peer); 41 | 42 | bool waitGroup(const std::vector& events); 43 | 44 | void setForwarding(bool forward); 45 | void addForwardTarget(ForwardTarget&& t); 46 | 47 | private: 48 | Group group_; 49 | bool forwarding_; 50 | std::vector forwards_; 51 | }; 52 | 53 | } // namespace rdd 54 | -------------------------------------------------------------------------------- /raster/framework/Signal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "accelerator/Function.h" 23 | 24 | namespace rdd { 25 | 26 | void setupSignal(int signo, void (*handler)(int)); 27 | void setupSignal(int signo, void (*handler)(int, siginfo_t*, void*)); 28 | 29 | void setupIgnoreSignal(int signo); 30 | void setupReloadSignal(int signo); 31 | void setupShutdownSignal(int signo); 32 | void setupMemoryProtectSignal(); 33 | 34 | void sendSignal(int signo, const char* pidfile); 35 | 36 | class Shutdown { 37 | public: 38 | Shutdown() {} 39 | 40 | void addTask(acc::VoidFunc&& callback) { 41 | callbacks_.push_back(std::move(callback)); 42 | } 43 | 44 | void run() { 45 | for (auto& cb : callbacks_) { 46 | cb(); 47 | } 48 | exit(0); 49 | } 50 | 51 | private: 52 | std::vector callbacks_; 53 | }; 54 | 55 | } // namespace rdd 56 | -------------------------------------------------------------------------------- /raster/plugins/flume/gen-cpp/Scribe_server.skeleton.cpp: -------------------------------------------------------------------------------- 1 | // This autogenerated skeleton file illustrates how to build a server. 2 | // You should copy it to another filename to avoid overwriting it. 3 | 4 | #include "Scribe.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace ::apache::thrift; 11 | using namespace ::apache::thrift::protocol; 12 | using namespace ::apache::thrift::transport; 13 | using namespace ::apache::thrift::server; 14 | 15 | using boost::shared_ptr; 16 | 17 | using namespace ::rdd; 18 | 19 | class ScribeHandler : virtual public ScribeIf { 20 | public: 21 | ScribeHandler() { 22 | // Your initialization goes here 23 | } 24 | 25 | ResultCode Log(const std::vector & messages) { 26 | // Your implementation goes here 27 | printf("Log\n"); 28 | } 29 | 30 | }; 31 | 32 | int main(int argc, char **argv) { 33 | int port = 9090; 34 | shared_ptr handler(new ScribeHandler()); 35 | shared_ptr processor(new ScribeProcessor(handler)); 36 | shared_ptr serverTransport(new TServerSocket(port)); 37 | shared_ptr transportFactory(new TBufferedTransportFactory()); 38 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 39 | 40 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 41 | server.serve(); 42 | return 0; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /raster/protocol/http/AsyncServer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/net/Service.h" 22 | #include "raster/protocol/http/Processor.h" 23 | 24 | namespace rdd { 25 | 26 | class HTTPAsyncServer : public Service { 27 | public: 28 | HTTPAsyncServer(acc::StringPiece name) : Service(name) {} 29 | ~HTTPAsyncServer() override {} 30 | 31 | void makeChannel(int port, const TimeoutOption& timeoutOpt) override; 32 | 33 | template 34 | void addHandler(const std::string& regex, Args&&... args) { 35 | handlers_.emplace(boost::regex(regex), 36 | std::make_shared(std::forward(args)...)); 37 | } 38 | 39 | std::shared_ptr matchHandler(const std::string& url) const; 40 | 41 | private: 42 | std::map> handlers_; 43 | }; 44 | 45 | } // namespace rdd 46 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPCommonHeaders.txt: -------------------------------------------------------------------------------- 1 | Accept 2 | Accept-Charset 3 | Accept-Datetime 4 | Accept-Encoding 5 | Accept-Language 6 | Accept-Ranges 7 | Access-Control-Allow-Credentials 8 | Access-Control-Allow-Headers 9 | Access-Control-Allow-Methods 10 | Access-Control-Allow-Origin 11 | Access-Control-Expose-Headers 12 | Access-Control-Max-Age 13 | Access-Control-Request-Headers 14 | Access-Control-Request-Method 15 | Age 16 | Allow 17 | Authorization 18 | Cache-Control 19 | Connection 20 | Content-Disposition 21 | Content-Encoding 22 | Content-Language 23 | Content-Length 24 | Content-Location 25 | Content-MD5 26 | Content-Range 27 | Content-Type 28 | Cookie 29 | DNT 30 | Date 31 | ETag 32 | Expect 33 | Expires 34 | From 35 | Front-End-Https 36 | Host 37 | If-Match 38 | If-Modified-Since 39 | If-None-Match 40 | If-Range 41 | If-Unmodified-Since 42 | Keep-Alive 43 | Last-Modified 44 | Link 45 | Location 46 | Max-Forwards 47 | Origin 48 | P3P 49 | Pragma 50 | Proxy-Authenticate 51 | Proxy-Authorization 52 | Proxy-Connection 53 | Range 54 | Referer 55 | Refresh 56 | Retry-After 57 | Server 58 | Set-Cookie 59 | Strict-Transport-Security 60 | TE 61 | Timestamp 62 | Trailer 63 | Transfer-Encoding 64 | Upgrade 65 | User-Agent 66 | VIP 67 | Vary 68 | Via 69 | WWW-Authenticate 70 | Warning 71 | X-Accel-Redirect 72 | X-Content-Security-Policy-Report-Only 73 | X-Content-Type-Options 74 | X-Forwarded-For 75 | X-Forwarded-Proto 76 | X-Frame-Options 77 | X-Powered-By 78 | X-Real-IP 79 | X-Requested-With 80 | X-Scheme 81 | X-UA-Compatible 82 | X-Wap-Profile 83 | X-XSS-Protection 84 | -------------------------------------------------------------------------------- /examples/parallel/gen-cpp/Parallel_server.skeleton.cpp: -------------------------------------------------------------------------------- 1 | // This autogenerated skeleton file illustrates how to build a server. 2 | // You should copy it to another filename to avoid overwriting it. 3 | 4 | #include "Parallel.h" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | using namespace ::apache::thrift; 11 | using namespace ::apache::thrift::protocol; 12 | using namespace ::apache::thrift::transport; 13 | using namespace ::apache::thrift::server; 14 | 15 | using boost::shared_ptr; 16 | 17 | using namespace ::rdd::parallel; 18 | 19 | class ParallelHandler : virtual public ParallelIf { 20 | public: 21 | ParallelHandler() { 22 | // Your initialization goes here 23 | } 24 | 25 | void run(Result& _return, const Query& query) { 26 | // Your implementation goes here 27 | printf("run\n"); 28 | } 29 | 30 | }; 31 | 32 | int main(int argc, char **argv) { 33 | int port = 9090; 34 | shared_ptr handler(new ParallelHandler()); 35 | shared_ptr processor(new ParallelProcessor(handler)); 36 | shared_ptr serverTransport(new TServerSocket(port)); 37 | shared_ptr transportFactory(new TBufferedTransportFactory()); 38 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 39 | 40 | TSimpleServer server(processor, serverTransport, transportFactory, protocolFactory); 41 | server.serve(); 42 | return 0; 43 | } 44 | 45 | -------------------------------------------------------------------------------- /raster/protocol/http/AsyncServer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/AsyncServer.h" 18 | 19 | namespace rdd { 20 | 21 | void HTTPAsyncServer::makeChannel(int port, const TimeoutOption& timeoutOpt) { 22 | Peer peer; 23 | peer.setFromLocalPort(port); 24 | channel_ = std::make_shared( 25 | peer, 26 | timeoutOpt, 27 | acc::make_unique(TransportDirection::DOWNSTREAM), 28 | acc::make_unique( 29 | [&](const std::string& url) { 30 | return matchHandler(url); 31 | })); 32 | } 33 | 34 | std::shared_ptr 35 | HTTPAsyncServer::matchHandler(const std::string& url) const { 36 | for (auto& kv : handlers_) { 37 | boost::cmatch match; 38 | if (boost::regex_match(url.c_str(), match, kv.first)) { 39 | return kv.second; 40 | } 41 | } 42 | return std::make_shared(); 43 | } 44 | 45 | } // namespace rdd 46 | -------------------------------------------------------------------------------- /raster/protocol/binary/SyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/NetUtil.h" 20 | #include "raster/protocol/binary/SyncTransport.h" 21 | 22 | namespace rdd { 23 | 24 | class BinarySyncClient { 25 | public: 26 | BinarySyncClient(const ClientOption& option); 27 | 28 | BinarySyncClient(const Peer& peer, 29 | const TimeoutOption& timeout); 30 | 31 | BinarySyncClient(const Peer& peer, 32 | uint64_t ctimeout = 100000, 33 | uint64_t rtimeout = 1000000, 34 | uint64_t wtimeout = 300000); 35 | 36 | virtual ~BinarySyncClient(); 37 | 38 | void close(); 39 | 40 | bool connect(); 41 | 42 | bool connected() const; 43 | 44 | bool fetch(acc::ByteRange& response, const acc::ByteRange& request); 45 | 46 | private: 47 | void init(); 48 | 49 | Peer peer_; 50 | TimeoutOption timeout_; 51 | std::unique_ptr transport_; 52 | }; 53 | 54 | } // namespace rdd 55 | -------------------------------------------------------------------------------- /raster/protocol/binary/SyncTransport.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/binary/SyncTransport.h" 18 | 19 | #include "accelerator/io/IOBuf.h" 20 | 21 | namespace rdd { 22 | 23 | void BinarySyncTransport::open() { 24 | socket_ = Socket::createSyncSocket(); 25 | socket_->setConnTimeout(timeout_.ctimeout); 26 | socket_->setRecvTimeout(timeout_.rtimeout); 27 | socket_->setSendTimeout(timeout_.wtimeout); 28 | socket_->connect(peer_); 29 | } 30 | 31 | bool BinarySyncTransport::isOpen() { 32 | return socket_->isConnected(); 33 | } 34 | 35 | void BinarySyncTransport::close() { 36 | socket_->close(); 37 | } 38 | 39 | void BinarySyncTransport::send(const acc::ByteRange& request) { 40 | sendHeader(request.size()); 41 | sendBody(acc::IOBuf::copyBuffer(request)); 42 | writeData(socket_.get()); 43 | } 44 | 45 | void BinarySyncTransport::recv(acc::ByteRange& response) { 46 | readData(socket_.get()); 47 | response = body->coalesce(); 48 | } 49 | 50 | } // namespace rdd 51 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPMethod.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "accelerator/Range.h" 22 | 23 | namespace rdd { 24 | 25 | // Ordered by frequency to minimize time spent in iteration 26 | #define HTTP_METHOD_GEN(x) \ 27 | x(GET), \ 28 | x(POST), \ 29 | x(OPTIONS), \ 30 | x(DELETE), \ 31 | x(HEAD), \ 32 | x(CONNECT), \ 33 | x(PUT), \ 34 | x(TRACE) 35 | 36 | #define HTTP_METHOD_ENUM(method) method 37 | 38 | /** 39 | * See the definitions in RFC2616 5.1.1 for the source of this list. 40 | */ 41 | enum class HTTPMethod { 42 | HTTP_METHOD_GEN(HTTP_METHOD_ENUM) 43 | }; 44 | 45 | #undef HTTP_METHOD_ENUM 46 | 47 | HTTPMethod stringToMethod(acc::StringPiece method); 48 | 49 | const std::string& methodToString(HTTPMethod method); 50 | 51 | std::ostream& operator<<(std::ostream& os, HTTPMethod method); 52 | 53 | } // namespace rdd 54 | -------------------------------------------------------------------------------- /raster/net/NetUtil.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/NetUtil.h" 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "accelerator/Logging.h" 24 | 25 | namespace rdd { 26 | 27 | std::string getNodeName() { 28 | struct utsname buf; 29 | if (uname(&buf) != -1) { 30 | return buf.nodename; 31 | } 32 | return "(unknown)"; 33 | } 34 | 35 | std::string getNodeIp() { 36 | static std::string ip; 37 | if (ip.empty()) { 38 | struct hostent* ent = gethostbyname(getNodeName().c_str()); 39 | if (ent != nullptr) { 40 | switch (ent->h_addrtype) { 41 | case AF_INET: 42 | case AF_INET6: 43 | { 44 | char str[128]; 45 | inet_ntop(ent->h_addrtype, ent->h_addr, str, sizeof(str)); 46 | ip = str; 47 | } 48 | break; 49 | default: break; 50 | } 51 | } 52 | } 53 | return ip.empty() ? "(unknown)" : ip; 54 | } 55 | 56 | } // namespace rdd 57 | -------------------------------------------------------------------------------- /raster/protocol/http/Processor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Processor.h" 20 | #include "raster/protocol/http/RequestHandler.h" 21 | 22 | namespace rdd { 23 | 24 | class HTTPProcessor : public Processor { 25 | public: 26 | HTTPProcessor( 27 | Event* event, 28 | const std::shared_ptr& handler) 29 | : Processor(event), handler_(handler) {} 30 | 31 | ~HTTPProcessor() override {} 32 | 33 | void run() override; 34 | 35 | protected: 36 | std::shared_ptr handler_; 37 | }; 38 | 39 | class HTTPProcessorFactory : public ProcessorFactory { 40 | public: 41 | typedef std::function< 42 | std::shared_ptr(const std::string&)> Router; 43 | 44 | HTTPProcessorFactory(Router router) : router_(std::move(router)) {} 45 | ~HTTPProcessorFactory() override {} 46 | 47 | std::unique_ptr create(Event* event) override; 48 | 49 | private: 50 | Router router_; 51 | }; 52 | 53 | } // namespace rdd 54 | -------------------------------------------------------------------------------- /raster/3rd/thrift/cxxfunctional.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef _THRIFT_CXXFUNCTIONAL_H_ 21 | #define _THRIFT_CXXFUNCTIONAL_H_ 1 22 | 23 | #include 24 | 25 | namespace apache { namespace thrift { namespace stdcxx { 26 | using ::std::function; 27 | using ::std::bind; 28 | 29 | namespace placeholders { 30 | using ::std::placeholders::_1; 31 | using ::std::placeholders::_2; 32 | using ::std::placeholders::_3; 33 | using ::std::placeholders::_4; 34 | using ::std::placeholders::_5; 35 | using ::std::placeholders::_6; 36 | } // apache::thrift::stdcxx::placeholders 37 | }}} // apache::thrift::stdcxx 38 | 39 | // Alias for thrift c++ compatibility namespace 40 | namespace tcxx = apache::thrift::stdcxx; 41 | 42 | #endif // #ifndef _THRIFT_CXXFUNCTIONAL_H_ 43 | -------------------------------------------------------------------------------- /raster/plugins/flume/FlumeClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "accelerator/thread/AtomicLinkedList.h" 24 | #include "raster/plugins/flume/gen-cpp/scribe_types.h" 25 | #include "raster/protocol/thrift/SyncClient.h" 26 | 27 | namespace rdd { 28 | 29 | class ScribeClient; 30 | class LogEntry; 31 | 32 | class FlumeClient { 33 | public: 34 | FlumeClient(const ClientOption& option, 35 | const std::string& category, 36 | const std::string& logDir); 37 | 38 | ~FlumeClient(); 39 | 40 | void add(std::string&& message); 41 | 42 | void send(); 43 | 44 | private: 45 | void sendInBatch(const std::vector& entries); 46 | void writeToDisk(const std::vector& entries); 47 | 48 | std::string category_; 49 | std::string logDir_; 50 | acc::AtomicLinkedList queue_; 51 | std::unique_ptr> client_; 52 | }; 53 | 54 | } // namespace rdd 55 | -------------------------------------------------------------------------------- /raster/framework/test/AutoTaskTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/framework/AutoTask.h" 18 | #include 19 | 20 | using namespace rdd; 21 | 22 | class Task : public AutoTask { 23 | public: 24 | Task() : times_(0) {} 25 | 26 | void update() { times_++; } 27 | size_t times() const { return times_; } 28 | 29 | private: 30 | size_t times_; 31 | }; 32 | 33 | TEST(AutoTaskManager, runTasks) { 34 | AutoTaskManager atm; 35 | Task a, b; 36 | atm.addTask(&a, 10000); 37 | atm.addTask(&b, 1000); 38 | EXPECT_EQ(0, a.times()); 39 | EXPECT_EQ(0, b.times()); 40 | atm.initStamp(); 41 | atm.runTasks(); 42 | EXPECT_EQ(1, a.times()); 43 | EXPECT_EQ(1, b.times()); 44 | usleep(1000); 45 | atm.runTasks(); 46 | EXPECT_EQ(1, a.times()); 47 | EXPECT_EQ(2, b.times()); 48 | usleep(4000); 49 | atm.runTasks(); 50 | EXPECT_EQ(1, a.times()); 51 | EXPECT_EQ(3, b.times()); 52 | usleep(5000); 53 | atm.runTasks(); 54 | EXPECT_EQ(2, a.times()); 55 | EXPECT_EQ(4, b.times()); 56 | } 57 | -------------------------------------------------------------------------------- /raster/coroutine/Context.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "accelerator/Function.h" 22 | 23 | namespace rdd { 24 | 25 | class Context { 26 | public: 27 | Context(acc::VoidFunc&& func, 28 | unsigned char* stackLimit, 29 | size_t stackSize) 30 | : func_(std::move(func)) { 31 | getcontext(&fiberContext_); 32 | fiberContext_.uc_stack.ss_sp = stackLimit; 33 | fiberContext_.uc_stack.ss_size = stackSize; 34 | makecontext(&fiberContext_, (void (*)())fiberFunc, 1, this); 35 | } 36 | 37 | void activate() { 38 | swapcontext(&mainContext_, &fiberContext_); 39 | } 40 | 41 | void deactivate() { 42 | swapcontext(&fiberContext_, &mainContext_); 43 | } 44 | 45 | private: 46 | static void fiberFunc(intptr_t arg) { 47 | auto ctx = reinterpret_cast(arg); 48 | ctx->func_(); 49 | } 50 | 51 | acc::VoidFunc func_; 52 | ucontext_t fiberContext_; 53 | ucontext_t mainContext_; 54 | }; 55 | 56 | } // namespace rdd 57 | -------------------------------------------------------------------------------- /raster/protocol/proto/IOBufInputStream.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/proto/IOBufInputStream.h" 18 | 19 | namespace rdd { 20 | 21 | IOBufInputStream::IOBufInputStream(acc::io::Cursor* cursor) 22 | : cursor_(cursor), 23 | size_(cursor_->totalLength()), 24 | position_(0) { 25 | } 26 | 27 | bool IOBufInputStream::Next(const void** data, int* size) { 28 | if (cursor_->isAtEnd()) { 29 | return false; 30 | } 31 | auto rng = cursor_->peekBytes(); 32 | *data = rng.data(); 33 | *size = rng.size(); 34 | position_ += rng.size(); 35 | return true; 36 | } 37 | 38 | void IOBufInputStream::BackUp(int count) { 39 | DCHECK_GE(count, 0); 40 | position_ -= cursor_->retreatAtMost(count); 41 | } 42 | 43 | bool IOBufInputStream::Skip(int count) { 44 | DCHECK_GE(count, 0); 45 | int n = cursor_->skipAtMost(count); 46 | position_ += n; 47 | return count == n; 48 | } 49 | 50 | google::protobuf::int64 IOBufInputStream::ByteCount() const { 51 | return position_; 52 | } 53 | 54 | } // namespace rdd 55 | -------------------------------------------------------------------------------- /raster/protocol/http/test/CookieTest.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/Cookie.h" 18 | #include 19 | 20 | using namespace rdd; 21 | 22 | TEST(Cookie, all) { 23 | Cookie cookie; 24 | 25 | cookie.set("fig", "newton"); 26 | cookie.set("sugar", "wafer"); 27 | EXPECT_STREQ("Set-Cookie: fig=newton\r\n" 28 | "Set-Cookie: sugar=wafer", cookie.str().c_str()); 29 | cookie.clear(); 30 | 31 | cookie.set("rocky", "road"); 32 | cookie.data["rocky"]->setAttr("path", "/cookie"); 33 | EXPECT_STREQ("Set-Cookie: rocky=road; Path=/cookie", cookie.str().c_str()); 34 | cookie.clear(); 35 | 36 | cookie.load("chips=ahoy; vienna=finger"); 37 | EXPECT_STREQ("Set-Cookie: chips=ahoy\r\n" 38 | "Set-Cookie: vienna=finger", cookie.str().c_str()); 39 | cookie.clear(); 40 | 41 | cookie.load("keebler=\"E=everybody; L=\\\"Loves\\\"; fudge=\\012;\";"); 42 | EXPECT_STREQ("Set-Cookie: keebler=\"E=everybody; L=\\\"Loves\\\"; fudge=\\012;\"", cookie.str().c_str()); 43 | cookie.clear(); 44 | } 45 | 46 | -------------------------------------------------------------------------------- /raster/net/EventPool.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/EventPool.h" 18 | 19 | namespace rdd { 20 | 21 | std::unique_ptr EventPool::get(const Peer& peer) { 22 | std::lock_guard guard(lock_); 23 | auto& q = pool_[peer]; 24 | if (!q.empty()) { 25 | auto event = std::move(q.back()); 26 | q.pop_back(); 27 | return event; 28 | } 29 | return nullptr; 30 | } 31 | 32 | void EventPool::giveBack(std::unique_ptr event) { 33 | std::lock_guard guard(lock_); 34 | pool_[event->peer()].push_back(std::move(event)); 35 | } 36 | 37 | size_t EventPool::count() const { 38 | std::lock_guard guard(lock_); 39 | size_t n = 0; 40 | for (auto& kv : pool_) { 41 | n += kv.second.size(); 42 | } 43 | return n; 44 | } 45 | 46 | EventPool* EventPoolManager::getPool(int id) { 47 | std::lock_guard guard(lock_); 48 | if (pool_.find(id) == pool_.end()) { 49 | pool_.emplace(id, acc::make_unique()); 50 | } 51 | return pool_[id].get(); 52 | } 53 | 54 | } // namespace rdd 55 | -------------------------------------------------------------------------------- /raster/net/Exception.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "accelerator/Conv.h" 23 | 24 | namespace rdd { 25 | 26 | class NetException : public std::exception { 27 | public: 28 | explicit NetException(std::string const& msg) 29 | : msg_(msg), code_(0) {} 30 | 31 | NetException(const NetException& other) 32 | : msg_(other.msg_), code_(other.code_) {} 33 | 34 | NetException(NetException&& other) noexcept 35 | : msg_(other.msg_), code_(other.code_) {} 36 | 37 | template 38 | explicit NetException(Args&&... args) 39 | : msg_(acc::to(std::forward(args)...)), 40 | code_(0) {} 41 | 42 | ~NetException(void) noexcept override {} 43 | 44 | const char* what(void) const noexcept override { 45 | return msg_.c_str(); 46 | } 47 | 48 | void setCode(int code) { code_ = code; } 49 | int getCode() const { return code_; } 50 | 51 | private: 52 | const std::string msg_; 53 | int code_; 54 | }; 55 | 56 | } // namespace rdd 57 | -------------------------------------------------------------------------------- /raster/protocol/http/SyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/NetUtil.h" 20 | #include "raster/protocol/http/SyncTransport.h" 21 | 22 | namespace rdd { 23 | 24 | class HTTPSyncClient { 25 | public: 26 | HTTPSyncClient(const ClientOption& option); 27 | 28 | HTTPSyncClient(const Peer& peer, 29 | const TimeoutOption& timeout); 30 | 31 | HTTPSyncClient(const Peer& peer, 32 | uint64_t ctimeout = 100000, 33 | uint64_t rtimeout = 1000000, 34 | uint64_t wtimeout = 300000); 35 | 36 | virtual ~HTTPSyncClient(); 37 | 38 | void close(); 39 | 40 | bool connect(); 41 | 42 | bool connected() const; 43 | 44 | bool fetch(const HTTPMessage& headers, std::unique_ptr body); 45 | 46 | HTTPMessage* headers() const; 47 | acc::IOBuf* body() const; 48 | HTTPHeaders* trailers() const; 49 | 50 | private: 51 | void init(); 52 | 53 | Peer peer_; 54 | TimeoutOption timeout_; 55 | std::unique_ptr transport_; 56 | }; 57 | 58 | } // namespace rdd 59 | -------------------------------------------------------------------------------- /raster/protocol/thrift/Util.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/thrift/Util.h" 18 | 19 | #include "accelerator/Logging.h" 20 | 21 | namespace rdd { 22 | namespace thrift { 23 | 24 | void setSeqId(::apache::thrift::transport::TMemoryBuffer* buf, int32_t seqid) { 25 | uint8_t* b; 26 | uint32_t n; 27 | buf->getBuffer(&b, &n); 28 | int32_t* p = (int32_t*)b; 29 | int32_t i = ntohl(*p++); 30 | i = i < 0 ? ntohl(*p++) : i + 1; 31 | if (n >= i + sizeof(int32_t)) { 32 | *(int32_t*)((uint8_t*)p + i) = htonl(seqid); 33 | } else { 34 | ACCLOG(WARN) << "invalid buf to set seqid"; 35 | } 36 | } 37 | 38 | int32_t getSeqId(::apache::thrift::transport::TMemoryBuffer* buf) { 39 | uint8_t* b; 40 | uint32_t n; 41 | buf->getBuffer(&b, &n); 42 | int32_t* p = (int32_t*)b; 43 | int32_t i = ntohl(*p++); 44 | i = i < 0 ? ntohl(*p++) : i + 1; 45 | if (n >= i + sizeof(int32_t)) { 46 | return ntohl(*(int32_t*)((uint8_t*)p + i)); 47 | } else { 48 | ACCLOG(WARN) << "invalid buf to get seqid"; 49 | return 0; 50 | } 51 | } 52 | 53 | } // namespace thrift 54 | } // namespace rdd 55 | -------------------------------------------------------------------------------- /raster/coroutine/FiberHub.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/coroutine/FiberHub.h" 18 | 19 | #include "raster/Portability.h" 20 | #include "raster/coroutine/FiberManager.h" 21 | 22 | DEFINE_uint64(fc_limit, 16384, // 1GB / 64KB 23 | "Limit # of fiber context (warning)."); 24 | 25 | DEFINE_uint64(fc_stack_size, 65536, // 64KB 26 | "Stack size of fiber context."); 27 | 28 | namespace rdd { 29 | 30 | void FiberHub::execute(Fiber* fiber, int poolId) { 31 | auto executor = getCPUThreadPoolExecutor(poolId); 32 | ACCLOG(V2) << executor->getThreadFactory()->namePrefix() 33 | << "* add " << *fiber; 34 | executor->add([&]() { FiberManager::run(fiber); }); 35 | } 36 | 37 | void FiberHub::execute(std::unique_ptr task, int poolId) { 38 | Fiber* fiber = task->fiber; 39 | if (!fiber) { 40 | if (Fiber::count() >= FLAGS_fc_limit) { 41 | ACCLOG(WARN) << "exceed fiber capacity"; 42 | // still add fiber 43 | } 44 | fiber = new Fiber(FLAGS_fc_stack_size, std::move(task)); 45 | } 46 | execute(fiber, poolId); 47 | } 48 | 49 | } // namespace rdd 50 | -------------------------------------------------------------------------------- /raster/protocol/http/AsyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/AsyncClient.h" 20 | #include "raster/protocol/http/HTTPHeaders.h" 21 | #include "raster/protocol/http/HTTPMessage.h" 22 | 23 | namespace rdd { 24 | 25 | class HTTPAsyncClient : public AsyncClient { 26 | public: 27 | HTTPAsyncClient(const ClientOption& option); 28 | 29 | HTTPAsyncClient(const Peer& peer, 30 | const TimeoutOption& timeout); 31 | 32 | HTTPAsyncClient(const Peer& peer, 33 | uint64_t ctimeout = 100000, 34 | uint64_t rtimeout = 1000000, 35 | uint64_t wtimeout = 300000); 36 | 37 | ~HTTPAsyncClient() override {} 38 | 39 | bool recv(); 40 | 41 | bool send(const HTTPMessage& headers, std::unique_ptr body); 42 | 43 | bool fetch(const HTTPMessage& headers, std::unique_ptr body); 44 | 45 | HTTPMessage* message() const; 46 | acc::IOBuf* body() const; 47 | HTTPHeaders* trailers() const; 48 | 49 | protected: 50 | std::shared_ptr makeChannel() override; 51 | }; 52 | 53 | } // namespace rdd 54 | -------------------------------------------------------------------------------- /raster/framework/HubAdaptor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/concurrency/CPUThreadPoolExecutor.h" 20 | #include "accelerator/concurrency/IOThreadPoolExecutor.h" 21 | #include "raster/net/Acceptor.h" 22 | #include "raster/net/NetHub.h" 23 | 24 | namespace rdd { 25 | 26 | class AsyncClient; 27 | 28 | class HubAdaptor : public NetHub { 29 | public: 30 | HubAdaptor(); 31 | 32 | void configThreads(const std::string& name, size_t threadCount); 33 | 34 | void addService(std::unique_ptr service); 35 | 36 | void configService( 37 | const std::string& name, 38 | int port, 39 | const TimeoutOption& timeoutOpt); 40 | 41 | void startService(); 42 | 43 | // FiberHub 44 | acc::CPUThreadPoolExecutor* getCPUThreadPoolExecutor(int poolId) override; 45 | // NetHub 46 | acc::EventLoop* getEventLoop() override; 47 | 48 | std::shared_ptr 49 | getSharedCPUThreadPoolExecutor(int poolId); 50 | 51 | private: 52 | std::unique_ptr ioPool_; 53 | std::map> cpuPoolMap_; 54 | 55 | Acceptor acceptor_; 56 | }; 57 | 58 | } // namespace rdd 59 | -------------------------------------------------------------------------------- /raster/protocol/http/RequestHandler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/protocol/http/HTTPException.h" 20 | #include "raster/protocol/http/HTTPMessage.h" 21 | #include "raster/protocol/http/ResponseBuilder.h" 22 | 23 | namespace rdd { 24 | 25 | class RequestHandler { 26 | public: 27 | virtual ~RequestHandler() {} 28 | 29 | virtual void onGet (); 30 | virtual void onPost (); 31 | virtual void onPut (); 32 | virtual void onHead (); 33 | virtual void onDelete (); 34 | virtual void onConnect (); 35 | virtual void onOptions (); 36 | virtual void onTrace (); 37 | 38 | virtual void prepare() {} 39 | virtual void finish() {} 40 | 41 | void handleException(const HTTPException& e); 42 | void handleException(const std::exception& e); 43 | void handleException(); 44 | 45 | void sendError(uint16_t code = 500); 46 | 47 | std::string getLocale(); 48 | std::string getBrowserLocale(); 49 | virtual std::string getUserLocale() { return ""; } 50 | 51 | HTTPMessage* headers; 52 | acc::IOBuf* body; 53 | HTTPHeaders* trailers; 54 | ResponseBuilder response; 55 | 56 | protected: 57 | std::string locale_; 58 | }; 59 | 60 | } // namespace rdd 61 | -------------------------------------------------------------------------------- /raster/3rd/thrift/protocol/TBase64Utils.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #ifndef _THRIFT_PROTOCOL_TBASE64UTILS_H_ 21 | #define _THRIFT_PROTOCOL_TBASE64UTILS_H_ 22 | 23 | #include 24 | #include 25 | 26 | namespace apache { 27 | namespace thrift { 28 | namespace protocol { 29 | 30 | // in must be at least len bytes 31 | // len must be 1, 2, or 3 32 | // buf must be a buffer of at least 4 bytes and may not overlap in 33 | // the data is not padded with '='; the caller can do this if desired 34 | void base64_encode(const uint8_t* in, uint32_t len, uint8_t* buf); 35 | 36 | // buf must be a buffer of at least 4 bytes and contain base64 encoded values 37 | // buf will be changed to contain output bytes 38 | // len is number of bytes to consume from input (must be 2, 3, or 4) 39 | // no '=' padding should be included in the input 40 | void base64_decode(uint8_t* buf, uint32_t len); 41 | } 42 | } 43 | } // apache::thrift::protocol 44 | 45 | #endif // #define _THRIFT_PROTOCOL_TBASE64UTILS_H_ 46 | -------------------------------------------------------------------------------- /raster/net/Group.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/Group.h" 18 | 19 | namespace rdd { 20 | 21 | using acc::SharedMutex; 22 | 23 | Group::Group(size_t capacity) : capacity_(capacity) { 24 | increase(1); 25 | } 26 | 27 | Group::Key Group::create(size_t groupSize) { 28 | SharedMutex::WriteHolder guard(lock_); 29 | if (groupKeys_.empty()) { 30 | capacity_ *= 2; 31 | increase(groupCounts_.size()); 32 | } 33 | assert(groupSize > 0); 34 | auto group = groupKeys_.top(); 35 | groupKeys_.pop(); 36 | groupCounts_[group] = groupSize; 37 | return group; 38 | } 39 | 40 | bool Group::finish(Group::Key group) { 41 | if (group == 0) { 42 | return true; 43 | } 44 | SharedMutex::WriteHolder guard(lock_); 45 | assert(groupCounts_[group] > 0); 46 | groupCounts_[group]--; 47 | if (groupCounts_[group] <= 0) { 48 | groupKeys_.push(group); 49 | return true; 50 | } 51 | return false; 52 | } 53 | 54 | size_t Group::count() const { 55 | SharedMutex::ReadHolder guard(lock_); 56 | return capacity_ - groupKeys_.size(); 57 | } 58 | 59 | void Group::increase(size_t bound) { 60 | for (auto i = capacity_; i >= bound; --i) { 61 | groupKeys_.push(i); 62 | } 63 | groupCounts_.resize(capacity_ + 1, 0); 64 | } 65 | 66 | } // namespace rdd 67 | -------------------------------------------------------------------------------- /raster/protocol/proto/AsyncServer.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/proto/AsyncServer.h" 18 | 19 | #include "raster/protocol/binary/Transport.h" 20 | #include "raster/protocol/proto/Processor.h" 21 | 22 | namespace rdd { 23 | 24 | void PBAsyncServer::makeChannel(int port, const TimeoutOption& timeout) { 25 | Peer peer; 26 | peer.setFromLocalPort(port); 27 | channel_ = std::make_shared( 28 | peer, 29 | timeout, 30 | acc::make_unique(), 31 | acc::make_unique(this)); 32 | } 33 | 34 | void PBAsyncServer::addService( 35 | const std::shared_ptr& service) { 36 | if (service) { 37 | services_[service->GetDescriptor()] = service; 38 | } 39 | } 40 | 41 | google::protobuf::Service* PBAsyncServer::getService( 42 | const google::protobuf::MethodDescriptor* method) { 43 | return services_[method->service()].get(); 44 | } 45 | 46 | void PBAsyncServer::addHandle(std::shared_ptr& handle) { 47 | handles_.insert(std::make_pair(handle->callId, handle)); 48 | } 49 | 50 | std::shared_ptr 51 | PBAsyncServer::removeHandle(const std::string& callId) { 52 | return handles_.erase_get(callId); 53 | } 54 | 55 | } // namespace rdd 56 | -------------------------------------------------------------------------------- /examples/http/Server.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017, Yeolar 3 | */ 4 | 5 | #include 6 | 7 | #include "raster/framework/Config.h" 8 | #include "raster/framework/Signal.h" 9 | #include "raster/framework/HubAdaptor.h" 10 | #include "raster/protocol/http/AsyncServer.h" 11 | #include "accelerator/Logging.h" 12 | #include "accelerator/Portability.h" 13 | #include "accelerator/ReflectObject.h" 14 | #include "accelerator/ScopeGuard.h" 15 | #include "accelerator/Uuid.h" 16 | 17 | static const char* VERSION = "1.1.0"; 18 | 19 | DEFINE_string(conf, "server.json", "Server config file"); 20 | 21 | using namespace rdd; 22 | 23 | class BaseHandler : public RequestHandler { 24 | public: 25 | BaseHandler() { 26 | ACCLOG(DEBUG) << "BaseHandler init"; 27 | } 28 | 29 | ~BaseHandler() override {} 30 | 31 | void onGet() override { 32 | response 33 | .status(200) 34 | .body("Yes, got it.\r\n") 35 | .sendWithEOM(); 36 | } 37 | }; 38 | 39 | int main(int argc, char* argv[]) { 40 | gflags::SetVersionString(VERSION); 41 | gflags::SetUsageMessage("Usage : ./httpserver"); 42 | gflags::ParseCommandLineFlags(&argc, &argv, true); 43 | 44 | setupIgnoreSignal(SIGPIPE); 45 | setupShutdownSignal(SIGINT); 46 | setupShutdownSignal(SIGTERM); 47 | 48 | auto httpserver = acc::make_unique("HTTP"); 49 | httpserver->addHandler("/"); 50 | 51 | acc::Singleton::get()->addService(std::move(httpserver)); 52 | 53 | config(FLAGS_conf.c_str(), { 54 | {configLogging, "logging"}, 55 | {configService, "service"}, 56 | {configThreadPool, "thread"}, 57 | {configNet, "net"}, 58 | {configMonitor, "monitor"}, 59 | {configJobGraph, "job.graph"} 60 | }); 61 | 62 | ACCLOG(INFO) << "rdd start ... ^_^"; 63 | acc::Singleton::get()->startService(); 64 | 65 | gflags::ShutDownCommandLineFlags(); 66 | 67 | return 0; 68 | } 69 | -------------------------------------------------------------------------------- /raster/framework/Config.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "accelerator/Function.h" 22 | #include "accelerator/json.h" 23 | 24 | namespace rdd { 25 | 26 | typedef std::pair ConfigTask; 27 | 28 | std::string generateDefault(); 29 | 30 | void configLogging(const acc::dynamic& j, bool reload); 31 | void configProcess(const acc::dynamic& j, bool reload); 32 | void configService(const acc::dynamic& j, bool reload); 33 | void configThreadPool(const acc::dynamic& j, bool reload); 34 | void configNet(const acc::dynamic& j, bool reload); 35 | void configMonitor(const acc::dynamic& j, bool reload); 36 | void configDegrader(const acc::dynamic& j, bool reload); 37 | void configSampler(const acc::dynamic& j, bool reload); 38 | void configJobGraph(const acc::dynamic& j, bool reload); 39 | 40 | class ConfigManager { 41 | public: 42 | ConfigManager() {} 43 | 44 | void setConfFile(const char* conf) { 45 | conf_ = conf; 46 | } 47 | 48 | void addTask(const ConfigTask& task) { 49 | tasks_.push_back(task); 50 | } 51 | 52 | void load(); 53 | 54 | private: 55 | const char* conf_; 56 | std::vector tasks_; 57 | bool inited_{false}; 58 | }; 59 | 60 | void config(const char* name, std::initializer_list confs); 61 | 62 | } // namespace rdd 63 | -------------------------------------------------------------------------------- /raster/protocol/proto/RpcController.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "accelerator/Function.h" 25 | #include "accelerator/io/Cursor.h" 26 | 27 | namespace rdd { 28 | 29 | class PBRpcController : public google::protobuf::RpcController { 30 | public: 31 | PBRpcController() {} 32 | ~PBRpcController() override {} 33 | 34 | // Client-side 35 | void Reset() override; 36 | bool Failed() const override; 37 | std::string ErrorText() const override; 38 | void StartCancel() override; 39 | 40 | // Server side 41 | void SetFailed(const std::string& reason) override; 42 | bool IsCanceled() const override; 43 | void NotifyOnCancel(google::protobuf::Closure* closure) override; 44 | 45 | void parseFrom(acc::io::Cursor& in); 46 | void serializeTo(acc::IOBufQueue& out) const; 47 | 48 | void copyFrom(const PBRpcController& o); 49 | 50 | void setCanceled() { canceled_ = true; } 51 | void setStartCancel(acc::VoidFunc&& cancelFunc); 52 | 53 | void complete(); 54 | 55 | private: 56 | bool canceled_{false}; 57 | bool failed_{false}; 58 | std::string failedReason_; 59 | std::set closures_; 60 | std::mutex closuresLock_; 61 | acc::VoidFunc cancelFunc_; 62 | }; 63 | 64 | } // namespace rdd 65 | -------------------------------------------------------------------------------- /raster/3rd/thrift/transport/TTransportException.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | #include "TTransportException.h" 21 | 22 | namespace apache { 23 | namespace thrift { 24 | namespace transport { 25 | 26 | const char* TTransportException::what() const throw() { 27 | if (message_.empty()) { 28 | switch (type_) { 29 | case UNKNOWN: 30 | return "TTransportException: Unknown transport exception"; 31 | case NOT_OPEN: 32 | return "TTransportException: Transport not open"; 33 | case TIMED_OUT: 34 | return "TTransportException: Timed out"; 35 | case END_OF_FILE: 36 | return "TTransportException: End of file"; 37 | case INTERRUPTED: 38 | return "TTransportException: Interrupted"; 39 | case BAD_ARGS: 40 | return "TTransportException: Invalid arguments"; 41 | case CORRUPTED_DATA: 42 | return "TTransportException: Corrupted Data"; 43 | case INTERNAL_ERROR: 44 | return "TTransportException: Internal error"; 45 | default: 46 | return "TTransportException: (Invalid exception type)"; 47 | } 48 | } else { 49 | return message_.c_str(); 50 | } 51 | } 52 | } 53 | } 54 | } // apache::thrift::transport 55 | -------------------------------------------------------------------------------- /raster/net/Channel.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Processor.h" 20 | #include "raster/net/Transport.h" 21 | 22 | namespace rdd { 23 | 24 | class Channel { 25 | public: 26 | Channel(const Peer& peer, 27 | const TimeoutOption& timeout, 28 | std::unique_ptr transportFactory = nullptr, 29 | std::unique_ptr processorFactory = nullptr) 30 | : id_(peer.port()), 31 | peer_(peer), 32 | timeout_(timeout), 33 | transportFactory_(std::move(transportFactory)), 34 | processorFactory_(std::move(processorFactory)) { 35 | } 36 | 37 | int id() const { return id_; } 38 | 39 | const Peer& peer() const { return peer_; } 40 | 41 | TimeoutOption timeoutOption() const { return timeout_; } 42 | 43 | TransportFactory* transportFactory() const { 44 | return transportFactory_.get(); 45 | } 46 | 47 | ProcessorFactory* processorFactory() const { 48 | return processorFactory_.get(); 49 | } 50 | 51 | private: 52 | int id_; 53 | Peer peer_; 54 | TimeoutOption timeout_; 55 | std::unique_ptr transportFactory_; 56 | std::unique_ptr processorFactory_; 57 | }; 58 | 59 | inline std::ostream& operator<<(std::ostream& os, const Channel& channel) { 60 | os << "channel[" << channel.id() << "]"; 61 | return os; 62 | } 63 | 64 | } // namespace rdd 65 | -------------------------------------------------------------------------------- /raster/protocol/proto/Processor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/net/Processor.h" 22 | #include "raster/protocol/proto/AsyncServer.h" 23 | 24 | namespace rdd { 25 | 26 | class PBProcessor : public Processor { 27 | public: 28 | PBProcessor(Event* event, PBAsyncServer* server) 29 | : Processor(event), server_(server) {} 30 | 31 | ~PBProcessor() override {} 32 | 33 | void run() override; 34 | 35 | private: 36 | void process( 37 | const std::string& callId, 38 | const google::protobuf::MethodDescriptor* method, 39 | const std::shared_ptr& request); 40 | 41 | void cancel(const std::string& callId); 42 | 43 | void finish(std::shared_ptr handle); 44 | 45 | void sendResponse( 46 | const std::string& callId, 47 | PBRpcController* controller, 48 | google::protobuf::Message* response); 49 | 50 | PBAsyncServer* server_; 51 | }; 52 | 53 | class PBProcessorFactory : public ProcessorFactory { 54 | public: 55 | PBProcessorFactory(PBAsyncServer* server) : server_(server) {} 56 | ~PBProcessorFactory() override {} 57 | 58 | std::unique_ptr create(Event* event) override { 59 | return acc::make_unique(event, server_); 60 | } 61 | 62 | private: 63 | PBAsyncServer* server_; 64 | }; 65 | 66 | } // namespace rdd 67 | -------------------------------------------------------------------------------- /raster/protocol/proto/SyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/net/NetUtil.h" 22 | #include "raster/protocol/proto/RpcChannel.h" 23 | #include "raster/protocol/proto/RpcController.h" 24 | 25 | namespace rdd { 26 | 27 | template 28 | class PBSyncClient { 29 | public: 30 | PBSyncClient(const ClientOption& option); 31 | 32 | PBSyncClient(const Peer& peer, 33 | const TimeoutOption& timeout); 34 | 35 | PBSyncClient(const Peer& peer, 36 | uint64_t ctimeout = 100000, 37 | uint64_t rtimeout = 1000000, 38 | uint64_t wtimeout = 300000); 39 | 40 | virtual ~PBSyncClient(); 41 | 42 | void close(); 43 | 44 | bool connect(); 45 | 46 | bool connected() const; 47 | 48 | template 49 | bool fetch(void (C::*func)(google::protobuf::RpcController*, 50 | const Req*, Res*, 51 | google::protobuf::Closure*), 52 | Res& _return, 53 | const Req& request); 54 | 55 | private: 56 | void init(); 57 | 58 | Peer peer_; 59 | TimeoutOption timeout_; 60 | std::unique_ptr rpcChannel_; 61 | std::unique_ptr controller_; 62 | std::unique_ptr service_; 63 | }; 64 | 65 | } // namespace rdd 66 | 67 | #include "raster/protocol/proto/SyncClient-inl.h" 68 | -------------------------------------------------------------------------------- /raster/framework/Sampler.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma one 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "accelerator/Algorithm.h" 25 | 26 | namespace rdd { 27 | 28 | class Sampler { 29 | public: 30 | virtual bool hit() = 0; 31 | }; 32 | 33 | class PercentSampler : public Sampler { 34 | public: 35 | PercentSampler() {} 36 | 37 | void setup(bool open, double percent); 38 | 39 | bool hit() override; 40 | 41 | private: 42 | std::atomic open_{false}; 43 | std::atomic percent_{0}; 44 | }; 45 | 46 | class SamplerManager { 47 | public: 48 | SamplerManager() {} 49 | 50 | template 51 | void setupSampler(const std::string& name, Args&&... args); 52 | 53 | bool hit(const std::string& name); 54 | 55 | private: 56 | std::map> samplers_; 57 | std::mutex lock_; 58 | }; 59 | 60 | template 61 | void SamplerManager::setupSampler(const std::string& name, Args&&... args) { 62 | std::lock_guard guard(lock_); 63 | if (!acc::containKey(samplers_, name)) { 64 | samplers_[name].reset(new Sam()); 65 | } 66 | Sampler* d = samplers_[name].get(); 67 | reinterpret_cast(d)->setup(std::forward(args)...); 68 | } 69 | 70 | } // namespace rdd 71 | 72 | #define RDDSAM_HIT(name) \ 73 | ::acc::Singleton< ::rdd::SamplerManager>::get()->hit(name) 74 | 75 | -------------------------------------------------------------------------------- /raster/net/Transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/io/IOBufQueue.h" 20 | #include "raster/net/Socket.h" 21 | 22 | namespace rdd { 23 | 24 | class Transport { 25 | public: 26 | enum IngressState { 27 | kInit, 28 | kOnReading, 29 | kFinish, 30 | kError, 31 | }; 32 | 33 | static const uint32_t kMinReadSize = 1460; 34 | static const uint32_t kMaxReadSize = 4000; 35 | 36 | virtual ~Transport() {} 37 | 38 | virtual void reset() = 0; 39 | 40 | virtual void processReadData() = 0; 41 | 42 | void setPeerAddress(const Peer& peer) { peerAddr_ = peer; } 43 | const Peer& peerAddress() const { return peerAddr_; } 44 | 45 | void setLocalAddress(const Peer& local) { localAddr_ = local; } 46 | const Peer& localAddress() const { return localAddr_; } 47 | 48 | void getReadBuffer(void** buf, size_t* bufSize); 49 | void readDataAvailable(size_t readSize); 50 | 51 | int readData(Socket* socket); 52 | int writeData(Socket* socket); 53 | 54 | void clone(Transport* other); 55 | 56 | protected: 57 | Peer peerAddr_; 58 | Peer localAddr_; 59 | IngressState state_; 60 | acc::IOBufQueue readBuf_{acc::IOBufQueue::cacheChainLength()}; 61 | acc::IOBufQueue writeBuf_{acc::IOBufQueue::cacheChainLength()}; 62 | }; 63 | 64 | class TransportFactory { 65 | public: 66 | virtual ~TransportFactory() {} 67 | virtual std::unique_ptr create() = 0; 68 | }; 69 | 70 | } // namespace rdd 71 | -------------------------------------------------------------------------------- /raster/plugins/flume/gen-cpp/scribe_types.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Autogenerated by Thrift Compiler (0.9.1) 3 | * 4 | * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING 5 | * @generated 6 | */ 7 | #ifndef scribe_TYPES_H 8 | #define scribe_TYPES_H 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | 17 | 18 | namespace rdd { 19 | 20 | enum ResultCode { 21 | OK = 0, 22 | TRY_LATER = 1 23 | }; 24 | 25 | extern const std::map _ResultCode_VALUES_TO_NAMES; 26 | 27 | typedef struct _LogEntry__isset { 28 | _LogEntry__isset() : category(false), message(false) {} 29 | bool category; 30 | bool message; 31 | } _LogEntry__isset; 32 | 33 | class LogEntry { 34 | public: 35 | 36 | static const char* ascii_fingerprint; // = "07A9615F837F7D0A952B595DD3020972"; 37 | static const uint8_t binary_fingerprint[16]; // = {0x07,0xA9,0x61,0x5F,0x83,0x7F,0x7D,0x0A,0x95,0x2B,0x59,0x5D,0xD3,0x02,0x09,0x72}; 38 | 39 | LogEntry() : category(), message() { 40 | } 41 | 42 | virtual ~LogEntry() throw() {} 43 | 44 | std::string category; 45 | std::string message; 46 | 47 | _LogEntry__isset __isset; 48 | 49 | void __set_category(const std::string& val) { 50 | category = val; 51 | } 52 | 53 | void __set_message(const std::string& val) { 54 | message = val; 55 | } 56 | 57 | bool operator == (const LogEntry & rhs) const 58 | { 59 | if (!(category == rhs.category)) 60 | return false; 61 | if (!(message == rhs.message)) 62 | return false; 63 | return true; 64 | } 65 | bool operator != (const LogEntry &rhs) const { 66 | return !(*this == rhs); 67 | } 68 | 69 | bool operator < (const LogEntry & ) const; 70 | 71 | uint32_t read(::apache::thrift::protocol::TProtocol* iprot); 72 | uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; 73 | 74 | }; 75 | 76 | void swap(LogEntry &a, LogEntry &b); 77 | 78 | } // namespace 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /raster/coroutine/FiberManager.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/coroutine/FiberManager.h" 18 | 19 | #include "accelerator/Logging.h" 20 | 21 | namespace rdd { 22 | 23 | __thread Fiber* FiberManager::fiber_ = nullptr; 24 | 25 | void FiberManager::update(Fiber* fiber) { 26 | fiber_ = fiber; 27 | } 28 | 29 | Fiber* FiberManager::get() { 30 | return fiber_; 31 | } 32 | 33 | void FiberManager::run(Fiber* fiber) { 34 | update(fiber); 35 | fiber->setStatus(Fiber::kRunable); 36 | fiber->execute(); 37 | switch (fiber->status()) { 38 | case Fiber::kBlock: { 39 | for (auto& fn : fiber->task()->blockCallbacks) { 40 | fn(); 41 | } 42 | break; 43 | } 44 | case Fiber::kExit: { 45 | fiber->task()->scheduleCallback(); 46 | delete fiber; 47 | break; 48 | } 49 | default: { 50 | ACCLOG(WARN) << *fiber << " status error"; 51 | delete fiber; 52 | break; 53 | } 54 | } 55 | } 56 | 57 | bool FiberManager::yield() { 58 | Fiber* fiber = get(); 59 | if (fiber) { 60 | fiber->yield(Fiber::kBlock); 61 | return true; 62 | } 63 | return false; 64 | } 65 | 66 | bool FiberManager::exit() { 67 | Fiber* fiber = get(); 68 | if (fiber) { 69 | fiber->yield(Fiber::kExit); 70 | return true; 71 | } 72 | return false; 73 | } 74 | 75 | Fiber::Task* getCurrentFiberTask() { 76 | Fiber* fiber = FiberManager::get(); 77 | return fiber ? fiber->task() : nullptr; 78 | } 79 | 80 | } // namespace rdd 81 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPMethod.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/HTTPMethod.h" 18 | 19 | #include 20 | 21 | #include "accelerator/Conv.h" 22 | #include "accelerator/thread/UnionBasedStatic.h" 23 | #include "raster/protocol/http/HTTPException.h" 24 | 25 | #define HTTP_METHOD_STR(method) #method 26 | 27 | namespace { 28 | 29 | // Method strings. This is a union-based static because this structure is 30 | // accessed from multiple threads and still needs to be accessible after exit() 31 | // is called to avoid crashing. 32 | typedef std::vector StringVector; 33 | DEFINE_UNION_STATIC_CONST_NO_INIT(StringVector, Vector, s_methodStrings); 34 | 35 | __attribute__((__constructor__)) 36 | void initMethodStrings() { 37 | new (const_cast(&s_methodStrings.data)) StringVector { 38 | HTTP_METHOD_GEN(HTTP_METHOD_STR) 39 | }; 40 | } 41 | 42 | } 43 | 44 | namespace rdd { 45 | 46 | HTTPMethod stringToMethod(acc::StringPiece method) { 47 | int index = 0; 48 | for (auto& cur : s_methodStrings.data) { 49 | if (caseInsensitiveEqual(cur, method)) { 50 | return HTTPMethod(index); 51 | } 52 | index++; 53 | } 54 | throw HTTPException(405, method); 55 | } 56 | 57 | const std::string& methodToString(HTTPMethod method) { 58 | return s_methodStrings.data[static_cast(method)]; 59 | } 60 | 61 | std::ostream& operator <<(std::ostream& out, HTTPMethod method) { 62 | out << methodToString(method); 63 | return out; 64 | } 65 | 66 | } // namespace rdd 67 | -------------------------------------------------------------------------------- /raster/net/Acceptor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/Acceptor.h" 18 | 19 | namespace rdd { 20 | 21 | Acceptor::Acceptor(std::shared_ptr hub) 22 | : hub_(hub), loop_(acc::make_unique()) { 23 | } 24 | 25 | void Acceptor::addService(std::unique_ptr service) { 26 | services_.emplace(service->name(), std::move(service)); 27 | } 28 | 29 | void Acceptor::configService( 30 | const std::string& name, int port, const TimeoutOption& timeout) { 31 | auto service = acc::get_deref_smart_ptr(services_, name); 32 | if (!service) { 33 | ACCLOG(FATAL) << "service: [" << name << "] not added"; 34 | return; 35 | } 36 | 37 | service->makeChannel(port, timeout); 38 | listen(service); 39 | } 40 | 41 | void Acceptor::listen(Service* service, int backlog) { 42 | int port = service->channel()->id(); 43 | auto socket = Socket::createAsyncSocket(); 44 | if (!socket || 45 | !(socket->bind(port)) || 46 | !(socket->listen(backlog))) { 47 | throw std::runtime_error("socket listen failed"); 48 | } 49 | 50 | auto event = new Event(service->channel(), std::move(socket)); 51 | event->setState(Event::kListen); 52 | event->setCompleteCallback([&](Event* ev) { hub_->execute(ev); }); 53 | event->setCloseCallback([&](Event* ev) { hub_->execute(ev); }); 54 | loop_->addEvent(event); 55 | ACCLOG(INFO) << *event << " listen on port=" << port; 56 | } 57 | 58 | void Acceptor::start() { 59 | loop_->loop(); 60 | } 61 | 62 | void Acceptor::stop() { 63 | loop_->stop(); 64 | } 65 | 66 | } // namespace rdd 67 | -------------------------------------------------------------------------------- /raster/net/Transport.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/Transport.h" 18 | 19 | namespace rdd { 20 | 21 | void Transport::getReadBuffer(void** buf, size_t* bufSize) { 22 | std::pair readSpace = 23 | readBuf_.preallocate(kMinReadSize, kMaxReadSize); 24 | *buf = readSpace.first; 25 | *bufSize = readSpace.second; 26 | } 27 | 28 | void Transport::readDataAvailable(size_t readSize) { 29 | ACCLOG(V3) << "read completed, bytes=" << readSize; 30 | readBuf_.postallocate(readSize); 31 | processReadData(); 32 | } 33 | 34 | int Transport::readData(Socket* socket) { 35 | state_ = kOnReading; 36 | while (state_ != kFinish) { 37 | void* buf; 38 | size_t bufSize; 39 | getReadBuffer(&buf, &bufSize); 40 | ssize_t r = socket->recv(buf, bufSize); 41 | if (r <= 0) { 42 | return r; 43 | } 44 | readDataAvailable(r); 45 | if (state_ == kError) { 46 | return -1; 47 | } 48 | } 49 | return 1; 50 | } 51 | 52 | int Transport::writeData(Socket* socket) { 53 | if (state_ == kError) { 54 | return -1; 55 | } 56 | const acc::IOBuf* buf; 57 | while ((buf = writeBuf_.front()) != nullptr && buf->length() != 0) { 58 | ssize_t r = socket->send(buf->data(), buf->length()); 59 | if (r < 0) { 60 | return r; 61 | } 62 | writeBuf_.trimStart(r); 63 | } 64 | return 1; 65 | } 66 | 67 | void Transport::clone(Transport* other) { 68 | state_ = other->state_; 69 | readBuf_.append(other->readBuf_.front()->clone()); 70 | writeBuf_.append(other->writeBuf_.front()->clone()); 71 | } 72 | 73 | } // namespace rdd 74 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPCommonHeaders.template.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Facebook, Inc. 3 | * Copyright 2017 Yeolar 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | #include 22 | 23 | namespace rdd { 24 | 25 | /** 26 | * Codes (hashes) of common HTTP header names 27 | */ 28 | enum HTTPHeaderCode : uint8_t { 29 | // code reserved to indicate the absence of an HTTP header 30 | HTTP_HEADER_NONE = 0, 31 | // code for any HTTP header name not in the list of common headers 32 | HTTP_HEADER_OTHER = 1, 33 | 34 | /* the following is a placeholder for the build script to generate a list 35 | * of enum values from the list in HTTPCommonHeaders.txt 36 | * 37 | * enum name of Some-Header is HTTP_HEADER_SOME_HEADER, 38 | * so an example fragment of the generated list could be: 39 | * ... 40 | * HTTP_HEADER_WARNING = 65, 41 | * HTTP_HEADER_WWW_AUTHENTICATE = 66, 42 | * HTTP_HEADER_X_BACKEND = 67, 43 | * HTTP_HEADER_X_BLOCKID = 68, 44 | * ... 45 | */ 46 | %%%%% 47 | 48 | }; 49 | 50 | class HTTPCommonHeaders { 51 | public: 52 | // Perfect hash function to match common HTTP header names 53 | static HTTPHeaderCode hash(const char* name, size_t len); 54 | 55 | inline static HTTPHeaderCode hash(const std::string& name) { 56 | return hash(name.data(), name.length()); 57 | } 58 | 59 | static std::string* initHeaderNames(); 60 | 61 | inline static const std::string* getPointerToHeaderName(HTTPHeaderCode code) { 62 | static const auto headerNames = initHeaderNames(); 63 | 64 | return headerNames + code; 65 | } 66 | }; 67 | 68 | } // namespace rdd 69 | -------------------------------------------------------------------------------- /raster/protocol/proto/AsyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/AsyncClient.h" 20 | #include "raster/protocol/proto/RpcChannel.h" 21 | 22 | namespace rdd { 23 | 24 | template 25 | class PBAsyncClient : public AsyncClient { 26 | public: 27 | PBAsyncClient(const ClientOption& option); 28 | 29 | PBAsyncClient(const Peer& peer, 30 | const TimeoutOption& timeout); 31 | 32 | PBAsyncClient(const Peer& peer, 33 | uint64_t ctimeout = 100000, 34 | uint64_t rtimeout = 1000000, 35 | uint64_t wtimeout = 300000); 36 | 37 | ~PBAsyncClient() override {} 38 | 39 | bool recv(); 40 | 41 | template 42 | bool send(void (C::*func)(google::protobuf::RpcController*, 43 | const Req*, Res*, 44 | google::protobuf::Closure*), 45 | Res& response, 46 | const Req& request); 47 | 48 | template 49 | bool fetch(void (C::*func)(google::protobuf::RpcController*, 50 | const Req*, Res*, 51 | google::protobuf::Closure*), 52 | Res& response, 53 | const Req& request); 54 | 55 | protected: 56 | virtual std::shared_ptr makeChannel(); 57 | 58 | private: 59 | void init(); 60 | 61 | std::unique_ptr rpcChannel_; 62 | std::unique_ptr controller_; 63 | std::unique_ptr service_; 64 | }; 65 | 66 | } // namespace rdd 67 | 68 | #include "raster/protocol/proto/AsyncClient-inl.h" 69 | -------------------------------------------------------------------------------- /raster/protocol/http/HTTPCommonHeaders.template.gperf: -------------------------------------------------------------------------------- 1 | %{ 2 | /* 3 | * Copyright (c) 2015, Facebook, Inc. 4 | * Copyright 2017 Yeolar 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | #include 20 | 21 | #include "raster/protocol/http/HTTPCommonHeaders.h" 22 | #include "accelerator/Logging.h" 23 | 24 | namespace rdd { 25 | 26 | %} 27 | 28 | %language=C++ 29 | %compare-lengths 30 | %ignore-case 31 | %struct-type 32 | %readonly-tables 33 | %global-table 34 | %enum 35 | %define class-name HTTPCommonHeadersInternal 36 | 37 | struct HTTPCommonHeaderName { const char* name; uint8_t code; }; 38 | 39 | 40 | /* the following is a placeholder for the build script to generate a list 41 | * of enum values from the list in HTTPCommonHeaders.txt 42 | */ 43 | %%%%% 44 | 45 | HTTPHeaderCode HTTPCommonHeaders::hash(const char* name, size_t len) { 46 | const HTTPCommonHeaderName* match = 47 | HTTPCommonHeadersInternal::in_word_set(name, len); 48 | return (match == nullptr) ? HTTP_HEADER_OTHER : HTTPHeaderCode(match->code); 49 | } 50 | 51 | std::string* HTTPCommonHeaders::initHeaderNames() { 52 | DCHECK_LE(MAX_HASH_VALUE, 255); 53 | auto headerNames = new std::string[256]; 54 | for (int j = MIN_HASH_VALUE; j <= MAX_HASH_VALUE; ++j) { 55 | uint8_t code = wordlist[j].code; 56 | const uint8_t OFFSET = 2; // first 2 values are reserved for special cases 57 | if (code >= OFFSET && code < TOTAL_KEYWORDS + OFFSET 58 | && wordlist[j].name[0] != '\0') { 59 | DCHECK_EQ(headerNames[code], std::string()); 60 | // this would mean a duplicate header code in the .gperf file 61 | 62 | headerNames[code] = wordlist[j].name; 63 | } 64 | } 65 | return headerNames; 66 | } 67 | 68 | } // namespace rdd 69 | -------------------------------------------------------------------------------- /raster/protocol/thrift/AsyncServer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Service.h" 20 | #include "raster/protocol/thrift/Processor.h" 21 | #include "raster/protocol/binary/Transport.h" 22 | 23 | namespace rdd { 24 | 25 | template 26 | class TAsyncServer : public Service { 27 | public: 28 | TAsyncServer(acc::StringPiece name) : Service(name) {} 29 | ~TAsyncServer() override {} 30 | 31 | void makeChannel(int port, const TimeoutOption& timeout) override { 32 | Peer peer; 33 | peer.setFromLocalPort(port); 34 | channel_ = std::make_shared( 35 | peer, 36 | timeout, 37 | acc::make_unique(), 38 | acc::make_unique>()); 39 | } 40 | 41 | If* handler() { 42 | auto pf = channel_->processorFactory(); 43 | return ((TProcessorFactory*)(pf))->handler(); 44 | } 45 | }; 46 | 47 | template 48 | class TZlibAsyncServer : public Service { 49 | public: 50 | TZlibAsyncServer(acc::StringPiece name) : Service(name) {} 51 | ~TZlibAsyncServer() override {} 52 | 53 | void makeChannel(int port, const TimeoutOption& timeout) override { 54 | Peer peer; 55 | peer.setFromLocalPort(port); 56 | channel_ = std::make_shared( 57 | peer, 58 | timeout, 59 | acc::make_unique(), 60 | acc::make_unique>()); 61 | } 62 | 63 | If* handler() { 64 | auto pf = channel_->processorFactory(); 65 | return ((TProcessorFactory*)(pf))->handler(); 66 | } 67 | }; 68 | 69 | } // namespace rdd 70 | -------------------------------------------------------------------------------- /examples/empty/Server.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (C) 2017, Yeolar 3 | */ 4 | 5 | #include 6 | 7 | #include "raster/framework/Config.h" 8 | #include "raster/framework/HubAdaptor.h" 9 | #include "raster/framework/Signal.h" 10 | #include "raster/protocol/thrift/AsyncServer.h" 11 | #include "accelerator/Logging.h" 12 | #include "accelerator/Portability.h" 13 | #include "accelerator/ScopeGuard.h" 14 | #include "accelerator/Uuid.h" 15 | #include "gen-cpp/Empty.h" 16 | 17 | static const char* VERSION = "1.1.0"; 18 | 19 | DEFINE_string(conf, "server.json", "Server config file"); 20 | 21 | using namespace rdd; 22 | using namespace rdd::empty; 23 | 24 | class EmptyHandler : virtual public EmptyIf { 25 | public: 26 | EmptyHandler() { 27 | ACCLOG(DEBUG) << "EmptyHandler init"; 28 | } 29 | 30 | void run(Result& _return, const Query& query) { 31 | if (!acc::StringPiece(query.traceid).startsWith("rdd")) { 32 | _return.__set_code(ResultCode::E_SOURCE__UNTRUSTED); 33 | ACCLOG(INFO) << "untrusted request: [" << query.traceid << "]"; 34 | } 35 | if (!checkOK(_return)) return; 36 | 37 | _return.__set_traceid(acc::generateUuid(query.traceid, "rdde")); 38 | _return.__set_code(ResultCode::OK); 39 | 40 | ACCTLOG(INFO, query.traceid) << "query: \"" << query.query << "\"" 41 | << " code=" << _return.code; 42 | if (!checkOK(_return)) return; 43 | } 44 | 45 | private: 46 | bool checkOK(const Result& result) { 47 | return result.code < 1000; 48 | } 49 | }; 50 | 51 | int main(int argc, char* argv[]) { 52 | gflags::SetVersionString(VERSION); 53 | gflags::SetUsageMessage("Usage : ./empty"); 54 | gflags::ParseCommandLineFlags(&argc, &argv, true); 55 | 56 | setupIgnoreSignal(SIGPIPE); 57 | setupShutdownSignal(SIGINT); 58 | setupShutdownSignal(SIGTERM); 59 | 60 | acc::Singleton::get()->addService( 61 | acc::make_unique>("Empty")); 62 | 63 | config(FLAGS_conf.c_str(), { 64 | {configLogging, "logging"}, 65 | {configService, "service"}, 66 | {configThreadPool, "thread"}, 67 | {configNet, "net"}, 68 | {configMonitor, "monitor"}, 69 | {configJobGraph, "job.graph"} 70 | }); 71 | 72 | ACCLOG(INFO) << "rdd start ... ^_^"; 73 | acc::Singleton::get()->startService(); 74 | 75 | gflags::ShutDownCommandLineFlags(); 76 | 77 | return 0; 78 | } 79 | -------------------------------------------------------------------------------- /raster/protocol/thrift/SyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | 21 | #include "raster/3rd/thrift/transport/TSocket.h" 22 | #include "raster/3rd/thrift/transport/TTransport.h" 23 | #include "raster/3rd/thrift/transport/TBufferTransports.h" 24 | #include "raster/3rd/thrift/protocol/TBinaryProtocol.h" 25 | #include "raster/net/NetUtil.h" 26 | 27 | namespace rdd { 28 | 29 | template < 30 | class C, 31 | class TTransport = apache::thrift::transport::TFramedTransport, 32 | class TProtocol = apache::thrift::protocol::TBinaryProtocol> 33 | class TSyncClient { 34 | public: 35 | TSyncClient(const ClientOption& option); 36 | 37 | TSyncClient(const Peer& peer, 38 | const TimeoutOption& timeout); 39 | 40 | TSyncClient(const Peer& peer, 41 | uint64_t ctimeout = 100000, 42 | uint64_t rtimeout = 1000000, 43 | uint64_t wtimeout = 300000); 44 | 45 | virtual ~TSyncClient(); 46 | 47 | void close(); 48 | 49 | bool connect(); 50 | 51 | bool connected() const; 52 | 53 | template 54 | bool fetch(void (C::*func)(Res&, const Req&...), 55 | Res& response, const Req&... requests); 56 | 57 | template 58 | bool fetch(Res (C::*func)(const Req&...), 59 | Res& response, const Req&... requests); 60 | 61 | private: 62 | void init(); 63 | 64 | Peer peer_; 65 | TimeoutOption timeout_; 66 | std::unique_ptr client_; 67 | boost::shared_ptr socket_; 68 | boost::shared_ptr transport_; 69 | boost::shared_ptr protocol_; 70 | }; 71 | 72 | } // namespace rdd 73 | 74 | #include "raster/protocol/thrift/SyncClient-inl.h" 75 | -------------------------------------------------------------------------------- /raster/protocol/proto/AsyncServer.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "accelerator/thread/LockedMap.h" 24 | #include "raster/net/Service.h" 25 | #include "raster/protocol/proto/RpcController.h" 26 | 27 | namespace rdd { 28 | 29 | class PBAsyncServer : public Service { 30 | public: 31 | PBAsyncServer(acc::StringPiece name) : Service(name) {} 32 | ~PBAsyncServer() override {} 33 | 34 | void makeChannel(int port, const TimeoutOption& timeout) override; 35 | 36 | void addService( 37 | const std::shared_ptr& service); 38 | 39 | google::protobuf::Service* getService( 40 | const google::protobuf::MethodDescriptor* method); 41 | 42 | struct Handle { 43 | Handle(const std::string& callId_, 44 | const std::shared_ptr& controller_, 45 | const std::shared_ptr& request_, 46 | const std::shared_ptr& response_) 47 | : callId(callId_), 48 | controller(controller_), 49 | request(request_), 50 | response(response_) {} 51 | 52 | std::string callId; 53 | std::shared_ptr controller; 54 | std::shared_ptr request; 55 | std::shared_ptr response; 56 | }; 57 | 58 | void addHandle(std::shared_ptr& handle); 59 | 60 | std::shared_ptr removeHandle(const std::string& callId); 61 | 62 | private: 63 | std::map< 64 | const google::protobuf::ServiceDescriptor*, 65 | std::shared_ptr> services_; 66 | acc::LockedMap> handles_; 67 | }; 68 | 69 | } // namespace rdd 70 | -------------------------------------------------------------------------------- /raster/coroutine/Fiber.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "accelerator/Time.h" 25 | #include "raster/coroutine/BoostContext.h" 26 | 27 | namespace rdd { 28 | 29 | #define RDD_FIBER_GEN(x) \ 30 | x(Init), \ 31 | x(Runable), \ 32 | x(Running), \ 33 | x(Block), \ 34 | x(Exit) 35 | 36 | #define RDD_FIBER_ENUM(status) k##status 37 | 38 | class Fiber { 39 | public: 40 | struct Task { 41 | virtual ~Task() {} 42 | virtual void handle() = 0; 43 | void run(); 44 | 45 | Fiber* fiber; 46 | std::list blockCallbacks; 47 | acc::VoidFunc scheduleCallback; 48 | }; 49 | 50 | public: 51 | enum Status { 52 | RDD_FIBER_GEN(RDD_FIBER_ENUM) 53 | }; 54 | 55 | static size_t count() { return count_; } 56 | 57 | Fiber(int stackSize, std::unique_ptr task); 58 | 59 | ~Fiber(); 60 | 61 | Task* task() const { return task_.get(); } 62 | 63 | int status() const { return status_; } 64 | void setStatus(int status); 65 | 66 | const char* statusName() const; 67 | 68 | void execute(); 69 | void yield(int status); 70 | 71 | uint64_t starttime() const; 72 | uint64_t cost() const; 73 | 74 | std::string timestampStr() const; 75 | 76 | Fiber(const Fiber&) = delete; 77 | Fiber& operator=(const Fiber&) = delete; 78 | 79 | private: 80 | static std::atomic count_; 81 | 82 | std::unique_ptr task_; 83 | 84 | unsigned char* stackLimit_; 85 | size_t stackSize_; 86 | Context context_; 87 | 88 | int status_{kInit}; 89 | std::vector timestamps_; 90 | }; 91 | 92 | std::ostream& operator<<(std::ostream& os, const Fiber& fiber); 93 | 94 | #undef RDD_FIBER_ENUM 95 | 96 | } // namespace rdd 97 | -------------------------------------------------------------------------------- /raster/framework/Degrader.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/framework/Degrader.h" 18 | 19 | #include "accelerator/Logging.h" 20 | #include "accelerator/MapUtil.h" 21 | #include "accelerator/Time.h" 22 | 23 | namespace rdd { 24 | 25 | using acc::SpinLockGuard; 26 | 27 | void CountDegrader::setup(bool open, uint32_t limit, uint32_t gap) { 28 | ACCLOG(INFO) << "Degrader: setup " 29 | << "open=" << open << ", limit=" << limit << ", gap=" << gap; 30 | SpinLockGuard guard(lock_); 31 | open_ = open; 32 | limit_ = limit; 33 | gap_ = gap; 34 | } 35 | 36 | bool CountDegrader::needDemote() { 37 | if (open_) { 38 | time_t ts = time(nullptr); 39 | SpinLockGuard guard(lock_); 40 | ts /= gap_; 41 | if (ts != ts_) { 42 | ts_ = ts; 43 | count_ = 0; 44 | } 45 | return ++count_ > limit_; 46 | } 47 | return false; 48 | } 49 | 50 | void RateDegrader::setup(bool open, uint32_t limit, double rate) { 51 | ACCLOG(INFO) << "Degrader: setup " 52 | << "open=" << open << ", limit=" << limit << ", rate=" << rate; 53 | SpinLockGuard guard(lock_); 54 | open_ = open; 55 | rate_ = rate; 56 | limit_ = limit; 57 | ticket_ = limit; 58 | } 59 | 60 | bool RateDegrader::needDemote() { 61 | if (open_) { 62 | uint64_t ts = acc::timestampNow(); 63 | SpinLockGuard guard(lock_); 64 | if (ts_ != 0) { 65 | uint32_t incr = std::max(ts - ts_, 0ul) * rate_; 66 | ticket_ = std::min(ticket_ + incr, limit_); 67 | } 68 | ts_ = ts; 69 | if (ticket_ == 0) { 70 | return true; 71 | } 72 | ticket_--; 73 | } 74 | return false; 75 | } 76 | 77 | bool DegraderManager::hit(const std::string& name) { 78 | std::lock_guard guard(lock_); 79 | auto degrader = acc::get_ptr(degraders_, name); 80 | return degrader ? degrader->get()->needDemote() : false; 81 | } 82 | 83 | } // namespace rdd 84 | -------------------------------------------------------------------------------- /raster/protocol/http/Processor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/http/Processor.h" 18 | 19 | #include "raster/protocol/http/HTTPMethod.h" 20 | 21 | namespace rdd { 22 | 23 | void HTTPProcessor::run() { 24 | auto transport = event_->transport(); 25 | transport->headers->dumpMessage(acc::logging::LOG_V1); 26 | 27 | handler_->headers = transport->headers.get(); 28 | handler_->body = transport->body.get(); 29 | handler_->trailers = transport->trailers.get(); 30 | handler_->response.setupTransport(transport); 31 | 32 | try { 33 | handler_->prepare(); 34 | switch (transport->headers->getMethod()) { 35 | case HTTPMethod::GET: 36 | handler_->onGet(); 37 | break; 38 | case HTTPMethod::POST: 39 | handler_->onPost(); 40 | break; 41 | case HTTPMethod::PUT: 42 | handler_->onPut(); 43 | break; 44 | case HTTPMethod::HEAD: 45 | handler_->onHead(); 46 | break; 47 | case HTTPMethod::DELETE: 48 | handler_->onDelete(); 49 | break; 50 | case HTTPMethod::CONNECT: 51 | handler_->onConnect(); 52 | break; 53 | case HTTPMethod::OPTIONS: 54 | handler_->onOptions(); 55 | break; 56 | case HTTPMethod::TRACE: 57 | handler_->onTrace(); 58 | break; 59 | } 60 | handler_->finish(); 61 | } catch (HTTPException& e) { 62 | handler_->handleException(e); 63 | } catch (std::exception& e) { 64 | handler_->handleException(e); 65 | } catch (...) { 66 | handler_->handleException(); 67 | } 68 | } 69 | 70 | std::unique_ptr HTTPProcessorFactory::create(Event* event) { 71 | auto transport = event->transport(); 72 | auto url = transport->headers->getURL(); 73 | return acc::make_unique(event, router_(url)); 74 | } 75 | 76 | } // namespace rdd 77 | -------------------------------------------------------------------------------- /raster/framework/HubAdaptor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/framework/HubAdaptor.h" 18 | 19 | #include "accelerator/stats/Monitor.h" 20 | 21 | namespace rdd { 22 | 23 | HubAdaptor::HubAdaptor() 24 | : acceptor_(std::shared_ptr(this)) { 25 | } 26 | 27 | void HubAdaptor::configThreads(const std::string& name, size_t threadCount) { 28 | if (name == "io") { 29 | auto factory = std::make_shared("IOThreadPool_"); 30 | ioPool_.reset(new acc::IOThreadPoolExecutor(threadCount, factory)); 31 | } else { 32 | auto factory = std::make_shared("CPUThreadPool" + name + "_"); 33 | cpuPoolMap_.emplace( 34 | acc::to(name), 35 | acc::make_unique(threadCount, factory)); 36 | } 37 | } 38 | 39 | void HubAdaptor::addService(std::unique_ptr service) { 40 | acceptor_.addService(std::move(service)); 41 | } 42 | 43 | void HubAdaptor::configService( 44 | const std::string& name, int port, const TimeoutOption& timeoutOpt) { 45 | acceptor_.configService(name, port, timeoutOpt); 46 | } 47 | 48 | void HubAdaptor::startService() { 49 | acceptor_.start(); 50 | } 51 | 52 | acc::CPUThreadPoolExecutor* 53 | HubAdaptor::getCPUThreadPoolExecutor(int poolId) { 54 | auto it = cpuPoolMap_.find(poolId); 55 | if (it != cpuPoolMap_.end()) { 56 | return it->second.get(); 57 | } 58 | ACCLOG(FATAL) << "CPUThreadPool" << poolId << " not found"; 59 | return nullptr; 60 | } 61 | 62 | std::shared_ptr 63 | HubAdaptor::getSharedCPUThreadPoolExecutor(int poolId) { 64 | auto it = cpuPoolMap_.find(poolId); 65 | if (it != cpuPoolMap_.end()) { 66 | return it->second; 67 | } 68 | ACCLOG(FATAL) << "CPUThreadPool" << poolId << " not found"; 69 | return nullptr; 70 | } 71 | 72 | acc::EventLoop* HubAdaptor::getEventLoop() { 73 | ioPool_->getEventLoop(); 74 | } 75 | 76 | } // namespace rdd 77 | -------------------------------------------------------------------------------- /raster/protocol/thrift/Processor.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include 20 | #include 21 | 22 | #include "raster/3rd/thrift/TProcessor.h" 23 | #include "raster/3rd/thrift/protocol/TBinaryProtocol.h" 24 | #include "raster/3rd/thrift/transport/TBufferTransports.h" 25 | #include "raster/3rd/thrift/transport/TTransportException.h" 26 | #include "raster/net/Processor.h" 27 | 28 | namespace rdd { 29 | 30 | class TProcessor : public Processor { 31 | public: 32 | TProcessor( 33 | Event* event, 34 | std::unique_ptr< ::apache::thrift::TProcessor> processor); 35 | 36 | ~TProcessor() override {} 37 | 38 | void run() override; 39 | 40 | protected: 41 | std::unique_ptr< ::apache::thrift::TProcessor> processor_; 42 | boost::shared_ptr< ::apache::thrift::transport::TMemoryBuffer> pibuf_; 43 | boost::shared_ptr< ::apache::thrift::transport::TMemoryBuffer> pobuf_; 44 | boost::shared_ptr< ::apache::thrift::protocol::TBinaryProtocol> piprot_; 45 | boost::shared_ptr< ::apache::thrift::protocol::TBinaryProtocol> poprot_; 46 | }; 47 | 48 | class TZlibProcessor : public TProcessor { 49 | public: 50 | TZlibProcessor( 51 | Event* event, 52 | std::unique_ptr< ::apache::thrift::TProcessor> processor) 53 | : TProcessor(event, std::move(processor)) {} 54 | 55 | ~TZlibProcessor() override {} 56 | 57 | void run() override; 58 | }; 59 | 60 | template 61 | class TProcessorFactory : public ProcessorFactory { 62 | public: 63 | TProcessorFactory() : handler_(new If()) {} 64 | ~TProcessorFactory() override {} 65 | 66 | std::unique_ptr create(Event* event) override { 67 | return acc::make_unique(event, acc::make_unique

(handler_)); 68 | } 69 | 70 | If* handler() { return handler_.get(); } 71 | 72 | private: 73 | boost::shared_ptr handler_; 74 | }; 75 | 76 | } // namespace rdd 77 | -------------------------------------------------------------------------------- /raster/protocol/thrift/AsyncClient.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/3rd/thrift/protocol/TBinaryProtocol.h" 20 | #include "raster/3rd/thrift/transport/TBufferTransports.h" 21 | #include "raster/net/AsyncClient.h" 22 | 23 | namespace rdd { 24 | 25 | template 26 | class TAsyncClient : public AsyncClient { 27 | public: 28 | TAsyncClient(const ClientOption& option); 29 | 30 | TAsyncClient(const Peer& peer, 31 | const TimeoutOption& timeout); 32 | 33 | TAsyncClient(const Peer& peer, 34 | uint64_t ctimeout = 100000, 35 | uint64_t rtimeout = 1000000, 36 | uint64_t wtimeout = 300000); 37 | 38 | ~TAsyncClient() override {} 39 | 40 | template 41 | bool recv(void (C::*recvFunc)(Res&), Res& _return); 42 | 43 | template 44 | bool recv(Res (C::*recvFunc)(void), Res& _return); 45 | 46 | template 47 | bool send(void (C::*sendFunc)(const Req&...), const Req&... requests); 48 | 49 | template 50 | bool fetch(void (C::*recvFunc)(Res&), Res& _return, 51 | void (C::*sendFunc)(const Req&...), const Req&... requests); 52 | 53 | template 54 | bool fetch(Res (C::*recvFunc)(void), Res& _return, 55 | void (C::*sendFunc)(const Req&...), const Req&... requests); 56 | 57 | protected: 58 | std::shared_ptr makeChannel() override; 59 | 60 | boost::shared_ptr< ::apache::thrift::transport::TMemoryBuffer> pibuf_; 61 | boost::shared_ptr< ::apache::thrift::transport::TMemoryBuffer> pobuf_; 62 | boost::shared_ptr< ::apache::thrift::protocol::TBinaryProtocol> piprot_; 63 | boost::shared_ptr< ::apache::thrift::protocol::TBinaryProtocol> poprot_; 64 | 65 | private: 66 | void init(); 67 | 68 | std::unique_ptr client_; 69 | }; 70 | 71 | } // namespace rdd 72 | 73 | #include "raster/protocol/thrift/AsyncClient-inl.h" 74 | -------------------------------------------------------------------------------- /raster/plugins/flume/FlumeClient.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/plugins/flume/FlumeClient.h" 18 | 19 | #include 20 | 21 | #include "accelerator/Time.h" 22 | #include "raster/plugins/flume/gen-cpp/Scribe.h" 23 | 24 | namespace rdd { 25 | 26 | FlumeClient::FlumeClient(const ClientOption& option, 27 | const std::string& category, 28 | const std::string& logDir) 29 | : category_(category), logDir_(logDir) { 30 | client_ = acc::make_unique>(option); 31 | } 32 | 33 | FlumeClient::~FlumeClient() { 34 | send(); 35 | } 36 | 37 | void FlumeClient::add(std::string&& message) { 38 | LogEntry entry; 39 | entry.category = category_; 40 | entry.message = std::move(message); 41 | entry.message += "\n"; 42 | queue_.insertHead(std::move(entry)); 43 | } 44 | 45 | void FlumeClient::send() { 46 | std::vector entries; 47 | queue_.sweep([&](LogEntry&& entry) { 48 | entries.push_back(std::move(entry)); 49 | }); 50 | sendInBatch(entries); 51 | } 52 | 53 | void FlumeClient::sendInBatch(const std::vector& entries) { 54 | if (!client_->connected() && !client_->connect()) { 55 | ACCLOG(ERROR) << "connect flume client failed"; 56 | writeToDisk(entries); 57 | return; 58 | } 59 | 60 | ResultCode r; 61 | if (!client_->fetch(&ScribeClient::Log, r, entries) || 62 | r != ResultCode::OK) { 63 | ACCLOG(ERROR) << "write to flume client failed"; 64 | writeToDisk(entries); 65 | } 66 | } 67 | 68 | void FlumeClient::writeToDisk(const std::vector& entries) { 69 | std::string path = acc::to( 70 | logDir_, "/", category_, ".flume.", acc::timeNowPrintf("%Y%m%d")); 71 | std::ofstream file(path.c_str(), std::ofstream::app); 72 | 73 | if (file.is_open()) { 74 | for (auto& entry : entries) { 75 | file << entry.message; 76 | } 77 | file.close(); 78 | } 79 | } 80 | 81 | } // namespace rdd 82 | -------------------------------------------------------------------------------- /raster/protocol/binary/Transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "accelerator/Memory.h" 20 | #include "accelerator/compression/ZlibStreamCompressor.h" 21 | #include "accelerator/compression/ZlibStreamDecompressor.h" 22 | #include "raster/net/Transport.h" 23 | 24 | namespace rdd { 25 | 26 | class BinaryTransport : public Transport { 27 | public: 28 | BinaryTransport() { reset(); } 29 | ~BinaryTransport() override {} 30 | 31 | void reset() override; 32 | 33 | void processReadData() override; 34 | 35 | size_t onIngress(const acc::IOBuf& buf); 36 | 37 | void sendHeader(uint32_t header); 38 | size_t sendBody(std::unique_ptr body); 39 | 40 | uint32_t header; 41 | std::unique_ptr body; 42 | 43 | private: 44 | uint8_t headerBuf_[4]; 45 | size_t headerSize_; 46 | bool headersComplete_; 47 | }; 48 | 49 | class BinaryTransportFactory : public TransportFactory { 50 | public: 51 | BinaryTransportFactory() {} 52 | ~BinaryTransportFactory() override {} 53 | 54 | std::unique_ptr create() override { 55 | return acc::make_unique(); 56 | } 57 | }; 58 | 59 | class ZlibTransport : public Transport { 60 | public: 61 | ZlibTransport() { reset(); } 62 | ~ZlibTransport() override {} 63 | 64 | void reset() override; 65 | 66 | void processReadData() override; 67 | 68 | size_t onIngress(const acc::IOBuf& buf); 69 | 70 | size_t sendBody(std::unique_ptr body); 71 | 72 | std::unique_ptr body; 73 | 74 | private: 75 | std::unique_ptr compressor_; 76 | std::unique_ptr decompressor_; 77 | }; 78 | 79 | class ZlibTransportFactory : public TransportFactory { 80 | public: 81 | ZlibTransportFactory() {} 82 | ~ZlibTransportFactory() override {} 83 | 84 | std::unique_ptr create() override { 85 | return acc::make_unique(); 86 | } 87 | }; 88 | 89 | } // namespace rdd 90 | -------------------------------------------------------------------------------- /raster/protocol/http/Transport.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #include "raster/net/Transport.h" 20 | #include "raster/protocol/http/HTTP1xCodec.h" 21 | 22 | namespace rdd { 23 | 24 | class HTTPTransport : public Transport, public HTTP1xCodec::Callback { 25 | public: 26 | HTTPTransport(TransportDirection direction) 27 | : codec_(direction) { 28 | codec_.setCallback(this); 29 | } 30 | 31 | ~HTTPTransport() override {} 32 | 33 | void reset() override; 34 | 35 | void processReadData() override; 36 | 37 | void sendHeaders(const HTTPMessage& headers, HTTPHeaderSize* size); 38 | size_t sendBody(std::unique_ptr body, bool includeEOM); 39 | size_t sendChunkHeader(size_t length); 40 | size_t sendChunkTerminator(); 41 | size_t sendTrailers(const HTTPHeaders& trailers); 42 | size_t sendEOM(); 43 | size_t sendAbort(); 44 | 45 | std::unique_ptr headers; 46 | std::unique_ptr body; 47 | std::unique_ptr trailers; 48 | 49 | private: 50 | // HTTP1xCodec::Callback 51 | void onMessageBegin(HTTPMessage* msg) override; 52 | void onHeadersComplete(std::unique_ptr msg) override; 53 | void onBody(std::unique_ptr chain) override; 54 | void onChunkHeader(size_t length) override; 55 | void onChunkComplete() override; 56 | void onTrailersComplete(std::unique_ptr trailers) override; 57 | void onMessageComplete() override; 58 | void onError(const HTTPException& error) override; 59 | 60 | HTTP1xCodec codec_; 61 | }; 62 | 63 | class HTTPTransportFactory : public TransportFactory { 64 | public: 65 | HTTPTransportFactory(TransportDirection direction) 66 | : direction_(direction) {} 67 | 68 | ~HTTPTransportFactory() override {} 69 | 70 | std::unique_ptr create() override { 71 | return acc::make_unique(direction_); 72 | } 73 | 74 | private: 75 | TransportDirection direction_; 76 | }; 77 | 78 | } // namespace rdd 79 | -------------------------------------------------------------------------------- /raster/framework/Degrader.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #pragma one 18 | 19 | #include 20 | #include 21 | #include 22 | #include 23 | 24 | #include "accelerator/Algorithm.h" 25 | #include "accelerator/thread/SpinLock.h" 26 | 27 | namespace rdd { 28 | 29 | class Degrader { 30 | public: 31 | virtual bool needDemote() = 0; 32 | }; 33 | 34 | class CountDegrader : public Degrader { 35 | public: 36 | CountDegrader() {} 37 | 38 | void setup(bool open, uint32_t limit, uint32_t gap); 39 | 40 | bool needDemote() override; 41 | 42 | private: 43 | std::atomic open_{false}; 44 | uint32_t limit_{0}; 45 | uint32_t gap_{0}; 46 | uint32_t count_{0}; 47 | time_t ts_{0}; 48 | acc::SpinLock lock_; 49 | }; 50 | 51 | class RateDegrader : public Degrader { 52 | public: 53 | RateDegrader() {} 54 | 55 | void setup(bool open, uint32_t limit, double rate); 56 | 57 | bool needDemote() override; 58 | 59 | private: 60 | std::atomic open_{false}; 61 | double rate_{0.0}; 62 | uint32_t limit_{0}; 63 | uint32_t ticket_{0}; 64 | time_t ts_{0}; 65 | acc::SpinLock lock_; 66 | }; 67 | 68 | class DegraderManager { 69 | public: 70 | DegraderManager() {} 71 | 72 | template 73 | void setupDegrader(const std::string& name, Args&&... args); 74 | 75 | bool hit(const std::string& name); 76 | 77 | private: 78 | std::map> degraders_; 79 | std::mutex lock_; 80 | }; 81 | 82 | template 83 | void DegraderManager::setupDegrader(const std::string& name, Args&&... args) { 84 | std::lock_guard guard(lock_); 85 | if (!acc::containKey(degraders_, name)) { 86 | degraders_[name].reset(new Deg()); 87 | } 88 | Degrader* d = degraders_[name].get(); 89 | reinterpret_cast(d)->setup(std::forward(args)...); 90 | } 91 | 92 | } // namespace rdd 93 | 94 | #define RDDDEG_HIT(name) \ 95 | ::acc::Singleton< ::rdd::DegraderManager>::get()->hit(name) 96 | 97 | -------------------------------------------------------------------------------- /raster/protocol/binary/SyncClient.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/binary/SyncClient.h" 18 | 19 | #include "accelerator/Logging.h" 20 | 21 | namespace rdd { 22 | 23 | BinarySyncClient::BinarySyncClient(const ClientOption& option) 24 | : peer_(option.peer), timeout_(option.timeout) { 25 | init(); 26 | } 27 | 28 | BinarySyncClient::BinarySyncClient(const Peer& peer, 29 | const TimeoutOption& timeout) 30 | : peer_(peer), timeout_(timeout) { 31 | init(); 32 | } 33 | 34 | BinarySyncClient::BinarySyncClient(const Peer& peer, 35 | uint64_t ctimeout, 36 | uint64_t rtimeout, 37 | uint64_t wtimeout) 38 | : peer_(peer), timeout_({ctimeout, rtimeout, wtimeout}) { 39 | init(); 40 | } 41 | 42 | BinarySyncClient::~BinarySyncClient() { 43 | close(); 44 | } 45 | 46 | void BinarySyncClient::close() { 47 | if (transport_->isOpen()) { 48 | transport_->close(); 49 | } 50 | } 51 | 52 | bool BinarySyncClient::connect() { 53 | try { 54 | transport_->open(); 55 | } 56 | catch (std::exception& e) { 57 | ACCLOG(ERROR) << "BinarySyncClient: connect " << peer_ 58 | << " failed, " << e.what(); 59 | return false; 60 | } 61 | ACCLOG(DEBUG) << "connect peer[" << peer_ << "]"; 62 | return true; 63 | } 64 | 65 | bool BinarySyncClient::connected() const { 66 | return transport_->isOpen(); 67 | } 68 | 69 | bool BinarySyncClient::fetch(acc::ByteRange& response, const acc::ByteRange& request) { 70 | try { 71 | transport_->send(request); 72 | transport_->recv(response); 73 | } 74 | catch (std::exception& e) { 75 | ACCLOG(ERROR) << "BinarySyncClient: fetch " << peer_ 76 | << " failed, " << e.what(); 77 | return false; 78 | } 79 | return true; 80 | } 81 | 82 | void BinarySyncClient::init() { 83 | transport_.reset(new BinarySyncTransport(peer_, timeout_)); 84 | ACCLOG(DEBUG) << "SyncClient: " << peer_; 85 | } 86 | 87 | } // namespace rdd 88 | -------------------------------------------------------------------------------- /raster/net/NetHub.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/net/NetHub.h" 18 | 19 | #include "raster/net/Channel.h" 20 | #include "raster/net/EventTask.h" 21 | 22 | namespace rdd { 23 | 24 | void NetHub::execute(Event* event) { 25 | if (event->task()) { 26 | size_t i = event->group(); 27 | event->setGroup(0); 28 | if (i == 0 || group_.finish(i)) { 29 | FiberHub::execute(event->task()->fiber, event->channel()->id()); 30 | } 31 | } 32 | int poolId = event->channel()->id(); 33 | auto task = acc::make_unique(event); 34 | task->scheduleCallback = [&]() { addEvent(event); }; 35 | FiberHub::execute(std::move(task), poolId); 36 | } 37 | 38 | void NetHub::addEvent(Event* event) { 39 | if (forwarding_ && event->socket()->isClient()) { 40 | for (auto& f : forwards_) { 41 | if (f.port == event->channel()->id() && f.flow > rand() % 100) { 42 | forwardEvent(event, f.fpeer); 43 | } 44 | } 45 | } 46 | getEventLoop()->addEvent(event); 47 | } 48 | 49 | void NetHub::forwardEvent(Event* event, const Peer& peer) { 50 | auto socket = Socket::createAsyncSocket(); 51 | if (!socket || 52 | !socket->connect(peer)) { 53 | return; 54 | } 55 | auto evcopy = new Event(event->channel(), std::move(socket)); 56 | evcopy->transport()->clone(event->transport()); 57 | evcopy->setForward(); 58 | evcopy->setState(Event::kWrited); 59 | getEventLoop()->addEvent(evcopy); 60 | } 61 | 62 | bool NetHub::waitGroup(const std::vector& events) { 63 | for (auto& event : events) { 64 | if (event->group() != 0) { 65 | ACCLOG(WARN) << "create group on grouping Event, giveup"; 66 | return false; 67 | } 68 | } 69 | size_t i = group_.create(events.size()); 70 | for (auto& event : events) { 71 | event->setGroup(i); 72 | } 73 | return true; 74 | } 75 | 76 | void NetHub::setForwarding(bool forward) { 77 | forwarding_ = forward; 78 | } 79 | 80 | void NetHub::addForwardTarget(ForwardTarget&& t) { 81 | forwards_.push_back(std::move(t)); 82 | } 83 | 84 | } // namespace rdd 85 | -------------------------------------------------------------------------------- /raster/coroutine/Fiber.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/coroutine/Fiber.h" 18 | 19 | #include "accelerator/Logging.h" 20 | #include "raster/coroutine/FiberManager.h" 21 | 22 | #define RDD_FIBER_STR(status) #status 23 | 24 | namespace { 25 | static const char* statusStrings[] = { 26 | RDD_FIBER_GEN(RDD_FIBER_STR) 27 | }; 28 | } 29 | 30 | namespace rdd { 31 | 32 | void Fiber::Task::run() { 33 | handle(); 34 | FiberManager::exit(); 35 | } 36 | 37 | std::atomic Fiber::count_(0); 38 | 39 | Fiber::Fiber(int stackSize, std::unique_ptr task) 40 | : task_(std::move(task)), 41 | stackLimit_(new unsigned char[stackSize]), 42 | stackSize_(stackSize), 43 | context_(std::bind(&Task::run, task_.get()), stackLimit_, stackSize_) { 44 | task_->fiber = this; 45 | timestamps_.push_back(std::move(acc::Timestamp(status_))); 46 | ++count_; 47 | } 48 | 49 | Fiber::~Fiber() { 50 | delete [] stackLimit_; 51 | --count_; 52 | } 53 | 54 | void Fiber::setStatus(int status) { 55 | status_ = status; 56 | timestamps_.push_back(std::move(acc::Timestamp(status, cost()))); 57 | } 58 | 59 | const char* Fiber::statusName() const { 60 | return statusStrings[status_]; 61 | } 62 | 63 | void Fiber::execute() { 64 | assert(status_ == kRunable); 65 | setStatus(kRunning); 66 | ACCLOG(V5) << *this << " execute"; 67 | context_.activate(); 68 | } 69 | 70 | void Fiber::yield(int status) { 71 | assert(status_ == kRunning); 72 | setStatus(status); 73 | ACCLOG(V5) << *this << " yield"; 74 | context_.deactivate(); 75 | } 76 | 77 | uint64_t Fiber::starttime() const { 78 | return timestamps_.front().stamp; 79 | } 80 | 81 | uint64_t Fiber::cost() const { 82 | return acc::timePassed(starttime()); 83 | } 84 | 85 | std::string Fiber::timestampStr() const { 86 | return acc::join("-", timestamps_); 87 | } 88 | 89 | std::ostream& operator<<(std::ostream& os, const Fiber& fiber) { 90 | os << "fc(" 91 | << (void*)(&fiber) << ", " 92 | << fiber.statusName() << ", " 93 | << fiber.timestampStr() << ")"; 94 | return os; 95 | } 96 | 97 | } // namespace rdd 98 | -------------------------------------------------------------------------------- /raster/protocol/binary/AsyncClient.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2017 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/binary/AsyncClient.h" 18 | 19 | #include "raster/framework/HubAdaptor.h" 20 | #include "raster/protocol/binary/Transport.h" 21 | 22 | namespace rdd { 23 | 24 | BinaryAsyncClient::BinaryAsyncClient(const ClientOption& option) 25 | : AsyncClient(acc::Singleton::try_get(), option) { 26 | channel_ = makeChannel(); 27 | } 28 | 29 | BinaryAsyncClient::BinaryAsyncClient(const Peer& peer, 30 | const TimeoutOption& timeout) 31 | : AsyncClient(acc::Singleton::try_get(), peer, timeout) { 32 | channel_ = makeChannel(); 33 | } 34 | 35 | BinaryAsyncClient::BinaryAsyncClient(const Peer& peer, 36 | uint64_t ctimeout, 37 | uint64_t rtimeout, 38 | uint64_t wtimeout) 39 | : AsyncClient(acc::Singleton::try_get(), 40 | peer, ctimeout, rtimeout, wtimeout) { 41 | channel_ = makeChannel(); 42 | } 43 | 44 | bool BinaryAsyncClient::recv(acc::ByteRange& response) { 45 | if (!event_ || event_->state() == Event::kFail) { 46 | return false; 47 | } 48 | auto transport = event_->transport(); 49 | response = transport->body->coalesce(); 50 | return true; 51 | } 52 | 53 | bool BinaryAsyncClient::send(const acc::ByteRange& request) { 54 | if (!event_) { 55 | return false; 56 | } 57 | auto transport = event_->transport(); 58 | transport->sendHeader(request.size()); 59 | transport->sendBody(acc::IOBuf::copyBuffer(request)); 60 | return true; 61 | } 62 | 63 | bool BinaryAsyncClient::fetch(acc::ByteRange& response, const acc::ByteRange& request) { 64 | return (send(request) && 65 | FiberManager::yield() && 66 | recv(response)); 67 | } 68 | 69 | std::shared_ptr BinaryAsyncClient::makeChannel() { 70 | return std::make_shared( 71 | peer_, 72 | timeout_, 73 | acc::make_unique()); 74 | } 75 | 76 | } // namespace rdd 77 | -------------------------------------------------------------------------------- /raster/protocol/thrift/Processor.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Yeolar 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #include "raster/protocol/thrift/Processor.h" 18 | 19 | #include "raster/protocol/binary/Transport.h" 20 | 21 | namespace rdd { 22 | 23 | TProcessor::TProcessor( 24 | Event* event, 25 | std::unique_ptr< ::apache::thrift::TProcessor> processor) 26 | : Processor(event), processor_(std::move(processor)) { 27 | pibuf_.reset(new apache::thrift::transport::TMemoryBuffer()); 28 | pobuf_.reset(new apache::thrift::transport::TMemoryBuffer()); 29 | piprot_.reset(new apache::thrift::protocol::TBinaryProtocol(pibuf_)); 30 | poprot_.reset(new apache::thrift::protocol::TBinaryProtocol(pobuf_)); 31 | } 32 | 33 | void TProcessor::run() { 34 | auto transport = event_->transport(); 35 | try { 36 | auto range = transport->body->coalesce(); 37 | pibuf_->resetBuffer((uint8_t*)range.data(), range.size()); 38 | processor_->process(piprot_, poprot_, nullptr); 39 | uint8_t* p; 40 | uint32_t n; 41 | pobuf_->getBuffer(&p, &n); 42 | transport->sendHeader(n); 43 | transport->sendBody(acc::IOBuf::copyBuffer(p, n));; 44 | } catch (apache::thrift::protocol::TProtocolException& e) { 45 | ACCLOG(WARN) << "catch exception: " << e.what(); 46 | } catch (std::exception& e) { 47 | ACCLOG(WARN) << "catch exception: " << e.what(); 48 | } catch (...) { 49 | ACCLOG(WARN) << "catch unknown exception"; 50 | } 51 | } 52 | 53 | void TZlibProcessor::run() { 54 | auto transport = event_->transport(); 55 | try { 56 | auto range = transport->body->coalesce(); 57 | pibuf_->resetBuffer((uint8_t*)range.data(), range.size()); 58 | processor_->process(piprot_, poprot_, nullptr); 59 | uint8_t* p; 60 | uint32_t n; 61 | pobuf_->getBuffer(&p, &n); 62 | transport->sendBody(acc::IOBuf::copyBuffer(p, n));; 63 | } catch (apache::thrift::protocol::TProtocolException& e) { 64 | ACCLOG(WARN) << "catch exception: " << e.what(); 65 | } catch (std::exception& e) { 66 | ACCLOG(WARN) << "catch exception: " << e.what(); 67 | } catch (...) { 68 | ACCLOG(WARN) << "catch unknown exception"; 69 | } 70 | } 71 | 72 | } // namespace rdd 73 | -------------------------------------------------------------------------------- /raster/protocol/http/ParseURL.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2015, Facebook, Inc. 3 | * Copyright 2017 Yeolar 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #pragma once 19 | 20 | #include 21 | 22 | #include "accelerator/Conv.h" 23 | #include "accelerator/Logging.h" 24 | #include "accelerator/Range.h" 25 | 26 | namespace rdd { 27 | 28 | /** 29 | * ParseURL can handle non-fully-formed URLs. 30 | * This class must not persist beyond the lifetime of the buffer 31 | * underlying the input acc::StringPiece 32 | */ 33 | class ParseURL { 34 | public: 35 | ParseURL() {} 36 | explicit ParseURL(acc::StringPiece urlVal) { 37 | init(urlVal); 38 | } 39 | 40 | void init(acc::StringPiece urlVal) { 41 | ACCCHECK(!initialized_); 42 | url_ = urlVal; 43 | parse(); 44 | initialized_ = true; 45 | } 46 | 47 | acc::StringPiece url() const { return url_; } 48 | acc::StringPiece scheme() const { return scheme_; } 49 | std::string authority() const { return authority_; } 50 | acc::StringPiece host() const { return host_; } 51 | uint16_t port() const { return port_; } 52 | acc::StringPiece path() const { return path_; } 53 | acc::StringPiece query() const { return query_; } 54 | acc::StringPiece fragment() const { return fragment_; } 55 | 56 | bool valid() const { return valid_; } 57 | bool hasHost() const { return valid() && !host_.empty(); } 58 | 59 | std::string hostAndPort() const { 60 | std::string rc = host_.str(); 61 | if (port_ != 0) { 62 | acc::toAppend(":", port_, &rc); 63 | } 64 | return rc; 65 | } 66 | 67 | acc::StringPiece hostNoBrackets() { 68 | stripBrackets(); 69 | return hostNoBrackets_; 70 | } 71 | 72 | bool hostIsIPAddress(); 73 | 74 | void stripBrackets(); 75 | 76 | private: 77 | void parse(); 78 | 79 | void parseNonFully(); 80 | 81 | bool parseAuthority(); 82 | 83 | acc::StringPiece url_; 84 | acc::StringPiece scheme_; 85 | std::string authority_; 86 | acc::StringPiece host_; 87 | acc::StringPiece hostNoBrackets_; 88 | acc::StringPiece path_; 89 | acc::StringPiece query_; 90 | acc::StringPiece fragment_; 91 | uint16_t port_{0}; 92 | bool valid_{false}; 93 | bool initialized_{false}; 94 | }; 95 | 96 | } // namespace rdd 97 | --------------------------------------------------------------------------------