├── debian ├── compat ├── docs ├── grape.dirs ├── source │ └── format ├── libcocaine-plugin-queue-driver.install ├── grape-components.install ├── grape-dev.install ├── grape.install ├── rules ├── copyright └── control ├── src ├── queue │ ├── queue.conf │ ├── queue.profile │ ├── CMakeLists.txt │ ├── test.cpp │ ├── queue.hpp │ ├── chunk.hpp │ └── app.cpp ├── testerhead-cpp │ ├── testerhead-cpp.profile │ ├── CMakeLists.txt │ ├── testerhead-cpp.conf │ └── app.cpp ├── data_array │ ├── CMakeLists.txt │ └── data_array.cpp ├── driver │ ├── CMakeLists.txt │ ├── module.cpp │ ├── driver.hpp │ └── cocaine-json-trait.hpp ├── queue-pump │ ├── CMakeLists.txt │ ├── queue-push.cpp │ ├── queue-pull.cpp │ └── queue-pump.cpp └── srw.py ├── example ├── profile-single.json ├── manifest.json ├── manifest-single.json ├── CMakeLists.txt ├── test.cpp ├── test_single.cpp └── test_start.cpp ├── launchpad ├── CMakeLists.txt ├── json-get-value.py ├── _queue_clear.py ├── testerhead-cpp ├── lookup-host.py ├── queue ├── nodes.py └── _queue_stats.py ├── test ├── sample_test.test.cpp ├── test_pump.test.cpp ├── CMakeLists.txt ├── data_array.test.cpp ├── chunk.test.cpp └── chunk_iterator.test.cpp ├── .gitignore ├── include └── grape │ ├── entry_id.hpp │ ├── rapidjson │ ├── internal │ │ ├── strfunc.h │ │ ├── stack.h │ │ └── pow10.h │ ├── filestream.h │ ├── stringbuffer.h │ ├── filewritestream.h │ ├── filereadstream.h │ ├── prettywriter.h │ ├── writer.h │ ├── rapidjson.h │ ├── allocators.h │ └── encodedstream.h │ ├── data_array.hpp │ ├── logger_adapter.hpp │ ├── elliptics_client_state.hpp │ └── concurrent-pump.hpp ├── cmake └── Modules │ ├── FindCocaine.cmake │ ├── FindCocaineNative.cmake │ ├── locate_library.cmake │ └── Findelliptics.cmake ├── CMakeLists.txt ├── grape-bf.spec └── README.md /debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /debian/docs: -------------------------------------------------------------------------------- 1 | #example/*.json 2 | -------------------------------------------------------------------------------- /debian/grape.dirs: -------------------------------------------------------------------------------- 1 | usr/lib/grape 2 | -------------------------------------------------------------------------------- /debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (native) 2 | -------------------------------------------------------------------------------- /debian/libcocaine-plugin-queue-driver.install: -------------------------------------------------------------------------------- 1 | usr/lib/cocaine/* 2 | -------------------------------------------------------------------------------- /debian/grape-components.install: -------------------------------------------------------------------------------- 1 | usr/lib/grape/{queue,testerhead-cpp} 2 | -------------------------------------------------------------------------------- /debian/grape-dev.install: -------------------------------------------------------------------------------- 1 | usr/include/grape/ 2 | usr/lib/libgrape_data_array.so 3 | -------------------------------------------------------------------------------- /debian/grape.install: -------------------------------------------------------------------------------- 1 | usr/lib/libgrape_data_array.so.* 2 | usr/lib/grape/launchpad/* 3 | -------------------------------------------------------------------------------- /src/queue/queue.conf: -------------------------------------------------------------------------------- 1 | { 2 | "type": "binary", 3 | "slave": "queue", 4 | 5 | "remotes": [ 6 | "localhost:1025:2" 7 | ], 8 | "groups": [2] 9 | } 10 | -------------------------------------------------------------------------------- /example/profile-single.json: -------------------------------------------------------------------------------- 1 | { 2 | "heartbeat-timeout" : 60, 3 | "pool-limit" : 5, 4 | "queue-limit" : 0, 5 | "grow-threshold" : 1, 6 | "concurrency" : 10 7 | } 8 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | include /usr/share/cdbs/1/class/cmake.mk 4 | include /usr/share/cdbs/1/rules/debhelper.mk 5 | 6 | DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo 7 | -------------------------------------------------------------------------------- /src/testerhead-cpp/testerhead-cpp.profile: -------------------------------------------------------------------------------- 1 | { 2 | "heartbeat-timeout" : 60, 3 | "pool-limit" : 5, 4 | "queue-limit" : 2000, 5 | "grow-threshold" : 1, 6 | "concurrency" : 10 7 | } 8 | -------------------------------------------------------------------------------- /src/queue/queue.profile: -------------------------------------------------------------------------------- 1 | { 2 | "heartbeat-timeout" : 60, 3 | "pool-limit" : 5, 4 | "queue-limit" : 10240000, 5 | "idle-timeout" : 0, 6 | "grow-threshold" : 1, 7 | "concurrency" : 10 8 | } 9 | -------------------------------------------------------------------------------- /launchpad/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | install(DIRECTORY . DESTINATION lib${LIB_SUFFIX}/grape/launchpad 2 | COMPONENT runtime 3 | USE_SOURCE_PERMISSIONS 4 | PATTERN "*.pyc" EXCLUDE 5 | PATTERN "CMakeLists.txt" EXCLUDE 6 | PATTERN "*.py" 7 | ) 8 | -------------------------------------------------------------------------------- /test/sample_test.test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | TEST(MathTest, DISABLED_TwoPlusTwoEqualsFour) { 5 | EXPECT_EQ(2 + 2, 4); 6 | } 7 | 8 | TEST(MathTest, DISABLED_TwoByTwoEqualsFour) { 9 | EXPECT_EQ(2 * 2, 5) << "WAT???" << std::endl; 10 | } 11 | -------------------------------------------------------------------------------- /launchpad/json-get-value.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import json 4 | 5 | def get_value(node, path): 6 | for i in path.split('.'): 7 | node = node[i] 8 | return node 9 | 10 | 11 | if __name__ == '__main__': 12 | import sys 13 | path = sys.argv[1] 14 | doc = json.load(sys.stdin) 15 | 16 | print get_value(doc, path) 17 | 18 | -------------------------------------------------------------------------------- /src/data_array/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(grape_data_array SHARED data_array.cpp) 2 | target_link_libraries(grape_data_array ${elliptics_LIBRARIES} ${MSGPACK_LIBRARIES}) 3 | 4 | set_target_properties(grape_data_array PROPERTIES 5 | VERSION ${grape_VERSION} 6 | SOVERSION ${grape_VERSION_MAJOR} 7 | ) 8 | 9 | install(TARGETS grape_data_array 10 | LIBRARY DESTINATION lib${LIB_SUFFIX} 11 | ) 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile 2 | CMakeCache.txt 3 | CMakeFiles/ 4 | cmake_install.cmake 5 | install_manifest.txt 6 | *.o 7 | *.a 8 | *~ 9 | *.exp 10 | *.so 11 | *.so.* 12 | *.a.* 13 | *.o.* 14 | *.in 15 | *.lo 16 | *.la 17 | root/ 18 | tags 19 | *.tar 20 | *.tar.* 21 | *.cocaine-plugin 22 | .*.sw* 23 | src/queue/block 24 | src/queue/queue 25 | src/queue/queue-tool 26 | src/testerhead-cpp/testerhead-cpp 27 | src/forward/forward 28 | -------------------------------------------------------------------------------- /src/testerhead-cpp/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_executable(testerhead-cpp app.cpp) 2 | set_target_properties(testerhead-cpp PROPERTIES 3 | COMPILE_FLAGS "-std=c++0x" 4 | ) 5 | target_link_libraries(testerhead-cpp 6 | ${Boost_PROGRAM_OPTIONS_LIBRARY} 7 | ${GRAPE_COMMON_LIBRARIES} 8 | grape_data_array 9 | ) 10 | 11 | install(TARGETS testerhead-cpp 12 | RUNTIME DESTINATION lib${LIB_SUFFIX}/grape 13 | COMPONENT runtime 14 | ) 15 | 16 | -------------------------------------------------------------------------------- /src/queue/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(queue STATIC queue.cpp chunk.cpp) 2 | target_link_libraries(queue ${GRAPE_COMMON_LIBRARIES} ${elliptics_LIBRARIES} grape_data_array) 3 | 4 | add_executable(queue-app app.cpp) 5 | set_target_properties(queue-app PROPERTIES 6 | OUTPUT_NAME "queue" 7 | ) 8 | target_link_libraries(queue-app queue ${GRAPE_COMMON_LIBRARIES} rt) 9 | 10 | install(TARGETS queue-app 11 | RUNTIME DESTINATION lib${LIB_SUFFIX}/grape 12 | COMPONENT runtime 13 | ) 14 | -------------------------------------------------------------------------------- /src/driver/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_library(queue-driver MODULE driver.cpp module.cpp) 2 | set_target_properties(queue-driver PROPERTIES 3 | PREFIX "" 4 | SUFFIX ".cocaine-plugin" 5 | COMPILE_FLAGS "-std=c++0x -Wall -Wextra" 6 | ) 7 | 8 | target_link_libraries(queue-driver 9 | rt 10 | ${elliptics_cpp_LIBRARIES} 11 | ${Cocaine_LIBRARIES} 12 | grape_data_array 13 | ) 14 | 15 | install(TARGETS queue-driver 16 | LIBRARY DESTINATION lib${LIB_SUFFIX}/cocaine 17 | COMPONENT runtime 18 | ) 19 | -------------------------------------------------------------------------------- /example/manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "binary", 3 | "engine" : { 4 | "heartbeat-timeout" : 60, 5 | "pool-limit" : 10, 6 | "queue-limit" : 2000, 7 | "grow-threshold" : 1, 8 | "slave" : "/tmp/build/bin/cocaine-slave" 9 | }, 10 | "args" : { 11 | "name" : "libgrape_etest.so", 12 | "config" : { 13 | "nodes" : { 14 | "localhost": 1025 15 | }, 16 | "groups" : [1, 2], 17 | "log-level" : 3, 18 | "log" : "/dev/stdout" 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /launchpad/_queue_clear.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | from nodes import connect 5 | from _queue_stats import exec_on_all_workers 6 | 7 | if __name__ == '__main__': 8 | import argparse 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument('-r', '--remote', help='address of seed node') 11 | parser.add_argument('-g', '--group', help='elliptics group') 12 | args = parser.parse_args() 13 | 14 | s = connect([args.remote], [int(args.group)]) 15 | exec_on_all_workers(s, 'queue@clear') 16 | -------------------------------------------------------------------------------- /example/manifest-single.json: -------------------------------------------------------------------------------- 1 | { 2 | "type" : "binary", 3 | "slave" : "/opt/elliptics/build/bin/cocaine-worker-generic", 4 | "args" : { 5 | "name" : "libgrape_etest_single.so", 6 | "config" : { 7 | "nodes" : { 8 | "localhost": 1025 9 | }, 10 | "groups" : [2], 11 | "log-level" : 1, 12 | "log" : "/tmp/grape-etest.log", 13 | "write-type" : "random1", 14 | "ioflags" : 3072, 15 | "cflags" : 0, 16 | "event-base" : "test-single@event", 17 | "event-num" : 5 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /include/grape/entry_id.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __ENTRY_ID_HPP 2 | #define __ENTRY_ID_HPP 3 | 4 | #include 5 | #include 6 | 7 | namespace ioremap { namespace grape { 8 | 9 | struct entry_id { 10 | int32_t chunk; 11 | int32_t pos; 12 | 13 | static entry_id from_dnet_raw_id(const dnet_raw_id *id) { 14 | entry_id result; 15 | memcpy(&result, id->id + DNET_ID_SIZE - sizeof(entry_id), sizeof(entry_id)); 16 | return result; 17 | } 18 | 19 | MSGPACK_DEFINE(chunk, pos); 20 | }; 21 | 22 | }} 23 | 24 | #endif // __ENTRY_ID_HPP 25 | -------------------------------------------------------------------------------- /example/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | project(grape_etest) 2 | add_library(grape_etest SHARED test.cpp) 3 | set_target_properties(grape_etest PROPERTIES COMPILE_FLAGS "-std=c++0x") 4 | target_link_libraries(grape_etest grape ${elliptics_LIBRARIES}) 5 | 6 | project(grape_etest_single) 7 | add_library(grape_etest_single SHARED test_single.cpp) 8 | target_link_libraries(grape_etest_single grape ${elliptics_LIBRARIES}) 9 | 10 | project(grape_test_start) 11 | add_executable(grape_test_start test_start.cpp) 12 | target_link_libraries(grape_test_start grape ${elliptics_LIBRARIES} ${Boost_PROGRAM_OPTIONS_LIBRARY}) 13 | -------------------------------------------------------------------------------- /src/queue-pump/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | #add_executable(queue-pump queue-pump.cpp) 2 | #target_link_libraries(queue-pump grape_data_array boost_program_options ${GRAPE_COMMON_LIBRARIES}) 3 | 4 | add_executable(queue-push queue-push.cpp) 5 | target_link_libraries(queue-push boost_program_options ${GRAPE_COMMON_LIBRARIES}) 6 | 7 | add_executable(queue-pull queue-pull.cpp) 8 | target_link_libraries(queue-pull grape_data_array boost_program_options ${GRAPE_COMMON_LIBRARIES}) 9 | 10 | install(TARGETS queue-push queue-pull 11 | RUNTIME DESTINATION lib${LIB_SUFFIX}/grape/launchpad 12 | COMPONENT runtime 13 | ) 14 | -------------------------------------------------------------------------------- /src/srw.py: -------------------------------------------------------------------------------- 1 | from cocaine.services import Service 2 | import struct 3 | 4 | event = 'queue@pop' 5 | data = '' 6 | 7 | data_size = len(data) 8 | event_size = len(event) 9 | 10 | flags = 1 11 | status = 0 12 | __key = 0 13 | src_key = 3 14 | 15 | packed = struct.pack('<64sQQiiii32s%ds%ds' % (event_size, data_size), 16 | "this is id", 17 | data_size, 18 | flags, 19 | event_size, 20 | status, 21 | __key, 22 | src_key, 23 | "this is address", 24 | event, 25 | data) 26 | 27 | response = Service('queue', hostname = 'localhost', port = 10053).enqueue('pop', packed, 'localhost-queue-1') 28 | print response.get() 29 | -------------------------------------------------------------------------------- /include/grape/rapidjson/internal/strfunc.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ 2 | #define RAPIDJSON_INTERNAL_STRFUNC_H_ 3 | 4 | namespace rapidjson { 5 | namespace internal { 6 | 7 | //! Custom strlen() which works on different character types. 8 | /*! \tparam Ch Character type (e.g. char, wchar_t, short) 9 | \param s Null-terminated input string. 10 | \return Number of characters in the string. 11 | \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. 12 | */ 13 | template 14 | inline SizeType StrLen(const Ch* s) { 15 | const Ch* p = s; 16 | while (*p != '\0') 17 | ++p; 18 | return SizeType(p - s); 19 | } 20 | 21 | } // namespace internal 22 | } // namespace rapidjson 23 | 24 | #endif // RAPIDJSON_INTERNAL_STRFUNC_H_ 25 | -------------------------------------------------------------------------------- /cmake/Modules/FindCocaine.cmake: -------------------------------------------------------------------------------- 1 | # Find Cocaine Native Framework - base for building c++ applications running under the Cocaine. 2 | # https://github.com/cocaine/cocaine-framework-native 3 | # 4 | # This module defines 5 | # Cocaine_FOUND - whether the cocaine-framework-native was found 6 | # Cocaine_LIBRARIES - it's libraries 7 | # Cocaine_INCLUDE_DIRS - it's include paths 8 | # Cocaine_CFLAGS - flags to compile with 9 | 10 | find_path(Cocaine_INCLUDE_DIR cocaine/context.hpp) 11 | 12 | find_library(Cocaine_LIBRARY cocaine-core) 13 | 14 | set(Cocaine_INCLUDE_DIRS "${Cocaine_INCLUDE_DIR}") 15 | set(Cocaine_LIBRARIES "${Cocaine_LIBRARY}") 16 | 17 | include(FindPackageHandleStandardArgs) 18 | find_package_handle_standard_args(Cocaine DEFAULT_MSG Cocaine_LIBRARY Cocaine_INCLUDE_DIR) 19 | 20 | mark_as_advanced(Cocaine_LIBRARY Cocaine_INCLUDE_DIR) 21 | -------------------------------------------------------------------------------- /cmake/Modules/FindCocaineNative.cmake: -------------------------------------------------------------------------------- 1 | # Find Cocaine Native Framework - base for building c++ applications running under the Cocaine. 2 | # https://github.com/cocaine/cocaine-framework-native 3 | # 4 | # This module defines 5 | # CocaineNative_FOUND - whether the cocaine-framework-native was found 6 | # CocaineNative_LIBRARIES - it's libraries 7 | # CocaineNative_INCLUDE_DIRS - it's include paths 8 | # CocaineNative_CFLAGS - flags to compile with 9 | 10 | find_path(CocaineNative_INCLUDE_DIR cocaine/framework/dispatch.hpp) 11 | 12 | find_library(CocaineNative_LIBRARY cocaine-framework) 13 | 14 | set(CocaineNative_INCLUDE_DIRS "${CocaineNative_INCLUDE_DIR}") 15 | set(CocaineNative_LIBRARIES "${CocaineNative_LIBRARY}") 16 | 17 | include(FindPackageHandleStandardArgs) 18 | find_package_handle_standard_args(CocaineNative DEFAULT_MSG CocaineNative_LIBRARY CocaineNative_INCLUDE_DIR) 19 | 20 | mark_as_advanced(CocaineNative_LIBRARY CocaineNative_INCLUDE_DIR) 21 | -------------------------------------------------------------------------------- /cmake/Modules/locate_library.cmake: -------------------------------------------------------------------------------- 1 | FUNCTION(LOCATE_LIBRARY VARIABLE HEADER LIBRARY) 2 | IF(${VARIABLE}_INCLUDE_DIRS AND ${VARIABLE}_LIBRARY_DIRS) 3 | RETURN() 4 | ENDIF() 5 | 6 | FIND_PATH(${VARIABLE}_INCLUDE_DIRS NAMES ${HEADER} PATH_SUFFIXES ${ARGN}) 7 | FIND_LIBRARY(${VARIABLE}_LIBRARIES NAMES ${LIBRARY} PATH_SUFFIXES ${ARGN}) 8 | message("include: " ${${VARIABLE}_INCLUDE_DIRS}) 9 | message("library: " ${${VARIABLE}_LIBRARIES}) 10 | GET_FILENAME_COMPONENT(${VARIABLE}_LIBRARY_DIRS ${${VARIABLE}_LIBRARIES} PATH CACHE) 11 | message("library dirs: " ${${VARIABLE}_LIBRARY_DIRS}) 12 | 13 | STRING(TOLOWER ${VARIABLE} LIBRARY_NAME) 14 | 15 | IF(NOT ${VARIABLE}_INCLUDE_DIRS OR NOT ${VARIABLE}_LIBRARY_DIRS) 16 | MESSAGE(FATAL_ERROR "${LIBRARY_NAME} development files are required to build.") 17 | ELSE() 18 | MESSAGE(STATUS "Found ${LIBRARY_NAME}: ${${VARIABLE}_LIBRARIES}") 19 | ENDIF() 20 | ENDFUNCTION() 21 | 22 | -------------------------------------------------------------------------------- /src/driver/module.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013+ Ruslan Nigmatullin 3 | 4 | This file is part of Grape. 5 | 6 | Cocaine is free software; you can redistribute it and/or modify 7 | it under the terms of the GNU Lesser General Public License as published by 8 | the Free Software Foundation; either version 3 of the License, or 9 | (at your option) any later version. 10 | 11 | Cocaine is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU Lesser General Public License for more details. 15 | 16 | You should have received a copy of the GNU Lesser General Public License 17 | along with this program. If not, see . 18 | */ 19 | 20 | #include "driver.hpp" 21 | 22 | extern "C" { 23 | void 24 | initialize(cocaine::api::repository_t& repository) { 25 | repository.insert("persistent-queue"); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/testerhead-cpp/testerhead-cpp.conf: -------------------------------------------------------------------------------- 1 | { 2 | "type": "binary", 3 | "slave": "testerhead-cpp", 4 | 5 | "remotes": [ 6 | "localhost:1025:2" 7 | ], 8 | 9 | "remotes_": [ 10 | "s20h.xxx.yandex.net:1025:2", 11 | "s21h.xxx.yandex.net:1025:2", 12 | "s22h.xxx.yandex.net:1025:2", 13 | "s23h.xxx.yandex.net:1025:2", 14 | "s24h.xxx.yandex.net:1025:2" 15 | ], 16 | 17 | "groups": [2], 18 | "logfile" : "/tmp/testerhead-cpp.log", 19 | "loglevel" : 0, 20 | 21 | "delay": 10, 22 | 23 | "drivers" : { 24 | "queue": { 25 | "type": "persistent-queue", 26 | "args": { 27 | "source-queue-app": "queue", 28 | "source-queue-id": "test-queue-id", 29 | "emit": "testerhead-cpp@test-event", 30 | 31 | "remotes": [ 32 | "localhost:1025:2" 33 | ], 34 | 35 | "remotes_": [ 36 | "s20h.xxx.yandex.net:1025:2", 37 | "s21h.xxx.yandex.net:1025:2", 38 | "s22h.xxx.yandex.net:1025:2", 39 | "s23h.xxx.yandex.net:1025:2", 40 | "s24h.xxx.yandex.net:1025:2" 41 | ], 42 | "groups": [2] 43 | } 44 | } 45 | } 46 | } 47 | 48 | -------------------------------------------------------------------------------- /src/queue/test.cpp: -------------------------------------------------------------------------------- 1 | #include "queue.hpp" 2 | 3 | #include 4 | 5 | int main(int argc, char *argv[]) 6 | { 7 | ioremap::grape::queue q("test-queue-id"); 8 | q.initialize("queue.conf"); 9 | 10 | for (int i = 0; i < 10; ++i) { 11 | std::string data = "this is a test: " + lexical_cast(i); 12 | std::cout << "<< " << data << std::endl; 13 | 14 | q.push(data); 15 | } 16 | 17 | for (int i = 0; i < 10; ++i) { 18 | ioremap::grape::data_array d = q.pop(3); 19 | 20 | if (d.empty()) 21 | break; 22 | 23 | size_t pos = 0; 24 | for (auto sz : d.sizes()) { 25 | std::cout << ">> " << d.data().substr(pos, sz) << std::endl; 26 | pos += sz; 27 | } 28 | 29 | ioremap::grape::data_array copy = ioremap::grape::data_array::deserialize(d.serialize()); 30 | pos = 0; 31 | for (auto sz : copy.sizes()) { 32 | std::cout << "copy >> " << d.data().substr(pos, sz) << std::endl; 33 | pos += sz; 34 | } 35 | } 36 | 37 | std::string end = "at the end (test push/pop in the same chunk)"; 38 | q.push(end); 39 | std::cout << "going to pop final message" << std::endl; 40 | std::cout << end << " : " << q.pop(10).data() << std::endl; 41 | } 42 | -------------------------------------------------------------------------------- /cmake/Modules/Findelliptics.cmake: -------------------------------------------------------------------------------- 1 | find_path(elliptics_INCLUDE_DIR elliptics/cppdef.h PATHS ${ELLIPTICS_PREFIX}/include /usr/include) 2 | find_path(cocaine_INCLUDE_DIR cocaine/common.hpp PATHS ${COCAINE_PREFIX}/include /usr/include) 3 | 4 | #find_library(elliptics_LIBRARY NAMES elliptics PATHS ${ELLIPTICS_PREFIX}/lib ${ELLIPTICS_PREFIX}/lib64 /usr/lib /usr/lib64) 5 | find_library(elliptics_cpp_LIBRARY NAMES elliptics_cpp PATHS ${ELLIPTICS_PREFIX}/lib ${ELLIPTICS_PREFIX}/lib64 /usr/lib /usr/lib64) 6 | find_library(elliptics_client_LIBRARY NAMES elliptics_client PATHS ${ELLIPTICS_PREFIX}/lib ${ELLIPTICS_PREFIX}/lib64 /usr/lib /usr/lib64) 7 | 8 | set(elliptics_LIBRARIES ${elliptics_LIBRARY} ${elliptics_cpp_LIBRARY} ${elliptics_client_LIBRARY}) 9 | set(elliptics_INCLUDE_DIRS ${elliptics_INCLUDE_DIR} ${cocaine_INCLUDE_DIR}) 10 | list(REMOVE_DUPLICATES elliptics_INCLUDE_DIRS) 11 | 12 | message(STATUS "elliptics_INCLUDE_DIR: " ${elliptics_INCLUDE_DIR}) 13 | message(STATUS "elliptics_INCLUDE_DIRS: " ${elliptics_INCLUDE_DIRS}) 14 | 15 | include(FindPackageHandleStandardArgs) 16 | # handle the QUIETLY and REQUIRED arguments and set LIBXML2_FOUND to TRUE 17 | # if all listed variables are TRUE 18 | find_package_handle_standard_args(elliptics DEFAULT_MSG elliptics_LIBRARIES elliptics_INCLUDE_DIRS) 19 | 20 | mark_as_advanced(elliptics_INCLUDE_DIRS elliptics_LIBRARIES) 21 | -------------------------------------------------------------------------------- /debian/copyright: -------------------------------------------------------------------------------- 1 | This package was debianized by: 2 | 3 | Evgeniy Polyakov on Wed, 18 Apr 2012 02:58:14 +0300 4 | 5 | It was downloaded from: 6 | 7 | 8 | 9 | Upstream Author(s): 10 | 11 | Evgeniy Polyakov 12 | 13 | Copyright: 14 | 15 | > 16 | 17 | License: 18 | 19 | This package is free software; you can redistribute it and/or modify 20 | it under the terms of the GNU General Public License as published by 21 | the Free Software Foundation; either version 2 of the License, or 22 | (at your option) any later version. 23 | 24 | This package is distributed in the hope that it will be useful, 25 | but WITHOUT ANY WARRANTY; without even the implied warranty of 26 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 27 | GNU General Public License for more details. 28 | 29 | You should have received a copy of the GNU General Public License 30 | along with this package; if not, write to the Free Software 31 | Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA 32 | 33 | On Debian systems, the complete text of the GNU General 34 | Public License can be found in `/usr/share/common-licenses/GPL'. 35 | 36 | The Debian packaging is: 37 | 38 | Copyright C) 2012+, Evgeniy Polyakov 39 | 40 | and is licensed under the GPL, see above. 41 | -------------------------------------------------------------------------------- /include/grape/rapidjson/filestream.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_FILESTREAM_H_ 2 | #define RAPIDJSON_FILESTREAM_H_ 3 | 4 | #include "rapidjson.h" 5 | #include 6 | 7 | namespace rapidjson { 8 | 9 | //! (Depreciated) Wrapper of C file stream for input or output. 10 | /*! 11 | This simple wrapper does not check the validity of the stream. 12 | \implements Stream 13 | \deprecated { This was only for basic testing in version 0.1, it is found that the performance is very low by using fgetc(). Use FileReadStream instead. } 14 | */ 15 | class FileStream { 16 | public: 17 | typedef char Ch; //!< Character type. Only support char. 18 | 19 | FileStream(FILE* fp) : fp_(fp), count_(0) { Read(); } 20 | char Peek() const { return current_; } 21 | char Take() { char c = current_; Read(); return c; } 22 | size_t Tell() const { return count_; } 23 | void Put(char c) { fputc(c, fp_); } 24 | void Flush() { fflush(fp_); } 25 | 26 | // Not implemented 27 | char* PutBegin() { return 0; } 28 | size_t PutEnd(char*) { return 0; } 29 | 30 | private: 31 | void Read() { 32 | RAPIDJSON_ASSERT(fp_ != 0); 33 | int c = fgetc(fp_); 34 | if (c != EOF) { 35 | current_ = (char)c; 36 | count_++; 37 | } 38 | else if (current_ != '\0') 39 | current_ = '\0'; 40 | } 41 | 42 | FILE* fp_; 43 | char current_; 44 | size_t count_; 45 | }; 46 | 47 | } // namespace rapidjson 48 | 49 | #endif // RAPIDJSON_FILESTREAM_H_ 50 | -------------------------------------------------------------------------------- /launchpad/testerhead-cpp: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=$(dirname $(readlink -f $0)) 4 | 5 | REMOTE_HOST=bscoll08.rt.bstest.yandex.net 6 | REMOTE=$REMOTE_HOST:1030:2 7 | LOCATOR_PORT=10060 8 | GROUP=4 9 | 10 | DIOC="dnet_ioclient -r $REMOTE -g $GROUP" 11 | 12 | APP=testerhead-cpp 13 | 14 | APPMANIFEST=${APPMANIFEST:-/etc/elliptics/apps/$APP.conf} 15 | APPPROFILE=${APPPROFILE:-/etc/elliptics/apps/$APP.profile} 16 | 17 | pack() { 18 | TARFILE=$1/$APP.tar 19 | tar -C $2 -cvf $TARFILE $APP 20 | tar -C $(dirname $APPMANIFEST) -rvf $TARFILE $(basename $APPMANIFEST) 21 | rm -f $TARFILE.bz2 22 | bzip2 $TARFILE 23 | } 24 | 25 | case $1 in 26 | pack-dev) 27 | pack $DIR $DIR/../grape/build-debug/src/testerhead-cpp 28 | ;; 29 | pack) 30 | pack $DIR /usr/lib/grape 31 | ;; 32 | upload) 33 | cocaine-tool app upload --host $REMOTE_HOST --port $LOCATOR_PORT -n $APP --manifest $APPMANIFEST --package $APP.tar.bz2 34 | cocaine-tool profile upload --host $REMOTE_HOST --port $LOCATOR_PORT -n $APP --profile $APPPROFILE 35 | ;; 36 | start) 37 | $DIOC -c "$APP@start-task" 38 | ;; 39 | stop) 40 | $DIOC -c "$APP@stop-task" 41 | ;; 42 | info) 43 | $DIOC -c "$APP@info" 44 | ;; 45 | ping) 46 | $DIOC -c "$APP@ping" 47 | echo 48 | ;; 49 | *) 50 | echo "usage: $(basename $0) {pack|upload|start|stop|info}" ;; 51 | esac 52 | #exit 0 53 | -------------------------------------------------------------------------------- /include/grape/rapidjson/stringbuffer.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_STRINGBUFFER_H_ 2 | #define RAPIDJSON_STRINGBUFFER_H_ 3 | 4 | #include "rapidjson.h" 5 | #include "internal/stack.h" 6 | 7 | namespace rapidjson { 8 | 9 | //! Represents an in-memory output stream. 10 | /*! 11 | \tparam Encoding Encoding of the stream. 12 | \tparam Allocator type for allocating memory buffer. 13 | \implements Stream 14 | */ 15 | template 16 | struct GenericStringBuffer { 17 | typedef typename Encoding::Ch Ch; 18 | 19 | GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 20 | 21 | void Put(Ch c) { *stack_.template Push() = c; } 22 | void Flush() {} 23 | 24 | void Clear() { stack_.Clear(); } 25 | 26 | const Ch* GetString() const { 27 | // Push and pop a null terminator. This is safe. 28 | *stack_.template Push() = '\0'; 29 | stack_.template Pop(1); 30 | 31 | return stack_.template Bottom(); 32 | } 33 | 34 | size_t GetSize() const { return stack_.GetSize(); } 35 | 36 | static const size_t kDefaultCapacity = 256; 37 | mutable internal::Stack stack_; 38 | }; 39 | 40 | typedef GenericStringBuffer > StringBuffer; 41 | 42 | //! Implement specialized version of PutN() with memset() for better performance. 43 | template<> 44 | inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { 45 | memset(stream.stack_.Push(n), c, n * sizeof(c)); 46 | } 47 | 48 | } // namespace rapidjson 49 | 50 | #endif // RAPIDJSON_STRINGBUFFER_H_ 51 | -------------------------------------------------------------------------------- /test/test_pump.test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | TEST(PumpTest, DISABLED_InitWorks) { 10 | ioremap::grape::concurrent_pump pump; 11 | pump.concurrency_limit = 10; 12 | } 13 | 14 | TEST(PumpTest, QueueRandomSleep) { 15 | std::atomic_int queue_size(0); 16 | const int messages_limit(21); 17 | const int concurrency_limit(5); 18 | 19 | std::atomic_int received_messages; 20 | std::atomic_int sent_messages; 21 | 22 | std::mutex mutex; 23 | std::condition_variable condition; 24 | 25 | ioremap::grape::concurrent_pump pump; 26 | pump.concurrency_limit = concurrency_limit; 27 | 28 | auto reader = [&] { 29 | received_messages = 0; 30 | while (true) { 31 | std::unique_lock lock(mutex); 32 | if (received_messages >= messages_limit && queue_size == 0) { 33 | break; 34 | } 35 | if (queue_size == 0) { 36 | continue; 37 | } 38 | 39 | // read message 40 | ++received_messages; 41 | --queue_size; 42 | pump.complete_request(); 43 | lock.unlock(); 44 | 45 | std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 50)); 46 | } 47 | }; 48 | 49 | sent_messages = 0; 50 | auto writer = [&] { 51 | std::unique_lock lock(mutex); 52 | if (sent_messages >= messages_limit) { 53 | pump.stop(); 54 | pump.complete_request(); 55 | return; 56 | } 57 | assert(queue_size < concurrency_limit); 58 | 59 | // write message 60 | ++sent_messages; 61 | ++queue_size; 62 | lock.unlock(); 63 | 64 | std::this_thread::sleep_for(std::chrono::milliseconds(rand() % 50)); 65 | }; 66 | 67 | std::thread tReader(reader); 68 | 69 | pump.run(writer); 70 | 71 | tReader.join(); 72 | 73 | EXPECT_EQ(sent_messages, messages_limit); 74 | EXPECT_EQ(received_messages, messages_limit); 75 | } 76 | -------------------------------------------------------------------------------- /launchpad/lookup-host.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import socket 5 | from nodes import connect 6 | from bisect import bisect_right 7 | 8 | def print_routing_table(table): 9 | for node_id, node_ip in table: 10 | print node_ip, node_id 11 | 12 | if __name__ == '__main__': 13 | import argparse 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('remote', help='address of seed node') 16 | parser.add_argument('id', help='hexadecimal id (or prefix of) to lookup responsible host for') 17 | parser.add_argument('-g', '--group', type=int, help='elliptics group') 18 | parser.add_argument('--loglevel', type=int, choices=xrange(5), default=1) 19 | parser.add_argument('--verbose', '-v', action='count') 20 | parser.set_defaults(group=1) 21 | args = parser.parse_args() 22 | 23 | # supplement id to even number of chars 24 | # (as hex form must represent whole number bytes) 25 | if len(args.id) % 2 != 0: 26 | args.id += '0' 27 | try: 28 | eid = map(ord, args.id.decode("hex")) 29 | except TypeError, e: 30 | print 'Invalid id:', e 31 | exit(1) 32 | 33 | session = connect([args.remote], [args.group], 34 | loglevel=args.loglevel, 35 | ) 36 | 37 | routing_table = session.get_routes() 38 | sorted_ids = sorted(routing_table, key=lambda x: x[0].id) 39 | 40 | if args.verbose > 1: 41 | print 'Routing table:' 42 | print_routing_table(sorted_ids) 43 | print 44 | 45 | def find_responsible(a, x): 46 | '''Find responsible routing item in the circular routing table. 47 | Find righmost value less then or equal to x. 48 | And last entry is responsible for [0, a[0]) span. 49 | ''' 50 | keys = [i[0].id for i in a] 51 | i = bisect_right(keys, x) 52 | return a[i-1] 53 | 54 | id, addr = find_responsible(sorted_ids, eid) 55 | 56 | if args.verbose > 0: 57 | print 'Responsible entry:' 58 | print addr, id 59 | print 60 | print 'Answer:' 61 | 62 | print '%s:%s' % (socket.gethostbyaddr(addr.host)[0], addr.port) 63 | 64 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: grape 2 | Section: net 3 | Priority: optional 4 | Maintainer: Ivan Chelyubeev 5 | Build-Depends: debhelper (>= 7), 6 | cdbs, 7 | cmake, 8 | elliptics-dev (>= 2.25), 9 | elliptics-client (>= 2.25), 10 | cocaine-framework-native-dev (>= 0.11), cocaine-framework-native-dev (<< 0.12), 11 | libcocaine-dev (>= 0.11), libcocaine-dev (<< 0.12), 12 | libboost-dev, 13 | libboost-system-dev, 14 | libboost-program-options-dev, 15 | libgtest-dev, 16 | libmsgpack-dev, 17 | libev-dev, 18 | Standards-Version: 3.8.0 19 | Homepage: https://github.com/reverbrain/grape 20 | Vcs-Git: git://github.com/reverbrain/grape.git 21 | Vcs-Browser: https://github.com/reverbrain/grape 22 | 23 | Package: grape 24 | Architecture: any 25 | Depends: ${shlibs:Depends}, ${misc:Depends}, python-prettytable 26 | Description: Realtime pipeline processing engine 27 | 28 | #Package: grape-dbg 29 | #Architecture: any 30 | #Section: debug 31 | #Depends: ${shlibs:Depends}, ${misc:Depends}, grape (= ${binary:Version}) 32 | #Description: Grape debug files 33 | # Grape debug files and symbols 34 | 35 | Package: grape-dev 36 | Architecture: any 37 | Depends: ${shlibs:Depends}, ${misc:Depends}, grape (= ${binary:Version}) 38 | Description: Realtime pipeline processing engine (includes) 39 | 40 | Package: libcocaine-plugin-queue-driver 41 | Architecture: any 42 | Depends: ${shlibs:Depends}, ${misc:Depends}, elliptics-client (>= 2.25), grape (= ${binary:Version}) 43 | Description: Grape queue driver (cocaine plugin) 44 | Grape queue driver runs as a cocaine plugin and can be turned on for applications, 45 | which want to pop events from persistent queue (application named 'queue') 46 | 47 | Package: grape-components 48 | Architecture: any 49 | Depends: ${shlibs:Depends}, ${misc:Depends}, grape (= ${binary:Version}), cocaine-framework-native (>= 0.11), cocaine-framework-native (<< 0.12) 50 | Description: Grape queue and other components (cocaine apps) 51 | Grape queue and other component apps 52 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required (VERSION 2.6) 2 | project (grape) 3 | 4 | add_definitions("-std=c++0x -Wreorder -Wreturn-type -Wunused-variable") 5 | 6 | # The version number. 7 | FILE (READ "${CMAKE_CURRENT_SOURCE_DIR}/debian/changelog" DEBCHANGELOG) 8 | string(REGEX MATCH "([0-9]+\\.[0-9]+\\.[0-9]+)" DEBFULLVERSION "${DEBCHANGELOG}") 9 | STRING (REGEX MATCH "([0-9]+\\.[0-9]+)" grape_VERSION_MAJOR "${DEBFULLVERSION}") 10 | STRING (REGEX MATCH "([0-9]+$)" grape_VERSION_MINOR "${DEBFULLVERSION}") 11 | set(grape_VERSION "${grape_VERSION_MAJOR}.${grape_VERSION_MINOR}") 12 | 13 | find_package(Boost REQUIRED system program_options) 14 | 15 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/Modules/") 16 | 17 | find_package(elliptics REQUIRED) 18 | find_package(CocaineNative REQUIRED) 19 | find_package(Cocaine REQUIRED) 20 | 21 | include(cmake/Modules/locate_library.cmake) 22 | locate_library(LIBEV "ev++.h" "ev" "libev") 23 | locate_library(MSGPACK "msgpack.hpp" "msgpack") 24 | 25 | set(GRAPE_COMMON_LIBRARIES 26 | ${elliptics_cpp_LIBRARY} 27 | ${CocaineNative_LIBRARIES} 28 | ${Cocaine_LIBRARY} 29 | ${LIBEV_LIBRARIES} 30 | ${Boost_SYSTEM_LIBRARY} 31 | ${Boost_THREAD_LIBRARY} 32 | ) 33 | 34 | include_directories(${PROJECT_SOURCE_DIR}/include ${elliptics_INCLUDE_DIRS} ${CocaineNative_INCLUDE_DIRS} ${LIBEV_INCLUDE_DIRS}) 35 | 36 | add_subdirectory(src/data_array) 37 | add_subdirectory(src/driver) 38 | add_subdirectory(src/queue) 39 | add_subdirectory(src/queue-pump) 40 | add_subdirectory(src/testerhead-cpp) 41 | 42 | #add_subdirectory(example) 43 | add_subdirectory(launchpad) 44 | 45 | # testing 46 | enable_testing() 47 | add_subdirectory(test) 48 | 49 | install(DIRECTORY include/ DESTINATION include) 50 | 51 | #set(ARCHIVE_NAME ${CMAKE_PROJECT_NAME}) 52 | #add_custom_target(dist 53 | # COMMAND git archive --prefix=${ARCHIVE_NAME}-${grape_VERSION}/ HEAD 54 | # | bzip2 > ${CMAKE_BINARY_DIR}/${ARCHIVE_NAME}-${grape_VERSION}.tar.bz2 55 | # WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) 56 | -------------------------------------------------------------------------------- /include/grape/rapidjson/filewritestream.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_FILEWRITESTREAM_H_ 2 | #define RAPIDJSON_FILEWRITESTREAM_H_ 3 | 4 | #include "rapidjson.h" 5 | #include 6 | 7 | namespace rapidjson { 8 | 9 | //! Wrapper of C file stream for input using fread(). 10 | /*! 11 | \implements Stream 12 | */ 13 | class FileWriteStream { 14 | public: 15 | typedef char Ch; //!< Character type. Only support char. 16 | 17 | FileWriteStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { 18 | RAPIDJSON_ASSERT(fp_ != 0); 19 | } 20 | 21 | void Put(char c) { 22 | if (current_ >= bufferEnd_) 23 | Flush(); 24 | 25 | *current_++ = c; 26 | } 27 | 28 | void PutN(char c, size_t n) { 29 | size_t avail = bufferEnd_ - current_; 30 | while (n > avail) { 31 | memset(current_, c, avail); 32 | current_ += avail; 33 | Flush(); 34 | n -= avail; 35 | avail = bufferEnd_ - current_; 36 | } 37 | 38 | if (n > 0) { 39 | memset(current_, c, n); 40 | current_ += n; 41 | } 42 | } 43 | 44 | void Flush() { 45 | if (current_ != buffer_) { 46 | fwrite(buffer_, 1, current_ - buffer_, fp_); 47 | current_ = buffer_; 48 | } 49 | } 50 | 51 | // Not implemented 52 | char Peek() const { RAPIDJSON_ASSERT(false); return 0; } 53 | char Take() { RAPIDJSON_ASSERT(false); return 0; } 54 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 55 | char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 56 | size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } 57 | 58 | private: 59 | FILE* fp_; 60 | char *buffer_; 61 | char *bufferEnd_; 62 | char *current_; 63 | }; 64 | 65 | //! Implement specialized version of PutN() with memset() for better performance. 66 | template<> 67 | inline void PutN(FileWriteStream& stream, char c, size_t n) { 68 | stream.PutN(c, n); 69 | } 70 | 71 | } // namespace rapidjson 72 | 73 | #endif // RAPIDJSON_FILESTREAM_H_ 74 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(Threads REQUIRED) 2 | include_directories(${Threads_INCLUDE_DIR}) 3 | 4 | # GTest 5 | find_package(GTest) 6 | 7 | if (NOT GTEST_FOUND) 8 | find_path(GTEST_SRC_DIR src/gtest.cc HINTS /usr/src/gtest) 9 | find_path(GTEST_INCLUDE_DIRS gtest.h HINTS /usr/include/gtest) 10 | if (GTEST_SRC_DIR AND GTEST_INCLUDE_DIRS) 11 | message(STATUS "GTest source package found. Compiling...") 12 | add_subdirectory(${GTEST_SRC_DIR} ${CMAKE_BINARY_DIR}/gtest) 13 | 14 | set(GTEST_FOUND 1) 15 | set(GTEST_MAIN_LIBRARIES ${CMAKE_BINARY_DIR}/gtest/libgtest_main.a) 16 | set(GTEST_LIBRARIES ${CMAKE_BINARY_DIR}/gtest/libgtest.a) 17 | set(GTEST_BOTH_LIBRARIES ${GTEST_LIBRARIES} ${GTEST_MAIN_LIBRARIES}) 18 | else (GTEST_SRC_DIR AND GTEST_INCLUDE_DIRS) 19 | message(SEND_ERROR "Cannot find required package: GTest") 20 | endif (GTEST_SRC_DIR AND GTEST_INCLUDE_DIRS) 21 | else (NOT GTEST_FOUND) 22 | endif (NOT GTEST_FOUND) 23 | 24 | include_directories(${GTEST_INCLUDE_DIRS}) 25 | set(GTEST_TARGETS gtest gtest_main) 26 | 27 | # Touch test's CMakeLists to update tests sources 28 | add_custom_target(check_tests ALL touch ${CMAKE_CURRENT_SOURCE_DIR}/CMakeLists.txt) 29 | 30 | # Show tests printouts 31 | set(CMAKE_CTEST_COMMAND ctest --output-on-failure) 32 | add_custom_target(run_tests ${CMAKE_CTEST_COMMAND}) 33 | 34 | #set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "bin/") 35 | 36 | include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR} ${CMAKE_SOURCE_DIR}) 37 | 38 | # Get all test source files 39 | file(GLOB_RECURSE test_src_files RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.test.cpp") 40 | 41 | foreach(test_src_file IN LISTS test_src_files) 42 | string(REGEX REPLACE "[.]test[.]cpp$" "_test" test_exec_file ${test_src_file}) 43 | 44 | add_executable(${test_exec_file} ${test_src_file}) 45 | target_link_libraries(${test_exec_file} ${CMAKE_THREAD_LIBS_INIT} ${GTEST_BOTH_LIBRARIES} 46 | queue grape_data_array rt 47 | ) 48 | add_dependencies(${test_exec_file} ${GTEST_TARGETS}) 49 | 50 | add_test(${test_exec_file} ${test_exec_file}) 51 | list(APPEND test_list ${test_exec_file}) 52 | endforeach(test_src_file) 53 | 54 | add_dependencies(run_tests ${test_list}) 55 | 56 | -------------------------------------------------------------------------------- /include/grape/rapidjson/filereadstream.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_FILEREADSTREAM_H_ 2 | #define RAPIDJSON_FILEREADSTREAM_H_ 3 | 4 | #include "rapidjson.h" 5 | #include 6 | 7 | namespace rapidjson { 8 | 9 | //! File byte stream for input using fread(). 10 | /*! 11 | \implements Stream 12 | */ 13 | class FileReadStream { 14 | public: 15 | typedef char Ch; //!< Character type (byte). 16 | 17 | //! Constructor. 18 | /*! 19 | \param fp File pointer opened for read. 20 | \param buffer user-supplied buffer. 21 | \param bufferSize size of buffer in bytes. Must >=4 bytes. 22 | */ 23 | FileReadStream(FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 24 | RAPIDJSON_ASSERT(fp_ != 0); 25 | RAPIDJSON_ASSERT(bufferSize >= 4); 26 | Read(); 27 | } 28 | 29 | Ch Peek() const { return *current_; } 30 | Ch Take() { Ch c = *current_; Read(); return c; } 31 | size_t Tell() const { return count_ + (current_ - buffer_); } 32 | 33 | // Not implemented 34 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 35 | void Flush() { RAPIDJSON_ASSERT(false); } 36 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 37 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 38 | 39 | // For encoding detection only. 40 | const Ch* Peek4() const { 41 | return (current_ + 4 <= bufferLast_) ? current_ : 0; 42 | } 43 | 44 | private: 45 | void Read() { 46 | if (current_ < bufferLast_) 47 | ++current_; 48 | else if (!eof_) { 49 | count_ += readCount_; 50 | readCount_ = fread(buffer_, 1, bufferSize_, fp_); 51 | bufferLast_ = buffer_ + readCount_ - 1; 52 | current_ = buffer_; 53 | 54 | if (readCount_ < bufferSize_) { 55 | buffer_[readCount_] = '\0'; 56 | ++bufferLast_; 57 | eof_ = true; 58 | } 59 | } 60 | } 61 | 62 | FILE* fp_; 63 | Ch *buffer_; 64 | size_t bufferSize_; 65 | Ch *bufferLast_; 66 | Ch *current_; 67 | size_t readCount_; 68 | size_t count_; //!< Number of characters read 69 | bool eof_; 70 | }; 71 | 72 | } // namespace rapidjson 73 | 74 | #endif // RAPIDJSON_FILESTREAM_H_ 75 | -------------------------------------------------------------------------------- /test/data_array.test.cpp: -------------------------------------------------------------------------------- 1 | // #include 2 | // #include 3 | // #include 4 | // #include 5 | 6 | #include 7 | #include 8 | 9 | using namespace ioremap::grape; 10 | 11 | TEST(DataArray, Iteration) { 12 | data_array a; 13 | 14 | std::vector data_entries; 15 | 16 | for (int i = 0; i < 10; ++i) { 17 | std::string data("entry-"); 18 | data += std::to_string(i); 19 | data_entries.push_back(data); 20 | } 21 | 22 | for (int i = 0; i < 10; ++i) { 23 | a.append(data_entries[i].data(), data_entries[i].size(), entry_id{8, i}); 24 | } 25 | 26 | { 27 | int n = 0; 28 | for (auto entry : a) { 29 | EXPECT_EQ(entry.entry_id.chunk, 8) << "bad chunk number in entry " << n; 30 | EXPECT_EQ(entry.entry_id.pos, n) << "bad pos number in entry " << n; 31 | EXPECT_EQ(std::string(entry.data, entry.size), data_entries[n]) << "bad data in entry " << n; 32 | ++n; 33 | } 34 | EXPECT_EQ(n, 10) << "wrong number of entries"; 35 | } 36 | { 37 | int n = 0; 38 | for (data_array::iterator i = a.begin(); i != a.end(); ++i) { 39 | EXPECT_EQ(i->entry_id.chunk, 8) << "bad chunk number in entry " << n; 40 | EXPECT_EQ(i->entry_id.pos, n) << "bad pos number in entry " << n; 41 | EXPECT_EQ(std::string(i->data, i->size), data_entries[n]) << "bad data in entry " << n; 42 | ++n; 43 | } 44 | EXPECT_EQ(n, 10) << "wrong number of entries"; 45 | } 46 | } 47 | 48 | TEST(DataArray, EnumerateArray) { 49 | data_array a; 50 | { 51 | a.append(std::string("entry"), entry_id{1, 2}); 52 | } 53 | std::vector entries(a.begin(), a.end()); 54 | EXPECT_EQ(std::string(entries[0].data, entries[0].size), "entry"); 55 | } 56 | 57 | TEST(DataArray, EnumerateConstArray) { 58 | data_array a; 59 | { 60 | a.append(std::string("entry"), entry_id{1, 2}); 61 | } 62 | const data_array b = a; 63 | std::vector entries(b.begin(), b.end()); 64 | EXPECT_EQ(std::string(entries[0].data, entries[0].size), "entry"); 65 | } 66 | -------------------------------------------------------------------------------- /include/grape/data_array.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __GRAPE_DATA_ARRAY_HPP 2 | #define __GRAPE_DATA_ARRAY_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | namespace ioremap { namespace grape { 14 | 15 | class data_array 16 | { 17 | private: 18 | std::vector m_id; 19 | std::vector m_size; 20 | std::string m_data; 21 | 22 | public: 23 | // virtual view into single array item 24 | struct entry 25 | { 26 | const char *data; 27 | size_t size; 28 | grape::entry_id entry_id; 29 | }; 30 | 31 | class iterator : public std::iterator 32 | { 33 | public: 34 | iterator(const iterator &other); 35 | 36 | iterator &operator =(const iterator &other); 37 | 38 | bool operator ==(const iterator &other) const; 39 | bool operator !=(const iterator &other) const; 40 | 41 | value_type operator *() const; 42 | value_type *operator ->() const; 43 | 44 | iterator &operator ++(); 45 | iterator operator ++(int); 46 | 47 | private: 48 | iterator(const data_array *array, bool at_end); 49 | 50 | const data_array *array; 51 | int index; 52 | size_t offset; 53 | 54 | void prepare_value() const; 55 | mutable value_type value; 56 | 57 | friend class data_array; 58 | }; 59 | 60 | void append(const char *data, size_t size, const entry_id &id); 61 | void append(const std::string &data, const entry_id &id); 62 | void append(const data_array::entry &entry); 63 | void extend(const data_array &d); 64 | 65 | bool empty() const; 66 | 67 | // "do whatever you want" interface 68 | const std::vector &ids() const; 69 | const std::vector &sizes() const; 70 | const std::string &data() const; 71 | 72 | // iteration interface 73 | 74 | iterator begin() const; 75 | iterator end() const; 76 | 77 | MSGPACK_DEFINE(m_id, m_size, m_data); 78 | }; 79 | 80 | template 81 | elliptics::data_pointer serialize(const T &obj) { 82 | msgpack::sbuffer sbuf; 83 | msgpack::pack(sbuf, obj); 84 | 85 | return elliptics::data_pointer::copy(sbuf.data(), sbuf.size()); 86 | } 87 | 88 | template 89 | T deserialize(const elliptics::data_pointer &d) { 90 | msgpack::unpacked msg; 91 | msgpack::unpack(&msg, (const char *)d.data(), d.size()); 92 | 93 | msgpack::object obj = msg.get(); 94 | 95 | T result; 96 | obj.convert(&result); 97 | 98 | return result; 99 | } 100 | 101 | }} 102 | 103 | #endif /* __GRAPE_DATA_ARRAY_HPP */ 104 | -------------------------------------------------------------------------------- /include/grape/logger_adapter.hpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | namespace ioremap { namespace grape { 7 | 8 | class logger_adapter : public cocaine::framework::logger_t 9 | { 10 | private: 11 | std::shared_ptr base_logger; 12 | public: 13 | logger_adapter(std::shared_ptr base_logger) 14 | : base_logger(base_logger) 15 | {} 16 | 17 | logger_adapter(ioremap::elliptics::logger base_logger) 18 | : base_logger(std::make_shared(base_logger)) 19 | {} 20 | 21 | virtual 22 | ~logger_adapter() { 23 | // pass 24 | } 25 | 26 | template 27 | void emit(cocaine::logging::priorities priority, const std::string &format, const Args&... args) { 28 | emit(priority, cocaine::format(format, args...)); 29 | } 30 | 31 | virtual 32 | void emit(cocaine::logging::priorities priority, 33 | const std::string &message) { 34 | if (message.back() != '\n') { 35 | base_logger->log(prio_to_dnet_log_level(priority), (message + "\n").c_str()); 36 | } 37 | else { 38 | base_logger->log(prio_to_dnet_log_level(priority), message.c_str()); 39 | } 40 | } 41 | 42 | virtual 43 | cocaine::logging::priorities verbosity() const { 44 | return dnet_log_level_to_prio(base_logger->get_log_level()); 45 | } 46 | 47 | // XXX: Following methods have been copy-pasted from elliptics' srw.cpp 48 | 49 | // INFO level has value 2 in elliptics and value 3 in cocaine, 50 | // nevertheless we want to support unified sense of INFO across both systems, 51 | // so we need to play with the mapping a bit. 52 | // 53 | // Specifically: 54 | // 1) cocaine warning and info levels are both mapped into eliptics info level 55 | // 2) elliptics notice level means cocaine info level 56 | 57 | static cocaine::logging::priorities dnet_log_level_to_prio(int level) { 58 | cocaine::logging::priorities prio = (cocaine::logging::priorities)level; 59 | // elliptics info level becomes cocaine warning level, 60 | // so we must to level it up 61 | if (prio == cocaine::logging::warning) { 62 | prio = cocaine::logging::info; 63 | } 64 | return prio; 65 | } 66 | 67 | static int prio_to_dnet_log_level(cocaine::logging::priorities prio) { 68 | int level = DNET_LOG_DATA; 69 | if (prio == cocaine::logging::debug) 70 | level = DNET_LOG_DEBUG; 71 | if (prio == cocaine::logging::info) 72 | level = DNET_LOG_INFO; 73 | if (prio == cocaine::logging::warning) 74 | level = DNET_LOG_INFO; 75 | if (prio == cocaine::logging::error) 76 | level = DNET_LOG_ERROR; 77 | return level; 78 | } 79 | }; 80 | 81 | }} 82 | -------------------------------------------------------------------------------- /src/queue/queue.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __QUEUE_HPP 2 | #define __QUEUE_HPP 3 | 4 | #include 5 | 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "chunk.hpp" 16 | 17 | namespace ioremap { namespace grape { 18 | 19 | struct queue_state { 20 | int chunk_id_push; 21 | int chunk_id_ack; 22 | }; 23 | 24 | struct queue_statistics { 25 | uint64_t push_count; 26 | uint64_t pop_count; 27 | uint64_t ack_count; 28 | uint64_t timeout_count; 29 | 30 | uint64_t state_write_count; 31 | 32 | chunk_stat chunks_popped; 33 | chunk_stat chunks_pushed; 34 | }; 35 | 36 | class queue { 37 | public: 38 | ELLIPTICS_DISABLE_COPY(queue); 39 | 40 | queue(const std::string &queue_id); 41 | 42 | void initialize(const std::string &config); 43 | 44 | // single entry methods 45 | void push(const elliptics::data_pointer &d); 46 | elliptics::data_pointer peek(entry_id *entry_id); 47 | void ack(const entry_id id); 48 | elliptics::data_pointer pop(); 49 | 50 | // multiple entries methods 51 | data_array peek(int num); 52 | void ack(const std::vector &ids); 53 | data_array pop(int num); 54 | 55 | // content manipulation 56 | void clear(); 57 | 58 | void reply(cocaine::framework::response_ptr response, const ioremap::elliptics::exec_context &context, 59 | const ioremap::elliptics::argument_data &d, 60 | ioremap::elliptics::exec_context::final_state state); 61 | void final(cocaine::framework::response_ptr response, const ioremap::elliptics::exec_context &context, const ioremap::elliptics::argument_data &d); 62 | 63 | const std::string &queue_id() const; 64 | const queue_state &state(); 65 | const queue_statistics &statistics(); 66 | void clear_counters(); 67 | 68 | private: 69 | int m_chunk_max; 70 | uint64_t m_ack_wait_timeout; 71 | uint64_t m_timeout_check_period; 72 | 73 | std::string m_queue_id; 74 | std::string m_queue_state_id; 75 | 76 | elliptics_client_state m_client_proto; 77 | std::shared_ptr m_reply_client; 78 | std::shared_ptr m_data_client; 79 | 80 | queue_state m_state; 81 | queue_statistics m_statistics; 82 | 83 | std::map m_chunks; 84 | std::map m_wait_ack; 85 | uint64_t m_last_timeout_check_time; 86 | 87 | void write_state(); 88 | 89 | void update_chunk_timeout(int chunk_id, shared_chunk chunk); 90 | 91 | void check_timeouts(); 92 | }; 93 | 94 | }} // namespace ioremap::grape 95 | 96 | #endif /* __QUEUE_HPP */ 97 | -------------------------------------------------------------------------------- /launchpad/queue: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | DIR=$(dirname $(readlink -f $0)) 4 | 5 | # elliptics endpoint 6 | REMOTE_HOST=${REMOTE_HOST:-bscoll08.rt.bstest.yandex.net} 7 | REMOTE_PORT=${REMOTE_PORT:-1025} 8 | REMOTE=$REMOTE_HOST:$REMOTE_PORT:2 9 | GROUP=${GROUP:-1} 10 | 11 | # cocaine endpoint 12 | # REMOTE_HOST plus 13 | LOCATOR_PORT=${LOCATOR_PORT:-10056} 14 | 15 | DIOC="dnet_ioclient -r $REMOTE -g $GROUP" 16 | 17 | APP=queue 18 | 19 | APPMANIFEST=${APPMANIFEST:-/etc/elliptics/apps/$APP.conf} 20 | APPPROFILE=${APPPROFILE:-/etc/elliptics/apps/$APP.profile} 21 | 22 | function sha() { 23 | printf -- $1 | sha512sum | cut -f1 -d' ' 24 | } 25 | 26 | #queue_id_hash=$(sha $queue_id) 27 | 28 | pack() { 29 | TARFILE=$1/$APP.tar 30 | tar -C $2 -cvf $TARFILE $APP 31 | tar -C $(dirname $APPMANIFEST) -rvf $TARFILE $(basename $APPMANIFEST) 32 | rm -f $TARFILE.bz2 33 | bzip2 $TARFILE 34 | } 35 | 36 | case $1 in 37 | pack-dev) 38 | #pack $DIR $DIR/../grape/build-debug/src/queue 39 | pack $DIR /home/ijon/proj/grape/build-debug/src/queue 40 | ;; 41 | pack) 42 | pack $DIR /usr/lib/grape 43 | ;; 44 | upload) 45 | cocaine-tool app upload --host $REMOTE_HOST --port $LOCATOR_PORT -n $APP --manifest $APPMANIFEST --package $APP.tar.bz2 46 | cocaine-tool profile upload --host $REMOTE_HOST --port $LOCATOR_PORT -n $APP --profile $APPPROFILE 47 | ;; 48 | start) 49 | # send start event to every distinct node in the group 50 | IDS=$(./nodes.py $REMOTE id) 51 | N=0 52 | for i in $IDS; do 53 | dnet_ioclient -r $REMOTE -g $GROUP -I $i -c "$APP@start-multiple-task ${PREFIX}$((N++))" 54 | done 55 | ;; 56 | stop) 57 | $DIOC -c "$APP@stop-task" 58 | ;; 59 | info) 60 | $DIOC -c "$APP@info" ${2:+-I$2} 61 | ;; 62 | stats_) 63 | $DIOC -c "$APP@stats" ${2:+-I$2} 64 | ;; 65 | stats) 66 | ./_queue_stats.py -r $REMOTE -g $GROUP 67 | ;; 68 | clear_) 69 | $DIOC -c "$APP@clear" 70 | ;; 71 | clear) 72 | ./_queue_clear.py -r $REMOTE -g $GROUP 73 | ;; 74 | push) 75 | $DIOC -c "$APP@push ${2:-abcdef}" 76 | ;; 77 | pop) 78 | $DIOC -c "$APP@pop" 79 | ;; 80 | peek) 81 | $DIOC -c "$APP@peek" 82 | ;; 83 | ack) 84 | $DIOC -c "$APP@ack" -I$queue_id_hash 85 | ;; 86 | ping) 87 | $DIOC -c "$APP@ping" 88 | ;; 89 | loglevel) 90 | $DIOC -U0 -M$2 91 | ;; 92 | _) 93 | $DIOC -c "$APP@$2" -I$queue_id_hash 94 | ;; 95 | *) 96 | echo "usage: $(basename $0) {pack|upload|start|stop|info|log|new-id|ping}" ;; 97 | esac 98 | #exit 0 99 | -------------------------------------------------------------------------------- /include/grape/rapidjson/internal/stack.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_INTERNAL_STACK_H_ 2 | #define RAPIDJSON_INTERNAL_STACK_H_ 3 | 4 | namespace rapidjson { 5 | namespace internal { 6 | 7 | /////////////////////////////////////////////////////////////////////////////// 8 | // Stack 9 | 10 | //! A type-unsafe stack for storing different types of data. 11 | /*! \tparam Allocator Allocator for allocating stack memory. 12 | */ 13 | template 14 | class Stack { 15 | public: 16 | Stack(Allocator* allocator, size_t stack_capacity) : allocator_(allocator), own_allocator_(0), stack_(0), stack_top_(0), stack_end_(0), stack_capacity_(stack_capacity) { 17 | RAPIDJSON_ASSERT(stack_capacity_ > 0); 18 | if (!allocator_) 19 | own_allocator_ = allocator_ = new Allocator(); 20 | stack_top_ = stack_ = (char*)allocator_->Malloc(stack_capacity_); 21 | stack_end_ = stack_ + stack_capacity_; 22 | } 23 | 24 | ~Stack() { 25 | Allocator::Free(stack_); 26 | delete own_allocator_; // Only delete if it is owned by the stack 27 | } 28 | 29 | void Clear() { /*stack_top_ = 0;*/ stack_top_ = stack_; } 30 | 31 | template 32 | T* Push(size_t count = 1) { 33 | // Expand the stack if needed 34 | if (stack_top_ + sizeof(T) * count >= stack_end_) { 35 | size_t new_capacity = stack_capacity_ * 2; 36 | size_t size = GetSize(); 37 | size_t new_size = GetSize() + sizeof(T) * count; 38 | if (new_capacity < new_size) 39 | new_capacity = new_size; 40 | stack_ = (char*)allocator_->Realloc(stack_, stack_capacity_, new_capacity); 41 | stack_capacity_ = new_capacity; 42 | stack_top_ = stack_ + size; 43 | stack_end_ = stack_ + stack_capacity_; 44 | } 45 | T* ret = (T*)stack_top_; 46 | stack_top_ += sizeof(T) * count; 47 | return ret; 48 | } 49 | 50 | template 51 | T* Pop(size_t count) { 52 | RAPIDJSON_ASSERT(GetSize() >= count * sizeof(T)); 53 | stack_top_ -= count * sizeof(T); 54 | return (T*)stack_top_; 55 | } 56 | 57 | template 58 | T* Top() { 59 | RAPIDJSON_ASSERT(GetSize() >= sizeof(T)); 60 | return (T*)(stack_top_ - sizeof(T)); 61 | } 62 | 63 | template 64 | T* Bottom() { return (T*)stack_; } 65 | 66 | Allocator& GetAllocator() { return *allocator_; } 67 | bool Empty() const { return stack_top_ == stack_; } 68 | size_t GetSize() const { return stack_top_ - stack_; } 69 | size_t GetCapacity() const { return stack_capacity_; } 70 | 71 | private: 72 | Allocator* allocator_; 73 | Allocator* own_allocator_; 74 | char *stack_; 75 | char *stack_top_; 76 | char *stack_end_; 77 | size_t stack_capacity_; 78 | }; 79 | 80 | } // namespace internal 81 | } // namespace rapidjson 82 | 83 | #endif // RAPIDJSON_STACK_H_ 84 | -------------------------------------------------------------------------------- /src/queue-pump/queue-push.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | using namespace boost::program_options; 10 | using namespace ioremap::grape; 11 | 12 | int main(int argc, char** argv) 13 | { 14 | options_description generic("Generic options"); 15 | generic.add_options() 16 | ("help", "help message") 17 | ; 18 | 19 | options_description elliptics("Elliptics options"); 20 | elliptics.add_options() 21 | ("remote,r", value(), "remote addr to initially connect to") 22 | ("group,g", value>()->multitoken(), "group(s) to connect to") 23 | ("loglevel", value()->default_value(0), "client loglevel") 24 | ("net-thread-num", value()->default_value(0), "client 'net thread' pool size") 25 | ("io-thread-num", value()->default_value(0), "client 'io thread' pool size") 26 | ; 27 | 28 | options_description other("Options"); 29 | elliptics.add_options() 30 | ("concurrency,n", value()->default_value(1), "concurrency limit") 31 | ("limit,l", value()->default_value(0), "upper limit") 32 | ; 33 | 34 | options_description opts; 35 | opts.add(generic).add(elliptics).add(other); 36 | 37 | parsed_options parsed_opts = parse_command_line(argc, argv, opts); 38 | variables_map args; 39 | store(parsed_opts, args); 40 | notify(args); 41 | 42 | if (args.count("help")) { 43 | std::cout << "Queue support utility." << "\n"; 44 | std::cout << opts << "\n"; 45 | return 1; 46 | } 47 | if (!args.count("remote")) 48 | { 49 | std::cerr << "--remote option required" << "\n"; 50 | return 1; 51 | } 52 | 53 | if (!args.count("group")) { 54 | std::cerr << "--group option required" << "\n"; 55 | return 1; 56 | } 57 | 58 | std::vector remotes; 59 | remotes.push_back(args["remote"].as()); 60 | 61 | std::vector groups = args["group"].as>(); 62 | 63 | std::string logfile = "/dev/stderr"; 64 | 65 | int loglevel = args["loglevel"].as(); 66 | int net_thread_num = args["net-thread-num"].as(); 67 | int io_thread_num = args["io-thread-num"].as(); 68 | 69 | int concurrency = args["concurrency"].as(); 70 | int limit = args["limit"].as(); 71 | 72 | auto clientlib = elliptics_client_state::create( 73 | remotes, groups, logfile, loglevel, 74 | 0, 0, net_thread_num, io_thread_num 75 | ); 76 | 77 | const std::string queue_name("queue"); 78 | 79 | // write queue indefinitely, with ever increasing number 80 | queue_writer pump(clientlib.create_session(), queue_name, concurrency); 81 | int counter = 0; 82 | pump.run([&counter, &limit] () { 83 | if (limit > 0 && counter >= limit) { 84 | return ioremap::elliptics::data_pointer(); 85 | } 86 | return ioremap::elliptics::data_pointer::copy(std::to_string(counter++)); 87 | }); 88 | 89 | return 0; 90 | } 91 | -------------------------------------------------------------------------------- /launchpad/nodes.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | sys.path.append('bindings/python/') 6 | import elliptics 7 | 8 | class PassthroughWrapper(object): 9 | ''' Wrapper to assure session/node destroy sequence: session first, node last ''' 10 | def __init__(self, node, session): 11 | self.node = node 12 | self.session = session 13 | 14 | def __getattr__(self, name): 15 | return getattr(self.session, name) 16 | 17 | def __del__(self): 18 | del self.session 19 | del self.node 20 | 21 | def connect(endpoints, groups, **kw): 22 | remotes = [] 23 | for r in endpoints: 24 | parts = r.split(":") 25 | remotes.append((parts[0], int(parts[1]))) 26 | 27 | def rename(new, old): 28 | if old in kw: 29 | kw[new] = kw.pop(old) 30 | 31 | kw.pop('elog', None) 32 | kw.pop('cfg', None) 33 | kw.pop('remotes', None) 34 | rename('log_file', 'logfile') 35 | rename('log_level', 'loglevel') 36 | 37 | n = elliptics.create_node(**kw) 38 | 39 | # def create_node(**kw): 40 | # log = elliptics.Logger(kw.get('logfile', '/dev/stderr'), kw.get('loglevel', 1)) 41 | # config = elliptics.Config() 42 | # config.config.wait_timeout = kw.get('wait-timeout', 60) 43 | # return elliptics.Node(log, config) 44 | # n = create_node(**kw) 45 | 46 | for r in remotes: 47 | try: 48 | n.add_remote(r[0], r[1]) 49 | except Exception as e: 50 | pass 51 | 52 | s = elliptics.Session(n) 53 | s.add_groups(groups) 54 | #XXX: Is it time to drop PassthroughWrapper binder? 55 | return PassthroughWrapper(n, s) 56 | 57 | def node_id_map(routes): 58 | return dict([(i[1], i[0]) for i in routes]) 59 | 60 | def nodes(routes): 61 | return sorted(set([i[1] for i in routes])) 62 | 63 | def ids(routes): 64 | return node_id_map(routes).values() 65 | 66 | def node_id(routes): 67 | return sorted(node_id_map(routes).items(), key=lambda x: x[1].id) 68 | 69 | if __name__ == '__main__': 70 | import argparse 71 | parser = argparse.ArgumentParser() 72 | parser.add_argument('remote', help='address of seed node') 73 | parser.add_argument('what', choices=['ip', 'id', 'all'], help='what information to print') 74 | parser.add_argument('-g', '--group', type=int, help='elliptics group') 75 | parser.add_argument('--loglevel', type=int, choices=xrange(5), default=1) 76 | parser.set_defaults(group=1) 77 | args = parser.parse_args() 78 | 79 | session = connect([args.remote], [args.group], 80 | loglevel=args.loglevel, 81 | io_thread_num=3, 82 | ) 83 | 84 | routing_table = session.get_routes() 85 | #for node_id,node_ip in routing_table: 86 | # print node_ip, ''.join(['{0:02x}'.format(k) for k in node_id.id]) 87 | 88 | if args.what == 'ip': 89 | for i in nodes(routing_table): 90 | print i 91 | elif args.what == 'id': 92 | for i in ids(routing_table): 93 | print ''.join(['{0:02x}'.format(k) for k in i.id]) 94 | elif args.what == 'all': 95 | for node_ip,node_id in node_id(routing_table): 96 | print node_ip, ''.join(['{0:02x}'.format(k) for k in node_id.id]) 97 | 98 | -------------------------------------------------------------------------------- /grape-bf.spec: -------------------------------------------------------------------------------- 1 | Summary: Grape 2 | Name: grape 3 | Version: 0.7.12 4 | Release: 1%{?dist} 5 | 6 | License: GPLv2+ 7 | Group: System Environment/Libraries 8 | URL: https://github.com/reverbrain/grape 9 | Source0: %{name}-%{version}.tar.bz2 10 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 11 | 12 | %if %{defined rhel} && 0%{?rhel} < 6 13 | BuildRequires: gcc44 gcc44-c++ 14 | %define boost_ver 141 15 | %else 16 | %define boost_ver %{nil} 17 | %endif 18 | BuildRequires: cmake 19 | BuildRequires: elliptics-devel >= 2.25, elliptics-client-devel >= 2.25 20 | BuildRequires: cocaine-framework-native-devel >= 0.11, cocaine-framework-native-devel < 0.12 21 | BuildRequires: libcocaine-core2-devel >= 0.11, libcocaine-core2-devel < 0.12 22 | BuildRequires: boost%{boost_ver}-devel 23 | BuildRequires: gtest-devel 24 | BuildRequires: msgpack-devel 25 | 26 | #FIXME: This is proxy dependency for cocaine-framework-native, 27 | # we only need to be able to link apps with binary libev4 28 | BuildRequires: libev-devel 29 | 30 | Obsoletes: srw 31 | 32 | Requires: python-prettytable 33 | 34 | %description 35 | Realtime pipeline processing engine 36 | 37 | 38 | %package devel 39 | Summary: Development files for %{name} 40 | Group: Development/Libraries 41 | Requires: %{name} = %{version}-%{release} 42 | 43 | %description devel 44 | Realtime pipeline processing engine (development files) 45 | 46 | 47 | %package components 48 | Summary: Grape queue and other components (cocaine apps) 49 | Group: Development/Libraries 50 | Requires: %{name} = %{version}-%{release} 51 | 52 | %description components 53 | Grape queue and other component apps 54 | 55 | %package -n cocaine-plugin-queue-driver 56 | Summary: Grape queue driver (cocaine plugin) 57 | Group: Development/Libraries 58 | Requires: %{name} = %{version}-%{release} 59 | 60 | %description -n cocaine-plugin-queue-driver 61 | Grape queue driver runs as a cocaine plugin and can be turned on for applications, 62 | which want to pop events from persistent queue (application named 'queue') 63 | 64 | 65 | %prep 66 | %setup -q 67 | 68 | %build 69 | mkdir -p %{_target_platform} 70 | pushd %{_target_platform} 71 | 72 | %if %{defined rhel} && 0%{?rhel} < 6 73 | export CC=gcc44 74 | export CXX=g++44 75 | CXXFLAGS="-pthread -I/usr/include/boost141" LDFLAGS="-L/usr/lib64/boost141" %{cmake} -DBoost_LIB_DIR=/usr/lib64/boost141 -DBoost_INCLUDE_DIR=/usr/include/boost141 -DBoost_LIBRARYDIR=/usr/lib64/boost141 -DBOOST_LIBRARYDIR=/usr/lib64/boost141 -DCMAKE_CXX_COMPILER=g++44 -DCMAKE_C_COMPILER=gcc44 .. 76 | %else 77 | %{cmake} .. 78 | %endif 79 | 80 | popd 81 | 82 | make %{?_smp_mflags} -C %{_target_platform} 83 | 84 | %install 85 | rm -rf %{buildroot} 86 | make install DESTDIR=%{buildroot} -C %{_target_platform} 87 | 88 | %post -p /sbin/ldconfig 89 | %postun -p /sbin/ldconfig 90 | 91 | %clean 92 | rm -rf %{buildroot} 93 | 94 | 95 | %files 96 | %defattr(-,root,root,-) 97 | %{_libdir}/*grape*.so* 98 | %{_libdir}/grape/launchpad/* 99 | 100 | %files devel 101 | %defattr(-,root,root,-) 102 | %{_includedir}/grape/* 103 | %{_libdir}/*grape*.so 104 | 105 | %files components 106 | %defattr(-,root,root,-) 107 | %{_libdir}/queue 108 | %{_libdir}/testerhead-cpp 109 | 110 | %files -n cocaine-plugin-queue-driver 111 | %defattr(-,root,root,-) 112 | %{_libdir}/cocaine/queue-driver.cocaine-plugin 113 | 114 | 115 | # Primary place for the changelog is debian/changelog, 116 | # there is no tool to convert debian/changelog to rpm one and 117 | # it's silly to do that by hand -- so, no changelog at all 118 | #%changelog 119 | -------------------------------------------------------------------------------- /src/data_array/data_array.cpp: -------------------------------------------------------------------------------- 1 | #include "grape/data_array.hpp" 2 | 3 | namespace ioremap { namespace grape { 4 | 5 | void data_array::append(const char *data, size_t size, const entry_id &id) 6 | { 7 | size_t old_data_size = m_data.size(); 8 | 9 | try { 10 | m_data.insert(m_data.end(), data, data + size); 11 | m_size.push_back(size); 12 | m_id.push_back(id); 13 | } catch (...) { 14 | m_data.resize(old_data_size); 15 | throw; 16 | } 17 | } 18 | 19 | void data_array::append(const std::string &data, const entry_id &id) 20 | { 21 | append(data.data(), data.size(), id); 22 | } 23 | 24 | void data_array::append(const data_array::entry &entry) 25 | { 26 | append(entry.data, entry.size, entry.entry_id); 27 | } 28 | 29 | void data_array::extend(const data_array &d) 30 | { 31 | size_t old_data_size = m_data.size(); 32 | size_t old_sizes_size = m_size.size(); 33 | size_t old_ids_size = m_id.size(); 34 | 35 | try { 36 | m_data.insert(m_data.end(), d.data().begin(), d.data().end()); 37 | m_size.insert(m_size.end(), d.sizes().begin(), d.sizes().end()); 38 | m_id.insert(m_id.end(), d.ids().begin(), d.ids().end()); 39 | } catch (...) { 40 | m_data.resize(old_data_size); 41 | m_size.resize(old_sizes_size); 42 | m_id.resize(old_ids_size); 43 | throw; 44 | } 45 | } 46 | 47 | const std::vector &data_array::ids(void) const 48 | { 49 | return m_id; 50 | } 51 | 52 | const std::vector &data_array::sizes(void) const 53 | { 54 | return m_size; 55 | } 56 | 57 | const std::string &data_array::data(void) const 58 | { 59 | return m_data; 60 | } 61 | 62 | bool data_array::empty(void) const 63 | { 64 | return m_size.empty(); 65 | } 66 | 67 | data_array::iterator::iterator(const data_array *array, bool at_end) 68 | : array(array), index(0), offset(0) 69 | { 70 | if (at_end) { 71 | index = array->sizes().size(); 72 | offset = array->data().size(); 73 | } 74 | } 75 | 76 | data_array::iterator::iterator(const iterator &other) 77 | : array(other.array), index(other.index), offset(other.offset) 78 | { 79 | } 80 | 81 | data_array::iterator &data_array::iterator::operator =(const iterator &other) 82 | { 83 | array = other.array; 84 | index = other.index; 85 | offset = other.offset; 86 | return *this; 87 | } 88 | 89 | bool data_array::iterator::operator ==(const iterator &other) const 90 | { 91 | return array == other.array && index == other.index && offset == other.offset; 92 | } 93 | 94 | bool data_array::iterator::operator !=(const iterator &other) const 95 | { 96 | return !operator ==(other); 97 | } 98 | 99 | void data_array::iterator::prepare_value() const 100 | { 101 | value.data = (const char *)array->data().data() + offset; 102 | value.size = array->sizes()[index]; 103 | value.entry_id = array->ids()[index]; 104 | } 105 | 106 | data_array::iterator::value_type data_array::iterator::operator *() const 107 | { 108 | prepare_value(); 109 | return value; 110 | } 111 | 112 | data_array::iterator::value_type *data_array::iterator::operator ->() const 113 | { 114 | prepare_value(); 115 | return &value; 116 | } 117 | 118 | data_array::iterator &data_array::iterator::operator ++() 119 | { 120 | int size = array->sizes()[index]; 121 | offset += size; 122 | ++index; 123 | return *this; 124 | } 125 | 126 | data_array::iterator data_array::iterator::operator ++(int) 127 | { 128 | iterator tmp = *this; 129 | operator ++(); 130 | return tmp; 131 | } 132 | 133 | data_array::iterator data_array::begin() const 134 | { 135 | return iterator(this, false); 136 | } 137 | 138 | data_array::iterator data_array::end() const 139 | { 140 | return iterator(this, true); 141 | } 142 | 143 | }} 144 | -------------------------------------------------------------------------------- /test/chunk.test.cpp: -------------------------------------------------------------------------------- 1 | // #include 2 | // #include 3 | // #include 4 | // #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include 13 | #include "src/queue/chunk.hpp" 14 | 15 | class ChunkMeta : public ::testing::Test { 16 | public: 17 | static const int meta_size = 100; 18 | ioremap::grape::chunk_meta meta; 19 | std::vector chunk_sizes; 20 | 21 | ChunkMeta() 22 | : meta(meta_size) 23 | {} 24 | 25 | virtual void SetUp() { 26 | extern void grape_queue_module_set_logger(std::shared_ptr); 27 | ioremap::elliptics::file_logger log("/dev/stderr", 0); 28 | grape_queue_module_set_logger(std::make_shared(log)); 29 | 30 | for (int i = 0; i < meta_size; ++i) { 31 | chunk_sizes.push_back( (rand() % 100) + 1 ); 32 | } 33 | } 34 | }; 35 | 36 | TEST_F(ChunkMeta, CheckConstructedMetaSize) { 37 | ASSERT_EQ(meta.data().size(), sizeof(ioremap::grape::chunk_disk) + meta_size * sizeof(ioremap::grape::chunk_entry)); 38 | } 39 | 40 | TEST_F(ChunkMeta, CheckMetaPushesCorrectSizes) { 41 | for (int i = 0; i < meta_size; ++i) { 42 | meta.push(chunk_sizes[i]); 43 | } 44 | 45 | for (int i = 0; i < meta_size; ++i) { 46 | ASSERT_EQ(meta[i].size, chunk_sizes[i]); 47 | } 48 | } 49 | 50 | TEST_F(ChunkMeta, CheckByteOffsetWorks) { 51 | for (int i = 0; i < meta_size; ++i) { 52 | meta.push(chunk_sizes[i]); 53 | } 54 | 55 | uint64_t offset = 0; 56 | for (int i = 0; i < meta_size; ++i) { 57 | ASSERT_EQ(offset, meta.byte_offset(i)); 58 | offset += meta[i].size; 59 | } 60 | } 61 | 62 | TEST_F(ChunkMeta, CheckPopThrowsOnEmptyMeta) { 63 | ASSERT_THROW(meta.pop(), ioremap::elliptics::error); 64 | } 65 | 66 | TEST_F(ChunkMeta, CheckMetaIsFull) { 67 | for (int i = 0; i < meta_size; ++i) { 68 | meta.push(chunk_sizes[i]); 69 | } 70 | ASSERT_TRUE(meta.full()); 71 | } 72 | 73 | TEST_F(ChunkMeta, CheckMetaIsExhausted) { 74 | for (int i = 0; i < meta_size; ++i) { 75 | meta.push(chunk_sizes[i]); 76 | } 77 | for (int i = 0; i < meta_size; ++i) { 78 | meta.pop(); 79 | } 80 | ASSERT_TRUE(meta.exhausted()); 81 | } 82 | 83 | TEST_F(ChunkMeta, CheckPopThrowsOnExhaustedMeta) { 84 | for (int i = 0; i < meta_size; ++i) { 85 | meta.push(chunk_sizes[i]); 86 | } 87 | for (int i = 0; i < meta_size; ++i) { 88 | meta.pop(); 89 | } 90 | 91 | ASSERT_THROW(meta.pop(), ioremap::elliptics::error); 92 | } 93 | 94 | TEST_F(ChunkMeta, CheckAckThrowsOnEmptyMeta) { 95 | ASSERT_THROW(meta.ack(0, ioremap::grape::chunk_entry::STATE_ACKED), ioremap::elliptics::error); 96 | } 97 | 98 | TEST_F(ChunkMeta, CheckAckThrowsOnNotPoppedMeta) { 99 | for (int i = 0; i < meta_size; ++i) { 100 | meta.push(chunk_sizes[i]); 101 | } 102 | 103 | ASSERT_THROW(meta.ack(0, ioremap::grape::chunk_entry::STATE_ACKED), ioremap::elliptics::error); 104 | } 105 | 106 | TEST_F(ChunkMeta, CheckMetaIsComplete) { 107 | for (int i = 0; i < meta_size; ++i) { 108 | meta.push(chunk_sizes[i]); 109 | } 110 | 111 | for (int i = 0; i < meta_size; ++i) { 112 | meta.pop(); 113 | } 114 | 115 | for (int i = 0; i < meta_size; ++i) { 116 | meta.ack(i, ioremap::grape::chunk_entry::STATE_ACKED); 117 | } 118 | 119 | ASSERT_TRUE(meta.complete()); 120 | } 121 | 122 | TEST_F(ChunkMeta, CheckAckNoThrowOnAckedMeta) { 123 | for (int i = 0; i < meta_size; ++i) { 124 | meta.push(chunk_sizes[i]); 125 | } 126 | 127 | for (int i = 0; i < meta_size; ++i) { 128 | meta.pop(); 129 | } 130 | 131 | for (int i = 0; i < meta_size; ++i) { 132 | meta.ack(i, ioremap::grape::chunk_entry::STATE_ACKED); 133 | } 134 | 135 | for (int i = 0; i < meta_size; ++i) { 136 | ASSERT_NO_THROW(meta.ack(i, ioremap::grape::chunk_entry::STATE_ACKED)); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/queue-pump/queue-pull.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | using namespace boost::program_options; 9 | using namespace ioremap::grape; 10 | 11 | int main(int argc, char** argv) 12 | { 13 | options_description generic("Generic options"); 14 | generic.add_options() 15 | ("help", "help message") 16 | ; 17 | 18 | options_description elliptics("Elliptics options"); 19 | elliptics.add_options() 20 | ("remote,r", value(), "remote addr to initially connect to") 21 | ("group,g", value>()->multitoken(), "group(s) to connect to") 22 | ("loglevel", value()->default_value(0), "client loglevel") 23 | ("net-thread-num", value()->default_value(0), "client 'net thread' pool size") 24 | ("io-thread-num", value()->default_value(0), "client 'io thread' pool size") 25 | ; 26 | 27 | options_description other("Options"); 28 | other.add_options() 29 | ("concurrency,n", value()->default_value(1), "concurrency limit") 30 | ("request-size,s", value()->default_value(100), "request size") 31 | ("limit,l", value()->default_value(0), "data pull limit") 32 | ("bulk-mode,b", "use bulk queue reader instead of queue reader") 33 | ; 34 | 35 | options_description opts; 36 | opts.add(generic).add(elliptics).add(other); 37 | 38 | parsed_options parsed_opts = parse_command_line(argc, argv, opts); 39 | variables_map args; 40 | store(parsed_opts, args); 41 | notify(args); 42 | 43 | if (args.count("help")) { 44 | std::cout << "Queue support utility." << "\n"; 45 | std::cout << opts << "\n"; 46 | return 1; 47 | } 48 | 49 | if (!args.count("remote")) 50 | { 51 | std::cerr << "--remote option required" << "\n"; 52 | return 1; 53 | } 54 | 55 | if (!args.count("group")) { 56 | std::cerr << "--group option required" << "\n"; 57 | return 1; 58 | } 59 | 60 | std::vector remotes; 61 | remotes.push_back(args["remote"].as()); 62 | 63 | std::vector groups = args["group"].as>(); 64 | 65 | std::string logfile = "/dev/stderr"; 66 | 67 | int loglevel = args["loglevel"].as(); 68 | int net_thread_num = args["net-thread-num"].as(); 69 | int io_thread_num = args["io-thread-num"].as(); 70 | 71 | int concurrency = args["concurrency"].as(); 72 | int request_size = args["request-size"].as();; 73 | int limit = args["limit"].as(); 74 | 75 | auto clientlib = elliptics_client_state::create( 76 | remotes, groups, logfile, loglevel, 77 | 0, 0, net_thread_num, io_thread_num 78 | ); 79 | 80 | const std::string queue_name("queue"); 81 | 82 | if (args.count("bulk-mode")) { 83 | int counter = 0; 84 | auto session = clientlib.create_session(); 85 | bulk_queue_reader pump(session, queue_name, request_size, concurrency); 86 | pump.run([&limit, &counter, &session, &queue_name] (ioremap::elliptics::exec_context context, ioremap::grape::data_array array) -> int { 87 | fprintf(stderr, "processing %ld entries\n", array.sizes().size()); 88 | if (limit > 0) { 89 | counter += array.sizes().size(); 90 | } 91 | auto result = bulk_queue_reader::REQUEST_CONTINUE; 92 | if (limit > 0 && counter >= limit) { 93 | result = bulk_queue_reader::REQUEST_STOP; 94 | } 95 | 96 | std::vector entries(array.begin(), array.end()); 97 | ioremap::grape::bulk_queue_reader::queue_ack(session, queue_name, context, entries); 98 | 99 | return result; 100 | }); 101 | 102 | if (limit > 0) { 103 | fprintf(stderr, "specified limit %d, actually read %d\n", limit, counter); 104 | } 105 | 106 | } else { 107 | // read queue indefinitely 108 | queue_reader pump(clientlib.create_session(), queue_name, request_size, concurrency); 109 | pump.run([] (ioremap::grape::entry_id entry_id, ioremap::elliptics::data_pointer data) -> bool { 110 | fprintf(stderr, "entry %d-%d, byte size %ld\n", entry_id.chunk, entry_id.pos, data.size()); 111 | return true; 112 | }); 113 | } 114 | 115 | return 0; 116 | } 117 | -------------------------------------------------------------------------------- /example/test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include 6 | #include 7 | 8 | using namespace ioremap::grape; 9 | 10 | extern "C" { 11 | void *initialize(const char *config, const size_t size); 12 | } 13 | 14 | class test_node0_t : public elliptics_node_t { 15 | public: 16 | test_node0_t(const std::string &config) : elliptics_node_t(config) {} 17 | 18 | void handle(struct sph *sph) { 19 | char *payload = (char *)(sph + 1); 20 | char *real_data = payload + sph->event_size; 21 | 22 | struct sph orig_sph = *sph; 23 | 24 | std::string event = xget_event(sph, payload); 25 | 26 | xlog(__LOG_NOTICE, "grape::test-node0: %s: data-size: %zd, event-size: %d: data: '%.*s'\n", 27 | event.c_str(), sph->data_size, sph->event_size, 28 | (int)sph->data_size, real_data); 29 | 30 | if (event == "test-app@event0") 31 | emit(orig_sph, "1", "test-app@event1", std::string(real_data, sph->data_size) + "1"); 32 | if (event == "test-app@event1") 33 | emit(orig_sph, "2", "test-app@event2", std::string(real_data, sph->data_size) + "2"); 34 | if (event == "test-app@event2") 35 | emit(orig_sph, "finish", "test-app@finish", std::string(real_data, sph->data_size) + "3"); 36 | 37 | if (event == "test-app@finish") { 38 | /* 39 | * Block waiting for execution of the second application in a row 40 | * This application has to be started already 41 | */ 42 | std::string second_result = emit_blocked("key", "test-app-second@event0", std::string(real_data, sph->data_size)); 43 | 44 | /* 45 | * Reply adds not only your data, but also the whole sph header to the original caller's waiting container 46 | */ 47 | reply(orig_sph, "test-app@finish", second_result, true); 48 | } 49 | } 50 | }; 51 | 52 | class test_node1_t : public elliptics_node_t { 53 | public: 54 | test_node1_t(const std::string &config) : elliptics_node_t(config) {} 55 | 56 | void handle(struct sph *sph) { 57 | char *payload = (char *)(sph + 1); 58 | char *real_data = payload + sph->event_size; 59 | 60 | struct sph orig_sph = *sph; 61 | 62 | std::string event = xget_event(sph, payload); 63 | 64 | xlog(__LOG_NOTICE, "grape::test-node1: %s: data-size: %zd, event-size: %d: data: '%.*s'\n", 65 | event.c_str(), sph->data_size, sph->event_size, 66 | (int)sph->data_size, real_data); 67 | 68 | std::string data = std::string(real_data, sph->data_size) + event + "|"; 69 | if (event == "test-app-second@event0") 70 | emit(orig_sph, "1", "test-app-second@event1", data); 71 | if (event == "test-app-second@event1") 72 | emit(orig_sph, "2", "test-app-second@event2", data); 73 | if (event == "test-app-second@event2") 74 | emit(orig_sph, "finish", "test-app-second@finish", data); 75 | 76 | if (event == "test-app-second@finish") { 77 | 78 | /* 79 | * Reply adds not only your data, but also the whole sph header to the original caller's waiting container 80 | */ 81 | reply(orig_sph, "test-app-second@finish", data, true); 82 | } 83 | } 84 | }; 85 | 86 | static void test_add_app0(topology_t *top, const std::string &cfg) 87 | { 88 | test_node0_t *node0 = new test_node0_t(cfg); 89 | 90 | top->add_slot("test-app@event0", node0); 91 | top->add_slot("test-app@event1", node0); 92 | top->add_slot("test-app@event2", node0); 93 | top->add_slot("test-app@finish", node0); 94 | } 95 | 96 | static void test_add_app1(topology_t *top, const std::string &cfg) 97 | { 98 | 99 | test_node1_t *node1 = new test_node1_t(cfg); 100 | 101 | top->add_slot("test-app-second@event0", node1); 102 | top->add_slot("test-app-second@event1", node1); 103 | top->add_slot("test-app-second@event2", node1); 104 | top->add_slot("test-app-second@finish", node1); 105 | } 106 | 107 | void *initialize(const char *config, const size_t size) 108 | { 109 | Json::Reader reader; 110 | Json::Value root; 111 | 112 | reader.parse(config, root); 113 | std::string cfg(config, size); 114 | 115 | /* 116 | * Everything below is a proof-of-concept code 117 | * Do not use it as C++ codying cook-book, 118 | * but rather properly code exception handling 119 | */ 120 | 121 | 122 | topology_t *top = new topology_t(root["log"].asString().c_str(), root["log-level"].asInt()); 123 | 124 | test_add_app0(top, cfg); 125 | test_add_app1(top, cfg); 126 | 127 | return (void *)top; 128 | } 129 | -------------------------------------------------------------------------------- /src/driver/driver.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013+ Ruslan Nigmatullin 3 | 4 | This file is part of Grape. 5 | 6 | Cocaine is free software; you can redistribute it and/or modify 7 | it under the terms of the GNU Lesser General Public License as published by 8 | the Free Software Foundation; either version 3 of the License, or 9 | (at your option) any later version. 10 | 11 | Cocaine is distributed in the hope that it will be useful, 12 | but WITHOUT ANY WARRANTY; without even the implied warranty of 13 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 | GNU Lesser General Public License for more details. 15 | 16 | You should have received a copy of the GNU Lesser General Public License 17 | along with this program. If not, see . 18 | */ 19 | 20 | #ifndef __GRAPE_QUEUE_DRIVER_HPP 21 | #define __GRAPE_QUEUE_DRIVER_HPP 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include "grape/elliptics_client_state.hpp" 35 | 36 | namespace cocaine { namespace driver { 37 | 38 | class queue_driver: public api::driver_t { 39 | public: 40 | typedef api::driver_t category_type; 41 | 42 | public: 43 | struct downstream_t: public cocaine::api::stream_t { 44 | downstream_t(queue_driver *parent); 45 | ~downstream_t(); 46 | 47 | virtual void write(const char *data, size_t size); 48 | virtual void error(int code, const std::string &message); 49 | virtual void close(); 50 | 51 | queue_driver *parent; 52 | uint64_t start_time; 53 | bool success; 54 | }; 55 | 56 | struct queue_request { 57 | int num; 58 | dnet_id id; 59 | int src_key; 60 | 61 | queue_request(void) : num(0), src_key(0) { 62 | memset(&id, 0, sizeof(dnet_id)); 63 | } 64 | }; 65 | 66 | queue_driver(context_t& context, io::reactor_t& reactor, app_t& app, const std::string& name, const Json::Value& args); 67 | 68 | virtual ~queue_driver(); 69 | 70 | virtual Json::Value info() const; 71 | 72 | void queue_dec(int num); 73 | void queue_inc(int num); 74 | 75 | void get_more_data(); 76 | 77 | private: 78 | cocaine::context_t& m_context; 79 | cocaine::app_t& m_app; 80 | std::shared_ptr m_log; 81 | 82 | elliptics_client_state m_client; 83 | std::vector m_queue_groups; 84 | int m_wait_timeout; 85 | int m_check_timeout; 86 | int m_request_size; 87 | double m_rate_upper_limit; 88 | double m_initial_rate_boost; 89 | 90 | void on_request_timer_event(ev::timer&, int); 91 | void on_rate_control_timer_event(ev::timer&, int); 92 | 93 | // request queue and callbacks 94 | void send_request(); 95 | void on_queue_request_data(std::shared_ptr req, const ioremap::elliptics::exec_result_entry &result); 96 | void on_queue_request_complete(std::shared_ptr req, const ioremap::elliptics::error_info &error); 97 | 98 | bool enqueue_data(const ioremap::elliptics::exec_context &context); 99 | void on_worker_complete(uint64_t start_time, bool success); 100 | 101 | private: 102 | // std::queue m_local_queue; 103 | // std::mutex m_local_queue_mutex; 104 | // std::mutex m_local_queue_processing_mutex; 105 | 106 | const std::string m_queue_name; 107 | std::string m_worker_event; 108 | std::string m_event_name; 109 | const std::string m_queue_pop_event; 110 | 111 | const double m_timeout; 112 | const double m_deadline; 113 | 114 | std::atomic_int m_queue_length; 115 | std::atomic_int m_queue_length_max; 116 | 117 | ev::timer m_request_timer; 118 | ev::timer m_rate_control_timer; 119 | 120 | double m_rate_focus_backoff; 121 | double m_request_speed_backoff; 122 | double m_delay_safe_interval; 123 | double m_initial_growth_time; 124 | double m_exponential_factor; 125 | 126 | std::atomic last_process_done_time; // in microseconds 127 | std::atomic processed_time; // in microseconds 128 | std::atomic process_count; 129 | std::atomic receive_count; 130 | std::atomic request_count; 131 | double rate_focus; 132 | int growth_step; 133 | double growth_time; 134 | 135 | uint64_t last_request_time; // in microseconds 136 | 137 | std::atomic_int m_queue_src_key; 138 | 139 | friend class downstream_t; 140 | }; 141 | 142 | }} 143 | 144 | #endif /* __GRAPE_QUEUE_DRIVER_HPP */ 145 | -------------------------------------------------------------------------------- /include/grape/rapidjson/internal/pow10.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_POW10_ 2 | #define RAPIDJSON_POW10_ 3 | 4 | namespace rapidjson { 5 | namespace internal { 6 | 7 | //! Computes integer powers of 10 in double (10.0^n). 8 | /*! This function uses lookup table for fast and accurate results. 9 | \param n positive/negative exponent. Must <= 308. 10 | \return 10.0^n 11 | */ 12 | inline double Pow10(int n) { 13 | static const double e[] = { // 1e-308...1e308: 617 * 8 bytes = 4936 bytes 14 | 1e-308,1e-307,1e-306,1e-305,1e-304,1e-303,1e-302,1e-301,1e-300, 15 | 1e-299,1e-298,1e-297,1e-296,1e-295,1e-294,1e-293,1e-292,1e-291,1e-290,1e-289,1e-288,1e-287,1e-286,1e-285,1e-284,1e-283,1e-282,1e-281,1e-280, 16 | 1e-279,1e-278,1e-277,1e-276,1e-275,1e-274,1e-273,1e-272,1e-271,1e-270,1e-269,1e-268,1e-267,1e-266,1e-265,1e-264,1e-263,1e-262,1e-261,1e-260, 17 | 1e-259,1e-258,1e-257,1e-256,1e-255,1e-254,1e-253,1e-252,1e-251,1e-250,1e-249,1e-248,1e-247,1e-246,1e-245,1e-244,1e-243,1e-242,1e-241,1e-240, 18 | 1e-239,1e-238,1e-237,1e-236,1e-235,1e-234,1e-233,1e-232,1e-231,1e-230,1e-229,1e-228,1e-227,1e-226,1e-225,1e-224,1e-223,1e-222,1e-221,1e-220, 19 | 1e-219,1e-218,1e-217,1e-216,1e-215,1e-214,1e-213,1e-212,1e-211,1e-210,1e-209,1e-208,1e-207,1e-206,1e-205,1e-204,1e-203,1e-202,1e-201,1e-200, 20 | 1e-199,1e-198,1e-197,1e-196,1e-195,1e-194,1e-193,1e-192,1e-191,1e-190,1e-189,1e-188,1e-187,1e-186,1e-185,1e-184,1e-183,1e-182,1e-181,1e-180, 21 | 1e-179,1e-178,1e-177,1e-176,1e-175,1e-174,1e-173,1e-172,1e-171,1e-170,1e-169,1e-168,1e-167,1e-166,1e-165,1e-164,1e-163,1e-162,1e-161,1e-160, 22 | 1e-159,1e-158,1e-157,1e-156,1e-155,1e-154,1e-153,1e-152,1e-151,1e-150,1e-149,1e-148,1e-147,1e-146,1e-145,1e-144,1e-143,1e-142,1e-141,1e-140, 23 | 1e-139,1e-138,1e-137,1e-136,1e-135,1e-134,1e-133,1e-132,1e-131,1e-130,1e-129,1e-128,1e-127,1e-126,1e-125,1e-124,1e-123,1e-122,1e-121,1e-120, 24 | 1e-119,1e-118,1e-117,1e-116,1e-115,1e-114,1e-113,1e-112,1e-111,1e-110,1e-109,1e-108,1e-107,1e-106,1e-105,1e-104,1e-103,1e-102,1e-101,1e-100, 25 | 1e-99, 1e-98, 1e-97, 1e-96, 1e-95, 1e-94, 1e-93, 1e-92, 1e-91, 1e-90, 1e-89, 1e-88, 1e-87, 1e-86, 1e-85, 1e-84, 1e-83, 1e-82, 1e-81, 1e-80, 26 | 1e-79, 1e-78, 1e-77, 1e-76, 1e-75, 1e-74, 1e-73, 1e-72, 1e-71, 1e-70, 1e-69, 1e-68, 1e-67, 1e-66, 1e-65, 1e-64, 1e-63, 1e-62, 1e-61, 1e-60, 27 | 1e-59, 1e-58, 1e-57, 1e-56, 1e-55, 1e-54, 1e-53, 1e-52, 1e-51, 1e-50, 1e-49, 1e-48, 1e-47, 1e-46, 1e-45, 1e-44, 1e-43, 1e-42, 1e-41, 1e-40, 28 | 1e-39, 1e-38, 1e-37, 1e-36, 1e-35, 1e-34, 1e-33, 1e-32, 1e-31, 1e-30, 1e-29, 1e-28, 1e-27, 1e-26, 1e-25, 1e-24, 1e-23, 1e-22, 1e-21, 1e-20, 29 | 1e-19, 1e-18, 1e-17, 1e-16, 1e-15, 1e-14, 1e-13, 1e-12, 1e-11, 1e-10, 1e-9, 1e-8, 1e-7, 1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1, 1e+0, 30 | 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 31 | 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 32 | 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 33 | 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 34 | 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 35 | 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, 36 | 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, 37 | 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, 38 | 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, 39 | 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, 40 | 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, 41 | 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, 42 | 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, 43 | 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, 44 | 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, 45 | 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 46 | }; 47 | RAPIDJSON_ASSERT(n <= 308); 48 | return n < -308 ? 0.0 : e[n + 308]; 49 | } 50 | 51 | } // namespace internal 52 | } // namespace rapidjson 53 | 54 | #endif // RAPIDJSON_POW10_ 55 | -------------------------------------------------------------------------------- /example/test_single.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | using namespace ioremap::grape; 13 | 14 | namespace { 15 | enum test_write_type { 16 | TEST_WRITE_NONE = 0, 17 | TEST_WRITE_RANDOM, 18 | }; 19 | } 20 | 21 | extern "C" { 22 | void *initialize(cocaine::logging::log_t *logger, const char *config, const size_t size); 23 | } 24 | 25 | class test_node0_t : public elliptics_node_t { 26 | public: 27 | test_node0_t(const std::string &config) : 28 | elliptics_node_t(config), m_wtype(TEST_WRITE_NONE), m_cflags(0), m_ioflags(0), m_event_num(0) { 29 | struct timeval tv; 30 | 31 | gettimeofday(&tv, NULL); 32 | srand(tv.tv_sec + tv.tv_usec); 33 | 34 | Json::Reader reader; 35 | Json::Value root; 36 | 37 | reader.parse(config, root); 38 | 39 | try { 40 | std::string write_type = root["write-type"].asString(); 41 | if (write_type == "random") 42 | m_wtype = TEST_WRITE_RANDOM; 43 | } catch (...) { 44 | } 45 | 46 | try { 47 | m_ioflags = root["ioflags"].asInt(); 48 | } catch (...) { 49 | } 50 | 51 | try { 52 | m_cflags = root["cflags"].asInt64(); 53 | } catch (...) { 54 | } 55 | 56 | Json::Value groups(root["groups"]); 57 | if (!groups.empty() && groups.isArray()) { 58 | std::transform(groups.begin(), groups.end(), std::back_inserter(m_groups), json_digitizer()); 59 | } 60 | 61 | m_event_base = root["event-base"].asString(); 62 | m_event_num = root["event-num"].asInt(); 63 | } 64 | 65 | void handle(struct sph *sph) { 66 | char *payload = (char *)(sph + 1); 67 | 68 | char *real_data = payload + sph->event_size; 69 | std::string data(real_data, sph->data_size); 70 | 71 | struct sph orig_sph = *sph; 72 | 73 | std::string event = xget_event(sph, payload); 74 | std::string emit_event; 75 | 76 | bool final = false; 77 | 78 | if (event == m_event_base + "-start") { 79 | emit_event = m_event_base + boost::lexical_cast(0); 80 | } else { 81 | int pos = 0; 82 | sscanf(event.c_str(), (m_event_base + "%d").c_str(), &pos); 83 | 84 | pos++; 85 | if (pos < m_event_num) 86 | emit_event = m_event_base + boost::lexical_cast(pos); 87 | else 88 | final = true; 89 | } 90 | 91 | io(); 92 | if (emit_event.size() > 0) { 93 | std::string rand_key = boost::lexical_cast(rand()) + "test-single"; 94 | emit(orig_sph, rand_key, emit_event, data); 95 | } 96 | 97 | if (final) { 98 | char date_str[64]; 99 | struct tm tm; 100 | struct timeval tv; 101 | 102 | gettimeofday(&tv, NULL); 103 | localtime_r((time_t *)&tv.tv_sec, &tm); 104 | strftime(date_str, sizeof(date_str), "%F %R:%S", &tm); 105 | 106 | std::ostringstream reply_data; 107 | reply_data << date_str << "." << tv.tv_usec << ": " << event << ": " << data; 108 | /* 109 | * Reply adds not only your data, but also the whole sph header to the original caller's waiting container 110 | */ 111 | reply(orig_sph, event, reply_data.str(), final); 112 | } 113 | 114 | xlog(__LOG_INFO, "%s: grape::test-node0: %s -> %s: data-size: %zd, final: %d\n", 115 | dnet_dump_id_str(sph->src.id), 116 | event.c_str(), emit_event.c_str(), sph->data_size, final); 117 | } 118 | 119 | private: 120 | std::vector m_groups; 121 | test_write_type m_wtype; 122 | uint64_t m_cflags; 123 | uint32_t m_ioflags; 124 | std::string m_event_base; 125 | int m_event_num; 126 | 127 | void io(void) { 128 | if (m_wtype == TEST_WRITE_NONE) 129 | return; 130 | 131 | if (m_groups.empty()) 132 | return; 133 | 134 | std::ostringstream key; 135 | key << rand(); 136 | 137 | std::string data; 138 | data.resize(100); 139 | 140 | ioremap::elliptics::session s(*m_node); 141 | 142 | s.set_groups(m_groups); 143 | s.set_ioflags(m_ioflags); 144 | s.set_cflags(m_cflags); 145 | 146 | try { 147 | s.read_data(key.str(), 0, 0); 148 | } catch (...) { 149 | } 150 | 151 | try { 152 | s.write_data(key.str(), data, 0); 153 | } catch (...) { 154 | } 155 | 156 | xlog(__LOG_NOTICE, "grape::test-node0::io: %s", key.str().c_str()); 157 | } 158 | }; 159 | 160 | void *initialize(cocaine::logging::log_t *logger, const char *config, const size_t size) 161 | { 162 | std::string cfg(config, size); 163 | 164 | Json::Reader reader; 165 | Json::Value root; 166 | 167 | reader.parse(config, root); 168 | 169 | /* 170 | * Everything below is a proof-of-concept code 171 | * Do not use it as C++ codying cook-book, 172 | * but rather properly code exception handling 173 | */ 174 | 175 | topology_t *top = new topology_t(root["log"].asString().c_str(), root["log-level"].asInt()); 176 | 177 | test_node0_t *node0 = new test_node0_t(cfg); 178 | std::string base = root["event-base"].asString(); 179 | int num = root["event-num"].asInt(); 180 | 181 | for (int i = 0; i < num; ++i) 182 | top->add_slot(base + boost::lexical_cast(i), node0); 183 | 184 | top->add_slot(base + "-start", node0); 185 | 186 | return (void *)top; 187 | } 188 | -------------------------------------------------------------------------------- /src/driver/cocaine-json-trait.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2011-2013 Andrey Sibiryov 3 | Copyright (c) 2011-2013 Other contributors as noted in the AUTHORS file. 4 | 5 | This file is part of Cocaine. 6 | 7 | Cocaine is free software; you can redistribute it and/or modify 8 | it under the terms of the GNU Lesser General Public License as published by 9 | the Free Software Foundation; either version 3 of the License, or 10 | (at your option) any later version. 11 | 12 | Cocaine is distributed in the hope that it will be useful, 13 | but WITHOUT ANY WARRANTY; without even the implied warranty of 14 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 15 | GNU Lesser General Public License for more details. 16 | 17 | You should have received a copy of the GNU Lesser General Public License 18 | along with this program. If not, see . 19 | */ 20 | 21 | #ifndef COCAINE_JSON_TYPE_TRAITS_HPP 22 | #define COCAINE_JSON_TYPE_TRAITS_HPP 23 | 24 | #include 25 | 26 | #include "cocaine/traits.hpp" 27 | 28 | namespace cocaine { namespace io { 29 | 30 | template<> 31 | struct type_traits { 32 | template 33 | static inline 34 | void 35 | pack(msgpack::packer& packer, const Json::Value& source) { 36 | switch(source.type()) { 37 | case Json::objectValue: { 38 | packer.pack_map(source.size()); 39 | 40 | Json::Value::Members keys(source.getMemberNames()); 41 | 42 | for(Json::Value::Members::const_iterator it = keys.begin(); 43 | it != keys.end(); 44 | ++it) 45 | { 46 | packer << *it; 47 | pack(packer, source[*it]); 48 | } 49 | 50 | break; 51 | } 52 | 53 | case Json::arrayValue: 54 | packer.pack_array(source.size()); 55 | 56 | for(Json::Value::const_iterator it = source.begin(); 57 | it != source.end(); 58 | ++it) 59 | { 60 | pack(packer, *it); 61 | } 62 | 63 | break; 64 | 65 | case Json::booleanValue: 66 | packer << source.asBool(); 67 | break; 68 | 69 | case Json::stringValue: 70 | packer << source.asString(); 71 | break; 72 | 73 | case Json::realValue: 74 | packer << source.asDouble(); 75 | break; 76 | 77 | case Json::intValue: 78 | packer << source.asLargestInt(); 79 | break; 80 | 81 | case Json::uintValue: 82 | packer << source.asLargestUInt(); 83 | break; 84 | 85 | case Json::nullValue: 86 | packer << msgpack::type::nil(); 87 | break; 88 | } 89 | } 90 | 91 | static inline 92 | void 93 | unpack(const msgpack::object& object, Json::Value& target) { 94 | switch(object.type) { 95 | case msgpack::type::MAP: { 96 | msgpack::object_kv *ptr = object.via.map.ptr, 97 | *const end = ptr + object.via.map.size; 98 | 99 | for(; ptr < end; ++ptr) { 100 | if(ptr->key.type != msgpack::type::RAW) { 101 | // NOTE: The keys should be strings, as the object 102 | // representation of the property maps is still a 103 | // JSON object. 104 | throw msgpack::type_error(); 105 | } 106 | 107 | unpack( 108 | ptr->val, 109 | target[ptr->key.as()] 110 | ); 111 | } 112 | 113 | break; 114 | } 115 | 116 | case msgpack::type::ARRAY: { 117 | msgpack::object *ptr = object.via.array.ptr, 118 | *const end = ptr + object.via.array.size; 119 | 120 | for(unsigned int index = 0; ptr < end; ++ptr, ++index) { 121 | unpack(*ptr, target[index]); 122 | } 123 | 124 | break; 125 | } 126 | 127 | case msgpack::type::RAW: 128 | target = object.as(); 129 | break; 130 | 131 | case msgpack::type::DOUBLE: 132 | target = object.as(); 133 | break; 134 | 135 | case msgpack::type::POSITIVE_INTEGER: 136 | target = static_cast(object.as()); 137 | break; 138 | 139 | case msgpack::type::NEGATIVE_INTEGER: 140 | target = static_cast(object.as()); 141 | break; 142 | 143 | case msgpack::type::BOOLEAN: 144 | target = object.as(); 145 | break; 146 | 147 | case msgpack::type::NIL: 148 | break; 149 | } 150 | } 151 | }; 152 | 153 | }} // namespace cocaine::io 154 | 155 | #endif 156 | -------------------------------------------------------------------------------- /launchpad/_queue_stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import json 5 | from itertools import imap, ifilter 6 | import elliptics 7 | from nodes import connect, node_id 8 | import prettytable 9 | 10 | class Error(Exception): 11 | pass 12 | 13 | def get_app_worker_count(s, key, app): 14 | result = s.exec_(key, event='%s@info' % (app)).get()[0] 15 | info = json.loads(str(result.context.data)) 16 | if 'slaves' in info: 17 | return info['slaves']['capacity'] 18 | else: 19 | raise Error(info['error']) 20 | 21 | #NOTE: manually sends separate commands to each node 22 | def app_worker_info(s, app): 23 | result = [] 24 | for addr, key in s.routes.addresses_with_id(): 25 | try: 26 | worker_count = get_app_worker_count(s, key, app) 27 | except Error, e: 28 | print 'ERROR: %s, %s' % (addr, e) 29 | worker_count = 1 30 | for worker in range(worker_count): 31 | result.append((addr, key, worker)) 32 | return result 33 | 34 | #NOTE: fanout sending using key=None 35 | #FIXME: still does not work: context.src_id is empty, address is broken 36 | #def app_worker_info(s, app): 37 | # result = [] 38 | # for i in s.exec_(None, '%s@info' % (app)).get(): 39 | # c = i.context 40 | # print i.address, c.src_id, c.src_key 41 | # info = json.loads(i.context.data) 42 | # worker_count = info['slaves']['capacity'] 43 | # for worker in range(worker_count): 44 | # result.append((i.context.address, i.context.src_id, worker)) 45 | # return result 46 | 47 | def exec_on_all_workers(s, event, data=None, ordered=True): 48 | app, command = event.split('@') 49 | 50 | worker_info = app_worker_info(s, app) 51 | if ordered: 52 | # retain existing order by worker number and add order by ip addresses 53 | worker_info = sorted(worker_info, key=lambda x: str(x[0]) + str(x[2])) 54 | 55 | asyncs = [] 56 | for addr, direct_id, worker in worker_info: 57 | asyncs.append((s.exec_(direct_id, src_key=worker, event=event, data=data or ''), addr, worker)) 58 | 59 | results = [] 60 | for async_result, addr, worker in asyncs: 61 | try: 62 | async_result.wait() 63 | rlist = async_result.get() 64 | if len(rlist) > 0: 65 | r = rlist[0] 66 | #print r.address, r.context.data 67 | results.append(r.context.data) 68 | else: 69 | print 'ERROR: %s, worker %d: no data returned' % (addr, worker) 70 | except elliptics.Error, e: 71 | print 'ERROR: %s, worker %d: %s' % (addr, worker, e) 72 | #results.append(None) 73 | 74 | return results 75 | 76 | def get_stats(s): 77 | return [json.loads(str(i)) for i in exec_on_all_workers(s, 'queue@stats')] 78 | 79 | def select_stats(stats, names): 80 | return [ifilter(lambda x: x[0] in names, i.iteritems()) for i in stats] 81 | 82 | def accumulate(*args): 83 | raw = [i[1] for i in args] 84 | return args[0][0], sum(raw), raw 85 | 86 | def summarize(stats): 87 | return stats and imap(accumulate, *stats) 88 | 89 | def avg(*args): 90 | raw = [i[1] for i in args] 91 | return args[0][0], sum(raw) / len(args), raw 92 | 93 | def average(stats): 94 | return stats and imap(avg, *stats) 95 | 96 | def pass_raw(stats): 97 | def none(*args): 98 | raw = [i[1] for i in args] 99 | return args[0][0], None, raw 100 | return stats and imap(none, *stats) 101 | 102 | 103 | if __name__ == '__main__': 104 | import argparse 105 | parser = argparse.ArgumentParser() 106 | parser.add_argument('-r', '--remote', help='address of seed node') 107 | parser.add_argument('-g', '--group', type=int, help='elliptics group') 108 | parser.add_argument('--loglevel', type=int, choices=xrange(5), default=1) 109 | args = parser.parse_args() 110 | 111 | session = connect([args.remote], [args.group], loglevel=args.loglevel) 112 | stats = get_stats(session) 113 | 114 | 115 | ATTRS = [ 116 | ('queue_id', pass_raw), 117 | ('high-id', pass_raw), 118 | ('low-id', pass_raw), 119 | ('push.count', summarize), 120 | ('pop.count', summarize), 121 | ('ack.count', summarize), 122 | ('timeout.count', summarize), 123 | ('push.rate', summarize), 124 | ('pop.rate', summarize), 125 | ('ack.rate', summarize), 126 | ('push.time', average), 127 | ('pop.time', average), 128 | ('ack.time', average), 129 | ] 130 | 131 | def print_attrs(stats, attrs): 132 | #x = prettytable.PrettyTable(['Name', 'Aggregate'] + [''] * len(stats[0][0][1])) 133 | #x.set_style(prettytable.PLAIN_COLUMN) 134 | #x.aligns = ['l', 'r', 'l'] 135 | firsttime = True 136 | x = None 137 | for name, proc in attrs: 138 | for name, acc, raw in proc(select_stats(stats, [name])): 139 | #print '%s\t= %r\t: %r' % (name, acc, raw) 140 | if firsttime: 141 | x = prettytable.PrettyTable(['Name', 'Aggregate'] + ['Sub'] * len(raw)) 142 | x.aligns = ['l', 'r'] + ['r'] * len(raw) 143 | firsttime = False 144 | x.add_row([name, acc] + raw) 145 | if x is not None: 146 | x.printt(border=False) 147 | 148 | print_attrs(stats, ATTRS) 149 | 150 | -------------------------------------------------------------------------------- /src/queue/chunk.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __CHUNK_HPP 2 | #define __CHUNK_HPP 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | namespace { 11 | inline uint64_t microseconds_now() { 12 | timespec t; 13 | clock_gettime(CLOCK_MONOTONIC_RAW, &t); 14 | return t.tv_sec * 1000000 + t.tv_nsec / 1000; 15 | } 16 | } 17 | 18 | namespace ioremap { namespace grape { 19 | 20 | struct chunk_entry { 21 | static const int STATE_ACKED = 1; 22 | 23 | int size; 24 | int state; 25 | }; 26 | 27 | struct chunk_disk { 28 | int max; // size 29 | int low; // indicies: low/high marks 30 | int high; 31 | int acked; 32 | struct chunk_entry entries[]; 33 | }; 34 | 35 | class chunk_meta { 36 | public: 37 | ELLIPTICS_DISABLE_COPY(chunk_meta); 38 | 39 | chunk_meta(int max); 40 | 41 | // Increases high mark. 42 | // Returns true when given chunk is full 43 | bool push(int size); 44 | // Increases low mark 45 | void pop(); 46 | // Marks entry at @pos position with @state state. 47 | // Returns true when given chunk is fully acked 48 | bool ack(int32_t pos, int state); 49 | 50 | std::string &data(); 51 | void assign(char *data, size_t size); 52 | 53 | int low_mark() const; 54 | int high_mark() const; 55 | int acked() const; 56 | 57 | bool full() const; 58 | bool exhausted() const; 59 | bool complete() const; 60 | 61 | chunk_entry operator[] (int32_t pos) const; 62 | uint64_t byte_offset(int32_t pos) const; 63 | 64 | private: 65 | std::string m_data; 66 | struct chunk_disk *m_ptr; 67 | }; 68 | 69 | struct iteration { 70 | uint64_t byte_offset; 71 | int entry_index; 72 | 73 | iteration() 74 | : byte_offset(0), entry_index(0) 75 | {} 76 | }; 77 | 78 | struct iterator { 79 | enum iteration_mode { 80 | REPLAY = 0, 81 | FORWARD, 82 | }; 83 | const iteration_mode mode; 84 | iteration &state; 85 | chunk_meta &meta; 86 | 87 | iterator(iteration_mode mode, iteration &state, chunk_meta &meta) 88 | : mode(mode), state(state), meta(meta) 89 | {} 90 | 91 | virtual void begin() = 0; 92 | virtual void advance() = 0; 93 | virtual bool at_end() = 0; 94 | }; 95 | 96 | struct forward_iterator : public iterator { 97 | 98 | forward_iterator(iteration &state, chunk_meta &meta) 99 | : iterator(FORWARD, state, meta) 100 | {} 101 | 102 | virtual void begin() { 103 | state.entry_index = meta.low_mark(); 104 | state.byte_offset = meta.byte_offset(state.entry_index); 105 | } 106 | virtual void advance() { 107 | int size = meta[state.entry_index].size; 108 | // advance low_mark 109 | meta.pop(); 110 | state.entry_index = meta.low_mark(); 111 | state.byte_offset += size; 112 | } 113 | virtual bool at_end() { 114 | return (state.entry_index >= meta.high_mark()); 115 | } 116 | }; 117 | 118 | struct replay_iterator : public iterator { 119 | 120 | replay_iterator(iteration &state, chunk_meta &meta) 121 | : iterator(REPLAY, state, meta) 122 | {} 123 | 124 | void step() { 125 | int size = meta[state.entry_index].size; 126 | ++state.entry_index; 127 | state.byte_offset += size; 128 | } 129 | void skip_acked() { 130 | while (!at_end() && meta[state.entry_index].state == chunk_entry::STATE_ACKED) { 131 | step(); 132 | } 133 | } 134 | 135 | virtual void begin() { 136 | state.byte_offset = 0; 137 | state.entry_index = 0; 138 | skip_acked(); 139 | } 140 | virtual void advance() { 141 | step(); 142 | skip_acked(); 143 | } 144 | virtual bool at_end() { 145 | return (state.entry_index >= meta.low_mark()); 146 | } 147 | }; 148 | 149 | struct chunk_stat { 150 | uint64_t write_data; 151 | uint64_t write_meta; 152 | uint64_t read; 153 | uint64_t remove; 154 | uint64_t push; 155 | uint64_t pop; 156 | uint64_t ack; 157 | }; 158 | 159 | class chunk { 160 | public: 161 | ELLIPTICS_DISABLE_COPY(chunk); 162 | 163 | chunk(elliptics::session &session, const std::string &queue_id, int chunk_id, int max); 164 | ~chunk(); 165 | 166 | bool load_meta(); 167 | void write_meta(); 168 | const chunk_meta &meta(); 169 | 170 | // single entry methods 171 | bool push(const elliptics::data_pointer &d); // returns true if chunk is full 172 | elliptics::data_pointer pop(int32_t *pos); 173 | bool ack(int32_t pos, bool write); 174 | 175 | // multiple entries methods 176 | data_array pop(int num); 177 | 178 | void reset_iteration(); 179 | bool expect_no_more(); 180 | 181 | void remove(); 182 | 183 | struct chunk_stat stat(void); 184 | void add(struct chunk_stat *st); 185 | 186 | int id() const; 187 | void reset_time(uint64_t timeout); 188 | uint64_t get_time(void); 189 | 190 | private: 191 | std::string m_traceid; 192 | int m_chunk_id; 193 | elliptics::key m_data_key; 194 | dnet_io_attr m_data_io; 195 | elliptics::key m_meta_key; 196 | dnet_io_attr m_meta_io; 197 | elliptics::session m_session_data; 198 | elliptics::session m_session_meta; 199 | 200 | struct chunk_stat m_stat; 201 | 202 | iteration iteration_state; 203 | std::unique_ptr iter; 204 | 205 | // whole chunk data is cached here 206 | // cache is being filled when ::pop is invoked and @m_pop_offset is >= than cache size 207 | elliptics::data_pointer m_data; 208 | 209 | chunk_meta m_meta; 210 | 211 | double m_fire_time; 212 | 213 | void reset_iteration_mode(); 214 | bool update_data_cache(); 215 | }; 216 | 217 | typedef std::shared_ptr shared_chunk; 218 | 219 | }} // namespace ioremap::grape 220 | 221 | #endif /* __CHUNK_HPP */ 222 | -------------------------------------------------------------------------------- /example/test_start.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | 5 | #include 6 | 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | namespace po = boost::program_options; 19 | namespace pt = boost::posix_time; 20 | using namespace ioremap; 21 | 22 | class starter : public grape::elliptics_node_t { 23 | public: 24 | starter(Json::Value &root, const std::string &config, const std::string &start_event, int thread_num, int request_num, 25 | bool run_io): 26 | elliptics_node_t(config), 27 | m_limit(request_num), 28 | m_jconf(root), 29 | m_start_event(start_event) 30 | { 31 | struct timeval tv; 32 | 33 | gettimeofday(&tv, NULL); 34 | srand(tv.tv_sec + tv.tv_usec); 35 | 36 | for (int i = 0; i < thread_num; ++i) 37 | m_tgroup.create_thread(boost::bind(run_io ? &starter::loop_io : &starter::loop, this)); 38 | } 39 | 40 | ~starter() { 41 | m_tgroup.join_all(); 42 | } 43 | 44 | void handle(struct sph *) { 45 | } 46 | 47 | private: 48 | boost::thread_group m_tgroup; 49 | boost::detail::atomic_count m_limit; 50 | std::string m_base_event; 51 | Json::Value m_jconf; 52 | std::string m_start_event; 53 | 54 | long gettid(void) { 55 | return syscall(SYS_gettid); 56 | } 57 | 58 | void loop(void) { 59 | elliptics::session s(*m_node); 60 | 61 | Json::Value groups(m_jconf["groups"]); 62 | if (!groups.empty() && groups.isArray()) { 63 | std::vector gr; 64 | std::transform(groups.begin(), groups.end(), std::back_inserter(gr), grape::json_digitizer()); 65 | s.set_groups(gr); 66 | } 67 | 68 | std::string binary; 69 | std::string data("Test data"); 70 | 71 | while (--m_limit >= 0) { 72 | std::string key = lexical_cast(rand()) + "starter"; 73 | struct dnet_id id; 74 | s.transform(key, id); 75 | id.group_id = 0; 76 | std::string reply = s.exec_unlocked(&id, m_start_event, data, binary); 77 | } 78 | } 79 | 80 | void loop_io(void) { 81 | elliptics::session s(*m_node); 82 | 83 | Json::Value groups(m_jconf["groups"]); 84 | if (!groups.empty() && groups.isArray()) { 85 | std::vector gr; 86 | std::transform(groups.begin(), groups.end(), std::back_inserter(gr), grape::json_digitizer()); 87 | s.set_groups(gr); 88 | } 89 | 90 | uint64_t ioflags = m_jconf["ioflags"].asInt(); 91 | s.set_ioflags(ioflags); 92 | 93 | std::string data("Test data"); 94 | std::string key = lexical_cast(rand()) + "starter"; 95 | 96 | s.write_data(key, data, 0); 97 | 98 | s.set_cflags(DNET_FLAGS_NOLOCK); 99 | while (--m_limit >= 0) { 100 | try { 101 | s.read_data(key, 0, 0); 102 | } catch (...) { 103 | } 104 | } 105 | } 106 | }; 107 | 108 | int main(int argc, char *argv[]) 109 | { 110 | int thread_num; 111 | long request_num; 112 | int log_level; 113 | int connection_num; 114 | std::string start_event; 115 | std::string mpath, log; 116 | 117 | po::options_description desc("Allowed options"); 118 | 119 | desc.add_options() 120 | ("help,h", "this help message") 121 | ("thread,t", po::value(&thread_num)->default_value(10), "number of threads") 122 | ("manifest,m", po::value(&mpath), "manifest file") 123 | ("log,l", po::value(&log), "log file") 124 | ("log-level,L", po::value(&log_level)->default_value(2), "log level") 125 | ("request,r", po::value(&request_num)->default_value(1000000)) 126 | ("start-event,s", po::value(&start_event), "starting event") 127 | ("connections,c", po::value(&connection_num)->default_value(1), "number of network connections") 128 | ("run-io,i", "run io test instead of exec") 129 | ; 130 | 131 | po::variables_map vm; 132 | po::store(po::parse_command_line(argc, argv, desc), vm); 133 | po::notify(vm); 134 | 135 | if (vm.count("help")) { 136 | std::cout << desc << std::endl; 137 | return -1; 138 | } 139 | 140 | if (!vm.count("manifest")) { 141 | std::cerr << "You must provide manifest path\n" << desc << std::endl; 142 | return -1; 143 | } 144 | 145 | if (!vm.count("start-event")) { 146 | std::cerr << "You must provide starting event path\n" << desc << std::endl; 147 | return -1; 148 | } 149 | 150 | bool run_io = vm.count("run-io") ? true : false; 151 | 152 | Json::Reader reader; 153 | Json::Value root; 154 | 155 | std::ifstream min(mpath.c_str()); 156 | reader.parse(min, root); 157 | 158 | if (vm.count("log")) 159 | root["args"]["config"]["log"] = log; 160 | if (vm.count("log-level")) 161 | root["args"]["config"]["log-level"] = log_level; 162 | 163 | grape::logger::instance()->init(root["args"]["config"]["log"].asString(), root["args"]["config"]["log-level"].asInt(), true); 164 | 165 | Json::FastWriter writer; 166 | std::string config = writer.write(root["args"]["config"]); 167 | 168 | pt::ptime time_start(pt::microsec_clock::local_time()); 169 | { 170 | std::vector > proc; 171 | 172 | for (int i = 0; i < connection_num; ++i) { 173 | boost::shared_ptr start(new starter(root["args"]["config"], config, 174 | start_event, thread_num, request_num / connection_num, run_io)); 175 | proc.push_back(start); 176 | } 177 | } 178 | pt::ptime time_end(pt::microsec_clock::local_time()); 179 | 180 | pt::time_duration duration(time_end - time_start); 181 | 182 | std::cout << "Total time: " << duration << ", rps: " << request_num * 1000000 / duration.total_microseconds() << std::endl; 183 | 184 | return 0; 185 | } 186 | -------------------------------------------------------------------------------- /test/chunk_iterator.test.cpp: -------------------------------------------------------------------------------- 1 | // #include 2 | // #include 3 | // #include 4 | // #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #include 13 | #include "src/queue/chunk.hpp" 14 | 15 | const int meta_size = 100; 16 | 17 | class ChunkIterator : public ::testing::Test { 18 | public: 19 | static const int max_entry_size = 1000; 20 | ioremap::grape::chunk_meta meta; 21 | ioremap::grape::iteration state; 22 | 23 | std::vector chunk_sizes; 24 | 25 | ChunkIterator() 26 | : meta(meta_size) 27 | {} 28 | 29 | virtual void SetUp() { 30 | extern void grape_queue_module_set_logger(std::shared_ptr); 31 | ioremap::elliptics::file_logger log("/dev/stderr", 0); 32 | grape_queue_module_set_logger(std::make_shared(log)); 33 | 34 | chunk_sizes.resize(meta_size); 35 | for (int i = 0; i < meta_size; ++i) { 36 | chunk_sizes[i] = (rand() % max_entry_size) + 1; 37 | } 38 | 39 | for (int i = 0; i < meta_size; ++i) { 40 | meta.push(chunk_sizes[i]); 41 | } 42 | } 43 | }; 44 | 45 | TEST_F(ChunkIterator, CheckForwardIteratorPassesMeta) { 46 | ioremap::grape::forward_iterator forward_iter(state, meta); 47 | 48 | for (forward_iter.begin(); !forward_iter.at_end(); forward_iter.advance()) { 49 | ASSERT_EQ(state.byte_offset, meta.byte_offset(state.entry_index)); 50 | ASSERT_EQ(chunk_sizes[state.entry_index], meta[state.entry_index].size); 51 | } 52 | 53 | ASSERT_EQ(state.entry_index, meta_size); 54 | } 55 | 56 | TEST_F(ChunkIterator, CheckForwardIteratorReusesState) { 57 | int passed_count = 0; 58 | 59 | while (true) { 60 | ioremap::grape::forward_iterator forward_iter(state, meta); 61 | forward_iter.begin(); 62 | 63 | ASSERT_EQ(passed_count, state.entry_index); 64 | 65 | if (forward_iter.at_end()) { 66 | ASSERT_EQ(state.entry_index, meta_size); 67 | break; 68 | } 69 | 70 | int skip_size = (rand() % (meta_size - state.entry_index)) + 1; 71 | for (int step = 0; step < skip_size; ++step) { 72 | ASSERT_EQ(chunk_sizes[passed_count + step], meta[state.entry_index].size); 73 | forward_iter.advance(); 74 | } 75 | 76 | passed_count += skip_size; 77 | } 78 | } 79 | 80 | TEST_F(ChunkIterator, CheckReplayIteratorIsAtEndOnEmptyMeta) { 81 | ioremap::grape::replay_iterator replay_iter(state, meta); 82 | 83 | ASSERT_EQ(state.entry_index, 0); 84 | ASSERT_TRUE(replay_iter.at_end()); 85 | } 86 | 87 | TEST_F(ChunkIterator, CheckReplayIteratorIsAtEndOnCompleteMeta) { 88 | for (int i = 0; i < meta_size; ++i) { 89 | meta.pop(); 90 | meta.ack(i, ioremap::grape::chunk_entry::STATE_ACKED); 91 | } 92 | 93 | ioremap::grape::replay_iterator replay_iter(state, meta); 94 | replay_iter.begin(); 95 | ASSERT_TRUE(replay_iter.at_end()); 96 | ASSERT_EQ(state.entry_index, meta_size); 97 | } 98 | 99 | TEST_F(ChunkIterator, CheckReplayIteratorStopsOnUnackedEntry) { 100 | const int acked_entry_count = meta_size / 2; 101 | const int nonacked_entry_count = 5; 102 | for (int i = 0; i < acked_entry_count; ++i) { 103 | meta.pop(); 104 | meta.ack(i, ioremap::grape::chunk_entry::STATE_ACKED); 105 | } 106 | for (int i = 0; i < nonacked_entry_count; ++i) { 107 | meta.pop(); 108 | } 109 | 110 | ioremap::grape::replay_iterator replay_iter(state, meta); 111 | replay_iter.begin(); 112 | ASSERT_EQ(state.entry_index, acked_entry_count); 113 | while (!replay_iter.at_end()) { 114 | replay_iter.advance(); 115 | } 116 | ASSERT_TRUE(replay_iter.at_end()); 117 | ASSERT_EQ(state.entry_index, acked_entry_count + nonacked_entry_count); 118 | } 119 | 120 | TEST_F(ChunkIterator, CheckReplayIteratorPassesExhausted) { 121 | for (int i = 0; i < meta_size; ++i) { 122 | meta.pop(); 123 | } 124 | 125 | ioremap::grape::replay_iterator replay_iter(state, meta); 126 | 127 | int passed_count = 0; 128 | replay_iter.begin(); 129 | while(!replay_iter.at_end()) { 130 | ASSERT_EQ(passed_count, state.entry_index); 131 | ASSERT_EQ(state.byte_offset, meta.byte_offset(state.entry_index)); 132 | ASSERT_EQ(chunk_sizes[passed_count], meta[state.entry_index].size); 133 | ++passed_count; 134 | replay_iter.advance(); 135 | } 136 | 137 | int expect_passed = meta_size; 138 | ASSERT_EQ(passed_count, expect_passed); 139 | 140 | ASSERT_EQ(state.entry_index, meta_size); 141 | ASSERT_EQ(state.entry_index, meta.low_mark()); 142 | } 143 | 144 | TEST_F(ChunkIterator, CheckForwardAndReplayIteratorsWorkTogether) { 145 | int passed_count = 0; 146 | 147 | while (true) { 148 | ioremap::grape::forward_iterator forward_iter(state, meta); 149 | forward_iter.begin(); 150 | 151 | ASSERT_EQ(passed_count, state.entry_index); 152 | 153 | if (forward_iter.at_end()) { 154 | break; 155 | } 156 | 157 | int skip_size = (rand() % (meta_size - state.entry_index)) + 1; 158 | 159 | // skip block (contains of next skip_size elements) 160 | for (int step = 0; step < skip_size; ++step) { 161 | ASSERT_EQ(passed_count + step, state.entry_index); 162 | ASSERT_EQ(chunk_sizes[passed_count + step], meta[state.entry_index].size); 163 | forward_iter.advance(); 164 | } 165 | 166 | // ack all but first element in the block 167 | for (int step = 1; step < skip_size; ++step) { 168 | meta.ack(passed_count + step, ioremap::grape::chunk_entry::STATE_ACKED); 169 | } 170 | 171 | ioremap::grape::replay_iterator replay_iter(state, meta); 172 | replay_iter.begin(); 173 | 174 | // replay_iter should point on first element in the block 175 | ASSERT_EQ(passed_count, state.entry_index); 176 | ASSERT_EQ(chunk_sizes[passed_count], meta[state.entry_index].size); 177 | 178 | replay_iter.advance(); 179 | 180 | // all other elements in the block have been acked 181 | ASSERT_TRUE(replay_iter.at_end()); 182 | 183 | // ack first element in the block 184 | meta.ack(passed_count, ioremap::grape::chunk_entry::STATE_ACKED); 185 | 186 | passed_count += skip_size; 187 | } 188 | 189 | // this way all elements have been acked 190 | int expect_passed = meta_size; 191 | ASSERT_EQ(expect_passed, passed_count); 192 | } 193 | 194 | -------------------------------------------------------------------------------- /include/grape/rapidjson/prettywriter.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_PRETTYWRITER_H_ 2 | #define RAPIDJSON_PRETTYWRITER_H_ 3 | 4 | #include "writer.h" 5 | 6 | namespace rapidjson { 7 | 8 | //! Writer with indentation and spacing. 9 | /*! 10 | \tparam OutputStream Type of ouptut os. 11 | \tparam Encoding Encoding of both source strings and output. 12 | \tparam Allocator Type of allocator for allocating memory of stack. 13 | */ 14 | template, typename TargetEncoding = UTF8<>, typename Allocator = MemoryPoolAllocator<> > 15 | class PrettyWriter : public Writer { 16 | public: 17 | typedef Writer Base; 18 | typedef typename Base::Ch Ch; 19 | 20 | //! Constructor 21 | /*! \param os Output os. 22 | \param allocator User supplied allocator. If it is null, it will create a private one. 23 | \param levelDepth Initial capacity of 24 | */ 25 | PrettyWriter(OutputStream& os, Allocator* allocator = 0, size_t levelDepth = Base::kDefaultLevelDepth) : 26 | Base(os, allocator, levelDepth), indentChar_(' '), indentCharCount_(4) {} 27 | 28 | //! Set custom indentation. 29 | /*! \param indentChar Character for indentation. Must be whitespace character (' ', '\t', '\n', '\r'). 30 | \param indentCharCount Number of indent characters for each indentation level. 31 | \note The default indentation is 4 spaces. 32 | */ 33 | PrettyWriter& SetIndent(Ch indentChar, unsigned indentCharCount) { 34 | RAPIDJSON_ASSERT(indentChar == ' ' || indentChar == '\t' || indentChar == '\n' || indentChar == '\r'); 35 | indentChar_ = indentChar; 36 | indentCharCount_ = indentCharCount; 37 | return *this; 38 | } 39 | 40 | //@name Implementation of Handler. 41 | //@{ 42 | 43 | PrettyWriter& Null() { PrettyPrefix(kNullType); Base::WriteNull(); return *this; } 44 | PrettyWriter& Bool(bool b) { PrettyPrefix(b ? kTrueType : kFalseType); Base::WriteBool(b); return *this; } 45 | PrettyWriter& Int(int i) { PrettyPrefix(kNumberType); Base::WriteInt(i); return *this; } 46 | PrettyWriter& Uint(unsigned u) { PrettyPrefix(kNumberType); Base::WriteUint(u); return *this; } 47 | PrettyWriter& Int64(int64_t i64) { PrettyPrefix(kNumberType); Base::WriteInt64(i64); return *this; } 48 | PrettyWriter& Uint64(uint64_t u64) { PrettyPrefix(kNumberType); Base::WriteUint64(u64); return *this; } 49 | PrettyWriter& Double(double d) { PrettyPrefix(kNumberType); Base::WriteDouble(d); return *this; } 50 | 51 | PrettyWriter& String(const Ch* str, SizeType length, bool copy = false) { 52 | (void)copy; 53 | PrettyPrefix(kStringType); 54 | Base::WriteString(str, length); 55 | return *this; 56 | } 57 | 58 | PrettyWriter& StartObject() { 59 | PrettyPrefix(kObjectType); 60 | new (Base::level_stack_.template Push()) typename Base::Level(false); 61 | Base::WriteStartObject(); 62 | return *this; 63 | } 64 | 65 | PrettyWriter& EndObject(SizeType memberCount = 0) { 66 | (void)memberCount; 67 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); 68 | RAPIDJSON_ASSERT(!Base::level_stack_.template Top()->inArray); 69 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 70 | 71 | if (!empty) { 72 | Base::os_.Put('\n'); 73 | WriteIndent(); 74 | } 75 | Base::WriteEndObject(); 76 | if (Base::level_stack_.Empty()) // end of json text 77 | Base::os_.Flush(); 78 | return *this; 79 | } 80 | 81 | PrettyWriter& StartArray() { 82 | PrettyPrefix(kArrayType); 83 | new (Base::level_stack_.template Push()) typename Base::Level(true); 84 | Base::WriteStartArray(); 85 | return *this; 86 | } 87 | 88 | PrettyWriter& EndArray(SizeType memberCount = 0) { 89 | (void)memberCount; 90 | RAPIDJSON_ASSERT(Base::level_stack_.GetSize() >= sizeof(typename Base::Level)); 91 | RAPIDJSON_ASSERT(Base::level_stack_.template Top()->inArray); 92 | bool empty = Base::level_stack_.template Pop(1)->valueCount == 0; 93 | 94 | if (!empty) { 95 | Base::os_.Put('\n'); 96 | WriteIndent(); 97 | } 98 | Base::WriteEndArray(); 99 | if (Base::level_stack_.Empty()) // end of json text 100 | Base::os_.Flush(); 101 | return *this; 102 | } 103 | 104 | //@} 105 | 106 | //! Simpler but slower overload. 107 | PrettyWriter& String(const Ch* str) { return String(str, internal::StrLen(str)); } 108 | 109 | protected: 110 | void PrettyPrefix(Type type) { 111 | (void)type; 112 | if (Base::level_stack_.GetSize() != 0) { // this value is not at root 113 | typename Base::Level* level = Base::level_stack_.template Top(); 114 | 115 | if (level->inArray) { 116 | if (level->valueCount > 0) { 117 | Base::os_.Put(','); // add comma if it is not the first element in array 118 | Base::os_.Put('\n'); 119 | } 120 | else 121 | Base::os_.Put('\n'); 122 | WriteIndent(); 123 | } 124 | else { // in object 125 | if (level->valueCount > 0) { 126 | if (level->valueCount % 2 == 0) { 127 | Base::os_.Put(','); 128 | Base::os_.Put('\n'); 129 | } 130 | else { 131 | Base::os_.Put(':'); 132 | Base::os_.Put(' '); 133 | } 134 | } 135 | else 136 | Base::os_.Put('\n'); 137 | 138 | if (level->valueCount % 2 == 0) 139 | WriteIndent(); 140 | } 141 | if (!level->inArray && level->valueCount % 2 == 0) 142 | RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name 143 | level->valueCount++; 144 | } 145 | else 146 | RAPIDJSON_ASSERT(type == kObjectType || type == kArrayType); 147 | } 148 | 149 | void WriteIndent() { 150 | size_t count = (Base::level_stack_.GetSize() / sizeof(typename Base::Level)) * indentCharCount_; 151 | PutN(Base::os_, indentChar_, count); 152 | } 153 | 154 | Ch indentChar_; 155 | unsigned indentCharCount_; 156 | }; 157 | 158 | } // namespace rapidjson 159 | 160 | #endif // RAPIDJSON_RAPIDJSON_H_ 161 | -------------------------------------------------------------------------------- /include/grape/elliptics_client_state.hpp: -------------------------------------------------------------------------------- 1 | #ifndef ELLIPTICS_CLIENT_STATE_HPP__ 2 | #define ELLIPTICS_CLIENT_STATE_HPP__ 3 | 4 | #include 5 | 6 | //XXX: work around rapidjson's filth 7 | #pragma GCC diagnostic push 8 | #pragma GCC diagnostic ignored "-Wpragmas" 9 | #pragma GCC diagnostic ignored "-Wunused-local-typedefs" 10 | #include "grape/rapidjson/document.h" 11 | #include "grape/rapidjson/prettywriter.h" 12 | #include "grape/rapidjson/stringbuffer.h" 13 | #include "grape/rapidjson/filestream.h" 14 | #pragma GCC diagnostic pop 15 | 16 | #include 17 | #include 18 | 19 | #include 20 | 21 | using namespace ioremap; 22 | 23 | class configuration_error : public elliptics::error 24 | { 25 | public: 26 | explicit configuration_error(const std::string &message) throw() 27 | : error(EINVAL, message) 28 | {} 29 | }; 30 | 31 | static inline void read_groups_array(std::vector *result, const char *name, const rapidjson::Value &value) { 32 | if (const auto *m = value.FindMember(name)) { 33 | if (m->value.IsArray()) { 34 | std::transform(m->value.Begin(), m->value.End(), 35 | std::back_inserter(*result), 36 | std::bind(&rapidjson::Value::GetInt, std::placeholders::_1) 37 | ); 38 | } else { 39 | std::ostringstream str; 40 | str << name << "value must be of array type"; 41 | throw configuration_error(str.str().c_str()); 42 | } 43 | } 44 | } 45 | 46 | struct elliptics_client_state { 47 | std::shared_ptr logger; 48 | std::shared_ptr node; 49 | std::vector groups; 50 | 51 | elliptics::session create_session() { 52 | elliptics::session session(*node); 53 | session.set_groups(groups); 54 | return session; 55 | } 56 | 57 | // Configure from preparsed json config. 58 | // Config template: 59 | // { 60 | // remotes: ["localhost:1025:2"], 61 | // groups: [2], 62 | // logfile: "/dev/stderr", 63 | // loglevel: 0 64 | // } 65 | static elliptics_client_state create(const rapidjson::Document &args) { 66 | std::string logfile = "/dev/stderr"; 67 | uint loglevel = DNET_LOG_ERROR; 68 | std::vector remotes; 69 | std::vector groups; 70 | int wait_timeout = 0; 71 | int check_timeout = 0; 72 | int net_thread_num = 0; 73 | int io_thread_num = 0; 74 | 75 | try { 76 | { 77 | const auto *m = args.FindMember("logfile"); 78 | if (m && m->value.IsString()) { 79 | logfile = m->value.GetString(); 80 | } 81 | } 82 | { 83 | const auto *m = args.FindMember("loglevel"); 84 | if (m && m->value.IsInt()) { 85 | loglevel = m->value.GetInt(); 86 | } 87 | } 88 | { 89 | const auto *m = args.FindMember("wait-timeout"); 90 | if (m && m->value.IsInt()) { 91 | wait_timeout = m->value.GetInt(); 92 | } 93 | } 94 | { 95 | const auto *m = args.FindMember("check-timeout"); 96 | if (m && m->value.IsInt()) { 97 | check_timeout = m->value.GetInt(); 98 | } 99 | } 100 | { 101 | const auto *m = args.FindMember("net-thread-num"); 102 | if (m && m->value.IsInt()) { 103 | net_thread_num = m->value.GetInt(); 104 | } 105 | } 106 | { 107 | const auto *m = args.FindMember("io-thread-num"); 108 | if (m && m->value.IsInt()) { 109 | io_thread_num = m->value.GetInt(); 110 | } 111 | } 112 | 113 | const rapidjson::Value &remotesArray = args["remotes"]; 114 | std::transform(remotesArray.Begin(), remotesArray.End(), 115 | std::back_inserter(remotes), 116 | std::bind(&rapidjson::Value::GetString, std::placeholders::_1) 117 | ); 118 | const rapidjson::Value &groupsArray = args["groups"]; 119 | std::transform(groupsArray.Begin(), groupsArray.End(), 120 | std::back_inserter(groups), 121 | std::bind(&rapidjson::Value::GetInt, std::placeholders::_1) 122 | ); 123 | } catch (const std::exception &e) { 124 | throw configuration_error(e.what()); 125 | } 126 | 127 | return create(remotes, groups, logfile, loglevel, wait_timeout, check_timeout, net_thread_num, io_thread_num); 128 | } 129 | 130 | static elliptics_client_state create(const std::string &conf, rapidjson::Document &doc) { 131 | FILE *cf; 132 | 133 | cf = fopen(conf.c_str(), "r"); 134 | if (!cf) { 135 | std::ostringstream str; 136 | str << "failed to open config file '" << conf << "'"; 137 | throw configuration_error(str.str().c_str()); 138 | } 139 | 140 | try { 141 | rapidjson::FileStream fs(cf); 142 | 143 | doc.ParseStream, rapidjson::FileStream>(fs); 144 | if (doc.HasParseError()) { 145 | std::ostringstream str; 146 | str << "can not parse config file '" << conf << "': " << doc.GetParseError(); 147 | throw configuration_error(str.str().c_str()); 148 | } 149 | 150 | fclose(cf); 151 | cf = NULL; 152 | 153 | return create(doc); 154 | } catch (...) { 155 | if (cf) 156 | fclose(cf); 157 | 158 | throw; 159 | } 160 | } 161 | 162 | static elliptics_client_state create(const std::vector &remotes, 163 | const std::vector &groups, const std::string &logfile, int loglevel, 164 | int wait_timeout = 0, int check_timeout = 0, 165 | int net_thread_num = 0, int io_thread_num = 0) { 166 | if (remotes.size() == 0) { 167 | throw configuration_error("no remotes have been specified"); 168 | } 169 | if (groups.size() == 0) { 170 | throw configuration_error("no groups have been specified"); 171 | } 172 | 173 | dnet_config cfg; 174 | memset(&cfg, 0, sizeof(cfg)); 175 | if (net_thread_num) { 176 | cfg.net_thread_num = net_thread_num; 177 | } 178 | if (io_thread_num) { 179 | cfg.io_thread_num = io_thread_num; 180 | } 181 | 182 | elliptics_client_state result; 183 | result.logger.reset(new elliptics::file_logger(logfile.c_str(), loglevel)); 184 | result.node.reset(new elliptics::node(*result.logger, cfg)); 185 | result.groups = groups; 186 | 187 | if (wait_timeout != 0 || check_timeout != 0) { 188 | // if unset, use default values as in node.c:dnet_node_create() 189 | wait_timeout = wait_timeout ? wait_timeout : 5; 190 | check_timeout = check_timeout ? check_timeout : DNET_DEFAULT_CHECK_TIMEOUT_SEC; 191 | result.node->set_timeouts(wait_timeout, check_timeout); 192 | } 193 | 194 | if (remotes.size() == 1) { 195 | // any error is fatal if there is a single remote address 196 | result.node->add_remote(remotes.front().c_str()); 197 | 198 | } else { 199 | // add_remote throws errors if: 200 | // * it can not parse address 201 | // * it can not connect to a specified address 202 | // * there is address duplication (NOTE: is this still true?) 203 | // In any case we ignore all errors in hope that at least one would suffice. 204 | int added = 0; 205 | for (const auto &i : remotes) { 206 | try { 207 | result.node->add_remote(i.c_str()); 208 | ++added; 209 | } catch (const elliptics::error &e) { 210 | char buf[1024]; 211 | 212 | snprintf(buf, sizeof(buf), "could not connect to: %s: %s\n", 213 | i.c_str(), e.what()); 214 | 215 | result.logger->log(DNET_LOG_ERROR, buf); 216 | } 217 | } 218 | if (added == 0) { 219 | throw configuration_error("no remotes were added successfully"); 220 | } 221 | } 222 | 223 | return result; 224 | } 225 | }; 226 | 227 | #endif // ELLIPTICS_CLIENT_STATE_HPP__ 228 | -------------------------------------------------------------------------------- /include/grape/rapidjson/writer.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_WRITER_H_ 2 | #define RAPIDJSON_WRITER_H_ 3 | 4 | #include "rapidjson.h" 5 | #include "internal/stack.h" 6 | #include "internal/strfunc.h" 7 | #include // snprintf() or _sprintf_s() 8 | #include // placement new 9 | 10 | #ifdef _MSC_VER 11 | #pragma warning(push) 12 | #pragma warning(disable : 4127) // conditional expression is constant 13 | #endif 14 | 15 | namespace rapidjson { 16 | 17 | //! JSON writer 18 | /*! Writer implements the concept Handler. 19 | It generates JSON text by events to an output os. 20 | 21 | User may programmatically calls the functions of a writer to generate JSON text. 22 | 23 | On the other side, a writer can also be passed to objects that generates events, 24 | 25 | for example Reader::Parse() and Document::Accept(). 26 | 27 | \tparam OutputStream Type of output stream. 28 | \tparam SourceEncoding Encoding of both source strings. 29 | \tparam TargetEncoding Encoding of and output stream. 30 | \implements Handler 31 | */ 32 | template, typename TargetEncoding = UTF8<>, typename Allocator = MemoryPoolAllocator<> > 33 | class Writer { 34 | public: 35 | typedef typename SourceEncoding::Ch Ch; 36 | 37 | Writer(OutputStream& os, Allocator* allocator = 0, size_t levelDepth = kDefaultLevelDepth) : 38 | os_(os), level_stack_(allocator, levelDepth * sizeof(Level)) {} 39 | 40 | //@name Implementation of Handler 41 | //@{ 42 | Writer& Null() { Prefix(kNullType); WriteNull(); return *this; } 43 | Writer& Bool(bool b) { Prefix(b ? kTrueType : kFalseType); WriteBool(b); return *this; } 44 | Writer& Int(int i) { Prefix(kNumberType); WriteInt(i); return *this; } 45 | Writer& Uint(unsigned u) { Prefix(kNumberType); WriteUint(u); return *this; } 46 | Writer& Int64(int64_t i64) { Prefix(kNumberType); WriteInt64(i64); return *this; } 47 | Writer& Uint64(uint64_t u64) { Prefix(kNumberType); WriteUint64(u64); return *this; } 48 | Writer& Double(double d) { Prefix(kNumberType); WriteDouble(d); return *this; } 49 | 50 | Writer& String(const Ch* str, SizeType length, bool copy = false) { 51 | (void)copy; 52 | Prefix(kStringType); 53 | WriteString(str, length); 54 | return *this; 55 | } 56 | 57 | Writer& StartObject() { 58 | Prefix(kObjectType); 59 | new (level_stack_.template Push()) Level(false); 60 | WriteStartObject(); 61 | return *this; 62 | } 63 | 64 | Writer& EndObject(SizeType memberCount = 0) { 65 | (void)memberCount; 66 | RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); 67 | RAPIDJSON_ASSERT(!level_stack_.template Top()->inArray); 68 | level_stack_.template Pop(1); 69 | WriteEndObject(); 70 | if (level_stack_.Empty()) // end of json text 71 | os_.Flush(); 72 | return *this; 73 | } 74 | 75 | Writer& StartArray() { 76 | Prefix(kArrayType); 77 | new (level_stack_.template Push()) Level(true); 78 | WriteStartArray(); 79 | return *this; 80 | } 81 | 82 | Writer& EndArray(SizeType elementCount = 0) { 83 | (void)elementCount; 84 | RAPIDJSON_ASSERT(level_stack_.GetSize() >= sizeof(Level)); 85 | RAPIDJSON_ASSERT(level_stack_.template Top()->inArray); 86 | level_stack_.template Pop(1); 87 | WriteEndArray(); 88 | if (level_stack_.Empty()) // end of json text 89 | os_.Flush(); 90 | return *this; 91 | } 92 | //@} 93 | 94 | //! Simpler but slower overload. 95 | Writer& String(const Ch* str) { return String(str, internal::StrLen(str)); } 96 | 97 | protected: 98 | //! Information for each nested level 99 | struct Level { 100 | Level(bool inArray_) : inArray(inArray_), valueCount(0) {} 101 | bool inArray; //!< true if in array, otherwise in object 102 | size_t valueCount; //!< number of values in this level 103 | }; 104 | 105 | static const size_t kDefaultLevelDepth = 32; 106 | 107 | void WriteNull() { 108 | os_.Put('n'); os_.Put('u'); os_.Put('l'); os_.Put('l'); 109 | } 110 | 111 | void WriteBool(bool b) { 112 | if (b) { 113 | os_.Put('t'); os_.Put('r'); os_.Put('u'); os_.Put('e'); 114 | } 115 | else { 116 | os_.Put('f'); os_.Put('a'); os_.Put('l'); os_.Put('s'); os_.Put('e'); 117 | } 118 | } 119 | 120 | void WriteInt(int i) { 121 | if (i < 0) { 122 | os_.Put('-'); 123 | i = -i; 124 | } 125 | WriteUint((unsigned)i); 126 | } 127 | 128 | void WriteUint(unsigned u) { 129 | char buffer[10]; 130 | char *p = buffer; 131 | do { 132 | *p++ = (u % 10) + '0'; 133 | u /= 10; 134 | } while (u > 0); 135 | 136 | do { 137 | --p; 138 | os_.Put(*p); 139 | } while (p != buffer); 140 | } 141 | 142 | void WriteInt64(int64_t i64) { 143 | if (i64 < 0) { 144 | os_.Put('-'); 145 | i64 = -i64; 146 | } 147 | WriteUint64((uint64_t)i64); 148 | } 149 | 150 | void WriteUint64(uint64_t u64) { 151 | char buffer[20]; 152 | char *p = buffer; 153 | do { 154 | *p++ = char(u64 % 10) + '0'; 155 | u64 /= 10; 156 | } while (u64 > 0); 157 | 158 | do { 159 | --p; 160 | os_.Put(*p); 161 | } while (p != buffer); 162 | } 163 | 164 | //! \todo Optimization with custom double-to-string converter. 165 | void WriteDouble(double d) { 166 | char buffer[100]; 167 | #if _MSC_VER 168 | int ret = sprintf_s(buffer, sizeof(buffer), "%g", d); 169 | #else 170 | int ret = snprintf(buffer, sizeof(buffer), "%g", d); 171 | #endif 172 | RAPIDJSON_ASSERT(ret >= 1); 173 | for (int i = 0; i < ret; i++) 174 | os_.Put(buffer[i]); 175 | } 176 | 177 | void WriteString(const Ch* str, SizeType length) { 178 | static const char hexDigits[16] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; 179 | static const char escape[256] = { 180 | #define Z16 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 181 | //0 1 2 3 4 5 6 7 8 9 A B C D E F 182 | 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'b', 't', 'n', 'u', 'f', 'r', 'u', 'u', // 00 183 | 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', 'u', // 10 184 | 0, 0, '"', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 20 185 | Z16, Z16, // 30~4F 186 | 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'\\', 0, 0, 0, // 50 187 | Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16, Z16 // 60~FF 188 | #undef Z16 189 | }; 190 | 191 | os_.Put('\"'); 192 | GenericStringStream is(str); 193 | while (is.Tell() < length) { 194 | const Ch c = is.Peek(); 195 | if ((sizeof(Ch) == 1 || (unsigned)c < 256) && escape[(unsigned char)c]) { 196 | is.Take(); 197 | os_.Put('\\'); 198 | os_.Put(escape[(unsigned char)c]); 199 | if (escape[(unsigned char)c] == 'u') { 200 | os_.Put('0'); 201 | os_.Put('0'); 202 | os_.Put(hexDigits[(unsigned char)c >> 4]); 203 | os_.Put(hexDigits[(unsigned char)c & 0xF]); 204 | } 205 | } 206 | else 207 | Transcoder::Transcode(is, os_); 208 | } 209 | os_.Put('\"'); 210 | } 211 | 212 | void WriteStartObject() { os_.Put('{'); } 213 | void WriteEndObject() { os_.Put('}'); } 214 | void WriteStartArray() { os_.Put('['); } 215 | void WriteEndArray() { os_.Put(']'); } 216 | 217 | void Prefix(Type type) { 218 | (void)type; 219 | if (level_stack_.GetSize() != 0) { // this value is not at root 220 | Level* level = level_stack_.template Top(); 221 | if (level->valueCount > 0) { 222 | if (level->inArray) 223 | os_.Put(','); // add comma if it is not the first element in array 224 | else // in object 225 | os_.Put((level->valueCount % 2 == 0) ? ',' : ':'); 226 | } 227 | if (!level->inArray && level->valueCount % 2 == 0) 228 | RAPIDJSON_ASSERT(type == kStringType); // if it's in object, then even number should be a name 229 | level->valueCount++; 230 | } 231 | else 232 | RAPIDJSON_ASSERT(type == kObjectType || type == kArrayType); 233 | } 234 | 235 | OutputStream& os_; 236 | internal::Stack level_stack_; 237 | 238 | private: 239 | // Prohibit assignment for VC C4512 warning 240 | Writer& operator=(const Writer& w); 241 | }; 242 | 243 | } // namespace rapidjson 244 | 245 | #ifdef _MSC_VER 246 | #pragma warning(pop) 247 | #endif 248 | 249 | #endif // RAPIDJSON_RAPIDJSON_H_ 250 | -------------------------------------------------------------------------------- /include/grape/rapidjson/rapidjson.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_RAPIDJSON_H_ 2 | #define RAPIDJSON_RAPIDJSON_H_ 3 | 4 | // Copyright (c) 2011 Milo Yip (miloyip@gmail.com) 5 | // Version 0.1 6 | 7 | #include // malloc(), realloc(), free() 8 | #include // memcpy() 9 | 10 | /////////////////////////////////////////////////////////////////////////////// 11 | // RAPIDJSON_NO_INT64DEFINE 12 | 13 | // Here defines int64_t and uint64_t types in global namespace. 14 | // If user have their own definition, can define RAPIDJSON_NO_INT64DEFINE to disable this. 15 | #ifndef RAPIDJSON_NO_INT64DEFINE 16 | #ifdef _MSC_VER 17 | typedef __int64 int64_t; 18 | typedef unsigned __int64 uint64_t; 19 | #define RAPIDJSON_FORCEINLINE __forceinline 20 | #else 21 | #include 22 | #define RAPIDJSON_FORCEINLINE 23 | #endif 24 | #endif // RAPIDJSON_NO_INT64TYPEDEF 25 | 26 | /////////////////////////////////////////////////////////////////////////////// 27 | // RAPIDJSON_ENDIAN 28 | #define RAPIDJSON_LITTLEENDIAN 0 //!< Little endian machine 29 | #define RAPIDJSON_BIGENDIAN 1 //!< Big endian machine 30 | 31 | //! Endianness of the machine. 32 | /*! GCC provided macro for detecting endianness of the target machine. But other 33 | compilers may not have this. User can define RAPIDJSON_ENDIAN to either 34 | RAPIDJSON_LITTLEENDIAN or RAPIDJSON_BIGENDIAN. 35 | */ 36 | #ifndef RAPIDJSON_ENDIAN 37 | #ifdef __BYTE_ORDER__ 38 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ 39 | #define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN 40 | #else 41 | #define RAPIDJSON_ENDIAN RAPIDJSON_BIGENDIAN 42 | #endif // __BYTE_ORDER__ 43 | #else 44 | #define RAPIDJSON_ENDIAN RAPIDJSON_LITTLEENDIAN // Assumes little endian otherwise. 45 | #endif 46 | #endif // RAPIDJSON_ENDIAN 47 | 48 | 49 | /////////////////////////////////////////////////////////////////////////////// 50 | // RAPIDJSON_ALIGNSIZE 51 | 52 | //! Data alignment of the machine. 53 | /*! 54 | Some machine requires strict data alignment. 55 | Currently the default uses 4 bytes alignment. User can customize this. 56 | */ 57 | #ifndef RAPIDJSON_ALIGN 58 | #define RAPIDJSON_ALIGN(x) ((x + 3) & ~3) 59 | #endif 60 | 61 | /////////////////////////////////////////////////////////////////////////////// 62 | // RAPIDJSON_SSE2/RAPIDJSON_SSE42/RAPIDJSON_SIMD 63 | 64 | // Enable SSE2 optimization. 65 | //#define RAPIDJSON_SSE2 66 | 67 | // Enable SSE4.2 optimization. 68 | //#define RAPIDJSON_SSE42 69 | 70 | #if defined(RAPIDJSON_SSE2) || defined(RAPIDJSON_SSE42) 71 | #define RAPIDJSON_SIMD 72 | #endif 73 | 74 | /////////////////////////////////////////////////////////////////////////////// 75 | // RAPIDJSON_NO_SIZETYPEDEFINE 76 | 77 | #ifndef RAPIDJSON_NO_SIZETYPEDEFINE 78 | namespace rapidjson { 79 | //! Use 32-bit array/string indices even for 64-bit platform, instead of using size_t. 80 | /*! User may override the SizeType by defining RAPIDJSON_NO_SIZETYPEDEFINE. 81 | */ 82 | typedef unsigned SizeType; 83 | } // namespace rapidjson 84 | #endif 85 | 86 | /////////////////////////////////////////////////////////////////////////////// 87 | // RAPIDJSON_ASSERT 88 | 89 | //! Assertion. 90 | /*! By default, rapidjson uses C assert() for assertion. 91 | User can override it by defining RAPIDJSON_ASSERT(x) macro. 92 | */ 93 | #ifndef RAPIDJSON_ASSERT 94 | #include 95 | #define RAPIDJSON_ASSERT(x) assert(x) 96 | #endif // RAPIDJSON_ASSERT 97 | 98 | /////////////////////////////////////////////////////////////////////////////// 99 | // RAPIDJSON_STATIC_ASSERT 100 | 101 | // Adopt from boost 102 | #ifndef RAPIDJSON_STATIC_ASSERT 103 | namespace rapidjson { 104 | template struct STATIC_ASSERTION_FAILURE; 105 | template <> struct STATIC_ASSERTION_FAILURE { enum { value = 1 }; }; 106 | template struct StaticAssertTest {}; 107 | } // namespace rapidjson 108 | 109 | #define RAPIDJSON_JOIN(X, Y) RAPIDJSON_DO_JOIN(X, Y) 110 | #define RAPIDJSON_DO_JOIN(X, Y) RAPIDJSON_DO_JOIN2(X, Y) 111 | #define RAPIDJSON_DO_JOIN2(X, Y) X##Y 112 | 113 | #define RAPIDJSON_STATIC_ASSERT(x) typedef ::rapidjson::StaticAssertTest<\ 114 | sizeof(::rapidjson::STATIC_ASSERTION_FAILURE)>\ 115 | RAPIDJSON_JOIN(StaticAssertTypedef, __LINE__) 116 | #endif 117 | 118 | /////////////////////////////////////////////////////////////////////////////// 119 | // Helpers 120 | 121 | #define RAPIDJSON_MULTILINEMACRO_BEGIN do { 122 | #define RAPIDJSON_MULTILINEMACRO_END \ 123 | } while((void)0, 0) 124 | 125 | /////////////////////////////////////////////////////////////////////////////// 126 | // Allocators and Encodings 127 | 128 | #include "allocators.h" 129 | #include "encodings.h" 130 | 131 | namespace rapidjson { 132 | 133 | /////////////////////////////////////////////////////////////////////////////// 134 | // Stream 135 | 136 | /*! \class rapidjson::Stream 137 | \brief Concept for reading and writing characters. 138 | 139 | For read-only stream, no need to implement PutBegin(), Put(), Flush() and PutEnd(). 140 | 141 | For write-only stream, only need to implement Put() and Flush(). 142 | 143 | \code 144 | concept Stream { 145 | typename Ch; //!< Character type of the stream. 146 | 147 | //! Read the current character from stream without moving the read cursor. 148 | Ch Peek() const; 149 | 150 | //! Read the current character from stream and moving the read cursor to next character. 151 | Ch Take(); 152 | 153 | //! Get the current read cursor. 154 | //! \return Number of characters read from start. 155 | size_t Tell(); 156 | 157 | //! Begin writing operation at the current read pointer. 158 | //! \return The begin writer pointer. 159 | Ch* PutBegin(); 160 | 161 | //! Write a character. 162 | void Put(Ch c); 163 | 164 | //! Flush the buffer. 165 | void Flush(); 166 | 167 | //! End the writing operation. 168 | //! \param begin The begin write pointer returned by PutBegin(). 169 | //! \return Number of characters written. 170 | size_t PutEnd(Ch* begin); 171 | } 172 | \endcode 173 | */ 174 | 175 | //! Put N copies of a character to a stream. 176 | template 177 | inline void PutN(Stream& stream, Ch c, size_t n) { 178 | for (size_t i = 0; i < n; i++) 179 | stream.Put(c); 180 | } 181 | 182 | /////////////////////////////////////////////////////////////////////////////// 183 | // StringStream 184 | 185 | //! Read-only string stream. 186 | /*! \implements Stream 187 | */ 188 | template 189 | struct GenericStringStream { 190 | typedef typename Encoding::Ch Ch; 191 | 192 | GenericStringStream(const Ch *src) : src_(src), head_(src) {} 193 | 194 | Ch Peek() const { return *src_; } 195 | Ch Take() { return *src_++; } 196 | size_t Tell() const { return src_ - head_; } 197 | 198 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 199 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 200 | void Flush() { RAPIDJSON_ASSERT(false); } 201 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 202 | 203 | const Ch* src_; //!< Current read position. 204 | const Ch* head_; //!< Original head of the string. 205 | }; 206 | 207 | typedef GenericStringStream > StringStream; 208 | 209 | /////////////////////////////////////////////////////////////////////////////// 210 | // InsituStringStream 211 | 212 | //! A read-write string stream. 213 | /*! This string stream is particularly designed for in-situ parsing. 214 | \implements Stream 215 | */ 216 | template 217 | struct GenericInsituStringStream { 218 | typedef typename Encoding::Ch Ch; 219 | 220 | GenericInsituStringStream(Ch *src) : src_(src), dst_(0), head_(src) {} 221 | 222 | // Read 223 | Ch Peek() { return *src_; } 224 | Ch Take() { return *src_++; } 225 | size_t Tell() { return src_ - head_; } 226 | 227 | // Write 228 | Ch* PutBegin() { return dst_ = src_; } 229 | void Put(Ch c) { RAPIDJSON_ASSERT(dst_ != 0); *dst_++ = c; } 230 | void Flush() {} 231 | size_t PutEnd(Ch* begin) { return dst_ - begin; } 232 | 233 | Ch* src_; 234 | Ch* dst_; 235 | Ch* head_; 236 | }; 237 | 238 | typedef GenericInsituStringStream > InsituStringStream; 239 | 240 | /////////////////////////////////////////////////////////////////////////////// 241 | // Type 242 | 243 | //! Type of JSON value 244 | enum Type { 245 | kNullType = 0, //!< null 246 | kFalseType = 1, //!< false 247 | kTrueType = 2, //!< true 248 | kObjectType = 3, //!< object 249 | kArrayType = 4, //!< array 250 | kStringType = 5, //!< string 251 | kNumberType = 6, //!< number 252 | }; 253 | 254 | } // namespace rapidjson 255 | 256 | #endif // RAPIDJSON_RAPIDJSON_H_ 257 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Grape is a stepping stone to building data processing systems on top of Elliptics routing and [server-side code execution](http://doc.reverbrain.com/elliptics:serverside). 2 | 3 | Its main goal is to provide an active example of elliptics data processing capabilities and also to provide ready-to-use building blocks for such systems. 4 | 5 | Grape, as for now, consist of 2 components: 6 | 7 | * fault-tolerant persistent queue 8 | * and a connector that allows to direct queue output into user application running on elliptics cluster (see [event driver concept](http://doc.reverbrain.com/stub:cocaine-event-driver) in Cocaine docs) 9 | 10 | ### Queue 11 | Queue is a [cocaine](https://github.com/cocaine/cocaine-core) application running on elliptics node. Its deployment process follows [general process](http://doc.reverbrain.com/stub:cocaine-app-deployment-process) for cocaine applications. 12 | 13 | Once deployed and started queue accepts data entries pushed into it, stores them among nodes of elliptics cluster its working on, and gives data entries back on consumer request, maintaining entries original order. 14 | 15 | Queue supports fault-tolerance by using data replication and by implementing fault-replay mechanics: consumer must acknowledge processing status of every data entry that it retrieved from the queue - failing to do so will result in entry "replay", over and over again up until it'll be confirmed. 16 | 17 | (For further details about how this works internally see [TODO: How queue works]().) 18 | 19 | #### API 20 | Queue's API basically consist of three methods: `push`, `peek`, `ack`: 21 | 22 | * `push` pushes data entry to the top of the queue 23 | * `peek` gets data entry from the bottom of the queue 24 | * `ack` confirms that entry has been processed and could be dropped 25 | 26 | (Latter two are combined in additional short-circuit method `pop`.) 27 | 28 | These methods are implemented in two sets: simple one operates in single queue entries and more complex one operates in multi-entry blocks. 29 | 30 | ##### queue.push 31 | ``` 32 | dnet_id key; 33 | session->exec(&key, "queue@push", ioremap::elliptics::data_pointer::from_raw("abcd")).wait(); 34 | ``` 35 | Pushes data entry ("abcd") to the queue running under the base name "queue" at node responsible for the specified `dnet_id`. 36 | 37 | There is no multi-entry variant for this method. 38 | 39 | ##### queue.peek 40 | ``` 41 | dnet_id key; 42 | ioremap::elliptics::exec_context context = session->exec( 43 | &key, "queue@peek", ioremap::elliptics::data_pointer() 44 | ).get_one().context(); 45 | ioremap::elliptics::data_pointer entry_data = context.data(); 46 | ioremap::grape::entry_id entry_id = ioremap::grape::entry_id::from_dnet_raw_id(context.src_id()); 47 | ``` 48 | Peeks data entry from the queue running under the base name "queue" at node responsible for the specified `dnet_id`. 49 | 50 | Returns entry id embedded in `src_id` field of the response. Also returns queue's supplemental subid in the `src_key` field (that subid makes possible to acknowledge entry back and thus must be preserved). Both fields are accessible through `exec_context`. 51 | 52 | (Details of the [TODO: request and response fields](https://github.com/reverbrain/elliptics/blob/master/include/elliptics/srw.h#L30) of the exec command explained separately.) 53 | 54 | ##### queue.ack 55 | ``` 56 | session->exec(context, "queue@ack", ioremap::elliptics::data_pointer()).wait(); 57 | ``` 58 | or equivalent: 59 | ``` 60 | session->exec(context.src_id(), context.src_key(), "queue@ack", ioremap::elliptics::data_pointer()).wait(); 61 | ``` 62 | Acknowledges entry received by a previous `peek`. 63 | 64 | Entry id must be sent embedded in `dnet_id` of the request. `src_key` must be set to that received by a previous `peek`. 65 | 66 | ##### queue.peek-multi 67 | ``` 68 | dnet_id key; 69 | ioremap::elliptics::exec_context context = session->exec( 70 | &key, "queue@peek-multi", ioremap::elliptics::data_pointer("100") 71 | ).get_one().context(); 72 | auto array = ioremap::grape::deserialize(context.data()); 73 | ioremap::elliptics::data_pointer d = array.data(); 74 | size_t offset = 0; 75 | for (size_t i = 0; i < array.sizes().size(); ++i) { 76 | int bytesize = array.sizes()[i]; 77 | // process data: (d.data() + offset, bytesize) 78 | offset += bytesize; 79 | } 80 | ``` 81 | Peeks multiple data entries from the queue running under the base name "queue" at node responsible for the specified `dnet_id`. 82 | 83 | Peek-multi has an argument: hint about number of entries, which must be presented in a string form. 84 | 85 | Returns serialized `ioremap::grape::data_array` structure which holds entries' data packed into byte array and array with entries' byte sizes and array with entries' ids. 86 | 87 | `ioremap::grape::data_array` is declared in a header file `include/grape/data_array.hpp`. 88 | 89 | ##### queue.ack-multi 90 | ``` 91 | ioremap::grape::data_array array = ...; 92 | session->exec(context, "queue@ack-multi", ioremap::grape::serialize(array.ids())).wait(); 93 | ``` 94 | Acknowledges entries received by a previous `peek` (may be several). 95 | 96 | ##### queue.pop and queue.pop-multi 97 | Short circuit methods `pop` and `pop-multi` has a combined effect of `peek` and `ack` called in one go. They are simple to use but also lose acking and replaying properties. 98 | 99 | #### Additional methods 100 | Queue also implements few techical methods (in addition to common [TODO: Cocaine and Elliptics app managment]() capabilities): 101 | 102 | * `ping` can be used to see if queue is currently active (or activate it for that matter) 103 | * `stats` shows internal state and statistics queue gathers about itself 104 | 105 | #### Configuration 106 | 107 | Queue reads its configuration from the file `queue.conf`. This file must be included in deployment tarball along with an app executable (see following section on Deployment). 108 | 109 | `queue.conf` must contain configuration for the elliptics client (used to return replies on inbound events) and can include queue configuration options. 110 | 111 | There is only one configuration option for now: 112 | 113 | * `chunk-max-size` (int) - specifies how many entries will contain single chunk in the queue (default value: 10000) 114 | 115 | #### Deployment 116 | Deployment process of the queue follows [general process](http://doc.reverbrain.com/stub:cocaine-app-deployment-process) for cocaine applications. For launching the queue user needs three files: 117 | 118 | * `queue` application file (which is an executable) 119 | * `queue.conf` config file (which is also a manifest file) 120 | * `queue.profile` execution profile file 121 | 122 | `queue` app could be taken from the binary package `grape-components` or built from the sources. Config and profile files also exist both in source repository and included in the same package. 123 | 124 | Here we presume that user have installation of elliptics running on `localhost:1025` in group `2` (how to do it see [Elliptics: Server setup tutorial](http://doc.reverbrain.com/elliptics:server-tutorial)). 125 | 126 | `queue.conf` content: 127 | ``` 128 | { 129 | "type": "binary", 130 | "slave": "queue", 131 | 132 | "remotes": [ 133 | "localhost:1025:2" 134 | ], 135 | "groups": [2] 136 | } 137 | ``` 138 | 139 | `queue.profile` content: 140 | ``` 141 | { 142 | "heartbeat-timeout" : 60, 143 | "pool-limit" : 1, 144 | "queue-limit" : 1000, 145 | "grow-threshold" : 1, 146 | "concurrency" : 10, 147 | "idle-timeout": 0 148 | } 149 | ``` 150 | 151 | Steps to launch a queue: 152 | 153 | 1. Create tarball with queue executable and config files: 154 | 155 | ``` 156 | tar cvjf queue.tar.bz2 queue queue.conf 157 | ``` 158 | 159 | 2. Upload tarball, manifest (same as config) and profile 160 | 161 | ``` 162 | cocaine-tool -n queue -m queue.conf -p queue.tar.bz2 app:upload 163 | cocaine-tool -n queue -m queue.profile profile:upload 164 | ``` 165 | 166 | 3. Deploy the app (get it ready to run) 167 | 168 | ``` 169 | dnet_ioclient -r localhost:1025:2 -g 2 -c "queue@start-multiple-task local" 170 | ``` 171 | 172 | (More details about what these commands do exactly see in [TODO: Cocaine: application deployment]() and [TODO: Elliptics task management]().) 173 | 174 | Now queue is deployed (on every node that this elliptics installation includes, most possible that would be a single node here) and will actually start as soon as it'll receive its first command (or event). 175 | 176 | Activate the queue: 177 | ``` 178 | dnet_ioclient -r localhost:1025:2 -g 2 -c "queue@ping" 179 | ``` 180 | Queue is up and running if reply would be: 181 | ``` 182 | 127.0.0.1:1025: queue@ping "ok" 183 | ``` 184 | 185 | --- 186 | 187 | Links: 188 | 189 | * Elliptics: http://www.reverbrain.com/elliptics/ 190 | * Cocaine: https://github.com/cocaine/cocaine-core 191 | * Google group: https://groups.google.com/forum/?fromgroups=#!forum/reverbrain 192 | -------------------------------------------------------------------------------- /include/grape/rapidjson/allocators.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_ALLOCATORS_H_ 2 | #define RAPIDJSON_ALLOCATORS_H_ 3 | 4 | #include "rapidjson.h" 5 | 6 | namespace rapidjson { 7 | 8 | /////////////////////////////////////////////////////////////////////////////// 9 | // Allocator 10 | 11 | /*! \class rapidjson::Allocator 12 | \brief Concept for allocating, resizing and freeing memory block. 13 | 14 | Note that Malloc() and Realloc() are non-static but Free() is static. 15 | 16 | So if an allocator need to support Free(), it needs to put its pointer in 17 | the header of memory block. 18 | 19 | \code 20 | concept Allocator { 21 | static const bool kNeedFree; //!< Whether this allocator needs to call Free(). 22 | 23 | // Allocate a memory block. 24 | // \param size of the memory block in bytes. 25 | // \returns pointer to the memory block. 26 | void* Malloc(size_t size); 27 | 28 | // Resize a memory block. 29 | // \param originalPtr The pointer to current memory block. Null pointer is permitted. 30 | // \param originalSize The current size in bytes. (Design issue: since some allocator may not book-keep this, explicitly pass to it can save memory.) 31 | // \param newSize the new size in bytes. 32 | void* Realloc(void* originalPtr, size_t originalSize, size_t newSize); 33 | 34 | // Free a memory block. 35 | // \param pointer to the memory block. Null pointer is permitted. 36 | static void Free(void *ptr); 37 | }; 38 | \endcode 39 | */ 40 | 41 | /////////////////////////////////////////////////////////////////////////////// 42 | // CrtAllocator 43 | 44 | //! C-runtime library allocator. 45 | /*! This class is just wrapper for standard C library memory routines. 46 | \implements Allocator 47 | */ 48 | class CrtAllocator { 49 | public: 50 | static const bool kNeedFree = true; 51 | void* Malloc(size_t size) { return malloc(size); } 52 | void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { (void)originalSize; return realloc(originalPtr, newSize); } 53 | static void Free(void *ptr) { free(ptr); } 54 | }; 55 | 56 | /////////////////////////////////////////////////////////////////////////////// 57 | // MemoryPoolAllocator 58 | 59 | //! Default memory allocator used by the parser and DOM. 60 | /*! This allocator allocate memory blocks from pre-allocated memory chunks. 61 | 62 | It does not free memory blocks. And Realloc() only allocate new memory. 63 | 64 | The memory chunks are allocated by BaseAllocator, which is CrtAllocator by default. 65 | 66 | User may also supply a buffer as the first chunk. 67 | 68 | If the user-buffer is full then additional chunks are allocated by BaseAllocator. 69 | 70 | The user-buffer is not deallocated by this allocator. 71 | 72 | \tparam BaseAllocator the allocator type for allocating memory chunks. Default is CrtAllocator. 73 | \implements Allocator 74 | */ 75 | template 76 | class MemoryPoolAllocator { 77 | public: 78 | static const bool kNeedFree = false; //!< Tell users that no need to call Free() with this allocator. (concept Allocator) 79 | 80 | //! Constructor with chunkSize. 81 | /*! \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. 82 | \param baseAllocator The allocator for allocating memory chunks. 83 | */ 84 | MemoryPoolAllocator(size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : 85 | chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(0), baseAllocator_(baseAllocator), ownBaseAllocator_(0) 86 | { 87 | if (!baseAllocator_) 88 | ownBaseAllocator_ = baseAllocator_ = new BaseAllocator(); 89 | AddChunk(chunk_capacity_); 90 | } 91 | 92 | //! Constructor with user-supplied buffer. 93 | /*! The user buffer will be used firstly. When it is full, memory pool allocates new chunk with chunk size. 94 | 95 | The user buffer will not be deallocated when this allocator is destructed. 96 | 97 | \param buffer User supplied buffer. 98 | \param size Size of the buffer in bytes. It must at least larger than sizeof(ChunkHeader). 99 | \param chunkSize The size of memory chunk. The default is kDefaultChunkSize. 100 | \param baseAllocator The allocator for allocating memory chunks. 101 | */ 102 | MemoryPoolAllocator(char *buffer, size_t size, size_t chunkSize = kDefaultChunkCapacity, BaseAllocator* baseAllocator = 0) : 103 | chunkHead_(0), chunk_capacity_(chunkSize), userBuffer_(buffer), baseAllocator_(baseAllocator), ownBaseAllocator_(0) 104 | { 105 | RAPIDJSON_ASSERT(buffer != 0); 106 | RAPIDJSON_ASSERT(size > sizeof(ChunkHeader)); 107 | chunkHead_ = (ChunkHeader*)buffer; 108 | chunkHead_->capacity = size - sizeof(ChunkHeader); 109 | chunkHead_->size = 0; 110 | chunkHead_->next = 0; 111 | } 112 | 113 | //! Destructor. 114 | /*! This deallocates all memory chunks, excluding the user-supplied buffer. 115 | */ 116 | ~MemoryPoolAllocator() { 117 | Clear(); 118 | delete ownBaseAllocator_; 119 | } 120 | 121 | //! Deallocates all memory chunks, excluding the user-supplied buffer. 122 | void Clear() { 123 | while(chunkHead_ != 0 && chunkHead_ != (ChunkHeader *)userBuffer_) { 124 | ChunkHeader* next = chunkHead_->next; 125 | baseAllocator_->Free(chunkHead_); 126 | chunkHead_ = next; 127 | } 128 | } 129 | 130 | //! Computes the total capacity of allocated memory chunks. 131 | /*! \return total capacity in bytes. 132 | */ 133 | size_t Capacity() { 134 | size_t capacity = 0; 135 | for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) 136 | capacity += c->capacity; 137 | return capacity; 138 | } 139 | 140 | //! Computes the memory blocks allocated. 141 | /*! \return total used bytes. 142 | */ 143 | size_t Size() { 144 | size_t size = 0; 145 | for (ChunkHeader* c = chunkHead_; c != 0; c = c->next) 146 | size += c->size; 147 | return size; 148 | } 149 | 150 | //! Allocates a memory block. (concept Allocator) 151 | void* Malloc(size_t size) { 152 | size = RAPIDJSON_ALIGN(size); 153 | if (chunkHead_->size + size > chunkHead_->capacity) 154 | AddChunk(chunk_capacity_ > size ? chunk_capacity_ : size); 155 | 156 | char *buffer = (char *)(chunkHead_ + 1) + chunkHead_->size; 157 | chunkHead_->size += size; 158 | return buffer; 159 | } 160 | 161 | //! Resizes a memory block (concept Allocator) 162 | void* Realloc(void* originalPtr, size_t originalSize, size_t newSize) { 163 | if (originalPtr == 0) 164 | return Malloc(newSize); 165 | 166 | // Do not shrink if new size is smaller than original 167 | if (originalSize >= newSize) 168 | return originalPtr; 169 | 170 | // Simply expand it if it is the last allocation and there is sufficient space 171 | if (originalPtr == (char *)(chunkHead_ + 1) + chunkHead_->size - originalSize) { 172 | size_t increment = newSize - originalSize; 173 | increment = RAPIDJSON_ALIGN(increment); 174 | if (chunkHead_->size + increment <= chunkHead_->capacity) { 175 | chunkHead_->size += increment; 176 | return originalPtr; 177 | } 178 | } 179 | 180 | // Realloc process: allocate and copy memory, do not free original buffer. 181 | void* newBuffer = Malloc(newSize); 182 | RAPIDJSON_ASSERT(newBuffer != 0); // Do not handle out-of-memory explicitly. 183 | return memcpy(newBuffer, originalPtr, originalSize); 184 | } 185 | 186 | //! Frees a memory block (concept Allocator) 187 | static void Free(void *ptr) { (void)ptr; } // Do nothing 188 | 189 | private: 190 | //! Creates a new chunk. 191 | /*! \param capacity Capacity of the chunk in bytes. 192 | */ 193 | void AddChunk(size_t capacity) { 194 | ChunkHeader* chunk = (ChunkHeader*)baseAllocator_->Malloc(sizeof(ChunkHeader) + capacity); 195 | chunk->capacity = capacity; 196 | chunk->size = 0; 197 | chunk->next = chunkHead_; 198 | chunkHead_ = chunk; 199 | } 200 | 201 | static const int kDefaultChunkCapacity = 64 * 1024; //!< Default chunk capacity. 202 | 203 | //! Chunk header for perpending to each chunk. 204 | /*! Chunks are stored as a singly linked list. 205 | */ 206 | struct ChunkHeader { 207 | size_t capacity; //!< Capacity of the chunk in bytes (excluding the header itself). 208 | size_t size; //!< Current size of allocated memory in bytes. 209 | ChunkHeader *next; //!< Next chunk in the linked list. 210 | }; 211 | 212 | ChunkHeader *chunkHead_; //!< Head of the chunk linked-list. Only the head chunk serves allocation. 213 | size_t chunk_capacity_; //!< The minimum capacity of chunk when they are allocated. 214 | char *userBuffer_; //!< User supplied buffer. 215 | BaseAllocator* baseAllocator_; //!< base allocator for allocating memory chunks. 216 | BaseAllocator* ownBaseAllocator_; //!< base allocator created by this object. 217 | }; 218 | 219 | } // namespace rapidjson 220 | 221 | #endif // RAPIDJSON_ENCODINGS_H_ 222 | -------------------------------------------------------------------------------- /include/grape/rapidjson/encodedstream.h: -------------------------------------------------------------------------------- 1 | #ifndef RAPIDJSON_ENCODEDSTREAM_H_ 2 | #define RAPIDJSON_ENCODEDSTREAM_H_ 3 | 4 | #include "rapidjson.h" 5 | 6 | namespace rapidjson { 7 | 8 | //! Input byte stream wrapper with a statically bound encoding. 9 | /*! 10 | \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. 11 | \tparam InputByteStream Type of input byte stream. For example, FileReadStream. 12 | */ 13 | template 14 | class EncodedInputStream { 15 | RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); 16 | public: 17 | typedef typename Encoding::Ch Ch; 18 | 19 | EncodedInputStream(InputByteStream& is) : is_(is) { 20 | current_ = Encoding::TakeBOM(is_); 21 | } 22 | 23 | Ch Peek() const { return current_; } 24 | Ch Take() { Ch c = current_; current_ = Encoding::Take(is_); return c; } 25 | size_t Tell() const { return is_.Tell(); } 26 | 27 | // Not implemented 28 | void Put(Ch c) { RAPIDJSON_ASSERT(false); } 29 | void Flush() { RAPIDJSON_ASSERT(false); } 30 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 31 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 32 | 33 | private: 34 | // Prohibit assignment for VC C4512 warning 35 | EncodedInputStream& operator=(const EncodedInputStream&); 36 | 37 | InputByteStream& is_; 38 | Ch current_; 39 | }; 40 | 41 | //! Output byte stream wrapper with statically bound encoding. 42 | /*! 43 | \tparam Encoding The interpretation of encoding of the stream. Either UTF8, UTF16LE, UTF16BE, UTF32LE, UTF32BE. 44 | \tparam InputByteStream Type of input byte stream. For example, FileWriteStream. 45 | */ 46 | template 47 | class EncodedOutputStream { 48 | RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); 49 | public: 50 | typedef typename Encoding::Ch Ch; 51 | 52 | EncodedOutputStream(OutputByteStream& os, bool putBOM = true) : os_(os) { 53 | if (putBOM) 54 | Encoding::PutBOM(os_); 55 | } 56 | 57 | void Put(Ch c) { Encoding::Put(os_, c); } 58 | void Flush() { os_.Flush(); } 59 | 60 | // Not implemented 61 | Ch Peek() const { RAPIDJSON_ASSERT(false); } 62 | Ch Take() { RAPIDJSON_ASSERT(false); } 63 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 64 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 65 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 66 | 67 | private: 68 | // Prohibit assignment for VC C4512 warning 69 | EncodedOutputStream& operator=(const EncodedOutputStream&); 70 | 71 | OutputByteStream& os_; 72 | }; 73 | 74 | #define RAPIDJSON_ENCODINGS_FUNC(x) UTF8::x, UTF16LE::x, UTF16BE::x, UTF32LE::x, UTF32BE::x 75 | 76 | //! Input stream wrapper with dynamically bound encoding and automatic encoding detection. 77 | /*! 78 | \tparam CharType Type of character for reading. 79 | \tparam InputByteStream type of input byte stream to be wrapped. 80 | */ 81 | template 82 | class AutoUTFInputStream { 83 | RAPIDJSON_STATIC_ASSERT(sizeof(typename InputByteStream::Ch) == 1); 84 | public: 85 | typedef CharType Ch; 86 | 87 | //! Constructor. 88 | /*! 89 | \param is input stream to be wrapped. 90 | \param type UTF encoding type if it is not detected from the stream. 91 | */ 92 | AutoUTFInputStream(InputByteStream& is, UTFType type = kUTF8) : is_(&is), type_(type), hasBOM_(false) { 93 | DetectType(); 94 | static const TakeFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Take) }; 95 | takeFunc_ = f[type_]; 96 | current_ = takeFunc_(*is_); 97 | } 98 | 99 | UTFType GetType() const { return type_; } 100 | bool HasBOM() const { return hasBOM_; } 101 | 102 | Ch Peek() const { return current_; } 103 | Ch Take() { Ch c = current_; current_ = takeFunc_(*is_); return c; } 104 | size_t Tell() const { return is_->Tell(); } 105 | 106 | // Not implemented 107 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 108 | void Flush() { RAPIDJSON_ASSERT(false); } 109 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 110 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 111 | 112 | private: 113 | // Detect encoding type with BOM or RFC 4627 114 | void DetectType() { 115 | // BOM (Byte Order Mark): 116 | // 00 00 FE FF UTF-32BE 117 | // FF FE 00 00 UTF-32LE 118 | // FE FF UTF-16BE 119 | // FF FE UTF-16LE 120 | // EF BB BF UTF-8 121 | 122 | const unsigned char* c = (const unsigned char *)is_->Peek4(); 123 | if (!c) 124 | return; 125 | 126 | unsigned bom = c[0] | (c[1] << 8) | (c[2] << 16) | (c[3] << 24); 127 | hasBOM_ = false; 128 | if (bom == 0xFFFE0000) { type_ = kUTF32BE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } 129 | else if (bom == 0x0000FEFF) { type_ = kUTF32LE; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); is_->Take(); } 130 | else if ((bom & 0xFFFF) == 0xFFFE) { type_ = kUTF16BE; hasBOM_ = true; is_->Take(); is_->Take(); } 131 | else if ((bom & 0xFFFF) == 0xFEFF) { type_ = kUTF16LE; hasBOM_ = true; is_->Take(); is_->Take(); } 132 | else if ((bom & 0xFFFFFF) == 0xBFBBEF) { type_ = kUTF8; hasBOM_ = true; is_->Take(); is_->Take(); is_->Take(); } 133 | 134 | // RFC 4627: Section 3 135 | // "Since the first two characters of a JSON text will always be ASCII 136 | // characters [RFC0020], it is possible to determine whether an octet 137 | // stream is UTF-8, UTF-16 (BE or LE), or UTF-32 (BE or LE) by looking 138 | // at the pattern of nulls in the first four octets." 139 | // 00 00 00 xx UTF-32BE 140 | // 00 xx 00 xx UTF-16BE 141 | // xx 00 00 00 UTF-32LE 142 | // xx 00 xx 00 UTF-16LE 143 | // xx xx xx xx UTF-8 144 | 145 | if (!hasBOM_) { 146 | unsigned pattern = (c[0] ? 1 : 0) | (c[1] ? 2 : 0) | (c[2] ? 4 : 0) | (c[3] ? 8 : 0); 147 | switch (pattern) { 148 | case 0x08: type_ = kUTF32BE; break; 149 | case 0x0A: type_ = kUTF16BE; break; 150 | case 0x01: type_ = kUTF32LE; break; 151 | case 0x05: type_ = kUTF16LE; break; 152 | case 0x0F: type_ = kUTF8; break; 153 | } 154 | } 155 | 156 | // RUntime check whether the size of character type is sufficient. It only perform checks with assertion. 157 | switch (type_) { 158 | case kUTF8: 159 | // Do nothing 160 | break; 161 | case kUTF16LE: 162 | case kUTF16BE: 163 | RAPIDJSON_ASSERT(sizeof(Ch) >= 2); 164 | break; 165 | case kUTF32LE: 166 | case kUTF32BE: 167 | RAPIDJSON_ASSERT(sizeof(Ch) >= 4); 168 | break; 169 | } 170 | } 171 | 172 | typedef Ch (*TakeFunc)(InputByteStream& is); 173 | InputByteStream* is_; 174 | UTFType type_; 175 | Ch current_; 176 | TakeFunc takeFunc_; 177 | bool hasBOM_; 178 | }; 179 | 180 | //! Output stream wrapper with dynamically bound encoding and automatic encoding detection. 181 | /*! 182 | \tparam CharType Type of character for writing. 183 | \tparam InputByteStream type of output byte stream to be wrapped. 184 | */ 185 | template 186 | class AutoUTFOutputStream { 187 | RAPIDJSON_STATIC_ASSERT(sizeof(typename OutputByteStream::Ch) == 1); 188 | public: 189 | typedef CharType Ch; 190 | 191 | //! Constructor. 192 | /*! 193 | \param os output stream to be wrapped. 194 | \param type UTF encoding type. 195 | \param putBOM Whether to write BOM at the beginning of the stream. 196 | */ 197 | AutoUTFOutputStream(OutputByteStream& os, UTFType type, bool putBOM) : os_(&os), type_(type) { 198 | // RUntime check whether the size of character type is sufficient. It only perform checks with assertion. 199 | switch (type_) { 200 | case kUTF16LE: 201 | case kUTF16BE: 202 | RAPIDJSON_ASSERT(sizeof(Ch) >= 2); 203 | break; 204 | case kUTF32LE: 205 | case kUTF32BE: 206 | RAPIDJSON_ASSERT(sizeof(Ch) >= 4); 207 | break; 208 | case kUTF8: 209 | // Do nothing 210 | break; 211 | } 212 | 213 | static const PutFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(Put) }; 214 | putFunc_ = f[type_]; 215 | 216 | if (putBOM) 217 | PutBOM(); 218 | } 219 | 220 | UTFType GetType() const { return type_; } 221 | 222 | void Put(Ch c) { putFunc_(*os_, c); } 223 | void Flush() { os_->Flush(); } 224 | 225 | // Not implemented 226 | Ch Peek() const { RAPIDJSON_ASSERT(false); } 227 | Ch Take() { RAPIDJSON_ASSERT(false); } 228 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 229 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 230 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 231 | 232 | private: 233 | void PutBOM() { 234 | typedef void (*PutBOMFunc)(OutputByteStream&); 235 | static const PutBOMFunc f[] = { RAPIDJSON_ENCODINGS_FUNC(PutBOM) }; 236 | f[type_](*os_); 237 | } 238 | 239 | typedef void (*PutFunc)(OutputByteStream&, Ch); 240 | 241 | OutputByteStream* os_; 242 | UTFType type_; 243 | PutFunc putFunc_; 244 | }; 245 | 246 | #undef RAPIDJSON_ENCODINGS_FUNC 247 | 248 | } // namespace rapidjson 249 | 250 | #endif // RAPIDJSON_FILESTREAM_H_ 251 | -------------------------------------------------------------------------------- /src/queue-pump/queue-pump.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | struct request { 14 | dnet_id id; 15 | int src_key; 16 | 17 | request() : src_key(0) { 18 | memset(&id, 0, sizeof(dnet_id)); 19 | } 20 | request(int src_key) : src_key(src_key) { 21 | memset(&id, 0, sizeof(dnet_id)); 22 | } 23 | }; 24 | 25 | class queue_pump 26 | { 27 | public: 28 | typedef std::function processing_function; 29 | 30 | private: 31 | ioremap::elliptics::session client; 32 | const std::string queue_name; 33 | const int request_size; 34 | processing_function proc; 35 | 36 | std::atomic_int next_request_id; 37 | std::atomic_int running_requests; 38 | std::mutex mutex; 39 | std::condition_variable condition; 40 | 41 | public: 42 | queue_pump(ioremap::elliptics::session client, const std::string &queue_name, int request_size) 43 | : client(client) 44 | , queue_name(queue_name) 45 | , request_size(request_size) 46 | , next_request_id(0) 47 | , running_requests(0) 48 | {} 49 | 50 | void run(processing_function func) { 51 | proc = func; 52 | 53 | srand(time(NULL)); 54 | 55 | const int concurrency_limit = 1; 56 | 57 | while(1) { 58 | while(running_requests < concurrency_limit) { 59 | ++running_requests; 60 | queue_peek(client, next_request_id++, request_size); 61 | //fprintf(stderr, "%d running_requests\n", (int)running_requests); 62 | } 63 | std::unique_lock lock(mutex); 64 | condition.wait(lock, [this]{return running_requests < concurrency_limit;}); 65 | } 66 | } 67 | 68 | void queue_peek(ioremap::elliptics::session client, int req_unique_id, int arg) 69 | { 70 | client.set_exceptions_policy(ioremap::elliptics::session::no_exceptions); 71 | 72 | std::string queue_key = std::to_string(req_unique_id) + std::to_string(rand()); 73 | 74 | auto req = std::make_shared(req_unique_id); 75 | client.transform(queue_key, req->id); 76 | 77 | client.exec(&req->id, req->src_key, "queue@peek-multi", std::to_string(arg)) 78 | .connect( 79 | std::bind(&queue_pump::data_received, this, req, std::placeholders::_1), 80 | std::bind(&queue_pump::request_complete, this, req, std::placeholders::_1) 81 | ); 82 | } 83 | 84 | void queue_ack(ioremap::elliptics::session client, 85 | std::shared_ptr req, 86 | ioremap::elliptics::exec_context context, 87 | const std::vector &ids) 88 | { 89 | client.set_exceptions_policy(ioremap::elliptics::session::no_exceptions); 90 | 91 | size_t count = ids.size(); 92 | client.exec(context, "queue@ack-multi", ioremap::grape::serialize(ids)) 93 | .connect( 94 | ioremap::elliptics::async_result::result_function(), 95 | [req, count] (const ioremap::elliptics::error_info &error) { 96 | if (error) { 97 | fprintf(stderr, "%s %d, %ld entries not acked: %s\n", dnet_dump_id(&req->id), req->src_key, count, error.message().c_str()); 98 | } else { 99 | fprintf(stderr, "%s %d, %ld entries acked\n", dnet_dump_id(&req->id), req->src_key, count); 100 | } 101 | } 102 | ); 103 | } 104 | 105 | void data_received(std::shared_ptr req, const ioremap::elliptics::exec_result_entry &result) 106 | { 107 | if (result.error()) { 108 | fprintf(stderr, "%s %d: error: %s\n", dnet_dump_id(&req->id), req->src_key, result.error().message().c_str()); 109 | return; 110 | } 111 | 112 | ioremap::elliptics::exec_context context = result.context(); 113 | 114 | // queue.peek returns no data when queue is empty. 115 | if (context.data().empty()) { 116 | return; 117 | } 118 | 119 | // Received context must be used for acking to the same queue instance. 120 | // 121 | // But before that context.src_key must be restored back 122 | // to the original src_key used in the original request to the queue, 123 | // or else our worker's ack will not be routed to the exact same 124 | // queue worker that issued reply with this context. 125 | // 126 | // (src_key of the request gets replaced by job id server side, 127 | // so reply does not carries the same src_key as a request. 128 | // Which is unfortunate.) 129 | context.set_src_key(req->src_key); 130 | 131 | process_block(req, context); 132 | } 133 | 134 | void request_complete(std::shared_ptr req, const ioremap::elliptics::error_info &error) 135 | { 136 | if (error) { 137 | fprintf(stderr, "%s %d: queue request completion error: %s\n", dnet_dump_id(&req->id), req->src_key, error.message().c_str()); 138 | } else { 139 | //fprintf(stderr, "%s %d: queue request completed\n", dnet_dump_id(&req->id), req->src_key); 140 | } 141 | 142 | --running_requests; 143 | condition.notify_one(); 144 | } 145 | 146 | void process_block(std::shared_ptr req, const ioremap::elliptics::exec_context &context) 147 | { 148 | fprintf(stderr, "%s %d, received data, byte size %ld\n", 149 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 150 | context.data().size() 151 | ); 152 | 153 | auto array = ioremap::grape::deserialize(context.data()); 154 | auto d = ioremap::elliptics::data_pointer::from_raw(array.data()); 155 | size_t count = array.sizes().size(); 156 | 157 | // process entries 158 | fprintf(stderr, "%s %d, processing %ld entries\n", 159 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 160 | count 161 | ); 162 | 163 | size_t offset = 0; 164 | for (size_t i = 0; i < count; ++i) { 165 | const ioremap::grape::entry_id &entry_id = array.ids()[i]; 166 | int bytesize = array.sizes()[i]; 167 | 168 | //TODO: check result of the proc() 169 | proc(entry_id, d.slice(offset, bytesize)); 170 | 171 | offset += bytesize; 172 | } 173 | 174 | // acknowledge entries 175 | fprintf(stderr, "%s %d, acking %ld entries\n", 176 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 177 | count 178 | ); 179 | 180 | queue_ack(client, req, context, array.ids()); 181 | } 182 | 183 | }; 184 | 185 | using namespace boost::program_options; 186 | 187 | int main(int argc, char** argv) 188 | { 189 | options_description generic("Generic options"); 190 | generic.add_options() 191 | ("help", "help message") 192 | ; 193 | 194 | options_description elliptics("Elliptics options"); 195 | elliptics.add_options() 196 | ("remote,r", value(), "remote elliptics node addr to connect to") 197 | ("group,g", value>()->multitoken(), "group(s) to connect to") 198 | ("loglevel", value()->default_value(0), "elliptics node loglevel") 199 | ; 200 | 201 | /* options_description lowlevel("Queue data access and lowlevel actions"); 202 | lowlevel.add_options() 203 | ("id", value(), "id of the queue to work with") 204 | ("get-state", "find and show state record") 205 | ("get-block", value(), "find and show block record") 206 | //("get", value(), "get item at specified index") 207 | ; 208 | 209 | options_description highlevel("Queue API methods"); 210 | highlevel.add_options() 211 | ("new", "recreate queue before anything else") 212 | ("dump-state", "call queue.dump_state") 213 | ("push", value>()->implicit_value(std::vector(1, "abcdf"), "abcdf"), "call queue.push") 214 | //NOTE: construct below is a hack to allow multiple occurencies of --pop and without any args 215 | ("pop", value>()->zero_tokens(), "call queue.pop") 216 | ; 217 | */ 218 | options_description opts; 219 | opts.add(generic).add(elliptics);//.add(lowlevel).add(highlevel); 220 | 221 | parsed_options parsed_opts = parse_command_line(argc, argv, opts); 222 | variables_map args; 223 | store(parsed_opts, args); 224 | notify(args); 225 | 226 | if (args.count("help")) { 227 | std::cout << "Queue support utility." << "\n"; 228 | std::cout << opts << "\n"; 229 | return 1; 230 | } 231 | 232 | // if (!args.count("id")) { 233 | // std::cerr << "--id option required" << "\n"; 234 | // return 1; 235 | // } 236 | // int id(args["id"].as()); 237 | 238 | if (!args.count("remote")) 239 | { 240 | std::cerr << "--remote option required" << "\n"; 241 | return 1; 242 | } 243 | 244 | if (!args.count("group")) { 245 | std::cerr << "--group option required" << "\n"; 246 | return 1; 247 | } 248 | 249 | std::vector remotes; 250 | remotes.push_back(args["remote"].as()); 251 | 252 | std::vector groups = args["group"].as>(); 253 | 254 | std::string logfile = "/dev/stderr"; 255 | 256 | int loglevel = args["loglevel"].as(); 257 | 258 | auto clientlib = elliptics_client_state::create(remotes, groups, logfile, loglevel); 259 | 260 | const std::string queue_name("queue"); 261 | const int request_size = 100; 262 | 263 | // read queue indefinitely 264 | queue_pump pump(clientlib.create_session(), queue_name, request_size); 265 | pump.run([] (ioremap::grape::entry_id entry_id, ioremap::elliptics::data_pointer data) -> bool { 266 | fprintf(stderr, "entry %d-%d, byte size %ld\n", entry_id.chunk, entry_id.pos, data.size()); 267 | return true; 268 | }); 269 | 270 | return 0; 271 | } 272 | -------------------------------------------------------------------------------- /src/testerhead-cpp/app.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace ioremap::elliptics; 9 | 10 | class app_context { 11 | public: 12 | // proxy to the logging service 13 | std::shared_ptr m_log; 14 | 15 | // elliptics client generator 16 | elliptics_client_state _elliptics_client_state; 17 | 18 | // reply delay, in milliseconds 19 | int _delay; 20 | std::string _queue_ack_event; 21 | 22 | app_context(cocaine::framework::dispatch_t& dispatch); 23 | 24 | // void single_entry(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response); 25 | // void multi_entry(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response); 26 | void process(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response); 27 | }; 28 | 29 | app_context::app_context(cocaine::framework::dispatch_t& dispatch) { 30 | // obtain logging facility 31 | m_log = dispatch.service_manager()->get_system_logger(); 32 | COCAINE_LOG_INFO(m_log, "application start: %s", dispatch.id().c_str()); 33 | 34 | _delay = 0; 35 | _queue_ack_event = "queue@ack"; 36 | 37 | // configure 38 | //FIXME: replace this with config storage service when it's done 39 | { 40 | rapidjson::Document doc; 41 | _elliptics_client_state = elliptics_client_state::create("testerhead-cpp.conf", doc); 42 | 43 | if (doc.HasMember("delay")) { 44 | _delay = doc["delay"].GetInt() * 1000; 45 | } 46 | if (doc.HasMember("source-queue-ack-event")) { 47 | _queue_ack_event = doc["source-queue-ack-event"].GetString(); 48 | } 49 | } 50 | 51 | // register event handlers 52 | 53 | //XXX: it seems that "unregistered" entry has higher precedence over all 54 | // more specific ones: "unregistered" highjacks all events. 55 | // It's so counter intuitive, and most propably is a bug. 56 | // cocaine-framework-native version 0.10.5~prerelease~0. 57 | 58 | // on("single-entry", &app_context::single_entry); 59 | // on("multi-entry", &app_context::multi_entry); 60 | dispatch.on("testerhead-cpp@single-entry", this, &app_context::process); 61 | dispatch.on("testerhead-cpp@multi-entry", this, &app_context::process); 62 | } 63 | 64 | void app_context::process(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response) 65 | { 66 | session client = _elliptics_client_state.create_session(); 67 | 68 | exec_context context = exec_context::from_raw(chunks[0].c_str(), chunks[0].size()); 69 | 70 | // auto reply_ack = [&client, &context] () { 71 | // client.reply(context, std::string(), exec_context::final); 72 | // }; 73 | // auto reply_error = [&client, &context] (const char *msg) { 74 | // client.reply(context, std::string(msg), exec_context::final); 75 | // }; 76 | // auto reply = [&client, &context] (data_pointer d) { 77 | // client.reply(context, d, exec_context::final); 78 | // }; 79 | 80 | std::string app; 81 | std::string event; 82 | { 83 | char *p = strchr((char*)cocaine_event.c_str(), '@'); 84 | app.assign(cocaine_event.c_str(), p - cocaine_event.c_str()); 85 | event.assign(p + 1); 86 | } 87 | 88 | const std::string action_id = cocaine::format("%s %d", dnet_dump_id_str(context.src_id()->id), context.src_key()); 89 | 90 | COCAINE_LOG_INFO(m_log, "%s, event: %s, data-size: %ld", 91 | action_id.c_str(), 92 | event.c_str(), context.data().size() 93 | ); 94 | 95 | if (_delay) { 96 | usleep(_delay); 97 | } 98 | 99 | if (event == "single-entry") { 100 | ioremap::grape::entry_id id = ioremap::grape::entry_id::from_dnet_raw_id(context.src_id()); 101 | COCAINE_LOG_INFO(m_log, "received entry: %d-%d", id.chunk, id.pos); 102 | 103 | // acking success 104 | if (_queue_ack_event != "none") { 105 | client.set_exceptions_policy(session::no_exceptions); 106 | 107 | // dnet_id queue_id; 108 | // dnet_setup_id(&queue_id, 0, context.src_id()->id); 109 | 110 | COCAINE_LOG_INFO(m_log, "%s, acking entry %d-%d to queue %s", 111 | action_id.c_str(), id.chunk, id.pos, dnet_dump_id_str(context.src_id()->id)); 112 | 113 | client.exec(context, _queue_ack_event, data_pointer()).connect( 114 | async_result::result_function(), 115 | [this, action_id, id] (const error_info &error) { 116 | if (error) { 117 | COCAINE_LOG_ERROR(m_log, "%s, entry %d-%d not acked", 118 | action_id.c_str(), id.chunk, id.pos); 119 | } else { 120 | COCAINE_LOG_INFO(m_log, "%s, entry %d-%d acked", 121 | action_id.c_str(), id.chunk, id.pos); 122 | } 123 | } 124 | ); 125 | } 126 | 127 | } else if (event == "multi-entry") { 128 | // acking success 129 | if (_queue_ack_event != "none") { 130 | client.set_exceptions_policy(session::no_exceptions); 131 | 132 | auto d = ioremap::grape::deserialize(context.data()); 133 | size_t count = d.ids().size(); 134 | 135 | COCAINE_LOG_INFO(m_log, "%s, acking multi entry, size %ld, to queue %s", 136 | action_id.c_str(), count, dnet_dump_id_str(context.src_id()->id)); 137 | 138 | // send back only a vector with ids 139 | client.exec(context, _queue_ack_event, ioremap::grape::serialize(d.ids())).connect( 140 | async_result::result_function(), 141 | [this, action_id, count] (const error_info &error) { 142 | if (error) { 143 | COCAINE_LOG_ERROR(m_log, "%s, %ld entries not acked: %s", 144 | action_id.c_str(), count, error.message().c_str()); 145 | } else { 146 | COCAINE_LOG_INFO(m_log, "%s, %ld entries acked", action_id.c_str(), count); 147 | } 148 | } 149 | ); 150 | } 151 | } else if (event == "ping") { 152 | client.reply(context, std::string("ok"), ioremap::elliptics::exec_context::final); 153 | } 154 | } 155 | /* 156 | void app_context::single_entry(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response) 157 | { 158 | session client = _elliptics_client_state.create_session(); 159 | 160 | exec_context context = exec_context::from_raw(chunks[0].c_str(), chunks[0].size()); 161 | 162 | // auto reply_ack = [&client, &context] () { 163 | // client.reply(context, std::string(), exec_context::final); 164 | // }; 165 | // auto reply_error = [&client, &context] (const char *msg) { 166 | // client.reply(context, std::string(msg), exec_context::final); 167 | // }; 168 | auto reply = [&client, &context] (data_pointer d) { 169 | client.reply(context, d, exec_context::final); 170 | }; 171 | 172 | ioremap::grape::entry_id id = ioremap::grape::entry_id::from_dnet_raw_id(context.src_id()); 173 | COCAINE_LOG_INFO(m_log, "event: '%s', entry: %d-%d, data: '%s'", 174 | cocaine_event.c_str(), 175 | id.chunk, id.pos, 176 | context.data().to_string().c_str() 177 | ); 178 | 179 | if (_delay) { 180 | usleep(_delay); 181 | } 182 | 183 | // acking success 184 | // if (m_ack_on_success) { 185 | client.set_exceptions_policy(session::no_exceptions); 186 | 187 | dnet_id queue_id; 188 | dnet_setup_id(&queue_id, 0, context.src_id()->id); 189 | queue_id.type = 0; 190 | 191 | COCAINE_LOG_INFO(m_log, "acking entry %d-%d to queue %s", id.chunk, id.pos, dnet_dump_id_str(context.src_id()->id)); 192 | 193 | client.exec(&queue_id, _queue_ack_event, data_pointer()).connect( 194 | async_result::result_function(), 195 | [this, id] (const error_info &error) { 196 | if (error) { 197 | COCAINE_LOG_ERROR(m_log, "entry %d-%d not acked", id.chunk, id.pos); 198 | } else { 199 | COCAINE_LOG_INFO(m_log, "entry %d-%d acked", id.chunk, id.pos); 200 | } 201 | } 202 | ); 203 | // } 204 | } 205 | 206 | void app_context::multi_entry(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response) 207 | { 208 | session client = _elliptics_client_state.create_session(); 209 | 210 | exec_context context = exec_context::from_raw(chunks[0].c_str(), chunks[0].size()); 211 | 212 | // auto reply_ack = [&client, &context] () { 213 | // client.reply(context, std::string(), exec_context::final); 214 | // }; 215 | // auto reply_error = [&client, &context] (const char *msg) { 216 | // client.reply(context, std::string(msg), exec_context::final); 217 | // }; 218 | // auto reply = [&client, &context] (data_pointer d) { 219 | // client.reply(context, d, exec_context::final); 220 | // }; 221 | 222 | ioremap::grape::entry_id id = ioremap::grape::entry_id::from_dnet_raw_id(context.src_id()); 223 | COCAINE_LOG_INFO(m_log, "event: '%s', entry: %d-%d, data: '%s'", 224 | cocaine_event.c_str(), 225 | id.chunk, id.pos, 226 | context.data().to_string().c_str() 227 | ); 228 | 229 | if (_delay) { 230 | usleep(_delay); 231 | } 232 | 233 | // acking success 234 | // if (m_ack_on_success) { 235 | client.set_exceptions_policy(session::no_exceptions); 236 | 237 | dnet_id queue_id; 238 | dnet_setup_id(&queue_id, 0, context.src_id()->id); 239 | queue_id.type = 0; 240 | 241 | ioremap::grape::data_array d = ioremap::grape::data_array::deserialize(context.data()); 242 | size_t count = d.ids().size(); 243 | 244 | //FIXME: drop data, leave in the reply only ids 245 | client.exec(&queue_id, _queue_ack_event, context.data()).connect( 246 | async_result::result_function(), 247 | [this, count] (const error_info &error) { 248 | if (error) { 249 | COCAINE_LOG_ERROR(m_log, "%ld entries not acked", count); 250 | } else { 251 | COCAINE_LOG_INFO(m_log, "%ld entries acked", count); 252 | } 253 | } 254 | ); 255 | // } 256 | } 257 | */ 258 | int main(int argc, char **argv) 259 | { 260 | return cocaine::framework::run(argc, argv); 261 | } 262 | -------------------------------------------------------------------------------- /src/queue/app.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include 4 | #include 5 | #include 6 | 7 | #include "queue.hpp" 8 | 9 | namespace { 10 | 11 | // inline uint64_t microseconds_now() { 12 | // timespec t; 13 | // clock_gettime(CLOCK_MONOTONIC_RAW, &t); 14 | // return t.tv_sec * 1000000 + t.tv_nsec / 1000; 15 | // } 16 | 17 | template 18 | double modified_moving_average(double avg, double input) { 19 | avg -= avg / N; 20 | avg += input / N; 21 | return avg; 22 | } 23 | 24 | double exponential_moving_average(double avg, double input, double alpha) { 25 | return alpha * input + (1.0 - alpha) * avg; 26 | //return avg + alpha * (input - avg); 27 | } 28 | 29 | struct rate_stat 30 | { 31 | uint64_t last_update; // in microseconds 32 | double avg; 33 | 34 | rate_stat() : last_update(microseconds_now()), avg(0.0) {} 35 | 36 | void update(size_t num) { 37 | uint64_t now = microseconds_now(); 38 | double elapsed = double(now - last_update) / 1000000; // in seconds 39 | double alpha = (elapsed > 1.0) ? 1.0 : elapsed; 40 | avg = exponential_moving_average(avg, ((double)num / elapsed), alpha); 41 | last_update = now; 42 | } 43 | 44 | double get() { 45 | return avg; 46 | } 47 | }; 48 | 49 | struct time_stat 50 | { 51 | uint64_t start_time; // in microseconds 52 | double avg; 53 | 54 | time_stat() : avg(0.0) {} 55 | 56 | void start() { 57 | start_time = microseconds_now(); 58 | } 59 | void stop() { 60 | avg = modified_moving_average<19>(avg, microseconds_now() - start_time); 61 | } 62 | 63 | double get() { 64 | return avg; 65 | } 66 | }; 67 | 68 | } 69 | 70 | class queue_app_context { 71 | public: 72 | queue_app_context(cocaine::framework::dispatch_t& dispatch); 73 | virtual ~queue_app_context(); 74 | 75 | void process(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response); 76 | 77 | private: 78 | typedef ioremap::grape::data_array peek_multi_type; 79 | typedef std::vector ack_multi_type; 80 | 81 | std::string m_id; 82 | std::shared_ptr m_log; 83 | std::shared_ptr m_queue; 84 | 85 | rate_stat m_push_rate; 86 | rate_stat m_pop_rate; 87 | rate_stat m_ack_rate; 88 | 89 | time_stat m_push_time; 90 | time_stat m_pop_time; 91 | time_stat m_ack_time; 92 | }; 93 | 94 | queue_app_context::queue_app_context(cocaine::framework::dispatch_t& dispatch) 95 | : m_id(dispatch.id()) 96 | , m_log(dispatch.service_manager()->get_system_logger()) 97 | { 98 | //FIXME: pass logger explicitly everywhere 99 | extern void grape_queue_module_set_logger(std::shared_ptr); 100 | grape_queue_module_set_logger(m_log); 101 | 102 | m_queue.reset(new ioremap::grape::queue(m_id)); 103 | m_queue->initialize("queue.conf"); 104 | COCAINE_LOG_INFO(m_log, "%s: queue has been successfully configured", m_id.c_str()); 105 | 106 | // register event handlers 107 | dispatch.on("ping", this, &queue_app_context::process); 108 | dispatch.on("push", this, &queue_app_context::process); 109 | dispatch.on("pop-multi", this, &queue_app_context::process); 110 | dispatch.on("pop-multiple-string", this, &queue_app_context::process); 111 | dispatch.on("pop", this, &queue_app_context::process); 112 | dispatch.on("peek", this, &queue_app_context::process); 113 | dispatch.on("peek-multi", this, &queue_app_context::process); 114 | dispatch.on("ack", this, &queue_app_context::process); 115 | dispatch.on("ack-multi", this, &queue_app_context::process); 116 | dispatch.on("clear", this, &queue_app_context::process); 117 | dispatch.on("stats-clear", this, &queue_app_context::process); 118 | dispatch.on("stats", this, &queue_app_context::process); 119 | } 120 | 121 | queue_app_context::~queue_app_context() 122 | { 123 | } 124 | 125 | void queue_app_context::process(const std::string &cocaine_event, const std::vector &chunks, cocaine::framework::response_ptr response) 126 | { 127 | ioremap::elliptics::exec_context context = ioremap::elliptics::exec_context::from_raw(chunks[0].c_str(), chunks[0].size()); 128 | 129 | const std::string &event = cocaine_event; 130 | 131 | const std::string action_id = cocaine::format("%s %d, %s", dnet_dump_id_str(context.src_id()->id), context.src_key(), m_id.c_str()); 132 | 133 | COCAINE_LOG_INFO(m_log, "%s, event: %s, data-size: %ld", 134 | action_id.c_str(), 135 | event.c_str(), context.data().size() 136 | ); 137 | 138 | if (event == "ping") { 139 | m_queue->final(response, context, "ok"); 140 | 141 | } else if (event == "push") { 142 | ioremap::elliptics::data_pointer d = context.data(); 143 | // skip adding zero length data, because there is no value in that 144 | // queue has no method to request size and we can use zero reply in pop 145 | // to indicate queue emptiness 146 | if (!d.empty()) { 147 | m_push_time.start(); 148 | m_queue->push(d); 149 | m_push_time.stop(); 150 | COCAINE_LOG_INFO(m_log, "%s, push time %ld", 151 | action_id.c_str(), 152 | microseconds_now() - m_push_time.start_time 153 | ); 154 | m_push_rate.update(1); 155 | } 156 | m_queue->final(response, context, ioremap::elliptics::data_pointer()); 157 | 158 | } else if (event == "pop-multi") { 159 | int num = stoi(context.data().to_string()); 160 | 161 | m_pop_time.start(); 162 | m_ack_time.start(); 163 | peek_multi_type d = m_queue->pop(num); 164 | m_ack_time.stop(); 165 | m_pop_time.stop(); 166 | if (!d.empty()) { 167 | m_queue->final(response, context, ioremap::grape::serialize(d)); 168 | m_pop_rate.update(d.sizes().size()); 169 | m_ack_rate.update(d.sizes().size()); 170 | } else { 171 | m_queue->final(response, context, ioremap::elliptics::data_pointer()); 172 | } 173 | 174 | COCAINE_LOG_INFO(m_log, "%s, completed event: %s, size: %ld, popped: %d/%d (multiple: '%s')", 175 | action_id.c_str(), 176 | event.c_str(), context.data().size(), 177 | d.sizes().size(), num, context.data().to_string().c_str() 178 | ); 179 | 180 | } else if (event == "pop") { 181 | m_pop_time.start(); 182 | m_ack_time.start(); 183 | m_queue->final(response, context, m_queue->pop()); 184 | m_ack_time.stop(); 185 | m_pop_time.stop(); 186 | m_pop_rate.update(1); 187 | m_ack_rate.update(1); 188 | 189 | } else if (event == "peek") { 190 | m_pop_time.start(); 191 | ioremap::grape::entry_id entry_id; 192 | ioremap::elliptics::data_pointer d = m_queue->peek(&entry_id); 193 | 194 | COCAINE_LOG_INFO(m_log, "%s, peeked entry: %d-%d: (%ld)'%s'", 195 | action_id.c_str(), 196 | entry_id.chunk, entry_id.pos, d.size(), d.to_string() 197 | ); 198 | 199 | // embed entry id directly into sph of the reply 200 | dnet_raw_id *src = context.src_id(); 201 | memcpy(&src->id[DNET_ID_SIZE - sizeof(entry_id)], &entry_id, sizeof(entry_id)); 202 | 203 | m_queue->final(response, context, d); 204 | 205 | m_pop_time.stop(); 206 | m_pop_rate.update(1); 207 | 208 | } else if (event == "peek-multi") { 209 | m_pop_time.start(); 210 | int num = stoi(context.data().to_string()); 211 | 212 | peek_multi_type d = m_queue->peek(num); 213 | 214 | if (!d.empty()) { 215 | m_queue->final(response, context, ioremap::grape::serialize(d)); 216 | m_pop_rate.update(d.sizes().size()); 217 | } else { 218 | m_queue->final(response, context, ioremap::elliptics::data_pointer()); 219 | } 220 | 221 | m_pop_time.stop(); 222 | 223 | COCAINE_LOG_INFO(m_log, "%s, peeked %ld entries (asked %d)", 224 | action_id.c_str(), 225 | d.sizes().size(), num 226 | ); 227 | 228 | } else if (event == "ack") { 229 | m_ack_time.start(); 230 | ioremap::elliptics::data_pointer d = context.data(); 231 | ioremap::grape::entry_id entry_id = ioremap::grape::entry_id::from_dnet_raw_id(context.src_id()); 232 | 233 | m_queue->ack(entry_id); 234 | m_queue->final(response, context, ioremap::elliptics::data_pointer()); 235 | 236 | m_ack_time.stop(); 237 | m_ack_rate.update(1); 238 | 239 | COCAINE_LOG_INFO(m_log, "%s, acked entry %d-%d", 240 | action_id.c_str(), 241 | entry_id.chunk, entry_id.pos 242 | ); 243 | 244 | } else if (event == "ack-multi") { 245 | m_ack_time.start(); 246 | auto d = ioremap::grape::deserialize(context.data()); 247 | 248 | m_queue->ack(d); 249 | m_queue->final(response, context, ioremap::elliptics::data_pointer()); 250 | 251 | m_ack_time.stop(); 252 | m_ack_rate.update(d.size()); 253 | 254 | COCAINE_LOG_INFO(m_log, "%s, acked %ld entries", 255 | action_id.c_str(), 256 | d.size() 257 | ); 258 | 259 | } else if (event == "clear") { 260 | // clear queue content 261 | m_queue->clear(); 262 | m_queue->final(response, context, "ok"); 263 | 264 | } else if (event == "stats-clear") { 265 | m_queue->clear_counters(); 266 | m_queue->final(response, context, "ok"); 267 | 268 | } else if (event == "stats") { 269 | rapidjson::StringBuffer stream; 270 | rapidjson::PrettyWriter writer(stream); 271 | rapidjson::Document root; 272 | 273 | root.SetObject(); 274 | 275 | rapidjson::Value name; 276 | std::string qname = m_queue->queue_id(); 277 | name.SetString(qname.c_str(), qname.size()); 278 | 279 | ioremap::grape::queue_state state = m_queue->state(); 280 | ioremap::grape::queue_statistics st = m_queue->statistics(); 281 | 282 | root.AddMember("queue_id", name, root.GetAllocator()); 283 | 284 | root.AddMember("high-id", state.chunk_id_push, root.GetAllocator()); 285 | root.AddMember("low-id", state.chunk_id_ack, root.GetAllocator()); 286 | 287 | root.AddMember("push.count", st.push_count, root.GetAllocator()); 288 | root.AddMember("pop.count", st.pop_count, root.GetAllocator()); 289 | root.AddMember("ack.count", st.ack_count, root.GetAllocator()); 290 | root.AddMember("push.rate", m_push_rate.get(), root.GetAllocator()); 291 | root.AddMember("pop.rate", m_pop_rate.get(), root.GetAllocator()); 292 | root.AddMember("ack.rate", m_ack_rate.get(), root.GetAllocator()); 293 | root.AddMember("push.time", m_push_time.get(), root.GetAllocator()); 294 | root.AddMember("pop.time", m_pop_time.get(), root.GetAllocator()); 295 | root.AddMember("ack.time", m_ack_time.get(), root.GetAllocator()); 296 | root.AddMember("timeout.count", st.timeout_count, root.GetAllocator()); 297 | root.AddMember("state.write_count", st.state_write_count, root.GetAllocator()); 298 | 299 | root.AddMember("chunks_popped.write_data", st.chunks_popped.write_data, root.GetAllocator()); 300 | root.AddMember("chunks_popped.write_meta", st.chunks_popped.write_meta, root.GetAllocator()); 301 | root.AddMember("chunks_popped.read", st.chunks_popped.read, root.GetAllocator()); 302 | root.AddMember("chunks_popped.remove", st.chunks_popped.remove, root.GetAllocator()); 303 | root.AddMember("chunks_popped.push", st.chunks_popped.push, root.GetAllocator()); 304 | root.AddMember("chunks_popped.pop", st.chunks_popped.pop, root.GetAllocator()); 305 | root.AddMember("chunks_popped.ack", st.chunks_popped.ack, root.GetAllocator()); 306 | 307 | root.AddMember("chunks_pushed.write_data", st.chunks_pushed.write_data, root.GetAllocator()); 308 | root.AddMember("chunks_pushed.write_meta", st.chunks_pushed.write_meta, root.GetAllocator()); 309 | root.AddMember("chunks_pushed.read", st.chunks_pushed.read, root.GetAllocator()); 310 | root.AddMember("chunks_pushed.remove", st.chunks_pushed.remove, root.GetAllocator()); 311 | root.AddMember("chunks_pushed.push", st.chunks_pushed.push, root.GetAllocator()); 312 | root.AddMember("chunks_pushed.pop", st.chunks_pushed.pop, root.GetAllocator()); 313 | root.AddMember("chunks_pushed.ack", st.chunks_pushed.ack, root.GetAllocator()); 314 | 315 | root.Accept(writer); 316 | 317 | m_queue->final(response, context, ioremap::elliptics::data_pointer::from_raw(const_cast(stream.GetString()), stream.GetSize())); 318 | 319 | } else { 320 | std::string msg = event + ": unknown event"; 321 | m_queue->final(response, context, msg); 322 | } 323 | 324 | COCAINE_LOG_INFO(m_log, "%s, event: %s, data-size: %ld, completed", 325 | action_id.c_str(), 326 | event.c_str(), context.data().size() 327 | ); 328 | } 329 | 330 | int main(int argc, char **argv) 331 | { 332 | try { 333 | return cocaine::framework::run(argc, argv); 334 | } catch (const std::exception &e) { 335 | std::ofstream tmp("/tmp/queue.out"); 336 | 337 | std::ostringstream out; 338 | out << "queue failed: " << e.what(); 339 | 340 | tmp.write(out.str().c_str(), out.str().size()); 341 | } 342 | } 343 | -------------------------------------------------------------------------------- /include/grape/concurrent-pump.hpp: -------------------------------------------------------------------------------- 1 | #ifndef __CONCURRENT_HPP 2 | #define __CONCURRENT_HPP 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | namespace ioremap { namespace grape { 16 | 17 | class concurrent_pump 18 | { 19 | private: 20 | std::atomic_int running_requests; 21 | std::mutex mutex; 22 | std::condition_variable condition; 23 | std::atomic_bool cont; 24 | 25 | public: 26 | int concurrency_limit; 27 | bool wait_on_exit; 28 | 29 | concurrent_pump() 30 | : running_requests(0) 31 | , cont(true) 32 | , concurrency_limit(1) 33 | , wait_on_exit(true) 34 | {} 35 | 36 | void run(std::function make_request) { 37 | cont = true; 38 | while (true) { 39 | std::unique_lock lock(mutex); 40 | condition.wait(lock, [this]{return running_requests < concurrency_limit;}); 41 | if (!cont) { 42 | break; 43 | } 44 | ++running_requests; 45 | lock.unlock(); 46 | 47 | make_request(); 48 | } 49 | 50 | if (wait_on_exit) { 51 | std::unique_lock lock(mutex); 52 | condition.wait(lock, [this]{return running_requests == 0;}); 53 | } 54 | } 55 | 56 | // Should be called when response on request is received 57 | void complete_request() { 58 | std::unique_lock lock(mutex); 59 | --running_requests; 60 | condition.notify_one(); 61 | } 62 | 63 | // Should be called if runloop must be stopped 64 | void stop() { 65 | cont = false; 66 | } 67 | }; 68 | 69 | struct request { 70 | dnet_id id; 71 | int src_key; 72 | 73 | request() : src_key(-1) { 74 | memset(&id, 0, sizeof(dnet_id)); 75 | } 76 | request(int src_key) : src_key(src_key) { 77 | memset(&id, 0, sizeof(dnet_id)); 78 | } 79 | }; 80 | 81 | template 82 | class base_queue_reader 83 | { 84 | protected: 85 | concurrent_pump runloop; 86 | 87 | ioremap::elliptics::session client; 88 | const std::string queue_name; 89 | const int request_size; 90 | std::atomic_int next_request_id; 91 | 92 | std::shared_ptr log; 93 | 94 | int concurrency_limit; 95 | 96 | public: 97 | base_queue_reader(ioremap::elliptics::session client, const std::string &queue_name, int request_size, int concurrency_limit) 98 | : client(client) 99 | , queue_name(queue_name) 100 | , request_size(request_size) 101 | , next_request_id(0) 102 | , concurrency_limit(concurrency_limit) 103 | { 104 | srand(time(NULL)); 105 | log = std::make_shared(client.get_node().get_log()); 106 | } 107 | 108 | void run() { 109 | runloop.concurrency_limit = concurrency_limit; 110 | runloop.run([this] () { 111 | base_queue_reader::queue_peek(client, next_request_id++, request_size); 112 | }); 113 | } 114 | 115 | void queue_ack(ioremap::elliptics::exec_context context, 116 | std::shared_ptr req, 117 | const std::vector &ids) 118 | { 119 | std::string log_prefix = cocaine::format("%s %d", dnet_dump_id(&req->id), req->src_key); 120 | base_queue_ack(client, queue_name, context, log, ids, log_prefix); 121 | } 122 | 123 | static void queue_ack(ioremap::elliptics::session client, 124 | const std::string &queue_name, 125 | ioremap::elliptics::exec_context context, 126 | const std::vector &ids, 127 | const std::string& log_prefix = "") 128 | { 129 | auto log = std::make_shared(client.get_node().get_log()); 130 | base_queue_ack(client, queue_name, context, log, ids, log_prefix); 131 | } 132 | 133 | static void queue_ack(ioremap::elliptics::session client, 134 | const std::string &queue_name, 135 | ioremap::elliptics::exec_context context, 136 | const std::vector &entries, 137 | const std::string& log_prefix = "") 138 | { 139 | auto log = std::make_shared(client.get_node().get_log()); 140 | 141 | std::vector entry_ids; 142 | std::transform(entries.begin(), entries.end(), 143 | std::back_inserter(entry_ids), 144 | [] (const data_array::entry &entry) { return entry.entry_id; } 145 | ); 146 | 147 | base_queue_ack(client, queue_name, context, log, entry_ids, log_prefix); 148 | } 149 | 150 | static inline void base_queue_ack(ioremap::elliptics::session client, 151 | const std::string &queue_name, 152 | ioremap::elliptics::exec_context context, 153 | std::shared_ptr log, 154 | const std::vector &ids, 155 | const std::string &log_prefix) 156 | { 157 | client.set_exceptions_policy(ioremap::elliptics::session::no_exceptions); 158 | 159 | size_t count = ids.size(); 160 | client.exec(context, queue_name + "@ack-multi", serialize(ids)) 161 | .connect(ioremap::elliptics::async_result::result_function(), 162 | [log, log_prefix, count] (const ioremap::elliptics::error_info &error) { 163 | if (error) { 164 | COCAINE_LOG_ERROR(log, "%s: %ld entries not acked: %s", log_prefix.c_str(), count, error.message().c_str()); 165 | } else { 166 | COCAINE_LOG_INFO(log, "%s: %ld entries acked", log_prefix.c_str(), count); 167 | } 168 | } 169 | ); 170 | } 171 | 172 | void queue_peek(ioremap::elliptics::session client, int req_unique_id, int arg) 173 | { 174 | client.set_exceptions_policy(ioremap::elliptics::session::no_exceptions); 175 | 176 | std::string queue_key = std::to_string(req_unique_id) + std::to_string(rand()); 177 | 178 | auto req = std::make_shared(req_unique_id); 179 | client.transform(queue_key, req->id); 180 | 181 | client.exec(&req->id, req->src_key, queue_name + "@peek-multi", std::to_string(arg)) 182 | .connect( 183 | std::bind(&base_queue_reader::data_received, this, req, std::placeholders::_1), 184 | std::bind(&base_queue_reader::request_complete, this, req, std::placeholders::_1) 185 | ); 186 | } 187 | 188 | void data_received(std::shared_ptr req, const ioremap::elliptics::exec_result_entry &result) 189 | { 190 | if (result.error()) { 191 | COCAINE_LOG_ERROR(log, "%s %d: error: %s", dnet_dump_id(&req->id), req->src_key, result.error().message().c_str()); 192 | return; 193 | } 194 | 195 | ioremap::elliptics::exec_context context = result.context(); 196 | 197 | // queue.peek returns no data when queue is empty. 198 | if (context.data().empty()) { 199 | return; 200 | } 201 | 202 | // Received context must be used for acking to the same queue instance. 203 | // 204 | // But before that context.src_key must be restored back 205 | // to the original src_key used in the original request to the queue, 206 | // or else our worker's ack will not be routed to the exact same 207 | // queue worker that issued reply with this context. 208 | // 209 | // (src_key of the request gets replaced by job id server side, 210 | // so reply does not carries the same src_key as a request. 211 | // Which is unfortunate.) 212 | context.set_src_key(req->src_key); 213 | 214 | COCAINE_LOG_INFO(log, "%s %d: received data, byte size %ld", 215 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 216 | context.data().size() 217 | ); 218 | 219 | auto array = deserialize(context.data()); 220 | 221 | { 222 | size_t count = array.sizes().size(); 223 | COCAINE_LOG_INFO(log, "%s %d: processing %ld entries", 224 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 225 | count 226 | ); 227 | COCAINE_LOG_INFO(log, "%s %d: array %p", dnet_dump_id_str(context.src_id()->id), context.src_key(), array.data().data()); 228 | } 229 | 230 | static_cast(this)->process_data_array(req, context, array); 231 | } 232 | 233 | void request_complete(std::shared_ptr req, const ioremap::elliptics::error_info &error) { 234 | //TODO: add reaction to hard errors like No such device or address: -6 235 | if (error) { 236 | COCAINE_LOG_ERROR(log, "%s %d: queue request completion error: %s", dnet_dump_id(&req->id), req->src_key, error.message().c_str()); 237 | } else { 238 | COCAINE_LOG_INFO(log, "%s %d: queue request completed", dnet_dump_id(&req->id), req->src_key); 239 | } 240 | 241 | runloop.complete_request(); 242 | } 243 | }; 244 | 245 | class queue_reader: public base_queue_reader 246 | { 247 | public: 248 | typedef std::function processing_function; 249 | processing_function proc; 250 | 251 | queue_reader(ioremap::elliptics::session client, const std::string &queue_name, int request_size, int concurrency_limit) 252 | : base_queue_reader(client, queue_name, request_size, concurrency_limit) 253 | {} 254 | 255 | void run(processing_function func) { 256 | proc = func; 257 | base_queue_reader::run(); 258 | } 259 | 260 | void process_data_array(std::shared_ptr req, ioremap::elliptics::exec_context context, data_array &array) { 261 | auto d = ioremap::elliptics::data_pointer::from_raw(array.data()); 262 | size_t count = array.sizes().size(); 263 | 264 | std::vector ack_ids; 265 | 266 | size_t offset = 0; 267 | for (size_t i = 0; i < count; ++i) { 268 | const entry_id &id = array.ids()[i]; 269 | int bytesize = array.sizes()[i]; 270 | 271 | //TODO: check result of the proc() 272 | if (proc(id, d.slice(offset, bytesize))) { 273 | ack_ids.push_back(id); 274 | } 275 | 276 | offset += bytesize; 277 | } 278 | 279 | // acknowledge entries 280 | COCAINE_LOG_INFO(log, "%s %d: acking %ld entries", 281 | dnet_dump_id_str(context.src_id()->id), context.src_key(), 282 | ack_ids.size() 283 | ); 284 | 285 | queue_ack(context, req, ack_ids); 286 | } 287 | }; 288 | 289 | class bulk_queue_reader: public base_queue_reader 290 | { 291 | public: 292 | typedef std::function processing_function; 293 | processing_function proc; 294 | 295 | static const int REQUEST_CONTINUE = 0; 296 | static const int REQUEST_ACK = 1 << 0; 297 | static const int REQUEST_STOP = 1 << 1; 298 | 299 | bulk_queue_reader(ioremap::elliptics::session client, const std::string &queue_name, int request_size, int concurrency_limit) 300 | : base_queue_reader(client, queue_name, request_size, concurrency_limit) 301 | {} 302 | 303 | void run(processing_function func) { 304 | proc = func; 305 | base_queue_reader::run(); 306 | } 307 | 308 | void handle_process_result(int result, std::shared_ptr req, ioremap::elliptics::exec_context context, data_array &array) { 309 | if (result & REQUEST_ACK) { 310 | queue_ack(context, req, array.ids()); 311 | } 312 | if (result & REQUEST_STOP) { 313 | runloop.stop(); 314 | } 315 | } 316 | 317 | void process_data_array(std::shared_ptr req, ioremap::elliptics::exec_context context, data_array &array) { 318 | int proc_result = proc(context, array); 319 | handle_process_result(proc_result, req, context, array); 320 | } 321 | }; 322 | 323 | class queue_writer 324 | { 325 | public: 326 | typedef ioremap::elliptics::data_pointer generator_result_type; 327 | typedef std::function generation_function; 328 | 329 | private: 330 | concurrent_pump runloop; 331 | 332 | ioremap::elliptics::session client; 333 | const std::string queue_name; 334 | std::atomic_int next_request_id; 335 | std::shared_ptr log; 336 | 337 | int concurrency_limit; 338 | 339 | generation_function gen; 340 | 341 | public: 342 | queue_writer(ioremap::elliptics::session client, const std::string &queue_name, int concurrency_limit = 1) 343 | : client(client) 344 | , queue_name(queue_name) 345 | , next_request_id(0) 346 | , concurrency_limit(concurrency_limit) 347 | { 348 | log = std::make_shared(client.get_node().get_log()); 349 | srand(time(NULL)); 350 | } 351 | 352 | void run(generation_function func) { 353 | gen = func; 354 | runloop.concurrency_limit = concurrency_limit; 355 | runloop.run([this] () { 356 | generator_result_type d = gen(); 357 | if (d.empty()) { 358 | runloop.stop(); 359 | } 360 | queue_push(client, next_request_id++, d); 361 | }); 362 | } 363 | 364 | void queue_push(ioremap::elliptics::session client, int req_unique_id, ioremap::elliptics::data_pointer d) 365 | { 366 | client.set_exceptions_policy(ioremap::elliptics::session::no_exceptions); 367 | 368 | std::string queue_key = std::to_string(req_unique_id) + std::to_string(rand()); 369 | 370 | auto req = std::make_shared(req_unique_id); 371 | client.transform(queue_key, req->id); 372 | 373 | client.exec(&req->id, req->src_key, queue_name + "@push", d) 374 | .connect( 375 | ioremap::elliptics::async_result::result_function(), 376 | std::bind(&queue_writer::request_complete, this, req, std::placeholders::_1) 377 | ); 378 | } 379 | 380 | void request_complete(std::shared_ptr req, const ioremap::elliptics::error_info &error) 381 | { 382 | if (error) { 383 | COCAINE_LOG_ERROR(log, "%s %d: queue request completion error: %s", dnet_dump_id(&req->id), req->src_key, error.message().c_str()); 384 | } else { 385 | COCAINE_LOG_INFO(log, "%s %d: queue request completed", dnet_dump_id(&req->id), req->src_key); 386 | } 387 | 388 | runloop.complete_request(); 389 | } 390 | }; 391 | 392 | }} // namespace ioremap::grape 393 | 394 | #endif //__CONCURRENT_HPP 395 | 396 | --------------------------------------------------------------------------------