├── debian ├── compat ├── libmastermind.install ├── python-mastermind-cache.install ├── libmastermind-dev.install ├── rules └── control ├── bindings ├── CMakeLists.txt └── python │ ├── CMakeLists.txt │ └── main.cpp ├── README.md ├── tests ├── pytest_runner.sh ├── bindings │ └── python │ │ ├── test.py │ │ ├── messages.py │ │ └── unittestpy.py ├── teamcity_cppunit.h ├── teamcity_messages.h ├── teamcity_cppunit.cpp ├── manager.sh ├── teamcity_messages.cpp └── test.cpp ├── cmake └── FindPythonExtensionDir.cmake ├── include └── libmastermind │ ├── common.hpp │ ├── couple_sequence.hpp │ ├── error.hpp │ └── mastermind.hpp ├── src ├── namespace_p.hpp ├── utils.hpp ├── couple_sequence_p.hpp ├── couple_weights_p.hpp ├── utils.cpp ├── cached_keys.hpp ├── couple_sequence.cpp ├── namespace_state_p.hpp ├── couple_weights.cpp ├── namespace.cpp ├── error.cpp ├── cache_p.hpp ├── namespace_state_view.cpp ├── mastermind_impl.hpp ├── namespace_state.cpp └── mastermind.cpp ├── CMakeLists.txt ├── foreign └── cocaine │ └── traits │ └── dynamic.hpp └── LICENSE /debian/compat: -------------------------------------------------------------------------------- 1 | 7 2 | -------------------------------------------------------------------------------- /bindings/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(python) 2 | -------------------------------------------------------------------------------- /debian/libmastermind.install: -------------------------------------------------------------------------------- 1 | usr/lib/libmastermind.so.* 2 | -------------------------------------------------------------------------------- /debian/python-mastermind-cache.install: -------------------------------------------------------------------------------- 1 | usr/lib/python*/dist-packages/* 2 | -------------------------------------------------------------------------------- /debian/libmastermind-dev.install: -------------------------------------------------------------------------------- 1 | usr/lib/libmastermind.so 2 | usr/include/libmastermind/* 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | libelliptics_proxy 2 | ================== 3 | 4 | Library that implements smart proxy for elliptics 5 | [Documentation](doc/en/Overview.md) 6 | -------------------------------------------------------------------------------- /tests/pytest_runner.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | manager="$1/manager.sh" 4 | 5 | $manager prepare 4 6 | $manager start 7 | 8 | python $1/bindings/python/test.py 9 | 10 | $manager clear 11 | 12 | -------------------------------------------------------------------------------- /cmake/FindPythonExtensionDir.cmake: -------------------------------------------------------------------------------- 1 | execute_process(COMMAND "${PYTHON_EXECUTABLE}" -c "from distutils import sysconfig; print sysconfig.get_python_lib(prefix='${CMAKE_INSTALL_PREFIX}')," 2 | RESULT_VARIABLE ret_var 3 | OUTPUT_VARIABLE PYTHON_EXTENSION_DIR 4 | OUTPUT_STRIP_TRAILING_WHITESPACE) 5 | -------------------------------------------------------------------------------- /debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | include /usr/share/cdbs/1/class/cmake.mk 4 | include /usr/share/cdbs/1/rules/debhelper.mk 5 | 6 | MASTERMIND_VERSION := "$(shell dpkg-parsechangelog | sed -n -r -e 's/^Version: ([^.]+\.[^.]+).*/\1/p')" 7 | 8 | DEB_DH_MAKESHLIBS_ARGS := -V "libmastermind (>= $(MASTERMIND_VERSION))" 9 | 10 | DEB_CMAKE_EXTRA_FLAGS := -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_SKIP_RPATH=OFF 11 | 12 | DEB_DH_STRIP_ARGS := --dbg-package=libmastermind 13 | 14 | #DEB_MAKE_CHECK_TARGET := test ARGS="-V" 15 | 16 | -------------------------------------------------------------------------------- /include/libmastermind/common.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COMMON__HPP 21 | #define LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COMMON__HPP 22 | 23 | #include 24 | 25 | namespace mastermind { 26 | 27 | typedef int group_t; 28 | typedef std::vector groups_t; 29 | 30 | } // namespace mastermind 31 | 32 | #endif /* LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COMMON__HPP */ 33 | 34 | -------------------------------------------------------------------------------- /bindings/python/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | find_package(Boost REQUIRED COMPONENTS python) 2 | #find_package(PythonLibs REQUIRED) 3 | find_package(PythonInterp REQUIRED) 4 | find_package(PythonExtensionDir REQUIRED) 5 | #message(STATUS "Python includes are situated in (${PYTHON_INCLUDE_PATH}, ${PYTHON_INCLUDE_DIRS})") 6 | 7 | INCLUDE_DIRECTORIES(BEFORE ${PROJECT_SOURCE_DIR}/include) 8 | include_directories(${Boost_INCLUDE_DIRS}) 9 | link_directories(${Boost_LIBRARY_DIRS}) 10 | 11 | find_package(PythonLibs REQUIRED) 12 | include_directories(${PYTHON_INCLUDE_PATH}) 13 | include_directories(${PYTHON_INCLUDE_DIRS}) 14 | link_directories(${PYTHON_LIBRARIES}) 15 | 16 | add_library(mastermind_cache SHARED main.cpp) 17 | target_link_libraries(mastermind_cache ${Boost_LIBRARIES} ${PYTHON_LIBRARIES} mastermind) 18 | 19 | set_target_properties(mastermind_cache PROPERTIES 20 | PREFIX "" 21 | OUTPUT_NAME "mastermind_cache" 22 | VERSION ${MAJOR_VERSION} 23 | SOVERSION ${MINOR_VERSION} 24 | ) 25 | 26 | install(TARGETS mastermind_cache 27 | LIBRARY DESTINATION ${PYTHON_EXTENSION_DIR}/ 28 | ARCHIVE DESTINATION ${PYTHON_EXTENSION_DIR}/ 29 | BUNDLE DESTINATION library 30 | ) 31 | 32 | #install(CODE "EXECUTE_PROCESS( 33 | # COMMAND ${PYTHON_EXECUTABLE} setup.py install --prefix=$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX} 34 | # WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/ 35 | #)") 36 | -------------------------------------------------------------------------------- /debian/control: -------------------------------------------------------------------------------- 1 | Source: libmastermind 2 | Section: net 3 | Priority: optional 4 | Maintainer: Anton Kortunov 5 | Build-Depends: debhelper (>= 5), 6 | cdbs, 7 | cmake (>= 2.6), 8 | libboost-dev, 9 | libboost-python-dev, 10 | libboost-thread-dev, 11 | libboost-iostreams-dev, 12 | cocaine-framework-native (>= 0.11.1), cocaine-framework-native (<< 0.12), 13 | cocaine-framework-native-dev (>= 0.11.1), cocaine-framework-native-dev (<< 0.12), 14 | libcocaine-core2 (>= 0.11), libcocaine-core2 (<< 0.12), 15 | libcocaine-dev (>= 0.11), libcocaine-dev (<< 0.12), 16 | libmsgpack-dev, 17 | libboost-system-dev, 18 | libjsoncpp-dev, 19 | libkora-util-dev (= 1.1.0-rc1) 20 | Standards-Version: 3.9.1 21 | 22 | Package: libmastermind 23 | Architecture: any 24 | Depends: 25 | ${shlibs:Depends}, 26 | cocaine-framework-native (>= 0.11.1), cocaine-framework-native (<< 0.12), 27 | libcocaine-core2 (>= 0.11), libcocaine-core2 (<< 0.12), 28 | libkora-util1 (= 1.1.0-rc1), 29 | Description: Client for Mastermind service 30 | 31 | Package: libmastermind-dev 32 | Architecture: any 33 | Depends: libmastermind (= ${Source-Version}), libkora-util-dev (= 1.1.0-rc1) 34 | Description: Client for Mastermind service (developer files) 35 | 36 | Package: python-mastermind-cache 37 | Architecture: any 38 | Depends: libmastermind (= ${Source-Version}) 39 | Description: Client for Mastermind service (python binding) 40 | Replaces: mastermind-cache 41 | Provides: mastermind-cache 42 | Conflicts: mastermind-cache 43 | 44 | -------------------------------------------------------------------------------- /tests/bindings/python/test.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import sys 3 | sys.path.append("bindings/python/") 4 | 5 | from unittestpy import TeamcityTestRunner 6 | from elliptics_proxy import * 7 | 8 | import unittest 9 | 10 | r = remote('localhost', 1025) 11 | c = config() 12 | c.groups.append(1) 13 | c.groups.append(2) 14 | c.log_path = 'log' 15 | c.remotes.append(r) 16 | proxy = elliptics_proxy_t(c) 17 | 18 | class TestTeamcityMessages(unittest.TestCase): 19 | 20 | def test_sync(self): 21 | data = 'test data of test_sync' 22 | key = 'test_sync_key' 23 | lr = proxy.write(key, data) 24 | assert len(lr) == 2 25 | dc2 = proxy.read(key) 26 | assert dc2.data == data 27 | proxy.remove(key) 28 | 29 | def test_async(self): 30 | data = 'test data for test_async' 31 | key = 'test_async_key' 32 | dc = data_container_t(data) 33 | dc.timestamp = timespec(123, 456789) 34 | 35 | awr = proxy.write_async(key, dc) 36 | lr = awr.get() 37 | assert len(lr) == 2 38 | 39 | arr = proxy.read_async(key, embeded = True, groups = [1]); 40 | ldc = arr.get() 41 | dc = ldc[0] 42 | assert dc.data == data 43 | ts2 = dc.timestamp 44 | assert ts2.tv_sec == 123 45 | assert ts2.tv_nsec == 456789 46 | 47 | arr = proxy.read_async(key, embeded = True, groups = [2]); 48 | ldc = arr.get() 49 | dc = ldc[0] 50 | assert dc.data == data 51 | ts2 = dc.timestamp 52 | assert ts2.tv_sec == 123 53 | assert ts2.tv_nsec == 456789 54 | 55 | arm = proxy.remove_async(key); 56 | arm.wait() 57 | 58 | if __name__ == '__main__': 59 | runner = TeamcityTestRunner() 60 | unittest.main(testRunner=runner) 61 | -------------------------------------------------------------------------------- /tests/bindings/python/messages.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import sys 3 | 4 | 5 | class TeamcityServiceMessages(object): 6 | quote = {"'": "|'", "|": "||", "\n": "|n", "\r": "|r", ']': '|]'} 7 | 8 | def __init__(self, output=sys.stdout): 9 | self.output = output 10 | 11 | def escapeValue(self, value): 12 | return "".join([self.quote.get(x, x) for x in value]) 13 | 14 | def message(self, messageName, **properties): 15 | self.output.write("\n##teamcity[" + messageName) 16 | for k in sorted(properties.keys()): 17 | self.output.write(" %s='%s'" % (k, self.escapeValue(properties[k]))) 18 | self.output.write("]\n") 19 | 20 | def testSuiteStarted(self, suiteName): 21 | self.message('testSuiteStarted', name=suiteName) 22 | 23 | def testSuiteFinished(self, suiteName): 24 | self.message('testSuiteFinished', name=suiteName) 25 | 26 | def testStarted(self, testName): 27 | self.message('testStarted', name=testName) 28 | 29 | def testFinished(self, testName): 30 | self.message('testFinished', name=testName) 31 | 32 | def testIgnored(self, testName, message=''): 33 | self.message('testIgnored', name=testName, message=message) 34 | 35 | def testFailed(self, testName, message='', details=''): 36 | self.message('testFailed', name=testName, 37 | message=message, details=details) 38 | 39 | def testStdOut(self, testName, out): 40 | self.message('testStdOut', name=testName, out=out) 41 | 42 | def testStdErr(self, testName, out): 43 | self.message('testStdErr', name=testName, out=out) 44 | -------------------------------------------------------------------------------- /src/namespace_p.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__NAMESPACE_P_HPP 21 | #define SRC__NAMESPACE_P_HPP 22 | 23 | #include "libmastermind/mastermind.hpp" 24 | 25 | namespace mastermind { 26 | 27 | struct namespace_settings_t::data { 28 | 29 | data(); 30 | data(const data &d); 31 | data(data &&d); 32 | 33 | std::string name; 34 | int groups_count; 35 | std::string success_copies_num; 36 | std::string auth_key; 37 | std::vector static_couple; 38 | 39 | std::string auth_key_for_write; 40 | std::string auth_key_for_read; 41 | 42 | std::string sign_token; 43 | std::string sign_path_prefix; 44 | std::string sign_port; 45 | 46 | int redirect_expire_time; 47 | int64_t redirect_content_length_threshold; 48 | 49 | bool is_active; 50 | 51 | bool can_choose_couple_to_upload; 52 | int64_t multipart_content_length_threshold; 53 | }; 54 | 55 | } // mastermind 56 | 57 | #endif /* SRC__NAMESPACE_P_HPP */ 58 | 59 | -------------------------------------------------------------------------------- /tests/teamcity_cppunit.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2011 JetBrains s.r.o. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | * $Revision: 88625 $ 16 | */ 17 | 18 | #ifndef H_TEAMCITY_CPPUNIT 19 | #define H_TEAMCITY_CPPUNIT 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include "teamcity_messages.h" 26 | 27 | namespace JetBrains { 28 | 29 | class TeamcityProgressListener: public CPPUNIT_NS::TestListener { 30 | TeamcityMessages messages; 31 | 32 | public: 33 | TeamcityProgressListener(const std::string& _flowid); 34 | TeamcityProgressListener(); 35 | ~TeamcityProgressListener(); 36 | 37 | void startTest(CPPUNIT_NS::Test *test); 38 | void addFailure(const CPPUNIT_NS::TestFailure &failure); 39 | void endTest(CPPUNIT_NS::Test *test); 40 | void startSuite(CPPUNIT_NS::Test *test); 41 | void endSuite(CPPUNIT_NS::Test *test); 42 | 43 | private: 44 | std::string flowid; 45 | 46 | // Prevents the use of the copy constructor. 47 | TeamcityProgressListener(const TeamcityProgressListener ©); 48 | 49 | // Prevents the use of the copy operator. 50 | void operator =(const TeamcityProgressListener ©); 51 | }; 52 | 53 | } 54 | 55 | #endif /* H_TEAMCITY_CPPUNIT */ 56 | -------------------------------------------------------------------------------- /tests/teamcity_messages.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2011 JetBrains s.r.o. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | * $Revision: 88625 $ 16 | */ 17 | 18 | #ifndef H_TEAMCITY_MESSAGES 19 | #define H_TEAMCITY_MESSAGES 20 | 21 | #include 22 | #include 23 | 24 | namespace JetBrains { 25 | 26 | std::string getFlowIdFromEnvironment(); 27 | bool underTeamcity(); 28 | 29 | class TeamcityMessages { 30 | std::ostream *m_out; 31 | 32 | protected: 33 | std::string escape(std::string s); 34 | 35 | void openMsg(const std::string &name); 36 | void writeProperty(std::string name, std::string value); 37 | void closeMsg(); 38 | 39 | public: 40 | TeamcityMessages(); 41 | 42 | void setOutput(std::ostream &); 43 | 44 | void suiteStarted(std::string name, std::string flowid = ""); 45 | void suiteFinished(std::string name, std::string flowid = ""); 46 | 47 | void testStarted(std::string name, std::string flowid = ""); 48 | void testFailed(std::string name, std::string message, std::string details, std::string flowid = ""); 49 | void testIgnored(std::string name, std::string message, std::string flowid = ""); 50 | void testFinished(std::string name, int durationMs = -1, std::string flowid = ""); 51 | }; 52 | 53 | } 54 | 55 | #endif /* H_TEAMCITY_MESSAGES */ 56 | -------------------------------------------------------------------------------- /tests/bindings/python/unittestpy.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | import traceback 3 | import sys 4 | from unittest import TestResult 5 | 6 | from messages import TeamcityServiceMessages 7 | 8 | 9 | # Added *k to some methods to get compatibility with nosetests 10 | class TeamcityTestResult(TestResult): 11 | def __init__(self, stream=sys.stdout): 12 | TestResult.__init__(self) 13 | 14 | self.output = stream 15 | 16 | self.createMessages() 17 | 18 | def createMessages(self): 19 | self.messages = TeamcityServiceMessages(self.output) 20 | 21 | def formatErr(self, err): 22 | exctype, value, tb = err 23 | return ''.join(traceback.format_exception(exctype, value, tb)) 24 | 25 | def getTestName(self, test): 26 | return test.shortDescription() or str(test) 27 | 28 | def addSuccess(self, test, *k): 29 | TestResult.addSuccess(self, test) 30 | 31 | self.output.write("ok\n") 32 | 33 | def addError(self, test, err, *k): 34 | TestResult.addError(self, test, err) 35 | 36 | err = self.formatErr(err) 37 | 38 | self.messages.testFailed(self.getTestName(test), 39 | message='Error', details=err) 40 | 41 | def addFailure(self, test, err, *k): 42 | TestResult.addFailure(self, test, err) 43 | 44 | err = self.formatErr(err) 45 | 46 | self.messages.testFailed(self.getTestName(test), 47 | message='Failure', details=err) 48 | 49 | def startTest(self, test): 50 | self.messages.testStarted(self.getTestName(test)) 51 | 52 | def stopTest(self, test): 53 | self.messages.testFinished(self.getTestName(test)) 54 | 55 | 56 | class TeamcityTestRunner(object): 57 | def __init__(self, stream=sys.stderr): 58 | self.stream = stream 59 | 60 | def _makeResult(self): 61 | return TeamcityTestResult(self.stream) 62 | 63 | def run(self, test): 64 | result = self._makeResult() 65 | test(result) 66 | return result 67 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required( VERSION 2.6 ) 2 | 3 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/") 4 | 5 | set(LIB mastermind) 6 | project(${LIB}) 7 | 8 | FILE (READ "${CMAKE_CURRENT_SOURCE_DIR}/debian/changelog" DEBCHANGELOG) 9 | 10 | string(REGEX MATCH "([0-9]+\\.[0-9]+)" DEBFULLVERSION "${DEBCHANGELOG}") 11 | string(REGEX MATCH "([0-9]+)" MAJOR_VERSION "${DEBFULLVERSION}") 12 | string (REGEX MATCH "([0-9]+$)" MINOR_VERSION "${DEBFULLVERSION}") 13 | 14 | set(CMAKE_CXX_FLAGS "-Wall -pedantic -std=c++0x -O2 -g") 15 | 16 | set(REQUIRED_LIBRARIES 17 | cocaine-core 18 | cocaine-framework 19 | boost_thread 20 | boost_system 21 | boost_iostreams 22 | jsoncpp 23 | kora-util 24 | ) 25 | 26 | include_directories(BEFORE 27 | ${PROJECT_SOURCE_DIR}/include 28 | ${PROJECT_SOURCE_DIR}/foreign 29 | ) 30 | 31 | aux_source_directory(src SOURCES) 32 | set(SOURCES 33 | ${SOURCES} 34 | ${PROJECT_SOURCE_DIR}/src/utils.hpp 35 | ) 36 | 37 | add_library(${LIB} SHARED ${SOURCES}) 38 | 39 | set_target_properties(${LIB} PROPERTIES 40 | VERSION "${MAJOR_VERSION}.${MINOR_VERSION}" 41 | SOVERSION ${MAJOR_VERSION} 42 | ) 43 | 44 | target_link_libraries(${LIB} 45 | ${REQUIRED_LIBRARIES}) 46 | 47 | add_subdirectory(bindings) 48 | 49 | install(TARGETS ${LIB} 50 | RUNTIME DESTINATION bin COMPONENT runtime 51 | LIBRARY DESTINATION lib COMPONENT runtime 52 | ARCHIVE DESTINATION lib COMPONENT developement 53 | ) 54 | 55 | install( 56 | DIRECTORY 57 | include/ 58 | DESTINATION include 59 | COMPONENT development) 60 | 61 | #set (TESTS_SOURCES 62 | # tests/test.cpp 63 | # tests/teamcity_cppunit.cpp 64 | # tests/teamcity_messages.cpp 65 | # tests/teamcity_cppunit.h 66 | # tests/teamcity_messages.h 67 | #) 68 | #set (TEST test_${LIB}) 69 | #add_executable (${TEST} ${TESTS_SOURCES}) 70 | #target_link_libraries (${TEST} ${LIB} cppunit) 71 | ##set_target_properties(${TEST} PROPERTIES 72 | # BUILD_WITH_INSTALL_RPATH 1 73 | # INSTALL_RPATH .) 74 | #set_target_properties(${TEST} PROPERTIES 75 | # INSTALL_RPATH "." 76 | # BUILD_WITH_INSTALL_RPATH ON) 77 | #enable_testing () 78 | #add_test (${TEST} ${TEST} ${PROJECT_SOURCE_DIR}/tests/) 79 | #set(PYTEST pytest_runner.sh) 80 | #add_test (${PYTEST} ${PYTEST} ${PROJECT_SOURCE_DIR}/tests/) 81 | #file(COPY 82 | # ${CMAKE_CURRENT_SOURCE_DIR}/tests/${PYTEST} 83 | # DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) 84 | -------------------------------------------------------------------------------- /include/libmastermind/couple_sequence.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COUPLE_SEQUENCE__HPP 21 | #define LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COUPLE_SEQUENCE__HPP 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace mastermind { 30 | 31 | class couple_info_t { 32 | public: 33 | group_t id; 34 | groups_t groups; 35 | 36 | protected: 37 | }; 38 | 39 | class couple_sequence_const_iterator_t 40 | : public std::iterator 41 | { 42 | public: 43 | typedef couple_sequence_const_iterator_t self_type; 44 | 45 | couple_sequence_const_iterator_t(); 46 | couple_sequence_const_iterator_t(const self_type &that); 47 | 48 | reference 49 | operator * () const; 50 | 51 | pointer 52 | operator -> () const; 53 | 54 | bool 55 | operator == (const self_type &other) const; 56 | 57 | bool 58 | operator != (const self_type &other) const; 59 | 60 | self_type & 61 | operator ++ (); 62 | 63 | self_type 64 | operator ++ (int); 65 | 66 | self_type & 67 | operator = (const self_type &that); 68 | 69 | protected: 70 | class data_t; 71 | 72 | std::shared_ptr data; 73 | }; 74 | 75 | class couple_sequence_t { 76 | public: 77 | typedef couple_sequence_const_iterator_t const_iterator; 78 | 79 | const_iterator 80 | begin() const; 81 | 82 | const_iterator 83 | end() const; 84 | 85 | size_t 86 | size() const; 87 | 88 | protected: 89 | class data_t; 90 | 91 | std::shared_ptr data; 92 | 93 | }; 94 | 95 | } // namespace mastermind 96 | 97 | #endif /* LIBMASTERMIND__INCLUDE__LIBMASTERMIND__COUPLE_SEQUENCE__HPP */ 98 | 99 | -------------------------------------------------------------------------------- /tests/teamcity_cppunit.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2011 JetBrains s.r.o. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | * $Revision: 88625 $ 16 | */ 17 | 18 | #include 19 | 20 | #include "cppunit/Test.h" 21 | #include "cppunit/Exception.h" 22 | 23 | #include "teamcity_cppunit.h" 24 | 25 | using namespace CPPUNIT_NS; 26 | using namespace std; 27 | 28 | namespace JetBrains { 29 | 30 | TeamcityProgressListener::TeamcityProgressListener() 31 | { 32 | flowid = getFlowIdFromEnvironment(); 33 | } 34 | 35 | TeamcityProgressListener::TeamcityProgressListener(const std::string& _flowid) 36 | { 37 | flowid = _flowid; 38 | } 39 | 40 | TeamcityProgressListener::~TeamcityProgressListener() 41 | {} 42 | 43 | void TeamcityProgressListener::startTest(Test *test) { 44 | messages.testStarted(test->getName(), flowid); 45 | } 46 | 47 | static string sourceLine2string(const SourceLine &sline) { 48 | stringstream ss; 49 | 50 | ss << sline.fileName() << ":" << sline.lineNumber(); 51 | 52 | return ss.str(); 53 | } 54 | 55 | void TeamcityProgressListener::addFailure(const TestFailure &failure) { 56 | const Exception *e = failure.thrownException(); 57 | 58 | string details = e->message().details(); 59 | 60 | if (e->sourceLine().isValid()) { 61 | details.append(" at "); 62 | details.append(sourceLine2string(e->sourceLine())); 63 | details.append("\n"); 64 | } 65 | 66 | messages.testFailed( 67 | failure.failedTest()->getName(), 68 | e->message().shortDescription(), 69 | details, 70 | flowid 71 | ); 72 | } 73 | 74 | void TeamcityProgressListener::endTest(Test *test) { 75 | messages.testFinished(test->getName(), -1, flowid); 76 | } 77 | 78 | void TeamcityProgressListener::startSuite(Test *test) { 79 | messages.suiteStarted(test->getName(), flowid); 80 | } 81 | 82 | void TeamcityProgressListener::endSuite(Test *test) { 83 | messages.suiteFinished(test->getName(), flowid); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/utils.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__UTILS_HPP 21 | #define SRC__UTILS_HPP 22 | 23 | #include "libmastermind/mastermind.hpp" 24 | #include "namespace_state_p.hpp" 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | namespace mastermind { 36 | 37 | typedef std::chrono::system_clock clock_type; 38 | typedef clock_type::duration duration_type; 39 | typedef clock_type::time_point time_point_type; 40 | 41 | class spent_time_printer_t { 42 | public: 43 | spent_time_printer_t(const std::string &handler_name, std::shared_ptr &logger); 44 | 45 | ~spent_time_printer_t(); 46 | 47 | private: 48 | std::string m_handler_name; 49 | std::shared_ptr &m_logger; 50 | std::chrono::system_clock::time_point m_beg_time; 51 | }; 52 | 53 | struct fake_group_info_t { 54 | group_t id; 55 | groups_t groups; 56 | uint64_t free_effective_space; 57 | std::string ns; 58 | namespace_state_init_t::data_t::couples_t::group_info_t::status_tag group_status; 59 | }; 60 | 61 | enum GROUP_INFO_STATUS { 62 | GROUP_INFO_STATUS_OK, 63 | GROUP_INFO_STATUS_BAD, 64 | GROUP_INFO_STATUS_COUPLED 65 | }; 66 | 67 | std::string 68 | ungzip(const std::string &gzip_string); 69 | 70 | } // namespace mastermind 71 | 72 | template 73 | std::ostream &operator <<(std::ostream &ostream, const std::vector &vector) { 74 | ostream << '['; 75 | { 76 | auto beg = vector.begin(); 77 | auto end = vector.end(); 78 | for (auto it = beg; it != end; ++it) { 79 | if (it != beg) { 80 | ostream << ", "; 81 | } 82 | 83 | ostream << *it; 84 | } 85 | } 86 | ostream << ']'; 87 | 88 | return ostream; 89 | } 90 | 91 | namespace msgpack { 92 | 93 | mastermind::group_info_response_t &operator >> (object o, mastermind::group_info_response_t &v); 94 | 95 | } // namespace msgpack 96 | 97 | #endif /* SRC__UTILS_HPP */ 98 | -------------------------------------------------------------------------------- /src/couple_sequence_p.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef LIBMASTERMIND__SRC__COUPLE_SEQUENCE_P__HPP 21 | #define LIBMASTERMIND__SRC__COUPLE_SEQUENCE_P__HPP 22 | 23 | #include "libmastermind/couple_sequence.hpp" 24 | #include "couple_weights_p.hpp" 25 | 26 | #include 27 | 28 | namespace mastermind { 29 | 30 | class couple_sequence_const_iterator_t::data_t 31 | { 32 | public: 33 | data_t(ns_state::weight::weighted_couples_info_t weighted_couples_info_) 34 | : weighted_couples_info(std::move(weighted_couples_info_)) 35 | , current_index(0) 36 | { 37 | try_extract_next(); 38 | } 39 | 40 | void 41 | try_extract_next() { 42 | if (weighted_couples_info.empty()) { 43 | return; 44 | } 45 | 46 | auto total_weight = weighted_couples_info.back().weight; 47 | double shoot_point = double(random()) / RAND_MAX * total_weight; 48 | auto it = std::lower_bound(weighted_couples_info.begin() 49 | , weighted_couples_info.end(), uint64_t(shoot_point)); 50 | 51 | couple_info_t couple_info; 52 | couple_info.id = it->couple_info->id; 53 | couple_info.groups = it->couple_info->groups; 54 | couples_info.emplace_back(couple_info); 55 | 56 | weighted_couples_info.erase(it); 57 | } 58 | 59 | ns_state::weight::weighted_couples_info_t weighted_couples_info; 60 | std::vector couples_info; 61 | size_t current_index; 62 | }; 63 | 64 | class couple_sequence_t::data_t 65 | { 66 | public: 67 | data_t(ns_state::weight::weighted_couples_info_t weighted_couples_info_) 68 | : weighted_couples_info(std::move(weighted_couples_info_)) 69 | {} 70 | 71 | ns_state::weight::weighted_couples_info_t weighted_couples_info; 72 | }; 73 | 74 | #define INIT_CLASS(name) \ 75 | class name##_init_t : public name##_t \ 76 | { \ 77 | public: \ 78 | typedef name##_t::data_t data_t; \ 79 | \ 80 | name##_init_t(std::shared_ptr data_) { \ 81 | data = std::move(data_); \ 82 | } \ 83 | } 84 | 85 | INIT_CLASS(couple_sequence_const_iterator); 86 | INIT_CLASS(couple_sequence); 87 | 88 | } // namespace mastermind 89 | 90 | #endif /* LIBMASTERMIND__SRC__COUPLE_SEQUENCE_P__HPP */ 91 | 92 | -------------------------------------------------------------------------------- /src/couple_weights_p.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "libmastermind/mastermind.hpp" 21 | 22 | #include "cocaine/traits/dynamic.hpp" 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | #include 29 | #include 30 | #include 31 | 32 | #ifndef LIBMASTERMIND__SRC__COUPLE_WEIGHTS_P__HPP 33 | #define LIBMASTERMIND__SRC__COUPLE_WEIGHTS_P__HPP 34 | 35 | namespace mastermind { 36 | namespace ns_state { 37 | 38 | namespace weight { 39 | 40 | class couple_info_t { 41 | public: 42 | couple_info_t() 43 | : id (-1) 44 | , weight(0) 45 | , memory(0) 46 | , coefficient(1) 47 | {} 48 | 49 | groups_t groups; 50 | group_t id; 51 | uint64_t weight; 52 | uint64_t memory; 53 | double coefficient; 54 | 55 | private: 56 | }; 57 | 58 | bool 59 | memory_comparator(const couple_info_t &lhs, const couple_info_t &rhs); 60 | 61 | typedef std::vector couples_info_t; 62 | 63 | class weighted_couple_info_t { 64 | public: 65 | typedef couples_info_t::const_iterator const_iterator_t; 66 | 67 | weighted_couple_info_t(uint64_t weight_, const_iterator_t couple_info_) 68 | : weight(weight_) 69 | , couple_info(std::move(couple_info_)) 70 | {} 71 | 72 | uint64_t weight; 73 | const_iterator_t couple_info; 74 | 75 | friend 76 | bool 77 | operator < (const weighted_couple_info_t &lhs, const weighted_couple_info_t &rhs) { 78 | return lhs.weight < rhs.weight; 79 | } 80 | 81 | friend 82 | bool 83 | operator < (const weighted_couple_info_t &lhs, const uint64_t &rhs_weight) { 84 | return lhs.weight < rhs_weight; 85 | } 86 | 87 | private: 88 | }; 89 | 90 | typedef std::vector weighted_couples_info_t; 91 | 92 | struct weights_t { 93 | weights_t(const kora::config_t &config, size_t groups_count_, bool ns_is_static_); 94 | 95 | weights_t(weights_t &&other); 96 | 97 | couple_info_t 98 | get(uint64_t size) const; 99 | 100 | weighted_couples_info_t 101 | get_all(uint64_t size) const; 102 | 103 | const couples_info_t & 104 | data() const; 105 | 106 | void 107 | set_coefficient(group_t couple_id, double coefficient); 108 | 109 | private: 110 | typedef std::mutex mutex_t; 111 | typedef std::lock_guard lock_guard_t; 112 | 113 | static 114 | couples_info_t 115 | create(const kora::config_t &config, size_t groups_count, bool ns_is_static); 116 | 117 | const size_t groups_count; 118 | couples_info_t couples_info; 119 | mutable mutex_t couples_info_mutex; 120 | }; 121 | 122 | } // namespace weight 123 | } // namespace ns_state 124 | } // namespace mastermind 125 | 126 | #endif /* LIBMASTERMIND__SRC__COUPLE_WEIGHTS_P__HPP */ 127 | 128 | -------------------------------------------------------------------------------- /tests/manager.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gpath=`pwd`/nodes; 4 | 5 | function check_node { 6 | g=1 7 | while [ "$g" -eq 1 ]; do 8 | dnet_ioclient -r localhost:$((1024 + $1)):2 2> manager.out 9 | res=$? 10 | #echo $res 11 | # starting 12 | if [ "$2" -eq 1 ] && [ "$res" -eq 0 ]; then 13 | g=0; 14 | elif [ "$2" -eq 2 ] && [ "$res" -ne 0 ]; then 15 | g=0; 16 | fi 17 | done 18 | } 19 | 20 | function start_node { 21 | path=$gpath/$1 22 | if ! [ -f $path/pid ]; then 23 | echo starting node $1... 24 | dnet_ioserv -c $path/elliptics.conf & 25 | echo $! > $path/pid 26 | check_node $1 1; 27 | echo " done" 28 | fi 29 | } 30 | 31 | function stop_node { 32 | path=$gpath/$1 33 | if [ -f $path/pid ]; then 34 | echo stopping node $1... 35 | kill `cat $path/pid` 36 | rm $path/pid 37 | check_node $1 2; 38 | echo " done" 39 | fi 40 | } 41 | 42 | function prepare_env { 43 | if [ "$#" -ne 2 ]; then 44 | exit 3; 45 | fi 46 | 47 | mkdir $gpath; 48 | echo $2 > $gpath/count; 49 | 50 | for index in `seq 1 $2`; do 51 | path=$gpath/$index; 52 | rm -rf $path; 53 | mkdir -p $path; 54 | mkdir $path/data; 55 | mkdir $path/kdb; 56 | filename="$path/elliptics.conf"; 57 | echo "" > $filename; 58 | echo "log = $path/log" >> $filename; 59 | echo "group = $index" >> $filename; 60 | echo "history = $path/kdb" >> $filename; 61 | echo "io_thread_num = 250" >> $filename; 62 | echo "net_thread_num = 100" >> $filename; 63 | echo "nonblocking_io_thread_num = 50" >> $filename; 64 | echo "log_level = 2" >> $filename; 65 | echo "join = 1" >> $filename; 66 | echo "remote = localhost:1025:2" >> $filename; 67 | echo "addr = localhost:$((1024+$index)):2" >> $filename; 68 | echo "wait_timeout = 30" >> $filename; 69 | echo "check_timeout = 50" >> $filename; 70 | echo "auth_cookie = unique_storage_cookie" >> $filename; 71 | echo "cache_size =" >> $filename; 72 | echo "server_net_prio = 0x20" >> $filename; 73 | echo "client_net_prio = 6" >> $filename; 74 | echo "flags = 8" >> $filename; 75 | echo "backend = blob" >> $filename; 76 | echo "blob_size = 10M" >> $filename; 77 | echo "records_in_blob = 5000000" >> $filename; 78 | echo "blob_flags = 2" >> $filename; 79 | echo "blob_cache_size = 0" >> $filename; 80 | echo "defrag_timeout = 3600" >> $filename; 81 | echo "defrag_percentage = 25" >> $filename; 82 | echo "sync = 30" >> $filename; 83 | echo "data = $path/data" >> $filename; 84 | done; 85 | } 86 | 87 | function start_nodes { 88 | if [ "$#" -eq 1 ]; then 89 | c=`cat $gpath/count`; 90 | for index in `seq 1 $c`; do 91 | start_node $index; 92 | done; 93 | elif [ "$2" -ge 1 ] && [ "$2" -le 3 ]; then 94 | start_node $2; 95 | fi 96 | } 97 | 98 | function stop_nodes { 99 | if [ "$#" -eq 1 ]; then 100 | c=`cat $gpath/count`; 101 | for index in `seq 1 $c`; do 102 | stop_node $index; 103 | done; 104 | elif [ "$2" -ge 1 ] && [ "$2" -le 3 ]; then 105 | stop_node $2; 106 | fi 107 | } 108 | 109 | function clear_env { 110 | stop_nodes $@; 111 | rm -rf $gpath; 112 | } 113 | 114 | if [ "$#" -lt 1 ]; then 115 | echo not enough args 116 | exit 1 117 | fi 118 | 119 | if test $1 = "prepare"; then 120 | clear_env $@; 121 | prepare_env $@; 122 | exit 0; 123 | elif test $1 = "clear"; then 124 | clear_env $@; 125 | exit 0; 126 | elif test $1 = "start"; then 127 | start_nodes $@; 128 | exit 0; 129 | elif test $1 = "stop"; then 130 | stop_nodes $@; 131 | exit 0; 132 | fi 133 | 134 | echo unknown command 135 | exit 2 136 | -------------------------------------------------------------------------------- /src/utils.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "namespace_p.hpp" 21 | #include "utils.hpp" 22 | #include "libmastermind/error.hpp" 23 | 24 | #include 25 | 26 | #include 27 | #include 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | namespace mastermind { 40 | 41 | spent_time_printer_t::spent_time_printer_t(const std::string &handler_name, std::shared_ptr &logger) 42 | : m_handler_name(handler_name) 43 | , m_logger(logger) 44 | , m_beg_time(std::chrono::system_clock::now()) 45 | { 46 | COCAINE_LOG_DEBUG(m_logger, "libmastermind: handling \'%s\'", m_handler_name.c_str()); 47 | } 48 | 49 | spent_time_printer_t::~spent_time_printer_t() { 50 | auto end_time = std::chrono::system_clock::now(); 51 | COCAINE_LOG_INFO(m_logger, "libmastermind: time spent for \'%s\': %d milliseconds" 52 | , m_handler_name.c_str() 53 | , static_cast(std::chrono::duration_cast(end_time - m_beg_time).count()) 54 | ); 55 | } 56 | 57 | std::string 58 | ungzip(const std::string &gzip_string) { 59 | namespace bi = boost::iostreams; 60 | 61 | bi::filtering_streambuf input; 62 | input.push(bi::gzip_decompressor()); 63 | 64 | std::istringstream iss(gzip_string); 65 | input.push(iss); 66 | 67 | std::ostringstream oss; 68 | bi::copy(input, std::ostreambuf_iterator(oss)); 69 | return oss.str(); 70 | } 71 | 72 | } // namespace mastermind 73 | 74 | namespace msgpack { 75 | mastermind::group_info_response_t &operator >> (object o, mastermind::group_info_response_t &v) { 76 | if (o.type != type::MAP) { 77 | throw type_error(); 78 | } 79 | 80 | msgpack::object_kv *p = o.via.map.ptr; 81 | msgpack::object_kv *const pend = o.via.map.ptr + o.via.map.size; 82 | 83 | for (; p < pend; ++p) { 84 | std::string key; 85 | 86 | p->key.convert(&key); 87 | 88 | // if (!key.compare("nodes")) { 89 | // p->val.convert(&(v.nodes)); 90 | // } 91 | if (!key.compare("couples")) { 92 | p->val.convert(&(v.couples)); 93 | } 94 | else if (!key.compare("status")) { 95 | std::string status; 96 | p->val.convert(&status); 97 | if (!status.compare("bad")) { 98 | v.status = mastermind::GROUP_INFO_STATUS_BAD; 99 | } else if (!status.compare("coupled")) { 100 | v.status = mastermind::GROUP_INFO_STATUS_COUPLED; 101 | } 102 | } else if (!key.compare("namespace")) { 103 | p->val.convert(&v.name_space); 104 | } 105 | } 106 | 107 | return v; 108 | } 109 | 110 | } // namespace msgpack 111 | -------------------------------------------------------------------------------- /src/cached_keys.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef LIBMASTERMIND__SRC__CACHED_KEY__HPP 21 | #define LIBMASTERMIND__SRC__CACHED_KEY__HPP 22 | 23 | #include "libmastermind/common.hpp" 24 | 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | namespace mastermind { 33 | 34 | class cached_keys_t { 35 | public: 36 | cached_keys_t() 37 | { 38 | } 39 | 40 | cached_keys_t(const kora::dynamic_t &dynamic) 41 | : groups_map(create_groups_map(dynamic)) 42 | { 43 | } 44 | 45 | groups_t 46 | get(const std::string &key, const std::string &couple_id) const { 47 | auto key_it = groups_map.find(key); 48 | 49 | if (groups_map.end() == key_it) { 50 | return {}; 51 | } 52 | 53 | auto couple_id_it = key_it->second.find(couple_id); 54 | 55 | if (key_it->second.end() == couple_id_it) { 56 | return {}; 57 | } 58 | 59 | return couple_id_it->second; 60 | } 61 | 62 | groups_t 63 | get(const std::string &key, group_t couple_id) const { 64 | return get(key, boost::lexical_cast(couple_id)); 65 | } 66 | 67 | private: 68 | typedef std::map> groups_map_t; 69 | 70 | static 71 | groups_map_t 72 | create_groups_map(const kora::dynamic_t &dynamic) { 73 | try { 74 | const auto &dynamic_keys = dynamic.as_object(); 75 | 76 | auto groups_map = groups_map_t{}; 77 | for (auto key_it = dynamic_keys.begin(), key_end = dynamic_keys.end() 78 | ; key_it != key_end; ++key_it) { 79 | const auto &dynamic_couple_ids = key_it->second.as_object(); 80 | 81 | auto couple_id_map = std::map{}; 82 | for (auto couple_id_it = dynamic_couple_ids.begin() 83 | , couple_id_end = dynamic_couple_ids.end() 84 | ; couple_id_it != couple_id_end; ++couple_id_it) { 85 | const auto &dynamic_info = couple_id_it->second.as_object(); 86 | const auto &dynamic_cache_groups = dynamic_info.at("cache_groups").as_array(); 87 | 88 | auto groups = groups_t{}; 89 | for (auto cache_groups_it = dynamic_cache_groups.begin() 90 | , cache_groups_end = dynamic_cache_groups.end() 91 | ; cache_groups_it != cache_groups_end; ++cache_groups_it) { 92 | groups.emplace_back(cache_groups_it->to()); 93 | } 94 | couple_id_map.insert(std::make_pair(couple_id_it->first, groups)); 95 | } 96 | 97 | groups_map.insert(std::make_pair(key_it->first, couple_id_map)); 98 | } 99 | 100 | return groups_map; 101 | } catch (const std::exception &ex) { 102 | throw std::runtime_error(std::string("cached_keys parse error: ") + ex.what()); 103 | } 104 | } 105 | 106 | groups_map_t groups_map; 107 | }; 108 | 109 | } // namespace mastermind 110 | 111 | #endif /* LIBMASTERMIND__SRC__CACHED_KEY__HPP */ 112 | 113 | -------------------------------------------------------------------------------- /src/couple_sequence.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "libmastermind/couple_sequence.hpp" 21 | #include "couple_sequence_p.hpp" 22 | 23 | mastermind::couple_sequence_const_iterator_t::couple_sequence_const_iterator_t() 24 | {} 25 | 26 | mastermind::couple_sequence_const_iterator_t::couple_sequence_const_iterator_t( 27 | const self_type &that) 28 | { 29 | if (that.data) { 30 | data = std::make_shared(*that.data); 31 | } 32 | } 33 | 34 | mastermind::couple_sequence_const_iterator_t::reference 35 | mastermind::couple_sequence_const_iterator_t::operator * () const { 36 | return data->couples_info[data->current_index]; 37 | } 38 | 39 | mastermind::couple_sequence_const_iterator_t::pointer 40 | mastermind::couple_sequence_const_iterator_t::operator -> () const { 41 | return &data->couples_info[data->current_index]; 42 | } 43 | 44 | bool 45 | mastermind::couple_sequence_const_iterator_t::operator == (const self_type &other) const { 46 | if (data && other.data) { 47 | return data->current_index == other.data->current_index; 48 | } 49 | 50 | if (!data && !other.data) { 51 | return true; 52 | } 53 | 54 | auto d = (data ? data : other.data); 55 | 56 | if (d->current_index == d->couples_info.size() 57 | && d->weighted_couples_info.empty()) { 58 | return true; 59 | } 60 | 61 | return false; 62 | } 63 | 64 | bool 65 | mastermind::couple_sequence_const_iterator_t::operator != (const self_type &other) const { 66 | return !(*this == other); 67 | } 68 | 69 | mastermind::couple_sequence_const_iterator_t::self_type & 70 | mastermind::couple_sequence_const_iterator_t::operator ++ () { 71 | data->try_extract_next(); 72 | data->current_index += 1; 73 | return *this; 74 | } 75 | 76 | mastermind::couple_sequence_const_iterator_t::self_type 77 | mastermind::couple_sequence_const_iterator_t::operator ++ (int) { 78 | self_type result; 79 | result.data = std::make_shared(*data); 80 | ++*this; 81 | return result; 82 | } 83 | 84 | mastermind::couple_sequence_const_iterator_t::self_type & 85 | mastermind::couple_sequence_const_iterator_t::operator = (const self_type &that) { 86 | data = std::make_shared(*that.data); 87 | return *this; 88 | } 89 | 90 | mastermind::couple_sequence_t::const_iterator 91 | mastermind::couple_sequence_t::begin() const { 92 | if (!data) { 93 | return end(); 94 | } 95 | 96 | auto d = std::make_shared( 97 | data->weighted_couples_info); 98 | return couple_sequence_const_iterator_init_t(std::move(d)); 99 | } 100 | 101 | mastermind::couple_sequence_t::const_iterator 102 | mastermind::couple_sequence_t::end() const { 103 | return couple_sequence_const_iterator_init_t(std::shared_ptr()); 104 | } 105 | 106 | size_t 107 | mastermind::couple_sequence_t::size() const { 108 | if (!data) { 109 | return 0; 110 | } 111 | 112 | return data->weighted_couples_info.size(); 113 | } 114 | 115 | -------------------------------------------------------------------------------- /tests/teamcity_messages.cpp: -------------------------------------------------------------------------------- 1 | /* Copyright 2011 JetBrains s.r.o. 2 | * 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | * 15 | * $Revision: 88625 $ 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #include "teamcity_messages.h" 22 | 23 | using namespace std; 24 | 25 | namespace JetBrains { 26 | 27 | std::string getFlowIdFromEnvironment() { 28 | const char *flowId = getenv("TEAMCITY_PROCESS_FLOW_ID"); 29 | return flowId == NULL ? "" : flowId; 30 | } 31 | 32 | bool underTeamcity() { 33 | return getenv("TEAMCITY_PROJECT_NAME") != NULL; 34 | } 35 | 36 | TeamcityMessages::TeamcityMessages() 37 | : m_out(&cout) 38 | {} 39 | 40 | void TeamcityMessages::setOutput(ostream &out) { 41 | m_out = &out; 42 | } 43 | 44 | string TeamcityMessages::escape(string s) { 45 | string result; 46 | 47 | for (size_t i = 0; i < s.length(); i++) { 48 | char c = s[i]; 49 | 50 | switch (c) { 51 | case '\n': result.append("|n"); break; 52 | case '\r': result.append("|r"); break; 53 | case '\'': result.append("|'"); break; 54 | case '|': result.append("||"); break; 55 | case ']': result.append("|]"); break; 56 | default: result.append(&c, 1); 57 | } 58 | } 59 | 60 | return result; 61 | } 62 | 63 | void TeamcityMessages::openMsg(const string &name) { 64 | // endl for http://jetbrains.net/tracker/issue/TW-4412 65 | *m_out << endl << "##teamcity[" << name; 66 | } 67 | 68 | void TeamcityMessages::closeMsg() { 69 | *m_out << "]"; 70 | // endl for http://jetbrains.net/tracker/issue/TW-4412 71 | *m_out << endl; 72 | m_out->flush(); 73 | } 74 | 75 | void TeamcityMessages::writeProperty(string name, string value) { 76 | *m_out << " " << name << "='" << escape(value) << "'"; 77 | } 78 | 79 | void TeamcityMessages::suiteStarted(string name, string flowid) { 80 | openMsg("testSuiteStarted"); 81 | writeProperty("name", name); 82 | if(flowid.length() > 0) { 83 | writeProperty("flowId", flowid); 84 | } 85 | 86 | closeMsg(); 87 | } 88 | 89 | void TeamcityMessages::suiteFinished(string name, string flowid) { 90 | openMsg("testSuiteFinished"); 91 | writeProperty("name", name); 92 | if(flowid.length() > 0) { 93 | writeProperty("flowId", flowid); 94 | } 95 | 96 | closeMsg(); 97 | } 98 | 99 | void TeamcityMessages::testStarted(string name, string flowid) { 100 | openMsg("testStarted"); 101 | writeProperty("name", name); 102 | if(flowid.length() > 0) { 103 | writeProperty("flowId", flowid); 104 | } 105 | 106 | closeMsg(); 107 | } 108 | 109 | void TeamcityMessages::testFinished(string name, int durationMs, string flowid) { 110 | openMsg("testFinished"); 111 | 112 | writeProperty("name", name); 113 | 114 | if(flowid.length() > 0) { 115 | writeProperty("flowId", flowid); 116 | } 117 | 118 | if(durationMs >= 0) { 119 | stringstream out; 120 | out << durationMs; 121 | writeProperty("duration", out.str()); 122 | } 123 | 124 | closeMsg(); 125 | } 126 | 127 | void TeamcityMessages::testFailed(string name, string message, string details, string flowid) { 128 | openMsg("testFailed"); 129 | writeProperty("name", name); 130 | writeProperty("message", message); 131 | writeProperty("details", details); 132 | if(flowid.length() > 0) { 133 | writeProperty("flowId", flowid); 134 | } 135 | 136 | closeMsg(); 137 | } 138 | 139 | void TeamcityMessages::testIgnored(std::string name, std::string message, string flowid) { 140 | openMsg("testIgnored"); 141 | writeProperty("name", name); 142 | writeProperty("message", message); 143 | if(flowid.length() > 0) { 144 | writeProperty("flowId", flowid); 145 | } 146 | 147 | closeMsg(); 148 | } 149 | 150 | } 151 | -------------------------------------------------------------------------------- /src/namespace_state_p.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__NAMESPACE_STATE_P__HPP 21 | #define SRC__NAMESPACE_STATE_P__HPP 22 | 23 | #include "libmastermind/mastermind.hpp" 24 | #include "couple_weights_p.hpp" 25 | 26 | #include "cocaine/traits/dynamic.hpp" 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | namespace mastermind { 36 | 37 | class namespace_state_t::data_t { 38 | public: 39 | struct settings_t { 40 | settings_t(const std::string &name, const kora::config_t &config 41 | , const user_settings_factory_t &factory); 42 | 43 | settings_t(settings_t &&other); 44 | 45 | size_t groups_count; 46 | std::string success_copies_num; 47 | 48 | struct auth_keys_t { 49 | std::string read; 50 | std::string write; 51 | } auth_keys; 52 | 53 | groups_t static_groups; 54 | 55 | user_settings_ptr_t user_settings_ptr; 56 | }; 57 | 58 | struct couples_t { 59 | struct group_info_t; 60 | struct groupset_info_t; 61 | struct couple_info_t; 62 | 63 | typedef std::map group_info_map_t; 64 | typedef std::map groupset_info_map_t; 65 | typedef std::map couple_info_map_t; 66 | 67 | typedef group_info_map_t::const_iterator group_info_map_iterator_t; 68 | typedef groupset_info_map_t::const_iterator groupset_map_iterator_t; 69 | typedef couple_info_map_t::const_iterator couple_info_map_iterator_t; 70 | 71 | struct group_info_t { 72 | enum class status_tag { 73 | UNKNOWN, COUPLED 74 | }; 75 | 76 | group_t id; 77 | 78 | status_tag status; 79 | 80 | couple_info_map_iterator_t couple_info_map_iterator; 81 | }; 82 | 83 | struct groupset_info_t { 84 | enum class type_tag { 85 | UNKNOWN, LRC 86 | }; 87 | 88 | enum class status_tag { 89 | UNKNOWN, BAD 90 | }; 91 | 92 | std::string id; 93 | 94 | groups_t groups; 95 | type_tag type; 96 | status_tag status; 97 | uint64_t free_effective_space; 98 | uint64_t free_reserved_space; 99 | 100 | kora::dynamic_t hosts; 101 | kora::dynamic_t settings; 102 | }; 103 | 104 | struct couple_info_t { 105 | enum class status_tag { 106 | UNKNOWN, BAD 107 | }; 108 | 109 | std::string id; 110 | 111 | groups_t groups; 112 | status_tag status; 113 | uint64_t free_effective_space; 114 | uint64_t free_reserved_space; 115 | 116 | kora::dynamic_t hosts; 117 | 118 | std::vector groups_info_map_iterator; 119 | 120 | groupset_info_map_t groupset_info_map; 121 | std::vector read_preference; 122 | }; 123 | 124 | couples_t(const kora::config_t &config); 125 | 126 | couples_t(couples_t &&other); 127 | 128 | group_info_map_t group_info_map; 129 | couple_info_map_t couple_info_map; 130 | }; 131 | 132 | struct statistics_t { 133 | statistics_t(const kora::config_t &config); 134 | 135 | bool 136 | ns_is_full() const; 137 | 138 | private: 139 | bool is_full; 140 | }; 141 | 142 | /*struct weights_t { 143 | weights_t(const kora::config_t &config, size_t groups_count_) {} 144 | groups_t get(size_t size) const { return groups_t(); } 145 | };*/ 146 | 147 | data_t(std::string name, const kora::config_t &config 148 | , const user_settings_factory_t &factory); 149 | 150 | data_t(data_t &&other); 151 | 152 | void check_consistency(); 153 | 154 | std::string name; 155 | 156 | settings_t settings; 157 | couples_t couples; 158 | ns_state::weight::weights_t weights; 159 | statistics_t statistics; 160 | 161 | std::string extract; 162 | }; 163 | 164 | class namespace_state_init_t 165 | : public namespace_state_t { 166 | public: 167 | namespace_state_init_t(std::shared_ptr data_); 168 | 169 | struct data_t : namespace_state_t::data_t { 170 | data_t(std::string name, const kora::config_t &config 171 | , const user_settings_factory_t &factory); 172 | 173 | data_t(data_t &&other); 174 | }; 175 | 176 | }; 177 | 178 | 179 | } // namespace mastermind 180 | 181 | #endif /* SRC__NAMESPACE_STATE_P__HPP */ 182 | 183 | -------------------------------------------------------------------------------- /include/libmastermind/error.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef INCLUDE__LIBMASTERMIND__ERROR_H 21 | #define INCLUDE__LIBMASTERMIND__ERROR_H 22 | 23 | #include 24 | 25 | #include 26 | 27 | namespace mastermind { 28 | 29 | namespace libmastermind_error { 30 | 31 | enum libmastermind_error_t { 32 | couple_not_found, 33 | not_enough_memory, 34 | unknown_namespace, 35 | invalid_groups_count, 36 | cache_is_expired 37 | }; 38 | 39 | } // namespace libmastermind_error 40 | 41 | class libmastermind_category_impl 42 | : public std::error_category 43 | { 44 | public: 45 | const char *name() const noexcept; 46 | std::string message(int ev) const; 47 | }; 48 | 49 | const std::error_category &libmastermind_category(); 50 | 51 | std::error_code make_error_code(libmastermind_error::libmastermind_error_t e); 52 | std::error_condition make_error_condition(libmastermind_error::libmastermind_error_t e); 53 | 54 | //IMPORTANT: ensure that the following exception list is in sync with 55 | // the exception list in bindings/python/main.cpp:init_exception_translator() 56 | 57 | class couple_not_found_error 58 | : public std::system_error 59 | { 60 | public: 61 | couple_not_found_error(); 62 | }; 63 | 64 | class not_enough_memory_error 65 | : public std::system_error 66 | { 67 | public: 68 | not_enough_memory_error(); 69 | }; 70 | 71 | class unknown_namespace_error 72 | : public std::system_error 73 | { 74 | public: 75 | unknown_namespace_error(); 76 | }; 77 | 78 | class invalid_groups_count_error 79 | : public std::system_error 80 | { 81 | public: 82 | invalid_groups_count_error(); 83 | }; 84 | 85 | class cache_is_expired_error 86 | : public std::system_error 87 | { 88 | public: 89 | cache_is_expired_error(); 90 | }; 91 | 92 | enum class mastermind_errc { 93 | update_loop_already_started = 1 94 | , update_loop_already_stopped 95 | , unknown_feedback 96 | , namespace_state_not_found 97 | , unknown_group 98 | , unknown_groupset 99 | , remotes_empty 100 | }; 101 | 102 | const std::error_category & 103 | mastermind_category(); 104 | 105 | } // namespace mastermind 106 | 107 | namespace std { 108 | 109 | template<> 110 | struct is_error_code_enum 111 | : public true_type 112 | {}; 113 | 114 | template<> 115 | struct is_error_code_enum 116 | : public true_type 117 | {}; 118 | 119 | std::error_code 120 | make_error_code(mastermind::mastermind_errc e); 121 | 122 | std::error_condition 123 | make_error_condition(mastermind::mastermind_errc e); 124 | 125 | } // namespace std 126 | 127 | namespace mastermind { 128 | 129 | class mastermind_error : public std::runtime_error 130 | { 131 | public: 132 | mastermind_error(std::error_code error_code_); 133 | 134 | const std::error_code & 135 | code() const; 136 | 137 | private: 138 | std::error_code error_code; 139 | }; 140 | 141 | class update_loop_already_started : public mastermind_error 142 | { 143 | public: 144 | update_loop_already_started(); 145 | }; 146 | 147 | class update_loop_already_stopped : public mastermind_error 148 | { 149 | public: 150 | update_loop_already_stopped(); 151 | }; 152 | 153 | class unknown_feedback : public mastermind_error 154 | { 155 | public: 156 | unknown_feedback(group_t couple_id_, int feedback_); 157 | 158 | group_t 159 | couple_id() const; 160 | 161 | int 162 | feedback() const; 163 | 164 | private: 165 | group_t m_couple_id; 166 | int m_feedback; 167 | }; 168 | 169 | class namespace_state_not_found_error : public mastermind_error 170 | { 171 | public: 172 | namespace_state_not_found_error(); 173 | }; 174 | 175 | class unknown_group_error : public mastermind_error 176 | { 177 | public: 178 | unknown_group_error(int group_); 179 | 180 | int 181 | group() const; 182 | 183 | private: 184 | int m_group; 185 | }; 186 | 187 | class unknown_groupset_error : public mastermind_error 188 | { 189 | public: 190 | unknown_groupset_error(std::string groupset_); 191 | ~unknown_groupset_error() noexcept {} 192 | 193 | const std::string & 194 | groupset() const; 195 | 196 | private: 197 | std::string m_groupset; 198 | }; 199 | 200 | class remotes_empty_error : public mastermind_error 201 | { 202 | public: 203 | remotes_empty_error(); 204 | }; 205 | 206 | } // namespace mastermind 207 | 208 | #endif /* INCLUDE__LIBMASTERMIND__ERROR_H */ 209 | -------------------------------------------------------------------------------- /src/couple_weights.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "couple_weights_p.hpp" 21 | #include "utils.hpp" 22 | 23 | #include 24 | 25 | #include 26 | #include 27 | #include 28 | 29 | namespace mastermind { 30 | namespace ns_state { 31 | namespace weight { 32 | 33 | bool 34 | memory_comparator(const couple_info_t &lhs, const couple_info_t &rhs) { 35 | return lhs.memory > rhs.memory; 36 | } 37 | 38 | weights_t::weights_t(const kora::config_t &config 39 | , size_t groups_count_, bool ns_is_static_) 40 | try 41 | : groups_count(groups_count_) 42 | , couples_info(create(config, groups_count, ns_is_static_)) 43 | { 44 | } catch (const std::exception &ex) { 45 | throw std::runtime_error(std::string("cannot create weights-state: ") + ex.what()); 46 | } 47 | 48 | weights_t::weights_t(weights_t &&other) 49 | : groups_count(other.groups_count) 50 | , couples_info(std::move(other.couples_info)) 51 | { 52 | } 53 | 54 | couples_info_t 55 | weights_t::create( 56 | const kora::config_t &config, size_t groups_count, bool ns_is_static) { 57 | const auto key = boost::lexical_cast(groups_count); 58 | 59 | const auto &object = config.underlying_object().as_object(); 60 | auto it = object.find(key); 61 | 62 | if (object.end() == it) { 63 | // TODO: log 64 | return {}; 65 | } 66 | 67 | if (!it->second.is_array()) { 68 | // TODO: log 69 | return {}; 70 | } 71 | 72 | const auto &couples = it->second.as_array(); 73 | couples_info_t couples_info; 74 | 75 | for (auto it = couples.begin(), end = couples.end(); it != end; ++it) { 76 | try { 77 | const auto &couple = it->as_array(); 78 | couple_info_t couple_info; 79 | 80 | { 81 | const auto &dynamic_groups = couple[0].as_array(); 82 | 83 | for (auto it = dynamic_groups.begin(), end = dynamic_groups.end(); 84 | it != end; ++it) { 85 | couple_info.groups.emplace_back(it->to()); 86 | } 87 | } 88 | 89 | couple_info.weight = couple[1].to(); 90 | couple_info.memory = couple[2].to(); 91 | couple_info.id = *std::min_element(couple_info.groups.begin(), couple_info.groups.end()); 92 | 93 | couples_info.emplace_back(std::move(couple_info)); 94 | } catch (const std::exception &ex) { 95 | // TODO: log 96 | } 97 | } 98 | 99 | std::sort(couples_info.begin(), couples_info.end(), memory_comparator); 100 | 101 | return couples_info; 102 | } 103 | 104 | couple_info_t 105 | weights_t::get(uint64_t size) const { 106 | auto weighted_groups = get_all(size); 107 | 108 | auto total_weight = weighted_groups.back().weight; 109 | double shoot_point = double(random()) / RAND_MAX * total_weight; 110 | auto it = std::lower_bound(weighted_groups.begin(), weighted_groups.end() 111 | , uint64_t(shoot_point)); 112 | 113 | if (it == weighted_groups.end()) { 114 | throw couple_not_found_error(); 115 | } 116 | 117 | return *it->couple_info; 118 | } 119 | 120 | weighted_couples_info_t 121 | weights_t::get_all(uint64_t size) const { 122 | weighted_couples_info_t weighted_couples_info; 123 | weighted_couples_info.reserve(couples_info.size()); 124 | uint64_t total_weight = 0; 125 | 126 | { 127 | lock_guard_t lock_guard(couples_info_mutex); 128 | for (auto it = couples_info.begin(), end = couples_info.end(); 129 | it != end; ++it) { 130 | 131 | if (it->memory < size) { 132 | break; 133 | } 134 | 135 | uint64_t weight = it->weight * it->coefficient; 136 | 137 | if (weight == 0) { 138 | continue; 139 | } 140 | 141 | total_weight += weight; 142 | 143 | weighted_couples_info.emplace_back(total_weight, it); 144 | } 145 | } 146 | 147 | if (weighted_couples_info.empty()) { 148 | throw not_enough_memory_error(); 149 | } 150 | 151 | return weighted_couples_info; 152 | } 153 | 154 | const couples_info_t & 155 | weights_t::data() const { 156 | return couples_info; 157 | } 158 | 159 | void 160 | weights_t::set_coefficient(group_t couple_id, double coefficient) { 161 | for (auto it = couples_info.begin(), end = couples_info.end(); it != end; ++it) { 162 | const auto &groups = it->groups; 163 | auto cit = std::find(groups.begin(), groups.end(), couple_id); 164 | 165 | if (cit != groups.end()) { 166 | lock_guard_t lock_guard(couples_info_mutex); 167 | it->coefficient = std::min(it->coefficient, coefficient); 168 | break; 169 | } 170 | } 171 | } 172 | 173 | } // namespace weight 174 | } // namespace ns_state 175 | } // namespace mastermind 176 | 177 | -------------------------------------------------------------------------------- /src/namespace.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "namespace_p.hpp" 21 | 22 | namespace mastermind { 23 | 24 | namespace_settings_t::data::data() 25 | : redirect_expire_time(0) 26 | , redirect_content_length_threshold(-1) 27 | , is_active(false) 28 | , can_choose_couple_to_upload(false) 29 | , multipart_content_length_threshold(0) 30 | { 31 | } 32 | 33 | namespace_settings_t::data::data(const namespace_settings_t::data &d) 34 | : name(d.name) 35 | , groups_count(d.groups_count) 36 | , success_copies_num(d.success_copies_num) 37 | , auth_key(d.auth_key) 38 | , static_couple(d.static_couple) 39 | , auth_key_for_write(d.auth_key_for_write) 40 | , auth_key_for_read(d.auth_key_for_read) 41 | , sign_token(d.sign_token) 42 | , sign_path_prefix(d.sign_path_prefix) 43 | , sign_port(d.sign_port) 44 | , redirect_expire_time(d.redirect_expire_time) 45 | , redirect_content_length_threshold(d.redirect_content_length_threshold) 46 | , is_active(d.is_active) 47 | , can_choose_couple_to_upload(d.can_choose_couple_to_upload) 48 | , multipart_content_length_threshold(d.multipart_content_length_threshold) 49 | { 50 | } 51 | 52 | namespace_settings_t::data::data(namespace_settings_t::data &&d) 53 | : name(std::move(d.name)) 54 | , groups_count(std::move(d.groups_count)) 55 | , success_copies_num(std::move(d.success_copies_num)) 56 | , auth_key(std::move(d.auth_key)) 57 | , static_couple(std::move(d.static_couple)) 58 | , auth_key_for_write(std::move(d.auth_key_for_write)) 59 | , auth_key_for_read(std::move(d.auth_key_for_read)) 60 | , sign_token(std::move(d.sign_token)) 61 | , sign_path_prefix(std::move(d.sign_path_prefix)) 62 | , sign_port(std::move(d.sign_port)) 63 | , redirect_expire_time(d.redirect_expire_time) 64 | , redirect_content_length_threshold(d.redirect_content_length_threshold) 65 | , is_active(d.is_active) 66 | , can_choose_couple_to_upload(d.can_choose_couple_to_upload) 67 | , multipart_content_length_threshold(d.multipart_content_length_threshold) 68 | { 69 | } 70 | 71 | namespace_settings_t::namespace_settings_t() { 72 | } 73 | 74 | namespace_settings_t::namespace_settings_t(const namespace_settings_t &ns) 75 | : m_data(new data(*ns.m_data)) 76 | { 77 | } 78 | namespace_settings_t::namespace_settings_t(namespace_settings_t &&ns) 79 | : m_data(std::move(ns.m_data)) 80 | { 81 | } 82 | 83 | namespace_settings_t::namespace_settings_t(data &&d) 84 | : m_data(new data(std::move(d))) 85 | { 86 | } 87 | 88 | namespace_settings_t &namespace_settings_t::operator =(namespace_settings_t &&ns) { 89 | m_data = std::move(ns.m_data); 90 | return *this; 91 | } 92 | 93 | namespace_settings_t::~namespace_settings_t() { 94 | } 95 | 96 | const std::string &namespace_settings_t::name() const { 97 | return m_data->name; 98 | } 99 | 100 | int namespace_settings_t::groups_count() const { 101 | return m_data->groups_count; 102 | } 103 | 104 | const std::string &namespace_settings_t::success_copies_num () const { 105 | return m_data->success_copies_num; 106 | } 107 | 108 | const std::string &namespace_settings_t::auth_key () const { 109 | return m_data->auth_key.empty() ? m_data->auth_key_for_write : m_data->auth_key; 110 | } 111 | 112 | const std::vector &namespace_settings_t::static_couple () const { 113 | return m_data->static_couple; 114 | } 115 | 116 | const std::string &namespace_settings_t::sign_token () const { 117 | return m_data->sign_token; 118 | } 119 | 120 | const std::string &namespace_settings_t::sign_path_prefix () const { 121 | return m_data->sign_path_prefix; 122 | } 123 | 124 | const std::string &namespace_settings_t::sign_port () const { 125 | return m_data->sign_port; 126 | } 127 | 128 | const std::string &namespace_settings_t::auth_key_for_write() const { 129 | return m_data->auth_key_for_write.empty() ? m_data->auth_key : m_data->auth_key_for_write; 130 | } 131 | 132 | const std::string &namespace_settings_t::auth_key_for_read() const { 133 | return m_data->auth_key_for_read; 134 | } 135 | 136 | int namespace_settings_t::redirect_expire_time() const { 137 | return m_data->redirect_expire_time; 138 | } 139 | 140 | int64_t namespace_settings_t::redirect_content_length_threshold() const { 141 | return m_data->redirect_content_length_threshold; 142 | } 143 | 144 | bool namespace_settings_t::is_active() const { 145 | return m_data->is_active; 146 | } 147 | 148 | bool namespace_settings_t::can_choose_couple_to_upload() const { 149 | return m_data->can_choose_couple_to_upload; 150 | } 151 | 152 | int64_t namespace_settings_t::multipart_content_length_threshold() const { 153 | return m_data->multipart_content_length_threshold; 154 | } 155 | 156 | } //mastermind 157 | 158 | -------------------------------------------------------------------------------- /foreign/cocaine/traits/dynamic.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright (c) 2013-2014 Andrey Goryachev 3 | Copyright (c) 2011-2014 Other contributors as noted in the AUTHORS file. 4 | This file is part of Cocaine. 5 | Cocaine is free software; you can redistribute it and/or modify 6 | it under the terms of the GNU Lesser General Public License as published by 7 | the Free Software Foundation; either version 3 of the License, or 8 | (at your option) any later version. 9 | Cocaine is distributed in the hope that it will be useful, 10 | but WITHOUT ANY WARRANTY; without even the implied warranty of 11 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 | GNU Lesser General Public License for more details. 13 | You should have received a copy of the GNU Lesser General Public License 14 | along with this program. If not, see . 15 | */ 16 | 17 | #ifndef COCAINE_DYNAMIC_SERIALIZATION_TRAITS_HPP 18 | #define COCAINE_DYNAMIC_SERIALIZATION_TRAITS_HPP 19 | 20 | #include 21 | 22 | //#include 23 | #include 24 | 25 | namespace cocaine { namespace io { 26 | 27 | using kora::dynamic_t; 28 | 29 | // Dynamic objects essentially have the same structure as msgpack objects, so these serialization 30 | // traits are pretty much straightforward. 31 | 32 | namespace aux { 33 | 34 | template 35 | struct pack_dynamic: 36 | public boost::static_visitor<> 37 | { 38 | pack_dynamic(msgpack::packer& target): 39 | m_target(target) 40 | { } 41 | 42 | void 43 | operator()(const dynamic_t::null_t&) const { 44 | m_target << msgpack::type::nil(); 45 | } 46 | 47 | void 48 | operator()(const dynamic_t::bool_t& source) const { 49 | m_target << source; 50 | } 51 | 52 | void 53 | operator()(const dynamic_t::int_t& source) const { 54 | m_target << source; 55 | } 56 | 57 | void 58 | operator()(const dynamic_t::uint_t& source) const { 59 | m_target << source; 60 | } 61 | 62 | void 63 | operator()(const dynamic_t::double_t& source) const { 64 | m_target << source; 65 | } 66 | 67 | void 68 | operator()(const dynamic_t::string_t& source) const { 69 | m_target << source; 70 | } 71 | 72 | void 73 | operator()(const dynamic_t::array_t& source) const { 74 | m_target.pack_array(source.size()); 75 | 76 | for(size_t i = 0; i < source.size(); ++i) { 77 | source[i].apply(*this); 78 | } 79 | } 80 | 81 | void 82 | operator()(const dynamic_t::object_t& source) const { 83 | m_target.pack_map(source.size()); 84 | 85 | for(auto it = source.begin(); it != source.end(); ++it) { 86 | m_target << it->first; 87 | it->second.apply(*this); 88 | } 89 | } 90 | 91 | private: 92 | msgpack::packer& m_target; 93 | }; 94 | 95 | } // namespace aux 96 | 97 | template<> 98 | struct type_traits { 99 | template 100 | static inline 101 | void 102 | pack(msgpack::packer& target, const dynamic_t& source) { 103 | source.apply(aux::pack_dynamic(target)); 104 | } 105 | 106 | static inline 107 | void 108 | unpack(const msgpack::object& source, dynamic_t& target) { 109 | switch(source.type) { 110 | case msgpack::type::MAP: { 111 | dynamic_t::object_t container; 112 | 113 | msgpack::object_kv *ptr = source.via.map.ptr, 114 | *end = ptr + source.via.map.size; 115 | 116 | for(; ptr < end; ++ptr) { 117 | if(ptr->key.type != msgpack::type::RAW) { 118 | // NOTE: The keys should be strings. 119 | throw msgpack::type_error(); 120 | } 121 | 122 | unpack(ptr->val, container[ptr->key.as()]); 123 | } 124 | 125 | target = std::move(container); 126 | } break; 127 | 128 | case msgpack::type::ARRAY: { 129 | dynamic_t::array_t container; 130 | container.reserve(source.via.array.size); 131 | 132 | msgpack::object *ptr = source.via.array.ptr, 133 | *end = ptr + source.via.array.size; 134 | 135 | for(; ptr < end; ++ptr) { 136 | container.push_back(dynamic_t()); 137 | unpack(*ptr, container.back()); 138 | } 139 | 140 | target = std::move(container); 141 | } break; 142 | 143 | case msgpack::type::RAW: 144 | target = source.as(); 145 | break; 146 | 147 | case msgpack::type::DOUBLE: 148 | target = source.as(); 149 | break; 150 | 151 | case msgpack::type::POSITIVE_INTEGER: 152 | target = source.as(); 153 | break; 154 | 155 | case msgpack::type::NEGATIVE_INTEGER: 156 | target = source.as(); 157 | break; 158 | 159 | case msgpack::type::BOOLEAN: 160 | target = source.as(); 161 | break; 162 | 163 | case msgpack::type::NIL: 164 | target = dynamic_t::null_t(); 165 | break; 166 | } 167 | } 168 | }; 169 | 170 | }} // namespace cocaine::io 171 | 172 | #endif 173 | 174 | -------------------------------------------------------------------------------- /src/error.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "libmastermind/error.hpp" 21 | 22 | namespace std { 23 | 24 | std::error_code 25 | make_error_code(mastermind::mastermind_errc e) { 26 | return std::error_code(static_cast(e), mastermind::mastermind_category()); 27 | } 28 | 29 | std::error_condition 30 | make_error_condition(mastermind::mastermind_errc e) { 31 | return std::error_condition(static_cast(e), mastermind::mastermind_category()); 32 | } 33 | 34 | } // namespace std 35 | 36 | namespace mastermind { 37 | 38 | const char *libmastermind_category_impl::name() const noexcept { 39 | return "libmastermind"; 40 | } 41 | 42 | std::string libmastermind_category_impl::message(int ev) const { 43 | switch (ev) { 44 | case libmastermind_error::couple_not_found: 45 | return "Couple not found"; 46 | case libmastermind_error::not_enough_memory: 47 | return "There is no couple with enough disk space"; 48 | case libmastermind_error::unknown_namespace: 49 | return "Unknown namespace"; 50 | case libmastermind_error::invalid_groups_count: 51 | return "Cannot find couple with such count of groups"; 52 | case libmastermind_error::cache_is_expired: 53 | return "Expired cache cannot be used"; 54 | default: 55 | return "Unknown libmastermind error"; 56 | } 57 | } 58 | 59 | const std::error_category &libmastermind_category() { 60 | const static libmastermind_category_impl instance; 61 | return instance; 62 | } 63 | 64 | class mastermind_category_t 65 | : public std::error_category 66 | { 67 | public: 68 | const char *name() const noexcept { 69 | return "mastermind category"; 70 | } 71 | 72 | std::string 73 | message(int ev) const { 74 | switch(static_cast(ev)) { 75 | case mastermind_errc::update_loop_already_started: 76 | return "update loop already started"; 77 | case mastermind_errc::update_loop_already_stopped: 78 | return "update loop already stopped"; 79 | case mastermind_errc::unknown_feedback: 80 | return "unknown feedback"; 81 | case mastermind_errc::namespace_state_not_found: 82 | return "namespace state not found"; 83 | case mastermind_errc::unknown_group: 84 | return "unknown group"; 85 | case mastermind_errc::unknown_groupset: 86 | return "unknown groupset"; 87 | case mastermind_errc::remotes_empty: 88 | return "remotes list is empty"; 89 | default: 90 | return "unknown mastermind error"; 91 | } 92 | } 93 | }; 94 | 95 | const std::error_category & 96 | mastermind_category() { 97 | const static mastermind_category_t instance; 98 | return instance; 99 | } 100 | 101 | 102 | std::error_code make_error_code(libmastermind_error::libmastermind_error_t e) { 103 | return std::error_code(static_cast(e), libmastermind_category()); 104 | } 105 | 106 | std::error_condition make_error_condition(libmastermind_error::libmastermind_error_t e) { 107 | return std::error_condition(static_cast(e), libmastermind_category()); 108 | } 109 | 110 | couple_not_found_error::couple_not_found_error() 111 | : std::system_error(make_error_code(libmastermind_error::couple_not_found)) 112 | {} 113 | 114 | not_enough_memory_error::not_enough_memory_error() 115 | : std::system_error(make_error_code(libmastermind_error::not_enough_memory)) 116 | {} 117 | 118 | unknown_namespace_error::unknown_namespace_error() 119 | : std::system_error(make_error_code(libmastermind_error::unknown_namespace)) 120 | {} 121 | 122 | invalid_groups_count_error::invalid_groups_count_error() 123 | : std::system_error(make_error_code(libmastermind_error::invalid_groups_count)) 124 | {} 125 | 126 | cache_is_expired_error::cache_is_expired_error() 127 | : std::system_error(make_error_code(libmastermind_error::cache_is_expired)) 128 | {} 129 | 130 | mastermind_error::mastermind_error(std::error_code error_code_) 131 | : std::runtime_error(error_code_.message()) 132 | , error_code(std::move(error_code_)) 133 | {} 134 | 135 | const std::error_code & 136 | mastermind_error::code() const { 137 | return error_code; 138 | } 139 | 140 | update_loop_already_started::update_loop_already_started() 141 | : mastermind_error(std::make_error_code( 142 | mastermind::mastermind_errc::update_loop_already_started)) 143 | {} 144 | 145 | update_loop_already_stopped::update_loop_already_stopped() 146 | : mastermind_error(std::make_error_code( 147 | mastermind::mastermind_errc::update_loop_already_stopped)) 148 | {} 149 | 150 | unknown_feedback::unknown_feedback(group_t couple_id_, int feedback_) 151 | : mastermind_error(std::make_error_code( 152 | mastermind::mastermind_errc::unknown_feedback)) 153 | , m_couple_id(couple_id_) 154 | , m_feedback(feedback_) 155 | {} 156 | 157 | group_t 158 | unknown_feedback::couple_id() const { 159 | return m_couple_id; 160 | } 161 | 162 | int 163 | unknown_feedback::feedback() const { 164 | return m_feedback; 165 | } 166 | 167 | namespace_state_not_found_error::namespace_state_not_found_error() 168 | : mastermind_error(std::make_error_code( 169 | mastermind::mastermind_errc::namespace_state_not_found)) 170 | {} 171 | 172 | unknown_group_error::unknown_group_error(int group_) 173 | : mastermind_error(std::make_error_code(mastermind::mastermind_errc::unknown_group)) 174 | , m_group(group_) 175 | {} 176 | 177 | int 178 | unknown_group_error::group() const { 179 | return m_group; 180 | } 181 | 182 | unknown_groupset_error::unknown_groupset_error(std::string groupset_) 183 | : mastermind_error(std::make_error_code(mastermind::mastermind_errc::unknown_groupset)) 184 | , m_groupset(std::move(groupset_)) 185 | {} 186 | 187 | const std::string & 188 | unknown_groupset_error::groupset() const { 189 | return m_groupset; 190 | } 191 | 192 | remotes_empty_error::remotes_empty_error() 193 | : mastermind_error(std::make_error_code(mastermind::mastermind_errc::remotes_empty)) 194 | {} 195 | 196 | } // namespace mastermind 197 | 198 | -------------------------------------------------------------------------------- /src/cache_p.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef LIBMASTERMIND__SRC__CACHE_P__HPP 21 | #define LIBMASTERMIND__SRC__CACHE_P__HPP 22 | 23 | #include "cocaine/traits/dynamic.hpp" 24 | #include "utils.hpp" 25 | 26 | #include 27 | 28 | #include 29 | 30 | #include 31 | #include 32 | #include 33 | 34 | namespace mastermind { 35 | 36 | template 37 | class cache_t 38 | { 39 | public: 40 | typedef T value_type; 41 | typedef std::tuple tuple_value_type; 42 | typedef std::shared_ptr shared_value_type; 43 | 44 | typedef std::function factory_t; 46 | 47 | cache_t(std::string name_ = "") 48 | : last_update_time(clock_type::now()) 49 | , m_is_expired(false) 50 | , name(std::move(name_)) 51 | , shared_value(std::make_shared(value_type(), kora::dynamic_t::null)) 52 | { 53 | } 54 | 55 | cache_t(value_type value_, std::string name_ = "") 56 | : last_update_time(clock_type::now()) 57 | , m_is_expired(false) 58 | , name(std::move(name_)) 59 | , shared_value(std::make_shared(std::move(value_) 60 | , kora::dynamic_t::null)) 61 | { 62 | } 63 | 64 | cache_t(value_type value_, kora::dynamic_t raw_value_, std::string name_ = "") 65 | : last_update_time(clock_type::now()) 66 | , m_is_expired(false) 67 | , name(std::move(name_)) 68 | , shared_value(std::make_shared(std::move(value_) 69 | , std::move(raw_value_))) 70 | { 71 | } 72 | 73 | cache_t(kora::dynamic_t raw_value_, const factory_t &factory, std::string name_ = "") 74 | : last_update_time(clock_type::now()) 75 | , m_is_expired(false) 76 | , name(std::move(name_)) 77 | { 78 | static const std::string LAST_UPDATE_TIME = "last-update-time"; 79 | static const std::string VALUE = "value"; 80 | 81 | const auto &raw_value_object = raw_value_.as_object(); 82 | 83 | { 84 | auto seconds = raw_value_object[LAST_UPDATE_TIME].to(); 85 | last_update_time = time_point_type(std::chrono::seconds(seconds)); 86 | } 87 | 88 | const auto &raw_value = raw_value_object[VALUE]; 89 | shared_value = std::make_shared(factory(name, raw_value), raw_value); 90 | } 91 | 92 | kora::dynamic_t 93 | serialize() const { 94 | static const std::string LAST_UPDATE_TIME = "last-update-time"; 95 | static const std::string VALUE = "value"; 96 | 97 | kora::dynamic_t result = kora::dynamic_t::empty_object; 98 | auto &result_object = result.as_object(); 99 | 100 | result_object[LAST_UPDATE_TIME] = std::chrono::duration_cast( 101 | last_update_time.time_since_epoch()).count(); 102 | 103 | result_object[VALUE] = get_raw_value(); 104 | 105 | return result; 106 | } 107 | 108 | time_point_type 109 | get_last_update_time() const { 110 | return last_update_time; 111 | } 112 | 113 | bool 114 | is_expired() const { 115 | return m_is_expired; 116 | } 117 | 118 | void 119 | set_expire(bool is_expired_) { 120 | m_is_expired = is_expired_; 121 | } 122 | 123 | const std::string & 124 | get_name() const { 125 | return name; 126 | } 127 | 128 | value_type & 129 | get_value() { 130 | if (is_expired()) { 131 | throw cache_is_expired_error(); 132 | } 133 | 134 | return std::get<0>(*shared_value); 135 | } 136 | 137 | const value_type & 138 | get_value() const { 139 | if (is_expired()) { 140 | throw cache_is_expired_error(); 141 | } 142 | 143 | return std::get<0>(*shared_value); 144 | } 145 | 146 | const value_type & 147 | get_value_unsafe() const { 148 | return std::get<0>(*shared_value); 149 | } 150 | 151 | std::shared_ptr 152 | get_shared_value() { 153 | if (is_expired()) { 154 | throw cache_is_expired_error(); 155 | } 156 | 157 | return std::shared_ptr(shared_value, &get_value()); 158 | } 159 | 160 | std::shared_ptr 161 | get_shared_value() const { 162 | if (is_expired()) { 163 | throw cache_is_expired_error(); 164 | } 165 | 166 | return std::shared_ptr(shared_value, &get_value()); 167 | } 168 | 169 | const kora::dynamic_t & 170 | get_raw_value() const { 171 | return std::get<1>(*shared_value); 172 | } 173 | 174 | private: 175 | time_point_type last_update_time; 176 | bool m_is_expired; 177 | 178 | std::string name; 179 | shared_value_type shared_value; 180 | }; 181 | 182 | template 183 | class synchronized_cache_t 184 | { 185 | public: 186 | typedef cache_t cache_type; 187 | 188 | synchronized_cache_t(cache_type cache_) 189 | : cache(std::move(cache_)) 190 | { 191 | } 192 | 193 | void 194 | set(cache_type cache_) { 195 | std::unique_lock lock(mutex); 196 | 197 | // Cache object should be destoryed when mutex is unlocked to prevent deadlocks 198 | auto local_cache = std::move(cache); 199 | cache = std::move(cache_); 200 | 201 | lock.unlock(); 202 | } 203 | 204 | cache_type 205 | copy() const { 206 | std::lock_guard lock_guard(mutex); 207 | (void) lock_guard; 208 | 209 | return cache; 210 | } 211 | 212 | private: 213 | mutable std::mutex mutex; 214 | cache_type cache; 215 | }; 216 | 217 | template 218 | class synchronized_cache_map_t 219 | { 220 | public: 221 | typedef cache_t cache_type; 222 | typedef std::map cache_map_t; 223 | 224 | synchronized_cache_map_t() 225 | { 226 | } 227 | 228 | void 229 | set(const std::string &key, cache_type cache) { 230 | std::unique_lock lock(mutex); 231 | 232 | auto insert_result = cache_map.insert(std::make_pair(key, cache)); 233 | 234 | if (std::get<1>(insert_result)) { 235 | // That was a new key, nothing to do anymore 236 | return; 237 | } 238 | 239 | auto it = std::get<0>(insert_result); 240 | 241 | // Cache object should be destoryed when mutex is unlocked to prevent deadlocks 242 | auto local_cache = std::move(it->second); 243 | it->second = std::move(cache); 244 | 245 | lock.unlock(); 246 | } 247 | 248 | cache_map_t 249 | copy() const { 250 | std::lock_guard lock_guard(mutex); 251 | (void) lock_guard; 252 | 253 | return cache_map; 254 | } 255 | 256 | cache_type 257 | copy(const std::string &key) const { 258 | std::lock_guard lock_guard(mutex); 259 | (void) lock_guard; 260 | 261 | auto it = cache_map.find(key); 262 | 263 | if (it == cache_map.end()) { 264 | throw unknown_namespace_error(); 265 | } 266 | 267 | return it->second; 268 | } 269 | 270 | bool 271 | remove(const std::string &key) { 272 | std::unique_lock lock(mutex); 273 | 274 | auto it = cache_map.find(key); 275 | 276 | if (cache_map.end() == it) { 277 | return false; 278 | } 279 | 280 | // Cache object should be destoryed when mutex is unlocked to prevent deadlocks 281 | auto local_cache = std::move(it->second); 282 | cache_map.erase(it); 283 | 284 | lock.unlock(); 285 | return true; 286 | } 287 | 288 | 289 | private: 290 | mutable std::mutex mutex; 291 | cache_map_t cache_map; 292 | }; 293 | 294 | } // namespace mastermind 295 | 296 | #endif /* LIBMASTERMIND__SRC__CACHE_P__HPP */ 297 | 298 | -------------------------------------------------------------------------------- /src/namespace_state_view.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "namespace_state_p.hpp" 21 | #include "couple_sequence_p.hpp" 22 | 23 | namespace mastermind { 24 | 25 | size_t 26 | namespace_state_t::settings_t::groups_count() const { 27 | return namespace_state.data->settings.groups_count; 28 | } 29 | 30 | const std::string & 31 | namespace_state_t::settings_t::success_copies_num() const { 32 | return namespace_state.data->settings.success_copies_num; 33 | } 34 | 35 | const namespace_state_t::user_settings_t & 36 | namespace_state_t::settings_t::user_settings() const { 37 | if (!namespace_state.data->settings.user_settings_ptr) { 38 | throw std::runtime_error("uninitialized user settings cannot be used"); 39 | } 40 | 41 | return *namespace_state.data->settings.user_settings_ptr; 42 | } 43 | 44 | namespace_state_t::settings_t::settings_t(const namespace_state_t &namespace_state_) 45 | : namespace_state(namespace_state_) 46 | { 47 | } 48 | 49 | struct namespace_state_t::groupset_t::data_t : public namespace_state_t::data_t::couples_t::groupset_info_t 50 | { 51 | }; 52 | 53 | namespace_state_t::groupset_t::groupset_t(const data_t &data_) 54 | : data(data_) 55 | { 56 | } 57 | 58 | uint64_t 59 | namespace_state_t::groupset_t::free_effective_space() const { 60 | return data.free_effective_space; 61 | } 62 | 63 | uint64_t 64 | namespace_state_t::groupset_t::free_reserved_space() const { 65 | return data.free_reserved_space; 66 | } 67 | 68 | std::string 69 | namespace_state_t::groupset_t::type() const { 70 | if (data.type == namespace_state_t::data_t::couples_t::groupset_info_t::type_tag::LRC) { 71 | return "lrc"; 72 | } 73 | return "UNKNOWN"; 74 | } 75 | 76 | std::string 77 | namespace_state_t::groupset_t::status() const { 78 | if (data.status == namespace_state_t::data_t::couples_t::groupset_info_t::status_tag::BAD) { 79 | return "BAD"; 80 | } 81 | return "UNKNOWN"; 82 | } 83 | 84 | std::string 85 | namespace_state_t::groupset_t::id() const { 86 | return data.id; 87 | } 88 | 89 | const std::vector & 90 | namespace_state_t::groupset_t::group_ids() const { 91 | return data.groups; 92 | } 93 | 94 | const kora::dynamic_t & 95 | namespace_state_t::groupset_t::hosts() const { 96 | return data.hosts; 97 | } 98 | 99 | const kora::dynamic_t & 100 | namespace_state_t::groupset_t::settings() const { 101 | return data.settings; 102 | } 103 | 104 | std::vector 105 | namespace_state_t::couples_t::get_couple_read_preference(group_t group) const { 106 | auto it = namespace_state.data->couples.group_info_map.find(group); 107 | 108 | if (it == namespace_state.data->couples.group_info_map.end() || 109 | it->second.couple_info_map_iterator->second.read_preference.empty()) { 110 | return {"replicas"}; 111 | } 112 | 113 | return it->second.couple_info_map_iterator->second.read_preference; 114 | } 115 | 116 | namespace_state_t::groupset_t 117 | namespace_state_t::couples_t::get_couple_groupset(group_t group, const std::string &groupset_id) const { 118 | auto cit = namespace_state.data->couples.group_info_map.find(group); 119 | 120 | if (cit == namespace_state.data->couples.group_info_map.end()) { 121 | throw unknown_group_error{group}; 122 | } 123 | 124 | const auto & map = cit->second.couple_info_map_iterator->second.groupset_info_map; 125 | auto pit = map.find(groupset_id); 126 | if (pit == map.end()) { 127 | throw unknown_groupset_error{groupset_id}; 128 | } 129 | 130 | return groupset_t(static_cast(pit->second)); 131 | } 132 | 133 | std::vector 134 | namespace_state_t::couples_t::get_couple_groupset_ids(group_t group) const { 135 | auto cit = namespace_state.data->couples.group_info_map.find(group); 136 | 137 | if (cit == namespace_state.data->couples.group_info_map.end()) { 138 | throw unknown_group_error{group}; 139 | } 140 | 141 | const auto & map = cit->second.couple_info_map_iterator->second.groupset_info_map; 142 | 143 | std::vector groupset_ids; 144 | 145 | for (auto it = map.begin(); it != map.end(); it++) { 146 | groupset_ids.emplace_back(it->first); 147 | } 148 | 149 | // TODO: remove when "replicas" becomes a separate groupset in 'groupset_info_map' 150 | { 151 | auto it = find(groupset_ids.begin(), groupset_ids.end(), "replicas"); 152 | if (it == groupset_ids.end()) { 153 | groupset_ids.emplace_back("replicas"); 154 | } 155 | } 156 | 157 | return groupset_ids; 158 | } 159 | 160 | groups_t 161 | namespace_state_t::couples_t::get_couple_groups(group_t group) const { 162 | auto it = namespace_state.data->couples.group_info_map.find(group); 163 | 164 | if (it == namespace_state.data->couples.group_info_map.end()) { 165 | return groups_t(); 166 | } 167 | 168 | return it->second.couple_info_map_iterator->second.groups; 169 | } 170 | 171 | groups_t 172 | namespace_state_t::couples_t::get_groups(group_t group) const { 173 | auto groups = get_couple_groups(group); 174 | 175 | if (groups.empty()) { 176 | return {group}; 177 | } 178 | return groups; 179 | } 180 | 181 | uint64_t 182 | namespace_state_t::couples_t::free_effective_space(group_t group) const { 183 | auto it = namespace_state.data->couples.group_info_map.find(group); 184 | 185 | if (it == namespace_state.data->couples.group_info_map.end()) { 186 | return 0; 187 | } 188 | 189 | return it->second.couple_info_map_iterator->second.free_effective_space; 190 | } 191 | 192 | uint64_t 193 | namespace_state_t::couples_t::free_reserved_space(group_t group) const { 194 | auto it = namespace_state.data->couples.group_info_map.find(group); 195 | 196 | if (it == namespace_state.data->couples.group_info_map.end()) { 197 | return 0; 198 | } 199 | 200 | return it->second.couple_info_map_iterator->second.free_reserved_space; 201 | } 202 | 203 | kora::dynamic_t 204 | namespace_state_t::couples_t::hosts(group_t group) const { 205 | auto it = namespace_state.data->couples.group_info_map.find(group); 206 | 207 | if (it == namespace_state.data->couples.group_info_map.end()) { 208 | throw unknown_group_error{group}; 209 | } 210 | 211 | return it->second.couple_info_map_iterator->second.hosts; 212 | } 213 | 214 | namespace_state_t::couples_t::couples_t(const namespace_state_t &namespace_state_) 215 | : namespace_state(namespace_state_) 216 | { 217 | } 218 | 219 | groups_t 220 | namespace_state_t::weights_t::groups(uint64_t size) const { 221 | return namespace_state.data->weights.get(size).groups; 222 | } 223 | 224 | couple_sequence_t 225 | namespace_state_t::weights_t::couple_sequence(uint64_t size) const { 226 | auto data = std::make_shared( 227 | namespace_state.data->weights.get_all(size)); 228 | return couple_sequence_init_t(std::move(data)); 229 | } 230 | 231 | void 232 | namespace_state_t::weights_t::set_feedback(group_t couple_id 233 | , feedback_tag feedback) { 234 | switch (feedback) { 235 | case feedback_tag::available: 236 | namespace_state.data->weights.set_coefficient(couple_id, 1); 237 | break; 238 | case feedback_tag::partly_unavailable: 239 | namespace_state.data->weights.set_coefficient(couple_id, 0.1); 240 | break; 241 | case feedback_tag::temporary_unavailable: 242 | namespace_state.data->weights.set_coefficient(couple_id, 0.01); 243 | break; 244 | case feedback_tag::permanently_unavailable: 245 | namespace_state.data->weights.set_coefficient(couple_id, 0); 246 | break; 247 | default: 248 | throw unknown_feedback(couple_id, static_cast(feedback)); 249 | } 250 | 251 | } 252 | 253 | namespace_state_t::weights_t::weights_t(const namespace_state_t &namespace_state_) 254 | : namespace_state(namespace_state_) 255 | { 256 | } 257 | 258 | bool 259 | namespace_state_t::statistics_t::ns_is_full() { 260 | return namespace_state.data->statistics.ns_is_full(); 261 | } 262 | 263 | namespace_state_t::statistics_t::statistics_t(const namespace_state_t &namespace_state_) 264 | : namespace_state(namespace_state_) 265 | { 266 | } 267 | 268 | namespace_state_t::settings_t 269 | namespace_state_t::settings() const { 270 | return settings_t(*this); 271 | } 272 | 273 | namespace_state_t::couples_t 274 | namespace_state_t::couples() const { 275 | return couples_t(*this); 276 | } 277 | 278 | namespace_state_t::weights_t 279 | namespace_state_t::weights() const { 280 | return weights_t(*this); 281 | } 282 | 283 | namespace_state_t::statistics_t 284 | namespace_state_t::statistics() const { 285 | return statistics_t(*this); 286 | } 287 | 288 | const std::string &namespace_state_t::name() const { 289 | return data->name; 290 | } 291 | 292 | namespace_state_t::operator bool() const { 293 | return static_cast(data); 294 | } 295 | 296 | } // namespace mastermind 297 | -------------------------------------------------------------------------------- /include/libmastermind/mastermind.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef INCLUDE__LIBMASTERMIND__MASTERMIND_HPP 21 | #define INCLUDE__LIBMASTERMIND__MASTERMIND_HPP 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | 29 | #include 30 | #include 31 | 32 | #include 33 | 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | namespace mastermind { 40 | 41 | struct group_info_response_t { 42 | std::vector nodes; 43 | std::vector couples; 44 | int status; 45 | std::string name_space; 46 | }; 47 | 48 | struct namespace_settings_t { 49 | struct data; 50 | 51 | namespace_settings_t(); 52 | namespace_settings_t(const namespace_settings_t &ns); 53 | namespace_settings_t(namespace_settings_t &&ns); 54 | namespace_settings_t(data &&d); 55 | 56 | ~namespace_settings_t(); 57 | 58 | namespace_settings_t &operator =(namespace_settings_t &&ns); 59 | 60 | const std::string &name() const; 61 | int groups_count() const; 62 | const std::string &success_copies_num() const; 63 | const std::string &auth_key() const; 64 | const std::vector &static_couple() const; 65 | const std::string &sign_token() const; 66 | const std::string &sign_path_prefix() const; 67 | const std::string &sign_port() const; 68 | const std::string &auth_key_for_write() const; 69 | const std::string &auth_key_for_read() const; 70 | bool is_active() const; 71 | 72 | bool can_choose_couple_to_upload() const; 73 | int64_t multipart_content_length_threshold() const; 74 | 75 | int redirect_expire_time() const; 76 | int64_t redirect_content_length_threshold() const; 77 | 78 | private: 79 | std::unique_ptr m_data; 80 | }; 81 | 82 | class namespace_state_t { 83 | public: 84 | class user_settings_t { 85 | public: 86 | virtual ~user_settings_t(); 87 | }; 88 | 89 | typedef std::unique_ptr user_settings_ptr_t; 90 | 91 | typedef 92 | std::function 93 | user_settings_factory_t; 94 | 95 | class settings_t { 96 | public: 97 | size_t groups_count() const; 98 | const std::string &success_copies_num() const; 99 | 100 | const user_settings_t &user_settings() const; 101 | 102 | private: 103 | friend class namespace_state_t; 104 | 105 | settings_t(const namespace_state_t &namespace_state_); 106 | 107 | const namespace_state_t &namespace_state; 108 | }; 109 | 110 | class couples_t; 111 | 112 | class groupset_t { 113 | public: 114 | uint64_t free_effective_space() const; 115 | uint64_t free_reserved_space() const; 116 | 117 | std::string type() const; 118 | std::string status() const; 119 | std::string id() const; 120 | 121 | const std::vector &group_ids() const; 122 | const kora::dynamic_t &hosts() const; 123 | const kora::dynamic_t &settings() const; 124 | 125 | private: 126 | friend class couples_t; 127 | 128 | struct data_t; 129 | groupset_t(const data_t &data_); 130 | 131 | const data_t &data; 132 | }; 133 | 134 | class couples_t { 135 | public: 136 | std::vector get_couple_read_preference(group_t group) const; 137 | groupset_t get_couple_groupset(group_t group, const std::string &groupset_id) const; 138 | 139 | std::vector get_couple_groupset_ids(group_t group) const; 140 | 141 | groups_t get_couple_groups(group_t group) const; 142 | groups_t get_groups(group_t group) const; 143 | 144 | uint64_t free_effective_space(group_t group) const; 145 | uint64_t free_reserved_space(group_t group) const; 146 | kora::dynamic_t hosts(group_t group) const; 147 | private: 148 | friend class namespace_state_t; 149 | 150 | couples_t(const namespace_state_t &namespace_state_); 151 | 152 | const namespace_state_t &namespace_state; 153 | }; 154 | 155 | class weights_t { 156 | public: 157 | enum class feedback_tag { 158 | available 159 | , partly_unavailable 160 | , temporary_unavailable 161 | , permanently_unavailable 162 | }; 163 | 164 | groups_t groups(uint64_t size = 0) const; 165 | couple_sequence_t couple_sequence(uint64_t size = 0) const; 166 | void set_feedback(group_t couple_id, feedback_tag feedback); 167 | 168 | private: 169 | friend class namespace_state_t; 170 | 171 | weights_t(const namespace_state_t &namespace_state_); 172 | 173 | const namespace_state_t &namespace_state; 174 | }; 175 | 176 | class statistics_t { 177 | public: 178 | bool 179 | ns_is_full(); 180 | private: 181 | friend class namespace_state_t; 182 | 183 | statistics_t(const namespace_state_t &namespace_state_); 184 | 185 | const namespace_state_t &namespace_state; 186 | }; 187 | 188 | settings_t settings() const; 189 | 190 | couples_t couples() const; 191 | 192 | weights_t weights() const; 193 | 194 | statistics_t statistics() const; 195 | 196 | const std::string &name() const; 197 | 198 | operator bool() const; 199 | 200 | protected: 201 | class data_t; 202 | 203 | std::shared_ptr data; 204 | 205 | private: 206 | }; 207 | 208 | class mastermind_t { 209 | public: 210 | typedef std::pair remote_t; 211 | typedef std::vector remotes_t; 212 | 213 | mastermind_t(const remotes_t &remotes, 214 | const std::shared_ptr &logger, 215 | int group_info_update_period = 60); 216 | mastermind_t(const std::string &host, uint16_t port, 217 | const std::shared_ptr &logger, 218 | int group_info_update_period = 60); 219 | mastermind_t(const remotes_t &remotes, 220 | const std::shared_ptr &logger, 221 | int group_info_update_period, std::string cache_path, int expired_time, 222 | std::string worker_name); 223 | mastermind_t(const remotes_t &remotes, 224 | const std::shared_ptr &logger, 225 | int group_info_update_period, std::string cache_path, 226 | int warning_time, int expire_time, 227 | std::string worker_name, 228 | int enqueue_timeout, 229 | int reconnect_timeout); 230 | mastermind_t(const remotes_t &remotes, 231 | const std::shared_ptr &logger, 232 | int group_info_update_period, std::string cache_path, 233 | int warning_time, int expire_time, 234 | std::string worker_name, 235 | int enqueue_timeout, 236 | int reconnect_timeout, 237 | namespace_state_t::user_settings_factory_t user_settings_factory 238 | ); 239 | mastermind_t(const remotes_t &remotes, 240 | const std::shared_ptr &logger, 241 | int group_info_update_period, std::string cache_path, 242 | int warning_time, int expire_time, 243 | std::string worker_name, 244 | int enqueue_timeout, 245 | int reconnect_timeout, 246 | bool auto_start 247 | ); 248 | 249 | ~mastermind_t(); 250 | 251 | void 252 | start(); 253 | 254 | void 255 | stop(); 256 | 257 | bool 258 | is_running() const; 259 | 260 | bool 261 | is_valid() const; 262 | 263 | std::vector get_metabalancer_groups(uint64_t count = 0, const std::string &name_space = std::string("default"), uint64_t size = 0); 264 | group_info_response_t get_metabalancer_group_info(int group); 265 | std::map> get_symmetric_groups(); 266 | std::vector get_symmetric_groups(int group); 267 | std::vector get_couple_by_group(int group); 268 | std::vector get_couple(int couple_id, const std::string &ns); 269 | std::vector > get_bad_groups(); 270 | std::vector get_all_groups(); 271 | std::vector get_cache_groups(const std::string &key); 272 | std::vector get_namespaces_settings(); 273 | std::vector get_elliptics_remotes(); 274 | std::vector, uint64_t, uint64_t>> get_couple_list(const std::string &ns); 275 | 276 | uint64_t free_effective_space_in_couple_by_group(size_t group); 277 | 278 | namespace_state_t 279 | get_namespace_state(const std::string &name) const; 280 | 281 | namespace_state_t 282 | find_namespace_state(group_t group) const; 283 | 284 | groups_t 285 | get_cached_groups(const std::string &elliptics_id, group_t couple_id) const; 286 | 287 | std::string json_group_weights(); 288 | std::string json_symmetric_groups(); 289 | std::string json_bad_groups(); 290 | std::string json_cache_groups(); 291 | std::string json_metabalancer_info(); 292 | std::string json_namespaces_settings(); 293 | std::string json_namespace_statistics(const std::string &ns); 294 | 295 | void 296 | set_user_settings_factory(namespace_state_t::user_settings_factory_t user_settings_factory); 297 | 298 | void cache_force_update(); 299 | void set_update_cache_callback(const std::function &callback); 300 | void set_update_cache_ext1_callback(const std::function &callback); 301 | 302 | private: 303 | struct data; 304 | std::unique_ptr m_data; 305 | }; 306 | 307 | } // namespace mastermind 308 | 309 | #endif /* INCLUDE__LIBMASTERMIND__MASTERMIND_HPP */ 310 | 311 | -------------------------------------------------------------------------------- /src/mastermind_impl.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #ifndef SRC__MASTERMIND_IMPL_HPP 21 | #define SRC__MASTERMIND_IMPL_HPP 22 | 23 | #include "libmastermind/mastermind.hpp" 24 | #include "utils.hpp" 25 | #include "cache_p.hpp" 26 | #include "namespace_state_p.hpp" 27 | #include "cached_keys.hpp" 28 | 29 | #include 30 | #include 31 | #include 32 | #include 33 | #include 34 | #include 35 | #include 36 | 37 | #include 38 | #include 39 | #include 40 | #include 41 | 42 | #include 43 | 44 | #include 45 | 46 | namespace mastermind { 47 | 48 | struct mastermind_t::data { 49 | data(const remotes_t &remotes, const std::shared_ptr &logger, 50 | int group_info_update_period, std::string cache_path, 51 | int warning_time_, int expire_time_, std::string worker_name, 52 | int enqueue_timeout_, int reconnect_timeout_, 53 | namespace_state_t::user_settings_factory_t user_settings_factory_, bool auto_start); 54 | ~data(); 55 | 56 | std::shared_ptr 57 | get_namespace_state(const std::string &name) const; 58 | 59 | void 60 | start(); 61 | 62 | void 63 | stop(); 64 | 65 | bool 66 | is_running() const; 67 | 68 | bool 69 | is_valid() const; 70 | 71 | void reconnect(); 72 | 73 | template 74 | std::string 75 | simple_enqueue(const std::string &event, const T &chunk); 76 | 77 | // deprecated 78 | template 79 | bool simple_enqueue_old(const std::string &event, const T &chunk, R &result); 80 | 81 | kora::dynamic_t 82 | enqueue(const std::string &event); 83 | 84 | kora::dynamic_t 85 | enqueue_gzip(const std::string &event); 86 | 87 | kora::dynamic_t 88 | enqueue(const std::string &event, kora::dynamic_t args); 89 | 90 | template 91 | std::string 92 | enqueue_with_reconnect(const std::string &event, const T &chunk); 93 | 94 | // deprecated 95 | template 96 | void enqueue_old(const std::string &event, const T &chunk, R &result); 97 | 98 | void 99 | collect_namespaces_states(); 100 | 101 | bool collect_cached_keys(); 102 | bool collect_elliptics_remotes(); 103 | 104 | void collect_info_loop_impl(); 105 | void collect_info_loop(); 106 | 107 | void 108 | cache_expire(); 109 | 110 | template 111 | bool 112 | check_cache_for_expire(const std::string &title, const cache_t &cache 113 | , const duration_type &preferable_life_time, const duration_type &warning_time 114 | , const duration_type &expire_time); 115 | 116 | void 117 | generate_fake_caches(); 118 | 119 | void serialize(); 120 | 121 | bool 122 | namespace_state_is_deleted(const kora::dynamic_t &raw_value); 123 | 124 | namespace_state_init_t::data_t 125 | create_namespaces_states(const std::string &name, const kora::dynamic_t &raw_value); 126 | 127 | cached_keys_t 128 | create_cached_keys(const std::string &name, const kora::dynamic_t &raw_value); 129 | 130 | std::vector 131 | create_elliptics_remotes(const std::string &name, const kora::dynamic_t &raw_value); 132 | 133 | namespace_settings_t 134 | create_namespace_settings(const std::string &name, const kora::dynamic_t &raw_value); 135 | 136 | void deserialize(); 137 | 138 | void 139 | set_user_settings_factory(namespace_state_t::user_settings_factory_t user_settings_factory_); 140 | 141 | void cache_force_update(); 142 | void set_update_cache_callback(const std::function &callback); 143 | void set_update_cache_ext1_callback(const std::function &callback); 144 | 145 | void 146 | process_callbacks(); 147 | 148 | std::shared_ptr m_logger; 149 | 150 | remotes_t m_remotes; 151 | remote_t m_current_remote; 152 | size_t m_next_remote; 153 | std::string m_cache_path; 154 | std::string m_worker_name; 155 | 156 | int m_metabase_timeout; 157 | uint64_t m_metabase_current_stamp; 158 | 159 | 160 | typedef synchronized_cache_map_t namespaces_states_t; 161 | typedef synchronized_cache_t> elliptics_remotes_t; 162 | 163 | namespaces_states_t namespaces_states; 164 | synchronized_cache_t cached_keys; 165 | elliptics_remotes_t elliptics_remotes; 166 | 167 | synchronized_cache_t> namespaces_settings; 168 | synchronized_cache_t> bad_groups; 169 | synchronized_cache_t> fake_groups_info; 170 | 171 | const int m_group_info_update_period; 172 | std::thread m_weight_cache_update_thread; 173 | std::condition_variable m_weight_cache_condition_variable; 174 | std::mutex m_mutex; 175 | std::function m_cache_update_callback; 176 | bool m_done; 177 | std::mutex m_reconnect_mutex; 178 | 179 | std::chrono::seconds warning_time; 180 | std::chrono::seconds expire_time; 181 | 182 | std::chrono::milliseconds enqueue_timeout; 183 | std::chrono::milliseconds reconnect_timeout; 184 | 185 | namespace_state_t::user_settings_factory_t user_settings_factory; 186 | 187 | bool cache_is_expired; 188 | // m_cache_update_callback with cache expiration info 189 | std::function cache_update_ext1_callback; 190 | 191 | std::shared_ptr m_app; 192 | std::shared_ptr m_service_manager; 193 | }; 194 | 195 | template 196 | std::string 197 | mastermind_t::data::simple_enqueue(const std::string &event, const T &chunk) { 198 | try { 199 | auto g = m_app->enqueue(event, chunk); 200 | g.wait_for(enqueue_timeout); 201 | 202 | if (g.ready() == false) { 203 | throw std::runtime_error("enqueue timeout"); 204 | } 205 | 206 | return g.next(); 207 | } catch (const std::exception &ex) { 208 | throw std::runtime_error("cannot process event " + event + ": " + ex.what()); 209 | } 210 | } 211 | 212 | template 213 | bool mastermind_t::data::simple_enqueue_old(const std::string &event, const T &chunk, R &result) { 214 | try { 215 | auto g = m_app->enqueue(event, chunk); 216 | g.wait_for(enqueue_timeout); 217 | 218 | if (g.ready() == false) { 219 | return false; 220 | } 221 | 222 | auto chunk = g.next(); 223 | 224 | result = cocaine::framework::unpack(chunk); 225 | return true; 226 | } catch (const std::exception &ex) { 227 | COCAINE_LOG_ERROR(m_logger, "libmastermind: enqueue_impl: %s", ex.what()); 228 | } 229 | return false; 230 | } 231 | 232 | template 233 | std::string 234 | mastermind_t::data::enqueue_with_reconnect(const std::string &event, const T &chunk) { 235 | try { 236 | bool tried_to_reconnect = false; 237 | 238 | if (!m_service_manager || !m_app 239 | || m_app->status() != cocaine::framework::service_status::connected) { 240 | COCAINE_LOG_INFO(m_logger, "libmastermind: enqueue: preconnect"); 241 | tried_to_reconnect = true; 242 | reconnect(); 243 | } 244 | 245 | try { 246 | return simple_enqueue(event, chunk); 247 | } catch (const std::exception &ex) { 248 | COCAINE_LOG_ERROR(m_logger 249 | , "libmastermind: cannot process enqueue (1st try): %s" 250 | , ex.what()); 251 | } 252 | 253 | if (tried_to_reconnect) { 254 | throw std::runtime_error("reconnect is useless"); 255 | } 256 | 257 | reconnect(); 258 | 259 | try { 260 | return simple_enqueue(event, chunk); 261 | } catch (const std::exception &ex) { 262 | COCAINE_LOG_ERROR(m_logger 263 | , "libmastermind: cannot process enqueue (2nd try): %s" 264 | , ex.what()); 265 | } 266 | 267 | throw std::runtime_error("bad connection"); 268 | } catch (const std::exception &ex) { 269 | throw std::runtime_error(std::string("cannot process enqueue: ") + ex.what()); 270 | } 271 | } 272 | 273 | template 274 | void mastermind_t::data::enqueue_old(const std::string &event, const T &chunk, R &result) { 275 | bool tried_to_reconnect = false; 276 | try { 277 | if (!m_service_manager || !m_app || m_app->status() != cocaine::framework::service_status::connected) { 278 | COCAINE_LOG_INFO(m_logger, "libmastermind: enqueue: preconnect"); 279 | tried_to_reconnect = true; 280 | reconnect(); 281 | } 282 | 283 | if (simple_enqueue_old(event, chunk, result)) { 284 | return; 285 | } 286 | 287 | if (tried_to_reconnect) { 288 | throw std::runtime_error("cannot process enqueue"); 289 | } 290 | 291 | reconnect(); 292 | 293 | if (simple_enqueue_old(event, chunk, result)) { 294 | return; 295 | } 296 | 297 | throw std::runtime_error("cannot reprocess enqueue"); 298 | } catch (const cocaine::framework::service_error_t &ex) { 299 | COCAINE_LOG_ERROR(m_logger, "libmastermind: enqueue: %s", ex.what()); 300 | throw; 301 | } catch (const std::exception &ex) { 302 | COCAINE_LOG_ERROR(m_logger, "libmastermind: enqueue: %s", ex.what()); 303 | throw; 304 | } 305 | } 306 | 307 | template 308 | bool 309 | mastermind_t::data::check_cache_for_expire(const std::string &title, const cache_t &cache 310 | , const duration_type &preferable_life_time, const duration_type &warning_time 311 | , const duration_type &expire_time) { 312 | bool is_expired = cache.is_expired(); 313 | 314 | auto life_time = std::chrono::duration_cast( 315 | clock_type::now() - cache.get_last_update_time()); 316 | 317 | if (expire_time <= life_time) { 318 | COCAINE_LOG_ERROR(m_logger 319 | , "cache \"%s\" has been expired; life-time=%ds" 320 | , title.c_str(), static_cast(life_time.count())); 321 | is_expired = true; 322 | } else if (warning_time <= life_time) { 323 | COCAINE_LOG_ERROR(m_logger 324 | , "cache \"%s\" will be expired soon; life-time=%ds" 325 | , title.c_str(), static_cast(life_time.count())); 326 | } else if (preferable_life_time <= life_time) { 327 | COCAINE_LOG_ERROR(m_logger 328 | , "cache \"%s\" is too old; life-time=%ds" 329 | , title.c_str(), static_cast(life_time.count())); 330 | } 331 | 332 | return is_expired; 333 | } 334 | 335 | } // namespace mastermind 336 | 337 | #endif /* SRC__MASTERMIND_IMPL_HPP */ 338 | -------------------------------------------------------------------------------- /src/namespace_state.cpp: -------------------------------------------------------------------------------- 1 | #include "namespace_state_p.hpp" 2 | #include "utils.hpp" 3 | 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | mastermind::namespace_state_t::user_settings_t::~user_settings_t() { 11 | } 12 | 13 | mastermind::namespace_state_t::data_t::settings_t::settings_t(const std::string &name 14 | , const kora::config_t &state , const user_settings_factory_t &factory) 15 | try 16 | : groups_count(state.at("groups-count")) 17 | , success_copies_num(state.at("success-copies-num")) 18 | { 19 | if (state.has("auth-keys")) { 20 | const auto &auth_keys_state = state.at("auth-keys"); 21 | 22 | auth_keys.read = auth_keys_state.at("read", ""); 23 | auth_keys.write = auth_keys_state.at("write", ""); 24 | } 25 | 26 | if (state.has("static-couple")) { 27 | const auto &static_couple_config = state.at("static-couple"); 28 | 29 | for (size_t index = 0, size = static_couple_config.size(); index != size; ++index) { 30 | static_groups.emplace_back(static_couple_config.at(index)); 31 | } 32 | } 33 | 34 | if (factory) { 35 | user_settings_ptr = factory(name, state); 36 | } 37 | } catch (const std::exception &ex) { 38 | throw std::runtime_error(std::string("cannot create settings-state: ") + ex.what()); 39 | } 40 | 41 | mastermind::namespace_state_t::data_t::settings_t::settings_t(settings_t &&other) 42 | : groups_count(other.groups_count) 43 | , success_copies_num(std::move(other.success_copies_num)) 44 | , auth_keys(std::move(other.auth_keys)) 45 | , user_settings_ptr(std::move(other.user_settings_ptr)) 46 | { 47 | } 48 | 49 | mastermind::namespace_state_t::data_t::couples_t::couples_t(const kora::config_t &state) 50 | try 51 | { 52 | for (size_t index = 0, size = state.size(); index != size; ++index) { 53 | const auto &couple_info_state = state.at(index); 54 | 55 | auto couple_id = couple_info_state.at("id"); 56 | 57 | auto ci_insert_result = couple_info_map.insert(std::make_pair( 58 | couple_id, couple_info_t())); 59 | 60 | if (std::get<1>(ci_insert_result) == false) { 61 | throw std::runtime_error("reuse the same couple_id=" + couple_id); 62 | } 63 | 64 | auto &couple_info = std::get<0>(ci_insert_result)->second; 65 | 66 | couple_info.id = couple_id; 67 | 68 | { 69 | const auto &dynamic_tuple = couple_info_state.at("tuple") 70 | .underlying_object().as_array(); 71 | 72 | for (auto it = dynamic_tuple.begin(), end = dynamic_tuple.end(); 73 | it != end; ++it) { 74 | couple_info.groups.emplace_back(it->to()); 75 | } 76 | } 77 | 78 | { 79 | auto status = couple_info_state.at("couple_status"); 80 | 81 | if (status == "BAD") { 82 | couple_info.status = couple_info_t::status_tag::BAD; 83 | } else { 84 | couple_info.status = couple_info_t::status_tag::UNKNOWN; 85 | } 86 | } 87 | 88 | couple_info.free_effective_space = couple_info_state.at("free_effective_space", 0); 89 | couple_info.free_reserved_space = couple_info_state.at("free_reserved_space", 0); 90 | 91 | couple_info.hosts = couple_info_state.at("hosts").underlying_object(); 92 | 93 | const auto &groups_info_state = couple_info_state.at("groups"); 94 | 95 | for (size_t index = 0, size = groups_info_state.size(); index != size; ++index) { 96 | const auto &group_info_state = groups_info_state.at(index); 97 | 98 | auto group_id = group_info_state.at("id"); 99 | 100 | auto gi_insert_result = group_info_map.insert(std::make_pair( 101 | group_id, group_info_t())); 102 | 103 | if (std::get<1>(gi_insert_result) == false) { 104 | throw std::runtime_error("resuse the same group_id=" 105 | + boost::lexical_cast(group_id)); 106 | } 107 | 108 | auto &group_info = std::get<0>(gi_insert_result)->second; 109 | 110 | group_info.id = group_id; 111 | 112 | { 113 | auto status = group_info_state.at("status"); 114 | 115 | if (status == "COUPLED") { 116 | group_info.status = group_info_t::status_tag::COUPLED; 117 | } else { 118 | group_info.status = group_info_t::status_tag::UNKNOWN; 119 | } 120 | } 121 | 122 | group_info.couple_info_map_iterator = std::get<0>(ci_insert_result); 123 | couple_info.groups_info_map_iterator.emplace_back(std::get<0>(gi_insert_result)); 124 | } 125 | 126 | if (couple_info_state.has("groupsets")) { 127 | const auto &groupsets_state = couple_info_state.at("groupsets").underlying_object().as_object(); 128 | for (auto it = groupsets_state.begin(); it != groupsets_state.end(); ++it) { 129 | const auto & groupset_info_state = it->second.as_object(); 130 | auto & groupset_info = couple_info.groupset_info_map[it->first]; 131 | 132 | groupset_info.id = groupset_info_state["id"].as_string(); 133 | 134 | const auto & groups_state = groupset_info_state["group_ids"].as_array(); 135 | groupset_info.groups.resize(groups_state.size()); 136 | std::transform(groups_state.begin(), groups_state.end(), groupset_info.groups.begin(), 137 | [] (const kora::dynamic_t & num) { return num.as_uint(); }); 138 | 139 | auto type = groupset_info_state.at("type", "UNKNOWN").as_string(); 140 | if (type == "lrc") { 141 | groupset_info.type = groupset_info_t::type_tag::LRC; 142 | } else { 143 | groupset_info.type = groupset_info_t::type_tag::UNKNOWN; 144 | } 145 | 146 | auto status = groupset_info_state.at("status", "BAD").as_string(); 147 | if (status == "BAD") { 148 | groupset_info.status = groupset_info_t::status_tag::BAD; 149 | } else { 150 | groupset_info.status = groupset_info_t::status_tag::UNKNOWN; 151 | } 152 | 153 | groupset_info.free_effective_space = groupset_info_state.at("free_effective_space", 0).as_uint(); 154 | groupset_info.free_reserved_space = groupset_info_state.at("free_reserved_space", 0).as_uint(); 155 | 156 | groupset_info.hosts = groupset_info_state.at("hosts").as_object(); 157 | groupset_info.settings = groupset_info_state.at("settings").as_object(); 158 | } 159 | 160 | if (!groupsets_state.empty()) { 161 | const auto &readpref_state = couple_info_state.at("read_preference").underlying_object().as_array(); 162 | couple_info.read_preference.resize(readpref_state.size()); 163 | std::transform(readpref_state.begin(), readpref_state.end(), couple_info.read_preference.begin(), 164 | [] (const kora::dynamic_t & pref) { return pref.as_string(); }); 165 | } 166 | } 167 | } 168 | } catch (const std::exception &ex) { 169 | throw std::runtime_error(std::string("cannot create couples-state: ") + ex.what()); 170 | } 171 | 172 | mastermind::namespace_state_t::data_t::couples_t::couples_t(couples_t &&other) 173 | : group_info_map(std::move(other.group_info_map)) 174 | , couple_info_map(std::move(other.couple_info_map)) 175 | { 176 | } 177 | 178 | mastermind::namespace_state_t::data_t::statistics_t::statistics_t(const kora::config_t &config) 179 | try 180 | : is_full(config.at("is_full", false)) 181 | { 182 | // TODO: log whether is_full is provided by mastermind 183 | } catch (const std::exception &ex) { 184 | throw std::runtime_error(std::string("cannot create statistics-state: ") + ex.what()); 185 | } 186 | 187 | bool 188 | mastermind::namespace_state_t::data_t::statistics_t::ns_is_full() const { 189 | return is_full; 190 | } 191 | 192 | mastermind::namespace_state_t::data_t::data_t(std::string name_, const kora::config_t &config 193 | , const user_settings_factory_t &factory) 194 | try 195 | : name(std::move(name_)) 196 | , settings(name, config.at("settings"), factory) 197 | , couples(config.at("couples")) 198 | , weights(config.at("weights"), settings.groups_count, !settings.static_groups.empty()) 199 | , statistics(config.at("statistics")) 200 | { 201 | check_consistency(); 202 | 203 | } catch (const std::exception &ex) { 204 | throw std::runtime_error("cannot create ns-state " + name + ": " + ex.what()); 205 | } 206 | 207 | mastermind::namespace_state_t::data_t::data_t(data_t &&other) 208 | : name(std::move(other.name)) 209 | , settings(std::move(other.settings)) 210 | , couples(std::move(other.couples)) 211 | , weights(std::move(other.weights)) 212 | , statistics(std::move(other.statistics)) 213 | , extract(std::move(other.extract)) 214 | { 215 | } 216 | 217 | void 218 | mastermind::namespace_state_t::data_t::check_consistency() { 219 | std::ostringstream oss; 220 | 221 | oss << "namespace=" << name; 222 | oss << " groups-count=" << settings.groups_count; 223 | 224 | { 225 | if (couples.couple_info_map.empty()) { 226 | throw std::runtime_error("couples list is empty"); 227 | } 228 | } 229 | 230 | { 231 | size_t nonzero_weights = 0; 232 | 233 | const auto &weights_data = weights.data(); 234 | 235 | for (auto it = weights_data.begin(), end = weights_data.end(); it != end; ++it) { 236 | auto weight = it->weight; 237 | 238 | if (weight != 0) { 239 | nonzero_weights += 1; 240 | } 241 | 242 | { 243 | auto couple_it = couples.couple_info_map.cend(); 244 | const auto &groups = it->groups; 245 | 246 | if (groups.size() != settings.groups_count) { 247 | std::ostringstream oss; 248 | oss 249 | << "groups.size is not equal to groups_count(" << settings.groups_count 250 | << "), groups=" << groups; 251 | throw std::runtime_error(oss.str()); 252 | } 253 | 254 | for (auto git = groups.begin(), gend = groups.end(); git != gend; ++git) { 255 | auto group_info_it = couples.group_info_map.find(*git); 256 | 257 | if (group_info_it == couples.group_info_map.end()) { 258 | throw std::runtime_error("there is no group-info for group " 259 | + boost::lexical_cast(*git)); 260 | } 261 | 262 | if (couple_it != couples.couple_info_map.end()) { 263 | if (couple_it != group_info_it->second.couple_info_map_iterator) { 264 | std::ostringstream oss; 265 | oss << "inconsisten couple: " << groups; 266 | throw std::runtime_error(oss.str()); 267 | } 268 | } else { 269 | couple_it = group_info_it->second.couple_info_map_iterator; 270 | } 271 | } 272 | 273 | if (couple_it->second.groups.size() != groups.size()) { 274 | std::ostringstream oss; 275 | oss << "inconsisten couple: " << groups; 276 | throw std::runtime_error(oss.str()); 277 | } 278 | } 279 | } 280 | 281 | bool is_static = false; 282 | 283 | if (nonzero_weights == 0 && !statistics.ns_is_full()) { 284 | if (settings.static_groups.empty()) { 285 | throw std::runtime_error("no weighted couples were obtained from mastermind"); 286 | } else { 287 | // Because namespace has static couple 288 | nonzero_weights = 1; 289 | is_static = true; 290 | } 291 | } 292 | 293 | oss << " couples-for-write=" << nonzero_weights; 294 | 295 | if (is_static) { 296 | oss << " [static]"; 297 | } 298 | 299 | if (statistics.ns_is_full()) { 300 | oss << " [full]"; 301 | } 302 | } 303 | 304 | oss << " couples=" << couples.couple_info_map.size(); 305 | 306 | extract = oss.str(); 307 | } 308 | 309 | mastermind::namespace_state_init_t::namespace_state_init_t( 310 | std::shared_ptr data_) { 311 | data = std::move(data_); 312 | } 313 | 314 | mastermind::namespace_state_init_t::data_t::data_t(std::string name 315 | , const kora::config_t &config, const user_settings_factory_t &factory) 316 | : namespace_state_t::data_t(std::move(name), config, factory) 317 | { 318 | } 319 | 320 | mastermind::namespace_state_init_t::data_t::data_t(data_t &&other) 321 | : namespace_state_t::data_t(std::move(other)) 322 | { 323 | } 324 | 325 | -------------------------------------------------------------------------------- /src/mastermind.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2014 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "libmastermind/mastermind.hpp" 21 | 22 | #include "mastermind_impl.hpp" 23 | 24 | #include 25 | 26 | #include 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | namespace mastermind { 33 | 34 | mastermind_t::mastermind_t( 35 | const remotes_t &remotes, 36 | const std::shared_ptr &logger, 37 | int group_info_update_period 38 | ) 39 | : m_data(new data(remotes, logger, group_info_update_period, 40 | "/var/tmp/libmastermind.cache", 41 | std::numeric_limits::max(), std::numeric_limits::max(), "mastermind", 42 | 4000, 4000, namespace_state_t::user_settings_factory_t(), true)) 43 | { 44 | } 45 | 46 | mastermind_t::mastermind_t( 47 | const std::string &host, 48 | uint16_t port, 49 | const std::shared_ptr &logger, 50 | int group_info_update_period 51 | ) 52 | : m_data(new data(remotes_t{std::make_pair(host, port)}, 53 | logger, group_info_update_period, "/var/tmp/libmastermind.cache", 54 | std::numeric_limits::max(), std::numeric_limits::max(), "mastermind", 55 | 4000, 4000, namespace_state_t::user_settings_factory_t(), true)) 56 | { 57 | } 58 | 59 | mastermind_t::mastermind_t( 60 | const remotes_t &remotes, 61 | const std::shared_ptr &logger, 62 | int group_info_update_period, 63 | std::string cache_path, 64 | int expired_time, 65 | std::string worker_name 66 | ) 67 | : m_data(new data(remotes, logger, group_info_update_period, std::move(cache_path), 68 | std::numeric_limits::max(), expired_time, std::move(worker_name), 69 | 4000, 4000, namespace_state_t::user_settings_factory_t(), true)) 70 | { 71 | } 72 | 73 | mastermind_t::mastermind_t(const remotes_t &remotes, 74 | const std::shared_ptr &logger, 75 | int group_info_update_period, std::string cache_path, 76 | int warning_time, int expire_time, 77 | std::string worker_name, 78 | int enqueue_timeout, 79 | int reconnect_timeout) 80 | : m_data(new data(remotes, logger, group_info_update_period, std::move(cache_path), 81 | warning_time, expire_time, std::move(worker_name), 82 | enqueue_timeout, reconnect_timeout, 83 | namespace_state_t::user_settings_factory_t(), true)) 84 | { 85 | } 86 | 87 | mastermind_t::mastermind_t(const remotes_t &remotes, 88 | const std::shared_ptr &logger, 89 | int group_info_update_period, std::string cache_path, 90 | int warning_time, int expire_time, 91 | std::string worker_name, 92 | int enqueue_timeout, 93 | int reconnect_timeout, 94 | namespace_state_t::user_settings_factory_t user_settings_factory 95 | ) 96 | : m_data(new data(remotes, logger, group_info_update_period, std::move(cache_path), 97 | warning_time, expire_time, std::move(worker_name), 98 | enqueue_timeout, reconnect_timeout, 99 | std::move(user_settings_factory), true)) 100 | { 101 | } 102 | 103 | mastermind_t::mastermind_t(const remotes_t &remotes, 104 | const std::shared_ptr &logger, 105 | int group_info_update_period, std::string cache_path, 106 | int warning_time, int expire_time, 107 | std::string worker_name, 108 | int enqueue_timeout, 109 | int reconnect_timeout, 110 | bool auto_start 111 | ) 112 | : m_data(new data(remotes, logger, group_info_update_period, std::move(cache_path), 113 | warning_time, expire_time, std::move(worker_name), 114 | enqueue_timeout, reconnect_timeout, 115 | namespace_state_t::user_settings_factory_t(), auto_start)) 116 | { 117 | } 118 | 119 | mastermind_t::~mastermind_t() 120 | { 121 | if (m_data->is_running()) { 122 | m_data->stop(); 123 | } 124 | } 125 | 126 | void 127 | mastermind_t::start() { 128 | m_data->start(); 129 | } 130 | 131 | void 132 | mastermind_t::stop() { 133 | m_data->stop(); 134 | } 135 | 136 | bool 137 | mastermind_t::is_running() const { 138 | return m_data->is_running(); 139 | } 140 | 141 | bool 142 | mastermind_t::is_valid() const { 143 | return m_data->is_valid(); 144 | } 145 | 146 | std::vector mastermind_t::get_metabalancer_groups(uint64_t count, const std::string &name_space, uint64_t size) { 147 | try { 148 | auto cache = m_data->namespaces_states.copy(name_space); 149 | 150 | if (count != cache.get_value().settings.groups_count) { 151 | throw invalid_groups_count_error(); 152 | } 153 | 154 | auto couple = cache.get_value().weights.get(size); 155 | 156 | { 157 | std::ostringstream oss; 158 | oss 159 | << "libmastermind: get_metabalancer_groups: request={group-count=" << count 160 | << ", namespace=" << name_space << ", size=" << size 161 | << "}; response={" 162 | << "couple=["; 163 | { 164 | auto &&groups = couple.groups; 165 | for (auto beg = groups.begin(), it = beg, end = groups.end(); it != end; ++it) { 166 | if (beg != it) oss << ", "; 167 | oss << *it; 168 | } 169 | } 170 | oss << "], weight=" << couple.weight << ", free-space=" << couple.memory 171 | << "};"; 172 | 173 | auto msg = oss.str(); 174 | COCAINE_LOG_INFO(m_data->m_logger, "%s", msg.c_str()); 175 | } 176 | 177 | return couple.groups; 178 | } catch(const std::system_error &ex) { 179 | COCAINE_LOG_ERROR( 180 | m_data->m_logger, 181 | "libmastermind: cannot obtain couple for: count = %llu; namespace = %s; size = %llu; \"%s\"", 182 | count, name_space.c_str(), size, ex.code().message().c_str()); 183 | throw; 184 | } 185 | } 186 | 187 | group_info_response_t mastermind_t::get_metabalancer_group_info(int group) { 188 | try { 189 | group_info_response_t resp; 190 | 191 | { 192 | std::lock_guard lock(m_data->m_mutex); 193 | (void) lock; 194 | 195 | m_data->enqueue_old("get_group_info", group, resp); 196 | } 197 | return resp; 198 | } catch(const std::system_error &ex) { 199 | COCAINE_LOG_ERROR( 200 | m_data->m_logger, 201 | "libmastermind: get_metabalancer_group_info: group = %d; \"%s\"", 202 | group, ex.code().message().c_str()); 203 | throw; 204 | } 205 | } 206 | 207 | std::map> mastermind_t::get_symmetric_groups() { 208 | try { 209 | auto cache = m_data->fake_groups_info.copy(); 210 | 211 | std::map> result; 212 | 213 | for (auto it = cache.get_value().begin(), end = cache.get_value().end(); 214 | it != end; ++it) { 215 | result.insert(std::make_pair(it->first, it->second.groups)); 216 | } 217 | 218 | return result; 219 | } catch(const std::system_error &ex) { 220 | COCAINE_LOG_ERROR(m_data->m_logger, "libmastermind: get_symmetric_groups: \"%s\"", ex.code().message().c_str()); 221 | throw; 222 | } 223 | } 224 | 225 | std::vector mastermind_t::get_symmetric_groups(int group) { 226 | std::vector result = get_couple_by_group(group); 227 | 228 | if (result.empty()) { 229 | result = {group}; 230 | } 231 | 232 | if (m_data->m_logger->verbosity() >= cocaine::logging::debug) { 233 | std::ostringstream oss; 234 | oss 235 | << "libmastermind: get_symmetric_groups: request={group=" << group 236 | << "}; response={" 237 | << "couple=["; 238 | { 239 | auto &&groups = result; 240 | for (auto beg = groups.begin(), it = beg, end = groups.end(); it != end; ++it) { 241 | if (beg != it) oss << ", "; 242 | oss << *it; 243 | } 244 | } 245 | oss << "]};"; 246 | 247 | auto msg = oss.str(); 248 | COCAINE_LOG_DEBUG(m_data->m_logger, "%s", msg.c_str()); 249 | } 250 | 251 | return result; 252 | } 253 | 254 | std::vector mastermind_t::get_couple_by_group(int group) { 255 | auto cache = m_data->fake_groups_info.copy(); 256 | std::vector result; 257 | 258 | auto git = cache.get_value().find(group); 259 | 260 | if (git != cache.get_value().end()) { 261 | result = git->second.groups; 262 | } 263 | 264 | return result; 265 | } 266 | 267 | std::vector mastermind_t::get_couple(int couple_id, const std::string &ns) { 268 | COCAINE_LOG_INFO(m_data->m_logger, "libmastermind: get_couple: couple_id=%d ns=%s" 269 | , couple_id, ns); 270 | 271 | auto cache = m_data->fake_groups_info.copy(); 272 | std::vector result; 273 | 274 | auto git = cache.get_value().find(couple_id); 275 | 276 | if (git == cache.get_value().end()) { 277 | COCAINE_LOG_ERROR(m_data->m_logger 278 | , "libmastermind: get_couple: cannot find couple by the couple_id"); 279 | return std::vector(); 280 | } 281 | 282 | if (git->second.group_status != 283 | namespace_state_init_t::data_t::couples_t::group_info_t::status_tag::COUPLED) { 284 | COCAINE_LOG_ERROR(m_data->m_logger 285 | , "libmastermind: get_couple: couple status is not COUPLED: %d" 286 | , static_cast(git->second.group_status)); 287 | return std::vector(); 288 | } 289 | 290 | if (git->second.ns != ns) { 291 | COCAINE_LOG_ERROR(m_data->m_logger 292 | , "libmastermind: get_couple: couple belongs to another namespace: %s" 293 | , git->second.ns); 294 | return std::vector(); 295 | } 296 | 297 | { 298 | std::ostringstream oss; 299 | oss << "libmastermind: get_couple: couple was found: ["; 300 | { 301 | const auto &couple = git->second.groups; 302 | for (auto beg = couple.begin(), it = beg, end = couple.end(); it != end; ++it) { 303 | if (beg != it) oss << ", "; 304 | oss << *it; 305 | } 306 | } 307 | oss << "]};"; 308 | 309 | auto msg = oss.str(); 310 | COCAINE_LOG_INFO(m_data->m_logger, "%s", msg.c_str()); 311 | } 312 | 313 | return git->second.groups; 314 | } 315 | 316 | std::vector > mastermind_t::get_bad_groups() { 317 | try { 318 | auto cache = m_data->bad_groups.copy(); 319 | return cache.get_value(); 320 | } catch(const std::system_error &ex) { 321 | COCAINE_LOG_ERROR(m_data->m_logger, "libmastermind: get_bad_groups: \"%s\"", ex.code().message().c_str()); 322 | throw; 323 | } 324 | } 325 | 326 | std::vector mastermind_t::get_all_groups() { 327 | std::vector res; 328 | 329 | auto groups = get_symmetric_groups(); 330 | for (auto it = groups.begin(); it != groups.end(); ++it) { 331 | res.push_back(it->first); 332 | } 333 | 334 | return res; 335 | } 336 | 337 | std::vector mastermind_t::get_cache_groups(const std::string &key) { 338 | COCAINE_LOG_ERROR(m_data-> m_logger 339 | , "libmastermind: get_cache_groups: using of obsolete method"); 340 | return {}; 341 | } 342 | 343 | std::vector mastermind_t::get_namespaces_settings() { 344 | try { 345 | auto cache = m_data->namespaces_settings.copy(); 346 | return cache.get_value(); 347 | } catch(const std::system_error &ex) { 348 | COCAINE_LOG_ERROR( 349 | m_data->m_logger, 350 | "libmastermind: get_namespaces_settings; \"%s\"", 351 | ex.code().message().c_str()); 352 | throw; 353 | } 354 | } 355 | 356 | std::vector mastermind_t::get_elliptics_remotes() { 357 | auto cache = m_data->elliptics_remotes.copy(); 358 | 359 | if (cache.is_expired()) { 360 | return std::vector(); 361 | } 362 | 363 | return cache.get_value(); 364 | } 365 | 366 | std::vector, uint64_t, uint64_t>> mastermind_t::get_couple_list( 367 | const std::string &ns) { 368 | auto namespace_states = m_data->namespaces_states.copy(ns); 369 | 370 | if (namespace_states.is_expired()) { 371 | return std::vector, uint64_t, uint64_t>>(); 372 | } 373 | 374 | const auto &weights = namespace_states.get_value().weights.data(); 375 | 376 | std::map, uint64_t, uint64_t>> result_map; 377 | 378 | for (auto it = weights.begin(), end = weights.end(); it != end; ++it) { 379 | auto weight = it->weight; 380 | auto memory = it->memory; 381 | const auto &couple = it->groups; 382 | auto group_id = it->id; 383 | 384 | result_map.insert(std::make_pair(group_id 385 | , std::make_tuple(couple, weight, memory))); 386 | } 387 | 388 | { 389 | const auto &couples = namespace_states.get_value().couples.couple_info_map; 390 | 391 | for (auto it = couples.begin(), end = couples.end(); it != end; ++it) { 392 | const auto &couple_info = it->second; 393 | const auto &groups = couple_info.groups; 394 | 395 | auto couple_id = *std::min_element(groups.begin(), groups.end()); 396 | 397 | result_map.insert(std::make_pair(couple_id 398 | , std::make_tuple(groups, static_cast(0) 399 | , couple_info.free_effective_space))); 400 | } 401 | } 402 | 403 | std::vector, uint64_t, uint64_t>> result; 404 | result.reserve(result_map.size()); 405 | 406 | for (auto it = result_map.begin(), end = result_map.end(); it != end; ++it) { 407 | result.emplace_back(std::move(it->second)); 408 | } 409 | 410 | return result; 411 | } 412 | 413 | uint64_t mastermind_t::free_effective_space_in_couple_by_group(size_t group) { 414 | auto cache = m_data->fake_groups_info.copy(); 415 | 416 | if (cache.is_expired()) { 417 | return 0; 418 | } 419 | 420 | auto git = cache.get_value().find(group); 421 | if (git == cache.get_value().end()) { 422 | return 0; 423 | } 424 | 425 | return git->second.free_effective_space; 426 | } 427 | 428 | namespace_state_t 429 | mastermind_t::get_namespace_state(const std::string &name) const { 430 | auto data = m_data->get_namespace_state(name); 431 | return namespace_state_init_t(data); 432 | } 433 | 434 | namespace_state_t 435 | mastermind_t::find_namespace_state(group_t group) const { 436 | auto cache = m_data->fake_groups_info.copy(); 437 | auto it = cache.get_value().find(group); 438 | 439 | if (it == cache.get_value().end()) { 440 | throw namespace_state_not_found_error{}; 441 | } 442 | 443 | return get_namespace_state(it->second.ns); 444 | } 445 | 446 | groups_t 447 | mastermind_t::get_cached_groups(const std::string &elliptics_id, group_t couple_id) const { 448 | auto cache = m_data->cached_keys.copy(); 449 | return cache.get_value().get(elliptics_id, couple_id); 450 | } 451 | 452 | std::string mastermind_t::json_group_weights() { 453 | auto cache = m_data->namespaces_states.copy(); 454 | 455 | kora::dynamic_t raw_group_weights = kora::dynamic_t::empty_object; 456 | auto &raw_group_weights_object = raw_group_weights.as_object(); 457 | 458 | for (auto it = cache.begin(), end = cache.end(); it != end; ++it) { 459 | if (it->second.is_expired()) { 460 | continue; 461 | } 462 | 463 | const auto &ns_state = it->second.get_value(); 464 | const auto &ns_raw_state = it->second.get_raw_value(); 465 | 466 | raw_group_weights_object[ns_state.name] = ns_raw_state.as_object()["weights"]; 467 | } 468 | 469 | return kora::to_pretty_json(raw_group_weights); 470 | } 471 | 472 | std::string mastermind_t::json_symmetric_groups() { 473 | auto cache = m_data->fake_groups_info.copy(); 474 | 475 | kora::dynamic_t raw_symmetric_groups = kora::dynamic_t::empty_object; 476 | auto &raw_symmetric_groups_object = raw_symmetric_groups.as_object(); 477 | 478 | for (auto it = cache.get_value().begin(), end = cache.get_value().end(); it != end; ++it) { 479 | raw_symmetric_groups_object[boost::lexical_cast(it->first)] 480 | = it->second.groups; 481 | } 482 | 483 | return kora::to_pretty_json(raw_symmetric_groups); 484 | } 485 | 486 | std::string mastermind_t::json_bad_groups() { 487 | auto cache = m_data->bad_groups.copy(); 488 | 489 | std::ostringstream oss; 490 | oss << "{" << std::endl; 491 | auto ite = cache.get_value().end(); 492 | if (cache.get_value().begin() != cache.get_value().end()) --ite; 493 | for (auto it = cache.get_value().begin(); it != cache.get_value().end(); ++it) { 494 | oss << "\t["; 495 | for (auto it2 = it->begin(); it2 != it->end(); ++it2) { 496 | if (it2 != it->begin()) { 497 | oss << ", "; 498 | } 499 | oss << *it2; 500 | } 501 | oss << "]"; 502 | if (it != ite) { 503 | oss << ','; 504 | } 505 | oss << std::endl; 506 | } 507 | oss << "}"; 508 | 509 | return oss.str(); 510 | } 511 | 512 | std::string mastermind_t::json_cache_groups() { 513 | auto cache = m_data->cached_keys.copy(); 514 | const auto &dynamic = cache.get_raw_value(); 515 | 516 | return kora::to_pretty_json(dynamic); 517 | } 518 | 519 | std::string mastermind_t::json_metabalancer_info() { 520 | auto cache = m_data->namespaces_states.copy(); 521 | 522 | kora::dynamic_t raw_metabalancer_info = kora::dynamic_t::empty_object; 523 | auto &raw_metabalancer_info_object = raw_metabalancer_info.as_object(); 524 | 525 | for (auto it = cache.begin(), end = cache.end(); it != end; ++it) { 526 | if (it->second.is_expired()) { 527 | continue; 528 | } 529 | 530 | const auto &ns_state = it->second.get_value(); 531 | const auto &ns_raw_state = it->second.get_raw_value(); 532 | 533 | raw_metabalancer_info_object[ns_state.name] = ns_raw_state.as_object()["couples"]; 534 | } 535 | 536 | return kora::to_pretty_json(raw_metabalancer_info); 537 | } 538 | 539 | std::string mastermind_t::json_namespaces_settings() { 540 | auto cache = m_data->namespaces_states.copy(); 541 | 542 | kora::dynamic_t raw_namespaces_settings = kora::dynamic_t::empty_object; 543 | auto &raw_namespaces_settings_object = raw_namespaces_settings.as_object(); 544 | 545 | for (auto it = cache.begin(), end = cache.end(); it != end; ++it) { 546 | if (it->second.is_expired()) { 547 | continue; 548 | } 549 | 550 | const auto &ns_state = it->second.get_value(); 551 | const auto &ns_raw_state = it->second.get_raw_value(); 552 | 553 | raw_namespaces_settings_object[ns_state.name] = ns_raw_state.as_object()["settings"]; 554 | } 555 | 556 | return kora::to_pretty_json(raw_namespaces_settings); 557 | } 558 | 559 | std::string mastermind_t::json_namespace_statistics(const std::string &ns) { 560 | auto cache = m_data->namespaces_states.copy(); 561 | auto it = cache.find(ns); 562 | 563 | if (it == cache.end()) { 564 | return {}; 565 | } 566 | 567 | return kora::to_pretty_json(it->second.get_raw_value().as_object()["statistics"]); 568 | } 569 | 570 | void 571 | mastermind_t::set_user_settings_factory(namespace_state_t::user_settings_factory_t user_settings_factory) { 572 | m_data->set_user_settings_factory(std::move(user_settings_factory)); 573 | } 574 | 575 | void mastermind_t::cache_force_update() { 576 | m_data->cache_force_update(); 577 | } 578 | 579 | void mastermind_t::set_update_cache_callback(const std::function &callback) { 580 | m_data->set_update_cache_callback(callback); 581 | } 582 | 583 | void 584 | mastermind_t::set_update_cache_ext1_callback(const std::function &callback) { 585 | m_data->set_update_cache_ext1_callback(callback); 586 | } 587 | 588 | } // namespace mastermind 589 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /tests/test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | 14 | #include 15 | #include 16 | #include 17 | #include "teamcity_cppunit.h" 18 | 19 | 20 | using namespace elliptics; 21 | 22 | class elliptics_manager_t { 23 | public: 24 | virtual elliptics_proxy_t::config config() = 0; 25 | virtual ~elliptics_manager_t() {} 26 | }; 27 | 28 | class script_manager_t : public elliptics_manager_t { 29 | public: 30 | script_manager_t(const std::string &script) 31 | : m_script(script) 32 | { 33 | std::system((script + " prepare 4").c_str()); 34 | nodes(); 35 | } 36 | 37 | ~script_manager_t() { 38 | std::system((m_script + " clear").c_str()); 39 | } 40 | 41 | elliptics_proxy_t::config config() { 42 | elliptics_proxy_t::config elconf; 43 | elconf.groups.push_back(1); 44 | elconf.groups.push_back(2); 45 | //elconf.cocaine_config = std::string("/home/derikon/cocaine/cocaine_config.json"); 46 | elconf.remotes.push_back(elliptics_proxy_t::remote("localhost", 1025, 2)); 47 | return elconf; 48 | } 49 | 50 | private: 51 | 52 | enum node_action { start, stop }; 53 | 54 | template 55 | void node(size_t num) { 56 | std::ostringstream oss; 57 | oss << m_script; 58 | if(na == start) oss << " start "; 59 | else if (na == stop) oss << " stop "; 60 | oss << num; 61 | std::system(oss.str().c_str()); 62 | } 63 | 64 | template 65 | void nodes(std::initializer_list nums) { 66 | for(auto it = nums.begin(); it != nums.end(); ++it) 67 | node(*it); 68 | } 69 | 70 | template 71 | void nodes() { 72 | std::ostringstream oss; 73 | oss << m_script; 74 | if(na == start) oss << " start "; 75 | else if (na == stop) oss << " stop "; 76 | std::system(oss.str().c_str()); 77 | } 78 | 79 | std::string m_script; 80 | }; 81 | 82 | class simple_manager_t : public elliptics_manager_t { 83 | public: 84 | simple_manager_t(const std::string &hostname, int port, int family, 85 | const std::vector &groups) 86 | : m_hostname(hostname) 87 | , m_port(port) 88 | , m_family(family) 89 | , m_groups(groups) 90 | { 91 | } 92 | 93 | elliptics_proxy_t::config config() { 94 | elliptics_proxy_t::config elconf; 95 | std::copy(m_groups.begin(), m_groups.end(), std::back_inserter(elconf.groups)); 96 | //elconf.cocaine_config = std::string("/home/derikon/cocaine/cocaine_config.json"); 97 | elconf.remotes.push_back(elliptics_proxy_t::remote(m_hostname, m_port, m_family)); 98 | return elconf; 99 | } 100 | private: 101 | std::string m_hostname; 102 | int m_port; 103 | int m_family; 104 | std::vector m_groups; 105 | }; 106 | 107 | class elliptics_tests_t : public CppUnit::TestFixture { 108 | public: 109 | elliptics_tests_t(const std::shared_ptr &elliptics_manager) 110 | : m_elliptics_manager(elliptics_manager) 111 | { 112 | elliptics_proxy_t::config elconf = m_elliptics_manager->config(); 113 | elconf.log_mask = 1; 114 | elconf.success_copies_num = SUCCESS_COPIES_TYPE__ALL; 115 | elconf.chunk_size = 16; 116 | 117 | m_proxy.reset(new elliptics_proxy_t(elconf)); 118 | sleep(1); 119 | } 120 | 121 | ~elliptics_tests_t() { 122 | m_proxy.reset(); 123 | } 124 | 125 | void remove() { 126 | elliptics::key_t key("key_remove"); 127 | std::string data("test data"); 128 | std::vector lr = m_proxy->write(key, data); 129 | CPPUNIT_ASSERT(lr.size() == 2); 130 | m_proxy->remove(key); 131 | } 132 | 133 | void write_read() { 134 | elliptics::key_t key("key_write_read"); 135 | std::string data("test data"); 136 | std::vector lr = m_proxy->write(key, data); 137 | CPPUNIT_ASSERT(lr.size() == 2); 138 | data_container_t dc = m_proxy->read(key); 139 | CPPUNIT_ASSERT(dc.data.to_string() == data); 140 | } 141 | 142 | void write_chunk_read() { 143 | elliptics::key_t key("key_write_chunk_read"); 144 | std::string data("long data to be sure chunk write is used"); 145 | std::vector lr = m_proxy->write(key, data); 146 | CPPUNIT_ASSERT(lr.size() == 2); 147 | data_container_t dc = m_proxy->read(key); 148 | CPPUNIT_ASSERT(dc.data.to_string() == data); 149 | } 150 | 151 | void write_parts_read() { 152 | elliptics::key_t key("key_write_parts_read"); 153 | std::string data("test data"); 154 | 155 | std::vector lr; 156 | 157 | lr = m_proxy->write(key, data.substr(0, 3), _ioflags = DNET_IO_FLAGS_PREPARE, _size = data.size()); 158 | CPPUNIT_ASSERT(lr.size() == 2); 159 | 160 | lr = m_proxy->write(key, data.substr(3, 3), _ioflags = DNET_IO_FLAGS_PLAIN_WRITE, _offset = 3); 161 | CPPUNIT_ASSERT(lr.size() == 2); 162 | 163 | lr = m_proxy->write(key, data.substr(6, 3), _ioflags = DNET_IO_FLAGS_COMMIT, _size = data.size(), _offset = 6); 164 | CPPUNIT_ASSERT(lr.size() == 2); 165 | 166 | data_container_t dc = m_proxy->read(key); 167 | CPPUNIT_ASSERT(dc.data.to_string() == data); 168 | } 169 | 170 | void bulk_write_read() { 171 | std::vector keys = {std::string("key_bulk_write_read_1"), 172 | std::string("key_bulk_write_read_2")}; 173 | std::vector data = {std::string("test data 1"), 174 | std::string("test data 2")}; 175 | 176 | { 177 | auto results = m_proxy->bulk_write (keys, data); 178 | 179 | CPPUNIT_ASSERT(results.size() == 2); 180 | 181 | for (auto it = results.begin (), end = results.end (); it != end; ++it) { 182 | CPPUNIT_ASSERT(it->second.size() == 2); 183 | } 184 | } 185 | 186 | { 187 | auto result = m_proxy->bulk_read (keys); 188 | 189 | CPPUNIT_ASSERT(result.size() == 2); 190 | 191 | for (size_t index = 0; index != 2; ++index) { 192 | CPPUNIT_ASSERT(result[keys[index]].data.to_string() == data[index].data.to_string()); 193 | } 194 | } 195 | } 196 | 197 | void async_write_read() { 198 | elliptics::key_t key1("key_async_write_read_1"); 199 | elliptics::key_t key2("key_async_write_read_2"); 200 | 201 | std::string data1("test data 1"); 202 | std::string data2("test data 2"); 203 | 204 | std::vector l; 205 | auto awr1 = m_proxy->write_async(key1, data1); 206 | auto awr2 = m_proxy->write_async(key2, data2); 207 | 208 | l = awr1.get (); 209 | CPPUNIT_ASSERT(l.size() == 2); 210 | 211 | l = awr2.get (); 212 | CPPUNIT_ASSERT(l.size() == 2); 213 | 214 | data_container_t dc; 215 | auto arr1 = m_proxy->read_async(key1); 216 | auto arr2 = m_proxy->read_async(key2); 217 | 218 | dc = arr1.get_one(); 219 | CPPUNIT_ASSERT(dc.data.to_string() == data1); 220 | 221 | dc = arr2.get_one(); 222 | CPPUNIT_ASSERT(dc.data.to_string() == data2); 223 | } 224 | 225 | void write_read_with_embeds() { 226 | elliptics::key_t key("key_write_read_with_embeds"); 227 | std::string data("test data"); 228 | timespec ts; 229 | ts.tv_sec = 123; 230 | ts.tv_nsec = 456789; 231 | 232 | { 233 | data_container_t dc(data); 234 | dc.set(ts); 235 | auto lr = m_proxy->write(key, dc); 236 | CPPUNIT_ASSERT(lr.size() == 2); 237 | } 238 | { 239 | auto dc = m_proxy->read(key, _embeded = true); 240 | CPPUNIT_ASSERT(data == dc.data.to_string()); 241 | timespec ts2 = *dc.get(); 242 | CPPUNIT_ASSERT(ts2.tv_sec == ts.tv_sec); 243 | CPPUNIT_ASSERT(ts2.tv_nsec == ts.tv_nsec); 244 | } 245 | } 246 | 247 | void write_read_with_smart_embeds() { 248 | elliptics::key_t key("key_write_read_with_smart_embeds"); 249 | std::string data("test data"); 250 | timespec ts; 251 | ts.tv_sec = 123; 252 | ts.tv_nsec = 456789; 253 | 254 | { 255 | data_container_t dc(data); 256 | dc.set(ts); 257 | auto lr = m_proxy->write(key, dc); 258 | CPPUNIT_ASSERT(lr.size() == 2); 259 | } 260 | { 261 | auto dc = m_proxy->read(key); 262 | CPPUNIT_ASSERT(data == dc.data.to_string()); 263 | timespec ts2 = *dc.get(); 264 | CPPUNIT_ASSERT(ts2.tv_sec == ts.tv_sec); 265 | CPPUNIT_ASSERT(ts2.tv_nsec == ts.tv_nsec); 266 | } 267 | } 268 | 269 | void lookup() { 270 | elliptics::key_t key("key_lookup"); 271 | std::string data("data"); 272 | std::vector groups = {2, 3}; 273 | auto lr = m_proxy->write(key, data, _groups = groups); 274 | CPPUNIT_ASSERT(lr.size() == 2); 275 | auto l = m_proxy->lookup(key); 276 | CPPUNIT_ASSERT(l.port() == 1026); 277 | } 278 | 279 | void write_g4_scnALL() { 280 | auto lrs = write("write_g4_scnALL", 0, 0, 0, 0, {1, 2, 3, 4}, SUCCESS_COPIES_TYPE__ALL); 281 | CPPUNIT_ASSERT(lrs.size() == 4); 282 | } 283 | 284 | void write_g3_scnALL() { 285 | auto lrs = write("write_g3_scnALL", 0, 0, 0, 0, {1, 2, 3}, SUCCESS_COPIES_TYPE__ALL); 286 | CPPUNIT_ASSERT(lrs.size() == 3); 287 | } 288 | 289 | void write_g2_1_scnALL() { 290 | std::vector lrs; 291 | try { lrs = write("write_g2_1_scnALL", 0, 0, 0, 0, {1, 2, 101}, SUCCESS_COPIES_TYPE__ALL); } catch (...) {} 292 | CPPUNIT_ASSERT(lrs.size() == 0); 293 | } 294 | 295 | void write_g3_scnQUORUM() { 296 | auto lrs = write("write_g3_scnQUORUM", 0, 0, 0, 0, {1, 2, 3}, SUCCESS_COPIES_TYPE__QUORUM); 297 | CPPUNIT_ASSERT(lrs.size() == 3); 298 | } 299 | 300 | void write_g2_1_scnQUORUM() { 301 | auto lrs = write("write_g2_1_scnQUORUM", 0, 0, 0, 0, {1, 2, 101}, SUCCESS_COPIES_TYPE__QUORUM); 302 | CPPUNIT_ASSERT(lrs.size() == 2); 303 | } 304 | 305 | void write_g1_2_scnQUORUM() { 306 | std::vector lrs; 307 | try { lrs = write("write_g1_2_scnQUORUM", 0, 0, 0, 0, {1, 101, 102}, SUCCESS_COPIES_TYPE__QUORUM); } catch (...) {} 308 | CPPUNIT_ASSERT(lrs.size() == 0); 309 | } 310 | 311 | void write_g2_1_scnANY() { 312 | auto lrs = write("write_g2_1_scnANY", 0, 0, 0, 0, {1, 2, 102}, SUCCESS_COPIES_TYPE__ANY); 313 | CPPUNIT_ASSERT(lrs.size() == 2); 314 | } 315 | 316 | void write_g1_2_scnANY() { 317 | auto lrs = write("write_g1_2_scnANY", 0, 0, 0, 0, {1, 101, 102}, SUCCESS_COPIES_TYPE__ANY); 318 | CPPUNIT_ASSERT(lrs.size() == 1); 319 | } 320 | 321 | void write_g0_3_scnANY() { 322 | std::vector lrs; 323 | try { lrs = write("write_g0_3_scnANY", 0, 0, 0, 0, {101, 102, 103}, SUCCESS_COPIES_TYPE__ANY); } catch (...) {} 324 | CPPUNIT_ASSERT(lrs.size() == 0); 325 | } 326 | private: 327 | std::vector write(const char *key, uint64_t offset, uint64_t size, uint64_t cflags, uint64_t ioflags, std::vector groups, int success_copies_num) { 328 | return m_proxy->write(elliptics::key_t(key), std::string("test data"), _offset = offset, _size = size, _cflags = cflags, _ioflags = ioflags, _groups = groups, _success_copies_num = success_copies_num); 329 | } 330 | 331 | std::shared_ptr m_elliptics_manager; 332 | std::shared_ptr m_proxy; 333 | }; 334 | 335 | #define ADD_TEST(name, func) suite.addTest(new elliptics_caller_t(name, &elliptics_tests_t:: func, elliptics_tests)) 336 | #define ADD_TEST1(func) ADD_TEST(#func, func) 337 | 338 | void print_usage() { 339 | std::cout << "Usage: appname [--remote hostname:port:family groups]" << std::endl; 340 | } 341 | 342 | int main(int argc, char* argv[]) 343 | { 344 | std::shared_ptr elliptics_manager; 345 | if (strcmp(argv[1], "--remote") == 0) { 346 | if (argc != 4) { 347 | print_usage(); 348 | return 0; 349 | } 350 | 351 | typedef boost::char_separator separator_t; 352 | typedef boost::tokenizer tokenizer_t; 353 | 354 | std::string remote = argv[2]; 355 | separator_t sep(":"); 356 | tokenizer_t tok(remote, sep); 357 | 358 | std::string hostname; 359 | int port; 360 | int family; 361 | 362 | tokenizer_t::iterator tit = tok.begin(); 363 | hostname = *tit; 364 | port = boost::lexical_cast(*(++tit)); 365 | family = boost::lexical_cast(*(++tit)); 366 | 367 | if (++tit != tok.end()) { 368 | std::cout << "Incorrect remote" << std::endl; 369 | print_usage(); 370 | return 0; 371 | } 372 | 373 | std::string str_groups = argv[3]; 374 | std::vector groups; 375 | tokenizer_t tok2(str_groups, sep); 376 | for (tokenizer_t::iterator it = tok2.begin(), end = tok2.end(); end != it; ++it) { 377 | groups.push_back(boost::lexical_cast(*it)); 378 | } 379 | 380 | elliptics_manager.reset(new simple_manager_t(hostname, port, family, groups)); 381 | } else { 382 | elliptics_manager.reset(new script_manager_t(std::string(argv[1]) + "/manager.sh")); 383 | } 384 | 385 | CppUnit::TestResult controller; 386 | 387 | CppUnit::TestResultCollector result; 388 | controller.addListener(&result); 389 | 390 | std::unique_ptr listener(new JetBrains::TeamcityProgressListener()); 391 | controller.addListener(listener.get()); 392 | CppUnit::TestSuite suite; 393 | 394 | elliptics_tests_t elliptics_tests(elliptics_manager); 395 | typedef CppUnit::TestCaller elliptics_caller_t; 396 | 397 | ADD_TEST("write and remove", remove); 398 | ADD_TEST("simple write and read", write_read); 399 | ADD_TEST("chunked write and read", write_chunk_read); 400 | ADD_TEST("prepare-commit write and read", write_parts_read); 401 | ADD_TEST("bulk write and bulk read", bulk_write_read); 402 | ADD_TEST("async write and async read", async_write_read); 403 | ADD_TEST("lookup", lookup); 404 | ADD_TEST("write and read with embeds", write_read_with_embeds); 405 | ADD_TEST("write and read with smart embeds", write_read_with_smart_embeds); 406 | 407 | ADD_TEST("write into 4 groups; with SUCCESS_COPIES_TYPE__ALL", write_g4_scnALL); 408 | ADD_TEST("write into 3 groups; with SUCCESS_COPIES_TYPE__ALL", write_g3_scnALL); 409 | ADD_TEST("write into 3 groups, 1 group is shut down; with SUCCESS_COPIES_TYPE__ALL", write_g2_1_scnALL); 410 | ADD_TEST("write into 3 groups; with SUCCESS_COPIES_TYPE__QUORUM", write_g3_scnQUORUM); 411 | ADD_TEST("write into 3 groups, 1 group is shut down; with SUCCESS_COPIES_TYPE__QUORUM", write_g2_1_scnQUORUM); 412 | ADD_TEST("write into 3 groups, 2 groups are shut down; with SUCCESS_COPIES_TYPE__QUORUM", write_g1_2_scnQUORUM); 413 | ADD_TEST("write into 3 groups, 1 group is shut down; with SUCCESS_COPIES_TYPE__ANY", write_g2_1_scnANY); 414 | ADD_TEST("write into 3 groups, 2 groups are shut down; with SUCCESS_COPIES_TYPE__ANY", write_g1_2_scnANY); 415 | ADD_TEST("write into 3 groups, 3 groups are shut down; with SUCCESS_COPIES_TYPE__ANY", write_g0_3_scnANY); 416 | 417 | suite.run(&controller); 418 | return result.wasSuccessful() ? 0 : 1;; 419 | #if 0 420 | //test_lookup (); 421 | //test_read_async (); 422 | test_async (); 423 | //test_sync (); 424 | return 0; 425 | elliptics_proxy_t::config c; 426 | c.groups.push_back(1); 427 | c.groups.push_back(2); 428 | c.log_mask = 1; 429 | //c.cocaine_config = std::string("/home/toshik/cocaine/cocaine_config.json"); 430 | 431 | //c.remotes.push_back(EllipticsProxy::remote("elisto22f.dev.yandex.net", 1025)); 432 | c.remotes.push_back(elliptics_proxy_t::remote("derikon.dev.yandex.net", 1025, 2)); 433 | 434 | elliptics_proxy_t proxy(c); 435 | 436 | sleep(1); 437 | elliptics::key_t k(std::string("test")); 438 | 439 | std::string data("test3"); 440 | 441 | std::vector keys = {std::string ("key5"), std::string ("key6")}; 442 | std::vector data_arr = {"data1", "data2"}; 443 | //std::vector keys = {std::string ("key1")}; 444 | //std::vector data_arr = {"data1"}; 445 | std::vector groups = {1, 2}; 446 | 447 | { 448 | auto results = proxy.bulk_write (keys, data_arr, _groups = groups); 449 | for (auto it = results.begin (), end = results.end (); it != end; ++it) { 450 | std::cout << it->first.to_string () << ':' << std::endl; 451 | for (auto it2 = it->second.begin (), end2 = it->second.end (); it2 != end2; ++it2) { 452 | std::cout << "\tgroup: " << it2->group << "\tpath: " << it2->hostname 453 | << ":" << it2->port << it2->path << std::endl; 454 | } 455 | } 456 | } 457 | 458 | /*for (int i = 0; i != 2; ++i) { 459 | auto result = proxy.write (keys [i], data_arr [i], _groups = groups, _replication_count = groups.size ()); 460 | std::cout << keys [i].str () << ':' << std::endl; 461 | for (auto it = result.begin (), end = result.end (); it != end; ++it) { 462 | std::cout << "\tgroup: " << it->group << "\tpath: " << it->hostname 463 | << ":" << it->port << it->path << std::endl; 464 | } 465 | }*/ 466 | 467 | /*std::vector g = {2}; 468 | 469 | for (int i = 0; i != 2; ++i){ 470 | auto result = proxy.read (keys [0], _groups = std::vector (1, i + 1)); 471 | std::cout << result.data << std::endl; 472 | }*/ 473 | 474 | { 475 | auto result = proxy.bulk_read (keys, _groups = groups); 476 | //std::cout << "Read data: " << result.data << std::endl; 477 | for (auto it = result.begin (), end = result.end (); it != end; ++it) { 478 | //std::cout << it->first.str () << '\t' << it->second.data << std::endl; 479 | std::cout << it->second.data << std::endl; 480 | } 481 | } 482 | 483 | return 0; 484 | 485 | /* 486 | std::vector lg = proxy.get_metabalancer_groups(3); 487 | 488 | std::cout << "Got groups: " << std::endl; 489 | for (std::vector::const_iterator it = lg.begin(); it != lg.end(); it++) 490 | std::cout << *it << " "; 491 | std::cout << std::endl; 492 | */ 493 | 494 | group_info_response_t gi = proxy.get_metabalancer_group_info(103); 495 | std::cout << "Got info from mastermind: status: " << gi.status << ", " << gi.nodes.size() << " groups: "; 496 | for (std::vector::const_iterator it = gi.couples.begin(); it != gi.couples.end(); it++) 497 | std::cout << *it << " "; 498 | std::cout << std::endl; 499 | 500 | std::vector remotes = proxy.lookup_addr(k, gi.couples); 501 | for (std::vector::const_iterator it = remotes.begin(); it != remotes.end(); ++it) 502 | std::cout << it->host << " "; 503 | std::cout << std::endl; 504 | 505 | struct dnet_id id; 506 | 507 | memset(&id, 0, sizeof(id)); 508 | for (int i = 0; i < DNET_ID_SIZE; i++) 509 | id.id[i] = i; 510 | 511 | elliptics::key_t key1(id); 512 | 513 | memset(&id, 0, sizeof(id)); 514 | for (int i = 0; i < DNET_ID_SIZE; i++) 515 | id.id[i] = i+16; 516 | 517 | elliptics::key_t key2(id); 518 | 519 | 520 | std::cout << "ID1: " << dnet_dump_id_len (&key1.id (), 6) << " " 521 | << " ID2: " << dnet_dump_id_len (&key2.id (), 6) << std::endl; 522 | std::cout << "Key1: " << key1.to_string () << " " << (key1 < key2) << " Key2: " << key2.to_string () << std::endl; 523 | 524 | 525 | 526 | 527 | 528 | return 0; 529 | 530 | std::vector::const_iterator it; 531 | std::vector l = proxy.write(k, data); 532 | std::cout << "written " << l.size() << " copies" << std::endl; 533 | for (it = l.begin(); it != l.end(); ++it) { 534 | std::cout << " path: " << it->hostname << ":" << it->port << it->path << std::endl; 535 | } 536 | 537 | for (int i = 0; i < 20; i++) { 538 | lookup_result_t l1 = proxy.lookup(k); 539 | std::cout << "lookup path: " << l1.hostname << ":" << l1.port << l1.path << std::endl; 540 | } 541 | 542 | read_result_t res = proxy.read(k); 543 | 544 | std::cout << "!!!!!!!!!!!!!!!!!!!!!!!!!!!" << res.data << std::endl; 545 | 546 | return 0; 547 | #endif 548 | } 549 | 550 | -------------------------------------------------------------------------------- /bindings/python/main.cpp: -------------------------------------------------------------------------------- 1 | /* 2 | Client library for mastermind 3 | Copyright (C) 2013-2015 Yandex 4 | 5 | This program is free software; you can redistribute it and/or 6 | modify it under the terms of the GNU General Public License 7 | as published by the Free Software Foundation; either version 2 8 | of the License, or (at your option) any later version. 9 | 10 | This program is distributed in the hope that it will be useful, 11 | but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 | GNU General Public License for more details. 14 | 15 | You should have received a copy of the GNU General Public License 16 | along with this program; if not, write to the Free Software 17 | Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. 18 | */ 19 | 20 | #include "libmastermind/mastermind.hpp" 21 | 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #include 31 | #include 32 | 33 | namespace bp = boost::python; 34 | namespace mm = mastermind; 35 | 36 | namespace mastermind { 37 | namespace binding { 38 | 39 | class gil_guard_t { 40 | public: 41 | gil_guard_t() 42 | : gstate(PyGILState_Ensure()) 43 | , is_released(false) 44 | { 45 | } 46 | 47 | ~gil_guard_t() { 48 | release(); 49 | } 50 | 51 | gil_guard_t(const gil_guard_t &) = delete; 52 | 53 | gil_guard_t(gil_guard_t &&) = delete; 54 | 55 | gil_guard_t & 56 | operator = (const gil_guard_t &) = delete; 57 | 58 | gil_guard_t & 59 | operator = (gil_guard_t &&) = delete; 60 | 61 | void 62 | release() { 63 | if (is_released) { 64 | return; 65 | } 66 | 67 | PyGILState_Release(gstate); 68 | is_released = true; 69 | } 70 | 71 | private: 72 | PyGILState_STATE gstate; 73 | bool is_released; 74 | }; 75 | 76 | class py_allow_threads_scoped 77 | { 78 | public: 79 | py_allow_threads_scoped() 80 | : save(PyEval_SaveThread()) 81 | {} 82 | 83 | void disallow() 84 | { 85 | PyEval_RestoreThread(save); 86 | save = NULL; 87 | } 88 | 89 | ~py_allow_threads_scoped() 90 | { 91 | if (save) 92 | PyEval_RestoreThread(save); 93 | } 94 | private: 95 | PyThreadState* save; 96 | }; 97 | 98 | namespace detail { 99 | 100 | bp::object 101 | convert(const kora::dynamic_t &d, const gil_guard_t &gil_guard); 102 | 103 | bp::object 104 | convert(const kora::dynamic_t::array_t &v, const gil_guard_t &gil_guard) { 105 | bp::list result; 106 | 107 | for (auto it = v.begin(), end = v.end(); it != end; ++it) { 108 | result.append(convert(*it, gil_guard)); 109 | } 110 | 111 | return result; 112 | } 113 | 114 | bp::object 115 | convert(const kora::dynamic_t::object_t &v, const gil_guard_t &gil_guard) { 116 | bp::dict result; 117 | 118 | for (auto it = v.begin(), end = v.end(); it != end; ++it) { 119 | result[it->first] = convert(it->second, gil_guard); 120 | } 121 | 122 | return result; 123 | } 124 | 125 | bp::object 126 | convert(const kora::dynamic_t &d, const gil_guard_t &gil_guard) { 127 | if (d.is_null()) { 128 | return bp::object{}; 129 | } else if (d.is_bool()) { 130 | return bp::object{d.as_bool()}; 131 | } else if (d.is_int()) { 132 | return bp::object{d.as_int()}; 133 | } else if (d.is_uint()) { 134 | return bp::object{d.as_uint()}; 135 | } else if (d.is_double()) { 136 | return bp::object{d.as_double()}; 137 | } else if (d.is_string()) { 138 | return bp::object{d.as_string()}; 139 | } else if (d.is_array()) { 140 | return convert(d.as_array(), gil_guard); 141 | } else if (d.is_object()) { 142 | return convert(d.as_object(), gil_guard); 143 | } 144 | 145 | return bp::object{}; 146 | } 147 | 148 | } // namespace detail 149 | 150 | class logger_t : public cocaine::framework::logger_t { 151 | public: 152 | typedef cocaine::logging::priorities verbosity_t; 153 | 154 | logger_t(const gil_guard_t &) 155 | : logging(new bp::object{bp::import("logging")}) 156 | , logger(new bp::object{logging->attr("getLogger")("mastermind_cache")}) 157 | { 158 | } 159 | 160 | void emit(verbosity_t priority, const std::string& message) { 161 | gil_guard_t gil_guard; 162 | logger->attr(get_name(priority))("%s", message.c_str()); 163 | } 164 | 165 | verbosity_t verbosity() const { 166 | return verbosity_t::debug; 167 | } 168 | 169 | ~logger_t() { 170 | gil_guard_t gil_guard; 171 | logger.reset(); 172 | logging.reset(); 173 | } 174 | 175 | private: 176 | static 177 | const char * 178 | get_name(verbosity_t priority) { 179 | switch (priority) { 180 | case verbosity_t::debug: 181 | return "debug"; 182 | case verbosity_t::info: 183 | return "info"; 184 | case verbosity_t::warning: 185 | return "warning"; 186 | case verbosity_t::error: 187 | return "error"; 188 | default: 189 | return "unknown"; 190 | } 191 | } 192 | 193 | std::unique_ptr logging; 194 | std::unique_ptr logger; 195 | }; 196 | 197 | class namespace_state_t { 198 | public: 199 | class user_settings_t : public mm::namespace_state_t::user_settings_t { 200 | public: 201 | user_settings_t(bp::object dict_, const gil_guard_t &) 202 | : dict(new bp::object{std::move(dict_)}) 203 | { 204 | } 205 | 206 | ~user_settings_t() { 207 | gil_guard_t gil_guard; 208 | dict.reset(); 209 | } 210 | 211 | std::unique_ptr dict; 212 | }; 213 | 214 | class couples_t { 215 | public: 216 | couples_t(mm::namespace_state_t impl_) 217 | : impl(std::move(impl_)) 218 | {} 219 | 220 | bp::list 221 | get_couple_read_preference(int group) const { 222 | auto pref = impl.couples().get_couple_read_preference(group); 223 | 224 | bp::list result; 225 | 226 | for (auto it = pref.begin(), end = pref.end(); it != end; ++it) { 227 | result.append(*it); 228 | } 229 | 230 | return result; 231 | } 232 | 233 | bp::dict 234 | get_couple_groupset(int group, const std::string &groupset_id) const { 235 | auto groupset = impl.couples().get_couple_groupset(group, groupset_id); 236 | 237 | bp::dict result; 238 | 239 | result["free_effective_space"] = groupset.free_effective_space(); 240 | result["free_reserved_space"] = groupset.free_reserved_space(); 241 | 242 | result["type"] = groupset.type(); 243 | result["status"] = groupset.status(); 244 | result["id"] = groupset.id(); 245 | 246 | bp::list group_ids; 247 | for (int g : groupset.group_ids()) { 248 | group_ids.append(g); 249 | } 250 | 251 | result["group_ids"] = group_ids; 252 | 253 | gil_guard_t gil_guard; 254 | result["hosts"] = detail::convert(groupset.hosts(), gil_guard); 255 | result["settings"] = detail::convert(groupset.settings(), gil_guard); 256 | 257 | return result; 258 | } 259 | 260 | bp::list 261 | get_couple_groupset_ids(int group) const { 262 | auto groupset_ids = impl.couples().get_couple_groupset_ids(group); 263 | 264 | bp::list result; 265 | 266 | for (auto it = groupset_ids.begin(), end = groupset_ids.end(); it != end; ++it) { 267 | result.append(*it); 268 | } 269 | 270 | return result; 271 | } 272 | 273 | // the method is always called from python's thread only 274 | bp::list 275 | get_couple_groups(int group) const { 276 | auto groups = impl.couples().get_couple_groups(group); 277 | 278 | bp::list result; 279 | 280 | for (auto it = groups.begin(), end = groups.end(); it != end; ++it) { 281 | result.append(*it); 282 | } 283 | 284 | return result; 285 | } 286 | 287 | // the method is always called from python's thread only 288 | bp::list 289 | get_groups(int group) const { 290 | auto groups = impl.couples().get_groups(group); 291 | 292 | bp::list result; 293 | 294 | for (auto it = groups.begin(), end = groups.end(); it != end; ++it) { 295 | result.append(*it); 296 | } 297 | 298 | return result; 299 | } 300 | 301 | bp::object 302 | hosts(int group) const { 303 | auto hosts = impl.couples().hosts(group); 304 | 305 | gil_guard_t gil_guard; 306 | return detail::convert(hosts, gil_guard); 307 | } 308 | 309 | private: 310 | mm::namespace_state_t impl; 311 | }; 312 | 313 | class settings_t { 314 | public: 315 | settings_t(mm::namespace_state_t impl_) 316 | : impl(std::move(impl_)) 317 | {} 318 | 319 | size_t 320 | groups_count() const { 321 | return impl.settings().groups_count(); 322 | } 323 | 324 | const std::string & 325 | success_copies_num() const { 326 | return impl.settings().success_copies_num(); 327 | } 328 | 329 | // the method is always called from python's thread only 330 | const bp::object & 331 | user_settings() const { 332 | return *(static_cast(impl.settings().user_settings()).dict); 333 | } 334 | 335 | private: 336 | mm::namespace_state_t impl; 337 | }; 338 | 339 | namespace_state_t(mm::namespace_state_t impl_) 340 | : impl(std::move(impl_)) 341 | {} 342 | 343 | couples_t 344 | couples() const { 345 | return {impl}; 346 | } 347 | 348 | settings_t 349 | settings() const { 350 | return {impl}; 351 | } 352 | 353 | const std::string & 354 | name() const { 355 | return impl.name(); 356 | } 357 | 358 | private: 359 | mm::namespace_state_t impl; 360 | }; 361 | 362 | class mastermind_t { 363 | public: 364 | // the constructor is always called from python's thread only 365 | mastermind_t(const std::string &remotes, int update_period, std::string cache_path 366 | , int warning_time, int expire_time, std::string worker_name 367 | , int enqueue_timeout, int reconnect_timeout, bp::object ns_filter_, bool auto_start) 368 | { 369 | gil_guard_t gil_guard; 370 | auto native_remotes = parse_remotes(remotes); 371 | auto logger = std::make_shared(gil_guard); 372 | set_ns_filter(std::move(ns_filter_)); 373 | gil_guard.release(); 374 | 375 | py_allow_threads_scoped gil_release; 376 | impl = std::make_shared( 377 | native_remotes, logger, update_period, std::move(cache_path) 378 | , warning_time, expire_time, std::move(worker_name) 379 | , enqueue_timeout, reconnect_timeout, false); 380 | 381 | impl->set_user_settings_factory([this] (const std::string &name 382 | , const kora::config_t &config) { 383 | return user_settings_factory(name, config); 384 | }); 385 | 386 | if (auto_start) { 387 | start_impl(gil_release); 388 | } 389 | } 390 | 391 | // the destructor is always called from python's thread only 392 | ~mastermind_t() { 393 | if (is_running()) { 394 | stop(); 395 | } 396 | } 397 | 398 | void 399 | start() { 400 | py_allow_threads_scoped gil_release; 401 | start_impl(gil_release); 402 | } 403 | 404 | void 405 | stop() { 406 | py_allow_threads_scoped gil_release; 407 | impl->stop(); 408 | } 409 | 410 | bool 411 | is_running() const { 412 | return impl->is_running(); 413 | } 414 | 415 | bool 416 | is_valid() const { 417 | return impl->is_valid(); 418 | } 419 | 420 | namespace_state_t 421 | get_namespace_state(const std::string &name) const { 422 | return {impl->get_namespace_state(name)}; 423 | } 424 | 425 | namespace_state_t 426 | find_namespace_state(int group) const { 427 | return {impl->find_namespace_state(group)}; 428 | } 429 | 430 | // the method is always called from python's thread only 431 | void 432 | set_ns_filter(bp::object ns_filter_) { 433 | ns_filter = ns_filter_; 434 | } 435 | 436 | private: 437 | static 438 | std::vector 439 | parse_remotes(const std::string &remotes) { 440 | typedef boost::char_separator separator_t; 441 | typedef boost::tokenizer tokenizer_t; 442 | 443 | std::vector result; 444 | 445 | separator_t sep1(","); 446 | tokenizer_t tok1(remotes, sep1); 447 | 448 | separator_t sep2(":"); 449 | 450 | for (auto it = tok1.begin(), end = tok1.end(); it != end; ++it) { 451 | tokenizer_t tok2(*it, sep2); 452 | auto jt = tok2.begin(); 453 | 454 | if (tok2.end() == jt) { 455 | throw std::runtime_error("remotes are malformed"); 456 | } 457 | 458 | auto host = *jt++; 459 | uint16_t port = 10053; 460 | 461 | if (tok2.end() != jt) { 462 | port = boost::lexical_cast(*jt++); 463 | } 464 | 465 | if (tok2.end() != jt) { 466 | throw std::runtime_error("remotes are malformed"); 467 | } 468 | 469 | result.emplace_back(std::make_pair(std::move(host), port)); 470 | } 471 | 472 | return result; 473 | } 474 | 475 | mm::namespace_state_t::user_settings_ptr_t 476 | user_settings_factory(const std::string &name, const kora::config_t &config) { 477 | gil_guard_t gil_guard; 478 | 479 | auto settings = detail::convert(config.underlying_object().as_object(), gil_guard); 480 | 481 | if (ns_filter) { 482 | try { 483 | if (!ns_filter(name, settings)) { 484 | return nullptr; 485 | } 486 | } catch (const std::exception &ex) { 487 | // ns_filter can throw python object that must be destroyed during gil is locked 488 | // that is during gil_guard is not out of scope. 489 | throw std::runtime_error(std::string{"ns_filter error: "} + ex.what()); 490 | } 491 | } 492 | 493 | std::unique_ptr result{ 494 | new namespace_state_t::user_settings_t{std::move(settings), gil_guard} 495 | }; 496 | 497 | return mm::namespace_state_t::user_settings_ptr_t(std::move(result)); 498 | } 499 | 500 | void 501 | start_impl(py_allow_threads_scoped &) { 502 | impl->start(); 503 | } 504 | 505 | std::shared_ptr impl; 506 | bp::object ns_filter; 507 | }; 508 | 509 | namespace exception { 510 | 511 | PyObject * 512 | make_exception_class(const std::string &class_name, PyObject *base_type = PyExc_Exception) { 513 | std::string scope_name = bp::extract(bp::scope().attr("__name__")); 514 | auto full_name = scope_name + '.' + class_name; 515 | char *raw_full_name = &full_name.front(); 516 | 517 | auto *exception_type = PyErr_NewException(raw_full_name, base_type, 0); 518 | 519 | if (!exception_type) { 520 | bp::throw_error_already_set(); 521 | } 522 | 523 | // bp::scope().attr(class_name) = bp::handle<>(bp::borrowed(exception_type)); 524 | bp::scope().attr(class_name.c_str()) = bp::handle<>(exception_type); 525 | return exception_type; 526 | } 527 | 528 | namespace detail { 529 | 530 | template 531 | std::string 532 | exception_message(const Ex &ex) { 533 | return ex.what(); 534 | } 535 | 536 | template <> 537 | std::string 538 | exception_message(const mm::unknown_feedback &ex) { 539 | std::ostringstream oss; 540 | oss << ex.what() << ": couple_id=" << ex.couple_id() << "; feedback=" << ex.feedback(); 541 | return oss.str(); 542 | } 543 | 544 | template <> 545 | std::string 546 | exception_message(const mm::unknown_group_error &ex) { 547 | std::ostringstream oss; 548 | oss << ex.what() << ": group=" << ex.group(); 549 | return oss.str(); 550 | } 551 | 552 | template <> 553 | std::string 554 | exception_message(const mm::unknown_groupset_error &ex) { 555 | std::ostringstream oss; 556 | oss << ex.what() << ": groupset=" << ex.groupset(); 557 | return oss.str(); 558 | } 559 | 560 | } // namespace detail 561 | 562 | template 563 | void 564 | register_exception_translator(const std::string &class_name, PyObject *base_type) { 565 | auto *exception_class = make_exception_class(class_name, base_type); 566 | 567 | auto translate = [exception_class] (const Ex &ex) { 568 | // PyErr_SetString(exception_class, ex.what()); 569 | PyErr_SetString(exception_class, detail::exception_message(ex).c_str()); 570 | }; 571 | 572 | bp::register_exception_translator(std::move(translate)); 573 | } 574 | 575 | } // namespace exception 576 | 577 | void 578 | init_exception_translator() { 579 | auto *mastermind_cache_error 580 | = exception::make_exception_class("MastermindCacheError"); 581 | 582 | //IMPORTANT: ensure that the following exception list is in sync with 583 | // the exception list in include/libmastermind/error.hpp 584 | 585 | exception::register_exception_translator( 586 | "CoupleNotFoundError", mastermind_cache_error); 587 | 588 | exception::register_exception_translator( 589 | "NotEnoughMemoryError", mastermind_cache_error); 590 | 591 | exception::register_exception_translator( 592 | "UnknownNamespaceError", mastermind_cache_error); 593 | 594 | exception::register_exception_translator( 595 | "InvalidGroupsCountError", mastermind_cache_error); 596 | 597 | exception::register_exception_translator( 598 | "CacheIsExpiredError", mastermind_cache_error); 599 | 600 | exception::register_exception_translator( 601 | "UpdateLoopAlreadyStartedError", mastermind_cache_error); 602 | 603 | exception::register_exception_translator( 604 | "UpdateLoopAlreadyStopped", mastermind_cache_error); 605 | 606 | exception::register_exception_translator( 607 | "UnknownFeedback", mastermind_cache_error); 608 | 609 | exception::register_exception_translator( 610 | "UnknownGroupError", mastermind_cache_error); 611 | 612 | exception::register_exception_translator( 613 | "UnknownGroupsetError", mastermind_cache_error); 614 | 615 | exception::register_exception_translator( 616 | "NamespaceNotFoundError", mastermind_cache_error); 617 | 618 | exception::register_exception_translator( 619 | "RemotesEmptyError", mastermind_cache_error); 620 | 621 | } 622 | 623 | } // namespace binding 624 | } // namespace mastermind 625 | 626 | namespace mb = mastermind::binding; 627 | 628 | BOOST_PYTHON_MODULE(mastermind_cache) { 629 | PyEval_InitThreads(); 630 | 631 | mb::init_exception_translator(); 632 | 633 | bp::class_("Couples", bp::no_init) 634 | .def("get_couple_read_preference" 635 | , &mb::namespace_state_t::couples_t::get_couple_read_preference 636 | , (bp::arg("group"))) 637 | .def("get_couple_groupset" 638 | , &mb::namespace_state_t::couples_t::get_couple_groupset 639 | , (bp::arg("group"), bp::arg("groupset"))) 640 | .def("get_couple_groupset_ids" 641 | , &mb::namespace_state_t::couples_t::get_couple_groupset_ids 642 | , (bp::arg("group"))) 643 | .def("get_couple_groups" 644 | , &mb::namespace_state_t::couples_t::get_couple_groups 645 | , (bp::arg("group"))) 646 | .def("get_groups" 647 | , &mb::namespace_state_t::couples_t::get_groups 648 | , (bp::arg("group"))) 649 | .def("hosts" 650 | , &mb::namespace_state_t::couples_t::hosts 651 | , (bp::arg("group"))) 652 | ; 653 | 654 | bp::class_("Settings", bp::no_init) 655 | .def("groups_count", &mb::namespace_state_t::settings_t::groups_count) 656 | .def("success_copies_num", bp::make_function( 657 | &mb::namespace_state_t::settings_t::success_copies_num 658 | , bp::return_value_policy())) 659 | .def("user_settings", bp::make_function( 660 | &mb::namespace_state_t::settings_t::user_settings 661 | , bp::return_value_policy())) 662 | ; 663 | 664 | bp::class_("NamespaceState", bp::no_init) 665 | .def("couples", &mb::namespace_state_t::couples) 666 | .def("settings", &mb::namespace_state_t::settings) 667 | .def("name", bp::make_function(&mb::namespace_state_t::name 668 | , bp::return_value_policy())) 669 | ; 670 | 671 | bp::class_("MastermindCache" 672 | , bp::init( 674 | (bp::arg("remotes"), bp::arg("update_period") = 60 675 | , bp::arg("cache_path") = std::string{} 676 | , bp::arg("warning_time") = std::numeric_limits::max() 677 | , bp::arg("expire_time") = std::numeric_limits::max() 678 | , bp::arg("worker_name") = std::string{"mastermind2.26"} 679 | , bp::arg("enqueue_timeout") = 4000 680 | , bp::arg("reconnect_timeout") = 4000 681 | , bp::arg("ns_filter") = bp::object() 682 | , bp::arg("auto_start") = true 683 | ))) 684 | .def("start", &mb::mastermind_t::start) 685 | .def("stop", &mb::mastermind_t::stop) 686 | .def("is_running", &mb::mastermind_t::is_running) 687 | .def("is_valid", &mb::mastermind_t::is_valid) 688 | .def("get_namespace_state", &mb::mastermind_t::get_namespace_state 689 | , (bp::arg("name"))) 690 | .def("find_namespace_state", &mb::mastermind_t::find_namespace_state 691 | , (bp::arg("group"))) 692 | .def("set_ns_filter", &mb::mastermind_t::set_ns_filter, (bp::arg("ns_filter"))) 693 | ; 694 | } 695 | 696 | --------------------------------------------------------------------------------