├── .gitignore ├── contrib └── deb.oxen.io.gpg ├── TODO.txt ├── oxenmq ├── version.h.in ├── fmt.h ├── auth.h ├── connections.h ├── message.h ├── pubsub.h ├── oxenmq-internal.h ├── jobs.cpp ├── address.h ├── batch.h ├── address.cpp ├── auth.cpp └── connections.cpp ├── cmake ├── local-libzmq │ ├── README.txt │ ├── FindZeroMQ.cmake │ └── LocalLibzmq.cmake └── libatomic.cmake ├── liboxenmq.pc.in ├── .gitmodules ├── tests ├── CMakeLists.txt ├── main.cpp ├── test_socket_limit.cpp ├── common.h ├── test_inject.cpp ├── test_timer.cpp ├── test_batch.cpp ├── test_requests.cpp ├── test_tagged_threads.cpp ├── test_address.cpp ├── test_failures.cpp └── test_pubsub.cpp ├── LICENSE ├── .drone.jsonnet ├── CMakeLists.txt └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /build* 2 | /compile_commands.json 3 | /obj-*-linux-gnu/ 4 | -------------------------------------------------------------------------------- /contrib/deb.oxen.io.gpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/oxen-io/oxen-mq/HEAD/contrib/deb.oxen.io.gpg -------------------------------------------------------------------------------- /TODO.txt: -------------------------------------------------------------------------------- 1 | 2 | - split out proxy code & data into a private lokimq/proxy.h header so that the main header doesn't 3 | need to include so much. 4 | 5 | -------------------------------------------------------------------------------- /oxenmq/version.h.in: -------------------------------------------------------------------------------- 1 | namespace oxenmq { 2 | constexpr int VERSION_MAJOR = @PROJECT_VERSION_MAJOR@; 3 | constexpr int VERSION_MINOR = @PROJECT_VERSION_MINOR@; 4 | constexpr int VERSION_PATCH = @PROJECT_VERSION_PATCH@; 5 | } 6 | -------------------------------------------------------------------------------- /cmake/local-libzmq/README.txt: -------------------------------------------------------------------------------- 1 | LocalLibzmq.cmake - builds a local static copy 2 | 3 | FindZeroMQ.cmake - overrides cppzmq's version with a simple file that just sets up the minimum bits 4 | cppzmq needs to use our static copy build above. 5 | -------------------------------------------------------------------------------- /cmake/local-libzmq/FindZeroMQ.cmake: -------------------------------------------------------------------------------- 1 | 2 | # cppzmq expects these targets: 3 | add_library(libzmq INTERFACE IMPORTED GLOBAL) 4 | add_library(libzmq-static INTERFACE IMPORTED GLOBAL) 5 | target_link_libraries(libzmq INTERFACE libzmq_vendor) 6 | target_link_libraries(libzmq-static INTERFACE libzmq_vendor) 7 | set(ZeroMQ_FOUND ON) 8 | 9 | -------------------------------------------------------------------------------- /liboxenmq.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | exec_prefix=${prefix} 3 | libdir=@CMAKE_INSTALL_FULL_LIBDIR@ 4 | includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ 5 | 6 | Name: liboxenmq 7 | Description: ZeroMQ-based communication library 8 | Version: @PROJECT_VERSION@ 9 | 10 | Libs: -L${libdir} -loxenmq 11 | Libs.private: @PRIVATE_LIBS@ 12 | Requires: liboxenc 13 | Requires.private: libzmq libsodium 14 | Cflags: -I${includedir} 15 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "cppzmq"] 2 | path = cppzmq 3 | url = https://github.com/zeromq/cppzmq.git 4 | [submodule "Catch2"] 5 | path = tests/Catch2 6 | url = https://github.com/catchorg/Catch2.git 7 | [submodule "oxen-encoding"] 8 | path = oxen-encoding 9 | url = https://github.com/session-foundation/oxen-encoding.git 10 | [submodule "oxen-logging"] 11 | path = oxen-logging 12 | url = https://github.com/oxen-io/oxen-logging.git 13 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | add_subdirectory(Catch2) 3 | 4 | add_executable(tests 5 | main.cpp 6 | test_address.cpp 7 | test_batch.cpp 8 | test_connect.cpp 9 | test_commands.cpp 10 | test_failures.cpp 11 | test_inject.cpp 12 | test_pubsub.cpp 13 | test_requests.cpp 14 | test_socket_limit.cpp 15 | test_tagged_threads.cpp 16 | test_timer.cpp 17 | ) 18 | 19 | find_package(Threads) 20 | 21 | target_link_libraries(tests Catch2::Catch2 oxenmq Threads::Threads oxen::logging) 22 | 23 | set_target_properties(tests PROPERTIES 24 | CXX_STANDARD 20 25 | CXX_STANDARD_REQUIRED ON 26 | CXX_EXTENSIONS OFF 27 | ) 28 | 29 | add_custom_target(check COMMAND tests) 30 | -------------------------------------------------------------------------------- /oxenmq/fmt.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include "address.h" 6 | #include "auth.h" 7 | #include "connections.h" 8 | 9 | template <> 10 | struct fmt::formatter : fmt::formatter { 11 | auto format(oxenmq::AuthLevel v, format_context& ctx) const { 12 | return formatter::format(to_string(v), ctx); 13 | } 14 | }; 15 | template <> 16 | struct fmt::formatter : fmt::formatter { 17 | auto format(const oxenmq::ConnectionID& conn, format_context& ctx) const { 18 | return formatter::format(conn.to_string(), ctx); 19 | } 20 | }; 21 | template <> 22 | struct fmt::formatter : fmt::formatter { 23 | auto format(const oxenmq::address& addr, format_context& ctx) const { 24 | return formatter::format(addr.full_address(), ctx); 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /tests/main.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | #include 4 | 5 | int main(int argc, char* argv[]) 6 | { 7 | Catch::Session session; 8 | 9 | using namespace Catch::Clara; 10 | std::string log_level = "critical", log_file = "stderr"; 11 | 12 | auto cli = 13 | session.cli() | Opt(log_level, "level")["--log-level"]("oxen-logging log level to apply to the test run") | 14 | Opt(log_file, "file")["--log-file"]( 15 | "oxen-logging log file to output logs to, or one of or one of stdout/-/stderr/syslog."); 16 | 17 | session.cli(cli); 18 | 19 | namespace log = oxen::log; 20 | if (int rc = session.applyCommandLine(argc, argv); rc != 0) 21 | return rc; 22 | 23 | log::Level lvl = log::level_from_string(log_level); 24 | 25 | constexpr std::array print_vals = {"stdout", "-", "", "stderr", "nocolor", "stdout-nocolor", "stderr-nocolor"}; 26 | log::Type type; 27 | if (std::count(print_vals.begin(), print_vals.end(), log_file)) 28 | type = log::Type::Print; 29 | else if (log_file == "syslog") 30 | type = log::Type::System; 31 | else 32 | type = log::Type::File; 33 | 34 | oxen::log::add_sink(type, log_file, "[%T.%f] [%*] [\x1b[1m%n\x1b[0m:%^%l%$|\x1b[3m%g:%#\x1b[0m] %v"); 35 | oxen::log::reset_level(lvl); 36 | 37 | return session.run(); 38 | } 39 | -------------------------------------------------------------------------------- /tests/test_socket_limit.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | 4 | using namespace oxenmq; 5 | 6 | TEST_CASE("zmq socket limit", "[zmq][socket-limit]") { 7 | // Make sure setting .MAX_SOCKETS works as expected. (This test was added when a bug was fixed 8 | // that was causing it not to be applied). 9 | std::string listen = random_localhost(); 10 | OxenMQ server{ 11 | "", "", // generate ephemeral keys 12 | false, // not a service node 13 | [](auto) { return ""; }, 14 | }; 15 | server.listen_plain(listen); 16 | server.start(); 17 | 18 | std::atomic failed = 0, good = 0, failed_toomany = 0; 19 | OxenMQ client; 20 | client.MAX_SOCKETS = 15; 21 | client.start(); 22 | 23 | std::vector conns; 24 | address server_addr{listen}; 25 | for (int i = 0; i < 16; i++) 26 | client.connect_remote(server_addr, 27 | [&](auto) { good++; }, 28 | [&](auto cid, auto msg) { 29 | if (msg == "connect() failed: Too many open files") 30 | failed_toomany++; 31 | else 32 | failed++; 33 | }); 34 | 35 | 36 | wait_for([&] { return good > 0 && failed_toomany > 0; }); 37 | { 38 | auto lock = catch_lock(); 39 | REQUIRE( good > 0 ); 40 | REQUIRE( failed == 0 ); 41 | REQUIRE( failed_toomany > 0 ); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019-2020, The Loki Project 2 | 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its contributors 16 | may be used to endorse or promote products derived from this software without 17 | specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 20 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 21 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /cmake/libatomic.cmake: -------------------------------------------------------------------------------- 1 | function(check_working_cxx_atomics64 varname) 2 | set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) 3 | if(MSVC OR MSVC_VERSION) 4 | set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std:c++20") 5 | else() 6 | set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -std=c++20") 7 | endif() 8 | check_cxx_source_compiles(" 9 | #include 10 | #include 11 | std::atomic x{0}; 12 | int main() { 13 | uint64_t i = x.load(std::memory_order_relaxed); 14 | return 0; 15 | } 16 | " ${varname}) 17 | set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS}) 18 | endfunction() 19 | 20 | if(CMAKE_SYSTEM_NAME MATCHES "Linux") 21 | check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITHOUT_LIB) 22 | 23 | if(HAVE_CXX_ATOMICS64_WITHOUT_LIB) 24 | message(STATUS "Have working 64bit atomics") 25 | return() 26 | endif() 27 | 28 | if (NOT MSVC AND NOT MSVC_VERSION) 29 | check_library_exists(atomic __atomic_load_8 "" HAVE_CXX_LIBATOMICS64) 30 | if (HAVE_CXX_LIBATOMICS64) 31 | message(STATUS "Have 64bit atomics via library") 32 | list(APPEND CMAKE_REQUIRED_LIBRARIES "atomic") 33 | check_working_cxx_atomics64(HAVE_CXX_ATOMICS64_WITH_LIB) 34 | if (HAVE_CXX_ATOMICS64_WITH_LIB) 35 | message(STATUS "Can link with libatomic") 36 | link_libraries(-latomic) 37 | return() 38 | endif() 39 | endif() 40 | endif() 41 | if (MSVC OR MSVC_VERSION) 42 | message(FATAL_ERROR "Host compiler must support 64-bit std::atomic! (What does MSVC do to inline atomics?)") 43 | else() 44 | message(FATAL_ERROR "Host compiler must support 64-bit std::atomic!") 45 | endif() 46 | endif() 47 | -------------------------------------------------------------------------------- /cmake/local-libzmq/LocalLibzmq.cmake: -------------------------------------------------------------------------------- 1 | set(LIBZMQ_PREFIX ${CMAKE_BINARY_DIR}/libzmq) 2 | set(ZeroMQ_VERSION 4.3.5) 3 | set(LIBZMQ_URL https://github.com/zeromq/libzmq/releases/download/v${ZeroMQ_VERSION}/zeromq-${ZeroMQ_VERSION}.tar.gz) 4 | set(LIBZMQ_HASH SHA512=a71d48aa977ad8941c1609947d8db2679fc7a951e4cd0c3a1127ae026d883c11bd4203cf315de87f95f5031aec459a731aec34e5ce5b667b8d0559b157952541) 5 | 6 | message(${LIBZMQ_URL}) 7 | 8 | if(LIBZMQ_TARBALL_URL) 9 | # make a build time override of the tarball url so we can fetch it if the original link goes away 10 | set(LIBZMQ_URL ${LIBZMQ_TARBALL_URL}) 11 | endif() 12 | 13 | 14 | file(MAKE_DIRECTORY ${LIBZMQ_PREFIX}/include) 15 | 16 | set(libzmq_compiler_args) 17 | foreach(lang C CXX) 18 | foreach(thing COMPILER FLAGS COMPILER_LAUNCHER) 19 | if(DEFINED CMAKE_${lang}_${thing}) 20 | list(APPEND libzmq_compiler_args "-DCMAKE_${lang}_${thing}=${CMAKE_${lang}_${thing}}") 21 | endif() 22 | endforeach() 23 | endforeach() 24 | 25 | if(CMAKE_OSX_DEPLOYMENT_TARGET) 26 | list(APPEND libzmq_compiler_args "-DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET}") 27 | endif() 28 | 29 | include(ExternalProject) 30 | include(ProcessorCount) 31 | ExternalProject_Add(libzmq_external 32 | PREFIX ${LIBZMQ_PREFIX} 33 | URL ${LIBZMQ_URL} 34 | URL_HASH ${LIBZMQ_HASH} 35 | CMAKE_ARGS ${libzmq_compiler_args} 36 | -DCMAKE_BUILD_TYPE=Release 37 | -DCMAKE_POLICY_VERSION_MINIMUM=3.5 38 | -DWITH_LIBSODIUM=ON -DENABLE_CURVE=ON -DZMQ_BUILD_TESTS=OFF -DWITH_PERF_TOOL=OFF -DENABLE_DRAFTS=OFF 39 | -DBUILD_SHARED=OFF -DBUILD_STATIC=ON -DWITH_DOC=OFF -DCMAKE_INSTALL_PREFIX=${LIBZMQ_PREFIX} 40 | BUILD_BYPRODUCTS ${LIBZMQ_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libzmq.a 41 | ) 42 | 43 | add_library(libzmq_vendor STATIC IMPORTED GLOBAL) 44 | add_dependencies(libzmq_vendor libzmq_external) 45 | set_target_properties(libzmq_vendor PROPERTIES 46 | INTERFACE_INCLUDE_DIRECTORIES ${LIBZMQ_PREFIX}/include 47 | IMPORTED_LOCATION ${LIBZMQ_PREFIX}/${CMAKE_INSTALL_LIBDIR}/libzmq.a) 48 | -------------------------------------------------------------------------------- /tests/common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "oxenmq/oxenmq.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "oxenmq/fmt.h" 11 | 12 | using namespace oxenmq; 13 | 14 | // Apple's mutexes, thread scheduling, and IO handling are garbage and it shows up with lots of 15 | // spurious failures in this test suite (because it expects a system to not suck that badly), so we 16 | // multiply the time-sensitive bits by this factor as a hack to make the test suite work. 17 | constexpr int TIME_DILATION = 18 | #ifdef __APPLE__ 19 | 5; 20 | #elif defined(__x86_64__) 21 | 1; 22 | #else 23 | 2; 24 | #endif 25 | 26 | static auto startup = std::chrono::steady_clock::now(); 27 | 28 | /// Returns a localhost connection string to listen on. It can be considered random, though in 29 | /// practice in the current implementation is sequential starting at 25432. 30 | inline std::string random_localhost() { 31 | static std::atomic last = 25432; 32 | last++; 33 | assert(last); // We should never call this enough to overflow 34 | return "tcp://127.0.0.1:" + std::to_string(last); 35 | } 36 | 37 | 38 | // Catch2 macros aren't thread safe, so guard with a mutex 39 | inline std::unique_lock catch_lock() { 40 | static std::mutex mutex; 41 | return std::unique_lock{mutex}; 42 | } 43 | 44 | /// Waits up to 200ms for something to happen. 45 | template 46 | inline void wait_for(Func f, std::chrono::milliseconds wait_time = 200ms) { 47 | auto start = std::chrono::steady_clock::now(); 48 | auto end = start + wait_time * TIME_DILATION; 49 | while (std::chrono::steady_clock::now() < end) { 50 | if (f()) 51 | break; 52 | std::this_thread::sleep_for(10ms * TIME_DILATION); 53 | } 54 | auto lock = catch_lock(); 55 | UNSCOPED_INFO( 56 | "done waiting after " << (std::chrono::steady_clock::now() - start).count() << "ns"); 57 | } 58 | 59 | /// Waits on an atomic bool for up to 100ms for an initial connection, which is more than enough 60 | /// time for an initial connection + request. 61 | inline void wait_for_conn(std::atomic& c) { 62 | wait_for([&c] { return c.load(); }); 63 | } 64 | 65 | /// Waits enough time for us to receive a reply from a localhost remote. 66 | inline void reply_sleep() { 67 | std::this_thread::sleep_for(10ms * TIME_DILATION); 68 | } 69 | 70 | namespace oxenmq { 71 | class TestSuiteHelper { 72 | public: 73 | static size_t num_peers(const OxenMQ& omq) { return omq.peers.size(); } 74 | }; 75 | } // namespace oxenmq 76 | -------------------------------------------------------------------------------- /oxenmq/auth.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include 5 | 6 | namespace oxenmq { 7 | 8 | /// Authentication levels for command categories and connections 9 | enum class AuthLevel { 10 | denied, ///< Not actually an auth level, but can be returned by the AllowFunc to deny an incoming connection. 11 | none, ///< No authentication at all; any random incoming ZMQ connection can invoke this command. 12 | basic, ///< Basic authentication commands require a login, or a node that is specifically configured to be a public node (e.g. for public RPC). 13 | admin, ///< Advanced authentication commands require an admin user, either via explicit login or by implicit login from localhost. This typically protects administrative commands like shutting down, starting mining, or access sensitive data. 14 | }; 15 | 16 | /// The access level for a command category 17 | struct Access { 18 | /// Minimum access level required 19 | AuthLevel auth; 20 | /// If true only remote SNs may call the category commands 21 | bool remote_sn; 22 | /// If true the category requires that the local node is a SN 23 | bool local_sn; 24 | 25 | /// Constructor. Intentionally allows implicit conversion from an AuthLevel so that an 26 | /// AuthLevel can be passed anywhere an Access is required (the resulting Access will have both 27 | /// remote and local sn set to false). 28 | Access(AuthLevel auth = AuthLevel::none, bool remote_sn = false, bool local_sn = false) 29 | : auth{auth}, remote_sn{remote_sn}, local_sn{local_sn} {} 30 | }; 31 | 32 | /// Simple hash implementation for a string that is *already* a hash-like value (such as a pubkey). 33 | /// Falls back to std::hash if given a string smaller than a size_t. 34 | struct already_hashed { 35 | size_t operator()(const std::string& s) const { 36 | if (s.size() < sizeof(size_t)) 37 | return std::hash{}(s); 38 | size_t hash; 39 | std::memcpy(&hash, &s[0], sizeof(hash)); 40 | return hash; 41 | } 42 | }; 43 | 44 | /// std::unordered_set specialization for specifying pubkeys (used, in particular, by 45 | /// OxenMQ::set_active_sns and OxenMQ::update_active_sns); this is a std::string unordered_set that 46 | /// also uses a specialized trivial hash function that uses part of the value itself (i.e. the 47 | /// pubkey) directly as a hash value. (This is nice and fast for uniformly distributed values like 48 | /// pubkeys and a terrible hash choice for anything else). 49 | using pubkey_set = std::unordered_set; 50 | 51 | inline constexpr std::string_view to_string(AuthLevel a) { 52 | switch (a) { 53 | case AuthLevel::denied: return "denied"; 54 | case AuthLevel::none: return "none"; 55 | case AuthLevel::basic: return "basic"; 56 | case AuthLevel::admin: return "admin"; 57 | default: return "(unknown)"; 58 | } 59 | } 60 | 61 | inline AuthLevel auth_from_string(std::string_view a) { 62 | if (a == "none") return AuthLevel::none; 63 | if (a == "basic") return AuthLevel::basic; 64 | if (a == "admin") return AuthLevel::admin; 65 | return AuthLevel::denied; 66 | } 67 | 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /tests/test_inject.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | 3 | using namespace oxenmq; 4 | 5 | TEST_CASE("injected external commands", "[injected]") { 6 | std::string listen = random_localhost(); 7 | OxenMQ server{ 8 | "", "", // generate ephemeral keys 9 | false, // not a service node 10 | [](auto) { return ""; } 11 | }; 12 | server.set_general_threads(1); 13 | server.listen_curve(listen); 14 | 15 | std::atomic hellos = 0; 16 | std::atomic done = false; 17 | server.add_category("public", AuthLevel::none, 3); 18 | server.add_command("public", "hello", [&](Message& m) { 19 | hellos++; 20 | while (!done) std::this_thread::sleep_for(10ms); 21 | }); 22 | 23 | server.start(); 24 | 25 | OxenMQ client{}; 26 | client.start(); 27 | 28 | std::atomic got{false}; 29 | bool success = false; 30 | 31 | // Deliberately using a deprecated command here, disable -Wdeprecated-declarations 32 | #ifdef __GNUG__ 33 | #pragma GCC diagnostic push 34 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" 35 | #endif 36 | auto c = client.connect_remote(listen, 37 | [&](auto conn) { success = true; got = true; }, 38 | [&](auto conn, std::string_view) { got = true; }, 39 | server.get_pubkey()); 40 | 41 | #ifdef __GNUG__ 42 | #pragma GCC diagnostic pop 43 | #endif 44 | 45 | wait_for_conn(got); 46 | { 47 | auto lock = catch_lock(); 48 | REQUIRE( got ); 49 | REQUIRE( success ); 50 | } 51 | 52 | // First make sure that basic message respects the 3 thread limit 53 | client.send(c, "public.hello"); 54 | client.send(c, "public.hello"); 55 | client.send(c, "public.hello"); 56 | client.send(c, "public.hello"); 57 | wait_for([&] { return hellos >= 3; }); 58 | std::this_thread::sleep_for(20ms); 59 | { 60 | auto lock = catch_lock(); 61 | REQUIRE( hellos == 3 ); 62 | } 63 | done = true; 64 | wait_for([&] { return hellos >= 4; }); 65 | { 66 | auto lock = catch_lock(); 67 | REQUIRE( hellos == 4 ); 68 | } 69 | 70 | // Now try injecting external commands 71 | done = false; 72 | hellos = 0; 73 | client.send(c, "public.hello"); 74 | wait_for([&] { return hellos >= 1; }); 75 | server.inject_task("public", "(injected)", "localhost", [&] { hellos += 10; while (!done) std::this_thread::sleep_for(10ms); }); 76 | wait_for([&] { return hellos >= 11; }); 77 | client.send(c, "public.hello"); 78 | wait_for([&] { return hellos >= 12; }); 79 | server.inject_task("public", "(injected)", "localhost", [&] { hellos += 10; while (!done) std::this_thread::sleep_for(10ms); }); 80 | server.inject_task("public", "(injected)", "localhost", [&] { hellos += 10; while (!done) std::this_thread::sleep_for(10ms); }); 81 | server.inject_task("public", "(injected)", "localhost", [&] { hellos += 10; while (!done) std::this_thread::sleep_for(10ms); }); 82 | wait_for([&] { return hellos >= 12; }); 83 | std::this_thread::sleep_for(20ms); 84 | { 85 | auto lock = catch_lock(); 86 | REQUIRE( hellos == 12 ); 87 | } 88 | done = true; 89 | wait_for([&] { return hellos >= 42; }); 90 | { 91 | auto lock = catch_lock(); 92 | REQUIRE( hellos == 42 ); 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /oxenmq/connections.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "auth.h" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | namespace oxenmq { 10 | 11 | struct ConnectionID; 12 | 13 | namespace detail { 14 | template 15 | oxenc::bt_dict build_send(ConnectionID to, std::string_view cmd, T&&... opts); 16 | } 17 | 18 | /// Opaque data structure representing a connection which supports ==, !=, < and std::hash. For 19 | /// connections to service node this is the service node pubkey (and you can pass a 32-byte string 20 | /// anywhere a ConnectionID is called for). For non-SN remote connections you need to keep a copy 21 | /// of the ConnectionID returned by connect_remote(). 22 | struct ConnectionID { 23 | // Default construction; creates a ConnectionID with an invalid internal ID that will not match 24 | // an actual connection. 25 | ConnectionID() : ConnectionID(0) {} 26 | // Construction from a service node pubkey 27 | ConnectionID(std::string pubkey_) : id{SN_ID}, pk{std::move(pubkey_)} { 28 | if (pk.size() != 32) 29 | throw std::runtime_error{"Invalid pubkey: expected 32 bytes"}; 30 | } 31 | // Construction from a service node pubkey 32 | ConnectionID(std::string_view pubkey_) : ConnectionID(std::string{pubkey_}) {} 33 | 34 | ConnectionID(const ConnectionID&) = default; 35 | ConnectionID(ConnectionID&&) = default; 36 | ConnectionID& operator=(const ConnectionID&) = default; 37 | ConnectionID& operator=(ConnectionID&&) = default; 38 | 39 | // Returns true if this is a ConnectionID (false for a default-constructed, invalid id) 40 | explicit operator bool() const { 41 | return id != 0; 42 | } 43 | 44 | // Two ConnectionIDs are equal if they are both SNs and have matching pubkeys, or they are both 45 | // not SNs and have matching internal IDs and routes. (Pubkeys do not have to match for 46 | // non-SNs). 47 | bool operator==(const ConnectionID &o) const { 48 | if (sn() && o.sn()) 49 | return pk == o.pk; 50 | return id == o.id && route == o.route; 51 | } 52 | bool operator!=(const ConnectionID &o) const { return !(*this == o); } 53 | bool operator<(const ConnectionID &o) const { 54 | if (sn() && o.sn()) 55 | return pk < o.pk; 56 | return id < o.id || (id == o.id && route < o.route); 57 | } 58 | 59 | // Returns true if this ConnectionID represents a SN connection 60 | bool sn() const { return id == SN_ID; } 61 | 62 | // Returns this connection's pubkey, if any. (Note that all curve connections have pubkeys, not 63 | // only SNs). 64 | const std::string& pubkey() const { return pk; } 65 | 66 | // Returns a copy of the ConnectionID with the route set to empty. 67 | ConnectionID unrouted() { return ConnectionID{id, pk, ""}; } 68 | 69 | std::string to_string() const; 70 | 71 | 72 | private: 73 | ConnectionID(int64_t id) : id{id} {} 74 | ConnectionID(int64_t id, std::string pubkey, std::string route = "") 75 | : id{id}, pk{std::move(pubkey)}, route{std::move(route)} {} 76 | 77 | constexpr static int64_t SN_ID = -1; 78 | int64_t id = 0; 79 | std::string pk; 80 | std::string route; 81 | friend class OxenMQ; 82 | friend struct std::hash; 83 | template 84 | friend oxenc::bt_dict detail::build_send(ConnectionID to, std::string_view cmd, T&&... opts); 85 | }; 86 | 87 | } // namespace oxenmq 88 | namespace std { 89 | template <> struct hash { 90 | size_t operator()(const oxenmq::ConnectionID &c) const { 91 | return c.sn() ? oxenmq::already_hashed{}(c.pk) : 92 | std::hash{}(c.id) + std::hash{}(c.route); 93 | } 94 | }; 95 | } // namespace std 96 | -------------------------------------------------------------------------------- /tests/test_timer.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq/oxenmq.h" 2 | #include "common.h" 3 | #include 4 | #include 5 | 6 | TEST_CASE("timer test", "[timer][basic]") { 7 | oxenmq::OxenMQ omq{}; 8 | 9 | omq.set_general_threads(1); 10 | omq.set_batch_threads(1); 11 | 12 | std::atomic ticks = 0; 13 | auto timer = omq.add_timer([&] { ticks++; }, 5ms); 14 | omq.start(); 15 | auto start = std::chrono::steady_clock::now(); 16 | wait_for([&] { return ticks.load() > 3; }); 17 | { 18 | auto elapsed_ms = std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count(); 19 | auto lock = catch_lock(); 20 | REQUIRE( ticks.load() > 3 ); 21 | REQUIRE( elapsed_ms < 50 * TIME_DILATION ); 22 | } 23 | } 24 | 25 | TEST_CASE("timer squelch", "[timer][squelch]") { 26 | oxenmq::OxenMQ omq{}; 27 | 28 | omq.set_general_threads(3); 29 | omq.set_batch_threads(3); 30 | 31 | std::atomic first = true; 32 | std::atomic done = false; 33 | std::atomic ticks = 0; 34 | 35 | // Set up a timer with squelch on; the job shouldn't get rescheduled until the first call 36 | // finishes, by which point we set `done` and so should get exactly 1 tick. 37 | auto timer = omq.add_timer([&] { 38 | if (first.exchange(false)) { 39 | std::this_thread::sleep_for(30ms * TIME_DILATION); 40 | ticks++; 41 | done = true; 42 | } else if (!done) { 43 | ticks++; 44 | } 45 | }, 5ms * TIME_DILATION, true /* squelch */); 46 | omq.start(); 47 | 48 | wait_for([&] { return done.load(); }); 49 | { 50 | auto lock = catch_lock(); 51 | REQUIRE( done.load() ); 52 | REQUIRE( ticks.load() == 1 ); 53 | } 54 | 55 | // Start another timer with squelch *off*; the subsequent jobs should get scheduled even while 56 | // the first one blocks 57 | std::atomic first2 = true; 58 | std::atomic done2 = false; 59 | std::atomic ticks2 = 0; 60 | auto timer2 = omq.add_timer([&] { 61 | if (first2.exchange(false)) { 62 | std::this_thread::sleep_for(40ms * TIME_DILATION); 63 | done2 = true; 64 | } else if (!done2) { 65 | ticks2++; 66 | } 67 | }, 5ms, false /* squelch */); 68 | 69 | wait_for([&] { return done2.load(); }); 70 | { 71 | auto lock = catch_lock(); 72 | REQUIRE( ticks2.load() > 2 ); 73 | REQUIRE( done2.load() ); 74 | } 75 | } 76 | 77 | TEST_CASE("timer cancel", "[timer][cancel]") { 78 | oxenmq::OxenMQ omq{}; 79 | 80 | omq.set_general_threads(1); 81 | omq.set_batch_threads(1); 82 | 83 | std::atomic ticks = 0; 84 | 85 | // We set up *and cancel* this timer before omq starts, so it should never fire 86 | auto notimer = omq.add_timer([&] { ticks += 1000; }, 5ms * TIME_DILATION); 87 | omq.cancel_timer(notimer); 88 | 89 | TimerID timer = omq.add_timer([&] { 90 | if (++ticks == 3) 91 | omq.cancel_timer(timer); 92 | }, 5ms * TIME_DILATION); 93 | 94 | omq.start(); 95 | 96 | wait_for([&] { return ticks.load() >= 3; }); 97 | { 98 | auto lock = catch_lock(); 99 | REQUIRE( ticks.load() == 3 ); 100 | } 101 | 102 | // Test the alternative taking an lvalue reference instead of returning by value (see oxenmq.h 103 | // for why this is sometimes needed). 104 | std::atomic ticks3 = 0; 105 | std::weak_ptr w_timer3; 106 | { 107 | auto timer3 = std::make_shared(); 108 | auto& t3ref = *timer3; // Get this reference *before* we move the shared pointer into the lambda 109 | omq.add_timer(t3ref, [&ticks3, &omq, timer3=std::move(timer3)] { 110 | if (ticks3 == 0) 111 | ticks3++; 112 | else if (ticks3 > 1) { 113 | omq.cancel_timer(*timer3); 114 | ticks3++; 115 | } 116 | }, 1ms); 117 | } 118 | 119 | wait_for([&] { return ticks3.load() >= 1; }); 120 | { 121 | auto lock = catch_lock(); 122 | REQUIRE( ticks3.load() == 1 ); 123 | } 124 | ticks3++; 125 | wait_for([&] { return ticks3.load() >= 3 && w_timer3.expired(); }); 126 | { 127 | auto lock = catch_lock(); 128 | REQUIRE( ticks3.load() == 3 ); 129 | REQUIRE( w_timer3.expired() ); 130 | } 131 | } 132 | 133 | -------------------------------------------------------------------------------- /tests/test_batch.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq/batch.h" 2 | #include "common.h" 3 | #include 4 | 5 | double do_my_task(int input) { 6 | if (input % 10 == 7) 7 | throw std::domain_error("I don't do '7s, sorry"); 8 | if (input == 1) 9 | return 5.0; 10 | return 3.0 * input; 11 | } 12 | 13 | std::promise> done; 14 | 15 | void continue_big_task(std::vector> results) { 16 | double sum = 0; 17 | int exc_count = 0; 18 | for (auto& r : results) { 19 | try { 20 | sum += r.get(); 21 | } catch (const std::exception& e) { 22 | exc_count++; 23 | } 24 | } 25 | done.set_value({sum, exc_count}); 26 | } 27 | 28 | void start_big_task(oxenmq::OxenMQ& omq) { 29 | size_t num_jobs = 32; 30 | 31 | oxenmq::Batch batch; 32 | batch.reserve(num_jobs); 33 | 34 | for (size_t i = 0; i < num_jobs; i++) 35 | batch.add_job([i]() { return do_my_task(i); }); 36 | 37 | batch.completion(&continue_big_task); 38 | 39 | omq.batch(std::move(batch)); 40 | } 41 | 42 | 43 | TEST_CASE("batching many small jobs", "[batch-many]") { 44 | oxenmq::OxenMQ omq{ 45 | "", "", // generate ephemeral keys 46 | false, // not a service node 47 | [](auto) { return ""; }, 48 | }; 49 | omq.set_general_threads(4); 50 | omq.set_batch_threads(4); 51 | omq.start(); 52 | 53 | start_big_task(omq); 54 | auto sum = done.get_future().get(); 55 | auto lock = catch_lock(); 56 | REQUIRE( sum.first == 1337.0 ); 57 | REQUIRE( sum.second == 3 ); 58 | } 59 | 60 | TEST_CASE("batch exception propagation", "[batch-exceptions]") { 61 | oxenmq::OxenMQ omq{ 62 | "", "", // generate ephemeral keys 63 | false, // not a service node 64 | [](auto) { return ""; }, 65 | }; 66 | omq.set_general_threads(4); 67 | omq.set_batch_threads(4); 68 | omq.start(); 69 | 70 | std::promise done_promise; 71 | std::future done_future = done_promise.get_future(); 72 | 73 | using Catch::Matchers::Message; 74 | 75 | SECTION( "value return" ) { 76 | oxenmq::Batch batch; 77 | for (int i : {1, 2}) 78 | batch.add_job([i]() { if (i == 1) return 42; throw std::domain_error("bad value " + std::to_string(i)); }); 79 | batch.completion([&done_promise](auto results) { 80 | auto lock = catch_lock(); 81 | REQUIRE( results.size() == 2 ); 82 | REQUIRE( results[0].get() == 42 ); 83 | REQUIRE_THROWS_MATCHES( results[1].get() == 0, std::domain_error, Message("bad value 2") ); 84 | done_promise.set_value(); 85 | }); 86 | omq.batch(std::move(batch)); 87 | done_future.get(); 88 | } 89 | 90 | SECTION( "lvalue return" ) { 91 | oxenmq::Batch batch; 92 | int forty_two = 42; 93 | for (int i : {1, 2}) 94 | batch.add_job([i,&forty_two]() -> int& { 95 | if (i == 1) 96 | return forty_two; 97 | throw std::domain_error("bad value " + std::to_string(i)); 98 | }); 99 | batch.completion([&done_promise,&forty_two](auto results) { 100 | auto lock = catch_lock(); 101 | REQUIRE( results.size() == 2 ); 102 | auto& r = results[0].get(); 103 | REQUIRE( &r == &forty_two ); 104 | REQUIRE( r == 42 ); 105 | REQUIRE_THROWS_MATCHES( results[1].get(), std::domain_error, Message("bad value 2") ); 106 | done_promise.set_value(); 107 | }); 108 | omq.batch(std::move(batch)); 109 | done_future.get(); 110 | } 111 | 112 | SECTION( "void return" ) { 113 | oxenmq::Batch batch; 114 | for (int i : {1, 2}) 115 | batch.add_job([i]() { if (i != 1) throw std::domain_error("bad value " + std::to_string(i)); }); 116 | batch.completion([&done_promise](auto results) { 117 | auto lock = catch_lock(); 118 | REQUIRE( results.size() == 2 ); 119 | REQUIRE_NOTHROW( results[0].get() ); 120 | REQUIRE_THROWS_MATCHES( results[1].get(), std::domain_error, Message("bad value 2") ); 121 | done_promise.set_value(); 122 | }); 123 | omq.batch(std::move(batch)); 124 | done_future.get(); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /oxenmq/message.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "connections.h" 4 | 5 | namespace oxenmq { 6 | 7 | class OxenMQ; 8 | 9 | /// Encapsulates an incoming message from a remote connection with message details plus extra 10 | /// info need to send a reply back through the proxy thread via the `reply()` method. Note that 11 | /// this object gets reused: callbacks should use but not store any reference beyond the callback. 12 | class Message { 13 | public: 14 | OxenMQ& oxenmq; ///< The owning OxenMQ object 15 | std::vector data; ///< The provided command data parts, if any. 16 | ConnectionID conn; ///< The connection info for routing a reply; also contains the pubkey/sn status. 17 | std::string reply_tag; ///< If the invoked command is a request command this is the required reply tag that will be prepended by `send_reply()`. 18 | Access access; ///< The access level of the invoker. This can be higher than the access level of the command, for example for an admin invoking a basic command. 19 | std::string remote; ///< Some sort of remote address from which the request came. Often "IP" for TCP connections and "localhost:UID:GID:PID" for unix socket connections. 20 | 21 | /// Constructor 22 | Message(OxenMQ& omq, ConnectionID cid, Access access, std::string remote) 23 | : oxenmq{omq}, conn{std::move(cid)}, access{std::move(access)}, remote{std::move(remote)} {} 24 | 25 | // Non-copyable 26 | Message(const Message&) = delete; 27 | Message& operator=(const Message&) = delete; 28 | 29 | /// Sends a command back to whomever sent this message. Arguments are forwarded to send() but 30 | /// with send_option::optional{} added if the originator is not a SN. For SN messages (i.e. 31 | /// where `sn` is true) this is a "strong" reply by default in that the proxy will attempt to 32 | /// establish a new connection to the SN if no longer connected. For non-SN messages the reply 33 | /// will be attempted using the available routing information, but if the connection has already 34 | /// been closed the reply will be dropped. 35 | /// 36 | /// If you want to send a non-strong reply even when the remote is a service node then add 37 | /// an explicit `send_option::optional()` argument. 38 | template 39 | void send_back(std::string_view command, Args&&... args); 40 | 41 | /// Sends a reply to a request. This takes no command: the command is always the built-in 42 | /// "REPLY" command, followed by the unique reply tag, then any reply data parts. All other 43 | /// arguments are as in `send_back()`. You should only send one reply for a command expecting 44 | /// replies, though this is not enforced: attempting to send multiple replies will simply be 45 | /// dropped when received by the remote. (Note, however, that it is possible to send multiple 46 | /// messages -- e.g. you could send a reply and then also call send_back() and/or send_request() 47 | /// to send more requests back to the sender). 48 | template 49 | void send_reply(Args&&... args); 50 | 51 | /// Sends a request back to whomever sent this message. This is effectively a wrapper around 52 | /// omq.request() that takes care of setting up the recipient arguments. 53 | template 54 | void send_request(std::string_view command, ReplyCallback&& callback, Args&&... args); 55 | 56 | /** Class returned by `send_later()` that can be used to call `back()`, `reply()`, or 57 | * `request()` beyond the lifetime of the Message instance as if calling `msg.send_back()`, 58 | * `msg.send_reply()`, or `msg.send_request()`. For example: 59 | * 60 | * auto send = msg.send_later(); 61 | * // ... later, perhaps in a lambda or scheduled job: 62 | * send.reply("content"); 63 | * 64 | * is equivalent to 65 | * 66 | * msg.send_reply("content"); 67 | * 68 | * except that it is valid even if `msg` is no longer valid. 69 | */ 70 | class DeferredSend { 71 | public: 72 | OxenMQ& oxenmq; ///< The owning OxenMQ object 73 | ConnectionID conn; ///< The connection info for routing a reply; also contains the pubkey/sn status 74 | std::string reply_tag; ///< If the invoked command is a request command this is the required reply tag that will be prepended by `reply()`. 75 | 76 | explicit DeferredSend(Message& m) : oxenmq{m.oxenmq}, conn{m.conn}, reply_tag{m.reply_tag} {} 77 | 78 | template 79 | void operator()(Args &&...args) const { 80 | if (reply_tag.empty()) 81 | back(std::forward(args)...); 82 | else 83 | reply(std::forward(args)...); 84 | }; 85 | 86 | 87 | /// Equivalent to msg.send_back(...), but can be invoked later. 88 | template 89 | void back(std::string_view command, Args&&... args) const; 90 | 91 | /// Equivalent to msg.send_reply(...), but can be invoked later. 92 | template 93 | void reply(Args&&... args) const; 94 | 95 | /// Equivalent to msg.send_request(...), but can be invoked later. 96 | template 97 | void request(std::string_view command, ReplyCallback&& callback, Args&&... args) const; 98 | }; 99 | 100 | /// Returns a DeferredSend object that can be used to send replies to this message even if the 101 | /// message expires. Typically this is used when sending a reply requires waiting on another 102 | /// task to complete without needing to block the handler thread. 103 | DeferredSend send_later() { return DeferredSend{*this}; } 104 | }; 105 | 106 | } 107 | -------------------------------------------------------------------------------- /tests/test_requests.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | 4 | using namespace oxenmq; 5 | 6 | TEST_CASE("basic requests", "[requests]") { 7 | std::string listen = random_localhost(); 8 | OxenMQ server{ 9 | "", "", // generate ephemeral keys 10 | false, // not a service node 11 | [](auto) { return ""; }, 12 | }; 13 | server.listen_curve(listen); 14 | 15 | std::atomic hellos{0}, his{0}; 16 | 17 | server.add_category("public", Access{AuthLevel::none}); 18 | server.add_request_command("public", "hello", [&](Message& m) { 19 | m.send_reply("123"); 20 | }); 21 | server.start(); 22 | 23 | OxenMQ client{}; 24 | 25 | client.start(); 26 | 27 | std::atomic connected{false}, failed{false}; 28 | std::string pubkey; 29 | 30 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 31 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 32 | [&](auto, auto) { failed = true; }); 33 | 34 | wait_for([&] { return connected || failed; }); 35 | { 36 | auto lock = catch_lock(); 37 | REQUIRE( connected ); 38 | REQUIRE_FALSE( failed ); 39 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 40 | } 41 | 42 | std::atomic got_reply{false}; 43 | bool success; 44 | std::vector data; 45 | client.request(c, "public.hello", [&](bool ok, std::vector data_) { 46 | got_reply = true; 47 | success = ok; 48 | data = std::move(data_); 49 | }); 50 | 51 | reply_sleep(); 52 | { 53 | auto lock = catch_lock(); 54 | REQUIRE( got_reply.load() ); 55 | REQUIRE( success ); 56 | REQUIRE( data == std::vector{{"123"}} ); 57 | } 58 | } 59 | 60 | TEST_CASE("request from server to client", "[requests]") { 61 | std::string listen = random_localhost(); 62 | OxenMQ server{ 63 | "", "", // generate ephemeral keys 64 | false, // not a service node 65 | [](auto) { return ""; }, 66 | }; 67 | server.listen_curve(listen); 68 | 69 | std::atomic hellos{0}, his{0}; 70 | 71 | server.add_category("public", Access{AuthLevel::none}); 72 | server.add_request_command("public", "hello", [&](Message& m) { 73 | m.send_reply("123"); 74 | }); 75 | server.start(); 76 | 77 | OxenMQ client{}; 78 | 79 | client.start(); 80 | 81 | std::atomic connected{false}, failed{false}; 82 | std::string pubkey; 83 | 84 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 85 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 86 | [&](auto, auto) { failed = true; }); 87 | 88 | int i; 89 | for (i = 0; i < 5; i++) { 90 | if (connected.load()) 91 | break; 92 | std::this_thread::sleep_for(50ms); 93 | } 94 | { 95 | auto lock = catch_lock(); 96 | REQUIRE( connected.load() ); 97 | REQUIRE( !failed.load() ); 98 | REQUIRE( i <= 1 ); 99 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 100 | } 101 | 102 | std::atomic got_reply{false}; 103 | bool success; 104 | std::vector data; 105 | client.request(c, "public.hello", [&](bool ok, std::vector data_) { 106 | got_reply = true; 107 | success = ok; 108 | data = std::move(data_); 109 | }); 110 | 111 | std::this_thread::sleep_for(50ms); 112 | { 113 | auto lock = catch_lock(); 114 | REQUIRE( got_reply.load() ); 115 | REQUIRE( success ); 116 | REQUIRE( data == std::vector{{"123"}} ); 117 | } 118 | } 119 | 120 | TEST_CASE("request timeouts", "[requests][timeout]") { 121 | std::string listen = random_localhost(); 122 | OxenMQ server{ 123 | "", "", // generate ephemeral keys 124 | false, // not a service node 125 | [](auto) { return ""; }, 126 | }; 127 | server.listen_curve(listen); 128 | 129 | std::atomic hellos{0}, his{0}; 130 | 131 | server.add_category("public", Access{AuthLevel::none}); 132 | server.add_request_command("public", "blackhole", [&](Message& m) { /* doesn't reply */ }); 133 | server.start(); 134 | 135 | OxenMQ client{}; 136 | 137 | client.CONN_CHECK_INTERVAL = 10ms; // impatience (don't set this low in production code) 138 | client.start(); 139 | 140 | std::atomic connected{false}, failed{false}; 141 | std::string pubkey; 142 | 143 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 144 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 145 | [&](auto, auto) { failed = true; }); 146 | 147 | wait_for([&] { return connected || failed; }); 148 | 149 | REQUIRE( connected ); 150 | REQUIRE_FALSE( failed ); 151 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 152 | 153 | std::atomic got_triggered{false}; 154 | bool success; 155 | std::vector data; 156 | client.request(c, "public.blackhole", [&](bool ok, std::vector data_) { 157 | got_triggered = true; 158 | success = ok; 159 | data = std::move(data_); 160 | }, 161 | oxenmq::send_option::request_timeout{10ms} 162 | ); 163 | 164 | std::atomic got_triggered2{false}; 165 | client.request(c, "public.blackhole", [&](bool ok, std::vector data_) { 166 | got_triggered = true; 167 | success = ok; 168 | data = std::move(data_); 169 | }, 170 | oxenmq::send_option::request_timeout{200ms} 171 | ); 172 | 173 | std::this_thread::sleep_for(100ms); 174 | REQUIRE( got_triggered ); 175 | REQUIRE_FALSE( got_triggered2 ); 176 | REQUIRE_FALSE( success ); 177 | REQUIRE( data == std::vector{{"TIMEOUT"}} ); 178 | 179 | } 180 | -------------------------------------------------------------------------------- /.drone.jsonnet: -------------------------------------------------------------------------------- 1 | local docker_base = 'registry.oxen.rocks/'; 2 | 3 | local default_deps_nocxx = ['libsodium-dev', 'libzmq3-dev', 'liboxenc-dev']; 4 | 5 | local submodule_commands = ['git fetch --tags', 'git submodule update --init --recursive --depth=1']; 6 | 7 | local submodules = { 8 | name: 'submodules', 9 | image: 'drone/git', 10 | commands: submodule_commands, 11 | }; 12 | 13 | local apt_get_quiet = 'apt-get -o=Dpkg::Use-Pty=0 -q '; 14 | 15 | 16 | local generic_build(build_type, cmake_extra, werror=false, tests=true) 17 | = [ 18 | 'mkdir build', 19 | 'cd build', 20 | 'cmake .. -G Ninja -DCMAKE_COLOR_DIAGNOSTICS=ON -DCMAKE_BUILD_TYPE=' + build_type + 21 | ' -DWARNINGS_AS_ERRORS=' + (if werror then 'ON' else 'OFF') + 22 | ' -DOXENMQ_BUILD_TESTS=' + (if tests then 'ON' else 'OFF') + 23 | ' ' + cmake_extra, 24 | 'ninja -v', 25 | 'cd ..', 26 | ] 27 | + (if tests then [ 28 | 'cd build', 29 | './tests/tests --colour-mode ansi', 30 | 'cd ..', 31 | ] else []); 32 | 33 | 34 | local debian_pipeline(name, 35 | image, 36 | arch='amd64', 37 | deps=['g++'] + default_deps_nocxx, 38 | cmake_extra='', 39 | build_type='Release', 40 | extra_cmds=[], 41 | werror=false, 42 | distro='$$(lsb_release -sc)', 43 | allow_fail=false) = { 44 | kind: 'pipeline', 45 | type: 'docker', 46 | name: name, 47 | platform: { arch: arch }, 48 | environment: { CLICOLOR_FORCE: '1' }, // Lets color through ninja (1.9+) 49 | steps: [ 50 | submodules, 51 | { 52 | name: 'build', 53 | image: image, 54 | pull: 'always', 55 | [if allow_fail then 'failure']: 'ignore', 56 | commands: [ 57 | 'echo "Building on ${DRONE_STAGE_MACHINE}"', 58 | 'echo "man-db man-db/auto-update boolean false" | debconf-set-selections', 59 | apt_get_quiet + 'update', 60 | apt_get_quiet + 'install -y eatmydata', 61 | 'eatmydata ' + apt_get_quiet + ' install --no-install-recommends -y lsb-release', 62 | 'cp contrib/deb.oxen.io.gpg /etc/apt/trusted.gpg.d', 63 | 'echo deb http://deb.oxen.io ' + distro + ' main >/etc/apt/sources.list.d/oxen.list', 64 | 'eatmydata ' + apt_get_quiet + ' update', 65 | 'eatmydata ' + apt_get_quiet + 'dist-upgrade -y', 66 | 'eatmydata ' + apt_get_quiet + 'install -y cmake git ninja-build pkg-config ccache ' + std.join(' ', deps), 67 | ] 68 | + generic_build(build_type, cmake_extra, werror=werror) 69 | + extra_cmds, 70 | }, 71 | ], 72 | }; 73 | 74 | local clang(version) = debian_pipeline( 75 | 'Debian sid/clang-' + version + ' (amd64)', 76 | docker_base + 'debian-sid-clang', 77 | distro='sid', 78 | deps=['clang-' + version, 'clang-tools-' + version] + default_deps_nocxx, 79 | cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + ' -DCMAKE_CXX_COMPILER=clang++-' + version + ' ' 80 | ); 81 | 82 | local full_llvm(version) = debian_pipeline( 83 | 'Debian sid/llvm-' + version + ' (amd64)', 84 | docker_base + 'debian-sid-clang', 85 | distro='sid', 86 | deps=['clang-' + version, 'clang-tools-' + version, 'lld-' + version, 'libc++-' + version + '-dev', 'libc++abi-' + version + '-dev'] 87 | + default_deps_nocxx, 88 | cmake_extra='-DCMAKE_C_COMPILER=clang-' + version + 89 | ' -DCMAKE_CXX_COMPILER=clang++-' + version + 90 | ' -DCMAKE_CXX_FLAGS=-stdlib=libc++ ' + 91 | std.join(' ', [ 92 | '-DCMAKE_' + type + '_LINKER_FLAGS=-fuse-ld=lld-' + version 93 | for type in ['EXE', 'MODULE', 'SHARED', 'STATIC'] 94 | ]) 95 | ); 96 | 97 | local mac_builder(name, 98 | build_type='Release', 99 | arch='amd64', 100 | cmake_extra='-DCMAKE_CXX_COMPILER_LAUNCHER=ccache ', 101 | extra_cmds=[], 102 | tests=true) = { 103 | kind: 'pipeline', 104 | type: 'exec', 105 | name: name, 106 | platform: { os: 'darwin', arch: arch }, 107 | steps: [ 108 | { name: 'submodules', commands: submodule_commands }, 109 | { 110 | name: 'build', 111 | environment: { SSH_KEY: { from_secret: 'SSH_KEY' } }, 112 | commands: [ 113 | 'echo "Building on ${DRONE_STAGE_MACHINE}"', 114 | // If you don't do this then the C compiler doesn't have an include path containing 115 | // basic system headers. WTF apple: 116 | 'export SDKROOT="$(xcrun --sdk macosx --show-sdk-path)"', 117 | 'ulimit -n 1024', // Because macOS has a stupid tiny default ulimit 118 | ] 119 | + generic_build(build_type, cmake_extra) 120 | + extra_cmds, 121 | }, 122 | ], 123 | }; 124 | 125 | [ 126 | debian_pipeline('Debian sid (amd64)', docker_base + 'debian-sid', distro='sid'), 127 | debian_pipeline('Debian sid/Debug (amd64)', docker_base + 'debian-sid', build_type='Debug', distro='sid'), 128 | clang(17), 129 | full_llvm(19), 130 | debian_pipeline('Debian sid (ARM64)', docker_base + 'debian-sid', arch='arm64', distro='sid'), 131 | debian_pipeline('Debian stable (i386)', docker_base + 'debian-stable/i386'), 132 | debian_pipeline('Debian stable (armhf)', docker_base + 'debian-stable/arm32v7', arch='arm64'), 133 | debian_pipeline('Debian bullseye (amd64)', docker_base + 'debian-bullseye'), 134 | debian_pipeline('Debian bullseye (armhf)', docker_base + 'debian-bullseye/arm32v7', arch='arm64'), 135 | debian_pipeline('Ubuntu focal (amd64)', 136 | docker_base + 'ubuntu-focal', 137 | deps=default_deps_nocxx + ['g++-10'], 138 | cmake_extra='-DCMAKE_C_COMPILER=gcc-10 -DCMAKE_CXX_COMPILER=g++-10'), 139 | debian_pipeline('Ubuntu noble (amd64)', docker_base + 'ubuntu-noble'), 140 | mac_builder('MacOS (amd64) Release', build_type='Release', arch='amd64'), 141 | mac_builder('MacOS (amd64) Debug', build_type='Debug', arch='amd64'), 142 | mac_builder('MacOS (arm64) Release', build_type='Release', arch='arm64'), 143 | mac_builder('MacOS (arm64) Debug', build_type='Debug', arch='arm64'), 144 | ] 145 | -------------------------------------------------------------------------------- /tests/test_tagged_threads.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq/batch.h" 2 | #include "common.h" 3 | #include 4 | 5 | TEST_CASE("tagged thread start functions", "[tagged][start]") { 6 | oxenmq::OxenMQ omq{}; 7 | 8 | omq.set_general_threads(2); 9 | omq.set_batch_threads(2); 10 | auto t_abc = omq.add_tagged_thread("abc"); 11 | std::atomic start_called = false; 12 | auto t_def = omq.add_tagged_thread("def", [&] { start_called = true; }); 13 | 14 | std::this_thread::sleep_for(20ms); 15 | { 16 | auto lock = catch_lock(); 17 | REQUIRE_FALSE( start_called ); 18 | } 19 | 20 | omq.start(); 21 | wait_for([&] { return start_called.load(); }); 22 | { 23 | auto lock = catch_lock(); 24 | REQUIRE( start_called ); 25 | } 26 | } 27 | 28 | TEST_CASE("tagged threads quit-before-start", "[tagged][quit]") { 29 | auto omq = std::make_unique(); 30 | auto t_abc = omq->add_tagged_thread("abc"); 31 | REQUIRE_NOTHROW(omq.reset()); 32 | } 33 | 34 | TEST_CASE("batch jobs to tagged threads", "[tagged][batch]") { 35 | oxenmq::OxenMQ omq{}; 36 | 37 | omq.set_general_threads(2); 38 | omq.set_batch_threads(2); 39 | std::thread::id id_abc, id_def; 40 | auto t_abc = omq.add_tagged_thread("abc", [&] { id_abc = std::this_thread::get_id(); }); 41 | auto t_def = omq.add_tagged_thread("def", [&] { id_def = std::this_thread::get_id(); }); 42 | omq.start(); 43 | 44 | std::atomic done = false; 45 | std::thread::id id; 46 | omq.job([&] { id = std::this_thread::get_id(); done = true; }); 47 | wait_for([&] { return done.load(); }); 48 | { 49 | auto lock = catch_lock(); 50 | REQUIRE( id != id_abc ); 51 | REQUIRE( id != id_def ); 52 | } 53 | 54 | done = false; 55 | omq.job([&] { id = std::this_thread::get_id(); done = true; }, t_abc); 56 | wait_for([&] { return done.load(); }); 57 | { 58 | auto lock = catch_lock(); 59 | REQUIRE( id == id_abc ); 60 | } 61 | 62 | done = false; 63 | omq.job([&] { id = std::this_thread::get_id(); done = true; }, t_def); 64 | wait_for([&] { return done.load(); }); 65 | { 66 | auto lock = catch_lock(); 67 | REQUIRE( id == id_def ); 68 | } 69 | 70 | std::atomic sleep = true; 71 | auto sleeper = [&] { for (int i = 0; sleep && i < 10; i++) { std::this_thread::sleep_for(25ms); } }; 72 | omq.job(sleeper); 73 | omq.job(sleeper); 74 | // This one should stall: 75 | std::atomic bad = false; 76 | omq.job([&] { bad = true; }); 77 | 78 | std::this_thread::sleep_for(50ms); 79 | 80 | done = false; 81 | omq.job([&] { id = std::this_thread::get_id(); done = true; }, t_abc); 82 | wait_for([&] { return done.load(); }); 83 | { 84 | auto lock = catch_lock(); 85 | REQUIRE( done.load() ); 86 | REQUIRE_FALSE( bad.load() ); 87 | } 88 | 89 | done = false; 90 | // We can queue up a bunch of jobs which should all happen in order, and all on the abc thread. 91 | std::vector v; 92 | for (int i = 0; i < 100; i++) { 93 | omq.job([&] { if (std::this_thread::get_id() == id_abc) v.push_back(v.size()); }, t_abc); 94 | } 95 | omq.job([&] { done = true; }, t_abc); 96 | wait_for([&] { return done.load(); }); 97 | { 98 | auto lock = catch_lock(); 99 | REQUIRE( done.load() ); 100 | REQUIRE_FALSE( bad.load() ); 101 | REQUIRE( v.size() == 100 ); 102 | for (int i = 0; i < 100; i++) 103 | REQUIRE( v[i] == i ); 104 | } 105 | sleep = false; 106 | wait_for([&] { return bad.load(); }); 107 | { 108 | auto lock = catch_lock(); 109 | REQUIRE( bad.load() ); 110 | } 111 | } 112 | 113 | TEST_CASE("batch job completion on tagged threads", "[tagged][batch-completion]") { 114 | oxenmq::OxenMQ omq{}; 115 | 116 | omq.set_general_threads(4); 117 | omq.set_batch_threads(4); 118 | std::thread::id id_abc; 119 | auto t_abc = omq.add_tagged_thread("abc", [&] { id_abc = std::this_thread::get_id(); }); 120 | omq.start(); 121 | 122 | oxenmq::Batch batch; 123 | for (int i = 1; i < 10; i++) 124 | batch.add_job([i, &id_abc]() { if (std::this_thread::get_id() == id_abc) return 0; return i; }); 125 | 126 | std::atomic result_sum = -1; 127 | batch.completion([&](auto result) { 128 | int sum = 0; 129 | for (auto& r : result) 130 | sum += r.get(); 131 | result_sum = std::this_thread::get_id() == id_abc ? sum : -sum; 132 | }, t_abc); 133 | omq.batch(std::move(batch)); 134 | wait_for([&] { return result_sum.load() != -1; }); 135 | { 136 | auto lock = catch_lock(); 137 | REQUIRE( result_sum == 45 ); 138 | } 139 | } 140 | 141 | 142 | TEST_CASE("timer job completion on tagged threads", "[tagged][timer]") { 143 | oxenmq::OxenMQ omq{}; 144 | 145 | omq.set_general_threads(4); 146 | omq.set_batch_threads(4); 147 | 148 | std::thread::id id_abc; 149 | auto t_abc = omq.add_tagged_thread("abc", [&] { id_abc = std::this_thread::get_id(); }); 150 | omq.start(); 151 | 152 | std::atomic ticks = 0; 153 | std::atomic abc_ticks = 0; 154 | omq.add_timer([&] { ticks++; }, 10ms); 155 | omq.add_timer([&] { if (std::this_thread::get_id() == id_abc) abc_ticks++; }, 10ms, true, t_abc); 156 | 157 | wait_for([&] { return ticks.load() > 2 && abc_ticks > 2; }); 158 | { 159 | auto lock = catch_lock(); 160 | REQUIRE( ticks.load() > 2 ); 161 | REQUIRE( abc_ticks.load() > 2 ); 162 | } 163 | } 164 | 165 | TEST_CASE("destruction during start with tagged workers", "[tagged][destruction]") { 166 | // Reproducible bug from oxend, where a start() failure due to binding to an already-used bind 167 | // address resulted in a hang during oxenmq shutdown after the start failed, when there were 168 | // also tagged threads present. 169 | 170 | // Make a conflicting listener: 171 | std::string listen = random_localhost(); 172 | oxenmq::OxenMQ omq0{"", "", false, [](auto) { return ""; }}; 173 | omq0.listen_curve(listen); 174 | omq0.start(); 175 | 176 | oxenmq::OxenMQ omq{"", "", false, [](auto) { return ""; }}; 177 | omq.listen_curve(listen); 178 | 179 | SECTION("no tagged thread") {} 180 | SECTION("with tagged thread") { 181 | omq.add_tagged_thread("xxx"); 182 | } 183 | 184 | REQUIRE_THROWS_AS( 185 | omq.start(), 186 | zmq::error_t); 187 | 188 | // HANGS HERE AT `omq` DESTRUCTION WITH BUG 189 | } 190 | 191 | 192 | -------------------------------------------------------------------------------- /oxenmq/pubsub.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "connections.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | namespace oxenmq { 14 | 15 | using namespace std::chrono_literals; 16 | 17 | namespace detail { 18 | struct no_data_t final {}; 19 | inline constexpr no_data_t no_data{}; 20 | 21 | template 22 | struct SubData { 23 | std::chrono::steady_clock::time_point expiry; 24 | UserData user_data; 25 | explicit SubData(std::chrono::steady_clock::time_point _exp) 26 | : expiry{_exp}, user_data{} {} 27 | }; 28 | 29 | template <> 30 | struct SubData { 31 | std::chrono::steady_clock::time_point expiry; 32 | }; 33 | } 34 | 35 | 36 | /** 37 | * OMQ Subscription class. Handles pub/sub connections such that the user only needs to call 38 | * methods to subscribe and publish. 39 | * 40 | * FIXME: do we want an unsubscribe, or is expiry / conn management sufficient? 41 | * 42 | * Type UserData can contain whatever information the user may need at publish time, for example if 43 | * the subscription is for logs the subscriber can specify log levels or categories, and the 44 | * publisher can choose to send or not based on those. The UserData type, if provided and non-void, 45 | * must be default constructible, must be comparable with ==, and must be movable. 46 | */ 47 | template 48 | class Subscription { 49 | static constexpr bool have_user_data = !std::is_void_v; 50 | using UserData_if_present = std::conditional_t; 51 | using subdata_t = detail::SubData; 52 | 53 | std::unordered_map subs; 54 | std::shared_mutex _mutex; 55 | const std::string description; // description of the sub for logging 56 | const std::chrono::milliseconds sub_duration; // extended by re-subscribe 57 | 58 | public: 59 | 60 | Subscription() = delete; 61 | Subscription(std::string description, std::chrono::milliseconds sub_duration = 30min) 62 | : description{std::move(description)}, sub_duration{sub_duration} {} 63 | 64 | // returns true if new sub, false if refresh sub. throws on error. `data` will be checked 65 | // against the existing data: if there is existing data and it compares `==` to the given value, 66 | // false is returned (and the existing data is not replaced). Otherwise the given data gets 67 | // stored for this connection (replacing existing data, if present), and true is returned. 68 | bool subscribe(const ConnectionID& conn, UserData_if_present data) { 69 | std::unique_lock lock{_mutex}; 70 | auto expiry = std::chrono::steady_clock::now() + sub_duration; 71 | auto [value, added] = subs.emplace(conn, subdata_t{expiry}); 72 | if (added) { 73 | if constexpr (have_user_data) 74 | value->second.user_data = std::move(data); 75 | return true; 76 | } 77 | 78 | value->second.expiry = expiry; 79 | 80 | if constexpr (have_user_data) { 81 | // if user_data changed, consider it a new sub rather than refresh, and update 82 | // user_data in the mapped value. 83 | if (!(value->second.user_data == data)) { 84 | value->second.user_data = std::move(data); 85 | return true; 86 | } 87 | } 88 | return false; 89 | } 90 | 91 | // no-user-data version, only available for Subscription (== Subscription without a 92 | // UserData type). 93 | template = 0> 94 | bool subscribe(const ConnectionID& conn) { 95 | return subscribe(conn, detail::no_data); 96 | } 97 | 98 | // unsubscribe a connection ID. return the user data, if a sub was present. 99 | template = 0> 100 | std::optional unsubscribe(const ConnectionID& conn) { 101 | std::unique_lock lock{_mutex}; 102 | 103 | auto node = subs.extract(conn); 104 | if (!node.empty()) 105 | return node.mapped().user_data; 106 | 107 | return std::nullopt; 108 | } 109 | 110 | // no-user-data version, only available for Subscription (== Subscription without a 111 | // UserData type). 112 | template = 0> 113 | bool unsubscribe(const ConnectionID& conn) { 114 | std::unique_lock lock{_mutex}; 115 | auto node = subs.extract(conn); 116 | return !node.empty(); // true if removed, false if wasn't present 117 | } 118 | 119 | // force removal of expired subscriptions. removal will otherwise only happen on publish 120 | void remove_expired() { 121 | std::unique_lock lock{_mutex}; 122 | auto now = std::chrono::steady_clock::now(); 123 | for (auto itr = subs.begin(); itr != subs.end();) { 124 | if (itr->second.expiry < now) 125 | itr = subs.erase(itr); 126 | else 127 | itr++; 128 | } 129 | } 130 | 131 | // Func is any callable which takes: 132 | // - (const ConnectionID&, const UserData&) for Subscription with non-void UserData 133 | // - (const ConnectionID&) for Subscription. 134 | template 135 | void publish(Func&& func) { 136 | std::vector to_remove; 137 | { 138 | std::shared_lock lock(_mutex); 139 | if (subs.empty()) 140 | return; 141 | 142 | auto now = std::chrono::steady_clock::now(); 143 | 144 | for (const auto& [conn, sub] : subs) { 145 | if (sub.expiry < now) 146 | to_remove.push_back(conn); 147 | else if constexpr (have_user_data) 148 | func(conn, sub.user_data); 149 | else 150 | func(conn); 151 | } 152 | } 153 | 154 | if (to_remove.empty()) 155 | return; 156 | 157 | std::unique_lock lock{_mutex}; 158 | auto now = std::chrono::steady_clock::now(); 159 | for (auto& conn : to_remove) { 160 | auto it = subs.find(conn); 161 | if (it != subs.end() && it->second.expiry < now /* recheck: client might have resubscribed in between locks */) { 162 | subs.erase(it); 163 | } 164 | } 165 | } 166 | 167 | }; 168 | 169 | 170 | } // namespace oxenmq 171 | 172 | // vim:sw=4:et 173 | -------------------------------------------------------------------------------- /oxenmq/oxenmq-internal.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | #include 5 | #include 6 | 7 | #include "fmt.h" 8 | #include "oxenmq.h" 9 | 10 | namespace oxenmq { 11 | 12 | namespace log = oxen::log; 13 | 14 | constexpr char SN_ADDR_COMMAND[] = "inproc://sn-command"; 15 | constexpr char SN_ADDR_WORKERS[] = "inproc://sn-workers"; 16 | constexpr char SN_ADDR_SELF[] = "inproc://sn-self"; 17 | constexpr char ZMQ_ADDR_ZAP[] = "inproc://zeromq.zap.01"; 18 | 19 | #ifdef OXENMQ_USE_EPOLL 20 | 21 | constexpr auto EPOLL_COMMAND_ID = std::numeric_limits::max(); 22 | constexpr auto EPOLL_WORKER_ID = std::numeric_limits::max() - 1; 23 | constexpr auto EPOLL_ZAP_ID = std::numeric_limits::max() - 2; 24 | 25 | #endif 26 | 27 | /// Destructor for create_message(std::string&&) that zmq calls when it's done with the message. 28 | extern "C" inline void message_buffer_destroy(void*, void* hint) { 29 | delete reinterpret_cast(hint); 30 | } 31 | 32 | /// Creates a message without needing to reallocate the provided string data 33 | inline zmq::message_t create_message(std::string&& data) { 34 | auto* buffer = new std::string(std::move(data)); 35 | return zmq::message_t{&(*buffer)[0], buffer->size(), message_buffer_destroy, buffer}; 36 | }; 37 | 38 | /// Create a message copying from a string_view 39 | inline zmq::message_t create_message(std::string_view data) { 40 | return zmq::message_t{data.begin(), data.end()}; 41 | } 42 | 43 | template 44 | bool send_message_parts(zmq::socket_t& sock, It begin, It end) { 45 | while (begin != end) { 46 | zmq::message_t& msg = *begin++; 47 | if (!sock.send( 48 | msg, 49 | begin == end ? zmq::send_flags::dontwait 50 | : zmq::send_flags::dontwait | zmq::send_flags::sndmore)) 51 | return false; 52 | } 53 | return true; 54 | } 55 | 56 | template 57 | bool send_message_parts(zmq::socket_t& sock, Container&& c) { 58 | return send_message_parts(sock, c.begin(), c.end()); 59 | } 60 | 61 | /// Sends a message with an initial route. `msg` and `data` can be empty: if `msg` is empty then 62 | /// the msg frame will be an empty message; if `data` is empty then the data frame will be omitted. 63 | /// `flags` is passed through to zmq: typically given `zmq::send_flags::dontwait` to throw rather 64 | /// than block if a message can't be queued. 65 | inline bool send_routed_message( 66 | zmq::socket_t& socket, std::string route, std::string msg = {}, std::string data = {}) { 67 | assert(!route.empty()); 68 | std::array msgs{{create_message(std::move(route))}}; 69 | if (!msg.empty()) 70 | msgs[1] = create_message(std::move(msg)); 71 | if (!data.empty()) 72 | msgs[2] = create_message(std::move(data)); 73 | return send_message_parts( 74 | socket, msgs.begin(), data.empty() ? std::prev(msgs.end()) : msgs.end()); 75 | } 76 | 77 | // Sends some stuff to a socket directly. If dontwait is true then we throw instead of blocking if 78 | // the message cannot be accepted by zmq (i.e. because the outgoing buffer is full). 79 | inline bool send_direct_message(zmq::socket_t& socket, std::string msg, std::string data = {}) { 80 | std::array msgs{{create_message(std::move(msg))}}; 81 | if (!data.empty()) 82 | msgs[1] = create_message(std::move(data)); 83 | return send_message_parts( 84 | socket, msgs.begin(), data.empty() ? std::prev(msgs.end()) : msgs.end()); 85 | } 86 | 87 | // Receive all the parts of a single message from the given socket. Returns true if a message was 88 | // received, false if called with flags=zmq::recv_flags::dontwait and no message was available. 89 | inline bool recv_message_parts( 90 | zmq::socket_t& sock, 91 | std::vector& parts, 92 | const zmq::recv_flags flags = zmq::recv_flags::none) { 93 | do { 94 | zmq::message_t msg; 95 | if (!sock.recv(msg, flags)) 96 | return false; 97 | parts.push_back(std::move(msg)); 98 | } while (parts.back().more()); 99 | return true; 100 | } 101 | 102 | // Same as above, but using a fixed sized array; this is only used for internal jobs (e.g. control 103 | // messages) where we know the message parts should never exceed a given size (this function does 104 | // not bounds check except in debug builds). Returns the number of message parts received, or 0 on 105 | // read error. 106 | template 107 | inline size_t recv_message_parts( 108 | zmq::socket_t& sock, 109 | std::array& parts, 110 | const zmq::recv_flags flags = zmq::recv_flags::none) { 111 | for (size_t count = 0;; count++) { 112 | assert(count < N); 113 | if (!sock.recv(parts[count], flags)) 114 | return 0; 115 | if (!parts[count].more()) 116 | return count + 1; 117 | } 118 | } 119 | 120 | inline const char* get_peer_address(zmq::message_t& msg) { 121 | try { 122 | return msg.gets("Peer-Address"); 123 | } catch (...) { 124 | } 125 | return "(unknown)"; 126 | } 127 | 128 | // For logging: extracts the address on demand 129 | struct peer_address { 130 | zmq::message_t& msg; 131 | }; 132 | 133 | // Returns a string view of the given message data. It's the caller's responsibility to keep the 134 | // referenced message alive. If you want a std::string instead just call `m.to_string()` 135 | inline std::string_view view(const zmq::message_t& m) { 136 | return {m.data(), m.size()}; 137 | } 138 | 139 | // Extracts and builds the "send" part of a message for proxy_send/proxy_reply 140 | inline std::list build_send_parts( 141 | oxenc::bt_list_consumer send, std::string_view route) { 142 | std::list parts; 143 | if (!route.empty()) 144 | parts.push_back(create_message(route)); 145 | while (!send.is_finished()) 146 | parts.push_back(create_message(send.consume_string())); 147 | return parts; 148 | } 149 | 150 | /// Sends a control message to a specific destination by prefixing the worker name (or identity) 151 | /// then appending the command and optional data (if non-empty). (This is needed when sending the 152 | /// control message to a router socket, i.e. inside the proxy thread). 153 | inline void route_control( 154 | zmq::socket_t& sock, 155 | std::string_view identity, 156 | std::string_view cmd, 157 | const std::string& data = {}) { 158 | sock.send(create_message(identity), zmq::send_flags::sndmore); 159 | detail::send_control(sock, cmd, data); 160 | } 161 | 162 | struct log_hex { 163 | std::string_view orig; 164 | }; 165 | 166 | } // namespace oxenmq 167 | 168 | template <> 169 | struct fmt::formatter : formatter { 170 | auto format(const oxenmq::log_hex& l, format_context& ctx) const { 171 | return formatter::format(oxenc::to_hex(l.orig), ctx); 172 | } 173 | }; 174 | 175 | template <> 176 | struct fmt::formatter : formatter { 177 | auto format(oxenmq::peer_address& pa, format_context& ctx) const { 178 | return formatter::format(oxenmq::get_peer_address(pa.msg), ctx); 179 | } 180 | }; 181 | -------------------------------------------------------------------------------- /tests/test_address.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq/address.h" 2 | #include "common.h" 3 | 4 | const std::string pk = "\xf1\x6b\xa5\x59\x10\x39\xf0\x89\xb4\x2a\x83\x41\x75\x09\x30\x94\x07\x4d\x0d\x93\x7a\x79\xe5\x3e\x5c\xe7\x30\xf9\x46\xe1\x4b\x88"; 5 | const std::string pk_hex = "f16ba5591039f089b42a834175093094074d0d937a79e53e5ce730f946e14b88"; 6 | const std::string pk_HEX = "F16BA5591039F089B42A834175093094074D0D937A79E53E5CE730F946E14B88"; 7 | const std::string pk_b32z = "6fi4kseo88aeupbkopyzknjo1odw4dcuxjh6kx1hhhax1tzbjqry"; 8 | const std::string pk_B32Z = "6FI4KSEO88AEUPBKOPYZKNJO1ODW4DCUXJH6KX1HHHAX1TZBJQRY"; 9 | const std::string pk_b64 = "8WulWRA58Im0KoNBdQkwlAdNDZN6eeU+XOcw+UbhS4g"; // NB: padding '=' omitted 10 | 11 | TEST_CASE("tcp addresses", "[address][tcp]") { 12 | address a{"tcp://1.2.3.4:5678"}; 13 | REQUIRE( a.host == "1.2.3.4" ); 14 | REQUIRE( a.port == 5678 ); 15 | REQUIRE_FALSE( a.curve() ); 16 | REQUIRE( a.tcp() ); 17 | REQUIRE( a.zmq_address() == "tcp://1.2.3.4:5678" ); 18 | REQUIRE( a.full_address() == "tcp://1.2.3.4:5678" ); 19 | REQUIRE( a.qr_address() == "TCP://1.2.3.4:5678" ); 20 | 21 | REQUIRE_THROWS_AS( address{"tcp://1:1:1"}, std::invalid_argument ); 22 | REQUIRE_THROWS_AS( address{"tcpz://localhost:123"}, std::invalid_argument ); 23 | REQUIRE_THROWS_AS( address{"tcp://abc"}, std::invalid_argument ); 24 | REQUIRE_THROWS_AS( address{"tcpz://localhost:0"}, std::invalid_argument ); 25 | REQUIRE_THROWS_AS( address{"tcpz://[::1:1080"}, std::invalid_argument ); 26 | 27 | address b = address::tcp("example.com", 80); 28 | REQUIRE( b.host == "example.com" ); 29 | REQUIRE( b.port == 80 ); 30 | REQUIRE_FALSE( b.curve() ); 31 | REQUIRE( b.tcp() ); 32 | REQUIRE( b.zmq_address() == "tcp://example.com:80" ); 33 | REQUIRE( b.full_address() == "tcp://example.com:80" ); 34 | REQUIRE( b.qr_address() == "TCP://EXAMPLE.COM:80" ); 35 | 36 | address c{"tcp://[::1]:1111"}; 37 | REQUIRE( c.host == "[::1]" ); 38 | REQUIRE( c.port == 1111 ); 39 | } 40 | 41 | TEST_CASE("unix sockets", "[address][ipc]") { 42 | address a{"ipc:///path/to/foo"}; 43 | REQUIRE( a.socket == "/path/to/foo" ); 44 | REQUIRE_FALSE( a.curve() ); 45 | REQUIRE_FALSE( a.tcp() ); 46 | REQUIRE( a.zmq_address() == "ipc:///path/to/foo" ); 47 | REQUIRE( a.full_address() == "ipc:///path/to/foo" ); 48 | 49 | address b = address::ipc("../foo"); 50 | REQUIRE( b.socket == "../foo" ); 51 | REQUIRE_FALSE( b.curve() ); 52 | REQUIRE_FALSE( b.tcp() ); 53 | REQUIRE( b.zmq_address() == "ipc://../foo" ); 54 | REQUIRE( b.full_address() == "ipc://../foo" ); 55 | } 56 | 57 | TEST_CASE("pubkey formats", "[address][curve][pubkey]") { 58 | address a{"tcp+curve://a:1/" + pk_hex}; 59 | address b{"curve://a:1/" + pk_b32z}; 60 | address c{"curve://a:1/" + pk_b64}; 61 | address d{"CURVE://A:1/" + pk_B32Z}; 62 | REQUIRE( a.curve() ); 63 | REQUIRE( a.host == "a" ); 64 | REQUIRE( a.port == 1 ); 65 | REQUIRE((b.curve() && c.curve() && d.curve())); 66 | REQUIRE( a.pubkey == pk ); 67 | REQUIRE( b.pubkey == pk ); 68 | REQUIRE( c.pubkey == pk ); 69 | REQUIRE( d.pubkey == pk ); 70 | 71 | address e{"ipc+curve://my.sock/" + pk_hex}; 72 | address f{"ipc+curve://../my.sock/" + pk_b32z}; 73 | address g{"ipc+curve:///my.sock/" + pk_B32Z}; 74 | address h{"ipc+curve://./my.sock/" + pk_b64}; 75 | REQUIRE( e.curve() ); 76 | REQUIRE( e.ipc() ); 77 | REQUIRE_FALSE( e.tcp() ); 78 | REQUIRE((f.curve() && g.curve() && h.curve())); 79 | REQUIRE( e.socket == "my.sock" ); 80 | REQUIRE( f.socket == "../my.sock" ); 81 | REQUIRE( g.socket == "/my.sock" ); 82 | REQUIRE( h.socket == "./my.sock" ); 83 | REQUIRE( e.pubkey == pk ); 84 | REQUIRE( f.pubkey == pk ); 85 | REQUIRE( g.pubkey == pk ); 86 | REQUIRE( h.pubkey == pk ); 87 | 88 | REQUIRE( d.full_address(address::encoding::hex) == "curve://a:1/" + pk_hex ); 89 | REQUIRE( c.full_address(address::encoding::base32z) == "curve://a:1/" + pk_b32z ); 90 | REQUIRE( b.full_address(address::encoding::BASE32Z) == "curve://a:1/" + pk_B32Z ); 91 | REQUIRE( a.full_address(address::encoding::base64) == "curve://a:1/" + pk_b64 ); 92 | 93 | REQUIRE( h.full_address(address::encoding::hex) == "ipc+curve://./my.sock/" + pk_hex ); 94 | REQUIRE( g.full_address(address::encoding::base32z) == "ipc+curve:///my.sock/" + pk_b32z ); 95 | REQUIRE( f.full_address(address::encoding::BASE32Z) == "ipc+curve://../my.sock/" + pk_B32Z ); 96 | REQUIRE( e.full_address(address::encoding::base64) == "ipc+curve://my.sock/" + pk_b64 ); 97 | 98 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock/" + pk_hex.substr(0, 63)}, std::invalid_argument); 99 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock/" + pk_b32z.substr(0, 51)}, std::invalid_argument); 100 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock/" + pk_B32Z.substr(0, 51)}, std::invalid_argument); 101 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock/" + pk_b64.substr(0, 42)}, std::invalid_argument); 102 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock"}, std::invalid_argument); 103 | REQUIRE_THROWS_AS(address{"ipc+curve://my.sock/"}, std::invalid_argument); 104 | } 105 | 106 | 107 | TEST_CASE("tcp QR-code friendly addresses", "[address][tcp][qr]") { 108 | address a{"tcp://public.loki.foundation:12345"}; 109 | address a_qr{"TCP://PUBLIC.LOKI.FOUNDATION:12345"}; 110 | address b{"tcp://PUBLIC.LOKI.FOUNDATION:12345"}; 111 | REQUIRE( a == a_qr ); 112 | REQUIRE( a != b ); 113 | REQUIRE( a.host == "public.loki.foundation" ); 114 | REQUIRE( a.qr_address() == "TCP://PUBLIC.LOKI.FOUNDATION:12345" ); 115 | 116 | address c = address::tcp_curve("public.loki.foundation", 12345, pk); 117 | REQUIRE( c.qr_address() == "CURVE://PUBLIC.LOKI.FOUNDATION:12345/" + pk_B32Z ); 118 | REQUIRE( address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/" + pk_B32Z} == c ); 119 | // We don't produce with upper-case hex, but we accept it: 120 | REQUIRE( address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/" + pk_HEX} == c ); 121 | 122 | // lower case not permitted: ▾ 123 | REQUIRE_THROWS_AS(address{"CURVE://PUBLIC.LOKI.FOUNDATiON:12345/" + pk_B32Z}, std::invalid_argument); 124 | // also only accept upper-base base32z and hex: 125 | REQUIRE_THROWS_AS(address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/" + pk_b32z}, std::invalid_argument); 126 | REQUIRE_THROWS_AS(address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/" + pk_hex}, std::invalid_argument); 127 | // don't accept base64 even if it's upper-case (because case-converting it changes the value) 128 | REQUIRE_THROWS_AS(address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"}, std::invalid_argument); 129 | REQUIRE_THROWS_AS(address{"CURVE://PUBLIC.LOKI.FOUNDATION:12345/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}, std::invalid_argument); 130 | } 131 | 132 | TEST_CASE("address hashing", "[address][hash]") { 133 | address a{"tcp://public.loki.foundation:12345"}; 134 | address b{"tcp+curve://public.loki.foundation:12345/" + pk_hex}; 135 | address c{"ipc:///tmp/some.sock"}; 136 | address d{"ipc:///tmp/some.other.sock"}; 137 | 138 | std::hash hasher{}; 139 | REQUIRE( hasher(a) != hasher(b) ); 140 | REQUIRE( hasher(a) != hasher(c) ); 141 | REQUIRE( hasher(a) != hasher(d) ); 142 | REQUIRE( hasher(b) != hasher(c) ); 143 | REQUIRE( hasher(b) != hasher(d) ); 144 | REQUIRE( hasher(c) != hasher(d) ); 145 | 146 | std::unordered_set set; 147 | set.insert(a); 148 | set.insert(b); 149 | set.insert(c); 150 | set.insert(d); 151 | 152 | CHECK( set.size() == 4 ); 153 | std::unordered_map count; 154 | for (const auto& addr : set) 155 | count[addr]++; 156 | 157 | REQUIRE( count.size() == 4 ); 158 | CHECK( count[a] == 1 ); 159 | CHECK( count[b] == 1 ); 160 | CHECK( count[c] == 1 ); 161 | CHECK( count[d] == 1 ); 162 | } 163 | -------------------------------------------------------------------------------- /oxenmq/jobs.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq.h" 2 | #include "batch.h" 3 | #include "oxenmq-internal.h" 4 | 5 | namespace oxenmq { 6 | 7 | static auto cat = log::Cat("oxenmq"); 8 | 9 | void OxenMQ::proxy_batch(detail::Batch* batch) { 10 | const auto [jobs, tagged_threads] = batch->size(); 11 | log::trace(cat, "proxy queuing batch job with {} jobs{}", jobs, tagged_threads ? " (job uses tagged thread(s))" : ""); 12 | if (!tagged_threads) { 13 | for (size_t i = 0; i < jobs; i++) 14 | batch_jobs.emplace_back(batch, i); 15 | } else { 16 | // Some (or all) jobs have a specific thread target so queue any such jobs in the tagged 17 | // worker queue. 18 | auto threads = batch->threads(); 19 | for (size_t i = 0; i < jobs; i++) { 20 | auto& jobs = threads[i] > 0 21 | ? std::get(tagged_workers[threads[i] - 1]) 22 | : batch_jobs; 23 | jobs.emplace_back(batch, i); 24 | } 25 | } 26 | 27 | proxy_skip_one_poll = true; 28 | } 29 | 30 | void OxenMQ::job(std::function f, std::optional thread) { 31 | if (thread && thread->_id == -1) 32 | throw std::logic_error{"job() cannot be used to queue an in-proxy job"}; 33 | auto* j = new Job(std::move(f), thread); 34 | auto* baseptr = static_cast(j); 35 | detail::send_control(get_control_socket(), "BATCH", oxenc::bt_serialize(reinterpret_cast(baseptr))); 36 | } 37 | 38 | void OxenMQ::proxy_schedule_reply_job(std::function f) { 39 | auto* j = new Job(std::move(f)); 40 | reply_jobs.emplace_back(static_cast(j), 0); 41 | proxy_skip_one_poll = true; 42 | } 43 | 44 | void OxenMQ::proxy_run_batch_jobs(batch_queue& jobs, const int reserved, int& active, bool reply) { 45 | while (!jobs.empty() && active_workers() < max_workers && 46 | (active < reserved || active_workers() < general_workers)) { 47 | proxy_run_worker(get_idle_worker().load(std::move(jobs.front()), reply)); 48 | jobs.pop_front(); 49 | active++; 50 | } 51 | } 52 | 53 | // Called either within the proxy thread, or before the proxy thread has been created; actually adds 54 | // the timer. If the timer object hasn't been set up yet it gets set up here. 55 | void OxenMQ::proxy_timer(int id, std::function job, std::chrono::milliseconds interval, bool squelch, int thread) { 56 | if (!timers) 57 | timers.reset(zmq_timers_new()); 58 | 59 | int zmq_timer_id = zmq_timers_add(timers.get(), 60 | interval.count(), 61 | [](int timer_id, void* self) { static_cast(self)->_queue_timer_job(timer_id); }, 62 | this); 63 | if (zmq_timer_id == -1) 64 | throw zmq::error_t{}; 65 | timer_jobs[zmq_timer_id] = { std::move(job), squelch, false, thread }; 66 | timer_zmq_id[id] = zmq_timer_id; 67 | } 68 | 69 | void OxenMQ::proxy_timer(oxenc::bt_list_consumer timer_data) { 70 | auto timer_id = timer_data.consume_integer(); 71 | std::unique_ptr> func{reinterpret_cast*>(timer_data.consume_integer())}; 72 | auto interval = std::chrono::milliseconds{timer_data.consume_integer()}; 73 | auto squelch = timer_data.consume_integer(); 74 | auto thread = timer_data.consume_integer(); 75 | if (!timer_data.is_finished()) 76 | throw std::runtime_error("Internal error: proxied timer request contains unexpected data"); 77 | proxy_timer(timer_id, std::move(*func), interval, squelch, thread); 78 | } 79 | 80 | void OxenMQ::_queue_timer_job(int timer_id) { 81 | auto it = timer_jobs.find(timer_id); 82 | if (it == timer_jobs.end()) { 83 | log::warning(cat, "Could not find timer job {}", timer_id); 84 | return; 85 | } 86 | auto& [func, squelch, running, thread] = it->second; 87 | if (squelch && running) { 88 | log::debug(cat, "Not running timer job {} because a job for that timer is still running", timer_id); 89 | return; 90 | } 91 | 92 | if (thread == -1) { // Run directly in proxy thread 93 | try { func(); } 94 | catch (const std::exception &e) { log::warning(cat, "timer job {} raised an exception: {}", timer_id, e.what()); } 95 | catch (...) { log::warning(cat, "timer job {} raised a non-std exception", timer_id); } 96 | return; 97 | } 98 | 99 | detail::Batch* b; 100 | if (squelch) { 101 | auto* bv = new Batch; 102 | bv->add_job(func, thread); 103 | running = true; 104 | bv->completion([this,timer_id](auto results) { 105 | try { results[0].get(); } 106 | catch (const std::exception &e) { log::warning(cat, "timer job {} raised an exception: {}", timer_id, e.what()); } 107 | catch (...) { log::warning(cat, "timer job {} raised a non-std exception", timer_id); } 108 | auto it = timer_jobs.find(timer_id); 109 | if (it != timer_jobs.end()) 110 | it->second.running = false; 111 | }, OxenMQ::run_in_proxy); 112 | b = bv; 113 | } else { 114 | b = new Job(func, thread); 115 | } 116 | log::trace(cat, "b: {}, {}; thread = {}", b->size().first, b->size().second, thread); 117 | assert(b->size() == std::make_pair(size_t{1}, thread > 0)); 118 | auto& queue = thread > 0 119 | ? std::get(tagged_workers[thread - 1]) 120 | : batch_jobs; 121 | queue.emplace_back(static_cast(b), 0); 122 | } 123 | 124 | void OxenMQ::add_timer(TimerID& timer, std::function job, std::chrono::milliseconds interval, bool squelch, std::optional thread) { 125 | int th_id = thread ? thread->_id : 0; 126 | timer._id = next_timer_id++; 127 | if (proxy_thread.joinable()) { 128 | detail::send_control(get_control_socket(), "TIMER", oxenc::bt_serialize(oxenc::bt_list{{ 129 | timer._id, 130 | detail::serialize_object(std::move(job)), 131 | interval.count(), 132 | squelch, 133 | th_id}})); 134 | } else { 135 | proxy_timer(timer._id, std::move(job), interval, squelch, th_id); 136 | } 137 | } 138 | 139 | TimerID OxenMQ::add_timer(std::function job, std::chrono::milliseconds interval, bool squelch, std::optional thread) { 140 | TimerID tid; 141 | add_timer(tid, std::move(job), interval, squelch, std::move(thread)); 142 | return tid; 143 | } 144 | 145 | void OxenMQ::proxy_timer_del(int id) { 146 | if (!timers) 147 | return; 148 | auto it = timer_zmq_id.find(id); 149 | if (it == timer_zmq_id.end()) 150 | return; 151 | zmq_timers_cancel(timers.get(), it->second); 152 | timer_zmq_id.erase(it); 153 | } 154 | 155 | void OxenMQ::cancel_timer(TimerID timer_id) { 156 | if (proxy_thread.joinable()) { 157 | detail::send_control(get_control_socket(), "TIMER_DEL", oxenc::bt_serialize(timer_id._id)); 158 | } else { 159 | proxy_timer_del(timer_id._id); 160 | } 161 | } 162 | 163 | void OxenMQ::TimersDeleter::operator()(void* timers) { zmq_timers_destroy(&timers); } 164 | 165 | TaggedThreadID OxenMQ::add_tagged_thread(std::string name, std::function start) { 166 | if (proxy_thread.joinable()) 167 | throw std::logic_error{"Cannot add tagged threads after calling `start()`"}; 168 | 169 | if (name == "_proxy"sv || name.empty() || name.find('\0') != std::string::npos) 170 | throw std::logic_error{"Invalid tagged thread name `" + name + "'"}; 171 | 172 | auto& [run, busy, queue] = tagged_workers.emplace_back(); 173 | busy = false; 174 | run.worker_id = tagged_workers.size(); // We want index + 1 (b/c 0 is used for non-tagged jobs) 175 | run.worker_routing_name = "t" + std::to_string(run.worker_id); 176 | run.worker_routing_id = "t" + std::string{reinterpret_cast(&run.worker_id), sizeof(run.worker_id)}; 177 | log::trace(cat, "Created new tagged thread {} with routing id {}", name, run.worker_routing_name); 178 | 179 | run.worker_thread = std::thread{&OxenMQ::worker_thread, this, run.worker_id, name, std::move(start)}; 180 | 181 | return TaggedThreadID{static_cast(run.worker_id)}; 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | set(CMAKE_EXPORT_COMPILE_COMMANDS ON) 3 | 4 | find_program(CCACHE_PROGRAM ccache) 5 | if(CCACHE_PROGRAM) 6 | foreach(lang C CXX) 7 | if(NOT DEFINED CMAKE_${lang}_COMPILER_LAUNCHER AND NOT CMAKE_${lang}_COMPILER MATCHES ".*/ccache") 8 | message(STATUS "Enabling ccache for ${lang}") 9 | set(CMAKE_${lang}_COMPILER_LAUNCHER ${CCACHE_PROGRAM} CACHE STRING "") 10 | endif() 11 | endforeach() 12 | endif() 13 | 14 | cmake_minimum_required(VERSION 3.10...3.31) 15 | 16 | # Has to be set before `project()`, and ignored on non-macos: 17 | set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12 CACHE STRING "macOS deployment target (Apple clang only)") 18 | 19 | project(liboxenmq 20 | VERSION 1.3.0 21 | LANGUAGES CXX C) 22 | 23 | include(GNUInstallDirs) 24 | include(cmake/libatomic.cmake) 25 | 26 | message(STATUS "oxenmq v${PROJECT_VERSION}") 27 | 28 | set(OXENMQ_LIBVERSION 0) 29 | 30 | 31 | if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) 32 | set(oxenmq_IS_TOPLEVEL_PROJECT TRUE) 33 | else() 34 | set(oxenmq_IS_TOPLEVEL_PROJECT FALSE) 35 | endif() 36 | 37 | 38 | option(BUILD_SHARED_LIBS "Build shared libraries instead of static ones" ON) 39 | set(oxenmq_INSTALL_DEFAULT OFF) 40 | if(BUILD_SHARED_LIBS OR oxenmq_IS_TOPLEVEL_PROJECT) 41 | set(oxenmq_INSTALL_DEFAULT ON) 42 | endif() 43 | set(oxenmq_EPOLL_DEFAULT OFF) 44 | if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND NOT CMAKE_CROSSCOMPILING) 45 | set(oxenmq_EPOLL_DEFAULT ON) 46 | endif() 47 | 48 | option(OXENMQ_BUILD_TESTS "Building and perform oxenmq tests" ${oxenmq_IS_TOPLEVEL_PROJECT}) 49 | option(OXENMQ_INSTALL "Add oxenmq libraries and headers to cmake install target; defaults to ON if BUILD_SHARED_LIBS is enabled or we are the top-level project; OFF for a static subdirectory build" ${oxenmq_INSTALL_DEFAULT}) 50 | option(OXENMQ_INSTALL_CPPZMQ "Install cppzmq header with oxenmq/ headers (requires OXENMQ_INSTALL)" ON) 51 | option(OXENMQ_USE_EPOLL "Build with epoll support for socket polling (requires Linux)" ${oxenmq_EPOLL_DEFAULT}) 52 | 53 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") 54 | 55 | set(CMAKE_CXX_STANDARD 20) 56 | set(CMAKE_CXX_STANDARD_REQUIRED ON) 57 | set(CMAKE_CXX_EXTENSIONS OFF) 58 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 59 | 60 | configure_file(oxenmq/version.h.in oxenmq/version.h @ONLY) 61 | configure_file(liboxenmq.pc.in liboxenmq.pc @ONLY) 62 | 63 | 64 | add_library(oxenmq 65 | oxenmq/address.cpp 66 | oxenmq/auth.cpp 67 | oxenmq/connections.cpp 68 | oxenmq/jobs.cpp 69 | oxenmq/oxenmq.cpp 70 | oxenmq/proxy.cpp 71 | oxenmq/worker.cpp 72 | ) 73 | set_target_properties(oxenmq PROPERTIES SOVERSION ${OXENMQ_LIBVERSION}) 74 | if(OXENMQ_USE_EPOLL) 75 | target_compile_definitions(oxenmq PRIVATE OXENMQ_USE_EPOLL) 76 | endif() 77 | 78 | set(THREADS_PREFER_PTHREAD_FLAG ON) 79 | find_package(Threads REQUIRED) 80 | target_link_libraries(oxenmq PRIVATE Threads::Threads) 81 | 82 | 83 | if(TARGET oxen::logging) 84 | add_library(_oxenmq_external_logging INTERFACE IMPORTED) 85 | target_link_libraries(_oxenmq_external_logging INTERFACE oxen::logging) 86 | target_link_libraries(oxenmq PRIVATE _oxenmq_external_logging) 87 | message(STATUS "using pre-existing oxen::logging target") 88 | elseif(BUILD_SHARED_LIBS) 89 | include(FindPkgConfig) 90 | pkg_check_modules(OXENLOGGING liboxen-logging>=1.2.0) 91 | if(OXENLOGGING_FOUND) 92 | # If we load oxen-logging via system lib then we won't necessarily have fmt/spdlog targets, 93 | # but this script will give us them: 94 | include(oxen-logging/cmake/load_fmt_spdlog.cmake) 95 | 96 | add_library(omq_logging_fmt_spdlog INTERFACE) 97 | target_link_libraries(omq_logging_fmt_spdlog INTERFACE oxen-logging ${OXEN_LOGGING_FMT_TARGET} ${OXEN_LOGGING_SPDLOG_TARGET}) 98 | add_library(oxen::logging ALIAS omq_logging_fmt_spdlog) 99 | target_link_libraries(oxenmq PUBLIC oxen::logging) 100 | endif() 101 | endif() 102 | if(NOT TARGET oxen::logging) 103 | if(APPLE) 104 | # LTO is usually broken on macOS, just like most things under the surface. 105 | set(USE_LTO OFF CACHE INTERNAL "") 106 | endif() 107 | set(OXEN_LOGGING_FORCE_SUBMODULES ON CACHE INTERNAL "") 108 | add_subdirectory(oxen-logging) 109 | target_link_libraries(oxenmq PUBLIC oxen::logging) 110 | endif() 111 | 112 | 113 | if(TARGET oxenc::oxenc) 114 | add_library(_oxenmq_external_oxenc INTERFACE IMPORTED) 115 | target_link_libraries(_oxenmq_external_oxenc INTERFACE oxenc::oxenc) 116 | target_link_libraries(oxenmq PUBLIC _oxenmq_external_oxenc) 117 | message(STATUS "using pre-existing oxenc::oxenc target") 118 | elseif(BUILD_SHARED_LIBS) 119 | include(FindPkgConfig) 120 | pkg_check_modules(oxenc liboxenc>=1.4.0 IMPORTED_TARGET) 121 | 122 | if(oxenc_FOUND) 123 | # Work around cmake bug 22180 (PkgConfig::tgt not set if no flags needed) 124 | if(TARGET PkgConfig::oxenc OR CMAKE_VERSION VERSION_GREATER_EQUAL "3.21") 125 | target_link_libraries(oxenmq PUBLIC PkgConfig::oxenc) 126 | endif() 127 | else() 128 | add_subdirectory(oxen-encoding) 129 | target_link_libraries(oxenmq PUBLIC oxenc::oxenc) 130 | endif() 131 | else() 132 | add_subdirectory(oxen-encoding) 133 | target_link_libraries(oxenmq PUBLIC oxenc::oxenc) 134 | endif() 135 | 136 | # libzmq is nearly impossible to link statically from a system-installed static library: it depends 137 | # on a ton of other libraries, some of which are not all statically available. If the caller wants 138 | # to mess with this, so be it: they can set up a libzmq target and we'll use it. Otherwise if they 139 | # asked us to do things statically, don't even try to find a system lib and just build it. 140 | set(oxenmq_build_static_libzmq OFF) 141 | if(TARGET libzmq) 142 | target_link_libraries(oxenmq PUBLIC libzmq) 143 | elseif(BUILD_SHARED_LIBS) 144 | include(FindPkgConfig) 145 | pkg_check_modules(libzmq libzmq>=4.3 IMPORTED_TARGET) 146 | 147 | if(libzmq_FOUND) 148 | # Debian sid includes a -isystem in the mit-krb package that, starting with pkg-config 0.29.2, 149 | # breaks cmake's pkgconfig module because it stupidly thinks "-isystem" is a path, so if we find 150 | # -isystem in the include dirs then hack it out. 151 | get_property(zmq_inc TARGET PkgConfig::libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES) 152 | list(FIND zmq_inc "-isystem" broken_isystem) 153 | if(NOT broken_isystem EQUAL -1) 154 | list(REMOVE_AT zmq_inc ${broken_isystem}) 155 | set_property(TARGET PkgConfig::libzmq PROPERTY INTERFACE_INCLUDE_DIRECTORIES ${zmq_inc}) 156 | endif() 157 | 158 | target_link_libraries(oxenmq PUBLIC PkgConfig::libzmq) 159 | else() 160 | set(oxenmq_build_static_libzmq ON) 161 | endif() 162 | else() 163 | set(oxenmq_build_static_libzmq ON) 164 | endif() 165 | 166 | if(oxenmq_build_static_libzmq) 167 | message(STATUS "libzmq >= 4.3 not found or static build requested, building bundled version") 168 | list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/local-libzmq") 169 | include(LocalLibzmq) 170 | target_link_libraries(oxenmq PUBLIC libzmq_vendor) 171 | endif() 172 | 173 | target_include_directories(oxenmq 174 | PUBLIC 175 | $ 176 | $ 177 | $ 178 | ) 179 | 180 | target_compile_options(oxenmq PRIVATE -Wall -Wextra) 181 | 182 | option(WARNINGS_AS_ERRORS "treat all warnings as errors" OFF) 183 | if(WARNINGS_AS_ERRORS) 184 | target_compile_options(oxenmq PRIVATE -Werror) 185 | endif() 186 | 187 | if (CMAKE_CXX_COMPILER_ID MATCHES "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 13.0.0) 188 | message(STATUS "Disabling -Werror for restrict-qualified parameters") 189 | target_compile_options(oxenmq PRIVATE -Wno-error=restrict) 190 | endif() 191 | 192 | function(link_dep_libs target linktype libdirs) 193 | foreach(lib ${ARGN}) 194 | find_library(link_lib-${lib} NAMES ${lib} PATHS ${libdirs}) 195 | if(link_lib-${lib}) 196 | target_link_libraries(${target} ${linktype} ${link_lib-${lib}}) 197 | endif() 198 | endforeach() 199 | endfunction() 200 | 201 | # If the caller has already set up a sodium target then we will just link to it, otherwise we go 202 | # looking for it. 203 | if(TARGET sodium) 204 | target_link_libraries(oxenmq PUBLIC sodium) 205 | if(oxenmq_build_static_libzmq) 206 | target_link_libraries(libzmq_vendor INTERFACE sodium) 207 | endif() 208 | else() 209 | include(FindPkgConfig) 210 | pkg_check_modules(sodium REQUIRED libsodium IMPORTED_TARGET) 211 | 212 | if(BUILD_SHARED_LIBS) 213 | target_link_libraries(oxenmq PUBLIC PkgConfig::sodium) 214 | if(oxenmq_build_static_libzmq) 215 | target_link_libraries(libzmq_vendor INTERFACE PkgConfig::sodium) 216 | endif() 217 | else() 218 | link_dep_libs(oxenmq PUBLIC "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES}) 219 | target_include_directories(oxenmq PUBLIC ${sodium_STATIC_INCLUDE_DIRS}) 220 | if(oxenmq_build_static_libzmq) 221 | link_dep_libs(libzmq_vendor INTERFACE "${sodium_STATIC_LIBRARY_DIRS}" ${sodium_STATIC_LIBRARIES}) 222 | target_include_directories(libzmq_vendor INTERFACE ${sodium_STATIC_INCLUDE_DIRS}) 223 | endif() 224 | endif() 225 | endif() 226 | 227 | add_library(oxenmq::oxenmq ALIAS oxenmq) 228 | 229 | if(OXENMQ_INSTALL) 230 | install( 231 | TARGETS oxenmq 232 | EXPORT oxenmqConfig 233 | DESTINATION ${CMAKE_INSTALL_LIBDIR} 234 | ) 235 | 236 | install( 237 | FILES oxenmq/address.h 238 | oxenmq/auth.h 239 | oxenmq/batch.h 240 | oxenmq/connections.h 241 | oxenmq/fmt.h 242 | oxenmq/message.h 243 | oxenmq/oxenmq.h 244 | oxenmq/pubsub.h 245 | ${CMAKE_CURRENT_BINARY_DIR}/oxenmq/version.h 246 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq 247 | ) 248 | 249 | if(OXENMQ_INSTALL_CPPZMQ) 250 | install( 251 | FILES cppzmq/zmq.hpp 252 | DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/oxenmq 253 | ) 254 | endif() 255 | 256 | 257 | install( 258 | FILES ${CMAKE_CURRENT_BINARY_DIR}/liboxenmq.pc 259 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig 260 | ) 261 | 262 | endif() 263 | 264 | if(OXENMQ_BUILD_TESTS) 265 | add_subdirectory(tests) 266 | endif() 267 | -------------------------------------------------------------------------------- /oxenmq/address.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020-2021, The Oxen Project 2 | // 3 | // All rights reserved. 4 | // 5 | // Redistribution and use in source and binary forms, with or without modification, are 6 | // permitted provided that the following conditions are met: 7 | // 8 | // 1. Redistributions of source code must retain the above copyright notice, this list of 9 | // conditions and the following disclaimer. 10 | // 11 | // 2. Redistributions in binary form must reproduce the above copyright notice, this list 12 | // of conditions and the following disclaimer in the documentation and/or other 13 | // materials provided with the distribution. 14 | // 15 | // 3. Neither the name of the copyright holder nor the names of its contributors may be 16 | // used to endorse or promote products derived from this software without specific 17 | // prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 20 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 21 | // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 | // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 | // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 26 | // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 27 | // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #pragma once 30 | #include 31 | #include 32 | #include 33 | #include 34 | 35 | namespace oxenmq { 36 | 37 | using namespace std::literals; 38 | 39 | /** OxenMQ address abstraction class. This class uses and extends standard ZMQ addresses allowing 40 | * extra parameters to be passed in in a relative standard way. 41 | * 42 | * External ZMQ addresses generally have two forms that we are concerned with: one for TCP and one 43 | * for Unix sockets: 44 | * 45 | * tcp://HOST:PORT -- HOST can be a hostname, IPv4 address, or IPv6 address in [...] 46 | * ipc://PATH -- PATH can be absolute (ipc:///path/to/some.sock) or relative (ipc://some.sock) 47 | * 48 | * but this doesn't carry enough info: in particular, we can connect with two very different 49 | * protocols: curve25519-encrypted, or plaintext, but for curve25519-encrypted we require the 50 | * remote's public key as well to verify the connection. 51 | * 52 | * This class, then, handles this by allowing addresses of: 53 | * 54 | * Standard ZMQ address: these carry no pubkey and so the connection will be unencrypted: 55 | * 56 | * tcp://HOSTNAME:PORT 57 | * ipc://PATH 58 | * 59 | * Non-ZMQ address formats that specify that the connection shall be x25519 encrypted: 60 | * 61 | * curve://HOSTNAME:PORT/PUBKEY -- PUBKEY must be specified in hex (64 characters), base32z (52) 62 | * or base64 (43 or 44 with one '=' trailing padding) 63 | * ipc+curve:///path/to/my.sock/PUBKEY -- same requirements on PUBKEY as above. 64 | * tcp+curve://(whatever) -- alias for curve://(whatever) 65 | * 66 | * We also accept special upper-case TCP-only variants which *only* accept uppercase characters and 67 | * a few required symbols (:, /, $, ., and -) in the string: 68 | * 69 | * TCP://HOSTNAME:PORT 70 | * CURVE://HOSTNAME:PORT/B32ZPUBKEY 71 | * 72 | * These versions are explicitly meant to be used with QR codes; the upper-case-only requirement 73 | * allows a smaller QR code by allowing QR's alphanumeric mode (which allows only [A-Z0-9 $%*+./:-]) 74 | * to be used. Such a QR-friendly address can be created from the qr_address() method. To support 75 | * literal IPv6 addresses we surround the address with $...$ instead of the usual [...]. 76 | * 77 | * Note that this class does very little validate the host argument at all, and no socket path 78 | * validation whatsoever. The only constraint on host is when parsing an encoded address: we check 79 | * that it contains no : at all, or must be a [bracketed] expression that contains only hex 80 | * characters, :'s, or .'s. Otherwise, if you pass broken crap into the hostname, expect broken 81 | * crap out. 82 | */ 83 | struct address { 84 | /// Supported address protocols: TCP connections (tcp), or unix sockets (ipc). 85 | enum class proto { 86 | tcp, 87 | tcp_curve, 88 | ipc, 89 | ipc_curve 90 | }; 91 | /// Supported public key encodings (used when regenerating an augmented address). 92 | enum class encoding { 93 | hex, ///< hexadecimal encoded 94 | base32z, ///< base32z encoded 95 | base64, ///< base64 encoded (*without* trailing = padding) 96 | BASE32Z ///< upper-case base32z encoding, meant for QR encoding 97 | }; 98 | 99 | /// The protocol: one of the `protocol` enum values for tcp or ipc (unix sockets), with or 100 | /// without _curve encryption. 101 | proto protocol = proto::tcp; 102 | /// The host for tcp connections; can be a hostname or IP address. If this is an IPv6 it must be surrounded with [ ]. 103 | std::string host; 104 | /// The port (for tcp connections) 105 | uint16_t port = 0; 106 | /// The socket path (for unix socket connections) 107 | std::string socket; 108 | /// If a curve connection, this is the required remote public key (in bytes) 109 | std::string pubkey; 110 | 111 | /// Default constructor; this gives you an unusable address. 112 | address() = default; 113 | 114 | /** 115 | * Constructs an address by parsing a string_view containing one of the formats listed in the 116 | * class description. This is intentionally implicitly constructible so that you can pass a 117 | * string_view into anything expecting an `address`. 118 | * 119 | * Throw std::invalid_argument if the given address is not parseable. 120 | */ 121 | address(std::string_view addr); 122 | 123 | /** Constructs an address from a remote string and a separate pubkey. Typically `remote` is a 124 | * basic ZMQ connect string, though this is not enforced. Any pubkey information embedded in 125 | * the remote string will be discarded and replaced with the given pubkey string. The result 126 | * will be curve encrypted if `pubkey` is non-empty, plaintext if `pubkey` is empty. 127 | * 128 | * Throws an exception if either addr or pubkey is invalid. 129 | * 130 | * Exactly equivalent to `address a{remote}; a.set_pubkey(pubkey);` 131 | */ 132 | address(std::string_view addr, std::string_view pubkey) : address(addr) { set_pubkey(pubkey); } 133 | 134 | /// Replaces the address's pubkey (if any) with the given pubkey (or no pubkey if empty). If 135 | /// changing from pubkey to no-pubkey or no-pubkey to pubkey then the protocol is update to 136 | /// switch to or from curve encryption. 137 | /// 138 | /// pubkey should be the 32-byte binary pubkey, or an empty string to remove an existing pubkey. 139 | /// 140 | /// Returns the object itself, so that you can chain it. 141 | address& set_pubkey(std::string_view pubkey); 142 | 143 | /// Constructs and builds the ZMQ connection address from the stored connection details. This 144 | /// does not contain any of the curve-related details; those must be specified separately when 145 | /// interfacing with ZMQ. 146 | std::string zmq_address() const; 147 | 148 | /// Returns true if the connection was specified as a curve-encryption-enabled connection, false 149 | /// otherwise. 150 | bool curve() const { return protocol == proto::tcp_curve || protocol == proto::ipc_curve; } 151 | 152 | /// True if the protocol is TCP (either with or without curve) 153 | bool tcp() const { return protocol == proto::tcp || protocol == proto::tcp_curve; } 154 | 155 | /// True if the protocol is unix socket (either with or without curve) 156 | bool ipc() const { return !tcp(); } 157 | 158 | /// Returns the full "augmented" address string (i.e. that could be passed in to the 159 | /// constructor). This will be equivalent (but not necessarily identical) to an augmented 160 | /// string passed into the constructor. Takes an optional encoding format for the pubkey (if 161 | /// any), which defaults to base32z. 162 | std::string full_address(encoding enc = encoding::base32z) const; 163 | 164 | /// Returns a QR-code friendly address string. This returns an all-uppercase version of the 165 | /// address with "TCP://" or "CURVE://" for the protocol string, and uses upper-case base32z 166 | /// encoding for the pubkey (for curve addresses). For literal IPv6 addresses we surround the 167 | /// address with $ instead of [...] 168 | /// 169 | /// \throws std::logic_error if called on a unix socket address. 170 | std::string qr_address() const; 171 | 172 | /// Returns `.pubkey` but encoded in the given format 173 | std::string encode_pubkey(encoding enc) const; 174 | 175 | /// Returns true if two addresses are identical (i.e. same protocol and relevant protocol 176 | /// arguments). 177 | /// 178 | /// Note that it is possible for addresses to connect to the same socket without being 179 | /// identical: for example, using "foo.sock" and "./foo.sock", or writing IPv6 addresses (or 180 | /// even IPv4 addresses) in slightly different ways). Such equivalent but non-equal values will 181 | /// result in a false return here. 182 | /// 183 | /// Note also that we ignore irrelevant arguments: for example, we don't care whether pubkeys 184 | /// match when comparing two non-curve TCP addresses. 185 | bool operator==(const address& other) const; 186 | /// Negation of == 187 | bool operator!=(const address& other) const { return !operator==(other); } 188 | 189 | /// Factory function that constructs a TCP address from a host and port. The connection will be 190 | /// plaintext. If the host is an IPv6 address it *must* be surrounded with [ and ]. 191 | static address tcp(std::string host, uint16_t port); 192 | 193 | /// Factory function that constructs a curve-encrypted TCP address from a host, port, and remote 194 | /// pubkey. The pubkey must be 32 bytes. As above, IPv6 addresses must be specified as [addr]. 195 | static address tcp_curve(std::string host, uint16_t, std::string pubkey); 196 | 197 | /// Factory function that constructs a unix socket address from a path. The connection will be 198 | /// plaintext (which is usually fine for a socket since unix sockets are local machine). 199 | static address ipc(std::string path); 200 | 201 | /// Factory function that constructs a unix socket address from a path and remote pubkey. The 202 | /// connection will be curve25519 encrypted; the remote pubkey must be 32 bytes. 203 | static address ipc_curve(std::string path, std::string pubkey); 204 | }; 205 | 206 | } // namespace oxenmq 207 | 208 | namespace std { 209 | template<> struct hash { 210 | std::size_t operator()(const oxenmq::address& a) const noexcept { 211 | return std::hash{}(a.full_address(oxenmq::address::encoding::hex)); 212 | } 213 | }; 214 | } // namespace std 215 | -------------------------------------------------------------------------------- /tests/test_failures.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | #include 4 | 5 | using namespace oxenmq; 6 | 7 | TEST_CASE("failure responses - UNKNOWNCOMMAND", "[failure][UNKNOWNCOMMAND]") { 8 | std::string listen = random_localhost(); 9 | OxenMQ server{ 10 | "", "", // generate ephemeral keys 11 | false, // not a service node 12 | [](auto) { return ""; } 13 | }; 14 | server.listen_plain(listen); 15 | server.start(); 16 | 17 | // Use a raw socket here because I want to see the raw commands coming on the wire 18 | zmq::context_t client_ctx; 19 | zmq::socket_t client{client_ctx, zmq::socket_type::dealer}; 20 | client.connect(listen); 21 | // Handshake: we send HI, they reply HELLO. 22 | client.send(zmq::message_t{"HI", 2}, zmq::send_flags::none); 23 | { 24 | zmq::message_t hello; 25 | auto recvd = client.recv(hello); 26 | 27 | auto lock = catch_lock(); 28 | REQUIRE( recvd ); 29 | REQUIRE( hello.to_string() == "HELLO" ); 30 | REQUIRE_FALSE( hello.more() ); 31 | } 32 | 33 | client.send(zmq::message_t{"a.a", 3}, zmq::send_flags::none); 34 | zmq::message_t resp; 35 | auto recvd = client.recv(resp); 36 | 37 | auto lock = catch_lock(); 38 | REQUIRE( recvd ); 39 | REQUIRE( resp.to_string() == "UNKNOWNCOMMAND" ); 40 | REQUIRE( resp.more() ); 41 | REQUIRE( client.recv(resp) ); 42 | REQUIRE( resp.to_string() == "a.a" ); 43 | REQUIRE_FALSE( resp.more() ); 44 | } 45 | 46 | TEST_CASE("failure responses - NO_REPLY_TAG", "[failure][NO_REPLY_TAG]") { 47 | std::string listen = random_localhost(); 48 | OxenMQ server{ 49 | "", "", // generate ephemeral keys 50 | false, // not a service node 51 | [](auto) { return ""; } 52 | }; 53 | server.listen_plain(listen); 54 | server.add_category("x", AuthLevel::none) 55 | .add_request_command("r", [] (auto& m) { m.send_reply("a"); }); 56 | server.start(); 57 | 58 | // Use a raw socket here because I want to see the raw commands coming on the wire 59 | zmq::context_t client_ctx; 60 | zmq::socket_t client{client_ctx, zmq::socket_type::dealer}; 61 | client.connect(listen); 62 | // Handshake: we send HI, they reply HELLO. 63 | client.send(zmq::message_t{"HI", 2}, zmq::send_flags::none); 64 | { 65 | zmq::message_t hello; 66 | auto recvd = client.recv(hello); 67 | 68 | auto lock = catch_lock(); 69 | REQUIRE( recvd ); 70 | REQUIRE( hello.to_string() == "HELLO" ); 71 | REQUIRE_FALSE( hello.more() ); 72 | } 73 | 74 | client.send(zmq::message_t{"x.r", 3}, zmq::send_flags::none); 75 | zmq::message_t resp; 76 | auto recvd = client.recv(resp); 77 | 78 | { 79 | auto lock = catch_lock(); 80 | REQUIRE( recvd ); 81 | REQUIRE( resp.to_string() == "NO_REPLY_TAG" ); 82 | REQUIRE( resp.more() ); 83 | REQUIRE( client.recv(resp) ); 84 | REQUIRE( resp.to_string() == "x.r" ); 85 | REQUIRE_FALSE( resp.more() ); 86 | } 87 | 88 | client.send(zmq::message_t{"x.r", 3}, zmq::send_flags::sndmore); 89 | client.send(zmq::message_t{"foo", 3}, zmq::send_flags::none); 90 | recvd = client.recv(resp); 91 | { 92 | auto lock = catch_lock(); 93 | REQUIRE( recvd ); 94 | REQUIRE( resp.to_string() == "REPLY" ); 95 | REQUIRE( resp.more() ); 96 | REQUIRE( client.recv(resp) ); 97 | REQUIRE( resp.to_string() == "foo" ); 98 | REQUIRE( resp.more() ); 99 | REQUIRE( client.recv(resp) ); 100 | REQUIRE( resp.to_string() == "a" ); 101 | REQUIRE_FALSE( resp.more() ); 102 | } 103 | } 104 | 105 | TEST_CASE("failure responses - FORBIDDEN", "[failure][FORBIDDEN]") { 106 | std::string listen = random_localhost(); 107 | OxenMQ server{ 108 | "", "", // generate ephemeral keys 109 | false, // not a service node 110 | [](auto) { return ""; } 111 | }; 112 | server.listen_plain(listen, [](auto, auto, auto) { 113 | static int count = 0; 114 | ++count; 115 | return count == 1 ? AuthLevel::none : count == 2 ? AuthLevel::basic : AuthLevel::admin; 116 | }); 117 | server.add_category("x", AuthLevel::basic) 118 | .add_command("x", [] (auto& m) { m.send_back("a"); }); 119 | server.add_category("y", AuthLevel::admin) 120 | .add_command("x", [] (auto& m) { m.send_back("b"); }); 121 | server.start(); 122 | 123 | zmq::context_t client_ctx; 124 | std::array clients; 125 | // Client 0 should get none auth level, client 1 should get basic, client 2 should get admin 126 | for (auto& client : clients) { 127 | client = {client_ctx, zmq::socket_type::dealer}; 128 | client.connect(listen); 129 | // Handshake: we send HI, they reply HELLO. 130 | client.send(zmq::message_t{"HI", 2}, zmq::send_flags::none); 131 | { 132 | zmq::message_t hello; 133 | auto recvd = client.recv(hello); 134 | 135 | auto lock = catch_lock(); 136 | REQUIRE( recvd ); 137 | REQUIRE( hello.to_string() == "HELLO" ); 138 | REQUIRE_FALSE( hello.more() ); 139 | } 140 | } 141 | 142 | for (auto& c : clients) 143 | c.send(zmq::message_t{"x.x", 3}, zmq::send_flags::none); 144 | 145 | zmq::message_t resp; 146 | auto recvd = clients[0].recv(resp); 147 | { 148 | auto lock = catch_lock(); 149 | REQUIRE( recvd ); 150 | REQUIRE( resp.to_string() == "FORBIDDEN" ); 151 | REQUIRE( resp.more() ); 152 | REQUIRE( clients[0].recv(resp) ); 153 | REQUIRE( resp.to_string() == "x.x" ); 154 | REQUIRE_FALSE( resp.more() ); 155 | } 156 | for (int i : {1, 2}) { 157 | recvd = clients[i].recv(resp); 158 | auto lock = catch_lock(); 159 | REQUIRE( recvd ); 160 | REQUIRE( resp.to_string() == "a" ); 161 | REQUIRE_FALSE( resp.more() ); 162 | } 163 | 164 | for (auto& c : clients) 165 | c.send(zmq::message_t{"y.x", 3}, zmq::send_flags::none); 166 | 167 | for (int i : {0, 1}) { 168 | recvd = clients[i].recv(resp); 169 | auto lock = catch_lock(); 170 | REQUIRE( recvd ); 171 | REQUIRE( resp.to_string() == "FORBIDDEN" ); 172 | REQUIRE( resp.more() ); 173 | REQUIRE( clients[i].recv(resp) ); 174 | REQUIRE( resp.to_string() == "y.x" ); 175 | REQUIRE_FALSE( resp.more() ); 176 | } 177 | recvd = clients[2].recv(resp); 178 | { 179 | auto lock = catch_lock(); 180 | REQUIRE( recvd ); 181 | REQUIRE( resp.to_string() == "b" ); 182 | REQUIRE_FALSE( resp.more() ); 183 | } 184 | } 185 | 186 | TEST_CASE("failure responses - NOT_A_SERVICE_NODE", "[failure][NOT_A_SERVICE_NODE]") { 187 | std::string listen = random_localhost(); 188 | OxenMQ server{ 189 | "", "", // generate ephemeral keys 190 | false, // not a service node 191 | [](auto) { return ""; } 192 | }; 193 | server.listen_plain(listen, [](auto, auto, auto) { 194 | static int count = 0; 195 | ++count; 196 | return count == 1 ? AuthLevel::none : count == 2 ? AuthLevel::basic : AuthLevel::admin; 197 | }); 198 | server.add_category("x", Access{AuthLevel::none, false, true}) 199 | .add_command("x", [] (auto&) {}) 200 | .add_request_command("r", [] (auto& m) { m.send_reply(); }) 201 | ; 202 | server.start(); 203 | 204 | zmq::context_t client_ctx; 205 | zmq::socket_t client{client_ctx, zmq::socket_type::dealer}; 206 | client.connect(listen); 207 | // Handshake: we send HI, they reply HELLO. 208 | client.send(zmq::message_t{"HI", 2}, zmq::send_flags::none); 209 | { 210 | zmq::message_t hello; 211 | auto recvd = client.recv(hello); 212 | 213 | auto lock = catch_lock(); 214 | REQUIRE( recvd ); 215 | REQUIRE( hello.to_string() == "HELLO" ); 216 | REQUIRE_FALSE( hello.more() ); 217 | } 218 | 219 | client.send(zmq::message_t{"x.x", 3}, zmq::send_flags::none); 220 | 221 | zmq::message_t resp; 222 | auto recvd = client.recv(resp); 223 | { 224 | auto lock = catch_lock(); 225 | REQUIRE( recvd ); 226 | REQUIRE( resp.to_string() == "NOT_A_SERVICE_NODE" ); 227 | REQUIRE( resp.more() ); 228 | REQUIRE( client.recv(resp) ); 229 | REQUIRE( resp.to_string() == "x.x" ); 230 | REQUIRE_FALSE( resp.more() ); 231 | } 232 | 233 | client.send(zmq::message_t{"x.r", 3}, zmq::send_flags::sndmore); 234 | client.send(zmq::message_t{"xyz123", 6}, zmq::send_flags::none); // reply tag 235 | 236 | recvd = client.recv(resp); 237 | { 238 | auto lock = catch_lock(); 239 | REQUIRE( recvd ); 240 | REQUIRE( resp.to_string() == "NOT_A_SERVICE_NODE" ); 241 | REQUIRE( resp.more() ); 242 | REQUIRE( client.recv(resp) ); 243 | REQUIRE( resp.to_string() == "REPLY" ); 244 | REQUIRE( resp.more() ); 245 | REQUIRE( client.recv(resp) ); 246 | REQUIRE( resp.to_string() == "xyz123" ); 247 | REQUIRE_FALSE( resp.more() ); 248 | } 249 | } 250 | 251 | TEST_CASE("failure responses - FORBIDDEN_SN", "[failure][FORBIDDEN_SN]") { 252 | std::string listen = random_localhost(); 253 | OxenMQ server{ 254 | "", "", // generate ephemeral keys 255 | false, // not a service node 256 | [](auto) { return ""; } 257 | }; 258 | server.listen_plain(listen, [](auto, auto, auto) { 259 | static int count = 0; 260 | ++count; 261 | return count == 1 ? AuthLevel::none : count == 2 ? AuthLevel::basic : AuthLevel::admin; 262 | }); 263 | server.add_category("x", Access{AuthLevel::none, true, false}) 264 | .add_command("x", [] (auto&) {}) 265 | .add_request_command("r", [] (auto& m) { m.send_reply(); }) 266 | ; 267 | server.start(); 268 | 269 | zmq::context_t client_ctx; 270 | zmq::socket_t client{client_ctx, zmq::socket_type::dealer}; 271 | client.connect(listen); 272 | // Handshake: we send HI, they reply HELLO. 273 | client.send(zmq::message_t{"HI", 2}, zmq::send_flags::none); 274 | { 275 | zmq::message_t hello; 276 | auto recvd = client.recv(hello); 277 | 278 | auto lock = catch_lock(); 279 | REQUIRE( recvd ); 280 | REQUIRE( hello.to_string() == "HELLO" ); 281 | REQUIRE_FALSE( hello.more() ); 282 | } 283 | 284 | client.send(zmq::message_t{"x.x", 3}, zmq::send_flags::none); 285 | 286 | zmq::message_t resp; 287 | auto recvd = client.recv(resp); 288 | { 289 | auto lock = catch_lock(); 290 | REQUIRE( recvd ); 291 | REQUIRE( resp.to_string() == "FORBIDDEN_SN" ); 292 | REQUIRE( resp.more() ); 293 | REQUIRE( client.recv(resp) ); 294 | REQUIRE( resp.to_string() == "x.x" ); 295 | REQUIRE_FALSE( resp.more() ); 296 | } 297 | 298 | client.send(zmq::message_t{"x.r", 3}, zmq::send_flags::sndmore); 299 | client.send(zmq::message_t{"xyz123", 6}, zmq::send_flags::none); // reply tag 300 | 301 | recvd = client.recv(resp); 302 | { 303 | auto lock = catch_lock(); 304 | REQUIRE( recvd ); 305 | REQUIRE( resp.to_string() == "FORBIDDEN_SN" ); 306 | REQUIRE( resp.more() ); 307 | REQUIRE( client.recv(resp) ); 308 | REQUIRE( resp.to_string() == "REPLY" ); 309 | REQUIRE( resp.more() ); 310 | REQUIRE( client.recv(resp) ); 311 | REQUIRE( resp.to_string() == "xyz123" ); 312 | REQUIRE_FALSE( resp.more() ); 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /oxenmq/batch.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2020-2021, The Oxen Project 2 | // 3 | // All rights reserved. 4 | // 5 | // Redistribution and use in source and binary forms, with or without modification, are 6 | // permitted provided that the following conditions are met: 7 | // 8 | // 1. Redistributions of source code must retain the above copyright notice, this list of 9 | // conditions and the following disclaimer. 10 | // 11 | // 2. Redistributions in binary form must reproduce the above copyright notice, this list 12 | // of conditions and the following disclaimer in the documentation and/or other 13 | // materials provided with the distribution. 14 | // 15 | // 3. Neither the name of the copyright holder nor the names of its contributors may be 16 | // used to endorse or promote products derived from this software without specific 17 | // prior written permission. 18 | // 19 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 20 | // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 21 | // MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL 22 | // THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 24 | // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | // INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 26 | // STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF 27 | // THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | 29 | #pragma once 30 | #include 31 | #include 32 | #include 33 | #include "oxenmq.h" 34 | 35 | namespace oxenmq { 36 | 37 | namespace detail { 38 | 39 | enum class BatchState { 40 | running, // there are still jobs to run (or running) 41 | complete, // the batch is complete but still has a completion job to call 42 | done // the batch is complete and has no completion function 43 | }; 44 | 45 | struct BatchStatus { 46 | BatchState state; 47 | int thread; 48 | }; 49 | 50 | // Virtual base class for Batch 51 | class Batch { 52 | public: 53 | // Returns the number of jobs in this batch and whether any of them are thread-specific 54 | virtual std::pair size() const = 0; 55 | // Returns a vector of exactly the same length of size().first containing the tagged thread ids 56 | // of the batch jobs or 0 for general jobs. 57 | virtual std::vector threads() const = 0; 58 | // Called in a worker thread to run the job 59 | virtual void run_job(int i) = 0; 60 | // Called in the main proxy thread when the worker returns from finishing a job. The return 61 | // value tells us whether the current finishing job finishes off the batch: `running` to tell us 62 | // there are more jobs; `complete` to tell us that the jobs are done but the completion function 63 | // needs to be called; and `done` to signal that the jobs are done and there is no completion 64 | // function. 65 | virtual BatchStatus job_finished() = 0; 66 | // Called by a worker; not scheduled until all jobs are done. 67 | virtual void job_completion() = 0; 68 | 69 | virtual ~Batch() = default; 70 | }; 71 | 72 | } 73 | 74 | /** 75 | * Simple class that can either hold a result or an exception and retrieves the result (or raises 76 | * the exception) via a .get() method. 77 | * 78 | * This is designed to be like a very stripped down version of a std::promise/std::future pair. We 79 | * reimplemented it, however, because by ditching all the thread synchronization that promise/future 80 | * guarantees we can substantially reduce call overhead (by a factor of ~8 according to benchmarking 81 | * code). Since OxenMQ's proxy<->worker communication channel already gives us thread that overhead 82 | * would just be wasted. 83 | * 84 | * @tparam R the value type held by the result; must be default constructible. Note, however, that 85 | * there are specializations provided for lvalue references types and `void` (which obviously don't 86 | * satisfy this). 87 | */ 88 | template 89 | class job_result { 90 | R value; 91 | std::exception_ptr exc; 92 | 93 | public: 94 | /// Sets the value. Should be called only once, or not at all if set_exception was called. 95 | void set_value(R&& v) { value = std::move(v); } 96 | 97 | /// Sets the exception, which will be rethrown when `get()` is called. Should be called 98 | /// only once, or not at all if set_value() was called. 99 | void set_exception(std::exception_ptr e) { exc = std::move(e); } 100 | 101 | /// Retrieves the value. If an exception was set instead of a value then that exception is 102 | /// thrown instead. Note that the interval value is moved out of the held value so you should 103 | /// not call this multiple times. 104 | R get() { 105 | if (exc) std::rethrow_exception(exc); 106 | return std::move(value); 107 | } 108 | }; 109 | 110 | /** job_result specialization for reference types */ 111 | template 112 | class job_result::value>> { 113 | std::remove_reference_t* value_ptr; 114 | std::exception_ptr exc; 115 | 116 | public: 117 | void set_value(R v) { value_ptr = &v; } 118 | void set_exception(std::exception_ptr e) { exc = std::move(e); } 119 | R get() { 120 | if (exc) std::rethrow_exception(exc); 121 | return *value_ptr; 122 | } 123 | }; 124 | 125 | /** job_result specialization for void; there is no value, but exceptions are still captured 126 | * (rethrown when `get()` is called). 127 | */ 128 | template<> 129 | class job_result { 130 | std::exception_ptr exc; 131 | 132 | public: 133 | void set_exception(std::exception_ptr e) { exc = std::move(e); } 134 | // Returns nothing, but rethrows if there is a captured exception. 135 | void get() { if (exc) std::rethrow_exception(exc); } 136 | }; 137 | 138 | /// Helper class used to set up batches of jobs to be scheduled via the oxenmq job handler. 139 | /// 140 | /// @tparam R - the return type of the individual jobs 141 | /// 142 | template 143 | class Batch final : private detail::Batch { 144 | friend class OxenMQ; 145 | public: 146 | /// The completion function type, called after all jobs have finished. 147 | using CompletionFunc = std::function> results)>; 148 | 149 | // Default constructor 150 | Batch() = default; 151 | 152 | // movable 153 | Batch(Batch&&) = default; 154 | Batch &operator=(Batch&&) = default; 155 | 156 | // non-copyable 157 | Batch(const Batch&) = delete; 158 | Batch &operator=(const Batch&) = delete; 159 | 160 | private: 161 | std::vector, int>> jobs; 162 | std::vector> results; 163 | CompletionFunc complete; 164 | std::size_t jobs_outstanding = 0; 165 | int complete_in_thread = 0; 166 | bool started = false; 167 | bool tagged_thread_jobs = false; 168 | 169 | void check_not_started() { 170 | if (started) 171 | throw std::logic_error("Cannot add jobs or completion function after starting a oxenmq::Batch!"); 172 | } 173 | 174 | public: 175 | /// Preallocates space in the internal vector that stores jobs. 176 | void reserve(std::size_t num) { 177 | jobs.reserve(num); 178 | results.reserve(num); 179 | } 180 | 181 | /// Adds a job. This takes any callable object that is invoked with no arguments and returns R 182 | /// (the Batch return type). The tasks will be scheduled and run when the next worker thread is 183 | /// available. The called function may throw exceptions (which will be propagated to the 184 | /// completion function through the job_result values). There is no guarantee on the order of 185 | /// invocation of the jobs. 186 | /// 187 | /// \param job the callback 188 | /// \param thread an optional TaggedThreadID indicating a thread in which this job must run 189 | void add_job(std::function job, std::optional thread = std::nullopt) { 190 | check_not_started(); 191 | if (thread && thread->_id == -1) 192 | // There are some special case internal jobs where we allow this, but they use the 193 | // private method below that doesn't have this check. 194 | throw std::logic_error{"Cannot add a proxy thread batch job -- this makes no sense"}; 195 | add_job(std::move(job), thread ? thread->_id : 0); 196 | } 197 | 198 | /// Sets the completion function to invoke after all jobs have finished. If this is not set 199 | /// then jobs simply run and results are discarded. 200 | /// 201 | /// \param comp - function to call when all jobs have finished 202 | /// \param thread - optional tagged thread in which to schedule the completion job. If not 203 | /// provided then the completion job is scheduled in the pool of batch job threads. 204 | /// 205 | /// `thread` can be provided the value &OxenMQ::run_in_proxy to invoke the completion function 206 | /// *IN THE PROXY THREAD* itself after all jobs have finished. Be very, very careful: this 207 | /// should be a nearly trivial job that does not require any substantial CPU time and does not 208 | /// block for any reason. This is only intended for the case where the completion job is so 209 | /// trivial that it will take less time than simply queuing the job to be executed by another 210 | /// thread. 211 | void completion(CompletionFunc comp, std::optional thread = std::nullopt) { 212 | check_not_started(); 213 | if (complete) 214 | throw std::logic_error("Completion function can only be set once"); 215 | complete = std::move(comp); 216 | complete_in_thread = thread ? thread->_id : 0; 217 | } 218 | 219 | private: 220 | 221 | void add_job(std::function job, int thread_id) { 222 | jobs.emplace_back(std::move(job), thread_id); 223 | results.emplace_back(); 224 | jobs_outstanding++; 225 | if (thread_id != 0) 226 | tagged_thread_jobs = true; 227 | } 228 | 229 | std::pair size() const override { 230 | return {jobs.size(), tagged_thread_jobs}; 231 | } 232 | 233 | std::vector threads() const override { 234 | std::vector t; 235 | t.reserve(jobs.size()); 236 | for (auto& j : jobs) 237 | t.push_back(j.second); 238 | return t; 239 | }; 240 | 241 | template 242 | void set_value(job_result& r, std::function& f) { r.set_value(f()); } 243 | void set_value(job_result&, std::function& f) { f(); } 244 | 245 | void run_job(const int i) override { 246 | // called by worker thread 247 | auto& r = results[i]; 248 | try { 249 | set_value(r, jobs[i].first); 250 | } catch (...) { 251 | r.set_exception(std::current_exception()); 252 | } 253 | } 254 | 255 | detail::BatchStatus job_finished() override { 256 | --jobs_outstanding; 257 | if (jobs_outstanding) 258 | return {detail::BatchState::running, 0}; 259 | if (complete) 260 | return {detail::BatchState::complete, complete_in_thread}; 261 | return {detail::BatchState::done, 0}; 262 | } 263 | 264 | void job_completion() override { 265 | return complete(std::move(results)); 266 | } 267 | }; 268 | 269 | // Similar to Batch, but doesn't support a completion function and only handles a single task. 270 | class Job final : private detail::Batch { 271 | friend class OxenMQ; 272 | public: 273 | /// Constructs the Job to run a single task. Takes any callable invokable with no arguments and 274 | /// having no return value. The task will be scheduled and run when the next worker thread is 275 | /// available. Any exceptions thrown by the job will be caught and squelched (the exception 276 | /// terminates/completes the job). 277 | 278 | explicit Job(std::function f, std::optional thread = std::nullopt) 279 | : Job{std::move(f), thread ? thread->_id : 0} 280 | { 281 | if (thread && thread->_id == -1) 282 | // There are some special case internal jobs where we allow this, but they use the 283 | // private ctor below that doesn't have this check. 284 | throw std::logic_error{"Cannot add a proxy thread job -- this makes no sense"}; 285 | } 286 | 287 | // movable 288 | Job(Job&&) = default; 289 | Job &operator=(Job&&) = default; 290 | 291 | // non-copyable 292 | Job(const Job&) = delete; 293 | Job &operator=(const Job&) = delete; 294 | 295 | private: 296 | explicit Job(std::function f, int thread_id) 297 | : job{std::move(f), thread_id} {} 298 | 299 | std::pair, int> job; 300 | bool done = false; 301 | 302 | std::pair size() const override { return {1, job.second != 0}; } 303 | std::vector threads() const override { return {job.second}; } 304 | 305 | void run_job(const int /*i*/) override { 306 | try { job.first(); } 307 | catch (...) {} 308 | } 309 | 310 | detail::BatchStatus job_finished() override { return {detail::BatchState::done, 0}; } 311 | 312 | void job_completion() override {} // Never called because we return ::done (not ::complete) above. 313 | 314 | }; 315 | 316 | template 317 | void OxenMQ::batch(Batch&& batch) { 318 | if (batch.size().first == 0) 319 | throw std::logic_error("Cannot batch a a job batch with 0 jobs"); 320 | // Need to send this over to the proxy thread via the base class pointer. It assumes ownership. 321 | auto* baseptr = static_cast(new Batch(std::move(batch))); 322 | detail::send_control(get_control_socket(), "BATCH", oxenc::bt_serialize(reinterpret_cast(baseptr))); 323 | } 324 | 325 | } 326 | -------------------------------------------------------------------------------- /oxenmq/address.cpp: -------------------------------------------------------------------------------- 1 | #include "address.h" 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | namespace oxenmq { 12 | 13 | constexpr size_t enc_length(address::encoding enc) { 14 | return enc == address::encoding::hex ? 64 : 15 | enc == address::encoding::base64 ? 43 : // this can be 44 with a padding byte, but we don't need it 16 | 52 /*base32z*/; 17 | }; 18 | 19 | // Parses an encoding pubkey from the given string_view. Advanced the string_view beyond the 20 | // consumed pubkey data, and returns the pubkey (as a 32-byte string). Throws if no valid pubkey 21 | // was found at the beginning of addr. We look for hex, base32z, or base64 pubkeys *unless* qr is 22 | // given: for QR-friendly we only accept hex or base32z (since QR cannot handle base64's alphabet). 23 | std::string decode_pubkey(std::string_view& in, bool qr) { 24 | std::string pubkey; 25 | if (in.size() >= 64 && oxenc::is_hex(in.substr(0, 64))) { 26 | pubkey = oxenc::from_hex(in.substr(0, 64)); 27 | in.remove_prefix(64); 28 | } else if (in.size() >= 52 && oxenc::is_base32z(in.substr(0, 52))) { 29 | pubkey = oxenc::from_base32z(in.substr(0, 52)); 30 | in.remove_prefix(52); 31 | } else if (!qr && in.size() >= 43 && oxenc::is_base64(in.substr(0, 43))) { 32 | pubkey = oxenc::from_base64(in.substr(0, 43)); 33 | in.remove_prefix(43); 34 | if (!in.empty() && in.front() == '=') 35 | in.remove_prefix(1); // allow (and eat) a padding byte at the end 36 | } else { 37 | throw std::invalid_argument{"No pubkey found"}; 38 | } 39 | return pubkey; 40 | } 41 | 42 | // Parse the host, port, and optionally pubkey from a string view, mutating it to remove the parsed 43 | // sections. qr should be true if we should accept $IPv6$ as a QR-encoding-friendly alternative to 44 | // [IPv6] (the returned host will have the $ replaced, i.e. [IPv6]). 45 | std::tuple parse_tcp(std::string_view& addr, bool qr, bool expect_pubkey) { 46 | std::tuple result; 47 | auto& host = std::get<0>(result); 48 | if (addr.front() == '[' || (qr && addr.front() == '$')) { // IPv6 addr (though this is far from complete validation) 49 | auto pos = addr.find_first_not_of(":.1234567890abcdefABCDEF", 1); 50 | if (pos == std::string_view::npos) 51 | throw std::invalid_argument("Could not find terminating ] while parsing an IPv6 address"); 52 | if (!(addr[pos] == ']' || (qr && addr[pos] == '$'))) 53 | throw std::invalid_argument{"Expected " + (qr ? "$"s : "]"s) + " to close IPv6 address but found " + std::string(1, addr[pos])}; 54 | host = std::string{addr.substr(0, pos+1)}; 55 | if (qr) { 56 | if (host.front() == '$') 57 | host.front() = '['; 58 | if (host.back() == '$') 59 | host.back() = ']'; 60 | } 61 | addr.remove_prefix(pos+1); 62 | } else { 63 | auto port_pos = addr.find(':'); 64 | if (port_pos == std::string_view::npos) 65 | throw std::invalid_argument{"Could not determine host (no following ':port' found)"}; 66 | if (port_pos == 0) 67 | throw std::invalid_argument{"Host cannot be empty"}; 68 | host = std::string{addr.substr(0, port_pos)}; 69 | addr.remove_prefix(port_pos); 70 | } 71 | 72 | if (qr) 73 | // Lower-case the host because upper case hostnames are ugly 74 | for (char& c : host) 75 | if (c >= 'A' && c <= 'Z') 76 | c = c - 'A' + 'a'; 77 | 78 | if (addr.size() < 2 || addr[0] != ':') 79 | throw std::invalid_argument{"Could not find :port in address string"}; 80 | addr.remove_prefix(1); 81 | auto pos = addr.find_first_not_of("1234567890"); 82 | if (pos == 0) 83 | throw std::invalid_argument{"Could not find numeric port in address string"}; 84 | if (pos == std::string_view::npos) 85 | pos = addr.size(); 86 | size_t processed; 87 | int port_int = std::stoi(std::string{addr.substr(0, pos)}, &processed); 88 | if (port_int == 0 || processed != pos) 89 | throw std::invalid_argument{"Could not parse numeric port in address string"}; 90 | if (port_int < 0 || port_int > std::numeric_limits::max()) 91 | throw std::invalid_argument{"Invalid port: port must be in range 1-65535"}; 92 | std::get<1>(result) = static_cast(port_int); 93 | addr.remove_prefix(pos); 94 | 95 | if (expect_pubkey) { 96 | if (addr.size() < 1 + enc_length(qr ? address::encoding::base32z : address::encoding::base64) 97 | || addr.front() != '/') 98 | throw std::invalid_argument{"Invalid address: expected /PUBKEY after port"}; 99 | addr.remove_prefix(1); 100 | 101 | std::get<2>(result) = decode_pubkey(addr, qr); 102 | if (!addr.empty()) 103 | throw std::invalid_argument{"Invalid address: found unexpected trailing data after pubkey"}; 104 | } else if (!addr.empty()) { 105 | throw std::invalid_argument{"Invalid address: found unexpected trailing data after port"}; 106 | } 107 | 108 | return result; 109 | } 110 | 111 | // Parse the socket path and (possibly) pubkey, mutating it to remove the parsed sections. 112 | // Currently the /pubkey *must* be at the end of the string, but this might not always be the case 113 | // (e.g. we could in the future support query string-like arguments). 114 | std::pair parse_unix(std::string_view& addr, bool expect_pubkey) { 115 | std::pair result; 116 | if (expect_pubkey) { 117 | size_t b64_len = addr.size() > 0 && addr.back() == '=' ? 44 : 43; 118 | if (addr.size() > 64 && addr[addr.size() - 65] == '/' && oxenc::is_hex(addr.substr(addr.size() - 64))) { 119 | result.first = std::string{addr.substr(0, addr.size() - 65)}; 120 | result.second = oxenc::from_hex(addr.substr(addr.size() - 64)); 121 | } else if (addr.size() > 52 && addr[addr.size() - 53] == '/' && oxenc::is_base32z(addr.substr(addr.size() - 52))) { 122 | result.first = std::string{addr.substr(0, addr.size() - 53)}; 123 | result.second = oxenc::from_base32z(addr.substr(addr.size() - 52)); 124 | } else if (addr.size() > b64_len && addr[addr.size() - b64_len - 1] == '/' && oxenc::is_base64(addr.substr(addr.size() - b64_len))) { 125 | result.first = std::string{addr.substr(0, addr.size() - b64_len - 1)}; 126 | result.second = oxenc::from_base64(addr.substr(addr.size() - b64_len)); 127 | } else { 128 | throw std::invalid_argument{"icp+curve:// requires a trailing /PUBKEY value, got: " + std::string{addr}}; 129 | } 130 | } else { 131 | // Anything goes 132 | result.first = std::string{addr}; 133 | } 134 | 135 | // Any path above consumes everything: 136 | addr.remove_prefix(addr.size()); 137 | 138 | return result; 139 | } 140 | 141 | address::address(std::string_view addr) { 142 | auto protoend = addr.find("://"sv); 143 | if (protoend == std::string_view::npos || protoend == 0) 144 | throw std::invalid_argument("Invalid address: no protocol found"); 145 | auto pro = addr.substr(0, protoend); 146 | addr.remove_prefix(protoend + 3); 147 | if (addr.empty()) 148 | throw std::invalid_argument("Invalid address: no value specified after protocol"); 149 | bool qr = false; 150 | if (pro == "tcp") protocol = proto::tcp; 151 | else if (pro == "tcp+curve" || pro == "curve") protocol = proto::tcp_curve; 152 | else if (pro == "ipc") protocol = proto::ipc; 153 | else if (pro == "ipc+curve") protocol = proto::ipc_curve; 154 | else if (pro == "TCP") { 155 | protocol = proto::tcp; 156 | qr = true; 157 | } else if (pro == "CURVE") { 158 | protocol = proto::tcp_curve; 159 | qr = true; 160 | } else { 161 | throw std::invalid_argument("Invalid protocol '" + std::string{pro} + "'"); 162 | } 163 | 164 | if (qr) { 165 | // The QR variations only allow QR-alphanumeric characters (upper-case letters, numbers, and 166 | // a few symbols): 167 | for (char c : addr) { 168 | // QR alphanumeric also allows space, %, *, +, but we don't need or allow any of those here. 169 | if (!((c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '$' || c == ':' || c == '/' || c == '.' || c == '-')) 170 | throw std::invalid_argument("Found non-QR-alphanumeric value in QR TCP:// or CURVE:// address"); 171 | } 172 | } 173 | 174 | if (tcp()) 175 | std::tie(host, port, pubkey) = parse_tcp(addr, qr, curve()); 176 | else 177 | std::tie(socket, pubkey) = parse_unix(addr, curve()); 178 | 179 | if (!addr.empty()) 180 | throw std::invalid_argument{"Invalid trailing garbage '" + std::string{addr} + "' in address"}; 181 | } 182 | 183 | address& address::set_pubkey(std::string_view pk) { 184 | if (pk.size() == 0) { 185 | if (protocol == proto::tcp_curve) protocol = proto::tcp; 186 | else if (protocol == proto::ipc_curve) protocol = proto::ipc; 187 | } else if (pk.size() == 32) { 188 | if (protocol == proto::tcp) protocol = proto::tcp_curve; 189 | else if (protocol == proto::ipc) protocol = proto::ipc_curve; 190 | } else { 191 | throw std::invalid_argument{"Invalid pubkey passed to set_pubkey(): require 0- or 32-byte pubkey"}; 192 | } 193 | pubkey = pk; 194 | return *this; 195 | } 196 | 197 | std::string address::encode_pubkey(encoding enc) const { 198 | std::string pk; 199 | if (enc == encoding::hex) 200 | pk = oxenc::to_hex(pubkey); 201 | else if (enc == encoding::base32z) 202 | pk = oxenc::to_base32z(pubkey); 203 | else if (enc == encoding::BASE32Z) { 204 | pk = oxenc::to_base32z(pubkey); 205 | for (char& c : pk) 206 | if (c >= 'a' && c <= 'z') 207 | c = c - 'a' + 'A'; 208 | } else if (enc == encoding::base64) { 209 | pk = oxenc::to_base64(pubkey); 210 | if (pk.size() == 44 && pk.back() == '=') 211 | pk.resize(43); 212 | } else { 213 | throw std::logic_error{"Invalid encoding"}; 214 | } 215 | return pk; 216 | } 217 | 218 | std::string address::full_address(encoding enc) const { 219 | std::string result; 220 | std::string pk; 221 | if (curve()) 222 | pk = encode_pubkey(enc); 223 | 224 | if (protocol == proto::tcp) { 225 | result.reserve(6 /*tcp:// */ + host.size() + 6 /*:port*/); 226 | result += "tcp://"; 227 | result += host; 228 | result += ':'; 229 | result += std::to_string(port); 230 | } else if (protocol == proto::tcp_curve) { 231 | result.reserve(8 /*curve:// */ + host.size() + 6 /*:port*/ + 1 /* / */ + pk.size()); 232 | result += "curve://"; 233 | result += host; 234 | result += ':'; 235 | result += std::to_string(port); 236 | result += '/'; 237 | result += pk; 238 | } else if (protocol == proto::ipc) { 239 | result.reserve(6 /*ipc:// */ + socket.size()); 240 | result += "ipc://"; 241 | result += socket; 242 | } else if (protocol == proto::ipc_curve) { 243 | result.reserve(12 /*ipc+curve:// */ + socket.size() + 1 /* / */ + pk.size()); 244 | result += "ipc+curve://"; 245 | result += socket; 246 | result += '/'; 247 | result += pk; 248 | } else { 249 | throw std::logic_error{"Invalid protocol"}; 250 | } 251 | 252 | return result; 253 | } 254 | 255 | std::string address::zmq_address() const { 256 | std::string result; 257 | if (tcp()) { 258 | result.reserve(6 /*tcp:// */ + host.size() + 6 /*:port*/); 259 | result += "tcp://"; 260 | result += host; 261 | result += ':'; 262 | result += std::to_string(port); 263 | } else { 264 | result.reserve(6 /*ipc:// */ + socket.size()); 265 | result += "ipc://"; 266 | result += socket; 267 | } 268 | return result; 269 | } 270 | 271 | std::string address::qr_address() const { 272 | if (protocol != proto::tcp && protocol != proto::tcp_curve) 273 | throw std::logic_error("Cannot construct a QR-friendly address for a non-TCP address"); 274 | if (host.empty()) 275 | throw std::logic_error("Cannot construct a QR-friendly address with an empty TCP host"); 276 | std::string result; 277 | result.reserve((curve() ? 8 /*CURVE:// */ : 6 /*TCP:// */) + host.size() + 6 /*:port*/ + 278 | (curve() ? 1 + enc_length(encoding::BASE32Z) : 0)); 279 | result += curve() ? "CURVE://" : "TCP://"; 280 | std::string uc_host = host; 281 | for (auto& c : uc_host) 282 | if (c >= 'a' && c <= 'z') 283 | c = c - 'a' + 'A'; 284 | 285 | if (uc_host.front() == '[' && uc_host.back() == ']') { 286 | uc_host.front() = '$'; 287 | uc_host.back() = '$'; 288 | } 289 | result += uc_host; 290 | result += ':'; 291 | result += std::to_string(port); 292 | 293 | if (curve()) { 294 | result += '/'; 295 | result += encode_pubkey(encoding::BASE32Z); 296 | } 297 | 298 | return result; 299 | } 300 | 301 | bool address::operator==(const address& other) const { 302 | if (protocol != other.protocol) 303 | return false; 304 | if (tcp()) 305 | if (host != other.host || port != other.port) 306 | return false; 307 | if (ipc()) 308 | if (socket != other.socket) 309 | return false; 310 | if (curve()) 311 | if (pubkey != other.pubkey) 312 | return false; 313 | return true; 314 | } 315 | 316 | address address::tcp(std::string host, uint16_t port) { 317 | address a; 318 | a.protocol = proto::tcp; 319 | a.host = std::move(host); 320 | a.port = port; 321 | return a; 322 | } 323 | 324 | address address::tcp_curve(std::string host, uint16_t port, std::string pubkey) { 325 | address a; 326 | a.protocol = proto::tcp_curve; 327 | a.host = std::move(host); 328 | a.port = port; 329 | a.pubkey = std::move(pubkey); 330 | return a; 331 | } 332 | 333 | address address::ipc(std::string path) { 334 | address a; 335 | a.protocol = proto::ipc; 336 | a.socket = std::move(path); 337 | return a; 338 | } 339 | 340 | address address::ipc_curve(std::string path, std::string pubkey) { 341 | address a; 342 | a.protocol = proto::ipc_curve; 343 | a.socket = std::move(path); 344 | a.pubkey = std::move(pubkey); 345 | return a; 346 | } 347 | 348 | } 349 | -------------------------------------------------------------------------------- /oxenmq/auth.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq.h" 2 | #include 3 | #include "oxenmq-internal.h" 4 | #include 5 | #include 6 | 7 | namespace oxenmq { 8 | 9 | namespace { 10 | 11 | // Builds a ZMTP metadata key-value pair. These will be available on every message from that peer. 12 | // Keys must start with X- and be <= 255 characters. 13 | std::string zmtp_metadata(std::string_view key, std::string_view value) { 14 | assert(key.size() > 2 && key.size() <= 255 && key[0] == 'X' && key[1] == '-'); 15 | 16 | std::string result; 17 | result.reserve(1 + key.size() + 4 + value.size()); 18 | result += static_cast(key.size()); // Size octet of key 19 | result.append(&key[0], key.size()); // key data 20 | for (int i = 24; i >= 0; i -= 8) // 4-byte size of value in network order 21 | result += static_cast((value.size() >> i) & 0xff); 22 | result.append(&value[0], value.size()); // value data 23 | 24 | return result; 25 | } 26 | 27 | auto cat = log::Cat("oxenmq"); 28 | 29 | } 30 | 31 | 32 | bool OxenMQ::proxy_check_auth(int64_t conn_id, bool outgoing, const peer_info& peer, 33 | zmq::message_t& cmd, const cat_call_t& cat_call, std::vector& data) { 34 | auto command = view(cmd); 35 | std::string reply; 36 | 37 | if (!cat_call.first) { 38 | log::info(cat, "Invalid command '{}' sent by remote [{}]/{}", command, log_hex(peer.pubkey), peer_address(cmd)); 39 | reply = "UNKNOWNCOMMAND"; 40 | } else if (peer.auth_level < cat_call.first->access.auth) { 41 | log::info(cat, "Access denied to {} for peer [{}]/{}: peer auth level {} < {}", command, log_hex(peer.pubkey), peer_address(cmd), peer.auth_level, cat_call.first->access.auth); 42 | reply = "FORBIDDEN"; 43 | } else if (cat_call.first->access.local_sn && !local_service_node) { 44 | log::info(cat, "Access denied to {} for peer [{}]/{}: that command is only available when this OxenMQ is running in service node mode", command, log_hex(peer.pubkey), peer_address(cmd)); 45 | reply = "NOT_A_SERVICE_NODE"; 46 | } else if (cat_call.first->access.remote_sn && !peer.service_node) { 47 | log::info(cat, "Access denied to {} for peer [{}]/{}: remote is not recognized as a service node", command, log_hex(peer.pubkey), peer_address(cmd)); 48 | reply = "FORBIDDEN_SN"; 49 | } else if (cat_call.second->second /*is_request*/ && data.empty()) { 50 | log::info(cat, "Received an invalid request for '{}' with no reply tag from remote [{}]/{}", command, log_hex(peer.pubkey), peer_address(cmd)); 51 | reply = "NO_REPLY_TAG"; 52 | } else { 53 | return true; 54 | } 55 | 56 | std::vector msgs; 57 | msgs.reserve(4); 58 | if (!outgoing) 59 | msgs.push_back(create_message(peer.route)); 60 | msgs.push_back(create_message(reply)); 61 | if (cat_call.second && cat_call.second->second /*request command*/ && !data.empty()) { 62 | msgs.push_back(create_message("REPLY"sv)); 63 | msgs.push_back(create_message(view(data.front()))); // reply tag 64 | } else { 65 | msgs.push_back(create_message(view(cmd))); 66 | } 67 | 68 | try { 69 | send_message_parts(connections.at(conn_id), msgs); 70 | } catch (const zmq::error_t& err) { 71 | /* can't send: possibly already disconnected. Ignore. */ 72 | log::debug(cat, "Couldn't send auth failure message {} to peer [{}]/{}: {}", reply, log_hex(peer.pubkey), peer_address(cmd), err.what()); 73 | } 74 | 75 | return false; 76 | } 77 | 78 | void OxenMQ::set_active_sns(pubkey_set pubkeys) { 79 | if (proxy_thread.joinable()) { 80 | auto data = oxenc::bt_serialize(detail::serialize_object(std::move(pubkeys))); 81 | detail::send_control(get_control_socket(), "SET_SNS", data); 82 | } else { 83 | proxy_set_active_sns(std::move(pubkeys)); 84 | } 85 | } 86 | void OxenMQ::proxy_set_active_sns(std::string_view data) { 87 | proxy_set_active_sns(detail::deserialize_object(oxenc::bt_deserialize(data))); 88 | } 89 | void OxenMQ::proxy_set_active_sns(pubkey_set pubkeys) { 90 | pubkey_set added, removed; 91 | for (auto it = pubkeys.begin(); it != pubkeys.end(); ) { 92 | auto& pk = *it; 93 | if (pk.size() != 32) { 94 | log::warning(cat, "Invalid private key of length {} ({}) passed to set_active_sns", pk.size(), log_hex(pk)); 95 | it = pubkeys.erase(it); 96 | continue; 97 | } 98 | if (!active_service_nodes.count(pk)) 99 | added.insert(std::move(pk)); 100 | ++it; 101 | } 102 | if (added.empty() && active_service_nodes.size() == pubkeys.size()) { 103 | log::debug(cat, "set_active_sns(): new set of SNs is unchanged, skipping update"); 104 | return; 105 | } 106 | for (const auto& pk : active_service_nodes) { 107 | if (!pubkeys.count(pk)) 108 | removed.insert(pk); 109 | if (active_service_nodes.size() + added.size() - removed.size() == pubkeys.size()) 110 | break; 111 | } 112 | proxy_update_active_sns_clean(std::move(added), std::move(removed)); 113 | } 114 | 115 | void OxenMQ::update_active_sns(pubkey_set added, pubkey_set removed) { 116 | if (proxy_thread.joinable()) { 117 | std::array data; 118 | data[0] = detail::serialize_object(std::move(added)); 119 | data[1] = detail::serialize_object(std::move(removed)); 120 | detail::send_control(get_control_socket(), "UPDATE_SNS", oxenc::bt_serialize(data)); 121 | } else { 122 | proxy_update_active_sns(std::move(added), std::move(removed)); 123 | } 124 | } 125 | void OxenMQ::proxy_update_active_sns(oxenc::bt_list_consumer data) { 126 | auto added = detail::deserialize_object(data.consume_integer()); 127 | auto remed = detail::deserialize_object(data.consume_integer()); 128 | proxy_update_active_sns(std::move(added), std::move(remed)); 129 | } 130 | void OxenMQ::proxy_update_active_sns(pubkey_set added, pubkey_set removed) { 131 | // We take a caller-provided set of added/removed then filter out any junk (bad pks, conflicting 132 | // values, pubkeys that already(added) or do not(removed) exist), then pass the purified lists 133 | // to the _clean version. 134 | 135 | for (auto it = removed.begin(); it != removed.end(); ) { 136 | const auto& pk = *it; 137 | if (pk.size() != 32) { 138 | log::warning(cat, "Invalid private key of length {} ({}) passed to update_active_sns (removed)", pk.size(), log_hex(pk)); 139 | it = removed.erase(it); 140 | } else if (!active_service_nodes.count(pk) || added.count(pk) /* added wins if in both */) { 141 | it = removed.erase(it); 142 | } else { 143 | ++it; 144 | } 145 | } 146 | 147 | for (auto it = added.begin(); it != added.end(); ) { 148 | const auto& pk = *it; 149 | if (pk.size() != 32) { 150 | log::warning(cat, "Invalid private key of length {} ({}) passed to update_active_sns (added)", pk.size(), log_hex(pk)); 151 | it = added.erase(it); 152 | } else if (active_service_nodes.count(pk)) { 153 | it = added.erase(it); 154 | } else { 155 | ++it; 156 | } 157 | } 158 | 159 | proxy_update_active_sns_clean(std::move(added), std::move(removed)); 160 | } 161 | 162 | void OxenMQ::proxy_update_active_sns_clean(pubkey_set added, pubkey_set removed) { 163 | log::debug(cat, "Updating SN auth status with +{}/-{} pubkeys", added.size(), removed.size()); 164 | 165 | // For anything we remove we want close the connection to the SN (if outgoing), and remove the 166 | // stored peer_info (incoming or outgoing). 167 | for (const auto& pk : removed) { 168 | ConnectionID c{pk}; 169 | active_service_nodes.erase(pk); 170 | auto range = peers.equal_range(c); 171 | for (auto it = range.first; it != range.second; ) { 172 | bool outgoing = it->second.outgoing(); 173 | auto conn_id = it->second.conn_id; 174 | it = peers.erase(it); 175 | if (outgoing) { 176 | log::debug(cat, "Closing outgoing connection to {}", c); 177 | proxy_close_connection(conn_id, CLOSE_LINGER); 178 | } 179 | } 180 | } 181 | 182 | // For pubkeys we add there's nothing special to be done beyond adding them to the pubkey set 183 | for (auto& pk : added) 184 | active_service_nodes.insert(std::move(pk)); 185 | } 186 | 187 | void OxenMQ::process_zap_requests() { 188 | for (std::vector frames; recv_message_parts(zap_auth, frames, zmq::recv_flags::dontwait); frames.clear()) { 189 | log::debug(cat, "Processing ZAP authentication request"); 190 | 191 | // https://rfc.zeromq.org/spec:27/ZAP/ 192 | // 193 | // The request message SHALL consist of the following message frames: 194 | // 195 | // The version frame, which SHALL contain the three octets "1.0". 196 | // The request id, which MAY contain an opaque binary blob. 197 | // The domain, which SHALL contain a (non-empty) string. 198 | // The address, the origin network IP address. 199 | // The identity, the connection Identity, if any. 200 | // The mechanism, which SHALL contain a string. 201 | // The credentials, which SHALL be zero or more opaque frames. 202 | // 203 | // The reply message SHALL consist of the following message frames: 204 | // 205 | // The version frame, which SHALL contain the three octets "1.0". 206 | // The request id, which MAY contain an opaque binary blob. 207 | // The status code, which SHALL contain a string. 208 | // The status text, which MAY contain a string. 209 | // The user id, which SHALL contain a string. 210 | // The metadata, which MAY contain a blob. 211 | // 212 | // (NB: there are also null address delimiters at the beginning of each mentioned in the 213 | // RFC, but those have already been removed through the use of a REP socket) 214 | 215 | std::vector response_vals(6); 216 | response_vals[0] = "1.0"; // version 217 | if (frames.size() >= 2) 218 | response_vals[1] = std::string{view(frames[1])}; // unique identifier 219 | std::string &status_code = response_vals[2], &status_text = response_vals[3]; 220 | 221 | if (frames.size() < 6 || view(frames[0]) != "1.0") { 222 | log::error(cat, "Bad ZAP authentication request: version != 1.0 or invalid ZAP message parts"); 223 | status_code = "500"; 224 | status_text = "Internal error: invalid auth request"; 225 | } else { 226 | auto auth_domain = view(frames[2]); 227 | size_t bind_id = (size_t) -1; 228 | try { 229 | bind_id = oxenc::bt_deserialize(view(frames[2])); 230 | } catch (...) {} 231 | 232 | if (bind_id >= bind.size()) { 233 | log::error(cat, "Bad ZAP authentication request: invalid auth domain '{}'", auth_domain); 234 | status_code = "400"; 235 | status_text = "Unknown authentication domain: " + std::string{auth_domain}; 236 | } else if (bind[bind_id].curve 237 | ? !(frames.size() == 7 && view(frames[5]) == "CURVE") 238 | : !(frames.size() == 6 && view(frames[5]) == "NULL")) { 239 | log::error(cat, "Bad ZAP authentication request: invalid {} authentication request", 240 | bind[bind_id].curve ? "CURVE" : "NULL"); 241 | status_code = "500"; 242 | status_text = "Invalid authentication request mechanism"; 243 | } else if (bind[bind_id].curve && frames[6].size() != 32) { 244 | log::error(cat, "Bad ZAP authentication request: invalid request pubkey"); 245 | status_code = "500"; 246 | status_text = "Invalid public key size for CURVE authentication"; 247 | } else { 248 | auto ip = view(frames[3]); 249 | // If we're in dual stack mode IPv4 address might be IPv4-mapped IPv6 address (e.g. 250 | // ::ffff:192.168.0.1); if so, remove the prefix to get a proper IPv4 address: 251 | if (ip.size() >= 14 && ip.substr(0, 7) == "::ffff:"sv && ip.find_last_not_of("0123456789."sv) == 6) 252 | ip = ip.substr(7); 253 | 254 | std::string_view pubkey; 255 | bool sn = false; 256 | if (bind[bind_id].curve) { 257 | pubkey = view(frames[6]); 258 | sn = active_service_nodes.count(std::string{pubkey}); 259 | } 260 | auto auth = bind[bind_id].allow(ip, pubkey, sn); 261 | auto& user_id = response_vals[4]; 262 | if (bind[bind_id].curve) { 263 | user_id.reserve(64); 264 | oxenc::to_hex(pubkey.begin(), pubkey.end(), std::back_inserter(user_id)); 265 | } 266 | 267 | if (auth <= AuthLevel::denied || auth > AuthLevel::admin) { 268 | log::info(cat, "Access denied for incoming {} {} connection from {} at {} with initial auth level {}", 269 | view(frames[5]), 270 | sn ? " service node" : " client", 271 | !user_id.empty() ? user_id : "[nouser]", ip, 272 | auth); 273 | status_code = "400"; 274 | status_text = "Access denied"; 275 | user_id.clear(); 276 | } else { 277 | log::debug(cat, "Accepted incoming {} {} connection with authentication level {} from {} at {}", 278 | view(frames[5]), 279 | sn ? " service node" : " client", 280 | auth, 281 | !user_id.empty() ? user_id : "[nouser]", 282 | ip); 283 | 284 | auto& metadata = response_vals[5]; 285 | metadata += zmtp_metadata("X-AuthLevel", to_string(auth)); 286 | 287 | status_code = "200"; 288 | status_text = ""; 289 | } 290 | } 291 | } 292 | 293 | log::trace(cat, "ZAP request result: {} {}", status_code, status_text); 294 | 295 | std::vector response; 296 | response.reserve(response_vals.size()); 297 | for (auto &r : response_vals) response.push_back(create_message(std::move(r))); 298 | send_message_parts(zap_auth, response.begin(), response.end()); 299 | } 300 | } 301 | 302 | } 303 | -------------------------------------------------------------------------------- /oxenmq/connections.cpp: -------------------------------------------------------------------------------- 1 | #include "oxenmq.h" 2 | #include "oxenmq-internal.h" 3 | #include 4 | #include 5 | #include 6 | 7 | #ifdef OXENMQ_USE_EPOLL 8 | extern "C" { 9 | #include 10 | #include 11 | } 12 | #endif 13 | 14 | namespace oxenmq { 15 | 16 | namespace { 17 | 18 | auto cat = log::Cat("oxenmq"); 19 | 20 | void add_pollitem(std::vector& pollitems, zmq::socket_t& sock) { 21 | pollitems.emplace_back(); 22 | auto &p = pollitems.back(); 23 | p.socket = static_cast(sock); 24 | p.fd = 0; 25 | p.events = ZMQ_POLLIN; 26 | } 27 | 28 | } // anonymous namespace 29 | 30 | void OxenMQ::rebuild_pollitems() { 31 | 32 | #ifdef OXENMQ_USE_EPOLL 33 | if (using_epoll) { 34 | if (epoll_fd != -1) 35 | close(epoll_fd); 36 | epoll_fd = epoll_create1(0); 37 | 38 | struct epoll_event ev; 39 | ev.events = EPOLLIN | EPOLLET; 40 | ev.data.u64 = EPOLL_COMMAND_ID; 41 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, command.get(zmq::sockopt::fd), &ev); 42 | 43 | ev.data.u64 = EPOLL_WORKER_ID; 44 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, workers_socket.get(zmq::sockopt::fd), &ev); 45 | 46 | ev.data.u64 = EPOLL_ZAP_ID; 47 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, zap_auth.get(zmq::sockopt::fd), &ev); 48 | 49 | for (auto& [id, s] : connections) { 50 | ev.data.u64 = id; 51 | epoll_ctl(epoll_fd, EPOLL_CTL_ADD, s.get(zmq::sockopt::fd), &ev); 52 | } 53 | connections_updated = false; 54 | return; 55 | } 56 | #endif 57 | 58 | // No epoll, or epoll not enabled. 59 | pollitems.clear(); 60 | add_pollitem(pollitems, command); 61 | add_pollitem(pollitems, workers_socket); 62 | add_pollitem(pollitems, zap_auth); 63 | 64 | for (auto& [id, s] : connections) 65 | add_pollitem(pollitems, s); 66 | connections_updated = false; 67 | } 68 | 69 | void OxenMQ::setup_external_socket(zmq::socket_t& socket) { 70 | socket.set(zmq::sockopt::reconnect_ivl, (int) RECONNECT_INTERVAL.count()); 71 | socket.set(zmq::sockopt::reconnect_ivl_max, (int) RECONNECT_INTERVAL_MAX.count()); 72 | socket.set(zmq::sockopt::handshake_ivl, (int) HANDSHAKE_TIME.count()); 73 | socket.set(zmq::sockopt::maxmsgsize, MAX_MSG_SIZE); 74 | if (IPV6) 75 | socket.set(zmq::sockopt::ipv6, 1); 76 | 77 | if (CONN_HEARTBEAT > 0s) { 78 | socket.set(zmq::sockopt::heartbeat_ivl, (int) CONN_HEARTBEAT.count()); 79 | if (CONN_HEARTBEAT_TIMEOUT > 0s) 80 | socket.set(zmq::sockopt::heartbeat_timeout, (int) CONN_HEARTBEAT_TIMEOUT.count()); 81 | } 82 | } 83 | 84 | void OxenMQ::setup_outgoing_socket(zmq::socket_t& socket, std::string_view remote_pubkey, bool use_ephemeral_routing_id) { 85 | 86 | setup_external_socket(socket); 87 | 88 | if (!remote_pubkey.empty()) { 89 | socket.set(zmq::sockopt::curve_serverkey, remote_pubkey); 90 | socket.set(zmq::sockopt::curve_publickey, pubkey); 91 | socket.set(zmq::sockopt::curve_secretkey, privkey); 92 | } 93 | 94 | if (!use_ephemeral_routing_id) { 95 | std::string routing_id; 96 | routing_id.reserve(33); 97 | routing_id += 'L'; // Prefix because routing id's starting with \0 are reserved by zmq (and our pubkey might start with \0) 98 | routing_id.append(pubkey.begin(), pubkey.end()); 99 | socket.set(zmq::sockopt::routing_id, routing_id); 100 | } 101 | // else let ZMQ pick a random one 102 | } 103 | 104 | 105 | void OxenMQ::setup_incoming_socket(zmq::socket_t& listener, bool curve, std::string_view pubkey, std::string_view privkey, size_t bind_index) { 106 | 107 | setup_external_socket(listener); 108 | 109 | listener.set(zmq::sockopt::zap_domain, oxenc::bt_serialize(bind_index)); 110 | if (curve) { 111 | listener.set(zmq::sockopt::curve_server, true); 112 | listener.set(zmq::sockopt::curve_publickey, pubkey); 113 | listener.set(zmq::sockopt::curve_secretkey, privkey); 114 | } 115 | listener.set(zmq::sockopt::router_handover, true); 116 | listener.set(zmq::sockopt::router_mandatory, true); 117 | } 118 | 119 | // Deprecated versions: 120 | ConnectionID OxenMQ::connect_remote(std::string_view remote, ConnectSuccess on_connect, 121 | ConnectFailure on_failure, AuthLevel auth_level, std::chrono::milliseconds timeout) { 122 | return connect_remote(address{remote}, std::move(on_connect), std::move(on_failure), 123 | auth_level, connect_option::timeout{timeout}); 124 | } 125 | 126 | ConnectionID OxenMQ::connect_remote(std::string_view remote, ConnectSuccess on_connect, 127 | ConnectFailure on_failure, std::string_view pubkey, AuthLevel auth_level, 128 | std::chrono::milliseconds timeout) { 129 | return connect_remote(address{remote}.set_pubkey(pubkey), std::move(on_connect), 130 | std::move(on_failure), auth_level, connect_option::timeout{timeout}); 131 | } 132 | 133 | void OxenMQ::disconnect(ConnectionID id, std::chrono::milliseconds linger) { 134 | detail::send_control(get_control_socket(), "DISCONNECT", oxenc::bt_serialize({ 135 | {"conn_id", id.id}, 136 | {"linger_ms", linger.count()}, 137 | {"pubkey", id.pk}, 138 | })); 139 | } 140 | 141 | std::pair 142 | OxenMQ::proxy_connect_sn(std::string_view remote, std::string_view connect_hint, bool optional, bool incoming_only, bool outgoing_only, bool use_ephemeral_routing_id, std::chrono::milliseconds keep_alive) { 143 | ConnectionID remote_cid{remote}; 144 | auto its = peers.equal_range(remote_cid); 145 | peer_info* peer = nullptr; 146 | for (auto it = its.first; it != its.second; ++it) { 147 | if (incoming_only && it->second.route.empty()) 148 | continue; // outgoing connection but we were asked to only use incoming connections 149 | if (outgoing_only && !it->second.route.empty()) 150 | continue; 151 | peer = &it->second; 152 | break; 153 | } 154 | 155 | if (peer) { 156 | log::trace(cat, "proxy asked to connect to {}; reusing existing connection", log_hex(remote)); 157 | if (peer->route.empty() /* == outgoing*/) { 158 | if (peer->idle_expiry < keep_alive) { 159 | log::debug(cat, "updating existing outgoing peer connection idle expiry time from {} to {}", peer->idle_expiry, keep_alive); 160 | peer->idle_expiry = keep_alive; 161 | } 162 | peer->activity(); 163 | } 164 | return {&connections[peer->conn_id], peer->route}; 165 | } else if (optional || incoming_only) { 166 | log::debug(cat, "proxy asked for optional or incoming connection, but no appropriate connection exists so aborting connection attempt"); 167 | return {nullptr, ""s}; 168 | } 169 | 170 | // No connection so establish a new one 171 | log::debug(cat, "proxy establishing new outbound connection to {}", log_hex(remote)); 172 | std::string addr; 173 | bool to_self = false && remote == pubkey; // FIXME; need to use a separate listening socket for this, otherwise we can't easily 174 | // tell it wasn't from a remote. 175 | if (to_self) { 176 | // special inproc connection if self that doesn't need any external connection 177 | addr = SN_ADDR_SELF; 178 | } else { 179 | addr = std::string{connect_hint}; 180 | if (addr.empty()) 181 | addr = sn_lookup(remote); 182 | else 183 | log::debug(cat, "using connection hint {}", connect_hint); 184 | 185 | if (addr.empty()) { 186 | log::error(cat, "peer lookup failed for {}", log_hex(remote)); 187 | return {nullptr, ""s}; 188 | } 189 | } 190 | 191 | log::debug(cat, "{} (me) connecting to {} to reach {}", log_hex(pubkey), addr, log_hex(remote)); 192 | std::optional socket; 193 | try { 194 | socket.emplace(context, zmq::socket_type::dealer); 195 | setup_outgoing_socket(*socket, remote, use_ephemeral_routing_id); 196 | socket->connect(addr); 197 | } catch (const zmq::error_t& e) { 198 | // Note that this failure cases indicates something serious went wrong that means zmq isn't 199 | // even going to try connecting (for example an unparseable remote address). 200 | log::error(cat, "Outgoing connection to {} failed: {}", addr, e.what()); 201 | return {nullptr, ""s}; 202 | } 203 | 204 | auto& p = peers.emplace(std::move(remote_cid), peer_info{})->second; 205 | p.service_node = true; 206 | p.pubkey = std::string{remote}; 207 | p.conn_id = next_conn_id++; 208 | p.idle_expiry = keep_alive; 209 | p.activity(); 210 | connections_updated = true; 211 | outgoing_sn_conns.emplace_hint(outgoing_sn_conns.end(), p.conn_id, ConnectionID{remote}); 212 | auto it = connections.emplace_hint(connections.end(), p.conn_id, *std::move(socket)); 213 | 214 | return {&it->second, ""s}; 215 | } 216 | 217 | std::pair OxenMQ::proxy_connect_sn(oxenc::bt_dict_consumer data) { 218 | std::string_view hint, remote_pk; 219 | std::chrono::milliseconds keep_alive; 220 | bool optional = false, incoming_only = false, outgoing_only = false, ephemeral_rid = EPHEMERAL_ROUTING_ID; 221 | 222 | // Alphabetical order 223 | if (data.skip_until("ephemeral_rid")) 224 | ephemeral_rid = data.consume_integer(); 225 | if (data.skip_until("hint")) 226 | hint = data.consume_string_view(); 227 | if (data.skip_until("incoming")) 228 | incoming_only = data.consume_integer(); 229 | if (data.skip_until("keep_alive")) 230 | keep_alive = std::chrono::milliseconds{data.consume_integer()}; 231 | if (data.skip_until("optional")) 232 | optional = data.consume_integer(); 233 | if (data.skip_until("outgoing_only")) 234 | outgoing_only = data.consume_integer(); 235 | if (!data.skip_until("pubkey")) 236 | throw std::runtime_error("Internal error: Invalid proxy_connect_sn command; pubkey missing"); 237 | remote_pk = data.consume_string_view(); 238 | 239 | return proxy_connect_sn(remote_pk, hint, optional, incoming_only, outgoing_only, ephemeral_rid, keep_alive); 240 | } 241 | 242 | /// Closes outgoing connections and removes all references. Note that this will call `erase()` 243 | /// which can invalidate iterators on the various connection containers - if you don't want that, 244 | /// delete it first so that the container won't contain the element being deleted. 245 | void OxenMQ::proxy_close_connection(int64_t id, std::chrono::milliseconds linger) { 246 | auto it = connections.find(id); 247 | if (it == connections.end()) { 248 | log::warning(cat, "internal error: connection to close ({}) doesn't exist!", id); 249 | return; 250 | } 251 | log::debug(cat, "Closing conn {}", id); 252 | it->second.set(zmq::sockopt::linger, linger > 0ms ? (int) linger.count() : 0); 253 | connections.erase(it); 254 | connections_updated = true; 255 | 256 | outgoing_sn_conns.erase(id); 257 | } 258 | 259 | void OxenMQ::proxy_expire_idle_peers() { 260 | for (auto it = peers.begin(); it != peers.end(); ) { 261 | auto &info = it->second; 262 | if (info.outgoing()) { 263 | auto idle = std::chrono::steady_clock::now() - info.last_activity; 264 | if (idle > info.idle_expiry) { 265 | log::debug( 266 | cat, 267 | "Closing outgoing connection to {}: idle time ({}) reached connection " 268 | "timeout ({})", 269 | it->first, 270 | std::chrono::duration_cast(idle), 271 | info.idle_expiry); 272 | proxy_close_connection(info.conn_id, CLOSE_LINGER); 273 | it = peers.erase(it); 274 | } else { 275 | log::trace( 276 | cat, 277 | "Not closing {}: {} <= {}", 278 | it->first, 279 | std::chrono::duration_cast(idle), 280 | info.idle_expiry); 281 | ++it; 282 | continue; 283 | } 284 | } else { 285 | ++it; 286 | } 287 | } 288 | } 289 | 290 | void OxenMQ::proxy_conn_cleanup() { 291 | log::trace(cat, "starting proxy connections cleanup"); 292 | 293 | // Drop idle connections (if we haven't done it in a while) 294 | log::trace(cat, "closing idle connections"); 295 | proxy_expire_idle_peers(); 296 | 297 | auto now = std::chrono::steady_clock::now(); 298 | 299 | // FIXME - check other outgoing connections to see if they died and if so purge them 300 | 301 | log::trace(cat, "Timing out pending outgoing connections"); 302 | // Check any pending outgoing connections for timeout 303 | for (auto it = pending_connects.begin(); it != pending_connects.end(); ) { 304 | auto& pc = *it; 305 | if (std::get(pc) < now) { 306 | auto id = std::get(pc); 307 | ConnectionID cid{id}; 308 | job([cid, callback = std::move(std::get(pc))] { callback(cid, "connection attempt timed out"); }); 309 | it = pending_connects.erase(it); // Don't let the below erase it (because it invalidates iterators) 310 | proxy_close_connection(id, 0ms); 311 | peers.erase(cid); 312 | } else { 313 | ++it; 314 | } 315 | } 316 | 317 | log::trace(cat, "Timing out pending requests"); 318 | // Remove any expired pending requests and schedule their callback with a failure 319 | for (auto it = pending_requests.begin(); it != pending_requests.end(); ) { 320 | auto& callback = it->second; 321 | if (callback.first < now) { 322 | log::debug(cat, "pending request {} expired, invoking callback with failure status and removing", log_hex(it->first)); 323 | job([callback = std::move(callback.second)] { callback(false, {{"TIMEOUT"s}}); }); 324 | it = pending_requests.erase(it); 325 | } else { 326 | ++it; 327 | } 328 | } 329 | 330 | log::trace(cat, "done proxy connections cleanup"); 331 | }; 332 | 333 | void OxenMQ::proxy_connect_remote(oxenc::bt_dict_consumer data) { 334 | AuthLevel auth_level = AuthLevel::none; 335 | long long conn_id = -1; 336 | ConnectSuccess on_connect; 337 | ConnectFailure on_failure; 338 | std::string remote; 339 | std::string remote_pubkey; 340 | std::chrono::milliseconds timeout = REMOTE_CONNECT_TIMEOUT; 341 | bool ephemeral_rid = EPHEMERAL_ROUTING_ID; 342 | 343 | if (data.skip_until("auth_level")) 344 | auth_level = static_cast(data.consume_integer>()); 345 | if (data.skip_until("conn_id")) 346 | conn_id = data.consume_integer(); 347 | if (data.skip_until("connect")) 348 | on_connect = detail::deserialize_object(data.consume_integer()); 349 | if (data.skip_until("ephemeral_rid")) 350 | ephemeral_rid = data.consume_integer(); 351 | if (data.skip_until("failure")) 352 | on_failure = detail::deserialize_object(data.consume_integer()); 353 | if (data.skip_until("pubkey")) { 354 | remote_pubkey = data.consume_string(); 355 | assert(remote_pubkey.size() == 32 || remote_pubkey.empty()); 356 | } 357 | if (data.skip_until("remote")) 358 | remote = data.consume_string(); 359 | if (data.skip_until("timeout")) 360 | timeout = std::chrono::milliseconds{data.consume_integer()}; 361 | 362 | if (conn_id == -1 || remote.empty()) 363 | throw std::runtime_error("Internal error: CONNECT_REMOTE proxy command missing required 'conn_id' and/or 'remote' value"); 364 | 365 | log::trace( 366 | cat, 367 | "Establishing remote connection to {} {}{}", 368 | remote, 369 | remote_pubkey.empty() ? "(NULL auth)" : "via CURVE expecting pubkey ", 370 | log_hex(remote_pubkey)); 371 | 372 | std::optional sock; 373 | try { 374 | sock.emplace(context, zmq::socket_type::dealer); 375 | setup_outgoing_socket(*sock, remote_pubkey, ephemeral_rid); 376 | sock->connect(remote); 377 | } catch (const zmq::error_t &e) { 378 | proxy_schedule_reply_job([conn_id, on_failure=std::move(on_failure), what="connect() failed: "s+e.what()] { 379 | on_failure(conn_id, std::move(what)); 380 | }); 381 | return; 382 | } 383 | 384 | auto &s = connections.emplace_hint(connections.end(), conn_id, std::move(*sock))->second; 385 | connections_updated = true; 386 | log::debug(cat, "Opened new zmq socket to {}", remote, ", conn_id ", conn_id, "; sending HI"); 387 | send_direct_message(s, "HI"); 388 | pending_connects.emplace_back(conn_id, std::chrono::steady_clock::now() + timeout, 389 | std::move(on_connect), std::move(on_failure)); 390 | auto& peer = peers.emplace(ConnectionID{conn_id, remote_pubkey}, peer_info{})->second; 391 | peer.pubkey = std::move(remote_pubkey); 392 | peer.service_node = false; 393 | peer.auth_level = auth_level; 394 | peer.conn_id = conn_id; 395 | peer.idle_expiry = 24h * 10 * 365; // "forever" 396 | peer.activity(); 397 | } 398 | 399 | void OxenMQ::proxy_disconnect(oxenc::bt_dict_consumer data) { 400 | ConnectionID connid{-1}; 401 | std::chrono::milliseconds linger = 1s; 402 | 403 | if (data.skip_until("conn_id")) 404 | connid.id = data.consume_integer(); 405 | if (data.skip_until("linger_ms")) 406 | linger = std::chrono::milliseconds(data.consume_integer()); 407 | if (data.skip_until("pubkey")) 408 | connid.pk = data.consume_string(); 409 | 410 | if (connid.sn() && connid.pk.size() != 32) 411 | throw std::runtime_error("Error: invalid disconnect of SN without a valid pubkey"); 412 | 413 | proxy_disconnect(std::move(connid), linger); 414 | } 415 | void OxenMQ::proxy_disconnect(ConnectionID conn, std::chrono::milliseconds linger) { 416 | log::trace(cat, "Disconnecting outgoing connection to {}", conn); 417 | auto pr = peers.equal_range(conn); 418 | for (auto it = pr.first; it != pr.second; ++it) { 419 | auto& peer = it->second; 420 | if (peer.outgoing()) { 421 | log::debug(cat, "Closing outgoing connection to {}", conn); 422 | proxy_close_connection(peer.conn_id, linger); 423 | peers.erase(it); 424 | return; 425 | } 426 | } 427 | log::warning(cat, "Failed to disconnect {}: no such outgoing connection", conn); 428 | } 429 | 430 | std::string ConnectionID::to_string() const { 431 | std::string result; 432 | if (!pk.empty()) { 433 | if (sn()) 434 | return fmt::format("SN {}", oxenc::to_hex(pk)); 435 | return fmt::format("non-SN authenticated remote [{}] {}", id, oxenc::to_hex(pk)); 436 | } 437 | return fmt::format("unauthenticated remote [{}]", id); 438 | } 439 | 440 | } 441 | -------------------------------------------------------------------------------- /tests/test_pubsub.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "oxenmq/pubsub.h" 3 | #include 4 | 5 | #include 6 | 7 | using namespace oxenmq; 8 | using namespace std::chrono_literals; 9 | 10 | static auto cat = oxen::log::Cat("oxenmq.test"); 11 | 12 | TEST_CASE("sub OK", "[pubsub]") { 13 | std::string listen = random_localhost(); 14 | OxenMQ server{ 15 | "", "", // generate ephemeral keys 16 | false, // not a service node 17 | [](auto) { return ""; }, 18 | }; 19 | server.listen_curve(listen); 20 | 21 | Subscription<> greetings{"greetings"}; 22 | 23 | std::atomic is_new{false}; 24 | server.add_category("public", Access{AuthLevel::none}); 25 | server.add_request_command("public", "greetings", [&](Message& m) { 26 | is_new = greetings.subscribe(m.conn); 27 | m.send_reply("OK"); 28 | }); 29 | server.start(); 30 | 31 | OxenMQ client{}; 32 | 33 | std::atomic reply_count{0}; 34 | client.add_category("notify", Access{AuthLevel::none}); 35 | client.add_command("notify", "greetings", [&](Message& m) { 36 | const auto& data = m.data; 37 | if (!data.size()) 38 | { 39 | oxen::log::error(cat, "client received public.greetings with empty data"); 40 | return; 41 | } 42 | if (data[0] == "hello") 43 | reply_count++; 44 | }); 45 | 46 | client.start(); 47 | 48 | std::atomic connected{false}, failed{false}; 49 | std::string pubkey; 50 | 51 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 52 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 53 | [&](auto, auto) { failed = true; }); 54 | 55 | wait_for([&] { return connected || failed; }); 56 | { 57 | auto lock = catch_lock(); 58 | REQUIRE( connected ); 59 | REQUIRE_FALSE( failed ); 60 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 61 | } 62 | 63 | std::atomic got_reply{false}; 64 | bool success; 65 | std::vector data; 66 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 67 | got_reply = true; 68 | success = ok; 69 | data = std::move(data_); 70 | }); 71 | 72 | reply_sleep(); 73 | { 74 | auto lock = catch_lock(); 75 | REQUIRE( got_reply.load() ); 76 | REQUIRE( success ); 77 | REQUIRE( data == std::vector{{"OK"}} ); 78 | } 79 | 80 | greetings.publish([&](auto& conn) { 81 | server.send(conn, "notify.greetings", "hello"); 82 | }); 83 | 84 | reply_sleep(); 85 | { 86 | auto lock = catch_lock(); 87 | REQUIRE( reply_count == 1 ); 88 | } 89 | 90 | greetings.publish([&](auto& conn) { 91 | server.send(conn, "notify.greetings", "hello"); 92 | }); 93 | 94 | reply_sleep(); 95 | { 96 | auto lock = catch_lock(); 97 | REQUIRE( reply_count == 2 ); 98 | } 99 | 100 | } 101 | 102 | TEST_CASE("user data", "[pubsub]") { 103 | std::string listen = random_localhost(); 104 | OxenMQ server{ 105 | "", "", // generate ephemeral keys 106 | false, // not a service node 107 | [](auto) { return ""; }, 108 | }; 109 | server.listen_curve(listen); 110 | 111 | Subscription greetings{"greetings"}; 112 | 113 | std::atomic is_new{false}; 114 | server.add_category("public", Access{AuthLevel::none}); 115 | server.add_request_command("public", "greetings", [&](Message& m) { 116 | is_new = greetings.subscribe(m.conn, std::string{m.data[0]}); 117 | m.send_reply("OK"); 118 | }); 119 | server.start(); 120 | 121 | OxenMQ client{}; 122 | 123 | std::string response{"foo"}; 124 | std::atomic reply_count{0}; 125 | std::atomic foo_count{0}; 126 | client.add_category("notify", Access{AuthLevel::none}); 127 | client.add_command("notify", "greetings", [&](Message& m) { 128 | const auto& data = m.data; 129 | if (!data.size()) 130 | { 131 | oxen::log::error(cat, "client received public.greetings with empty data"); 132 | return; 133 | } 134 | if (data[0] == response) 135 | reply_count++; 136 | if (data[0] == "foo") 137 | foo_count++; 138 | }); 139 | 140 | client.start(); 141 | 142 | std::atomic connected{false}, failed{false}; 143 | std::string pubkey; 144 | 145 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 146 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 147 | [&](auto, auto) { failed = true; }); 148 | 149 | wait_for([&] { return connected || failed; }); 150 | { 151 | auto lock = catch_lock(); 152 | REQUIRE( connected ); 153 | REQUIRE_FALSE( failed ); 154 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 155 | } 156 | 157 | std::atomic got_reply{false}; 158 | std::atomic success; 159 | std::vector data; 160 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 161 | got_reply = true; 162 | success = ok; 163 | data = std::move(data_); 164 | }, response); 165 | 166 | reply_sleep(); 167 | { 168 | auto lock = catch_lock(); 169 | REQUIRE( got_reply.load() ); 170 | REQUIRE( success ); 171 | REQUIRE( is_new ); 172 | REQUIRE( data == std::vector{{"OK"}} ); 173 | } 174 | 175 | got_reply = false; 176 | success = false; 177 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 178 | got_reply = true; 179 | success = ok; 180 | data = std::move(data_); 181 | }, response); 182 | 183 | reply_sleep(); 184 | { 185 | auto lock = catch_lock(); 186 | REQUIRE( got_reply.load() ); 187 | REQUIRE( success ); 188 | REQUIRE_FALSE( is_new ); 189 | REQUIRE( data == std::vector{{"OK"}} ); 190 | } 191 | 192 | greetings.publish([&](auto& conn, std::string user) { 193 | server.send(conn, "notify.greetings", user); 194 | }); 195 | 196 | reply_sleep(); 197 | { 198 | auto lock = catch_lock(); 199 | REQUIRE( reply_count == 1 ); 200 | REQUIRE( foo_count == 1 ); 201 | } 202 | 203 | got_reply = false; 204 | success = false; 205 | response = "bar"; 206 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 207 | got_reply = true; 208 | success = ok; 209 | data = std::move(data_); 210 | }, response); 211 | 212 | reply_sleep(); 213 | { 214 | auto lock = catch_lock(); 215 | REQUIRE( got_reply.load() ); 216 | REQUIRE( success ); 217 | REQUIRE( is_new ); 218 | REQUIRE( data == std::vector{{"OK"}} ); 219 | } 220 | 221 | greetings.publish([&](auto& conn, std::string user) { 222 | server.send(conn, "notify.greetings", user); 223 | }); 224 | 225 | reply_sleep(); 226 | { 227 | auto lock = catch_lock(); 228 | REQUIRE( reply_count == 2 ); 229 | REQUIRE( foo_count == 1 ); 230 | } 231 | 232 | } 233 | 234 | TEST_CASE("unsubscribe", "[pubsub]") { 235 | std::string listen = random_localhost(); 236 | OxenMQ server{ 237 | "", "", // generate ephemeral keys 238 | false, // not a service node 239 | [](auto) { return ""; }, 240 | }; 241 | server.listen_curve(listen); 242 | 243 | Subscription<> greetings{"greetings"}; 244 | 245 | std::atomic was_subbed{false}; 246 | server.add_category("public", Access{AuthLevel::none}); 247 | server.add_request_command("public", "greetings", [&](Message& m) { 248 | greetings.subscribe(m.conn); 249 | m.send_reply("OK"); 250 | }); 251 | server.add_request_command("public", "goodbye", [&](Message& m) { 252 | was_subbed = greetings.unsubscribe(m.conn); 253 | m.send_reply("OK"); 254 | }); 255 | server.start(); 256 | 257 | OxenMQ client{}; 258 | 259 | std::atomic reply_count{0}; 260 | client.add_category("notify", Access{AuthLevel::none}); 261 | client.add_command("notify", "greetings", [&](Message& m) { 262 | const auto& data = m.data; 263 | if (!data.size()) 264 | { 265 | oxen::log::error(cat, "client received public.greetings with empty data"); 266 | return; 267 | } 268 | if (data[0] == "hello") 269 | reply_count++; 270 | }); 271 | 272 | client.start(); 273 | 274 | std::atomic connected{false}, failed{false}; 275 | std::string pubkey; 276 | 277 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 278 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 279 | [&](auto, auto) { failed = true; }); 280 | 281 | wait_for([&] { return connected || failed; }); 282 | { 283 | auto lock = catch_lock(); 284 | REQUIRE( connected ); 285 | REQUIRE_FALSE( failed ); 286 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 287 | } 288 | 289 | std::atomic got_reply{false}; 290 | std::atomic success; 291 | std::vector data; 292 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 293 | got_reply = true; 294 | success = ok; 295 | data = std::move(data_); 296 | }); 297 | 298 | reply_sleep(); 299 | { 300 | auto lock = catch_lock(); 301 | REQUIRE( got_reply.load() ); 302 | REQUIRE( success ); 303 | REQUIRE( data == std::vector{{"OK"}} ); 304 | } 305 | 306 | greetings.publish([&](auto& conn) { 307 | server.send(conn, "notify.greetings", "hello"); 308 | }); 309 | 310 | reply_sleep(); 311 | { 312 | auto lock = catch_lock(); 313 | REQUIRE( reply_count == 1 ); 314 | } 315 | 316 | got_reply = false; 317 | success = false; 318 | client.request(c, "public.goodbye", [&](bool ok, std::vector data_) { 319 | got_reply = true; 320 | success = ok; 321 | data = std::move(data_); 322 | }); 323 | 324 | reply_sleep(); 325 | { 326 | auto lock = catch_lock(); 327 | REQUIRE( got_reply.load() ); 328 | REQUIRE( success ); 329 | REQUIRE( data == std::vector{{"OK"}} ); 330 | REQUIRE( was_subbed ); 331 | } 332 | 333 | greetings.publish([&](auto& conn) { 334 | server.send(conn, "notify.greetings", "hello"); 335 | }); 336 | 337 | reply_sleep(); 338 | { 339 | auto lock = catch_lock(); 340 | REQUIRE( reply_count == 1 ); 341 | } 342 | 343 | got_reply = false; 344 | success = false; 345 | client.request(c, "public.goodbye", [&](bool ok, std::vector data_) { 346 | got_reply = true; 347 | success = ok; 348 | data = std::move(data_); 349 | }); 350 | 351 | reply_sleep(); 352 | { 353 | auto lock = catch_lock(); 354 | REQUIRE( got_reply.load() ); 355 | REQUIRE( success ); 356 | REQUIRE( data == std::vector{{"OK"}} ); 357 | REQUIRE( was_subbed == false); 358 | } 359 | 360 | } 361 | 362 | TEST_CASE("expire", "[pubsub]") { 363 | std::string listen = random_localhost(); 364 | OxenMQ server{ 365 | "", "", // generate ephemeral keys 366 | false, // not a service node 367 | [](auto) { return ""; }, 368 | }; 369 | server.listen_curve(listen); 370 | 371 | Subscription<> greetings{"greetings", 250ms}; 372 | 373 | std::atomic was_subbed{false}; 374 | server.add_category("public", Access{AuthLevel::none}); 375 | server.add_request_command("public", "greetings", [&](Message& m) { 376 | greetings.subscribe(m.conn); 377 | m.send_reply("OK"); 378 | }); 379 | server.add_request_command("public", "goodbye", [&](Message& m) { 380 | was_subbed = greetings.unsubscribe(m.conn); 381 | m.send_reply("OK"); 382 | }); 383 | server.start(); 384 | 385 | OxenMQ client{}; 386 | 387 | std::atomic reply_count{0}; 388 | client.add_category("notify", Access{AuthLevel::none}); 389 | client.add_command("notify", "greetings", [&](Message& m) { 390 | const auto& data = m.data; 391 | if (!data.size()) 392 | { 393 | oxen::log::error(cat, "client received public.greetings with empty data"); 394 | return; 395 | } 396 | if (data[0] == "hello") 397 | reply_count++; 398 | }); 399 | 400 | client.start(); 401 | 402 | std::atomic connected{false}, failed{false}; 403 | std::string pubkey; 404 | 405 | auto c = client.connect_remote(address{listen, server.get_pubkey()}, 406 | [&](auto conn) { pubkey = conn.pubkey(); connected = true; }, 407 | [&](auto, auto) { failed = true; }); 408 | 409 | wait_for([&] { return connected || failed; }); 410 | { 411 | auto lock = catch_lock(); 412 | REQUIRE( connected ); 413 | REQUIRE_FALSE( failed ); 414 | REQUIRE( oxenc::to_hex(pubkey) == oxenc::to_hex(server.get_pubkey()) ); 415 | } 416 | 417 | std::atomic got_reply{false}; 418 | bool success; 419 | std::vector data; 420 | client.request(c, "public.greetings", [&](bool ok, std::vector data_) { 421 | got_reply = true; 422 | success = ok; 423 | data = std::move(data_); 424 | }); 425 | 426 | reply_sleep(); 427 | { 428 | auto lock = catch_lock(); 429 | REQUIRE( got_reply.load() ); 430 | REQUIRE( success ); 431 | REQUIRE( data == std::vector{{"OK"}} ); 432 | } 433 | 434 | // should be expired by now 435 | std::this_thread::sleep_for(500ms); 436 | 437 | greetings.remove_expired(); 438 | 439 | got_reply = false; 440 | success = false; 441 | client.request(c, "public.goodbye", [&](bool ok, std::vector data_) { 442 | got_reply = true; 443 | success = ok; 444 | data = std::move(data_); 445 | }); 446 | 447 | reply_sleep(); 448 | { 449 | auto lock = catch_lock(); 450 | REQUIRE( got_reply.load() ); 451 | REQUIRE( success ); 452 | REQUIRE( data == std::vector{{"OK"}} ); 453 | REQUIRE( was_subbed == false); 454 | } 455 | 456 | } 457 | 458 | TEST_CASE("multiple subs", "[pubsub]") { 459 | std::string listen = random_localhost(); 460 | OxenMQ server{ 461 | "", "", // generate ephemeral keys 462 | false, // not a service node 463 | [](auto) { return ""; }, 464 | }; 465 | server.listen_curve(listen); 466 | 467 | Subscription<> greetings{"greetings"}; 468 | 469 | std::atomic is_new{false}; 470 | server.add_category("public", Access{AuthLevel::none}); 471 | server.add_request_command("public", "greetings", [&](Message& m) { 472 | is_new = greetings.subscribe(m.conn); 473 | m.send_reply("OK"); 474 | }); 475 | server.start(); 476 | 477 | /* client 1 */ 478 | std::atomic reply_count_c1{0}; 479 | std::atomic connected_c1{false}, failed_c1{false}; 480 | std::atomic got_reply_c1{false}; 481 | bool success_c1; 482 | std::vector data_c1; 483 | std::string pubkey_c1; 484 | OxenMQ client1{}; 485 | 486 | client1.add_category("notify", Access{AuthLevel::none}); 487 | client1.add_command("notify", "greetings", [&](Message& m) { 488 | const auto& data = m.data; 489 | if (!data.size()) 490 | { 491 | oxen::log::error(cat, "client received public.greetings with empty data"); 492 | return; 493 | } 494 | if (data[0] == "hello") 495 | reply_count_c1++; 496 | }); 497 | 498 | client1.start(); 499 | 500 | auto c1 = client1.connect_remote(address{listen, server.get_pubkey()}, 501 | [&](auto conn) { pubkey_c1 = conn.pubkey(); connected_c1 = true; }, 502 | [&](auto, auto) { failed_c1 = true; }); 503 | 504 | wait_for([&] { return connected_c1 || failed_c1; }); 505 | { 506 | auto lock = catch_lock(); 507 | REQUIRE( connected_c1 ); 508 | REQUIRE_FALSE( failed_c1 ); 509 | REQUIRE( oxenc::to_hex(pubkey_c1) == oxenc::to_hex(server.get_pubkey()) ); 510 | } 511 | 512 | client1.request(c1, "public.greetings", [&](bool ok, std::vector data_) { 513 | got_reply_c1 = true; 514 | success_c1 = ok; 515 | data_c1 = std::move(data_); 516 | }); 517 | 518 | reply_sleep(); 519 | { 520 | auto lock = catch_lock(); 521 | REQUIRE( got_reply_c1.load() ); 522 | REQUIRE( success_c1 ); 523 | REQUIRE( data_c1 == std::vector{{"OK"}} ); 524 | } 525 | /* end client 1 */ 526 | 527 | /* client 2 */ 528 | std::atomic reply_count_c2{0}; 529 | std::atomic connected_c2{false}, failed_c2{false}; 530 | std::atomic got_reply_c2{false}; 531 | bool success_c2; 532 | std::vector data_c2; 533 | std::string pubkey_c2; 534 | OxenMQ client2{}; 535 | 536 | client2.add_category("notify", Access{AuthLevel::none}); 537 | client2.add_command("notify", "greetings", [&](Message& m) { 538 | const auto& data = m.data; 539 | if (!data.size()) 540 | { 541 | oxen::log::error(cat, "client received public.greetings with empty data"); 542 | return; 543 | } 544 | if (data[0] == "hello") 545 | reply_count_c2++; 546 | }); 547 | 548 | client2.start(); 549 | 550 | auto c2 = client2.connect_remote(address{listen, server.get_pubkey()}, 551 | [&](auto conn) { pubkey_c2 = conn.pubkey(); connected_c2 = true; }, 552 | [&](auto, auto) { failed_c2 = true; }); 553 | 554 | wait_for([&] { return connected_c2 || failed_c2; }); 555 | { 556 | auto lock = catch_lock(); 557 | REQUIRE( connected_c2 ); 558 | REQUIRE_FALSE( failed_c2 ); 559 | REQUIRE( oxenc::to_hex(pubkey_c2) == oxenc::to_hex(server.get_pubkey()) ); 560 | } 561 | 562 | client2.request(c2, "public.greetings", [&](bool ok, std::vector data_) { 563 | got_reply_c2 = true; 564 | success_c2 = ok; 565 | data_c2 = std::move(data_); 566 | }); 567 | 568 | reply_sleep(); 569 | { 570 | auto lock = catch_lock(); 571 | REQUIRE( got_reply_c2.load() ); 572 | REQUIRE( success_c2 ); 573 | REQUIRE( data_c2 == std::vector{{"OK"}} ); 574 | } 575 | /* end client2 */ 576 | 577 | greetings.publish([&](auto& conn) { 578 | server.send(conn, "notify.greetings", "hello"); 579 | }); 580 | 581 | reply_sleep(); 582 | { 583 | auto lock = catch_lock(); 584 | REQUIRE( reply_count_c1 == 1 ); 585 | REQUIRE( reply_count_c2 == 1 ); 586 | } 587 | 588 | greetings.publish([&](auto& conn) { 589 | server.send(conn, "notify.greetings", "hello"); 590 | }); 591 | 592 | reply_sleep(); 593 | { 594 | auto lock = catch_lock(); 595 | REQUIRE( reply_count_c1 == 2 ); 596 | REQUIRE( reply_count_c2 == 2 ); 597 | } 598 | 599 | } 600 | 601 | 602 | // vim:sw=4:et 603 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # OxenMQ - high-level zeromq-based message passing for network-based projects 2 | 3 | This C++17 library contains an abstraction layer around ZeroMQ to provide a high-level interface to 4 | authentication, RPC, and message passing. It is used extensively within Oxen projects (hence the 5 | name) as the underlying communication mechanism of SN-to-SN communication ("quorumnet"), the RPC 6 | interface used by wallets and local daemon commands, communication channels between oxend and 7 | auxiliary services (storage server, lokinet), and also provides local multithreaded job scheduling 8 | within a process. 9 | 10 | Messages channels can be encrypted (using x25519) or not -- however opening an encrypted channel 11 | requires knowing the server pubkey. Within Oxen, all SN-to-SN traffic is encrypted, and other 12 | traffic can be encrypted as needed. 13 | 14 | This library makes minimal use of mutexes, and none in the hot paths of the code, instead mostly 15 | relying on ZMQ sockets for synchronization; for more information on this (and why this is generally 16 | much better performing and more scalable) see the ZMQ guide documentation on the topic. 17 | 18 | ## Basic message structure 19 | 20 | OxenMQ messages come in two fundamental forms: "commands", consisting of a command named and 21 | optional arguments, and "requests", consisting of a request name, a request tag, and optional 22 | arguments. 23 | 24 | All channels are capable of bidirectional communication, and multiple messages can be in transit in 25 | either direction at any time. OxenMQ sets up a "listener" and "client" connections, but these only 26 | determine how connections are established: once established, commands can be issued by either party. 27 | 28 | The command/request string is one of two types: 29 | 30 | `category.command` - for commands/requests registered by the OxenMQ caller (e.g. oxend). Here 31 | `category` must be at least one character not containing a `.` and `command` may be anything. These 32 | categories and commands are registered according to general function and authentication level (more 33 | on this below). For example, for oxend categories are: 34 | 35 | - `system` - is for RPC commands related to the system administration such as mining, getting 36 | sensitive statistics, accessing SN private keys, remote shutdown, etc. 37 | - `sn` - is for SN-to-SN communication such as blink quorum and uptime proof obligation votes. 38 | - `blink` - is for public blink commands (i.e. blink submission) and is only provided by nodes 39 | running as service nodes. 40 | - `blockchain` - is for remote blockchain access such as retrieving blocks and transactions as well 41 | as subscribing to updates for new blocks, transactions, and service node states. 42 | 43 | The difference between a request and a command is that a request includes an additional opaque tag 44 | value which is used to identify a reply. For example you could register a `general.backwards` 45 | request that takes a string that receives a reply containing that string reversed. When invoking 46 | the request via OxenMQ you provide a callback to be invoked when the reply arrives. On the wire 47 | this looks like: 48 | 49 | <<< [general.backwards] [v71.&a] [hello world] 50 | >>> [REPLY] [v71.&a] [dlrow olleh] 51 | 52 | where each [] denotes a message part and `v71.&a` is a unique randomly generated identifier handled 53 | by OxenMQ (both the invoker and the recipient code only see the `hello world`/`dlrow olleh` message 54 | parts). 55 | 56 | In contrast, regular registered commands have no identifier or expected reply callback. For example 57 | you could register a `general.pong` commands that takes an argument and prints it out. So requests 58 | and output would look like this: 59 | 60 | >>> [general.pong] [hi] 61 | hi 62 | >>> [general.pong] [there] 63 | there 64 | 65 | You could also create a `ping` command that instructs someone to pong you with a random word -- i.e. 66 | give him a ping and she sends you a pong command: 67 | 68 | <<< [general.ping] 69 | >>> [general.pong] [omg] 70 | omg 71 | 72 | Although this *looks* like a reply it isn't quite the same because there is no connection between 73 | the ping and the pong (and, as above, pongs can be issued directly). In particular this means if 74 | you send multiple pings to the same recipient: 75 | 76 | <<< [general.ping] 77 | <<< [general.ping] 78 | >>> [general.pong] [world] 79 | >>> [general.pong] [hello] 80 | 81 | you would have no way to know whether the first pong is in reply to the first or second ping. We 82 | could amend this to include a number to be echoed back: 83 | 84 | <<< [general.ping] [1] 85 | <<< [general.ping] [2] 86 | >>> [general.pong] [2] [world] 87 | >>> [general.pong] [1] [hello] 88 | 89 | and now, in the pong, we could keep track of which number goes with which outgoing ping. This is 90 | the basic idea behind using a reply instead of command, except that you don't register the `pong` 91 | command at all (there is a generic "REPLY" command for all replies), and the index values are 92 | handled for you transparently. 93 | 94 | ## Command arguments 95 | 96 | Optional command/request arguments are always strings on the wire. The OxenMQ-using developer is 97 | free to create whatever encoding she wants, and these can vary across commands. For example 98 | `wallet.tx` might be a request that returns a transaction in binary, while `wallet.tx_info` might 99 | return tx metadata in JSON, and `p2p.send_tx` might encode tx data and metadata in a bt-encoded 100 | data string. 101 | 102 | No structure at all is imposed on message data to allow maximum flexibility; it is entirely up to 103 | the calling code to handle all encoding/decoding duties. 104 | 105 | Internal commands passed between OxenMQ-managed threads use either plain strings or bt-encoded 106 | dictionaries. See `oxenmq/bt_serialize.h` if you want a bt serializer/deserializer. 107 | 108 | ## Sending commands 109 | 110 | Sending a command to a peer is done by using a connection ID, and generally falls into either a 111 | `send()` method or a `request()` method. 112 | 113 | omq.send(conn, "category.command", "some data"); 114 | omq.request(conn, "category.command", [](bool success, std::vector data) { 115 | if (success) { std::cout << "Remote replied: " << data.at(0) << "\n"; } }); 116 | 117 | The connection ID generally has two possible values: 118 | 119 | - a string containing a service node pubkey. In this mode OxenMQ will look for the given SN in 120 | already-established connections, reusing a connection if one exists. If no connection already 121 | exists, a new connection to the given SN is attempted (this requires constructing the OxenMQ 122 | object with a callback to determine SN remote addresses). 123 | - a ConnectionID object, typically returned by the `connect_remote` method (although there are other 124 | places to get one, such as from the `Message` object passed to a command: see the following 125 | section). 126 | 127 | ```C++ 128 | // Send to a service node, establishing a connection if necessary: 129 | std::string my_sn = ...; // 32-byte pubkey of a known SN 130 | omq.send(my_sn, "sn.explode", "{ \"seconds\": 30 }"); 131 | 132 | // Connect to a remote by address then send it something 133 | auto conn = omq.connect_remote("tcp://127.0.0.1:4567", 134 | [](ConnectionID c) { std::cout << "Connected!\n"; }, 135 | [](ConnectionID c, string_view f) { std::cout << "Connect failed: " << f << "\n" }); 136 | omq.request(conn, "rpc.get_height", [](bool s, std::vector d) { 137 | if (s && d.size() == 1) 138 | std::cout << "Current height: " << d[0] << "\n"; 139 | else 140 | std::cout << "Timeout fetching height!"; 141 | }); 142 | ``` 143 | 144 | ## Command invocation 145 | 146 | The application registers categories and registers commands within these categories with callbacks. 147 | The callbacks are passed a OxenMQ::Message object from which the message (plus various connection 148 | information) can be obtained. There is no structure imposed at all on the data passed in subsequent 149 | message parts: it is up to the command itself to deserialize however it wishes (e.g. JSON, 150 | bt-encoded, or any other encoding). 151 | 152 | The Message object also provides methods for replying to the caller. Simple replies queue a reply 153 | if the client is still connected. Replies to service nodes can also be "strong" replies: when 154 | replying to a SN that has closed connection with a strong reply we will attempt to reestablish a 155 | connection to deliver the message. In order for this to work the OxenMQ caller must provide a 156 | lookup function to retrieve the remote address given a SN x25519 pubkey. 157 | 158 | ### Callbacks 159 | 160 | Invoked command functions are always invoked with exactly one arguments: a non-const OxenMQ::Message 161 | reference from which the connection info, OxenMQ object, and message data can be obtained. 162 | 163 | The Message object also contains a `ConnectionID` object as the public `conn` member; it is safe to 164 | take a copy of this and then use it later to send commands to this peer. (For example, a wallet 165 | might issue a command to a node requesting that it be sent any new transactions that arrive; the 166 | node could store a copy of the ConnectionID, then use these copies when any such transaction 167 | arrives). 168 | 169 | ## Authentication 170 | 171 | Each category has access control consisting of three values: 172 | 173 | - Auth level, one of: 174 | - None - no authentication required at all, any remote client may invoke this command 175 | - Basic - this requires a basic authentication level (None access is implied) 176 | - Admin - this requires administrative access (Basic access is implied) 177 | - ServiceNode (bool) - if true this requires that the remote connection has proven its identity as 178 | an active service node (via its x25519 key). 179 | - LocalServiceNode (bool) - if true this requires that the local node is running in service node 180 | mode (note that it is *not* required that the local SN be *active*). 181 | 182 | Authentication level components are cumulative: for example, a category with Basic auth + 183 | ServiceNode=true + LocalServiceNode=true would only be access if all three conditions are met. 184 | 185 | The authentication mechanism works in two ways: defaults based on configuration, and explicit 186 | logins. 187 | 188 | Configuration defaults allows controlling the default access for an incoming connection based on its 189 | remote address. Typically this is used to allow connections from localhost (or a unix domain 190 | socket) to automatically be an Admin connection without requiring explicit authentication. This 191 | also allows configuration of how public connections should be treated: for example, an oxend running 192 | as a public RPC server would do so by granting Basic access to all incoming connections. 193 | 194 | Explicit logins allow the daemon to specify username/passwords with mapping to Basic or Admin 195 | authentication levels. 196 | 197 | Thus, for example, a daemon could be configured to be allow Basic remote access with authentication 198 | (i.e. requiring a username/password login given out to people who should be able to access). 199 | 200 | For example, in oxend the categories described above have authentication levels of: 201 | 202 | - `system` - Admin 203 | - `sn` - ServiceNode 204 | - `blink` - LocalServiceNode 205 | - `blockchain` - Basic 206 | 207 | ### Service Node authentication 208 | 209 | In order to handle ServiceNode authentication, OxenMQ uses an Allow callback invoked during 210 | connection to determine both whether to allow the connection, and to determine whether the incoming 211 | connection is an active service node. 212 | 213 | Note that this status persists for the life of the connection (i.e. it is not rechecked on each 214 | command invocation). If you require stronger protection against being called by 215 | decommissioned/deregistered service nodes from a connection established when the SN was active then 216 | the callback itself will need to verify when invoked. 217 | 218 | ## Command aliases 219 | 220 | Command aliases can be specified. For example, an alias `a.b` -> `c.d` will rewrite any incoming 221 | `a.b` command to `c.d` before handling it. These are applied *before* any authentication is 222 | performed. 223 | 224 | The main purpose here is for backwards compatibility either for renaming category or commands, or 225 | for changing command access levels by moving it from one category to another. It's recommended that 226 | such aliases be used only temporarily for version transitions. 227 | 228 | ## Threads 229 | 230 | OxenMQ operates a pool of worker threads to handle jobs. The simplest use just allocates new jobs 231 | to a free worker thread, and we have a "general threads" value to configure how many such threads 232 | are available. 233 | 234 | You may, however, also reserve a minimum number of workers per command category. For example, you 235 | could reserve 1 thread for the `sys` category and 2 for the `qnet` category plus 8 general threads. 236 | The general threads will be used most of the time for any categories (including `sys` and `qnet`), 237 | but there will always be at least 1/2 worker threads either currently working on or available for 238 | incoming `system`/`sn` commands. General thread gets used first; only if all general threads are 239 | currently busy *and* a category has unused reserved threads will an additional thread be used. 240 | 241 | Note that these actual reserved threads are not exclusive: reserving M of N total threads for a 242 | category simply ensures that no more than (N-M) threads are being used for other categories at any 243 | given time, but the actual jobs may run on any worker thread. 244 | 245 | As mentioned above, OxenMQ tries to avoid exceeding the configured general threads value (G) 246 | whenever possible: the only time we will dispatch a job to a worker thread when we have >= G threads 247 | already running is when a new command arrives, the category reserves M threads, and the thread pool 248 | is currently processing fewer than M jobs for that category. 249 | 250 | Some examples: assume A and B are commands that take sufficiently long to run that we receive all 251 | commands before the first job is finished. Suppose that A's category reserves 2 threads, B's 252 | category has no reserved threads, and we have 4 general threads configured. 253 | 254 | Example 1: commands arrive in order AABBBB. This will not exceed 4 threads: when the third B 255 | arrives there are already 4 jobs running so it gets queued. 256 | 257 | Example 2: commands arrive in order AABBAA. This also won't exceed 4 threads: when the third A 258 | arrives there are already 4 jobs running and two of them are already A jobs, so there are no 259 | remaining slots for A jobs. 260 | 261 | Example 3: BBBAAA. This won't exceed 5 threads: the first four get started normally. When the 262 | second A arrives there are 4 threads running, but only 1 of them is an A thus there is still a free 263 | slot for A jobs so we start the second A on a fifth thread. The third A, however, has no A jobs 264 | available so gets queued. 265 | 266 | Example 4: BBBBBBAAA. At most 6 jobs. The 5th and 6th B's get queued (all general workers are 267 | busy). The first and second get started (there are two unused reserved A slots), the third one gets 268 | queued. The 5th and 6th B's are already interesting on their own: they won't be started until there 269 | are only three active jobs; the third A won't be started until *either* there are only three active 270 | jobs, or one of the other A's finish. 271 | 272 | Thus the general thread count should be regarded as the "normal" thread limit and reserved threads 273 | allow an extra burst of thread activity *only if* all general threads are busy with other categories 274 | when a command with reserve threads arrived. 275 | 276 | ## Internal batch jobs 277 | 278 | A common pattern is one where a single thread suddenly has some work that can be be parallelized. 279 | You could employ some blocking, locking, mutex + condition variable monstrosity, but you shouldn't. 280 | 281 | Instead OxenMQ provides a mechanism for this by allowing you to submit a batch of jobs with a 282 | completion callback. All jobs will be queued and, when the last one finishes, the finalization 283 | callback will be queued to continue with the task. 284 | 285 | These batch jobs are quite different from ordinary network commands, as described above: they have 286 | no authentication and can only be submitted by the program itself to its own worker threads. They 287 | share worker threads with all other commands, as described above, but have their own separate 288 | reserved thread value (for all intents and purposes this works just like a category reserved count). 289 | 290 | From the caller point of view this requires splitting the logic into two parts, a "Before" that sets 291 | up the batch, a "Job" that does the work (multiple times), and an "After" that continues once all 292 | jobs are finished. 293 | 294 | For example, the following example shows how you might use it to convert from input values (0 to 49) 295 | to some other output value: 296 | 297 | ```C++ 298 | double do_my_task(int input) { 299 | if (input % 10 == 7) 300 | throw std::domain_error("I don't do '7s, sorry"); 301 | if (input == 1) 302 | return 5.0; 303 | return 3.0 * input; 304 | } 305 | 306 | void continue_big_task(std::vector> results) { 307 | double sum = 0; 308 | for (auto& r : results) { 309 | try { 310 | sum += r.get(); 311 | } catch (const std::exception& e) { 312 | std::cout << "Oh noes! " << e.what() << "\n"; 313 | } 314 | } 315 | std::cout << "All done, sum = " << sum << "\n"; 316 | 317 | // Output: 318 | // Oh noes! I don't do '7s, sorry 319 | // Oh noes! I don't do '7s, sorry 320 | // Oh noes! I don't do '7s, sorry 321 | // All done, sum = 1337 322 | } 323 | 324 | void start_big_task() { 325 | size_t num_jobs = 32; 326 | 327 | oxenmq::Batch batch; 328 | batch.reserve(num_jobs); 329 | 330 | for (size_t i = 0; i < num_jobs; i++) 331 | batch.add_job([i]() { return do_my_task(i); }); 332 | 333 | batch.completion(&continue_big_task); 334 | 335 | omq.batch(std::move(batch)); 336 | // ... to be continued in `continue_big_task` after all the jobs finish 337 | 338 | // Can do other things here, but note that continue_big_task could run 339 | // *before* anything else here finishes. 340 | } 341 | ``` 342 | 343 | This code deliberately does not support blocking to wait for the tasks to finish: if you want such a 344 | poor design (which is a recipe for deadlocks: imagine jobs that queuing other jobs that can end up 345 | exhausting the worker threads with waiting jobs) then you can implement it yourself; OxenMQ isn't 346 | going to help you hurt yourself like that. 347 | 348 | ### Single-job queuing 349 | 350 | As a shortcut there is a `omq.job(...)` method that schedules a single task (with no return value) 351 | in the batch job queue. This is useful when some event requires triggering some other event, but 352 | you don't need to wait for or collect its result. (Internally this is just a convenience method 353 | around creating a single-job, no-completion Batch job). 354 | 355 | You generally do *not* want to use single (or batched) jobs for operations that can block 356 | indefinitely (such as network operations) because each such job waiting on some external condition 357 | ties up a thread. You are better off in such a case using your own asynchronous operations and 358 | either using your own thread or a periodic timer (see below) to shepherd those operations. 359 | 360 | ## Timers 361 | 362 | OxenMQ supports scheduling periodic tasks via the `add_timer()` function. These timers have an 363 | interval and are scheduled as (single-job) batches when the timer fires. They also support 364 | "squelching" (enabled by default) that supresses the job being scheduled if a previously scheduled 365 | job is already scheduled or running. 366 | --------------------------------------------------------------------------------