├── .gitignore ├── .gitmodules ├── CMakeLists.txt ├── LICENSE ├── README.md ├── broker.hpp ├── consts.h ├── consumer.hpp ├── doc └── design_draft.md ├── entity.hpp ├── file_storage.hpp ├── itoa_jeaiii.hpp ├── main.cpp ├── mem_storage.hpp ├── producer.hpp ├── qimq.vcxproj ├── qimq.vcxproj.filters ├── storage.hpp └── test ├── CMakeLists.txt ├── doctest ├── doctest.hpp └── tests │ └── demo_test.cpp ├── main.cpp ├── test_pull_ack ├── main.cpp └── test_pull_ack.vcxproj ├── test_pull_msg ├── CMakeLists.txt └── main.cpp └── test_send_msg ├── CMakeLists.txt └── main.cpp /.gitignore: -------------------------------------------------------------------------------- 1 | # Prerequisites 2 | *.d 3 | 4 | # Compiled Object files 5 | *.slo 6 | *.lo 7 | *.o 8 | *.obj 9 | 10 | # Precompiled Headers 11 | *.gch 12 | *.pch 13 | 14 | # Compiled Dynamic libraries 15 | *.so 16 | *.dylib 17 | *.dll 18 | 19 | # Fortran module files 20 | *.mod 21 | *.smod 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.exe 31 | *.out 32 | *.app 33 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "rest_rpc"] 2 | path = rest_rpc 3 | url = https://github.com/qicosmos/rest_rpc.git 4 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(qimq) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++17") 5 | 6 | include_directories(${Boost_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} 7 | "rest_rpc/include" 8 | "rest_rpc/third/msgpack/include" 9 | ) 10 | 11 | add_executable(qimq main.cpp) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 qicosmos 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # qimq 2 | a modern c++17 high performance, cross-platform distributed mq 3 | -------------------------------------------------------------------------------- /broker.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "entity.hpp" 5 | #include "mem_storage.hpp" 6 | #include "file_storage.hpp" 7 | using namespace rest_rpc; 8 | using namespace rpc_service; 9 | 10 | namespace qimq { 11 | template 12 | class broker_t { 13 | public: 14 | broker_t(short port, size_t size, size_t timeout_seconds = 15, 15 | size_t check_seconds = 10): rpc_server_(port, size, timeout_seconds), worker_(ios_){ 16 | init(); 17 | } 18 | 19 | ~broker_t() { 20 | ios_.stop(); 21 | wait_thd_.join(); 22 | } 23 | 24 | void run() { 25 | rpc_server_.run(); 26 | } 27 | 28 | send_result send_msg(rpc_conn conn, int64_t msg_id, std::string val) { 29 | auto code = store_.add(msg_id_, val); 30 | send_result result{ code, msg_id_++ }; 31 | return result; 32 | } 33 | 34 | std::string pull(rpc_conn conn, int64_t msg_id, std::string client_id) { 35 | auto& val = store_.get(msg_id); 36 | 37 | start_wait_ack(conn, msg_id, std::move(client_id), val); 38 | 39 | return val; 40 | } 41 | 42 | void start_wait_ack(rpc_conn conn, int64_t msg_id, std::string client_id, const std::string& val) { 43 | std::shared_ptr wait = std::make_shared(ios_, 5000); 44 | std::weak_ptr wp(wait); 45 | { 46 | std::unique_lock lock(wait_mtx_); 47 | wait_ack_map_.emplace(msg_id, wait); 48 | } 49 | 50 | wait->set_callback([this, conn, wp, msg_id, &val, client_id = std::move(client_id)](int try_times) { 51 | if (try_times <= 5) { 52 | auto call = wp.lock(); 53 | if (call) { 54 | std::cout << "time out, try times: " << try_times << " " << call.use_count() << "\n"; 55 | auto c = conn.lock(); 56 | if (c) { 57 | //retry to push message to the client 58 | rpc_server_.publish_by_token("pull", std::move(client_id), send_result{ok, msg_id, val}); 59 | call->start_timer(); 60 | } 61 | } 62 | } 63 | else { 64 | std::cout << "retry 5 times failed\n"; 65 | } 66 | }); 67 | wait->start_timer(); 68 | } 69 | 70 | void pull_ack(rpc_conn conn, int64_t msg_id, consume_result result) { 71 | decltype(wait_ack_map_.begin()) it; 72 | 73 | { 74 | std::unique_lock lock(wait_mtx_); 75 | it = wait_ack_map_.find(msg_id); 76 | if (it == wait_ack_map_.end()) { 77 | //the msg_id is not exist 78 | return; 79 | } 80 | } 81 | 82 | auto wait = it->second.lock(); 83 | wait->cancel(); 84 | 85 | { 86 | std::unique_lock lock(wait_mtx_); 87 | wait_ack_map_.erase(it); 88 | } 89 | 90 | if (result.code != 0) { 91 | std::cout << "pull error\n"; 92 | } 93 | std::cout << "msg_id: " << msg_id << " has been got by consumer\n"; 94 | } 95 | 96 | void consume_ack(rpc_conn conn, int64_t msg_id, consume_result result) { 97 | if (result.code != 0) { 98 | std::cout << "consume error\n"; 99 | } 100 | 101 | std::cout << "msg has been consumed\n"; 102 | 103 | bool r = store_.remove(msg_id); 104 | if (r) { 105 | std::cout << "remove msg_id: "<< msg_id <<" ok\n"; 106 | } 107 | else { 108 | std::cout << "remove msg_id: " << msg_id << " failed\n"; 109 | std::cout << "remove msg_id: " << msg_id << " failed\n"; 110 | } 111 | } 112 | private: 113 | void init() { 114 | rpc_server_.register_handler("send_msg", &broker_t::send_msg, this); 115 | rpc_server_.register_handler("pull", &broker_t::pull, this); 116 | rpc_server_.register_handler("pull_ack", &broker_t::pull_ack, this); 117 | rpc_server_.register_handler("consume_ack", &broker_t::consume_ack, this); 118 | 119 | wait_thd_ = std::thread([this] { 120 | ios_.run(); 121 | 122 | std::cout << "quit\n"; 123 | }); 124 | } 125 | 126 | class wait_t : asio::noncopyable, public std::enable_shared_from_this { 127 | public: 128 | wait_t(asio::io_service& ios, size_t timeout) : timer_(ios), 129 | timeout_(timeout) { 130 | } 131 | ~wait_t() { 132 | std::cout << "destruct\n"; 133 | } 134 | void start_timer() { 135 | if (timeout_ == 0) { 136 | return; 137 | } 138 | 139 | timer_.expires_from_now(std::chrono::milliseconds(timeout_)); 140 | auto self = this->shared_from_this(); 141 | timer_.async_wait([this, self](boost::system::error_code ec) { 142 | if (ec) { 143 | callback(false); 144 | return; 145 | } 146 | 147 | has_timeout_ = true; 148 | callback(true); 149 | }); 150 | } 151 | 152 | void set_callback(std::function callback) { 153 | cb_ = std::move(callback); 154 | } 155 | 156 | bool has_timeout() const { 157 | return has_timeout_; 158 | } 159 | 160 | void cancel() { 161 | if (timeout_ == 0) { 162 | return; 163 | } 164 | 165 | boost::system::error_code ec; 166 | timer_.cancel(ec); 167 | } 168 | 169 | private: 170 | void callback(bool has_timeout) { 171 | if (!has_timeout) 172 | return; 173 | 174 | try_times_++; 175 | 176 | if (cb_) { 177 | cb_(try_times_); 178 | } 179 | } 180 | 181 | boost::asio::steady_timer timer_; 182 | std::function cb_; 183 | size_t timeout_; 184 | bool has_timeout_ = false; 185 | int try_times_ = 0; 186 | }; 187 | 188 | rpc_server rpc_server_; 189 | T store_; 190 | std::atomic msg_id_ = store_.empty() ? 0 : store_.size(); 191 | boost::asio::io_service ios_; 192 | boost::asio::io_service::work worker_; 193 | std::thread wait_thd_; 194 | std::unordered_map> wait_ack_map_; 195 | std::shared_mutex wait_mtx_; 196 | }; 197 | } -------------------------------------------------------------------------------- /consts.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | namespace qimq { 4 | static std::string EMPTY_STR = ""; 5 | } -------------------------------------------------------------------------------- /consumer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | namespace qimq { 3 | 4 | } -------------------------------------------------------------------------------- /doc/design_draft.md: -------------------------------------------------------------------------------- 1 | 本文是对美团的《[消息队列设计精要](https://tech.meituan.com/2016/07/01/mq-design.html)》一文的总结，并认可文中的rpc消息队列的设计思想，对原作者的分享表示感谢。我在这里将总结分享给大家，也希望和大家一起讨论消息队列的技术细节！ 2 | 3 | ## 最简单的消息队列是一个消息转发器和两次RPC： 4 | 1. producer发送消息给broker的RPC 5 | 2. consumer消费确认的RPC 6 | 3. broker将消息落地后再转发到接受者 7 | 8 | 把这三个环节解决了，消息队列就可以实现了。 9 | 10 | ## 需要解决的技术要点： 11 | 12 | **1.producer向broker的可靠消息投递** 13 | 14 | producer发送消息之前先在本地落地，broker收到消息之后也要先落地，之后再告诉producer发送成功，producer再删除本地消息。如果没有收到发送成功或者失败的结果就定时重发（重发次数要做限制），保证消息一定送达。 15 | 16 | **2.broker持久化收到的消息** 17 | 18 | broker收到消息后先持久化，这个持久化可以是文件系统，分布式kv或数据库。broker收到消费确认之后再从持久化存储中移除消息，这里的移除是逻辑上的移除，不是真正的移除，便于消息回溯。持久化消息也便于做消息队列的重启恢复。 19 | 20 | **3.消费确认** 21 | 22 | 把消息的送达和消息的处理分开，简单说需要消费者两次确认，一次送达，一次处理完成。 23 | 24 | 消费者主动ACK处理结果，如果需要重发还可以约定重发的时间。 25 | 26 | **4.重复消息的处理** 27 | 28 | 主要是鉴别重复消息和减少重复投递 29 | 30 | broker记录message id, 直到消息被消费后后清除，重复id到来不处理； 31 | 32 | broker投递到consumer的消息在需要重发之前询问consumer消息处理完了吗，如果询问无果再重发 33 | 34 | **5.幂等性** 35 | 36 | 由于有重复消息，所以要解决幂等性的问题。一个简单的方法是共享存储，broker多机器共享一个DB或者一个分布式文件/kv系统，则处理消息自然是幂等的。 37 | 38 | **6.高可用** 39 | 40 | 主备或者raft保证高可用 -------------------------------------------------------------------------------- /entity.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | 4 | namespace qimq { 5 | struct send_result { 6 | int code; 7 | int64_t msg_id; 8 | std::string data; 9 | std::string msg; 10 | 11 | MSGPACK_DEFINE(code, msg_id, data, msg); 12 | }; 13 | 14 | //struct message_t { 15 | // int64_t msg_id; 16 | // std::string msg; 17 | // MSGPACK_DEFINE(msg_id, msg); 18 | //}; 19 | 20 | using pull_result = send_result; 21 | 22 | using consume_result = send_result; 23 | 24 | enum error_code { 25 | ok = 0, 26 | has_exist, 27 | add_failed, 28 | at_capacity, 29 | 30 | }; 31 | } -------------------------------------------------------------------------------- /file_storage.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include "storage.hpp" 3 | #include "consts.h" 4 | #ifdef USE_ROCKSDB 5 | #include "rocksdb/db.h" 6 | #include "rocksdb/slice.h" 7 | #include "rocksdb/options.h" 8 | 9 | using namespace rocksdb; 10 | #endif // USE_ROCKSDB 11 | 12 | #include "itoa_jeaiii.hpp" 13 | 14 | namespace qimq { 15 | #ifdef USE_ROCKSDB 16 | class file_storage : public storage_t { 17 | public: 18 | ~file_storage() { 19 | delete db_; 20 | } 21 | 22 | file_storage() { 23 | Options options; 24 | // Optimize RocksDB. This is the easiest way to get RocksDB to perform well 25 | options.IncreaseParallelism(); 26 | //options.OptimizeLevelStyleCompaction(); 27 | // create the DB if it's not already present 28 | options.create_if_missing = true; 29 | 30 | // open DB 31 | Status s = DB::Open(options, "qimq_store", &db_); 32 | if (!s.ok()) { 33 | std::cout << s.ToString() << "\n"; 34 | } 35 | } 36 | 37 | [[nodiscard]] error_code add(const int64_t& key, std::string val) { 38 | char temp[20]; 39 | i64toa_jeaiii(key, temp); 40 | 41 | std::string value; 42 | auto s1 = db_->Get(ReadOptions(), temp, &value); 43 | if (!s1.IsNotFound()) { 44 | return error_code::has_exist; 45 | } 46 | 47 | auto s = db_->Put(w_options_, temp, std::move(val)); 48 | return s.ok() ? error_code::ok : error_code::add_failed; 49 | } 50 | 51 | std::string& get(const int64_t& key) { 52 | char temp[20]; 53 | i64toa_jeaiii(key, temp); 54 | 55 | std::string value; 56 | auto s1 = db_->Get(ReadOptions(), temp, &value); 57 | if (s1.IsNotFound()) { 58 | return EMPTY_STR; 59 | } 60 | 61 | return value; 62 | } 63 | 64 | [[nodiscard]] bool remove(const int64_t& key) { 65 | char temp[20]; 66 | i64toa_jeaiii(key, temp); 67 | 68 | auto s = db_->Delete(w_options_, temp); 69 | return s.ok(); 70 | } 71 | 72 | [[nodiscard]] int64_t size() { 73 | int64_t size = 0; 74 | rocksdb::Iterator* it = db_->NewIterator(r_options_); 75 | for (it->SeekToFirst(); it->Valid(); it->Next()) { 76 | size++; 77 | //cout << it->key().ToString() << ": " << it->value().ToString() << endl; 78 | } 79 | 80 | delete it; 81 | return it->status().ok() ? size : -1; 82 | } 83 | 84 | [[nodiscard]] bool empty() { 85 | rocksdb::Iterator* it = db_->NewIterator(r_options_); 86 | it->SeekToFirst(); 87 | bool is_empty = !(it->Valid()); 88 | delete it; 89 | return is_empty; 90 | } 91 | 92 | [[nodiscard]] bool has(const int64_t& key) { 93 | char temp[20]; 94 | i64toa_jeaiii(key, temp); 95 | 96 | std::string value; 97 | auto s = db_->Get(ReadOptions(), temp, &value); 98 | return !s.IsNotFound(); 99 | } 100 | 101 | private: 102 | //std::shared_mutex mtx_; 103 | DB* db_; 104 | const WriteOptions w_options_{}; 105 | const rocksdb::ReadOptions r_options_{}; 106 | }; 107 | #endif // USE_ROCKSDB 108 | } -------------------------------------------------------------------------------- /itoa_jeaiii.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright (c) 2017 James Edward Anhalt III - https://github.com/jeaiii/itoa 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | #include 26 | 27 | // form a 4.32 fixed point number: t = u * 2^32 / 10^log10(u) 28 | // use as much precision as possible when needed (log10(u) >= 5) 29 | // so shift up then down afterwards by log10(u) * log2(10) ~= 53/16 30 | // need to round up before and or after in some cases 31 | // once we have the fixed point number we can read off the digit in the upper 32 bits 32 | // and multiply the lower 32 bits by 10 to get the next digit and so on 33 | // we can do 2 digits at a time by multiplying by 100 each time 34 | 35 | // TODO: 36 | // x64 optimized verison (no need to line up on 32bit boundary, so can multiply by 5 instead of 10 using lea instruction) 37 | // full 64 bit LG() 38 | // try splitting the number into chucks that can be processed independently 39 | // try odd digit first 40 | // try writing 4 chars at a time 41 | 42 | #if 0 43 | // 1 char at a time 44 | 45 | #define W(N, I) b[N] = char(I) + '0' 46 | #define A(N) t = (uint64_t(1) << (32 + N / 5 * N * 53 / 16)) / uint32_t(1e##N) + 1 - N / 9, t *= u, t >>= N / 5 * N * 53 / 16, t += N / 5 * 4, W(0, t >> 32) 47 | #define D(N) t = uint64_t(10) * uint32_t(t), W(N, t >> 32) 48 | 49 | #define L0 W(0, u) 50 | #define L1 A(1), D(1) 51 | #define L2 A(2), D(1), D(2) 52 | #define L3 A(3), D(1), D(2), D(3) 53 | #define L4 A(4), D(1), D(2), D(3), D(4) 54 | #define L5 A(5), D(1), D(2), D(3), D(4), D(5) 55 | #define L6 A(6), D(1), D(2), D(3), D(4), D(5), D(6) 56 | #define L7 A(7), D(1), D(2), D(3), D(4), D(5), D(6), D(7) 57 | #define L8 A(8), D(1), D(2), D(3), D(4), D(5), D(6), D(7), D(8) 58 | #define L9 A(9), D(1), D(2), D(3), D(4), D(5), D(6), D(7), D(8), D(9) 59 | 60 | #else 61 | // 2 chars at a time 62 | 63 | struct pair { char t, o; }; 64 | #define P(T) T, '0', T, '1', T, '2', T, '3', T, '4', T, '5', T, '6', T, '7', T, '8', T, '9' 65 | static const pair s_pairs[] = { P('0'), P('1'), P('2'), P('3'), P('4'), P('5'), P('6'), P('7'), P('8'), P('9') }; 66 | 67 | #define W(N, I) *(pair*)&b[N] = s_pairs[I] 68 | #define A(N) t = (uint64_t(1) << (32 + N / 5 * N * 53 / 16)) / uint32_t(1e##N) + 1 + N/6 - N/8, t *= u, t >>= N / 5 * N * 53 / 16, t += N / 6 * 4, W(0, t >> 32) 69 | #define S(N) b[N] = char(uint64_t(10) * uint32_t(t) >> 32) + '0' 70 | #define D(N) t = uint64_t(100) * uint32_t(t), W(N, t >> 32) 71 | 72 | #define L0 b[0] = char(u) + '0' 73 | #define L1 W(0, u) 74 | #define L2 A(1), S(2) 75 | #define L3 A(2), D(2) 76 | #define L4 A(3), D(2), S(4) 77 | #define L5 A(4), D(2), D(4) 78 | #define L6 A(5), D(2), D(4), S(6) 79 | #define L7 A(6), D(2), D(4), D(6) 80 | #define L8 A(7), D(2), D(4), D(6), S(8) 81 | #define L9 A(8), D(2), D(4), D(6), D(8) 82 | 83 | #endif 84 | 85 | #define LN(N) (L##N, b += N + 1) 86 | #define LZ LN 87 | // if you want to '\0' terminate 88 | //#define LZ(N) &(L##N, b[N + 1] = '\0') 89 | 90 | #define LG(F) (u<100 ? u<10 ? F(0) : F(1) : u<1000000 ? u<10000 ? u<1000 ? F(2) : F(3) : u<100000 ? F(4) : F(5) : u<100000000 ? u<10000000 ? F(6) : F(7) : u<1000000000 ? F(8) : F(9)) 91 | 92 | char* u32toa_jeaiii(uint32_t u, char* b) 93 | { 94 | uint64_t t; 95 | return LG(LZ); 96 | } 97 | 98 | char* i32toa_jeaiii(int32_t i, char* b) 99 | { 100 | uint32_t u = i < 0 ? *b++ = '-', 0 - uint32_t(i) : i; 101 | uint64_t t; 102 | return LG(LZ); 103 | } 104 | 105 | char* u64toa_jeaiii(uint64_t n, char* b) 106 | { 107 | uint32_t u; 108 | uint64_t t; 109 | 110 | if (uint32_t(n >> 32) == 0) 111 | return u = uint32_t(n), LG(LZ); 112 | 113 | uint64_t a = n / 100000000; 114 | 115 | if (uint32_t(a >> 32) == 0) 116 | { 117 | u = uint32_t(a); 118 | LG(LN); 119 | } 120 | else 121 | { 122 | u = uint32_t(a / 100000000); 123 | LG(LN); 124 | u = a % 100000000; 125 | LN(7); 126 | } 127 | 128 | u = n % 100000000; 129 | return LZ(7); 130 | } 131 | 132 | char* i64toa_jeaiii(int64_t i, char* b) 133 | { 134 | uint64_t n = i < 0 ? *b++ = '-', 0 - uint64_t(i) : i; 135 | return u64toa_jeaiii(n, b); 136 | } 137 | -------------------------------------------------------------------------------- /main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include "broker.hpp" 3 | using namespace qimq; 4 | 5 | int main() { 6 | broker_t broker(9000, std::thread::hardware_concurrency()); 7 | broker.run(); 8 | return 0; 9 | } -------------------------------------------------------------------------------- /mem_storage.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include 4 | #include "storage.hpp" 5 | #include "consts.h" 6 | 7 | namespace qimq { 8 | class mem_storage : public storage_t { 9 | public: 10 | 11 | [[nodiscard]] error_code add(const int64_t& key, std::string val) { 12 | std::unique_lock lock(mtx_); 13 | auto it = map_.find(key); 14 | if(it!= map_.end()){ 15 | return error_code::has_exist; 16 | } 17 | 18 | map_.emplace(std::move(key), std::move(val)); 19 | return error_code::ok; 20 | } 21 | 22 | std::string& get(const int64_t& key) { 23 | std::unique_lock lock(mtx_); 24 | auto it = map_.find(key); 25 | if (it == map_.end()) 26 | return EMPTY_STR; 27 | 28 | return it->second; 29 | } 30 | 31 | [[nodiscard]] bool remove(const int64_t& key) { 32 | std::unique_lock lock(mtx_); 33 | map_.erase(key); 34 | return true; 35 | } 36 | 37 | [[nodiscard]] int64_t size() { 38 | std::unique_lock lock(mtx_); 39 | return map_.size(); 40 | } 41 | 42 | [[nodiscard]] bool empty() { 43 | std::unique_lock lock(mtx_); 44 | return map_.empty(); 45 | } 46 | 47 | [[nodiscard]] bool has(const int64_t& key) { 48 | std::unique_lock lock(mtx_); 49 | return map_.find(key) != map_.end(); 50 | } 51 | 52 | private: 53 | std::shared_mutex mtx_; 54 | std::map map_; 55 | }; 56 | } -------------------------------------------------------------------------------- /producer.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace qimq { 4 | 5 | } -------------------------------------------------------------------------------- /qimq.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 15.0 23 | {7EBA3832-4E24-4E01-B4EE-2CD5C17D1AF8} 24 | qimq 25 | 10.0 26 | 27 | 28 | 29 | Application 30 | true 31 | v142 32 | MultiByte 33 | 34 | 35 | Application 36 | false 37 | v142 38 | true 39 | MultiByte 40 | 41 | 42 | Application 43 | true 44 | v142 45 | MultiByte 46 | 47 | 48 | Application 49 | false 50 | v142 51 | true 52 | MultiByte 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | rest_rpc/include;rest_rpc/third/msgpack/include;$(IncludePath) 74 | 75 | 76 | rest_rpc/include;rest_rpc/third/msgpack/include;$(IncludePath) 77 | 78 | 79 | rest_rpc/include;rest_rpc/third/msgpack/include;$(IncludePath) 80 | 81 | 82 | rest_rpc/include;rest_rpc/third/msgpack/include;$(IncludePath) 83 | 84 | 85 | 86 | Level3 87 | Disabled 88 | true 89 | true 90 | stdcpp17 91 | 92 | 93 | 94 | 95 | Level3 96 | Disabled 97 | true 98 | true 99 | stdcpp17 100 | MultiThreadedDebugDLL 101 | 102 | 103 | rocksdb.lib;%(AdditionalDependencies) 104 | 105 | 106 | 107 | 108 | Level3 109 | MaxSpeed 110 | true 111 | true 112 | true 113 | true 114 | stdcpp17 115 | 116 | 117 | true 118 | true 119 | 120 | 121 | 122 | 123 | Level3 124 | MaxSpeed 125 | true 126 | true 127 | true 128 | true 129 | stdcpp17 130 | MultiThreadedDLL 131 | USE_ROCKSDB;%(PreprocessorDefinitions) 132 | 133 | 134 | true 135 | true 136 | rocksdb.lib;zlib.lib;rpcrt4.lib;Shlwapi.lib;snappy.lib;%(AdditionalDependencies) 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | -------------------------------------------------------------------------------- /qimq.vcxproj.filters: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | {4FC737F1-C7A5-4376-A066-2A32D752A2FF} 6 | cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx 7 | 8 | 9 | {93995380-89BD-4b04-88EB-625FBE52EBFB} 10 | h;hh;hpp;hxx;hm;inl;inc;ipp;xsd 11 | 12 | 13 | {67DA6AB6-F800-4c08-8B7A-83BB121AAD01} 14 | rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms 15 | 16 | 17 | 18 | 19 | 源文件 20 | 21 | 22 | 23 | 24 | 头文件 25 | 26 | 27 | 头文件 28 | 29 | 30 | 头文件 31 | 32 | 33 | 头文件 34 | 35 | 36 | 头文件 37 | 38 | 39 | 头文件 40 | 41 | 42 | 头文件 43 | 44 | 45 | 头文件 46 | 47 | 48 | -------------------------------------------------------------------------------- /storage.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #include 3 | #include "entity.hpp" 4 | 5 | namespace qimq { 6 | template 7 | class storage_t { 8 | public: 9 | int add(const int64_t& key, T val) { 10 | return static_cast(this)->add(key, val); 11 | } 12 | 13 | T& get(const int64_t& key) { 14 | return static_cast(this)->get(key); 15 | } 16 | 17 | bool remove(const int64_t& key) { 18 | return static_cast(this)->remove(key); 19 | } 20 | 21 | int64_t size() { 22 | return static_cast(this)->size(); 23 | } 24 | 25 | bool empty() { 26 | return static_cast(this)->empty(); 27 | } 28 | 29 | bool has(const int64_t& key) { 30 | static_cast(this)->has(key); 31 | } 32 | }; 33 | } -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.13) 2 | project(test) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++17") 5 | 6 | include_directories(${Boost_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} 7 | "../rest_rpc/include" 8 | "../rest_rpc/third/msgpack/include" 9 | ) 10 | 11 | add_executable(test main.cpp) -------------------------------------------------------------------------------- /test/doctest/tests/demo_test.cpp: -------------------------------------------------------------------------------- 1 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN 2 | #include "doctest.h" 3 | 4 | int factorial(int number) { return number <= 1 ? number : factorial(number - 1) * number; } 5 | 6 | TEST_CASE("testing the factorial function") { 7 | CHECK(factorial(1) == 1); 8 | CHECK(factorial(2) == 2); 9 | CHECK(factorial(3) == 6); 10 | CHECK(factorial(10) == 3628800); 11 | } -------------------------------------------------------------------------------- /test/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | using namespace rest_rpc; 8 | using namespace rest_rpc::rpc_service; 9 | 10 | int main() { 11 | std::cout << "Hello, World!" << std::endl; 12 | return 0; 13 | } -------------------------------------------------------------------------------- /test/test_pull_ack/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN 9 | #include "doctest.hpp" 10 | 11 | using namespace rest_rpc; 12 | using namespace rest_rpc::rpc_service; 13 | 14 | struct result { 15 | int code; 16 | int64_t msg_id; 17 | std::string data; 18 | std::string msg; 19 | 20 | MSGPACK_DEFINE(code, msg_id, data, msg); 21 | }; 22 | 23 | bool test_pull_ack() 24 | { 25 | rpc_client client("127.0.0.1", 9000); 26 | bool r = client.connect(); 27 | REQUIRE(r); 28 | 29 | std::string uuid = "3345-d91c-4b9f-9129-101597cd3dd9"; 30 | bool isRetry = false; 31 | 32 | client.subscribe("pull", uuid, [&](string_view data) { 33 | isRetry = true; 34 | }); 35 | 36 | // assume pull success 37 | int msgId = 0; 38 | auto retVal = client.call("pull", msgId, uuid); 39 | REQUIRE(!retVal.empty()); 40 | 41 | try { 42 | result consume_result{ 0, msgId }; 43 | client.call("pull_ack", msgId, consume_result); 44 | } 45 | catch (std::exception & e) { 46 | std::cout << "some error:" << e.what(); 47 | return false; 48 | } 49 | 50 | std::cout << "consuming: " << retVal << std::endl; 51 | 52 | std::thread thd([&client]() { 53 | std::this_thread::sleep_for(std::chrono::milliseconds(25000)); 54 | }); 55 | 56 | thd.join(); 57 | return (isRetry == false); 58 | } 59 | 60 | TEST_CASE("test pull ack") 61 | { 62 | CHECK(test_pull_ack() == true); 63 | } -------------------------------------------------------------------------------- /test/test_pull_ack/test_pull_ack.vcxproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | Win32 7 | 8 | 9 | Release 10 | Win32 11 | 12 | 13 | Debug 14 | x64 15 | 16 | 17 | Release 18 | x64 19 | 20 | 21 | 22 | 16.0 23 | {D30EE916-485D-4619-912E-D5985B1D3B87} 24 | testpullack 25 | 10.0 26 | 27 | 28 | 29 | Application 30 | true 31 | v142 32 | Unicode 33 | 34 | 35 | Application 36 | false 37 | v142 38 | true 39 | Unicode 40 | 41 | 42 | Application 43 | true 44 | v142 45 | Unicode 46 | 47 | 48 | Application 49 | false 50 | v142 51 | true 52 | Unicode 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | true 74 | $(Boost_INCLUDE_DIRS);$(IncludePath);../../rest_rpc/include;../../rest_rpc/third/msgpack/include;../doctest 75 | $(Boost_INCLUDE_DIRS)\VS2019\lib;$(LibraryPath) 76 | 77 | 78 | true 79 | 80 | 81 | false 82 | 83 | 84 | false 85 | 86 | 87 | 88 | Level3 89 | true 90 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 91 | true 92 | 93 | 94 | Console 95 | true 96 | 97 | 98 | 99 | 100 | Level3 101 | true 102 | _DEBUG;_CONSOLE;%(PreprocessorDefinitions) 103 | true 104 | 105 | 106 | Console 107 | true 108 | 109 | 110 | 111 | 112 | Level3 113 | true 114 | true 115 | true 116 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 117 | true 118 | 119 | 120 | Console 121 | true 122 | true 123 | true 124 | 125 | 126 | 127 | 128 | Level3 129 | true 130 | true 131 | true 132 | NDEBUG;_CONSOLE;%(PreprocessorDefinitions) 133 | true 134 | 135 | 136 | Console 137 | true 138 | true 139 | true 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | -------------------------------------------------------------------------------- /test/test_pull_msg/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(test_pull_msg) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++17") 5 | 6 | include_directories(${Boost_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} 7 | "../../rest_rpc/include" 8 | "../../rest_rpc/third/msgpack/include" 9 | ) 10 | 11 | add_executable(test_pull_msg main.cpp) -------------------------------------------------------------------------------- /test/test_pull_msg/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace rest_rpc; 9 | using namespace rest_rpc::rpc_service; 10 | 11 | void test_pull_msg(int64_t msgid) { 12 | try { 13 | rpc_client client("127.0.0.1", 9000); 14 | bool r = client.connect(); 15 | if (!r) { 16 | std::cout << "connect timeout" << std::endl; 17 | return; 18 | } 19 | 20 | std::string uuid = "1746a8d4-d91c-4b9f-9129-101597cd3dd9"; 21 | auto result = client.call("pull", msgid ,uuid ); 22 | std::cout << result << std::endl; 23 | } 24 | catch (const std::exception & e) { 25 | std::cout << e.what() << std::endl; 26 | } 27 | } 28 | 29 | int main() { 30 | std::cout << "pull msg test start!" << std::endl; 31 | test_pull_msg(0); 32 | return 0; 33 | } -------------------------------------------------------------------------------- /test/test_send_msg/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 3.0) 2 | project(test_send_msg) 3 | 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pthread -std=c++17") 5 | 6 | include_directories(${Boost_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR} 7 | "../../rest_rpc/include" 8 | "../../rest_rpc/third/msgpack/include" 9 | ) 10 | 11 | add_executable(test_send_msg main.cpp) -------------------------------------------------------------------------------- /test/test_send_msg/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include "../../broker.hpp" 7 | 8 | using namespace rest_rpc; 9 | using namespace rest_rpc::rpc_service; 10 | 11 | void test_send_msg(int64_t msgid) { 12 | try { 13 | rpc_client client("127.0.0.1", 9000); 14 | bool r = client.connect(); 15 | if (!r) { 16 | std::cout << "connect timeout" << std::endl; 17 | return; 18 | } 19 | 20 | auto result = client.call("send_msg", msgid, "hello"); 21 | std::cout << "msg_id: " << result.msg_id << std::endl; 22 | } 23 | catch (const std::exception & e) { 24 | std::cout << e.what() << std::endl; 25 | } 26 | } 27 | 28 | int main() { 29 | std::cout << "send msg test start!" << std::endl; 30 | test_send_msg(0); 31 | return 0; 32 | } --------------------------------------------------------------------------------