├── .gitignore ├── .travis.yml ├── LICENSE ├── README.en.md ├── README.md ├── gen_test_data.sh ├── src ├── Makefile ├── art │ ├── art.cpp │ ├── art.hpp │ ├── node.cpp │ └── node.hpp ├── blink │ ├── b_link_tree.cpp │ ├── b_link_tree.hpp │ ├── bounded_mapping_queue.hpp │ ├── db.cpp │ ├── db.hpp │ ├── hash_entry.hpp │ ├── log.hpp │ ├── page.cpp │ ├── page.hpp │ ├── page_pool.cpp │ ├── page_pool.hpp │ ├── pool_manager.cpp │ ├── pool_manager.hpp │ ├── slice.hpp │ ├── task.hpp │ └── thread_pool_mapping.hpp ├── include │ ├── atomic.hpp │ ├── bounded_list.hpp │ ├── bounded_queue.hpp │ ├── cond.hpp │ ├── guard.hpp │ ├── latch.hpp │ ├── mutex.hpp │ ├── spin_lock.hpp │ ├── thread.hpp │ ├── thread_pool.hpp │ └── utility.hpp ├── network │ ├── buffer.cpp │ ├── buffer.hpp │ ├── callback.hpp │ ├── channel.cpp │ ├── channel.hpp │ ├── connection.cpp │ ├── connection.hpp │ ├── endpoint.cpp │ ├── endpoint.hpp │ ├── eventbase.cpp │ ├── eventbase.hpp │ ├── poller.cpp │ ├── poller.hpp │ ├── server.cpp │ ├── server.hpp │ ├── signal.hpp │ ├── socket.cpp │ ├── socket.hpp │ └── time.hpp ├── palm │ ├── barrier.hpp │ ├── batch.cpp │ ├── batch.hpp │ ├── batcher.cpp │ ├── batcher.hpp │ ├── palm_tree.cpp │ └── palm_tree.hpp ├── raft │ ├── arg.hpp │ ├── log.hpp │ ├── mushroom_log.hpp │ ├── raft_server.cpp │ └── raft_server.hpp ├── rpc │ ├── future.hpp │ ├── marshaller.hpp │ ├── rpc.hpp │ ├── rpc_connection.cpp │ ├── rpc_connection.hpp │ ├── rpc_server.cpp │ └── rpc_server.hpp └── run ├── test ├── art.cpp ├── barrier.cpp ├── batch.cpp ├── batcher.cpp ├── bloom.cpp ├── distributed_index.cpp ├── mushroom_log_vector.cpp ├── mushroom_multi_thread.cpp ├── mushroom_with_queue.cpp ├── network_client.cpp ├── network_server.cpp ├── page.cpp ├── palm.cpp ├── raft.cpp ├── rpc_call.hpp ├── rpc_client.cpp ├── rpc_server.cpp └── unit.h ├── test_data.cpp └── version.md /.gitignore: -------------------------------------------------------------------------------- 1 | *_test 2 | *.o 3 | .tags* 4 | paxos 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | 3 | language: cpp 4 | 5 | matrix: 6 | include: 7 | - os: linux 8 | 9 | script: 10 | - g++ -std=c++11 -O3 test_data.cpp -o test_data 11 | - cd src && make -j4 all 12 | 13 | notifications: 14 | email: never 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2018 UncP. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are 5 | met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | * Redistributions in binary form must reproduce the above 10 | copyright notice, this list of conditions and the following disclaimer 11 | in the documentation and/or other materials provided with the 12 | distribution. 13 | * Neither the name of Mushroom nor the names of UncP may be used 14 | to endorse or promote products derived from this software 15 | without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 18 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 19 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 20 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 21 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 22 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 23 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 27 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.en.md: -------------------------------------------------------------------------------- 1 | ### this repo is not updated any more，please go to [aili](https://github.com/UncP/aili) 2 | *** 3 | 4 | ## Mushroom: Distributed In-Memory Index 5 | [中文版 README](./README.md) 6 | 7 | [![Author](https://img.shields.io/badge/Author-UncP-brightgreen.svg)](./LICENSE) 8 | [![Version](https://img.shields.io/badge/Version-1.2.2-blue.svg)]() 9 | [![Build](https://img.shields.io/badge/Build-Passing-brightgreen.svg)](https://travis-ci.org/UncP/Mushroom) 10 | [![License](https://img.shields.io/badge/License-BSD-red.svg)](./LICENSE) 11 | 12 | #### Mushroom is a Linux light-weight distributed in-memory index written in C++11, consists of concurrent Blink tree, TCP communication library, RPC framework and Raft consensus algorithm. 13 | 14 | ### Behold, the power of Mushroom! 15 | 16 | ### Try Mushroom 17 | `first run gen_test_data.sh to generate test data, then enter src directory`
18 | `./run index 100, test distributed index on single machine，100 represents index number(raft is not well optimized, please limit number to 1-1000)`
19 | `./run blink thread 100, test multi-thread b link tree, number can be 1 to 10 million`
20 | `./run blink queue 100, test multi-thread b link tree, number can be 1 to 10 million`
21 | `./run art 100, test adaptive radix tree, number can be 1 to 10 million`
22 | 23 | 24 | ### Version Information 25 | | Version | Improvements | 26 | |:------:|:--------------------------:| 27 | | 0.1.0 | | 28 | | 0.2.0 | two-phase lock based concurrent index | 29 | | 0.2.1 | latch manager optimization | 30 | | 0.3.0 | implement prefix compaction, reducing Blink tree memory about 9.1 % | 31 | | 0.4.0 | implement **Mapping Queue**, reducing total program memory up to 50 %| 32 | | 0.4.1 | merge latch manager and page manager，reduce 1 lock per operation | 33 | | 0.4.2 | change the way B link tree root split | 34 | | 0.4.3 | new test strategy, threads conduct operations without going through the queue | 35 | | 0.4.4 | latch manager refactoring | 36 | | 0.5.0 | fix **BUG**(atomic operation bug) that exists from 0.4.1 to 0.4.4 | 37 | | 0.6.0 | multi-process supported, fix Blink tree search **bug**| 38 | | 0.6.1 | two-phase hashing page manager, implement lazy page allocation| 39 | | 0.6.2 | reduce dependency on standard library, speed up compile, reduce program size about 42.1%| 40 | | 0.6.4 |using posix spin lock, Optimize MushroomDB and BLinkTree structure| 41 | | 0.7.0 | Log-Structured Merge Tree | 42 | | 0.8.0 | TCP Communication Library & RPC Framework | 43 | | 0.9.0 | Raft | 44 | | 0.9.1 | ACID (batch operation) & modify locking strategy | 45 | | 0.9.2 | optimize Raft state transfer, improve liveness | 46 | | 1.0.0 | distributed in-memory index | 47 | | 1.1.0 | Blink tree node occupies 2/3 of node's space, originally 1/2, reducing memory use about 12% | 48 | | 1.1.1 | optimize raft memory use | 49 | | 1.2.0 | Adaptive Radix Tree | 50 | | 1.2.1 | remove latch manager, each page has its own latch, improve performance by 10% | 51 | | 1.2.2 | introducing bloom filter for write/read optimization | 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### 这个 repo 已弃用，请移步 [aili](https://github.com/UncP/aili) 查看最新进展 2 | *** 3 | 4 | ## Mushroom（蘑菇）：分布式内存索引 5 | [English Version of README](./README.en.md) 6 | 7 | [![Author](https://img.shields.io/badge/Author-UncP-brightgreen.svg)](https://github.com/UncP) 8 | [![Version](https://img.shields.io/badge/Version-1.2.2-blue.svg)]() 9 | [![Build](https://img.shields.io/badge/Build-Passing-brightgreen.svg)](https://travis-ci.org/UncP/Mushroom) 10 | [![License](https://img.shields.io/badge/License-BSD-red.svg)](./LICENSE) 11 | 12 | #### Mushroom是一个C++11编写、不依赖第三方库的轻量级Linux环境分布式内存索引，它由并发Blink树索引，TCP通信库，RPC框架，Raft一致性算法组成。 13 | 14 | ### Behold, the power of Mushroom! 15 | 16 | 17 | ### 尝试 18 | `首先运行脚本gen_test_data.sh生成测试数据，然后进入src目录`
19 | `./run blink queue 100，测试多线程索引（通过队列），数量可以是1到1000万`
20 | `./run blink thread 100，测试多线程索引，数量可以是1到1000万`
21 | `./run art 100，测试Adaptive Radix Tree，数量可以是1到1000万`
22 | `【失效】./run index 100，单机测试分布式索引，100是索引数量（因为Raft没有深入优化过，所以请将数量控制在1-1000）`
23 | 24 | 25 | ### 版本信息 26 | | 版本 | 备注 | 27 | |:------:|:---------------------------:| 28 | | 0.1.0 | | 29 | | 0.2.0 | 读写锁并发索引 | 30 | | 0.2.1 | 锁管理器优化 | 31 | | 0.3.0 | 引入前缀压缩，Blink树占用内存减少约 9.1 %| 32 | | 0.4.0 | 实现映射队列，减少程序使用内存超过 50 %| 33 | | 0.4.1 | 合并锁管理器与页面管理器，使每次操作减少1把锁| 34 | | 0.4.2 | 修改根节点分裂方式 | 35 | | 0.4.3 | 增加测试策略，多线程不经过队列直接进行任务| 36 | | 0.4.4 | 重构锁管理器 | 37 | | 0.5.0 | 修复从版本0.4.1到0.4.4一直存在的**bug**（原子操作bug）| 38 | | 0.6.0 | 共享内存映射支持多进程，修复搜索**bug**，正确实现并发Blink树| 39 | | 0.6.1 | 二次哈希页面管理器，实现页面的懒惰分配| 40 | | 0.6.2 | 减少对标准库的依赖，加快编译速度，减少程序体积约42.1%| 41 | | 0.6.4 | 使用posix自旋锁，优化MushroomDB和BLinkTree结构 | 42 | | 0.7.0 | 日志结构合并树（LSM Tree） | 43 | | 0.8.0 | TCP通信库、RPC框架 | 44 | | 0.9.0 | Raft | 45 | | 0.9.1 | ACID (批操作)、修改加锁策略 | 46 | | 0.9.2 | 优化Raft状态变化，提高Liveness | 47 | | 1.0.0 | 分布式内存索引 | 48 | | 1.1.0 | Blink树结点从1/2满提高到2/3满，降低内存使用率约12% | 49 | | 1.1.1 | 优化Raft内存使用（牺牲可读性） | 50 | | 1.2.0 | Adaptive Radix Tree | 51 | | 1.2.1 | 移除锁管理器，每个页面一把锁，性能提升约10% | 52 | | 1.2.2 | 引入布隆过滤器进行读写优化（然并卵） | 53 | 54 | 55 | ### 其他 56 | + 你可以在这个[知乎专栏](https://zhuanlan.zhihu.com/b-tree)里找到**Mushroom**的介绍 57 | + 分布式索引只是完成了单机功能，我写完就放弃优化了，所以较新的版本里`./run index`这个命令并不支持 58 | 59 | *** 60 | 61 | PS: Mushroom 是一次致力于获得**极致**存储性能的**激进**尝试，在阅读具体代码之前，请注意： 62 | 1. 编程规范、代码可读性、内存使用、接口是否友好等等一切无条件为性能做出让步 63 | 2. 但是核心 B link 树的并发算法实现很简洁 64 | 3. 对于内存的使用很极端，比如**映射队列**，介绍文章[上](https://zhuanlan.zhihu.com/p/26856329)以及[下](https://zhuanlan.zhihu.com/p/26919167) 65 | 4. 有一些指针的 Hack 操作，比如柔性数组 66 | 5. 删除和插入都是写操作，所以只有 Put（写）和 Get（读）操作（即不支持物理删除） 67 | -------------------------------------------------------------------------------- /gen_test_data.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | g++ -std=c++11 -O3 test_data.cpp -o test_data && ./test_data && rm test_data 4 | # mv data/10000000_0 data/10000000 5 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | CPP = $(CXX) 2 | CPPFLAGS = -std=c++11 -Wall -Wextra -O3 3 | LDFLAGS = -pthread 4 | 5 | BlinkFile = $(shell find blink -name '*.cpp') 6 | NetworkFile = $(shell find network -name '*.cpp') 7 | RaftFile = $(shell find raft -name '*.cpp') 8 | RpcFile = $(shell find rpc -name '*.cpp') 9 | ARTFILE = $(shell find art -name '*.cpp') 10 | PalmFile = $(shell find palm -name '*.cpp') 11 | 12 | MOBJ = $(BlinkFile:.cpp=.o) 13 | NOBJ = $(NetworkFile:.cpp=.o) 14 | RaftOBJ = $(RaftFile:.cpp=.o) 15 | RpcOBJ = $(RpcFile:.cpp=.o) 16 | AOBJ = $(ARTFILE:.cpp=.o) 17 | POBJ = $(PalmFile:.cpp=.o) 18 | 19 | default: palm_tree_test 20 | 21 | all: blinktree_with_queue_test blinktree_multi_thread_test palm_tree_test 22 | 23 | palm_tree_test: $(POBJ) ./blink/pool_manager.o ./blink/page_pool.o ./blink/page.o ../test/palm.cpp 24 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 25 | 26 | batcher_test: ../test/batcher.cpp ./palm/batcher.o ./blink/page.o 27 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 28 | 29 | batch_test: ../test/batch.cpp ./palm/batch.o 30 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 31 | 32 | barrier_test: ../test/barrier.cpp 33 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 34 | 35 | art_tree_test: $(AOBJ) ../test/art.cpp 36 | $(CPP) $(CPPFLAGS) -o $@ $^ -msse2 37 | 38 | mushroom_log_vector_test: raft/mushroom_log_vector.cpp ../test/mushroom_log_vector.cpp 39 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 40 | 41 | distributed_index_test: $(MOBJ) $(NOBJ) $(RpcOBJ) $(RaftOBJ) ../test/distributed_index.cpp 42 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 43 | 44 | raft_test: $(NOBJ) $(RpcOBJ) $(RaftOBJ) ../test/raft.cpp 45 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 46 | 47 | rpc_client_test: $(NOBJ) $(RpcOBJ) ../test/rpc_client.cpp 48 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 49 | 50 | rpc_server_test: $(NOBJ) $(RpcOBJ) ../test/rpc_server.cpp 51 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 52 | 53 | client_test: $(NOBJ) ../test/network_client.cpp 54 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 55 | 56 | server_test: $(NOBJ) ../test/network_server.cpp 57 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 58 | 59 | page_test: $(MOBJ) ../test/page.cpp 60 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 61 | 62 | blinktree_with_queue_test: $(MOBJ) ../test/mushroom_with_queue.cpp 63 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 64 | 65 | blinktree_multi_thread_test: $(MOBJ) ../test/mushroom_multi_thread.cpp 66 | $(CPP) $(CPPFLAGS) -o $@ $^ $(LDFLAGS) 67 | 68 | .cpp.o: 69 | $(CPP) $(CPPFLAGS) -c $< -o $@ $(DFLAGS) $(LDFLAGS) 70 | 71 | tag: 72 | cd .. && ctags -R -f .tags 73 | 74 | clean: 75 | rm */*.o 76 | rm -rf *_test 77 | -------------------------------------------------------------------------------- /src/art/art.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-21 10:40:44 6 | **/ 7 | 8 | #include 9 | 10 | #include "node.hpp" 11 | #include "art.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | ART::ART():root_(0) { } 16 | 17 | static Node** Descend(Node *cur, char byte) 18 | { 19 | union { 20 | Node4 *p4; 21 | Node16 *p16; 22 | Node48 *p48; 23 | Node256 *p256; 24 | }p; 25 | switch(cur->Type()) { 26 | case NODE4 : p.p4 = (Node4 *)cur; return p.p4 ->Descend(byte); 27 | case NODE16 : p.p16 = (Node16 *)cur; return p.p16 ->Descend(byte); 28 | case NODE48 : p.p48 = (Node48 *)cur; return p.p48 ->Descend(byte); 29 | case NODE256 : p.p256 = (Node256 *)cur; return p.p256->Descend(byte); 30 | default : assert(0); 31 | } 32 | } 33 | 34 | static void AddChild(Node *cur, Node **ref, uint8_t byte, void *child) 35 | { 36 | union { 37 | Node4 *p4; 38 | Node16 *p16; 39 | Node48 *p48; 40 | Node256 *p256; 41 | }p; 42 | switch(cur->Type()) { 43 | case NODE4: 44 | p.p4 = (Node4 *)cur; 45 | if (p.p4->Full()) { 46 | Node16 *node16 = new Node16(p.p4); 47 | *ref = node16; 48 | delete cur; 49 | node16->AddChild(byte, child); 50 | } else { 51 | p.p4->AddChild(byte, child); 52 | } 53 | break; 54 | case NODE16: 55 | p.p16 = (Node16 *)cur; 56 | if (p.p16->Full()) { 57 | Node48 *node48 = new Node48(p.p16); 58 | *ref = node48; 59 | delete cur; 60 | node48->AddChild(byte, child); 61 | } else { 62 | p.p16->AddChild(byte, child); 63 | } 64 | break; 65 | case NODE48: 66 | p.p48 = (Node48 *)cur; 67 | if (p.p48->Full()) { 68 | Node256 *node256 = new Node256(p.p48); 69 | *ref = node256; 70 | delete cur; 71 | node256->AddChild(byte, child); 72 | } else { 73 | p.p48->AddChild(byte, child); 74 | } 75 | break; 76 | case NODE256: 77 | p.p256 = (Node256 *)cur; 78 | p.p256->AddChild(byte, child); 79 | break; 80 | default: 81 | assert(0); 82 | } 83 | } 84 | 85 | static bool Insert(Node *cur, Node **ref, const uint8_t *key, uint32_t depth, uint32_t len, uint32_t val) 86 | { 87 | if (!cur) { 88 | *ref = (Node *)SET_LEAF(NewLeaf(key, len, val)); 89 | return true; 90 | } 91 | 92 | if (IS_LEAF(cur)) { 93 | Leaf *leaf = LEAF_RAW(cur); 94 | if (leaf->Match(key, len)) return false; 95 | 96 | Leaf *leaf2 = NewLeaf(key, len, val); 97 | uint32_t prefix_len = leaf->CommonPrefix(depth, leaf2); 98 | 99 | Node4 *node4 = new Node4(); 100 | *ref = (Node *)node4; 101 | 102 | node4->SetPrefix(key + depth, prefix_len); 103 | 104 | node4->AddChild(leaf->KeyAt(depth + prefix_len), SET_LEAF(leaf)); 105 | node4->AddChild(leaf2->KeyAt(depth + prefix_len), SET_LEAF(leaf2)); 106 | return true; 107 | } 108 | 109 | if (cur->PrefixLen()) { 110 | uint32_t prefix_diff = cur->MismatchPrefix(key, len, depth); 111 | if (prefix_diff >= cur->PrefixLen()) { 112 | depth += cur->PrefixLen(); 113 | goto Recurse_Search; 114 | } 115 | 116 | Node4 *node4 = new Node4(); 117 | *ref = (Node *)node4; 118 | node4->SetPrefix(cur->Prefix(), prefix_diff); 119 | 120 | if (node4->PrefixLen() <= Node::MAX_PREFIX_LEN) { 121 | node4->AddChild(cur->Prefix()[prefix_diff], cur); 122 | cur->AdjustPrefix(prefix_diff); 123 | } else { 124 | Leaf *leaf = Node::Minimum(cur); 125 | node4->AddChild(leaf->KeyAt(depth + prefix_diff), cur); 126 | cur->AdjustPrefix(prefix_diff, leaf->Key() + depth); 127 | } 128 | 129 | Leaf *leaf = NewLeaf(key, len, val); 130 | node4->AddChild(key[depth + prefix_diff], SET_LEAF(leaf)); 131 | return true; 132 | } 133 | 134 | Recurse_Search: 135 | Node **child = Descend(cur, key[depth]); 136 | if (child) 137 | return Insert(*child, child, key, depth + 1, len, val); 138 | 139 | Leaf *leaf = NewLeaf(key, len, val); 140 | AddChild(cur, ref, key[depth], SET_LEAF(leaf)); 141 | return true; 142 | } 143 | 144 | bool ART::Put(const uint8_t *key, uint32_t len, uint32_t val) 145 | { 146 | return Insert(root_, &root_, key, 0, len, val); 147 | } 148 | 149 | bool ART::Get(const uint8_t *key, uint32_t len, uint32_t *val) 150 | { 151 | Node *cur = root_; 152 | uint32_t depth = 0; 153 | while (cur) { 154 | if (IS_LEAF(cur)) { 155 | Leaf *leaf = LEAF_RAW(cur); 156 | if (leaf->Match(key, len)) { 157 | *val = leaf->Value(); 158 | return true; 159 | } 160 | return false; 161 | } 162 | 163 | if (cur->PrefixLen()) { 164 | uint32_t prefix_len = cur->CheckPrefix(key, len, depth); 165 | if (prefix_len != std::min(Node::MAX_PREFIX_LEN, cur->PrefixLen())) 166 | return false; 167 | depth += cur->PrefixLen(); 168 | } 169 | 170 | Node **child = Descend(cur, key[depth]); 171 | cur = child ? *child : 0; 172 | ++depth; 173 | } 174 | return false; 175 | } 176 | 177 | ART::~ART() 178 | { 179 | Free(root_); 180 | } 181 | 182 | } // namespace Mushroom 183 | -------------------------------------------------------------------------------- /src/art/art.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-20 18:42:03 6 | **/ 7 | 8 | #ifndef _ART_HPP_ 9 | #define _ART_HPP_ 10 | 11 | class Node; 12 | 13 | namespace Mushroom { 14 | 15 | class ART 16 | { 17 | public: 18 | ART(); 19 | 20 | bool Put(const uint8_t *key, uint32_t len, uint32_t val); 21 | 22 | bool Get(const uint8_t *key, uint32_t len, uint32_t *val); 23 | 24 | ~ART(); 25 | 26 | private: 27 | Node *root_; 28 | }; 29 | 30 | } // namespace Mushroom 31 | 32 | #endif /* _ART_HPP_ */ -------------------------------------------------------------------------------- /src/art/node.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-30 10:10:10 6 | **/ 7 | 8 | #include 9 | 10 | #include "node.hpp" 11 | 12 | namespace Mushroom { 13 | 14 | uint32_t Node::MAX_PREFIX_LEN = 8; 15 | 16 | void Free(Node *node) 17 | { 18 | if (!node) return ; 19 | 20 | if (IS_LEAF(node)) { 21 | DeleteLeaf(LEAF_RAW(node)); 22 | return ; 23 | } 24 | 25 | int i; 26 | union { 27 | Node4 *p4; 28 | Node16 *p16; 29 | Node48 *p48; 30 | Node256 *p256; 31 | }p; 32 | switch (node->Type()) { 33 | case NODE4: 34 | p.p4 = (Node4 *)node; 35 | for (i = 0; i < node->Count(); ++i) 36 | Free(p.p4->ChildAt(i)); 37 | break; 38 | case NODE16: 39 | p.p16 = (Node16 *)node; 40 | for (i = 0; i < node->Count(); ++i) 41 | Free(p.p16->ChildAt(i)); 42 | break; 43 | case NODE48: 44 | p.p48 = (Node48 *)node; 45 | for (i = 0; i < node->Count(); ++i) 46 | Free(p.p48->ChildAt(i)); 47 | break; 48 | case NODE256: 49 | p.p256 = (Node256 *)node; 50 | for (i = 0; i < 256; ++i) 51 | if (p.p256->ChildAt(i)) 52 | Free(p.p256->ChildAt(i)); 53 | break; 54 | default: 55 | assert(0); 56 | } 57 | delete node; 58 | } 59 | 60 | Leaf* Node::Minimum(const Node *node) 61 | { 62 | if (!node) return 0; 63 | if (IS_LEAF(node)) return LEAF_RAW(node); 64 | 65 | int idx; 66 | union { 67 | const Node48 *p48; 68 | const Node256 *p256; 69 | }p; 70 | switch (node->Type()) { 71 | case NODE4: 72 | return Minimum(((const Node4 *)node)->ChildAt(0)); 73 | case NODE16: 74 | return Minimum(((const Node16 *)node)->ChildAt(0)); 75 | case NODE48: 76 | idx = 0; 77 | p.p48 = (const Node48 *)node; 78 | while (!p.p48->KeyAt(idx)) ++idx; 79 | idx = p.p48->KeyAt(idx) - 1; 80 | return Minimum(p.p48->ChildAt(idx)); 81 | case NODE256: 82 | idx = 0; 83 | p.p256 = (const Node256 *)node; 84 | while (!p.p256->ChildAt(idx)) ++idx; 85 | return Minimum(p.p256->ChildAt(idx)); 86 | default: 87 | assert(0); 88 | } 89 | } 90 | 91 | } // namespace Mushroom 92 | -------------------------------------------------------------------------------- /src/art/node.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-21 09:42:33 6 | **/ 7 | 8 | #ifndef _NODE_HPP_ 9 | #define _NODE_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace Mushroom { 17 | 18 | enum NodeType { NODE4 = 0x0, NODE16, NODE48, NODE256, LEAF }; 19 | 20 | #define IS_LEAF(x) (uintptr_t(x) & 1) 21 | #define SET_LEAF(x) (void *)(uintptr_t(x) | 1) 22 | #define LEAF_RAW(x) (Leaf *)(uintptr_t(x) & ~1) 23 | 24 | class Leaf 25 | { 26 | public: 27 | Leaf(const uint8_t *key, uint32_t len, uint32_t val):val_(val), len_(len) { 28 | memcpy(key_, key, len_); 29 | } 30 | 31 | inline bool Match(const uint8_t *key, uint32_t len) const { 32 | if (len != len_) return false; 33 | return !memcmp(key_, key, len_); 34 | } 35 | 36 | uint32_t CommonPrefix(uint32_t depth, const Leaf *that) const { 37 | int max_cmp = std::min(this->len_, that->len_) - depth; 38 | int idx; 39 | for (idx = 0; idx < max_cmp; ++idx) 40 | if (this->key_[depth + idx] != that->key_[depth + idx]) 41 | return idx; 42 | return idx; 43 | } 44 | 45 | inline uint32_t KeyLen() const { return len_; } 46 | 47 | inline uint8_t KeyAt(uint32_t idx) const { 48 | return key_[idx]; 49 | } 50 | 51 | inline uint32_t Value() const { return val_; } 52 | 53 | inline const uint8_t* Key() const { return key_; } 54 | 55 | private: 56 | uint32_t val_; 57 | uint32_t len_; 58 | uint8_t key_[0]; 59 | }; 60 | 61 | inline Leaf* NewLeaf(const uint8_t *key, uint32_t len, uint32_t val) { 62 | return new (new char[sizeof(Leaf) + len]) Leaf(key, len, val); 63 | } 64 | 65 | inline void DeleteLeaf(Leaf *leaf) { 66 | delete [] (char *)leaf; 67 | } 68 | 69 | class Node 70 | { 71 | public: 72 | Node(NodeType type):type_(type), count_(0), len_(0) { } 73 | 74 | Node(NodeType type, uint8_t count, uint32_t len, const uint8_t *prefix) 75 | :type_(type), count_(count), len_(len) { 76 | memcpy(prefix_, prefix, std::min(MAX_PREFIX_LEN, len_)); 77 | } 78 | 79 | inline NodeType Type() const { return (NodeType)type_; } 80 | 81 | inline uint8_t Count() const { return count_; } 82 | 83 | inline void SetPrefix(const uint8_t *prefix, uint32_t len) { 84 | len_ = len; 85 | memcpy(prefix_, prefix, std::min(MAX_PREFIX_LEN, len_)); 86 | } 87 | 88 | inline uint32_t CheckPrefix(const uint8_t *key, uint32_t len, uint32_t depth) { 89 | int max_cmp = std::min(int(std::min(len_, MAX_PREFIX_LEN)), int(len - depth)); 90 | int idx; 91 | for (idx = 0; idx < max_cmp; ++idx) 92 | if (prefix_[idx] != key[depth + idx]) 93 | return idx; 94 | return idx; 95 | } 96 | 97 | inline void AdjustPrefix(uint32_t len) { 98 | len_ -= len + 1; 99 | memmove(prefix_, prefix_ + len + 1, std::min(MAX_PREFIX_LEN, len_)); 100 | } 101 | 102 | inline void AdjustPrefix(uint32_t len, const uint8_t *key) { 103 | len_ -= len + 1; 104 | memcpy(prefix_, key + len + 1, std::min(MAX_PREFIX_LEN, len_)); 105 | } 106 | 107 | inline const uint8_t* Prefix() const { return prefix_; } 108 | 109 | inline uint32_t PrefixLen() const { return len_; } 110 | 111 | uint32_t MismatchPrefix(const uint8_t *key, uint32_t len, uint32_t depth) { 112 | int max_cmp = std::min(int(std::min(MAX_PREFIX_LEN, len_)), int(len - depth)); 113 | int idx; 114 | for (idx = 0; idx < max_cmp; ++idx) { 115 | if (prefix_[idx] != key[depth + idx]) 116 | return idx; 117 | } 118 | 119 | if (len_ > MAX_PREFIX_LEN) { 120 | Leaf *leaf = Minimum(this); 121 | max_cmp = std::min(leaf->KeyLen(), len) - depth; 122 | for (; idx < max_cmp; ++idx) { 123 | if (leaf->KeyAt(depth + idx) != key[depth + idx]) 124 | return idx; 125 | } 126 | } 127 | return idx; 128 | } 129 | 130 | Node(const Node &) = delete; 131 | Node& operator=(const Node &) = delete; 132 | 133 | static uint32_t MAX_PREFIX_LEN; 134 | 135 | static Leaf* Minimum(const Node *node); 136 | 137 | protected: 138 | inline void IncrCount() { ++count_; } 139 | 140 | private: 141 | uint8_t type_; 142 | uint8_t count_; 143 | uint32_t len_; 144 | uint8_t prefix_[8]; // MAX_PREFIX_LEN 145 | }; 146 | 147 | class Node4 : public Node 148 | { 149 | public: 150 | Node4():Node(NODE4) { memset(key_, 0, 4); } 151 | 152 | inline bool Full() const { return Count() == 4; } 153 | 154 | inline const uint8_t* Key() const { return key_; } 155 | 156 | inline const Node* Child() const { return (const Node *)child_; } 157 | 158 | inline Node* ChildAt(uint32_t idx) const { 159 | return child_[idx]; 160 | } 161 | 162 | Node** Descend(uint8_t byte) { 163 | for (int i = 0; i < Count(); ++i) 164 | if (key_[i] == byte) 165 | return &child_[i]; 166 | return 0; 167 | } 168 | 169 | void AddChild(uint8_t byte, void *child) { 170 | int idx; 171 | for (idx = 0; idx < Count(); ++idx) 172 | if (byte < key_[idx]) 173 | break; 174 | memmove(key_ + idx + 1, key_ + idx, Count() - idx); 175 | memmove(child_ + idx + 1, child_ + idx, sizeof(Node *) * (Count() - idx)); 176 | key_[idx] = byte; 177 | child_[idx] = (Node *)child; 178 | IncrCount(); 179 | } 180 | 181 | private: 182 | uint8_t key_[4]; 183 | Node *child_[4]; 184 | }; 185 | 186 | class Node16 : public Node 187 | { 188 | public: 189 | Node16(const Node4 *node4) 190 | :Node(NODE16, node4->Count(), node4->PrefixLen(), node4->Prefix()) { 191 | memset(key_, 0, 16); 192 | memcpy(key_, node4->Key(), node4->Count()); 193 | memcpy(child_, node4->Child(), sizeof(Node *) * node4->Count()); 194 | } 195 | 196 | inline bool Full() const { return Count() == 16; } 197 | 198 | inline const Node* Child() const { return (const Node *)child_; } 199 | 200 | inline uint8_t KeyAt(uint32_t idx) const { 201 | return key_[idx]; 202 | } 203 | 204 | inline Node* ChildAt(uint32_t idx) const { 205 | return child_[idx]; 206 | } 207 | 208 | Node** Descend(uint8_t byte) { 209 | __m128i cmp = _mm_cmpeq_epi8(_mm_set1_epi8(byte), _mm_loadu_si128((__m128i *)key_)); 210 | int bit = _mm_movemask_epi8(cmp) & ((1 << Count()) - 1); 211 | if (bit) return &child_[__builtin_ctz(bit)]; 212 | else return 0; 213 | } 214 | 215 | void AddChild(uint8_t byte, void *child) { 216 | __m128i cmp = _mm_cmplt_epi8(_mm_set1_epi8(byte), _mm_loadu_si128((__m128i *)key_)); 217 | int bit = _mm_movemask_epi8(cmp) & ((1 << Count()) - 1); 218 | int idx; 219 | if (bit) { 220 | idx = __builtin_ctz(bit); 221 | memmove(key_ + idx + 1, key_ + idx, Count() - idx); 222 | memmove(child_ + idx + 1, child_ + idx, sizeof(Node *) * (Count() - idx)); 223 | } else { 224 | idx = Count(); 225 | } 226 | key_[idx] = byte; 227 | child_[idx] = (Node *)child; 228 | IncrCount(); 229 | } 230 | 231 | private: 232 | uint8_t key_[16]; 233 | Node *child_[16]; 234 | }; 235 | 236 | class Node48 : public Node 237 | { 238 | public: 239 | Node48(const Node16 *node16) 240 | :Node(NODE48, node16->Count(), node16->PrefixLen(), node16->Prefix()) { 241 | memset(index_, 0, 256); memset(child_, 0, sizeof(Node *) * 48); 242 | memcpy(child_, node16->Child(), sizeof(Node *) * node16->Count()); 243 | for (uint8_t i = 0; i < node16->Count(); ++i) 244 | index_[node16->KeyAt(i)] = i + 1; 245 | } 246 | 247 | inline bool Full() const { return Count() == 48; } 248 | 249 | inline uint8_t KeyAt(uint32_t idx) const { 250 | return index_[idx]; 251 | } 252 | 253 | inline Node* ChildAt(uint32_t idx) const { 254 | return child_[idx]; 255 | } 256 | 257 | Node** Descend(uint8_t byte) { 258 | uint8_t idx = index_[byte]; 259 | if (idx) 260 | return &child_[idx - 1]; 261 | return 0; 262 | } 263 | 264 | void AddChild(uint8_t byte, void *child) { 265 | int idx = 0; 266 | while (child_[idx]) ++idx; 267 | index_[byte] = idx + 1; 268 | child_[idx] = (Node *)child; 269 | IncrCount(); 270 | } 271 | 272 | private: 273 | uint8_t index_[256]; 274 | Node *child_[48]; 275 | }; 276 | 277 | class Node256 : public Node 278 | { 279 | public: 280 | Node256(const Node48 *node48) 281 | :Node(NODE256, node48->Count(), node48->PrefixLen(), node48->Prefix()) { 282 | memset(child_, 0, sizeof(Node *) * 256); 283 | for (uint32_t i = 0; i < 256; ++i) 284 | if (node48->KeyAt(i)) 285 | child_[i] = node48->ChildAt(node48->KeyAt(i) - 1); 286 | } 287 | 288 | inline Node* ChildAt(uint32_t idx) const { 289 | return child_[idx]; 290 | } 291 | 292 | Node** Descend(uint8_t byte) { 293 | return &child_[byte]; 294 | } 295 | 296 | void AddChild(uint8_t byte, void *child) { 297 | child_[byte] = (Node *)child; 298 | IncrCount(); 299 | } 300 | 301 | private: 302 | Node *child_[256]; 303 | }; 304 | 305 | void Free(Node *node); 306 | 307 | } // namespace Mushroom 308 | 309 | #endif /* _NODE_HPP_ */ -------------------------------------------------------------------------------- /src/blink/b_link_tree.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-07 20:12:13 6 | **/ 7 | 8 | #include 9 | 10 | #include "b_link_tree.hpp" 11 | #include "page.hpp" 12 | #include "pool_manager.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | BLinkTree::BLinkTree(uint32_t key_len) 17 | :pool_manager_(new PoolManager()), root_(0), key_len_(key_len) 18 | { 19 | degree_ = Page::CalculateDegree(key_len_); 20 | Set set; 21 | set.page_ = pool_manager_->NewPage(Page::ROOT, key_len_, 0, degree_); 22 | set.page_->InsertInfiniteKey(); 23 | } 24 | 25 | BLinkTree::~BLinkTree() 26 | { 27 | delete pool_manager_; 28 | } 29 | 30 | void BLinkTree::Free() 31 | { 32 | pool_manager_->Free(); 33 | } 34 | 35 | void BLinkTree::DescendToLeaf(const KeySlice *key, Set &set, LockType type) 36 | { 37 | set.page_no_ = root_.get(); 38 | set.page_ = pool_manager_->GetPage(set.page_no_); 39 | uint8_t level = set.page_->level_; 40 | for (; level;) { 41 | set.page_->LockShared(); 42 | page_t pre_no = set.page_->page_no_; 43 | set.page_no_ = set.page_->Descend(key); 44 | set.page_->UnlockShared(); 45 | set.page_ = pool_manager_->GetPage(set.page_no_); 46 | if (set.page_->level_ != level) { 47 | set.stack_[set.depth_++] = pre_no; 48 | --level; 49 | } 50 | } 51 | if (type == WriteLock) 52 | set.page_->Lock(); 53 | else 54 | set.page_->LockShared(); 55 | } 56 | 57 | bool BLinkTree::Split(Set &set, KeySlice *key) 58 | { 59 | if (set.page_->type_ != Page::ROOT) { 60 | Page *right = pool_manager_->NewPage(set.page_->type_, set.page_->key_len_, 61 | set.page_->level_, set.page_->degree_); 62 | set.page_->Split(right, key); 63 | set.page_->Unlock(); 64 | assert(set.depth_); 65 | set.page_no_ = set.stack_[--set.depth_]; 66 | set.page_ = pool_manager_->GetPage(set.page_no_); 67 | set.page_->Lock(); 68 | Insert(set, key); 69 | return true; 70 | } else { 71 | uint8_t level = set.page_->level_; 72 | Page *new_root = pool_manager_->NewPage(Page::ROOT, key_len_, level + 1, degree_); 73 | Page *right = pool_manager_->NewPage(level ? Page::BRANCH : Page::LEAF, 74 | set.page_->key_len_, level, degree_); 75 | 76 | new_root->InsertInfiniteKey(); 77 | new_root->AssignFirst(set.page_->page_no_); 78 | 79 | set.page_->type_ = level ? Page::BRANCH : Page::LEAF; 80 | set.page_->Split(right, key); 81 | 82 | page_t page_no = 0; 83 | new_root->Insert(key, page_no); 84 | root_ = new_root->page_no_; 85 | return false; 86 | } 87 | } 88 | 89 | void BLinkTree::Insert(Set &set, KeySlice *key) 90 | { 91 | InsertStatus status; 92 | for (; (status = set.page_->Insert(key, set.page_no_));) { 93 | assert(status == MoveRight); 94 | Latch *pre = set.page_->GetLatch(); 95 | set.page_ = pool_manager_->GetPage(set.page_no_); 96 | set.page_->Lock(); 97 | pre->Unlock(); 98 | } 99 | } 100 | 101 | // this is the entrance for putting a key in b link tree 102 | bool BLinkTree::Put(KeySlice *key) 103 | { 104 | Set set; 105 | 106 | DescendToLeaf(key, set, WriteLock); 107 | 108 | Insert(set, key); 109 | 110 | for (; set.page_->NeedSplit() && Split(set, key); ) 111 | continue; 112 | 113 | set.page_->Unlock(); 114 | return true; 115 | } 116 | 117 | // this is the entrance for getting a key in b link tree 118 | bool BLinkTree::Get(KeySlice *key) 119 | { 120 | Set set; 121 | 122 | DescendToLeaf(key, set, ReadLock); 123 | 124 | for (uint16_t idx = 0; !set.page_->Search(key, &idx);) { 125 | if (idx != set.page_->total_key_) { 126 | set.page_->UnlockShared(); 127 | assert(0); 128 | return false; 129 | } 130 | set.page_no_ = set.page_->Next(); 131 | Latch *pre = set.page_->GetLatch(); 132 | set.page_ = pool_manager_->GetPage(set.page_no_); 133 | set.page_->LockShared(); 134 | pre->Unlock(); 135 | } 136 | 137 | set.page_->UnlockShared(); 138 | return true; 139 | } 140 | 141 | } // namespace Mushroom 142 | -------------------------------------------------------------------------------- /src/blink/b_link_tree.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-07 20:10:30 6 | **/ 7 | 8 | #ifndef _B_LINK_TREE_HPP_ 9 | #define _B_LINK_TREE_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "../include/atomic.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class KeySlice; 17 | class Page; 18 | class PoolManager; 19 | 20 | enum LockType { WriteLock, ReadLock }; 21 | 22 | class BLinkTree : private NoCopy 23 | { 24 | public: 25 | static const uint32_t MAX_KEY_LENGTH = 255; 26 | 27 | BLinkTree(uint32_t key_len); 28 | 29 | ~BLinkTree(); 30 | 31 | bool Put(KeySlice *key); 32 | 33 | bool Get(KeySlice *key); 34 | 35 | void Free(); 36 | 37 | private: 38 | struct Set { 39 | Set():depth_(0) { } 40 | page_t page_no_; 41 | Page *page_; 42 | page_t stack_[8]; 43 | uint32_t depth_; 44 | }; 45 | 46 | void DescendToLeaf(const KeySlice *key, Set &set, LockType type); 47 | 48 | bool Split(Set &set, KeySlice *key); 49 | 50 | void Insert(Set &set, KeySlice *key); 51 | 52 | PoolManager *pool_manager_; 53 | 54 | Atomic root_; 55 | 56 | uint8_t key_len_; 57 | uint16_t degree_; 58 | }; 59 | 60 | } // namespace Mushroom 61 | 62 | #endif /* _B_LINK_TREE_HPP_ */ -------------------------------------------------------------------------------- /src/blink/bounded_mapping_queue.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-05 22:48:10 6 | **/ 7 | 8 | #ifndef _BOUNDED_MAPPING_QUEUE_HPP_ 9 | #define _BOUNDED_MAPPING_QUEUE_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "../include/mutex.hpp" 13 | #include "../include/cond.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | template 18 | class BoundedMappingQueue : private NoCopyTemplate 19 | { 20 | public: 21 | BoundedMappingQueue(int capacity, const std::function &constructor); 22 | 23 | ~BoundedMappingQueue(); 24 | 25 | inline T* Get(); 26 | 27 | inline void Push(); 28 | 29 | inline T* Pop(int *pos); 30 | 31 | inline void Put(int pos); 32 | 33 | void Clear(); 34 | 35 | private: 36 | bool clear_; 37 | int capacity_; 38 | T* *queue_; 39 | int *avail_; 40 | int *work_; 41 | int front_; 42 | int avail_back_; 43 | int work_back_; 44 | Mutex mutex_; 45 | Cond ready_; 46 | Cond empty_; 47 | }; 48 | 49 | template 50 | BoundedMappingQueue::BoundedMappingQueue(int capacity,const std::function &constructor) 51 | :clear_(false), capacity_(capacity), front_(0), avail_back_(0), work_back_(0) 52 | { 53 | if (capacity_ <= 0) 54 | capacity_ = 8; 55 | if (capacity_ > 1024) 56 | capacity_ = 1024; 57 | 58 | queue_ = new T*[capacity_]; 59 | for (int i = 0; i != capacity_; ++i) 60 | queue_[i] = constructor(); 61 | 62 | avail_ = new int[capacity_]; 63 | for (int i = 0; i != capacity_; ++i) 64 | avail_[i] = i; 65 | 66 | work_ = new int[capacity_]; 67 | for (int i = 0; i != capacity_; ++i) 68 | work_[i] = -1; 69 | } 70 | 71 | template 72 | BoundedMappingQueue::~BoundedMappingQueue() 73 | { 74 | Clear(); 75 | 76 | delete [] avail_; 77 | delete [] work_; 78 | 79 | while (capacity_) delete queue_[--capacity_]; 80 | 81 | delete [] queue_; 82 | } 83 | 84 | template 85 | void BoundedMappingQueue::Clear() 86 | { 87 | mutex_.Lock(); 88 | 89 | if (clear_) { 90 | mutex_.Unlock(); 91 | return ; 92 | } 93 | 94 | while (front_ != avail_back_ || front_ != work_back_) 95 | empty_.Wait(mutex_); 96 | 97 | clear_ = true; 98 | 99 | mutex_.Unlock(); 100 | ready_.Broadcast(); 101 | } 102 | 103 | template 104 | inline T* BoundedMappingQueue::Get() 105 | { 106 | mutex_.Lock(); 107 | while (avail_[front_] < 0) 108 | empty_.Wait(mutex_); 109 | return queue_[avail_[front_]]; 110 | } 111 | 112 | template 113 | inline void BoundedMappingQueue::Push() 114 | { 115 | work_[front_] = avail_[front_]; 116 | 117 | avail_[front_] = -1; 118 | if (++front_ == capacity_) front_ = 0; 119 | 120 | mutex_.Unlock(); 121 | ready_.Signal(); 122 | } 123 | 124 | template 125 | inline T* BoundedMappingQueue::Pop(int *pos) 126 | { 127 | mutex_.Lock(); 128 | while (work_[work_back_] < 0 && !clear_) 129 | ready_.Wait(mutex_); 130 | 131 | if (clear_) { 132 | mutex_.Unlock(); 133 | *pos = -1; 134 | return 0; 135 | } 136 | 137 | *pos = work_[work_back_]; 138 | T *ret = queue_[*pos]; 139 | 140 | work_[work_back_] = -1; 141 | if (++work_back_ == capacity_) work_back_ = 0; 142 | 143 | mutex_.Unlock(); 144 | return ret; 145 | } 146 | 147 | template 148 | inline void BoundedMappingQueue::Put(int pos) 149 | { 150 | mutex_.Lock(); 151 | 152 | avail_[avail_back_] = pos; 153 | if (++avail_back_ == capacity_) avail_back_ = 0; 154 | 155 | mutex_.Unlock(); 156 | empty_.Signal(); 157 | } 158 | 159 | } // namespace Mushroom 160 | 161 | #endif /* _BOUNDED_MAPPING_QUEUE_HPP_ */ -------------------------------------------------------------------------------- /src/blink/db.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-10 15:34:43 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "db.hpp" 12 | #include "pool_manager.hpp" 13 | #include "b_link_tree.hpp" 14 | #include "pool_manager.hpp" 15 | #include "page.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | MushroomDB::MushroomDB(const char *name, uint32_t key_len, uint32_t page_size, 20 | uint32_t pool_size, uint32_t hash_bits, uint32_t seg_bits) 21 | { 22 | assert(name); 23 | 24 | PoolManager::SetManagerInfo(page_size, pool_size, hash_bits, seg_bits); 25 | 26 | tree_ = new BLinkTree(key_len); 27 | } 28 | 29 | MushroomDB::~MushroomDB() 30 | { 31 | delete tree_; 32 | } 33 | 34 | bool MushroomDB::Put(KeySlice *key) 35 | { 36 | return tree_->Put(key); 37 | } 38 | 39 | bool MushroomDB::Get(KeySlice *key) 40 | { 41 | return tree_->Get(key); 42 | } 43 | 44 | void MushroomDB::Close() 45 | { 46 | tree_->Free(); 47 | } 48 | 49 | } // namespace Mushroom 50 | -------------------------------------------------------------------------------- /src/blink/db.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-10 15:32:20 6 | **/ 7 | 8 | #ifndef _MUSHROOM_DB_HPP_ 9 | #define _MUSHROOM_DB_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class KeySlice; 16 | class BLinkTree; 17 | 18 | class MushroomDB : private NoCopy 19 | { 20 | public: 21 | MushroomDB(const char *name, uint32_t key_len, uint32_t page_size, uint32_t pool_size, 22 | uint32_t hash_bits, uint32_t seg_bits); 23 | 24 | bool Put(KeySlice *key); 25 | 26 | bool Get(KeySlice *key); 27 | 28 | void Close(); 29 | 30 | ~MushroomDB(); 31 | 32 | private: 33 | BLinkTree *tree_; 34 | }; 35 | 36 | } // namespace Mushroom 37 | 38 | #endif /* _MUSHROOM_DB_HPP_ */ -------------------------------------------------------------------------------- /src/blink/hash_entry.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-07-19 16:55:10 6 | **/ 7 | 8 | #include "../include/spin_lock.hpp" 9 | 10 | namespace Mushroom { 11 | 12 | class HashEntry { 13 | public: 14 | HashEntry():slot_(0) { } 15 | 16 | inline void Lock() { 17 | lock_.Lock(); 18 | } 19 | 20 | inline bool TryLock() { 21 | return lock_.TryLock(); 22 | } 23 | 24 | inline void Unlock() { 25 | lock_.Unlock(); 26 | } 27 | 28 | inline void SetSlot(uint16_t slot) { 29 | slot_ = slot; 30 | } 31 | 32 | inline uint16_t GetSlot() const { 33 | return slot_; 34 | } 35 | 36 | private: 37 | SpinLock lock_; 38 | uint16_t slot_; 39 | }; 40 | 41 | } // namespace Mushroom 42 | -------------------------------------------------------------------------------- /src/blink/log.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-29 14:22:34 6 | **/ 7 | 8 | #ifndef _MUSHROOM_LOG_HPP_ 9 | #define _MUSHROOM_LOG_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class KeySlice; 16 | 17 | class Log : private NoCopy 18 | { 19 | public: 20 | Log(uint8_t op, const KeySlice *key):op_(op), key_(key) { } 21 | 22 | private: 23 | uint8_t op_; 24 | const KeySlice *key_; 25 | }; 26 | 27 | } // namespace Mushroom 28 | 29 | #endif /* _MUSHROOM_LOG_HPP_ */ -------------------------------------------------------------------------------- /src/blink/page.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-22 09:10:01 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "page.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | uint32_t Page::PageSize = 4096; 16 | 17 | void Page::SetPageInfo(uint32_t page_size) 18 | { 19 | PageSize = page_size; 20 | } 21 | 22 | uint16_t Page::CalculateDegree(uint8_t key_len, uint8_t pre_len) 23 | { 24 | Page *page = 0; 25 | uint16_t offset = (char *)page->data_ - (char *)page + pre_len; 26 | return (PageSize - offset) / (KeySlice::ValLen + IndexByte + key_len); 27 | } 28 | 29 | Page::Page(page_t page_no, uint8_t type, uint8_t key_len, uint8_t level, uint16_t degree) 30 | { 31 | memset(this, 0, PageSize); 32 | latch_.Init(); 33 | page_no_ = page_no; 34 | degree_ = degree; 35 | type_ = (uint8_t)type; 36 | key_len_ = key_len; 37 | level_ = level; 38 | } 39 | 40 | void Page::InsertInfiniteKey() 41 | { 42 | TempSlice(key); 43 | memset(key->key_, 0xFF, key_len_); 44 | page_t page_no = 0; 45 | assert(Insert(key, page_no) == InsertOk); 46 | } 47 | 48 | void Page::AssignFirst(page_t first) 49 | { 50 | first_ = first; 51 | } 52 | 53 | bool Page::Traverse(const KeySlice *key, uint16_t *idx, KeySlice **slice, int type) const 54 | { 55 | uint16_t low = 0, high = total_key_, mid = 0; 56 | uint16_t *index = Index(); 57 | if (pre_len_) { 58 | int res = memcmp(key->key_, data_, pre_len_); 59 | if (res < 0) { 60 | *idx = 0; 61 | return false; 62 | } else if (res > 0) { 63 | *idx = high--; 64 | *slice = Key(index, high); 65 | return false; 66 | } 67 | } 68 | KeySlice *curr = 0; 69 | while (low != high) { 70 | mid = low + ((high - low) >> 1); 71 | curr = Key(index, mid); 72 | int res = memcmp(key->key_ + pre_len_, curr->key_, key_len_); 73 | if (res < 0) { 74 | high = mid; 75 | } else if (res > 0) { 76 | low = mid + 1; 77 | } else { 78 | if (type) { 79 | *idx = mid; 80 | *slice = Key(index, *idx); 81 | return true; 82 | } else { 83 | low = mid + 1; 84 | } 85 | } 86 | } 87 | *idx = high; 88 | if (high) *slice = Key(index, high-1); 89 | return false; 90 | } 91 | 92 | page_t Page::Descend(const KeySlice *key) const 93 | { 94 | uint16_t index; 95 | KeySlice *slice = 0; 96 | Traverse(key, &index, &slice, 0); 97 | return index ? slice->page_no_ : first_; 98 | } 99 | 100 | bool Page::Search(const KeySlice *key, uint16_t *index) const 101 | { 102 | KeySlice *slice = 0; 103 | return Traverse(key, index, &slice); 104 | } 105 | 106 | InsertStatus Page::Insert(const KeySlice *key, page_t &page_no) 107 | { 108 | uint16_t pos; 109 | KeySlice *slice = 0; 110 | bool flag = Traverse(key, &pos, &slice); 111 | if (flag) 112 | return ExistedKey; 113 | if (pos == total_key_ && pos) { 114 | page_no = Next(); 115 | assert(page_no); 116 | return MoveRight; 117 | } 118 | 119 | uint16_t end = total_key_ * (KeySlice::ValLen + key_len_) + pre_len_; 120 | memcpy(data_ + end, &key->page_no_, KeySlice::ValLen); 121 | memcpy(data_ + end + KeySlice::ValLen, key->key_ + pre_len_, key_len_); 122 | 123 | uint16_t *index = Index(); 124 | --index; 125 | if (pos) memmove(&index[0], &index[1], pos << 1); 126 | index[pos] = end; 127 | ++total_key_; 128 | return InsertOk; 129 | } 130 | 131 | void Page::Split(Page *that, KeySlice *slice) 132 | { 133 | uint16_t left = total_key_ >> 1, right = total_key_ - left, index = left; 134 | uint16_t *l_idx = this->Index(); 135 | uint16_t *r_idx = that->Index(); 136 | KeySlice *fence = Key(l_idx, left++); 137 | 138 | if (pre_len_) { 139 | memcpy(that->data_, this->data_, pre_len_); 140 | that->pre_len_ = this->pre_len_; 141 | memcpy(slice->key_, data_, pre_len_); 142 | } 143 | 144 | slice->page_no_ = that->page_no_; 145 | memcpy(slice->key_ + pre_len_, fence->key_, key_len_); 146 | 147 | if (level_) { 148 | that->AssignFirst(fence->page_no_); 149 | r_idx -= --right; 150 | ++index; 151 | } else { 152 | r_idx -= right; 153 | } 154 | 155 | uint16_t slot_len = KeySlice::ValLen + key_len_; 156 | for (uint16_t i = index, j = 0; i != total_key_; ++i, ++j) { 157 | r_idx[j] = that->pre_len_ + j * slot_len; 158 | KeySlice *l = this->Key(l_idx, i); 159 | KeySlice *r = that->Key(r_idx, j); 160 | memcpy(r, l, KeySlice::ValLen + key_len_); 161 | } 162 | 163 | fence->page_no_ = that->page_no_; 164 | 165 | uint16_t limit = left * (KeySlice::ValLen + this->key_len_) + this->pre_len_, j = 0; 166 | for (uint16_t i = left; i < total_key_ && j < left; ++i) { 167 | if (l_idx[i] < limit) { 168 | for (; j < left; ++j) { 169 | if (l_idx[j] >= limit) { 170 | KeySlice *o = Key(l_idx, i); 171 | KeySlice *n = Key(l_idx, j); 172 | l_idx[j] = l_idx[i]; 173 | memcpy(o, n, KeySlice::ValLen + key_len_); 174 | ++j; 175 | break; 176 | } 177 | } 178 | } 179 | } 180 | uint16_t offset = this->total_key_ - left; 181 | memmove(&l_idx[offset], &l_idx[0], left << 1); 182 | 183 | this->total_key_ = left; 184 | that->total_key_ = right; 185 | } 186 | 187 | bool Page::Full() const 188 | { 189 | return total_key_ == degree_; 190 | } 191 | 192 | bool Page::NeedSplit() 193 | { 194 | if (!Full()) return false; 195 | uint16_t *index = Index(); 196 | const char *first = Key(index, 0)->key_; 197 | const char *last = Key(index, total_key_ - 1)->key_; 198 | char prefix[key_len_]; 199 | uint8_t pre_len = 0; 200 | for (; first[pre_len] == last[pre_len]; ++pre_len) 201 | prefix[pre_len] = first[pre_len]; 202 | if (!pre_len) 203 | return true; 204 | uint16_t degree = CalculateDegree(key_len_ - pre_len, pre_len + pre_len_); 205 | if (degree <= degree_) 206 | return true; 207 | char buf[PageSize]; 208 | Page *copy = (Page *)buf; 209 | memcpy(copy, this, PageSize); 210 | memcpy(data_ + pre_len_, prefix, pre_len); 211 | char *curr = data_ + pre_len_ + pre_len; 212 | uint16_t *cindex = copy->Index(); 213 | uint16_t suf_len = key_len_ - pre_len; 214 | for (uint16_t i = 0; i != total_key_; ++i, ++index) { 215 | KeySlice *key = copy->Key(cindex, i); 216 | *index = curr - data_; 217 | memcpy(curr, &key->page_no_, KeySlice::ValLen); 218 | curr += KeySlice::ValLen; 219 | memcpy(curr, key->key_ + pre_len, suf_len); 220 | curr += suf_len; 221 | } 222 | pre_len_ += pre_len; 223 | key_len_ -= pre_len; 224 | degree_ = degree; 225 | return false; 226 | } 227 | 228 | std::string Page::ToString(bool f, bool f2) const 229 | { 230 | std::ostringstream os; 231 | os << "type: "; 232 | if (type_ == LEAF) os << "leaf "; 233 | if (type_ == BRANCH) os << "branch "; 234 | if (type_ == ROOT) os << "root "; 235 | os << "no: " << page_no_ << " "; 236 | os << "fir: " << first_ << " "; 237 | os << "tot: " << total_key_ << " "; 238 | os << "lvl: " << (int)level_ << " "; 239 | os << "keylen: " << (int)key_len_ << " "; 240 | os << "deg: " << degree_ << "\n"; 241 | 242 | if (pre_len_) { 243 | os << "pre_len: " << (int)pre_len_ << " "; 244 | os << "prefix: " << std::string(data_, pre_len_) << "\n"; 245 | } 246 | 247 | uint16_t *index = Index(); 248 | if (!f) { 249 | os << Key(index, 0)->ToString(key_len_); 250 | os << Key(index, total_key_ - 1)->ToString(key_len_); 251 | } else { 252 | // for (uint16_t i = 0; i != total_key_; ++i) 253 | // os << index[i] << " "; 254 | // os << "\n"; 255 | for (uint16_t i = 0; i != total_key_; ++i) { 256 | KeySlice *key = Key(index, i); 257 | if (f2) 258 | os << key->page_no_ << " "; 259 | os << key->ToString(key_len_); 260 | } 261 | } 262 | os << "\nnext: " << Key(index, total_key_-1)->page_no_ << "\n"; 263 | return os.str(); 264 | } 265 | 266 | } // namespace Mushroom 267 | -------------------------------------------------------------------------------- /src/blink/page.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-22 09:09:54 6 | **/ 7 | 8 | #ifndef _BTREE_PAGE_HPP_ 9 | #define _BTREE_PAGE_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../include/utility.hpp" 15 | #include "slice.hpp" 16 | #include "../include/latch.hpp" 17 | 18 | namespace Mushroom { 19 | 20 | typedef enum { InsertOk = 0x0, ExistedKey = 0x1, MoveRight = 0x2 } InsertStatus; 21 | 22 | typedef enum { UpdateOk = 0x0, Promote = 0x1, MoveNext = 0x2 } UpdateStatus; 23 | 24 | class KeySlice; 25 | class BLinkTree; 26 | 27 | class Page : private NoCopy 28 | { 29 | friend class BLinkTree; 30 | public: 31 | static uint32_t PageSize; 32 | 33 | static enum { ROOT = 1, BRANCH = 2, LEAF = 4 } Type; 34 | 35 | static const uint32_t IndexByte = 2; 36 | 37 | static void SetPageInfo(uint32_t page_size); 38 | 39 | static uint16_t CalculateDegree(uint8_t key_len, uint8_t pre_len = 0); 40 | 41 | void InsertInfiniteKey(); 42 | 43 | Page(page_t id, uint8_t type, uint8_t key_len, uint8_t level, uint16_t degree); 44 | 45 | inline page_t Next() const { 46 | KeySlice *key = (KeySlice *)(data_ + Index()[total_key_-1]); 47 | return key->page_no_; 48 | } 49 | 50 | inline void SetPageNo(page_t page_no) { page_no_ = page_no; } 51 | 52 | inline page_t PageNo() const { return page_no_; } 53 | 54 | inline uint16_t TotalKey() const { return total_key_; } 55 | 56 | inline uint16_t Degree() const { return degree_; } 57 | 58 | void AssignFirst(page_t first); 59 | 60 | page_t Descend(const KeySlice *key) const; 61 | 62 | bool Search(const KeySlice *key, uint16_t *index) const; 63 | 64 | InsertStatus Insert(const KeySlice *key, page_t &page_no); 65 | 66 | void Insert(Page *that, KeySlice *key); 67 | 68 | void Split(Page *that, KeySlice *slice); 69 | 70 | bool Full() const; 71 | 72 | bool NeedSplit(); 73 | 74 | inline Latch* GetLatch() { return &latch_; } 75 | 76 | inline void LockShared() { latch_.ReadLock(); } 77 | 78 | inline void Lock() { latch_.WriteLock(); } 79 | 80 | inline void UnlockShared() { latch_.Unlock(); } 81 | 82 | inline void Unlock() { latch_.Unlock(); } 83 | 84 | inline uint16_t* Index() const { 85 | return (uint16_t *)((char *)this + (PageSize - (total_key_ * IndexByte))); 86 | } 87 | 88 | inline KeySlice* Key(const uint16_t *index, uint16_t pos) const { 89 | return (KeySlice *)(data_ + index[pos]); 90 | } 91 | 92 | std::string ToString(bool f, bool f2) const; 93 | 94 | private: 95 | bool Traverse(const KeySlice *key, uint16_t *idx, KeySlice **slice, int type = 1) const; 96 | 97 | Latch latch_; 98 | page_t page_no_; 99 | page_t first_; 100 | uint16_t total_key_; 101 | uint16_t degree_; 102 | uint8_t type_; 103 | uint8_t level_; 104 | uint8_t key_len_; 105 | uint8_t pre_len_; 106 | char data_[0]; 107 | }; 108 | 109 | } // namespace Mushroom 110 | 111 | #endif /* _BTREE_PAGE_HPP_ */ -------------------------------------------------------------------------------- /src/blink/page_pool.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-03-25 12:58:32 6 | **/ 7 | 8 | #include "page_pool.hpp" 9 | 10 | namespace Mushroom { 11 | 12 | uint32_t PagePool::SegBits; 13 | uint32_t PagePool::SegSize; 14 | uint32_t PagePool::SegMask; 15 | 16 | } // namespace Mushroom 17 | -------------------------------------------------------------------------------- /src/blink/page_pool.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-03-25 12:56:16 6 | **/ 7 | 8 | #ifndef _PAGE_POOL_HPP_ 9 | #define _PAGE_POOL_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | #include "page.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | class PoolManager; 19 | 20 | class PagePool : private NoCopy 21 | { 22 | friend class PoolManager; 23 | 24 | public: 25 | static void SetPoolInfo(uint32_t seg_bits) { 26 | SegBits = seg_bits; 27 | SegSize = 1 << seg_bits; 28 | SegMask = SegSize - 1; 29 | } 30 | 31 | PagePool() { } 32 | 33 | inline void Initialize(page_t base) { 34 | base_ = base; 35 | mem_ = new char[Page::PageSize << SegBits]; 36 | memset(mem_, 0, Page::PageSize << SegBits); 37 | } 38 | 39 | inline Page* GetPage(page_t page_no) { 40 | return (Page *)(mem_ + Page::PageSize * (page_no & SegMask)); 41 | } 42 | 43 | inline void Destroy() { 44 | delete [] mem_; 45 | } 46 | 47 | static uint32_t SegSize; 48 | private: 49 | static uint32_t SegBits; 50 | static uint32_t SegMask; 51 | 52 | page_t base_; 53 | char *mem_; 54 | PagePool *next_; 55 | }; 56 | 57 | } // namespace Mushroom 58 | 59 | #endif /* _PAGE_POOL_HPP_ */ -------------------------------------------------------------------------------- /src/blink/pool_manager.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-03-19 13:13:52 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "hash_entry.hpp" 14 | #include "page.hpp" 15 | #include "page_pool.hpp" 16 | #include "pool_manager.hpp" 17 | 18 | namespace Mushroom { 19 | 20 | uint32_t PoolManager::HashMask; 21 | uint32_t PoolManager::PoolSize; 22 | 23 | void PoolManager::SetManagerInfo(uint32_t page_size, uint32_t pool_size, uint32_t hash_bits, 24 | uint32_t seg_bits) 25 | { 26 | Page::SetPageInfo(page_size); 27 | PagePool::SetPoolInfo(seg_bits); 28 | PoolSize = pool_size; 29 | HashMask = (1 << hash_bits) - 1; 30 | } 31 | 32 | PoolManager::PoolManager():cur_page_(0), total_pool_(0) 33 | { 34 | entries_ = new HashEntry[HashMask+1]; 35 | pool_ = new PagePool[PoolSize]; 36 | } 37 | 38 | PoolManager::~PoolManager() 39 | { 40 | delete [] pool_; 41 | delete [] entries_; 42 | } 43 | 44 | void PoolManager::Link(uint16_t hash, uint16_t victim) 45 | { 46 | PagePool *pool = pool_ + victim; 47 | 48 | uint16_t slot = entries_[hash].GetSlot(); 49 | if (slot) 50 | pool->next_ = pool_ + slot; 51 | 52 | entries_[hash].SetSlot(victim); 53 | } 54 | 55 | Page* PoolManager::GetPage(page_t page_no) 56 | { 57 | page_t base = page_no & ~PagePool::SegMask; 58 | page_t hash = (page_no >> PagePool::SegBits) & HashMask; 59 | Page *page = 0; 60 | 61 | entries_[hash].Lock(); 62 | 63 | uint16_t slot = entries_[hash].GetSlot(); 64 | if (slot) { 65 | PagePool *pool = pool_ + slot; 66 | for (; pool; pool = pool->next_) 67 | if (pool->base_ == base) 68 | break; 69 | if (pool) { 70 | page = pool->GetPage(page_no); 71 | entries_[hash].Unlock(); 72 | return page; 73 | } 74 | } 75 | 76 | uint16_t victim = ++total_pool_; 77 | assert(victim < PoolSize); 78 | 79 | pool_[victim].Initialize(base); 80 | Link(hash, victim); 81 | page = pool_[victim].GetPage(page_no); 82 | entries_[hash].Unlock(); 83 | return page; 84 | } 85 | 86 | Page* PoolManager::NewPage(uint8_t type, uint8_t key_len, uint8_t level, uint16_t degree) 87 | { 88 | page_t page_no = cur_page_++; 89 | Page *page = GetPage(page_no); 90 | return new (page) Page(page_no, type, key_len, level, degree); 91 | } 92 | 93 | void PoolManager::Free() 94 | { 95 | for (uint16_t i = 1, end = total_pool_.get(); i <= end; ++i) 96 | pool_[i].Destroy(); 97 | } 98 | 99 | } // namespace Mushroom 100 | -------------------------------------------------------------------------------- /src/blink/pool_manager.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-03-19 13:09:33 6 | **/ 7 | 8 | #ifndef _POOL_MANAGER_HPP_ 9 | #define _POOL_MANAGER_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "../include/atomic.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class Page; 17 | class HashEntry; 18 | class PagePool; 19 | 20 | class PoolManager : private NoCopy 21 | { 22 | public: 23 | static void SetManagerInfo(uint32_t page_size, uint32_t pool_size, uint32_t hash_bits, 24 | uint32_t seg_bits); 25 | 26 | PoolManager(); 27 | 28 | ~PoolManager(); 29 | 30 | Page* GetPage(page_t page_no); 31 | 32 | Page* NewPage(uint8_t type, uint8_t key_len, uint8_t level, uint16_t degree); 33 | 34 | void Free(); 35 | 36 | private: 37 | static uint32_t HashMask; 38 | static uint32_t PoolSize; 39 | 40 | void Link(uint16_t hash, uint16_t victim); 41 | 42 | Atomic cur_page_; 43 | atomic_16_t total_pool_; 44 | HashEntry *entries_; 45 | PagePool *pool_; 46 | }; 47 | 48 | } // namespace Mushroom 49 | 50 | #endif /* _POOL_MANAGER_HPP_ */ -------------------------------------------------------------------------------- /src/blink/slice.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-09 15:37:50 6 | **/ 7 | 8 | #ifndef _SLICE_HPP_ 9 | #define _SLICE_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../include/utility.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | class KeySlice : private NoCopy 19 | { 20 | public: 21 | static const uint32_t KeyLen = 16; 22 | static const uint32_t ValLen = sizeof(page_t); 23 | static const uint32_t KeySize = KeyLen + ValLen; 24 | 25 | std::string ToString(uint32_t len = KeyLen) const { 26 | return std::string(key_, len) + "\n"; 27 | } 28 | 29 | page_t page_no_; 30 | char key_[0]; 31 | }; 32 | 33 | inline KeySlice* NewKeySlice() { 34 | return (KeySlice *)new char[KeySlice::KeySize]; 35 | } 36 | 37 | inline void DeleteKeySlice(KeySlice *key) { 38 | delete [] (char *)key; 39 | } 40 | 41 | #define TempSlice(name) \ 42 | char buf_##name[KeySlice::KeySize]; \ 43 | memset(buf_##name, 0, KeySlice::KeySize); \ 44 | KeySlice *name = (KeySlice *)buf_##name; 45 | 46 | } // namespace Mushroom 47 | 48 | #endif /* _SLICE_HPP_ */ -------------------------------------------------------------------------------- /src/blink/task.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-18 15:30:39 6 | **/ 7 | 8 | #ifndef _MUSHROOM_TASK_HPP_ 9 | #define _MUSHROOM_TASK_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "slice.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class MushroomDB; 17 | 18 | class MushroomTask : private NoCopy 19 | { 20 | public: 21 | MushroomTask():fun_(0), db_(0), key_(NewKeySlice()) { } 22 | 23 | ~MushroomTask() { DeleteKeySlice(key_); } 24 | 25 | inline void Assign(bool (MushroomDB::*(fun))(KeySlice *), MushroomDB *db, KeySlice *key) { 26 | fun_ = fun; 27 | db_ = db; 28 | memcpy(key_, key, KeySlice::KeySize); 29 | } 30 | 31 | bool operator()() { return (db_->*fun_)(key_); } 32 | 33 | private: 34 | bool (MushroomDB::*(fun_))(KeySlice *); 35 | MushroomDB *db_; 36 | KeySlice *key_; 37 | }; 38 | 39 | } // namespace Mushroom 40 | 41 | #endif /* _MUSHROOM_TASK_HPP_ */ -------------------------------------------------------------------------------- /src/blink/thread_pool_mapping.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-10-16 18:51:28 6 | **/ 7 | 8 | #ifndef _THREAD_POOL_MAPPING_HPP_ 9 | #define _THREAD_POOL_MAPPING_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "../include/thread.hpp" 13 | #include "bounded_mapping_queue.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | template 18 | class ThreadPoolMapping : private NoCopyTemplate 19 | { 20 | public: 21 | ThreadPoolMapping(BoundedMappingQueue *queue, int thread_num); 22 | 23 | ~ThreadPoolMapping(); 24 | 25 | void Run(); 26 | 27 | void Clear(); 28 | 29 | private: 30 | bool working_; 31 | int thread_num_; 32 | BoundedMappingQueue *queue_; 33 | Thread **threads_; 34 | }; 35 | 36 | template 37 | ThreadPoolMapping::ThreadPoolMapping(BoundedMappingQueue *queue, int thread_num) 38 | :working_(false), thread_num_(thread_num), queue_(queue) 39 | { 40 | if (thread_num_ <= 0) 41 | thread_num_ = 1; 42 | if (thread_num_ > 4) 43 | thread_num_ = 4; 44 | 45 | threads_ = new Thread*[thread_num_]; 46 | 47 | working_ = true; 48 | 49 | for (int i = 0; i != thread_num_; ++i) { 50 | threads_[i] = new Thread([this] { this->Run(); }); 51 | threads_[i]->Start(); 52 | } 53 | } 54 | 55 | template 56 | void ThreadPoolMapping::Run() 57 | { 58 | for (;;) { 59 | int pos; 60 | T* task = queue_->Pop(&pos); 61 | 62 | if (!task) break; 63 | 64 | (*task)(); 65 | 66 | queue_->Put(pos); 67 | } 68 | } 69 | 70 | template 71 | void ThreadPoolMapping::Clear() 72 | { 73 | if (!working_) return ; 74 | 75 | queue_->Clear(); 76 | 77 | working_ = false; 78 | 79 | for (int i = 0; i != thread_num_; ++i) 80 | threads_[i]->Stop(); 81 | } 82 | 83 | template 84 | ThreadPoolMapping::~ThreadPoolMapping() 85 | { 86 | Clear(); 87 | 88 | for (int i = 0; i != thread_num_; ++i) 89 | delete threads_[i]; 90 | 91 | delete [] threads_; 92 | } 93 | 94 | } // namespace Mushroom 95 | 96 | #endif /* _THREAD_POOL_MAPPING_HPP_ */ -------------------------------------------------------------------------------- /src/include/atomic.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-04 12:54:54 6 | **/ 7 | 8 | #ifndef _ATOMIC_HPP_ 9 | #define _ATOMIC_HPP_ 10 | 11 | #include 12 | 13 | namespace Mushroom { 14 | 15 | template 16 | class Atomic 17 | { 18 | public: 19 | Atomic() { } 20 | 21 | Atomic(T val):val_(val) { } 22 | 23 | inline T get() { 24 | return __sync_val_compare_and_swap(&val_, 0, 0); 25 | } 26 | 27 | inline T operator++() { 28 | return __sync_add_and_fetch(&val_, 1); 29 | } 30 | 31 | inline T operator--() { 32 | return __sync_sub_and_fetch(&val_, 1); 33 | } 34 | 35 | inline T operator++(int) { 36 | return __sync_fetch_and_add(&val_, 1); 37 | } 38 | 39 | inline T operator--(int) { 40 | return __sync_fetch_and_sub(&val_, 1); 41 | } 42 | 43 | inline Atomic& operator=(T new_val) { 44 | __sync_val_compare_and_swap(&val_, val_, new_val); 45 | return *this; 46 | } 47 | 48 | private: 49 | volatile T val_; 50 | }; 51 | 52 | typedef Atomic atomic_8_t; 53 | typedef Atomic atomic_16_t; 54 | typedef Atomic atomic_32_t; 55 | 56 | } // namespace Mushroom 57 | 58 | #endif /* _ATOMIC_HPP_ */ -------------------------------------------------------------------------------- /src/include/bounded_list.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-14 16:09:39 6 | **/ 7 | 8 | #ifndef _BOUNDED_LIST_HPP_ 9 | #define _BOUNDED_LIST_HPP_ 10 | 11 | #include 12 | 13 | #include "utility.hpp" 14 | #include "mutex.hpp" 15 | #include "cond.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | template 20 | class BoundedList : private NoCopyTemplate 21 | { 22 | public: 23 | BoundedList(int capacity, const std::function &constructor); 24 | 25 | ~BoundedList(); 26 | 27 | inline T* Get(); 28 | 29 | inline void Push(T *t); 30 | 31 | inline T* Pop(); 32 | 33 | inline void Put(T *); 34 | 35 | void Clear(); 36 | 37 | private: 38 | bool clear_; 39 | int capacity_; 40 | std::list free_; 41 | std::list busy_; 42 | Mutex free_mutex_; 43 | Mutex busy_mutex_; 44 | Cond free_cond_; 45 | Cond busy_cond_; 46 | }; 47 | 48 | template 49 | BoundedList::BoundedList(int capacity, const std::function &constructor) 50 | :clear_(false), capacity_(capacity) 51 | { 52 | if (capacity_ <= 0) 53 | capacity_ = 8; 54 | if (capacity_ > 1024) 55 | capacity_ = 1024; 56 | 57 | for (int i = 0; i < capacity_; ++i) 58 | free_.push_back(constructor()); 59 | } 60 | 61 | template 62 | BoundedList::~BoundedList() 63 | { 64 | Clear(); 65 | 66 | for (auto e : free_) 67 | delete e; 68 | } 69 | 70 | template 71 | void BoundedList::Clear() 72 | { 73 | free_mutex_.Lock(); 74 | 75 | if (clear_) { 76 | free_mutex_.Unlock(); 77 | return ; 78 | } 79 | 80 | while (int(free_.size()) != capacity_) 81 | free_cond_.Wait(free_mutex_); 82 | 83 | clear_ = true; 84 | 85 | free_mutex_.Unlock(); 86 | busy_cond_.Broadcast(); 87 | } 88 | 89 | template 90 | inline T* BoundedList::Get() 91 | { 92 | free_mutex_.Lock(); 93 | while (free_.empty()) 94 | free_cond_.Wait(free_mutex_); 95 | T* t = free_.front(); 96 | free_.pop_front(); 97 | free_mutex_.Unlock(); 98 | return t; 99 | } 100 | 101 | template 102 | inline void BoundedList::Push(T *t) 103 | { 104 | busy_mutex_.Lock(); 105 | busy_.push_back(t); 106 | busy_mutex_.Unlock(); 107 | busy_cond_.Signal(); 108 | } 109 | 110 | template 111 | inline T* BoundedList::Pop() 112 | { 113 | busy_mutex_.Lock(); 114 | while (busy_.empty() && !clear_) 115 | busy_cond_.Wait(busy_mutex_); 116 | if (clear_) { 117 | busy_mutex_.Unlock(); 118 | return 0; 119 | } 120 | T* t = busy_.front(); 121 | busy_.pop_front(); 122 | busy_mutex_.Unlock(); 123 | return t; 124 | } 125 | 126 | template 127 | inline void BoundedList::Put(T *t) 128 | { 129 | free_mutex_.Lock(); 130 | free_.push_back(t); 131 | free_mutex_.Unlock(); 132 | free_cond_.Signal(); 133 | } 134 | 135 | } // namespace Mushroom 136 | 137 | #endif /* _BOUNDED_LIST_HPP_ */ -------------------------------------------------------------------------------- /src/include/bounded_queue.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-09 21:14:07 6 | **/ 7 | 8 | #ifndef _BOUNDED_QUEUE_HPP_ 9 | #define _BOUNDED_QUEUE_HPP_ 10 | 11 | #include "utility.hpp" 12 | #include "mutex.hpp" 13 | #include "cond.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | template 18 | class BoundedQueue : private NoCopyTemplate 19 | { 20 | public: 21 | BoundedQueue(int capacity); 22 | 23 | ~BoundedQueue(); 24 | 25 | inline void Push(T &&); 26 | 27 | inline T Pop(); 28 | 29 | inline void Clear(); 30 | 31 | private: 32 | bool clear_; 33 | int beg_; 34 | int end_; 35 | int capacity_; 36 | T *queue_; 37 | Mutex mutex_; 38 | Cond empty_; 39 | Cond full_; 40 | }; 41 | 42 | template 43 | BoundedQueue::BoundedQueue(int capacity):clear_(false), beg_(0), end_(0),capacity_(capacity) 44 | { 45 | if (capacity_ <= 0) 46 | capacity_ = 8; 47 | if (capacity_ > 1024) 48 | capacity_ = 1024; 49 | 50 | queue_ = new T[capacity_]; 51 | } 52 | 53 | template 54 | BoundedQueue::~BoundedQueue() 55 | { 56 | delete [] queue_; 57 | } 58 | 59 | template 60 | inline void BoundedQueue::Clear() 61 | { 62 | mutex_.Lock(); 63 | if (clear_) { 64 | mutex_.Unlock(); 65 | return ; 66 | } 67 | clear_ = true; 68 | empty_.Broadcast(); 69 | mutex_.Unlock(); 70 | } 71 | 72 | template 73 | inline void BoundedQueue::Push(T &&t) 74 | { 75 | mutex_.Lock(); 76 | while (((end_+1)%capacity_) == beg_) 77 | full_.Wait(mutex_); 78 | queue_[end_] = t; 79 | if (++end_ == capacity_) 80 | end_ = 0; 81 | mutex_.Unlock(); 82 | empty_.Signal(); 83 | } 84 | 85 | template 86 | inline T BoundedQueue::Pop() 87 | { 88 | mutex_.Lock(); 89 | while (beg_ == end_ && !clear_) 90 | empty_.Wait(mutex_); 91 | if (clear_) { 92 | mutex_.Unlock(); 93 | return T(); 94 | } 95 | T t = std::move(queue_[beg_]); 96 | if (++beg_ == capacity_) 97 | beg_ = 0; 98 | mutex_.Unlock(); 99 | full_.Signal(); 100 | return t; 101 | } 102 | 103 | } // namespace Mushroom 104 | 105 | #endif /* _BOUNDED_QUEUE_HPP_ */ -------------------------------------------------------------------------------- /src/include/cond.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-05 22:54:44 6 | **/ 7 | 8 | #ifndef _COND_HPP_ 9 | #define _COND_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "utility.hpp" 15 | #include "mutex.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | class Cond : private NoCopy 20 | { 21 | public: 22 | Cond() { 23 | assert(!pthread_cond_init(cond_, 0)); 24 | } 25 | 26 | inline void Wait(Mutex &mutex) { 27 | pthread_cond_wait(cond_, mutex.mutex_); 28 | } 29 | 30 | inline void Signal() { 31 | pthread_cond_signal(cond_); 32 | } 33 | 34 | inline void Broadcast() { 35 | pthread_cond_broadcast(cond_); 36 | } 37 | 38 | inline bool TimedWait(Mutex &mutex, int64_t millisecond) { 39 | struct timeval tv; 40 | gettimeofday(&tv, 0); 41 | timespec abstime; 42 | abstime.tv_sec = tv.tv_sec; 43 | int64_t nsec = int64_t(tv.tv_usec) * 1000 + millisecond * 1000000; 44 | if (nsec >= 1000000000) { 45 | ++abstime.tv_sec; 46 | nsec -= 1000000000; 47 | } 48 | abstime.tv_nsec = nsec; 49 | return pthread_cond_timedwait(cond_, mutex.mutex_, &abstime) == ETIMEDOUT; 50 | } 51 | 52 | ~Cond() { 53 | assert(!pthread_cond_destroy(cond_)); 54 | } 55 | 56 | private: 57 | pthread_cond_t cond_[1]; 58 | }; 59 | 60 | } // namespace Mushroom 61 | 62 | #endif /* _COND_HPP_ */ -------------------------------------------------------------------------------- /src/include/guard.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-19 10:24:14 6 | **/ 7 | 8 | #ifndef _GUARD_HPP_ 9 | #define _GUARD_HPP_ 10 | 11 | #include "utility.hpp" 12 | #include "mutex.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class Guard : private NoCopy 17 | { 18 | public: 19 | Guard(Mutex &mutex):mutex_(mutex) { mutex_.Lock(); } 20 | 21 | ~Guard() { mutex_.Unlock(); } 22 | 23 | private: 24 | Mutex &mutex_; 25 | }; 26 | 27 | } // namespace Mushroom 28 | 29 | #endif /* _GUARD_HPP_ */ -------------------------------------------------------------------------------- /src/include/latch.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-09-05 16:03:53 6 | **/ 7 | 8 | #ifndef _LATCH_HPP_ 9 | #define _LATCH_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "utility.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | class Latch : private NoCopy { 19 | public: 20 | Latch() { } 21 | 22 | inline void Init() { 23 | assert(!pthread_rwlock_init(latch_, 0)); 24 | } 25 | 26 | inline void ReadLock() { 27 | pthread_rwlock_rdlock(latch_); 28 | } 29 | 30 | inline void WriteLock() { 31 | pthread_rwlock_wrlock(latch_); 32 | } 33 | 34 | inline void Unlock() { 35 | pthread_rwlock_unlock(latch_); 36 | } 37 | 38 | inline void Destroy() { 39 | assert(!pthread_rwlock_destroy(latch_)); 40 | } 41 | 42 | private: 43 | pthread_rwlock_t latch_[1]; 44 | }; 45 | 46 | } // namespace Mushroom 47 | 48 | #endif /* _LATCH_HPP_ */ -------------------------------------------------------------------------------- /src/include/mutex.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-05 22:53:02 6 | **/ 7 | 8 | #ifndef _MUTEX_HPP_ 9 | #define _MUTEX_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "utility.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | class Cond; 19 | 20 | class Mutex : private NoCopy 21 | { 22 | friend class Cond; 23 | public: 24 | Mutex() { 25 | assert(!pthread_mutex_init(mutex_, 0)); 26 | } 27 | 28 | inline void Lock() { 29 | pthread_mutex_lock(mutex_); 30 | } 31 | 32 | inline void Unlock() { 33 | pthread_mutex_unlock(mutex_); 34 | } 35 | 36 | ~Mutex() { 37 | assert(!pthread_mutex_destroy(mutex_)); 38 | } 39 | 40 | private: 41 | pthread_mutex_t mutex_[1]; 42 | }; 43 | 44 | } // namespace Mushroom 45 | 46 | #endif /* _MUTEX_HPP_ */ -------------------------------------------------------------------------------- /src/include/spin_lock.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-05 22:56:41 6 | **/ 7 | 8 | #ifndef _SPIN_LOCK_HPP_ 9 | #define _SPIN_LOCK_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "utility.hpp" 15 | 16 | #ifdef __APPLE__ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #ifndef PTHREAD_PROCESS_SHARED 23 | #define PTHREAD_PROCESS_SHARED 1 24 | #endif 25 | #ifndef PTHREAD_PROCESS_PRIVATE 26 | #define PTHREAD_PROCESS_PRIVATE 2 27 | #endif 28 | 29 | #ifndef UNUSED 30 | #define UNUSED(expr) (void)(expr) 31 | #endif 32 | 33 | typedef std::atomic_flag pthread_spinlock_t; 34 | 35 | static inline int pthread_spin_destroy(pthread_spinlock_t *lock) { 36 | UNUSED(lock); 37 | return 0; 38 | } 39 | 40 | static inline int pthread_spin_lock(pthread_spinlock_t *lock) { 41 | while (lock->test_and_set(std::memory_order_acquire)) 42 | std::this_thread::yield(); 43 | return 0; 44 | } 45 | 46 | static inline int pthread_spin_trylock(pthread_spinlock_t *lock) { 47 | if (lock->test_and_set(std::memory_order_acquire)) 48 | return 0; 49 | else 50 | return EBUSY; 51 | } 52 | 53 | static inline int pthread_spin_unlock(pthread_spinlock_t *lock) { 54 | lock->clear(std::memory_order_release); 55 | return 0; 56 | } 57 | 58 | static inline int pthread_spin_init(pthread_spinlock_t *lock, int pshared) { 59 | UNUSED(pshared); 60 | pthread_spin_unlock(lock); 61 | return 0; 62 | } 63 | 64 | #endif /* __APPLE__ */ 65 | 66 | namespace Mushroom { 67 | 68 | class SpinLock : private NoCopy 69 | { 70 | public: 71 | SpinLock() { 72 | assert(!pthread_spin_init(lock_, 0)); 73 | } 74 | 75 | inline void Lock() { 76 | pthread_spin_lock(lock_); 77 | } 78 | 79 | inline bool TryLock() { 80 | return !pthread_spin_trylock(lock_); 81 | } 82 | 83 | inline void Unlock() { 84 | pthread_spin_unlock(lock_); 85 | } 86 | 87 | ~SpinLock() { 88 | assert(!pthread_spin_destroy(lock_)); 89 | } 90 | 91 | private: 92 | pthread_spinlock_t lock_[1]; 93 | }; 94 | 95 | } // namespace Mushroom 96 | 97 | #endif /* _SPIN_LOCK_HPP_ */ -------------------------------------------------------------------------------- /src/include/thread.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-04 09:18:22 6 | **/ 7 | 8 | #ifndef _THREAD_HPP_ 9 | #define _THREAD_HPP_ 10 | 11 | #include 12 | 13 | #include "utility.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | class Thread : private NoCopy 18 | { 19 | public: 20 | Thread(const Func &func):func_(func) { } 21 | 22 | inline void Start() { 23 | std::thread tmp(func_); 24 | thread_.swap(tmp); 25 | } 26 | 27 | inline void Stop() { 28 | thread_.join(); 29 | } 30 | 31 | private: 32 | Func func_; 33 | std::thread thread_; 34 | }; 35 | 36 | } // namespace Mushroom 37 | 38 | #endif /* _THREAD_HPP_ */ -------------------------------------------------------------------------------- /src/include/thread_pool.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-09 21:16:59 6 | **/ 7 | 8 | #ifndef _THREAD_POOL_HPP_ 9 | #define _THREAD_POOL_HPP_ 10 | 11 | #include "utility.hpp" 12 | #include "bounded_queue.hpp" 13 | #include "thread.hpp" 14 | #include "atomic.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | template 19 | class ThreadPool : private NoCopyTemplate 20 | { 21 | public: 22 | ThreadPool(BoundedQueue *queue, int thread_num); 23 | 24 | ~ThreadPool(); 25 | 26 | void Run(); 27 | 28 | void Clear(); 29 | 30 | private: 31 | atomic_8_t working_; 32 | int thread_num_; 33 | BoundedQueue *queue_; 34 | Thread **threads_; 35 | }; 36 | 37 | template 38 | ThreadPool::ThreadPool(BoundedQueue *queue, int thread_num) 39 | :working_(0), thread_num_(thread_num), queue_(queue) 40 | { 41 | if (thread_num_ <= 0) 42 | thread_num_ = 1; 43 | else if (thread_num_ > 8) 44 | thread_num_ = 8; 45 | 46 | threads_ = new Thread*[thread_num_]; 47 | 48 | for (int i = 0; i != thread_num_; ++i) { 49 | threads_[i] = new Thread([this] { this->Run(); }); 50 | threads_[i]->Start(); 51 | } 52 | working_ = 1; 53 | } 54 | 55 | template 56 | void ThreadPool::Run() 57 | { 58 | for (;;) { 59 | T task = queue_->Pop(); 60 | 61 | if (!working_.get()) 62 | break; 63 | 64 | task(); 65 | } 66 | } 67 | 68 | template 69 | void ThreadPool::Clear() 70 | { 71 | if (!working_.get()) 72 | return ; 73 | 74 | working_ = 0; 75 | 76 | queue_->Clear(); 77 | 78 | for (int i = 0; i != thread_num_; ++i) 79 | threads_[i]->Stop(); 80 | } 81 | 82 | template 83 | ThreadPool::~ThreadPool() 84 | { 85 | for (int i = 0; i != thread_num_; ++i) 86 | delete threads_[i]; 87 | 88 | delete [] threads_; 89 | } 90 | 91 | } // namespace Mushroom 92 | 93 | #endif /* _THREAD_POOL_HPP_ */ -------------------------------------------------------------------------------- /src/include/utility.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-06 22:38:46 6 | **/ 7 | 8 | #ifndef _UTILITY_HPP_ 9 | #define _UTILITY_HPP_ 10 | 11 | #include 12 | 13 | namespace Mushroom { 14 | 15 | class NoCopy { 16 | public: 17 | NoCopy() = default; 18 | private: 19 | NoCopy(const NoCopy &) = delete; 20 | NoCopy& operator=(const NoCopy &) = delete; 21 | }; 22 | 23 | template 24 | class NoCopyTemplate 25 | { 26 | public: 27 | NoCopyTemplate() = default; 28 | private: 29 | NoCopyTemplate(const NoCopyTemplate &) = delete; 30 | NoCopyTemplate& operator=(const NoCopyTemplate &) = delete; 31 | }; 32 | 33 | typedef uint32_t valptr; 34 | typedef uint32_t page_t; 35 | 36 | typedef std::function Task; 37 | typedef std::function Func; 38 | 39 | } // namespace Mushroom 40 | 41 | #endif /* _UTILITY_HPP_ */ -------------------------------------------------------------------------------- /src/network/buffer.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-27 09:44:47 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "buffer.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | Buffer::Buffer():data_(new char[4096]), cap_(4096) 16 | { 17 | memset(data_, 0, cap_); 18 | Clear(); 19 | } 20 | 21 | Buffer::~Buffer() 22 | { 23 | delete [] data_; 24 | } 25 | 26 | uint32_t Buffer::size() const 27 | { 28 | return size_; 29 | } 30 | 31 | bool Buffer::empty() const 32 | { 33 | return !size_; 34 | } 35 | 36 | char* Buffer::data() const 37 | { 38 | data_[end_] = '\0'; 39 | return data_ + beg_; 40 | } 41 | 42 | char* Buffer::begin() const 43 | { 44 | return data_ + beg_; 45 | } 46 | 47 | char* Buffer::end() const 48 | { 49 | return data_ + end_; 50 | } 51 | 52 | uint32_t Buffer::space() const 53 | { 54 | return cap_ - end_; 55 | } 56 | 57 | void Buffer::Clear() 58 | { 59 | beg_ = 0; 60 | end_ = 0; 61 | size_ = 0; 62 | } 63 | 64 | void Buffer::Reset() 65 | { 66 | if (empty()) { 67 | assert(beg_ == end_); 68 | beg_ = 0; 69 | end_ = 0; 70 | size_ = 0; 71 | } 72 | } 73 | 74 | void Buffer::Adjust() 75 | { 76 | memmove(data_, begin(), size_); 77 | beg_ = 0; 78 | end_ = size_; 79 | } 80 | 81 | void Buffer::AdvanceHead(uint32_t len) 82 | { 83 | assert(size_ >= len); 84 | beg_ += len; 85 | size_ -= len; 86 | } 87 | 88 | void Buffer::AdvanceTail(uint32_t len) 89 | { 90 | assert(end_ + len <= cap_); 91 | end_ += len; 92 | size_ += len; 93 | } 94 | 95 | void Buffer::Unget(uint32_t len) 96 | { 97 | assert(beg_ >= len); 98 | beg_ -= len; 99 | size_ += len; 100 | } 101 | 102 | void Buffer::Read(const char *data, uint32_t len) 103 | { 104 | if (end_ + len > cap_) { 105 | uint32_t new_cap = cap_; 106 | while (end_ + len > new_cap) 107 | new_cap <<= 1; 108 | char *buf = new char[new_cap]; 109 | memcpy(buf, begin(), size_); 110 | delete [] data_; 111 | data_ = buf; 112 | beg_ = 0; 113 | end_ = size_; 114 | cap_ = new_cap; 115 | } 116 | assert(end_ + len <= cap_); 117 | memcpy(end(), data, len); 118 | end_ += len; 119 | size_ += len; 120 | } 121 | 122 | void Buffer::Write(char *data, uint32_t len) 123 | { 124 | assert(size_ >= len); 125 | memcpy(data, begin(), len); 126 | beg_ += len; 127 | size_ -= len; 128 | } 129 | 130 | } // namespace Mushroom 131 | -------------------------------------------------------------------------------- /src/network/buffer.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-27 09:44:37 6 | **/ 7 | 8 | #ifndef _BUFFER_HPP_ 9 | #define _BUFFER_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class Buffer : private NoCopy 16 | { 17 | public: 18 | Buffer(); 19 | 20 | ~Buffer(); 21 | 22 | uint32_t size() const; 23 | 24 | bool empty() const; 25 | 26 | char* data() const; 27 | 28 | char* begin() const; 29 | 30 | char* end() const; 31 | 32 | uint32_t space() const; 33 | 34 | void Read(const char *data, uint32_t len); 35 | 36 | void Write(char *data, uint32_t len); 37 | 38 | void AdvanceHead(uint32_t len); 39 | 40 | void AdvanceTail(uint32_t len); 41 | 42 | void Reset(); 43 | 44 | void Clear(); 45 | 46 | void Adjust(); 47 | 48 | void Unget(uint32_t len); 49 | 50 | private: 51 | char *data_; 52 | uint32_t size_; 53 | uint32_t cap_; 54 | uint32_t beg_; 55 | uint32_t end_; 56 | }; 57 | 58 | } // namespace Mushroom 59 | 60 | #endif /* _BUFFER_HPP_ */ -------------------------------------------------------------------------------- /src/network/callback.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 17:25:03 6 | **/ 7 | 8 | #ifndef _CALLBACK_HPP_ 9 | #define _CALLBACK_HPP_ 10 | 11 | #include 12 | 13 | namespace Mushroom { 14 | 15 | class Connection; 16 | 17 | typedef std::function ReadCallBack; 18 | typedef std::function WriteCallBack; 19 | 20 | typedef std::function ConnectCallBack; 21 | 22 | } // namespace Mushroom 23 | 24 | #endif /* _CALLBACK_HPP_ */ -------------------------------------------------------------------------------- /src/network/channel.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 17:07:49 6 | **/ 7 | 8 | #include "channel.hpp" 9 | #include "poller.hpp" 10 | 11 | namespace Mushroom { 12 | 13 | Channel::Channel(int fd, Poller *poller, const ReadCallBack &readcb, 14 | const WriteCallBack &writecb) 15 | :fd_(fd), poller_(poller), readcb_(readcb), writecb_(writecb) 16 | { 17 | events_ = ReadEvent; 18 | poller_->AddChannel(this); 19 | } 20 | 21 | Channel::~Channel() 22 | { 23 | poller_->RemoveChannel(this); 24 | } 25 | 26 | int Channel::fd() const 27 | { 28 | return fd_; 29 | } 30 | 31 | uint32_t Channel::events() const 32 | { 33 | return events_; 34 | } 35 | 36 | void Channel::OnRead(const ReadCallBack &readcb) 37 | { 38 | readcb_ = readcb; 39 | } 40 | 41 | void Channel::OnWrite(const WriteCallBack &writecb) 42 | { 43 | writecb_ = writecb; 44 | } 45 | 46 | bool Channel::CanRead() const 47 | { 48 | return events_ & ReadEvent; 49 | } 50 | 51 | bool Channel::CanWrite() const 52 | { 53 | return events_ & WriteEvent; 54 | } 55 | 56 | void Channel::EnableRead(bool flag) 57 | { 58 | if (flag) 59 | events_ |= ReadEvent; 60 | else 61 | events_ &= ~ReadEvent; 62 | 63 | poller_->UpdateChannel(this); 64 | } 65 | 66 | void Channel::EnableWrite(bool flag) 67 | { 68 | if (flag) 69 | events_ |= WriteEvent; 70 | else 71 | events_ &= ~WriteEvent; 72 | 73 | poller_->UpdateChannel(this); 74 | } 75 | 76 | void Channel::HandleRead() 77 | { 78 | readcb_(); 79 | } 80 | 81 | void Channel::HandleWrite() 82 | { 83 | writecb_(); 84 | } 85 | 86 | } // namespace Mushroom 87 | -------------------------------------------------------------------------------- /src/network/channel.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 17:07:38 6 | **/ 7 | 8 | #ifndef _CHANNEL_HPP_ 9 | #define _CHANNEL_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "callback.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class Poller; 17 | 18 | class Channel : private NoCopy 19 | { 20 | public: 21 | friend class Connection; 22 | 23 | Channel(int fd, Poller *poller, const ReadCallBack &readcb, const WriteCallBack &writecb); 24 | 25 | ~Channel(); 26 | 27 | int fd() const; 28 | 29 | uint32_t events() const; 30 | 31 | bool CanRead() const; 32 | 33 | bool CanWrite() const; 34 | 35 | void OnRead(const ReadCallBack &readcb); 36 | 37 | void OnWrite(const WriteCallBack &writecb); 38 | 39 | void EnableRead(bool flag); 40 | 41 | void EnableWrite(bool flag); 42 | 43 | void HandleRead(); 44 | 45 | void HandleWrite(); 46 | 47 | private: 48 | int fd_; 49 | uint32_t events_; 50 | Poller *poller_; 51 | 52 | ReadCallBack readcb_; 53 | WriteCallBack writecb_; 54 | }; 55 | 56 | } // namespace Mushroom 57 | 58 | #endif /* _CHANNEL_HPP_ */ -------------------------------------------------------------------------------- /src/network/connection.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-25 22:11:08 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "connection.hpp" 14 | #include "channel.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | Connection::Connection(const EndPoint &server, Poller *poller) 19 | :readcb_(0), writecb_(0) 20 | { 21 | assert(socket_.Create()); 22 | if (!socket_.Connect(server)) { 23 | printf("socket connect server %s failed, %s :(\n", 24 | server.ToString().c_str(), strerror(errno)); 25 | return ; 26 | } 27 | assert(socket_.SetNonBlock()); 28 | channel_ = new Channel(socket_.fd(), poller, 29 | [this]() { this->HandleRead(); }, [this]() { this->HandleWrite(); }); 30 | connected_ = true; 31 | } 32 | 33 | Connection::Connection(const Socket &socket, Poller *poller) 34 | :socket_(socket), readcb_(0), writecb_(0) 35 | { 36 | assert(socket_.SetNonBlock()); 37 | channel_ = new Channel(socket_.fd(), poller, 38 | [this]() { this->HandleRead(); }, [this]() { this->HandleWrite(); }); 39 | connected_ = true; 40 | } 41 | 42 | Connection::~Connection() 43 | { 44 | Close(); 45 | } 46 | 47 | bool Connection::Close() 48 | { 49 | if (socket_.Valid()) { 50 | connected_ = false; 51 | delete channel_; 52 | return socket_.Close(); 53 | } 54 | return true; 55 | } 56 | 57 | bool Connection::Success() const 58 | { 59 | return connected_; 60 | } 61 | 62 | Buffer& Connection::GetInput() 63 | { 64 | return input_; 65 | } 66 | 67 | Buffer& Connection::GetOutput() 68 | { 69 | return output_; 70 | } 71 | 72 | void Connection::OnRead(const ReadCallBack &readcb) 73 | { 74 | readcb_ = readcb; 75 | } 76 | 77 | void Connection::OnWrite(const WriteCallBack &writecb) 78 | { 79 | writecb_ = writecb; 80 | } 81 | 82 | void Connection::HandleRead() 83 | { 84 | if (!connected_) { 85 | printf("connection has closed :(\n"); 86 | return ; 87 | } 88 | input_.Reset(); 89 | bool blocked = false; 90 | uint32_t read = socket_.Read(input_.end(), input_.space(), &blocked); 91 | if (!read && !blocked) { 92 | Close(); 93 | return ; 94 | } 95 | input_.AdvanceTail(read); 96 | if (readcb_ && read) 97 | readcb_(); 98 | } 99 | 100 | void Connection::HandleWrite() 101 | { 102 | if (!connected_) { 103 | printf("connection has closed :(\n"); 104 | return ; 105 | } 106 | bool blocked = false; 107 | uint32_t write = socket_.Write(output_.begin(), output_.size(), &blocked); 108 | if (!write && !blocked) { 109 | Close(); 110 | return ; 111 | } 112 | output_.AdvanceHead(write); 113 | if (output_.empty()) { 114 | if (writecb_) 115 | writecb_(); 116 | if (channel_->CanWrite()) 117 | channel_->EnableWrite(false); 118 | } 119 | } 120 | 121 | void Connection::Send(const char *str) 122 | { 123 | Send(str, strlen(str)); 124 | } 125 | 126 | void Connection::Send(Buffer &buffer) 127 | { 128 | output_.Read(buffer.begin(), buffer.size()); 129 | buffer.Clear(); 130 | SendOutput(); 131 | } 132 | 133 | void Connection::Send(const char *str, uint32_t len) 134 | { 135 | output_.Read(str, len); 136 | SendOutput(); 137 | } 138 | 139 | void Connection::SendOutput() 140 | { 141 | if (!connected_) { 142 | // Error("connection has closed :("); 143 | output_.Clear(); 144 | return ; 145 | } 146 | bool blocked = false; 147 | uint32_t write = socket_.Write(output_.begin(), output_.size(), &blocked); 148 | if (!write && !blocked) { 149 | Close(); 150 | return ; 151 | } 152 | output_.AdvanceHead(write); 153 | if (output_.size()) { 154 | output_.Adjust(); 155 | // if (!channel_->CanWrite()) 156 | // channel_->EnableWrite(true); 157 | } 158 | } 159 | 160 | } // namespace Mushroom 161 | -------------------------------------------------------------------------------- /src/network/connection.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-25 22:11:01 6 | **/ 7 | 8 | #ifndef _CONNECTION_HPP_ 9 | #define _CONNECTION_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "callback.hpp" 13 | #include "socket.hpp" 14 | #include "endpoint.hpp" 15 | #include "buffer.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | class Channel; 20 | class Poller; 21 | 22 | class Connection : private NoCopy 23 | { 24 | public: 25 | Connection(const Socket &socket, Poller *poller); 26 | 27 | Connection(const EndPoint &server, Poller *poller); 28 | 29 | virtual ~Connection(); 30 | 31 | bool Success() const; 32 | 33 | Buffer& GetInput(); 34 | 35 | Buffer& GetOutput(); 36 | 37 | void HandleRead(); 38 | 39 | void HandleWrite(); 40 | 41 | void OnRead(const ReadCallBack &readcb); 42 | 43 | void OnWrite(const WriteCallBack &writecb); 44 | 45 | bool Close(); 46 | 47 | void Send(const char *str); 48 | 49 | void Send(Buffer &buffer); 50 | 51 | void Send(const char *str, uint32_t len); 52 | 53 | void SendOutput(); 54 | 55 | protected: 56 | Socket socket_; 57 | bool connected_; 58 | Channel *channel_; 59 | Buffer input_; 60 | Buffer output_; 61 | 62 | ReadCallBack readcb_; 63 | WriteCallBack writecb_; 64 | }; 65 | 66 | } // namespace Mushroom 67 | 68 | #endif /* _CONNECTION_HPP_ */ -------------------------------------------------------------------------------- /src/network/endpoint.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 23:25:45 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "endpoint.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | EndPoint::EndPoint(uint16_t port, uint32_t address):port_(port), address_(address) { } 16 | 17 | EndPoint::EndPoint(uint16_t port, const char *str):port_(port) 18 | { 19 | assert(inet_pton(AF_INET, str, &address_) == 1); 20 | } 21 | 22 | EndPoint::~EndPoint() { } 23 | 24 | uint16_t EndPoint::Port() const 25 | { 26 | return port_; 27 | } 28 | 29 | uint32_t EndPoint::Address() const 30 | { 31 | return address_; 32 | } 33 | 34 | std::string EndPoint::ToString() const 35 | { 36 | char buf[MaxLen]; 37 | assert(inet_ntop(AF_INET, &address_, buf, MaxLen)); 38 | return std::string(buf) + " port " + std::to_string(port_); 39 | } 40 | 41 | } // namespace Mushroom 42 | -------------------------------------------------------------------------------- /src/network/endpoint.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 23:25:32 6 | **/ 7 | 8 | #ifndef _ENDPOINT_HPP_ 9 | #define _ENDPOINT_HPP_ 10 | 11 | #include 12 | 13 | namespace Mushroom { 14 | 15 | class EndPoint 16 | { 17 | public: 18 | static const uint32_t MaxLen = 16; 19 | 20 | EndPoint(uint16_t port, uint32_t address); 21 | 22 | EndPoint(uint16_t port, const char *str); 23 | 24 | ~EndPoint(); 25 | 26 | uint16_t Port() const; 27 | 28 | uint32_t Address() const; 29 | 30 | std::string ToString() const; 31 | 32 | private: 33 | uint16_t port_; 34 | uint32_t address_; 35 | }; 36 | 37 | } // namespace Mushroom 38 | 39 | #endif /* _ENDPOINT_HPP_ */ -------------------------------------------------------------------------------- /src/network/eventbase.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-06 23:47:45 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "../include/bounded_queue.hpp" 12 | #include "../include/thread_pool.hpp" 13 | #include "time.hpp" 14 | #include "eventbase.hpp" 15 | #include "poller.hpp" 16 | #include "channel.hpp" 17 | #include "socket.hpp" 18 | 19 | namespace Mushroom { 20 | 21 | static const int MaxTimeout = 0x7FFFFFFF; 22 | 23 | EventBase::EventBase(int thread_num, int queue_size) 24 | :running_(true), poller_(new Poller()), next_time_out_(MaxTimeout), seq_(0), 25 | queue_(new BoundedQueue(queue_size)), pool_(new ThreadPool(queue_, thread_num)) 26 | { 27 | int r = pipe(wake_up_); 28 | assert(!r); 29 | assert(Socket(wake_up_[0]).SetNonBlock()); 30 | channel_ = new Channel(wake_up_[0], poller_, 0, 0); 31 | channel_->OnRead([this]() { 32 | char buf; 33 | ssize_t r = read(channel_->fd(), &buf, sizeof(buf)); 34 | if (r == sizeof(buf)) { 35 | } else { 36 | assert(0); 37 | } 38 | }); 39 | pid_ = pthread_self(); 40 | } 41 | 42 | EventBase::~EventBase() 43 | { 44 | close(wake_up_[1]); 45 | delete channel_; 46 | delete poller_; 47 | delete pool_; 48 | delete queue_; 49 | } 50 | 51 | Poller* EventBase::GetPoller() 52 | { 53 | return poller_; 54 | } 55 | 56 | void EventBase::Loop() 57 | { 58 | while (running_) { 59 | poller_->LoopOnce(std::min(5000, next_time_out_)); 60 | HandleTimeout(); 61 | } 62 | if (next_time_out_ != MaxTimeout) { 63 | poller_->LoopOnce(next_time_out_); 64 | HandleTimeout(); 65 | } else { 66 | poller_->LoopOnce(0); 67 | } 68 | repeat_.clear(); 69 | pending_.clear(); 70 | pool_->Clear(); 71 | } 72 | 73 | void EventBase::Exit() 74 | { 75 | mutex_.Lock(); 76 | if (!running_) { 77 | mutex_.Unlock(); 78 | return ; 79 | } 80 | running_ = false; 81 | WakeUp(); 82 | mutex_.Unlock(); 83 | } 84 | 85 | void EventBase::WakeUp() 86 | { 87 | char buf; 88 | ssize_t r = write(wake_up_[1], &buf, sizeof(buf)); 89 | assert(r > 0); 90 | } 91 | 92 | void EventBase::Refresh() 93 | { 94 | if (pending_.empty()) { 95 | next_time_out_ = MaxTimeout; 96 | } else { 97 | auto &it = pending_.begin()->first; 98 | int64_t tmp = it.first - Time::Now(); 99 | next_time_out_ = (tmp <= 0) ? 0 : int(tmp); 100 | } 101 | } 102 | 103 | void EventBase::RunNow(Task &&task) 104 | { 105 | RunAfter(0, std::move(task)); 106 | } 107 | 108 | TimerId EventBase::RunAfter(int64_t milli_sec, Task &&task) 109 | { 110 | mutex_.Lock(); 111 | if (!running_) { 112 | mutex_.Unlock(); 113 | return TimerId(); 114 | } 115 | TimerId id { Time::Now() + milli_sec, seq_++}; 116 | pending_.insert({id, std::move(task)}); 117 | if (pthread_self() != pid_) 118 | WakeUp(); 119 | else 120 | Refresh(); 121 | mutex_.Unlock(); 122 | return id; 123 | } 124 | 125 | TimerId EventBase::RunEvery(int64_t milli_sec, Task &&task) 126 | { 127 | mutex_.Lock(); 128 | if (!running_) { 129 | mutex_.Unlock(); 130 | return TimerId(); 131 | } 132 | uint32_t seq = seq_++; 133 | TimeRep rep {milli_sec, Time::Now()}; 134 | TimerId id {rep.second, seq}; 135 | repeat_.insert({seq, rep}); 136 | pending_.insert({id, std::move(task)}); 137 | if (pthread_self() != pid_) 138 | WakeUp(); 139 | else 140 | Refresh(); 141 | mutex_.Unlock(); 142 | return {milli_sec, seq}; 143 | } 144 | 145 | void EventBase::RescheduleAfter(TimerId *id, int64_t milli_sec, Task &&task) 146 | { 147 | mutex_.Lock(); 148 | if (!running_) { 149 | mutex_.Unlock(); 150 | return ; 151 | } 152 | TimerId nid { Time::Now() + milli_sec, seq_++}; 153 | auto it = pending_.find(*id); 154 | if (it != pending_.end()) 155 | pending_.erase(it); 156 | pending_.insert({nid, std::move(task)}); 157 | *id = nid; 158 | if (pthread_self() != pid_) 159 | WakeUp(); 160 | else 161 | Refresh(); 162 | mutex_.Unlock(); 163 | } 164 | 165 | void EventBase::RescheduleAfter(const TimerId &id, int64_t milli_sec) 166 | { 167 | mutex_.Lock(); 168 | if (!running_) { 169 | mutex_.Unlock(); 170 | return ; 171 | } 172 | TimerId nid { Time::Now() + milli_sec, seq_++}; 173 | auto it = pending_.find(id); 174 | if (it == pending_.end()) { 175 | mutex_.Unlock(); 176 | return ; 177 | } 178 | Task task = std::move(it->second); 179 | pending_.erase(it); 180 | pending_.insert({nid, std::move(task)}); 181 | if (pthread_self() != pid_) 182 | WakeUp(); 183 | else 184 | Refresh(); 185 | mutex_.Unlock(); 186 | } 187 | 188 | void EventBase::Cancel(const TimerId &id) 189 | { 190 | mutex_.Lock(); 191 | auto rit = repeat_.find(id.second); 192 | if (rit != repeat_.end()) { 193 | repeat_.erase(rit); 194 | auto it = pending_.find({rit->second.second, id.second}); 195 | if (it != pending_.end()) 196 | pending_.erase(it); 197 | } else { 198 | auto it = pending_.find(id); 199 | if (it != pending_.end()) 200 | pending_.erase(it); 201 | } 202 | mutex_.Unlock(); 203 | } 204 | 205 | void EventBase::HandleTimeout() 206 | { 207 | TimerId now { Time::Now(), 0xFFFFFFFF}; 208 | std::vector expired; 209 | mutex_.Lock(); 210 | for (; running_ && !pending_.empty() && pending_.begin()->first <= now; ) { 211 | expired.push_back(pending_.begin()->second); 212 | const TimerId &id = pending_.begin()->first; 213 | auto it = repeat_.find(id.second); 214 | if (it != repeat_.end()) { 215 | TimerId nid { now.first + it->second.first, id.second }; 216 | it->second.second = nid.first; 217 | pending_.insert({nid, std::move(pending_.begin()->second)}); 218 | } 219 | pending_.erase(pending_.begin()); 220 | } 221 | Refresh(); 222 | mutex_.Unlock(); 223 | for (uint32_t i = 0; i < expired.size(); ++i) 224 | queue_->Push(std::move(expired[i])); 225 | } 226 | 227 | } // namespace Mushroom 228 | -------------------------------------------------------------------------------- /src/network/eventbase.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-06 23:45:21 6 | **/ 7 | 8 | #ifndef _EVENT_BASE_HPP_ 9 | #define _EVENT_BASE_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | #include "../include/mutex.hpp" 15 | 16 | namespace Mushroom { 17 | 18 | class Poller; 19 | class Channel; 20 | template class ThreadPool; 21 | template class BoundedQueue; 22 | 23 | typedef std::pair TimerId; 24 | typedef std::pair TimeRep; 25 | 26 | class EventBase : private NoCopy 27 | { 28 | public: 29 | EventBase(int thread_num, int queue_size); 30 | 31 | ~EventBase(); 32 | 33 | void Loop(); 34 | 35 | void Exit(); 36 | 37 | Poller* GetPoller(); 38 | 39 | void RunNow(Task &&task); 40 | 41 | TimerId RunAfter(int64_t milli_sec, Task &&task); 42 | 43 | TimerId RunEvery(int64_t milli_sec, Task &&task); 44 | 45 | void RescheduleAfter(TimerId *id, int64_t milli_sec, Task &&task); 46 | 47 | void RescheduleAfter(const TimerId &id, int64_t milli_sec); 48 | 49 | void Cancel(const TimerId &timer_id); 50 | 51 | private: 52 | void HandleTimeout(); 53 | 54 | void WakeUp(); 55 | 56 | void Repeat(); 57 | 58 | void Refresh(); 59 | 60 | bool running_; 61 | int wake_up_[2]; 62 | Channel *channel_; 63 | Poller *poller_; 64 | 65 | uint64_t pid_; 66 | int next_time_out_; 67 | 68 | uint32_t seq_; 69 | 70 | BoundedQueue *queue_; 71 | ThreadPool *pool_; 72 | 73 | Mutex mutex_; 74 | std::map repeat_; 75 | std::map pending_; 76 | }; 77 | 78 | } // namespace Mushroom 79 | 80 | #endif /* _EVENT_BASE_HPP_ */ -------------------------------------------------------------------------------- /src/network/poller.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-24 11:31:12 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "poller.hpp" 13 | #include "channel.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | Poller::Poller() 18 | { 19 | assert((fd_ = epoll_create1(EPOLL_CLOEXEC)) >= 0); 20 | } 21 | 22 | Poller::~Poller() 23 | { 24 | close(fd_); 25 | } 26 | 27 | void Poller::AddChannel(Channel *channel) 28 | { 29 | struct epoll_event event; 30 | memset(&event, 0, sizeof(event)); 31 | event.events = channel->events(); 32 | event.data.ptr = channel; 33 | assert(!epoll_ctl(fd_, EPOLL_CTL_ADD, channel->fd(), &event)); 34 | } 35 | 36 | void Poller::UpdateChannel(Channel *channel) 37 | { 38 | struct epoll_event event; 39 | memset(&event, 0, sizeof(event)); 40 | event.events = channel->events(); 41 | event.data.ptr = channel; 42 | assert(!epoll_ctl(fd_, EPOLL_CTL_MOD, channel->fd(), &event)); 43 | } 44 | 45 | void Poller::RemoveChannel(Channel *channel) 46 | { 47 | struct epoll_event event; 48 | memset(&event, 0, sizeof(event)); 49 | event.events = channel->events(); 50 | event.data.ptr = channel; 51 | assert(!epoll_ctl(fd_, EPOLL_CTL_DEL, channel->fd(), &event)); 52 | } 53 | 54 | void Poller::LoopOnce(int ms) 55 | { 56 | int ready = epoll_wait(fd_, events_, MaxEvents, ms); 57 | assert(!(ready == -1 && errno != EINTR)); 58 | for (; --ready >= 0; ) { 59 | Channel *channel = (Channel *)events_[ready].data.ptr; 60 | uint32_t event = events_[ready].events; 61 | if (event & ReadEvent) { 62 | channel->HandleRead(); 63 | } else if (event & WriteEvent) { 64 | channel->HandleWrite(); 65 | } else { 66 | assert(0); 67 | } 68 | } 69 | } 70 | 71 | } // namespace Mushroom 72 | -------------------------------------------------------------------------------- /src/network/poller.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-24 11:21:25 6 | **/ 7 | 8 | #ifndef _POLLER_HPP_ 9 | #define _POLLER_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | 15 | const uint32_t ReadEvent = EPOLLIN; 16 | const uint32_t WriteEvent = EPOLLOUT; 17 | const uint32_t MaxEvents = 1024; 18 | 19 | namespace Mushroom { 20 | 21 | class Channel; 22 | 23 | class Poller : private NoCopy 24 | { 25 | public: 26 | Poller(); 27 | 28 | ~Poller(); 29 | 30 | void AddChannel(Channel *channel); 31 | 32 | void UpdateChannel(Channel *channel); 33 | 34 | void RemoveChannel(Channel *channel); 35 | 36 | void LoopOnce(int ms); 37 | 38 | private: 39 | int fd_; 40 | struct epoll_event events_[MaxEvents]; 41 | }; 42 | 43 | } // namespace Mushroom 44 | 45 | #endif /* _POLLER_HPP_ */ -------------------------------------------------------------------------------- /src/network/server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 10:50:45 6 | **/ 7 | 8 | #include "eventbase.hpp" 9 | #include "server.hpp" 10 | #include "channel.hpp" 11 | #include "connection.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | Server::Server(EventBase *event_base, uint16_t port) 16 | :port_(port), event_base_(event_base), listen_(0), connectcb_(0) { } 17 | 18 | Server::~Server() 19 | { 20 | if (listen_) 21 | Close(); 22 | for (auto e : connections_) 23 | delete e; 24 | } 25 | 26 | void Server::Close() 27 | { 28 | delete listen_; 29 | listen_ = 0; 30 | socket_.Close(); 31 | 32 | for (auto e : connections_) 33 | e->Close(); 34 | } 35 | 36 | void Server::Start() 37 | { 38 | assert(socket_.Create() && socket_.SetResuseAddress() && socket_.Bind(port_) && 39 | socket_.Listen()); 40 | listen_ = new Channel(socket_.fd(), event_base_->GetPoller(), [this]() { HandleAccept(); }, 0); 41 | } 42 | 43 | void Server::OnConnect(const ConnectCallBack &connectcb) 44 | { 45 | connectcb_ = connectcb; 46 | } 47 | 48 | void Server::HandleAccept() 49 | { 50 | int fd = socket_.Accept(); 51 | assert(fd > 0); 52 | Connection *con = new Connection(Socket(fd), event_base_->GetPoller()); 53 | if (connectcb_) 54 | connectcb_(con); 55 | connections_.push_back(con); 56 | } 57 | 58 | std::vector& Server::Connections() 59 | { 60 | return connections_; 61 | } 62 | 63 | uint16_t Server::Port() const 64 | { 65 | return port_; 66 | } 67 | 68 | } // namespaceMushroom 69 | -------------------------------------------------------------------------------- /src/network/server.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 10:50:39 6 | **/ 7 | 8 | #ifndef _SERVER_HPP_ 9 | #define _SERVER_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | #include "callback.hpp" 15 | #include "socket.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | class Channel; 20 | class EventBase; 21 | 22 | class Server : private NoCopy 23 | { 24 | public: 25 | Server(EventBase *event_base, uint16_t port); 26 | 27 | virtual ~Server(); 28 | 29 | void Start(); 30 | 31 | void Close(); 32 | 33 | void OnConnect(const ConnectCallBack &connectcb); 34 | 35 | std::vector& Connections(); 36 | 37 | uint16_t Port() const; 38 | 39 | protected: 40 | uint16_t port_; 41 | EventBase *event_base_; 42 | Socket socket_; 43 | Channel *listen_; 44 | 45 | std::vector connections_; 46 | ConnectCallBack connectcb_; 47 | 48 | private: 49 | void HandleAccept(); 50 | }; 51 | 52 | } // namespace Mushroom 53 | 54 | #endif /* _SERVER_HPP_ */ -------------------------------------------------------------------------------- /src/network/signal.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 22:44:03 6 | **/ 7 | 8 | #ifndef _SIGNAL_HPP_ 9 | #define _SIGNAL_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | namespace Mushroom { 16 | 17 | class Signal 18 | { 19 | public: 20 | static void Register(int sig, const std::function &handler) { 21 | handlers_[sig] = handler; 22 | signal(sig, signal_handler); 23 | } 24 | 25 | static void signal_handler(int sig) { 26 | handlers_[sig](); 27 | } 28 | 29 | private: 30 | static std::unordered_map> handlers_; 31 | }; 32 | 33 | std::unordered_map> Signal::handlers_; 34 | 35 | } // namespace Mushroom 36 | 37 | #endif /* _SIGNAL_HPP_ */ -------------------------------------------------------------------------------- /src/network/socket.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 10:23:53 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "socket.hpp" 18 | 19 | namespace Mushroom { 20 | 21 | Socket::Socket():fd_(-1) { } 22 | 23 | Socket::Socket(int fd):fd_(fd) { } 24 | 25 | Socket::~Socket() { } 26 | 27 | int Socket::fd() const 28 | { 29 | return fd_; 30 | } 31 | 32 | bool Socket::Valid() const 33 | { 34 | return fd_ != -1; 35 | } 36 | 37 | bool Socket::Create() 38 | { 39 | fd_ = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); 40 | return fd_ != -1; 41 | } 42 | 43 | bool Socket::Close() 44 | { 45 | bool flag = true; 46 | if (fd_ != -1) { 47 | flag = !close(fd_); 48 | fd_ = -1; 49 | } 50 | return flag; 51 | } 52 | 53 | bool Socket::Connect(const EndPoint &end_point) 54 | { 55 | struct sockaddr_in server; 56 | memset(&server, 0, sizeof(server)); 57 | server.sin_family = AF_INET; 58 | server.sin_port = htons(end_point.Port()); 59 | server.sin_addr.s_addr = end_point.Address(); 60 | return !connect(fd_, (const struct sockaddr *)&server, sizeof(server)); 61 | } 62 | 63 | bool Socket::Bind(uint16_t port) 64 | { 65 | struct sockaddr_in server; 66 | memset(&server, 0, sizeof(server)); 67 | server.sin_family = AF_INET; 68 | server.sin_port = htons(port); 69 | server.sin_addr.s_addr = htonl(INADDR_ANY); 70 | return !bind(fd_, (const struct sockaddr *)&server, sizeof(server)); 71 | } 72 | 73 | bool Socket::Listen() 74 | { 75 | return !listen(fd_, 32); 76 | } 77 | 78 | int Socket::Accept() 79 | { 80 | struct sockaddr_in client; 81 | memset(&client, 0, sizeof(client)); 82 | socklen_t len = sizeof(client); 83 | int fd = accept(fd_, (struct sockaddr *)&client, &len); 84 | return fd; 85 | } 86 | 87 | bool Socket::SetOption(int value, bool flag) 88 | { 89 | return !setsockopt(fd_, SOL_SOCKET, value, &flag, sizeof(flag)); 90 | } 91 | 92 | bool Socket::GetOption(int value, int *ret) 93 | { 94 | socklen_t len = sizeof(*ret); 95 | return !getsockopt(fd_, SOL_SOCKET, value, ret, &len); 96 | } 97 | 98 | bool Socket::SetResuseAddress() 99 | { 100 | int flag = 1; 101 | return !setsockopt(fd_, SOL_SOCKET, SO_REUSEADDR, &flag, sizeof(flag)); 102 | } 103 | 104 | bool Socket::GetPeerName(EndPoint *endpoint) 105 | { 106 | struct sockaddr_in addr; 107 | memset(&addr, 0, sizeof(addr)); 108 | socklen_t len = sizeof(addr); 109 | if (!getsockname(fd_, (struct sockaddr *)&addr, &len)) { 110 | *endpoint = EndPoint(ntohs(addr.sin_port), addr.sin_addr.s_addr); 111 | return true; 112 | } 113 | return false; 114 | } 115 | 116 | bool Socket::GetSockName(EndPoint *endpoint) 117 | { 118 | struct sockaddr_in addr; 119 | memset(&addr, 0, sizeof(addr)); 120 | socklen_t len = sizeof(addr); 121 | if (!getpeername(fd_, (struct sockaddr *)&addr, &len)) { 122 | *endpoint = EndPoint(ntohs(addr.sin_port), addr.sin_addr.s_addr); 123 | return true; 124 | } 125 | return false; 126 | } 127 | 128 | bool Socket::AddFlag(int flag) 129 | { 130 | int value = fcntl(fd_, F_GETFL, 0); 131 | assert(value != -1); 132 | return !fcntl(fd_, F_SETFL, value | flag); 133 | } 134 | 135 | bool Socket::SetNonBlock() 136 | { 137 | int value = fcntl(fd_, F_GETFL, 0); 138 | assert(value != -1); 139 | return !fcntl(fd_, F_SETFL, value | O_NONBLOCK); 140 | } 141 | 142 | uint32_t Socket::Write(const char *data, uint32_t len, bool *blocked) 143 | { 144 | uint32_t written = 0; 145 | for (; written < len;) { 146 | ssize_t r = write(fd_, data + written, len - written); 147 | if (r > 0) { 148 | written += r; 149 | continue; 150 | } else if (r == -1) { 151 | if (errno == EINTR) 152 | continue; 153 | if (errno == EAGAIN || errno == EWOULDBLOCK) { 154 | *blocked = true; 155 | break; 156 | } 157 | } 158 | printf("write error, %s :(\n", strerror(errno)); 159 | break; 160 | } 161 | return written; 162 | } 163 | 164 | uint32_t Socket::Read(char *data, uint32_t len, bool *blocked) 165 | { 166 | uint32_t has_read = 0; 167 | ssize_t r; 168 | for (; has_read < len && (r = read(fd_, data + has_read, len - has_read));) { 169 | if (r == -1) { 170 | if (errno == EINTR) 171 | continue; 172 | if (errno == EAGAIN || errno == EWOULDBLOCK) 173 | *blocked = true; 174 | else 175 | printf("read error, %s :(\n", strerror(errno)); 176 | break; 177 | } 178 | has_read += r; 179 | } 180 | return has_read; 181 | } 182 | 183 | } // namespace Mushroom 184 | -------------------------------------------------------------------------------- /src/network/socket.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-23 10:23:22 6 | **/ 7 | 8 | #ifndef _SOCKET_HPP_ 9 | #define _SOCKET_HPP_ 10 | 11 | #include "endpoint.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class Socket 16 | { 17 | public: 18 | Socket(); 19 | 20 | Socket(int fd); 21 | 22 | ~Socket(); 23 | 24 | int fd() const; 25 | 26 | bool Valid() const; 27 | 28 | bool Create(); 29 | 30 | bool Close(); 31 | 32 | bool Connect(const EndPoint &end_point); 33 | 34 | bool Bind(uint16_t port); 35 | 36 | bool Listen(); 37 | 38 | int Accept(); 39 | 40 | uint32_t Write(const char *data, uint32_t len, bool *blocked); 41 | 42 | uint32_t Read(char *data, uint32_t len, bool *blocked); 43 | 44 | bool SetOption(int value, bool flag); 45 | 46 | bool GetOption(int value, int *ret); 47 | 48 | bool GetPeerName(EndPoint *endpoint); 49 | 50 | bool GetSockName(EndPoint *endpoint); 51 | 52 | bool AddFlag(int flag); 53 | 54 | bool SetNonBlock(); 55 | 56 | bool SetResuseAddress(); 57 | 58 | private: 59 | int fd_; 60 | }; 61 | 62 | } // namespace Mushroom 63 | 64 | #endif /* _SOCKET_HPP_ */ -------------------------------------------------------------------------------- /src/network/time.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-14 21:19:07 6 | **/ 7 | 8 | #ifndef _TIME_HPP_ 9 | #define _TIME_HPP_ 10 | 11 | #include 12 | 13 | namespace Mushroom { 14 | 15 | class Time 16 | { 17 | public: 18 | static int64_t Now() { 19 | return NowMicro() / 1000; 20 | } 21 | 22 | static int64_t NowMicro() { 23 | struct timeval tv; 24 | gettimeofday(&tv, 0); 25 | return (int64_t(tv.tv_sec) * 1000000 + tv.tv_usec); 26 | } 27 | }; 28 | 29 | } // namespace Mushroom 30 | 31 | #endif /* _TIME_HPP_ */ -------------------------------------------------------------------------------- /src/palm/barrier.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-3-25 23:21:00 6 | **/ 7 | 8 | #ifndef _BARRIER_HPP_ 9 | #define _BARRIER_HPP_ 10 | 11 | #include "../include/mutex.hpp" 12 | #include "../include/cond.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class Barrier 17 | { 18 | public: 19 | Barrier(int num):num_(num), cnt_(num_), gen_(0) { } 20 | 21 | void Wait() { 22 | mutex_.Lock(); 23 | uint32_t gen = gen_; 24 | if (--cnt_ == 0) { 25 | ++gen_; 26 | cnt_ = num_; 27 | mutex_.Unlock(); 28 | cond_.Broadcast(); 29 | } 30 | while (gen == gen_) 31 | cond_.Wait(mutex_); 32 | mutex_.Unlock(); 33 | } 34 | 35 | private: 36 | Mutex mutex_; 37 | Cond cond_; 38 | const uint32_t num_; 39 | uint32_t cnt_; 40 | uint32_t gen_; 41 | }; 42 | 43 | } // namespace Mushroom 44 | 45 | #endif /* _BARRIER_HPP_ */ -------------------------------------------------------------------------------- /src/palm/batch.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-28 15:36:00 6 | **/ 7 | 8 | #include "batch.hpp" 9 | #include "../blink/slice.hpp" 10 | 11 | namespace Mushroom { 12 | 13 | uint32_t Batch::Size = 64; 14 | 15 | void Batch::SetSize(uint32_t size) 16 | { 17 | Size = size; 18 | } 19 | 20 | Batch::Batch() 21 | { 22 | batch_ = new KeySlice*[Size]; 23 | for (uint32_t i = 0; i < Size; ++i) 24 | batch_[i] = NewKeySlice(); 25 | } 26 | 27 | Batch::~Batch() 28 | { 29 | for (int32_t i = (int32_t)Size - 1; i >= 0; --i) 30 | DeleteKeySlice(batch_[i]); 31 | delete [] batch_; 32 | } 33 | 34 | // not check boundary for performance reason 35 | void Batch::SetKeySlice(uint32_t idx, const char *key) 36 | { 37 | memcpy(batch_[idx]->key_, key, KeySlice::KeyLen); 38 | } 39 | 40 | // not check boundary for performance reason 41 | const KeySlice* Batch::GetKeySlice(uint32_t idx) const 42 | { 43 | return batch_[idx]; 44 | } 45 | 46 | } // Mushroom 47 | -------------------------------------------------------------------------------- /src/palm/batch.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-28 15:33:00 6 | **/ 7 | 8 | #ifndef _BATCH_HPP_ 9 | #define _BATCH_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class KeySlice; 16 | 17 | class Batch : private NoCopy 18 | { 19 | public: 20 | static uint32_t Size; 21 | 22 | static void SetSize(uint32_t size); 23 | 24 | Batch(); 25 | 26 | ~Batch(); 27 | 28 | void SetKeySlice(uint32_t idx, const char *key); 29 | 30 | const KeySlice* GetKeySlice(uint32_t idx) const; 31 | 32 | private: 33 | KeySlice **batch_; 34 | }; 35 | 36 | } // Mushroom 37 | 38 | #endif /* _BATCH_HPP_ */ -------------------------------------------------------------------------------- /src/palm/batcher.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-28 16:46:00 6 | **/ 7 | 8 | #include 9 | 10 | #include "batcher.hpp" 11 | #include "../blink/slice.hpp" 12 | #include "../blink/page.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | Batcher::Batcher():idx_(0) 17 | { 18 | char *tmp = new char[Page::PageSize]; 19 | uint16_t degree = Page::CalculateDegree(KeySlice::KeyLen); 20 | page_ = new (tmp) Page(0, 0, KeySlice::KeyLen, 0, degree); 21 | page_->InsertInfiniteKey(); 22 | } 23 | 24 | Batcher::~Batcher() 25 | { 26 | idx_ = 0; 27 | delete [] (char *)page_; 28 | } 29 | 30 | bool Batcher::InsertKeySlice(const KeySlice *slice) 31 | { 32 | if (page_->Full()) return false; 33 | page_t page_no; 34 | return page_->Insert(slice, page_no) == InsertOk; 35 | } 36 | 37 | uint32_t Batcher::TotalKey() const { 38 | return page_->TotalKey() - 1; 39 | } 40 | 41 | uint32_t Batcher::Capacity() const { 42 | return page_->Degree() - 1; 43 | } 44 | 45 | void Batcher::BeforeGet() { 46 | idx_ = page_->Index(); 47 | } 48 | 49 | // before any Get call, must call BeforeGet 50 | const KeySlice* Batcher::GetKeySlice(uint32_t idx) const { 51 | return page_->Key(idx_, idx); 52 | } 53 | 54 | std::string Batcher::ToString() const { 55 | return page_->ToString(true, true); 56 | } 57 | 58 | } // Mushroom 59 | -------------------------------------------------------------------------------- /src/palm/batcher.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-28 16:46:00 6 | **/ 7 | 8 | #ifndef _BATCHER_HPP_ 9 | #define _BATCHER_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | class KeySlice; 18 | class Page; 19 | 20 | // batcher is just a wrapper of page, we use it to store keys in order before 21 | // divide them into batch 22 | class Batcher : private NoCopy 23 | { 24 | public: 25 | Batcher(); 26 | 27 | ~Batcher(); 28 | 29 | bool InsertKeySlice(const KeySlice *slice); 30 | 31 | uint32_t TotalKey() const; 32 | 33 | uint32_t Capacity() const; 34 | 35 | void BeforeGet(); 36 | 37 | const KeySlice* GetKeySlice(uint32_t idx) const; 38 | 39 | std::string ToString() const; 40 | 41 | private: 42 | Page *page_; 43 | uint16_t *idx_; 44 | }; 45 | 46 | } // Mushroom 47 | 48 | #endif /* _BATCHER_HPP_ */ -------------------------------------------------------------------------------- /src/palm/palm_tree.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-27 17:23:00 6 | **/ 7 | 8 | #include "palm_tree.hpp" 9 | #include "../blink/page.hpp" 10 | #include "../blink/pool_manager.hpp" 11 | #include "barrier.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | PalmTree::PalmTree(uint32_t key_len, uint32_t threads) 16 | :pool_manager_(new PoolManager()), barrier_(new Barrier(threads)), root_(0), key_len_(key_len) 17 | { 18 | degree_ = Page::CalculateDegree(key_len_); 19 | Set set; 20 | set.page_ = pool_manager_->NewPage(Page::ROOT, key_len_, 0, degree_); 21 | set.page_->InsertInfiniteKey(); 22 | } 23 | 24 | PalmTree::~PalmTree() 25 | { 26 | delete barrier_; 27 | delete pool_manager_; 28 | } 29 | 30 | void PalmTree::Free() 31 | { 32 | pool_manager_->Free(); 33 | } 34 | 35 | bool PalmTree::Put(KeySlice *key) 36 | { 37 | 38 | } 39 | 40 | bool PalmTree::Get(KeySlice *key) 41 | { 42 | 43 | } 44 | 45 | } // Mushroom 46 | -------------------------------------------------------------------------------- /src/palm/palm_tree.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-27 17:22:00 6 | **/ 7 | 8 | #ifndef _PALM_TREE_HPP_ 9 | #define _PALM_TREE_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | class KeySlice; 16 | class Page; 17 | class PoolManager; 18 | class Barrier; 19 | 20 | class PalmTree : private NoCopy 21 | { 22 | public: 23 | static const uint32_t MAX_KEY_LENGTH = 255; 24 | 25 | PalmTree(uint32_t key_len, uint32_t threads); 26 | 27 | bool Put(KeySlice *key); 28 | 29 | bool Get(KeySlice *key); 30 | 31 | void Free(); 32 | 33 | ~PalmTree(); 34 | 35 | private: 36 | struct Set { 37 | Set():depth_(0) { } 38 | page_t page_no_; 39 | Page *page_; 40 | page_t stack_[8]; 41 | uint32_t depth_; 42 | }; 43 | 44 | PoolManager *pool_manager_; 45 | 46 | Barrier *barrier_; 47 | 48 | page_t root_; 49 | 50 | uint8_t key_len_; 51 | uint16_t degree_; 52 | }; 53 | 54 | } // Mushroom 55 | 56 | #endif /* _PALM_TREE_HPP_ */ -------------------------------------------------------------------------------- /src/raft/arg.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-08 09:15:27 6 | **/ 7 | 8 | #ifndef _RAFT_ARG_HPP_ 9 | #define _RAFT_ARG_HPP_ 10 | 11 | #include "../rpc/marshaller.hpp" 12 | // #include "mushroom_log.hpp" 13 | #include "log.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | struct RequestVoteArgs 18 | { 19 | RequestVoteArgs() { } 20 | RequestVoteArgs(uint32_t term, uint32_t id, uint32_t last_index, uint32_t last_term) 21 | :term_(term), id_(id), last_index_(last_index), last_term_(last_term) { } 22 | uint32_t term_; 23 | int32_t id_; 24 | int32_t last_index_; 25 | uint32_t last_term_; 26 | }; 27 | 28 | struct RequestVoteReply 29 | { 30 | RequestVoteReply() { } 31 | uint32_t term_; 32 | uint32_t granted_; 33 | }; 34 | 35 | struct AppendEntryArgs 36 | { 37 | AppendEntryArgs() { } 38 | 39 | AppendEntryArgs(uint32_t term, int32_t id, uint32_t prev_term, int32_t prev_index, 40 | int32_t leader_commit):term_(term), id_(id), prev_term_(prev_term), 41 | prev_index_(prev_index), leader_commit_(leader_commit) { } 42 | uint32_t term_; 43 | int32_t id_; 44 | uint32_t prev_term_; 45 | int32_t prev_index_; 46 | int32_t leader_commit_; 47 | std::vector entries_; 48 | }; 49 | 50 | struct AppendEntryReply 51 | { 52 | AppendEntryReply() { } 53 | uint32_t term_; 54 | int32_t idx_; 55 | }; 56 | 57 | inline Marshaller& operator<<(Marshaller &marshaller, const RequestVoteArgs &args) 58 | { 59 | marshaller << args.term_; 60 | marshaller << args.id_; 61 | marshaller << args.last_index_; 62 | marshaller << args.last_term_; 63 | return marshaller; 64 | } 65 | 66 | inline Marshaller& operator>>(Marshaller &marshaller, RequestVoteArgs &args) 67 | { 68 | marshaller >> args.term_; 69 | marshaller >> args.id_; 70 | marshaller >> args.last_index_; 71 | marshaller >> args.last_term_; 72 | return marshaller; 73 | } 74 | 75 | inline Marshaller& operator<<(Marshaller &marshaller, const RequestVoteReply &reply) 76 | { 77 | marshaller << reply.term_; 78 | marshaller << reply.granted_; 79 | return marshaller; 80 | } 81 | 82 | inline Marshaller& operator>>(Marshaller &marshaller, RequestVoteReply &reply) 83 | { 84 | marshaller >> reply.term_; 85 | marshaller >> reply.granted_; 86 | return marshaller; 87 | } 88 | 89 | inline Marshaller& operator<<(Marshaller &marshaller, const AppendEntryArgs &args) 90 | { 91 | marshaller << args.term_; 92 | marshaller << args.id_; 93 | marshaller << args.prev_term_; 94 | marshaller << args.prev_index_; 95 | marshaller << args.leader_commit_; 96 | marshaller << args.entries_; 97 | return marshaller; 98 | } 99 | 100 | inline Marshaller& operator>>(Marshaller &marshaller, AppendEntryArgs &args) 101 | { 102 | marshaller >> args.term_; 103 | marshaller >> args.id_; 104 | marshaller >> args.prev_term_; 105 | marshaller >> args.prev_index_; 106 | marshaller >> args.leader_commit_; 107 | marshaller >> args.entries_; 108 | return marshaller; 109 | } 110 | 111 | inline Marshaller& operator<<(Marshaller &marshaller, const AppendEntryReply &reply) 112 | { 113 | marshaller << reply.term_; 114 | marshaller << reply.idx_; 115 | return marshaller; 116 | } 117 | 118 | inline Marshaller& operator>>(Marshaller &marshaller, AppendEntryReply &reply) 119 | { 120 | marshaller >> reply.term_; 121 | marshaller >> reply.idx_; 122 | return marshaller; 123 | } 124 | 125 | } // namespace Mushroom 126 | 127 | #endif /* _RAFT_ARG_HPP_ */ -------------------------------------------------------------------------------- /src/raft/log.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-07 22:19:41 6 | **/ 7 | 8 | #ifndef _RAFT_LOG_HPP_ 9 | #define _RAFT_LOG_HPP_ 10 | 11 | #include "../rpc/marshaller.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | struct Log 16 | { 17 | Log() { } 18 | Log(uint32_t number):number_(number) { } 19 | 20 | bool operator!=(const Log& that) { return term_ != that.term_ || number_ != that.number_; } 21 | 22 | uint32_t term_; 23 | uint32_t number_; 24 | }; 25 | 26 | inline Marshaller& operator<<(Marshaller &marshaller, const Log &log) 27 | { 28 | marshaller << log.term_; 29 | marshaller << log.number_; 30 | return marshaller; 31 | } 32 | 33 | inline Marshaller& operator>>(Marshaller &marshaller, Log &log) 34 | { 35 | marshaller >> log.term_; 36 | marshaller >> log.number_; 37 | return marshaller; 38 | } 39 | 40 | } // namespace Mushroom 41 | 42 | #endif /* _RAFT_LOG_HPP_ */ 43 | -------------------------------------------------------------------------------- /src/raft/mushroom_log.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-03 10:51:13 6 | **/ 7 | 8 | #ifndef _MUSHROOM_LOG_HPP_ 9 | #define _MUSHROOM_LOG_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "../rpc/marshaller.hpp" 13 | #include "../blink/slice.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | struct MushroomLog 18 | { 19 | MushroomLog():key_(NewKeySlice()) { } 20 | 21 | MushroomLog(const MushroomLog &that) { 22 | term_ = that.term_; 23 | key_ = NewKeySlice(); 24 | memcpy(key_, that.key_, KeySlice::KeySize); 25 | } 26 | 27 | MushroomLog& operator=(const MushroomLog &that) { 28 | term_ = that.term_; 29 | key_ = NewKeySlice(); 30 | memcpy(key_, that.key_, KeySlice::KeySize); 31 | return *this; 32 | } 33 | 34 | ~MushroomLog() { DeleteKeySlice(key_); } 35 | 36 | uint32_t term_; 37 | KeySlice *key_; 38 | }; 39 | 40 | inline Marshaller& operator<<(Marshaller &marshaller, const MushroomLog &log) 41 | { 42 | marshaller << log.term_; 43 | marshaller.Read(log.key_, KeySlice::KeySize); 44 | return marshaller; 45 | } 46 | 47 | inline Marshaller& operator>>(Marshaller &marshaller, MushroomLog &log) 48 | { 49 | marshaller >> log.term_; 50 | log.key_ = NewKeySlice(); 51 | marshaller.Write(log.key_, KeySlice::KeySize); 52 | return marshaller; 53 | } 54 | 55 | } // namespace Mushroom 56 | 57 | #endif /* _MUSHROOM_LOG_HPP_ */ -------------------------------------------------------------------------------- /src/raft/raft_server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-22 21:21:08 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "raft_server.hpp" 13 | #include "../rpc/future.hpp" 14 | #include "../rpc/rpc_connection.hpp" 15 | #include "../network/time.hpp" 16 | // #include "mushroom_log.hpp" 17 | #include "log.hpp" 18 | #include "arg.hpp" 19 | 20 | namespace Mushroom { 21 | 22 | uint32_t RaftServer::TimeoutBase = 150; 23 | uint32_t RaftServer::TimeoutTop = 300; 24 | uint32_t RaftServer::ElectionTimeoutBase = 300; 25 | uint32_t RaftServer::HeartbeatInterval = 30; 26 | 27 | RaftServer::RaftServer(EventBase *event_base, uint16_t port, int32_t id) 28 | :RpcServer(event_base, port), id_(id), state_(Follower), running_(0), term_(0), vote_for_(-1), 29 | commit_(-1), applied_(-1), apply_func_(0) 30 | { 31 | Register("RaftServer::Vote", this, &RaftServer::Vote); 32 | Register("RaftServer::AppendEntry", this, &RaftServer::AppendEntry); 33 | RpcServer::Start(); 34 | } 35 | 36 | RaftServer::~RaftServer() 37 | { 38 | for (auto e : peers_) 39 | delete e; 40 | } 41 | 42 | void RaftServer::Close() 43 | { 44 | mutex_.Lock(); 45 | if (!running_) { 46 | mutex_.Unlock(); 47 | return ; 48 | } 49 | 50 | // Info("closing raft server %d", id_); 51 | 52 | RpcServer::Close(); 53 | 54 | running_ = 0; 55 | if (state_ == Follower) 56 | event_base_->Cancel(election_id_); 57 | else if (state_ == Candidate) 58 | event_base_->RescheduleAfter(timeout_id_, 0); 59 | else 60 | event_base_->Cancel(heartbeat_id_); 61 | 62 | // for (auto e : peers_) 63 | // e->Close(); 64 | 65 | mutex_.Unlock(); 66 | } 67 | 68 | bool RaftServer::IsLeader(uint32_t *term) 69 | { 70 | mutex_.Lock(); 71 | bool ret = (state_ == Leader); 72 | *term = term_; 73 | mutex_.Unlock(); 74 | return ret; 75 | } 76 | 77 | int32_t RaftServer::Id() 78 | { 79 | return id_; 80 | } 81 | 82 | uint32_t RaftServer::Term() 83 | { 84 | mutex_.Lock(); 85 | uint32_t ret = term_; 86 | mutex_.Unlock(); 87 | return ret; 88 | } 89 | 90 | void RaftServer::Status(bool print_log, bool print_next) 91 | { 92 | mutex_.Lock(); 93 | printf("leader: %d term: %u cmit: %d size: %lu\n", 94 | (state_ == Leader), term_, commit_, logs_.size()); 95 | for (auto &e : logs_) 96 | printf("%u ", e.term_); 97 | printf("\n"); 98 | mutex_.Unlock(); 99 | } 100 | 101 | bool RaftServer::Start(/*Mushroom*/Log log, uint32_t *index) 102 | { 103 | mutex_.Lock(); 104 | if (!running_ || state_ != Leader) { 105 | mutex_.Unlock(); 106 | return false; 107 | } 108 | *index = logs_.size(); 109 | log.term_ = term_; 110 | logs_.push_back(log); 111 | mutex_.Unlock(); 112 | // event_base_->RunNow([this]() { SendAppendEntry(false); }); 113 | SendAppendEntry(false); 114 | return true; 115 | } 116 | 117 | bool RaftServer::LogAt(uint32_t index, /*Mushroom*/Log &log) 118 | { 119 | mutex_.Lock(); 120 | if (int32_t(index) > commit_) { 121 | mutex_.Unlock(); 122 | return false; 123 | } 124 | log = logs_[index]; 125 | mutex_.Unlock(); 126 | return true; 127 | } 128 | 129 | void RaftServer::AddPeer(RpcConnection *peer) 130 | { 131 | mutex_.Lock(); 132 | peers_.push_back(peer); 133 | next_.push_back(0); 134 | match_.push_back(0); 135 | mutex_.Unlock(); 136 | } 137 | 138 | std::vector& RaftServer::Peers() 139 | { 140 | return peers_; 141 | } 142 | 143 | void RaftServer::SetApplyFunc(ApplyFunc &&func) 144 | { 145 | apply_func_ = func; 146 | } 147 | 148 | void RaftServer::Start() 149 | { 150 | mutex_.Lock(); 151 | running_ = 1; 152 | RescheduleElection(); 153 | mutex_.Unlock(); 154 | } 155 | 156 | int64_t RaftServer::GetElectionTimeout() 157 | { 158 | static std::default_random_engine engine(time(0)); 159 | static std::uniform_int_distribution dist(TimeoutBase, TimeoutTop); 160 | return dist(engine); 161 | } 162 | 163 | void RaftServer::RescheduleElection() 164 | { 165 | assert(state_ == Follower); 166 | event_base_->RescheduleAfter(&election_id_, GetElectionTimeout(), [this]() { 167 | SendRequestVote(); 168 | }); 169 | } 170 | 171 | void RaftServer::BecomeFollower(uint32_t term) 172 | { 173 | // Info("%d becoming follower, term %u", id_, term); 174 | if (state_ == Leader) 175 | event_base_->Cancel(heartbeat_id_); 176 | else if (state_ == Candidate) 177 | event_base_->RescheduleAfter(timeout_id_, 0); 178 | state_ = Follower; 179 | term_ = term; 180 | vote_for_ = -1; 181 | RescheduleElection(); 182 | } 183 | 184 | void RaftServer::BecomeCandidate() 185 | { 186 | // Info("becoming candidate %d %u", id_, term_); 187 | ++term_; 188 | state_ = Candidate; 189 | vote_for_ = id_; 190 | votes_ = 1; 191 | } 192 | 193 | void RaftServer::BecomeLeader() 194 | { 195 | // Info("%d becoming leader, term %u", id_, term_); 196 | state_ = Leader; 197 | event_base_->RescheduleAfter(timeout_id_, 0); 198 | for (auto &e : next_) 199 | e = commit_ + 1; 200 | for (auto &e : match_) 201 | e = -1; 202 | heartbeat_id_ = event_base_->RunEvery(HeartbeatInterval, [this]() { 203 | SendAppendEntry(true); 204 | }); 205 | } 206 | 207 | void RaftServer::Vote(const RequestVoteArgs *args, RequestVoteReply *reply) 208 | { 209 | mutex_.Lock(); 210 | reply->granted_ = 0; 211 | const RequestVoteArgs &arg = *args; 212 | int32_t last_idx = logs_.size() - 1; 213 | uint32_t last_term = (last_idx >= 0) ? logs_[last_idx].term_ : 0; 214 | uint32_t prev_term = term_; 215 | if (!running_ || arg.term_ < term_) 216 | goto end; 217 | 218 | if (arg.term_ > term_) 219 | BecomeFollower(arg.term_); 220 | 221 | if (vote_for_ != -1 && vote_for_ != arg.id_) 222 | goto end; 223 | 224 | if (arg.last_term_ < last_term) 225 | goto end; 226 | if (arg.last_term_ == last_term && last_idx > arg.last_index_) 227 | goto end; 228 | 229 | // Info("%d vote for %d", id_, arg.id_); 230 | 231 | reply->granted_ = 1; 232 | vote_for_ = arg.id_; 233 | 234 | if (prev_term == term_) 235 | RescheduleElection(); 236 | 237 | end: 238 | reply->term_ = term_; 239 | mutex_.Unlock(); 240 | } 241 | 242 | void RaftServer::SendRequestVote() 243 | { 244 | mutex_.Lock(); 245 | if (!running_) { 246 | mutex_.Unlock(); 247 | return ; 248 | } 249 | BecomeCandidate(); 250 | int32_t last_idx = logs_.size() - 1; 251 | RequestVoteArgs args(term_, id_, last_idx, last_idx >= 0 ? logs_[last_idx].term_ : 0); 252 | 253 | // Info("election: term %u id %d size %d lst_tm %u", args.term_, args.id_, args.last_index_, 254 | // args.last_term_); 255 | 256 | uint32_t size = peers_.size(); 257 | Future *futures = new Future[size]; 258 | for (uint32_t i = 0; i < size; ++i) { 259 | Future *fu = futures + i; 260 | peers_[i]->Call("RaftServer::Vote", &args, fu); 261 | fu->OnCallback([this, fu]() { 262 | ReceiveRequestVoteReply(fu->Value()); 263 | }); 264 | } 265 | 266 | timeout_id_ = event_base_->RunAfter(ElectionTimeoutBase + GetElectionTimeout(), 267 | [this, futures, size]() { 268 | for (uint32_t i = 0; i != size; ++i) { 269 | peers_[i]->RemoveFuture(&futures[i]); 270 | futures[i].Cancel(); 271 | } 272 | delete [] futures; 273 | mutex_.Lock(); 274 | if (running_ && state_ == Candidate) 275 | event_base_->RunNow([this]() { SendRequestVote(); }); 276 | mutex_.Unlock(); 277 | }); 278 | mutex_.Unlock(); 279 | } 280 | 281 | void RaftServer::ReceiveRequestVoteReply(const RequestVoteReply &reply) 282 | { 283 | mutex_.Lock(); 284 | if (!running_ || state_ != Candidate) 285 | goto end; 286 | if (reply.term_ == term_ && reply.granted_) { 287 | if (++votes_ > ((peers_.size() + 1) / 2)) 288 | BecomeLeader(); 289 | } else if (reply.term_ > term_) { 290 | BecomeFollower(reply.term_); 291 | } 292 | end: 293 | mutex_.Unlock(); 294 | } 295 | 296 | void RaftServer::AppendEntry(const AppendEntryArgs *args, AppendEntryReply *reply) 297 | { 298 | mutex_.Lock(); 299 | const AppendEntryArgs &arg = *args; 300 | int32_t prev_i = arg.prev_index_; 301 | uint32_t prev_t = arg.prev_term_; 302 | uint32_t prev_j = 0; 303 | if (!running_) 304 | goto end; 305 | 306 | if (arg.term_ < term_) 307 | goto end; 308 | 309 | if ((arg.term_ > term_) || (arg.term_ == term_ && state_ == Candidate)) 310 | BecomeFollower(arg.term_); 311 | else 312 | RescheduleElection(); 313 | 314 | if (prev_i >= int32_t(logs_.size())) 315 | goto index; 316 | if (prev_i >= 0 && logs_[prev_i].term_ != prev_t) { 317 | assert(commit_ < prev_i); 318 | logs_.erase(logs_.begin() + prev_i, logs_.end()); 319 | goto index; 320 | } 321 | 322 | ++prev_i; 323 | for (; prev_i < int32_t(logs_.size()) && prev_j < arg.entries_.size(); ++prev_i, ++prev_j) { 324 | if (logs_[prev_i].term_ != arg.entries_[prev_j].term_) { 325 | assert(commit_ < prev_i); 326 | logs_.erase(logs_.begin() + prev_i, logs_.end()); 327 | break; 328 | } 329 | } 330 | if (prev_j < arg.entries_.size()) 331 | logs_.insert(logs_.end(), arg.entries_.begin() + prev_j, arg.entries_.end()); 332 | 333 | if (arg.leader_commit_ > commit_) { 334 | commit_ = std::min(arg.leader_commit_, int32_t(logs_.size()) - 1); 335 | // for (; applied_ < commit_;) apply_func_(logs_[++applied_]); 336 | } 337 | 338 | index: 339 | reply->idx_ = logs_.size() - 1; 340 | 341 | end: 342 | reply->term_ = term_; 343 | mutex_.Unlock(); 344 | } 345 | 346 | void RaftServer::SendAppendEntry(bool heartbeat) 347 | { 348 | mutex_.Lock(); 349 | if (!running_ || state_ != Leader) { 350 | mutex_.Unlock(); 351 | return ; 352 | } 353 | uint32_t size = peers_.size(); 354 | AppendEntryArgs args[size]; 355 | Future *futures = new Future[size]; 356 | for (size_t i = 0; i < peers_.size(); ++i) { 357 | int32_t prev = next_[i] - 1; 358 | args[i] = {term_, id_, prev >= 0 ? logs_[prev].term_ : 0, prev, commit_}; 359 | if (!heartbeat && next_[i] < int32_t(logs_.size())) 360 | args[i].entries_.insert(args[i].entries_.end(), logs_.begin() + next_[i], logs_.end()); 361 | Future *fu = futures + i; 362 | peers_[i]->Call("RaftServer::AppendEntry", &args[i], fu); 363 | fu->OnCallback([this, i, fu]() { 364 | ReceiveAppendEntryReply(i, fu->Value()); 365 | }); 366 | } 367 | mutex_.Unlock(); 368 | event_base_->RunAfter(TimeoutTop, [this, futures, size]() { 369 | for (uint32_t i = 0; i != size; ++i) { 370 | peers_[i]->RemoveFuture(&futures[i]); 371 | futures[i].Cancel(); 372 | } 373 | delete [] futures; 374 | }); 375 | } 376 | 377 | void RaftServer::ReceiveAppendEntryReply(uint32_t i, const AppendEntryReply &reply) 378 | { 379 | mutex_.Lock(); 380 | uint32_t vote = 1; 381 | if (!running_|| state_ != Leader) 382 | goto end; 383 | if (reply.term_ > term_) { 384 | BecomeFollower(reply.term_); 385 | goto end; 386 | } 387 | if (reply.term_ != term_) 388 | goto end; 389 | if (next_[i] == reply.idx_ + 1) 390 | goto end; 391 | next_[i] = reply.idx_ + 1; 392 | match_[i] = next_[i] - 1; 393 | 394 | if (commit_ >= reply.idx_ || logs_[reply.idx_].term_ != term_) 395 | goto end; 396 | for (uint32_t i = 0; i < peers_.size(); ++i) 397 | if (match_[i] >= reply.idx_) 398 | ++vote; 399 | if (vote > ((peers_.size() + 1) / 2)) { 400 | commit_ = reply.idx_; 401 | // for (; applied_ < commit_;) apply_func_(logs_[++applied_]); 402 | } 403 | end: 404 | mutex_.Unlock(); 405 | } 406 | 407 | } // namespace Mushroom 408 | -------------------------------------------------------------------------------- /src/raft/raft_server.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-22 21:19:57 6 | **/ 7 | 8 | #ifndef _RAFT_SERVER_HPP_ 9 | #define _RAFT_SERVER_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | #include "../include/mutex.hpp" 15 | #include "../include/cond.hpp" 16 | #include "../rpc/rpc_server.hpp" 17 | #include "../network/eventbase.hpp" 18 | 19 | namespace Mushroom { 20 | 21 | class Log; 22 | class MushroomLog; 23 | class RpcConnection; 24 | class RequestVoteArgs; 25 | class RequestVoteReply; 26 | class AppendEntryArgs; 27 | class AppendEntryReply; 28 | 29 | class RaftServer : public RpcServer 30 | { 31 | public: 32 | enum State { Follower = 0x0, Candidate, Leader }; 33 | 34 | typedef std::function ApplyFunc; 35 | 36 | RaftServer(EventBase *event_base, uint16_t port, int32_t id); 37 | 38 | ~RaftServer(); 39 | 40 | bool IsLeader(uint32_t *term); 41 | 42 | int32_t Id(); 43 | 44 | uint32_t Term(); 45 | 46 | void Status(bool print_log = false, bool print_next = false); 47 | 48 | void SetApplyFunc(ApplyFunc &&func); 49 | 50 | void Start(); 51 | 52 | bool Start(/*Mushroom*/Log log, uint32_t *index); 53 | 54 | bool LogAt(uint32_t index, /*Mushroom*/Log &log); 55 | 56 | void Close(); 57 | 58 | void AddPeer(RpcConnection *peer); 59 | 60 | std::vector& Peers(); 61 | 62 | void Vote(const RequestVoteArgs *args, RequestVoteReply *reply); 63 | 64 | void AppendEntry(const AppendEntryArgs *args, AppendEntryReply *reply); 65 | 66 | static uint32_t ElectionTimeoutBase; 67 | 68 | private: 69 | static uint32_t TimeoutBase; 70 | static uint32_t TimeoutTop; 71 | static uint32_t HeartbeatInterval; 72 | 73 | static int64_t GetElectionTimeout(); 74 | 75 | void RescheduleElection(); 76 | 77 | void SendRequestVote(); 78 | 79 | void RequestVote(); 80 | 81 | void SendAppendEntry(bool heartbeat); 82 | 83 | void BecomeFollower(uint32_t term); 84 | 85 | void BecomeCandidate(); 86 | 87 | void BecomeLeader(); 88 | 89 | void UpdateCommitIndex(); 90 | 91 | void ReceiveRequestVoteReply(const RequestVoteReply &reply); 92 | 93 | void ReceiveAppendEntryReply(uint32_t i, const AppendEntryReply &reply); 94 | 95 | using RpcServer::Register; 96 | using RpcServer::event_base_; 97 | 98 | int32_t id_; 99 | 100 | uint8_t state_; 101 | uint8_t running_; 102 | 103 | uint32_t term_; 104 | int32_t vote_for_; 105 | int32_t commit_; 106 | int32_t applied_; 107 | 108 | uint32_t votes_; 109 | 110 | // std::vector logs_; 111 | std::vector logs_; 112 | 113 | std::vector next_; 114 | std::vector match_; 115 | 116 | std::vector peers_; 117 | 118 | Mutex mutex_; 119 | 120 | TimerId election_id_; 121 | TimerId heartbeat_id_; 122 | TimerId timeout_id_; 123 | 124 | ApplyFunc apply_func_; 125 | }; 126 | 127 | } // namespace Mushroom 128 | 129 | #endif /* _RAFT_SERVER_HPP_ */ 130 | -------------------------------------------------------------------------------- /src/rpc/future.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-10 20:22:22 6 | **/ 7 | 8 | #ifndef _FUTURE_HPP_ 9 | #define _FUTURE_HPP_ 10 | 11 | #include 12 | 13 | #include "../include/utility.hpp" 14 | #include "../include/mutex.hpp" 15 | #include "../include/cond.hpp" 16 | 17 | namespace Mushroom { 18 | 19 | template 20 | class Future : private NoCopyTemplate 21 | { 22 | public: 23 | enum Status { Pending = 0x0, Ok = 0x1, Timeout = 0x2 }; 24 | 25 | Future():status_(Pending), cb_(0) { } 26 | 27 | inline void SetId(uint32_t id) { id_ = id; } 28 | 29 | inline uint32_t GetId() const { return id_; } 30 | 31 | inline void OnCallback(Func &&cb) { cb_ = cb; } 32 | 33 | inline T& Value() { return value_; } 34 | 35 | inline void Wait() { 36 | mutex_.Lock(); 37 | while (status_ == Pending) 38 | cond_.Wait(mutex_); 39 | mutex_.Unlock(); 40 | } 41 | 42 | inline bool ok() { 43 | mutex_.Lock(); 44 | bool ret = (status_ == Ok); 45 | mutex_.Unlock(); 46 | return ret; 47 | } 48 | 49 | inline void Notify(Marshaller &mar) { 50 | mar >> value_; 51 | mutex_.Lock(); 52 | if (status_ == Pending) { 53 | status_ = Ok; 54 | cond_.Signal(); 55 | } else { 56 | mutex_.Unlock(); 57 | return ; 58 | } 59 | if (cb_) cb_(); 60 | mutex_.Unlock(); 61 | } 62 | 63 | inline void Cancel() { 64 | mutex_.Lock(); 65 | if (status_ == Pending) { 66 | status_ = Timeout; 67 | cond_.Signal(); 68 | } 69 | mutex_.Unlock(); 70 | } 71 | 72 | private: 73 | uint8_t status_; 74 | uint32_t id_; 75 | T value_; 76 | Mutex mutex_; 77 | Cond cond_; 78 | Func cb_; 79 | }; 80 | 81 | } // namespace Mushroom 82 | 83 | #endif /* _FUTURE_HPP_ */ -------------------------------------------------------------------------------- /src/rpc/marshaller.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 09:17:51 6 | **/ 7 | 8 | #ifndef _MARSHALLER_HPP_ 9 | #define _MARSHALLER_HPP_ 10 | 11 | #include 12 | 13 | #include "../network/buffer.hpp" 14 | 15 | namespace Mushroom { 16 | 17 | class Marshaller 18 | { 19 | public: 20 | Marshaller():input_(0), output_(0) { } 21 | 22 | Marshaller(Buffer *input, Buffer *output):input_(input), output_(output) { } 23 | 24 | template 25 | inline void MarshalArgs(uint32_t id, uint32_t rid, const T *args); 26 | 27 | template 28 | inline void MarshalReply(uint32_t rid, const T *reply); 29 | 30 | inline void Read(const void *str, uint32_t len); 31 | 32 | inline void Write(void *str, uint32_t len); 33 | 34 | inline uint32_t HasCompleteArgs(); 35 | 36 | inline void Dump(uint32_t size); 37 | 38 | private: 39 | inline void Unget(uint32_t size); 40 | 41 | Buffer *input_; 42 | Buffer *output_; 43 | }; 44 | 45 | inline Marshaller& operator<<(Marshaller &marshaller, const uint8_t &v) { 46 | marshaller.Read(&v, 1); 47 | return marshaller; 48 | } 49 | 50 | inline Marshaller& operator<<(Marshaller &marshaller, const int32_t &v) { 51 | marshaller.Read(&v, 4); 52 | return marshaller; 53 | } 54 | 55 | inline Marshaller& operator<<(Marshaller &marshaller, const uint32_t &v) { 56 | marshaller.Read(&v, 4); 57 | return marshaller; 58 | } 59 | 60 | template 61 | inline Marshaller& operator<<(Marshaller &marshaller, const std::vector &v) { 62 | uint32_t e = v.size(); 63 | marshaller << e; 64 | for (uint32_t i = 0; i < e; ++i) 65 | marshaller << v[i]; 66 | return marshaller; 67 | } 68 | 69 | inline Marshaller& operator>>(Marshaller &marshaller, uint8_t &v) { 70 | marshaller.Write(&v, 1); 71 | return marshaller; 72 | } 73 | 74 | inline Marshaller& operator>>(Marshaller &marshaller, int32_t &v) { 75 | marshaller.Write(&v, 4); 76 | return marshaller; 77 | } 78 | 79 | inline Marshaller& operator>>(Marshaller &marshaller, uint32_t &v) { 80 | marshaller.Write(&v, 4); 81 | return marshaller; 82 | } 83 | 84 | template 85 | inline Marshaller& operator>>(Marshaller &marshaller, std::vector &v) { 86 | uint32_t e; 87 | marshaller >> e; 88 | v.reserve(e); 89 | for (uint32_t i = 0; i < e; ++i) { 90 | T t; 91 | marshaller >> t; 92 | v.push_back(t); 93 | } 94 | return marshaller; 95 | } 96 | 97 | template 98 | inline void Marshaller::MarshalArgs(uint32_t id, uint32_t rid, const T *args) 99 | { 100 | output_->Reset(); 101 | uint32_t *len = (uint32_t *)output_->end(); 102 | output_->AdvanceTail(4); 103 | uint32_t before = output_->size(); 104 | *this << id; 105 | *this << rid; 106 | *this << *args; 107 | *len = output_->size() - before; 108 | } 109 | 110 | template 111 | inline void Marshaller::MarshalReply(uint32_t rid, const T *reply) 112 | { 113 | output_->Reset(); 114 | uint32_t *len = (uint32_t *)output_->end(); 115 | output_->AdvanceTail(4); 116 | uint32_t before = output_->size(); 117 | *this << rid; 118 | *this << *reply; 119 | *len = output_->size() - before; 120 | } 121 | 122 | inline void Marshaller::Read(const void *str, uint32_t len) 123 | { 124 | output_->Read((const char *)str, len); 125 | } 126 | 127 | inline void Marshaller::Write(void *str, uint32_t len) 128 | { 129 | input_->Write((char *)str, len); 130 | } 131 | 132 | inline void Marshaller::Unget(uint32_t size) 133 | { 134 | input_->Unget(size); 135 | } 136 | 137 | inline void Marshaller::Dump(uint32_t size) 138 | { 139 | input_->AdvanceHead(size); 140 | } 141 | 142 | inline uint32_t Marshaller::HasCompleteArgs() 143 | { 144 | if (input_->size() < 4) 145 | return 0; 146 | uint32_t packet_size; 147 | *this >> packet_size; 148 | if (input_->size() >= packet_size) { 149 | return packet_size; 150 | } else { 151 | Unget(4); 152 | return 0; 153 | } 154 | } 155 | 156 | } // namespace Mushroom 157 | 158 | #endif /* _MARSHALLER_HPP_ */ -------------------------------------------------------------------------------- /src/rpc/rpc.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-01 20:04:39 6 | **/ 7 | 8 | #ifndef _RPC_HPP_ 9 | #define _RPC_HPP_ 10 | 11 | #include "../include/utility.hpp" 12 | #include "marshaller.hpp" 13 | 14 | namespace Mushroom { 15 | 16 | class RPC 17 | { 18 | public: 19 | RPC():service_(0) { } 20 | 21 | template 22 | inline uint32_t Generate(const char *str, T1 *obj, void (T1::*(fun))(const T2*, T3*)); 23 | 24 | inline void GetReady(Marshaller &marshaller) { marshaller_ = marshaller; } 25 | 26 | inline void operator()() { service_(); } 27 | 28 | inline static uint32_t Hash(const char *str); 29 | 30 | private: 31 | Marshaller marshaller_; 32 | Func service_; 33 | }; 34 | 35 | template 36 | inline uint32_t RPC::Generate(const char *str, T1 *obj, void (T1::*(fun))(const T2*, T3*)) { 37 | service_ = [this, obj, fun]() { 38 | uint32_t rid; 39 | marshaller_ >> rid; 40 | T2 args; 41 | marshaller_ >> args; 42 | T3 reply; 43 | (obj->*fun)(&args, &reply); 44 | marshaller_.MarshalReply(rid, &reply); 45 | }; 46 | return Hash(str); 47 | } 48 | 49 | inline uint32_t RPC::Hash(const char *str) { 50 | uint32_t ret = 0; 51 | char *p = (char *)str; 52 | while (*p) 53 | ret += uint32_t(*p++); 54 | return ret; 55 | } 56 | 57 | } // namespace Mushroom 58 | 59 | #endif /* _RPC_HPP_ */ -------------------------------------------------------------------------------- /src/rpc/rpc_connection.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-02 21:22:15 6 | **/ 7 | 8 | #include 9 | 10 | #include "rpc_connection.hpp" 11 | 12 | namespace Mushroom { 13 | 14 | atomic_32_t RpcConnection::RpcId(0); 15 | 16 | RpcConnection::RpcConnection(const EndPoint &server, Poller *poller, float error_rate) 17 | :Connection(server, poller), disable_(0), error_rate_(error_rate), marshaller_(&input_, &output_) 18 | { 19 | readcb_ = [this]() { 20 | uint32_t packet_size; 21 | if (disable_.get()) { 22 | input_.Clear(); 23 | return ; 24 | } 25 | for (; (packet_size = marshaller_.HasCompleteArgs());) { 26 | uint32_t rid; 27 | marshaller_ >> rid; 28 | mutex_.Lock(); 29 | auto it = futures_.find(rid); 30 | if (it == futures_.end()) { 31 | mutex_.Unlock(); 32 | marshaller_.Dump(packet_size - 4); 33 | // Info("rpc id %u not called or expired :(", rid); 34 | } else { 35 | Func func(std::move(it->second)); 36 | futures_.erase(it); 37 | mutex_.Unlock(); 38 | func(); 39 | } 40 | } 41 | if (input_.size()) 42 | input_.Adjust(); 43 | }; 44 | } 45 | 46 | RpcConnection::RpcConnection(const Socket &socket, Poller *poller) 47 | :Connection(socket, poller), disable_(0), error_rate_(0), marshaller_(&input_, &output_) { } 48 | 49 | RpcConnection::~RpcConnection() { } 50 | 51 | bool RpcConnection::Close() 52 | { 53 | Disable(); 54 | return Connection::Close(); 55 | } 56 | 57 | void RpcConnection::Disable() 58 | { 59 | disable_ = 1; 60 | } 61 | 62 | bool RpcConnection::Disabled() 63 | { 64 | return disable_.get(); 65 | } 66 | 67 | void RpcConnection::Enable() 68 | { 69 | disable_ = 0; 70 | } 71 | 72 | Marshaller& RpcConnection::GetMarshaller() 73 | { 74 | return marshaller_; 75 | } 76 | 77 | } // namespace Mushroom 78 | -------------------------------------------------------------------------------- /src/rpc/rpc_connection.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-02 21:22:07 6 | **/ 7 | 8 | #ifndef _RPC_CONNECTION_HPP_ 9 | #define _RPC_CONNECTION_HPP_ 10 | 11 | #include 12 | // #include 13 | // #include 14 | 15 | #include "../include/atomic.hpp" 16 | #include "../include/mutex.hpp" 17 | #include "../network/connection.hpp" 18 | #include "rpc.hpp" 19 | #include "marshaller.hpp" 20 | #include "future.hpp" 21 | 22 | // static std::default_random_engine engine(time(0)); 23 | // static std::uniform_real_distribution dist(0, 1); 24 | 25 | namespace Mushroom { 26 | 27 | class RpcConnection : public Connection 28 | { 29 | public: 30 | static atomic_32_t RpcId; 31 | 32 | RpcConnection(const EndPoint &server, Poller *poller, float error_rate); 33 | 34 | RpcConnection(const Socket &socket, Poller *poller); 35 | 36 | ~RpcConnection(); 37 | 38 | template 39 | inline void Call(const char *str, const T1 *args, Future *fu); 40 | 41 | template 42 | inline void RemoveFuture(Future *fu); 43 | 44 | bool Close(); 45 | 46 | Marshaller& GetMarshaller(); 47 | 48 | bool Disabled(); 49 | 50 | void Disable(); 51 | 52 | void Enable(); 53 | 54 | using Connection::OnRead; 55 | 56 | private: 57 | using Connection::Send; 58 | using Connection::OnWrite; 59 | 60 | atomic_32_t disable_; 61 | float error_rate_; 62 | 63 | Mutex mutex_; 64 | std::map futures_; 65 | 66 | Marshaller marshaller_; 67 | }; 68 | 69 | template 70 | inline void RpcConnection::Call(const char *str, const T1 *args, Future *fu) 71 | { 72 | uint32_t id = RPC::Hash(str); 73 | uint32_t rid = RpcId++; 74 | fu->SetId(rid); 75 | mutex_.Lock(); 76 | futures_.insert({rid, std::move([fu, this]() { fu->Notify(marshaller_); })}); 77 | if (!disable_.get()) { // && dist(engine) > error_rate_ 78 | marshaller_.MarshalArgs(id, rid, args); 79 | SendOutput(); 80 | } 81 | mutex_.Unlock(); 82 | } 83 | 84 | template 85 | inline void RpcConnection::RemoveFuture(Future *fu) 86 | { 87 | mutex_.Lock(); 88 | auto it = futures_.find(fu->GetId()); 89 | if (it != futures_.end()) 90 | futures_.erase(it); 91 | mutex_.Unlock(); 92 | } 93 | 94 | } // namespace Mushroom 95 | 96 | #endif /* _RPC_CONNECTION_HPP_ */ -------------------------------------------------------------------------------- /src/rpc/rpc_server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 11:23:08 6 | **/ 7 | 8 | #include "rpc_server.hpp" 9 | #include "../network/eventbase.hpp" 10 | #include "rpc_connection.hpp" 11 | #include "../network/channel.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | RpcServer::RpcServer(EventBase *event_base, uint16_t port) 16 | :Server(event_base, port), rpc_count_(0) { } 17 | 18 | RpcServer::~RpcServer() 19 | { 20 | for (auto e : services_) 21 | delete e.second; 22 | } 23 | 24 | void RpcServer::Start() 25 | { 26 | Server::Start(); 27 | listen_->OnRead([this]() { HandleAccept(); }); 28 | } 29 | 30 | void RpcServer::Close() 31 | { 32 | Server::Close(); 33 | } 34 | 35 | void RpcServer::HandleAccept() 36 | { 37 | int fd = socket_.Accept(); 38 | assert(fd > 0); 39 | RpcConnection *con = new RpcConnection(Socket(fd), event_base_->GetPoller()); 40 | connections_.push_back((Connection *)con); 41 | con->OnRead([con, this]() { 42 | if (con->Disabled()) { 43 | con->GetInput().Clear(); 44 | return ; 45 | } 46 | Marshaller &mar = con->GetMarshaller(); 47 | bool has = false; 48 | for (; mar.HasCompleteArgs();) { 49 | uint32_t id; 50 | mar >> id; 51 | auto it = services_.find(id); 52 | assert(it != services_.end()); 53 | RPC *rpc = it->second; 54 | rpc->GetReady(mar); 55 | (*rpc)(); 56 | has = true; 57 | ++rpc_count_; 58 | } 59 | Buffer &in = con->GetInput(); 60 | if (in.size()) 61 | in.Adjust(); 62 | if (has) 63 | con->SendOutput(); 64 | }); 65 | } 66 | 67 | uint32_t RpcServer::RpcCount() 68 | { 69 | return rpc_count_.get(); 70 | } 71 | 72 | } // namespace Mushroom 73 | -------------------------------------------------------------------------------- /src/rpc/rpc_server.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-30 11:23:02 6 | **/ 7 | 8 | #ifndef _RPC_SERVER_HPP_ 9 | #define _RPC_SERVER_HPP_ 10 | 11 | #include 12 | #include 13 | 14 | #include "../include/utility.hpp" 15 | #include "../include/atomic.hpp" 16 | #include "../network/server.hpp" 17 | #include "rpc.hpp" 18 | 19 | namespace Mushroom { 20 | 21 | class EventBase; 22 | 23 | class RpcServer : public Server 24 | { 25 | public: 26 | RpcServer(EventBase *event_base, uint16_t port); 27 | 28 | virtual ~RpcServer(); 29 | 30 | void Start(); 31 | 32 | void Close(); 33 | 34 | template 35 | void Register(const char *str, T1 *obj, void (T1::*(fun))(const T2*, T3*)); 36 | 37 | uint32_t RpcCount(); 38 | 39 | private: 40 | std::unordered_map services_; 41 | 42 | atomic_32_t rpc_count_; 43 | 44 | void HandleAccept(); 45 | }; 46 | 47 | template 48 | void RpcServer::Register(const char *str, T1 *obj, void (T1::*(fun))(const T2*, T3*)) 49 | { 50 | RPC *rpc = new RPC(); 51 | uint32_t id = rpc->Generate(str, obj, fun); 52 | assert(services_.find(id) == services_.end()); 53 | services_.insert({id, rpc}); 54 | } 55 | 56 | } // namespace Mushroom 57 | 58 | #endif /* _RPC_SERVER_HPP_ */ -------------------------------------------------------------------------------- /src/run: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | if [ "$1" = "art" ] 3 | then 4 | make art_test && ./art_test $2 5 | elif [ "$1" = "index" ] 6 | then 7 | make distributed_index_test && ./distributed_index_test 4096 4800 10 4 $2 8 | elif [ "$1" = "raft" ] 9 | then 10 | make raft_test 11 | total=1 12 | arg= 13 | if [ $2 -gt 0 ] 14 | then 15 | total=$2 16 | else 17 | arg=$2 18 | fi 19 | if [ $3 -gt 0 ] 20 | then 21 | total=$3 22 | fi 23 | fail=0 24 | success=0 25 | for i in `seq $total`; do 26 | echo 'Running Test '$i' of '$total'' 27 | ./raft_test $arg 28 | # ./raft_test $arg > $i 29 | if [ $? -ne 0 ] 30 | then 31 | fail=`expr $fail + 1` 32 | echo 'Test Failed at '$i'' 33 | else 34 | success=`expr $success + 1` 35 | # rm $i 36 | fi 37 | done 38 | echo 'Total '$total' Fail '$fail' Success '$success'' 39 | elif [ "$1" = "blink" ] 40 | then 41 | if [ "$2" = "queue" ] 42 | then 43 | make blinktree_with_queue_test && ./blinktree_with_queue_test 4096 4800 10 4 $3 44 | elif [ "$2" = "thread" ] 45 | then 46 | make blinktree_multi_thread_test && ./blinktree_multi_thread_test 4096 4800 10 4 $3 47 | else 48 | echo "wrong argument :(" 49 | fi 50 | elif [ "$1" = "network" ] 51 | then 52 | if [ "$2" = "client" ] 53 | then 54 | make client_test && ./client_test 55 | elif [ "$2" = "server" ] 56 | then 57 | make server_test && ./server_test 58 | else 59 | echo "wrong argument :(" 60 | fi 61 | elif [ "$1" = "rpc" ] 62 | then 63 | if [ "$2" = "client" ] 64 | then 65 | make rpc_client_test && ./rpc_client_test $3 66 | elif [ "$2" = "server" ] 67 | then 68 | make rpc_server_test && ./rpc_server_test 69 | else 70 | echo "wrong argument :(" 71 | fi 72 | else 73 | echo "wrong argument :(" 74 | fi 75 | -------------------------------------------------------------------------------- /test/art.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-21 10:42:40 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "../src/art/art.hpp" 15 | 16 | const char *file = "../data/10000000"; 17 | static const int key_len = 16; 18 | static int total; 19 | 20 | using namespace Mushroom; 21 | 22 | double Put(ART *art) 23 | { 24 | int fd = open(file, O_RDONLY); 25 | assert(fd > 0); 26 | char buf[8192]; 27 | int cur = 0, ptr = 0, count = 0; 28 | bool flag = true; 29 | 30 | auto beg = std::chrono::high_resolution_clock::now(); 31 | for (; flag && (ptr = pread(fd, buf, 8192, cur)) > 0; cur += ptr) { 32 | while (--ptr && buf[ptr] != '\n' && buf[ptr] != '\0') buf[ptr] = '\0'; 33 | if (ptr) buf[ptr++] = '\0'; 34 | else break; 35 | for (int i = 0; i < ptr; ++i) { 36 | char *tmp = buf + i; 37 | i += key_len; 38 | assert(buf[i] == '\n' || buf[i] == '\0'); 39 | buf[i] = '\0'; 40 | 41 | art->Put((const uint8_t *)tmp, key_len, (uint32_t)count); 42 | 43 | if (++count == total) { 44 | flag = false; 45 | break; 46 | } 47 | } 48 | } 49 | close(fd); 50 | 51 | auto end = std::chrono::high_resolution_clock::now(); 52 | auto t = std::chrono::duration>(end - beg).count(); 53 | return t; 54 | } 55 | 56 | double Get(ART *art) 57 | { 58 | int fd = open(file, O_RDONLY); 59 | assert(fd > 0); 60 | char buf[8192]; 61 | int cur = 0, ptr = 0, count = 0; 62 | bool flag = true; 63 | 64 | auto beg = std::chrono::high_resolution_clock::now(); 65 | for (; flag && (ptr = pread(fd, buf, 8192, cur)) > 0; cur += ptr) { 66 | while (--ptr && buf[ptr] != '\n' && buf[ptr] != '\0') buf[ptr] = '\0'; 67 | if (ptr) buf[ptr++] = '\0'; 68 | else break; 69 | for (int i = 0; i < ptr; ++i) { 70 | char *tmp = buf + i; 71 | i += key_len; 72 | assert(buf[i] == '\n' || buf[i] == '\0'); 73 | buf[i] = '\0'; 74 | 75 | uint32_t val; 76 | assert(art->Get((const uint8_t *)tmp, key_len, &val) && int(val) == count); 77 | 78 | if (++count == total) { 79 | flag = false; 80 | break; 81 | } 82 | } 83 | } 84 | close(fd); 85 | 86 | auto end = std::chrono::high_resolution_clock::now(); 87 | auto t = std::chrono::duration>(end - beg).count(); 88 | return t; 89 | } 90 | 91 | int main(int argc, char **argv) 92 | { 93 | total = argc == 2 ? atoi(argv[1]) : 1; 94 | 95 | ART art; 96 | 97 | auto t1 = Put(&art); 98 | 99 | auto t2 = Get(&art); 100 | 101 | printf("\033[31mtotal: %d\033[0m\n\033[32mput time: %f s\033[0m\n", total, t1); 102 | printf("\033[34mget time: %f s\033[0m\n", t2); 103 | 104 | return 0; 105 | } 106 | -------------------------------------------------------------------------------- /test/barrier.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-27 17:43:00 6 | **/ 7 | 8 | #include 9 | 10 | #include "../src/palm/barrier.hpp" 11 | 12 | static const int threads = 5; 13 | static Mushroom::Barrier barrier(threads); 14 | 15 | void* f1(void *) 16 | { 17 | printf("hello\n"); 18 | barrier.Wait(); 19 | return (void *)0; 20 | } 21 | 22 | void* f2(void *) 23 | { 24 | printf("world\n"); 25 | barrier.Wait(); 26 | return (void *)0; 27 | } 28 | 29 | int main() 30 | { 31 | pthread_t ids[threads]; 32 | for (int i = 0; i < threads; ++i) { 33 | assert(pthread_create(&ids[i], 0, f1, 0) == 0); 34 | } 35 | for (int i = 0; i != threads; ++i) 36 | assert(pthread_join(ids[i], 0) == 0); 37 | 38 | for (int i = 0; i < threads; ++i) { 39 | assert(pthread_create(&ids[i], 0, f2, 0) == 0); 40 | } 41 | for (int i = 0; i != threads; ++i) 42 | assert(pthread_join(ids[i], 0) == 0); 43 | return 0; 44 | } -------------------------------------------------------------------------------- /test/batch.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-28 15:36:00 6 | **/ 7 | 8 | #include 9 | 10 | #include "../src/blink/slice.hpp" 11 | #include "../src/palm/batch.hpp" 12 | 13 | int main() 14 | { 15 | Mushroom::Batch::SetSize(8); 16 | Mushroom::Batch batch; 17 | 18 | const char *str = "hello world >:)<"; 19 | batch.SetKeySlice(7, str); 20 | auto slice = batch.GetKeySlice(7); 21 | 22 | assert(!memcmp(str, slice->key_, Mushroom::KeySlice::KeyLen)); 23 | printf("%s", slice->ToString().c_str()); 24 | return 0; 25 | } -------------------------------------------------------------------------------- /test/batcher.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-07-29 17:02:00 6 | **/ 7 | 8 | #include 9 | 10 | #include "../src/blink/slice.hpp" 11 | #include "../src/palm/batcher.hpp" 12 | 13 | int main() 14 | { 15 | using namespace Mushroom; 16 | Batcher batcher; 17 | for (uint32_t i = 0; i < batcher.Capacity(); ++i) { 18 | std::string s = std::to_string(i); 19 | s.insert(0, 3 - s.size(), '0'); 20 | s = "batcher_test_" + s; 21 | TempSlice(key); 22 | memcpy(key->key_, s.c_str(), KeySlice::KeyLen); 23 | assert(batcher.InsertKeySlice(key)); 24 | } 25 | assert(batcher.TotalKey() == batcher.Capacity()); 26 | printf("%s\n", batcher.ToString().c_str()); 27 | return 0; 28 | } -------------------------------------------------------------------------------- /test/bloom.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-09-05 22:48:34 6 | **/ 7 | 8 | #include 9 | 10 | #include "unit.h" 11 | #include "../src/blink/bloom.hpp" 12 | 13 | using namespace Mushroom; 14 | 15 | static int NextLength(int length) { 16 | if (length < 10) { 17 | length += 1; 18 | } else if (length < 100) { 19 | length += 10; 20 | } else if (length < 1000) { 21 | length += 100; 22 | } else { 23 | length += 1000; 24 | } 25 | return length; 26 | } 27 | 28 | TEST(Empty_BloomFilter) 29 | { 30 | BloomFilter f(0); 31 | ASSERT_FALSE(f.Match("hello", 5)); 32 | ASSERT_FALSE(f.Match("world", 5)); 33 | } 34 | 35 | TEST(Simple_BloomFilter) 36 | { 37 | BloomFilter f(2); 38 | ASSERT_FALSE(f.Match("hello", 5)); 39 | ASSERT_FALSE(f.Match("world", 5)); 40 | f.Add("hello", 5); 41 | f.Add("world", 5); 42 | ASSERT_TRUE(f.Match("hello", 5)); 43 | ASSERT_TRUE(f.Match("world", 5)); 44 | } 45 | 46 | TEST(VaryingLengths_BloomFilter) { 47 | char buffer[sizeof(int)]; 48 | 49 | // Count number of filters that significantly exceed the false positive rate 50 | int mediocre_filters = 0; 51 | int good_filters = 0; 52 | 53 | for (int length = 1; length <= 10000; length = NextLength(length)) { 54 | BloomFilter f(length); 55 | for (int i = 0; i < length; i++) { 56 | memcpy(buffer, &i, sizeof(int)); 57 | f.Add(buffer, sizeof(int)); 58 | } 59 | 60 | ASSERT_LE((int)f.size(), ((length * 10 / 8) + 40)); 61 | 62 | for (int i = 0; i < length; i++) { 63 | memcpy(buffer, &i, sizeof(int)); 64 | ASSERT_TRUE(f.Match(buffer, sizeof(int))); 65 | } 66 | 67 | int result = 0; 68 | for (int i = 0; i < 10000; i++) { 69 | int tmp = i + 1000000000; 70 | memcpy(buffer, &tmp, sizeof(int)); 71 | if (f.Match(buffer, sizeof(int))) 72 | ++result; 73 | } 74 | double rate = double(result) / 10000.0; 75 | 76 | printf("false positives: %5.2f%% @ length = %6d ; bytes = %6d\n", 77 | rate * 100.0, length, static_cast(f.size())); 78 | // ASSERT_LE(rate, 0.02); // Must not be over 2% 79 | if (rate > 0.0125) mediocre_filters++; // Allowed, but not too often 80 | else good_filters++; 81 | } 82 | printf("Filters: %d good, %d mediocre\n", good_filters, mediocre_filters); 83 | ASSERT_LE(mediocre_filters, good_filters / 5); 84 | } 85 | 86 | int main() 87 | { 88 | return RUN_ALL_TESTS(0); 89 | } 90 | -------------------------------------------------------------------------------- /test/distributed_index.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-03 09:37:22 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "../src/mushroom/db.hpp" 13 | #include "../src/network/signal.hpp" 14 | #include "../src/rpc/rpc_connection.hpp" 15 | #include "../src/raft/raft_server.hpp" 16 | #include "../src/include/thread.hpp" 17 | #include "../src/raft/mushroom_log.hpp" 18 | 19 | using namespace std; 20 | using namespace Mushroom; 21 | 22 | static EventBase *base = 0; 23 | static Thread *loop = 0; 24 | static vector rafts; 25 | static uint16_t port_base = 7000; 26 | 27 | static void MakeRaftConfig(int total) { 28 | base = new EventBase(4, 64); 29 | loop = new Thread([&]() { base->Loop(); }); 30 | rafts.resize(total); 31 | for (int i = 0; i < total; ++i) 32 | rafts[i] = new RaftServer(base, port_base++, i); 33 | for (int i = 0; i < total; ++i) { 34 | for (int j = 0; j < total; ++j) { 35 | if (i == j) continue; 36 | rafts[i]->AddPeer(new RpcConnection(EndPoint(rafts[j]->Port(), "127.0.0.1"), 37 | base->GetPoller(), 0)); 38 | } 39 | } 40 | loop->Start(); 41 | for (auto e : rafts) 42 | e->Start(); 43 | } 44 | 45 | static void FreeRaftSet() { 46 | for (auto e : rafts) 47 | e->Close(); 48 | if (base) base->Exit(); 49 | if (loop) loop->Stop(); 50 | for (auto e : rafts) 51 | delete e; 52 | delete base; 53 | delete loop; 54 | base = 0; 55 | loop = 0; 56 | rafts.clear(); 57 | } 58 | 59 | int main(int argc, char **argv) 60 | { 61 | const char *file = "../data/10000000"; 62 | 63 | assert(argc > 4); 64 | uint32_t page_size = atoi(argv[1]) ? atoi(argv[1]) : 4096; 65 | uint32_t pool_size = atoi(argv[2]) ? atoi(argv[2]) : 4800; 66 | uint32_t hash_bits = atoi(argv[3]) ? atoi(argv[3]) : 10; 67 | uint32_t seg_bits = atoi(argv[4]) ? atoi(argv[4]) : 4; 68 | 69 | int total = (argc == 6) ? atoi(argv[5]) : 1; 70 | const int key_len = 16; 71 | 72 | int all = 3; 73 | MakeRaftConfig(all); 74 | Signal::Register(SIGINT, []() { FreeRaftSet(); }); 75 | 76 | vector db; 77 | for (int i = 0; i < all; i++) { 78 | db.push_back(new MushroomDB("mushroom_test", key_len, page_size, pool_size, hash_bits, 79 | seg_bits, log_page)); 80 | MushroomDB *cur = db[i]; 81 | rafts[i]->SetApplyFunc([cur](MushroomLog &log) { 82 | return cur->Put(log.key_); 83 | }); 84 | } 85 | 86 | sleep(1); 87 | 88 | MushroomLog log; 89 | int fd = open(file, O_RDONLY); 90 | assert(fd > 0); 91 | char buf[8192]; 92 | int curr = 0, ptr = 0, count = 0; 93 | bool flag = true; 94 | for (; (ptr = pread(fd, buf, 8192, curr)) > 0 && flag; curr += ptr) { 95 | while (--ptr && buf[ptr] != '\n' && buf[ptr] != '\0') buf[ptr] = '\0'; 96 | if (ptr) buf[ptr++] = '\0'; 97 | else break; 98 | for (int i = 0; i < ptr;) { 99 | int j = 0; 100 | char *tmp = buf + i; 101 | for (; buf[i] != '\n' && buf[i] != '\0'; ++i, ++j) ; 102 | tmp[j] = '\0'; 103 | log.key_->page_no_ = 0; 104 | memcpy(log.key_->key_, tmp, key_len); 105 | uint32_t index; 106 | for (auto e : rafts) 107 | if (e->Start(log, &index)) 108 | break; 109 | if (++count == total) { 110 | flag = false; 111 | break; 112 | } 113 | if (!(count % 100)) 114 | sleep(1); 115 | ++i; 116 | } 117 | } 118 | close(fd); 119 | 120 | sleep(2); 121 | FreeRaftSet(); 122 | flag = true; 123 | for (uint32_t i = 1; i < db.size(); ++i) 124 | if (!(*db[0] == *db[i])) 125 | flag = false; 126 | for (auto e : db) { 127 | e->Close(); 128 | delete e; 129 | } 130 | printf("\033[31mtotal: %d\033[0m\n", total); 131 | if (!flag) 132 | printf("\033[31mfailed :(\033[0m\n"); 133 | else 134 | printf("\033[32msuccess :)\033[0m\n"); 135 | return 0; 136 | } 137 | -------------------------------------------------------------------------------- /test/mushroom_log_vector.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-06-14 15:45:13 6 | **/ 7 | 8 | #include "unit.h" 9 | #include "../src/raft/mushroom_log_vector.hpp" 10 | 11 | using namespace Mushroom; 12 | 13 | TEST(Append) 14 | { 15 | MushroomLogVector vec; 16 | MushroomLog *log = NewMushroomLog(); 17 | log->term_ = 1234; 18 | const char *str = "hello world ! :)"; 19 | log->key_->page_no_ = 4321; 20 | memcpy(log->key_->key_, str, 16); 21 | int total = 10000; 22 | for (int i = 0; i < total; ++i) 23 | vec.Append(*log); 24 | DeleteMushroomLog(log); 25 | ASSERT_TRUE(int(vec.size()) == total); 26 | } 27 | 28 | TEST(Get) 29 | { 30 | MushroomLogVector vec; 31 | MushroomLog *log = NewMushroomLog(); 32 | log->term_ = 1234; 33 | const char *str = "hello world ! :)"; 34 | log->key_->page_no_ = 43210; 35 | memcpy(log->key_->key_, str, 16); 36 | int total = 100; 37 | for (int i = 0; i < total; ++i) 38 | vec.Append(*log); 39 | for (int i = 0; i < total; ++i) { 40 | MushroomLog &l = vec[total - 1]; 41 | ASSERT_TRUE(l.term_ == log->term_); 42 | ASSERT_TRUE(!memcmp(&l, log, KeySlice::KeySize)); 43 | } 44 | DeleteMushroomLog(log); 45 | } 46 | 47 | int main() 48 | { 49 | return RUN_ALL_TESTS('\0'); 50 | } 51 | -------------------------------------------------------------------------------- /test/mushroom_multi_thread.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-03-16 17:04:27 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "../src/blink/slice.hpp" 16 | #include "../src/blink/db.hpp" 17 | 18 | using namespace Mushroom; 19 | 20 | static const int key_len = 16; 21 | 22 | static const char *files[] = { 23 | "../data/25000000_0", 24 | "../data/25000000_1", 25 | "../data/25000000_2", 26 | "../data/25000000_3" 27 | }; 28 | 29 | struct ThreadArg 30 | { 31 | ThreadArg() { } 32 | int i; 33 | int all; 34 | MushroomDB *db; 35 | bool (MushroomDB::*(fun))(KeySlice *); 36 | }; 37 | 38 | void* Do(void *arg) 39 | { 40 | int all = ((ThreadArg *)arg)->all; 41 | MushroomDB *db = ((ThreadArg *)arg)->db; 42 | bool (MushroomDB::*(fun))(KeySlice *); 43 | fun = ((ThreadArg *)arg)->fun; 44 | 45 | TempSlice(key); 46 | int fd = open(files[((ThreadArg *)arg)->i], O_RDONLY); 47 | assert(fd > 0); 48 | char buf[8192]; 49 | int curr = 0, ptr = 0, count = 0; 50 | bool flag = true; 51 | for (; (ptr = pread(fd, buf, 8192, curr)) > 0 && flag; curr += ptr) { 52 | while (--ptr && buf[ptr] != '\n' && buf[ptr] != '\0') buf[ptr] = '\0'; 53 | if (ptr) buf[ptr++] = '\0'; 54 | else break; 55 | for (int i = 0; i < ptr; ++i) { 56 | char *tmp = buf + i; 57 | i += key_len; 58 | assert(buf[i] == '\n' || buf[i] == '\0'); 59 | buf[i] = '\0'; 60 | key->page_no_ = 0; 61 | memcpy(key->key_, tmp, key_len); 62 | (db->*fun)(key); 63 | if (++count == all) { 64 | flag = false; 65 | break; 66 | } 67 | } 68 | } 69 | close(fd); 70 | return 0; 71 | } 72 | 73 | int main(int argc, char **argv) 74 | { 75 | assert(argc > 4); 76 | uint32_t page_size = atoi(argv[1]) ? atoi(argv[1]) : 4096; 77 | uint32_t pool_size = atoi(argv[2]) ? atoi(argv[2]) : 4800; 78 | uint32_t hash_bits = atoi(argv[3]) ? atoi(argv[3]) : 10; 79 | uint32_t seg_bits = atoi(argv[4]) ? atoi(argv[4]) : 4; 80 | 81 | const int total = (argc == 6) ? atoi(argv[5]) : 1; 82 | 83 | MushroomDB *db = new MushroomDB("mushroom_test", 84 | key_len, page_size, pool_size, hash_bits, seg_bits); 85 | 86 | int thread_num = 4; 87 | auto beg = std::chrono::high_resolution_clock::now(); 88 | int all = total == 1 ? 1 : total / thread_num; 89 | pthread_t ids[thread_num]; 90 | ThreadArg *args = new ThreadArg[thread_num]; 91 | // Put 92 | for (int i = 0; i != thread_num; ++i) { 93 | args[i].i = i; 94 | args[i].all = all; 95 | args[i].db = db; 96 | args[i].fun = &MushroomDB::Put; 97 | assert(pthread_create(&ids[i], 0, Do, &args[i]) == 0); 98 | } 99 | for (int i = 0; i != thread_num; ++i) 100 | assert(pthread_join(ids[i], 0) == 0); 101 | auto end = std::chrono::high_resolution_clock::now(); 102 | auto t1 = std::chrono::duration>(end - beg).count(); 103 | 104 | // Get 105 | beg = std::chrono::high_resolution_clock::now(); 106 | for (int i = 0; i != thread_num; ++i) { 107 | args[i].i = i; 108 | args[i].all = all; 109 | args[i].db = db; 110 | args[i].fun = &MushroomDB::Get; 111 | assert(pthread_create(&ids[i], 0, Do, &args[i]) == 0); 112 | } 113 | for (int i = 0; i != thread_num; ++i) 114 | assert(pthread_join(ids[i], 0) == 0); 115 | end = std::chrono::high_resolution_clock::now(); 116 | auto t2 = std::chrono::duration>(end - beg).count(); 117 | 118 | delete [] args; 119 | db->Close(); 120 | delete db; 121 | 122 | printf("\033[31mtotal: %d\033[0m\n\033[32mput time: %f s\033[0m\n", all * thread_num, t1); 123 | printf("\033[34mget time: %f s\033[0m\n", t2); 124 | return 0; 125 | } 126 | -------------------------------------------------------------------------------- /test/mushroom_with_queue.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2016-11-20 12:37:41 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "../src/blink/slice.hpp" 14 | #include "../src/blink/db.hpp" 15 | #include "../src/blink/task.hpp" 16 | #include "../src/blink/bounded_mapping_queue.hpp" 17 | #include "../src/blink/thread_pool_mapping.hpp" 18 | 19 | using namespace Mushroom; 20 | 21 | static const int key_len = 16; 22 | static int total; 23 | 24 | double Do(const char *file, MushroomDB *db, bool (MushroomDB::*(fun))(KeySlice *)) 25 | { 26 | BoundedMappingQueue *queue = new BoundedMappingQueue(1024, []() { 27 | return new MushroomTask(); 28 | }); 29 | 30 | ThreadPoolMapping pool(queue, 4); 31 | 32 | TempSlice(key); 33 | int fd = open(file, O_RDONLY); 34 | assert(fd > 0); 35 | char buf[8192]; 36 | int curr = 0, ptr = 0, count = 0; 37 | bool flag = true; 38 | 39 | auto beg = std::chrono::high_resolution_clock::now(); 40 | for (; (ptr = pread(fd, buf, 8192, curr)) > 0 && flag; curr += ptr) { 41 | while (--ptr && buf[ptr] != '\n' && buf[ptr] != '\0') buf[ptr] = '\0'; 42 | if (ptr) buf[ptr++] = '\0'; 43 | else break; 44 | for (int i = 0; i < ptr; ++i) { 45 | char *tmp = buf + i; 46 | i += key_len; 47 | assert(buf[i] == '\n' || buf[i] == '\0'); 48 | buf[i] = '\0'; 49 | 50 | key->page_no_ = 0; 51 | memcpy(key->key_, tmp, key_len); 52 | 53 | MushroomTask *task = queue->Get(); 54 | task->Assign(fun, db, key); 55 | queue->Push(); 56 | 57 | if (++count == total) { 58 | flag = false; 59 | break; 60 | } 61 | } 62 | } 63 | close(fd); 64 | 65 | pool.Clear(); 66 | delete queue; 67 | 68 | auto end = std::chrono::high_resolution_clock::now(); 69 | auto t = std::chrono::duration>(end - beg).count(); 70 | return t; 71 | } 72 | 73 | int main(int argc, char **argv) 74 | { 75 | const char *file = "../data/10000000"; 76 | 77 | assert(argc > 4); 78 | uint32_t page_size = atoi(argv[1]) ? atoi(argv[1]) : 4096; 79 | uint32_t pool_size = atoi(argv[2]) ? atoi(argv[2]) : 4800; 80 | uint32_t hash_bits = atoi(argv[3]) ? atoi(argv[3]) : 10; 81 | uint32_t seg_bits = atoi(argv[4]) ? atoi(argv[4]) : 4; 82 | 83 | total = (argc == 6) ? atoi(argv[5]) : 1; 84 | 85 | MushroomDB db("mushroom_test", key_len, page_size, pool_size, hash_bits, seg_bits); 86 | 87 | double t1 = Do(file, &db, &MushroomDB::Put); 88 | 89 | double t2 = Do(file, &db, &MushroomDB::Get); 90 | 91 | db.Close(); 92 | 93 | printf("\033[31mtotal: %d\033[0m\n\033[32mput time: %f s\033[0m\n", total, t1); 94 | printf("\033[34mget time: %f s\033[0m\n", t2); 95 | 96 | return 0; 97 | } 98 | -------------------------------------------------------------------------------- /test/network_client.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-28 14:34:58 6 | **/ 7 | 8 | #include 9 | 10 | #include "../src/log/log.hpp" 11 | #include "../src/network/signal.hpp" 12 | #include "../src/network/eventbase.hpp" 13 | #include "../src/network/connection.hpp" 14 | 15 | using namespace Mushroom; 16 | 17 | int main() 18 | { 19 | EventBase base(1, 8); 20 | Connection con(EndPoint(8000, "127.0.0.1"), base.GetPoller()); 21 | Signal::Register(SIGINT, [&base]() { base.Exit(); }); 22 | 23 | ExitIf(!con.Success(), ""); 24 | 25 | con.OnRead([&con]() { 26 | printf("read %u : %s\n", con.GetInput().size(), con.GetInput().data()); 27 | con.GetInput().Clear(); 28 | }); 29 | 30 | base.RunAfter(5000, [&base]() { 31 | base.Exit(); 32 | }); 33 | TimerId id1 = base.RunEvery(500, [&con]() { 34 | con.Send("hello world :)"); 35 | }); 36 | TimerId id2 = base.RunEvery(700, [&con]() { 37 | con.Send("hello world ;)"); 38 | }); 39 | base.RunAfter(1500, [&base, id1]() { 40 | base.Cancel(id1); 41 | }); 42 | base.RunAfter(3500, [&base, id2]() { 43 | base.Cancel(id2); 44 | }); 45 | base.Loop(); 46 | return 0; 47 | } 48 | -------------------------------------------------------------------------------- /test/network_server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-28 14:35:06 6 | **/ 7 | 8 | #include "../src/network/signal.hpp" 9 | #include "../src/network/eventbase.hpp" 10 | #include "../src/network/connection.hpp" 11 | #include "../src/network/server.hpp" 12 | 13 | using namespace Mushroom; 14 | 15 | int main() 16 | { 17 | EventBase base(1, 8); 18 | Signal::Register(SIGINT, [&] { base.Exit(); }); 19 | Server server(&base, 8000); 20 | server.Start(); 21 | server.OnConnect([](Connection *con) { 22 | con->OnRead([con]() { 23 | printf("read %u : %s\n", con->GetInput().size(), con->GetInput().data()); 24 | con->Send(con->GetInput()); 25 | }); 26 | }); 27 | base.Loop(); 28 | return 0; 29 | } 30 | -------------------------------------------------------------------------------- /test/palm.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2018-7-27 17:39:00 6 | **/ 7 | 8 | #include "../src/palm/palm_tree.hpp" 9 | 10 | int main() 11 | { 12 | return 0; 13 | } 14 | -------------------------------------------------------------------------------- /test/raft.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-08 14:22:46 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "unit.h" 14 | 15 | #include "../src/network/signal.hpp" 16 | #include "../src/rpc/rpc_connection.hpp" 17 | #include "../src/raft/log.hpp" 18 | #include "../src/raft/raft_server.hpp" 19 | #include "../src/include/thread.hpp" 20 | #include "../src/network/time.hpp" 21 | 22 | using namespace std; 23 | using namespace Mushroom; 24 | 25 | static EventBase *base = 0; 26 | static Thread *loop = 0; 27 | static vector rafts; 28 | static vector connected; 29 | static uint16_t port_base = 7000; 30 | 31 | namespace RaftTest { 32 | 33 | static void FreeRaftSet() { 34 | for (auto e : rafts) 35 | if (e) e->Close(); 36 | if (base) base->Exit(); 37 | if (loop) loop->Stop(); 38 | for (auto e : rafts) 39 | delete e; 40 | delete base; 41 | delete loop; 42 | base = 0; 43 | loop = 0; 44 | rafts.clear(); 45 | connected.clear(); 46 | } 47 | 48 | static void MakeRaftSet(int total) { 49 | FreeRaftSet(); 50 | base = new EventBase(4, 64); 51 | loop = new Thread([&]() { base->Loop(); }); 52 | rafts.resize(total); 53 | connected.resize(total); 54 | for (int i = 0; i < total; ++i) 55 | rafts[i] = new RaftServer(base, port_base++, i); 56 | for (int i = 0; i < total; ++i) { 57 | for (int j = 0; j < total; ++j) { 58 | if (i == j) continue; 59 | rafts[i]->AddPeer(new RpcConnection(EndPoint(rafts[j]->Port(), "127.0.0.1"), 60 | base->GetPoller(), 0)); 61 | } 62 | connected[i] = true; 63 | } 64 | loop->Start(); 65 | for (auto e : rafts) 66 | e->Start(); 67 | } 68 | 69 | static void WaitForElection(float factor) { 70 | if (factor == 0.f) 71 | return ; 72 | usleep(factor * (2 * RaftServer::ElectionTimeoutBase) * 1000); 73 | } 74 | 75 | static void CheckOneLeaderAfter(float factor, int32_t *number, int32_t *id) { 76 | WaitForElection(factor); 77 | map> map; 78 | *id = -1; 79 | *number = 0; 80 | uint32_t last = 0; 81 | for (uint32_t i = 0; i < rafts.size(); ++i) { 82 | if (!connected[i]) 83 | continue; 84 | uint32_t term; 85 | if (rafts[i]->IsLeader(&term)) 86 | map[term].push_back(rafts[i]->Id()); 87 | last = last < term ? term : last; 88 | } 89 | if (map.size()) { 90 | auto &leaders = map[last]; 91 | *number = leaders.size(); 92 | if (*number == 1) 93 | *id = leaders[0]; 94 | } 95 | } 96 | 97 | static bool CheckNoLeaderAfter(float factor) { 98 | int32_t number; 99 | int32_t id; 100 | CheckOneLeaderAfter(factor, &number, &id); 101 | return number == 0; 102 | } 103 | 104 | static void DisableServer(int32_t id) { 105 | connected[id] = false; 106 | for (auto e : rafts[id]->Peers()) 107 | e->Disable(); 108 | for (auto e : rafts[id]->Connections()) 109 | ((RpcConnection *)e)->Disable(); 110 | } 111 | 112 | static void EnableServer(int32_t id) { 113 | connected[id] = true; 114 | for (auto e : rafts[id]->Peers()) 115 | e->Enable(); 116 | for (auto e : rafts[id]->Connections()) 117 | ((RpcConnection *)e)->Enable(); 118 | } 119 | 120 | /* 121 | static void StartServer(uint32_t idx) 122 | { 123 | rafts[idx]->Start(); 124 | EnableServer(idx); 125 | connected[idx] = true; 126 | } 127 | 128 | static void CrashServer(uint32_t idx) 129 | { 130 | connected[idx] = false; 131 | DisableServer(idx); 132 | rafts[idx]->Reset(); 133 | } 134 | */ 135 | static void DisableServerFor(float factor, int32_t id) { 136 | DisableServer(id); 137 | WaitForElection(factor); 138 | EnableServer(id); 139 | } 140 | 141 | static bool CommittedAt(uint32_t index, uint32_t *commit, int *count) 142 | { 143 | *commit = -1; 144 | *count = 0; 145 | uint32_t pre = ~0; 146 | for (auto e : rafts) { 147 | Log log; 148 | if (e && !e->LogAt(index, log)) continue; 149 | if (*count && pre != *commit) { 150 | printf("not match at %u, %u : %u\n", index, pre, *commit); 151 | return false; 152 | } 153 | pre = *commit; 154 | *count += 1; 155 | } 156 | return true; 157 | } 158 | 159 | static uint32_t One(uint32_t number, int expect) 160 | { 161 | int count; 162 | int64_t now = Time::Now(); 163 | for (; Time::Now() < (now + 5000);) { 164 | uint32_t index = ~0u; 165 | for (uint32_t i = 0; i < rafts.size(); ++i) { 166 | if (!connected[i]) continue; 167 | if (rafts[i] && rafts[i]->Start(Log(number), &index)) 168 | break; 169 | } 170 | if (index == ~0u) { 171 | usleep(200 * 1000); 172 | continue; 173 | } 174 | for (int k = 0; k < 20; ++k) { 175 | usleep(50 * 1000); 176 | uint32_t commit; 177 | if (!CommittedAt(index, &commit, &count)) 178 | continue; 179 | if (count >= expect && commit == number) 180 | return index; 181 | } 182 | } 183 | printf("%u failed to reach agreement, %d : %d\n", number, expect, count); 184 | return ~0u; 185 | } 186 | 187 | static void PrintAllServer(bool print_log = true, bool print_next = true) 188 | { 189 | for (auto e : rafts) 190 | if (e) e->Status(print_log, print_next); 191 | } 192 | 193 | static void PrintRaftServer(int32_t index) 194 | { 195 | rafts[index]->Status(true, false); 196 | } 197 | 198 | static uint32_t RpcCount() 199 | { 200 | uint32_t ret = 0; 201 | for (auto e : rafts) 202 | ret += e->RpcCount(); 203 | return ret; 204 | } 205 | 206 | static uint32_t Wait(uint32_t index, int expect, uint32_t term) 207 | { 208 | uint32_t to = 10; 209 | uint32_t commit; 210 | int count; 211 | for (int i = 0; i < 10; ++i) { 212 | if (!CommittedAt(index, &commit, &count)) 213 | continue; 214 | if (count >= expect) 215 | break; 216 | usleep(to * 1000); 217 | if (to < 1000) to *= 2; 218 | for (auto e : rafts) 219 | if (e->Term() > term) 220 | return ~0; 221 | } 222 | CommittedAt(index, &commit, &count); 223 | if (count < expect) 224 | return ~0; 225 | return commit; 226 | } 227 | 228 | } // namespace RaftTest 229 | 230 | using namespace RaftTest; 231 | 232 | TEST(ElectionWithNoNetworkFaliure) 233 | { 234 | MakeRaftSet(5); 235 | int32_t number; 236 | int32_t id; 237 | CheckOneLeaderAfter(1, &number, &id); 238 | ASSERT_EQ(number, 1); 239 | } 240 | 241 | TEST(ReelectionAfterNetworkFailure) 242 | { 243 | uint32_t total = 3; 244 | MakeRaftSet(total); 245 | int32_t number; 246 | int32_t leader1; 247 | CheckOneLeaderAfter(1, &number, &leader1); 248 | ASSERT_EQ(number, 1); 249 | 250 | DisableServer(leader1); 251 | WaitForElection(1); 252 | int32_t leader2; 253 | CheckOneLeaderAfter(2, &number, &leader2); 254 | ASSERT_EQ(number, 1); 255 | 256 | EnableServer(leader1); 257 | int32_t leader3; 258 | CheckOneLeaderAfter(0.5, &number, &leader3); 259 | ASSERT_EQ(number, 1); 260 | ASSERT_EQ(leader2, leader3); 261 | 262 | DisableServer(leader2); 263 | DisableServer((leader2+1)%total); 264 | ASSERT_TRUE(CheckNoLeaderAfter(1)); 265 | 266 | EnableServer((leader2+1)%total); 267 | int32_t leader4; 268 | CheckOneLeaderAfter(2, &number, &leader4); 269 | ASSERT_EQ(number, 1); 270 | 271 | EnableServer(leader2); 272 | int32_t leader5; 273 | CheckOneLeaderAfter(1, &number, &leader5); 274 | ASSERT_EQ(number, 1); 275 | ASSERT_EQ(leader4, leader5); 276 | } 277 | 278 | TEST(AgreementWithoutNetworkFailure) 279 | { 280 | uint32_t total = 3; 281 | MakeRaftSet(total); 282 | int32_t number; 283 | int32_t id; 284 | CheckOneLeaderAfter(2, &number, &id); 285 | ASSERT_EQ(number, 1); 286 | 287 | for (uint32_t i = 0; i < 10u; ++i) { 288 | uint32_t commit; 289 | int count; 290 | ASSERT_TRUE(CommittedAt(i, &commit, &count)); 291 | ASSERT_EQ(count, 0); 292 | uint32_t index = One(i, total); 293 | ASSERT_EQ(index, i); 294 | } 295 | } 296 | 297 | TEST(AgreementWithFollowerDisconnected) 298 | { 299 | uint32_t total = 3; 300 | MakeRaftSet(total); 301 | int32_t number; 302 | int32_t leader; 303 | CheckOneLeaderAfter(2, &number, &leader); 304 | ASSERT_EQ(number, 1); 305 | 306 | uint32_t lg = 0; 307 | ASSERT_NE(One(lg++, total), ~0u); 308 | ASSERT_NE(One(lg++, total), ~0u); 309 | 310 | DisableServer((leader+1)%total); 311 | ASSERT_NE(One(lg++, total-1), ~0u); 312 | ASSERT_NE(One(lg++, total-1), ~0u); 313 | 314 | DisableServerFor(1, leader); 315 | int32_t leader2; 316 | CheckOneLeaderAfter(2, &number, &leader2); 317 | ASSERT_EQ(number, 1); 318 | 319 | ASSERT_NE(One(lg++, total-1), ~0u); 320 | ASSERT_NE(One(lg++, total-1), ~0u); 321 | 322 | EnableServer((leader+1)%total); 323 | WaitForElection(0.5); 324 | ASSERT_NE(One(lg++, total), ~0u); 325 | 326 | DisableServerFor(1, leader2); 327 | int32_t leader3; 328 | CheckOneLeaderAfter(2, &number, &leader3); 329 | ASSERT_EQ(number, 1); 330 | ASSERT_NE(One(lg++, total), ~0u); 331 | } 332 | 333 | TEST(AgreementWithHalfFollowerDisconnected) 334 | { 335 | uint32_t total = 5; 336 | MakeRaftSet(total); 337 | 338 | uint32_t lg = 0; 339 | uint32_t idx = 0; 340 | ASSERT_EQ(idx++, One(lg++, total)); 341 | 342 | int32_t number; 343 | int32_t leader; 344 | CheckOneLeaderAfter(0, &number, &leader); 345 | ASSERT_EQ(number, 1); 346 | 347 | DisableServer((leader+1)%total); 348 | DisableServer((leader+2)%total); 349 | DisableServer((leader+3)%total); 350 | 351 | uint32_t index; 352 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 353 | ASSERT_EQ(index, 1u); 354 | 355 | WaitForElection(1); 356 | uint32_t commit; 357 | int count; 358 | CommittedAt(index, &commit, &count); 359 | ASSERT_EQ(count, 0); 360 | 361 | EnableServer((leader+1)%total); 362 | EnableServer((leader+2)%total); 363 | EnableServer((leader+3)%total); 364 | 365 | int32_t leader2; 366 | CheckOneLeaderAfter(2, &number, &leader2); 367 | ASSERT_EQ(number, 1); 368 | 369 | uint32_t index2; 370 | ASSERT_TRUE(rafts[leader2]->Start(Log(lg++), &index2)); 371 | ASSERT_TRUE(index2 >= 1 && index2 < 3); 372 | 373 | ASSERT_NE(One(lg++, total), ~0u); 374 | WaitForElection(1); 375 | ASSERT_NE(One(lg++, total), ~0u); 376 | } 377 | 378 | TEST(RejoinOfPartitionedLeader) 379 | { 380 | uint32_t total = 3; 381 | MakeRaftSet(total); 382 | 383 | uint32_t lg = 0; 384 | uint32_t idx = 0; 385 | ASSERT_EQ(idx++, One(lg++, total)); 386 | 387 | int32_t number; 388 | int32_t leader; 389 | CheckOneLeaderAfter(0, &number, &leader); 390 | ASSERT_EQ(number, 1); 391 | 392 | DisableServer(leader); 393 | 394 | uint32_t index; 395 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 396 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 397 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 398 | 399 | WaitForElection(1); 400 | ASSERT_EQ(idx++, One(lg++, total-1)); 401 | 402 | int32_t leader2; 403 | CheckOneLeaderAfter(0, &number, &leader2); 404 | ASSERT_EQ(number, 1); 405 | 406 | DisableServer(leader2); 407 | EnableServer(leader); 408 | WaitForElection(2); 409 | ASSERT_EQ(idx++, One(lg++, total-1)); 410 | 411 | EnableServer(leader2); 412 | ASSERT_EQ(idx++, One(lg++, total)); 413 | } 414 | 415 | TEST(LeaderBackupIncorrectLog) 416 | { 417 | uint32_t total = 5; 418 | MakeRaftSet(total); 419 | 420 | uint32_t lg = 0; 421 | ASSERT_NE(One(lg++, total), ~0u); 422 | 423 | int32_t number; 424 | int32_t leader; 425 | CheckOneLeaderAfter(0, &number, &leader); 426 | ASSERT_EQ(number, 1); 427 | 428 | DisableServer((leader+2)%total); 429 | DisableServer((leader+3)%total); 430 | DisableServer((leader+4)%total); 431 | 432 | int all = 30; 433 | uint32_t index; 434 | for (int i = 0; i < all; ++i) 435 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 436 | 437 | DisableServer((leader+0)%total); 438 | DisableServer((leader+1)%total); 439 | 440 | EnableServer((leader+2)%total); 441 | EnableServer((leader+3)%total); 442 | EnableServer((leader+4)%total); 443 | 444 | for (int i = 0; i < all; ++i) 445 | ASSERT_NE(One(lg++, total-2), ~0u); 446 | 447 | int32_t leader2; 448 | CheckOneLeaderAfter(0, &number, &leader2); 449 | ASSERT_EQ(number, 1); 450 | 451 | int32_t other = (leader + 2) % total; 452 | if (other == leader2) 453 | other = (leader2 + 1) % total; 454 | DisableServer(other); 455 | 456 | for (int i = 0; i < all; ++i) 457 | ASSERT_TRUE(rafts[leader2]->Start(Log(lg++), &index)); 458 | WaitForElection(0.5); 459 | 460 | for (int i = 0; i < int(total); ++i) 461 | DisableServer(i); 462 | EnableServer((leader+0)%total); 463 | EnableServer((leader+1)%total); 464 | EnableServer(other); 465 | 466 | WaitForElection(2); 467 | for (int i = 0; i < all; ++i) 468 | ASSERT_NE(One(lg++, total-2), ~0u); 469 | 470 | for (int i = 0; i < int(total); ++i) 471 | EnableServer(i); 472 | 473 | ASSERT_NE(One(lg++, total), ~0u); 474 | } 475 | 476 | TEST(RpcCount) 477 | { 478 | uint32_t total = 3; 479 | MakeRaftSet(total); 480 | 481 | int32_t number; 482 | int32_t leader; 483 | CheckOneLeaderAfter(0.5, &number, &leader); 484 | ASSERT_EQ(number, 1); 485 | uint32_t count = RpcCount(); 486 | ASSERT_LT(count, 30u); 487 | 488 | uint32_t lg = 0; 489 | uint32_t all1, all2; 490 | loop: 491 | for (int i = 0; i < 3; ++i) { 492 | if (i) WaitForElection(1); 493 | all1 = RpcCount(); 494 | uint32_t index, term; 495 | ASSERT_TRUE(rafts[leader]->Start(Log(lg++), &index)); 496 | term = rafts[leader]->Term(); 497 | int iter = 10; 498 | vector logs; 499 | for (int j = 1; j < iter+2; ++j) { 500 | uint32_t tmp_idx; 501 | logs.push_back(lg); 502 | if (!rafts[leader]->Start(Log(lg++), &tmp_idx)) 503 | goto loop; 504 | uint32_t tmp_trm = rafts[leader]->Term(); 505 | if (tmp_trm != term) 506 | goto loop; 507 | ASSERT_EQ(index+j, tmp_idx); 508 | } 509 | for (int j = 1; j < iter+1; ++j) { 510 | uint32_t number = Wait(index+j, total, term); 511 | if (number == ~0u) 512 | goto loop; 513 | ASSERT_EQ(number, logs[j-1]); 514 | } 515 | all2 = 0; 516 | for (auto e : rafts) { 517 | if (e->Term() != term) 518 | goto loop; 519 | all2 += e->RpcCount(); 520 | } 521 | ASSERT_LE(int(all2-all1), (iter+1+3)*3); 522 | break; 523 | } 524 | 525 | WaitForElection(1); 526 | uint32_t all3 = RpcCount(); 527 | ASSERT_LE(all3 - all2, 90u); 528 | } 529 | 530 | TEST(LeaderFrequentlyChange) 531 | { 532 | uint32_t total = 5; 533 | uint32_t up = total; 534 | MakeRaftSet(total); 535 | std::default_random_engine eng(time(0)); 536 | std::uniform_int_distribution dist(0, 999); 537 | 538 | ASSERT_NE(One(0, total), ~0u); 539 | 540 | uint32_t iter = 30; 541 | for (uint32_t i = 1; i <= iter; ++i) { 542 | int32_t leader = -1; 543 | for (uint32_t j = 0; j < total; ++j) { 544 | if (!connected[j]) continue; 545 | uint32_t index; 546 | if (rafts[j] && rafts[j]->Start(Log(i), &index)) { 547 | leader = int32_t(j); 548 | break; 549 | } 550 | } 551 | if (dist(eng) < 100) 552 | usleep((dist(eng) % RaftServer::ElectionTimeoutBase) * 1000); 553 | else 554 | usleep((dist(eng) % 13) * 1000); 555 | 556 | if (leader != -1) { 557 | DisableServer(leader); 558 | WaitForElection(0.5); 559 | --up; 560 | } 561 | 562 | if (up < ((total + 1) / 2)) { 563 | int idx = dist(eng) % total; 564 | if (!connected[idx]) { 565 | EnableServer(idx); 566 | ++up; 567 | } 568 | } 569 | } 570 | 571 | for (uint32_t i = 0; i < total; ++i) 572 | if (!connected[i]) 573 | EnableServer(i); 574 | 575 | ASSERT_NE(One(iter+1, total), ~0u); 576 | } 577 | 578 | /* 579 | TEST(LeaderCrashFrequently) 580 | { 581 | uint32_t total = 5; 582 | uint32_t up = total; 583 | MakeRaftSet(total); 584 | std::default_random_engine eng(time(0)); 585 | std::uniform_int_distribution dist(0, 999); 586 | 587 | ASSERT_NE(One(0, total), ~0u); 588 | 589 | uint32_t iter = 100; 590 | for (uint32_t i = 1; i <= iter; ++i) { 591 | int32_t leader = -1; 592 | for (uint32_t j = 0; j < total; ++j) { 593 | if (!connected[j]) continue; 594 | uint32_t index; 595 | if (rafts[j] && rafts[j]->Start(i, &index)) { 596 | leader = int32_t(j); 597 | break; 598 | } 599 | } 600 | if (dist(eng) < 100) 601 | usleep((dist(eng) % (RaftServer::ElectionTimeout / 2)) * 1000); 602 | else 603 | usleep((dist(eng) % 13) * 1000); 604 | 605 | if (leader != -1) { 606 | // printf("crash %d\n", leader); 607 | CrashServer(leader); 608 | --up; 609 | } 610 | 611 | if (up < ((total + 1) / 2)) { 612 | int idx = dist(eng) % total; 613 | if (!connected[idx]) { 614 | // printf("start %d\n", idx); 615 | StartServer(idx); 616 | ++up; 617 | } 618 | } 619 | } 620 | 621 | for (uint32_t i = 0; i < total; ++i) 622 | if (!connected[i]) 623 | StartServer(i); 624 | 625 | ASSERT_NE(One(iter+1, total), ~0u); 626 | 627 | PrintAllServer(); 628 | } 629 | */ 630 | 631 | int main(int argc, char **argv) 632 | { 633 | Signal::Register(SIGINT, []() { FreeRaftSet(); }); 634 | int r = RUN_ALL_TESTS(argc == 2 ? argv[1] : '\0'); 635 | FreeRaftSet(); 636 | return r; 637 | } 638 | -------------------------------------------------------------------------------- /test/rpc_call.hpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-01 20:54:10 6 | **/ 7 | 8 | #ifndef _RPC_CALL_HPP_ 9 | #define _RPC_CALL_HPP_ 10 | 11 | #include "../src/rpc/marshaller.hpp" 12 | 13 | namespace Mushroom { 14 | 15 | struct Test 16 | { 17 | Test() { } 18 | 19 | struct Pair { 20 | Pair() { } 21 | Pair(int32_t num1, int32_t num2):num1(num1), num2(num2) { } 22 | int32_t num1; 23 | int32_t num2; 24 | }; 25 | 26 | void Add(const Pair *args, int32_t *reply) { 27 | *reply = args->num1 - args->num2; 28 | } 29 | 30 | void Minus(const Pair *args, int32_t *reply) { 31 | *reply = args->num1 + args->num2; 32 | } 33 | 34 | void Mult(const Pair *args, int32_t *reply) { 35 | *reply = args->num1 * args->num2; 36 | } 37 | }; 38 | 39 | inline Marshaller& operator<<(Marshaller &marshaller, const Test::Pair &v) 40 | { 41 | marshaller << v.num1; 42 | marshaller << v.num2; 43 | return marshaller; 44 | } 45 | 46 | inline Marshaller& operator>>(Marshaller &marshaller, Test::Pair &v) 47 | { 48 | marshaller >> v.num1; 49 | marshaller >> v.num2; 50 | return marshaller; 51 | } 52 | 53 | } // namespace Mushroom 54 | 55 | #endif /* _RPC_CALL_HPP_ */ -------------------------------------------------------------------------------- /test/rpc_client.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-04-29 18:54:45 6 | **/ 7 | 8 | #include 9 | #include 10 | 11 | #include "rpc_call.hpp" 12 | #include "../src/log/log.hpp" 13 | #include "../src/network/signal.hpp" 14 | #include "../src/rpc/rpc_connection.hpp" 15 | #include "../src/network/eventbase.hpp" 16 | #include "../src/include/thread.hpp" 17 | 18 | using namespace Mushroom; 19 | 20 | int main(int argc, char **argv) 21 | { 22 | EventBase base(1, 8); 23 | RpcConnection con(EndPoint(7000, "127.0.0.1"), base.GetPoller(), 0.0); 24 | 25 | ExitIf(!con.Success(), ""); 26 | 27 | Thread loop([&]() { 28 | base.Loop(); 29 | }); 30 | 31 | loop.Start(); 32 | 33 | Test test; 34 | Test::Pair args(2047, 65535); 35 | 36 | int total = (argc == 2) ? atoi(argv[1]) : 1; 37 | std::vector> futures(total); 38 | for (int i = 0; i < total; ++i) 39 | con.Call("Test::Add", &args, &futures[i]); 40 | 41 | Signal::Register(SIGINT, [&base, &futures]() { 42 | base.Exit(); 43 | for (auto &e : futures) 44 | e.Cancel(); 45 | }); 46 | 47 | for (auto &e : futures) 48 | e.Wait(); 49 | 50 | con.Close(); 51 | base.Exit(); 52 | 53 | int32_t reply2; 54 | test.Add(&args, &reply2); 55 | int success = 0, failure = 0, bad = 0; 56 | for (int i = 0; i < total; ++i) { 57 | if (futures[i].ok()) { 58 | int32_t &reply = futures[i].Value(); 59 | if (reply == reply2) 60 | ++success; 61 | else 62 | ++failure; 63 | } else { 64 | ++bad; 65 | } 66 | } 67 | 68 | printf("\033[33mtotal : %d\033[0m\n", total); 69 | printf("\033[32msuccess: %d\033[0m\n", success); 70 | printf("\033[31mfailure: %d\033[0m\n", failure); 71 | printf("\033[34mbad : %d\033[0m\n", bad); 72 | 73 | loop.Stop(); 74 | return 0; 75 | } 76 | -------------------------------------------------------------------------------- /test/rpc_server.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-01 19:52:00 6 | **/ 7 | 8 | #include "rpc_call.hpp" 9 | #include "../src/network/signal.hpp" 10 | #include "../src/rpc/rpc_server.hpp" 11 | #include "../src/network/eventbase.hpp" 12 | 13 | using namespace Mushroom; 14 | 15 | int main() 16 | { 17 | EventBase base(1, 8); 18 | RpcServer server(&base, 7000); 19 | Signal::Register(SIGINT, [&] { base.Exit(); }); 20 | Test test; 21 | server.Register("Test::Add", &test, &Test::Add); 22 | server.Register("Test::Minus", &test, &Test::Minus); 23 | server.Register("Test::Mult", &test, &Test::Mult); 24 | server.Start(); 25 | base.Loop(); 26 | return 0; 27 | } 28 | -------------------------------------------------------------------------------- /test/unit.h: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Github: www.github.com/UncP/Mushroom 4 | * > License: BSD-3 5 | * > Time: 2017-05-19 10:05:56 6 | **/ 7 | 8 | #ifndef _UNIT_TEST_HPP_ 9 | #define _UNIT_TEST_HPP_ 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #define TEST_NAME(test_name) test_name##_TEST 18 | 19 | #define TEST(test_name) \ 20 | class TEST_NAME(test_name) : public TestCase \ 21 | { \ 22 | public: \ 23 | TEST_NAME(test_name)(const char *name):TestCase(name) { } \ 24 | virtual void Run(); \ 25 | private: \ 26 | static TestCase* const test_case_; \ 27 | }; \ 28 | TestCase* const TEST_NAME(test_name)::test_case_ = UnitTest::GetInstance()->RegisterTestCase( \ 29 | new TEST_NAME(test_name)(#test_name)); \ 30 | void TEST_NAME(test_name)::Run() 31 | 32 | #define RUN_ALL_TESTS(str) UnitTest::GetInstance()->Run(str); 33 | 34 | class TestCase 35 | { 36 | public: 37 | TestCase(const char *case_name) : case_name_(case_name) { }; 38 | virtual void Run() = 0; 39 | virtual ~TestCase() { } 40 | 41 | const char *case_name_; 42 | bool test_result_; 43 | }; 44 | 45 | class UnitTest 46 | { 47 | public: 48 | static UnitTest* GetInstance() { 49 | static UnitTest unit_test; 50 | return &unit_test; 51 | } 52 | 53 | TestCase* RegisterTestCase(TestCase *testcase) { 54 | test_cases_.push_back(testcase); 55 | return testcase; 56 | } 57 | 58 | bool Run(const char *str) { 59 | test_result_ = true; 60 | 61 | printf("\033[33m[ Start ] Unit Tests\033[0m\n\n"); 62 | 63 | all_ = 0; 64 | for (auto it = test_cases_.begin(); it != test_cases_.end(); ++it) { 65 | TestCase *test_case = *it; 66 | if (str && !strstr(test_case->case_name_, str)) 67 | continue; 68 | ++all_; 69 | current_test_case_ = test_case; 70 | current_test_case_->test_result_ = true; 71 | 72 | printf("\033[34m[ Run ] \033[0m%s\n", test_case->case_name_); 73 | 74 | test_case->Run(); 75 | 76 | if (test_case->test_result_) 77 | printf("\033[32m[ Pass ] \033[0m%s\n", test_case->case_name_); 78 | else 79 | printf("\033[31m[ Fail ] \033[0m%s\n", test_case->case_name_); 80 | 81 | if (test_case->test_result_) { 82 | ++passed_num_; 83 | } else { 84 | ++failed_num_; 85 | test_result_ = false; 86 | } 87 | } 88 | 89 | printf("\n\033[33m[ ALL ] \033[33;1m%d\033[0m\n", all_); 90 | printf("\033[32m[ PASS ] \033[32;1m%d\033[0m\n", passed_num_); 91 | printf("\033[31m[ FAIL ] \033[31;1m%d\033[0m\n", failed_num_); 92 | return !test_result_; 93 | } 94 | 95 | ~UnitTest() { 96 | for (auto i = test_cases_.begin(); i != test_cases_.end(); ++i) 97 | delete *i; 98 | } 99 | 100 | TestCase *current_test_case_; 101 | bool test_result_; 102 | int all_; 103 | int passed_num_; 104 | int failed_num_; 105 | std::vector test_cases_; 106 | }; 107 | 108 | enum OperatorType 109 | { 110 | OPERATOR_TYPE_EQ, 111 | OPERATOR_TYPE_NE, 112 | OPERATOR_TYPE_GT, 113 | OPERATOR_TYPE_LT, 114 | OPERATOR_TYPE_GE, 115 | OPERATOR_TYPE_LE 116 | }; 117 | 118 | template 119 | bool CheckNumericalData(ElemType left_value, ElemType right_value, 120 | const char *str_left_value, const char *str_right_value, 121 | const char *file_name, const unsigned long line_num, OperatorType operator_type) 122 | { 123 | bool condition = false; 124 | char str_operator[5] = {0}; 125 | 126 | switch (operator_type) { 127 | case OPERATOR_TYPE_EQ: 128 | if (typeid(ElemType) == typeid(double)) 129 | condition = !(std::fabs(left_value - right_value) < 1e-8); 130 | else if (typeid(ElemType) == typeid(float)) 131 | condition = !(std::fabs(left_value - right_value) < 1e-6); 132 | else 133 | condition = !(left_value == right_value); 134 | strcpy(str_operator, " == "); 135 | break; 136 | 137 | case OPERATOR_TYPE_NE: 138 | if (typeid(ElemType) == typeid(double)) 139 | condition = !(std::fabs(left_value - right_value) > 1e-8); 140 | else if (typeid(ElemType) == typeid(float)) 141 | condition = !(std::fabs(left_value - right_value) > 1e-6); 142 | else 143 | condition = !(left_value != right_value); 144 | strcpy(str_operator, " != "); 145 | break; 146 | 147 | case OPERATOR_TYPE_GT: 148 | condition = !(left_value > right_value); 149 | strcpy(str_operator, " > "); 150 | break; 151 | 152 | case OPERATOR_TYPE_LT: 153 | condition = !(left_value < right_value); 154 | strcpy(str_operator, " < "); 155 | break; 156 | 157 | case OPERATOR_TYPE_LE: 158 | condition = !(left_value <= right_value); 159 | strcpy(str_operator, " <= "); 160 | break; 161 | 162 | case OPERATOR_TYPE_GE: 163 | condition = !(left_value >= right_value); 164 | strcpy(str_operator, " >= "); 165 | break; 166 | } 167 | 168 | if (condition) { 169 | UnitTest::GetInstance()->current_test_case_->test_result_ = false; 170 | printf("\033[36;1m%s\033[0m: \033[31;1m%lu\033[0m: ", file_name, line_num); 171 | printf("\033[33;1mExpect: \033[0m%s%s%s\n", str_left_value, str_operator, str_right_value); 172 | } 173 | return !condition; 174 | } 175 | 176 | bool CheckStrData(const char *left_value, const char *right_value, 177 | const char *str_left_value, const char *str_right_value, 178 | const char *file_name, const unsigned long line_num, OperatorType operator_type) 179 | { 180 | bool condition = false; 181 | char str_operator[5] = {0}; 182 | 183 | if (operator_type == OPERATOR_TYPE_EQ) { 184 | condition = !((strcmp(left_value, right_value) == 0)); 185 | strcpy(str_operator, " == "); 186 | } 187 | else if (operator_type == OPERATOR_TYPE_NE) { 188 | condition = !((strcmp(left_value, right_value) != 0)); 189 | strcpy(str_operator, " != "); 190 | } 191 | 192 | if (condition) { 193 | UnitTest::GetInstance()->current_test_case_->test_result_ = false; 194 | printf("\033[36;1m%s\033[0m: \033[31;1m%lu\033[0m: ", file_name, line_num); 195 | printf("\033[33;1mExpect: \033[0m%s%s%s\n", str_left_value, str_operator, str_right_value); 196 | } 197 | 198 | return !condition; 199 | } 200 | 201 | #define CHECK_NUMERICAL_DATA(left_value, right_value, operator_type) \ 202 | CheckNumericalData(left_value, right_value, #left_value, #right_value, __FILE__, __LINE__, operator_type) 203 | #define CHECK_STR_DATA(left_value, right_value, operator_type) \ 204 | CheckStrData(left_value, right_value, #left_value, #right_value, __FILE__, __LINE__, operator_type) 205 | 206 | #define EXPECT_EQ(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_EQ) 207 | #define EXPECT_NE(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_NE) 208 | #define EXPECT_GT(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_GT) 209 | #define EXPECT_LT(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_LT) 210 | #define EXPECT_GE(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_GE) 211 | #define EXPECT_LE(left_value, right_value) CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_LE) 212 | 213 | #define ASSERT_EQ(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_EQ)) return; 214 | #define ASSERT_NE(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_NE)) return; 215 | #define ASSERT_GT(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_GT)) return; 216 | #define ASSERT_LT(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_LT)) return; 217 | #define ASSERT_GE(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_GE)) return; 218 | #define ASSERT_LE(left_value, right_value) if (!CHECK_NUMERICAL_DATA(left_value, right_value, OPERATOR_TYPE_LE)) return; 219 | 220 | #define EXPECT_TRUE(condition) CHECK_NUMERICAL_DATA(static_cast(condition), true, OPERATOR_TYPE_EQ) 221 | #define EXPECT_FALSE(condition) CHECK_NUMERICAL_DATA(static_cast(condition), false, OPERATOR_TYPE_EQ) 222 | #define ASSERT_TRUE(condition) if (!CHECK_NUMERICAL_DATA(static_cast(condition), true, OPERATOR_TYPE_EQ)) return; 223 | #define ASSERT_FALSE(condition) if (!CHECK_NUMERICAL_DATA(static_cast(condition), false, OPERATOR_TYPE_EQ)) return; 224 | #define EXPECT_STREQ(left_value, right_value) CHECK_STR_DATA(left_value, right_value, OPERATOR_TYPE_EQ) 225 | #define EXPECT_STRNE(left_value, right_value) CHECK_STR_DATA(left_value, right_value, OPERATOR_TYPE_NE) 226 | #define ASSERT_STREQ(left_value, right_value) if (!CHECK_STR_DATA(left_value, right_value, OPERATOR_TYPE_EQ)) return; 227 | #define ASSERT_STRNE(left_value, right_value) if (!CHECK_STR_DATA(left_value, right_value, OPERATOR_TYPE_NE)) return; 228 | 229 | #endif /* _UNIT_TEST_HPP_ */ -------------------------------------------------------------------------------- /test_data.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * > Author: UncP 3 | * > Mail: 770778010@qq.com 4 | * > Github: https://www.github.com/UncP/Mushroom 5 | * > Created Time: 2017-03-25 14:30:45 6 | **/ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace Mushroom { 19 | 20 | class MushroomDBTestData 21 | { 22 | public: 23 | MushroomDBTestData(time_t seed):seed_(seed) { } 24 | 25 | void Generate(int total, int file_num, int key_len) { 26 | const std::string choice = 27 | std::string("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+-*/"); 28 | std::default_random_engine generator(seed_); 29 | std::uniform_int_distribution distribution(0, choice.size()-1); 30 | 31 | if ((total % file_num)) { 32 | printf("fail to generate test data :(\n"); 33 | return ; 34 | } 35 | total /= file_num; 36 | std::ostringstream os; 37 | os << total; 38 | if (access("data", F_OK)) 39 | assert(mkdir("data", S_IRUSR | S_IWUSR | S_IXUSR | S_IROTH) >= 0); 40 | std::string base("data/"+os.str()); 41 | for (int i = 0; i != file_num; ++i) { 42 | std::ostringstream o; 43 | o << i; 44 | std::ofstream out(base+"_"+o.str()); 45 | char key[key_len+1]; 46 | key[key_len] = '\n'; 47 | for (int j = 0; j != total; ++j) { 48 | for (int k = 0; k != key_len; ++k) 49 | key[k] = choice[distribution(generator)]; 50 | out.write(key, key_len+1); 51 | } 52 | out.close(); 53 | } 54 | } 55 | 56 | private: 57 | time_t seed_; 58 | }; 59 | 60 | } // namespace Mushroom 61 | 62 | int main() 63 | { 64 | using namespace Mushroom; 65 | 66 | MushroomDBTestData data(time(0)); 67 | // total_key file_number key_size 68 | data.Generate(100000000, 4, 16); 69 | // data.Generate(10000000, 1, 16); 70 | return 0; 71 | } 72 | -------------------------------------------------------------------------------- /version.md: -------------------------------------------------------------------------------- 1 | ### v1.2.2 2 | 1. 保留前缀压缩的同时引入布隆过滤器 3 | 2. 布隆过滤器的大小选择： 4 | * 对于第一个节点，filter初始设为100 5 | * 对于分裂的节点，根据key的数量去确定大小，filter的大小以百为单位 6 | 3. 节点分裂 7 | 每次插入完毕后对节点进行判断 8 | 1. 如果当前key的数量等于filter的数量， 尝试扩大filter的数量，如果空间不够，分裂节点 9 | 2. 如果当前key的数量等于degree，尝试进行前缀压缩，如果无法压缩，分裂节点 10 | --------------------------------------------------------------------------------