├── .clang-format ├── .gitignore ├── CMakeLists.txt ├── README.md ├── build └── .placeholder ├── script ├── setup.sh └── suggest_conf.py └── src └── mica ├── alloc ├── hugetlbfs_shm.cc └── hugetlbfs_shm.h ├── common.h ├── datagram ├── datagram_client.h ├── datagram_client_impl.h ├── datagram_protocol.h ├── datagram_server.h └── datagram_server_impl.h ├── directory ├── directory_client.cc ├── directory_client.h ├── etcd_client.h └── etcdcpp │ ├── etcd.hpp │ ├── etcd_mod.cc │ ├── etcd_mod.h │ └── rapid_reply.hpp ├── network ├── dpdk.h ├── dpdk_impl.h ├── network_addr.h └── packet_io.h ├── pool ├── circular_log.h ├── circular_log_impl.h ├── pool.h ├── segregated_fit.h └── segregated_fit_impl.h ├── processor ├── partitions.h ├── partitions_impl │ ├── info.h │ ├── init.h │ ├── process.h │ └── recent_key_hash.h ├── processor.h ├── request_accessor.h ├── simple_processor.h └── types.h ├── table ├── ltable.h ├── ltable_impl │ ├── bucket.h │ ├── del.h │ ├── get.h │ ├── increment.h │ ├── info.h │ ├── init.h │ ├── item.h │ ├── lock.h │ ├── move_to_head.h │ ├── prefetch.h │ ├── set.h │ ├── specialization.h │ └── test.h ├── sqlite.h ├── table.h └── types.h ├── test ├── cdf.cc ├── microbench.cc ├── microbench.json ├── netbench.cc ├── netbench.json ├── server.cc ├── server.json ├── server_sqlite.cc ├── server_sqlite.json ├── test_atomics.cc ├── test_hash.cc ├── test_load.cc ├── test_load.json ├── test_prefetch.cc ├── test_processor.cc ├── test_processor.json ├── test_rand.cc ├── test_table.cc ├── test_table.json ├── test_tsc_sync.cc └── test_zipf.cc └── util ├── SafeInt ├── SafeInt3.hpp └── SafeInt3_mod.hpp ├── barrier.h ├── cityhash ├── city.cc ├── city.h ├── city_mod.cc ├── citycrc.h └── citycrc_mod.h ├── config.cc ├── config.h ├── hash.h ├── latency.h ├── lcore.cc ├── lcore.h ├── memcpy.h ├── pcg ├── include │ ├── pcg_extras.hpp │ ├── pcg_random.hpp │ └── pcg_uint128.hpp ├── pcg_extras.hpp ├── pcg_random.hpp └── pcg_uint128.hpp ├── philox └── philox_random.h ├── queue.h ├── rand.h ├── rand_pcg.h ├── rand_philox.h ├── rapidjson ├── allocators.h ├── document.h ├── encodedstream.h ├── encodings.h ├── error │ ├── en.h │ └── error.h ├── filereadstream.h ├── filewritestream.h ├── internal │ ├── biginteger.h │ ├── diyfp.h │ ├── dtoa.h │ ├── ieee754.h │ ├── itoa.h │ ├── meta.h │ ├── pow10.h │ ├── stack.h │ ├── strfunc.h │ ├── strtod.h │ └── swap.h ├── memorybuffer.h ├── memorystream.h ├── msinttypes │ ├── inttypes.h │ └── stdint.h ├── pointer.h ├── prettywriter.h ├── rapidjson.h ├── reader.h ├── stringbuffer.h └── writer.h ├── rate_limiter.h ├── roundup.h ├── rte_memcpy ├── rte_memcpy.h └── rte_memcpy_mod.h ├── safe_cast.h ├── siphash └── siphash24.c ├── stopwatch.cc ├── stopwatch.h ├── tsc.h ├── type_traits.h ├── zipf.cc └── zipf.h /.clang-format: -------------------------------------------------------------------------------- 1 | # 2 | # https://google-styleguide.googlecode.com/svn/trunk/cppguide.html 3 | # 4 | # Exceptions: 5 | # * Allow the Interface postfix for a class if the class does not define any methods or member variables. 6 | # * Allow rvalue references to express universal references. 7 | # * Use #pragma once in addition to the include guards. 8 | # * Allow using variable-length arrays if each is guaranteed <= 1 KiB. 9 | # * Do not reflow comments; do it manually. 10 | # 11 | 12 | BasedOnStyle: Google 13 | 14 | Language: Cpp 15 | DerivePointerAlignment: false 16 | PointerAlignment: Left 17 | PointerBindsToType: true 18 | 19 | # Requires clang-format v4.0+ 20 | ReflowComments: false 21 | SortIncludes: false 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | 3 | .tags* 4 | tags 5 | .ycm_extra_conf.py 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | MICA 2 2 | ====== 3 | 4 | A fast in-memory key-value store. 5 | 6 | Requirements 7 | ------------ 8 | 9 | * Linux x86\_64 >= 3.0 10 | * Intel CPU >= Sandy Bridge (Haswell or more recent recommended) 11 | * Hugepage (2 GiB) support 12 | 13 | Dependencies for compilation 14 | ---------------------------- 15 | 16 | * g++ >= 5.3 17 | * cmake >= 2.8 18 | * make >= 3.81 19 | * libnuma-dev >= 2.0 20 | * libcurl-dev >= 7.35 21 | * DPDK >= 16.11 22 | * libsqlite3-dev >= 3.0 (with -DSQLITE=ON) 23 | 24 | Dependencies for execution 25 | -------------------------- 26 | 27 | * bash >= 4.0 28 | * python >= 3.4 29 | * etcd >= 2.2 30 | 31 | Compiling DPDK 32 | -------------- 33 | 34 | * cd dpdk-16.11 35 | * make config T=x86_64-native-linuxapp-gcc 36 | # Optimization: try to increase "CONFIG_RTE_MEMPOOL_CACHE_MAX_SIZE" to 4096 in build/.config (but it can also break mempool initialization) 37 | * make -j 38 | 39 | Compiling MICA 40 | -------------- 41 | 42 | * cd mica2/build 43 | * ln -s ../../dpdk-16.11 ./dpdk 44 | * cmake .. 45 | * make -j 46 | 47 | Setting up the general environment 48 | ---------------------------------- 49 | 50 | * cd mica2/build 51 | * ln -s src/mica/test/*.json . 52 | * ../script/setup.sh 8192 8192 # 2 NUMA nodes, 16 Ki pages (32 GiB) 53 | * killall etcd; ../../etcd-v2.2.1-linux-amd64/etcd & 54 | 55 | Setting up the DPDK environment 56 | ------------------------------- 57 | 58 | * sudo modprobe uio 59 | * sudo insmod dpdk/build/kmod/igb_uio.ko 60 | * dpdk/tools/dpdk_nic_bind.py --status 61 | * sudo dpdk/tools/dpdk_nic_bind.py --force -b igb_uio 0000:02:00.0 0000:02:00.1 0000:04:00.0 0000:04:00.1 0000:83:00.0 0000:83:00.1 0000:84:00.0 0000:84:00.1 62 | * sudo dpdk/tools/dpdk_nic_bind.py --force -b igb_uio 0000:01:00.0 0000:01:00.1 0000:03:00.0 0000:03:00.1 0000:42:00.0 0000:42:00.1 0000:43:00.0 0000:43:00.1 63 | * sudo dpdk/tools/dpdk_nic_bind.py --force -b igb_uio 0000:03:00.0 0000:03:00.1 64 | 65 | Running microbench 66 | ------------------ 67 | 68 | * cd mica2/build 69 | * sudo ./microbench 0.00 # 0.00 = uniform key popularity 70 | 71 | Authors 72 | ------- 73 | 74 | Hyeontaek Lim (hl@cs.cmu.edu) 75 | 76 | License 77 | ------- 78 | 79 | Copyright 2014, 2015, 2016, 2017 Carnegie Mellon University 80 | 81 | Licensed under the Apache License, Version 2.0 (the "License"); 82 | you may not use this file except in compliance with the License. 83 | You may obtain a copy of the License at 84 | 85 | http://www.apache.org/licenses/LICENSE-2.0 86 | 87 | Unless required by applicable law or agreed to in writing, software 88 | distributed under the License is distributed on an "AS IS" BASIS, 89 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 90 | See the License for the specific language governing permissions and 91 | limitations under the License. 92 | 93 | -------------------------------------------------------------------------------- /build/.placeholder: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/efficient/mica2/d54007c1c1f4e6bafb05b9aab700a2f697ea6b73/build/.placeholder -------------------------------------------------------------------------------- /script/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Modified from Intel DPDK's tools/setup.sh 4 | 5 | # BSD LICENSE 6 | # 7 | # Copyright(c) 2010-2013 Intel Corporation. All rights reserved. 8 | # All rights reserved. 9 | # 10 | # Redistribution and use in source and binary forms, with or without 11 | # modification, are permitted provided that the following conditions 12 | # are met: 13 | # 14 | # * Redistributions of source code must retain the above copyright 15 | # notice, this list of conditions and the following disclaimer. 16 | # * Redistributions in binary form must reproduce the above copyright 17 | # notice, this list of conditions and the following disclaimer in 18 | # the documentation and/or other materials provided with the 19 | # distribution. 20 | # * Neither the name of Intel Corporation nor the names of its 21 | # contributors may be used to endorse or promote products derived 22 | # from this software without specific prior written permission. 23 | # 24 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 25 | # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 26 | # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 27 | # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 28 | # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 29 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 30 | # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 31 | # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 32 | # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 33 | # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 34 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 35 | 36 | mnthuge=/mnt/huge 37 | 38 | disable_oom_kills() 39 | { 40 | echo "Disabling OOM kills" 41 | sudo sysctl -q -w vm.overcommit_memory=1 42 | 43 | # This is OK for [8192, 8192] page configuration. 44 | sudo sysctl -q -w kernel.shmmax=34359738368 45 | sudo sysctl -q -w kernel.shmall=34359738368 46 | } 47 | 48 | drop_shm() 49 | { 50 | echo "Dropping SHM entries" 51 | 52 | for i in $(ipcs -m | awk '{ print $1; }'); do 53 | if [[ $i =~ 0x.* ]]; then 54 | sudo ipcrm -M $i 2>/dev/null 55 | fi 56 | done 57 | } 58 | 59 | drop_cache() 60 | { 61 | echo "Dropping the page cache" 62 | 63 | echo "echo 3 > /proc/sys/vm/drop_caches" > .echo_tmp 64 | sudo sh .echo_tmp 65 | rm -f .echo_tmp 66 | } 67 | 68 | setup_mmap_limits() 69 | { 70 | sudo sysctl -q -w vm.max_map_count=2147483647 71 | } 72 | 73 | remove_mnt_huge() 74 | { 75 | #echo "Unmounting $mnthuge and removing directory" 76 | 77 | #grep -s $mnthuge /proc/mounts > /dev/null 78 | #if [ $? -eq 0 ] ; then 79 | # sudo umount $mnthuge 80 | #fi 81 | 82 | #if [ -d $mnthuge ] ; then 83 | # sudo rm -R $mnthuge 84 | #fi 85 | 86 | echo "Unmounting hugetlbfs" 87 | for target in `mount -t hugetlbfs | awk '{ print $3 }'`; do 88 | sudo umount $target 89 | if [ -d $target ] ; then 90 | sudo rm -R $target 91 | fi 92 | done 93 | } 94 | 95 | clear_huge_pages() 96 | { 97 | echo "Removing currently reserved hugepages" 98 | 99 | echo > .echo_tmp 100 | for d in /sys/devices/system/node/node? ; do 101 | echo "echo 0 > $d/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 102 | done 103 | sudo sh .echo_tmp 104 | rm -f .echo_tmp 105 | 106 | remove_mnt_huge 107 | } 108 | 109 | create_mnt_huge() 110 | { 111 | echo "Creating $mnthuge and mounting as hugetlbfs" 112 | 113 | sudo mkdir -p $mnthuge 114 | 115 | grep -s $mnthuge /proc/mounts > /dev/null 116 | if [ $? -ne 0 ] ; then 117 | sudo mount -t hugetlbfs nodev $mnthuge 118 | fi 119 | } 120 | 121 | set_numa_pages() 122 | { 123 | clear_huge_pages 124 | 125 | echo "Reserving hugepages" 126 | 127 | for d in /sys/devices/system/node/node? ; do 128 | echo > .echo_tmp 129 | node=$(basename $d) 130 | Pages=$1 131 | echo -n "Number of pages for $node: $Pages requested, " 132 | shift 133 | echo "echo $Pages > $d/hugepages/hugepages-2048kB/nr_hugepages" >> .echo_tmp 134 | sudo sh .echo_tmp 135 | echo "$(cat "$d/hugepages/hugepages-2048kB/nr_hugepages") actual" 136 | done 137 | rm -f .echo_tmp 138 | 139 | create_mnt_huge 140 | } 141 | 142 | 143 | setup_mmap_limits 144 | disable_oom_kills 145 | drop_shm 146 | drop_cache 147 | 148 | set_numa_pages $* 149 | 150 | echo Done! 151 | -------------------------------------------------------------------------------- /script/suggest_conf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | 5 | seen_physical_core_ids = set() 6 | lcore_ids = [] 7 | 8 | for line in open('/proc/cpuinfo').readlines(): 9 | if re.match(r'^processor\s*:.*', line) is not None: 10 | processor_id = int(line.partition(':')[2]) 11 | elif re.match(r'^physical id\s*:.*', line) is not None: 12 | physical_id = int(line.partition(':')[2]) 13 | elif re.match(r'^core id\s*:.*', line) is not None: 14 | core_id = int(line.partition(':')[2]) 15 | 16 | if not line.strip(): 17 | if (physical_id, core_id) not in seen_physical_core_ids: 18 | seen_physical_core_ids.add((physical_id, core_id)) 19 | lcore_ids.append(processor_id) 20 | 21 | print('[partitions]') 22 | print('lcores = %s' % str(lcore_ids)) 23 | print('partition_count = %d' % len(lcore_ids)) 24 | -------------------------------------------------------------------------------- /src/mica/alloc/hugetlbfs_shm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_ALLOC_HUGETLB_SHM_H_ 3 | #define MICA_ALLOC_HUGETLB_SHM_H_ 4 | 5 | #include 6 | #include 7 | #include "mica/common.h" 8 | #include "mica/util/config.h" 9 | #include "mica/util/roundup.h" 10 | 11 | // Configuration file entries for HugeTLBFS_SHM: 12 | // 13 | // * hugetlbfs_path (string): The HugeTLBFS directory. 14 | // Default = "/mnt/huge" 15 | // * filename_prefix (string): The prefix of the filenames on HugeTLBFS. 16 | // Default = "mica_shm_" 17 | // * num_pages_to_init (integer): The maximum number of pages to initialize 18 | // across all NUMA domains. This is recommended to be set high because it 19 | // may not possible to find a sufficient number of free pages on each NUMA 20 | // domain if this number is too small. Default = 1048576 (virtually all 21 | // available pages) 22 | // * num_pages_to_free (array of integers): The number of pages to free for 23 | // other applications in each NUMA domain (e.g., Intel DPDK). Default = [0, 24 | // ..., 0] 25 | // * num_pages_to_reserve (array of integers): The number of pages to reserve 26 | // for use by HugeTLBFS_SHM. This actually does nothing in the 27 | // initialization process, but it shows a warning if HugeTLBFS_SHM is not 28 | // given that number of pages after initialization. Default = [0, ..., 0] 29 | // * clean_files_on_init (bool): If true, delete all files whose filename 30 | // starts with filename_prefix. Creating the files again takes some time, so 31 | // enable this only when the old memory state must be discarded. Default = 32 | // false 33 | // * clean_other_files_on_init (bool): Similar to clean_files_on_init, but 34 | // delete all files whose filename does not starts with filename_prefix. 35 | // This is required to make num_pages_to_free work. Default = true 36 | // * verbose (bool): Print verbose messages. Default = false 37 | 38 | namespace mica { 39 | namespace alloc { 40 | class HugeTLBFS_SHM { 41 | public: 42 | HugeTLBFS_SHM(const ::mica::util::Config& config); 43 | ~HugeTLBFS_SHM(); 44 | 45 | static constexpr size_t kInvalidId = std::numeric_limits::max(); 46 | 47 | static size_t roundup(size_t size) { 48 | return ::mica::util::roundup<2 * 1048576>(size); 49 | } 50 | 51 | void* find_free_address(size_t size); 52 | 53 | size_t alloc(size_t length, size_t numa_node); 54 | bool schedule_release(size_t entry_id); 55 | 56 | bool map(size_t entry_id, void* ptr, size_t offset, size_t length); 57 | bool unmap(void* ptr); 58 | 59 | size_t get_memuse() const { return used_memory_; } 60 | void dump_page_info(); 61 | 62 | void* malloc_contiguous(size_t size, size_t numa_node); 63 | void* malloc_contiguous_local(size_t size); 64 | void free_contiguous(void* ptr); 65 | 66 | void* malloc_striped(size_t size); 67 | void free_striped(void* ptr); 68 | 69 | private: 70 | void initialize(); 71 | 72 | void clean_files(); 73 | void clean_other_files(); 74 | void make_path(size_t page_id, char* out_path); 75 | 76 | void lock(); 77 | void unlock(); 78 | 79 | void check_release(size_t entry_id); 80 | 81 | void* malloc_contiguous_any(size_t size); 82 | 83 | static constexpr size_t kPageSize = 2 * 1048576; 84 | 85 | struct Page { 86 | size_t file_id; // ID used for path generation. 87 | void* addr; 88 | void* paddr; 89 | size_t numa_node; 90 | bool in_use; 91 | }; 92 | 93 | struct Entry { 94 | size_t refcount; // reference by mapping 95 | bool to_remove; // remove entry when refcount == 0 96 | size_t length; 97 | size_t num_pages; 98 | std::vector page_ids; 99 | }; 100 | 101 | struct Mapping { 102 | size_t entry_id; 103 | void* addr; 104 | size_t length; 105 | size_t page_offset; 106 | size_t num_pages; 107 | }; 108 | 109 | ::mica::util::Config config_; 110 | 111 | std::string hugetlbfs_path_; 112 | std::string filename_prefix_; 113 | size_t num_numa_nodes_; 114 | size_t num_pages_to_init_; 115 | std::vector num_pages_to_free_; 116 | std::vector num_pages_to_reserve_; 117 | bool clean_files_on_init_; 118 | bool clean_other_files_on_init_; 119 | bool verbose_; 120 | 121 | uint64_t state_lock_; 122 | std::vector pages_; 123 | std::vector entries_; 124 | std::vector mappings_; 125 | size_t used_memory_; 126 | }; 127 | } 128 | } 129 | 130 | #endif 131 | -------------------------------------------------------------------------------- /src/mica/common.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_COMMON_H_ 3 | #define MICA_COMMON_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #endif 10 | -------------------------------------------------------------------------------- /src/mica/directory/directory_client.cc: -------------------------------------------------------------------------------- 1 | // #pragma once 2 | #ifndef MICA_DIRECTORY_DIRECTORY_CLIENT_CC_ 3 | #define MICA_DIRECTORY_DIRECTORY_CLIENT_CC_ 4 | 5 | #include 6 | #include 7 | #include "mica/util/safe_cast.h" 8 | #include "mica/directory/directory_client.h" 9 | 10 | namespace mica { 11 | namespace directory { 12 | DirectoryClient::DirectoryClient(const ::mica::util::Config& config) 13 | : config_(config), registered_(false) { 14 | etcd_addr_ = config_.get("etcd_addr").get_str("localhost"); 15 | etcd_port_ = ::mica::util::safe_cast( 16 | config_.get("etcd_port").get_uint64(2379)); 17 | etcd_prefix_ = config_.get("etcd_prefix").get_str("/mica"); 18 | 19 | { 20 | auto c = config_.get("hostname"); 21 | if (c.exists()) 22 | hostname_ = c.get_str(); 23 | else { 24 | char buf[128]; 25 | int ret = gethostname(buf, sizeof(buf)); 26 | if (ret < 0) { 27 | fprintf(stderr, "error: gethostname() failed\n"); 28 | return; 29 | } 30 | hostname_ = buf; 31 | } 32 | } 33 | 34 | if (config_.get("append_pid").get_bool(true)) { 35 | char pid[16]; 36 | snprintf(pid, sizeof(pid), ".%u", getpid()); 37 | hostname_ += pid; 38 | } 39 | 40 | ttl_ = ::mica::util::safe_cast(config_.get("ttl").get_uint64(2)); 41 | 42 | verbose_ = config_.get("verbose").get_bool(false); 43 | 44 | try { 45 | etcd_client_ = std::make_unique(etcd_addr_, etcd_port_); 46 | } catch (...) { 47 | fprintf( 48 | stderr, 49 | "error: failed to initialize etcd client to the server at %s:%" PRIu16 50 | "\n", 51 | etcd_addr_.c_str(), etcd_port_); 52 | } 53 | 54 | if (verbose_) printf("hostname: %s\n", hostname_.c_str()); 55 | } 56 | 57 | DirectoryClient::~DirectoryClient() { 58 | if (registered_) unregister_server(); 59 | } 60 | 61 | std::string DirectoryClient::get_hostname() const { return hostname_; } 62 | 63 | void DirectoryClient::register_server(std::string info) { 64 | // Allow registering the server multiple times (with new info). 65 | info_ = info; 66 | 67 | try { 68 | auto reply = etcd_client_->Set( 69 | (etcd_prefix_ + "/servers/" + hostname_).c_str(), info_.c_str(), ttl_); 70 | 71 | if (verbose_) 72 | printf("DirectoryClient::register_server(): %s\n", 73 | reply.body().dump().c_str()); 74 | } catch (...) { 75 | fprintf(stderr, "error: failed to register the server node\n"); 76 | } 77 | 78 | registered_ = true; 79 | } 80 | 81 | void DirectoryClient::refresh_server() const { 82 | // TODO: Avoid sending the same information. 83 | try { 84 | auto reply = etcd_client_->Set( 85 | (etcd_prefix_ + "/servers/" + hostname_).c_str(), info_.c_str(), ttl_); 86 | 87 | if (verbose_) 88 | printf("DirectoryClient::refresh_server(): %s\n", 89 | reply.body().dump().c_str()); 90 | } catch (...) { 91 | fprintf(stderr, "error: failed to refresh the server node\n"); 92 | } 93 | } 94 | 95 | std::vector DirectoryClient::get_server_list() const { 96 | try { 97 | std::string prefix = etcd_prefix_ + "/servers/"; 98 | auto reply = etcd_client_->Get((prefix).c_str()); 99 | 100 | if (verbose_) 101 | printf("DirectoryClient::get_server_list(): %s\n", 102 | reply.body().dump().c_str()); 103 | 104 | std::vector v; 105 | auto c = reply.body().get("node").get("nodes"); 106 | if (!c.is_array()) { 107 | fprintf(stderr, "DirectoryClient::get_server_list(): found no servers\n"); 108 | return v; 109 | } 110 | for (size_t i = 0; i < c.size(); i++) { 111 | v.push_back(c.get(i).get("key").get_str().substr(prefix.size())); 112 | if (verbose_) 113 | printf("DirectoryClient::get_server_list(): found server name: %s\n", 114 | v.back().c_str()); 115 | } 116 | return v; 117 | } catch (...) { 118 | fprintf(stderr, "error: failed to get the server node list\n"); 119 | return std::vector(); 120 | } 121 | } 122 | 123 | std::string DirectoryClient::get_server_info(std::string name) const { 124 | try { 125 | auto reply = etcd_client_->Get((etcd_prefix_ + "/servers/" + name).c_str()); 126 | 127 | if (verbose_) 128 | printf("DirectoryClient::get_server_info(): %s\n", 129 | reply.body().dump().c_str()); 130 | 131 | return reply.body().get("node").get("value").get_str(""); 132 | } catch (...) { 133 | fprintf(stderr, "error: failed to get the server node information: %s\n", 134 | name.c_str()); 135 | return std::string(); 136 | } 137 | } 138 | 139 | void DirectoryClient::unregister_server() { 140 | assert(registered_); 141 | 142 | try { 143 | auto reply = etcd_client_->Delete((etcd_prefix_ + "/servers/").c_str()); 144 | 145 | if (verbose_) 146 | printf("DirectoryClient::unregister_server(): %s\n", 147 | reply.body().dump().c_str()); 148 | } catch (...) { 149 | fprintf(stderr, "error: failed to unregister the server node\n"); 150 | } 151 | 152 | registered_ = false; 153 | } 154 | } 155 | } 156 | 157 | #endif -------------------------------------------------------------------------------- /src/mica/directory/directory_client.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_DIRECTORY_DIRECTORY_CLIENT_H_ 3 | #define MICA_DIRECTORY_DIRECTORY_CLIENT_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "mica/common.h" 9 | #include "mica/util/config.h" 10 | #include "mica/directory/etcd_client.h" 11 | 12 | // Configuration file entries for DirectoryClient: 13 | // 14 | // * etcd_addr (string): The IP address or hostname of etcd. Default = 15 | // "localhost" 16 | // * etcd_port (integer): The port number of etcd. Default = 2379 17 | // * etcd_prefix (string): The key prefix in etcd. Default = "/mica" 18 | // * hostname (string): The unique hostname of the local node. Must not 19 | // contain any slash or dot. Default = (detected by gethostname()) 20 | // * append_pid (bool): Append "." + getpid() to the hostname. Default = true 21 | // * ttl (integer): TTL of the server information in seconds. Default = 2 22 | // * verbose (bool): Print verbose messages. Default = false 23 | 24 | namespace mica { 25 | namespace directory { 26 | class DirectoryClient { 27 | public: 28 | DirectoryClient(const ::mica::util::Config& config); 29 | ~DirectoryClient(); 30 | 31 | std::string get_hostname() const; 32 | 33 | void register_server(std::string info); 34 | void unregister_server(); 35 | 36 | void refresh_server() const; 37 | 38 | std::vector get_server_list() const; 39 | std::string get_server_info(std::string name) const; 40 | 41 | private: 42 | ::mica::util::Config config_; 43 | 44 | std::string etcd_addr_; 45 | uint16_t etcd_port_; 46 | std::string etcd_prefix_; 47 | std::string hostname_; 48 | uint32_t ttl_; 49 | bool verbose_; 50 | 51 | std::unique_ptr etcd_client_; 52 | 53 | bool registered_; 54 | std::string info_; 55 | }; 56 | } 57 | } 58 | 59 | #endif -------------------------------------------------------------------------------- /src/mica/directory/etcd_client.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_DIRECTORY_ETCD_CLIENT_H_ 3 | #define MICA_DIRECTORY_ETCD_CLIENT_H_ 4 | 5 | #include "mica/util/config.h" 6 | #pragma GCC diagnostic push 7 | #pragma GCC diagnostic ignored "-Wconversion" 8 | #pragma GCC diagnostic ignored "-Wsign-conversion" 9 | #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" 10 | #include "mica/directory/etcdcpp/etcd_mod.h" 11 | #pragma GCC diagnostic pop 12 | 13 | namespace mica { 14 | namespace directory { 15 | class EtcdReply { 16 | public: 17 | EtcdReply(const std::string& reply) 18 | : header_(), body_(::mica::util::Config::load(reply, "")) {} 19 | 20 | EtcdReply(const std::string& header, const std::string& reply) 21 | : header_(header), 22 | body_(::mica::util::Config::load(reply, "")) {} 23 | 24 | ~EtcdReply() {} 25 | 26 | bool ok() const { return !body_.get("errorCode").exists(); } 27 | 28 | const std::string& header() const { return header_; } 29 | const ::mica::util::Config& body() const { return body_; } 30 | 31 | private: 32 | std::string header_; 33 | const ::mica::util::Config body_; 34 | }; 35 | 36 | typedef ::etcd::Client EtcdClient; 37 | } 38 | } 39 | 40 | #endif 41 | -------------------------------------------------------------------------------- /src/mica/directory/etcdcpp/rapid_reply.hpp: -------------------------------------------------------------------------------- 1 | /* 2 | 3 | The MIT License (MIT) 4 | 5 | Copyright (C) 2015 Suryanathan Padmanabhan 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in 15 | all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 23 | THE SOFTWARE. 24 | 25 | */ 26 | 27 | #ifndef __RAPID_REPLY_HPP_INCLUDED__ 28 | #define __RAPID_REPLY_HPP_INCLUDED__ 29 | 30 | #include 31 | #include "etcd.hpp" 32 | 33 | // JSON PARSER INCLUDES 34 | #include "rapidjson/document.h" 35 | #include "rapidjson/writer.h" 36 | #include "rapidjson/stringbuffer.h" 37 | 38 | namespace example { 39 | ///////////////////////////////////////////////////////////////////////////// 40 | 41 | /** 42 | * @brief An example json reply wrapper. This can be easily replaced with a user 43 | * defined wrapper as song as it implements a similar interface 44 | */ 45 | class RapidReply { 46 | public: 47 | // Types 48 | typedef std::map KvPairs; 49 | 50 | // LIFECYCLE 51 | RapidReply(const std::string& reply) 52 | :document_(), 53 | header_() { 54 | _Parse(reply); 55 | } 56 | 57 | RapidReply( 58 | const std::string& header, 59 | const std::string& reply) 60 | :document_(), 61 | header_(header) { 62 | _Parse(reply); 63 | } 64 | 65 | // OPERATIONS 66 | void Print() const { 67 | rapidjson::StringBuffer strbuf; 68 | rapidjson::Writer writer(strbuf); 69 | document_.Accept(writer); 70 | std::cerr << strbuf.GetString() << '\n'; 71 | } 72 | 73 | void GetAll(KvPairs& kvPairs) { 74 | if (! document_.HasMember(kNode)) { 75 | } 76 | //ToDo check whether prefix should be "/" 77 | return _GetAll(document_[kNode], kvPairs); 78 | } 79 | 80 | etcd::Action GetAction() { 81 | if (! document_.HasMember(kAction)) { 82 | return etcd::Action::UNKNOWN; 83 | } 84 | CAM_II iter = kActionMap.find (document_[kAction].GetString()); 85 | if (iter == kActionMap.end()) { 86 | return etcd::Action::UNKNOWN; 87 | } 88 | return iter->second; 89 | } 90 | 91 | etcd::Index GetModifiedIndex() const { 92 | if ((! document_.HasMember(kNode)) || 93 | (!document_[kNode].HasMember(kModifiedIndex))) { 94 | throw std::runtime_error("possibly timed out"); 95 | } 96 | return document_[kNode][kModifiedIndex].GetUint64(); 97 | } 98 | 99 | private: 100 | // TYPES 101 | typedef etcd::ResponseActionMap::const_iterator CAM_II; 102 | 103 | // CONSTANTS 104 | const char *kErrorCode ="errorCode"; 105 | const char *kMessage = "message"; 106 | const char *kCause ="cause"; 107 | const char *kNode = "node"; 108 | const char *kModifiedIndex = "modifiedIndex"; 109 | const char *kNodes = "nodes"; 110 | const char *kDir = "dir"; 111 | const char *kKey = "key"; 112 | const char *kValue = "value"; 113 | const char* kAction = "action"; 114 | const etcd::ResponseActionMap kActionMap; 115 | 116 | // DATA MEMBERS 117 | rapidjson::Document document_; 118 | std::string header_; 119 | 120 | // OPERATIONS 121 | void _CheckError() { 122 | if (document_.HasMember(kErrorCode)) { 123 | throw etcd::ReplyException(document_[kErrorCode].GetInt(), 124 | document_[kMessage].GetString(), 125 | document_[kCause].GetString()); 126 | } 127 | } 128 | 129 | void _Parse(const std::string& json) { 130 | document_.Parse(json.c_str()); 131 | _CheckError(); 132 | } 133 | 134 | void _GetAll(const rapidjson::Value& doc, KvPairs& kvPairs) { 135 | if (doc.HasMember(kDir) && (doc[kDir].GetBool() == true)) { 136 | if (!doc.HasMember(kNodes)) 137 | return; // directory doesn't have nodes 138 | const rapidjson::Value& nodes = doc[kNodes]; 139 | assert(nodes.IsArray()); 140 | for (size_t i = 0; i < nodes.Size(); ++i) 141 | _GetAll(nodes[i], kvPairs); 142 | } else { 143 | if (doc.HasMember(kKey)) { 144 | if (doc.HasMember(kValue)) { 145 | kvPairs.emplace(doc[kKey].GetString(), doc[kValue].GetString()); 146 | } else { 147 | kvPairs.emplace(doc[kKey].GetString(), ""); 148 | } 149 | } 150 | } 151 | } 152 | }; 153 | 154 | } // namespace example 155 | 156 | #endif // __RAPID_REPLY_HPP_INCLUDED__ 157 | -------------------------------------------------------------------------------- /src/mica/network/dpdk.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_NETWORK_DPDK_H_ 3 | #define MICA_NETWORK_DPDK_H_ 4 | 5 | #include "mica/common.h" 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include "mica/util/config.h" 18 | #include "mica/util/lcore.h" 19 | #include "mica/network/packet_io.h" 20 | #include "mica/network/network_addr.h" 21 | 22 | // Configuration file entries for DPDK: 23 | // 24 | // * lcores (array): A list of lcore IDs to allow using with DPDK EAL. 25 | // * ports (array): A list of the port information dict: 26 | // * port_id (integer): The port ID. 27 | // * max_queue_count (integer): The maximum number of queues to use. 28 | // * ipv4_addr (string): The IP address to use. 29 | // * mac_addr (string): The MAC address to use. Default = (the first 30 | // detected MAC address) 31 | // * endpoints (array): A list of [lcore_id, port_id] pairs. Default = (At most 32 | // 1 lcore per 5 Gb/s). 33 | 34 | namespace mica { 35 | namespace network { 36 | struct BasicDPDKConfig { 37 | // The maximum number of NUMA domains to support. 38 | static constexpr uint16_t kMaxNUMACount = 8; 39 | 40 | // The maximum number of endpoints to support. 41 | static constexpr uint16_t kMaxEndpointCount = 256; 42 | 43 | // The number of packets to send or receive at once. 44 | // static constexpr uint16_t kMaxBurstSize = 32; 45 | 46 | // The number of RX/TX descriptors in each queue. 47 | static constexpr uint16_t kRXDescCount = 128; 48 | static constexpr uint16_t kTXDescCount = 512; 49 | 50 | // The number of spare packet buffer count per queue. 51 | static constexpr uint16_t kSpareMBufCount = 52 | 4096 - kRXDescCount - kTXDescCount; 53 | 54 | // The minimum required link speed (Gbps). 55 | static constexpr uint32_t kMinLinkSpeed = 10; 56 | 57 | // Be verbose. 58 | static constexpr bool kVerbose = false; 59 | }; 60 | 61 | template 62 | class DPDK : public PacketIOInterface { 63 | public: 64 | struct PacketBuffer : public rte_mbuf { 65 | public: 66 | uint16_t get_length() const { return rte_pktmbuf_data_len(this); } 67 | uint16_t get_headroom() const { return rte_pktmbuf_headroom(this); } 68 | uint16_t get_tailroom() const { return rte_pktmbuf_tailroom(this); } 69 | 70 | char* get_data() { return rte_pktmbuf_mtod(this, char*); } 71 | const char* get_data() const { return rte_pktmbuf_mtod(this, const char*); } 72 | 73 | void set_length(uint16_t len) { 74 | // Assume a single segment packet (implied by ETH_TXQ_FLAGS_NOMULTSEGS). 75 | assert(rte_pktmbuf_is_contiguous(this)); 76 | rte_pktmbuf_pkt_len(this) = rte_pktmbuf_data_len(this) = len; 77 | assert(rte_pktmbuf_is_contiguous(this)); 78 | } 79 | char* prepend(uint16_t len) { return rte_pktmbuf_prepend(this, len); } 80 | char* append(uint16_t len) { return rte_pktmbuf_append(this, len); } 81 | char* adj(uint16_t len) { return rte_pktmbuf_adj(this, len); } 82 | char* trim(uint16_t len) { return rte_pktmbuf_trim(this, len); } 83 | 84 | PacketBuffer(const PacketBuffer& o) = delete; 85 | PacketBuffer& operator=(const PacketBuffer& o) = delete; 86 | }; 87 | 88 | typedef uint32_t EndpointId; 89 | static constexpr EndpointId kInvalidEndpointId = 90 | std::numeric_limits::max(); 91 | 92 | static constexpr uint16_t kMaxEndpointCount = StaticConfig::kMaxEndpointCount; 93 | 94 | struct EndpointInfo { 95 | uint16_t owner_lcore_id; 96 | 97 | volatile uint64_t rx_bursts; 98 | volatile uint64_t rx_packets; 99 | 100 | volatile uint64_t tx_bursts; 101 | volatile uint64_t tx_packets; 102 | volatile uint64_t tx_dropped; 103 | 104 | // Specific to DPDK. 105 | // Values copied from Port. 106 | ether_addr mac_addr; 107 | uint32_t ipv4_addr; 108 | uint16_t numa_id; 109 | 110 | // UDP port for flow direction. 111 | uint16_t udp_port; 112 | 113 | private: 114 | friend DPDK; 115 | 116 | uint16_t port_id; 117 | uint16_t queue_id; 118 | } __attribute__((aligned(128))); 119 | 120 | // static constexpr uint16_t kMaxBurstSize = StaticConfig::kMaxBurstSize; 121 | 122 | DPDK(const ::mica::util::Config& config); 123 | ~DPDK(); 124 | 125 | std::vector get_endpoints() const; 126 | const EndpointInfo& get_endpoint_info(EndpointId eid) const; 127 | 128 | void start(); 129 | void stop(); 130 | 131 | PacketBuffer* allocate(); 132 | PacketBuffer* clone(PacketBuffer* buf); 133 | void release(PacketBuffer* buf); 134 | 135 | uint16_t receive(EndpointId eid, PacketBuffer** bufs, uint16_t buf_count); 136 | uint16_t send(EndpointId eid, PacketBuffer** bufs, uint16_t buf_count); 137 | 138 | private: 139 | ::mica::util::Config config_; 140 | 141 | void init_eal(uint64_t core_mask); 142 | void init_mempool(); 143 | 144 | static uint16_t get_port_numa_id(uint16_t port_id); 145 | 146 | void add_endpoint(uint16_t lcore_id, uint16_t port_id); 147 | 148 | struct Port { 149 | uint8_t valid; 150 | 151 | ether_addr mac_addr; 152 | uint32_t ipv4_addr; 153 | uint16_t numa_id; 154 | 155 | uint16_t max_queue_count; 156 | uint16_t next_available_queue_id; 157 | }; 158 | 159 | int rte_argc_; 160 | char* rte_argv_[100]; 161 | 162 | rte_mempool* mempools_[StaticConfig::kMaxNUMACount]; 163 | std::vector ports_; 164 | 165 | uint16_t endpoint_count_; 166 | EndpointInfo endpoint_info_[StaticConfig::kMaxEndpointCount]; 167 | 168 | bool started_; 169 | }; 170 | } 171 | } 172 | 173 | #include "mica/network/dpdk_impl.h" 174 | 175 | #endif 176 | -------------------------------------------------------------------------------- /src/mica/network/network_addr.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_NETWORK_ADDR_H_ 3 | #define MICA_NETWORK_ADDR_H_ 4 | 5 | namespace mica { 6 | namespace network { 7 | class NetworkAddress { 8 | public: 9 | static uint32_t parse_ipv4_addr(const char* s) { 10 | char* p = const_cast(s); 11 | uint32_t v = 0; 12 | for (size_t i = 0; i < 4; i++, p++) { 13 | assert(p && *p != '\0'); 14 | v = (v << 8) + static_cast( 15 | ::mica::util::safe_cast(strtoul(p, &p, 10))); 16 | } 17 | return v; 18 | } 19 | 20 | static ether_addr parse_mac_addr(const char* s) { 21 | char* p = const_cast(s); 22 | ether_addr v; 23 | for (size_t i = 0; i < 6; i++, p++) { 24 | assert(p && *p != '\0'); 25 | v.addr_bytes[i] = static_cast( 26 | ::mica::util::safe_cast(strtoul(p, &p, 16))); 27 | } 28 | return v; 29 | } 30 | 31 | static std::string str_ipv4_addr(uint32_t addr) { 32 | char buf[16]; 33 | snprintf(buf, sizeof(buf), "%d.%d.%d.%d", addr >> 24, (addr >> 16) & 255, 34 | (addr >> 8) & 255, addr & 255); 35 | return std::string(buf); 36 | } 37 | 38 | static std::string str_mac_addr(const ether_addr& addr) { 39 | char buf[18]; 40 | snprintf(buf, sizeof(buf), "%02x:%02x:%02x:%02x:%02x:%02x", 41 | addr.addr_bytes[0], addr.addr_bytes[1], addr.addr_bytes[2], 42 | addr.addr_bytes[3], addr.addr_bytes[4], addr.addr_bytes[5]); 43 | return std::string(buf); 44 | } 45 | }; 46 | } 47 | } 48 | 49 | #endif -------------------------------------------------------------------------------- /src/mica/network/packet_io.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_NETWORK_PACKET_IO_H_ 3 | #define MICA_NETWORK_PACKET_IO_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | 8 | namespace mica { 9 | namespace network { 10 | class PacketBufferInterface { 11 | public: 12 | uint16_t get_length() const; 13 | uint16_t get_headroom() const; 14 | uint16_t get_tailroom() const; 15 | 16 | char* get_data(); 17 | const char* get_data() const; 18 | 19 | void set_length(uint16_t len); 20 | char* prepend(uint16_t len); 21 | char* append(uint16_t len); 22 | char* adj(uint16_t len); 23 | char* trim(uint16_t len); 24 | }; 25 | 26 | class PacketIOInterface { 27 | public: 28 | typedef PacketBufferInterface PacketBuffer; 29 | 30 | typedef uint32_t EndpointId; 31 | static constexpr EndpointId kInvalidEndpointId = 32 | std::numeric_limits::max(); 33 | 34 | struct EndpointInfo { 35 | uint16_t owner_lcore_id; 36 | 37 | volatile uint64_t rx_bursts; 38 | volatile uint64_t rx_packets; 39 | 40 | volatile uint64_t tx_bursts; 41 | volatile uint64_t tx_packets; 42 | volatile uint64_t tx_dropped; 43 | } __attribute__((aligned(128))); 44 | 45 | // static constexpr uint16_t kMaxBurstSize = 1; 46 | 47 | std::vector get_endpoints() const; 48 | const EndpointInfo& get_endpoint_info(EndpointId eid) const; 49 | 50 | void start(); 51 | void stop(); 52 | 53 | PacketBuffer* allocate(); 54 | PacketBuffer* clone(PacketBuffer* buf); 55 | void release(PacketBuffer* buf); 56 | 57 | uint16_t receive(EndpointId eid, PacketBuffer** bufs, uint16_t buf_count); 58 | uint16_t send(EndpointId eid, PacketBuffer** bufs, uint16_t buf_count); 59 | }; 60 | } 61 | } 62 | 63 | #endif -------------------------------------------------------------------------------- /src/mica/pool/circular_log.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_POOL_CIRCULAR_LOG_H_ 3 | #define MICA_POOL_CIRCULAR_LOG_H_ 4 | 5 | #include "mica/pool/pool.h" 6 | #include "mica/util/config.h" 7 | #include "mica/util/barrier.h" 8 | #include "mica/util/roundup.h" 9 | #include "mica/util/safe_cast.h" 10 | #include "mica/alloc/hugetlbfs_shm.h" 11 | 12 | // Configuration file entries for CircularLog: 13 | // 14 | // * size (integer): The total size in bytes to use. 15 | // * concurrent_read (bool): If true, enable concurrent reads by multiple 16 | // threads. 17 | // * concurrent_write (bool): If true, enable concurrent writes by multiple 18 | // threads. 19 | // * numa_node (integer): The ID of the NUMA node to store the data. 20 | 21 | namespace mica { 22 | namespace pool { 23 | struct BasicCircularLogConfig { 24 | // Support concurrent access. The actual concurrent access is enabled by 25 | // concurrent_read and concurrent_write in the configuration. 26 | static constexpr bool kConcurrent = true; 27 | 28 | // Be verbose. 29 | static constexpr bool kVerbose = false; 30 | 31 | // The memory allocator type. 32 | typedef ::mica::alloc::HugeTLBFS_SHM Alloc; 33 | }; 34 | 35 | class CircularLogTag {}; 36 | 37 | template 38 | class CircularLog : public PoolInterface { 39 | public: 40 | typedef CircularLogTag Tag; 41 | 42 | typedef typename StaticConfig::Alloc Alloc; 43 | 44 | CircularLog(const ::mica::util::Config& config, Alloc* alloc); 45 | ~CircularLog(); 46 | 47 | typedef uint64_t Offset; 48 | static constexpr size_t kOffsetWidth = 48; 49 | static constexpr Offset kInsufficientSpace = 50 | std::numeric_limits::max(); 51 | 52 | void lock(); 53 | void unlock(); 54 | 55 | void reset(); 56 | Offset allocate(size_t size); 57 | void release(Offset offset); 58 | 59 | void prefetch_item(Offset offset) const; 60 | 61 | char* get_item(Offset offset); 62 | const char* get_item(Offset offset) const; 63 | 64 | char* get_item(Offset offset, std::size_t* out_len); 65 | const char* get_item(Offset offset, std::size_t* out_len) const; 66 | 67 | // CircularLogTag specific 68 | bool is_valid(Offset offset) const; 69 | // uint64_t get_head() const; 70 | uint64_t get_tail() const; 71 | uint64_t get_mask() const; 72 | uint64_t get_size() const; 73 | 74 | private: 75 | void check_invariants() const; 76 | 77 | void pop_head(); 78 | uint64_t push_tail(uint64_t item_size); 79 | 80 | static constexpr size_t kMinimumSize = 2 * 1048576; 81 | static constexpr size_t kWrapAroundSize = 2 * 1048576; 82 | static constexpr size_t kOffsetMask = (size_t(1) << kOffsetWidth) - 1; 83 | 84 | struct Item { 85 | uint64_t size; 86 | char data[0]; 87 | }; 88 | 89 | ::mica::util::Config config_; 90 | Alloc* alloc_; 91 | 92 | uint8_t concurrent_access_mode_; 93 | char* data_; 94 | uint64_t size_; // a power of two 95 | uint64_t mask_; // size - 1; this mask is used only when converting the 96 | // offset to the actual location of the item 97 | 98 | // Padding to separate static and dynamic fields. 99 | char padding0[128]; 100 | 101 | uint8_t lock_; 102 | 103 | // internally, pool uses full 64-bit numbers for head and tail 104 | // however, the valid range for item_offset is limited to 105 | // (kOffsetMask + 1) 106 | // we resolve this inconsistency by applying MEHCACHED_ITEM_OFFSET_MASK mask 107 | // whenever returning the offset to the outside or using a masked offset 108 | // given from the outside 109 | uint64_t head_; // start offset of items 110 | uint64_t tail_; // end offset of items 111 | } __attribute__((aligned(128))); // To prevent false sharing caused by 112 | // adjacent cacheline prefetching. 113 | } 114 | } 115 | 116 | #include "mica/pool/circular_log_impl.h" 117 | 118 | #endif 119 | -------------------------------------------------------------------------------- /src/mica/pool/pool.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_POOL_POOL_H_ 3 | #define MICA_POOL_POOL_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | 8 | namespace mica { 9 | namespace pool { 10 | class PoolInterface { 11 | public: 12 | typedef void Tag; 13 | 14 | typedef uint64_t Offset; 15 | static constexpr size_t kOffsetWidth = 16 | static_cast(std::numeric_limits::digits); 17 | static constexpr Offset kInsufficientSpace = 18 | std::numeric_limits::max(); 19 | 20 | void lock(); 21 | void unlock(); 22 | 23 | void reset(); 24 | Offset allocate(size_t size); 25 | void release(Offset offset); 26 | 27 | void prefetch_item(Offset offset) const; 28 | 29 | char* get_item(Offset offset); 30 | const char* get_item(Offset offset) const; 31 | 32 | char* get_item(Offset offset, std::size_t* out_len); 33 | const char* get_item(Offset offset, std::size_t* out_len) const; 34 | }; 35 | } 36 | } 37 | 38 | #endif -------------------------------------------------------------------------------- /src/mica/pool/segregated_fit.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_POOL_SEGREGATED_FIT_H_ 3 | #define MICA_POOL_SEGREGATED_FIT_H_ 4 | 5 | #include 6 | #include "mica/pool/pool.h" 7 | #include "mica/util/config.h" 8 | #include "mica/util/barrier.h" 9 | #include "mica/util/memcpy.h" 10 | #include "mica/util/roundup.h" 11 | #include "mica/util/safe_cast.h" 12 | #include "mica/alloc/hugetlbfs_shm.h" 13 | 14 | // Configuration file entries for SegregatedFit: 15 | // 16 | // * size (integer): The total size in bytes to use. 17 | // * concurrent_read (bool): If true, enable concurrent reads by multiple 18 | // threads. 19 | // * concurrent_write (bool): If true, enable concurrent writes by multiple 20 | // threads. 21 | // * numa_node (integer): The ID of the NUMA node to store the data. 22 | 23 | namespace mica { 24 | namespace pool { 25 | struct BasicSegregatedFitConfig { 26 | // Support concurrent access. The actual concurrent access is enabled by 27 | // concurrent_read and concurrent_write in the configuration. 28 | static constexpr bool kConcurrent = true; 29 | 30 | // Be verbose. 31 | static constexpr bool kVerbose = false; 32 | 33 | // The number of size classes for freelists. 34 | static constexpr size_t kNumClasses = 32; 35 | // The byte increment between freelist size classes. 36 | // The maximum object size supported without linear scanning is 37 | // kMinimumSize + (kNumClasses - 1) * kClassIncrement - kOverhead 38 | static constexpr size_t kClassIncrement = 8; 39 | 40 | // The memory allocator type. 41 | typedef ::mica::alloc::HugeTLBFS_SHM Alloc; 42 | }; 43 | 44 | class SegregatedFitTag {}; 45 | 46 | // memory allocation using segregated fit (similar to Lea) 47 | template 48 | class SegregatedFit : public PoolInterface { 49 | public: 50 | typedef SegregatedFitTag Tag; 51 | 52 | typedef typename StaticConfig::Alloc Alloc; 53 | 54 | SegregatedFit(const ::mica::util::Config& config, Alloc* alloc); 55 | ~SegregatedFit(); 56 | 57 | typedef uint64_t Offset; 58 | static constexpr size_t kOffsetWidth = 40; 59 | static constexpr Offset kInsufficientSpace = 60 | std::numeric_limits::max(); 61 | 62 | void lock(); 63 | void unlock(); 64 | 65 | void reset(); 66 | Offset allocate(size_t item_size); 67 | void release(Offset offset); 68 | 69 | char* allocate_raw(size_t item_size); 70 | void release_raw(char* p); 71 | 72 | void prefetch_item(Offset offset) const; 73 | 74 | char* get_item(Offset offset); 75 | const char* get_item(Offset offset) const; 76 | 77 | char* get_item(Offset offset, std::size_t* out_len); 78 | const char* get_item(Offset offset, std::size_t* out_len) const; 79 | 80 | private: 81 | static constexpr size_t size_to_class_roundup(uint64_t size); 82 | static constexpr size_t size_to_class_rounddown(uint64_t size); 83 | 84 | void insert_free_chunk(char* chunk_start, uint64_t chunk_size); 85 | 86 | void remove_free_chunk_from_free_list(char* chunk_start, uint64_t chunk_size); 87 | bool remove_free_chunk_from_head(uint64_t minimum_chunk_size, 88 | char** out_chunk_start, 89 | uint64_t* out_chunk_size); 90 | 91 | void coalese_free_chunk_left(char** chunk_start, uint64_t* chunk_size); 92 | void coalese_free_chunk_right(char** chunk_start, uint64_t* chunk_size); 93 | 94 | static constexpr uint64_t get_tag_size(uint64_t vec); 95 | static constexpr uint64_t get_tag_status(uint64_t vec); 96 | static constexpr uint64_t make_tag_vec(uint64_t size, uint64_t status); 97 | 98 | // Per-item space overhead solely caused by SegregatedFit 99 | static constexpr size_t kOverhead = 24; 100 | // Minimum pool size = 32 bytes (must be able to hold 4 size_t variables) 101 | static constexpr size_t kMinimumSize = 32; 102 | // Maximum pool size = 40-bit wide size (can be up to 63-bit wide) 103 | static constexpr size_t kMaximumSize = (1UL << 40) - 1; 104 | 105 | static constexpr uint64_t kStatusFree = 0; 106 | static constexpr uint64_t kStatusOccupied = 1; 107 | 108 | ::mica::util::Config config_; 109 | Alloc* alloc_; 110 | 111 | uint8_t concurrent_access_mode_; 112 | char* data_; // the base address of the reserved memory 113 | 114 | // Padding to separate static and dynamic fields. 115 | char padding0[128]; 116 | 117 | uint8_t lock_; 118 | char* free_head_[StaticConfig::kNumClasses]; // the head free pointer 119 | // of each class 120 | uint64_t size_; // the total size 121 | } __attribute__((aligned(128))); // To prevent false sharing caused by 122 | // adjacent cacheline prefetching. 123 | } 124 | } 125 | 126 | #include "mica/pool/segregated_fit_impl.h" 127 | 128 | #endif 129 | -------------------------------------------------------------------------------- /src/mica/processor/partitions_impl/init.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_PROCESSOR_PARTITIONS_IMPL_INIT_H_ 3 | #define MICA_PROCESSOR_PARTITIONS_IMPL_INIT_H_ 4 | 5 | namespace mica { 6 | namespace processor { 7 | template 8 | Partitions::Partitions(const ::mica::util::Config& config, 9 | Alloc* alloc) 10 | : config_(config), alloc_(alloc) { 11 | { 12 | auto c = config.get("lcores"); 13 | for (size_t i = 0; i < c.size(); i++) { 14 | uint16_t lcore_id = 15 | ::mica::util::safe_cast(c.get(i).get_uint64()); 16 | assert(lcore_id < StaticConfig::kMaxLCoreCount); 17 | lcores_.push_back(lcore_id); 18 | } 19 | } 20 | assert(lcores_.size() > 0); 21 | 22 | partition_count_ = ::mica::util::safe_cast( 23 | config.get("partition_count").get_uint64()); 24 | assert(static_cast(partition_count_) <= 25 | StaticConfig::kMaxPartitionCount); 26 | 27 | total_size_ = config.get("total_size").get_uint64(); 28 | total_item_count_ = config.get("total_item_count").get_uint64(); 29 | 30 | extra_collision_avoidance_ = 31 | config.get("extra_collision_avoidance").get_double(-1.); 32 | 33 | mth_threshold_ = config.get("mth_threshold").get_double(0.5); 34 | 35 | concurrent_read_ = config.get("concurrent_read").get_bool() ? 1 : 0; 36 | concurrent_write_ = config.get("concurrent_write").get_bool() ? 1 : 0; 37 | 38 | // pipeline_size_ = 39 | // ::mica::util::safe_cast(config.get("pipeline_size").get_uint64()); 40 | uint8_t stage_gap = 41 | ::mica::util::safe_cast(config.get("stage_gap").get_uint64(2)); 42 | // assert(pipeline_size_ > stage_gap_ * (1 + 1 + 1)); 43 | for (size_t lcore_id = 0; lcore_id < StaticConfig::kMaxLCoreCount; lcore_id++) 44 | load_stats_[lcore_id].stage_gap = stage_gap; 45 | 46 | target_stage_gap_time_ = ::mica::util::safe_cast( 47 | config.get("target_stage_gap_time").get_uint64(0)); 48 | 49 | initialize(); 50 | } 51 | 52 | template 53 | Partitions::~Partitions() { 54 | for (size_t i = 0; i < partition_count_; i++) { 55 | delete tables_[i]; 56 | delete pools_[i]; 57 | } 58 | } 59 | 60 | template 61 | void Partitions::initialize() { 62 | reset_load_stats(); 63 | 64 | assert(::mica::util::next_power_of_two(StaticConfig::kRecentKeyHashBuckets) == 65 | StaticConfig::kRecentKeyHashBuckets); 66 | assert(::mica::util::next_power_of_two( 67 | StaticConfig::kRecentKeyHashAssociativity) == 68 | StaticConfig::kRecentKeyHashAssociativity); 69 | ::mica::util::memset(recent_key_hashes_, 0, sizeof(recent_key_hashes_)); 70 | 71 | size_t current_lcore_index = 0; 72 | for (size_t i = 0; i < partition_count_; i++) { 73 | uint16_t current_lcore_id = lcores_[current_lcore_index]; 74 | current_lcore_index = (current_lcore_index + 1) % lcores_.size(); 75 | 76 | auto pool_config = ::mica::util::Config::empty_dict( 77 | std::string() + "[derived from " + config_.get_path() + 78 | " by Partitions]"); 79 | pool_config.insert_uint64("size", total_size_ / partition_count_); 80 | pool_config.insert_bool("concurrent_read", concurrent_read_); 81 | pool_config.insert_bool("concurrent_write", concurrent_write_); 82 | pool_config.insert_uint64("numa_node", 83 | ::mica::util::lcore.numa_id(current_lcore_id)); 84 | 85 | auto table_config = ::mica::util::Config::empty_dict( 86 | std::string() + "[derived from " + config_.get_path() + 87 | " by Partitions]"); 88 | size_t item_count = total_item_count_ / partition_count_; 89 | table_config.insert_uint64("item_count", item_count); 90 | if (extra_collision_avoidance_ >= 0.) 91 | table_config.insert_double("extra_collision_avoidance", 92 | extra_collision_avoidance_); 93 | table_config.insert_bool("concurrent_read", concurrent_read_); 94 | table_config.insert_bool("concurrent_write", concurrent_write_); 95 | table_config.insert_uint64("numa_node", 96 | ::mica::util::lcore.numa_id(current_lcore_id)); 97 | table_config.insert_double("mth_threshold", mth_threshold_); 98 | 99 | owner_lcore_ids_[i] = current_lcore_id; 100 | pools_[i] = new Pool(pool_config, alloc_); 101 | tables_[i] = new Table(table_config, alloc_, pools_[i]); 102 | 103 | if (StaticConfig::kVerbose) { 104 | printf("= Partition %zu at lcore %hu\n", i, current_lcore_id); 105 | printf("== Pool config\n"); 106 | printf("%s\n", pool_config.dump().c_str()); 107 | printf("== Table config\n"); 108 | printf("%s\n", table_config.dump().c_str()); 109 | printf("\n"); 110 | } 111 | } 112 | } 113 | } 114 | } 115 | 116 | #endif -------------------------------------------------------------------------------- /src/mica/processor/partitions_impl/recent_key_hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_PROCESSOR_PARTITIONS_RECENT_KEY_HASH_H_ 3 | #define MICA_PROCESSOR_PARTITIONS_RECENT_KEY_HASH_H_ 4 | 5 | namespace mica { 6 | namespace processor { 7 | template 8 | bool Partitions::is_recent_key_hash(uint16_t lcore_id, 9 | uint64_t key_hash) const { 10 | // TODO: Introduce seeded hashing to key_hash so that we can avoid collisions 11 | // of a few popular items competing for the same bucket. 12 | // TODO: Adjust associativity dynamically depending on the request style. 13 | // Low associativity for either extremely skew (e.g., single key) or no skew 14 | // (i.e., no benefit), high associativity for moderate skew. 15 | 16 | // We need to shift key_hash only by 16 bits to obtain uint16_t, but 32-bit 17 | // shifting is faster by 1--2 clocks. 18 | size_t index = 19 | ((key_hash >> 32) & (StaticConfig::kRecentKeyHashBuckets - 1)) * 20 | StaticConfig::kRecentKeyHashAssociativity; 21 | uint16_t tag = static_cast(key_hash); 22 | 23 | if (StaticConfig::kRecentKeyHashAssociativity == 1) { 24 | return recent_key_hashes_[lcore_id].v[index] == tag; 25 | } else { 26 | for (size_t i = 0; i < StaticConfig::kRecentKeyHashAssociativity; i++) 27 | if (recent_key_hashes_[lcore_id].v[index + i] == tag) return true; 28 | return false; 29 | } 30 | } 31 | 32 | template 33 | void Partitions::update_recent_key_hash(uint16_t lcore_id, 34 | uint64_t key_hash) { 35 | size_t index = 36 | ((key_hash >> 32) & (StaticConfig::kRecentKeyHashBuckets - 1)) * 37 | StaticConfig::kRecentKeyHashAssociativity; 38 | uint16_t tag = static_cast(key_hash); 39 | 40 | if (recent_key_hashes_[lcore_id].v[index] == tag) return; 41 | 42 | if (StaticConfig::kRecentKeyHashAssociativity > 1) { 43 | size_t i; 44 | for (i = 1; i < StaticConfig::kRecentKeyHashAssociativity - 1; i++) 45 | if (recent_key_hashes_[lcore_id].v[index + i] == tag) break; 46 | 47 | // Shift key hashes to the right by 1 slot. 48 | for (size_t j = i; j > 0; j--) 49 | recent_key_hashes_[lcore_id].v[index + j] = 50 | recent_key_hashes_[lcore_id].v[index + j - 1]; 51 | } 52 | 53 | // Put the new entry. 54 | recent_key_hashes_[lcore_id].v[index] = tag; 55 | } 56 | } 57 | } 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /src/mica/processor/processor.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_PROCESSOR_PROCESSOR_H_ 3 | #define MICA_PROCESSOR_PROCESSOR_H_ 4 | 5 | #include "mica/processor/request_accessor.h" 6 | 7 | namespace mica { 8 | namespace processor { 9 | template 10 | class ProcessorInterface { 11 | public: 12 | template 13 | void process(RequestAccessor& ra); 14 | 15 | // template 16 | // void process(RequestAccessor& ra) const; 17 | 18 | size_t get_table_count() const; 19 | 20 | const Table* get_table(size_t index) const; 21 | Table* get_table(size_t index); 22 | 23 | bool get_concurrent_read() const; 24 | bool get_concurrent_write() const; 25 | }; 26 | } 27 | } 28 | #endif -------------------------------------------------------------------------------- /src/mica/processor/request_accessor.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_PROCESSOR_REQUEST_ACCESSOR_H_ 3 | #define MICA_PROCESSOR_REQUEST_ACCESSOR_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/processor/types.h" 7 | #include "mica/table/types.h" 8 | 9 | namespace mica { 10 | namespace processor { 11 | class RequestAccessorInterface {}; 12 | 13 | typedef ::mica::table::Result Result; 14 | 15 | class RandomAccessRequestAccessorInterface : public RequestAccessorInterface { 16 | public: 17 | bool prepare(size_t index); 18 | 19 | Operation get_operation(size_t index); 20 | 21 | uint64_t get_key_hash(size_t index); 22 | const char* get_key(size_t index); 23 | size_t get_key_length(size_t index); 24 | 25 | const char* get_value(size_t index); 26 | size_t get_value_length(size_t index); 27 | 28 | char* get_out_value(size_t index); 29 | size_t get_out_value_length(size_t index); 30 | 31 | void set_out_value_length(size_t index, size_t len); 32 | 33 | void set_result(size_t index, Result result); 34 | 35 | void retire(size_t index); 36 | }; 37 | 38 | class RequestArrayAccessor : public RandomAccessRequestAccessorInterface { 39 | public: 40 | RequestArrayAccessor(size_t count, const Operation* req_types, 41 | const uint64_t* key_hashes, const char** keys, 42 | const size_t* key_lengths, const char** values, 43 | const size_t* value_lengths, Result** out_results, 44 | char** out_values, size_t** out_value_lengths) 45 | : count_(count), 46 | req_types_(req_types), 47 | key_hashes_(key_hashes), 48 | keys_(keys), 49 | key_lengths_(key_lengths), 50 | values_(values), 51 | value_lengths_(value_lengths), 52 | out_results_(out_results), 53 | out_values_(out_values), 54 | out_value_lengths_(out_value_lengths) { 55 | assert(req_types_); 56 | assert(key_hashes_); 57 | assert(keys_); 58 | assert(key_lengths_); 59 | assert(values_); 60 | assert(value_lengths_); 61 | assert(out_values_); 62 | assert(out_value_lengths_); 63 | } 64 | 65 | bool prepare(size_t index) { 66 | if (index >= count_) return false; 67 | // __builtin_prefetch(keys_[index], 0, 0); 68 | // __builtin_prefetch(values_[index], 0, 0); 69 | // __builtin_prefetch(out_values_[index], 0, 0); 70 | return true; 71 | } 72 | 73 | Operation get_operation(size_t index) { 74 | assert(index < count_); 75 | return req_types_[index]; 76 | } 77 | 78 | uint64_t get_key_hash(size_t index) { 79 | assert(index < count_); 80 | return key_hashes_[index]; 81 | } 82 | 83 | const char* get_key(size_t index) { 84 | assert(index < count_); 85 | return keys_[index]; 86 | } 87 | 88 | size_t get_key_length(size_t index) { 89 | assert(index < count_); 90 | return key_lengths_[index]; 91 | } 92 | 93 | const char* get_value(size_t index) { 94 | assert(index < count_); 95 | return values_[index]; 96 | } 97 | 98 | size_t get_value_length(size_t index) { 99 | assert(index < count_); 100 | return value_lengths_[index]; 101 | } 102 | 103 | char* get_out_value(size_t index) { 104 | assert(index < count_); 105 | return out_values_[index]; 106 | } 107 | 108 | size_t get_out_value_length(size_t index) { 109 | assert(index < count_); 110 | return *out_value_lengths_[index]; 111 | } 112 | 113 | void set_out_value_length(size_t index, size_t len) { 114 | assert(index < count_); 115 | *out_value_lengths_[index] = len; 116 | } 117 | 118 | void set_result(size_t index, Result result) { 119 | assert(index < count_); 120 | *out_results_[index] = result; 121 | } 122 | 123 | void retire(size_t index) { 124 | assert(index < count_); 125 | (void)index; 126 | } 127 | 128 | private: 129 | size_t count_; 130 | const Operation* req_types_; 131 | const uint64_t* key_hashes_; 132 | const char** keys_; 133 | const size_t* key_lengths_; 134 | const char** values_; 135 | const size_t* value_lengths_; 136 | Result** out_results_; 137 | char** out_values_; 138 | size_t** out_value_lengths_; 139 | }; 140 | } 141 | } 142 | 143 | #endif -------------------------------------------------------------------------------- /src/mica/processor/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_PROCESSOR_TYPES_H_ 3 | #define MICA_PROCESSOR_TYPES_H_ 4 | 5 | #include "mica/common.h" 6 | 7 | namespace mica { 8 | namespace processor { 9 | enum class Operation { 10 | kReset = 0, 11 | kNoopRead, 12 | kNoopWrite, 13 | kAdd, 14 | kSet, 15 | kGet, 16 | kTest, 17 | kDelete, 18 | kIncrement, 19 | }; 20 | } 21 | } 22 | 23 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/del.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_DEL_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_DEL_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::del(uint64_t key_hash, const char* key, 9 | size_t key_length) { 10 | assert(key_length <= kMaxKeyLength); 11 | 12 | uint32_t bucket_index = calc_bucket_index(key_hash); 13 | uint16_t tag = calc_tag(key_hash); 14 | 15 | Bucket* bucket = buckets_ + bucket_index; 16 | 17 | lock_bucket(bucket); 18 | 19 | Bucket* located_bucket; 20 | size_t item_index = 21 | find_item_index(bucket, key_hash, tag, key, key_length, &located_bucket); 22 | if (item_index == StaticConfig::kBucketSize) { 23 | unlock_bucket(bucket); 24 | stat_inc(&Stats::delete_notfound); 25 | return Result::kNotFound; 26 | } 27 | 28 | if (std::is_base_of<::mica::pool::CircularLogTag, 29 | typename Pool::Tag>::value) { 30 | } else // if (std::is_base_of<::mica::pool::SegregatedFitTag, 31 | // Pool::Tag>::value) 32 | { 33 | pool_->lock(); 34 | } 35 | 36 | pool_->release(get_item_offset(located_bucket->item_vec[item_index])); 37 | 38 | if (std::is_base_of<::mica::pool::CircularLogTag, 39 | typename Pool::Tag>::value) { 40 | } else // if (std::is_base_of<::mica::pool::SegregatedFitTag, 41 | // Pool::Tag>::value) 42 | { 43 | pool_->unlock(); 44 | } 45 | 46 | located_bucket->item_vec[item_index] = 0; 47 | stat_dec(&Stats::count); 48 | 49 | fill_hole(located_bucket, item_index); 50 | 51 | unlock_bucket(bucket); 52 | 53 | stat_inc(&Stats::delete_found); 54 | return Result::kSuccess; 55 | } 56 | } 57 | } 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/get.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_GET_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_GET_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::get(uint64_t key_hash, const char* key, 9 | size_t key_length, char* out_value, 10 | size_t in_value_length, 11 | size_t* out_value_length, 12 | bool allow_mutation) const { 13 | assert(key_length <= kMaxKeyLength); 14 | 15 | uint32_t bucket_index = calc_bucket_index(key_hash); 16 | uint16_t tag = calc_tag(key_hash); 17 | 18 | const Bucket* bucket = buckets_ + bucket_index; 19 | 20 | bool partial_value; 21 | 22 | while (true) { 23 | uint32_t version_start = read_version_begin(bucket); 24 | 25 | const Bucket* located_bucket; 26 | size_t item_index = find_item_index(bucket, key_hash, tag, key, key_length, 27 | &located_bucket); 28 | if (item_index == StaticConfig::kBucketSize) { 29 | if (version_start != read_version_end(bucket)) continue; 30 | stat_inc(&Stats::get_notfound); 31 | return Result::kNotFound; 32 | } 33 | 34 | uint64_t item_vec = located_bucket->item_vec[item_index]; 35 | uint64_t item_offset = get_item_offset(item_vec); 36 | 37 | // we may read garbage data, but all operations relying on them are safe 38 | // here 39 | const Item* item = 40 | reinterpret_cast(pool_->get_item(item_offset)); 41 | uint32_t kv_length_vec = item->kv_length_vec; 42 | 43 | // We used to read key_length again in the old version, but we do not need 44 | // it because if this bucket is not changed, the stored key_length will be 45 | // the same as the key_length argument, and if this bucket is changed, we 46 | // can ignore (possibly invalid) key_length for now and retry again later. 47 | 48 | // key_length = get_key_length(kv_length_vec); 49 | // An invalid value means that a concurrent change is ongoing. 50 | // if (key_length > kMaxKeyLength) continue; 51 | 52 | size_t value_length = get_value_length(kv_length_vec); 53 | bool invalid_value_length = false; 54 | 55 | if (value_length <= kMaxValueLength) { 56 | *out_value_length = std::min(value_length, in_value_length); 57 | partial_value = (value_length != *out_value_length); 58 | ::mica::util::memcpy<8>(out_value, 59 | item->data + ::mica::util::roundup<8>(key_length), 60 | *out_value_length); 61 | } else { 62 | // An invalid value means that a concurrent change is ongoing. 63 | invalid_value_length = true; 64 | // Just to suppress a compiler warning. 65 | partial_value = false; 66 | } 67 | 68 | if (invalid_value_length || !Specialization::is_valid(pool_, item_offset)) { 69 | if (version_start != read_version_end(bucket)) continue; 70 | 71 | if (allow_mutation) { 72 | auto mut_this = const_cast*>(this); 73 | auto mut_bucket = const_cast(bucket); 74 | auto mut_located_bucket = const_cast(located_bucket); 75 | // remove stale item; this may remove some wrong item, but we do not 76 | // care because (1) if this key has been deleted and reinserted, it is 77 | // none of our business here (2) if this key has been deleted and a 78 | // different key was inserted, we delete innocent key, but without any 79 | // fatal issue. this will slow down query speed for outdated matching 80 | // key at first, but improves it later by skipping the value copy step 81 | mut_this->lock_bucket(mut_bucket); 82 | if (located_bucket->item_vec[item_index] != 0) { 83 | mut_located_bucket->item_vec[item_index] = 0; 84 | stat_dec(&Stats::count); 85 | } 86 | mut_this->unlock_bucket(mut_bucket); 87 | } 88 | 89 | stat_inc(&Stats::get_notfound); 90 | return Result::kNotFound; 91 | } 92 | 93 | if (version_start != read_version_end(bucket)) continue; 94 | 95 | stat_inc(&Stats::get_found); 96 | 97 | if (allow_mutation) 98 | const_cast*>(this)->move_to_head( 99 | const_cast(bucket), const_cast(located_bucket), 100 | item, key_length, value_length, item_index, item_vec, item_offset); 101 | 102 | break; 103 | } 104 | 105 | if (partial_value) 106 | return Result::kPartialValue; 107 | else 108 | return Result::kSuccess; 109 | } 110 | } 111 | } 112 | 113 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/increment.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_INCREMENT_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_INCREMENT_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::increment(uint64_t key_hash, const char* key, 9 | size_t key_length, uint64_t increment, 10 | uint64_t* out_value) { 11 | assert(key_length <= kMaxKeyLength); 12 | 13 | uint32_t bucket_index = calc_bucket_index(key_hash); 14 | uint16_t tag = calc_tag(key_hash); 15 | 16 | Bucket* bucket = buckets_ + bucket_index; 17 | 18 | // TODO: add stats 19 | 20 | lock_bucket(bucket); 21 | 22 | Bucket* located_bucket; 23 | size_t item_index = 24 | find_item_index(bucket, key_hash, tag, key, key_length, &located_bucket); 25 | 26 | if (item_index == StaticConfig::kBucketSize) { 27 | unlock_bucket(bucket); 28 | // TODO: support seeding a new item with the given default value? 29 | return Result::kNotFound; // does not exist 30 | } 31 | 32 | uint64_t item_offset = get_item_offset(located_bucket->item_vec[item_index]); 33 | 34 | if (std::is_base_of<::mica::pool::CircularLogTag, 35 | typename Pool::Tag>::value) { 36 | // ensure that the item is still valid 37 | pool_->lock(); 38 | 39 | if (!Specialization::is_valid(pool_, item_offset)) { 40 | pool_->unlock(); 41 | unlock_bucket(bucket); 42 | // TODO: support seeding a new item with the given default value? 43 | return Result::kError; // exists in the table but not valid in the pool 44 | } 45 | } 46 | 47 | Item* item = reinterpret_cast(pool_->get_item(item_offset)); 48 | 49 | size_t value_length = get_value_length(item->kv_length_vec); 50 | if (value_length != sizeof(uint64_t)) { 51 | if (std::is_base_of<::mica::pool::CircularLogTag, 52 | typename Pool::Tag>::value) 53 | pool_->unlock(); 54 | unlock_bucket(bucket); 55 | return Result::kError; // invalid value size 56 | } 57 | 58 | uint64_t old_value; 59 | ::mica::util::memcpy<8>(reinterpret_cast(&old_value), 60 | item->data + ::mica::util::roundup<8>(key_length), 61 | sizeof(old_value)); 62 | 63 | uint64_t new_value = old_value + increment; 64 | ::mica::util::memcpy<8>(item->data + ::mica::util::roundup<8>(key_length), 65 | reinterpret_cast(&new_value), 66 | sizeof(new_value)); 67 | 68 | *out_value = new_value; 69 | // va(SimpleValueReader(reinterpret_cast(&new_value), 70 | // sizeof(new_value)); 71 | 72 | if (std::is_base_of<::mica::pool::CircularLogTag, typename Pool::Tag>::value) 73 | pool_->unlock(); 74 | unlock_bucket(bucket); 75 | 76 | return Result::kSuccess; 77 | } 78 | } 79 | } 80 | 81 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/info.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_DEBUG_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_DEBUG_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void LTable::print_bucket(const Bucket* bucket) const { 9 | printf("\n", (size_t)bucket); 10 | for (size_t item_index = 0; item_index < StaticConfig::kBucketSize; 11 | item_index++) 12 | printf(" item_vec[%zu]: tag=%hu, item_offset=%lu\n", item_index, 13 | get_tag(bucket->item_vec[item_index]), 14 | get_item_offset(bucket->item_vec[item_index])); 15 | } 16 | 17 | template 18 | void LTable::print_buckets() const { 19 | for (size_t bucket_index = 0; bucket_index < num_buckets_; bucket_index++) { 20 | Bucket* bucket = buckets_ + bucket_index; 21 | print_bucket(bucket); 22 | } 23 | printf("\n"); 24 | } 25 | 26 | template 27 | void LTable::print_stats() const { 28 | if (StaticConfig::kCollectStats) { 29 | printf("count: %10zu\n", stats_.count); 30 | printf("set_nooverwrite: %10zu | ", stats_.set_nooverwrite); 31 | printf("set_new: %10zu | ", stats_.set_new); 32 | printf("set_inplace: %10zu | ", stats_.set_inplace); 33 | printf("set_evicted: %10zu\n", stats_.set_evicted); 34 | printf("get_found: %10zu | ", stats_.get_found); 35 | printf("get_notfound: %10zu\n", stats_.get_notfound); 36 | printf("test_found: %10zu | ", stats_.test_found); 37 | printf("test_notfound: %10zu\n", stats_.test_notfound); 38 | printf("delete_found: %10zu | ", stats_.delete_found); 39 | printf("delete_notfound: %10zu\n", stats_.delete_notfound); 40 | printf("cleanup: %10zu\n", stats_.cleanup); 41 | printf("move_to_head_performed: %10zu | ", stats_.move_to_head_performed); 42 | printf("move_to_head_skipped: %10zu | ", stats_.move_to_head_skipped); 43 | printf("move_to_head_failed: %10zu\n", stats_.move_to_head_failed); 44 | } 45 | } 46 | 47 | template 48 | void LTable::reset_stats(bool reset_count) { 49 | if (StaticConfig::kCollectStats) { 50 | size_t count = stats_.count; 51 | ::mica::util::memset(&stats_, 0, sizeof(stats_)); 52 | if (!reset_count) stats_.count = count; 53 | } 54 | } 55 | 56 | template 57 | void LTable::stat_inc(size_t Stats::*counter) const { 58 | if (StaticConfig::kCollectStats) __sync_add_and_fetch(&(stats_.*counter), 1); 59 | } 60 | 61 | template 62 | void LTable::stat_dec(size_t Stats::*counter) const { 63 | if (StaticConfig::kCollectStats) __sync_sub_and_fetch(&(stats_.*counter), 1); 64 | } 65 | } 66 | } 67 | 68 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/init.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_INIT_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_INIT_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | LTable::LTable(const ::mica::util::Config& config, Alloc* alloc, 9 | Pool* pool) 10 | : config_(config), alloc_(alloc), pool_(pool) { 11 | size_t item_count = config.get("item_count").get_uint64(); 12 | // Compensate the load factor. 13 | item_count = item_count * 11 / 10; 14 | 15 | size_t num_buckets = 16 | (item_count + StaticConfig::kBucketSize - 1) / StaticConfig::kBucketSize; 17 | 18 | double extra_collision_avoidance = 19 | config.get("extra_collision_avoidance") 20 | .get_double(StaticConfig::kEviction ? 0. : 0.1); 21 | size_t num_extra_buckets = static_cast( 22 | static_cast(num_buckets) * extra_collision_avoidance); 23 | 24 | bool concurrent_read = config.get("concurrent_read").get_bool(); 25 | bool concurrent_write = config.get("concurrent_write").get_bool(); 26 | size_t numa_node = config.get("numa_node").get_uint64(); 27 | double mth_threshold = config.get("mth_threshold").get_double(0.5); 28 | 29 | assert(num_buckets > 0); 30 | 31 | size_t log_num_buckets = 0; 32 | while (((size_t)1 << log_num_buckets) < num_buckets) log_num_buckets++; 33 | num_buckets = (size_t)1 << log_num_buckets; 34 | assert(log_num_buckets <= 32); 35 | 36 | num_buckets_ = ::mica::util::safe_cast(num_buckets); 37 | num_buckets_mask_ = ::mica::util::safe_cast(num_buckets - 1); 38 | num_extra_buckets_ = ::mica::util::safe_cast(num_extra_buckets); 39 | 40 | { 41 | size_t shm_size = 42 | Alloc::roundup(sizeof(Bucket) * (num_buckets_ + num_extra_buckets_)); 43 | 44 | // TODO: Extend num_extra_buckets_ to meet shm_size. 45 | 46 | size_t alloc_id = alloc_->alloc(shm_size, numa_node); 47 | if (alloc_id == Alloc::kInvalidId) { 48 | fprintf(stderr, "failed to allocate memory\n"); 49 | assert(false); 50 | return; 51 | } 52 | while (true) { 53 | buckets_ = reinterpret_cast(alloc_->find_free_address(shm_size)); 54 | if (buckets_ == nullptr) assert(false); 55 | if (alloc_->map(alloc_id, buckets_, 0, shm_size)) break; 56 | } 57 | if (!alloc_->schedule_release(alloc_id)) assert(false); 58 | } 59 | extra_buckets_ = buckets_ + num_buckets_ - 60 | 1; // subtract by one to compensate 1-base indices 61 | // the rest extra_bucket information is initialized in reset() 62 | 63 | // we have to zero out buckets here because reset() tries to free non-zero 64 | // entries in the buckets 65 | ::mica::util::memset(buckets_, 0, 66 | sizeof(Bucket) * (num_buckets_ + num_extra_buckets_)); 67 | 68 | if (!concurrent_read) 69 | concurrent_access_mode_ = 0; 70 | else if (concurrent_read && !concurrent_write) 71 | concurrent_access_mode_ = 1; 72 | else 73 | concurrent_access_mode_ = 2; 74 | 75 | mth_threshold_ = static_cast( 76 | static_cast(Specialization::get_size(pool_)) * mth_threshold); 77 | 78 | rshift_ = 0; 79 | while ((((Bucket::kItemOffsetMask + 1) >> 1) >> rshift_) > num_buckets_) 80 | rshift_++; 81 | 82 | reset(); 83 | 84 | if (StaticConfig::kVerbose) { 85 | fprintf(stderr, "warning: kVerbose is defined (low performance)\n"); 86 | 87 | if (StaticConfig::kCollectStats) 88 | fprintf(stderr, "warning: kCollectStats is defined (low performance)\n"); 89 | 90 | if (StaticConfig::kConcurrent && !concurrent_read && !concurrent_write) 91 | fprintf(stderr, "warning: kConcurrent is defined (low performance)\n"); 92 | 93 | if (StaticConfig::kEviction) 94 | fprintf(stderr, "info: mth_threshold=%lf (%s)\n", mth_threshold, 95 | mth_threshold == 0. 96 | ? "LRU" 97 | : (mth_threshold == 1. ? "FIFO" : "approx-LRU")); 98 | 99 | // #ifdef MEHCACHED_USE_PH 100 | // fprintf(stderr, "MEHCACHED_USE_PH\n"); 101 | // #endif 102 | 103 | fprintf(stderr, "info: num_buckets = %u\n", num_buckets_); 104 | fprintf(stderr, "info: num_extra_buckets = %u\n", num_extra_buckets_); 105 | // fprintf(stderr, "pool_size = %zu\n", pool_size); 106 | 107 | fprintf(stderr, "\n"); 108 | } 109 | } 110 | 111 | template 112 | LTable::~LTable() { 113 | reset(); 114 | 115 | if (!alloc_->unmap(buckets_)) assert(false); 116 | 117 | // pool_->free(); 118 | } 119 | 120 | template 121 | void LTable::reset() { 122 | size_t bucket_index; 123 | 124 | pool_->lock(); 125 | 126 | for (bucket_index = 0; bucket_index < num_buckets_; bucket_index++) { 127 | Bucket* bucket = buckets_ + bucket_index; 128 | size_t item_index; 129 | for (item_index = 0; item_index < StaticConfig::kBucketSize; item_index++) { 130 | uint64_t item_vec = bucket->item_vec[item_index]; 131 | if (item_vec != 0) pool_->release(get_item_offset(item_vec)); 132 | } 133 | } 134 | pool_->unlock(); 135 | 136 | ::mica::util::memset(buckets_, 0, 137 | sizeof(Bucket) * (num_buckets_ + num_extra_buckets_)); 138 | 139 | // initialize a free list of extra buckets 140 | extra_bucket_free_list_.lock = 0; 141 | if (num_extra_buckets_ == 0) 142 | extra_bucket_free_list_.head = 0; // no extra bucket at all 143 | else { 144 | uint32_t extra_bucket_index; 145 | for (extra_bucket_index = 1; 146 | extra_bucket_index < 1 + num_extra_buckets_ - 1; extra_bucket_index++) 147 | (extra_buckets_ + extra_bucket_index)->next_extra_bucket_index = 148 | extra_bucket_index + 1; 149 | (extra_buckets_ + extra_bucket_index)->next_extra_bucket_index = 150 | 0; // no more free extra bucket 151 | 152 | extra_bucket_free_list_.head = 1; 153 | } 154 | 155 | pool_->lock(); 156 | pool_->reset(); 157 | pool_->unlock(); 158 | 159 | reset_stats(true); 160 | } 161 | } 162 | } 163 | 164 | #endif 165 | -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/item.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_ITEM_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_ITEM_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | uint32_t LTable::get_key_length(uint32_t kv_length_vec) { 9 | return kv_length_vec >> 24; 10 | } 11 | 12 | template 13 | uint32_t LTable::get_value_length(uint32_t kv_length_vec) { 14 | return kv_length_vec & Item::kValueMask; 15 | } 16 | 17 | template 18 | uint32_t LTable::make_kv_length_vec(uint32_t key_length, 19 | uint32_t value_length) { 20 | return (key_length << 24) | value_length; 21 | } 22 | 23 | template 24 | uint16_t LTable::calc_tag(uint64_t key_hash) { 25 | uint16_t tag = (uint16_t)(key_hash & Bucket::kTagMask); 26 | if (tag == 0) 27 | return 1; 28 | else 29 | return tag; 30 | } 31 | 32 | template 33 | void LTable::set_item(Item* item, uint64_t key_hash, 34 | const char* key, uint32_t key_length, 35 | const char* value, uint32_t value_length) { 36 | assert(key_length <= Item::kKeyMask); 37 | assert(value_length <= Item::kValueMask); 38 | 39 | item->kv_length_vec = make_kv_length_vec(key_length, value_length); 40 | item->key_hash = key_hash; 41 | ::mica::util::memcpy<8>(item->data, key, key_length); 42 | ::mica::util::memcpy<8>(item->data + ::mica::util::roundup<8>(key_length), 43 | value, value_length); 44 | } 45 | 46 | template 47 | void LTable::set_item_value(Item* item, const char* value, 48 | uint32_t value_length) { 49 | assert(value_length <= Item::kValueMask); 50 | 51 | uint32_t key_length = get_key_length(item->kv_length_vec); 52 | item->kv_length_vec = make_kv_length_vec(key_length, value_length); 53 | ::mica::util::memcpy<8>(item->data + ::mica::util::roundup<8>(key_length), 54 | value, value_length); 55 | } 56 | 57 | template 58 | bool LTable::compare_keys(const char* key1, size_t key1_len, 59 | const char* key2, size_t key2_len) { 60 | return key1_len == key2_len && 61 | ::mica::util::memcmp_equal<8>(key1, key2, 62 | ::mica::util::roundup<8>(key1_len)); 63 | } 64 | } 65 | } 66 | 67 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/lock.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_LOCK_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_LOCK_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void LTable::lock_bucket(Bucket* bucket) { 9 | if (StaticConfig::kConcurrent) { 10 | if (concurrent_access_mode_ == 1) { 11 | assert((*(volatile uint32_t*)&bucket->version & 1U) == 0U); 12 | (*(volatile uint32_t*)&bucket->version)++; 13 | ::mica::util::memory_barrier(); 14 | } else if (concurrent_access_mode_ == 2) { 15 | while (true) { 16 | uint32_t v = *(volatile uint32_t*)&bucket->version & ~1U; 17 | uint32_t new_v = v | 1U; 18 | if (__sync_bool_compare_and_swap((volatile uint32_t*)&bucket->version, 19 | v, new_v)) 20 | break; 21 | } 22 | } 23 | } 24 | } 25 | 26 | template 27 | void LTable::unlock_bucket(Bucket* bucket) { 28 | if (StaticConfig::kConcurrent) { 29 | if (concurrent_access_mode_ != 0) { 30 | ::mica::util::memory_barrier(); 31 | assert((*(volatile uint32_t*)&bucket->version & 1U) == 1U); 32 | // no need to use atomic add because this thread is the only one writing 33 | // to version 34 | (*(volatile uint32_t*)&bucket->version)++; 35 | } 36 | } 37 | } 38 | 39 | template 40 | void LTable::lock_extra_bucket_free_list() { 41 | if (StaticConfig::kConcurrent) { 42 | if (concurrent_access_mode_ == 2) { 43 | while (true) { 44 | if (__sync_bool_compare_and_swap( 45 | (volatile uint8_t*)&extra_bucket_free_list_.lock, 0U, 1U)) 46 | break; 47 | } 48 | } 49 | } 50 | } 51 | 52 | template 53 | void LTable::unlock_extra_bucket_free_list() { 54 | if (StaticConfig::kConcurrent) { 55 | if (concurrent_access_mode_ == 2) { 56 | ::mica::util::memory_barrier(); 57 | assert((*(volatile uint8_t*)&extra_bucket_free_list_.lock & 1U) == 1U); 58 | // no need to use atomic add because this thread is the only one writing 59 | // to version 60 | *(volatile uint8_t*)&extra_bucket_free_list_.lock = 0U; 61 | } 62 | } 63 | } 64 | 65 | template 66 | uint32_t LTable::read_version_begin(const Bucket* bucket) const { 67 | if (StaticConfig::kConcurrent) { 68 | if (concurrent_access_mode_ != 0) { 69 | while (true) { 70 | uint32_t v = *(volatile uint32_t*)&bucket->version; 71 | ::mica::util::memory_barrier(); 72 | if ((v & 1U) != 0U) continue; 73 | return v; 74 | } 75 | } else 76 | return 0U; 77 | } else { 78 | return 0U; 79 | } 80 | } 81 | 82 | template 83 | uint32_t LTable::read_version_end(const Bucket* bucket) const { 84 | if (StaticConfig::kConcurrent) { 85 | if (concurrent_access_mode_ != 0) { 86 | ::mica::util::memory_barrier(); 87 | uint32_t v = *(volatile uint32_t*)&bucket->version; 88 | return v; 89 | } else 90 | return 0U; 91 | } else { 92 | return 0U; 93 | } 94 | } 95 | } 96 | } 97 | 98 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/move_to_head.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_MOVE_TO_HEAD_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_MOVE_TO_HEAD_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void LTable::move_to_head(Bucket* bucket, Bucket* located_bucket, 9 | const Item* item, size_t key_length, 10 | size_t value_length, size_t item_index, 11 | uint64_t item_vec, 12 | uint64_t item_offset) { 13 | if (!std::is_base_of<::mica::pool::CircularLogTag, typename Pool::Tag>::value) 14 | return; 15 | 16 | // for move-to-head; 17 | // this does not use Pool::get_item()'s item_size but recalculates a new 18 | // item_size to discard currently unused space within the pool for this 19 | // item. 20 | uint32_t item_size = static_cast( 21 | sizeof(Item) + ::mica::util::roundup<8>(key_length) + 22 | ::mica::util::roundup<8>(value_length)); 23 | 24 | // small distance_from_tail = recently appended at tail 25 | // it is not required to acquire a lock to read pool_->tail 26 | // because 27 | // some inaccurate number is OK 28 | // note that we are accessing pool_->tail, which uses a 64-bit 29 | // number; this is OK because we apply pool_->.mask mask 30 | // perform move-to-head within the same pool only 31 | 32 | uint64_t distance_from_tail = 33 | (Specialization::get_tail(pool_) - item_offset) & 34 | Specialization::get_mask(pool_); 35 | if (distance_from_tail > mth_threshold_) { 36 | lock_bucket(bucket); 37 | pool_->lock(); 38 | 39 | // check if the original item is still there 40 | if (located_bucket->item_vec[item_index] == item_vec) { 41 | // allocate new space 42 | uint64_t new_item_offset = pool_->allocate(item_size); 43 | uint64_t new_tail = Specialization::get_tail(pool_); 44 | 45 | // see if the original item is still valid because 46 | // pool::allocate() by this thread or other threads may 47 | // have invalidated it 48 | if (Specialization::is_valid(pool_, item_offset)) { 49 | Item* new_item = 50 | reinterpret_cast(pool_->get_item(new_item_offset)); 51 | ::mica::util::memcpy<8>(reinterpret_cast(new_item), 52 | reinterpret_cast(item), item_size); 53 | 54 | located_bucket->item_vec[item_index] = 55 | make_item_vec(get_tag(item_vec), new_item_offset); 56 | 57 | // success 58 | stat_inc(&Stats::move_to_head_performed); 59 | } else { 60 | // failed -- original data become invalid in the pool 61 | stat_inc(&Stats::move_to_head_failed); 62 | } 63 | 64 | // we need to hold the lock until we finish writing 65 | pool_->unlock(); 66 | unlock_bucket(bucket); 67 | 68 | // XXX: this may be too late; before cleaning, other threads may 69 | // have read some invalid location 70 | // ideally, this should be done before writing actual data 71 | cleanup_bucket(new_item_offset, new_tail); 72 | } else { 73 | pool_->unlock(); 74 | unlock_bucket(bucket); 75 | 76 | // failed -- original data become invalid in the table 77 | stat_inc(&Stats::move_to_head_failed); 78 | } 79 | } else { 80 | stat_inc(&Stats::move_to_head_skipped); 81 | } 82 | } 83 | } 84 | } 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/prefetch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_PREFETCH_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_PREFETCH_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | void LTable::prefetch_table(uint64_t key_hash) const { 9 | uint32_t bucket_index = calc_bucket_index(key_hash); 10 | const Bucket* bucket = buckets_ + bucket_index; 11 | 12 | // bucket address is already 64-byte aligned 13 | __builtin_prefetch(bucket, 0, 0); 14 | 15 | if (StaticConfig::kBucketSize > 7) 16 | __builtin_prefetch(reinterpret_cast(bucket) + 64, 0, 0); 17 | 18 | // XXX: prefetch extra buckets, too? 19 | } 20 | 21 | template 22 | void LTable::prefetch_pool(uint64_t key_hash) const { 23 | uint32_t bucket_index = calc_bucket_index(key_hash); 24 | const Bucket* bucket = buckets_ + bucket_index; 25 | 26 | uint16_t tag = calc_tag(key_hash); 27 | 28 | size_t item_index; 29 | for (item_index = 0; item_index < StaticConfig::kBucketSize; item_index++) { 30 | uint64_t item_vec = bucket->item_vec[item_index]; 31 | if (get_tag(item_vec) != tag) continue; 32 | 33 | pool_->prefetch_item(get_item_offset(item_vec)); 34 | } 35 | } 36 | } 37 | } 38 | 39 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/set.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_SET_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_SET_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::set(uint64_t key_hash, const char* key, 9 | size_t key_length, const char* value, 10 | size_t value_length, bool overwrite) { 11 | assert(key_length <= kMaxKeyLength); 12 | assert(value_length <= kMaxValueLength); 13 | 14 | uint32_t bucket_index = calc_bucket_index(key_hash); 15 | uint16_t tag = calc_tag(key_hash); 16 | 17 | Bucket* bucket = buckets_ + bucket_index; 18 | 19 | bool overwriting; 20 | 21 | lock_bucket(bucket); 22 | 23 | Bucket* located_bucket; 24 | size_t item_index = 25 | find_item_index(bucket, key_hash, tag, key, key_length, &located_bucket); 26 | 27 | if (item_index != StaticConfig::kBucketSize) { 28 | if (!overwrite) { 29 | stat_inc(&Stats::set_nooverwrite); 30 | 31 | unlock_bucket(bucket); 32 | return Result::kExists; // already exist but cannot overwrite 33 | } else { 34 | overwriting = true; 35 | } 36 | } else { 37 | if (StaticConfig::kEviction && std::is_base_of<::mica::pool::CircularLogTag, 38 | typename Pool::Tag>::value) { 39 | // pick an item with the same tag; this is potentially the same item but 40 | // invalid due to insufficient log space 41 | // this helps limiting the effect of "ghost" items in the table when the 42 | // log space is slightly smaller than enough 43 | // and makes better use of the log space by not evicting unrelated old 44 | // items in the same bucket 45 | item_index = find_same_tag(bucket, tag, &located_bucket); 46 | 47 | if (item_index == StaticConfig::kBucketSize) { 48 | // if there is no matching tag, we just use the empty or oldest item 49 | item_index = get_empty_or_oldest(bucket, &located_bucket); 50 | } 51 | } else { 52 | item_index = get_empty(bucket, &located_bucket); 53 | if (item_index == StaticConfig::kBucketSize) { 54 | // no more space 55 | // TODO: add a statistics entry 56 | unlock_bucket(bucket); 57 | return Result::kInsufficientSpace; 58 | } 59 | } 60 | 61 | overwriting = false; 62 | } 63 | 64 | uint32_t new_item_size = static_cast( 65 | sizeof(Item) + ::mica::util::roundup<8>(key_length) + 66 | ::mica::util::roundup<8>(value_length)); 67 | uint64_t item_offset = 0; 68 | 69 | // we have to lock the pool because is_valid check must be correct in the 70 | // overwrite mode; 71 | // unlike reading, we cannot afford writing data at a wrong place 72 | if (std::is_base_of<::mica::pool::CircularLogTag, typename Pool::Tag>::value) 73 | pool_->lock(); 74 | 75 | if (overwriting) { 76 | item_offset = get_item_offset(located_bucket->item_vec[item_index]); 77 | 78 | size_t old_item_size; 79 | Item* item = 80 | reinterpret_cast(pool_->get_item(item_offset, &old_item_size)); 81 | 82 | if (Specialization::is_valid(pool_, item_offset)) { 83 | if (old_item_size >= new_item_size) { 84 | stat_inc(&Stats::set_inplace); 85 | 86 | set_item_value(item, value, (uint32_t)value_length); 87 | 88 | if (std::is_base_of<::mica::pool::CircularLogTag, 89 | typename Pool::Tag>::value) 90 | pool_->unlock(); 91 | 92 | unlock_bucket(bucket); 93 | 94 | return Result::kSuccess; 95 | } 96 | } 97 | } 98 | 99 | uint64_t new_item_offset = pool_->allocate(new_item_size); 100 | if (new_item_offset == Pool::kInsufficientSpace) { 101 | // no more space 102 | // TODO: add a statistics entry 103 | unlock_bucket(bucket); 104 | return Result::kInsufficientSpace; 105 | } 106 | uint64_t new_tail; 107 | if (std::is_base_of<::mica::pool::CircularLogTag, typename Pool::Tag>::value) 108 | new_tail = Specialization::get_tail(pool_); 109 | Item* new_item = reinterpret_cast(pool_->get_item(new_item_offset)); 110 | 111 | stat_inc(&Stats::set_new); 112 | 113 | set_item(new_item, key_hash, key, (uint32_t)key_length, value, 114 | (uint32_t)value_length); 115 | 116 | // unlocking is delayed until we finish writing data at the new location; 117 | // otherwise, the new location may be invalidated (in a very rare case) 118 | if (std::is_base_of<::mica::pool::CircularLogTag, typename Pool::Tag>::value) 119 | pool_->unlock(); 120 | 121 | if (located_bucket->item_vec[item_index] != 0) { 122 | stat_inc(&Stats::set_evicted); 123 | stat_dec(&Stats::count); 124 | } 125 | 126 | located_bucket->item_vec[item_index] = make_item_vec(tag, new_item_offset); 127 | 128 | unlock_bucket(bucket); 129 | 130 | if (std::is_base_of<::mica::pool::CircularLogTag, 131 | typename Pool::Tag>::value) { 132 | // XXX: this may be too late; before cleaning, other threads may have read 133 | // some invalid location 134 | // ideally, this should be done before writing actual data 135 | cleanup_bucket(new_item_offset, new_tail); 136 | 137 | // this is done after bucket is updated and unlocked 138 | if (overwriting) pool_->release(item_offset); 139 | } else // if (std::is_base_of<::mica::pool::SegregatedFitTag, 140 | // Pool::Tag>::value) 141 | { 142 | // this is done after bucket is updated and unlocked 143 | if (overwriting) { 144 | pool_->lock(); 145 | pool_->release(item_offset); 146 | pool_->unlock(); 147 | } 148 | } 149 | 150 | stat_inc(&Stats::count); 151 | 152 | return Result::kSuccess; 153 | } 154 | } 155 | } 156 | 157 | #endif -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/specialization.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_SPECIALIZATION_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_SPECIALIZATION_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | class LTablePoolSpecialization { 9 | public: 10 | template 11 | static uint64_t get_tail(const Pool* pool); 12 | 13 | template 14 | static uint64_t get_mask(const Pool* pool); 15 | 16 | template 17 | static uint64_t get_size(const Pool* pool); 18 | 19 | template 20 | static bool is_valid(const Pool* pool, typename Pool::Offset offset); 21 | }; 22 | 23 | template 24 | template 25 | uint64_t LTablePoolSpecialization::get_tail(const Pool* pool) { 26 | (void)pool; 27 | return 0; 28 | } 29 | 30 | template <> 31 | template 32 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_tail( 33 | const Pool* pool) { 34 | return pool->get_tail(); 35 | } 36 | 37 | template 38 | template 39 | uint64_t LTablePoolSpecialization::get_mask(const Pool* pool) { 40 | (void)pool; 41 | return 0; 42 | } 43 | 44 | template <> 45 | template 46 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_mask( 47 | const Pool* pool) { 48 | return pool->get_mask(); 49 | } 50 | 51 | template 52 | template 53 | uint64_t LTablePoolSpecialization::get_size(const Pool* pool) { 54 | (void)pool; 55 | return 0; 56 | } 57 | 58 | template <> 59 | template 60 | uint64_t LTablePoolSpecialization<::mica::pool::CircularLogTag>::get_size( 61 | const Pool* pool) { 62 | return pool->get_size(); 63 | } 64 | 65 | template 66 | template 67 | bool LTablePoolSpecialization::is_valid(const Pool* pool, 68 | typename Pool::Offset offset) { 69 | (void)pool; 70 | (void)offset; 71 | return true; 72 | } 73 | 74 | template <> 75 | template 76 | bool LTablePoolSpecialization<::mica::pool::CircularLogTag>::is_valid( 77 | const Pool* pool, typename Pool::Offset offset) { 78 | return pool->is_valid(offset); 79 | } 80 | } 81 | } 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /src/mica/table/ltable_impl/test.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_LTABLE_IMPL_TEST_H_ 3 | #define MICA_TABLE_LTABLE_IMPL_TEST_H_ 4 | 5 | namespace mica { 6 | namespace table { 7 | template 8 | Result LTable::test(uint64_t key_hash, const char* key, 9 | size_t key_length) const { 10 | assert(key_length <= kMaxKeyLength); 11 | 12 | uint32_t bucket_index = calc_bucket_index(key_hash); 13 | uint16_t tag = calc_tag(key_hash); 14 | 15 | const Bucket* bucket = buckets_ + bucket_index; 16 | 17 | while (true) { 18 | uint32_t version_start = read_version_begin(bucket); 19 | 20 | const Bucket* located_bucket; 21 | size_t item_index = find_item_index(bucket, key_hash, tag, key, key_length, 22 | &located_bucket); 23 | 24 | if (version_start != read_version_end(bucket)) continue; 25 | 26 | if (item_index != StaticConfig::kBucketSize) { 27 | stat_inc(&Stats::test_found); 28 | return Result::kSuccess; 29 | } else { 30 | stat_inc(&Stats::test_notfound); 31 | return Result::kNotFound; 32 | } 33 | } 34 | // Not reachable. 35 | } 36 | } 37 | } 38 | 39 | #endif -------------------------------------------------------------------------------- /src/mica/table/table.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_TABLE_H_ 3 | #define MICA_TABLE_TABLE_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/table/types.h" 7 | 8 | namespace mica { 9 | namespace table { 10 | class TableInterface { 11 | public: 12 | void reset(); 13 | 14 | Result del(uint64_t key_hash, const char* key, size_t key_length); 15 | 16 | Result get(uint64_t key_hash, const char* key, size_t key_length, 17 | char* out_value, size_t in_value_length, size_t* out_value_length, 18 | bool allow_mutation) const; 19 | 20 | Result increment(uint64_t key_hash, const char* key, size_t key_length, 21 | uint64_t increment, uint64_t* out_value); 22 | 23 | Result set(uint64_t key_hash, const char* key, size_t key_length, 24 | const char* value, size_t value_length, bool overwrite); 25 | 26 | Result test(uint64_t key_hash, const char* key, size_t key_length) const; 27 | 28 | void prefetch_table(uint64_t key_hash) const; 29 | void prefetch_pool(uint64_t key_hash) const; 30 | 31 | void print_buckets() const; 32 | void print_stats() const; 33 | void reset_stats(bool reset_count); 34 | }; 35 | } 36 | } 37 | 38 | #endif -------------------------------------------------------------------------------- /src/mica/table/types.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_TABLE_TYPES_H_ 3 | #define MICA_TABLE_TYPES_H_ 4 | 5 | #include "mica/common.h" 6 | 7 | namespace mica { 8 | namespace table { 9 | enum class Result { 10 | kSuccess = 0, 11 | kError, 12 | kInsufficientSpace, 13 | kExists, 14 | kNotFound, 15 | kPartialValue, 16 | kNotProcessed, 17 | kNotSupported, 18 | kTimedOut, 19 | kRejected, 20 | }; 21 | } 22 | } 23 | 24 | #endif -------------------------------------------------------------------------------- /src/mica/test/cdf.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "mica/util/latency.h" 7 | 8 | typedef uint16_t binary_time_t; 9 | 10 | int main(int argc, const char* argv[]) { 11 | FILE* in_file = stdin; 12 | FILE* out_file = stdout; 13 | FILE* binary_file = nullptr; 14 | size_t skip = 0; 15 | size_t n = static_cast(-1); 16 | 17 | bool create_binary = false; 18 | 19 | int c; 20 | opterr = 0; 21 | while ((c = getopt(argc, const_cast(argv), "i:o:b:s:n:")) != -1) { 22 | switch (c) { 23 | case 'i': 24 | in_file = fopen(optarg, "r"); 25 | if (in_file == nullptr) { 26 | perror("fopen"); 27 | return EXIT_FAILURE; 28 | } 29 | break; 30 | case 'o': 31 | out_file = fopen(optarg, "w"); 32 | if (out_file == nullptr) { 33 | perror("fopen"); 34 | return EXIT_FAILURE; 35 | } 36 | break; 37 | case 'b': 38 | binary_file = fopen(optarg, "rb"); 39 | if (binary_file == nullptr) { 40 | binary_file = fopen(optarg, "w+b"); 41 | if (binary_file == nullptr) { 42 | perror("fopen"); 43 | return EXIT_FAILURE; 44 | } 45 | create_binary = true; 46 | } 47 | break; 48 | case 's': 49 | skip = static_cast(atol(optarg)); 50 | break; 51 | case 'n': 52 | n = static_cast(atol(optarg)); 53 | break; 54 | case '?': 55 | if (isprint(optopt)) 56 | fprintf(stderr, "incomplete option -%c\n", optopt); 57 | else 58 | fprintf(stderr, "error parsing arguments\n"); 59 | return EXIT_FAILURE; 60 | } 61 | } 62 | 63 | if (optind == argc) { 64 | printf( 65 | "%s [-i IN-FILE] [-o OUT-FILE] [-b BINARY-IN-OUT-FILE] [-s SKIP] [-n " 66 | "COUNT] " 67 | "(min|max|avg|cnt|0.5|0.99)...\n", 68 | argv[0]); 69 | return EXIT_FAILURE; 70 | } 71 | 72 | ::mica::util::Latency lat; 73 | 74 | if (create_binary) { 75 | char buf[64]; 76 | while (true) { 77 | if (fgets(buf, sizeof(buf), in_file) == nullptr) break; 78 | 79 | auto raw_l = atol(buf); 80 | auto l = static_cast(raw_l); 81 | if (static_cast(l) != raw_l) { 82 | fprintf(stderr, "error: binary_time_t overflew!\n"); 83 | return EXIT_FAILURE; 84 | } 85 | while (true) { 86 | size_t wrote = fwrite(&l, sizeof(l), 1, binary_file); 87 | if (wrote == 1) break; 88 | } 89 | } 90 | fflush(binary_file); 91 | } 92 | 93 | if (binary_file == nullptr) { 94 | char buf[64]; 95 | while (true) { 96 | if (fgets(buf, sizeof(buf), in_file) == nullptr) break; 97 | if (skip > 0) { 98 | skip--; 99 | continue; 100 | } 101 | if (--n == 0) break; 102 | if (buf[0] == '\n') continue; 103 | 104 | auto l = static_cast(atol(buf)); 105 | lat.update(l); 106 | } 107 | } else { 108 | int ret = fseek(binary_file, 0, SEEK_END); 109 | if (ret == EBADF || ret == EINVAL) { 110 | perror("fseek"); 111 | return EXIT_FAILURE; 112 | } 113 | long pos = ftell(binary_file); 114 | if (pos == EBADF || pos == EINVAL) { 115 | perror("ftell"); 116 | return EXIT_FAILURE; 117 | } 118 | size_t file_size = static_cast(pos); 119 | size_t max_n = file_size / sizeof(binary_time_t); 120 | 121 | auto p = mmap(nullptr, file_size, PROT_READ, MAP_PRIVATE, 122 | fileno(binary_file), 0); 123 | if (p == MAP_FAILED) { 124 | perror("mmap"); 125 | return EXIT_FAILURE; 126 | } 127 | 128 | size_t i = skip; 129 | while (i < max_n) { 130 | if (--n == 0) break; 131 | 132 | auto l = 133 | static_cast(reinterpret_cast(p)[i]); 134 | lat.update(l); 135 | i++; 136 | } 137 | 138 | munmap(p, file_size); 139 | } 140 | 141 | for (auto i = optind; i < argc; i++) { 142 | const char* arg = argv[i]; 143 | if (strcasecmp(arg, "min") == 0) 144 | fprintf(out_file, "%s: %" PRIu64 "\n", arg, lat.min()); 145 | else if (strcasecmp(arg, "max") == 0) 146 | fprintf(out_file, "%s: %" PRIu64 "\n", arg, lat.max()); 147 | else if (strcasecmp(arg, "avg") == 0) 148 | fprintf(out_file, "%s: %lf\n", arg, lat.avg_f()); 149 | else if (strcasecmp(arg, "cnt") == 0) 150 | fprintf(out_file, "%s: %" PRIu64 "\n", arg, lat.count()); 151 | else 152 | fprintf(out_file, "%s: %" PRIu64 "\n", arg, lat.perc(atof(arg))); 153 | } 154 | 155 | fflush(out_file); 156 | return EXIT_SUCCESS; 157 | } 158 | -------------------------------------------------------------------------------- /src/mica/test/microbench.json: -------------------------------------------------------------------------------- 1 | { 2 | "alloc": { 3 | /*"clean_files_on_init": true, 4 | "verbose": true*/ 5 | }, 6 | 7 | "processor": { 8 | "lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 9 | "partition_count": 16, 10 | 11 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], 12 | "partition_count": 20,*/ 13 | 14 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], 15 | "partition_count": 28,*/ 16 | 17 | "total_size": 922746880, 18 | "total_item_count": 31457280, 19 | 20 | "concurrent_read": false, 21 | /*"concurrent_read": true,*/ 22 | "concurrent_write": false, 23 | /*"concurrent_write": true,*/ 24 | 25 | "stage_gap": 2 26 | /*"target_stage_gap_time": 300*/ 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/mica/test/netbench.cc: -------------------------------------------------------------------------------- 1 | #include "mica/datagram/datagram_client.h" 2 | #include "mica/util/lcore.h" 3 | #include "mica/util/hash.h" 4 | #include "mica/util/zipf.h" 5 | 6 | typedef ::mica::alloc::HugeTLBFS_SHM Alloc; 7 | 8 | struct DPDKConfig : public ::mica::network::BasicDPDKConfig { 9 | static constexpr bool kVerbose = true; 10 | }; 11 | 12 | struct DatagramClientConfig 13 | : public ::mica::datagram::BasicDatagramClientConfig { 14 | typedef ::mica::network::DPDK Network; 15 | // static constexpr bool kSkipRX = true; 16 | // static constexpr bool kIgnoreServerPartition = true; 17 | // static constexpr bool kVerbose = true; 18 | }; 19 | 20 | typedef ::mica::datagram::DatagramClient Client; 21 | 22 | typedef ::mica::table::Result Result; 23 | 24 | template 25 | static uint64_t hash(const T* key, size_t key_length) { 26 | return ::mica::util::hash(key, key_length); 27 | } 28 | 29 | class ResponseHandler 30 | : public ::mica::datagram::ResponseHandlerInterface { 31 | public: 32 | void handle(Client::RequestDescriptor rd, Result result, const char* value, 33 | size_t value_length, const Argument& arg) { 34 | (void)rd; 35 | (void)result; 36 | (void)value; 37 | (void)value_length; 38 | (void)arg; 39 | } 40 | }; 41 | 42 | struct Args { 43 | uint16_t lcore_id; 44 | ::mica::util::Config* config; 45 | Alloc* alloc; 46 | Client* client; 47 | double zipf_theta; 48 | } __attribute__((aligned(128))); 49 | 50 | int worker_proc(void* arg) { 51 | auto args = reinterpret_cast(arg); 52 | 53 | Client& client = *args->client; 54 | 55 | ::mica::util::lcore.pin_thread(args->lcore_id); 56 | 57 | printf("worker running on lcore %" PRIu16 "\n", args->lcore_id); 58 | 59 | client.probe_reachability(); 60 | 61 | ResponseHandler rh; 62 | 63 | size_t num_items = 192 * 1048576; 64 | 65 | // double get_ratio = 0.95; 66 | double get_ratio = 0.50; 67 | 68 | uint32_t get_threshold = (uint32_t)(get_ratio * (double)((uint32_t)-1)); 69 | 70 | ::mica::util::Rand op_type_rand(static_cast(args->lcore_id) + 1000); 71 | ::mica::util::ZipfGen zg(num_items, args->zipf_theta, 72 | static_cast(args->lcore_id)); 73 | ::mica::util::Stopwatch sw; 74 | sw.init_start(); 75 | sw.init_end(); 76 | 77 | uint64_t key_i; 78 | uint64_t key_hash; 79 | size_t key_length = sizeof(key_i); 80 | char* key = reinterpret_cast(&key_i); 81 | 82 | uint64_t value_i; 83 | size_t value_length = sizeof(value_i); 84 | char* value = reinterpret_cast(&value_i); 85 | 86 | bool use_noop = false; 87 | // bool use_noop = true; 88 | 89 | uint64_t last_handle_response_time = sw.now(); 90 | // Check the response after sending some requests. 91 | // Ideally, packets per batch for both RX and TX should be similar. 92 | uint64_t response_check_interval = 20 * sw.c_1_usec(); 93 | 94 | uint64_t seq = 0; 95 | while (true) { 96 | // Determine the operation type. 97 | uint32_t op_r = op_type_rand.next_u32(); 98 | bool is_get = op_r <= get_threshold; 99 | 100 | // Generate the key. 101 | key_i = zg.next(); 102 | key_hash = hash(key, key_length); 103 | 104 | uint64_t now = sw.now(); 105 | while (!client.can_request(key_hash) || 106 | sw.diff_in_cycles(now, last_handle_response_time) >= 107 | response_check_interval) { 108 | last_handle_response_time = now; 109 | client.handle_response(rh); 110 | } 111 | 112 | if (!use_noop) { 113 | if (is_get) 114 | client.get(key_hash, key, key_length); 115 | else { 116 | value_i = seq; 117 | client.set(key_hash, key, key_length, value, value_length, true); 118 | } 119 | } else { 120 | if (is_get) 121 | client.noop_read(key_hash, key, key_length); 122 | else { 123 | value_i = seq; 124 | client.noop_write(key_hash, key, key_length, value, value_length); 125 | } 126 | } 127 | 128 | seq++; 129 | } 130 | 131 | return 0; 132 | } 133 | 134 | int main(int argc, const char* argv[]) { 135 | if (argc != 2) { 136 | printf("%s ZIPF-THETA\n", argv[0]); 137 | return EXIT_FAILURE; 138 | } 139 | 140 | double zipf_theta = atof(argv[1]); 141 | 142 | ::mica::util::lcore.pin_thread(0); 143 | 144 | auto config = ::mica::util::Config::load_file("netbench.json"); 145 | 146 | Alloc alloc(config.get("alloc")); 147 | 148 | DatagramClientConfig::Network network(config.get("network")); 149 | network.start(); 150 | 151 | Client::DirectoryClient dir_client(config.get("dir_client")); 152 | 153 | Client client(config.get("client"), &network, &dir_client); 154 | client.discover_servers(); 155 | 156 | uint16_t lcore_count = 157 | static_cast(::mica::util::lcore.lcore_count()); 158 | 159 | std::vector args(lcore_count); 160 | for (uint16_t lcore_id = 0; lcore_id < lcore_count; lcore_id++) { 161 | args[lcore_id].lcore_id = lcore_id; 162 | args[lcore_id].config = &config; 163 | args[lcore_id].alloc = &alloc; 164 | args[lcore_id].client = &client; 165 | args[lcore_id].zipf_theta = zipf_theta; 166 | } 167 | 168 | for (uint16_t lcore_id = 1; lcore_id < lcore_count; lcore_id++) { 169 | if (!rte_lcore_is_enabled(static_cast(lcore_id))) continue; 170 | rte_eal_remote_launch(worker_proc, &args[lcore_id], lcore_id); 171 | } 172 | worker_proc(&args[0]); 173 | 174 | network.stop(); 175 | 176 | return EXIT_SUCCESS; 177 | } 178 | -------------------------------------------------------------------------------- /src/mica/test/netbench.json: -------------------------------------------------------------------------------- 1 | { 2 | "dir_client": { 3 | "etcd_addr": "YOUR_ETCD_HOST", 4 | "etcd_port": 50505 5 | /*"verbose": true*/ 6 | }, 7 | 8 | "alloc": { 9 | "num_pages_to_free": [500, 500] 10 | /*"verbose": true*/ 11 | }, 12 | 13 | "network": { 14 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],*/ 15 | /*"lcores": [0, 2],*/ 16 | "lcores": [0], 17 | 18 | "ports": [ 19 | {"port_id": 0, "ipv4_addr": "10.0.1.1"} 20 | /*{"port_id": 1, "ipv4_addr": "10.0.1.2"}, 21 | {"port_id": 2, "ipv4_addr": "10.0.1.3"}, 22 | {"port_id": 3, "ipv4_addr": "10.0.1.4"}, 23 | {"port_id": 4, "ipv4_addr": "10.0.1.5"}, 24 | {"port_id": 5, "ipv4_addr": "10.0.1.6"}, 25 | {"port_id": 6, "ipv4_addr": "10.0.1.7"}, 26 | {"port_id": 7, "ipv4_addr": "10.0.1.8"}*/ 27 | ], 28 | 29 | "endpoints": [ 30 | /*[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [0, 5], [0, 6], [0, 7], 31 | [1, 0], [1, 1], [1, 2], [1, 3], [1, 4], [1, 5], [1, 6], [1, 7], 32 | [2, 0], [2, 1], [2, 2], [2, 3], [2, 4], [2, 5], [2, 6], [2, 7], 33 | [3, 0], [3, 1], [3, 2], [3, 3], [3, 4], [3, 5], [3, 6], [3, 7], 34 | [4, 0], [4, 1], [4, 2], [4, 3], [4, 4], [4, 5], [4, 6], [4, 7], 35 | [5, 0], [5, 1], [5, 2], [5, 3], [5, 4], [5, 5], [5, 6], [5, 7], 36 | [6, 0], [6, 1], [6, 2], [6, 3], [6, 4], [6, 5], [6, 6], [6, 7], 37 | [7, 0], [7, 1], [7, 2], [7, 3], [7, 4], [7, 5], [7, 6], [7, 7], 38 | [8, 0], [8, 1], [8, 2], [8, 3], [8, 4], [8, 5], [8, 6], [8, 7], 39 | [9, 0], [9, 1], [9, 2], [9, 3], [9, 4], [9, 5], [9, 6], [9, 7], 40 | [10, 0], [10, 1], [10, 2], [10, 3], [10, 4], [10, 5], [10, 6], [10, 7], 41 | [11, 0], [11, 1], [11, 2], [11, 3], [11, 4], [11, 5], [11, 6], [11, 7], 42 | [12, 0], [12, 1], [12, 2], [12, 3], [12, 4], [12, 5], [12, 6], [12, 7], 43 | [13, 0], [13, 1], [13, 2], [13, 3], [13, 4], [13, 5], [13, 6], [13, 7], 44 | [14, 0], [14, 1], [14, 2], [14, 3], [14, 4], [14, 5], [14, 6], [14, 7], 45 | [15, 0], [15, 1], [15, 2], [15, 3], [15, 4], [15, 5], [15, 6], [15, 7]*/ 46 | 47 | /*[0, 0], [0, 1], [0, 2], [0, 3], [0, 4], [0, 5], [0, 6], [0, 7], 48 | [2, 0], [2, 1], [2, 2], [2, 3], [2, 4], [2, 5], [2, 6], [2, 7]*/ 49 | 50 | /*[0, 0], [2, 0]*/ 51 | [0, 0] 52 | ], 53 | 54 | /*"dpdk_args": ["-n", "4", "--socket-mem=2048,2048"]*/ 55 | "dpdk_args": ["-n", "4", "--socket-mem=1000,1000"] 56 | }, 57 | 58 | "client": {} 59 | } 60 | -------------------------------------------------------------------------------- /src/mica/test/server.cc: -------------------------------------------------------------------------------- 1 | #include "mica/datagram/datagram_server.h" 2 | #include "mica/util/lcore.h" 3 | 4 | struct DPDKConfig : public ::mica::network::BasicDPDKConfig { 5 | static constexpr bool kVerbose = true; 6 | }; 7 | 8 | struct PartitionsConfig : public ::mica::processor::BasicPartitionsConfig { 9 | static constexpr bool kSkipPrefetchingForRecentKeyHashes = false; 10 | // static constexpr bool kVerbose = true; 11 | }; 12 | 13 | struct DatagramServerConfig 14 | : public ::mica::datagram::BasicDatagramServerConfig { 15 | typedef ::mica::processor::Partitions Processor; 16 | typedef ::mica::network::DPDK Network; 17 | // static constexpr bool kVerbose = true; 18 | }; 19 | 20 | typedef ::mica::datagram::DatagramServer Server; 21 | 22 | int main() { 23 | ::mica::util::lcore.pin_thread(0); 24 | 25 | auto config = ::mica::util::Config::load_file("server.json"); 26 | 27 | Server::DirectoryClient dir_client(config.get("dir_client")); 28 | 29 | DatagramServerConfig::Processor::Alloc alloc(config.get("alloc")); 30 | DatagramServerConfig::Processor processor(config.get("processor"), &alloc); 31 | 32 | DatagramServerConfig::Network network(config.get("network")); 33 | network.start(); 34 | 35 | Server server(config.get("server"), &processor, &network, &dir_client); 36 | server.run(); 37 | 38 | network.stop(); 39 | 40 | return EXIT_SUCCESS; 41 | } 42 | -------------------------------------------------------------------------------- /src/mica/test/server.json: -------------------------------------------------------------------------------- 1 | { 2 | "dir_client": { 3 | "etcd_addr": "YOUR_ETCD_HOST", 4 | "etcd_port": 50505 5 | /*"verbose": true*/ 6 | }, 7 | 8 | "alloc": { 9 | /*"num_pages_to_free": [1024, 1024]*/ 10 | /*"num_pages_to_free": [1024]*/ 11 | "num_pages_to_free": [2048] 12 | /*"verbose": true*/ 13 | }, 14 | "processor": { 15 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27], 16 | "partition_count": 56,*/ 17 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23], 18 | "partition_count": 48,*/ 19 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], 20 | "partition_count": 40,*/ 21 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 22 | "partition_count": 32,*/ 23 | /*"lcores": [0, 2], 24 | "partition_count": 4,*/ 25 | "lcores": [0, 1], 26 | "partition_count": 2, 27 | 28 | "total_size": 12884901888, /* 12 GiB */ 29 | "total_item_count": 201326592, /* 192 Mi */ 30 | 31 | "concurrent_read": false, 32 | /*"concurrent_read": true,*/ 33 | "concurrent_write": false 34 | /*"concurrent_write": true,*/ 35 | 36 | /*"stage_gap": 2*/ 37 | }, 38 | 39 | "network": { 40 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27],*/ 41 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],*/ 42 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19],*/ 43 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],*/ 44 | /*"lcores": [0, 2],*/ 45 | "lcores": [0, 1], 46 | 47 | "ports": [ 48 | {"port_id": 0, "ipv4_addr": "10.0.0.1"} 49 | /*{"port_id": 1, "ipv4_addr": "10.0.0.2"}, 50 | {"port_id": 2, "ipv4_addr": "10.0.0.3"}, 51 | {"port_id": 3, "ipv4_addr": "10.0.0.4"}, 52 | {"port_id": 4, "ipv4_addr": "10.0.0.5"}, 53 | {"port_id": 5, "ipv4_addr": "10.0.0.6"}, 54 | {"port_id": 6, "ipv4_addr": "10.0.0.7"}, 55 | {"port_id": 7, "ipv4_addr": "10.0.0.8"}*/ 56 | ], 57 | 58 | "endpoints": [ 59 | [0, 0], 60 | [1, 0] 61 | ], 62 | 63 | /* 64 | "endpoints": [ 65 | [0, 0], 66 | [1, 0], 67 | [2, 0], 68 | [3, 1], 69 | [4, 1], 70 | [5, 1], 71 | [6, 2], 72 | [7, 2], 73 | [8, 2], 74 | [9, 3], 75 | [10, 3], 76 | [11, 3], 77 | [12, 4], 78 | [13, 4], 79 | [14, 4], 80 | [15, 5], 81 | [16, 5], 82 | [17, 5], 83 | [18, 6], 84 | [19, 6], 85 | [20, 6], 86 | [21, 7], 87 | [22, 7], 88 | [23, 7] 89 | ] 90 | */ 91 | 92 | /*"dpdk_args": ["-n", "4", "--socket-mem=2048,2048"]*/ 93 | "dpdk_args": ["-n", "4", "--socket-mem=2048"] 94 | }, 95 | 96 | "server": { 97 | "rebalance_interval": 0 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /src/mica/test/server_sqlite.cc: -------------------------------------------------------------------------------- 1 | #include "mica/datagram/datagram_server.h" 2 | #include "mica/processor/simple_processor.h" 3 | #include "mica/table/sqlite.h" 4 | #include "mica/util/lcore.h" 5 | 6 | struct DPDKConfig : public ::mica::network::BasicDPDKConfig { 7 | static constexpr bool kVerbose = true; 8 | }; 9 | 10 | struct SimpleProcessorConfig 11 | : public ::mica::processor::BasicSimpleProcessorConfig { 12 | // static constexpr bool kVerbose = true; 13 | typedef ::mica::table::Sqlite<> Table; 14 | }; 15 | 16 | struct DatagramServerConfig 17 | : public ::mica::datagram::BasicDatagramServerConfig { 18 | typedef ::mica::processor::SimpleProcessor Processor; 19 | typedef ::mica::network::DPDK Network; 20 | // static constexpr bool kVerbose = true; 21 | }; 22 | 23 | typedef ::mica::datagram::DatagramServer Server; 24 | 25 | int main() { 26 | ::mica::util::lcore.pin_thread(0); 27 | 28 | auto config = ::mica::util::Config::load_file("server_sqlite.json"); 29 | 30 | Server::DirectoryClient dir_client(config.get("dir_client")); 31 | 32 | SimpleProcessorConfig::Table table(config.get("table")); 33 | DatagramServerConfig::Processor processor(config.get("processor"), &table); 34 | 35 | DatagramServerConfig::Network network(config.get("network")); 36 | network.start(); 37 | 38 | Server server(config.get("server"), &processor, &network, &dir_client); 39 | server.run(); 40 | 41 | network.stop(); 42 | 43 | return EXIT_SUCCESS; 44 | } -------------------------------------------------------------------------------- /src/mica/test/server_sqlite.json: -------------------------------------------------------------------------------- 1 | { 2 | "dir_client": { 3 | "etcd_addr": "YOUR_ETCD_HOST", 4 | "etcd_port": 50505 5 | /*"verbose": true*/ 6 | }, 7 | 8 | "alloc": { 9 | /*"num_pages_to_free": [1024, 1024]*/ 10 | /*"num_pages_to_free": [1024]*/ 11 | "num_pages_to_free": [2048] 12 | /*"verbose": true*/ 13 | }, 14 | "processor": { 15 | }, 16 | "table": { 17 | "path": ":memory:" 18 | }, 19 | 20 | "network": { 21 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27],*/ 22 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],*/ 23 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19],*/ 24 | /*"lcores": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],*/ 25 | /*"lcores": [0, 2],*/ 26 | /*"lcores": [0, 1],*/ 27 | "lcores": [0], 28 | 29 | "ports": [ 30 | {"port_id": 0, "ipv4_addr": "10.0.0.1"} 31 | /*{"port_id": 1, "ipv4_addr": "10.0.0.2"}, 32 | {"port_id": 2, "ipv4_addr": "10.0.0.3"}, 33 | {"port_id": 3, "ipv4_addr": "10.0.0.4"}, 34 | {"port_id": 4, "ipv4_addr": "10.0.0.5"}, 35 | {"port_id": 5, "ipv4_addr": "10.0.0.6"}, 36 | {"port_id": 6, "ipv4_addr": "10.0.0.7"}, 37 | {"port_id": 7, "ipv4_addr": "10.0.0.8"}*/ 38 | ], 39 | 40 | "endpoints": [ 41 | /*[0, 0], 42 | [1, 0]*/ 43 | [0, 0] 44 | ], 45 | 46 | /* 47 | "endpoints": [ 48 | [0, 0], 49 | [1, 0], 50 | [2, 0], 51 | [3, 1], 52 | [4, 1], 53 | [5, 1], 54 | [6, 2], 55 | [7, 2], 56 | [8, 2], 57 | [9, 3], 58 | [10, 3], 59 | [11, 3], 60 | [12, 4], 61 | [13, 4], 62 | [14, 4], 63 | [15, 5], 64 | [16, 5], 65 | [17, 5], 66 | [18, 6], 67 | [19, 6], 68 | [20, 6], 69 | [21, 7], 70 | [22, 7], 71 | [23, 7] 72 | ] 73 | */ 74 | 75 | /*"dpdk_args": ["-n", "4", "--socket-mem=2048,2048"]*/ 76 | "dpdk_args": ["-n", "4", "--socket-mem=2048"] 77 | }, 78 | 79 | "server": { 80 | "rebalance_interval": 0 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /src/mica/test/test_hash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mica/util/lcore.h" 3 | #include "mica/util/stopwatch.h" 4 | #include "mica/util/hash.h" 5 | 6 | static ::mica::util::Stopwatch sw; 7 | 8 | template 9 | void benchmark(const char* name, T& t) { 10 | const uint64_t count = 1000000000LU; 11 | 12 | uint64_t v = 0; 13 | 14 | uint64_t start = sw.now(); 15 | 16 | for (uint64_t i = 0; i < count; i++) 17 | v += static_cast(t(&i, sizeof(uint64_t))); 18 | 19 | uint64_t end = sw.now(); 20 | 21 | double diff = sw.diff(end, start); 22 | 23 | printf("Using %s\n", name); 24 | printf("count: %" PRIu64 "\n", count); 25 | printf("elapsed: %.2lf seconds\n", diff); 26 | printf("throughput: %.2lf M/s\n", 27 | static_cast(count) / diff / 1000000.); 28 | printf("v: %" PRIu64 "\n", v); 29 | printf("\n"); 30 | } 31 | 32 | int main(int argc, const char* argv[]) { 33 | (void)argc; 34 | (void)argv; 35 | 36 | ::mica::util::lcore.pin_thread(0); 37 | 38 | sw.init_start(); 39 | sw.init_end(); 40 | 41 | benchmark("CityHash", ::mica::util::hash_cityhash); 42 | benchmark("SipHash", ::mica::util::hash_siphash); 43 | 44 | return EXIT_SUCCESS; 45 | } 46 | -------------------------------------------------------------------------------- /src/mica/test/test_load.cc: -------------------------------------------------------------------------------- 1 | #include "mica/table/ltable.h" 2 | #include "mica/util/hash.h" 3 | #include "mica/util/lcore.h" 4 | 5 | struct LTableConfig : public ::mica::table::BasicLossyLTableConfig { 6 | // struct LTableConfig : public ::mica::table::BasicLosslessLTableConfig { 7 | static constexpr bool kCollectStats = true; 8 | }; 9 | 10 | typedef ::mica::table::LTable Table; 11 | typedef ::mica::table::Result Result; 12 | 13 | template 14 | static uint64_t hash(const T* key, size_t key_length) { 15 | return ::mica::util::hash(key, key_length); 16 | } 17 | 18 | int main() { 19 | ::mica::util::lcore.pin_thread(0); 20 | 21 | auto config = ::mica::util::Config::load_file("test_load.json"); 22 | 23 | LTableConfig::Alloc alloc(config.get("alloc")); 24 | LTableConfig::Pool pool(config.get("pool"), &alloc); 25 | Table table(config.get("table"), &alloc, &pool); 26 | 27 | size_t num_items = 16 * 1048576; 28 | 29 | size_t key_i; 30 | size_t value_i; 31 | const char* key = reinterpret_cast(&key); 32 | char* value = reinterpret_cast(&value); 33 | 34 | bool first_failure = false; 35 | size_t first_failure_i = 0; 36 | size_t last_failure_i = 0; 37 | size_t success_count = 0; 38 | 39 | for (size_t i = 0; i < num_items; i++) { 40 | key_i = i; 41 | value_i = i; 42 | uint64_t key_hash = hash(&key_i, sizeof(key_i)); 43 | table.set(key_hash, key, sizeof(key_i), value, sizeof(value_i), false); 44 | } 45 | 46 | for (size_t i = 0; i < num_items; i++) { 47 | key_i = i; 48 | size_t value_len; 49 | uint64_t key_hash = hash(&key_i, sizeof(key_i)); 50 | 51 | if (table.get(key_hash, key, sizeof(key_i), value, sizeof(value_i), 52 | &value_len, false) == Result::kSuccess) 53 | success_count++; 54 | else { 55 | if (!first_failure) { 56 | first_failure = true; 57 | first_failure_i = i; 58 | } 59 | last_failure_i = i; 60 | } 61 | } 62 | 63 | printf("first_failure: %zu (%.2f%%)\n", first_failure_i, 64 | 100. * (double)first_failure_i / (double)num_items); 65 | printf("last_failure: %zu (%.2f%%)\n", last_failure_i, 66 | 100. * (double)last_failure_i / (double)num_items); 67 | printf("success_count: %zu (%.2f%%)\n", success_count, 68 | 100. * (double)success_count / (double)num_items); 69 | 70 | table.print_stats(); 71 | 72 | return EXIT_SUCCESS; 73 | } 74 | -------------------------------------------------------------------------------- /src/mica/test/test_load.json: -------------------------------------------------------------------------------- 1 | { 2 | "alloc": { 3 | /*"clean_files_on_init": true, 4 | "verbose": true*/ 5 | }, 6 | 7 | "pool": { 8 | "size": 2000000000, 9 | "concurrent_read": false, 10 | "concurrent_write": false, 11 | "numa_node": 0 12 | }, 13 | 14 | "table": { 15 | "item_count": 16777216, 16 | "concurrent_read": false, 17 | "concurrent_write": false, 18 | "numa_node": 0 19 | } 20 | } -------------------------------------------------------------------------------- /src/mica/test/test_prefetch.cc: -------------------------------------------------------------------------------- 1 | #include "mica/alloc/hugetlbfs_shm.h" 2 | #include "mica/util/config.h" 3 | #include "mica/util/lcore.h" 4 | #include "mica/util/rand.h" 5 | #include "mica/util/roundup.h" 6 | #include "mica/util/tsc.h" 7 | #include 8 | 9 | typedef ::mica::alloc::HugeTLBFS_SHM Alloc; 10 | 11 | int main() { 12 | auto config = ::mica::util::Config::load_file("microbench.json"); 13 | 14 | Alloc alloc(config.get("alloc")); 15 | 16 | ::mica::util::lcore.pin_thread(0); 17 | 18 | size_t request_count = 16 * 1048576; 19 | size_t table_location_count = 128 * 1048576; 20 | size_t pool_location_count = 1024 * 1048576; 21 | 22 | size_t* requests = reinterpret_cast( 23 | alloc.malloc_contiguous(request_count * sizeof(uint64_t), 0)); 24 | assert(requests); 25 | 26 | size_t* table_locations = reinterpret_cast( 27 | alloc.malloc_contiguous(table_location_count * sizeof(size_t), 0)); 28 | assert(table_locations); 29 | 30 | uint64_t* pool_locations = reinterpret_cast( 31 | alloc.malloc_contiguous(pool_location_count * sizeof(uint64_t), 0)); 32 | assert(pool_locations); 33 | 34 | ::mica::util::Rand r(1); 35 | 36 | printf("initializing data\n"); 37 | 38 | for (size_t i = 0; i < request_count; i++) 39 | requests[i] = 40 | static_cast(r.next_u32() & (table_location_count - 1)); 41 | 42 | for (size_t i = 0; i < table_location_count; i++) 43 | table_locations[i] = 44 | static_cast(r.next_u32() & (pool_location_count - 1)); 45 | 46 | for (size_t i = 0; i < pool_location_count; i++) 47 | pool_locations[i] = static_cast(r.next_u32()); 48 | 49 | size_t gap = 0; 50 | uint64_t v = 0; 51 | 52 | uint64_t best_elapsed = static_cast(-1); 53 | size_t best_gap = 0; 54 | 55 | while (gap <= 16) { 56 | printf("gap = %zu\n", gap); 57 | 58 | uint64_t start_t = ::mica::util::rdtsc(); 59 | 60 | for (size_t i_ = 0; i_ < request_count + gap * 2; i_++) { 61 | size_t index = i_; 62 | 63 | if (index < request_count) 64 | __builtin_prefetch(&table_locations[requests[index]], 0, 0); 65 | 66 | index -= gap; 67 | 68 | if (index < request_count) 69 | __builtin_prefetch(&pool_locations[table_locations[requests[index]]], 0, 70 | 0); 71 | 72 | index -= gap; 73 | 74 | if (index < request_count) 75 | v ^= pool_locations[table_locations[requests[index]]]; 76 | } 77 | 78 | uint64_t diff = ::mica::util::rdtsc() - start_t; 79 | 80 | printf(" elapsed = %" PRIu64 " clocks (%" PRIu64 " clocks/req)\n", diff, 81 | diff / request_count); 82 | printf(" gap time = %" PRIu64 " clocks\n", gap * diff / request_count); 83 | 84 | if (best_elapsed > diff) { 85 | best_elapsed = diff; 86 | best_gap = gap; 87 | } 88 | 89 | gap++; 90 | } 91 | 92 | printf("v = %lu (ignore this)\n", v); 93 | printf("\n"); 94 | 95 | printf("best gap = %zu\n", best_gap); 96 | printf(" elapsed = %" PRIu64 " clocks (%" PRIu64 " clocks/req)\n", 97 | best_elapsed, best_elapsed / request_count); 98 | printf(" gap time = %" PRIu64 " clocks\n", 99 | best_gap * best_elapsed / request_count); 100 | 101 | return EXIT_SUCCESS; 102 | } 103 | -------------------------------------------------------------------------------- /src/mica/test/test_processor.cc: -------------------------------------------------------------------------------- 1 | #include "mica/processor/partitions.h" 2 | #include "mica/util/hash.h" 3 | #include "mica/util/lcore.h" 4 | 5 | struct LTableConfig : public ::mica::table::BasicLossyLTableConfig { 6 | // struct LTableConfig : public ::mica::table::BasicLosslessLTableConfig { 7 | // static constexpr bool kVerbose = false; 8 | // static constexpr bool kCollectStats = false; 9 | }; 10 | 11 | struct PartitionsConfig : public ::mica::processor::BasicPartitionsConfig { 12 | static constexpr bool kVerbose = true; 13 | typedef ::mica::table::LTable Table; 14 | typedef Table::Alloc Alloc; 15 | }; 16 | 17 | typedef ::mica::processor::Partitions Processor; 18 | 19 | typedef ::mica::table::Result Result; 20 | typedef ::mica::processor::Operation Operation; 21 | 22 | template 23 | static uint64_t hash(const T* key, size_t key_length) { 24 | return ::mica::util::hash(key, key_length); 25 | } 26 | 27 | int main() { 28 | ::mica::util::lcore.pin_thread(0); 29 | 30 | auto config = ::mica::util::Config::load_file("test_processor.json"); 31 | 32 | PartitionsConfig::Alloc alloc(config.get("alloc")); 33 | Processor processor(config.get("processor"), &alloc); 34 | 35 | uint64_t key_i; 36 | uint64_t value_i; 37 | const char* key = reinterpret_cast(&key_i); 38 | char* value = reinterpret_cast(&value_i); 39 | size_t key_length = sizeof(key_i); 40 | size_t value_length = sizeof(value_i); 41 | 42 | Result out_result; 43 | size_t out_value_length; 44 | 45 | Operation ops[1] = {Operation::kReset}; // To be filled. 46 | uint64_t key_hashes[1] = {0}; // To be filled. 47 | const char* keys[1] = {key}; 48 | uint64_t key_lengths[1] = {key_length}; 49 | const char* values[1] = {value}; 50 | uint64_t value_lengths[1] = {value_length}; 51 | Result* out_results[1] = {&out_result}; 52 | char* out_values[1] = {value}; 53 | size_t* out_value_lengths[1] = {&out_value_length}; 54 | 55 | ::mica::processor::RequestArrayAccessor ra( 56 | 1, ops, key_hashes, keys, key_lengths, values, value_lengths, out_results, 57 | out_values, out_value_lengths); 58 | 59 | { 60 | ops[0] = Operation::kSet; 61 | key_i = 10; 62 | key_hashes[0] = hash(&key_i, sizeof(key_i)); 63 | value_i = 200; 64 | processor.process(ra); 65 | assert(out_result == Result::kSuccess); 66 | } 67 | 68 | { 69 | ops[0] = Operation::kGet; 70 | key_i = 10; 71 | key_hashes[0] = hash(&key_i, sizeof(key_i)); 72 | value_i = 0; 73 | out_value_length = sizeof(value); 74 | processor.process(ra); 75 | assert(out_result == Result::kSuccess); 76 | assert(out_value_length == value_length); 77 | assert(value_i == 200); 78 | } 79 | 80 | { 81 | ops[0] = Operation::kSet; 82 | key_i = 10; 83 | key_hashes[0] = hash(&key_i, sizeof(key_i)); 84 | value_i = 300; 85 | processor.process(ra); 86 | assert(out_result == Result::kSuccess); 87 | } 88 | 89 | { 90 | ops[0] = Operation::kGet; 91 | key_i = 10; 92 | key_hashes[0] = hash(&key_i, sizeof(key_i)); 93 | value_i = 0; 94 | out_value_length = sizeof(value); 95 | processor.process(ra); 96 | assert(out_result == Result::kSuccess); 97 | assert(out_value_length == value_length); 98 | assert(value_i == 300); 99 | } 100 | 101 | return EXIT_SUCCESS; 102 | } 103 | -------------------------------------------------------------------------------- /src/mica/test/test_processor.json: -------------------------------------------------------------------------------- 1 | { 2 | "alloc": { 3 | /*"clean_files_on_init": true, 4 | "verbose": true*/ 5 | }, 6 | 7 | "processor": { 8 | "lcores": [0, 1, 2, 3], 9 | "partition_count": 16, 10 | 11 | "total_size": 1000000, 12 | "total_item_count": 1000, 13 | 14 | "concurrent_read": true, 15 | "concurrent_write": true 16 | 17 | /*"stage_gap": 2*/ 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/mica/test/test_rand.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include "mica/util/lcore.h" 3 | #include "mica/util/stopwatch.h" 4 | #include "mica/util/rand.h" 5 | #include "mica/util/rand_pcg.h" 6 | #include "mica/util/rand_philox.h" 7 | 8 | static ::mica::util::Stopwatch sw; 9 | 10 | template 11 | void benchmark(const char* name) { 12 | const uint64_t count = 1000000000LU; 13 | 14 | T rand; 15 | uint64_t v = 0; 16 | 17 | uint64_t start = sw.now(); 18 | 19 | for (uint64_t i = 0; i < count; i++) 20 | v += static_cast(rand.next_u32()); 21 | 22 | uint64_t end = sw.now(); 23 | 24 | double diff = sw.diff(end, start); 25 | 26 | printf("Using %s\n", name); 27 | printf("count: %" PRIu64 "\n", count); 28 | printf("elapsed: %.2lf seconds\n", diff); 29 | printf("throughput: %.2lf M/s\n", 30 | static_cast(count) / diff / 1000000.); 31 | printf("v: %" PRIu64 "\n", v); 32 | printf("\n"); 33 | } 34 | 35 | int main(int argc, const char* argv[]) { 36 | (void)argc; 37 | (void)argv; 38 | 39 | ::mica::util::lcore.pin_thread(0); 40 | 41 | sw.init_start(); 42 | sw.init_end(); 43 | 44 | // For warming up 45 | benchmark<::mica::util::Rand>("Rand (for warming up; ignore this)"); 46 | 47 | benchmark<::mica::util::Rand>("Rand"); 48 | benchmark<::mica::util::RandPCG>("RandPCG"); 49 | benchmark<::mica::util::RandPhilox>("RandPhilox"); 50 | 51 | return EXIT_SUCCESS; 52 | } -------------------------------------------------------------------------------- /src/mica/test/test_table.cc: -------------------------------------------------------------------------------- 1 | #include "mica/table/ltable.h" 2 | #include "mica/util/hash.h" 3 | #include "mica/util/lcore.h" 4 | 5 | struct LTableConfig : public ::mica::table::BasicLossyLTableConfig { 6 | // struct LTableConfig : public ::mica::table::BasicLosslessLTableConfig { 7 | static constexpr bool kVerbose = true; 8 | static constexpr bool kCollectStats = true; 9 | }; 10 | 11 | typedef ::mica::table::LTable Table; 12 | typedef ::mica::table::Result Result; 13 | 14 | template 15 | static uint64_t hash(const T* key, size_t key_length) { 16 | return ::mica::util::hash(key, key_length); 17 | } 18 | 19 | int main() { 20 | ::mica::util::lcore.pin_thread(0); 21 | 22 | auto config = ::mica::util::Config::load_file("test_table.json"); 23 | 24 | LTableConfig::Alloc alloc(config.get("alloc")); 25 | LTableConfig::Pool pool(config.get("pool"), &alloc); 26 | Table table(config.get("table"), &alloc, &pool); 27 | 28 | uint64_t key_hash; 29 | uint64_t key_i; 30 | uint64_t value_i; 31 | const char* key = reinterpret_cast(&key_i); 32 | char* value = reinterpret_cast(&value_i); 33 | size_t key_length = sizeof(key_i); 34 | size_t value_length = sizeof(value_i); 35 | 36 | Result out_result; 37 | size_t out_value_length; 38 | 39 | { 40 | key_i = 10; 41 | key_hash = hash(&key_i, sizeof(key_i)); 42 | value_i = 200; 43 | out_result = 44 | table.set(key_hash, key, key_length, value, value_length, true); 45 | table.print_stats(); 46 | assert(out_result == Result::kSuccess); 47 | // table.print_buckets(); 48 | } 49 | 50 | { 51 | key_i = 10; 52 | key_hash = hash(&key_i, sizeof(key_i)); 53 | value_i = 0; 54 | out_value_length = 0; 55 | out_result = table.get(key_hash, key, key_length, value, value_length, 56 | &out_value_length, false); 57 | table.print_stats(); 58 | assert(out_result == Result::kSuccess); 59 | assert(out_value_length == value_length); 60 | assert(value_i == 200); 61 | } 62 | 63 | { 64 | key_i = 10; 65 | key_hash = hash(&key_i, sizeof(key_i)); 66 | value_i = 300; 67 | out_result = 68 | table.set(key_hash, key, key_length, value, value_length, true); 69 | table.print_stats(); 70 | assert(out_result == Result::kSuccess); 71 | // table.print_buckets(); 72 | } 73 | 74 | { 75 | key_i = 10; 76 | key_hash = hash(&key_i, sizeof(key_i)); 77 | value_i = 0; 78 | out_result = table.get(key_hash, key, key_length, value, value_length, 79 | &out_value_length, false); 80 | table.print_stats(); 81 | assert(out_result == Result::kSuccess); 82 | assert(out_value_length == value_length); 83 | assert(value_i == 300); 84 | } 85 | 86 | { 87 | key_i = 10; 88 | key_hash = hash(&key_i, sizeof(key_i)); 89 | out_result = table.del(key_hash, key, key_length); 90 | table.print_stats(); 91 | assert(out_result == Result::kSuccess); 92 | } 93 | 94 | (void)out_result; 95 | 96 | return EXIT_SUCCESS; 97 | } 98 | -------------------------------------------------------------------------------- /src/mica/test/test_table.json: -------------------------------------------------------------------------------- 1 | { 2 | "alloc": { 3 | /*"clean_files_on_init": true, 4 | "verbose": true*/ 5 | }, 6 | 7 | "pool": { 8 | "size": 134217728, 9 | "concurrent_read": true, 10 | "concurrent_write": true, 11 | "numa_node": 0 12 | }, 13 | 14 | "table": { 15 | "item_count": 1000, 16 | "concurrent_read": true, 17 | "concurrent_write": true, 18 | "numa_node": 0 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/mica/test/test_zipf.cc: -------------------------------------------------------------------------------- 1 | #include "mica/util/zipf.h" 2 | #include 3 | 4 | int main() { 5 | ::mica::util::ZipfGen::test(-1.); 6 | ::mica::util::ZipfGen::test(0.0); 7 | ::mica::util::ZipfGen::test(0.99); 8 | ::mica::util::ZipfGen::test(40.); 9 | 10 | return EXIT_SUCCESS; 11 | } 12 | -------------------------------------------------------------------------------- /src/mica/util/barrier.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_BARRIER_H_ 3 | #define MICA_UTIL_BARRIER_H_ 4 | 5 | #include "mica/common.h" 6 | 7 | namespace mica { 8 | namespace util { 9 | static void memory_barrier() { asm volatile("" ::: "memory"); } 10 | 11 | static void lfence() { asm volatile("lfence" ::: "memory"); } 12 | 13 | static void sfence() { asm volatile("sfence" ::: "memory"); } 14 | 15 | static void mfence() { asm volatile("mfence" ::: "memory"); } 16 | 17 | static void pause() { asm volatile("pause"); } 18 | 19 | static void clflush(volatile void* p) { asm volatile("clflush (%0)" ::"r"(p)); } 20 | 21 | static void cpuid(unsigned int* eax, unsigned int* ebx, unsigned int* ecx, 22 | unsigned int* edx) { 23 | asm volatile("cpuid" 24 | : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) 25 | : "0"(*eax), "2"(*ecx)); 26 | } 27 | } 28 | } 29 | 30 | #endif -------------------------------------------------------------------------------- /src/mica/util/cityhash/city.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // http://code.google.com/p/cityhash/ 24 | // 25 | // This file provides a few functions for hashing strings. All of them are 26 | // high-quality functions in the sense that they pass standard tests such 27 | // as Austin Appleby's SMHasher. They are also fast. 28 | // 29 | // For 64-bit x86 code, on short strings, we don't know of anything faster than 30 | // CityHash64 that is of comparable quality. We believe our nearest competitor 31 | // is Murmur3. For 64-bit x86 code, CityHash64 is an excellent choice for hash 32 | // tables and most other hashing (excluding cryptography). 33 | // 34 | // For 64-bit x86 code, on long strings, the picture is more complicated. 35 | // On many recent Intel CPUs, such as Nehalem, Westmere, Sandy Bridge, etc., 36 | // CityHashCrc128 appears to be faster than all competitors of comparable 37 | // quality. CityHash128 is also good but not quite as fast. We believe our 38 | // nearest competitor is Bob Jenkins' Spooky. We don't have great data for 39 | // other 64-bit CPUs, but for long strings we know that Spooky is slightly 40 | // faster than CityHash on some relatively recent AMD x86-64 CPUs, for example. 41 | // Note that CityHashCrc128 is declared in citycrc.h. 42 | // 43 | // For 32-bit x86 code, we don't know of anything faster than CityHash32 that 44 | // is of comparable quality. We believe our nearest competitor is Murmur3A. 45 | // (On 64-bit CPUs, it is typically faster to use the other CityHash variants.) 46 | // 47 | // Functions in the CityHash family are not suitable for cryptography. 48 | // 49 | // Please see CityHash's README file for more details on our performance 50 | // measurements and so on. 51 | // 52 | // WARNING: This code has been only lightly tested on big-endian platforms! 53 | // It is known to work well on little-endian platforms that have a small penalty 54 | // for unaligned reads, such as current Intel and AMD moderate-to-high-end CPUs. 55 | // It should work on all 32-bit and 64-bit platforms that allow unaligned reads; 56 | // bug reports are welcome. 57 | // 58 | // By the way, for some hash functions, given strings a and b, the hash 59 | // of a+b is easily derived from the hashes of a and b. This property 60 | // doesn't hold for any hash functions in this file. 61 | 62 | #ifndef CITY_HASH_H_ 63 | #define CITY_HASH_H_ 64 | 65 | #include // for size_t. 66 | #include 67 | #include 68 | 69 | typedef uint8_t uint8; 70 | typedef uint32_t uint32; 71 | typedef uint64_t uint64; 72 | typedef std::pair uint128; 73 | 74 | inline uint64 Uint128Low64(const uint128& x) { return x.first; } 75 | inline uint64 Uint128High64(const uint128& x) { return x.second; } 76 | 77 | // Hash function for a byte array. 78 | uint64 CityHash64(const char *buf, size_t len); 79 | 80 | // Hash function for a byte array. For convenience, a 64-bit seed is also 81 | // hashed into the result. 82 | uint64 CityHash64WithSeed(const char *buf, size_t len, uint64 seed); 83 | 84 | // Hash function for a byte array. For convenience, two seeds are also 85 | // hashed into the result. 86 | uint64 CityHash64WithSeeds(const char *buf, size_t len, 87 | uint64 seed0, uint64 seed1); 88 | 89 | // Hash function for a byte array. 90 | uint128 CityHash128(const char *s, size_t len); 91 | 92 | // Hash function for a byte array. For convenience, a 128-bit seed is also 93 | // hashed into the result. 94 | uint128 CityHash128WithSeed(const char *s, size_t len, uint128 seed); 95 | 96 | // Hash function for a byte array. Most useful in 32-bit binaries. 97 | uint32 CityHash32(const char *buf, size_t len); 98 | 99 | // Hash 128 input bits down to 64 bits of output. 100 | // This is intended to be a reasonably good hash function. 101 | inline uint64 Hash128to64(const uint128& x) { 102 | // Murmur-inspired hashing. 103 | const uint64 kMul = 0x9ddfea08eb382d69ULL; 104 | uint64 a = (Uint128Low64(x) ^ Uint128High64(x)) * kMul; 105 | a ^= (a >> 47); 106 | uint64 b = (Uint128High64(x) ^ a) * kMul; 107 | b ^= (b >> 47); 108 | b *= kMul; 109 | return b; 110 | } 111 | 112 | #endif // CITY_HASH_H_ 113 | -------------------------------------------------------------------------------- /src/mica/util/cityhash/citycrc.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /src/mica/util/cityhash/citycrc_mod.h: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2011 Google, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | // 21 | // CityHash, by Geoff Pike and Jyrki Alakuijala 22 | // 23 | // This file declares the subset of the CityHash functions that require 24 | // _mm_crc32_u64(). See the CityHash README for details. 25 | // 26 | // Functions in the CityHash family are not suitable for cryptography. 27 | 28 | #ifndef CITY_HASH_CRC_H_ 29 | #define CITY_HASH_CRC_H_ 30 | 31 | #include "city.h" 32 | 33 | // Hash function for a byte array. 34 | uint128 CityHashCrc128(const char *s, size_t len); 35 | 36 | // Hash function for a byte array. For convenience, a 128-bit seed is also 37 | // hashed into the result. 38 | uint128 CityHashCrc128WithSeed(const char *s, size_t len, uint128 seed); 39 | 40 | // Hash function for a byte array. Sets result[0] ... result[3]. 41 | void CityHashCrc256(const char *s, size_t len, uint64 *result); 42 | 43 | #endif // CITY_HASH_CRC_H_ 44 | -------------------------------------------------------------------------------- /src/mica/util/config.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_CONFIG_H_ 3 | #define MICA_UTIL_CONFIG_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include "mica/common.h" 10 | #pragma GCC diagnostic push 11 | #pragma GCC diagnostic ignored "-Wconversion" 12 | #pragma GCC diagnostic ignored "-Wsign-conversion" 13 | #pragma GCC diagnostic ignored "-Wzero-as-null-pointer-constant" 14 | #include "mica/util/rapidjson/document.h" 15 | #pragma GCC diagnostic pop 16 | 17 | namespace mica { 18 | namespace util { 19 | class Config { 20 | public: 21 | Config(); 22 | Config(const Config& config); 23 | ~Config() {} 24 | 25 | // This constructor is public to support emplace_back(), emplace(). 26 | Config(const std::shared_ptr& root, 27 | rapidjson::Value* current, std::string path); 28 | 29 | Config& operator=(const Config& config) = delete; 30 | 31 | static Config empty_array(std::string path); 32 | static Config empty_dict(std::string path); 33 | 34 | static Config load_file(std::string path); 35 | void dump_file(std::string path) const; 36 | 37 | static Config load(std::string json_text, std::string path); 38 | std::string dump() const; 39 | 40 | std::string get_path() const; 41 | 42 | bool exists() const; 43 | bool is_bool() const; 44 | bool is_int64() const; 45 | bool is_uint64() const; 46 | bool is_double() const; 47 | bool is_str() const; 48 | bool is_array() const; 49 | bool is_dict() const; 50 | 51 | bool get_bool() const; 52 | int64_t get_int64() const; 53 | uint64_t get_uint64() const; 54 | double get_double() const; 55 | std::string get_str() const; 56 | 57 | bool get_bool(bool default_v) const; 58 | int64_t get_int64(int64_t default_v) const; 59 | uint64_t get_uint64(uint64_t default_v) const; 60 | double get_double(double default_v) const; 61 | std::string get_str(const std::string& default_v) const; 62 | 63 | size_t size() const; 64 | const Config get(size_t index) const; 65 | Config get(size_t index); 66 | 67 | std::vector keys() const; 68 | const Config get(std::string key) const; 69 | Config get(std::string key); 70 | 71 | Config& push_back_bool(bool v); 72 | Config& push_back_int64(int64_t v); 73 | Config& push_back_uint64(uint64_t v); 74 | Config& push_back_double(double v); 75 | Config& push_back_string(const std::string& v); 76 | Config& push_back_array(const Config& v); 77 | Config& push_back_dict(const Config& v); 78 | 79 | Config& insert_bool(std::string key, bool v); 80 | Config& insert_int64(std::string key, int64_t v); 81 | Config& insert_uint64(std::string key, uint64_t v); 82 | Config& insert_double(std::string key, double v); 83 | Config& insert_string(std::string key, std::string v); 84 | Config& insert_array(std::string key, const Config& v); 85 | Config& insert_dict(std::string key, const Config& v); 86 | 87 | private: 88 | std::shared_ptr root_; 89 | rapidjson::Value* current_; 90 | std::string path_; 91 | }; 92 | } 93 | } 94 | 95 | #endif 96 | -------------------------------------------------------------------------------- /src/mica/util/hash.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_HASH_H_ 3 | #define MICA_UTIL_HASH_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/util/cityhash/citycrc_mod.h" 7 | 8 | int siphash(uint8_t* out, const uint8_t* in, uint64_t inlen, const uint8_t* k); 9 | static const uint8_t siphash_key[16] = { 10 | 0, 11 | }; 12 | 13 | namespace mica { 14 | namespace util { 15 | template 16 | static uint64_t hash_cityhash(const T* key, size_t len) { 17 | return CityHash64(reinterpret_cast(key), len); 18 | } 19 | 20 | template 21 | static uint64_t hash_siphash(const T* key, size_t len) { 22 | uint64_t v; 23 | siphash(reinterpret_cast(&v), reinterpret_cast(key), 24 | len, siphash_key); 25 | return v; 26 | } 27 | 28 | template 29 | static uint64_t hash(const T* key, size_t len) { 30 | return hash_cityhash(key, len); 31 | } 32 | } 33 | } 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/mica/util/latency.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_LATENCY_H_ 3 | #define MICA_UTIL_LATENCY_H_ 4 | 5 | #include 6 | #include 7 | #include "mica/common.h" 8 | #include "mica/util/memcpy.h" 9 | 10 | namespace mica { 11 | namespace util { 12 | class Latency { 13 | public: 14 | Latency() { reset(); } 15 | 16 | void reset() { ::mica::util::memset(this, 0, sizeof(Latency)); } 17 | 18 | void update(uint64_t us) { 19 | if (us < 128) 20 | bin0_[us]++; 21 | else if (us < 384) 22 | bin1_[(us - 128) / 2]++; 23 | else if (us < 896) 24 | bin2_[(us - 384) / 4]++; 25 | else if (us < 1920) 26 | bin3_[(us - 896) / 8]++; 27 | else if (us < 3968) 28 | bin4_[(us - 1920) / 16]++; 29 | else 30 | bin5_++; 31 | } 32 | 33 | Latency& operator+=(const Latency& o) { 34 | uint64_t i; 35 | for (i = 0; i < 128; i++) bin0_[i] += o.bin0_[i]; 36 | for (i = 0; i < 128; i++) bin1_[i] += o.bin1_[i]; 37 | for (i = 0; i < 128; i++) bin2_[i] += o.bin2_[i]; 38 | for (i = 0; i < 128; i++) bin3_[i] += o.bin3_[i]; 39 | for (i = 0; i < 128; i++) bin4_[i] += o.bin4_[i]; 40 | bin5_ += o.bin5_; 41 | return *this; 42 | } 43 | 44 | uint64_t count() const { 45 | uint64_t count = 0; 46 | uint64_t i; 47 | for (i = 0; i < 128; i++) count += bin0_[i]; 48 | for (i = 0; i < 128; i++) count += bin1_[i]; 49 | for (i = 0; i < 128; i++) count += bin2_[i]; 50 | for (i = 0; i < 128; i++) count += bin3_[i]; 51 | for (i = 0; i < 128; i++) count += bin4_[i]; 52 | count += bin5_; 53 | return count; 54 | } 55 | 56 | uint64_t sum() const { 57 | uint64_t sum = 0; 58 | uint64_t i; 59 | for (i = 0; i < 128; i++) sum += bin0_[i] * (0 + i * 1); 60 | for (i = 0; i < 128; i++) sum += bin1_[i] * (128 + i * 2); 61 | for (i = 0; i < 128; i++) sum += bin2_[i] * (384 + i * 4); 62 | for (i = 0; i < 128; i++) sum += bin3_[i] * (896 + i * 8); 63 | for (i = 0; i < 128; i++) sum += bin4_[i] * (1920 + i * 16); 64 | sum += bin5_ * 3968; 65 | return sum; 66 | } 67 | 68 | uint64_t avg() const { return sum() / std::max(uint64_t(1), count()); } 69 | 70 | double avg_f() const { 71 | return static_cast(sum()) / 72 | static_cast(std::max(uint64_t(1), count())); 73 | } 74 | 75 | uint64_t min() const { 76 | uint64_t i; 77 | for (i = 0; i < 128; i++) 78 | if (bin0_[i] != 0) return 0 + i * 1; 79 | for (i = 0; i < 128; i++) 80 | if (bin1_[i] != 0) return 128 + i * 2; 81 | for (i = 0; i < 128; i++) 82 | if (bin2_[i] != 0) return 384 + i * 4; 83 | for (i = 0; i < 128; i++) 84 | if (bin3_[i] != 0) return 896 + i * 8; 85 | for (i = 0; i < 128; i++) 86 | if (bin4_[i] != 0) return 1920 + i * 16; 87 | // if (bin5_ != 0) return 3968; 88 | return 3968; 89 | } 90 | 91 | uint64_t max() const { 92 | int64_t i; 93 | if (bin5_ != 0) return 3968; 94 | for (i = 127; i >= 0; i--) 95 | if (bin4_[i] != 0) return 1920 + static_cast(i) * 16; 96 | for (i = 127; i >= 0; i--) 97 | if (bin3_[i] != 0) return 896 + static_cast(i) * 8; 98 | for (i = 127; i >= 0; i--) 99 | if (bin2_[i] != 0) return 384 + static_cast(i) * 4; 100 | for (i = 127; i >= 0; i--) 101 | if (bin1_[i] != 0) return 128 + static_cast(i) * 2; 102 | for (i = 127; i >= 0; i--) 103 | if (bin0_[i] != 0) return 0 + static_cast(i) * 1; 104 | return 0; 105 | } 106 | 107 | uint64_t perc(double p) const { 108 | uint64_t i; 109 | int64_t thres = static_cast(p * static_cast(count())); 110 | for (i = 0; i < 128; i++) 111 | if ((thres -= static_cast(bin0_[i])) < 0) return 0 + i * 1; 112 | for (i = 0; i < 128; i++) 113 | if ((thres -= static_cast(bin1_[i])) < 0) return 128 + i * 2; 114 | for (i = 0; i < 128; i++) 115 | if ((thres -= static_cast(bin2_[i])) < 0) return 384 + i * 4; 116 | for (i = 0; i < 128; i++) 117 | if ((thres -= static_cast(bin3_[i])) < 0) return 896 + i * 8; 118 | for (i = 0; i < 128; i++) 119 | if ((thres -= static_cast(bin4_[i])) < 0) return 1920 + i * 16; 120 | return 3968; 121 | } 122 | 123 | void print(FILE* fp) const { 124 | uint64_t i; 125 | for (i = 0; i < 128; i++) 126 | if (bin0_[i] != 0) 127 | fprintf(fp, "%4" PRIu64 " %6" PRIu64 "\n", 0 + i * 1, bin0_[i]); 128 | for (i = 0; i < 128; i++) 129 | if (bin1_[i] != 0) 130 | fprintf(fp, "%4" PRIu64 " %6" PRIu64 "\n", 128 + i * 2, bin1_[i]); 131 | for (i = 0; i < 128; i++) 132 | if (bin2_[i] != 0) 133 | fprintf(fp, "%4" PRIu64 " %6" PRIu64 "\n", 384 + i * 4, bin2_[i]); 134 | for (i = 0; i < 128; i++) 135 | if (bin3_[i] != 0) 136 | fprintf(fp, "%4" PRIu64 " %6" PRIu64 "\n", 896 + i * 8, bin3_[i]); 137 | for (i = 0; i < 128; i++) 138 | if (bin4_[i] != 0) 139 | fprintf(fp, "%4" PRIu64 " %6" PRIu64 "\n", 1920 + i * 16, bin4_[i]); 140 | if (bin5_ != 0) fprintf(fp, "%4d %6" PRIu64 "\n", 3968, bin5_); 141 | } 142 | 143 | private: 144 | // [0, 128) us 145 | uint64_t bin0_[128]; 146 | // [128, 384) us 147 | uint64_t bin1_[128]; 148 | // [384, 896) us 149 | uint64_t bin2_[128]; 150 | // [896, 1920) us 151 | uint64_t bin3_[128]; 152 | // [1920, 3968) us 153 | uint64_t bin4_[128]; 154 | // [3968, inf) us 155 | uint64_t bin5_; 156 | }; 157 | } 158 | } 159 | 160 | #endif -------------------------------------------------------------------------------- /src/mica/util/lcore.cc: -------------------------------------------------------------------------------- 1 | // #pragma once 2 | #ifndef MICA_UTIL_LCORE_CC_ 3 | #define MICA_UTIL_LCORE_CC_ 4 | 5 | #include "mica/common.h" 6 | #include 7 | #include 8 | #include 9 | #ifdef USE_DPDK 10 | #include 11 | #include 12 | #endif 13 | #include "mica/util/lcore.h" 14 | 15 | namespace mica { 16 | namespace util { 17 | thread_local size_t LCore::this_lcore_id_ = LCore::kUnknown; 18 | 19 | LCore::LCore() { 20 | numa_count_ = static_cast(numa_num_configured_nodes()); 21 | 22 | for (auto i = 0; i < numa_num_configured_cpus(); i++) { 23 | lcore_to_numa_id_.push_back(static_cast(numa_node_of_cpu(i))); 24 | #ifndef NDEBUG 25 | // printf("LCore %d -> NUMA %zu\n", i, lcore_to_numa_id_[i]); 26 | #endif 27 | } 28 | } 29 | 30 | void LCore::pin_thread(size_t lcore_id) const { 31 | // #ifndef USE_DPDK 32 | cpu_set_t cpuset; 33 | 34 | CPU_ZERO(&cpuset); 35 | CPU_SET(lcore_id, &cpuset); 36 | 37 | auto s = pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &cpuset); 38 | if (s != 0) { 39 | fprintf(stderr, "error: could not set affinity\n"); 40 | assert(false); 41 | return; 42 | } 43 | // #else 44 | // if (rte_lcore_id() != lcore_id) { 45 | // fprintf( 46 | // stderr, 47 | // "error: cannot pin the thread to a different lcore while using 48 | // DPDK\n"); 49 | // assert(false); 50 | // return; 51 | // } 52 | // if (lcore_to_numa_id_[lcore_id] != 53 | // rte_lcore_to_socket_id(static_cast(lcore_id))) { 54 | // fprintf(stderr, 55 | // "error: numactl and DPDK have inconsistent lcore-to-socket " 56 | // "mappings\n"); 57 | // } 58 | // #endif 59 | 60 | this_lcore_id_ = lcore_id; 61 | } 62 | 63 | const LCore lcore; 64 | } 65 | } 66 | 67 | #endif -------------------------------------------------------------------------------- /src/mica/util/lcore.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_LCORE_H_ 3 | #define MICA_UTIL_LCORE_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "mica/common.h" 9 | 10 | namespace mica { 11 | namespace util { 12 | class LCore { 13 | public: 14 | LCore(); 15 | 16 | static constexpr size_t kUnknown = std::numeric_limits::max(); 17 | 18 | size_t numa_count() const { return numa_count_; } 19 | size_t lcore_count() const { return lcore_to_numa_id_.size(); } 20 | 21 | size_t numa_id(size_t lcore_id) const { 22 | assert(lcore_id < lcore_count()); 23 | return lcore_to_numa_id_[lcore_id]; 24 | } 25 | 26 | size_t numa_id() const { return numa_id(lcore_id()); } 27 | 28 | size_t lcore_id() const { return this_lcore_id_; } 29 | 30 | void pin_thread(size_t lcore_id) const; 31 | 32 | private: 33 | std::vector lcore_to_numa_id_; 34 | size_t numa_count_; 35 | static thread_local size_t this_lcore_id_; 36 | }; 37 | 38 | extern const LCore lcore; 39 | } 40 | } 41 | 42 | #endif -------------------------------------------------------------------------------- /src/mica/util/memcpy.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_MEMCPY_H_ 3 | #define MICA_UTIL_MEMCPY_H_ 4 | 5 | #include 6 | #include 7 | #include "mica/common.h" 8 | #include "mica/util/rte_memcpy/rte_memcpy_mod.h" 9 | #include "mica/util/roundup.h" 10 | 11 | namespace mica { 12 | namespace util { 13 | static void memset(void* dest, int c, size_t n) { ::memset(dest, c, n); } 14 | 15 | static void memcpy(void* dest, const void* src, size_t n) { 16 | ::mica::util::rte_memcpy_func(dest, src, n); 17 | } 18 | 19 | static void memmove(void* dest, const void* src, size_t n) { ::memmove(dest, src, n); } 20 | 21 | static int memcmp(const void* a, const void* b, size_t n) { return ::memcmp(a, b, n); } 22 | 23 | static bool memcmp_equal(const void* a, const void* b, size_t n) { 24 | return ::memcmp(a, b, n) == 0; 25 | } 26 | 27 | // Wrappers for any pointer. 28 | template 29 | void memset(T* dest, int c, size_t n) { 30 | ::mica::util::memset(reinterpret_cast(dest), c, n); 31 | } 32 | 33 | template 34 | void memcpy(T1* dest, const T2* src, size_t n) { 35 | ::mica::util::memcpy(reinterpret_cast(dest), 36 | reinterpret_cast(src), n); 37 | } 38 | 39 | template 40 | void memmove(T1* dest, const T2* src, size_t n) { 41 | ::mica::util::memmove(reinterpret_cast(dest), 42 | reinterpret_cast(src), n); 43 | } 44 | 45 | template 46 | int memcmp(const T1* a, const T2* b, size_t n) { 47 | return ::mica::util::memcmp(reinterpret_cast(a), 48 | reinterpret_cast(b), n); 49 | } 50 | 51 | template 52 | bool memcmp_equal(const T1* a, const T2* b, size_t n) { 53 | return ::mica::util::memcmp_equal(reinterpret_cast(a), 54 | reinterpret_cast(b), n); 55 | } 56 | 57 | template 58 | void memset(T* dest, int c, size_t n) { 59 | assert(n == ::mica::util::roundup(n)); 60 | assert(reinterpret_cast(dest) % Alignment == 0); 61 | ::mica::util::memset(dest, c, n); 62 | } 63 | 64 | template 65 | void memcpy(T1* dest, const T2* src, size_t n) { 66 | assert(n == ::mica::util::roundup(n)); 67 | assert(reinterpret_cast(dest) % Alignment == 0); 68 | assert(reinterpret_cast(src) % Alignment == 0); 69 | if (Alignment == 8) { 70 | switch (n >> 3) { 71 | case 4: 72 | *(uint64_t*)(reinterpret_cast(dest) + 24) = 73 | *(const uint64_t*)(reinterpret_cast(src) + 24); 74 | // fall-through 75 | case 3: 76 | *(uint64_t*)(reinterpret_cast(dest) + 16) = 77 | *(const uint64_t*)(reinterpret_cast(src) + 16); 78 | // fall-through 79 | case 2: 80 | *(uint64_t*)(reinterpret_cast(dest) + 8) = 81 | *(const uint64_t*)(reinterpret_cast(src) + 8); 82 | // fall-through 83 | case 1: 84 | *(uint64_t*)(reinterpret_cast(dest) + 0) = 85 | *(const uint64_t*)(reinterpret_cast(src) + 0); 86 | // fall-through 87 | case 0: 88 | break; 89 | default: 90 | ::mica::util::memcpy(dest, src, n); 91 | break; 92 | } 93 | } else 94 | ::mica::util::memcpy(dest, src, n); 95 | } 96 | 97 | template 98 | void memmove(T1* dest, const T2* src, size_t n) { 99 | assert(n == ::mica::util::roundup(n)); 100 | assert(reinterpret_cast(dest) % Alignment == 0); 101 | assert(reinterpret_cast(src) % Alignment == 0); 102 | ::mica::util::memmove(dest, src, n); 103 | } 104 | 105 | template 106 | int memcmp(const T1* a, const T2* b, size_t n) { 107 | assert(n == ::mica::util::roundup(n)); 108 | assert(reinterpret_cast(a) % Alignment == 0); 109 | assert(reinterpret_cast(b) % Alignment == 0); 110 | return ::mica::util::memcmp(a, b, n); 111 | } 112 | 113 | template 114 | bool memcmp_equal(const T1* a, const T2* b, size_t n) { 115 | assert(n == ::mica::util::roundup(n)); 116 | assert(reinterpret_cast(a) % Alignment == 0); 117 | assert(reinterpret_cast(b) % Alignment == 0); 118 | // printf("%p %p %zu\n", a, b, n); 119 | if (Alignment == 8) { 120 | switch (n >> 3) { 121 | case 4: 122 | if (*reinterpret_cast( 123 | reinterpret_cast(a) + 24) != 124 | *reinterpret_cast( 125 | reinterpret_cast(b) + 24)) 126 | return false; 127 | // fall-through 128 | case 3: 129 | if (*reinterpret_cast( 130 | reinterpret_cast(a) + 16) != 131 | *reinterpret_cast( 132 | reinterpret_cast(b) + 16)) 133 | return false; 134 | // fall-through 135 | case 2: 136 | if (*reinterpret_cast( 137 | reinterpret_cast(a) + 8) != 138 | *reinterpret_cast( 139 | reinterpret_cast(b) + 8)) 140 | return false; 141 | // fall-through 142 | case 1: 143 | if (*reinterpret_cast( 144 | reinterpret_cast(a) + 0) != 145 | *reinterpret_cast( 146 | reinterpret_cast(b) + 0)) 147 | return false; 148 | // fall-through 149 | case 0: 150 | return true; 151 | default: 152 | return ::mica::util::memcmp_equal(a, b, n); 153 | } 154 | } else 155 | return ::mica::util::memcmp_equal(a, b, n); 156 | } 157 | } 158 | } 159 | 160 | #endif -------------------------------------------------------------------------------- /src/mica/util/queue.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_QUEUE_H_ 3 | #define MICA_UTIL_QUEUE_H_ 4 | 5 | #include "mica/common.h" 6 | #include 7 | #include "mica/util/barrier.h" 8 | 9 | namespace mica { 10 | namespace util { 11 | template 13 | class Queue { 14 | public: 15 | Queue() : head_lock_(0), head_(0), tail_lock_(0), tail_(0) {} 16 | 17 | bool approximate_empty() const { 18 | ::mica::util::memory_barrier(); 19 | return head_ == tail_; 20 | } 21 | 22 | size_t approximate_size() const { 23 | ::mica::util::memory_barrier(); 24 | size_t head = head_; 25 | size_t tail = tail_; 26 | size_t size = tail - head; 27 | if (size > Capacity) size += Capacity; 28 | return size; 29 | } 30 | 31 | bool enqueue(const T& v) { 32 | if (MultipleProducer) 33 | while (!__sync_bool_compare_and_swap(&tail_lock_, 0, 1)) 34 | ::mica::util::pause(); 35 | 36 | size_t tail = tail_; 37 | size_t tail_next = tail + 1; 38 | if (tail_next == Capacity) tail_next = 0; 39 | 40 | // Full? 41 | if (tail_next == head_) { 42 | if (MultipleProducer) { 43 | tail_lock_ = 0; 44 | ::mica::util::memory_barrier(); 45 | } 46 | return false; 47 | } 48 | 49 | items_[tail] = v; 50 | tail_ = tail_next; 51 | 52 | if (MultipleProducer) { 53 | tail_lock_ = 0; 54 | ::mica::util::memory_barrier(); 55 | } 56 | return true; 57 | } 58 | 59 | bool dequeue(T* out_v) { 60 | if (MultipleConsumer) 61 | while (!__sync_bool_compare_and_swap(&head_lock_, 0, 1)) 62 | ::mica::util::pause(); 63 | 64 | size_t head = head_; 65 | size_t head_next = head + 1; 66 | if (head_next == Capacity) head_next = 0; 67 | 68 | // Empty? 69 | if (head == tail_) { 70 | if (MultipleConsumer) { 71 | head_lock_ = 0; 72 | ::mica::util::memory_barrier(); 73 | } 74 | return false; 75 | } 76 | 77 | *out_v = items_[head]; 78 | head_ = head_next; 79 | 80 | if (MultipleConsumer) { 81 | head_lock_ = 0; 82 | ::mica::util::memory_barrier(); 83 | } 84 | return true; 85 | } 86 | 87 | private: 88 | volatile uint8_t head_lock_ __attribute__((aligned(128))); 89 | size_t head_; 90 | 91 | volatile uint8_t tail_lock_ __attribute__((aligned(128))); 92 | size_t tail_; 93 | 94 | std::array items_ __attribute__((aligned(128))); 95 | } __attribute__((aligned(128))); 96 | 97 | template 98 | class SingleThreadedQueue { 99 | public: 100 | SingleThreadedQueue() : head_(0), tail_(0) {} 101 | 102 | bool empty() const { return head_ == tail_; } 103 | 104 | bool full() const { 105 | size_t tail_next = tail_ + 1; 106 | if (tail_next == Capacity) tail_next = 0; 107 | return tail_next == head_; 108 | } 109 | 110 | size_t size() const { 111 | size_t size = tail_ - head_; 112 | if (size > Capacity) size += Capacity; 113 | return size; 114 | } 115 | 116 | const T& head() const { return items_[head_]; } 117 | 118 | // The user must check empty() and read or copy head() before calling 119 | // pop_head(). 120 | void pop_head() { 121 | if (++head_ == Capacity) head_ = 0; 122 | } 123 | 124 | T& tail() { return items_[tail_]; } 125 | 126 | // The user must check full() and write to tail() before calling push_tail(). 127 | void push_tail() { 128 | if (++tail_ == Capacity) tail_ = 0; 129 | } 130 | 131 | private: 132 | size_t head_; 133 | size_t tail_; 134 | std::array items_; 135 | }; 136 | } 137 | } 138 | 139 | #endif 140 | -------------------------------------------------------------------------------- /src/mica/util/rand.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_RAND_H_ 3 | #define MICA_UTIL_RAND_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | 8 | namespace mica { 9 | namespace util { 10 | class Rand { 11 | public: 12 | explicit Rand() : state_(0) {} 13 | explicit Rand(uint64_t seed) : state_(seed) { assert(seed < (1UL << 48)); } 14 | Rand(const Rand& o) : state_(o.state_) {} 15 | Rand& operator=(const Rand& o) { 16 | state_ = o.state_; 17 | return *this; 18 | } 19 | 20 | uint32_t next_u32() { 21 | // same as Java's 22 | state_ = (state_ * 0x5deece66dUL + 0xbUL) & ((1UL << 48) - 1); 23 | return (uint32_t)(state_ >> (48 - 32)); 24 | } 25 | 26 | double next_f64() { 27 | // caution: this is maybe too non-random 28 | state_ = (state_ * 0x5deece66dUL + 0xbUL) & ((1UL << 48) - 1); 29 | return (double)state_ / (double)((1UL << 48) - 1); 30 | } 31 | 32 | private: 33 | uint64_t state_; 34 | }; 35 | } 36 | } 37 | 38 | #endif -------------------------------------------------------------------------------- /src/mica/util/rand_pcg.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_RAND_PCG_H_ 3 | #define MICA_UTIL_RAND_PCG_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | #pragma GCC diagnostic push 8 | #pragma GCC diagnostic ignored "-Wunused-but-set-parameter" 9 | #include "mica/util/pcg/pcg_random.hpp" 10 | #pragma GCC diagnostic pop 11 | 12 | namespace mica { 13 | namespace util { 14 | class RandPCG { 15 | public: 16 | explicit RandPCG() : state_(0) {} 17 | explicit RandPCG(uint64_t seed) : state_(seed) { assert(seed < (1UL << 63)); } 18 | RandPCG(const RandPCG& o) : state_(o.state_) {} 19 | RandPCG& operator=(const RandPCG& o) { 20 | state_ = o.state_; 21 | return *this; 22 | } 23 | 24 | uint32_t next_u32() { return state_(); } 25 | 26 | double next_f64() { return next_u32() / (double)((1UL << 32) - 1); } 27 | 28 | private: 29 | // pcg32 state_; 30 | pcg32_oneseq state_; 31 | // pcg32_fast state_; 32 | }; 33 | } 34 | } 35 | 36 | #endif -------------------------------------------------------------------------------- /src/mica/util/rand_philox.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_RAND_PHILOX_H_ 3 | #define MICA_UTIL_RAND_PHILOX_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/util/philox/philox_random.h" 7 | 8 | namespace mica { 9 | namespace util { 10 | class RandPhilox { 11 | public: 12 | explicit RandPhilox() : state_(0), result_(), i_(0) {} 13 | explicit RandPhilox(uint64_t seed) : state_(seed), result_(), i_(0) {} 14 | RandPhilox(const RandPhilox& o) 15 | : state_(o.state_), result_(o.result_), i_(o.i_) {} 16 | RandPhilox& operator=(const RandPhilox& o) { 17 | state_ = o.state_; 18 | result_ = o.result_; 19 | i_ = o.i_; 20 | return *this; 21 | } 22 | 23 | uint32_t next_u32() { 24 | if (!i_) { 25 | result_ = state_(); 26 | i_ = 4; 27 | } 28 | return result_[--i_]; 29 | } 30 | 31 | double next_f64() { return next_u32() / (double)((1UL << 32) - 1); } 32 | 33 | private: 34 | ::tensorflow::random::PhiloxRandom state_; 35 | ::tensorflow::random::PhiloxRandom::ResultType result_; 36 | int i_; 37 | }; 38 | } 39 | } 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/error/en.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_ERROR_EN_H__ 16 | #define RAPIDJSON_ERROR_EN_H__ 17 | 18 | #include "error.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | 22 | //! Maps error code of parsing into error message. 23 | /*! 24 | \ingroup RAPIDJSON_ERRORS 25 | \param parseErrorCode Error code obtained in parsing. 26 | \return the error message. 27 | \note User can make a copy of this function for localization. 28 | Using switch-case is safer for future modification of error codes. 29 | */ 30 | inline const RAPIDJSON_ERROR_CHARTYPE* GetParseError_En(ParseErrorCode parseErrorCode) { 31 | switch (parseErrorCode) { 32 | case kParseErrorNone: return RAPIDJSON_ERROR_STRING("No error."); 33 | 34 | case kParseErrorDocumentEmpty: return RAPIDJSON_ERROR_STRING("The document is empty."); 35 | case kParseErrorDocumentRootNotSingular: return RAPIDJSON_ERROR_STRING("The document root must not follow by other values."); 36 | 37 | case kParseErrorValueInvalid: return RAPIDJSON_ERROR_STRING("Invalid value."); 38 | 39 | case kParseErrorObjectMissName: return RAPIDJSON_ERROR_STRING("Missing a name for object member."); 40 | case kParseErrorObjectMissColon: return RAPIDJSON_ERROR_STRING("Missing a colon after a name of object member."); 41 | case kParseErrorObjectMissCommaOrCurlyBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or '}' after an object member."); 42 | 43 | case kParseErrorArrayMissCommaOrSquareBracket: return RAPIDJSON_ERROR_STRING("Missing a comma or ']' after an array element."); 44 | 45 | case kParseErrorStringUnicodeEscapeInvalidHex: return RAPIDJSON_ERROR_STRING("Incorrect hex digit after \\u escape in string."); 46 | case kParseErrorStringUnicodeSurrogateInvalid: return RAPIDJSON_ERROR_STRING("The surrogate pair in string is invalid."); 47 | case kParseErrorStringEscapeInvalid: return RAPIDJSON_ERROR_STRING("Invalid escape character in string."); 48 | case kParseErrorStringMissQuotationMark: return RAPIDJSON_ERROR_STRING("Missing a closing quotation mark in string."); 49 | case kParseErrorStringInvalidEncoding: return RAPIDJSON_ERROR_STRING("Invalid encoding in string."); 50 | 51 | case kParseErrorNumberTooBig: return RAPIDJSON_ERROR_STRING("Number too big to be stored in double."); 52 | case kParseErrorNumberMissFraction: return RAPIDJSON_ERROR_STRING("Miss fraction part in number."); 53 | case kParseErrorNumberMissExponent: return RAPIDJSON_ERROR_STRING("Miss exponent in number."); 54 | 55 | case kParseErrorTermination: return RAPIDJSON_ERROR_STRING("Terminate parsing due to Handler error."); 56 | case kParseErrorUnspecificSyntaxError: return RAPIDJSON_ERROR_STRING("Unspecific syntax error."); 57 | 58 | default: 59 | return RAPIDJSON_ERROR_STRING("Unknown error."); 60 | } 61 | } 62 | 63 | RAPIDJSON_NAMESPACE_END 64 | 65 | #endif // RAPIDJSON_ERROR_EN_H__ 66 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/filereadstream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEREADSTREAM_H_ 16 | #define RAPIDJSON_FILEREADSTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | #include 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! File byte stream for input using fread(). 24 | /*! 25 | \note implements Stream concept 26 | */ 27 | class FileReadStream { 28 | public: 29 | typedef char Ch; //!< Character type (byte). 30 | 31 | //! Constructor. 32 | /*! 33 | \param fp File pointer opened for read. 34 | \param buffer user-supplied buffer. 35 | \param bufferSize size of buffer in bytes. Must >=4 bytes. 36 | */ 37 | FileReadStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 38 | RAPIDJSON_ASSERT(fp_ != 0); 39 | RAPIDJSON_ASSERT(bufferSize >= 4); 40 | Read(); 41 | } 42 | 43 | Ch Peek() const { return *current_; } 44 | Ch Take() { Ch c = *current_; Read(); return c; } 45 | size_t Tell() const { return count_ + static_cast(current_ - buffer_); } 46 | 47 | // Not implemented 48 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 49 | void Flush() { RAPIDJSON_ASSERT(false); } 50 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 51 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 52 | 53 | // For encoding detection only. 54 | const Ch* Peek4() const { 55 | return (current_ + 4 <= bufferLast_) ? current_ : 0; 56 | } 57 | 58 | private: 59 | void Read() { 60 | if (current_ < bufferLast_) 61 | ++current_; 62 | else if (!eof_) { 63 | count_ += readCount_; 64 | readCount_ = fread(buffer_, 1, bufferSize_, fp_); 65 | bufferLast_ = buffer_ + readCount_ - 1; 66 | current_ = buffer_; 67 | 68 | if (readCount_ < bufferSize_) { 69 | buffer_[readCount_] = '\0'; 70 | ++bufferLast_; 71 | eof_ = true; 72 | } 73 | } 74 | } 75 | 76 | std::FILE* fp_; 77 | Ch *buffer_; 78 | size_t bufferSize_; 79 | Ch *bufferLast_; 80 | Ch *current_; 81 | size_t readCount_; 82 | size_t count_; //!< Number of characters read 83 | bool eof_; 84 | }; 85 | 86 | RAPIDJSON_NAMESPACE_END 87 | 88 | #endif // RAPIDJSON_FILESTREAM_H_ 89 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/filewritestream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_FILEWRITESTREAM_H_ 16 | #define RAPIDJSON_FILEWRITESTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | #include 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! Wrapper of C file stream for input using fread(). 24 | /*! 25 | \note implements Stream concept 26 | */ 27 | class FileWriteStream { 28 | public: 29 | typedef char Ch; //!< Character type. Only support char. 30 | 31 | FileWriteStream(std::FILE* fp, char* buffer, size_t bufferSize) : fp_(fp), buffer_(buffer), bufferEnd_(buffer + bufferSize), current_(buffer_) { 32 | RAPIDJSON_ASSERT(fp_ != 0); 33 | } 34 | 35 | void Put(char c) { 36 | if (current_ >= bufferEnd_) 37 | Flush(); 38 | 39 | *current_++ = c; 40 | } 41 | 42 | void PutN(char c, size_t n) { 43 | size_t avail = static_cast(bufferEnd_ - current_); 44 | while (n > avail) { 45 | std::memset(current_, c, avail); 46 | current_ += avail; 47 | Flush(); 48 | n -= avail; 49 | avail = static_cast(bufferEnd_ - current_); 50 | } 51 | 52 | if (n > 0) { 53 | std::memset(current_, c, n); 54 | current_ += n; 55 | } 56 | } 57 | 58 | void Flush() { 59 | if (current_ != buffer_) { 60 | size_t result = fwrite(buffer_, 1, static_cast(current_ - buffer_), fp_); 61 | if (result < static_cast(current_ - buffer_)) { 62 | // failure deliberately ignored at this time 63 | // added to avoid warn_unused_result build errors 64 | } 65 | current_ = buffer_; 66 | } 67 | } 68 | 69 | // Not implemented 70 | char Peek() const { RAPIDJSON_ASSERT(false); return 0; } 71 | char Take() { RAPIDJSON_ASSERT(false); return 0; } 72 | size_t Tell() const { RAPIDJSON_ASSERT(false); return 0; } 73 | char* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 74 | size_t PutEnd(char*) { RAPIDJSON_ASSERT(false); return 0; } 75 | 76 | private: 77 | // Prohibit copy constructor & assignment operator. 78 | FileWriteStream(const FileWriteStream&); 79 | FileWriteStream& operator=(const FileWriteStream&); 80 | 81 | std::FILE* fp_; 82 | char *buffer_; 83 | char *bufferEnd_; 84 | char *current_; 85 | }; 86 | 87 | //! Implement specialized version of PutN() with memset() for better performance. 88 | template<> 89 | inline void PutN(FileWriteStream& stream, char c, size_t n) { 90 | stream.PutN(c, n); 91 | } 92 | 93 | RAPIDJSON_NAMESPACE_END 94 | 95 | #endif // RAPIDJSON_FILESTREAM_H_ 96 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/internal/ieee754.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_IEEE754_ 16 | #define RAPIDJSON_IEEE754_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | class Double { 24 | public: 25 | Double() {} 26 | Double(double d) : d_(d) {} 27 | Double(uint64_t u) : u_(u) {} 28 | 29 | double Value() const { return d_; } 30 | uint64_t Uint64Value() const { return u_; } 31 | 32 | double NextPositiveDouble() const { 33 | RAPIDJSON_ASSERT(!Sign()); 34 | return Double(u_ + 1).Value(); 35 | } 36 | 37 | bool Sign() const { return (u_ & kSignMask) != 0; } 38 | uint64_t Significand() const { return u_ & kSignificandMask; } 39 | int Exponent() const { return static_cast(((u_ & kExponentMask) >> kSignificandSize) - kExponentBias); } 40 | 41 | bool IsNan() const { return (u_ & kExponentMask) == kExponentMask && Significand() != 0; } 42 | bool IsInf() const { return (u_ & kExponentMask) == kExponentMask && Significand() == 0; } 43 | bool IsNormal() const { return (u_ & kExponentMask) != 0 || Significand() == 0; } 44 | bool IsZero() const { return (u_ & (kExponentMask | kSignificandMask)) == 0; } 45 | 46 | uint64_t IntegerSignificand() const { return IsNormal() ? Significand() | kHiddenBit : Significand(); } 47 | int IntegerExponent() const { return (IsNormal() ? Exponent() : kDenormalExponent) - kSignificandSize; } 48 | uint64_t ToBias() const { return (u_ & kSignMask) ? ~u_ + 1 : u_ | kSignMask; } 49 | 50 | static unsigned EffectiveSignificandSize(int order) { 51 | if (order >= -1021) 52 | return 53; 53 | else if (order <= -1074) 54 | return 0; 55 | else 56 | return (unsigned)order + 1074; 57 | } 58 | 59 | private: 60 | static const int kSignificandSize = 52; 61 | static const int kExponentBias = 0x3FF; 62 | static const int kDenormalExponent = 1 - kExponentBias; 63 | static const uint64_t kSignMask = RAPIDJSON_UINT64_C2(0x80000000, 0x00000000); 64 | static const uint64_t kExponentMask = RAPIDJSON_UINT64_C2(0x7FF00000, 0x00000000); 65 | static const uint64_t kSignificandMask = RAPIDJSON_UINT64_C2(0x000FFFFF, 0xFFFFFFFF); 66 | static const uint64_t kHiddenBit = RAPIDJSON_UINT64_C2(0x00100000, 0x00000000); 67 | 68 | union { 69 | double d_; 70 | uint64_t u_; 71 | }; 72 | }; 73 | 74 | } // namespace internal 75 | RAPIDJSON_NAMESPACE_END 76 | 77 | #endif // RAPIDJSON_IEEE754_ 78 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/internal/pow10.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_POW10_ 16 | #define RAPIDJSON_POW10_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Computes integer powers of 10 in double (10.0^n). 24 | /*! This function uses lookup table for fast and accurate results. 25 | \param n non-negative exponent. Must <= 308. 26 | \return 10.0^n 27 | */ 28 | inline double Pow10(int n) { 29 | static const double e[] = { // 1e-0...1e308: 309 * 8 bytes = 2472 bytes 30 | 1e+0, 31 | 1e+1, 1e+2, 1e+3, 1e+4, 1e+5, 1e+6, 1e+7, 1e+8, 1e+9, 1e+10, 1e+11, 1e+12, 1e+13, 1e+14, 1e+15, 1e+16, 1e+17, 1e+18, 1e+19, 1e+20, 32 | 1e+21, 1e+22, 1e+23, 1e+24, 1e+25, 1e+26, 1e+27, 1e+28, 1e+29, 1e+30, 1e+31, 1e+32, 1e+33, 1e+34, 1e+35, 1e+36, 1e+37, 1e+38, 1e+39, 1e+40, 33 | 1e+41, 1e+42, 1e+43, 1e+44, 1e+45, 1e+46, 1e+47, 1e+48, 1e+49, 1e+50, 1e+51, 1e+52, 1e+53, 1e+54, 1e+55, 1e+56, 1e+57, 1e+58, 1e+59, 1e+60, 34 | 1e+61, 1e+62, 1e+63, 1e+64, 1e+65, 1e+66, 1e+67, 1e+68, 1e+69, 1e+70, 1e+71, 1e+72, 1e+73, 1e+74, 1e+75, 1e+76, 1e+77, 1e+78, 1e+79, 1e+80, 35 | 1e+81, 1e+82, 1e+83, 1e+84, 1e+85, 1e+86, 1e+87, 1e+88, 1e+89, 1e+90, 1e+91, 1e+92, 1e+93, 1e+94, 1e+95, 1e+96, 1e+97, 1e+98, 1e+99, 1e+100, 36 | 1e+101,1e+102,1e+103,1e+104,1e+105,1e+106,1e+107,1e+108,1e+109,1e+110,1e+111,1e+112,1e+113,1e+114,1e+115,1e+116,1e+117,1e+118,1e+119,1e+120, 37 | 1e+121,1e+122,1e+123,1e+124,1e+125,1e+126,1e+127,1e+128,1e+129,1e+130,1e+131,1e+132,1e+133,1e+134,1e+135,1e+136,1e+137,1e+138,1e+139,1e+140, 38 | 1e+141,1e+142,1e+143,1e+144,1e+145,1e+146,1e+147,1e+148,1e+149,1e+150,1e+151,1e+152,1e+153,1e+154,1e+155,1e+156,1e+157,1e+158,1e+159,1e+160, 39 | 1e+161,1e+162,1e+163,1e+164,1e+165,1e+166,1e+167,1e+168,1e+169,1e+170,1e+171,1e+172,1e+173,1e+174,1e+175,1e+176,1e+177,1e+178,1e+179,1e+180, 40 | 1e+181,1e+182,1e+183,1e+184,1e+185,1e+186,1e+187,1e+188,1e+189,1e+190,1e+191,1e+192,1e+193,1e+194,1e+195,1e+196,1e+197,1e+198,1e+199,1e+200, 41 | 1e+201,1e+202,1e+203,1e+204,1e+205,1e+206,1e+207,1e+208,1e+209,1e+210,1e+211,1e+212,1e+213,1e+214,1e+215,1e+216,1e+217,1e+218,1e+219,1e+220, 42 | 1e+221,1e+222,1e+223,1e+224,1e+225,1e+226,1e+227,1e+228,1e+229,1e+230,1e+231,1e+232,1e+233,1e+234,1e+235,1e+236,1e+237,1e+238,1e+239,1e+240, 43 | 1e+241,1e+242,1e+243,1e+244,1e+245,1e+246,1e+247,1e+248,1e+249,1e+250,1e+251,1e+252,1e+253,1e+254,1e+255,1e+256,1e+257,1e+258,1e+259,1e+260, 44 | 1e+261,1e+262,1e+263,1e+264,1e+265,1e+266,1e+267,1e+268,1e+269,1e+270,1e+271,1e+272,1e+273,1e+274,1e+275,1e+276,1e+277,1e+278,1e+279,1e+280, 45 | 1e+281,1e+282,1e+283,1e+284,1e+285,1e+286,1e+287,1e+288,1e+289,1e+290,1e+291,1e+292,1e+293,1e+294,1e+295,1e+296,1e+297,1e+298,1e+299,1e+300, 46 | 1e+301,1e+302,1e+303,1e+304,1e+305,1e+306,1e+307,1e+308 47 | }; 48 | RAPIDJSON_ASSERT(n >= 0 && n <= 308); 49 | return e[n]; 50 | } 51 | 52 | } // namespace internal 53 | RAPIDJSON_NAMESPACE_END 54 | 55 | #endif // RAPIDJSON_POW10_ 56 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/internal/strfunc.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_STRFUNC_H_ 16 | #define RAPIDJSON_INTERNAL_STRFUNC_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Custom strlen() which works on different character types. 24 | /*! \tparam Ch Character type (e.g. char, wchar_t, short) 25 | \param s Null-terminated input string. 26 | \return Number of characters in the string. 27 | \note This has the same semantics as strlen(), the return value is not number of Unicode codepoints. 28 | */ 29 | template 30 | inline SizeType StrLen(const Ch* s) { 31 | const Ch* p = s; 32 | while (*p) ++p; 33 | return SizeType(p - s); 34 | } 35 | 36 | } // namespace internal 37 | RAPIDJSON_NAMESPACE_END 38 | 39 | #endif // RAPIDJSON_INTERNAL_STRFUNC_H_ 40 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/internal/swap.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_INTERNAL_SWAP_H_ 16 | #define RAPIDJSON_INTERNAL_SWAP_H_ 17 | 18 | #include "../rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | namespace internal { 22 | 23 | //! Custom swap() to avoid dependency on C++ header 24 | /*! \tparam T Type of the arguments to swap, should be instantiated with primitive C++ types only. 25 | \note This has the same semantics as std::swap(). 26 | */ 27 | template 28 | inline void Swap(T& a, T& b) RAPIDJSON_NOEXCEPT { 29 | T tmp = a; 30 | a = b; 31 | b = tmp; 32 | } 33 | 34 | } // namespace internal 35 | RAPIDJSON_NAMESPACE_END 36 | 37 | #endif // RAPIDJSON_INTERNAL_SWAP_H_ 38 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/memorybuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYBUFFER_H_ 16 | #define RAPIDJSON_MEMORYBUFFER_H_ 17 | 18 | #include "rapidjson.h" 19 | #include "internal/stack.h" 20 | 21 | RAPIDJSON_NAMESPACE_BEGIN 22 | 23 | //! Represents an in-memory output byte stream. 24 | /*! 25 | This class is mainly for being wrapped by EncodedOutputStream or AutoUTFOutputStream. 26 | 27 | It is similar to FileWriteBuffer but the destination is an in-memory buffer instead of a file. 28 | 29 | Differences between MemoryBuffer and StringBuffer: 30 | 1. StringBuffer has Encoding but MemoryBuffer is only a byte buffer. 31 | 2. StringBuffer::GetString() returns a null-terminated string. MemoryBuffer::GetBuffer() returns a buffer without terminator. 32 | 33 | \tparam Allocator type for allocating memory buffer. 34 | \note implements Stream concept 35 | */ 36 | template 37 | struct GenericMemoryBuffer { 38 | typedef char Ch; // byte 39 | 40 | GenericMemoryBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 41 | 42 | void Put(Ch c) { *stack_.template Push() = c; } 43 | void Flush() {} 44 | 45 | void Clear() { stack_.Clear(); } 46 | void ShrinkToFit() { stack_.ShrinkToFit(); } 47 | Ch* Push(size_t count) { return stack_.template Push(count); } 48 | void Pop(size_t count) { stack_.template Pop(count); } 49 | 50 | const Ch* GetBuffer() const { 51 | return stack_.template Bottom(); 52 | } 53 | 54 | size_t GetSize() const { return stack_.GetSize(); } 55 | 56 | static const size_t kDefaultCapacity = 256; 57 | mutable internal::Stack stack_; 58 | }; 59 | 60 | typedef GenericMemoryBuffer<> MemoryBuffer; 61 | 62 | //! Implement specialized version of PutN() with memset() for better performance. 63 | template<> 64 | inline void PutN(MemoryBuffer& memoryBuffer, char c, size_t n) { 65 | std::memset(memoryBuffer.stack_.Push(n), c, n * sizeof(c)); 66 | } 67 | 68 | RAPIDJSON_NAMESPACE_END 69 | 70 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 71 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/memorystream.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_MEMORYSTREAM_H_ 16 | #define RAPIDJSON_MEMORYSTREAM_H_ 17 | 18 | #include "rapidjson.h" 19 | 20 | RAPIDJSON_NAMESPACE_BEGIN 21 | 22 | //! Represents an in-memory input byte stream. 23 | /*! 24 | This class is mainly for being wrapped by EncodedInputStream or AutoUTFInputStream. 25 | 26 | It is similar to FileReadBuffer but the source is an in-memory buffer instead of a file. 27 | 28 | Differences between MemoryStream and StringStream: 29 | 1. StringStream has encoding but MemoryStream is a byte stream. 30 | 2. MemoryStream needs size of the source buffer and the buffer don't need to be null terminated. StringStream assume null-terminated string as source. 31 | 3. MemoryStream supports Peek4() for encoding detection. StringStream is specified with an encoding so it should not have Peek4(). 32 | \note implements Stream concept 33 | */ 34 | struct MemoryStream { 35 | typedef char Ch; // byte 36 | 37 | MemoryStream(const Ch *src, size_t size) : src_(src), begin_(src), end_(src + size), size_(size) {} 38 | 39 | Ch Peek() const { return (src_ == end_) ? '\0' : *src_; } 40 | Ch Take() { return (src_ == end_) ? '\0' : *src_++; } 41 | size_t Tell() const { return static_cast(src_ - begin_); } 42 | 43 | Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; } 44 | void Put(Ch) { RAPIDJSON_ASSERT(false); } 45 | void Flush() { RAPIDJSON_ASSERT(false); } 46 | size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; } 47 | 48 | // For encoding detection only. 49 | const Ch* Peek4() const { 50 | return Tell() + 4 <= size_ ? src_ : 0; 51 | } 52 | 53 | const Ch* src_; //!< Current read position. 54 | const Ch* begin_; //!< Original head of the string. 55 | const Ch* end_; //!< End of stream. 56 | size_t size_; //!< Size of the stream. 57 | }; 58 | 59 | RAPIDJSON_NAMESPACE_END 60 | 61 | #endif // RAPIDJSON_MEMORYBUFFER_H_ 62 | -------------------------------------------------------------------------------- /src/mica/util/rapidjson/stringbuffer.h: -------------------------------------------------------------------------------- 1 | // Tencent is pleased to support the open source community by making RapidJSON available. 2 | // 3 | // Copyright (C) 2015 THL A29 Limited, a Tencent company, and Milo Yip. All rights reserved. 4 | // 5 | // Licensed under the MIT License (the "License"); you may not use this file except 6 | // in compliance with the License. You may obtain a copy of the License at 7 | // 8 | // http://opensource.org/licenses/MIT 9 | // 10 | // Unless required by applicable law or agreed to in writing, software distributed 11 | // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 12 | // CONDITIONS OF ANY KIND, either express or implied. See the License for the 13 | // specific language governing permissions and limitations under the License. 14 | 15 | #ifndef RAPIDJSON_STRINGBUFFER_H_ 16 | #define RAPIDJSON_STRINGBUFFER_H_ 17 | 18 | #include "rapidjson.h" 19 | 20 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 21 | #include // std::move 22 | #endif 23 | 24 | #include "internal/stack.h" 25 | 26 | RAPIDJSON_NAMESPACE_BEGIN 27 | 28 | //! Represents an in-memory output stream. 29 | /*! 30 | \tparam Encoding Encoding of the stream. 31 | \tparam Allocator type for allocating memory buffer. 32 | \note implements Stream concept 33 | */ 34 | template 35 | class GenericStringBuffer { 36 | public: 37 | typedef typename Encoding::Ch Ch; 38 | 39 | GenericStringBuffer(Allocator* allocator = 0, size_t capacity = kDefaultCapacity) : stack_(allocator, capacity) {} 40 | 41 | #if RAPIDJSON_HAS_CXX11_RVALUE_REFS 42 | GenericStringBuffer(GenericStringBuffer&& rhs) : stack_(std::move(rhs.stack_)) {} 43 | GenericStringBuffer& operator=(GenericStringBuffer&& rhs) { 44 | if (&rhs != this) 45 | stack_ = std::move(rhs.stack_); 46 | return *this; 47 | } 48 | #endif 49 | 50 | void Put(Ch c) { *stack_.template Push() = c; } 51 | void Flush() {} 52 | 53 | void Clear() { stack_.Clear(); } 54 | void ShrinkToFit() { 55 | // Push and pop a null terminator. This is safe. 56 | *stack_.template Push() = '\0'; 57 | stack_.ShrinkToFit(); 58 | stack_.template Pop(1); 59 | } 60 | Ch* Push(size_t count) { return stack_.template Push(count); } 61 | void Pop(size_t count) { stack_.template Pop(count); } 62 | 63 | const Ch* GetString() const { 64 | // Push and pop a null terminator. This is safe. 65 | *stack_.template Push() = '\0'; 66 | stack_.template Pop(1); 67 | 68 | return stack_.template Bottom(); 69 | } 70 | 71 | size_t GetSize() const { return stack_.GetSize(); } 72 | 73 | static const size_t kDefaultCapacity = 256; 74 | mutable internal::Stack stack_; 75 | 76 | private: 77 | // Prohibit copy constructor & assignment operator. 78 | GenericStringBuffer(const GenericStringBuffer&); 79 | GenericStringBuffer& operator=(const GenericStringBuffer&); 80 | }; 81 | 82 | //! String buffer with UTF8 encoding 83 | typedef GenericStringBuffer > StringBuffer; 84 | 85 | //! Implement specialized version of PutN() with memset() for better performance. 86 | template<> 87 | inline void PutN(GenericStringBuffer >& stream, char c, size_t n) { 88 | std::memset(stream.stack_.Push(n), c, n * sizeof(c)); 89 | } 90 | 91 | RAPIDJSON_NAMESPACE_END 92 | 93 | #endif // RAPIDJSON_STRINGBUFFER_H_ 94 | -------------------------------------------------------------------------------- /src/mica/util/rate_limiter.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_RATE_LIMITER_H_ 3 | #define MICA_UTIL_RATE_LIMITER_H_ 4 | 5 | #include "mica/common.h" 6 | #include 7 | #include 8 | #include "mica/util/rand.h" 9 | 10 | namespace mica { 11 | namespace util { 12 | class RateLimiter { 13 | public: 14 | void remove_tokens(double v); 15 | bool try_remove_tokens(double v); 16 | void remove_tokens_nowait(double v); 17 | void add_tokens(double v); 18 | }; 19 | 20 | class RegularRateLimiter : public RateLimiter { 21 | public: 22 | RegularRateLimiter(const Stopwatch& sw, double initial_tokens, 23 | double max_tokens, double new_tokens_per_cycle) 24 | : sw_(sw), 25 | tokens_(initial_tokens), 26 | max_tokens_(max_tokens), 27 | new_tokens_per_cycle_(new_tokens_per_cycle) { 28 | // printf("tokens=%lf\n", tokens_); 29 | // printf("max_tokens=%lf\n", max_tokens_); 30 | // printf("new_tokens_per_cycle=%lf\n", new_tokens_per_cycle); 31 | 32 | last_time_ = sw_.now(); 33 | } 34 | 35 | void remove_tokens(double v) { 36 | while (true) { 37 | update_tokens(); 38 | 39 | if (tokens_ < v) { 40 | ::mica::util::pause(); 41 | continue; 42 | } 43 | 44 | tokens_ -= v; 45 | } 46 | } 47 | 48 | bool try_remove_tokens(double v) { 49 | update_tokens(); 50 | 51 | if (tokens_ < v) return false; 52 | 53 | tokens_ -= v; 54 | return true; 55 | } 56 | 57 | void remove_tokens_nowait(double v) { tokens_ -= v; } 58 | 59 | void add_tokens(double v) { tokens_ += v; } 60 | 61 | private: 62 | void update_tokens() { 63 | uint64_t current_time = sw_.now(); 64 | 65 | double new_cycles = static_cast(current_time - last_time_); 66 | 67 | last_time_ = current_time; 68 | 69 | tokens_ = 70 | std::min(max_tokens_, tokens_ + new_cycles * new_tokens_per_cycle_); 71 | } 72 | 73 | private: 74 | const Stopwatch& sw_; 75 | 76 | double tokens_; 77 | double max_tokens_; 78 | double new_tokens_per_cycle_; 79 | 80 | uint64_t last_time_; 81 | }; 82 | 83 | class ExponentialRateLimiter : public RateLimiter { 84 | public: 85 | ExponentialRateLimiter(const Stopwatch& sw, double initial_tokens, 86 | double max_tokens, double new_tokens_per_cycle, 87 | uint64_t seed) 88 | : sw_(sw), 89 | tokens_(initial_tokens), 90 | max_tokens_(max_tokens), 91 | new_tokens_per_cycle_(new_tokens_per_cycle), 92 | cycles_per_new_token_(1. / new_tokens_per_cycle), 93 | rand_(seed) { 94 | // printf("tokens=%lf\n", tokens_); 95 | // printf("max_tokens=%lf\n", max_tokens_); 96 | // printf("new_tokens_per_cycle=%lf\n", new_tokens_per_cycle); 97 | 98 | last_time_ = sw_.now(); 99 | update_cycles_for_next_token(); 100 | } 101 | 102 | void remove_tokens(double v) { 103 | while (true) { 104 | update_tokens(); 105 | 106 | if (tokens_ < v) { 107 | ::mica::util::pause(); 108 | continue; 109 | } 110 | 111 | tokens_ -= v; 112 | } 113 | } 114 | 115 | bool try_remove_tokens(double v) { 116 | update_tokens(); 117 | 118 | if (tokens_ < v) return false; 119 | 120 | tokens_ -= v; 121 | return true; 122 | } 123 | 124 | void remove_tokens_nowait(double v) { tokens_ -= v; } 125 | 126 | void add_tokens(double v) { tokens_ += v; } 127 | 128 | private: 129 | void update_tokens() { 130 | uint64_t current_time = sw_.now(); 131 | 132 | uint64_t new_cycles = current_time - last_time_; 133 | while (new_cycles >= cycles_for_next_token_) { 134 | tokens_ += 1.; 135 | new_cycles -= cycles_for_next_token_; 136 | last_time_ += cycles_for_next_token_; 137 | 138 | update_cycles_for_next_token(); 139 | 140 | if (tokens_ >= max_tokens_) { 141 | tokens_ = max_tokens_; 142 | last_time_ = current_time; 143 | break; 144 | } 145 | } 146 | } 147 | 148 | void update_cycles_for_next_token() { 149 | double z; 150 | do { 151 | z = rand_.next_f64(); 152 | } while (z >= 1.); 153 | 154 | cycles_for_next_token_ = 155 | static_cast(-cycles_per_new_token_ * log(1. - z)); 156 | } 157 | 158 | private: 159 | const Stopwatch& sw_; 160 | 161 | double tokens_; 162 | double max_tokens_; 163 | double new_tokens_per_cycle_; 164 | 165 | double cycles_per_new_token_; 166 | Rand rand_; 167 | 168 | uint64_t last_time_; 169 | uint64_t cycles_for_next_token_; 170 | }; 171 | } 172 | } 173 | 174 | #endif -------------------------------------------------------------------------------- /src/mica/util/roundup.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_ROUNDUP_H_ 3 | #define MICA_UTIL_ROUNDUP_H_ 4 | 5 | #include "mica/common.h" 6 | 7 | namespace mica { 8 | namespace util { 9 | template 10 | static constexpr bool is_power_of_two(T x) { 11 | return x && ((x & T(x - 1)) == 0); 12 | } 13 | 14 | template 15 | static constexpr T roundup(T x) { 16 | static_assert(is_power_of_two(PowerOfTwoNumber), 17 | "PowerOfTwoNumber must be a power of 2"); 18 | return ((x) + T(PowerOfTwoNumber - 1)) & (~T(PowerOfTwoNumber - 1)); 19 | } 20 | 21 | #pragma GCC diagnostic push 22 | #pragma GCC diagnostic ignored "-Winline" 23 | 24 | template 25 | static constexpr T next_power_of_two_recursive(T x, T s) { 26 | return ((T(1) << s) < x) ? next_power_of_two_recursive(x, s + 1) 27 | : (T(1) << s); 28 | } 29 | 30 | template 31 | static constexpr T next_power_of_two(T x) { 32 | // T s(0); 33 | // while ((T(1) << s) < x) s++; 34 | // return T(1) << s; 35 | return next_power_of_two_recursive(x, T(0)); 36 | } 37 | 38 | #pragma GCC diagnostic pop 39 | } 40 | } 41 | 42 | #endif -------------------------------------------------------------------------------- /src/mica/util/safe_cast.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_SAFE_CAST_H_ 3 | #define MICA_UTIL_SAFE_CAST_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "mica/common.h" 9 | #define NEEDS_NULLPTR_DEFINED 0 10 | #include "mica/util/SafeInt/SafeInt3_mod.hpp" 11 | #undef NEEDS_NULLPTR_DEFINED 12 | 13 | namespace mica { 14 | namespace util { 15 | template 16 | static To safe_cast(From x) noexcept { 17 | try { 18 | return static_cast(SafeInt(x)); 19 | } catch (...) { 20 | fprintf(stderr, 21 | "error: unable to cast safely due to an out-of-range error\n"); 22 | assert(false); 23 | return To(0); 24 | } 25 | // return static_cast(x); 26 | } 27 | } 28 | } 29 | 30 | #endif -------------------------------------------------------------------------------- /src/mica/util/stopwatch.cc: -------------------------------------------------------------------------------- 1 | // #pragma once 2 | #ifndef MICA_UTIL_STOPWATCH_CC_ 3 | #define MICA_UTIL_STOPWATCH_CC_ 4 | 5 | #include "mica/util/stopwatch.h" 6 | #include "mica/util/barrier.h" 7 | 8 | namespace mica { 9 | namespace util { 10 | void Stopwatch::init_start() { 11 | init_t_ = now(); 12 | gettimeofday(&init_tv_, nullptr); 13 | } 14 | 15 | void Stopwatch::init_end() { 16 | struct timeval tv; 17 | 18 | const uint64_t min_time = 100000UL; // 100,000 us = 0.1 sec 19 | 20 | while (true) { 21 | gettimeofday(&tv, nullptr); 22 | 23 | uint64_t diff = 24 | static_cast(tv.tv_sec - init_tv_.tv_sec) * 1000000UL + 25 | static_cast(tv.tv_usec - init_tv_.tv_usec); 26 | 27 | if (diff >= min_time) { 28 | uint64_t t = now(); 29 | c_1_sec_ = (t - init_t_) * min_time * 10 / diff; 30 | c_1_msec_ = c_1_sec_ / 1000; 31 | c_1_usec_ = c_1_msec_ / 1000; 32 | c_1_nsec_ = c_1_usec_ / 1000; 33 | break; 34 | } 35 | 36 | pause(); 37 | } 38 | } 39 | } 40 | } 41 | 42 | #endif 43 | -------------------------------------------------------------------------------- /src/mica/util/stopwatch.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_STOPWATCH_H_ 3 | #define MICA_UTIL_STOPWATCH_H_ 4 | 5 | #include 6 | #include "mica/common.h" 7 | #include "mica/util/tsc.h" 8 | 9 | namespace mica { 10 | namespace util { 11 | class Stopwatch { 12 | public: 13 | void init_start(); 14 | void init_end(); 15 | 16 | uint64_t now() const { return rdtsc(); } 17 | 18 | uint64_t c_1_sec() const { return c_1_sec_; } 19 | uint64_t c_1_msec() const { return c_1_msec_; } 20 | uint64_t c_1_usec() const { return c_1_usec_; } 21 | uint64_t c_1_nsec() const { return c_1_nsec_; } // Usually not very accurate. 22 | 23 | uint64_t diff_in_cycles(uint64_t new_t, uint64_t old_t) const { 24 | return new_t - old_t; 25 | } 26 | 27 | uint64_t diff_in_ms(uint64_t new_t, uint64_t old_t) const { 28 | return (new_t - old_t) * 1000UL / c_1_sec_; 29 | } 30 | 31 | uint64_t diff_in_us(uint64_t new_t, uint64_t old_t) const { 32 | return (new_t - old_t) * 1000000UL / c_1_sec_; 33 | } 34 | 35 | uint64_t diff_in_ns(uint64_t new_t, uint64_t old_t) const { 36 | // XXX: This may overflow! 37 | return (new_t - old_t) * 1000000000UL / c_1_sec_; 38 | } 39 | 40 | double diff(uint64_t new_t, uint64_t old_t) const { 41 | return static_cast(diff_in_cycles(new_t, old_t)) / 42 | static_cast(c_1_sec_); 43 | } 44 | 45 | private: 46 | uint64_t c_1_sec_; 47 | uint64_t c_1_msec_; 48 | uint64_t c_1_usec_; 49 | uint64_t c_1_nsec_; 50 | 51 | struct timeval init_tv_; 52 | uint64_t init_t_; 53 | }; 54 | } 55 | } 56 | 57 | #endif -------------------------------------------------------------------------------- /src/mica/util/tsc.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_TSC_H_ 3 | #define MICA_UTIL_TSC_H_ 4 | 5 | #include "mica/common.h" 6 | 7 | namespace mica { 8 | namespace util { 9 | static uint64_t rdtsc() { 10 | uint64_t rax; 11 | uint64_t rdx; 12 | asm volatile("rdtsc" : "=a"(rax), "=d"(rdx)); 13 | return (rdx << 32) | rax; 14 | } 15 | 16 | static uint64_t rdtscp() { 17 | uint64_t rax; 18 | uint64_t rdx; 19 | uint32_t aux; 20 | asm volatile("rdtscp" : "=a"(rax), "=d"(rdx), "=c"(aux) : :); 21 | return (rdx << 32) | rax; 22 | } 23 | } 24 | } 25 | 26 | #endif -------------------------------------------------------------------------------- /src/mica/util/type_traits.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_TYPE_TRAITS_H_ 3 | #define MICA_UTIL_TYPE_TRAITS_H_ 4 | 5 | #include 6 | 7 | namespace std { 8 | // Fix compiliers that claim is_trivially_copyable>::value == false. 9 | template 10 | struct is_trivially_copyable> { 11 | static constexpr bool value = is_trivially_copyable::value; 12 | }; 13 | } 14 | 15 | #endif -------------------------------------------------------------------------------- /src/mica/util/zipf.cc: -------------------------------------------------------------------------------- 1 | // #pragma once 2 | #ifndef MICA_UTIL_ZIPF_CC_ 3 | #define MICA_UTIL_ZIPF_CC_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "mica/util/zipf.h" 9 | 10 | namespace mica { 11 | namespace util { 12 | double ZipfGen::pow_approx(double a, double b) { 13 | // from 14 | // http://martin.ankerl.com/2012/01/25/optimized-approximative-pow-in-c-and-cpp/ 15 | 16 | // calculate approximation with fraction of the exponent 17 | int e = (int)b; 18 | union { 19 | double d; 20 | int x[2]; 21 | } u = {a}; 22 | u.x[1] = (int)((b - (double)e) * (double)(u.x[1] - 1072632447) + 1072632447.); 23 | u.x[0] = 0; 24 | 25 | // exponentiation by squaring with the exponent's integer part 26 | // double r = u.d makes everything much slower, not sure why 27 | // TODO: use popcount? 28 | double r = 1.; 29 | while (e) { 30 | if (e & 1) r *= a; 31 | a *= a; 32 | e >>= 1; 33 | } 34 | 35 | return r * u.d; 36 | } 37 | 38 | ZipfGen::ZipfGen(uint64_t n, double theta, uint64_t rand_seed) { 39 | assert(n > 0); 40 | if (theta > 0.992 && theta < 1) 41 | fprintf(stderr, 42 | "warning: theta > 0.992 will be inaccurate due to approximation\n"); 43 | if (theta >= 1. && theta < 40.) { 44 | fprintf(stderr, "error: theta in [1., 40.) is not supported\n"); 45 | assert(false); 46 | theta_ = 0; // unused 47 | alpha_ = 0; // unused 48 | thres_ = 0; // unused 49 | return; 50 | } 51 | assert(theta == -1. || (theta >= 0. && theta < 1.) || theta >= 40.); 52 | n_ = n; 53 | theta_ = theta; 54 | if (theta == -1.) { 55 | seq_ = rand_seed % n; 56 | alpha_ = 0; // unused 57 | thres_ = 0; // unused 58 | } else if (theta > 0. && theta < 1.) { 59 | seq_ = 0; // unused 60 | alpha_ = 1. / (1. - theta); 61 | thres_ = 1. + pow_approx(0.5, theta); 62 | } else { 63 | seq_ = 0; // unused 64 | alpha_ = 0.; // unused 65 | thres_ = 0.; // unused 66 | } 67 | last_n_ = 0; 68 | zetan_ = 0.; 69 | eta_ = 0; 70 | // rand_state_[0] = (unsigned short)(rand_seed >> 0); 71 | // rand_state_[1] = (unsigned short)(rand_seed >> 16); 72 | // rand_state_[2] = (unsigned short)(rand_seed >> 32); 73 | rand_ = Rand(rand_seed); 74 | } 75 | 76 | ZipfGen::ZipfGen(const ZipfGen& src) { 77 | n_ = src.n_; 78 | theta_ = src.theta_; 79 | alpha_ = src.alpha_; 80 | thres_ = src.thres_; 81 | last_n_ = src.last_n_; 82 | dbl_n_ = src.dbl_n_; 83 | zetan_ = src.zetan_; 84 | eta_ = src.eta_; 85 | seq_ = src.seq_; 86 | rand_ = src.rand_; 87 | } 88 | 89 | ZipfGen::ZipfGen(const ZipfGen& src, uint64_t rand_seed) { 90 | n_ = src.n_; 91 | theta_ = src.theta_; 92 | alpha_ = src.alpha_; 93 | thres_ = src.thres_; 94 | last_n_ = src.last_n_; 95 | dbl_n_ = src.dbl_n_; 96 | zetan_ = src.zetan_; 97 | eta_ = src.eta_; 98 | seq_ = src.seq_; 99 | rand_ = Rand(rand_seed); 100 | } 101 | 102 | ZipfGen& ZipfGen::operator=(const ZipfGen& src) { 103 | n_ = src.n_; 104 | theta_ = src.theta_; 105 | alpha_ = src.alpha_; 106 | thres_ = src.thres_; 107 | last_n_ = src.last_n_; 108 | dbl_n_ = src.dbl_n_; 109 | zetan_ = src.zetan_; 110 | eta_ = src.eta_; 111 | seq_ = src.seq_; 112 | rand_ = src.rand_; 113 | return *this; 114 | } 115 | 116 | void ZipfGen::change_n(uint64_t n) { n_ = n; } 117 | 118 | double ZipfGen::zeta(uint64_t last_n, double last_sum, uint64_t n, 119 | double theta) { 120 | if (last_n > n) { 121 | last_n = 0; 122 | last_sum = 0.; 123 | } 124 | while (last_n < n) { 125 | last_sum += 1. / pow_approx((double)last_n + 1., theta); 126 | last_n++; 127 | } 128 | return last_sum; 129 | } 130 | 131 | uint64_t ZipfGen::next() { 132 | if (last_n_ != n_) { 133 | if (theta_ > 0. && theta_ < 1.) { 134 | zetan_ = zeta(last_n_, zetan_, n_, theta_); 135 | eta_ = (1. - pow_approx(2. / (double)n_, 1. - theta_)) / 136 | (1. - zeta(0, 0., 2, theta_) / zetan_); 137 | } 138 | last_n_ = n_; 139 | dbl_n_ = (double)n_; 140 | } 141 | 142 | if (theta_ == -1.) { 143 | uint64_t v = seq_; 144 | if (++seq_ >= n_) seq_ = 0; 145 | return v; 146 | } else if (theta_ == 0.) { 147 | double u = rand_.next_f64(); 148 | return (uint64_t)(dbl_n_ * u); 149 | } else if (theta_ >= 40.) { 150 | return 0UL; 151 | } else { 152 | // from J. Gray et al. Quickly generating billion-record synthetic 153 | // databases. In SIGMOD, 1994. 154 | 155 | // double u = erand48(rand_state_); 156 | double u = rand_.next_f64(); 157 | double uz = u * zetan_; 158 | if (uz < 1.) 159 | return 0UL; 160 | else if (uz < thres_) 161 | return 1UL; 162 | else { 163 | uint64_t v = 164 | (uint64_t)(dbl_n_ * pow_approx(eta_ * (u - 1.) + 1., alpha_)); 165 | if (v >= n_) v = n_ - 1; 166 | return v; 167 | } 168 | } 169 | } 170 | 171 | void ZipfGen::test(double theta) { 172 | double zetan = 0.; 173 | const uint64_t n = 1000000UL; 174 | uint64_t i; 175 | 176 | for (i = 0; i < n; i++) zetan += 1. / pow((double)i + 1., theta); 177 | 178 | if (theta < 1. || theta >= 40.) { 179 | ZipfGen zg(n, theta, 0); 180 | 181 | uint64_t num_key0 = 0; 182 | const uint64_t num_samples = 10000000UL; 183 | if (theta < 1. || theta >= 40.) { 184 | for (i = 0; i < num_samples; i++) 185 | if (zg.next() == 0) num_key0++; 186 | } 187 | 188 | printf("theta = %lf; using pow(): %.10lf", theta, 1. / zetan); 189 | if (theta < 1. || theta >= 40.) 190 | printf(", using approx-pow(): %.10lf", 191 | (double)num_key0 / (double)num_samples); 192 | printf("\n"); 193 | } 194 | } 195 | } 196 | } 197 | 198 | #endif -------------------------------------------------------------------------------- /src/mica/util/zipf.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | #ifndef MICA_UTIL_ZIPF_H_ 3 | #define MICA_UTIL_ZIPF_H_ 4 | 5 | #include "mica/common.h" 6 | #include "mica/util/rand.h" 7 | 8 | namespace mica { 9 | namespace util { 10 | class ZipfGen { 11 | public: 12 | ZipfGen(uint64_t n, double theta, uint64_t rand_seed); 13 | ZipfGen(const ZipfGen& src); 14 | ZipfGen(const ZipfGen& src, uint64_t rand_seed); 15 | ZipfGen& operator=(const ZipfGen& src); 16 | 17 | void change_n(uint64_t n); 18 | uint64_t next(); 19 | 20 | static void test(double theta); 21 | 22 | private: 23 | static double pow_approx(double a, double b); 24 | 25 | static double zeta(uint64_t last_n, double last_sum, uint64_t n, 26 | double theta); 27 | 28 | uint64_t n_; // number of items (input) 29 | double 30 | theta_; // skewness (input) in (0, 1); or, 0 = uniform, 1 = always zero 31 | double alpha_; // only depends on theta 32 | double thres_; // only depends on theta 33 | uint64_t last_n_; // last n used to calculate the following 34 | double dbl_n_; 35 | double zetan_; 36 | double eta_; 37 | // unsigned short rand_state[3]; // prng state 38 | uint64_t seq_; // for sequential number generation 39 | Rand rand_; 40 | } __attribute__((aligned(128))); // To prevent false sharing caused by 41 | // adjacent cacheline prefetching. 42 | } 43 | } 44 | 45 | #endif --------------------------------------------------------------------------------