├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── ChangeLog ├── LICENSE ├── README.md ├── benchmarks ├── CMakeLists.txt ├── bench_micro.c ├── bench_simul.c ├── benchmark_time.c ├── benchmark_time.h ├── rand.c ├── rand.h └── tools │ ├── .gitignore │ ├── benchconfig.py.example │ ├── latency_plot.py │ └── latency_run.py ├── cmake_uninstall.cmake.in ├── codecov.yml ├── doc ├── CMakeLists.txt └── vmemcache.md ├── libvmemcache.pc.in ├── packages.cmake ├── src ├── CMakeLists.txt ├── common.h ├── critnib.c ├── critnib.h ├── fast-hash.c ├── fast-hash.h ├── file.c ├── file.h ├── file_posix.c ├── libvmemcache.c ├── libvmemcache.h ├── libvmemcache.map ├── mmap.c ├── mmap.h ├── mmap_posix.c ├── os.h ├── os_posix.c ├── os_thread.h ├── os_thread_posix.c ├── out.c ├── out.h ├── ringbuf.c ├── ringbuf.h ├── sys │ └── queue.h ├── sys_util.h ├── util.c ├── util.h ├── util_posix.c ├── valgrind_internal.h ├── vmemcache.c ├── vmemcache.h ├── vmemcache_heap.c ├── vmemcache_heap.h ├── vmemcache_index.c ├── vmemcache_index.h ├── vmemcache_repl.c └── vmemcache_repl.h ├── tests ├── CMakeLists.txt ├── drd-log.supp ├── example.c ├── helgrind-log.supp ├── helpers.cmake ├── test-basic.cmake ├── test-bench-mt.cmake ├── test-bench-simul.cmake ├── test-example.cmake ├── test-heap-usage.cmake ├── test-mt.cmake ├── test-twolevel.cmake ├── test-utilization.cmake ├── test_helpers.h ├── twolevel.c ├── vmemcache_test_basic.c ├── vmemcache_test_heap_usage.c ├── vmemcache_test_mt.c └── vmemcache_test_utilization.c ├── travis.yml └── utils ├── check_license ├── check-headers.sh └── file-exceptions.sh ├── check_whitespace ├── cstyle ├── docker ├── build.sh ├── images │ ├── Dockerfile.fedora-28 │ ├── Dockerfile.ubuntu-18.04 │ ├── build-image.sh │ └── push-image.sh ├── pull-or-rebuild-image.sh └── run-build.sh └── md2man ├── default.man └── md2man.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.cmake 3 | *~ 4 | *.so 5 | *.3 6 | core 7 | Makefile 8 | CMakeCache.txt 9 | CMakeFiles 10 | Testing 11 | 12 | !.gitignore 13 | !.gitattributes 14 | !.clang-format 15 | !.travis.yml 16 | !.mailmap 17 | !.cstyleignore 18 | !.codecov.yml 19 | 20 | /build 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | travis.yml -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | cmake_minimum_required(VERSION 3.3) 5 | project(vmemcache C) 6 | 7 | set(VERSION_MAJOR 0) 8 | set(VERSION_MINOR 8) 9 | set(VERSION ${VERSION_MAJOR}.${VERSION_MINOR}) 10 | 11 | set(CMAKE_DISABLE_IN_SOURCE_BUILD ON) 12 | set(CMAKE_POSITION_INDEPENDENT_CODE ON) 13 | set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${CMAKE_SOURCE_DIR}) 14 | 15 | include(FindThreads) 16 | 17 | if(NOT CMAKE_BUILD_TYPE) 18 | set(CMAKE_BUILD_TYPE "RelWithDebInfo") 19 | endif (NOT CMAKE_BUILD_TYPE) 20 | 21 | include(CheckCCompilerFlag) 22 | include(GNUInstallDirs) 23 | 24 | if(NOT MSVC) 25 | find_package(PkgConfig QUIET) 26 | if(NOT PKG_CONFIG_FOUND) 27 | message(WARNING "Pkg-config not found. Detection of other dependencies may fail.") 28 | endif() 29 | endif() 30 | 31 | if(NOT WIN32) 32 | if(PKG_CONFIG_FOUND) 33 | pkg_check_modules(VALGRIND QUIET valgrind) 34 | else() 35 | find_package(VALGRIND QUIET) 36 | endif() 37 | 38 | if (NOT VALGRIND_FOUND) 39 | message(WARNING "Valgrind not found. Some tests will be skipped.") 40 | endif() 41 | endif() 42 | 43 | find_program(PANDOC NAMES pandoc) 44 | if(NOT PANDOC) 45 | message(WARNING "pandoc not found - documentation will not be generated") 46 | endif() 47 | 48 | set(CMAKE_C_STANDARD 99) 49 | 50 | include(CheckSymbolExists) 51 | CHECK_SYMBOL_EXISTS(getentropy unistd.h HAVE_GETENTROPY) 52 | 53 | # Checks whether flag is supported by current C compiler and appends 54 | # it to the relevant cmake variable. 55 | # 1st argument is a flag 56 | # 2nd (optional) argument is a build type (debug, release, relwithdebinfo) 57 | macro(add_c_flag flag) 58 | string(REPLACE - _ flag2 ${flag}) 59 | string(REPLACE " " _ flag2 ${flag2}) 60 | string(REPLACE = "_" flag2 ${flag2}) 61 | set(check_name "C_HAS_${flag2}") 62 | 63 | check_c_compiler_flag("${flag}" "${check_name}") 64 | 65 | if (${${check_name}}) 66 | if (${ARGC} EQUAL 1) 67 | set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${flag}") 68 | else() 69 | set(CMAKE_C_FLAGS_${ARGV1} "${CMAKE_C_FLAGS_${ARGV1}} ${flag}") 70 | endif() 71 | endif() 72 | endmacro() 73 | 74 | if(NOT MSVC) 75 | add_c_flag(-fno-common) 76 | add_c_flag(-Wall) 77 | add_c_flag(-Wconversion) 78 | add_c_flag(-Wfloat-equal) 79 | add_c_flag(-Wmissing-field-initializers) 80 | add_c_flag(-Wmissing-prototypes) 81 | add_c_flag(-Wmissing-variable-declarations) 82 | add_c_flag(-Wpointer-arith) 83 | add_c_flag(-Wsign-compare) 84 | add_c_flag(-Wsign-conversion) 85 | add_c_flag(-Wswitch-default) 86 | add_c_flag(-Wunused-macros) 87 | add_c_flag(-Wunreachable-code-return) 88 | add_c_flag(-Werror=incompatible-pointer-types) 89 | 90 | # Place each function or data item into its own section. Will be used to strip unneeded symbols. 91 | add_c_flag(-fdata-sections) 92 | add_c_flag(-ffunction-sections) 93 | 94 | add_c_flag(-ggdb DEBUG) 95 | add_c_flag(-DDEBUG DEBUG) 96 | 97 | add_c_flag(-ggdb RELWITHDEBINFO) 98 | add_c_flag(-fno-omit-frame-pointer RELWITHDEBINFO) 99 | 100 | check_c_compiler_flag(-Wl,-z,relro LINKER_HAS_RELRO) 101 | if(LINKER_HAS_RELRO) 102 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,relro") 103 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,relro") 104 | endif() 105 | 106 | check_c_compiler_flag(-Wl,--warn-common LINKER_HAS_WARN_COMMON) 107 | if(LINKER_HAS_WARN_COMMON) 108 | set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--warn-common") 109 | set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--warn-common") 110 | endif() 111 | 112 | add_c_flag("-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=2" RELEASE) 113 | endif() 114 | 115 | configure_file( 116 | "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" 117 | "${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake" 118 | IMMEDIATE @ONLY) 119 | 120 | add_custom_target(uninstall 121 | COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_BINARY_DIR}/cmake_uninstall.cmake) 122 | 123 | function(add_cstyle name) 124 | if(${ARGC} EQUAL 1) 125 | add_custom_target(cstyle-${name} 126 | COMMAND ${PERL_EXECUTABLE} 127 | ${CMAKE_SOURCE_DIR}/utils/cstyle 128 | ${CMAKE_CURRENT_SOURCE_DIR}/*.c 129 | ${CMAKE_CURRENT_SOURCE_DIR}/*.h) 130 | else() 131 | add_custom_target(cstyle-${name} 132 | COMMAND ${PERL_EXECUTABLE} 133 | ${CMAKE_SOURCE_DIR}/utils/cstyle ${ARGN}) 134 | endif() 135 | add_dependencies(cstyle cstyle-${name}) 136 | endfunction() 137 | 138 | # Generates check-whitespace-$name target and attaches it as a dependency 139 | # of global "check-whitespace" target. This target verifies C files in current 140 | # source dir do not have any whitespace errors. 141 | # If more arguments are used, then they are used as files to be checked 142 | # instead. 143 | # ${name} must be unique. 144 | 145 | function(add_check_whitespace name) 146 | if(${ARGC} EQUAL 1) 147 | add_custom_target(check-whitespace-${name} 148 | COMMAND ${PERL_EXECUTABLE} 149 | ${CMAKE_SOURCE_DIR}/utils/check_whitespace 150 | ${CMAKE_CURRENT_SOURCE_DIR}/*.c 151 | ${CMAKE_CURRENT_SOURCE_DIR}/*.h) 152 | else() 153 | add_custom_target(check-whitespace-${name} 154 | COMMAND ${PERL_EXECUTABLE} 155 | ${CMAKE_SOURCE_DIR}/utils/check_whitespace ${ARGN}) 156 | endif() 157 | add_dependencies(check-whitespace check-whitespace-${name}) 158 | endfunction() 159 | 160 | add_custom_target(checkers ALL) 161 | add_custom_target(cstyle) 162 | add_custom_target(check-whitespace) 163 | add_custom_target(check-license 164 | COMMAND ${CMAKE_SOURCE_DIR}/utils/check_license/check-headers.sh 165 | ${CMAKE_SOURCE_DIR} 166 | BSD-3-Clause) 167 | 168 | add_check_whitespace(other 169 | ${CMAKE_SOURCE_DIR}/utils/check_license/*.sh 170 | ${CMAKE_SOURCE_DIR}/README.md) 171 | 172 | option(STATS_ENABLED "statistics are enabled" ON) 173 | 174 | option(DEVELOPER_MODE "enable developer checks" OFF) 175 | if(DEVELOPER_MODE) 176 | add_c_flag(-Werror) 177 | add_dependencies(checkers cstyle) 178 | add_dependencies(checkers check-whitespace) 179 | endif(DEVELOPER_MODE) 180 | 181 | option(TRACE_TESTS 182 | "more verbose test outputs" OFF) 183 | 184 | configure_file(libvmemcache.pc.in libvmemcache.pc) 185 | install(FILES ${CMAKE_BINARY_DIR}/libvmemcache.pc 186 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig) 187 | 188 | enable_testing() 189 | add_subdirectory(src) 190 | add_subdirectory(tests) 191 | add_subdirectory(benchmarks) 192 | if (PANDOC) 193 | add_subdirectory(doc) 194 | install(FILES ${CMAKE_CURRENT_BINARY_DIR}/doc/vmemcache.3 195 | DESTINATION ${CMAKE_INSTALL_MANDIR}/man3) 196 | endif() 197 | 198 | if(NOT "${CPACK_GENERATOR}" STREQUAL "") 199 | include(${CMAKE_SOURCE_DIR}/packages.cmake) 200 | endif() 201 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | Wed Oct 21 2020 Piotr Balcer 2 | 3 | * Version 0.8.1 4 | 5 | This is a minor patch release that includes a couple minor maintance 6 | changes, plus a couple of small performance improvements. 7 | 8 | * Use fast-hash for sharding. 9 | * Fix comparison of bytes with 8th bit set in the index radix tree. 10 | * Use chattr for the cache's file to enable nocow. 11 | * Fix some issues on 32-bit platforms. 12 | * Migrate from obsolete __sync_fetch API to __atomic_fetch. 13 | * Increase guard size to 4096 bytes. 14 | * Various documentation and test improvements. 15 | * add vmemcache_exists() to check entry existence without side-effects 16 | * Regression fixes for recent compilers 17 | 18 | Fri Mar 29 2019 Piotr Balcer 19 | 20 | * Version 0.8 21 | This is the first official release of libvmemcache. It's an embeddable 22 | and lightweight in-memory caching solution designed to fully take 23 | advantage of large capacity memory, such as Persistent Memory with DAX, 24 | through memory mapping in an efficient and scalable way. 25 | 26 | Among other things, it includes: 27 | - Extent-based memory allocator which sidesteps the fragmentation 28 | problem that affects most in-memory databases and allows the cache 29 | to achieve very high space utilization for most workloads. 30 | - Buffered LRU, which combines a traditional LRU doubly-linked 31 | list with a non-blocking ring buffer to deliver high degree 32 | of scalability on modern multi-core CPUs. 33 | - Unique indexing structure, critnib, which delivers 34 | high-performance while being very space efficient. 35 | 36 | The reason this release has version 0.8 is because we are still looking 37 | for actual real world feedback before we stabilize the APIs and commit 38 | to maintaining backward compatibility. It does not mean that the library 39 | is unfinished or unstable. On the contrary, the cache is fully 40 | functional and we are confident in its quality. 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018-2019, Intel Corporation 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions 5 | are met: 6 | 7 | * Redistributions of source code must retain the above copyright 8 | notice, this list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in 12 | the documentation and/or other materials provided with the 13 | distribution. 14 | 15 | * Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived 17 | from this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | 31 | Everything in this source tree is covered by the previous license 32 | with the following exceptions: 33 | 34 | * src/fast_hash.c and src/fash_hash.h licensed unded MIT. 35 | 36 | * utils/cstyle (used only during development) licensed under CDDL. 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | libvmemcache: buffer based LRU cache 2 | ======================================= 3 | 4 | [![Build Status](https://travis-ci.org/pmem/vmemcache.svg?branch=master)](https://travis-ci.org/pmem/vmemcache) 5 | [![Coverage Status](https://codecov.io/github/pmem/vmemcache/coverage.svg?branch=master)](https://codecov.io/gh/pmem/vmemcache/branch/master) 6 | 7 | ## ⚠️ Discontinuation of the project 8 | The **vmemcache** project will no longer be maintained by Intel. 9 | - Intel has ceased development and contributions including, but not limited to, maintenance, bug fixes, new releases, 10 | or updates, to this project. 11 | - Intel no longer accepts patches to this project. 12 | - If you have an ongoing need to use this project, are interested in independently developing it, or would like to 13 | maintain patches for the open source software community, please create your own fork of this project. 14 | - You will find more information [here](https://pmem.io/blog/2022/11/update-on-pmdk-and-our-long-term-support-strategy/). 15 | 16 | ## Introduction 17 | 18 | **libvmemcache** is an embeddable and lightweight in-memory caching solution. 19 | It's designed to fully take advantage of large capacity memory, such as 20 | Persistent Memory with DAX, through memory mapping in an efficient 21 | and scalable way. 22 | 23 | The things that make it unique are: 24 | - Extent-based memory allocator which sidesteps the fragmentation 25 | problem that affects most in-memory databases and allows the cache 26 | to achieve very high space utilization for most workloads. 27 | - Buffered LRU, which combines a traditional LRU doubly-linked 28 | list with a non-blocking ring buffer to deliver high degree 29 | of scalability on modern multi-core CPUs. 30 | - Unique indexing structure, critnib, which delivers 31 | high-performance while being very space efficient. 32 | 33 | The cache is tuned to work optimally with relatively large value sizes. The 34 | smallest possible size is 256 bytes, but libvmemcache works best if the expected 35 | value sizes are above 1 kilobyte. 36 | 37 | ## Building The Source ## 38 | 39 | Requirements: 40 | - cmake >= 3.3 41 | 42 | Optional: 43 | - valgrind (for tests) 44 | - pandoc (for documentation) 45 | 46 | For all systems: 47 | 48 | ```sh 49 | $ git clone https://github.com/pmem/vmemcache.git 50 | $ cd vmemcache 51 | $ mkdir build 52 | $ cd build 53 | ``` 54 | 55 | And then: 56 | 57 | ### On RPM-based Linux distros (Fedora, openSUSE, RHEL, SLES) ### 58 | 59 | ```sh 60 | $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCPACK_GENERATOR=rpm 61 | $ make package 62 | $ sudo rpm -i libvmemcache*.rpm 63 | ``` 64 | 65 | ### On DEB-based Linux distros (Debian, Ubuntu) ### 66 | 67 | ```sh 68 | $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCPACK_GENERATOR=deb 69 | $ make package 70 | $ sudo dpkg -i libvmemcache*.deb 71 | ``` 72 | 73 | ### On other Linux distros ### 74 | ```sh 75 | $ cmake .. -DCMAKE_INSTALL_PREFIX=~/libvmemcache-bin 76 | $ make 77 | $ make install 78 | ``` 79 | 80 | # Statistics # 81 | 82 | Statistics are enabled by default. They can be disabled at the compile time 83 | of the libvmemcache library if the **STATS_ENABLED** CMake option is set to OFF. 84 | 85 | See the man page for more information about statistics. 86 | -------------------------------------------------------------------------------- /benchmarks/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | add_cstyle(benchmarks 5 | ${CMAKE_SOURCE_DIR}/benchmarks/*.c) 6 | 7 | add_check_whitespace(benchmarks 8 | ${CMAKE_SOURCE_DIR}/benchmarks/*.c) 9 | 10 | set(SOURCES 11 | benchmark_time.c 12 | rand.c 13 | ${CMAKE_SOURCE_DIR}/src/os_posix.c 14 | ${CMAKE_SOURCE_DIR}/src/os_thread_posix.c) 15 | 16 | add_executable(bench_micro bench_micro.c ${SOURCES}) 17 | 18 | target_include_directories(bench_micro PRIVATE 19 | ${CMAKE_SOURCE_DIR}/src 20 | ${CMAKE_SOURCE_DIR}/tests) 21 | 22 | target_link_libraries(bench_micro PRIVATE vmemcache) 23 | target_link_libraries(bench_micro PRIVATE ${CMAKE_THREAD_LIBS_INIT}) 24 | 25 | add_executable(bench_simul bench_simul.c ${SOURCES}) 26 | 27 | target_include_directories(bench_simul PRIVATE 28 | ${CMAKE_SOURCE_DIR}/src 29 | ${CMAKE_SOURCE_DIR}/tests) 30 | 31 | if (HAVE_GETENTROPY) 32 | target_compile_definitions(bench_simul PRIVATE HAVE_GETENTROPY) 33 | endif() 34 | 35 | if(STATS_ENABLED) 36 | target_compile_definitions(bench_simul PRIVATE STATS_ENABLED=1) 37 | endif() 38 | 39 | target_link_libraries(bench_simul PRIVATE vmemcache) 40 | target_link_libraries(bench_simul PRIVATE ${CMAKE_THREAD_LIBS_INIT}) 41 | -------------------------------------------------------------------------------- /benchmarks/benchmark_time.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2015-2018, Intel Corporation */ 3 | 4 | /* 5 | * benchmark_time.c -- benchmark_time module definitions 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "benchmark_time.h" 14 | #include "os.h" 15 | 16 | #define NSECPSEC 1000000000 17 | 18 | /* 19 | * benchmark_time_get -- get timestamp from clock source 20 | */ 21 | void 22 | benchmark_time_get(benchmark_time_t *time) 23 | { 24 | os_clock_gettime(CLOCK_MONOTONIC, time); 25 | } 26 | 27 | /* 28 | * benchmark_time_diff -- get time interval 29 | */ 30 | void 31 | benchmark_time_diff(benchmark_time_t *d, benchmark_time_t *t1, 32 | benchmark_time_t *t2) 33 | { 34 | long long nsecs = ((long long)t2->tv_sec - t1->tv_sec) * NSECPSEC 35 | + t2->tv_nsec - t1->tv_nsec; 36 | assert(nsecs >= 0); 37 | d->tv_sec = nsecs / NSECPSEC; 38 | d->tv_nsec = nsecs % NSECPSEC; 39 | } 40 | 41 | /* 42 | * benchmark_time_get_secs -- get total number of seconds 43 | */ 44 | double 45 | benchmark_time_get_secs(benchmark_time_t *t) 46 | { 47 | return (double)t->tv_sec + (double)t->tv_nsec / NSECPSEC; 48 | } 49 | 50 | /* 51 | * benchmark_time_get_nsecs -- get total number of nanoseconds 52 | */ 53 | unsigned long long 54 | benchmark_time_get_nsecs(benchmark_time_t *t) 55 | { 56 | unsigned long long ret = (unsigned long long)t->tv_nsec; 57 | 58 | ret += (unsigned long long)(t->tv_sec * NSECPSEC); 59 | 60 | return ret; 61 | } 62 | 63 | /* 64 | * benchmark_time_compare -- compare two moments in time 65 | */ 66 | int 67 | benchmark_time_compare(const benchmark_time_t *t1, const benchmark_time_t *t2) 68 | { 69 | if (t1->tv_sec == t2->tv_sec) 70 | return (int)((long long)t1->tv_nsec - (long long)t2->tv_nsec); 71 | else 72 | return (int)((long long)t1->tv_sec - (long long)t2->tv_sec); 73 | } 74 | 75 | /* 76 | * benchmark_time_set -- set time using number of nanoseconds 77 | */ 78 | void 79 | benchmark_time_set(benchmark_time_t *time, unsigned long long nsecs) 80 | { 81 | time->tv_sec = (long)nsecs / NSECPSEC; 82 | time->tv_nsec = (long)nsecs % NSECPSEC; 83 | } 84 | 85 | /* 86 | * number of samples used to calculate average time required to get a current 87 | * time from the system 88 | */ 89 | #define N_PROBES_GET_TIME 10000000UL 90 | 91 | /* 92 | * benchmark_get_avg_get_time -- calculates average time required to get the 93 | * current time from the system in nanoseconds 94 | */ 95 | unsigned long long 96 | benchmark_get_avg_get_time(void) 97 | { 98 | benchmark_time_t time; 99 | benchmark_time_t start; 100 | benchmark_time_t stop; 101 | 102 | benchmark_time_get(&start); 103 | for (size_t i = 0; i < N_PROBES_GET_TIME; i++) { 104 | benchmark_time_get(&time); 105 | } 106 | benchmark_time_get(&stop); 107 | 108 | benchmark_time_diff(&time, &start, &stop); 109 | 110 | unsigned long long avg = 111 | benchmark_time_get_nsecs(&time) / N_PROBES_GET_TIME; 112 | 113 | return avg; 114 | } 115 | -------------------------------------------------------------------------------- /benchmarks/benchmark_time.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2015-2018, Intel Corporation */ 3 | /* 4 | * benchmark_time.h -- declarations of benchmark_time module 5 | */ 6 | #include 7 | 8 | typedef struct timespec benchmark_time_t; 9 | 10 | void benchmark_time_get(benchmark_time_t *time); 11 | void benchmark_time_diff(benchmark_time_t *d, benchmark_time_t *t1, 12 | benchmark_time_t *t2); 13 | double benchmark_time_get_secs(benchmark_time_t *t); 14 | unsigned long long benchmark_time_get_nsecs(benchmark_time_t *t); 15 | int benchmark_time_compare(const benchmark_time_t *t1, 16 | const benchmark_time_t *t2); 17 | void benchmark_time_set(benchmark_time_t *time, unsigned long long nsecs); 18 | unsigned long long benchmark_get_avg_get_time(void); 19 | -------------------------------------------------------------------------------- /benchmarks/rand.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | /* 5 | * rand.c -- random utils 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "rand.h" 15 | 16 | /* 17 | * hash64 -- a u64 -> u64 hash 18 | */ 19 | uint64_t 20 | hash64(uint64_t x) 21 | { 22 | x += 0x9e3779b97f4a7c15; 23 | x = (x ^ (x >> 30)) * 0xbf58476d1ce4e5b9; 24 | x = (x ^ (x >> 27)) * 0x94d049bb133111eb; 25 | return x ^ (x >> 31); 26 | } 27 | 28 | /* 29 | * xoshiro256** random generator 30 | * 31 | * Fastest available good PRNG as of 2018 (sub-nanosecond per entry), produces 32 | * much better output than old stuff like rand() or Mersenne's Twister. 33 | * 34 | * By David Blackman and Sebastiano Vigna; PD/CC0 2018. 35 | */ 36 | 37 | static inline uint64_t rotl(const uint64_t x, int k) 38 | { 39 | /* optimized to a single instruction on x86 */ 40 | return (x << k) | (x >> (64 - k)); 41 | } 42 | 43 | /* 44 | * rnd64 -- return 64-bits of randomness 45 | */ 46 | uint64_t 47 | rnd64_r(rng_t *state) 48 | { 49 | uint64_t *s = (void *)state; 50 | 51 | const uint64_t result = rotl(s[1] * 5, 7) * 9; 52 | const uint64_t t = s[1] << 17; 53 | 54 | s[2] ^= s[0]; 55 | s[3] ^= s[1]; 56 | s[1] ^= s[2]; 57 | s[0] ^= s[3]; 58 | 59 | s[2] ^= t; 60 | 61 | s[3] = rotl(s[3], 45); 62 | 63 | return result; 64 | } 65 | 66 | /* 67 | * randomize -- initialize random generator 68 | * 69 | * Seed of 0 means random. 70 | */ 71 | void 72 | randomize_r(rng_t *state, uint64_t seed) 73 | { 74 | if (!seed) { 75 | #ifdef HAVE_GETENTROPY 76 | if (!getentropy(state, sizeof(rng_t))) 77 | return; /* nofail, but ENOSYS on kernel < 3.16 */ 78 | #endif 79 | seed = (uint64_t)getpid(); 80 | } 81 | 82 | uint64_t *s = (void *)state; 83 | s[0] = hash64(seed); 84 | s[1] = hash64(s[0]); 85 | s[2] = hash64(s[1]); 86 | s[3] = hash64(s[2]); 87 | } 88 | 89 | /* 90 | * n_lowest_bits -- return n lowest 1 bits 91 | * 92 | * When applied to random numbers, this puts them into nice uneven buckets. 93 | */ 94 | uint64_t 95 | n_lowest_bits(uint64_t x, int n) 96 | { 97 | uint64_t y = x; 98 | while (n-- > 0) 99 | y &= y - 1; 100 | return x ^ y; 101 | } 102 | -------------------------------------------------------------------------------- /benchmarks/rand.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | /* 5 | * rand.h -- random utils 6 | */ 7 | 8 | #ifndef RAND_H 9 | #define RAND_H 1 10 | 11 | #include 12 | 13 | typedef uint64_t rng_t[4]; 14 | 15 | uint64_t hash64(uint64_t x); 16 | uint64_t rnd64_r(rng_t *rng); 17 | void randomize_r(rng_t *rng, uint64_t seed); 18 | uint64_t n_lowest_bits(uint64_t x, int n); 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /benchmarks/tools/.gitignore: -------------------------------------------------------------------------------- 1 | benchconfig.py 2 | -------------------------------------------------------------------------------- /benchmarks/tools/benchconfig.py.example: -------------------------------------------------------------------------------- 1 | """ 2 | Example file for latency_run configuration. All provided configurations 3 | need to be dictionaries starting with 'bench_'. Keys and values are 4 | directly translated to 'bench_simul' command line arguments except: 5 | * 'testdir': test directory path (required) 6 | * 'numa_node': NUMA node number on which the benchmark is run (optional) 7 | """ 8 | 9 | bench_16_threads = { 10 | 'testdir': '/tmp', 11 | # 'numa_node': 0, 12 | 'n_threads': 16 13 | } 14 | 15 | bench_4_threads = { 16 | 'testdir': '/tmp', 17 | 'n_threads': 4, 18 | 'junk_start': 1 19 | } 20 | -------------------------------------------------------------------------------- /benchmarks/tools/latency_plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2019, Intel Corporation 4 | 5 | """ 6 | latency_plot.py - tool for drawing latency benchmarks plots based on 7 | output generated by 'bench_simul' and written to a file provided with 8 | 'latency_file' argument. 9 | """ 10 | 11 | import argparse 12 | import matplotlib.pyplot as plt 13 | 14 | 15 | MARKERS = ('o', '^', 's', 'D', 'X') 16 | CUR_MARKER = 0 17 | 18 | 19 | def _add_series(series, label, marker): 20 | """Add data series to plot""" 21 | plt.plot(series, label=label, marker=marker, linestyle=':', linewidth=0.5, 22 | markersize=4) 23 | 24 | 25 | def draw_plot(yscale='linear'): 26 | """Draw a plot of all previously added data series""" 27 | plt.yscale(yscale) 28 | plt.xticks(list(range(0, 101, 5))) 29 | plt.xlabel('percentile [%]') 30 | plt.grid(True) 31 | plt.ylabel('operation time [ns]') 32 | plt.legend() 33 | plt.show() 34 | 35 | 36 | def _parse_args(): 37 | """Parse command line arguments""" 38 | parser = argparse.ArgumentParser() 39 | parser.add_argument('out', nargs='*', help='Create a plot for all provided' 40 | ' output files') 41 | parser.add_argument('--yscale', '-y', help='Y-axis scale', 42 | default='linear') 43 | parser.add_argument('--hits', help='Draw hits', dest='hits', action='store_true') 44 | parser.add_argument('--no-hits', help='Do not draw hits', dest='hits', action='store_false') 45 | parser.set_defaults(hits=True) 46 | 47 | parser.add_argument('--ltrim', help='Remove a number of smallest latency values from the plot', default=0, type=int) 48 | parser.add_argument('--rtrim', help='Remove a number of biggest latency values from the plot', default=0, type=int) 49 | 50 | parser.add_argument('--misses', help='Draw misses', dest='misses', action='store_true') 51 | parser.add_argument('--no-misses', help='Do not draw misses', dest='misses', action='store_false') 52 | parser.set_defaults(misses=True) 53 | 54 | args = parser.parse_args() 55 | if not args.out: 56 | parser.error('at least one output need to be provided') 57 | return args 58 | 59 | 60 | def _read_out(path): 61 | """Read 'latency_file' output file""" 62 | with open(path, 'r') as f: 63 | out = f.readlines() 64 | hits = [float(h) for h in out[0].split(';')] 65 | misses = [float(m) for m in out[1].split(';')] 66 | return hits, misses 67 | 68 | 69 | def add_data(output, name, hits=True, misses=True, ltrim=0, rtrim=0): 70 | """Add data from 'latency_file' output file to plot""" 71 | global CUR_MARKER 72 | 73 | h, m = _read_out(output) 74 | if ltrim: 75 | h, m = h[ltrim:], m[ltrim:] 76 | if rtrim: 77 | h, m = h[:-rtrim], m[:-rtrim] 78 | if hits: 79 | _add_series(h, '{}_hits'.format(name), 80 | MARKERS[CUR_MARKER % len(MARKERS)]) 81 | if misses: 82 | _add_series(m, '{}_misses'.format(name), 83 | MARKERS[CUR_MARKER % len(MARKERS)]) 84 | 85 | # use different marker for each plotted benchmark data 86 | CUR_MARKER += 1 87 | 88 | 89 | def _main(): 90 | args = _parse_args() 91 | 92 | for out in args.out: 93 | add_data(out, out, args.hits, args.misses, args.ltrim, args.rtrim) 94 | draw_plot(args.yscale) 95 | 96 | 97 | if __name__ == '__main__': 98 | _main() 99 | -------------------------------------------------------------------------------- /benchmarks/tools/latency_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2019, Intel Corporation 4 | 5 | """ 6 | latency_run.py - tool for running latency benchmarks and plotting them 7 | with use of latency_plot.py. 8 | 9 | Benchmark configurations are stored in benchconfig.py file as separate 10 | dictionaries with names starting with 'bench_'. Configuration key 'testdir' 11 | is a benchmark test directory. All other keys and their values are directly 12 | translated into 'bench_simul' command line arguments. 13 | """ 14 | 15 | import sys 16 | import os 17 | import argparse 18 | import subprocess as sp 19 | import latency_plot as lp 20 | 21 | 22 | try: 23 | import benchconfig 24 | except ImportError: 25 | sys.exit('no benchconfig.py file provided') 26 | 27 | 28 | class Benchmark(): 29 | """Benchmark management""" 30 | def __init__(self, bin_, cfg): 31 | self.bin = bin_ 32 | self.cmd = '' 33 | self.handle_special_keys(cfg) 34 | for k, v in cfg.items(): 35 | self.cmd = self.cmd + ' {}={}'.format(k, v) 36 | if 'latency_samples' not in cfg: 37 | self.cmd = self.cmd + ' latency_samples=101' 38 | 39 | def handle_special_keys(self, cfg): 40 | """ 41 | Handle configuration keys that are NOT a direct 42 | representations of bench_simul command line arguments 43 | """ 44 | try: 45 | self.title = cfg.pop('title') 46 | cfg['latency_file'] = '{}.log'.format(self.title) 47 | self.out = cfg['latency_file'] 48 | self.cmd = '{} {}'.format(self.bin, cfg.pop('testdir')) 49 | if 'numa_node' in cfg: 50 | self.cmd = 'numactl -N {} {}'.format(cfg.pop('numa_node'), self.cmd) 51 | except KeyError as e: 52 | sys.exit('No "{}" key provided to configuration'.format(e.args[0])) 53 | 54 | 55 | def add_to_plot(self): 56 | """Add benchmark to plot""" 57 | lp.add_data(self.out, self.title) 58 | 59 | def run(self, verbose): 60 | """Run benchmark""" 61 | if verbose: 62 | print(self.cmd) 63 | 64 | proc = sp.run(self.cmd.split(' '), universal_newlines=True, 65 | stdout=sp.PIPE, stderr=sp.STDOUT) 66 | if proc.returncode != 0: 67 | sys.exit('benchmark failed: {}{}{}'.format(self.cmd, os.linesep, 68 | proc.stdout)) 69 | if verbose: 70 | print('{}{}{}'.format(self.cmd, os.linesep, proc.stdout)) 71 | 72 | 73 | def parse_config(): 74 | """Read configurations from benchconfig.py file""" 75 | cfgs = [] 76 | for k, v in vars(benchconfig).items(): 77 | if k.startswith('bench_') and isinstance(v, dict): 78 | v['title'] = k.split('bench_')[1] 79 | cfgs.append(v) 80 | 81 | if not cfgs: 82 | sys.exit('No configs found in benchconfig.py - all configs should ' 83 | 'be dictionaries starting with "bench_"') 84 | return cfgs 85 | 86 | 87 | def parse_args(): 88 | """Parse command line arguments""" 89 | parser = argparse.ArgumentParser() 90 | parser.add_argument('--bin', '-b', help='path to bench_simul binary', 91 | required=True) 92 | parser.add_argument('--config', '-c', nargs='*', help="run only selected " 93 | "configs from benchconfig (provide name without " 94 | "'bench' at the beginning)") 95 | parser.add_argument('--yscale', '-y', help='Y-axis scale', 96 | default='linear') 97 | parser.add_argument('--verbose', '-v', help='Print bench_simul output', 98 | action='store_true') 99 | args = parser.parse_args() 100 | return args 101 | 102 | 103 | def main(): 104 | args = parse_args() 105 | file_cfgs = parse_config() 106 | if args.config: 107 | cfgs = [c for c in file_cfgs if c['title'] in args.config] 108 | if len(args.config) != len(cfgs): 109 | titles = os.linesep.join([c['title'] for c in file_cfgs]) 110 | sys.exit('Invalid configuration provided, configurations defined' 111 | ' in benchconfig.py:{}{}'.format(os.linesep, titles)) 112 | else: 113 | cfgs = file_cfgs 114 | 115 | for c in cfgs: 116 | b = Benchmark(args.bin, c) 117 | b.run(args.verbose) 118 | b.add_to_plot() 119 | 120 | lp.draw_plot(args.yscale) 121 | 122 | 123 | if __name__ == '__main__': 124 | main() 125 | -------------------------------------------------------------------------------- /cmake_uninstall.cmake.in: -------------------------------------------------------------------------------- 1 | # From: https://cmake.org/Wiki/CMake_FAQ 2 | 3 | if(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") 4 | message(FATAL_ERROR "Cannot find install manifest: @CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") 5 | endif(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") 6 | 7 | file(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) 8 | string(REGEX REPLACE "\n" ";" files "${files}") 9 | foreach(file ${files}) 10 | message(STATUS "Uninstalling $ENV{DESTDIR}${file}") 11 | if(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") 12 | exec_program("@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" 13 | OUTPUT_VARIABLE rm_out 14 | RETURN_VALUE rm_retval 15 | ) 16 | if(NOT "${rm_retval}" STREQUAL 0) 17 | message(FATAL_ERROR "Problem when removing $ENV{DESTDIR}${file}") 18 | endif(NOT "${rm_retval}" STREQUAL 0) 19 | else(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") 20 | message(STATUS "File $ENV{DESTDIR}${file} does not exist.") 21 | endif(IS_SYMLINK "$ENV{DESTDIR}${file}" OR EXISTS "$ENV{DESTDIR}${file}") 22 | endforeach(file) 23 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | ignore: 2 | - tests/ 3 | - benchmarks/ 4 | 5 | comment: 6 | layout: "diff" 7 | behavior: default 8 | require_changes: yes 9 | -------------------------------------------------------------------------------- /doc/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | add_custom_target(doc ALL 5 | BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/vmemcache.3 6 | COMMENT "Building vmemcache.3" 7 | COMMAND ${CMAKE_SOURCE_DIR}/utils/md2man/md2man.sh 8 | ${CMAKE_CURRENT_SOURCE_DIR}/vmemcache.md 9 | ${CMAKE_SOURCE_DIR}/utils/md2man/default.man 10 | ${CMAKE_CURRENT_BINARY_DIR}/vmemcache.3 11 | ) 12 | -------------------------------------------------------------------------------- /doc/vmemcache.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: manual 3 | Content-Style: 'text/css' 4 | title: _MP(VMEMCACHE.3) 5 | collection: vmemcache 6 | header: VMEMCACHE 7 | ... 8 | 9 | [NAME](#name)
10 | [SYNOPSIS](#synopsis)
11 | [DESCRIPTION](#description)
12 | 13 | [comment]: <> (SPDX-License-Identifier: BSD-3-Clause) 14 | [comment]: <> (Copyright 2019, Intel Corporation) 15 | 16 | # NAME # 17 | 18 | **vmemcache** - buffer-based LRU cache 19 | 20 | # SYNOPSIS # 21 | 22 | ```c 23 | #include 24 | 25 | VMEMcache *vmemcache_new(); 26 | void vmemcache_delete(VMEMcache *cache); 27 | int vmemcache_set_eviction_policy(VMEMcache *cache, 28 | enum vmemcache_repl_p repl_p); 29 | int vmemcache_set_size(VMEMcache *cache, size_t size); 30 | int vmemcache_set_extent_size(VMEMcache *cache, size_t extent_size); 31 | int vmemcache_add(VMEMcache *cache, const char *path); 32 | 33 | void vmemcache_callback_on_evict(VMEMcache *cache, 34 | vmemcache_on_evict *evict, void *arg); 35 | void vmemcache_callback_on_miss(VMEMcache *cache, 36 | vmemcache_on_miss *miss, void *arg); 37 | 38 | ssize_t vmemcache_get(VMEMcache *cache, 39 | const void *key, size_t key_size, 40 | void *vbuf, size_t vbufsize, size_t offset, size_t *vsize); 41 | 42 | int vmemcache_put(VMEMcache *cache, 43 | const void *key, size_t key_size, 44 | const void *value, size_t value_size); 45 | 46 | int vmemcache_exists(VMEMcache *cache, 47 | const void *key, size_t key_size); 48 | 49 | int vmemcache_evict(VMEMcache *cache, const void *key, size_t ksize); 50 | 51 | int vmemcache_get_stat(VMEMcache *cache, 52 | enum vmemcache_statistic stat, 53 | void *value, size_t value_size); 54 | 55 | const char *vmemcache_errormsg(void); 56 | ``` 57 | 58 | # DESCRIPTION # 59 | 60 | **libvmemcache** is a volatile key-value store optimized for operating on 61 | NVDIMM based space, although it can work with any filesystem, 62 | stored in memory (tmpfs) or, less performant, on some kind of a disk. 63 | 64 | 65 | ##### Creation ##### 66 | 67 | `VMEMcache *vmemcache_new();` 68 | 69 | : Creates an empty unconfigured vmemcache instance. 70 | 71 | `int vmemcache_set_size(VMEMcache *cache, size_t size);` 72 | 73 | : Sets the size of the cache; it will be rounded **up** towards a whole page 74 | size alignment (4KB on x86). 75 | 76 | `int vmemcache_set_extent_size(VMEMcache *cache, size_t extent_size);` 77 | 78 | : Sets block size of the cache -- 256 bytes minimum, strongly recommended 79 | to be a multiple of 64 bytes. If the cache is backed by a non 80 | byte-addressable medium, the extent size should be 4096 (or a multiple) or 81 | performance will greatly suffer. 82 | 83 | `int vmemcache_set_eviction_policy(VMEMcache *cache, enum vmemcache_repl_p repl_p);` 84 | 85 | : Sets what should happen on a put into a full cache. 86 | 87 | + **VMEMCACHE_REPLACEMENT_NONE**: manual eviction only - puts into a full 88 | cache will fail 89 | + **VMEMCACHE_REPLACEMENT_LRU**: least recently accessed entry will be evicted 90 | to make space when needed 91 | 92 | `int vmemcache_add(VMEMcache *cache, const char *path);` 93 | 94 | : Associate the cache with a backing medium in the given *path*, which may be: 95 | 96 | + a `/dev/dax` device 97 | + a directory on a regular filesystem (which may or may not be mounted with 98 | -o dax, either on persistent memory or any other backing storage) 99 | 100 | `void vmemcache_delete(VMEMcache *cache);` 101 | 102 | : Frees any structures associated with the cache. 103 | 104 | 105 | ##### Use ##### 106 | 107 | `ssize_t vmemcache_get(VMEMcache *cache, const void *key, size_t key_size, void *vbuf, size_t vbufsize, size_t offset, size_t *vsize);` 108 | 109 | : Searches for an entry with the given *key*; it doesn't have to be 110 | zero-terminated or be text - any sequence of bytes of length *key_size* 111 | is okay. If found, the entry's value is copied to *vbuf* that has space 112 | for *vbufsize* bytes, optionally skipping *offset* bytes at the start. 113 | No matter if the copy was truncated or not, its true size is stored into 114 | *vsize*; *vsize* remains unmodified if the key was not found. 115 | 116 | Return value is number of bytes successfully copied, or -1 on error. 117 | In particular, if there's no entry for the given *key* in the cache, 118 | the errno will be ENOENT. 119 | 120 | 121 | `int vmemcache_put(VMEMcache *cache, const void *key, size_t key_size, const void *value, size_t value_size);` 122 | 123 | : Inserts the given key:value pair into the cache. Returns 0 on success, 124 | -1 on error. Inserting a key that already exists will fail with EEXIST. 125 | 126 | `int vmemcache_exists(VMEMcache *cache, const void *key, size_t key_size, size_t *vsize);` 127 | 128 | : Searches for an entry with the given *key*, and returns 1 if found, 129 | 0 if not found, and -1 if search couldn't be performed. The size of the 130 | found entry is stored into *vsize*; *vsize* remains unmodified if the 131 | key was not found. 132 | This function does not impact the replacement policy or statistics. 133 | 134 | `int vmemcache_evict(VMEMcache *cache, const void *key, size_t ksize);` 135 | 136 | : Removes the given key from the cache. If *key* is null and there is a 137 | replacement policy set, the oldest entry will be removed. Returns 0 if 138 | an entry has been evicted, -1 otherwise. 139 | 140 | 141 | ##### Callbacks ##### 142 | 143 | You can register a hook to be called during eviction or after a cache miss, 144 | using **vmemcache_callback_on_evict()** or **vmemcache_callback_on_miss()**, 145 | respectively: 146 | 147 | `void vmemcache_callback_on_evict(VMEMcache *cache, vmemcache_on_evict *evict, void *arg);` 148 | 149 | `void vmemcache_callback_on_miss(VMEMcache *cache, vmemcache_on_miss *miss, void *arg);` 150 | 151 | The extra *arg* will be passed to your function. 152 | 153 | A hook to be called during eviction has to have the following signature: 154 | 155 | `void vmemcache_on_evict(VMEMcache *cache, const void *key, size_t key_size, void *arg);` 156 | 157 | : Called when an entry is being removed from the cache. The eviction can't 158 | be prevented, but until the callback returns, the entry remains available 159 | for queries. The thread that triggered the eviction is blocked in the 160 | meantime. 161 | 162 | A hook to be called after a cache miss has to have the following signature: 163 | 164 | `void vmemcache_on_miss(VMEMcache *cache, const void *key, size_t key_size, void *arg);` 165 | 166 | : Called when a *get* query fails, to provide an opportunity to insert the 167 | missing key. If the callback calls *put* for that specific key, the *get* 168 | will return its value, even if it did not fit into the cache. 169 | 170 | 171 | ##### Misc ##### 172 | 173 | `int vmemcache_get_stat(VMEMcache *cache, enum vmemcache_statistic stat, void *value, size_t value_size);` 174 | 175 | : Obtains a piece of statistics about the cache. The *stat* may be: 176 | 177 | + **VMEMCACHE_STAT_PUT** 178 | -- count of puts 179 | + **VMEMCACHE_STAT_GET** 180 | -- count of gets 181 | + **VMEMCACHE_STAT_HIT** 182 | -- count of gets that were served from the cache 183 | + **VMEMCACHE_STAT_MISS** 184 | -- count of gets that were not present in the cache 185 | + **VMEMCACHE_STAT_EVICT** 186 | -- count of evictions 187 | + **VMEMCACHE_STAT_ENTRIES** 188 | -- *current* number of cache entries (key:value pairs) 189 | + **VMEMCACHE_STAT_DRAM_SIZE_USED** 190 | -- current amount of DRAM used 191 | + **VMEMCACHE_STAT_POOL_SIZE_USED** 192 | -- current usage of data pool 193 | + **VMEMCACHE_STAT_HEAP_ENTRIES** 194 | -- current number of discontiguous unused regions (ie, free space 195 | fragmentation) 196 | 197 | Statistics are enabled by default. They can be disabled at the compile time 198 | of the vmemcache library if the **STATS_ENABLED** CMake option is set to OFF. 199 | 200 | `const char *vmemcache_errormsg(void);` 201 | 202 | : Retrieves a human-friendly description of the last error. 203 | 204 | 205 | ##### Errors ##### 206 | 207 | On an error, a machine-usable description is passed in `errno`. It may be: 208 | 209 | + **EINVAL** -- nonsensical/invalid parameter 210 | + **ENOMEM** -- out of DRAM 211 | + **EEXIST** -- (put) entry for that key already exists 212 | + **ENOENT** -- (evict, get) no entry for that key 213 | + **ESRCH** -- (evict) could not find an evictable entry 214 | + **EAGAIN** -- (evict) an entry was used and could not be evicted, please try again 215 | + **ENOSPC** -- (create, put) not enough space in the memory pool 216 | -------------------------------------------------------------------------------- /libvmemcache.pc.in: -------------------------------------------------------------------------------- 1 | prefix=@CMAKE_INSTALL_PREFIX@ 2 | libdir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ 3 | version=@VERSION@ 4 | includedir=@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ 5 | 6 | Name: libvmemcache 7 | Description: libvmemcache - buffer based LRU cache 8 | Version: @VERSION@ 9 | URL: http://github.com/pmem/vmemcache 10 | Libs: -L@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@ -lvmemcache 11 | Cflags: -I@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_INCLUDEDIR@ 12 | -------------------------------------------------------------------------------- /packages.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | # 5 | # packages.cmake - CPack configuration for rpm and deb generation 6 | # 7 | 8 | string(TOUPPER "${CPACK_GENERATOR}" CPACK_GENERATOR) 9 | 10 | if(NOT ("${CPACK_GENERATOR}" STREQUAL "DEB" OR 11 | "${CPACK_GENERATOR}" STREQUAL "RPM")) 12 | message(FATAL_ERROR "Wrong CPACK_GENERATOR value, valid generators are: DEB, RPM") 13 | endif() 14 | 15 | set(CPACK_PACKAGING_INSTALL_PREFIX "${CMAKE_INSTALL_PREFIX}") 16 | set(CMAKE_INSTALL_TMPDIR /tmp CACHE PATH "Output dir for tmp") 17 | set(CPACK_COMPONENTS_ALL_IN_ONE) 18 | 19 | # Filter out some of directories from %dir section, which are expected 20 | # to exist in filesystem. Leaving them might lead to conflicts with other 21 | # packages (for example with 'filesystem' package on fedora which specify 22 | # /usr, /usr/local, etc.) 23 | set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION 24 | ${CPACK_PACKAGING_INSTALL_PREFIX} 25 | ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR} 26 | ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}/pkgconfig 27 | ${CPACK_PACKAGING_INSTALL_PREFIX}/${CMAKE_INSTALL_INCDIR} 28 | ${CPACK_PACKAGING_INSTALL_PREFIX}/share 29 | ${CPACK_PACKAGING_INSTALL_PREFIX}/share/doc) 30 | 31 | set(CPACK_PACKAGE_NAME "libvmemcache") 32 | set(CPACK_PACKAGE_VERSION ${VERSION}) 33 | set(CPACK_PACKAGE_VERSION_MAJOR ${VERSION_MAJOR}) 34 | set(CPACK_PACKAGE_VERSION_MINOR ${VERSION_MINOR}) 35 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Buffer-based LRU cache") 36 | set(CPACK_PACKAGE_VENDOR "self-built") 37 | 38 | set(CPACK_RPM_PACKAGE_NAME "libvmemcache") 39 | set(CPACK_RPM_PACKAGE_LICENSE "BSD") 40 | 41 | set(CPACK_DEBIAN_PACKAGE_NAME "libvmemcache") 42 | set(CPACK_DEBIAN_PACKAGE_VERSION ${CPACK_PACKAGE_VERSION}) 43 | set(CPACK_DEBIAN_PACKAGE_MAINTAINER "lukasz.dorau@intel.com") 44 | 45 | if("${CPACK_GENERATOR}" STREQUAL "RPM") 46 | set(CPACK_PACKAGE_FILE_NAME 47 | ${CPACK_RPM_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}.${CPACK_RPM_PACKAGE_ARCHITECTURE}) 48 | elseif("${CPACK_GENERATOR}" STREQUAL "DEB") 49 | set(CPACK_PACKAGE_FILE_NAME 50 | ${CPACK_DEBIAN_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}_${CPACK_DEBIAN_PACKAGE_ARCHITECTURE}) 51 | endif() 52 | 53 | set(targetDestDir ${CMAKE_INSTALL_TMPDIR}) 54 | include(CPack) 55 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | cmake_minimum_required(VERSION 3.3) 5 | project(vmemcache C) 6 | 7 | add_cstyle(src) 8 | add_check_whitespace(src) 9 | 10 | set(SOURCES 11 | out.c 12 | os_posix.c 13 | os_thread_posix.c 14 | util.c 15 | util_posix.c 16 | file.c 17 | file_posix.c 18 | fast-hash.c 19 | mmap.c 20 | mmap_posix.c 21 | libvmemcache.c 22 | critnib.c 23 | ringbuf.c 24 | vmemcache.c 25 | vmemcache_heap.c 26 | vmemcache_index.c 27 | vmemcache_repl.c) 28 | 29 | add_library(vmemcache SHARED ${SOURCES}) 30 | target_link_libraries(vmemcache PRIVATE 31 | ${CMAKE_THREAD_LIBS_INIT} 32 | -Wl,--version-script=${CMAKE_SOURCE_DIR}/src/libvmemcache.map) 33 | set_target_properties(vmemcache PROPERTIES SOVERSION 0) 34 | 35 | target_compile_definitions(vmemcache PRIVATE SRCVERSION="${VERSION}") 36 | 37 | if(STATS_ENABLED) 38 | target_compile_definitions(vmemcache PRIVATE STATS_ENABLED=1) 39 | endif() 40 | 41 | if(VALGRIND_FOUND) 42 | target_compile_definitions(vmemcache PRIVATE VALGRIND_ENABLED=1) 43 | endif() 44 | 45 | install(TARGETS vmemcache 46 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 47 | DESTINATION ${CMAKE_INSTALL_LIBDIR}/) 48 | install(FILES libvmemcache.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) 49 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2016-2019, Intel Corporation */ 3 | 4 | /* 5 | * common.h -- common definitions 6 | */ 7 | 8 | #ifndef COMMON_H 9 | #define COMMON_H 1 10 | 11 | #include "util.h" 12 | #include "out.h" 13 | #include "mmap.h" 14 | 15 | #ifdef __cplusplus 16 | extern "C" { 17 | #endif 18 | 19 | static inline void 20 | common_init(const char *log_prefix, const char *log_level_var, 21 | const char *log_file_var, int major_version, 22 | int minor_version) 23 | { 24 | util_init(); 25 | out_init(log_prefix, log_level_var, log_file_var, major_version, 26 | minor_version); 27 | } 28 | 29 | static inline void 30 | common_fini(void) 31 | { 32 | out_fini(); 33 | } 34 | 35 | #ifdef __cplusplus 36 | } 37 | #endif 38 | 39 | #endif 40 | -------------------------------------------------------------------------------- /src/critnib.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2020, Intel Corporation */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | #include "os_thread.h" 10 | #include "util.h" 11 | #include "out.h" 12 | #include "critnib.h" 13 | 14 | /* 15 | * WARNING: this implementation fails badly if you try to store two keys 16 | * where one is a prefix of another. Pass a struct { int len; char[] key; } 17 | * or such if such keys are possible. 18 | */ 19 | 20 | #define NIB ((1 << SLICE) - 1) 21 | 22 | #define KEYLEN(leaf) (leaf->key.ksize + sizeof(size_t)) 23 | 24 | typedef struct cache_entry critnib_leaf; 25 | 26 | /* 27 | * is_leaf -- (internal) check tagged pointer for leafness 28 | */ 29 | static inline bool 30 | is_leaf(struct critnib_node *n) 31 | { 32 | return (uintptr_t)n & 1; 33 | } 34 | 35 | /* 36 | * to_leaf -- (internal) untag a leaf pointer 37 | */ 38 | static inline critnib_leaf * 39 | to_leaf(struct critnib_node *n) 40 | { 41 | return (void *)((uintptr_t)n & ~1ULL); 42 | } 43 | 44 | /* 45 | * slice_index -- (internal) get index of radix child at a given shift 46 | */ 47 | static inline int 48 | slice_index(char b, bitn_t bit) 49 | { 50 | return (b >> bit) & NIB; 51 | } 52 | 53 | /* 54 | * critnib_new -- allocate a new hashmap 55 | */ 56 | struct critnib * 57 | critnib_new(void) 58 | { 59 | struct critnib *c = Zalloc(sizeof(struct critnib)); 60 | if (!c) 61 | return NULL; 62 | return c; 63 | } 64 | 65 | /* 66 | * delete_node -- (internal) recursively free a subtree 67 | */ 68 | static void 69 | delete_node(struct critnib_node *n, delete_entry_t del) 70 | { 71 | if (!n) 72 | return; 73 | if (is_leaf(n)) { 74 | if (del) 75 | del(to_leaf(n)); 76 | return; 77 | } 78 | for (int i = 0; i < SLNODES; i++) 79 | delete_node(n->child[i], del); 80 | Free(n); 81 | } 82 | 83 | /* 84 | * critnib_delete -- free a hashmap 85 | */ 86 | void 87 | critnib_delete(struct critnib *c, delete_entry_t del) 88 | { 89 | delete_node(c->root, del); 90 | Free(c); 91 | } 92 | 93 | /* 94 | * alloc_node -- (internal) alloc a node 95 | */ 96 | static struct critnib_node * 97 | alloc_node(struct critnib *c) 98 | { 99 | struct critnib_node *n = Zalloc(sizeof(struct critnib_node)); 100 | if (!n) 101 | return NULL; 102 | #ifdef STATS_ENABLED 103 | c->node_count++; 104 | #endif 105 | return n; 106 | } 107 | 108 | /* 109 | * any_leaf -- (internal) find any leaf in a subtree 110 | * 111 | * We know they're all identical up to the divergence point between a prefix 112 | * shared by all of them vs the new key we're inserting. 113 | */ 114 | static struct critnib_node * 115 | any_leaf(struct critnib_node *n) 116 | { 117 | for (int i = 0; i < SLNODES; i++) { 118 | struct critnib_node *m; 119 | if ((m = n->child[i])) 120 | return is_leaf(m) ? m : any_leaf(m); 121 | } 122 | return NULL; 123 | } 124 | 125 | /* 126 | * critnib_set -- insert a new entry 127 | */ 128 | int 129 | critnib_set(struct critnib *c, struct cache_entry *e) 130 | { 131 | const char *key = (void *)&e->key; 132 | byten_t key_len = (byten_t)KEYLEN(e); 133 | critnib_leaf *k = (void *)((uintptr_t)e | 1); 134 | 135 | struct critnib_node *n = c->root; 136 | if (!n) { 137 | c->root = (void *)k; 138 | return 0; 139 | } 140 | 141 | /* 142 | * Need to descend the tree twice: first to find a leaf that 143 | * represents a subtree whose all keys share a prefix at least as 144 | * long as the one common to the new key and that subtree. 145 | */ 146 | while (!is_leaf(n) && n->byte < key_len) { 147 | struct critnib_node *nn = 148 | n->child[slice_index(key[n->byte], n->bit)]; 149 | if (nn) 150 | n = nn; 151 | else { 152 | n = any_leaf(n); 153 | break; 154 | } 155 | } 156 | 157 | ASSERT(n); 158 | if (!is_leaf(n)) 159 | n = any_leaf(n); 160 | 161 | ASSERT(n); 162 | ASSERT(is_leaf(n)); 163 | critnib_leaf *nk = to_leaf(n); 164 | const char *nkey = (void *)&nk->key; 165 | 166 | /* Find the divergence point, accurate to a byte. */ 167 | byten_t common_len = ((byten_t)KEYLEN(nk) < key_len) 168 | ? (byten_t)KEYLEN(nk) : key_len; 169 | byten_t diff; 170 | for (diff = 0; diff < common_len; diff++) { 171 | if (nkey[diff] != key[diff]) 172 | break; 173 | } 174 | 175 | if (diff >= common_len) { 176 | /* 177 | * Either an update or a conflict between keys being a 178 | * prefix of each other. 179 | */ 180 | return EEXIST; 181 | } 182 | 183 | /* Calculate the divergence point within the single byte. */ 184 | char at = nkey[diff] ^ key[diff]; 185 | bitn_t sh = util_mssb_index((uint32_t)(uint8_t)at) 186 | & (bitn_t)~(SLICE - 1); 187 | 188 | /* Descend into the tree again. */ 189 | n = c->root; 190 | struct critnib_node **parent = &c->root; 191 | while (n && !is_leaf(n) && 192 | (n->byte < diff || (n->byte == diff && n->bit >= sh))) { 193 | parent = &n->child[slice_index(key[n->byte], n->bit)]; 194 | n = *parent; 195 | } 196 | 197 | /* 198 | * If the divergence point is at same nib as an existing node, and 199 | * the subtree there is empty, just place our leaf there and we're 200 | * done. Obviously this can't happen if SLICE == 1. 201 | */ 202 | if (!n) { 203 | *parent = (void *)k; 204 | return 0; 205 | } 206 | 207 | /* If not, we need to insert a new node in the middle of an edge. */ 208 | if (!(n = alloc_node(c))) 209 | return ENOMEM; 210 | 211 | n->child[slice_index(nkey[diff], sh)] = *parent; 212 | n->child[slice_index(key[diff], sh)] = (void *)k; 213 | n->byte = diff; 214 | n->bit = sh; 215 | *parent = n; 216 | return 0; 217 | } 218 | 219 | /* 220 | * critnib_get -- query a key 221 | */ 222 | void * 223 | critnib_get(struct critnib *c, const struct cache_entry *e) 224 | { 225 | const char *key = (void *)&e->key; 226 | byten_t key_len = (byten_t)KEYLEN(e); 227 | 228 | struct critnib_node *n = c->root; 229 | while (n && !is_leaf(n)) { 230 | if (n->byte >= key_len) 231 | return NULL; 232 | n = n->child[slice_index(key[n->byte], n->bit)]; 233 | } 234 | 235 | if (!n) 236 | return NULL; 237 | 238 | critnib_leaf *k = to_leaf(n); 239 | 240 | /* 241 | * We checked only nibs at divergence points, have to re-check the 242 | * whole key. 243 | */ 244 | return (key_len != KEYLEN(k) || memcmp(key, (void *)&k->key, 245 | key_len)) ? NULL : k; 246 | } 247 | 248 | /* 249 | * critnib_remove -- query and delete a key 250 | * 251 | * Neither the key nor its value are freed, just our private nodes. 252 | */ 253 | void * 254 | critnib_remove(struct critnib *c, const struct cache_entry *e) 255 | { 256 | const char *key = (void *)&e->key; 257 | byten_t key_len = (byten_t)KEYLEN(e); 258 | 259 | struct critnib_node **pp = NULL; 260 | struct critnib_node *n = c->root; 261 | struct critnib_node **parent = &c->root; 262 | 263 | /* First, do a get. */ 264 | while (n && !is_leaf(n)) { 265 | if (n->byte >= key_len) 266 | return NULL; 267 | pp = parent; 268 | parent = &n->child[slice_index(key[n->byte], n->bit)]; 269 | n = *parent; 270 | } 271 | 272 | if (!n) 273 | return NULL; 274 | 275 | critnib_leaf *k = to_leaf(n); 276 | if (key_len != KEYLEN(k) || memcmp(key, (void *)&k->key, key_len)) 277 | return NULL; 278 | 279 | /* Remove the entry (leaf). */ 280 | *parent = NULL; 281 | 282 | if (!pp) /* was root */ 283 | return k; 284 | 285 | /* Check if after deletion the node has just a single child left. */ 286 | n = *pp; 287 | struct critnib_node *only_child = NULL; 288 | for (int i = 0; i < SLNODES; i++) { 289 | if (n->child[i]) { 290 | if (only_child) /* Nope. */ 291 | return k; 292 | only_child = n->child[i]; 293 | } 294 | } 295 | 296 | /* Yes -- shorten the tree's edge. */ 297 | ASSERT(only_child); 298 | *pp = only_child; 299 | Free(n); 300 | #ifdef STATS_ENABLED 301 | c->node_count--; 302 | #endif 303 | return k; 304 | } 305 | -------------------------------------------------------------------------------- /src/critnib.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | #ifndef CRITNIB_H 5 | #define CRITNIB_H 6 | 7 | #include "vmemcache.h" 8 | #include "os_thread.h" 9 | 10 | /* 11 | * SLICE may be 1, 2, 4 or 8. 1 or 8 could be further optimized (critbit 12 | * and critbyte respectively); 4 (critnib) strikes a good balance between 13 | * speed and memory use. 14 | */ 15 | #define SLICE 4 16 | #define SLNODES (1 << SLICE) 17 | 18 | typedef uint32_t byten_t; 19 | typedef unsigned char bitn_t; 20 | 21 | struct critnib_node { 22 | struct critnib_node *child[SLNODES]; 23 | byten_t byte; 24 | bitn_t bit; 25 | }; 26 | 27 | struct critnib { 28 | struct critnib_node *root; 29 | os_rwlock_t lock; 30 | size_t leaf_count; /* entries */ 31 | size_t node_count; /* internal nodes only */ 32 | size_t DRAM_usage; /* ... of leaves (nodes are constant-sized) */ 33 | /* operation counts */ 34 | size_t put_count; 35 | size_t evict_count; 36 | size_t hit_count; 37 | size_t miss_count; 38 | }; 39 | 40 | struct cache_entry; 41 | 42 | struct critnib *critnib_new(void); 43 | void critnib_delete(struct critnib *c, delete_entry_t del); 44 | int critnib_set(struct critnib *c, struct cache_entry *e); 45 | void *critnib_get(struct critnib *c, const struct cache_entry *e); 46 | void *critnib_remove(struct critnib *c, const struct cache_entry *e); 47 | 48 | #endif 49 | -------------------------------------------------------------------------------- /src/fast-hash.c: -------------------------------------------------------------------------------- 1 | /* 2 | * The fast-hash algorithm is covered by the MIT License: 3 | * 4 | * Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com) 5 | * 6 | * Permission is hereby granted, free of charge, to any person 7 | * obtaining a copy of this software and associated documentation 8 | * files (the "Software"), to deal in the Software without 9 | * restriction, including without limitation the rights to use, copy, 10 | * modify, merge, publish, distribute, sublicense, and/or sell copies 11 | * of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | * SOFTWARE. 25 | */ 26 | 27 | #include "fast-hash.h" 28 | #include 29 | 30 | /* 31 | * mix -- (internal) helper for the fast-hash mixing step 32 | */ 33 | static inline uint64_t 34 | mix(uint64_t h) 35 | { 36 | h ^= h >> 23; 37 | h *= 0x2127599bf4325c37ULL; 38 | return h ^ h >> 47; 39 | } 40 | 41 | /* 42 | * hash -- calculate the hash of a piece of memory 43 | */ 44 | uint64_t 45 | hash(size_t key_size, const char *key) 46 | { 47 | /* fast-hash, by Zilong Tan */ 48 | const uint64_t m = 0x880355f21e6d1965ULL; 49 | const uint64_t *pos = (const uint64_t *)key; 50 | const uint64_t *end = pos + (key_size / 8); 51 | uint64_t h = key_size * m; 52 | 53 | while (pos != end) 54 | h = (h ^ mix(*pos++)) * m; 55 | 56 | if (key_size & 7) { 57 | uint64_t shift = (key_size & 7) * 8; 58 | uint64_t mask = (1ULL << shift) - 1; 59 | uint64_t v = htole64(*pos) & mask; 60 | h = (h ^ mix(v)) * m; 61 | } 62 | 63 | return mix(h); 64 | } 65 | -------------------------------------------------------------------------------- /src/fast-hash.h: -------------------------------------------------------------------------------- 1 | /* 2 | * The fast-hash algorithm is covered by the MIT License: 3 | * 4 | * Copyright (C) 2012 Zilong Tan (eric.zltan@gmail.com) 5 | * 6 | * Permission is hereby granted, free of charge, to any person 7 | * obtaining a copy of this software and associated documentation 8 | * files (the "Software"), to deal in the Software without 9 | * restriction, including without limitation the rights to use, copy, 10 | * modify, merge, publish, distribute, sublicense, and/or sell copies 11 | * of the Software, and to permit persons to whom the Software is 12 | * furnished to do so, subject to the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be 15 | * included in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 21 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 23 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | * SOFTWARE. 25 | */ 26 | 27 | #ifndef FAST_HASH_H 28 | #define FAST_HASH_H 1 29 | 30 | #include 31 | #include 32 | uint64_t hash(size_t key_size, const char *key); 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/file.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2018, Intel Corporation */ 3 | 4 | /* 5 | * file.h -- internal definitions for file module 6 | */ 7 | 8 | #ifndef PMDK_FILE_H 9 | #define PMDK_FILE_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include "os.h" 17 | 18 | #ifdef __cplusplus 19 | extern "C" { 20 | #endif 21 | 22 | #ifdef _WIN32 23 | #define NAME_MAX _MAX_FNAME 24 | #endif 25 | 26 | struct file_info { 27 | char filename[NAME_MAX + 1]; 28 | int is_dir; 29 | }; 30 | 31 | struct dir_handle { 32 | const char *path; 33 | #ifdef _WIN32 34 | HANDLE handle; 35 | char *_file; 36 | #else 37 | DIR *dirp; 38 | #endif 39 | }; 40 | 41 | enum file_type { 42 | OTHER_ERROR = -2, 43 | NOT_EXISTS = -1, 44 | TYPE_NORMAL = 1, 45 | TYPE_DEVDAX = 2 46 | }; 47 | 48 | int util_file_dir_open(struct dir_handle *a, const char *path); 49 | int util_file_dir_next(struct dir_handle *a, struct file_info *info); 50 | int util_file_dir_close(struct dir_handle *a); 51 | int util_file_dir_remove(const char *path); 52 | int util_file_exists(const char *path); 53 | enum file_type util_fd_get_type(int fd); 54 | enum file_type util_file_get_type(const char *path); 55 | int util_ddax_region_find(const char *path); 56 | ssize_t util_file_get_size(const char *path); 57 | size_t util_file_device_dax_alignment(const char *path); 58 | void *util_file_map_whole(const char *path); 59 | int util_file_zero(const char *path, os_off_t off, size_t len); 60 | ssize_t util_file_pread(const char *path, void *buffer, size_t size, 61 | os_off_t offset); 62 | ssize_t util_file_pwrite(const char *path, const void *buffer, size_t size, 63 | os_off_t offset); 64 | 65 | int util_tmpfile(const char *dir, const char *templ, int flags); 66 | int util_is_absolute_path(const char *path); 67 | 68 | int util_file_create(const char *path, size_t size, size_t minsize); 69 | int util_file_open(const char *path, size_t *size, size_t minsize, int flags); 70 | int util_unlink(const char *path); 71 | int util_unlink_flock(const char *path); 72 | int util_file_mkdir(const char *path, mode_t mode); 73 | 74 | int util_write_all(int fd, const char *buf, size_t count); 75 | 76 | #ifndef _WIN32 77 | #define util_read read 78 | #define util_write write 79 | #else 80 | /* XXX - consider adding an assertion on (count <= UINT_MAX) */ 81 | #define util_read(fd, buf, count) read(fd, buf, (unsigned)(count)) 82 | #define util_write(fd, buf, count) write(fd, buf, (unsigned)(count)) 83 | #define S_ISCHR(m) (((m) & S_IFMT) == S_IFCHR) 84 | #define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) 85 | #endif 86 | #ifdef __cplusplus 87 | } 88 | #endif 89 | #endif 90 | -------------------------------------------------------------------------------- /src/file_posix.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2019, Intel Corporation */ 3 | 4 | /* 5 | * file_posix.c -- Posix versions of file APIs 6 | */ 7 | 8 | /* for O_TMPFILE */ 9 | #define _GNU_SOURCE 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #include "os.h" 22 | #include "file.h" 23 | #include "out.h" 24 | 25 | #if 0 26 | #define MAX_SIZE_LENGTH 64 27 | #define DAX_REGION_ID_LEN 6 /* 5 digits + \0 */ 28 | #endif 29 | 30 | /* 31 | * util_tmpfile_mkstemp -- (internal) create temporary file 32 | * if O_TMPFILE not supported 33 | */ 34 | static int 35 | util_tmpfile_mkstemp(const char *dir, const char *templ) 36 | { 37 | /* the templ must start with a path separator */ 38 | ASSERTeq(templ[0], '/'); 39 | 40 | int oerrno; 41 | int fd = -1; 42 | 43 | char *fullname = alloca(strlen(dir) + strlen(templ) + 1); 44 | 45 | (void) strcpy(fullname, dir); 46 | (void) strcat(fullname, templ); 47 | 48 | sigset_t set, oldset; 49 | sigfillset(&set); 50 | (void) sigprocmask(SIG_BLOCK, &set, &oldset); 51 | 52 | mode_t prev_umask = umask(S_IRWXG | S_IRWXO); 53 | 54 | fd = os_mkstemp(fullname); 55 | 56 | umask(prev_umask); 57 | 58 | if (fd < 0) { 59 | ERR("!mkstemp"); 60 | goto err; 61 | } 62 | 63 | (void) os_unlink(fullname); 64 | (void) sigprocmask(SIG_SETMASK, &oldset, NULL); 65 | LOG(3, "unlinked file is \"%s\"", fullname); 66 | 67 | return fd; 68 | 69 | err: 70 | oerrno = errno; 71 | (void) sigprocmask(SIG_SETMASK, &oldset, NULL); 72 | if (fd != -1) 73 | (void) os_close(fd); 74 | errno = oerrno; 75 | return -1; 76 | } 77 | 78 | /* 79 | * util_tmpfile -- create temporary file 80 | */ 81 | int 82 | util_tmpfile(const char *dir, const char *templ, int flags) 83 | { 84 | LOG(3, "dir \"%s\" template \"%s\" flags %x", dir, templ, flags); 85 | 86 | /* only O_EXCL is allowed here */ 87 | ASSERT(flags == 0 || flags == O_EXCL); 88 | 89 | #ifdef O_TMPFILE 90 | int fd = open(dir, O_TMPFILE | O_RDWR | flags, S_IRUSR | S_IWUSR); 91 | /* 92 | * Open can fail if underlying file system does not support O_TMPFILE 93 | * flag. 94 | */ 95 | if (fd >= 0) 96 | return fd; 97 | if (errno != EOPNOTSUPP) { 98 | ERR("!open"); 99 | return -1; 100 | } 101 | #endif 102 | 103 | return util_tmpfile_mkstemp(dir, templ); 104 | } 105 | 106 | #if 0 107 | /* 108 | * util_is_absolute_path -- check if the path is an absolute one 109 | */ 110 | int 111 | util_is_absolute_path(const char *path) 112 | { 113 | LOG(3, "path: %s", path); 114 | 115 | if (path[0] == OS_DIR_SEPARATOR) 116 | return 1; 117 | else 118 | return 0; 119 | } 120 | 121 | /* 122 | * util_create_mkdir -- creates new dir 123 | */ 124 | int 125 | util_file_mkdir(const char *path, mode_t mode) 126 | { 127 | LOG(3, "path: %s mode: %o", path, mode); 128 | return mkdir(path, mode); 129 | } 130 | 131 | /* 132 | * util_file_dir_open -- open a directory 133 | */ 134 | int 135 | util_file_dir_open(struct dir_handle *handle, const char *path) 136 | { 137 | LOG(3, "handle: %p path: %s", handle, path); 138 | handle->dirp = opendir(path); 139 | return handle->dirp == NULL; 140 | } 141 | 142 | /* 143 | * util_file_dir_next -- read next file in directory 144 | */ 145 | int 146 | util_file_dir_next(struct dir_handle *handle, struct file_info *info) 147 | { 148 | LOG(3, "handle: %p info: %p", handle, info); 149 | struct dirent *d = readdir(handle->dirp); 150 | if (d == NULL) 151 | return 1; /* break */ 152 | info->filename[NAME_MAX] = '\0'; 153 | strncpy(info->filename, d->d_name, NAME_MAX + 1); 154 | if (info->filename[NAME_MAX] != '\0') 155 | return -1; /* filename truncated */ 156 | info->is_dir = d->d_type == DT_DIR; 157 | return 0; /* continue */ 158 | } 159 | 160 | /* 161 | * util_file_dir_close -- close a directory 162 | */ 163 | int 164 | util_file_dir_close(struct dir_handle *handle) 165 | { 166 | LOG(3, "path: %p", handle); 167 | return closedir(handle->dirp); 168 | } 169 | 170 | /* 171 | * util_file_dir_remove -- remove directory 172 | */ 173 | int 174 | util_file_dir_remove(const char *path) 175 | { 176 | LOG(3, "path: %s", path); 177 | return rmdir(path); 178 | } 179 | 180 | /* 181 | * device_dax_alignment -- (internal) checks the alignment of given Device DAX 182 | */ 183 | static size_t 184 | device_dax_alignment(const char *path) 185 | { 186 | LOG(3, "path \"%s\"", path); 187 | 188 | os_stat_t st; 189 | int olderrno; 190 | 191 | if (os_stat(path, &st) < 0) { 192 | ERR("!stat \"%s\"", path); 193 | return 0; 194 | } 195 | 196 | char spath[PATH_MAX]; 197 | snprintf(spath, PATH_MAX, "/sys/dev/char/%u:%u/device/align", 198 | os_major(st.st_rdev), os_minor(st.st_rdev)); 199 | 200 | LOG(4, "device align path \"%s\"", spath); 201 | 202 | int fd = os_open(spath, O_RDONLY); 203 | if (fd < 0) { 204 | ERR("!open \"%s\"", spath); 205 | return 0; 206 | } 207 | 208 | size_t size = 0; 209 | 210 | char sizebuf[MAX_SIZE_LENGTH + 1]; 211 | ssize_t nread; 212 | if ((nread = read(fd, sizebuf, MAX_SIZE_LENGTH)) < 0) { 213 | ERR("!read"); 214 | goto out; 215 | } 216 | 217 | sizebuf[nread] = 0; /* null termination */ 218 | 219 | char *endptr; 220 | 221 | olderrno = errno; 222 | errno = 0; 223 | 224 | /* 'align' is in decimal format */ 225 | size = strtoull(sizebuf, &endptr, 10); 226 | if (endptr == sizebuf || *endptr != '\n' || 227 | (size == ULLONG_MAX && errno == ERANGE)) { 228 | ERR("invalid device alignment %s", sizebuf); 229 | size = 0; 230 | goto out; 231 | } 232 | 233 | /* 234 | * If the alignment value is not a power of two, try with 235 | * hex format, as this is how it was printed in older kernels. 236 | * Just in case someone is using kernel <4.9. 237 | */ 238 | if ((size & (size - 1)) != 0) { 239 | size = strtoull(sizebuf, &endptr, 16); 240 | if (endptr == sizebuf || *endptr != '\n' || 241 | (size == ULLONG_MAX && errno == ERANGE)) { 242 | ERR("invalid device alignment %s", sizebuf); 243 | size = 0; 244 | goto out; 245 | } 246 | } 247 | 248 | errno = olderrno; 249 | 250 | out: 251 | olderrno = errno; 252 | (void) os_close(fd); 253 | errno = olderrno; 254 | 255 | LOG(4, "device alignment %zu", size); 256 | return size; 257 | } 258 | 259 | /* 260 | * util_file_device_dax_alignment -- returns internal Device DAX alignment 261 | */ 262 | size_t 263 | util_file_device_dax_alignment(const char *path) 264 | { 265 | LOG(3, "path \"%s\"", path); 266 | 267 | return device_dax_alignment(path); 268 | } 269 | 270 | /* 271 | * util_ddax_region_find -- returns Device DAX region id 272 | */ 273 | int 274 | util_ddax_region_find(const char *path) 275 | { 276 | LOG(3, "path \"%s\"", path); 277 | 278 | int dax_reg_id_fd; 279 | char dax_region_path[PATH_MAX]; 280 | char reg_id[DAX_REGION_ID_LEN]; 281 | char *end_addr; 282 | os_stat_t st; 283 | 284 | ASSERTne(path, NULL); 285 | if (os_stat(path, &st) < 0) { 286 | ERR("!stat \"%s\"", path); 287 | return -1; 288 | } 289 | 290 | dev_t dev_id = st.st_rdev; 291 | 292 | unsigned major = os_major(dev_id); 293 | unsigned minor = os_minor(dev_id); 294 | int ret = snprintf(dax_region_path, PATH_MAX, 295 | "/sys/dev/char/%u:%u/device/dax_region/id", 296 | major, minor); 297 | if (ret < 0) { 298 | ERR("snprintf(%p, %d, /sys/dev/char/%u:%u/device/" 299 | "dax_region/id, %u, %u): %d", 300 | dax_region_path, PATH_MAX, major, minor, major, minor, 301 | ret); 302 | return -1; 303 | } 304 | 305 | if ((dax_reg_id_fd = os_open(dax_region_path, O_RDONLY)) < 0) { 306 | LOG(1, "!open(\"%s\", O_RDONLY)", dax_region_path); 307 | return -1; 308 | } 309 | 310 | ssize_t len = read(dax_reg_id_fd, reg_id, DAX_REGION_ID_LEN); 311 | 312 | if (len == -1) { 313 | ERR("!read(%d, %p, %d)", dax_reg_id_fd, 314 | reg_id, DAX_REGION_ID_LEN); 315 | goto err; 316 | } else if (len < 2 || reg_id[len - 1] != '\n') { 317 | errno = EINVAL; 318 | ERR("!read(%d, %p, %d) invalid format", dax_reg_id_fd, 319 | reg_id, DAX_REGION_ID_LEN); 320 | goto err; 321 | } 322 | 323 | int olderrno = errno; 324 | errno = 0; 325 | long reg_num = strtol(reg_id, &end_addr, 10); 326 | if ((errno == ERANGE && (reg_num == LONG_MAX || reg_num == LONG_MIN)) || 327 | (errno != 0 && reg_num == 0)) { 328 | ERR("!strtol(%p, %p, 10)", reg_id, end_addr); 329 | goto err; 330 | } 331 | errno = olderrno; 332 | 333 | if (end_addr == reg_id) { 334 | ERR("!strtol(%p, %p, 10) no digits were found", 335 | reg_id, end_addr); 336 | goto err; 337 | } 338 | if (*end_addr != '\n') { 339 | ERR("!strtol(%s, %s, 10) invalid format", 340 | reg_id, end_addr); 341 | goto err; 342 | } 343 | 344 | os_close(dax_reg_id_fd); 345 | return (int)reg_num; 346 | 347 | err: 348 | os_close(dax_reg_id_fd); 349 | return -1; 350 | } 351 | #endif 352 | -------------------------------------------------------------------------------- /src/libvmemcache.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018, Intel Corporation */ 3 | 4 | /* 5 | * libvmemcache.c -- libvmemcache entry points 6 | */ 7 | 8 | #include 9 | #include 10 | 11 | #include "common.h" 12 | #include "libvmemcache.h" 13 | #include "vmemcache.h" 14 | 15 | /* 16 | * vmcache_init -- load-time initialization for vmcache 17 | * 18 | * Called automatically by the run-time loader. 19 | */ 20 | ATTR_CONSTRUCTOR 21 | void 22 | libvmemcache_init(void) 23 | { 24 | common_init(VMEMCACHE_PREFIX, VMEMCACHE_LEVEL_VAR, 25 | VMEMCACHE_FILE_VAR, VMEMCACHE_MAJOR_VERSION, 26 | VMEMCACHE_MINOR_VERSION); 27 | LOG(3, NULL); 28 | } 29 | 30 | /* 31 | * libvmemcache_fini -- libvmemcache cleanup routine 32 | * 33 | * Called automatically when the process terminates. 34 | */ 35 | ATTR_DESTRUCTOR 36 | void 37 | libvmemcache_fini(void) 38 | { 39 | LOG(3, NULL); 40 | common_fini(); 41 | } 42 | 43 | /* 44 | * vmemcache_errormsgU -- return last error message 45 | */ 46 | #ifndef _WIN32 47 | static inline 48 | #endif 49 | const char * 50 | vmemcache_errormsgU(void) 51 | { 52 | return out_get_errormsg(); 53 | } 54 | 55 | #ifndef _WIN32 56 | /* 57 | * vmemcache_errormsg -- return last error message 58 | */ 59 | const char * 60 | vmemcache_errormsg(void) 61 | { 62 | return vmemcache_errormsgU(); 63 | } 64 | #else 65 | /* 66 | * vmemcache_errormsgW -- return last error message as wchar_t 67 | */ 68 | const wchar_t * 69 | vmemcache_errormsgW(void) 70 | { 71 | return out_get_errormsgW(); 72 | } 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/libvmemcache.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * libvmemcache.h -- definitions of libvmemcache entry points 6 | * 7 | * This library provides near-zero waste volatile caching. 8 | */ 9 | 10 | #ifndef LIBVMEMCACHE_H 11 | #define LIBVMEMCACHE_H 1 12 | 13 | #include 14 | #include 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | 20 | #ifdef _WIN32 21 | 22 | #ifndef PMDK_UTF8_API 23 | 24 | #define vmemcache_new vmemcache_newW 25 | #define vmemcache_errormsg vmemcache_errormsgW 26 | 27 | #else 28 | 29 | #define vmemcache_new vmemcache_newU 30 | #define vmemcache_errormsg vmemcache_errormsgU 31 | 32 | #endif 33 | 34 | #endif 35 | 36 | /* 37 | * VMEMCACHE_MAJOR_VERSION and VMEMCACHE_MINOR_VERSION provide the current 38 | * version of the libvmemcache API as provided by this header file. 39 | */ 40 | #define VMEMCACHE_MAJOR_VERSION 0 41 | #define VMEMCACHE_MINOR_VERSION 8 42 | 43 | #define VMEMCACHE_MIN_POOL ((size_t)(1024 * 1024)) /* minimum pool size: 1MB */ 44 | #define VMEMCACHE_MIN_EXTENT ((size_t)256) /* minimum size of extent: 256B */ 45 | 46 | /* 47 | * opaque type, internal to libvmemcache 48 | */ 49 | typedef struct vmemcache VMEMcache; 50 | 51 | enum vmemcache_repl_p { 52 | VMEMCACHE_REPLACEMENT_NONE, 53 | VMEMCACHE_REPLACEMENT_LRU, 54 | 55 | VMEMCACHE_REPLACEMENT_NUM 56 | }; 57 | 58 | enum vmemcache_statistic { 59 | VMEMCACHE_STAT_PUT, /* total number of puts */ 60 | VMEMCACHE_STAT_GET, /* total number of gets */ 61 | VMEMCACHE_STAT_HIT, /* total number of hits */ 62 | VMEMCACHE_STAT_MISS, /* total number of misses */ 63 | VMEMCACHE_STAT_EVICT, /* total number of evicts */ 64 | VMEMCACHE_STAT_ENTRIES, /* current number of cache entries */ 65 | VMEMCACHE_STAT_DRAM_SIZE_USED, /* current size of DRAM used for keys */ 66 | VMEMCACHE_STAT_POOL_SIZE_USED, /* current size of memory pool */ 67 | /* used for values */ 68 | VMEMCACHE_STAT_HEAP_ENTRIES, /* current number of allocator heap */ 69 | /* entries */ 70 | VMEMCACHE_STATS_NUM /* total number of statistics */ 71 | }; 72 | 73 | enum vmemcache_bench_cfg { 74 | /* these will corrupt the data, good only for benchmarking */ 75 | VMEMCACHE_BENCH_INDEX_ONLY, /* disable anything but indexing */ 76 | VMEMCACHE_BENCH_NO_ALLOC, /* index+repl but no alloc */ 77 | VMEMCACHE_BENCH_NO_MEMCPY, /* alloc but don't copy data */ 78 | VMEMCACHE_BENCH_PREFAULT, /* prefault the whole pool */ 79 | }; 80 | 81 | typedef void vmemcache_on_evict(VMEMcache *cache, 82 | const void *key, size_t key_size, void *arg); 83 | 84 | typedef void vmemcache_on_miss(VMEMcache *cache, 85 | const void *key, size_t key_size, void *arg); 86 | 87 | VMEMcache * 88 | vmemcache_new(void); 89 | 90 | int vmemcache_set_eviction_policy(VMEMcache *cache, 91 | enum vmemcache_repl_p repl_p); 92 | int vmemcache_set_size(VMEMcache *cache, size_t size); 93 | int vmemcache_set_extent_size(VMEMcache *cache, size_t extent_size); 94 | 95 | #ifndef _WIN32 96 | int vmemcache_add(VMEMcache *cache, const char *path); 97 | #else 98 | int vmemcache_addU(VMEMcache *cache, const char *path); 99 | int vmemcache_addW(VMEMcache *cache, const wchar_t *path); 100 | #endif 101 | 102 | void vmemcache_delete(VMEMcache *cache); 103 | 104 | void vmemcache_callback_on_evict(VMEMcache *cache, 105 | vmemcache_on_evict *evict, void *arg); 106 | void vmemcache_callback_on_miss(VMEMcache *cache, 107 | vmemcache_on_miss *miss, void *arg); 108 | 109 | ssize_t /* returns the number of bytes read */ 110 | vmemcache_get(VMEMcache *cache, 111 | const void *key, size_t key_size, 112 | void *vbuf, /* user-provided buffer */ 113 | size_t vbufsize, /* size of vbuf */ 114 | size_t offset, /* offset inside of value from which to begin copying */ 115 | size_t *vsize /* real size of the object */); 116 | 117 | int vmemcache_exists(VMEMcache *cache, 118 | const void *key, size_t key_size, size_t *vsize); 119 | 120 | int vmemcache_put(VMEMcache *cache, 121 | const void *key, size_t key_size, 122 | const void *value, size_t value_size); 123 | 124 | int vmemcache_evict(VMEMcache *cache, const void *key, size_t ksize); 125 | 126 | int vmemcache_get_stat(VMEMcache *cache, 127 | enum vmemcache_statistic stat, 128 | void *value, 129 | size_t value_size); 130 | 131 | #ifndef _WIN32 132 | const char *vmemcache_errormsg(void); 133 | #else 134 | const char *vmemcache_errormsgU(void); 135 | const wchar_t *vmemcache_errormsgW(void); 136 | #endif 137 | 138 | /* UNSTABLE INTEFACE -- DO NOT USE! */ 139 | void vmemcache_bench_set(VMEMcache *cache, enum vmemcache_bench_cfg cfg, 140 | size_t val); 141 | 142 | #ifdef __cplusplus 143 | } 144 | #endif 145 | #endif /* libvmemcache.h */ 146 | -------------------------------------------------------------------------------- /src/libvmemcache.map: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | # 4 | # 5 | # src/libvmemcache.map -- linker map file for libvmemcache 6 | # 7 | LIBVMEMCACHE_1.0 { 8 | global: 9 | vmemcache_new; 10 | vmemcache_delete; 11 | vmemcache_set_eviction_policy; 12 | vmemcache_set_size; 13 | vmemcache_set_extent_size; 14 | vmemcache_add; 15 | vmemcache_put; 16 | vmemcache_get; 17 | vmemcache_exists; 18 | vmemcache_evict; 19 | vmemcache_callback_on_evict; 20 | vmemcache_callback_on_miss; 21 | vmemcache_get_stat; 22 | vmemcache_bench_set; 23 | vmemcache_errormsg; 24 | local: 25 | *; 26 | }; 27 | -------------------------------------------------------------------------------- /src/mmap.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2019, Intel Corporation */ 3 | 4 | /* 5 | * mmap.c -- mmap utilities 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "file.h" 19 | #include "mmap.h" 20 | #include "sys_util.h" 21 | #include "os.h" 22 | 23 | /* 24 | * util_map -- memory map a file 25 | * 26 | * This is just a convenience function that calls mmap() with the 27 | * appropriate arguments and includes our trace points. 28 | */ 29 | void * 30 | util_map(int fd, size_t len, int flags, int rdonly, size_t req_align, 31 | int *map_sync) 32 | { 33 | LOG(3, "fd %d len %zu flags %d rdonly %d req_align %zu map_sync %p", 34 | fd, len, flags, rdonly, req_align, map_sync); 35 | 36 | void *base; 37 | void *addr = util_map_hint(len, req_align); 38 | if (addr == MAP_FAILED) { 39 | ERR("cannot find a contiguous region of given size"); 40 | return NULL; 41 | } 42 | 43 | if (req_align) 44 | ASSERTeq((uintptr_t)addr % req_align, 0); 45 | 46 | int proto = rdonly ? PROT_READ : PROT_READ|PROT_WRITE; 47 | base = util_map_sync(addr, len, proto, flags, fd, 0, map_sync); 48 | if (base == MAP_FAILED) { 49 | ERR("!mmap %zu bytes", len); 50 | return NULL; 51 | } 52 | 53 | LOG(3, "mapped at %p", base); 54 | 55 | return base; 56 | } 57 | 58 | /* 59 | * util_unmap -- unmap a file 60 | * 61 | * This is just a convenience function that calls munmap() with the 62 | * appropriate arguments and includes our trace points. 63 | */ 64 | int 65 | util_unmap(void *addr, size_t len) 66 | { 67 | LOG(3, "addr %p len %zu", addr, len); 68 | 69 | /* 70 | * XXX Workaround for https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=169608 71 | */ 72 | #ifdef __FreeBSD__ 73 | if (!IS_PAGE_ALIGNED((uintptr_t)addr)) { 74 | errno = EINVAL; 75 | ERR("!munmap"); 76 | return -1; 77 | } 78 | #endif 79 | int retval = munmap(addr, len); 80 | if (retval < 0) 81 | ERR("!munmap"); 82 | 83 | return retval; 84 | } 85 | 86 | /* 87 | * chattr -- (internal) set file attributes 88 | */ 89 | static void 90 | chattr(int fd, int set, int clear) 91 | { 92 | int attr; 93 | 94 | if (ioctl(fd, FS_IOC_GETFLAGS, &attr) < 0) { 95 | LOG(3, "!ioctl(FS_IOC_GETFLAGS) failed"); 96 | return; 97 | } 98 | 99 | attr |= set; 100 | attr &= ~clear; 101 | 102 | if (ioctl(fd, FS_IOC_SETFLAGS, &attr) < 0) { 103 | LOG(3, "!ioctl(FS_IOC_SETFLAGS) failed"); 104 | return; 105 | } 106 | } 107 | 108 | /* 109 | * util_map_tmpfile -- reserve space in an unlinked file and memory-map it 110 | * 111 | * size must be multiple of page size. 112 | */ 113 | void * 114 | util_map_tmpfile(const char *dir, size_t size, size_t req_align) 115 | { 116 | int oerrno; 117 | 118 | if (((os_off_t)size) < 0) { 119 | ERR("invalid size (%zu) for os_off_t", size); 120 | errno = EFBIG; 121 | return NULL; 122 | } 123 | 124 | int fd = util_tmpfile(dir, OS_DIR_SEP_STR "vmem.XXXXXX", O_EXCL); 125 | if (fd == -1) { 126 | LOG(2, "cannot create temporary file in dir %s", dir); 127 | goto err; 128 | } 129 | 130 | chattr(fd, FS_NOCOW_FL, 0); 131 | 132 | if ((errno = os_posix_fallocate(fd, 0, (os_off_t)size)) != 0) { 133 | ERR("!posix_fallocate"); 134 | goto err; 135 | } 136 | 137 | void *base; 138 | if ((base = util_map(fd, size, MAP_SHARED, 139 | 0, req_align, NULL)) == NULL) { 140 | LOG(2, "cannot mmap temporary file"); 141 | goto err; 142 | } 143 | 144 | (void) os_close(fd); 145 | return base; 146 | 147 | err: 148 | oerrno = errno; 149 | if (fd != -1) 150 | (void) os_close(fd); 151 | errno = oerrno; 152 | return NULL; 153 | } 154 | -------------------------------------------------------------------------------- /src/mmap.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2019, Intel Corporation */ 3 | 4 | /* 5 | * mmap.h -- internal definitions for mmap module 6 | */ 7 | 8 | #ifndef PMDK_MMAP_H 9 | #define PMDK_MMAP_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #include "out.h" 19 | #include "sys/queue.h" 20 | #include "os.h" 21 | 22 | #ifdef __cplusplus 23 | extern "C" { 24 | #endif 25 | 26 | extern char *Mmap_mapfile; 27 | 28 | void *util_map_sync(void *addr, size_t len, int proto, int flags, int fd, 29 | os_off_t offset, int *map_sync); 30 | void *util_map(int fd, size_t len, int flags, int rdonly, 31 | size_t req_align, int *map_sync); 32 | int util_unmap(void *addr, size_t len); 33 | 34 | void *util_map_tmpfile(const char *dir, size_t size, size_t req_align); 35 | 36 | #ifdef __FreeBSD__ 37 | #define MAP_NORESERVE 0 38 | #define OS_MAPFILE "/proc/curproc/map" 39 | #else 40 | #define OS_MAPFILE "/proc/self/maps" 41 | #endif 42 | 43 | #ifndef MAP_SYNC 44 | #define MAP_SYNC 0x80000 45 | #endif 46 | 47 | #ifndef MAP_SHARED_VALIDATE 48 | #define MAP_SHARED_VALIDATE 0x03 49 | #endif 50 | 51 | /* 52 | * macros for micromanaging range protections for the debug version 53 | */ 54 | #ifdef DEBUG 55 | 56 | #define RANGE(addr, len, is_dev_dax, type) do {\ 57 | if (!is_dev_dax) ASSERT(util_range_##type(addr, len) >= 0);\ 58 | } while (0) 59 | 60 | #else 61 | 62 | #define RANGE(addr, len, is_dev_dax, type) do {} while (0) 63 | 64 | #endif 65 | 66 | #define RANGE_RO(addr, len, is_dev_dax) RANGE(addr, len, is_dev_dax, ro) 67 | #define RANGE_RW(addr, len, is_dev_dax) RANGE(addr, len, is_dev_dax, rw) 68 | #define RANGE_NONE(addr, len, is_dev_dax) RANGE(addr, len, is_dev_dax, none) 69 | 70 | /* pmem mapping type */ 71 | enum pmem_map_type { 72 | PMEM_DEV_DAX, /* device dax */ 73 | PMEM_MAP_SYNC, /* mapping with MAP_SYNC flag on dax fs */ 74 | 75 | MAX_PMEM_TYPE 76 | }; 77 | 78 | /* 79 | * this structure tracks the file mappings outstanding per file handle 80 | */ 81 | struct map_tracker { 82 | SORTEDQ_ENTRY(map_tracker) entry; 83 | uintptr_t base_addr; 84 | uintptr_t end_addr; 85 | int region_id; 86 | enum pmem_map_type type; 87 | #ifdef _WIN32 88 | /* Windows-specific data */ 89 | HANDLE FileHandle; 90 | HANDLE FileMappingHandle; 91 | DWORD Access; 92 | os_off_t Offset; 93 | size_t FileLen; 94 | #endif 95 | }; 96 | 97 | char *util_map_hint_unused(void *minaddr, size_t len, size_t align); 98 | char *util_map_hint(size_t len, size_t req_align); 99 | 100 | #define MEGABYTE ((uintptr_t)1 << 20) 101 | #define GIGABYTE ((uintptr_t)1 << 30) 102 | 103 | /* 104 | * util_map_hint_align -- choose the desired mapping alignment 105 | * 106 | * The smallest supported alignment is 2 megabytes because of the object 107 | * alignment requirements. Changing this value to 4 kilobytes constitues a 108 | * layout change. 109 | * 110 | * Use 1GB page alignment only if the mapping length is at least 111 | * twice as big as the page size. 112 | */ 113 | static inline size_t 114 | util_map_hint_align(size_t len, size_t req_align) 115 | { 116 | size_t align = 2 * MEGABYTE; 117 | if (req_align) 118 | align = req_align; 119 | else if (len >= 2 * GIGABYTE) 120 | align = GIGABYTE; 121 | 122 | return align; 123 | } 124 | 125 | #ifdef __cplusplus 126 | } 127 | #endif 128 | #endif 129 | -------------------------------------------------------------------------------- /src/mmap_posix.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2019, Intel Corporation */ 3 | 4 | /* 5 | * mmap_posix.c -- memory-mapped files for Posix 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include "mmap.h" 12 | #include "out.h" 13 | #include "os.h" 14 | 15 | #define PROCMAXLEN 2048 /* maximum expected line length in /proc files */ 16 | 17 | char *Mmap_mapfile = OS_MAPFILE; /* Should be modified only for testing */ 18 | 19 | #ifdef __FreeBSD__ 20 | static const char * const sscanf_os = "%p %p"; 21 | #else 22 | static const char * const sscanf_os = "%p-%p"; 23 | #endif 24 | 25 | /* 26 | * util_map_hint_unused -- use /proc to determine a hint address for mmap() 27 | * 28 | * This is a helper function for util_map_hint(). 29 | * It opens up /proc/self/maps and looks for the first unused address 30 | * in the process address space that is: 31 | * - greater or equal 'minaddr' argument, 32 | * - large enough to hold range of given length, 33 | * - aligned to the specified unit. 34 | * 35 | * Asking for aligned address like this will allow the DAX code to use large 36 | * mappings. It is not an error if mmap() ignores the hint and chooses 37 | * different address. 38 | */ 39 | char * 40 | util_map_hint_unused(void *minaddr, size_t len, size_t align) 41 | { 42 | LOG(3, "minaddr %p len %zu align %zu", minaddr, len, align); 43 | ASSERT(align > 0); 44 | 45 | FILE *fp; 46 | if ((fp = os_fopen(Mmap_mapfile, "r")) == NULL) { 47 | ERR("!%s", Mmap_mapfile); 48 | return MAP_FAILED; 49 | } 50 | 51 | char line[PROCMAXLEN]; /* for fgets() */ 52 | char *lo = NULL; /* beginning of current range in maps file */ 53 | char *hi = NULL; /* end of current range in maps file */ 54 | char *raddr = minaddr; /* ignore regions below 'minaddr' */ 55 | 56 | if (raddr == NULL) 57 | raddr += Pagesize; 58 | 59 | raddr = (char *)roundup((uintptr_t)raddr, align); 60 | 61 | while (fgets(line, PROCMAXLEN, fp) != NULL) { 62 | /* check for range line */ 63 | if (sscanf(line, sscanf_os, &lo, &hi) == 2) { 64 | LOG(4, "%p-%p", lo, hi); 65 | if (lo > raddr) { 66 | if ((uintptr_t)(lo - raddr) >= len) { 67 | LOG(4, "unused region of size %zu " 68 | "found at %p", 69 | lo - raddr, raddr); 70 | break; 71 | } else { 72 | LOG(4, "region is too small: %zu < %zu", 73 | lo - raddr, len); 74 | } 75 | } 76 | 77 | if (hi > raddr) { 78 | raddr = (char *)roundup((uintptr_t)hi, align); 79 | LOG(4, "nearest aligned addr %p", raddr); 80 | } 81 | 82 | if (raddr == NULL) { 83 | LOG(4, "end of address space reached"); 84 | break; 85 | } 86 | } 87 | } 88 | 89 | /* 90 | * Check for a case when this is the last unused range in the address 91 | * space, but is not large enough. (very unlikely) 92 | */ 93 | if ((raddr != NULL) && (UINTPTR_MAX - (uintptr_t)raddr < len)) { 94 | LOG(4, "end of address space reached"); 95 | raddr = MAP_FAILED; 96 | } 97 | 98 | fclose(fp); 99 | 100 | LOG(3, "returning %p", raddr); 101 | return raddr; 102 | } 103 | 104 | /* 105 | * util_map_hint -- determine hint address for mmap() 106 | * 107 | * The system picks the randomized mapping address. 108 | * ALSR in 64-bit Linux kernel uses 28-bit of randomness for mmap 109 | * (bit positions 12-39), which means the base mapping address is randomized 110 | * within [0..1024GB] range, with 4KB granularity. Assuming additional 111 | * 1GB alignment, it results in 1024 possible locations. 112 | */ 113 | char * 114 | util_map_hint(size_t len, size_t req_align) 115 | { 116 | LOG(3, "len %zu req_align %zu", len, req_align); 117 | 118 | char *hint_addr = MAP_FAILED; 119 | 120 | /* choose the desired alignment based on the requested length */ 121 | size_t align = util_map_hint_align(len, req_align); 122 | 123 | /* 124 | * Create dummy mapping to find an unused region of given size. 125 | * Request for increased size for later address alignment. 126 | * Use MAP_PRIVATE with read-only access to simulate 127 | * zero cost for overcommit accounting. Note: MAP_NORESERVE 128 | * flag is ignored if overcommit is disabled (mode 2). 129 | */ 130 | char *addr = mmap(NULL, len + align, PROT_READ, 131 | MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); 132 | if (addr != MAP_FAILED) { 133 | LOG(4, "system choice %p", addr); 134 | hint_addr = (char *)roundup((uintptr_t)addr, align); 135 | munmap(addr, len + align); 136 | } 137 | 138 | LOG(4, "hint %p", hint_addr); 139 | 140 | return hint_addr; 141 | } 142 | 143 | /* 144 | * util_map_sync -- memory map given file into memory, if MAP_SHARED flag is 145 | * provided it attempts to use MAP_SYNC flag. Otherwise it fallbacks to 146 | * mmap(2). 147 | */ 148 | void * 149 | util_map_sync(void *addr, size_t len, int proto, int flags, int fd, 150 | os_off_t offset, int *map_sync) 151 | { 152 | LOG(15, "addr %p len %zu proto %x flags %x fd %d offset %ld " 153 | "map_sync %p", addr, len, proto, flags, fd, offset, map_sync); 154 | 155 | if (map_sync) 156 | *map_sync = 0; 157 | 158 | /* if map_sync is NULL do not even try to mmap with MAP_SYNC flag */ 159 | if (!map_sync || flags & MAP_PRIVATE) 160 | return mmap(addr, len, proto, flags, fd, offset); 161 | 162 | /* MAP_SHARED */ 163 | void *ret = mmap(addr, len, proto, 164 | flags | MAP_SHARED_VALIDATE | MAP_SYNC, 165 | fd, offset); 166 | if (ret != MAP_FAILED) { 167 | LOG(4, "mmap with MAP_SYNC succeeded"); 168 | *map_sync = 1; 169 | return ret; 170 | } 171 | 172 | if (errno == EINVAL || errno == ENOTSUP) { 173 | LOG(4, "mmap with MAP_SYNC not supported"); 174 | return mmap(addr, len, proto, flags, fd, offset); 175 | } 176 | 177 | /* other error */ 178 | return MAP_FAILED; 179 | } 180 | -------------------------------------------------------------------------------- /src/os.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2017-2018, Intel Corporation */ 3 | 4 | /* 5 | * os.h -- os abstaction layer 6 | */ 7 | 8 | #ifndef PMDK_OS_H 9 | #define PMDK_OS_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #ifdef __cplusplus 16 | extern "C" { 17 | #endif 18 | 19 | #ifndef _WIN32 20 | #define OS_DIR_SEPARATOR '/' 21 | #define OS_DIR_SEP_STR "/" 22 | #else 23 | #define OS_DIR_SEPARATOR '\\' 24 | #define OS_DIR_SEP_STR "\\" 25 | #endif 26 | 27 | #ifndef _WIN32 28 | 29 | /* madvise() */ 30 | #ifdef __FreeBSD__ 31 | #define os_madvise minherit 32 | #define MADV_DONTFORK INHERIT_NONE 33 | #else 34 | #define os_madvise madvise 35 | #endif 36 | 37 | /* dlopen() */ 38 | #ifdef __FreeBSD__ 39 | #define RTLD_DEEPBIND 0 /* XXX */ 40 | #endif 41 | 42 | /* major(), minor() */ 43 | #ifdef __FreeBSD__ 44 | #define os_major (unsigned)major 45 | #define os_minor (unsigned)minor 46 | #else 47 | #define os_major major 48 | #define os_minor minor 49 | #endif 50 | 51 | #endif /* #ifndef _WIN32 */ 52 | 53 | struct iovec; 54 | 55 | /* os_flock */ 56 | #define OS_LOCK_SH 1 57 | #define OS_LOCK_EX 2 58 | #define OS_LOCK_NB 4 59 | #define OS_LOCK_UN 8 60 | 61 | #ifndef _WIN32 62 | typedef struct stat os_stat_t; 63 | #define os_fstat fstat 64 | #define os_lseek lseek 65 | #else 66 | typedef struct _stat64 os_stat_t; 67 | #define os_fstat _fstat64 68 | #define os_lseek _lseeki64 69 | #endif 70 | 71 | #define os_close close 72 | #define os_fclose fclose 73 | 74 | #ifndef _WIN32 75 | typedef off_t os_off_t; 76 | #else 77 | /* XXX: os_off_t defined in platform.h */ 78 | #endif 79 | int os_open(const char *pathname, int flags, ...); 80 | int os_fsync(int fd); 81 | int os_fsync_dir(const char *dir_name); 82 | int os_stat(const char *pathname, os_stat_t *buf); 83 | int os_unlink(const char *pathname); 84 | int os_access(const char *pathname, int mode); 85 | FILE *os_fopen(const char *pathname, const char *mode); 86 | FILE *os_fdopen(int fd, const char *mode); 87 | int os_chmod(const char *pathname, mode_t mode); 88 | int os_mkstemp(char *temp); 89 | int os_posix_fallocate(int fd, os_off_t offset, os_off_t len); 90 | int os_ftruncate(int fd, os_off_t length); 91 | int os_flock(int fd, int operation); 92 | ssize_t os_writev(int fd, const struct iovec *iov, int iovcnt); 93 | int os_clock_gettime(int id, struct timespec *ts); 94 | unsigned os_rand_r(unsigned *seedp); 95 | int os_unsetenv(const char *name); 96 | int os_setenv(const char *name, const char *value, int overwrite); 97 | char *os_getenv(const char *name); 98 | const char *os_strsignal(int sig); 99 | int os_execv(const char *path, char *const argv[]); 100 | 101 | /* 102 | * XXX: missing APis (used in ut_file.c) 103 | * 104 | * rename 105 | * read 106 | * write 107 | */ 108 | 109 | #ifdef __cplusplus 110 | } 111 | #endif 112 | 113 | #endif /* os.h */ 114 | -------------------------------------------------------------------------------- /src/os_posix.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2017-2019, Intel Corporation */ 3 | 4 | /* 5 | * os_posix.c -- abstraction layer for basic Posix functions 6 | */ 7 | 8 | #define _GNU_SOURCE 9 | 10 | #include 11 | #include 12 | #include 13 | #ifdef __FreeBSD__ 14 | #include 15 | #endif 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include "util.h" 25 | #include "out.h" 26 | #include "os.h" 27 | 28 | /* 29 | * os_open -- open abstraction layer 30 | */ 31 | int 32 | os_open(const char *pathname, int flags, ...) 33 | { 34 | int mode_required = (flags & O_CREAT) == O_CREAT; 35 | 36 | #ifdef O_TMPFILE 37 | mode_required |= (flags & O_TMPFILE) == O_TMPFILE; 38 | #endif 39 | 40 | if (mode_required) { 41 | va_list arg; 42 | va_start(arg, flags); 43 | /* Clang requires int due to auto-promotion */ 44 | int mode = va_arg(arg, int); 45 | va_end(arg); 46 | return open(pathname, flags, (mode_t)mode); 47 | } else { 48 | return open(pathname, flags); 49 | } 50 | } 51 | 52 | #if 0 53 | /* 54 | * os_fsync -- fsync abstraction layer 55 | */ 56 | int 57 | os_fsync(int fd) 58 | { 59 | return fsync(fd); 60 | } 61 | 62 | /* 63 | * os_fsync_dir -- fsync the directory 64 | */ 65 | int 66 | os_fsync_dir(const char *dir_name) 67 | { 68 | int fd = os_open(dir_name, O_RDONLY | O_DIRECTORY); 69 | if (fd < 0) 70 | return -1; 71 | 72 | int ret = os_fsync(fd); 73 | 74 | os_close(fd); 75 | 76 | return ret; 77 | } 78 | #endif 79 | 80 | /* 81 | * os_stat -- stat abstraction layer 82 | */ 83 | int 84 | os_stat(const char *pathname, os_stat_t *buf) 85 | { 86 | return stat(pathname, buf); 87 | } 88 | 89 | /* 90 | * os_unlink -- unlink abstraction layer 91 | */ 92 | int 93 | os_unlink(const char *pathname) 94 | { 95 | return unlink(pathname); 96 | } 97 | 98 | /* 99 | * os_access -- access abstraction layer 100 | */ 101 | int 102 | os_access(const char *pathname, int mode) 103 | { 104 | return access(pathname, mode); 105 | } 106 | 107 | /* 108 | * os_fopen -- fopen abstraction layer 109 | */ 110 | FILE * 111 | os_fopen(const char *pathname, const char *mode) 112 | { 113 | return fopen(pathname, mode); 114 | } 115 | 116 | #if 0 117 | /* 118 | * os_fdopen -- fdopen abstraction layer 119 | */ 120 | FILE * 121 | os_fdopen(int fd, const char *mode) 122 | { 123 | return fdopen(fd, mode); 124 | } 125 | 126 | /* 127 | * os_chmod -- chmod abstraction layer 128 | */ 129 | int 130 | os_chmod(const char *pathname, mode_t mode) 131 | { 132 | return chmod(pathname, mode); 133 | } 134 | #endif 135 | 136 | /* 137 | * os_mkstemp -- mkstemp abstraction layer 138 | */ 139 | int 140 | os_mkstemp(char *temp) 141 | { 142 | return mkstemp(temp); 143 | } 144 | 145 | /* 146 | * os_posix_fallocate -- posix_fallocate abstraction layer 147 | */ 148 | int 149 | os_posix_fallocate(int fd, os_off_t offset, off_t len) 150 | { 151 | 152 | #ifdef __FreeBSD__ 153 | struct stat fbuf; 154 | struct statfs fsbuf; 155 | /* 156 | * XXX Workaround for https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=223287 157 | * 158 | * FreeBSD implements posix_fallocate with a simple block allocation/zero 159 | * loop. If the requested size is unreasonably large, this can result in 160 | * an uninterruptable system call that will suck up all the space in the 161 | * file system and could take hours to fail. To avoid this, make a crude 162 | * check to see if the requested allocation is larger than the available 163 | * space in the file system (minus any blocks already allocated to the 164 | * file), and if so, immediately return ENOSPC. We do the check only if 165 | * the offset is 0; otherwise, trying to figure out how many additional 166 | * blocks are required is too complicated. 167 | * 168 | * This workaround is here mostly to fail "absurdly" large requests for 169 | * testing purposes; however, it is coded to allow normal (albeit slow) 170 | * operation if the space can actually be allocated. Because of the way 171 | * PMDK uses posix_fallocate, supporting Linux-style fallocate in 172 | * FreeBSD should be considered. 173 | */ 174 | if (offset == 0) { 175 | if (fstatfs(fd, &fsbuf) == -1 || fstat(fd, &fbuf) == -1) 176 | return errno; 177 | 178 | size_t reqd_blocks = 179 | (((size_t)len + (fsbuf.f_bsize - 1)) / fsbuf.f_bsize) 180 | - (size_t)fbuf.st_blocks; 181 | if (reqd_blocks > (size_t)fsbuf.f_bavail) 182 | return ENOSPC; 183 | } 184 | #endif 185 | 186 | return posix_fallocate(fd, offset, len); 187 | } 188 | 189 | #if 0 190 | /* 191 | * os_ftruncate -- ftruncate abstraction layer 192 | */ 193 | int 194 | os_ftruncate(int fd, os_off_t length) 195 | { 196 | return ftruncate(fd, length); 197 | } 198 | 199 | /* 200 | * os_flock -- flock abstraction layer 201 | */ 202 | int 203 | os_flock(int fd, int operation) 204 | { 205 | int opt = 0; 206 | if (operation & OS_LOCK_EX) 207 | opt |= LOCK_EX; 208 | if (operation & OS_LOCK_SH) 209 | opt |= LOCK_SH; 210 | if (operation & OS_LOCK_UN) 211 | opt |= LOCK_UN; 212 | if (operation & OS_LOCK_NB) 213 | opt |= LOCK_NB; 214 | 215 | return flock(fd, opt); 216 | } 217 | 218 | /* 219 | * os_writev -- writev abstraction layer 220 | */ 221 | ssize_t 222 | os_writev(int fd, const struct iovec *iov, int iovcnt) 223 | { 224 | return writev(fd, iov, iovcnt); 225 | } 226 | #endif 227 | 228 | /* 229 | * os_clock_gettime -- clock_gettime abstraction layer 230 | */ 231 | int 232 | os_clock_gettime(int id, struct timespec *ts) 233 | { 234 | return clock_gettime(id, ts); 235 | } 236 | 237 | #if 0 238 | /* 239 | * os_rand_r -- rand_r abstraction layer 240 | */ 241 | unsigned 242 | os_rand_r(unsigned *seedp) 243 | { 244 | return (unsigned)rand_r(seedp); 245 | } 246 | 247 | /* 248 | * os_unsetenv -- unsetenv abstraction layer 249 | */ 250 | int 251 | os_unsetenv(const char *name) 252 | { 253 | return unsetenv(name); 254 | } 255 | 256 | /* 257 | * os_setenv -- setenv abstraction layer 258 | */ 259 | int 260 | os_setenv(const char *name, const char *value, int overwrite) 261 | { 262 | return setenv(name, value, overwrite); 263 | } 264 | #endif 265 | 266 | /* 267 | * secure_getenv -- provide GNU secure_getenv for FreeBSD 268 | */ 269 | #ifndef __USE_GNU 270 | static char * 271 | secure_getenv(const char *name) 272 | { 273 | if (issetugid() != 0) 274 | return NULL; 275 | 276 | return getenv(name); 277 | } 278 | #endif 279 | 280 | /* 281 | * os_getenv -- getenv abstraction layer 282 | */ 283 | char * 284 | os_getenv(const char *name) 285 | { 286 | return secure_getenv(name); 287 | } 288 | 289 | #if 0 290 | /* 291 | * os_strsignal -- strsignal abstraction layer 292 | */ 293 | const char * 294 | os_strsignal(int sig) 295 | { 296 | return strsignal(sig); 297 | } 298 | 299 | int 300 | os_execv(const char *path, char *const argv[]) 301 | { 302 | return execv(path, argv); 303 | } 304 | #endif 305 | -------------------------------------------------------------------------------- /src/os_thread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015-2018, Intel Corporation 3 | * Copyright (c) 2016, Microsoft Corporation. All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without 6 | * modification, are permitted provided that the following conditions 7 | * are met: 8 | * 9 | * * Redistributions of source code must retain the above copyright 10 | * notice, this list of conditions and the following disclaimer. 11 | * 12 | * * Redistributions in binary form must reproduce the above copyright 13 | * notice, this list of conditions and the following disclaimer in 14 | * the documentation and/or other materials provided with the 15 | * distribution. 16 | * 17 | * * Neither the name of the copyright holder nor the names of its 18 | * contributors may be used to endorse or promote products derived 19 | * from this software without specific prior written permission. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 | */ 33 | 34 | /* 35 | * os_thread.h -- os thread abstraction layer 36 | */ 37 | 38 | #ifndef OS_THREAD_H 39 | #define OS_THREAD_H 1 40 | 41 | #include 42 | #include 43 | 44 | #ifdef __cplusplus 45 | extern "C" { 46 | #endif 47 | 48 | typedef union { 49 | long long align; 50 | char padding[44]; /* linux: 40 windows: 44 */ 51 | } os_mutex_t; 52 | 53 | typedef union { 54 | long long align; 55 | char padding[56]; /* linux: 56 windows: 13 */ 56 | } os_rwlock_t; 57 | 58 | typedef union { 59 | long long align; 60 | char padding[48]; /* linux: 48 windows: 12 */ 61 | } os_cond_t; 62 | 63 | typedef union { 64 | long long align; 65 | char padding[32]; /* linux: 8 windows: 32 */ 66 | } os_thread_t; 67 | 68 | typedef union { 69 | long long align; /* linux: long windows: 8 FreeBSD: 12 */ 70 | char padding[16]; /* 16 to be safe */ 71 | } os_once_t; 72 | 73 | #define OS_ONCE_INIT { .padding = {0} } 74 | 75 | typedef unsigned os_tls_key_t; 76 | 77 | typedef union { 78 | long long align; 79 | char padding[56]; /* linux: 56 windows: 8 */ 80 | } os_semaphore_t; 81 | 82 | typedef union { 83 | long long align; 84 | char padding[56]; /* linux: 56 windows: 8 */ 85 | } os_thread_attr_t; 86 | 87 | typedef union { 88 | long long align; 89 | char padding[512]; 90 | } os_cpu_set_t; 91 | 92 | #ifdef __FreeBSD__ 93 | #define cpu_set_t cpuset_t 94 | typedef uintptr_t os_spinlock_t; 95 | #else 96 | typedef volatile int os_spinlock_t; /* XXX: not implemented on windows */ 97 | #endif 98 | 99 | void os_cpu_zero(os_cpu_set_t *set); 100 | void os_cpu_set(size_t cpu, os_cpu_set_t *set); 101 | 102 | #ifndef _WIN32 103 | #define _When_(...) 104 | #endif 105 | int os_once(os_once_t *o, void (*func)(void)); 106 | 107 | int os_tls_key_create(os_tls_key_t *key, void (*destructor)(void *)); 108 | int os_tls_key_delete(os_tls_key_t key); 109 | int os_tls_set(os_tls_key_t key, const void *value); 110 | void *os_tls_get(os_tls_key_t key); 111 | 112 | int os_mutex_init(os_mutex_t *__restrict mutex); 113 | int os_mutex_destroy(os_mutex_t *__restrict mutex); 114 | _When_(return == 0, _Acquires_lock_(mutex->lock)) 115 | int os_mutex_lock(os_mutex_t *__restrict mutex); 116 | _When_(return == 0, _Acquires_lock_(mutex->lock)) 117 | int os_mutex_trylock(os_mutex_t *__restrict mutex); 118 | int os_mutex_unlock(os_mutex_t *__restrict mutex); 119 | 120 | /* XXX - non POSIX */ 121 | int os_mutex_timedlock(os_mutex_t *__restrict mutex, 122 | const struct timespec *abstime); 123 | 124 | int os_rwlock_init(os_rwlock_t *__restrict rwlock); 125 | int os_rwlock_destroy(os_rwlock_t *__restrict rwlock); 126 | int os_rwlock_rdlock(os_rwlock_t *__restrict rwlock); 127 | int os_rwlock_wrlock(os_rwlock_t *__restrict rwlock); 128 | int os_rwlock_tryrdlock(os_rwlock_t *__restrict rwlock); 129 | _When_(return == 0, _Acquires_exclusive_lock_(rwlock->lock)) 130 | int os_rwlock_trywrlock(os_rwlock_t *__restrict rwlock); 131 | _When_(rwlock->is_write != 0, _Requires_exclusive_lock_held_(rwlock->lock)) 132 | _When_(rwlock->is_write == 0, _Requires_shared_lock_held_(rwlock->lock)) 133 | int os_rwlock_unlock(os_rwlock_t *__restrict rwlock); 134 | int os_rwlock_timedrdlock(os_rwlock_t *__restrict rwlock, 135 | const struct timespec *abstime); 136 | int os_rwlock_timedwrlock(os_rwlock_t *__restrict rwlock, 137 | const struct timespec *abstime); 138 | 139 | int os_spin_init(os_spinlock_t *lock, int pshared); 140 | int os_spin_destroy(os_spinlock_t *lock); 141 | int os_spin_lock(os_spinlock_t *lock); 142 | int os_spin_unlock(os_spinlock_t *lock); 143 | int os_spin_trylock(os_spinlock_t *lock); 144 | 145 | int os_cond_init(os_cond_t *__restrict cond); 146 | int os_cond_destroy(os_cond_t *__restrict cond); 147 | int os_cond_broadcast(os_cond_t *__restrict cond); 148 | int os_cond_signal(os_cond_t *__restrict cond); 149 | int os_cond_timedwait(os_cond_t *__restrict cond, 150 | os_mutex_t *__restrict mutex, const struct timespec *abstime); 151 | int os_cond_wait(os_cond_t *__restrict cond, 152 | os_mutex_t *__restrict mutex); 153 | 154 | /* threading */ 155 | 156 | int os_thread_create(os_thread_t *thread, const os_thread_attr_t *attr, 157 | void *(*start_routine)(void *), void *arg); 158 | 159 | int os_thread_join(os_thread_t *thread, void **result); 160 | 161 | void os_thread_self(os_thread_t *thread); 162 | 163 | /* thread affinity */ 164 | 165 | int os_thread_setaffinity_np(os_thread_t *thread, size_t set_size, 166 | const os_cpu_set_t *set); 167 | 168 | int os_thread_atfork(void (*prepare)(void), void (*parent)(void), 169 | void (*child)(void)); 170 | 171 | 172 | int os_semaphore_init(os_semaphore_t *sem, unsigned value); 173 | int os_semaphore_destroy(os_semaphore_t *sem); 174 | int os_semaphore_wait(os_semaphore_t *sem); 175 | int os_semaphore_trywait(os_semaphore_t *sem); 176 | int os_semaphore_post(os_semaphore_t *sem); 177 | 178 | #ifdef __cplusplus 179 | } 180 | #endif 181 | #endif /* OS_THREAD_H */ 182 | -------------------------------------------------------------------------------- /src/out.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2014-2018, Intel Corporation */ 3 | 4 | /* 5 | * out.h -- definitions for "out" module 6 | */ 7 | 8 | #ifndef PMDK_OUT_H 9 | #define PMDK_OUT_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | #include "util.h" 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | /* 22 | * Suppress errors which are after appropriate ASSERT* macro for nondebug 23 | * builds. 24 | */ 25 | #if !defined(DEBUG) && (defined(__clang_analyzer__) || defined(__COVERITY__) ||\ 26 | defined(__KLOCWORK__)) 27 | #define OUT_FATAL_DISCARD_NORETURN __attribute__((noreturn)) 28 | #else 29 | #define OUT_FATAL_DISCARD_NORETURN 30 | #endif 31 | 32 | #ifndef EVALUATE_DBG_EXPRESSIONS 33 | #if defined(DEBUG) || defined(__clang_analyzer__) || defined(__COVERITY__) ||\ 34 | defined(__KLOCWORK__) 35 | #define EVALUATE_DBG_EXPRESSIONS 1 36 | #else 37 | #define EVALUATE_DBG_EXPRESSIONS 0 38 | #endif 39 | #endif 40 | 41 | #ifdef DEBUG 42 | 43 | #define OUT_LOG out_log 44 | #define OUT_NONL out_nonl 45 | #define OUT_FATAL out_fatal 46 | #define OUT_FATAL_ABORT out_fatal 47 | 48 | #else 49 | 50 | static __attribute__((always_inline)) inline void 51 | out_log_discard(const char *file, int line, const char *func, int level, 52 | const char *fmt, ...) 53 | { 54 | (void) file; 55 | (void) line; 56 | (void) func; 57 | (void) level; 58 | (void) fmt; 59 | } 60 | 61 | static __attribute__((always_inline)) inline void 62 | out_nonl_discard(int level, const char *fmt, ...) 63 | { 64 | (void) level; 65 | (void) fmt; 66 | } 67 | 68 | static __attribute__((always_inline)) OUT_FATAL_DISCARD_NORETURN inline void 69 | out_fatal_discard(const char *file, int line, const char *func, 70 | const char *fmt, ...) 71 | { 72 | (void) file; 73 | (void) line; 74 | (void) func; 75 | (void) fmt; 76 | } 77 | 78 | static __attribute__((always_inline)) NORETURN inline void 79 | out_fatal_abort(const char *file, int line, const char *func, 80 | const char *fmt, ...) 81 | { 82 | (void) file; 83 | (void) line; 84 | (void) func; 85 | (void) fmt; 86 | 87 | abort(); 88 | } 89 | 90 | #define OUT_LOG out_log_discard 91 | #define OUT_NONL out_nonl_discard 92 | #define OUT_FATAL out_fatal_discard 93 | #define OUT_FATAL_ABORT out_fatal_abort 94 | 95 | #endif 96 | 97 | #if defined(__KLOCWORK__) 98 | #define TEST_ALWAYS_TRUE_EXPR(cnd) 99 | #define TEST_ALWAYS_EQ_EXPR(cnd) 100 | #define TEST_ALWAYS_NE_EXPR(cnd) 101 | #else 102 | #define TEST_ALWAYS_TRUE_EXPR(cnd)\ 103 | if (__builtin_constant_p(cnd))\ 104 | ASSERT_COMPILE_ERROR_ON(cnd); 105 | #define TEST_ALWAYS_EQ_EXPR(lhs, rhs)\ 106 | if (__builtin_constant_p(lhs) && __builtin_constant_p(rhs))\ 107 | ASSERT_COMPILE_ERROR_ON((lhs) == (rhs)); 108 | #define TEST_ALWAYS_NE_EXPR(lhs, rhs)\ 109 | if (__builtin_constant_p(lhs) && __builtin_constant_p(rhs))\ 110 | ASSERT_COMPILE_ERROR_ON((lhs) != (rhs)); 111 | #endif 112 | 113 | /* produce debug/trace output */ 114 | #define LOG(level, ...) do { \ 115 | if (!EVALUATE_DBG_EXPRESSIONS) break;\ 116 | OUT_LOG(__FILE__, __LINE__, __func__, level, __VA_ARGS__);\ 117 | } while (0) 118 | 119 | /* produce debug/trace output without prefix and new line */ 120 | #define LOG_NONL(level, ...) do { \ 121 | if (!EVALUATE_DBG_EXPRESSIONS) break; \ 122 | OUT_NONL(level, __VA_ARGS__); \ 123 | } while (0) 124 | 125 | /* produce output and exit */ 126 | #define FATAL(...)\ 127 | OUT_FATAL_ABORT(__FILE__, __LINE__, __func__, __VA_ARGS__) 128 | 129 | /* assert a condition is true at runtime */ 130 | #define ASSERT_rt(cnd) do { \ 131 | if (!EVALUATE_DBG_EXPRESSIONS || (cnd)) break; \ 132 | OUT_FATAL(__FILE__, __LINE__, __func__, "assertion failure: %s", #cnd);\ 133 | } while (0) 134 | 135 | /* assertion with extra info printed if assertion fails at runtime */ 136 | #define ASSERTinfo_rt(cnd, info) do { \ 137 | if (!EVALUATE_DBG_EXPRESSIONS || (cnd)) break; \ 138 | OUT_FATAL(__FILE__, __LINE__, __func__, \ 139 | "assertion failure: %s (%s = %s)", #cnd, #info, info);\ 140 | } while (0) 141 | 142 | /* assert two integer values are equal at runtime */ 143 | #define ASSERTeq_rt(lhs, rhs) do { \ 144 | if (!EVALUATE_DBG_EXPRESSIONS || ((lhs) == (rhs))) break; \ 145 | OUT_FATAL(__FILE__, __LINE__, __func__,\ 146 | "assertion failure: %s (0x%llx) == %s (0x%llx)", #lhs,\ 147 | (unsigned long long)(lhs), #rhs, (unsigned long long)(rhs)); \ 148 | } while (0) 149 | 150 | /* assert two integer values are not equal at runtime */ 151 | #define ASSERTne_rt(lhs, rhs) do { \ 152 | if (!EVALUATE_DBG_EXPRESSIONS || ((lhs) != (rhs))) break; \ 153 | OUT_FATAL(__FILE__, __LINE__, __func__,\ 154 | "assertion failure: %s (0x%llx) != %s (0x%llx)", #lhs,\ 155 | (unsigned long long)(lhs), #rhs, (unsigned long long)(rhs)); \ 156 | } while (0) 157 | 158 | /* assert a condition is true */ 159 | #define ASSERT(cnd)\ 160 | do {\ 161 | /*\ 162 | * Detect useless asserts on always true expression. Please use\ 163 | * COMPILE_ERROR_ON(!cnd) or ASSERT_rt(cnd) in such cases.\ 164 | */\ 165 | TEST_ALWAYS_TRUE_EXPR(cnd);\ 166 | ASSERT_rt(cnd);\ 167 | } while (0) 168 | 169 | /* assertion with extra info printed if assertion fails */ 170 | #define ASSERTinfo(cnd, info)\ 171 | do {\ 172 | /* See comment in ASSERT. */\ 173 | TEST_ALWAYS_TRUE_EXPR(cnd);\ 174 | ASSERTinfo_rt(cnd, info);\ 175 | } while (0) 176 | 177 | /* assert two integer values are equal */ 178 | #define ASSERTeq(lhs, rhs)\ 179 | do {\ 180 | /* See comment in ASSERT. */\ 181 | TEST_ALWAYS_EQ_EXPR(lhs, rhs);\ 182 | ASSERTeq_rt(lhs, rhs);\ 183 | } while (0) 184 | 185 | /* assert two integer values are not equal */ 186 | #define ASSERTne(lhs, rhs)\ 187 | do {\ 188 | /* See comment in ASSERT. */\ 189 | TEST_ALWAYS_NE_EXPR(lhs, rhs);\ 190 | ASSERTne_rt(lhs, rhs);\ 191 | } while (0) 192 | 193 | #define ERR(...)\ 194 | out_err(__FILE__, __LINE__, __func__, __VA_ARGS__) 195 | 196 | void out_init(const char *log_prefix, const char *log_level_var, 197 | const char *log_file_var, int major_version, 198 | int minor_version); 199 | void out_fini(void); 200 | void out(const char *fmt, ...) FORMAT_PRINTF(1, 2); 201 | void out_nonl(int level, const char *fmt, ...) FORMAT_PRINTF(2, 3); 202 | void out_log(const char *file, int line, const char *func, int level, 203 | const char *fmt, ...) FORMAT_PRINTF(5, 6); 204 | void out_err(const char *file, int line, const char *func, 205 | const char *fmt, ...) FORMAT_PRINTF(4, 5); 206 | void NORETURN out_fatal(const char *file, int line, const char *func, 207 | const char *fmt, ...) FORMAT_PRINTF(4, 5); 208 | void out_set_print_func(void (*print_func)(const char *s)); 209 | void out_set_vsnprintf_func(int (*vsnprintf_func)(char *str, size_t size, 210 | const char *format, va_list ap)); 211 | 212 | #ifdef _WIN32 213 | #ifndef PMDK_UTF8_API 214 | #define out_get_errormsg out_get_errormsgW 215 | #else 216 | #define out_get_errormsg out_get_errormsgU 217 | #endif 218 | #endif 219 | 220 | #ifndef _WIN32 221 | const char *out_get_errormsg(void); 222 | #else 223 | const char *out_get_errormsgU(void); 224 | const wchar_t *out_get_errormsgW(void); 225 | #endif 226 | 227 | #ifdef __cplusplus 228 | } 229 | #endif 230 | 231 | #endif 232 | -------------------------------------------------------------------------------- /src/ringbuf.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2017-2019, Intel Corporation */ 3 | 4 | /* 5 | * ringbuf.c -- implementation of a simple multi-producer/multi-consumer (MPMC) 6 | * ring buffer. It uses atomic instructions for correctness and semaphores for 7 | * waiting. 8 | */ 9 | 10 | #include "valgrind_internal.h" 11 | 12 | #include "ringbuf.h" 13 | #include "util.h" 14 | #include "out.h" 15 | #include "os.h" 16 | #include "os_thread.h" 17 | #include "sys_util.h" 18 | 19 | #if 0 20 | /* 21 | * This number defines by how much the relevant semaphore will be increased to 22 | * unlock waiting threads and thus defines how many threads can wait on the 23 | * ring buffer at the same time. 24 | */ 25 | #define RINGBUF_MAX_CONSUMER_THREADS 1024 26 | #endif 27 | 28 | /* avoid false sharing by padding the variable */ 29 | #define CACHELINE_PADDING(type, name)\ 30 | union { type name; uint64_t name##_padding[8]; } 31 | 32 | struct ringbuf { 33 | CACHELINE_PADDING(uint64_t, read_pos); 34 | CACHELINE_PADDING(uint64_t, write_pos); 35 | 36 | CACHELINE_PADDING(os_semaphore_t, nfree); 37 | CACHELINE_PADDING(os_semaphore_t, nused); 38 | 39 | unsigned len; 40 | uint64_t len_mask; 41 | int running; 42 | 43 | void *data[]; 44 | }; 45 | 46 | /* 47 | * ringbuf_new -- creates a new ring buffer instance 48 | */ 49 | struct ringbuf * 50 | ringbuf_new(unsigned length) 51 | { 52 | LOG(4, NULL); 53 | 54 | /* length must be a power of two due to masking */ 55 | if (util_popcount(length) > 1) 56 | return NULL; 57 | 58 | struct ringbuf *rbuf = 59 | Zalloc(sizeof(*rbuf) + (length * sizeof(void *))); 60 | if (rbuf == NULL) 61 | return NULL; 62 | 63 | if (os_semaphore_init(&rbuf->nfree, length)) { 64 | Free(rbuf); 65 | return NULL; 66 | } 67 | 68 | if (os_semaphore_init(&rbuf->nused, 0)) { 69 | util_semaphore_destroy(&rbuf->nfree); 70 | Free(rbuf); 71 | return NULL; 72 | } 73 | 74 | rbuf->read_pos = 0; 75 | rbuf->write_pos = 0; 76 | 77 | rbuf->len = length; 78 | rbuf->len_mask = length - 1; 79 | rbuf->running = 1; 80 | 81 | return rbuf; 82 | } 83 | 84 | #if 0 85 | /* 86 | * ringbuf_length -- returns the length of the ring buffer 87 | */ 88 | unsigned 89 | ringbuf_length(struct ringbuf *rbuf) 90 | { 91 | LOG(4, NULL); 92 | 93 | return rbuf->len; 94 | } 95 | 96 | /* 97 | * ringbuf_stop -- if there are any threads stuck waiting on dequeue, unblocks 98 | * them. Those threads, if there are no new elements, will return NULL. 99 | */ 100 | void 101 | ringbuf_stop(struct ringbuf *rbuf) 102 | { 103 | LOG(4, NULL); 104 | 105 | /* wait for the buffer to become empty */ 106 | while (rbuf->read_pos != rbuf->write_pos) 107 | __sync_synchronize(); 108 | 109 | int ret = util_bool_compare_and_swap64(&rbuf->running, 1, 0); 110 | ASSERTeq(ret, 1); 111 | 112 | /* XXX just unlock all waiting threads somehow... */ 113 | for (int64_t i = 0; i < RINGBUF_MAX_CONSUMER_THREADS; ++i) 114 | util_semaphore_post(&rbuf->nused); 115 | } 116 | #endif 117 | 118 | /* 119 | * ringbuf_delete -- destroys an existing ring buffer instance 120 | */ 121 | void 122 | ringbuf_delete(struct ringbuf *rbuf) 123 | { 124 | LOG(4, NULL); 125 | 126 | ASSERTeq(rbuf->read_pos, rbuf->write_pos); 127 | util_semaphore_destroy(&rbuf->nfree); 128 | util_semaphore_destroy(&rbuf->nused); 129 | Free(rbuf); 130 | } 131 | 132 | /* 133 | * ringbuf_enqueue_atomic -- (internal) performs the lockfree insert of an 134 | * element into the ringbuf data array 135 | */ 136 | static void 137 | ringbuf_enqueue_atomic(struct ringbuf *rbuf, void *data) 138 | { 139 | LOG(4, NULL); 140 | 141 | size_t w = util_fetch_and_add64(&rbuf->write_pos, 1) & rbuf->len_mask; 142 | 143 | ASSERT(rbuf->running); 144 | 145 | /* 146 | * In most cases, this won't loop even once, but sometimes if the 147 | * semaphore is incremented concurrently in dequeue, we need to wait. 148 | */ 149 | while (!util_bool_compare_and_swap64(&rbuf->data[w], NULL, data)) 150 | ; 151 | 152 | VALGRIND_ANNOTATE_HAPPENS_BEFORE(&rbuf->data[w]); 153 | } 154 | 155 | #if 0 156 | /* 157 | * ringbuf_enqueue -- places a new value into the collection 158 | * 159 | * This function blocks if there's no space in the buffer. 160 | */ 161 | int 162 | ringbuf_enqueue(struct ringbuf *rbuf, void *data) 163 | { 164 | LOG(4, NULL); 165 | 166 | util_semaphore_wait(&rbuf->nfree); 167 | 168 | ringbuf_enqueue_atomic(rbuf, data); 169 | 170 | util_semaphore_post(&rbuf->nused); 171 | 172 | return 0; 173 | } 174 | #endif 175 | 176 | /* 177 | * ringbuf_tryenqueue -- places a new value into the collection 178 | * 179 | * This function fails if there's no space in the buffer. 180 | */ 181 | int 182 | ringbuf_tryenqueue(struct ringbuf *rbuf, void *data) 183 | { 184 | LOG(4, NULL); 185 | 186 | if (util_semaphore_trywait(&rbuf->nfree) != 0) 187 | return -1; 188 | 189 | ringbuf_enqueue_atomic(rbuf, data); 190 | 191 | util_semaphore_post(&rbuf->nused); 192 | 193 | return 0; 194 | } 195 | 196 | /* 197 | * ringbuf_dequeue_atomic -- performs a lockfree retrieval of data from ringbuf 198 | */ 199 | static void * 200 | ringbuf_dequeue_atomic(struct ringbuf *rbuf) 201 | { 202 | LOG(4, NULL); 203 | 204 | size_t r = util_fetch_and_add64(&rbuf->read_pos, 1) & rbuf->len_mask; 205 | /* 206 | * Again, in most cases, there won't be even a single loop, but if one 207 | * thread stalls while others perform work, it might happen that two 208 | * threads get the same read position. 209 | */ 210 | void *data = NULL; 211 | 212 | VALGRIND_ANNOTATE_HAPPENS_AFTER(&rbuf->data[r]); 213 | do { 214 | while ((data = rbuf->data[r]) == NULL) 215 | __sync_synchronize(); 216 | } while (!util_bool_compare_and_swap64(&rbuf->data[r], data, NULL)); 217 | 218 | return data; 219 | } 220 | 221 | #if 0 222 | /* 223 | * ringbuf_dequeue -- retrieves one value from the collection 224 | * 225 | * This function blocks if there are no values in the buffer. 226 | */ 227 | void * 228 | ringbuf_dequeue(struct ringbuf *rbuf) 229 | { 230 | LOG(4, NULL); 231 | 232 | util_semaphore_wait(&rbuf->nused); 233 | 234 | if (!rbuf->running) 235 | return NULL; 236 | 237 | void *data = ringbuf_dequeue_atomic(rbuf); 238 | 239 | util_semaphore_post(&rbuf->nfree); 240 | 241 | return data; 242 | } 243 | #endif 244 | 245 | /* 246 | * ringbuf_trydequeue -- retrieves one value from the collection 247 | * 248 | * This function fails if there are no values in the buffer. 249 | */ 250 | void * 251 | ringbuf_trydequeue(struct ringbuf *rbuf) 252 | { 253 | LOG(4, NULL); 254 | 255 | if (util_semaphore_trywait(&rbuf->nused) != 0) 256 | return NULL; 257 | 258 | if (!rbuf->running) 259 | return NULL; 260 | 261 | void *data = ringbuf_dequeue_atomic(rbuf); 262 | 263 | util_semaphore_post(&rbuf->nfree); 264 | 265 | return data; 266 | } 267 | 268 | /* 269 | * ringbuf_trydequeue_s -- valgrind-safe variant of the trydequeue function 270 | * 271 | * This function is needed for runtime race detection as a way to avoid false 272 | * positives due to usage of atomic instructions that might otherwise confuse 273 | * valgrind. 274 | */ 275 | void * 276 | ringbuf_trydequeue_s(struct ringbuf *rbuf, size_t data_size) 277 | { 278 | LOG(4, NULL); 279 | 280 | void *r = ringbuf_trydequeue(rbuf); 281 | if (r != NULL) 282 | VALGRIND_ANNOTATE_NEW_MEMORY(r, data_size); 283 | 284 | return r; 285 | } 286 | 287 | #if 0 288 | /* 289 | * ringbuf_dequeue_s -- valgrind-safe variant of the dequeue function 290 | * 291 | * This function is needed for runtime race detection as a way to avoid false 292 | * positives due to usage of atomic instructions that might otherwise confuse 293 | * valgrind. 294 | */ 295 | void * 296 | ringbuf_dequeue_s(struct ringbuf *rbuf, size_t data_size) 297 | { 298 | LOG(4, NULL); 299 | 300 | void *r = ringbuf_dequeue(rbuf); 301 | VALGRIND_ANNOTATE_NEW_MEMORY(r, data_size); 302 | 303 | return r; 304 | } 305 | #endif 306 | -------------------------------------------------------------------------------- /src/ringbuf.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2017-2019, Intel Corporation */ 3 | 4 | /* 5 | * ringbuf.h -- internal definitions for mpmc ring buffer 6 | */ 7 | 8 | #ifndef RINGBUF_H 9 | #define RINGBUF_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | struct ringbuf; 16 | 17 | struct ringbuf *ringbuf_new(unsigned length); 18 | void ringbuf_delete(struct ringbuf *rbuf); 19 | unsigned ringbuf_length(struct ringbuf *rbuf); 20 | void ringbuf_stop(struct ringbuf *rbuf); 21 | 22 | int ringbuf_enqueue(struct ringbuf *rbuf, void *data); 23 | int ringbuf_tryenqueue(struct ringbuf *rbuf, void *data); 24 | void *ringbuf_dequeue(struct ringbuf *rbuf); 25 | void *ringbuf_trydequeue(struct ringbuf *rbuf); 26 | void *ringbuf_dequeue_s(struct ringbuf *rbuf, size_t data_size); 27 | void *ringbuf_trydequeue_s(struct ringbuf *rbuf, size_t data_size); 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/sys_util.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2016-2018, Intel Corporation */ 3 | 4 | /* 5 | * sys_util.h -- internal utility wrappers around system functions 6 | */ 7 | 8 | #ifndef PMDK_SYS_UTIL_H 9 | #define PMDK_SYS_UTIL_H 1 10 | 11 | #include 12 | 13 | #include "os_thread.h" 14 | #include "out.h" 15 | 16 | #ifdef __cplusplus 17 | extern "C" { 18 | #endif 19 | 20 | /* 21 | * util_mutex_init -- os_mutex_init variant that never fails from 22 | * caller perspective. If os_mutex_init failed, this function aborts 23 | * the program. 24 | */ 25 | static inline void 26 | util_mutex_init(os_mutex_t *m) 27 | { 28 | int tmp = os_mutex_init(m); 29 | if (tmp) { 30 | errno = tmp; 31 | FATAL("!os_mutex_init"); 32 | } 33 | } 34 | 35 | /* 36 | * util_mutex_destroy -- os_mutex_destroy variant that never fails from 37 | * caller perspective. If os_mutex_destroy failed, this function aborts 38 | * the program. 39 | */ 40 | static inline void 41 | util_mutex_destroy(os_mutex_t *m) 42 | { 43 | int tmp = os_mutex_destroy(m); 44 | if (tmp) { 45 | errno = tmp; 46 | FATAL("!os_mutex_destroy"); 47 | } 48 | } 49 | 50 | /* 51 | * util_mutex_lock -- os_mutex_lock variant that never fails from 52 | * caller perspective. If os_mutex_lock failed, this function aborts 53 | * the program. 54 | */ 55 | static inline void 56 | util_mutex_lock(os_mutex_t *m) 57 | { 58 | int tmp = os_mutex_lock(m); 59 | if (tmp) { 60 | errno = tmp; 61 | FATAL("!os_mutex_lock"); 62 | } 63 | } 64 | 65 | /* 66 | * util_mutex_trylock -- os_mutex_trylock variant that never fails from 67 | * caller perspective (other than EBUSY). If util_mutex_trylock failed, this 68 | * function aborts the program. 69 | * Returns 0 if locked successfully, otherwise returns EBUSY. 70 | */ 71 | static inline int 72 | util_mutex_trylock(os_mutex_t *m) 73 | { 74 | int tmp = os_mutex_trylock(m); 75 | if (tmp && tmp != EBUSY) { 76 | errno = tmp; 77 | FATAL("!os_mutex_trylock"); 78 | } 79 | return tmp; 80 | } 81 | 82 | /* 83 | * util_mutex_unlock -- os_mutex_unlock variant that never fails from 84 | * caller perspective. If os_mutex_unlock failed, this function aborts 85 | * the program. 86 | */ 87 | static inline void 88 | util_mutex_unlock(os_mutex_t *m) 89 | { 90 | int tmp = os_mutex_unlock(m); 91 | if (tmp) { 92 | errno = tmp; 93 | FATAL("!os_mutex_unlock"); 94 | } 95 | } 96 | 97 | /* 98 | * util_rwlock_init -- os_rwlock_init variant that never fails from 99 | * caller perspective. If os_rwlock_init failed, this function aborts 100 | * the program. 101 | */ 102 | static inline void 103 | util_rwlock_init(os_rwlock_t *m) 104 | { 105 | int tmp = os_rwlock_init(m); 106 | if (tmp) { 107 | errno = tmp; 108 | FATAL("!os_rwlock_init"); 109 | } 110 | } 111 | 112 | /* 113 | * util_rwlock_rdlock -- os_rwlock_rdlock variant that never fails from 114 | * caller perspective. If os_rwlock_rdlock failed, this function aborts 115 | * the program. 116 | */ 117 | static inline void 118 | util_rwlock_rdlock(os_rwlock_t *m) 119 | { 120 | int tmp = os_rwlock_rdlock(m); 121 | if (tmp) { 122 | errno = tmp; 123 | FATAL("!os_rwlock_rdlock"); 124 | } 125 | } 126 | 127 | /* 128 | * util_rwlock_wrlock -- os_rwlock_wrlock variant that never fails from 129 | * caller perspective. If os_rwlock_wrlock failed, this function aborts 130 | * the program. 131 | */ 132 | static inline void 133 | util_rwlock_wrlock(os_rwlock_t *m) 134 | { 135 | int tmp = os_rwlock_wrlock(m); 136 | if (tmp) { 137 | errno = tmp; 138 | FATAL("!os_rwlock_wrlock"); 139 | } 140 | } 141 | 142 | /* 143 | * util_rwlock_unlock -- os_rwlock_unlock variant that never fails from 144 | * caller perspective. If os_rwlock_unlock failed, this function aborts 145 | * the program. 146 | */ 147 | static inline void 148 | util_rwlock_unlock(os_rwlock_t *m) 149 | { 150 | int tmp = os_rwlock_unlock(m); 151 | if (tmp) { 152 | errno = tmp; 153 | FATAL("!os_rwlock_unlock"); 154 | } 155 | } 156 | 157 | /* 158 | * util_rwlock_destroy -- os_rwlock_destroy variant that never fails from 159 | * caller perspective. If os_rwlock_destroy failed, this function aborts 160 | * the program. 161 | */ 162 | static inline void 163 | util_rwlock_destroy(os_rwlock_t *m) 164 | { 165 | int tmp = os_rwlock_destroy(m); 166 | if (tmp) { 167 | errno = tmp; 168 | FATAL("!os_rwlock_destroy"); 169 | } 170 | } 171 | 172 | /* 173 | * util_spin_init -- os_spin_init variant that logs on fail and sets errno. 174 | */ 175 | static inline int 176 | util_spin_init(os_spinlock_t *lock, int pshared) 177 | { 178 | int tmp = os_spin_init(lock, pshared); 179 | if (tmp) { 180 | errno = tmp; 181 | ERR("!os_spin_init"); 182 | } 183 | return tmp; 184 | } 185 | 186 | /* 187 | * util_spin_destroy -- os_spin_destroy variant that never fails from 188 | * caller perspective. If os_spin_destroy failed, this function aborts 189 | * the program. 190 | */ 191 | static inline void 192 | util_spin_destroy(os_spinlock_t *lock) 193 | { 194 | int tmp = os_spin_destroy(lock); 195 | if (tmp) { 196 | errno = tmp; 197 | FATAL("!os_spin_destroy"); 198 | } 199 | } 200 | 201 | /* 202 | * util_spin_lock -- os_spin_lock variant that never fails from caller 203 | * perspective. If os_spin_lock failed, this function aborts the program. 204 | */ 205 | static inline void 206 | util_spin_lock(os_spinlock_t *lock) 207 | { 208 | int tmp = os_spin_lock(lock); 209 | if (tmp) { 210 | errno = tmp; 211 | FATAL("!os_spin_lock"); 212 | } 213 | } 214 | 215 | /* 216 | * util_spin_unlock -- os_spin_unlock variant that never fails 217 | * from caller perspective. If os_spin_unlock failed, 218 | * this function aborts the program. 219 | */ 220 | static inline void 221 | util_spin_unlock(os_spinlock_t *lock) 222 | { 223 | int tmp = os_spin_unlock(lock); 224 | if (tmp) { 225 | errno = tmp; 226 | FATAL("!os_spin_unlock"); 227 | } 228 | } 229 | 230 | /* 231 | * util_semaphore_init -- os_semaphore_init variant that never fails 232 | * from caller perspective. If os_semaphore_init failed, 233 | * this function aborts the program. 234 | */ 235 | static inline void 236 | util_semaphore_init(os_semaphore_t *sem, unsigned value) 237 | { 238 | if (os_semaphore_init(sem, value)) 239 | FATAL("!os_semaphore_init"); 240 | } 241 | 242 | /* 243 | * util_semaphore_destroy -- deletes a semaphore instance 244 | */ 245 | static inline void 246 | util_semaphore_destroy(os_semaphore_t *sem) 247 | { 248 | if (os_semaphore_destroy(sem) != 0) 249 | FATAL("!os_semaphore_destroy"); 250 | } 251 | 252 | /* 253 | * util_semaphore_wait -- decreases the value of the semaphore 254 | */ 255 | static inline void 256 | util_semaphore_wait(os_semaphore_t *sem) 257 | { 258 | errno = 0; 259 | 260 | int ret; 261 | do { 262 | ret = os_semaphore_wait(sem); 263 | } while (errno == EINTR); /* signal interrupt */ 264 | 265 | if (ret != 0) 266 | FATAL("!os_semaphore_wait"); 267 | } 268 | 269 | /* 270 | * util_semaphore_trywait -- tries to decrease the value of the semaphore 271 | */ 272 | static inline int 273 | util_semaphore_trywait(os_semaphore_t *sem) 274 | { 275 | errno = 0; 276 | int ret; 277 | do { 278 | ret = os_semaphore_trywait(sem); 279 | } while (errno == EINTR); /* signal interrupt */ 280 | 281 | if (ret != 0 && errno != EAGAIN) 282 | FATAL("!os_semaphore_trywait"); 283 | 284 | return ret; 285 | } 286 | 287 | /* 288 | * util_semaphore_post -- increases the value of the semaphore 289 | */ 290 | static inline void 291 | util_semaphore_post(os_semaphore_t *sem) 292 | { 293 | if (os_semaphore_post(sem) != 0) 294 | FATAL("!os_semaphore_post"); 295 | } 296 | 297 | #ifdef __cplusplus 298 | } 299 | #endif 300 | 301 | #endif 302 | -------------------------------------------------------------------------------- /src/util_posix.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2015-2019, Intel Corporation */ 3 | 4 | /* 5 | * util_posix.c -- Abstraction layer for misc utilities (Posix implementation) 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include "util.h" 14 | #include "os.h" 15 | #include "out.h" 16 | 17 | /* pass through for Posix */ 18 | void 19 | util_strerror(int errnum, char *buff, size_t bufflen) 20 | { 21 | strerror_r(errnum, buff, bufflen); 22 | } 23 | 24 | #if 0 25 | /* 26 | * util_part_realpath -- get canonicalized absolute pathname 27 | * 28 | * As paths used in a poolset file have to be absolute (checked when parsing 29 | * a poolset file), here we only have to resolve symlinks. 30 | */ 31 | char * 32 | util_part_realpath(const char *path) 33 | { 34 | return realpath(path, NULL); 35 | } 36 | 37 | /* 38 | * util_compare_file_inodes -- compare device and inodes of two files; 39 | * this resolves hard links 40 | */ 41 | int 42 | util_compare_file_inodes(const char *path1, const char *path2) 43 | { 44 | struct stat sb1, sb2; 45 | if (os_stat(path1, &sb1)) { 46 | if (errno != ENOENT) { 47 | ERR("!stat failed for %s", path1); 48 | return -1; 49 | } 50 | LOG(1, "stat failed for %s", path1); 51 | errno = 0; 52 | return strcmp(path1, path2) != 0; 53 | } 54 | 55 | if (os_stat(path2, &sb2)) { 56 | if (errno != ENOENT) { 57 | ERR("!stat failed for %s", path2); 58 | return -1; 59 | } 60 | LOG(1, "stat failed for %s", path2); 61 | errno = 0; 62 | return strcmp(path1, path2) != 0; 63 | } 64 | 65 | return sb1.st_dev != sb2.st_dev || sb1.st_ino != sb2.st_ino; 66 | } 67 | 68 | /* 69 | * util_aligned_malloc -- allocate aligned memory 70 | */ 71 | void * 72 | util_aligned_malloc(size_t alignment, size_t size) 73 | { 74 | void *retval = NULL; 75 | 76 | errno = posix_memalign(&retval, alignment, size); 77 | 78 | return retval; 79 | } 80 | 81 | /* 82 | * util_aligned_free -- free allocated memory in util_aligned_malloc 83 | */ 84 | void 85 | util_aligned_free(void *ptr) 86 | { 87 | free(ptr); 88 | } 89 | #endif 90 | 91 | /* 92 | * util_getexecname -- return name of current executable 93 | */ 94 | char * 95 | util_getexecname(char *path, size_t pathlen) 96 | { 97 | ASSERT(pathlen != 0); 98 | ssize_t cc; 99 | 100 | #ifdef __FreeBSD__ 101 | #include 102 | #include 103 | 104 | int mib[4] = {CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1}; 105 | 106 | cc = (sysctl(mib, 4, path, &pathlen, NULL, 0) == -1) ? 107 | -1 : (ssize_t)pathlen; 108 | #else 109 | cc = readlink("/proc/self/exe", path, pathlen); 110 | #endif 111 | if (cc == -1) { 112 | strncpy(path, "unknown", pathlen); 113 | path[pathlen - 1] = '\0'; 114 | } else { 115 | path[cc] = '\0'; 116 | } 117 | 118 | return path; 119 | } 120 | -------------------------------------------------------------------------------- /src/vmemcache.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache.h -- internal definitions for vmemcache 6 | */ 7 | 8 | #ifndef VMEMCACHE_H 9 | #define VMEMCACHE_H 1 10 | 11 | #include 12 | #include 13 | 14 | #include "libvmemcache.h" 15 | #include "vmemcache_heap.h" 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #define VMEMCACHE_PREFIX "libvmemcache" 22 | #define VMEMCACHE_LEVEL_VAR "VMEMCACHE_LEVEL" 23 | #define VMEMCACHE_FILE_VAR "VMEMCACHE_FILE" 24 | 25 | struct index; 26 | struct repl_p; 27 | 28 | struct vmemcache { 29 | void *addr; /* mapping address */ 30 | size_t size; /* mapping size */ 31 | size_t extent_size; /* heap granularity */ 32 | struct heap *heap; /* heap address */ 33 | struct index *index; /* indexing structure */ 34 | enum vmemcache_repl_p repl_p; /* replacement policy */ 35 | struct repl_p *repl; /* replacement policy abstraction */ 36 | vmemcache_on_evict *on_evict; /* callback on evict */ 37 | void *arg_evict; /* argument for callback on evict */ 38 | vmemcache_on_miss *on_miss; /* callback on miss */ 39 | void *arg_miss; /* argument for callback on miss */ 40 | unsigned ready:1; /* is the cache ready for use? */ 41 | unsigned index_only:1; /* bench: disable repl+alloc */ 42 | unsigned no_alloc:1; /* bench: disable allocations */ 43 | unsigned no_memcpy:1; /* bench: don't copy actual data */ 44 | }; 45 | 46 | struct cache_entry { 47 | struct value { 48 | uint32_t refcount; 49 | int evicting; 50 | struct repl_p_entry *p_entry; 51 | size_t vsize; 52 | ptr_ext_t *extents; 53 | } value; 54 | 55 | struct key { 56 | size_t ksize; 57 | char key[]; 58 | } key; 59 | }; 60 | 61 | /* type of callback deleting a cache entry */ 62 | typedef void (*delete_entry_t)(struct cache_entry *entry); 63 | 64 | /* callback deleting a cache entry (of the above type 'delete_entry_t') */ 65 | void vmemcache_delete_entry_cb(struct cache_entry *entry); 66 | 67 | void vmemcache_entry_acquire(struct cache_entry *entry); 68 | void vmemcache_entry_release(VMEMcache *cache, struct cache_entry *entry); 69 | 70 | #ifdef __cplusplus 71 | } 72 | #endif 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /src/vmemcache_heap.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_heap.h -- internal definitions for vmemcache allocator module 6 | */ 7 | 8 | #ifndef VMEMCACHE_HEAP_H 9 | #define VMEMCACHE_HEAP_H 1 10 | 11 | #include 12 | #include 13 | 14 | /* type of the statistics */ 15 | typedef unsigned long long stat_t; 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | #define HEAP_ENTRY_IS_NULL(he) ((he.ptr) == NULL) 22 | 23 | /* just for type safety - see 'ptr' field in 'struct extent' below */ 24 | struct ptr_ext; 25 | typedef struct ptr_ext ptr_ext_t; 26 | 27 | /* extent structure ('struct heap_entry' without header and footer ) */ 28 | struct extent { 29 | ptr_ext_t *ptr; 30 | size_t size; 31 | }; 32 | 33 | struct heap; 34 | 35 | struct heap *vmcache_heap_create(void *addr, size_t size, size_t extent_size); 36 | void vmcache_heap_destroy(struct heap *heap); 37 | 38 | ssize_t vmcache_alloc(struct heap *heap, size_t size, 39 | ptr_ext_t **first_extent, 40 | ptr_ext_t **small_extent); 41 | 42 | void vmcache_free(struct heap *heap, ptr_ext_t *first_extent); 43 | 44 | stat_t vmcache_get_heap_used_size(struct heap *heap); 45 | stat_t vmcache_get_heap_entries_count(struct heap *heap); 46 | 47 | ptr_ext_t *vmcache_extent_get_next(ptr_ext_t *ptr); 48 | size_t vmcache_extent_get_size(ptr_ext_t *ptr); 49 | 50 | /* unsafe variant - the headers of extents cannot be modified */ 51 | #define EXTENTS_FOREACH(ext, extents) \ 52 | for ((ext).ptr = (extents), \ 53 | (ext).size = vmcache_extent_get_size((ext).ptr); \ 54 | (ext).ptr != NULL; \ 55 | (ext).ptr = vmcache_extent_get_next((ext).ptr), \ 56 | (ext).size = vmcache_extent_get_size((ext).ptr)) 57 | 58 | /* safe variant - the headers of extents can be modified (freed for example) */ 59 | #define EXTENTS_FOREACH_SAFE(ext, extents, __next) \ 60 | for ((ext).ptr = (extents), \ 61 | (ext).size = vmcache_extent_get_size((ext).ptr), \ 62 | (__next) = vmcache_extent_get_next((ext).ptr); \ 63 | (ext).ptr != NULL; \ 64 | (ext).ptr = (__next), \ 65 | (ext).size = vmcache_extent_get_size((ext).ptr), \ 66 | (__next) = vmcache_extent_get_next((__next))) 67 | 68 | #ifdef __cplusplus 69 | } 70 | #endif 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/vmemcache_index.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_index.c -- abstraction layer for vmemcache indexing API 6 | */ 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "vmemcache.h" 14 | #include "vmemcache_index.h" 15 | #include "critnib.h" 16 | #include "fast-hash.h" 17 | #include "sys_util.h" 18 | 19 | #ifdef STATS_ENABLED 20 | #define STAT_ADD(ptr, add) util_fetch_and_add64(ptr, add) 21 | #else 22 | #define STAT_ADD(ptr, add) do {} while (0) 23 | #endif 24 | 25 | /* must be a power of 2 */ 26 | #define NSHARDS 256 27 | 28 | struct index { 29 | struct critnib *bucket[NSHARDS]; 30 | int sharding; 31 | }; 32 | 33 | /* 34 | * shard_id -- (internal) hash the key and pick a shard bucket id 35 | */ 36 | static int 37 | shard_id(size_t key_size, const char *key) 38 | { 39 | return (int)hash(key_size, key) & (NSHARDS - 1); 40 | } 41 | 42 | /* 43 | * shard -- (internal) pick a shard bucket 44 | */ 45 | static struct critnib * 46 | shard(struct index *index, size_t key_size, const char *key) 47 | { 48 | if (index->sharding) 49 | return index->bucket[shard_id(key_size, key)]; 50 | 51 | return index->bucket[0]; 52 | } 53 | 54 | /* 55 | * vmcache_index_new -- initialize vmemcache indexing structure 56 | */ 57 | struct index * 58 | vmcache_index_new(void) 59 | { 60 | struct index *index = Malloc(sizeof(struct index)); 61 | if (!index) 62 | return NULL; 63 | 64 | index->sharding = env_yesno10("VMEMCACHE_SHARDING", 1); 65 | 66 | for (int i = 0; i < NSHARDS; i++) { 67 | struct critnib *c = critnib_new(); 68 | if (!c) { 69 | for (i--; i >= 0; i--) { 70 | util_rwlock_destroy(&index->bucket[i]->lock); 71 | critnib_delete(index->bucket[i], NULL); 72 | } 73 | Free(index); 74 | 75 | return NULL; 76 | } 77 | 78 | util_rwlock_init(&c->lock); 79 | index->bucket[i] = c; 80 | } 81 | 82 | return index; 83 | } 84 | 85 | /* 86 | * vmcache_index_delete -- destroy vmemcache indexing structure 87 | */ 88 | void 89 | vmcache_index_delete(struct index *index, delete_entry_t del_entry) 90 | { 91 | for (int i = 0; i < NSHARDS; i++) { 92 | util_rwlock_destroy(&index->bucket[i]->lock); 93 | critnib_delete(index->bucket[i], del_entry); 94 | } 95 | 96 | Free(index); 97 | } 98 | 99 | /* 100 | * vmcache_index_insert -- insert data into the vmemcache indexing structure 101 | */ 102 | int 103 | vmcache_index_insert(struct index *index, struct cache_entry *entry) 104 | { 105 | struct critnib *c = shard(index, entry->key.ksize, entry->key.key); 106 | 107 | util_rwlock_wrlock(&c->lock); 108 | 109 | int err = critnib_set(c, entry); 110 | if (err) { 111 | errno = err; 112 | util_rwlock_unlock(&c->lock); 113 | ERR("inserting to the index failed"); 114 | return -1; 115 | } 116 | 117 | #ifdef STATS_ENABLED 118 | c->leaf_count++; 119 | c->put_count++; 120 | c->DRAM_usage += malloc_usable_size(entry); 121 | #endif 122 | 123 | /* this is the first and the only one reference now (in the index) */ 124 | entry->value.refcount = 1; 125 | 126 | util_rwlock_unlock(&c->lock); 127 | 128 | return 0; 129 | } 130 | 131 | /* 132 | * vmcache_index_get -- get data from the vmemcache indexing structure 133 | */ 134 | int 135 | vmcache_index_get(struct index *index, const void *key, size_t ksize, 136 | struct cache_entry **entry, int bump_stat) 137 | { 138 | #define SIZE_1K 1024 139 | struct critnib *c = shard(index, ksize, key); 140 | 141 | struct cache_entry *e; 142 | 143 | *entry = NULL; 144 | 145 | if (ksize > SIZE_1K) { 146 | e = Malloc(sizeof(struct cache_entry) + ksize); 147 | if (e == NULL) { 148 | ERR("!Zalloc"); 149 | return -1; 150 | } 151 | } else { 152 | e = alloca(sizeof(struct cache_entry) + ksize); 153 | } 154 | 155 | e->key.ksize = ksize; 156 | memcpy(e->key.key, key, ksize); 157 | 158 | util_rwlock_rdlock(&c->lock); 159 | 160 | struct cache_entry *v = critnib_get(c, e); 161 | if (ksize > SIZE_1K) 162 | Free(e); 163 | if (v == NULL) { 164 | util_rwlock_unlock(&c->lock); 165 | 166 | if (bump_stat) 167 | STAT_ADD(&c->miss_count, 1); 168 | 169 | LOG(1, 170 | "vmcache_index_get: cannot find an element with the given key in the index"); 171 | return 0; 172 | } 173 | 174 | if (bump_stat) 175 | STAT_ADD(&c->hit_count, 1); 176 | 177 | vmemcache_entry_acquire(v); 178 | *entry = v; 179 | 180 | util_rwlock_unlock(&c->lock); 181 | 182 | return 0; 183 | } 184 | 185 | /* 186 | * vmcache_index_remove -- remove data from the vmemcache indexing structure 187 | */ 188 | int 189 | vmcache_index_remove(VMEMcache *cache, struct cache_entry *entry) 190 | { 191 | struct critnib *c = shard(cache->index, entry->key.ksize, 192 | entry->key.key); 193 | 194 | util_rwlock_wrlock(&c->lock); 195 | 196 | struct cache_entry *v = critnib_remove(c, entry); 197 | if (v == NULL) { 198 | util_rwlock_unlock(&c->lock); 199 | ERR( 200 | "vmcache_index_remove: cannot find an element with the given key in the index"); 201 | errno = EINVAL; 202 | return -1; 203 | } 204 | 205 | #ifdef STATS_ENABLED 206 | c->leaf_count--; 207 | c->evict_count++; 208 | c->DRAM_usage -= malloc_usable_size(entry); 209 | #endif 210 | 211 | vmemcache_entry_release(cache, entry); 212 | 213 | util_rwlock_unlock(&c->lock); 214 | 215 | return 0; 216 | } 217 | 218 | /* 219 | * vmemcache_index_get_stat -- query an index-held stat 220 | */ 221 | size_t 222 | vmemcache_index_get_stat(struct index *index, enum vmemcache_statistic stat) 223 | { 224 | size_t total = 0; 225 | 226 | switch (stat) { 227 | case VMEMCACHE_STAT_DRAM_SIZE_USED: 228 | { 229 | size_t nodes = 0; 230 | 231 | for (int i = 0; i < NSHARDS; i++) { 232 | nodes += index->bucket[i]->node_count; 233 | total += index->bucket[i]->DRAM_usage; 234 | } 235 | 236 | return total + nodes * sizeof(struct critnib_node); 237 | } 238 | 239 | case VMEMCACHE_STAT_PUT: 240 | for (int i = 0; i < NSHARDS; i++) 241 | total += index->bucket[i]->put_count; 242 | break; 243 | 244 | case VMEMCACHE_STAT_EVICT: 245 | for (int i = 0; i < NSHARDS; i++) 246 | total += index->bucket[i]->evict_count; 247 | break; 248 | 249 | case VMEMCACHE_STAT_HIT: 250 | for (int i = 0; i < NSHARDS; i++) 251 | total += index->bucket[i]->hit_count; 252 | break; 253 | 254 | case VMEMCACHE_STAT_MISS: 255 | for (int i = 0; i < NSHARDS; i++) 256 | total += index->bucket[i]->miss_count; 257 | break; 258 | 259 | case VMEMCACHE_STAT_ENTRIES: 260 | for (int i = 0; i < NSHARDS; i++) 261 | total += index->bucket[i]->leaf_count; 262 | break; 263 | 264 | default: 265 | FATAL("wrong stat type"); /* not callable from outside */ 266 | } 267 | 268 | return total; 269 | } 270 | -------------------------------------------------------------------------------- /src/vmemcache_index.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_index.h -- internal definitions for vmemcache indexing API 6 | */ 7 | 8 | #ifndef VMEMCACHE_INDEX_H 9 | #define VMEMCACHE_INDEX_H 1 10 | 11 | #include "libvmemcache.h" 12 | #include "critnib.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | struct cache_entry; 19 | 20 | struct index *vmcache_index_new(void); 21 | void vmcache_index_delete(struct index *index, delete_entry_t del_entry); 22 | int vmcache_index_insert(struct index *index, 23 | struct cache_entry *entry); 24 | int vmcache_index_get(struct index *index, const void *key, size_t ksize, 25 | struct cache_entry **entry, int bump_stat); 26 | int vmcache_index_remove(VMEMcache *cache, struct cache_entry *entry); 27 | size_t vmemcache_index_get_stat(struct index *index, 28 | enum vmemcache_statistic stat); 29 | 30 | #ifdef __cplusplus 31 | } 32 | #endif 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/vmemcache_repl.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_repl.c -- replacement policies for vmemcache 6 | */ 7 | 8 | #include 9 | 10 | #include "vmemcache.h" 11 | #include "vmemcache_repl.h" 12 | #include "util.h" 13 | #include "out.h" 14 | #include "sys/queue.h" 15 | #include "sys_util.h" 16 | #include "ringbuf.h" 17 | 18 | #define LEN_RING_BUF (1 << 12) 19 | 20 | struct repl_p_entry { 21 | TAILQ_ENTRY(repl_p_entry) node; 22 | void *data; 23 | struct repl_p_entry **ptr_entry; /* pointer to be zeroed when evicted */ 24 | }; 25 | 26 | struct repl_p_head { 27 | os_mutex_t lock; 28 | TAILQ_HEAD(head, repl_p_entry) first; 29 | struct ringbuf *ringbuf; 30 | }; 31 | 32 | /* forward declarations of replacement policy operations */ 33 | 34 | static int 35 | repl_p_none_new(struct repl_p_head **head); 36 | 37 | static void 38 | repl_p_none_delete(struct repl_p_head *head); 39 | 40 | static struct repl_p_entry * 41 | repl_p_none_insert(struct repl_p_head *head, void *element, 42 | struct repl_p_entry **ptr_entry); 43 | 44 | static void 45 | repl_p_none_use(struct repl_p_head *head, struct repl_p_entry **ptr_entry); 46 | 47 | static void * 48 | repl_p_none_evict(struct repl_p_head *head, struct repl_p_entry **ptr_entry); 49 | 50 | static int 51 | repl_p_lru_new(struct repl_p_head **head); 52 | 53 | static void 54 | repl_p_lru_delete(struct repl_p_head *head); 55 | 56 | static struct repl_p_entry * 57 | repl_p_lru_insert(struct repl_p_head *head, void *element, 58 | struct repl_p_entry **ptr_entry); 59 | 60 | static void 61 | repl_p_lru_use(struct repl_p_head *head, struct repl_p_entry **ptr_entry); 62 | 63 | static void * 64 | repl_p_lru_evict(struct repl_p_head *head, struct repl_p_entry **ptr_entry); 65 | 66 | /* replacement policy operations */ 67 | static const struct repl_p_ops repl_p_ops[VMEMCACHE_REPLACEMENT_NUM] = { 68 | { 69 | .repl_p_new = repl_p_none_new, 70 | .repl_p_delete = repl_p_none_delete, 71 | .repl_p_insert = repl_p_none_insert, 72 | .repl_p_use = repl_p_none_use, 73 | .repl_p_evict = repl_p_none_evict, 74 | .dram_per_entry = 0, 75 | }, 76 | { 77 | .repl_p_new = repl_p_lru_new, 78 | .repl_p_delete = repl_p_lru_delete, 79 | .repl_p_insert = repl_p_lru_insert, 80 | .repl_p_use = repl_p_lru_use, 81 | .repl_p_evict = repl_p_lru_evict, 82 | .dram_per_entry = sizeof(struct repl_p_entry), 83 | } 84 | }; 85 | 86 | /* 87 | * repl_p_init -- allocate and initialize the replacement policy structure 88 | */ 89 | struct repl_p * 90 | repl_p_init(enum vmemcache_repl_p rp) 91 | { 92 | struct repl_p *repl_p = Malloc(sizeof(struct repl_p)); 93 | if (repl_p == NULL) 94 | return NULL; 95 | 96 | repl_p->ops = &repl_p_ops[rp]; 97 | 98 | if (repl_p->ops->repl_p_new(&repl_p->head)) { 99 | Free(repl_p); 100 | return NULL; 101 | } 102 | 103 | return repl_p; 104 | } 105 | 106 | /* 107 | * repl_p_destroy -- destroy the replacement policy structure 108 | */ 109 | void 110 | repl_p_destroy(struct repl_p *repl_p) 111 | { 112 | ASSERTne(repl_p, NULL); 113 | 114 | repl_p->ops->repl_p_delete(repl_p->head); 115 | Free(repl_p); 116 | } 117 | 118 | /* 119 | * repl_p_none_new -- (internal) create a new "none" replacement policy 120 | */ 121 | static int 122 | repl_p_none_new(struct repl_p_head **head) 123 | { 124 | *head = NULL; 125 | return 0; 126 | } 127 | 128 | /* 129 | * repl_p_none_delete -- (internal) destroy the "none" replacement policy 130 | */ 131 | static void 132 | repl_p_none_delete(struct repl_p_head *head) 133 | { 134 | } 135 | 136 | /* 137 | * repl_p_none_insert -- (internal) insert a new element 138 | */ 139 | static struct repl_p_entry * 140 | repl_p_none_insert(struct repl_p_head *head, void *element, 141 | struct repl_p_entry **ptr_entry) 142 | { 143 | vmemcache_entry_acquire(element); 144 | return NULL; 145 | } 146 | 147 | /* 148 | * repl_p_none_use -- (internal) use the element 149 | */ 150 | static void 151 | repl_p_none_use(struct repl_p_head *head, struct repl_p_entry **ptr_entry) 152 | { 153 | } 154 | 155 | /* 156 | * repl_p_none_evict -- (internal) evict the element 157 | */ 158 | static void * 159 | repl_p_none_evict(struct repl_p_head *head, struct repl_p_entry **ptr_entry) 160 | { 161 | return ptr_entry; 162 | } 163 | 164 | 165 | /* 166 | * repl_p_lru_new -- (internal) create a new LRU replacement policy 167 | */ 168 | static int 169 | repl_p_lru_new(struct repl_p_head **head) 170 | { 171 | struct repl_p_head *h = Zalloc(sizeof(struct repl_p_head)); 172 | if (h == NULL) 173 | return -1; 174 | 175 | util_mutex_init(&h->lock); 176 | TAILQ_INIT(&h->first); 177 | h->ringbuf = ringbuf_new(LEN_RING_BUF); 178 | *head = h; 179 | 180 | return 0; 181 | } 182 | 183 | /* 184 | * dequeue_all -- (internal) dequeue all repl_p entries, 185 | * it MUST be run under a lock 186 | */ 187 | static void 188 | dequeue_all(struct repl_p_head *head) 189 | { 190 | struct repl_p_entry *e; 191 | int counter = 0; 192 | 193 | do { 194 | e = ringbuf_trydequeue_s(head->ringbuf, 195 | sizeof(struct repl_p_entry)); 196 | if (e == NULL) 197 | break; 198 | 199 | TAILQ_MOVE_TO_TAIL(&head->first, e, node); 200 | 201 | /* unlock the entry, so that it can be used again */ 202 | util_atomic_store_explicit64(e->ptr_entry, e, 203 | memory_order_relaxed); 204 | /* 205 | * We are limiting the number of iterations, 206 | * so that this loop ends for sure, because other thread 207 | * can insert new elements to the ring buffer in the same time. 208 | */ 209 | } while (++counter < LEN_RING_BUF); 210 | } 211 | 212 | /* 213 | * repl_p_lru_delete -- (internal) destroy the LRU replacement policy 214 | */ 215 | static void 216 | repl_p_lru_delete(struct repl_p_head *head) 217 | { 218 | dequeue_all(head); 219 | ringbuf_delete(head->ringbuf); 220 | 221 | while (!TAILQ_EMPTY(&head->first)) { 222 | struct repl_p_entry *entry = TAILQ_FIRST(&head->first); 223 | TAILQ_REMOVE(&head->first, entry, node); 224 | Free(entry); 225 | } 226 | 227 | util_mutex_destroy(&head->lock); 228 | Free(head); 229 | } 230 | 231 | /* 232 | * repl_p_lru_insert -- (internal) insert a new element 233 | */ 234 | static struct repl_p_entry * 235 | repl_p_lru_insert(struct repl_p_head *head, void *element, 236 | struct repl_p_entry **ptr_entry) 237 | { 238 | struct repl_p_entry *entry = Zalloc(sizeof(struct repl_p_entry)); 239 | if (entry == NULL) 240 | return NULL; 241 | 242 | entry->data = element; 243 | 244 | ASSERTne(ptr_entry, NULL); 245 | entry->ptr_entry = ptr_entry; 246 | 247 | /* 248 | * 'util_bool_compare_and_swap64' must always succeed here, 249 | * because this entry with ptr_entry=NULL has been considered as busy 250 | * so it has never been used so far. This is the first time we set 251 | * the 'entry->ptr_entry' to 'entry'. 252 | */ 253 | int rv = util_bool_compare_and_swap64(entry->ptr_entry, NULL, entry); 254 | if (rv == 0) { 255 | FATAL( 256 | "repl_p_lru_insert(): failed to initialize pointer to the LRU list"); 257 | } 258 | 259 | util_mutex_lock(&head->lock); 260 | 261 | vmemcache_entry_acquire(element); 262 | TAILQ_INSERT_TAIL(&head->first, entry, node); 263 | 264 | util_mutex_unlock(&head->lock); 265 | 266 | return entry; 267 | } 268 | 269 | /* 270 | * repl_p_lru_use -- (internal) use the element 271 | */ 272 | static void 273 | repl_p_lru_use(struct repl_p_head *head, struct repl_p_entry **ptr_entry) 274 | { 275 | struct repl_p_entry *entry; 276 | 277 | ASSERTne(ptr_entry, NULL); 278 | 279 | entry = *ptr_entry; 280 | if (entry == NULL) 281 | return; 282 | 283 | /* 284 | * Try to lock the entry by setting 'ptr_entry' to NULL 285 | * and enqueue it to the ring buffer, 286 | * so that it cannot be used nor evicted. 287 | */ 288 | if (!util_bool_compare_and_swap64(ptr_entry, entry, NULL)) 289 | return; 290 | 291 | /* 292 | * This the "in the middle of being used" state. 293 | * In this state - after bool_compare_and_swap() 294 | * and before ringbuf_tryenqueue() - the entry cannot be evicted. 295 | */ 296 | 297 | while (ringbuf_tryenqueue(head->ringbuf, entry) != 0) { 298 | util_mutex_lock(&head->lock); 299 | dequeue_all(head); 300 | util_mutex_unlock(&head->lock); 301 | } 302 | } 303 | 304 | /* 305 | * repl_p_lru_evict -- (internal) evict the element 306 | */ 307 | static void * 308 | repl_p_lru_evict(struct repl_p_head *head, struct repl_p_entry **ptr_entry) 309 | { 310 | struct repl_p_entry *entry; 311 | void *data = NULL; 312 | 313 | int is_LRU = (ptr_entry == NULL); 314 | 315 | util_mutex_lock(&head->lock); 316 | 317 | if (TAILQ_EMPTY(&head->first)) { 318 | errno = ESRCH; 319 | ERR("LRU queue is empty"); 320 | goto exit_unlock; 321 | } 322 | 323 | if (is_LRU) { 324 | entry = TAILQ_FIRST(&head->first); 325 | ptr_entry = entry->ptr_entry; 326 | } else { 327 | entry = *ptr_entry; 328 | } 329 | 330 | /* 331 | * Try to lock the entry by setting 'ptr_entry' to NULL, 332 | * so that it cannot be used nor evicted in other threads. 333 | */ 334 | if (entry != NULL && util_bool_compare_and_swap64(ptr_entry, 335 | entry, NULL)) 336 | goto evict_found_entry; 337 | 338 | /* 339 | * The first try failed. The entry could have been locked and enqueued 340 | * in the ring buffer, so let's flush the ring buffer and try again. 341 | */ 342 | dequeue_all(head); 343 | 344 | /* 345 | * If the entry was assigned as the LRU entry, let's assign it again, 346 | * because the LRU entry most likely has been changed in dequeue_all(). 347 | */ 348 | if (is_LRU) { 349 | entry = TAILQ_FIRST(&head->first); 350 | ptr_entry = entry->ptr_entry; 351 | } else { 352 | entry = *ptr_entry; 353 | } 354 | 355 | /* try to lock the entry the second time */ 356 | if (entry != NULL && util_bool_compare_and_swap64(ptr_entry, 357 | entry, NULL)) 358 | goto evict_found_entry; 359 | 360 | /* the second try failed */ 361 | 362 | if (!is_LRU) { 363 | /* the given entry is busy, give up */ 364 | errno = EAGAIN; 365 | ERR("entry is busy and cannot be evicted"); 366 | goto exit_unlock; 367 | } 368 | 369 | if (entry == NULL) { 370 | /* no entries in the LRU queue, give up */ 371 | errno = ESRCH; 372 | ERR("LRU queue is empty"); 373 | goto exit_unlock; 374 | } 375 | 376 | /* try to lock the next entries (repl_p_lru_evict can hardly fail) */ 377 | do { 378 | entry = TAILQ_NEXT(entry, node); 379 | if (entry == NULL) 380 | break; 381 | 382 | ptr_entry = entry->ptr_entry; 383 | } while (!util_bool_compare_and_swap64(ptr_entry, entry, NULL)); 384 | 385 | if (entry != NULL) 386 | goto evict_found_entry; 387 | 388 | /* 389 | * All entries in the LRU queue are locked. 390 | * The last chance is to try to dequeue an entry. 391 | */ 392 | entry = ringbuf_trydequeue_s(head->ringbuf, 393 | sizeof(struct repl_p_entry)); 394 | if (entry == NULL) { 395 | /* 396 | * Cannot find any entry to evict. 397 | * It means that all entries are heavily used 398 | * and they have to be "in the middle of being used" state now 399 | * (see repl_p_lru_use()). 400 | * There is nothing we can do but fail. 401 | */ 402 | errno = ESRCH; 403 | ERR("no entry eligible for eviction found"); 404 | goto exit_unlock; 405 | } 406 | 407 | evict_found_entry: 408 | TAILQ_REMOVE(&head->first, entry, node); 409 | 410 | data = entry->data; 411 | Free(entry); 412 | 413 | exit_unlock: 414 | util_mutex_unlock(&head->lock); 415 | return data; 416 | } 417 | -------------------------------------------------------------------------------- /src/vmemcache_repl.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_repl.h -- API of replacement policy for vmemcache 6 | */ 7 | 8 | #ifndef VMEMCACHE_REPL_H 9 | #define VMEMCACHE_REPL_H 1 10 | 11 | #include "libvmemcache.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | struct repl_p_head; 18 | struct repl_p_entry; 19 | 20 | struct repl_p_ops { 21 | /* create a new replacement policy list */ 22 | int 23 | (*repl_p_new)(struct repl_p_head **head); 24 | 25 | /* destroy the replacement policy list */ 26 | void 27 | (*repl_p_delete)(struct repl_p_head *head); 28 | 29 | /* insert a new element */ 30 | struct repl_p_entry * 31 | (*repl_p_insert)(struct repl_p_head *head, void *element, 32 | struct repl_p_entry **ptr_entry); 33 | 34 | /* evict an/the element */ 35 | void * 36 | (*repl_p_evict)(struct repl_p_head *head, 37 | struct repl_p_entry **ptr_entry); 38 | 39 | /* use the element */ 40 | void 41 | (*repl_p_use)(struct repl_p_head *head, 42 | struct repl_p_entry **ptr_entry); 43 | 44 | /* memory overhead per element */ 45 | size_t dram_per_entry; 46 | }; 47 | 48 | struct repl_p { 49 | const struct repl_p_ops *ops; 50 | struct repl_p_head *head; 51 | }; 52 | 53 | struct repl_p *repl_p_init(enum vmemcache_repl_p rp); 54 | void repl_p_destroy(struct repl_p *repl_p); 55 | 56 | #ifdef __cplusplus 57 | } 58 | #endif 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /tests/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | set(DEFAULT_TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/test) 5 | 6 | set(TEST_DIR ${DEFAULT_TEST_DIR} 7 | CACHE STRING "directory for vmemcache memory pool used for tests") 8 | 9 | set(DEVICE_DAX_PATH "" CACHE STRING 10 | "raw DAX device without a file system for tests. Example: /dev/dax0.0") 11 | 12 | set(GLOBAL_TEST_ARGS -DPARENT_DIR=${TEST_DIR}/) 13 | 14 | if(TRACE_TESTS) 15 | set(GLOBAL_TEST_ARGS ${GLOBAL_TEST_ARGS} --trace-expand) 16 | endif() 17 | 18 | add_cstyle(tests) 19 | add_check_whitespace(tests) 20 | 21 | set(vg_tracers memcheck helgrind drd) 22 | 23 | # Configures test ${name} 24 | function(test name file tracer location) 25 | if (${tracer} IN_LIST vg_tracers) 26 | if (NOT VALGRIND_FOUND) 27 | message(WARNING 28 | "Valgrind not found, test skipped: ${name}") 29 | return() 30 | endif() 31 | if (COVERAGE_BUILD) 32 | message(STATUS 33 | "This is the Coverage build, skipping Valgrind test: ${name}") 34 | return() 35 | endif() 36 | endif() 37 | 38 | if(${location} STREQUAL fs) 39 | set(TEST_POOL_LOCATION ${TEST_DIR}) 40 | elseif(${location} STREQUAL ddax) 41 | set(TEST_POOL_LOCATION ${DEVICE_DAX_PATH}) 42 | else() 43 | message(FATAL_ERROR "Unknown pool's location: ${location}") 44 | return() 45 | endif() 46 | 47 | add_test(NAME ${name} 48 | COMMAND ${CMAKE_COMMAND} 49 | ${GLOBAL_TEST_ARGS} 50 | -DTEST_NAME=${name} 51 | -DSRC_DIR=${CMAKE_CURRENT_SOURCE_DIR} 52 | -DBIN_DIR=${CMAKE_CURRENT_BINARY_DIR}/${file}-${tracer}-${location} 53 | -DCONFIG=$ 54 | -DTRACER=${tracer} 55 | -DTEST_POOL_LOCATION=${TEST_POOL_LOCATION} 56 | -P ${CMAKE_CURRENT_SOURCE_DIR}/${file}.cmake) 57 | 58 | set_tests_properties(${name} PROPERTIES 59 | ENVIRONMENT "LC_ALL=C;PATH=$ENV{PATH}" 60 | TIMEOUT 300) 61 | endfunction() 62 | 63 | # add and link an executable 64 | function(add_link_executable name sources libs) 65 | add_executable(${name} ${sources}) 66 | target_include_directories(${name} PRIVATE ${CMAKE_SOURCE_DIR}/src) 67 | if(STATS_ENABLED) 68 | target_compile_definitions(${name} PRIVATE STATS_ENABLED=1) 69 | endif() 70 | target_link_libraries(${name} PRIVATE ${libs}) 71 | endfunction() 72 | 73 | set(SOURCES_BASIC 74 | vmemcache_test_basic.c) 75 | 76 | set(SOURCES_MT 77 | vmemcache_test_mt.c 78 | ${CMAKE_SOURCE_DIR}/src/os_posix.c 79 | ${CMAKE_SOURCE_DIR}/src/os_thread_posix.c) 80 | 81 | set(SOURCES_UTLIIZATION 82 | vmemcache_test_utilization.c) 83 | 84 | set(SOURCES_HEAP_USAGE 85 | vmemcache_test_heap_usage.c) 86 | 87 | set(SOURCES_EXAMPLE 88 | example.c) 89 | 90 | set(SOURCES_TWOLEVEL 91 | twolevel.c) 92 | 93 | set(LIBS_BASIC 94 | vmemcache) 95 | 96 | set(LIBS_MT 97 | vmemcache 98 | ${CMAKE_THREAD_LIBS_INIT}) 99 | 100 | set(LIBS_UTLIIZATION 101 | vmemcache 102 | m) 103 | 104 | set(LIBS_HEAP_USAGE 105 | vmemcache 106 | ${CMAKE_DL_LIBS}) 107 | 108 | set(LIBS_EXAMPLE 109 | vmemcache) 110 | 111 | set(LIBS_TWOLEVEL 112 | vmemcache) 113 | 114 | add_link_executable(vmemcache_test_basic 115 | "${SOURCES_BASIC}" 116 | "${LIBS_BASIC}") 117 | 118 | add_link_executable(vmemcache_test_mt 119 | "${SOURCES_MT}" 120 | "${LIBS_MT}") 121 | 122 | add_link_executable(vmemcache_test_utilization 123 | "${SOURCES_UTLIIZATION}" 124 | "${LIBS_UTLIIZATION}") 125 | 126 | add_link_executable(example 127 | "${SOURCES_EXAMPLE}" 128 | "${LIBS_EXAMPLE}") 129 | 130 | add_link_executable(vmemcache_test_heap_usage 131 | "${SOURCES_HEAP_USAGE}" 132 | "${LIBS_HEAP_USAGE}") 133 | 134 | add_link_executable(twolevel 135 | "${SOURCES_TWOLEVEL}" 136 | "${LIBS_TWOLEVEL}") 137 | 138 | if(NOT "${TEST_DIR}" STREQUAL "") 139 | test("FS-test-basic" test-basic none fs) 140 | test("FS-test-basic-memcheck" test-basic memcheck fs) 141 | test("FS-test-mt" test-mt none fs) 142 | test("FS-test-mt-memcheck" test-mt memcheck fs) 143 | test("FS-test-mt-helgrind" test-mt helgrind fs) 144 | test("FS-test-mt-drd" test-mt drd fs) 145 | test("FS-test-bench-mt" test-bench-mt none fs) 146 | test("FS-test-bench-simul" test-bench-simul none fs) 147 | test("FS-test-bench-simul-memcheck" test-bench-simul memcheck fs) 148 | test("FS-test-bench-simul-helgrind" test-bench-simul helgrind fs) 149 | test("FS-test-bench-simul-drd" test-bench-simul drd fs) 150 | test("FS-test-utilization" test-utilization none fs) 151 | test("FS-test-utilization-memcheck" test-utilization memcheck fs) 152 | test("FS-test-heap-usage" test-heap-usage none fs) 153 | test("FS-test-heap-usage-memcheck" test-heap-usage memcheck fs) 154 | test("FS-test-twolevel" test-twolevel none fs) 155 | endif() 156 | 157 | if(NOT "${DEVICE_DAX_PATH}" STREQUAL "") 158 | test("DDAX-test-basic" test-basic none ddax) 159 | test("DDAX-test-basic-memcheck" test-basic memcheck ddax) 160 | test("DDAX-test-mt" test-mt none ddax) 161 | test("DDAX-test-mt-memcheck" test-mt memcheck ddax) 162 | test("DDAX-test-mt-helgrind" test-mt helgrind ddax) 163 | test("DDAX-test-mt-drd" test-mt drd ddax) 164 | test("DDAX-test-bench-mt" test-bench-mt none ddax) 165 | test("DDAX-test-bench-simul" test-bench-simul none ddax) 166 | test("DDAX-test-bench-simul-memcheck" test-bench-simul memcheck ddax) 167 | test("DDAX-test-bench-simul-helgrind" test-bench-simul helgrind ddax) 168 | test("DDAX-test-bench-simul-drd" test-bench-simul drd ddax) 169 | test("DDAX-test-utilization" test-utilization none ddax) 170 | test("DDAX-test-utilization-memcheck" test-utilization memcheck ddax) 171 | test("DDAX-test-heap-usage" test-heap-usage none ddax) 172 | test("DDAX-test-heap-usage-memcheck" test-heap-usage memcheck ddax) 173 | endif() 174 | 175 | test("example" test-example none fs) 176 | -------------------------------------------------------------------------------- /tests/drd-log.supp: -------------------------------------------------------------------------------- 1 | { 2 | 3 | drd:ConflictingAccess 4 | fun:*mempcpy 5 | ... 6 | fun:_IO_file_xsputn@@GLIBC* 7 | fun:fputs 8 | fun:out_print_func 9 | fun:out_common 10 | fun:out_log 11 | } 12 | { 13 | 14 | drd:ConflictingAccess 15 | fun:*memmove 16 | fun:_IO_file_xsputn@@GLIBC* 17 | fun:fputs 18 | fun:out_print_func 19 | fun:out_common 20 | fun:out_log 21 | } 22 | -------------------------------------------------------------------------------- /tests/example.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | #define STR_AND_LEN(x) (x), strlen(x) 9 | 10 | static VMEMcache *cache; 11 | 12 | static void 13 | on_miss(VMEMcache *cache, const void *key, size_t key_size, void *arg) 14 | { 15 | vmemcache_put(cache, STR_AND_LEN("meow"), 16 | STR_AND_LEN("Cthulhu fthagn")); 17 | } 18 | 19 | static void 20 | get(const char *key) 21 | { 22 | char buf[128]; 23 | ssize_t len = vmemcache_get(cache, STR_AND_LEN(key), 24 | buf, sizeof(buf), 0, NULL); 25 | if (len >= 0) 26 | printf("%.*s\n", (int)len, buf); 27 | else 28 | printf("(key not found: %s)\n", key); 29 | } 30 | 31 | int 32 | main() 33 | { 34 | cache = vmemcache_new(); 35 | if (vmemcache_add(cache, "/tmp")) { 36 | fprintf(stderr, "error: vmemcache_add: %s\n", 37 | vmemcache_errormsg()); 38 | return 1; 39 | } 40 | 41 | /* Query a non-existent key. */ 42 | get("meow"); 43 | 44 | /* Insert then query. */ 45 | vmemcache_put(cache, STR_AND_LEN("bark"), STR_AND_LEN("Lorem ipsum")); 46 | get("bark"); 47 | 48 | /* Install an on-miss handler. */ 49 | vmemcache_callback_on_miss(cache, on_miss, 0); 50 | get("meow"); 51 | 52 | vmemcache_delete(cache); 53 | return 0; 54 | } 55 | -------------------------------------------------------------------------------- /tests/helgrind-log.supp: -------------------------------------------------------------------------------- 1 | { 2 | 3 | Helgrind:Race 4 | fun:*mempcpy 5 | ... 6 | fun:_IO_file_xsputn@@GLIBC* 7 | fun:fputs 8 | fun:out_print_func 9 | fun:out_common 10 | fun:out_log 11 | } 12 | { 13 | 14 | Helgrind:Race 15 | fun:*memmove 16 | fun:_IO_file_xsputn@@GLIBC* 17 | fun:fputs 18 | fun:out_print_func 19 | fun:out_common 20 | fun:out_log 21 | } 22 | -------------------------------------------------------------------------------- /tests/helpers.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2017-2019, Intel Corporation 3 | 4 | cmake_minimum_required(VERSION 3.3) 5 | set(DIR ${PARENT_DIR}/${TEST_NAME}) 6 | 7 | if (WIN32) 8 | set(EXE_DIR ${CMAKE_CURRENT_BINARY_DIR}/../${CONFIG}) 9 | set(TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/../tests/${CONFIG}) 10 | else() 11 | set(EXE_DIR ${CMAKE_CURRENT_BINARY_DIR}/../) 12 | set(TEST_DIR ${CMAKE_CURRENT_BINARY_DIR}/../tests/) 13 | endif() 14 | 15 | function(setup) 16 | execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DIR}) 17 | execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${DIR}) 18 | execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${BIN_DIR}) 19 | execute_process(COMMAND ${CMAKE_COMMAND} -E make_directory ${BIN_DIR}) 20 | endfunction() 21 | 22 | function(cleanup) 23 | execute_process(COMMAND ${CMAKE_COMMAND} -E remove_directory ${DIR}) 24 | endfunction() 25 | 26 | # Executes test command ${name} and verifies its status matches ${expectation}. 27 | # Optional function arguments are passed as consecutive arguments to 28 | # the command. 29 | function(execute_arg input expectation name) 30 | message(STATUS "Executing: ${name} ${ARGN}") 31 | if("${input}" STREQUAL "") 32 | execute_process(COMMAND ${name} ${ARGN} 33 | RESULT_VARIABLE RET 34 | OUTPUT_FILE ${BIN_DIR}/out 35 | ERROR_FILE ${BIN_DIR}/err) 36 | else() 37 | execute_process(COMMAND ${name} ${ARGN} 38 | RESULT_VARIABLE RET 39 | INPUT_FILE ${input} 40 | OUTPUT_FILE ${BIN_DIR}/out 41 | ERROR_FILE ${BIN_DIR}/err) 42 | endif() 43 | message(STATUS "Test ${name}:") 44 | file(READ ${BIN_DIR}/out OUT) 45 | message(STATUS "Stdout:\n${OUT}") 46 | file(READ ${BIN_DIR}/err ERR) 47 | message(STATUS "Stderr:\n${ERR}") 48 | 49 | if(NOT RET EQUAL expectation) 50 | message(FATAL_ERROR "${name} ${ARGN} exit code ${RET} doesn't match expectation ${expectation}") 51 | endif() 52 | endfunction() 53 | 54 | function(run_under_valgrind vg_opt name) 55 | message(STATUS "Executing: valgrind ${vg_opt} ${name} ${ARGN}") 56 | execute_process(COMMAND valgrind ${vg_opt} ${name} ${ARGN} 57 | RESULT_VARIABLE RET 58 | OUTPUT_FILE ${BIN_DIR}/out 59 | ERROR_FILE ${BIN_DIR}/err) 60 | message(STATUS "Test ${name}:") 61 | file(READ ${BIN_DIR}/out OUT) 62 | message(STATUS "Stdout:\n${OUT}") 63 | file(READ ${BIN_DIR}/err ERR) 64 | message(STATUS "Stderr:\n${ERR}") 65 | 66 | if(NOT RET EQUAL 0) 67 | message(FATAL_ERROR 68 | "command 'valgrind ${name} ${ARGN}' failed:\n${ERR}") 69 | endif() 70 | 71 | set(text_passed "ERROR SUMMARY: 0 errors from 0 contexts") 72 | string(FIND "${ERR}" "${text_passed}" RET) 73 | if(RET EQUAL -1) 74 | message(FATAL_ERROR 75 | "command 'valgrind ${name} ${ARGN}' failed:\n${ERR}") 76 | endif() 77 | endfunction() 78 | 79 | function(execute expectation name) 80 | 81 | set(ENV{VMEMCACHE_FILE} "${BIN_DIR}/out.log") 82 | set(ENV{VMEMCACHE_LEVEL} "3") 83 | 84 | if (${TRACER} STREQUAL "none") 85 | execute_arg("" ${expectation} ${name} ${ARGN}) 86 | elseif (${TRACER} STREQUAL memcheck) 87 | set(VG_OPT "--leak-check=full") 88 | run_under_valgrind("${VG_OPT}" ${name} ${ARGN}) 89 | elseif (${TRACER} STREQUAL helgrind) 90 | set(HEL_SUPP "${SRC_DIR}/helgrind-log.supp") 91 | set(VG_OPT "--tool=helgrind" "--suppressions=${HEL_SUPP}") 92 | run_under_valgrind("${VG_OPT}" ${name} ${ARGN}) 93 | elseif (${TRACER} STREQUAL drd) 94 | set(DRD_SUPP "${SRC_DIR}/drd-log.supp") 95 | set(VG_OPT "--tool=drd" "--suppressions=${DRD_SUPP}") 96 | run_under_valgrind("${VG_OPT}" ${name} ${ARGN}) 97 | else () 98 | message(FATAL_ERROR "unknown tracer: ${TRACER}") 99 | endif () 100 | endfunction() 101 | -------------------------------------------------------------------------------- /tests/test-basic.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/vmemcache_test_basic ${TEST_POOL_LOCATION}) 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-bench-mt.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/../benchmarks/bench_micro "${TEST_POOL_LOCATION}") 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-bench-simul.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/../benchmarks/bench_simul "${TEST_POOL_LOCATION}" n_threads=4 ops_count=100 warm_up=0) 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-example.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/example) 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-heap-usage.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/vmemcache_test_heap_usage ${TEST_POOL_LOCATION}) 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-mt.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2018-2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | set(SEED 0) # set seed from time 9 | set(vg_thread_tracers helgrind drd) 10 | set(valgrind memcheck helgrind drd) 11 | 12 | if (${TRACER} IN_LIST vg_thread_tracers) 13 | set(N_THREADS 4) 14 | set(N_OPS 400) 15 | else() 16 | set(N_THREADS 10) 17 | set(N_OPS 10000) 18 | endif() 19 | 20 | if (${TRACER} IN_LIST valgrind) 21 | # skip tests that last very long under Valgrind 22 | execute(0 ${TEST_DIR}/vmemcache_test_mt ${TEST_POOL_LOCATION} ${N_THREADS} ${N_OPS} ${SEED} "skip") 23 | else() 24 | execute(0 ${TEST_DIR}/vmemcache_test_mt ${TEST_POOL_LOCATION} ${N_THREADS} ${N_OPS} ${SEED}) 25 | 26 | # additional tests for number of threads == 1 and 2 27 | execute(0 ${TEST_DIR}/vmemcache_test_mt ${TEST_POOL_LOCATION} 1) 28 | execute(0 ${TEST_DIR}/vmemcache_test_mt ${TEST_POOL_LOCATION} 2) 29 | endif() 30 | 31 | cleanup() 32 | -------------------------------------------------------------------------------- /tests/test-twolevel.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | setup() 7 | 8 | execute(0 ${TEST_DIR}/twolevel ${TEST_POOL_LOCATION} ${TEST_POOL_LOCATION}) 9 | 10 | cleanup() 11 | -------------------------------------------------------------------------------- /tests/test-utilization.cmake: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2019, Intel Corporation 3 | 4 | include(${SRC_DIR}/helpers.cmake) 5 | 6 | set(TIMEOUT_SEC 2) 7 | 8 | setup() 9 | 10 | execute(0 ${TEST_DIR}/vmemcache_test_utilization -d ${TEST_POOL_LOCATION} 11 | -t ${TIMEOUT_SEC} -n) 12 | 13 | cleanup() 14 | -------------------------------------------------------------------------------- /tests/test_helpers.h: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2018-2019, Intel Corporation */ 3 | 4 | /* 5 | * test_helpers.h -- header with helpers 6 | */ 7 | 8 | #ifndef TEST_HELPERS_H 9 | #define TEST_HELPERS_H 1 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #define UT_ERR(...) do {\ 20 | fprintf(stderr, "ERROR: " __VA_ARGS__);\ 21 | fprintf(stderr, "\n");\ 22 | } while (/*CONSTCOND*/0) 23 | 24 | #define UT_FATAL(...) do {\ 25 | fprintf(stderr, "FATAL ERROR at %s:%i in %s(): ",\ 26 | __FILE__, __LINE__, __func__);\ 27 | fprintf(stderr, __VA_ARGS__);\ 28 | fprintf(stderr, "\n");\ 29 | abort();\ 30 | } while (/*CONSTCOND*/0) 31 | 32 | #define UT_ASSERTeq(x, y) do if ((x) != (y)) {\ 33 | UT_FATAL("ASSERT FAILED : " #x " (%llu) ≠ %llu",\ 34 | (unsigned long long)(x), (unsigned long long)(y));\ 35 | } while (/*CONSTCOND*/0) 36 | 37 | #define UT_ASSERTin(x, min, max) do if ((x) < (min) || (x) > (max)) {\ 38 | UT_FATAL("ASSERT FAILED : " #x " = %llu not in [%llu,%llu]",\ 39 | (unsigned long long)(x),\ 40 | (unsigned long long)(min), (unsigned long long)(max));\ 41 | } while (/*CONSTCOND*/0) 42 | 43 | 44 | /* 45 | * str_to_unsigned -- (internal) convert string argument to unsigned int 46 | */ 47 | static inline int 48 | str_to_unsigned(const char *str, unsigned *value) 49 | { 50 | char *endptr = NULL; 51 | 52 | errno = 0; /* to distinguish success/failure after call */ 53 | 54 | unsigned long val = strtoul(str, &endptr, 10); 55 | if ((errno == ERANGE && val == ULONG_MAX) || 56 | (errno != 0 && val == 0) || 57 | (endptr == str) || (*endptr != '\0')) { 58 | UT_ERR("strtoul() failed to convert the string %s", str); 59 | return -1; 60 | } 61 | 62 | if (val > UINT_MAX) { 63 | UT_ERR("value %s is bigger than UINT_MAX (%u)", str, UINT_MAX); 64 | return -1; 65 | } 66 | 67 | *value = (unsigned)val; 68 | 69 | return 0; 70 | } 71 | 72 | /* 73 | * str_to_ull -- (internal) convert string argument to unsigned long long 74 | */ 75 | static inline int 76 | str_to_ull(const char *str, unsigned long long *value) 77 | { 78 | char *endptr = NULL; 79 | 80 | errno = 0; /* to distinguish success/failure after call */ 81 | 82 | unsigned long long val = strtoull(str, &endptr, 10); 83 | if ((errno == ERANGE && val == ULLONG_MAX) || 84 | (errno != 0 && val == 0) || 85 | (endptr == str) || (*endptr != '\0')) { 86 | UT_ERR("strtoull() failed to convert the string %s", str); 87 | return -1; 88 | } 89 | 90 | *value = (unsigned long long)val; 91 | 92 | return 0; 93 | } 94 | 95 | /* 96 | * get_granular_rand_size - (internal) generate random size value 97 | * with specified granularity 98 | */ 99 | static inline size_t 100 | get_granular_rand_size(size_t val_max, size_t granularity) 101 | { 102 | size_t val_size = 103 | (1 + (size_t) rand() / (RAND_MAX / (val_max / granularity) + 1)) * 104 | granularity; 105 | 106 | assert(val_size <= val_max); 107 | assert(val_size >= granularity); 108 | assert(val_size % granularity == 0 && 109 | "put value size must be a multiple of granularity"); 110 | 111 | return val_size; 112 | } 113 | 114 | #endif /* TEST_HELPERS_H */ 115 | -------------------------------------------------------------------------------- /tests/twolevel.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #define ERR(...) do { fprintf(stderr, __VA_ARGS__); exit(1); } while (0) 11 | 12 | #define SIZE_MB (1024 * 1024ULL) 13 | #define L1_CAPACITY (1 * SIZE_MB) 14 | #define L2_CAPACITY (10 * SIZE_MB) 15 | #define ZSIZE (SIZE_MB / 2) 16 | 17 | static void 18 | evict_demote(VMEMcache *cache, const void *key, size_t key_size, void *arg) 19 | { 20 | VMEMcache *colder = (VMEMcache *)arg; 21 | 22 | size_t vsize; 23 | /* First, obtain the value's size. */ 24 | if (vmemcache_get(cache, key, key_size, NULL, 0, 0, &vsize)) 25 | return; /* Somehow got deleted? -- can't happen. */ 26 | void *buf = malloc(vsize); 27 | if (!buf) 28 | return; 29 | /* Then, fetch the value. */ 30 | if (vmemcache_get(cache, key, key_size, buf, vsize, 0, NULL) == 31 | (ssize_t)vsize) { 32 | /* Again, it's not supposed to be missing. */ 33 | vmemcache_put(colder, key, key_size, buf, vsize); 34 | } 35 | free(buf); 36 | } 37 | 38 | static void 39 | miss_promote(VMEMcache *cache, const void *key, size_t key_size, void *arg) 40 | { 41 | VMEMcache *colder = (VMEMcache *)arg; 42 | 43 | size_t vsize; 44 | if (vmemcache_get(colder, key, key_size, NULL, 0, 0, &vsize)) { 45 | /* 46 | * Second-level cache miss. 47 | * 48 | * You may want to handle it somehow here. 49 | */ 50 | return; 51 | } 52 | void *buf = malloc(vsize); 53 | if (!buf) 54 | return; 55 | if (vmemcache_get(colder, key, key_size, buf, vsize, 0, NULL) == 56 | (ssize_t)vsize) { 57 | /* 58 | * Note that there's no lock, thus our entry may disappear 59 | * between these two get() calls. 60 | */ 61 | if (!vmemcache_put(cache, key, key_size, buf, vsize)) { 62 | /* 63 | * Put can legitimately fail: value too big for 64 | * upper-level cache, no space because all evictable 65 | * keys are busy, etc. 66 | * 67 | * The promotion likely cascades into one or more 68 | * demotions to migrate cold keys downwards, to make 69 | * space. 70 | */ 71 | /* 72 | * You may or may not want to evict from cold cache 73 | * here. 74 | */ 75 | vmemcache_evict(colder, key, key_size); 76 | } 77 | } 78 | free(buf); 79 | } 80 | 81 | static void 82 | get(VMEMcache *cache, const char *x, int expfail) 83 | { 84 | ssize_t ret = vmemcache_get(cache, x, strlen(x) + 1, NULL, 0, 0, NULL); 85 | if ((!ret) == expfail) { 86 | ERR("get(“%s”) %s when it shouldn't\n", x, expfail ? 87 | "succeeded" : "failed"); 88 | } 89 | } 90 | 91 | int 92 | main(int argc, const char **argv) 93 | { 94 | if (argc != 3) 95 | ERR("Usage: twolevel \n"); 96 | 97 | VMEMcache *pmem = vmemcache_new(); 98 | VMEMcache *dram = vmemcache_new(); 99 | if (!pmem || !dram) 100 | ERR("VMEMcache_new failed\n"); 101 | 102 | vmemcache_set_size(pmem, L2_CAPACITY); 103 | vmemcache_set_size(dram, L1_CAPACITY); 104 | 105 | if (vmemcache_add(pmem, argv[1])) 106 | ERR("vmemcache_add(“%s”) failed: %m\n", argv[1]); 107 | if (vmemcache_add(dram, argv[2])) 108 | ERR("vmemcache_add(“%s”) failed: %m\n", argv[2]); 109 | 110 | vmemcache_callback_on_evict(dram, evict_demote, pmem); 111 | vmemcache_callback_on_miss(dram, miss_promote, pmem); 112 | 113 | void *lotta_zeroes = mmap(NULL, ZSIZE, PROT_READ, 114 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, -1, 0); 115 | if (!lotta_zeroes) 116 | ERR("can't mmap zeroes: %m\n"); 117 | 118 | #define PUT(x) vmemcache_put(dram, x, strlen(x) + 1, lotta_zeroes, ZSIZE) 119 | #define GET(x) get(dram, x, 0) 120 | #define GETF(x) get(dram, x, 1) 121 | 122 | PUT("first"); 123 | PUT("second"); 124 | PUT("third"); 125 | GET("first"); 126 | GET("first"); 127 | GET("second"); 128 | GET("third"); 129 | GETF("nonexistent"); 130 | 131 | const int cap = (L1_CAPACITY / ZSIZE - 1) 132 | + (L2_CAPACITY / ZSIZE - 1) 133 | - 1; 134 | 135 | for (int i = 0; i < cap; i++) { 136 | char buf[12]; 137 | snprintf(buf, sizeof(buf), "%d", i); 138 | PUT(buf); 139 | } 140 | /* "first" and "second" should have been dropped, "third" is still in */ 141 | GETF("first"); 142 | GETF("second"); 143 | GET("third"); 144 | 145 | return 0; 146 | } 147 | -------------------------------------------------------------------------------- /tests/vmemcache_test_heap_usage.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_test_heap_usage.c -- libvmemcache heap usage tracing test. 6 | * The test passes if measured unit usage (usage per entry) is lower than 7 | * MAX_BYTES_PER_ENTRY 8 | */ 9 | 10 | /* enable RTLD_NEXT not defined by POSIX */ 11 | #define _GNU_SOURCE 12 | 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include "test_helpers.h" 22 | 23 | #define MAX_BYTES_PER_ENTRY 180 24 | 25 | static __thread int trace = 0; 26 | /* 27 | * TRACE_HEAP - heap usage is traced only for expressions wrapped inside 28 | * this macro 29 | */ 30 | #define TRACE_HEAP(fn) do { trace = 1; fn; trace = 0; } while (0) 31 | 32 | static void *(*actual_malloc)(size_t); 33 | static void *(*actual_realloc)(void *, size_t); 34 | static void (*actual_free)(void *); 35 | 36 | typedef struct { 37 | ssize_t usage; 38 | size_t entries; 39 | ssize_t unit_usage; 40 | int evicted; 41 | } heap_usage; 42 | 43 | static __thread heap_usage usage = {0, 0, 0, 0}; 44 | static int verbose = 0; 45 | 46 | /* 47 | * on_evict_cb -- (internal) 'on evict' callback 48 | */ 49 | static void 50 | on_evict_cb(VMEMcache *cache, const void *key, size_t key_size, 51 | void *arg) 52 | { 53 | heap_usage *usage = arg; 54 | usage->entries--; 55 | usage->evicted = 1; 56 | } 57 | 58 | /* 59 | * log_line -- (internal) print log line with current heap status 60 | */ 61 | static void 62 | log_line(heap_usage *usage, size_t size, void *ptr, const char *prefix) 63 | { 64 | printf("%s %zu bytes\t(%p)\theap usage: %zd bytes\n", 65 | prefix, size, ptr, usage->usage); 66 | } 67 | 68 | /* 69 | * malloc -- (internal) 'malloc' wrapper 70 | */ 71 | void 72 | *malloc(size_t size) 73 | { 74 | int tmp_trace = trace; 75 | trace = 0; 76 | 77 | if (actual_malloc == NULL) { 78 | actual_malloc = dlsym(RTLD_NEXT, "malloc"); 79 | if (actual_malloc == NULL) 80 | UT_FATAL("dlsym: could not load 'malloc' symbol"); 81 | } 82 | 83 | void *p = actual_malloc(size); 84 | if (p == NULL) 85 | goto end; 86 | 87 | if (tmp_trace) { 88 | size_t s = malloc_usable_size(p); 89 | usage.usage += (ssize_t)s; 90 | if (verbose) 91 | log_line(&usage, s, p, "allocating"); 92 | } 93 | 94 | end: 95 | trace = tmp_trace; 96 | return p; 97 | } 98 | 99 | /* 100 | * realloc -- (internal) 'realloc' wrapper 101 | */ 102 | void 103 | *realloc(void *ptr, size_t size) 104 | { 105 | int tmp_trace = trace; 106 | trace = 0; 107 | 108 | if (actual_realloc == NULL) { 109 | actual_realloc = dlsym(RTLD_NEXT, "realloc"); 110 | if (actual_realloc == NULL) 111 | UT_FATAL("dlsym: could not load 'realloc' symbol"); 112 | } 113 | 114 | if (tmp_trace) { 115 | size_t old_size = malloc_usable_size(ptr); 116 | usage.usage -= (ssize_t)old_size; 117 | } 118 | 119 | void *p = actual_realloc(ptr, size); 120 | if (p == NULL) 121 | goto end; 122 | 123 | if (tmp_trace) { 124 | size_t new_size = malloc_usable_size(p); 125 | usage.usage += (ssize_t)new_size; 126 | if (verbose) 127 | log_line(&usage, new_size, p, "allocating"); 128 | } 129 | 130 | end: 131 | trace = tmp_trace; 132 | return p; 133 | } 134 | 135 | /* 136 | * free -- (internal) 'free' wrapper 137 | */ 138 | void 139 | free(void *ptr) 140 | { 141 | int tmp_trace = trace; 142 | trace = 0; 143 | 144 | if (actual_free == NULL) { 145 | actual_free = dlsym(RTLD_NEXT, "free"); 146 | if (actual_free == NULL) 147 | UT_FATAL("dlsym: could not load 'free' symbol"); 148 | } 149 | 150 | if (tmp_trace) { 151 | size_t size = malloc_usable_size(ptr); 152 | usage.usage -= (ssize_t)size; 153 | if (verbose) 154 | log_line(&usage, size, ptr, "freeing"); 155 | } 156 | 157 | actual_free(ptr); 158 | trace = tmp_trace; 159 | } 160 | 161 | /* 162 | * test_heap_usage -- (internal) test heap usage 163 | */ 164 | static int 165 | test_heap_usage(const char *dir, heap_usage *usage) 166 | { 167 | int ret = 0; 168 | 169 | VMEMcache *cache; 170 | TRACE_HEAP(cache = vmemcache_new()); 171 | if (cache == NULL) 172 | UT_FATAL("vmemcache_new: %s", vmemcache_errormsg()); 173 | vmemcache_set_size(cache, VMEMCACHE_MIN_POOL); 174 | vmemcache_set_extent_size(cache, VMEMCACHE_MIN_EXTENT); 175 | TRACE_HEAP(ret = vmemcache_add(cache, dir)); 176 | if (ret) 177 | UT_FATAL("vmemcache_add: %s", vmemcache_errormsg()); 178 | 179 | TRACE_HEAP(vmemcache_callback_on_evict(cache, on_evict_cb, usage)); 180 | 181 | size_t key = 0; 182 | size_t vsize = 32; 183 | 184 | char *value = malloc(vsize); 185 | if (value == NULL) 186 | UT_FATAL("out of memory"); 187 | memset(value, 'a', vsize - 1); 188 | value[vsize - 1] = '\0'; 189 | 190 | int putret; 191 | while (!usage->evicted) { 192 | TRACE_HEAP(putret = vmemcache_put(cache, &key, sizeof(key), 193 | value, vsize)); 194 | if (putret) 195 | UT_FATAL("vmemcache put: %s. errno: %s", 196 | vmemcache_errormsg(), strerror(errno)); 197 | 198 | usage->entries++; 199 | key++; 200 | usage->unit_usage = 201 | usage->usage / (ssize_t)usage->entries; 202 | 203 | if (verbose) 204 | printf( 205 | "bytes per entry: %zu, (number of entries: %zu)\n", 206 | usage->unit_usage, usage->entries); 207 | } 208 | free(value); 209 | ssize_t unit_usage_full_cache = usage->unit_usage; 210 | 211 | TRACE_HEAP(vmemcache_delete(cache)); 212 | 213 | printf("heap usage per entry: %zd bytes\n", unit_usage_full_cache); 214 | if (unit_usage_full_cache > MAX_BYTES_PER_ENTRY) { 215 | UT_ERR( 216 | "heap usage per entry equals %zd bytes, should be lower than %d bytes", 217 | unit_usage_full_cache, MAX_BYTES_PER_ENTRY); 218 | ret = 1; 219 | } 220 | 221 | if (usage->usage != 0) 222 | UT_FATAL( 223 | "Final heap usage is different than 0 (%zd): possible memory leak", 224 | usage->usage); 225 | 226 | return ret; 227 | } 228 | 229 | int 230 | main(int argc, char **argv) 231 | { 232 | if (argc < 2) 233 | UT_FATAL("%s \n", argv[0]); 234 | 235 | if (argc == 3) { 236 | if (strcmp("verbose", argv[2]) == 0) 237 | verbose = 1; 238 | else 239 | UT_FATAL("Unknown argument: %s", argv[2]); 240 | } 241 | 242 | return test_heap_usage(argv[1], &usage); 243 | } 244 | -------------------------------------------------------------------------------- /tests/vmemcache_test_utilization.c: -------------------------------------------------------------------------------- 1 | // SPDX-License-Identifier: BSD-3-Clause 2 | /* Copyright 2019, Intel Corporation */ 3 | 4 | /* 5 | * vmemcache_test_utilization.c -- space utilization test source 6 | */ 7 | 8 | #include "test_helpers.h" 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | #include 20 | 21 | #define ALLOWED_RATIO 0.87 22 | #define MAX_KEYSIZE 30 23 | 24 | typedef struct { 25 | bool evicted; 26 | } on_evict_info; 27 | 28 | typedef struct { 29 | size_t pool_size; 30 | size_t extent_size; 31 | size_t val_max; 32 | char dir[PATH_MAX]; 33 | long seconds; 34 | unsigned seed; 35 | int print_output; 36 | } test_params; 37 | 38 | static const char *usage_str = "usage: %s " 39 | "-d " 40 | "[-p ] " 41 | "[-e ] " 42 | "[-v ] " 43 | "[-t ] " 44 | "[-m ] " 45 | "[-o ] " 46 | "[-s ] " 47 | "[-n] " 48 | "[-h]\n" 49 | "\t n - do not print out csv output (it is printed by default)\n"; 50 | 51 | /* 52 | * on_evict - (internal) on evict callback function 53 | */ 54 | static void 55 | on_evict(VMEMcache *cache, const void *key, size_t key_size, void *arg) 56 | { 57 | on_evict_info *info = (on_evict_info *)arg; 58 | info->evicted = true; 59 | } 60 | 61 | /* 62 | * parse_ull - (internal) try parsing unsigned long long command line argument 63 | */ 64 | static unsigned long long 65 | parse_ull(const char *valname, const char *prog) 66 | { 67 | unsigned long long val; 68 | if (str_to_ull(optarg, &val) != 0) { 69 | fprintf(stderr, "invalid %s value\n", valname); 70 | printf(usage_str, prog); 71 | exit(1); 72 | } 73 | return val; 74 | } 75 | 76 | /* 77 | * parse_unsigned - (internal) try parsing unsigned command line argument 78 | */ 79 | static unsigned 80 | parse_unsigned(const char *valname, const char *prog) 81 | { 82 | unsigned val; 83 | if (str_to_unsigned(optarg, &val) != 0) { 84 | fprintf(stderr, "invalid %s value\n", valname); 85 | printf(usage_str, prog); 86 | exit(1); 87 | } 88 | return val; 89 | } 90 | 91 | /* 92 | * argerror - (internal) exit with message on command line argument error 93 | */ 94 | static void 95 | argerror(const char *msg, const char *prog) 96 | { 97 | fprintf(stderr, "%s", msg); 98 | printf(usage_str, prog); 99 | exit(1); 100 | } 101 | 102 | /* 103 | * parse_args - (internal) parse command line arguments 104 | */ 105 | static test_params 106 | parse_args(int argc, char **argv) 107 | { 108 | test_params p = { 109 | .pool_size = VMEMCACHE_MIN_POOL, 110 | .extent_size = VMEMCACHE_MIN_EXTENT, 111 | .val_max = 0, 112 | .dir = "", 113 | .seconds = 0, 114 | .seed = 0, 115 | .print_output = 1, 116 | }; 117 | size_t val_max_factor = 70; 118 | 119 | const char *optstr = "hp:e:v:t:m:o:d:s:n"; 120 | int opt; 121 | long seconds = 0; 122 | long minutes = 0; 123 | long hours = 0; 124 | while ((opt = getopt(argc, argv, optstr)) != -1) { 125 | switch (opt) { 126 | case 'h': 127 | printf(usage_str, argv[0]); 128 | exit(0); 129 | case 'p': 130 | p.pool_size = 131 | (size_t)parse_ull("pool size", argv[0]); 132 | break; 133 | case 'e': 134 | p.extent_size = 135 | (size_t)parse_ull("extent size", argv[0]); 136 | break; 137 | case 'v': 138 | val_max_factor = 139 | (size_t)parse_ull("val max factor", argv[0]); 140 | break; 141 | case 't': 142 | seconds = parse_unsigned("seconds", argv[0]); 143 | break; 144 | case 'm': 145 | minutes = parse_unsigned("minutes", argv[0]); 146 | break; 147 | case 'o': 148 | hours = parse_unsigned("hours", argv[0]); 149 | break; 150 | case 's': 151 | p.seed = parse_unsigned("seed for rand()", argv[0]); 152 | break; 153 | case 'd': 154 | if (*optarg == 0) 155 | argerror("invalid dir argument\n", argv[0]); 156 | strcpy(p.dir, optarg); 157 | break; 158 | case 'n': 159 | p.print_output = 0; 160 | break; 161 | default: 162 | argerror("", argv[0]); 163 | break; 164 | } 165 | } 166 | 167 | if (*p.dir == 0) 168 | argerror("missing required dir argument\n", argv[0]); 169 | 170 | p.seconds = seconds + 60 * minutes + 3600 * hours; 171 | if (p.seconds <= 0) 172 | argerror("timeout must be greater than 0\n", argv[0]); 173 | 174 | p.val_max = val_max_factor * p.extent_size; 175 | 176 | if (p.seed == 0) 177 | p.seed = (unsigned)time(NULL); 178 | srand(p.seed); 179 | printf("seed = %u\n", p.seed); 180 | 181 | return p; 182 | } 183 | 184 | 185 | /* 186 | * put_until_timeout - (internal) put random-sized values into cache, 187 | * print utilization ratio as a csv 188 | */ 189 | static int 190 | put_until_timeout(VMEMcache *vc, const test_params *p) 191 | { 192 | int ret = 1; 193 | 194 | on_evict_info info = { false }; 195 | vmemcache_callback_on_evict(vc, on_evict, &info); 196 | 197 | /* print csv header */ 198 | if (p->print_output) 199 | printf("keynum,ratio\n"); 200 | 201 | float prev_ratio; 202 | float ratio = 0.0f; 203 | bool print_ratio = false; 204 | 205 | char *val = malloc(p->val_max); 206 | if (val == NULL) { 207 | fprintf(stderr, "malloc: cannot allocate memory (%zu bytes)\n", 208 | p->val_max); 209 | return ret; 210 | } 211 | 212 | size_t val_size; 213 | unsigned long long used_size; 214 | char key[MAX_KEYSIZE]; 215 | int len; 216 | size_t keynum = 0; 217 | 218 | long endtime = time(NULL) + p->seconds; 219 | while (endtime > time(NULL)) { 220 | /* create key */ 221 | len = sprintf(key, "%zu", keynum); 222 | if (len < 0) { 223 | fprintf(stderr, "sprintf return value: %d\n", len); 224 | goto exit_free; 225 | } 226 | 227 | /* generate value */ 228 | val_size = get_granular_rand_size(p->val_max, p->extent_size); 229 | 230 | /* put */ 231 | int ret = vmemcache_put(vc, key, (size_t)len, val, val_size); 232 | if (ret != 0) { 233 | fprintf(stderr, "vmemcache_put: %s\n", 234 | vmemcache_errormsg()); 235 | goto exit_free; 236 | } 237 | 238 | #ifdef STATS_ENABLED 239 | if (vmemcache_get_stat(vc, VMEMCACHE_STAT_POOL_SIZE_USED, 240 | &used_size, sizeof(used_size)) != 0) { 241 | fprintf(stderr, "vmemcache_get_stat: %s\n", 242 | vmemcache_errormsg()); 243 | goto exit_free; 244 | } 245 | #else 246 | /* 247 | * This test will always pass and show 100% utilization, 248 | * if statistics are disabled. 249 | */ 250 | used_size = p->pool_size; 251 | #endif /* STATS_ENABLED */ 252 | 253 | /* 254 | * Do not print the csv line if current ratio value is the same 255 | * (taking precision into account) as the previous one. The 256 | * intent is to avoid unnecessary bloating of the csv output. 257 | */ 258 | ratio = (float)used_size / (float)p->pool_size; 259 | if (p->print_output) { 260 | print_ratio = keynum == 0 || lroundf(ratio * 100) 261 | != lroundf(prev_ratio * 100); 262 | if (print_ratio) { 263 | printf("%zu,%.3f\n", keynum, ratio); 264 | prev_ratio = ratio; 265 | } 266 | } 267 | 268 | if (info.evicted && ratio < ALLOWED_RATIO) { 269 | fprintf(stderr, 270 | "insufficient space utilization. ratio: %.3f: seed %u\n", 271 | ratio, p->seed); 272 | goto exit_free; 273 | } 274 | 275 | ++keynum; 276 | } 277 | 278 | ret = 0; 279 | 280 | /* print the last csv line if already not printed */ 281 | if (p->print_output) { 282 | if (!print_ratio) 283 | printf("%zu,%.3f\n", keynum - 1, ratio); 284 | } else { 285 | printf("Passed\n"); 286 | } 287 | 288 | exit_free: 289 | free(val); 290 | 291 | return ret; 292 | } 293 | 294 | int 295 | main(int argc, char **argv) 296 | { 297 | test_params p = parse_args(argc, argv); 298 | 299 | VMEMcache *vc = vmemcache_new(); 300 | vmemcache_set_size(vc, p.pool_size); 301 | vmemcache_set_extent_size(vc, p.extent_size); 302 | vmemcache_set_eviction_policy(vc, VMEMCACHE_REPLACEMENT_LRU); 303 | if (vmemcache_add(vc, p.dir)) 304 | UT_FATAL("vmemcache_new: %s (%s)", vmemcache_errormsg(), p.dir); 305 | 306 | int ret = put_until_timeout(vc, &p); 307 | 308 | vmemcache_delete(vc); 309 | 310 | return ret; 311 | } 312 | -------------------------------------------------------------------------------- /travis.yml: -------------------------------------------------------------------------------- 1 | dist: trusty 2 | 3 | sudo: required 4 | 5 | language: c 6 | 7 | services: 8 | - docker 9 | 10 | env: 11 | matrix: 12 | - TYPE=normal OS=ubuntu OS_VER=18.04 PUSH_IMAGE=1 COVERAGE=1 13 | - TYPE=normal OS=fedora OS_VER=28 PUSH_IMAGE=1 14 | 15 | before_install: 16 | - echo $TRAVIS_COMMIT_RANGE 17 | - export HOST_WORKDIR=`pwd` 18 | - export GITHUB_REPO=pmem/vmemcache 19 | - export DOCKERHUB_REPO=pmem/vmemcache 20 | - cd utils/docker 21 | - ./pull-or-rebuild-image.sh 22 | - if [[ -f push_image_to_repo_flag ]]; then PUSH_THE_IMAGE=1; fi 23 | - if [[ -f skip_build_package_check ]]; then export SKIP_CHECK=1; fi 24 | - rm -f push_image_to_repo_flag skip_build_package_check 25 | 26 | script: 27 | - ./build.sh 28 | 29 | after_success: 30 | - if [[ $PUSH_THE_IMAGE -eq 1 ]]; then images/push-image.sh $OS-$OS_VER; fi 31 | -------------------------------------------------------------------------------- /utils/check_license/check-headers.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2020, Intel Corporation 4 | 5 | # check-headers.sh - check copyright and license in source files 6 | 7 | SELF=$0 8 | 9 | function usage() { 10 | echo "Usage: $SELF [-h|-v|-a]" 11 | echo " -h, --help this help message" 12 | echo " -v, --verbose verbose mode" 13 | echo " -a, --all check all files (only modified files are checked by default)" 14 | } 15 | 16 | if [ "$#" -lt 2 ]; then 17 | usage >&2 18 | exit 2 19 | fi 20 | 21 | SOURCE_ROOT=$1 22 | shift 23 | LICENSE=$1 24 | shift 25 | 26 | PATTERN=`mktemp` 27 | TMP=`mktemp` 28 | TMP2=`mktemp` 29 | TEMPFILE=`mktemp` 30 | rm -f $PATTERN $TMP $TMP2 31 | 32 | if [ "$1" == "-h" -o "$1" == "--help" ]; then 33 | usage 34 | exit 0 35 | fi 36 | 37 | export GIT="git -C ${SOURCE_ROOT}" 38 | $GIT rev-parse || exit 1 39 | 40 | if [ -f $SOURCE_ROOT/.git/shallow ]; then 41 | SHALLOW_CLONE=1 42 | echo 43 | echo "Warning: This is a shallow clone. Checking dates in copyright headers" 44 | echo " will be skipped in case of files that have no history." 45 | echo 46 | else 47 | SHALLOW_CLONE=0 48 | fi 49 | 50 | VERBOSE=0 51 | CHECK_ALL=0 52 | while [ "$1" != "" ]; do 53 | case $1 in 54 | -v|--verbose) 55 | VERBOSE=1 56 | ;; 57 | -a|--all) 58 | CHECK_ALL=1 59 | ;; 60 | esac 61 | shift 62 | done 63 | 64 | if [ $CHECK_ALL -eq 0 ]; then 65 | CURRENT_COMMIT=$($GIT log --pretty=%H -1) 66 | MERGE_BASE=$($GIT merge-base HEAD origin/master 2>/dev/null) 67 | [ -z $MERGE_BASE ] && \ 68 | MERGE_BASE=$($GIT log --pretty="%cN:%H" | grep GitHub | head -n1 | cut -d: -f2) 69 | [ -z $MERGE_BASE -o "$CURRENT_COMMIT" = "$MERGE_BASE" ] && \ 70 | CHECK_ALL=1 71 | fi 72 | 73 | if [ $CHECK_ALL -eq 1 ]; then 74 | echo "Checking copyright headers of all files..." 75 | GIT_COMMAND="ls-tree -r --name-only HEAD" 76 | else 77 | if [ $VERBOSE -eq 1 ]; then 78 | echo 79 | echo "Warning: will check copyright headers of modified files only," 80 | echo " in order to check all files issue the following command:" 81 | echo " $ $SELF -a" 82 | echo " (e.g.: $ $SELF $SOURCE_ROOT $LICENSE -a)" 83 | echo 84 | fi 85 | echo "Checking copyright headers of modified files only..." 86 | GIT_COMMAND="diff --name-only $MERGE_BASE $CURRENT_COMMIT" 87 | fi 88 | 89 | FILES=$($GIT $GIT_COMMAND | ${SOURCE_ROOT}/utils/check_license/file-exceptions.sh | \ 90 | grep -E -e '*\.[chs]$' -e '*\.[ch]pp$' -e '*\.sh$' \ 91 | -e '*\.link$' -e 'Makefile*' -e 'TEST*' \ 92 | -e '/common.inc$' -e '/match$' -e '/check_whitespace$' \ 93 | -e 'LICENSE$' -e 'CMakeLists.txt$' -e '*\.cmake$' | \ 94 | xargs) 95 | 96 | RV=0 97 | for file in $FILES ; do 98 | # The src_path is a path which should be used in every command except git. 99 | # git is called with -C flag so filepaths should be relative to SOURCE_ROOT 100 | src_path="${SOURCE_ROOT}/$file" 101 | [ ! -f $src_path ] && continue 102 | # ensure that file is UTF-8 encoded 103 | ENCODING=`file -b --mime-encoding $src_path` 104 | iconv -f $ENCODING -t "UTF-8" $src_path > $TEMPFILE 105 | 106 | if ! grep -q "SPDX-License-Identifier: $LICENSE" $src_path; then 107 | echo >&2 "error: no $LICENSE SPDX tag in file: $src_path" 108 | RV=1 109 | fi 110 | 111 | if [ $SHALLOW_CLONE -eq 0 ]; then 112 | $GIT log --no-merges --format="%ai %aE" -- $file | sort > $TMP 113 | else 114 | # mark the grafted commits (commits with no parents) 115 | $GIT log --no-merges --format="%ai %aE grafted-%p-commit" -- $file | sort > $TMP 116 | fi 117 | 118 | # skip checking dates for non-Intel commits 119 | [[ ! $(tail -n1 $TMP) =~ "@intel.com" ]] && continue 120 | 121 | # skip checking dates for new files 122 | [ $(cat $TMP | wc -l) -le 1 ] && continue 123 | 124 | # grep out the grafted commits (commits with no parents) 125 | # and skip checking dates for non-Intel commits 126 | grep -v -e "grafted--commit" $TMP | grep -e "@intel.com" > $TMP2 127 | 128 | [ $(cat $TMP2 | wc -l) -eq 0 ] && continue 129 | 130 | FIRST=`head -n1 $TMP2` 131 | LAST=` tail -n1 $TMP2` 132 | 133 | YEARS=`sed ' 134 | /Copyright [0-9-]\+.*, Intel Corporation/!d 135 | s/.*Copyright \([0-9]\+\)-\([0-9]\+\),.*/\1-\2/ 136 | s/.*Copyright \([0-9]\+\),.*/\1-\1/' $src_path` 137 | if [ -z "$YEARS" ]; then 138 | echo >&2 "No copyright years in $src_path" 139 | RV=1 140 | continue 141 | fi 142 | 143 | HEADER_FIRST=`echo $YEARS | cut -d"-" -f1` 144 | HEADER_LAST=` echo $YEARS | cut -d"-" -f2` 145 | 146 | COMMIT_FIRST=`echo $FIRST | cut -d"-" -f1` 147 | COMMIT_LAST=` echo $LAST | cut -d"-" -f1` 148 | if [ "$COMMIT_FIRST" != "" -a "$COMMIT_LAST" != "" ]; then 149 | if [ $HEADER_LAST -lt $COMMIT_LAST ]; then 150 | if [ $HEADER_FIRST -lt $COMMIT_FIRST ]; then 151 | COMMIT_FIRST=$HEADER_FIRST 152 | fi 153 | COMMIT_LAST=`date +%G` 154 | if [ $COMMIT_FIRST -eq $COMMIT_LAST ]; then 155 | NEW=$COMMIT_LAST 156 | else 157 | NEW=$COMMIT_FIRST-$COMMIT_LAST 158 | fi 159 | echo "$file:1: error: wrong copyright date: (is: $YEARS, should be: $NEW)" >&2 160 | RV=1 161 | fi 162 | else 163 | echo "error: unknown commit dates in file: $file" >&2 164 | RV=1 165 | fi 166 | done 167 | rm -f $TMP $TMP2 $TEMPFILE 168 | 169 | # check if error found 170 | if [ $RV -eq 0 ]; then 171 | echo "Copyright headers are OK." 172 | else 173 | echo "Error(s) in copyright headers found!" >&2 174 | fi 175 | exit $RV 176 | -------------------------------------------------------------------------------- /utils/check_license/file-exceptions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh -e 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2019, Intel Corporation 4 | # 5 | 6 | # file-exceptions.sh - filter out files not checked for copyright and license 7 | 8 | grep -v -E -e '/queue.h$|fast-hash' 9 | -------------------------------------------------------------------------------- /utils/check_whitespace: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env perl 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2015-2018, Intel Corporation 4 | 5 | # 6 | # check_whitespace -- scrub source tree for whitespace errors 7 | # 8 | 9 | use strict; 10 | use warnings; 11 | 12 | use File::Basename; 13 | use File::Find; 14 | use Encode; 15 | use v5.16; 16 | 17 | my $Me = $0; 18 | $Me =~ s,.*/,,; 19 | 20 | $SIG{HUP} = $SIG{INT} = $SIG{TERM} = $SIG{__DIE__} = sub { 21 | die @_ if $^S; 22 | 23 | my $errstr = shift; 24 | 25 | die "$Me: ERROR: $errstr"; 26 | }; 27 | 28 | my $Errcount = 0; 29 | 30 | # 31 | # err -- emit error, keep total error count 32 | # 33 | sub err { 34 | warn @_, "\n"; 35 | $Errcount++; 36 | } 37 | 38 | # 39 | # decode_file_as_string -- slurp an entire file into memory and decode 40 | # 41 | sub decode_file_as_string { 42 | my ($full, $file) = @_; 43 | my $fh; 44 | open($fh, '<', $full) or die "$full $!\n"; 45 | 46 | local $/; 47 | $_ = <$fh>; 48 | close $fh; 49 | 50 | # check known encodings or die 51 | my $decoded; 52 | my @encodings = ("UTF-8", "UTF-16", "UTF-16LE", "UTF-16BE"); 53 | 54 | foreach my $enc (@encodings) { 55 | eval { $decoded = decode( $enc, $_, Encode::FB_CROAK ) }; 56 | 57 | if (!$@) { 58 | $decoded =~ s/\R/\n/g; 59 | return $decoded; 60 | } 61 | } 62 | 63 | die "$Me: ERROR: Unknown file encoding"; 64 | } 65 | 66 | # 67 | # check_whitespace -- run the checks on the given file 68 | # 69 | sub check_whitespace { 70 | my ($full, $file) = @_; 71 | 72 | my $line = 0; 73 | my $eol; 74 | my $nf = 0; 75 | my $fstr = decode_file_as_string($full, $file); 76 | 77 | for (split /^/, $fstr) { 78 | $line++; 79 | $eol = /[\n]/s; 80 | if (/^\.nf$/) { 81 | err("$full:$line: ERROR: nested .nf") if $nf; 82 | $nf = 1; 83 | } elsif (/^\.fi$/) { 84 | $nf = 0; 85 | } elsif ($nf == 0) { 86 | chomp; 87 | err("$full:$line: ERROR: trailing whitespace") if /\s$/; 88 | err("$full:$line: ERROR: spaces before tabs") if / \t/; 89 | } 90 | } 91 | 92 | err("$full:$line: .nf without .fi") if $nf; 93 | err("$full:$line: noeol") unless $eol; 94 | } 95 | 96 | sub check_whitespace_with_exc { 97 | my ($full) = @_; 98 | 99 | $_ = $full; 100 | 101 | return 0 if /^[.\/]*src\/jemalloc.*/; 102 | return 0 if /^[.\/]*src\/sys\/queue\.h/; 103 | return 0 if /^[.\/]*src\/common\/valgrind\/.*\.h/; 104 | 105 | $_ = basename($full); 106 | 107 | return 0 unless /^(README.*|LICENSE.*|Makefile.*|CMakeLists.txt|.gitignore|TEST.*|RUNTESTS|check_whitespace|.*\.([chp13s]|sh|map|cpp|hpp|inc|PS1|ps1|py|md|cmake))$/; 108 | return 0 if -z; 109 | 110 | check_whitespace($full, $_); 111 | return 1; 112 | } 113 | 114 | my $verbose = 0; 115 | my $force = 0; 116 | my $recursive = 0; 117 | 118 | sub check { 119 | my ($file) = @_; 120 | my $r; 121 | 122 | if ($force) { 123 | $r = check_whitespace($file, basename($file)); 124 | } else { 125 | $r = check_whitespace_with_exc($file); 126 | } 127 | 128 | if ($verbose) { 129 | if ($r == 0) { 130 | printf("skipped $file\n"); 131 | } else { 132 | printf("checked $file\n"); 133 | } 134 | } 135 | } 136 | 137 | my @files = (); 138 | 139 | foreach my $arg (@ARGV) { 140 | if ($arg eq '-v') { 141 | $verbose = 1; 142 | next; 143 | } 144 | if ($arg eq '-f') { 145 | $force = 1; 146 | next; 147 | } 148 | if ($arg eq '-r') { 149 | $recursive = 1; 150 | next; 151 | } 152 | if ($arg eq '-g') { 153 | @files = `git ls-tree -r --name-only HEAD`; 154 | chomp(@files); 155 | next; 156 | } 157 | if ($arg eq '-h') { 158 | printf "Options: 159 | -g - check all files tracked by git 160 | -r dir - recursively check all files in specified directory 161 | -v verbose - print whether file was checked or not 162 | -f force - disable blacklist\n"; 163 | exit 1; 164 | } 165 | 166 | if ($recursive == 1) { 167 | find(sub { 168 | my $full = $File::Find::name; 169 | 170 | if (!$force && 171 | ($full eq './.git' || 172 | $full eq './src/jemalloc' || 173 | $full eq './src/debug' || 174 | $full eq './src/nondebug' || 175 | $full eq './rpmbuild' || 176 | $full eq './dpkgbuild')) { 177 | $File::Find::prune = 1; 178 | return; 179 | } 180 | 181 | return unless -f; 182 | 183 | push @files, $full; 184 | }, $arg); 185 | 186 | $recursive = 0; 187 | next; 188 | } 189 | 190 | push @files, $arg; 191 | } 192 | 193 | if (!@files) { 194 | printf "Empty file list!\n"; 195 | } 196 | 197 | foreach (@files) { 198 | check($_); 199 | } 200 | 201 | exit $Errcount; 202 | -------------------------------------------------------------------------------- /utils/docker/build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2017-2018, Intel Corporation 4 | 5 | # 6 | # build-local.sh - runs a Docker container from a Docker image with environment 7 | # prepared for running tests. 8 | # 9 | # 10 | # Notes: 11 | # - run this script from its location or set the variable 'HOST_WORKDIR' to 12 | # where the root of this project is on the host machine, 13 | # - set variables 'OS' and 'OS_VER' properly to a system you want to build this 14 | # repo on (for proper values take a look on the list of Dockerfiles at the 15 | # utils/docker/images directory), eg. OS=ubuntu, OS_VER=16.04. 16 | # 17 | 18 | set -e 19 | 20 | if [[ -z "$OS" || -z "$OS_VER" ]]; then 21 | echo "ERROR: The variables OS and OS_VER have to be set " \ 22 | "(eg. OS=ubuntu, OS_VER=16.04)." 23 | exit 1 24 | fi 25 | 26 | if [[ -z "$HOST_WORKDIR" ]]; then 27 | HOST_WORKDIR=$(readlink -f ../..) 28 | fi 29 | 30 | chmod -R a+w $HOST_WORKDIR 31 | 32 | imageName=${DOCKERHUB_REPO}:${OS}-${OS_VER} 33 | containerName=vmemcache-${OS}-${OS_VER} 34 | 35 | if [[ "$command" == "" ]]; then 36 | command="./run-build.sh"; 37 | fi 38 | 39 | if [ -n "$DNS_SERVER" ]; then DNS_SETTING=" --dns=$DNS_SERVER "; fi 40 | 41 | WORKDIR=/vmemcache 42 | SCRIPTSDIR=$WORKDIR/utils/docker 43 | 44 | echo Building ${OS}-${OS_VER} 45 | 46 | ci_env=`bash <(curl -s https://codecov.io/env)` 47 | # Run a container with 48 | # - environment variables set (--env) 49 | # - host directory containing source mounted (-v) 50 | # - working directory set (-w) 51 | docker run --privileged=true --name=$containerName -ti \ 52 | $DNS_SETTING \ 53 | ${docker_opts} \ 54 | $ci_env \ 55 | --env http_proxy=$http_proxy \ 56 | --env https_proxy=$https_proxy \ 57 | --env GITHUB_TOKEN=$GITHUB_TOKEN \ 58 | --env WORKDIR=$WORKDIR \ 59 | --env SCRIPTSDIR=$SCRIPTSDIR \ 60 | --env COVERAGE=$COVERAGE \ 61 | --env TRAVIS_REPO_SLUG=$TRAVIS_REPO_SLUG \ 62 | --env TRAVIS_BRANCH=$TRAVIS_BRANCH \ 63 | --env TRAVIS_EVENT_TYPE=$TRAVIS_EVENT_TYPE \ 64 | -v $HOST_WORKDIR:$WORKDIR \ 65 | -v /etc/localtime:/etc/localtime \ 66 | -w $SCRIPTSDIR \ 67 | $imageName $command 68 | -------------------------------------------------------------------------------- /utils/docker/images/Dockerfile.fedora-28: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2016-2020, Intel Corporation 3 | 4 | # 5 | # Dockerfile - a 'recipe' for Docker to build an image of fedora-based 6 | # environment prepared for running libvmemcache tests. 7 | # 8 | 9 | # Pull base image 10 | FROM fedora:28 11 | MAINTAINER piotr.balcer@intel.com 12 | 13 | # Install basic tools 14 | RUN dnf update -y \ 15 | && dnf install -y \ 16 | clang \ 17 | cmake \ 18 | gcc \ 19 | git \ 20 | hub \ 21 | libunwind-devel \ 22 | make \ 23 | man \ 24 | pandoc \ 25 | passwd \ 26 | rpm-build \ 27 | sudo \ 28 | tar \ 29 | wget \ 30 | which \ 31 | valgrind \ 32 | valgrind-devel \ 33 | && dnf clean all 34 | 35 | # Add user 36 | ENV USER user 37 | ENV USERPASS pass 38 | RUN useradd -m $USER 39 | RUN echo $USERPASS | passwd $USER --stdin 40 | RUN gpasswd wheel -a $USER 41 | USER $USER 42 | 43 | # Set required environment variables 44 | ENV OS fedora 45 | ENV OS_VER 28 46 | ENV PACKAGE_MANAGER rpm 47 | ENV NOTTY 1 48 | -------------------------------------------------------------------------------- /utils/docker/images/Dockerfile.ubuntu-18.04: -------------------------------------------------------------------------------- 1 | # SPDX-License-Identifier: BSD-3-Clause 2 | # Copyright 2016-2020, Intel Corporation 3 | 4 | # 5 | # Dockerfile - a 'recipe' for Docker to build an image of ubuntu-based 6 | # environment prepared for running libvmemcache tests. 7 | # 8 | 9 | # Pull base image 10 | FROM ubuntu:18.04 11 | MAINTAINER piotr.balcer@intel.com 12 | 13 | # Update the Apt cache and install basic tools 14 | RUN apt-get update \ 15 | && apt-get install -y software-properties-common \ 16 | clang \ 17 | cmake \ 18 | curl \ 19 | debhelper \ 20 | devscripts \ 21 | gcc \ 22 | gdb \ 23 | git \ 24 | libunwind8-dev \ 25 | pandoc \ 26 | pkg-config \ 27 | sudo \ 28 | wget \ 29 | whois \ 30 | valgrind \ 31 | && rm -rf /var/lib/apt/lists/* 32 | 33 | # Add user 34 | ENV USER user 35 | ENV USERPASS pass 36 | RUN useradd -m $USER -g sudo -p `mkpasswd $USERPASS` 37 | USER $USER 38 | 39 | # Set required environment variables 40 | ENV OS ubuntu 41 | ENV OS_VER 18.04 42 | ENV PACKAGE_MANAGER deb 43 | ENV NOTTY 1 44 | -------------------------------------------------------------------------------- /utils/docker/images/build-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2018, Intel Corporation 4 | 5 | # 6 | # build-image.sh - prepares a Docker image with -based 7 | # environment for testing libvmemcache, according 8 | # to the Dockerfile. file located 9 | # in the same directory. 10 | # 11 | # The script can be run locally. 12 | # 13 | 14 | set -e 15 | 16 | function usage { 17 | echo "Usage:" 18 | echo " build-image.sh " 19 | echo "where , for example, can be 'ubuntu-16.04', provided " \ 20 | "a Dockerfile named 'Dockerfile.ubuntu-16.04' exists in the " \ 21 | "current directory." 22 | } 23 | 24 | # Check if the first and second argument is nonempty 25 | if [[ -z "$1" || -z "$2" ]]; then 26 | usage 27 | exit 1 28 | fi 29 | 30 | # Check if the file Dockerfile.OS-VER exists 31 | if [[ ! -f "Dockerfile.$2" ]]; then 32 | echo "ERROR: wrong argument." 33 | usage 34 | exit 1 35 | fi 36 | 37 | # Build a Docker image tagged with ${DOCKERHUB_REPO}:OS-VER 38 | docker build -t $1:$2 \ 39 | --build-arg http_proxy=$http_proxy \ 40 | --build-arg https_proxy=$https_proxy \ 41 | -f Dockerfile.$2 . 42 | -------------------------------------------------------------------------------- /utils/docker/images/push-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2018, Intel Corporation 4 | 5 | # 6 | # push-image.sh - pushes the Docker image tagged with OS-VER 7 | # to the Docker Hub. 8 | # 9 | # The script utilizes $DOCKERHUB_USER and $DOCKERHUB_PASSWORD variables to log in to 10 | # Docker Hub. The variables can be set in the Travis project's configuration 11 | # for automated builds. 12 | # 13 | 14 | set -e 15 | 16 | function usage { 17 | echo "Usage:" 18 | echo " push-image.sh " 19 | echo "where , for example, can be 'ubuntu-16.04', provided " \ 20 | "a Docker image tagged with ${DOCKERHUB_REPO}:ubuntu-16.04 exists " \ 21 | "locally." 22 | } 23 | 24 | # Check if the first argument is nonempty 25 | if [[ -z "$1" ]]; then 26 | usage 27 | exit 1 28 | fi 29 | 30 | # Check if the image tagged with ${DOCKERHUB_REPO}:OS-VER exists locally 31 | if [[ ! $(docker images -a | awk -v pattern="^${DOCKERHUB_REPO}:$1\$" \ 32 | '$1":"$2 ~ pattern') ]] 33 | then 34 | echo "ERROR: wrong argument." 35 | usage 36 | exit 1 37 | fi 38 | 39 | # Log in to the Docker Hub 40 | docker login -u="$DOCKERHUB_USER" -p="$DOCKERHUB_PASSWORD" 41 | 42 | # Push the image to the repository 43 | docker push ${DOCKERHUB_REPO}:$1 44 | -------------------------------------------------------------------------------- /utils/docker/pull-or-rebuild-image.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2018, Intel Corporation 4 | 5 | # 6 | # pull-or-rebuild-image.sh - rebuilds the Docker image used in the 7 | # current Travis build if necessary. 8 | # 9 | # The script rebuilds the Docker image if the Dockerfile for the current 10 | # OS version (Dockerfile.${OS}-${OS_VER}) or any .sh script from the directory 11 | # with Dockerfiles were modified and committed. 12 | # 13 | # If the Travis build is not of the "pull_request" type (i.e. in case of 14 | # merge after pull_request) and it succeed, the Docker image should be pushed 15 | # to the Docker Hub repository. An empty file is created to signal that to 16 | # further scripts. 17 | # 18 | # If the Docker image does not have to be rebuilt, it will be pulled from 19 | # Docker Hub. 20 | # 21 | 22 | set -e 23 | 24 | if [[ "$TRAVIS_EVENT_TYPE" != "cron" && "$TRAVIS_BRANCH" != "coverity_scan" \ 25 | && "$COVERITY" -eq 1 ]]; then 26 | echo "INFO: Skip Coverity scan job if build is triggered neither by " \ 27 | "'cron' nor by a push to 'coverity_scan' branch" 28 | exit 0 29 | fi 30 | 31 | if [[ ( "$TRAVIS_EVENT_TYPE" == "cron" || "$TRAVIS_BRANCH" == "coverity_scan" )\ 32 | && "$COVERITY" -ne 1 ]]; then 33 | echo "INFO: Skip regular jobs if build is triggered either by 'cron'" \ 34 | " or by a push to 'coverity_scan' branch" 35 | exit 0 36 | fi 37 | 38 | if [[ -z "$OS" || -z "$OS_VER" ]]; then 39 | echo "ERROR: The variables OS and OS_VER have to be set properly " \ 40 | "(eg. OS=ubuntu, OS_VER=16.04)." 41 | exit 1 42 | fi 43 | 44 | if [[ -z "$HOST_WORKDIR" ]]; then 45 | echo "ERROR: The variable HOST_WORKDIR has to contain a path to " \ 46 | "the root of this project on the host machine" 47 | exit 1 48 | fi 49 | 50 | # TRAVIS_COMMIT_RANGE is usually invalid for force pushes - ignore such values 51 | # when used with non-upstream repository 52 | if [ -n "$TRAVIS_COMMIT_RANGE" -a $TRAVIS_REPO_SLUG != "${GITHUB_REPO}" ]; then 53 | if ! git rev-list $TRAVIS_COMMIT_RANGE; then 54 | TRAVIS_COMMIT_RANGE= 55 | fi 56 | fi 57 | 58 | # Find all the commits for the current build 59 | if [[ -n "$TRAVIS_COMMIT_RANGE" ]]; then 60 | commits=$(git rev-list $TRAVIS_COMMIT_RANGE) 61 | else 62 | commits=$TRAVIS_COMMIT 63 | fi 64 | echo "Commits in the commit range:" 65 | for commit in $commits; do echo $commit; done 66 | 67 | # Get the list of files modified by the commits 68 | files=$(for commit in $commits; do git diff-tree --no-commit-id --name-only \ 69 | -r $commit; done | sort -u) 70 | echo "Files modified within the commit range:" 71 | for file in $files; do echo $file; done 72 | 73 | # Path to directory with Dockerfiles and image building scripts 74 | images_dir_name=images 75 | base_dir=utils/docker/$images_dir_name 76 | 77 | # Check if committed file modifications require the Docker image to be rebuilt 78 | for file in $files; do 79 | # Check if modified files are relevant to the current build 80 | if [[ $file =~ ^($base_dir)\/Dockerfile\.($OS)-($OS_VER)$ ]] \ 81 | || [[ $file =~ ^($base_dir)\/.*\.sh$ ]] 82 | then 83 | # Rebuild Docker image for the current OS version 84 | echo "Rebuilding the Docker image for the Dockerfile.$OS-$OS_VER" 85 | pushd $images_dir_name 86 | ./build-image.sh ${DOCKERHUB_REPO} ${OS}-${OS_VER} 87 | popd 88 | 89 | # Check if the image has to be pushed to Docker Hub 90 | # (i.e. the build is triggered by commits to the ${GITHUB_REPO} 91 | # repository's master branch, and the Travis build is not 92 | # of the "pull_request" type). In that case, create the empty 93 | # file. 94 | if [[ $TRAVIS_REPO_SLUG == "${GITHUB_REPO}" \ 95 | && $TRAVIS_BRANCH == "master" \ 96 | && $TRAVIS_EVENT_TYPE != "pull_request" 97 | && $PUSH_IMAGE == "1" ]] 98 | then 99 | echo "The image will be pushed to Docker Hub" 100 | touch push_image_to_repo_flag 101 | else 102 | echo "Skip pushing the image to Docker Hub" 103 | fi 104 | 105 | if [[ $PUSH_IMAGE == "1" ]] 106 | then 107 | echo "Skip build package check if image has to be pushed" 108 | touch skip_build_package_check 109 | fi 110 | exit 0 111 | fi 112 | done 113 | 114 | # Getting here means rebuilding the Docker image is not required. 115 | # Pull the image from Docker Hub. 116 | docker pull ${DOCKERHUB_REPO}:${OS}-${OS_VER} 117 | -------------------------------------------------------------------------------- /utils/docker/run-build.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2019, Intel Corporation 4 | 5 | # 6 | # run-build.sh - is called inside a Docker container; prepares the environment 7 | # and starts a build of libvmemcache. 8 | # 9 | 10 | set -e 11 | 12 | [ "$WORKDIR" != "" ] && cd $WORKDIR 13 | INSTALL_DIR=/tmp/vmemcache 14 | 15 | mkdir -p $INSTALL_DIR 16 | 17 | # ----------------------------------------- 18 | # gcc & Debug 19 | 20 | echo 21 | echo " ##########################################" 22 | echo " # Running the configuration: gcc & Debug #" 23 | echo " ##########################################" 24 | echo 25 | 26 | mkdir -p build 27 | cd build 28 | 29 | CC=gcc \ 30 | cmake .. -DCMAKE_BUILD_TYPE=Debug \ 31 | -DDEVELOPER_MODE=1 \ 32 | -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ 33 | -DTRACE_TESTS=1 34 | 35 | make -j2 36 | ctest --output-on-failure 37 | 38 | make install 39 | make uninstall 40 | 41 | cd .. 42 | rm -r build 43 | 44 | # ----------------------------------------- 45 | # gcc & Release 46 | 47 | echo 48 | echo " ############################################" 49 | echo " # Running the configuration: gcc & Release #" 50 | echo " ############################################" 51 | echo 52 | 53 | mkdir build 54 | cd build 55 | 56 | CC=gcc \ 57 | cmake .. -DCMAKE_BUILD_TYPE=Release \ 58 | -DDEVELOPER_MODE=1 \ 59 | -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ 60 | -DTRACE_TESTS=1 61 | 62 | make -j2 63 | ctest --output-on-failure 64 | 65 | cd .. 66 | rm -r build 67 | 68 | # ----------------------------------------- 69 | # Clang & Debug 70 | 71 | echo 72 | echo " ############################################" 73 | echo " # Running the configuration: Clang & Debug #" 74 | echo " ############################################" 75 | echo 76 | 77 | mkdir build 78 | cd build 79 | 80 | CC=clang \ 81 | cmake .. -DCMAKE_BUILD_TYPE=Debug \ 82 | -DDEVELOPER_MODE=1 \ 83 | -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ 84 | -DTRACE_TESTS=1 85 | 86 | make -j2 87 | ctest --output-on-failure 88 | 89 | cd .. 90 | rm -r build 91 | 92 | # ----------------------------------------- 93 | # Clang & Release 94 | 95 | echo 96 | echo " ##############################################" 97 | echo " # Running the configuration: Clang & Release #" 98 | echo " ##############################################" 99 | echo 100 | 101 | mkdir build 102 | cd build 103 | 104 | CC=clang \ 105 | cmake .. -DCMAKE_BUILD_TYPE=Release \ 106 | -DDEVELOPER_MODE=1 \ 107 | -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR \ 108 | -DTRACE_TESTS=1 109 | 110 | make -j2 111 | ctest --output-on-failure 112 | 113 | cd .. 114 | rm -r build 115 | 116 | # ----------------------------------------- 117 | # deb & rpm 118 | 119 | echo 120 | echo " ##########################################" 121 | echo " # Running the configuration: deb & rpm #" 122 | echo " ##########################################" 123 | echo 124 | 125 | mkdir -p build 126 | cd build 127 | 128 | CC=gcc \ 129 | cmake .. -DCMAKE_BUILD_TYPE=Release \ 130 | -DCPACK_GENERATOR=$PACKAGE_MANAGER \ 131 | -DCMAKE_INSTALL_PREFIX=/usr \ 132 | -DTRACE_TESTS=1 133 | 134 | make -j2 135 | ctest --output-on-failure 136 | 137 | make package 138 | 139 | find . -iname "libvmemcache*.$PACKAGE_MANAGER" 140 | 141 | if [ $PACKAGE_MANAGER = "deb" ]; then 142 | echo "$ dpkg-deb --info ./libvmemcache*.deb" 143 | dpkg-deb --info ./libvmemcache*.deb 144 | 145 | echo "$ dpkg-deb -c ./libvmemcache*.deb" 146 | dpkg-deb -c ./libvmemcache*.deb 147 | 148 | echo "$ sudo -S dpkg -i ./libvmemcache*.deb" 149 | echo $USERPASS | sudo -S dpkg -i ./libvmemcache*.deb 150 | 151 | elif [ $PACKAGE_MANAGER = "rpm" ]; then 152 | echo "$ rpm -q --info ./libvmemcache*.rpm" 153 | rpm -q --info ./libvmemcache*.rpm && true 154 | 155 | echo "$ rpm -q --list ./libvmemcache*.rpm" 156 | rpm -q --list ./libvmemcache*.rpm && true 157 | 158 | echo "$ sudo -S rpm -ivh --force *.rpm" 159 | echo $USERPASS | sudo -S rpm -ivh --force *.rpm 160 | fi 161 | 162 | cd .. 163 | rm -rf build 164 | 165 | # ----------------------------------------- 166 | # Coverage 167 | if [[ $COVERAGE -eq 1 ]] ; then 168 | echo 169 | echo " #######################################" 170 | echo " # Running the configuration: Coverage #" 171 | echo " #######################################" 172 | echo 173 | 174 | mkdir build 175 | cd build 176 | 177 | CC=gcc \ 178 | cmake .. -DCMAKE_BUILD_TYPE=Debug \ 179 | -DTRACE_TESTS=1 \ 180 | -DCOVERAGE_BUILD=1 \ 181 | -DCMAKE_C_FLAGS=-coverage 182 | 183 | make -j2 184 | ctest --output-on-failure 185 | bash <(curl -s https://codecov.io/bash) -c 186 | 187 | cd .. 188 | 189 | rm -r build 190 | fi 191 | -------------------------------------------------------------------------------- /utils/md2man/default.man: -------------------------------------------------------------------------------- 1 | $if(has-tables)$ 2 | .\"t 3 | $endif$ 4 | $if(pandoc-version)$ 5 | .\" Automatically generated by Pandoc $pandoc-version$ 6 | .\" 7 | $endif$ 8 | $if(adjusting)$ 9 | .ad $adjusting$ 10 | $endif$ 11 | .TH "$title$" "$section$" "$date$" "PMDK - $version$" "PMDK Programmer's Manual" 12 | $if(hyphenate)$ 13 | .hy 14 | $else$ 15 | .nh \" Turn off hyphenation by default. 16 | $endif$ 17 | $for(header-includes)$ 18 | $header-includes$ 19 | $endfor$ 20 | .\" Copyright 2018-$year$, Intel Corporation 21 | .\" 22 | .\" Redistribution and use in source and binary forms, with or without 23 | .\" modification, are permitted provided that the following conditions 24 | .\" are met: 25 | .\" 26 | .\" * Redistributions of source code must retain the above copyright 27 | .\" notice, this list of conditions and the following disclaimer. 28 | .\" 29 | .\" * Redistributions in binary form must reproduce the above copyright 30 | .\" notice, this list of conditions and the following disclaimer in 31 | .\" the documentation and/or other materials provided with the 32 | .\" distribution. 33 | .\" 34 | .\" * Neither the name of the copyright holder nor the names of its 35 | .\" contributors may be used to endorse or promote products derived 36 | .\" from this software without specific prior written permission. 37 | .\" 38 | .\" THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 39 | .\" "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 40 | .\" LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 41 | .\" A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 42 | .\" OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 43 | .\" SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 44 | .\" LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 45 | .\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 46 | .\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 47 | .\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 48 | .\" OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 49 | $for(include-before)$ 50 | $include-before$ 51 | $endfor$ 52 | $body$ 53 | $for(include-after)$ 54 | $include-after$ 55 | $endfor$ 56 | $if(author)$ 57 | .SH AUTHORS 58 | $for(author)$$author$$sep$; $endfor$. 59 | $endif$ 60 | -------------------------------------------------------------------------------- /utils/md2man/md2man.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # SPDX-License-Identifier: BSD-3-Clause 3 | # Copyright 2016-2019, Intel Corporation 4 | # 5 | 6 | # 7 | # md2man.sh -- convert markdown to groff man pages 8 | # 9 | # usage: md2man.sh file template outfile 10 | # 11 | # This script converts markdown file into groff man page using pandoc. 12 | # It performs some pre- and post-processing for better results: 13 | # - parse input file for YAML metadata block and read man page title, 14 | # section and version 15 | # - cut-off metadata block and license 16 | # - unindent code blocks 17 | # 18 | 19 | set -e 20 | set -o pipefail 21 | 22 | filename=$1 23 | template=$2 24 | outfile=$3 25 | title=`sed -n 's/^title:\ _MP(*\([A-Za-z_-]*\).*$/\1/p' $filename` 26 | section=`sed -n 's/^title:.*\([0-9]\))$/\1/p' $filename` 27 | version=`sed -n 's/^date:\ *\(.*\)$/\1/p' $filename` 28 | 29 | dt=$(date +"%F") 30 | SOURCE_DATE_EPOCH="${SOURCE_DATE_EPOCH:-$(date +%s)}" 31 | YEAR=$(date -u -d "@$SOURCE_DATE_EPOCH" +%Y 2>/dev/null || 32 | date -u -r "$SOURCE_DATE_EPOCH" +%Y 2>/dev/null || date -u +%Y) 33 | dt=$(date -u -d "@$SOURCE_DATE_EPOCH" +%F 2>/dev/null || 34 | date -u -r "$SOURCE_DATE_EPOCH" +%F 2>/dev/null || date -u +%F) 35 | cat $filename | sed -n -e '/# NAME #/,$p' |\ 36 | pandoc -s -t man -o $outfile.tmp --template=$template \ 37 | -V title=$title -V section=$section \ 38 | -V date="$dt" -V version="$version" \ 39 | -V year="$YEAR" | 40 | sed '/^\.IP/{ 41 | N 42 | /\n\.nf/{ 43 | s/IP/PP/ 44 | } 45 | }' 46 | 47 | # don't overwrite the output file if the only thing that changed 48 | # is modification date (diff output has exactly 4 lines in this case) 49 | if [ -e $outfile ] 50 | then 51 | difflines=`diff $outfile $outfile.tmp | wc -l || true >2` 52 | onlydates=`diff $outfile $outfile.tmp | grep "$dt" | wc -l || true` 53 | if [ $difflines -eq 4 -a $onlydates -eq 1 ]; then 54 | rm $outfile.tmp 55 | else 56 | mv $outfile.tmp $outfile 57 | fi 58 | else 59 | mv $outfile.tmp $outfile 60 | fi 61 | --------------------------------------------------------------------------------