├── .gitignore ├── .travis.yml ├── COVERAGE.md ├── ChangeLog ├── LICENSE ├── Makefile.am ├── NOTICE ├── README.md ├── build.sh ├── configure.ac ├── m4 └── .gitignore ├── notes ├── TODO ├── coding_style.txt ├── eviction_strategies.md ├── memory_overhead.md ├── random_eviction.md └── slab_allocation.md ├── scripts ├── README.twctop.md ├── klog │ ├── klogParser │ │ ├── __init__.py │ │ └── klogFormat.py │ └── summary.py ├── twctop.rb └── twctop26.rb ├── src ├── Makefile.am ├── mc.c ├── mc_ascii.c ├── mc_ascii.h ├── mc_assoc.c ├── mc_assoc.h ├── mc_cache.c ├── mc_cache.h ├── mc_connection.c ├── mc_connection.h ├── mc_core.c ├── mc_core.h ├── mc_hash.c ├── mc_hash.h ├── mc_hotkey.c ├── mc_hotkey.h ├── mc_items.c ├── mc_items.h ├── mc_kc_map.c ├── mc_kc_map.h ├── mc_key_window.c ├── mc_key_window.h ├── mc_klog.c ├── mc_klog.h ├── mc_log.c ├── mc_log.h ├── mc_queue.h ├── mc_ring_array.c ├── mc_ring_array.h ├── mc_signal.c ├── mc_signal.h ├── mc_slabs.c ├── mc_slabs.h ├── mc_stats.c ├── mc_stats.h ├── mc_thread.c ├── mc_thread.h ├── mc_time.c ├── mc_time.h ├── mc_util.c └── mc_util.h └── tests ├── README.md ├── config ├── __init__.py ├── data │ ├── __init__.py │ └── default-template.py ├── defaults-template.py └── server │ ├── __init__.py │ ├── default-template.py │ └── mem4mb.py ├── data ├── __init__.py ├── data-template.default ├── data.high └── data.low ├── functional ├── 64bit.py ├── __init__.py ├── advanced.py ├── basic.py ├── expiry.py ├── startup.py └── stats.py ├── lib ├── __init__.py ├── common.py ├── logging.py ├── memcache.py └── utilities.py ├── performance ├── clients.py ├── monitor.py ├── setter.py └── shifter.py ├── protocol ├── __init__.py ├── badbasic.py └── badstartup.py ├── pytest.sh └── testrunner.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.lo 3 | *.o 4 | 5 | # Compiled Dynamic libraries 6 | *.so 7 | *.dylib 8 | 9 | # Compiled Static libraries 10 | *.la 11 | *.a 12 | 13 | # Compiled misc 14 | *.dep 15 | *.gcda 16 | *.gcno 17 | *.gcov 18 | 19 | # Packages 20 | *.tar.gz 21 | *.tar.bz2 22 | 23 | # Logs 24 | *.log 25 | 26 | # Temporary 27 | *.swp 28 | *.~ 29 | *.project 30 | *.cproject 31 | 32 | # Core and executable 33 | core* 34 | twemcache 35 | 36 | # Autotools 37 | .deps 38 | .libs 39 | 40 | /aclocal.m4 41 | /autom4te.cache 42 | /stamp-h1 43 | /autoscan.log 44 | /libtool 45 | 46 | /config/compile 47 | /config/config.guess 48 | /config/config.sub 49 | /config/depcomp 50 | /config/install-sh 51 | /config/ltmain.sh 52 | /config/missing 53 | /config 54 | 55 | /config.h 56 | /config.h.in 57 | /config.h.in~ 58 | /config.log 59 | /config.status 60 | /configure.scan 61 | /configure 62 | 63 | Makefile 64 | Makefile.in 65 | 66 | # Tests 67 | *.pyc 68 | tests/config/server/default.py 69 | tests/config/data/default.py 70 | tests/config/defaults.py 71 | tests/data/data.default 72 | tests/data/data.config 73 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | script: CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=full && make && sudo make install 3 | before_install: 4 | - sudo apt-get install libevent-dev 5 | -------------------------------------------------------------------------------- /COVERAGE.md: -------------------------------------------------------------------------------- 1 | # Twemcache Test Coverage 2 | 3 | To generate test coverage using gcov & lcov. 4 | 5 | ## Requirement 6 | Have both gcov and lcov installed. 7 | 8 | ## Build 9 | 10 | Configure Twemcache with no optimization and with gcov enabled 11 | ```sh 12 | CFLAGS="-ggdb3 -O0" ./configure --enable-debug=full --enable-gcov 13 | ``` 14 | 15 | ## Collect data (running at project root level) 16 | ```sh 17 | lcov -c --no-external --rc lcov_branch_coverage=1 -i -b src/ -d src/ -o twemcache_base.info 18 | make test 19 | lcov -c --no-external --rc lcov_branch_coverage=1 -b src/ -d src/ -o twemcache_test.info 20 | lcov -a twemcache_base.info -a twemcache_test.info --rc lcov_branch_coverage=1 -o twemcache_total.info 21 | genhtml --ignore-errors source twemcache_total.info --legend --output-directory=/tmp/twemcache 22 | ``` 23 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2003, Danga Interactive, Inc. 2 | Copyright (c) 2012, Twitter, Inc. 3 | All rights reserved. 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions 7 | are met: 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of the Twitter nor the names of its contributors 14 | may be used to endorse or promote products derived from this software 15 | without specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 | ARE DISCLAIMED. IN NO EVENT SHALL BE LIABLE FOR ANY 21 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 22 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 23 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 24 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 | THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure config.h.in config.h.in~ stamp-h.in 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | SUBDIRS = src 6 | 7 | EXTRA_DIST = README.md NOTICE LICENSE ChangeLog scripts notes 8 | 9 | test: 10 | env PYTHONPATH=$(pwd)/tests\:$(PYTHONPATH) ${SHELL} tests/pytest.sh 11 | 12 | clean-local: 13 | rm -f tests/config/defaults.py 14 | rm -f tests/config/server/default.py 15 | rm -f tests/config/data/default.py 16 | rm -f tests/data/data.default 17 | 18 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | /* 2 | * memcached - memory caching daemon 3 | * 4 | * http://www.danga.com/memcached/ 5 | * 6 | * Copyright 2003 Danga Interactive, Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Anatoly Vorobey 13 | * Brad Fitzpatrick 14 | */ 15 | 16 | /* 17 | * Hash table 18 | * 19 | * The hash function used here is by Bob Jenkins, 1996: 20 | * 21 | * "By Bob Jenkins, 1996. bob_jenkins@burtleburtle.net. 22 | * You may use this code any way you wish, private, educational, 23 | * or commercial. It's free." 24 | * 25 | * The rest of the file is licensed under the BSD license. See LICENSE. 26 | */ 27 | 28 | /* mc_queue.h 29 | * 30 | * Copyright (c) 1991, 1993 31 | * The Regents of the University of California. All rights reserved. 32 | * 33 | * Redistribution and use in source and binary forms, with or without 34 | * modification, are permitted provided that the following conditions 35 | * are met: 36 | * 1. Redistributions of source code must retain the above copyright 37 | * notice, this list of conditions and the following disclaimer. 38 | * 2. Redistributions in binary form must reproduce the above copyright 39 | * notice, this list of conditions and the following disclaimer in the 40 | * documentation and/or other materials provided with the distribution. 41 | * 4. Neither the name of the University nor the names of its contributors 42 | * may be used to endorse or promote products derived from this software 43 | * without specific prior written permission. 44 | * 45 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 46 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 49 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 | * SUCH DAMAGE. 56 | * 57 | * @(#)queue.h 8.5 (Berkeley) 8/20/94 58 | * $FreeBSD: src/sys/sys/queue.h,v 1.73 2010/02/20 01:05:30 emaste Exp $ 59 | */ 60 | 61 | /* tests/lib/memcache.py (python-memcached) 62 | * 63 | * Use and distribution licensed under the Python license. Full text available 64 | * at http://docs.python.org/license.html 65 | */ 66 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export CI_ROOT=$(pwd) 3 | rm -rf musl && mkdir musl 4 | rm -rf libevent && mkdir libevent 5 | wget https://artifactory.twitter.biz/libs-releases-local/musl-1.1.16.tar.gz 6 | packer fetch --use-tfe --cluster=smf1 cache-user libevent latest 7 | tar -xzf musl-1.1.16.tar.gz -C musl --strip-components=1 8 | cd musl 9 | ./configure --prefix=$CI_ROOT/musl/tmp --syslibdir=$CI_ROOT/musl/tmp/lib 10 | make 11 | make install 12 | export PATH=$PATH:$CI_ROOT/musl/tmp/bin 13 | cd $CI_ROOT 14 | tar -xzf libevent-2.0.22-stable.tar.gz -C libevent --strip-components=1 15 | cd libevent 16 | mkdir tmp 17 | ./configure --prefix=$CI_ROOT/libevent/tmp CC=musl-gcc --enable-static --disable-shared 18 | make 19 | make install 20 | cd $CI_ROOT 21 | autoreconf -fvi 22 | CFLAGS="-ggdb3 -O2" ./configure --enable-debug=log --with-libevent=$CI_ROOT/libevent/tmp CC=musl-gcc --enable-static 23 | make -j 24 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # Define the package version numbers and the bug reporting address 2 | m4_define([MC_MAJOR], 2) 3 | m4_define([MC_MINOR], 6) 4 | m4_define([MC_PATCH], 2) 5 | m4_define([MC_BUGS], [cache-team@twitter.com]) 6 | 7 | # Initialize autoconf 8 | AC_PREREQ([2.63]) 9 | AC_INIT([twemcache], [MC_MAJOR.MC_MINOR.MC_PATCH], [MC_BUGS]) 10 | AC_CONFIG_SRCDIR([src/mc.c]) 11 | AC_CONFIG_AUX_DIR([config]) 12 | AC_CONFIG_HEADERS([config.h:config.h.in]) 13 | AC_CONFIG_MACRO_DIR([m4]) 14 | AC_CANONICAL_SYSTEM 15 | 16 | # Initialize automake 17 | AM_INIT_AUTOMAKE(1.9 foreign) 18 | AM_SILENT_RULES([yes]) 19 | 20 | # Define macro variables for the package version numbers 21 | AC_DEFINE(MC_VERSION_MAJOR, MC_MAJOR, [Define the major version number]) 22 | AC_DEFINE(MC_VERSION_MINOR, MC_MINOR, [Define the minor version number]) 23 | AC_DEFINE(MC_VERSION_PATCH, MC_PATCH, [Define the patch version number]) 24 | AC_DEFINE(MC_VERSION_STRING, "MC_MAJOR.MC_MINOR.MC_PATCH", [Define the version string]) 25 | 26 | # Checks for language 27 | AC_LANG([C]) 28 | 29 | # Checks for OS and set OS variables 30 | AC_CANONICAL_HOST 31 | case $host_os in 32 | linux*) OS_LINUX=yes 33 | AC_DEFINE([OS_LINUX], [1], [Define to 1 if OS has Linux kernel]) 34 | ;; 35 | darwin*) OS_DARWIN=yes 36 | AC_DEFINE([OS_DARWIN], [1], [Define to 1 if OS has Darwin kernel]) 37 | ;; 38 | *) AC_MSG_ERROR([Your platform is not currently supported]) ;; 39 | esac 40 | AM_CONDITIONAL([RDYNAMIC], [test x$OS_LINUX = xyes]) 41 | 42 | # Checks for programs 43 | AC_PROG_CC 44 | AC_PROG_INSTALL 45 | AC_PROG_MAKE_SET 46 | AM_PROG_CC_C_O 47 | 48 | # Checks for typedefs, structures, and compiler characteristics 49 | AC_C_INLINE 50 | AC_C_CONST 51 | AC_TYPE_INT8_T 52 | AC_TYPE_INT16_T 53 | AC_TYPE_INT32_T 54 | AC_TYPE_INT64_T 55 | AC_TYPE_INTMAX_T 56 | AC_TYPE_INTPTR_T 57 | AC_TYPE_UINT8_T 58 | AC_TYPE_UINT16_T 59 | AC_TYPE_UINT32_T 60 | AC_TYPE_UINT64_T 61 | AC_TYPE_UINTMAX_T 62 | AC_TYPE_UINTPTR_T 63 | AC_TYPE_OFF_T 64 | AC_TYPE_PID_T 65 | AC_TYPE_SIZE_T 66 | AC_TYPE_SSIZE_T 67 | AC_TYPE_UID_T 68 | 69 | # Checks for header files 70 | AC_HEADER_STDBOOL 71 | AC_CHECK_HEADERS([fcntl.h float.h limits.h stddef.h stdlib.h string.h unistd.h]) 72 | AC_CHECK_HEADERS([inttypes.h stdint.h]) 73 | AC_CHECK_HEADERS([sys/ioctl.h sys/time.h sys/uio.h]) 74 | AC_CHECK_HEADERS([sys/socket.h sys/un.h netinet/in.h arpa/inet.h netdb.h]) 75 | 76 | # Checks for library functions 77 | AC_FUNC_FORK 78 | AC_FUNC_MALLOC 79 | AC_FUNC_REALLOC 80 | AC_CHECK_FUNCS([dup2]) 81 | AC_CHECK_FUNCS([gethostname]) 82 | AC_CHECK_FUNCS([gettimeofday]) 83 | AC_CHECK_FUNCS([strerror]) 84 | AC_CHECK_FUNCS([socket]) 85 | AC_CHECK_FUNCS([memchr memmove memset]) 86 | AC_CHECK_FUNCS([strchr strndup strtol strtoul strtoull]) 87 | AC_CHECK_FUNCS([mlockall]) 88 | AC_CHECK_FUNCS([getpagesizes]) 89 | AC_CHECK_FUNCS([memcntl]) 90 | AC_CHECK_FUNCS([backtrace]) 91 | 92 | # Search for library 93 | AC_SEARCH_LIBS([pthread_create], [pthread], [], 94 | [AC_MSG_ERROR([need posix thread library to be installed])]) 95 | 96 | # Check if we're a little-endian or a big-endian system 97 | AC_C_BIGENDIAN( 98 | [AC_DEFINE(HAVE_BIG_ENDIAN, 1, [Define to 1 if machine is big endian])], 99 | [AC_DEFINE(HAVE_LITTLE_ENDIAN, 1, [Define to 1 if machine is little endian])], 100 | [AC_MSG_ERROR([endianess of this machine is unknown])], 101 | [AC_MSG_ERROR([universial endianess not supported])] 102 | ) 103 | 104 | # Check whether to enable debug logs and asserts 105 | AC_MSG_CHECKING([whether to enable debug logs and asserts]) 106 | AC_ARG_ENABLE([debug], 107 | [AS_HELP_STRING( 108 | [--enable-debug=@<:@full|yes|log|no@:>@], 109 | [enable debug logs and asserts @<:@default=no@:>@]) 110 | ], 111 | [], 112 | [enable_debug=no]) 113 | AS_CASE([x$enable_debug], 114 | [xfull], [AC_DEFINE([HAVE_ASSERT_PANIC], [1], 115 | [Define to 1 if panic on an assert is enabled]) 116 | AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled]) 117 | AC_DEFINE([HAVE_MEM_SCRUB], [1], [Define to 1 if memory scrubbing is enabled]) 118 | ], 119 | [xyes], [AC_DEFINE([HAVE_ASSERT_LOG], [1], 120 | [Define to 1 if log on an assert is enabled]) 121 | AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled]) 122 | ], 123 | [xlog], [AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled])], 124 | [xno], [], 125 | [AC_MSG_FAILURE([invalid value ${enable_debug} for --enable-debug])]) 126 | AC_MSG_RESULT([$enable_debug]) 127 | 128 | # Check whether to disable stats 129 | AC_MSG_CHECKING([whether to disable stats]) 130 | AC_ARG_ENABLE([stats], 131 | [AS_HELP_STRING([--disable-stats], [disable stats collection])]) 132 | AS_IF( 133 | [test "x$enable_stats" = "xno"], 134 | [ 135 | AC_DEFINE([DISABLE_STATS], [1], [Define to 1 if stats collection is disabled]) 136 | AC_MSG_RESULT([yes]) 137 | ], 138 | [AC_MSG_RESULT([no])] 139 | ) 140 | 141 | # Check whether to disable command logger (klog) 142 | AC_MSG_CHECKING([whether to disable command logger (klog)]) 143 | AC_ARG_ENABLE([klog], 144 | [AS_HELP_STRING([--disable-klog], [disable klogger])]) 145 | AS_IF( 146 | [test "x$enable_klog" = "xno"], 147 | [ 148 | AC_DEFINE([DISABLE_KLOG], [1], [Define to 1 if klogger is disabled]) 149 | AC_MSG_RESULT([yes]) 150 | ], 151 | [AC_MSG_RESULT([no])] 152 | ) 153 | 154 | # Libevent detection; swiped from Tor, modified a bit 155 | trylibeventdir="" 156 | AC_ARG_WITH([libevent], 157 | [AS_HELP_STRING([--with-libevent=@<:@path@:>@], [specify path to libevent install])], 158 | [trylibeventdir=$withval], [trylibeventdir=] 159 | ) 160 | LIBEVENT_URL=https://libevent.org 161 | AC_CACHE_CHECK([for libevent directory], [ac_cv_libevent_dir], 162 | [ 163 | saved_LIBS="$LIBS" 164 | saved_LDFLAGS="$LDFLAGS" 165 | saved_CPPFLAGS="$CPPFLAGS" 166 | le_found=no 167 | for ledir in $trylibeventdir "" $prefix /usr/local 168 | do 169 | LDFLAGS="$saved_LDFLAGS" 170 | LIBS="-levent $saved_LIBS" 171 | 172 | # Skip the directory if it isn't there. 173 | AS_IF([test ! -z "$ledir" -a ! -d "$ledir"], [continue]) 174 | AS_IF( 175 | [test ! -z "$ledir"], 176 | [ 177 | AS_IF( 178 | [test -d "$ledir/lib"], 179 | [LDFLAGS="-L$ledir/lib $LDFLAGS"], 180 | [LDFLAGS="-L$ledir $LDFLAGS"] 181 | ) 182 | AS_IF( 183 | [test -d "$ledir/include"], 184 | [CPPFLAGS="-I$ledir/include $CPPFLAGS"], 185 | [CPPFLAGS="-I$ledir $CPPFLAGS"] 186 | ) 187 | ] 188 | ) 189 | 190 | # Can I compile and link it? 191 | AC_TRY_LINK( 192 | [ 193 | #include 194 | #include 195 | #include 196 | ], 197 | [ 198 | event_init(); 199 | ], 200 | [libevent_linked=yes], 201 | [libevent_linked=no] 202 | ) 203 | AS_IF( 204 | [test $libevent_linked = yes], 205 | [ 206 | AS_IF( 207 | [test ! -z "$ledir"], 208 | [ac_cv_libevent_dir=$ledir], 209 | [ac_cv_libevent_dir="(system)"] 210 | ) 211 | le_found=yes 212 | break 213 | ] 214 | ) 215 | done 216 | LIBS="$saved_LIBS" 217 | LDFLAGS="$saved_LDFLAGS" 218 | CPPFLAGS="$saved_CPPFLAGS" 219 | AS_IF( 220 | [test $le_found = no], 221 | [AC_MSG_ERROR([libevent required. Get it from $LIBEVENT_URL or specify its path using --with-libevent=/dir/])] 222 | ) 223 | ] 224 | ) 225 | AS_IF( 226 | [test $ac_cv_libevent_dir != "(system)"], 227 | [ 228 | AS_IF( 229 | [test -d "$ac_cv_libevent_dir/lib"], 230 | [ 231 | LDFLAGS="-L$ac_cv_libevent_dir/lib $LDFLAGS" 232 | le_libdir="$ac_cv_libevent_dir/lib" 233 | ], 234 | [ 235 | LDFLAGS="-L$ac_cv_libevent_dir $LDFLAGS" 236 | le_libdir="$ac_cv_libevent_dir" 237 | ] 238 | ) 239 | AS_IF( 240 | [test -d "$ac_cv_libevent_dir/include"], 241 | [CPPFLAGS="-I$ac_cv_libevent_dir/include $CPPFLAGS"] 242 | [CPPFLAGS="-I$ac_cv_libevent_dir $CPPFLAGS"] 243 | ) 244 | ] 245 | ) 246 | 247 | # Check whether to enable static linking 248 | AC_MSG_CHECKING([whether to enable static linking]) 249 | AC_ARG_ENABLE([static], 250 | [AS_HELP_STRING( 251 | [--enable-static=@<:@yes|libevent|no@:>@], 252 | [enable static linking @<:@default=no@:>@]) 253 | ], 254 | [], 255 | [enable_static=no]) 256 | AS_CASE([x$enable_static], 257 | [xyes], [ 258 | LDFLAGS="-static-libgcc -static $LDFLAGS" 259 | LIBS="-levent $LIBS -lrt" 260 | ], 261 | [xlibevent], [LIBS="-Wl,-Bstatic -levent -Wl,-Bdynamic $LIBS -lrt"], 262 | [xno], [LIBS="-levent $LIBS"], 263 | [AC_MSG_FAILURE([invalid value ${enable_static} for --enable-static])]) 264 | AC_MSG_RESULT([$enable_static]) 265 | 266 | AC_MSG_CHECKING([whether to enable test coverage]) 267 | AC_ARG_ENABLE([gcov], 268 | [AS_HELP_STRING( 269 | [--enable-gcov], 270 | [enable coverage testing with gcov]), 271 | ], 272 | [], 273 | [enable_gcov=no]) 274 | AS_CASE([x$enable_gcov], 275 | [xyes], [ 276 | AC_DEFINE([HAVE_GCOV], [1], [Define to 1 if gcov is enabled]) 277 | GCOV_CFLAGS="-O0 -fprofile-arcs -ftest-coverage" 278 | GCOV_LDFLAGS="-lgcov" 279 | ], 280 | [xno], [ 281 | GCOV_CFLAGS="" 282 | GCOV_LDFLAGS="" 283 | ]) 284 | AC_MSG_RESULT([$enable_gcov]) 285 | AC_SUBST(GCOV_CFLAGS) 286 | AC_SUBST(GCOV_LDFLAGS) 287 | 288 | # Define Makefiles 289 | AC_CONFIG_FILES([Makefile 290 | src/Makefile]) 291 | 292 | # Generate the "configure" script 293 | AC_OUTPUT 294 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twitter/twemcache/8cfc4ca5e76ed936bd3786c8cc43ed47e7778c08/m4/.gitignore -------------------------------------------------------------------------------- /notes/TODO: -------------------------------------------------------------------------------- 1 | * Add a cmd counter (counter for total # of commands) to stats 2 | * Reply strings like NOT_FOUND, EXISTS, etc should come from a #define table 3 | * separate command logic from item module, the latter should only care about create/destroy/link/replace.. items 4 | * Add stat to track the memory overhead of hash table and connection module 5 | * Refactor 'struct setting' to use the preprocessor and a #define table 6 | * Reintroduce the functionality of MEMCACHED_PORT_FILENAME 7 | * Reintroduce the functionality of MEMCACHED_HASH_BULK_MOVE 8 | * Move hashing of keys outside the cache_lock; exploring saving hash value in item struct 9 | * Analyze different hashing algorithms, choose the one that is cheapest 10 | * Put a cap the size of hashtables 11 | * Build a cache warmer or bootstraper, that populates twemcache from a dataset in a file 12 | -------------------------------------------------------------------------------- /notes/eviction_strategies.md: -------------------------------------------------------------------------------- 1 | ## Eviction Options 2 | Twemcache has three basic eviction strategies now when eviction is enabled: 3 | 1. item LRU eviction (`-M 1`) 4 | 2. slab random eviction (`-M 2`) 5 | 3. slab LRU eviction (`-M 4`) 6 | All eviction strategies have the same trigger, which is when a server runs out of existing free slots and quota for new slabs. 7 | 8 | Option No.1, commonly known as just "LRU eviction", has been around forever and is what all Memcached protocol implementation supports. It maintains an item LRU queue for each slab class, and accessing an item puts it to the back of the queue (but the update is rate-limited to avoid too much churn). When a server tries to evict, it scans from the head of the item LRU queue that the new item should go into, until an item not being currently accessed is found. Mainline Memcached allows at least one slab allocated to each slab class, even if that violates the number specified with `--max-memory`, to avoid total rejection of items between a particular size range. Twemcache does not make such attempt. Regardless, since this eviction strategy works only at the item-level and within the same slab class, it leads to the _slab calcification_ problem that threatens the long-term caching effectiveness as soon as item distribution among slab classes changes. 9 | 10 | Option No.2, sometimes shortened to "random eviction", is our first attempt to solve the issue with item LRU eviction. It was introduced in Twemcache 2.0, before Twemcache was even open sourced. Fundamentally, we need an eviction strategy that evicts at the same level as memory allocation, which is at the slab level. Random eviction requires a simple data structure called the _slab table_, which is an array of all slabs allocated by far. A server evicts by repeatedly picking a random slab until one that is not currently being accessed is found. Then all items in this slab are evicted before the server re-initialize it to be used by the next item. Random eviction is naive and by no means perfect in itself, but it is conceptually simple- it responses to memory requests instantaneously, and the laws of statistics ensure that a migrating item size distribution will lead to the same change to happen in slab distribution over time. In our opinion, it is perfect as a first attack on the slab calcification problem. It establishes a baseline for future exploration of a better solution. 11 | 12 | Option No.3, slab LRU eviction, is introduced in Twemcache 2.5.0. It furthers the search for a good slab-level eviction strategy and tries to be a little cleverer on choosing a slab to evict. It is based on the observation that with random eviction, a recently updated slab containing hot items may be evicted during the blind selection process. This naturally leads us to ask if we can adopt the same criteria as in item LRU eviction, but make it work at the slab-level. Slab LRU eviction is the result of such a thought. It uses a global slab LRU queue, which gets updated whenever an update to an item LRU queue is attempted (in other words, it shares the same update trigger as item LRU eviction). The updated slab is put to the tail of the queue, also in a rate-limited fashion, while eviction attempts are made from the head. It outperforms random eviction in terms of keeping hot data in memory when read requests are highly temporal and never reach beyond a small, recently updated portion of the data. On the other hand, it is more effective in the longer run compared to item LRU eviction if the item size distribution is dynamic. 13 | 14 | Twemcache allows any combination of these eviction strategies by setting the corresponding bit and pass the total numeric value to your `--eviction-strategy` option. Eviction strategies will be checked from the most significant bit to the least. 15 | 16 | ### A Note on Slab Automove 17 | Automove is the slab-level eviction strategy provided by mainline Memcached. The main difference between slab automove and slab random eviction is that the former only evicts a slab when it meets certain criteria. To quote from the [Release Note of Memcached 1.4.11](http://code.google.com/p/memcached/wiki/ReleaseNotes1411rc1), that criteria is "if a slab class is seen as having the highest eviction count 3 times 10 seconds apart, it will take a page from a slab class which has had zero evictions in the last 30 seconds and move the memory". From the phrasing, it is not difficult to tell that this is a more conservative strategy than both slab random eviction and slab LRU eviction. For example, the slab automover does not kick in if all slab classes have seen evictions within the past 30 seconds, or if there are two alternating slab classes witnessing the highest eviction in each 10 second window. Therefore, one should expect the involvement of slab automover to be dependent on the traffic pattern. 18 | We have a note that compares slab automove with random eviction through an experiment. Continue reading about the result in random_eviction.md 19 | -------------------------------------------------------------------------------- /notes/memory_overhead.md: -------------------------------------------------------------------------------- 1 | ## Question 2 | 3 | What is the memory overhead of twemcache when it configured to use 'X' GB of memory for all items (i.e. the -m option to twemcache is X / 1024) 4 | 5 | ## Tl; Dr 6 | 7 | For hash module, a 4G twemcache consumes 256M for hash table; a 8G would consume 512M and a 16G would consume 1GB! 8 | For connection module, 10K active connections we are consuming ~256M, for 50K, we will be consuming ~1.4G and for 100K we will be consuming ~2.6G 9 | 10 | ## Details 11 | 12 | In twemcache memory is consumed by the following modules: 13 | + Thread module 14 | + Stats module 15 | + Klogger module 16 | + Slabs module 17 | + Hash module 18 | + Connection module 19 | 20 | ### Thread Module 21 | 22 | Memory consumed by the thread module is negligible. 23 | 24 | ### Stats Module 25 | 26 | Memory consumed by the stats module can be computed as follows. The number of worker threads (specified by -t option) is by default set to 4. Besides worker threads, we have the dispatcher, aggregator and keylogger thread. So, when invoked with default options there are 7 threads. Each of these threads collect thread local stats. Each of these thread local stats do the following memory allocation: 27 | 28 | + sizeof(stats_tmetric) = 1504 29 | + stats_thread = alloc(sizeof(stats_tmetrics)); = 1504 = 1.5K 30 | + stats_slabs = alloc(MAX_NUM_OF_SLABCLASSES * sizeof(struct stats_metric *)) = 255 * 8 = 2040 = 2K 31 | + sizeof(stats_smetrics) = 1248 32 | + metric_array2 = mc_alloc(MAX_NUM_OF_SLABCLASSES * sizeof(stats_smetrics)) = 255 * 128 = 318240 = 318K 33 | 34 | So, in total ~2M consumed for setup of every thread, which is negligible. Also, for every connection we have a stats buffer, that grows depending on the # of stats that we need to send out: 35 | 36 | if (nsize != c->stats.size) { 37 | char *ptr = mc_realloc(c->stats.buffer, nsize); 38 | if (ptr) { 39 | c->stats.buffer = ptr; 40 | c->stats.size = nsize; 41 | 42 | With the current # stats, this value grows to 64K. But this is freed immediately by core_write_and_free() after the stats are sent out. So again, memory consumed by stats module is negligible 43 | 44 | ### Slabs Module 45 | 46 | Besides the memory value given by -m option, slab module consumes memory for the slabtable, which has one 8 byte entry for every slab in the system. So, for 4G twemcache (-m 4096), we have: 47 | 48 | + slab_info = {base = 0x0, curr = 0x0, nr_slabs = 4096, max_slabs = 4096, slabtable = 0xce8cd0} 49 | + slabtable = 8 * 4096 = 32K 50 | 51 | So, for X GB of twemcache, slabtable will take up, X * 1024 * 2^3 bytes, which is still in range of few low MBs and hence negligible. 52 | 53 | ### Hash Module 54 | 55 | Hash table initially starts of with 2^16 buckets and grows an number of buckets every time by a factor of 2 when the # hash items is greater that 1.5 times the number of hash buckets 56 | 57 | #define HASHSIZE(_n) (1UL << (_n)) 58 | static uint32_t hashpower = 16; 59 | 60 | hash_items++; 61 | if (!expanding && (hash_items > (HASHSIZE(hashpower) * 3) / 2)) { 62 | _assoc_expand(); 63 | } 64 | 65 | static void 66 | _assoc_expand(void) 67 | { 68 | uint32_t hashtable_sz = HASHSIZE(hashpower + 1); 69 | 70 | old_hashtable = primary_hashtable; 71 | primary_hashtable = _assoc_create_table(hashtable_sz); 72 | if (primary_hashtable != NULL) { 73 | log_debug(LOG_DEBUG, "expanding hash table from %u to %u buckets", 74 | HASHSIZE(hashpower), hashtable_sz); 75 | hashpower++; 76 | 77 | So, initially the hash table consumes 2^16 * 2^3 bytes = 512K bytes, and over time as more and more items gets added to the hash table, the hash table expands by a factor of 2. Lets calculate the maximum hash table size for a 4G twemcache (-m 4096); +For a 4G twemcache with 128 byte smallest item size+, the maximum # hash items are: 78 | 79 | + 4096 * (1024 * 1024 / 128) = 33554432.00 80 | + log_2(33554432.00) = 25.10 81 | + 2^25 * 1.5 = 50331648.0 82 | + 50331648.0 > 33554432.00 = 1 83 | 84 | So, the hash table size is = 2^25 * 2^3 = 256M. So, a 4G twemcache consumes 256M for hash table; a 8G would consume 512M and a 16G would consume 1GB! 85 | 86 | Lets calculate the maximum hash table size for a 16G twemcache (-m 65536) 87 | 88 | + 16 * 1024 * (1024 * 1024 / 128) = 134217728.00 89 | + log_2(134217728.00) = 27.11 90 | + 2^27 * 1.5 - 201326592.0 91 | + 134217728.00 < 201326592.0 =- 1 92 | 93 | So, for any twemcache instance with "X" GB of slab space, the maximum hash table size is: 94 | 95 | + max_slabs = X * 1024 96 | + min_item_size = 128 97 | + max_items_per_slab = (1024 * 1024 / min_item_size) 98 | + max_items = max_slabs * max_items_per_slab 99 | + hash_buckets = floor(log_2(max_items)) 100 | + hash_size is between 2^hash_buckets * 2^3 and 2^(hash_buckets + 1) * 2^3 101 | 102 | ### Connection Module 103 | 104 | Every connection objects allocates: 105 | 106 | + c = mc_zalloc(sizeof(struct conn)) = 496 bytes 107 | + c->rsize = read_buffer_size; 108 | + c->rbuf = mc_alloc((size_t)c->rsize) = 2048 109 | + c->wsize = DATA_BUFFER_SIZE; 110 | + c->wbuf = mc_alloc((size_t)c->wsize) = 2048 111 | + c->isize = ITEM_LIST_INITIAL; 112 | + c->ilist = mc_alloc(sizeof(struct item *) * c->isize) = 200 * 8 = 1600 113 | + c->suffixsize = SUFFIX_LIST_INITIAL; 114 | + c->suffixlist = mc_alloc(sizeof(char *) * c->suffixsize) = 20 * 8 = 160 bytes 115 | + c->iovsize = IOV_LIST_INITIAL; 116 | + c->iov = mc_alloc(sizeof(struct iovec) * c->iovsize) = 400 * 16 = 6400 117 | + c->msgsize = MSG_LIST_INITIAL; 118 | + c->msglist = mc_alloc(sizeof(struct msghdr) * c->msgsize) = 10 * 56 = 560 bytes 119 | 120 | Total initial bytes per conn = 13152 = ~13K. So, even if 50K connections are active, memory used will be = 13K * 10K = 130M. But because of the realloc strategy, each rbuf is 16K (in virtual memory), which it then shrinks to 2K in conn_shrink. This means that we have an additional 14K addition per conn, leading to around 27K 121 | 122 | new_rbuf = mc_realloc(c->rbuf, c->rsize * 2); 123 | if (new_rbuf == NULL) { 124 | stats_thread_incr(server_error); 125 | c->rbytes = 0; /* ignore what we read */ 126 | out_string(c, "SERVER_ERROR out of memory reading request"); 127 | c->write_and_go = CONN_CLOSING; 128 | return READ_MEMORY_ERROR; 129 | } 130 | c->rcurr = c->rbuf = new_rbuf; 131 | c->rsize *= 2; 132 | 133 | So, for 10K active connections we are consuming 270M, for 50M, we will be consuming 1350M and for 100K we will be consuming 2700M 134 | 135 | ## Conclusion 136 | 137 | The main contributors to memory overhead in twemcache are: 138 | 139 | + Connection module 140 | + Hash module 141 | 142 | For hash module, a 4G twemcache consumes 256M for hash table; a 8G would consume 512M and a 16G would consume 1GB! 143 | For connection module, 10K active connections we are consuming ~256M, for 50K, we will be consuming ~1.4G and for 100K we will be consuming ~2.6G 144 | 145 | -------------------------------------------------------------------------------- /notes/slab_allocation.md: -------------------------------------------------------------------------------- 1 | **Note:** To avoid any confusion, in this article, "Memcached Protocol" means the protocol adopted by all versions of Memcached regardless of implementation details; when unspecified, "Memcached" means both the original implementation ("mainline Memcached") as well as Twemcache; Twemcache-specific features are referred to as "Twemcache". 2 | 3 | ## Background 4 | Part of the Memcached Protocol design philosophy is to keep things simple. The protocol is an in-memory key-value store interface, and not much more than that. The strength of the software is its high performance and predictable latency under most conditions. 5 | 6 | ## Slab Allocation 7 | When it comes to memory management, the preference toward predictable performance leads to the decision of using a single memory allocation unit, called a _slab_. To still use memory relatively effectively, a slab-class structure is also adopted, so that a slab can be evenly sliced up into smaller _slots_; the size of each slot is determined by the slab class a slab belongs to. Garbage collection was entirely left out of the picture in the early days of Memcached, probably due to latency concerns; instead, a slot can be reused if users want to store an item of similar size. And even as the latest mainline Memcached and Twemcache provide the capabilities of moving slabs around in their own ways, both make no effort to preserve the items within the chosen slab (eviction candidate), and neither ever returns memory to the OS. 8 | 9 | ### Common Misunderstandings 10 | There are two common misunderstandings about memory management in Memcached: 1. people sometimes think it works like the buddy memory allocation, in which a larger memory chunk can be broken down into smaller ones when smaller ones are not available. This is not true. Once a slab is allocated and sliced up in a particular way, only items matching that slab class perfectly can be stored in it. No bigger items can go in, which is obvious; but items matching a lower slab class cannot go in either, which comes as a surprise to many. 2. people sometimes also think that by deleting items, they give memory back to the heap/OS, which can subsequently be re-allocated in the form of a new slab. This is not true either for the reason we mentioned earlier- Memcached never frees up a slab even when there is not an active item in it. 11 | 12 | So the take-away message here is that Memcached only evicts data but never frees memory. And slab calcification under a dynamic size distribution is unavoidable unless slabs can be reassigned. 13 | -------------------------------------------------------------------------------- /scripts/README.twctop.md: -------------------------------------------------------------------------------- 1 | ## Overview 2 | twctop is a tool that presents realtime cache stats in a intuitive way. Different 'views' of the same stats help users gaining insight into the cluster, which is particularly useful for debugging. When multiple hosts are included, stats are aggregated in a way that suits the current view. It is written in Ruby and consumes a YAML file with a particular format for historical reasons at Twitter. 3 | 4 | ## How to use twctop.rb 5 | 6 | To monitor a cluster: 7 | 8 | $ twctop.rb [options] 9 | 10 | or to monitor a single instance: 11 | 12 | $ twctop.rb [options] <-H hostname:port> 13 | 14 | ## Most Useful Options 15 | 16 | * -s, --sleep : 'watch'-style endless refreshing. After the initial stats query, ongoing rates since the last stats query are shown when applicable, rather than the absolutes since server start. A reasonable value for most pools in a low latency environment is 2. 17 | * -v, --view [slab|host|command]: Choose from one of the three views to start. Content shown in each view is explained below. After launch, one can still switch views by pressing the initial letter of each view. 18 | * -h, --help: full help message 19 | 20 | ## The YAML format 21 | The YAML file should look like the following, with the list servers replaced by the ones to be monitored: 22 | 23 | 
24 | production:
25 |   servers:
26 |     - 1.2.3.4:11111:20
27 |     - 1.2.3.5:11111:20
28 |
29 | 30 | ## The SLAB View 31 | 32 | The header on top shows slabs allocated, how many "slots" are used to store items, and how many the allocated slabs can host. 33 | 34 | The meaning of the columns in this view are listed below: 35 | * SLAB: Slab class, marked with the size of a single item within this slab class. 36 | * #SLAB: Number of slabs allocated into a slab class. 37 | * ITEM(%): What percentage of allocated slots are occupied. 38 | * KEYVAL(%): Data (key+value, no metadata) size compared to the memory size holding them. 39 | * VALUE(%): Size of values compared to that of key-value pairs, in percentage. 40 | * BAR: A figurative way of showing both how much memory has been used (i.e. cannot be used by other items) and how much has been allocated. It is also the easiest way to spot memory distribution among slabs. Bar widths are normalized. 41 | * REC(/s): Rate at which expired items are reclaimed (recycled) to be used by new items. 42 | * EVT(/s): Rate at which unexpired items are evicted (recycled) to be used by new items. 43 | * SEVT(/s): Rate at which slabs are (randomly) evicted to be used by new items (of any slab class). 44 | * LOCATE(/s): Rate at which existing items are found, this happens to both read, update commands and delete. 45 | * INSERT(/s): Rate at which new items are inserted, this happends to all write commands. 46 | 47 | ## The HOST View 48 | 49 | The header on top shows how much data are stored out of the total capacity. (Note: data include both key, value and item related information, but not unused slots or item metadata.) 50 | 51 | The meaning of the columns in this view are listed below: 52 | * INSTANCE: IP and port number of the memcached instance. 53 | * UTIL(%): Data size compared to maximum memory allowed, in percentage. 54 | * CONN: Number of open connections to the host. 55 | * LATENCY: Time to execute a "stats" command, including the time to read the entirety of response. 56 | * EXPIRE(/s): Rate at which expired items are reclaimed by the memcached instance. 57 | * EVICT(/s): Rate at which unexpired items are evicted to make room for new items. 58 | * BYTE_IN/s: Number of bytes received by an instance (TCP/UDP payload only). 59 | * BYTE_OUT/s: Number of bytes sent by an instance (TCP/UDP payload only). 60 | * R_REQ(/s): Rate at which read commands arrive. 61 | * W_REQ(/s): Rate at which write commands arrive. 62 | * D_REQ(/s): Rate at which delete commands arrive. 63 | * SVRERR(/s): Rate at which a server instance is throwing errors (OOM, for example). 64 | 65 | ## The COMMAND View 66 | 67 | The header on top shows how many requests are correctly processed, compared to the total count. 68 | 69 | The meaning of the columns in this view are listed below: 70 | * REQUEST(/s): Rate at which commands are sent. 71 | * MIX(%): Percentage in all requests. 72 | * SUCCESS(%): Percentage of commands returning success, for write commands only. 73 | * HIT(%): Percentage of requests hitting an existing item, for read, update and delete. 74 | * MISS(%): Percentage of requests hitting an existing item, for read, update and delete. 75 | * EXISTS(%): Percentage of requests failing because of existing items, for cas and add only. 76 | * ERROR(/s): Rate at which command errors (syntax error or other unknown problems). 77 | 78 | -------------------------------------------------------------------------------- /scripts/klog/klogParser/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/twitter/twemcache/8cfc4ca5e76ed936bd3786c8cc43ed47e7778c08/scripts/klog/klogParser/__init__.py -------------------------------------------------------------------------------- /scripts/klog/klogParser/klogFormat.py: -------------------------------------------------------------------------------- 1 | __doc__ = ''' 2 | Defines valid log format / pattern. 3 | cmd_pattern: normal command log entry format 4 | lgd_pattern: log discarding entry format 5 | 6 | hdr_patterns: a dictionary of patterns covering header formats of all the commands 7 | 8 | ''' 9 | 10 | import re 11 | 12 | _cmd_parts = [ 13 | r'(?P\S+)', # host %h 14 | r'\S+', # indent %l (unused) 15 | r'\[(?P