├── .gitignore ├── .travis.yml ├── ChangeLog ├── LICENSE ├── Makefile.am ├── NOTICE ├── README.md ├── conf ├── nutcracker.leaf.yml ├── nutcracker.root.yml └── nutcracker.yml ├── configure.ac ├── contrib ├── Makefile.am ├── yaml-0.1.4.tar.gz └── yaml-0.1.4 │ └── .gitignore ├── m4 └── .gitignore ├── man └── nutcracker.8 ├── notes ├── c-styleguide.txt ├── debug.txt ├── kqueue.pdf ├── memcache.txt ├── recommendation.md ├── redis.md ├── socket.txt ├── twemproxy新增功能说明书.docx └── twemproxy新增功能说明书.pdf ├── scripts ├── benchmark-mget.py ├── multi_get.sh ├── nutcracker.init ├── nutcracker.init.debian ├── nutcracker.spec ├── pipelined_read.sh ├── pipelined_write.sh ├── populate_memcached.sh ├── redis-check.py └── redis-check.sh └── src ├── Makefile.am ├── event ├── Makefile.am ├── nc_epoll.c ├── nc_event.h ├── nc_evport.c └── nc_kqueue.c ├── hashkit ├── Makefile.am ├── nc_crc16.c ├── nc_crc32.c ├── nc_fnv.c ├── nc_hashkit.h ├── nc_hsieh.c ├── nc_jenkins.c ├── nc_ketama.c ├── nc_md5.c ├── nc_modula.c ├── nc_murmur.c ├── nc_one_at_a_time.c └── nc_random.c ├── nc.c ├── nc_array.c ├── nc_array.h ├── nc_client.c ├── nc_client.h ├── nc_conf.c ├── nc_conf.h ├── nc_connection.c ├── nc_connection.h ├── nc_core.c ├── nc_core.h ├── nc_log.c ├── nc_log.h ├── nc_mbuf.c ├── nc_mbuf.h ├── nc_message.c ├── nc_message.h ├── nc_proxy.c ├── nc_proxy.h ├── nc_queue.h ├── nc_rbtree.c ├── nc_rbtree.h ├── nc_request.c ├── nc_response.c ├── nc_server.c ├── nc_server.h ├── nc_signal.c ├── nc_signal.h ├── nc_stats.c ├── nc_stats.h ├── nc_string.c ├── nc_string.h ├── nc_util.c ├── nc_util.h ├── nc_zookeeper.c ├── nc_zookeeper.h └── proto ├── Makefile.am ├── nc_memcache.c ├── nc_proto.h └── nc_redis.c /.gitignore: -------------------------------------------------------------------------------- 1 | # pyc 2 | *.pyc 3 | 4 | # Compiled Object files 5 | *.lo 6 | *.o 7 | 8 | # Compiled Dynamic libraries 9 | *.so 10 | 11 | # Compiled Static libraries 12 | *.la 13 | *.a 14 | 15 | # Compiled misc 16 | *.dep 17 | *.gcda 18 | *.gcno 19 | *.gcov 20 | 21 | # Packages 22 | *.tar.gz 23 | *.tar.bz2 24 | 25 | # Logs 26 | *.log 27 | 28 | # Temporary 29 | *.swp 30 | *.~ 31 | *.project 32 | *.cproject 33 | 34 | # Core and executable 35 | core* 36 | nutcracker 37 | 38 | # extracted yaml 39 | !/contrib/yaml-0.1.4.tar.gz 40 | 41 | # Autotools 42 | .deps 43 | .libs 44 | 45 | /aclocal.m4 46 | /autom4te.cache 47 | /stamp-h1 48 | /autoscan.log 49 | /libtool 50 | 51 | /config/config.guess 52 | /config/config.sub 53 | /config/depcomp 54 | /config/install-sh 55 | /config/ltmain.sh 56 | /config/missing 57 | /config 58 | 59 | /config.h 60 | /config.h.in 61 | /config.h.in~ 62 | /config.log 63 | /config.status 64 | /configure.scan 65 | /configure 66 | 67 | Makefile 68 | Makefile.in 69 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | script: CFLAGS="-ggdb3 -O0" autoreconf -fvi && ./configure --enable-debug=log && make && sudo make install 3 | 4 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2015-18-06 deep 2 | 3 | *twemproxy: version 0.4.6 release 4 | replace_server(redis) 5 | log rotate 6 | tcp keepalive 7 | administration 8 | configuration reload 9 | zookeeper 10 | 11 | 2014-18-10 idning 12 | 13 | * twemproxy: version 0.4.0 release 14 | mget improve (idning) 15 | many new commands supported: LEX, PFADD, PFMERGE, SORT, PING, QUIT, SCAN... (mattrobenolt, areina, idning) 16 | handle max open file limit(allenlz) 17 | add notice-log and use ms time in log(idning) 18 | fix bug in string_compare (andyqzb) 19 | fix deadlock in sighandler (idning) 20 | 21 | 2013-20-12 Manju Rajashekhar 22 | * twemproxy: version 0.3.0 release 23 | SRANDMEMBER support for the optional count argument (mkhq) 24 | Handle case where server responds while the request is still being sent (jdi-tagged) 25 | event ports (solaris/smartos) support 26 | add timestamp when the server was ejected 27 | support for set ex/px/nx/xx for redis 2.6.12 and up (ypocat) 28 | kqueue (bsd) support (ferenyx) 29 | fix parsing redis response to accept integer reply (charsyam) 30 | 31 | 2013-23-04 Manju Rajashekhar 32 | * twemproxy: version 0.2.4 release 33 | redis keys must be less than mbuf_data_size() in length (fifsky) 34 | Adds support for DUMP/RESTORE commands in Redis (remotezygote) 35 | Use of the weight value in the modula distribution (mezzatto) 36 | Add support to unix socket connections to servers (mezzatto) 37 | only check for duplicate server name and not 'host:port:weight' when 'name' is configured 38 | crc16 hash support added (mezzatto) 39 | 40 | 2013-31-01 Manju Rajashekhar 41 | * twemproxy: version 0.2.3 release 42 | RPOPLPUSH, SDIFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION, SUNIONSTORE, ZINTERSTORE, and ZUNIONSTORE support (dcartoon) 43 | EVAL and EVALSHA support (ferenyx) 44 | exit 1 if configuration file is invalid (cofyc) 45 | return non-zero exit status when nutcracker cannot start for some reason 46 | use server names in stats (charsyam) 47 | Fix failure to resolve long FQDN name resolve (conmame) 48 | add support for hash tags 49 | 50 | 2012-18-10 Manju Rajashekhar 51 | 52 | * twemproxy: version 0.2.2 release 53 | fix the off-by-one error when calculating redis key length 54 | 55 | 2012-12-10 Manju Rajashekhar 56 | 57 | * twemproxy: version 0.2.1 release 58 | don't use buf in conf_add_server 59 | allow an optional instance name for consistent hashing (charsyam) 60 | add --stats-addr=S option 61 | add stats-bind-any -a option (charsyam) 62 | 63 | 2012-12-03 Manju Rajashekhar 64 | 65 | * twemproxy: version 0.2.0 release 66 | add -D or --describe-stats command-line argument to print stats description 67 | redis support in twemproxy 68 | setup pre/post splitcopy and pre/post coalesce handlers in msg struct 69 | memcache pre_splitcopy, post_splitcopy, pre_coalesce and post_coalesce handlers 70 | every fragment of a msg vector keeps track of the first/last fragment, number of fragments and fragment owner 71 | set up msg parser handler for memcache connections 72 | refactor parsing code and create header file nc_proto.h 73 | stats_listen should use st->addr as the listening address string 74 | delete stats tracking memcache requests and responses; stats module no longer tracks protocol related stats 75 | 76 | 2012-10-27 Manju Rajashekhar 77 | 78 | * twemproxy: version 0.1.20 release 79 | on msg_repair, msg->pos should point to nbuf->pos and not nbuf->last 80 | refactor memcache parsing code into proto directory 81 | add redis option to configuration file 82 | fix macro definition strXcmp error for big endian 83 | fix log_hexdump and loga_hexdump 84 | 85 | 2012-07-31 Manju Rajashekhar 86 | 87 | * twemproxy: version 0.1.19 release 88 | close server connection on a stray response (yashh, bmatheny) 89 | 90 | 2012-06-19 Manju Rajashekhar 91 | 92 | * twemproxy: version 0.1.18 release 93 | command line option to set mbuf chunk size 94 | 95 | 2012-05-09 Manju Rajashekhar 96 | 97 | * twemproxy: version 0.1.17 release 98 | use _exit(0) instead of exit(0) when daemonizing 99 | use loga instead of log_stderr in nc_stacktrace 100 | 101 | 2012-02-09 Manju Rajashekhar 102 | 103 | * twemproxy: version 0.1.16 release 104 | twemproxy (aka nutcracker) is a fast and lightweight proxy for memcached protocol. 105 | 106 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure config.h.in config.h.in~ stamp-h.in 2 | 3 | ACLOCAL_AMFLAGS = -I m4 4 | 5 | SUBDIRS = contrib src 6 | 7 | dist_man_MANS = man/nutcracker.8 8 | 9 | EXTRA_DIST = README.md NOTICE LICENSE ChangeLog conf scripts notes 10 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | twemproxy is a fast and lightweight proxy for memcached protocol 2 | Copyright (C) 2012 Twitter, Inc. 3 | 4 | Portions of twemproxy were inspired from nginx: http://nginx.org/ 5 | 6 | The implementation of generic array (nc_array.[ch]) and red black tree 7 | (nc_rbtree.[ch]) also comes from nginx-0.8.55. 8 | 9 | /* 10 | * Copyright (C) 2002-2010 Igor Sysoev 11 | * 12 | * Redistribution and use in source and binary forms, with or without 13 | * modification, are permitted provided that the following conditions 14 | * are met: 15 | * 1. Redistributions of source code must retain the above copyright 16 | * notice, this list of conditions and the following disclaimer. 17 | * 2. Redistributions in binary form must reproduce the above copyright 18 | * notice, this list of conditions and the following disclaimer in the 19 | * documentation and/or other materials provided with the distribution. 20 | * 21 | * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 | * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 25 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 | * SUCH DAMAGE. 32 | */ 33 | 34 | The generic queue implementation comes from BSD 35 | 36 | /* 37 | * Copyright (c) 1991, 1993 38 | * The Regents of the University of California. All rights reserved. 39 | * 40 | * Redistribution and use in source and binary forms, with or without 41 | * modification, are permitted provided that the following conditions 42 | * are met: 43 | * 1. Redistributions of source code must retain the above copyright 44 | * notice, this list of conditions and the following disclaimer. 45 | * 2. Redistributions in binary form must reproduce the above copyright 46 | * notice, this list of conditions and the following disclaimer in the 47 | * documentation and/or other materials provided with the distribution. 48 | * 3. All advertising materials mentioning features or use of this software 49 | * must display the following acknowledgement: 50 | * This product includes software developed by the University of 51 | * California, Berkeley and its contributors. 52 | * 4. Neither the name of the University nor the names of its contributors 53 | * may be used to endorse or promote products derived from this software 54 | * without specific prior written permission. 55 | * 56 | * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 57 | * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 58 | * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 59 | * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 60 | * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 61 | * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 62 | * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 63 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 64 | * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 65 | * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 66 | * SUCH DAMAGE. 67 | */ 68 | 69 | The implementation of consistent hashing and individual hash algorithms were 70 | borrowed from libmemcached. 71 | 72 | Copyright (c) 2011, Data Differential (http://datadifferential.com/) 73 | Copyright (c) 2007-2010, TangentOrg (Brian Aker) 74 | All rights reserved. 75 | 76 | Redistribution and use in source and binary forms, with or without 77 | modification, are permitted provided that the following conditions are 78 | met: 79 | 80 | * Redistributions of source code must retain the above copyright 81 | notice, this list of conditions and the following disclaimer. 82 | 83 | * Redistributions in binary form must reproduce the above 84 | copyright notice, this list of conditions and the following disclaimer 85 | in the documentation and/or other materials provided with the 86 | distribution. 87 | 88 | * Neither the name of TangentOrg nor the names of its 89 | contributors may be used to endorse or promote products derived from 90 | this software without specific prior written permission. 91 | 92 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 93 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 94 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 95 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 96 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 97 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 98 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 99 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 100 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 101 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 102 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 103 | 104 | The source also includes libyaml (yaml-0.1.4) in contrib/ directory 105 | 106 | Copyright (c) 2006 Kirill Simonov 107 | 108 | Permission is hereby granted, free of charge, to any person obtaining a copy of 109 | this software and associated documentation files (the "Software"), to deal in 110 | the Software without restriction, including without limitation the rights to 111 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 112 | of the Software, and to permit persons to whom the Software is furnished to do 113 | so, subject to the following conditions: 114 | 115 | The above copyright notice and this permission notice shall be included in all 116 | copies or substantial portions of the Software. 117 | 118 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 119 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 120 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 121 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 122 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 123 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 124 | SOFTWARE. 125 | -------------------------------------------------------------------------------- /conf/nutcracker.leaf.yml: -------------------------------------------------------------------------------- 1 | leaf: 2 | listen: 127.0.0.1:22121 3 | hash: fnv1a_64 4 | distribution: ketama 5 | auto_eject_hosts: true 6 | server_retry_timeout: 2000 7 | server_failure_limit: 1 8 | servers: 9 | - 127.0.0.1:11212:1 10 | - 127.0.0.1:11213:1 11 | -------------------------------------------------------------------------------- /conf/nutcracker.root.yml: -------------------------------------------------------------------------------- 1 | root: 2 | listen: 127.0.0.1:22120 3 | hash: fnv1a_64 4 | distribution: ketama 5 | preconnect: true 6 | auto_eject_hosts: false 7 | servers: 8 | - 127.0.0.1:22121:1 9 | -------------------------------------------------------------------------------- /conf/nutcracker.yml: -------------------------------------------------------------------------------- 1 | alpha: 2 | listen: 127.0.0.1:22121 3 | hash: fnv1a_64 4 | distribution: ketama 5 | auto_eject_hosts: true 6 | redis: true 7 | server_retry_timeout: 2000 8 | server_failure_limit: 1 9 | servers: 10 | - 127.0.0.1:6379:1 11 | 12 | beta: 13 | listen: 127.0.0.1:22122 14 | hash: fnv1a_64 15 | hash_tag: "{}" 16 | distribution: ketama 17 | auto_eject_hosts: false 18 | timeout: 400 19 | redis: true 20 | servers: 21 | - 127.0.0.1:6380:1 server1 22 | - 127.0.0.1:6381:1 server2 23 | - 127.0.0.1:6382:1 server3 24 | - 127.0.0.1:6383:1 server4 25 | 26 | gamma: 27 | listen: 127.0.0.1:22123 28 | hash: fnv1a_64 29 | distribution: ketama 30 | timeout: 400 31 | backlog: 1024 32 | preconnect: true 33 | auto_eject_hosts: true 34 | server_retry_timeout: 2000 35 | server_failure_limit: 3 36 | servers: 37 | - 127.0.0.1:11212:1 38 | - 127.0.0.1:11213:1 39 | 40 | delta: 41 | listen: 127.0.0.1:22124 42 | hash: fnv1a_64 43 | distribution: ketama 44 | timeout: 100 45 | auto_eject_hosts: true 46 | server_retry_timeout: 2000 47 | server_failure_limit: 1 48 | servers: 49 | - 127.0.0.1:11214:1 50 | - 127.0.0.1:11215:1 51 | - 127.0.0.1:11216:1 52 | - 127.0.0.1:11217:1 53 | - 127.0.0.1:11218:1 54 | - 127.0.0.1:11219:1 55 | - 127.0.0.1:11220:1 56 | - 127.0.0.1:11221:1 57 | - 127.0.0.1:11222:1 58 | - 127.0.0.1:11223:1 59 | 60 | omega: 61 | listen: /tmp/gamma 62 | hash: hsieh 63 | distribution: ketama 64 | auto_eject_hosts: false 65 | servers: 66 | - 127.0.0.1:11214:100000 67 | - 127.0.0.1:11215:1 68 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # Define the package version numbers and the bug reporting address 2 | m4_define([NC_MAJOR], 0) 3 | m4_define([NC_MINOR], 4) 4 | m4_define([NC_PATCH], 6) 5 | m4_define([NC_BUGS], [manj@cs.stanford.edu]) 6 | 7 | # Initialize autoconf 8 | AC_PREREQ([2.64]) 9 | AC_INIT([nutcracker], [NC_MAJOR.NC_MINOR.NC_PATCH], [NC_BUGS]) 10 | AC_CONFIG_SRCDIR([src/nc.c]) 11 | AC_CONFIG_AUX_DIR([config]) 12 | AC_CONFIG_HEADERS([config.h:config.h.in]) 13 | AC_CONFIG_MACRO_DIR([m4]) 14 | 15 | # Initialize automake 16 | AM_INIT_AUTOMAKE([1.9 foreign]) 17 | 18 | # Define macro variables for the package version numbers 19 | AC_DEFINE(NC_VERSION_MAJOR, NC_MAJOR, [Define the major version number]) 20 | AC_DEFINE(NC_VERSION_MINOR, NC_MINOR, [Define the minor version number]) 21 | AC_DEFINE(NC_VERSION_PATCH, NC_PATCH, [Define the patch version number]) 22 | AC_DEFINE(NC_VERSION_STRING, "NC_MAJOR.NC_MINOR.NC_PATCH", [Define the version string]) 23 | 24 | # Checks for language 25 | AC_LANG([C]) 26 | 27 | # Checks for programs 28 | AC_PROG_AWK 29 | AC_PROG_CC 30 | AC_PROG_CPP 31 | AC_PROG_CXX 32 | AC_PROG_INSTALL 33 | AC_PROG_LN_S 34 | AC_PROG_MAKE_SET 35 | AC_PROG_RANLIB 36 | AC_PROG_LIBTOOL 37 | 38 | # Checks for typedefs, structures, and compiler characteristics 39 | AC_C_INLINE 40 | AC_TYPE_INT8_T 41 | AC_TYPE_INT16_T 42 | AC_TYPE_INT32_T 43 | AC_TYPE_INT64_T 44 | AC_TYPE_INTMAX_T 45 | AC_TYPE_INTPTR_T 46 | AC_TYPE_UINT8_T 47 | AC_TYPE_UINT16_T 48 | AC_TYPE_UINT32_T 49 | AC_TYPE_UINT64_T 50 | AC_TYPE_UINTMAX_T 51 | AC_TYPE_UINTPTR_T 52 | AC_TYPE_OFF_T 53 | AC_TYPE_PID_T 54 | AC_TYPE_SIZE_T 55 | AC_TYPE_SSIZE_T 56 | 57 | AC_C_BIGENDIAN( 58 | [], 59 | [AC_DEFINE(HAVE_LITTLE_ENDIAN, 1, [Define to 1 if machine is little endian])], 60 | [AC_MSG_ERROR([endianess of this machine is unknown])], 61 | [AC_MSG_ERROR([universial endianess not supported])] 62 | ) 63 | 64 | # Checks for header files 65 | AC_HEADER_STDBOOL 66 | AC_CHECK_HEADERS([fcntl.h float.h limits.h stddef.h stdlib.h string.h unistd.h]) 67 | AC_CHECK_HEADERS([inttypes.h stdint.h]) 68 | AC_CHECK_HEADERS([sys/ioctl.h sys/time.h sys/uio.h]) 69 | AC_CHECK_HEADERS([sys/socket.h sys/un.h netinet/in.h arpa/inet.h netdb.h]) 70 | AC_CHECK_HEADERS([execinfo.h], 71 | [AC_DEFINE(HAVE_BACKTRACE, [1], [Define to 1 if backtrace is supported])], []) 72 | AC_CHECK_HEADERS([sys/epoll.h], [], []) 73 | AC_CHECK_HEADERS([sys/event.h], [], []) 74 | 75 | # Checks for libraries 76 | AC_CHECK_LIB([m], [pow]) 77 | AC_CHECK_LIB([pthread], [pthread_create]) 78 | 79 | # Checks for library functions 80 | AC_FUNC_FORK 81 | AC_FUNC_MALLOC 82 | AC_FUNC_REALLOC 83 | AC_CHECK_FUNCS([dup2 gethostname gettimeofday strerror]) 84 | AC_CHECK_FUNCS([socket]) 85 | AC_CHECK_FUNCS([memchr memmove memset]) 86 | AC_CHECK_FUNCS([strchr strndup strtoul]) 87 | 88 | AC_CACHE_CHECK([if epoll works], [ac_cv_epoll_works], 89 | AC_TRY_RUN([ 90 | #include 91 | #include 92 | #include 93 | int 94 | main(int argc, char **argv) 95 | { 96 | int fd; 97 | 98 | fd = epoll_create(256); 99 | if (fd < 0) { 100 | perror("epoll_create:"); 101 | exit(1); 102 | } 103 | exit(0); 104 | } 105 | ], [ac_cv_epoll_works=yes], [ac_cv_epoll_works=no])) 106 | AS_IF([test "x$ac_cv_epoll_works" = "xyes"], 107 | [AC_DEFINE([HAVE_EPOLL], [1], [Define to 1 if epoll is supported])], []) 108 | 109 | AC_CACHE_CHECK([if kqueue works], [ac_cv_kqueue_works], 110 | AC_TRY_RUN([ 111 | #include 112 | #include 113 | #include 114 | #include 115 | #include 116 | int 117 | main(int argc, char **argv) 118 | { 119 | int fd; 120 | 121 | fd = kqueue(); 122 | if (fd < 0) { 123 | perror("kqueue:"); 124 | exit(1); 125 | } 126 | exit(0); 127 | } 128 | ], [ac_cv_kqueue_works=yes], [ac_cv_kqueue_works=no])) 129 | AS_IF([test "x$ac_cv_kqueue_works" = "xyes"], 130 | [AC_DEFINE([HAVE_KQUEUE], [1], [Define to 1 if kqueue is supported])], []) 131 | 132 | AC_CACHE_CHECK([if event ports works], [ac_cv_evports_works], 133 | AC_TRY_RUN([ 134 | #include 135 | #include 136 | #include 137 | int 138 | main(int argc, char **argv) 139 | { 140 | int fd; 141 | 142 | fd = port_create(); 143 | if (fd < 0) { 144 | perror("port_create:"); 145 | exit(1); 146 | } 147 | exit(0); 148 | } 149 | ], [ac_cv_evports_works=yes], [ac_cv_evports_works=no])) 150 | AS_IF([test "x$ac_cv_evports_works" = "xyes"], 151 | [AC_DEFINE([HAVE_EVENT_PORTS], [1], [Define to 1 if event ports is supported])], []) 152 | 153 | AS_IF([test "x$ac_cv_epoll_works" = "xno" && 154 | test "x$ac_cv_kqueue_works" = "xno" && 155 | test "x$ac_cv_evports_works" = "xno"], 156 | [AC_MSG_ERROR([either epoll or kqueue or event ports support is required])], []) 157 | 158 | AM_CONDITIONAL([OS_LINUX], [test "x$ac_cv_epoll_works" = "xyes"]) 159 | AM_CONDITIONAL([OS_BSD], [test "x$ac_cv_kqueue_works" = "xyes"]) 160 | AM_CONDITIONAL([OS_SOLARIS], [test "x$ac_cv_evports_works" = "xyes"]) 161 | 162 | # Package options 163 | AC_MSG_CHECKING([whether to enable debug logs and asserts]) 164 | AC_ARG_ENABLE([debug], 165 | [AS_HELP_STRING( 166 | [--enable-debug=@<:@full|yes|log|no@:>@], 167 | [enable debug logs and asserts @<:@default=no@:>@]) 168 | ], 169 | [], 170 | [enable_debug=no]) 171 | AS_CASE([x$enable_debug], 172 | [xfull], [AC_DEFINE([HAVE_ASSERT_PANIC], [1], 173 | [Define to 1 if panic on an assert is enabled]) 174 | AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled]) 175 | ], 176 | [xyes], [AC_DEFINE([HAVE_ASSERT_LOG], [1], 177 | [Define to 1 if log on an assert is enabled]) 178 | AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled]) 179 | ], 180 | [xlog], [AC_DEFINE([HAVE_DEBUG_LOG], [1], [Define to 1 if debug log is enabled])], 181 | [xno], [], 182 | [AC_MSG_FAILURE([invalid value ${enable_debug} for --enable-debug])]) 183 | AC_MSG_RESULT($enable_debug) 184 | 185 | AC_MSG_CHECKING([whether to disable stats]) 186 | AC_ARG_ENABLE([stats], 187 | [AS_HELP_STRING( 188 | [--disable-stats], 189 | [disable stats]) 190 | ], 191 | [disable_stats=yes], 192 | [disable_stats=no]) 193 | AS_IF([test "x$disable_stats" = xyes], 194 | [], 195 | [AC_DEFINE([HAVE_STATS], [1], [Define to 1 if stats is not disabled])]) 196 | AC_MSG_RESULT($disable_stats) 197 | 198 | AC_MSG_CHECKING([whether to enable zookeeper]) 199 | AC_ARG_WITH([zookeeper], 200 | AS_HELP_STRING([--with-zookeeper@<:@=no|yes|dir@:>@], 201 | [support zookeeper(default not support)]), 202 | [ 203 | if test "$withval" = "no"; then 204 | enable_zookeeper=no 205 | elif test "$withval" = "yes"; then 206 | enable_zookeeper=yes 207 | ZOOKEEPER_HOME="/usr" 208 | elif test "$withval" = ""; then 209 | enable_zookeeper=yes 210 | ZOOKEEPER_HOME="/usr" 211 | else 212 | enable_zookeeper=yes 213 | ZOOKEEPER_HOME="$withval" 214 | fi 215 | ], 216 | [enable_zookeeper=no]) 217 | AC_SUBST(ZOOKEEPER_HOME) 218 | AS_IF([test "x$enable_zookeeper" = xyes], 219 | [AC_DEFINE([HAVE_ZOOKEEPER], [1], [Define to 1 if zookeeper is enabled])], 220 | []) 221 | AC_MSG_RESULT($enable_zookeeper) 222 | 223 | AM_CONDITIONAL([APP_ZOOKEEPER], [test "x$enable_zookeeper" = "xyes"]) 224 | 225 | # Untar the yaml-0.1.4 in contrib/ before config.status is rerun 226 | AC_CONFIG_COMMANDS_PRE([tar xvfz contrib/yaml-0.1.4.tar.gz -C contrib]) 227 | 228 | # Call yaml-0.1.4 ./configure recursively 229 | AC_CONFIG_SUBDIRS([contrib/yaml-0.1.4]) 230 | 231 | # Define Makefiles 232 | AC_CONFIG_FILES([Makefile 233 | contrib/Makefile 234 | src/Makefile 235 | src/hashkit/Makefile 236 | src/proto/Makefile 237 | src/event/Makefile]) 238 | 239 | # Generate the "configure" script 240 | AC_OUTPUT 241 | -------------------------------------------------------------------------------- /contrib/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = yaml-0.1.4 2 | 3 | EXTRA_DIST = yaml-0.1.4.tar.gz 4 | -------------------------------------------------------------------------------- /contrib/yaml-0.1.4.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vipshop/twemproxy-vip/778e0d6b072312352ebf79d6c389671b5af07ab9/contrib/yaml-0.1.4.tar.gz -------------------------------------------------------------------------------- /contrib/yaml-0.1.4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything 2 | * 3 | 4 | # Except me 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /man/nutcracker.8: -------------------------------------------------------------------------------- 1 | .TH NUTCRACKER 8 "June 13, 2013" 2 | .SH NAME 3 | nutcracker \- Fast, light-weight proxy for memcached and Redis 4 | .SH SYNOPSIS 5 | .B nutcracker 6 | .RI [ options ] 7 | .SH DESCRIPTION 8 | \fBnutcracker\fP, also known as \fBtwemproxy\fP (pronounced "two-em-proxy"), is 9 | a fast and lightweight proxy for the memcached and Redis protocols. 10 | .PP 11 | It was primarily built to reduce the connection count on backend caching 12 | servers, but it has a number of features, such as: 13 | .IP \[bu] 14 | Maintains persistent server connections to backend servers. 15 | .IP \[bu] 16 | Enables pipelining of requests and responses. 17 | .IP \[bu] 18 | Supports multiple server pools simultaneously. 19 | .IP \[bu] 20 | Shard data automatically across multiple servers. 21 | .IP \[bu] 22 | Supports multiple hashing modes including consistent hashing and 23 | distribution. 24 | .IP \[bu] 25 | High-availability by disabling nodes on failures. 26 | .IP \[bu] 27 | Observability through stats exposed on stats monitoring port. 28 | .SH OPTIONS 29 | .TP 30 | .BR \-h ", " \-\-help 31 | Show usage information and exit. 32 | .TP 33 | .BR \-V ", " \-\-version 34 | Show version and exit. 35 | .TP 36 | .BR \-t ", " \-\-test-conf 37 | Test configuration for syntax errors and exit. 38 | .TP 39 | .BR \-D ", " \-\-describe-stats 40 | Print stats description and exit. 41 | .TP 42 | .BR \-v ", " \-\-verbosity=\fIN\fP 43 | Set logging level to \fIN\fP. (default: 5, min: 0, max: 11) 44 | .TP 45 | .BR \-o ", " \-\-output=\fIfilename\fP 46 | Set logging file to \fIfilename\fP. 47 | .TP 48 | .BR \-c ", " \-\-conf-file=\fIfilename\fP 49 | Set configuration file to \fIfilename\fP. 50 | .TP 51 | .BR \-s ", " \-\-stats-port=\fIport\fP 52 | Set stats monitoring port to \fIport\fP. 53 | (default: 22222) 54 | .TP 55 | .BR \-a ", " \-\-stats-addr=\fIaddress\fP 56 | Set stats monitoring IP to \fIaddress\fP. 57 | (default: 0.0.0.0) 58 | .TP 59 | .BR \-i ", " \-\-stats-interval=\fIinterval\fP 60 | Set stats aggregation interval in msec to \fIinterval\fP. 61 | (default: 30000 msec) 62 | .TP 63 | .BR \-m ", " \-\-mbuf-size=\fIsize\fP 64 | Set size of mbuf chunk in bytes to \fIsize\fP. (default: 16384 bytes) 65 | .TP 66 | .BR \-d ", " \-\-daemonize 67 | Run as a daemon. 68 | .TP 69 | .BR \-p ", " \-\-pid-file=\fIfilename\fP 70 | Set pid file to \fIfilename\fP. 71 | .SH SEE ALSO 72 | .BR memcached (8), 73 | .BR redis-server (1) 74 | .br 75 | .SH AUTHOR 76 | nutcracker was written by Twitter, Inc. 77 | -------------------------------------------------------------------------------- /notes/debug.txt: -------------------------------------------------------------------------------- 1 | - strace 2 | strace -o strace.txt -ttT -s 1024 -p `pgrep nutcracker` 3 | 4 | - libyaml (yaml-0.1.4) 5 | 6 | - yaml tokens: 7 | 8 | 0 YAML_NO_TOKEN, 9 | 1 YAML_STREAM_START_TOKEN, 10 | 2 YAML_STREAM_END_TOKEN, 11 | 3 YAML_VERSION_DIRECTIVE_TOKEN, 12 | 4 YAML_TAG_DIRECTIVE_TOKEN, 13 | 5 YAML_DOCUMENT_START_TOKEN, 14 | 6 YAML_DOCUMENT_END_TOKEN, 15 | 7 YAML_BLOCK_SEQUENCE_START_TOKEN, 16 | 8 YAML_BLOCK_MAPPING_START_TOKEN, 17 | 9 YAML_BLOCK_END_TOKEN, 18 | 10 YAML_FLOW_SEQUENCE_START_TOKEN, 19 | 11 YAML_FLOW_SEQUENCE_END_TOKEN, 20 | 12 YAML_FLOW_MAPPING_START_TOKEN, 21 | 13 YAML_FLOW_MAPPING_END_TOKEN, 22 | 14 YAML_BLOCK_ENTRY_TOKEN, 23 | 15 YAML_FLOW_ENTRY_TOKEN, 24 | 16 YAML_KEY_TOKEN, 25 | 17 YAML_VALUE_TOKEN, 26 | 18 YAML_ALIAS_TOKEN, 27 | 19 YAML_ANCHOR_TOKEN, 28 | 20 YAML_TAG_TOKEN, 29 | 21 YAML_SCALAR_TOKEN 30 | 31 | - yaml events 32 | 33 | 0 YAML_NO_EVENT, 34 | 1 YAML_STREAM_START_EVENT, 35 | 2 YAML_STREAM_END_EVENT, 36 | 3 YAML_DOCUMENT_START_EVENT, 37 | 4 YAML_DOCUMENT_END_EVENT, 38 | 5 YAML_ALIAS_EVENT, 39 | 6 YAML_SCALAR_EVENT, 40 | 7 YAML_SEQUENCE_START_EVENT, 41 | 8 YAML_SEQUENCE_END_EVENT, 42 | 9 YAML_MAPPING_START_EVENT, 43 | 10 YAML_MAPPING_END_EVENT 44 | 45 | - sys/queue.h 46 | 47 | queue.h is a generic linked list library adapted from BSD. It has three 48 | macro knobs that are useful for debugging: 49 | 50 | - QUEUE_MACRO_SCRUB nullifies links (next and prev pointers) of deleted 51 | elements and catches cases where we are attempting to do operations 52 | on an element that has already been unlinked. 53 | - QUEUE_MACRO_TRACE keeps track of __FILE__ and __LINE__ of last two 54 | updates to the list data structure. 55 | - QUEUE_MACRO_ASSERT verifies the sanity of list data structure on every 56 | operation. 57 | 58 | - valgrind 59 | valgrind --tool=memcheck --leak-check=yes 60 | 61 | - Core dump 62 | ulimit -c unlimited 63 | 64 | - Generate ENOMEM to test "Out of Memory" 65 | ulimit -m # limit maximum memory size 66 | ulimit -v # limit virtual memory 67 | 68 | - get nutcracker stats 69 | printf "" | socat - TCP:localhost:22222 | tee stats.txt 70 | printf "" | nc localhost 22222 | python -mjson.tool 71 | 72 | - Signalling and Logging 73 | SIGTTIN - To up the log level 74 | SIGTTOU - To down the log level 75 | SIGHUP - To reopen log file 76 | 77 | - Error codes: 78 | http://www.cs.utah.edu/dept/old/texinfo/glibc-manual-0.02/library_2.html 79 | /usr/include/asm-generic/errno-base.h 80 | /usr/include/asm-generic/errno.h 81 | 82 | - epoll (linux) 83 | 84 | union epoll_data { 85 | void *ptr; 86 | int fd; 87 | uint32_t u32; 88 | uint64_t u64; 89 | }; 90 | 91 | struct epoll_event { 92 | uint32_t events; /* epoll events */ 93 | struct epoll_data data; /* user data variable */ 94 | }; 95 | 96 | /* events */ 97 | EPOLLIN = 0x001, 98 | EPOLLPRI = 0x002, 99 | EPOLLOUT = 0x004, 100 | EPOLLERR = 0x008, 101 | EPOLLHUP = 0x010, 102 | EPOLLRDNORM = 0x040, 103 | EPOLLRDBAND = 0x080, 104 | EPOLLWRNORM = 0x100, 105 | EPOLLWRBAND = 0x200, 106 | EPOLLMSG = 0x400, 107 | EPOLLRDHUP = 0x2000, 108 | EPOLLONESHOT = (1 << 30), 109 | EPOLLET = (1 << 31) 110 | 111 | /* opcodes */ 112 | EPOLL_CTL_ADD = 1 /* add a file decriptor to the interface */ 113 | EPOLL_CTL_DEL = 2 /* remove a file decriptor from the interface */ 114 | EPOLL_CTL_MOD = 3 /* change file decriptor epoll_event structure */ 115 | 116 | - kqueue (bsd) 117 | 118 | struct kevent { 119 | uintptr_t ident; /* identifier for this event */ 120 | int16_t filter; /* filter for event */ 121 | uint16_t flags; /* general flags */ 122 | uint32_t fflags; /* filter-specific flags */ 123 | intptr_t data; /* filter-specific data */ 124 | void *udata; /* opaque user data identifier */ 125 | }; 126 | 127 | /* flags / events */ 128 | EV_ADD = 0x0001 /* action - add event to kq (implies enable) */ 129 | EV_DELETE = 0x0002 /* action - delete event from kq */ 130 | EV_ENABLE = 0x0004 /* action - enable event */ 131 | EV_DISABLE = 0x0008 /* action - disable event (not reported) */ 132 | EV_RECEIPT = 0x0040 /* action - force EV_ERROR on success, data == 0 */ 133 | 134 | EV_ONESHOT = 0x0010 /* flags - only report one occurrence */ 135 | EV_CLEAR = 0x0020 /* flags - clear event state after reporting */ 136 | EV_DISPATCH = 0x0080 /* flags - disable event after reporting */ 137 | EV_SYSFLAGS = 0xF000 /* flags - reserved by system */ 138 | EV_FLAG0 = 0x1000 /* flags - filter-specific flag */ 139 | EV_FLAG1 = 0x2000 /* flags - filter-specific flag */ 140 | 141 | EV_EOF = 0x8000 /* returned values - EOF detected */ 142 | EV_ERROR = 0x4000 /* returned values - error, data contains errno */ 143 | 144 | /* filters */ 145 | EVFILT_READ (-1) /* readable */ 146 | EVFILT_WRITE (-2) /* writable */ 147 | EVFILT_AIO (-3) /* attached to aio requests */ 148 | EVFILT_VNODE (-4) /* attached to vnodes */ 149 | EVFILT_PROC (-5) /* attached to struct proc */ 150 | EVFILT_SIGNAL (-6) /* attached to struct proc */ 151 | EVFILT_TIMER (-7) /* timers */ 152 | EVFILT_MACHPORT (-8) /* mach portsets */ 153 | EVFILT_FS (-9) /* filesystem events */ 154 | EVFILT_USER (-10) /* user events */ 155 | EVFILT_VM (-12) /* virtual memory events */ 156 | 157 | EV_CLEAR behaves like EPOLLET because it resets the event after it is 158 | returned; without this flag, the event would be repeatedly returned. 159 | 160 | - poll (unix) 161 | 162 | POLLIN 0x001 /* there is data to read */ 163 | POLLPRI 0x002 /* there is urgent data to read */ 164 | POLLOUT 0x004 /* writing now will not block */ 165 | 166 | POLLRDNORM 0x040 /* normal data may be read */ 167 | POLLRDBAND 0x080 /* priority data may be read */ 168 | POLLWRNORM 0x100 /* writing now will not block */ 169 | POLLWRBAND 0x200 /* priority data may be written */ 170 | 171 | POLLMSG 0x400 172 | POLLREMOVE 0x1000 173 | POLLRDHUP 0x2000 174 | 175 | POLLERR 0x008 /* error condition */ 176 | POLLHUP 0x010 /* hung up */ 177 | POLLNVAL 0x020 /* invalid polling request */ 178 | 179 | - event ports (solaris) 180 | 181 | typedef struct port_event { 182 | int portev_events; /* event data is source specific */ 183 | ushort_t portev_source; /* event source */ 184 | ushort_t portev_pad; /* port internal use */ 185 | uintptr_t portev_object; /* source specific object */ 186 | void *portev_user; /* user cookie */ 187 | } port_event_t; 188 | 189 | /* port sources */ 190 | PORT_SOURCE_AIO 1 191 | PORT_SOURCE_TIMER 2 192 | PORT_SOURCE_USER 3 193 | PORT_SOURCE_FD 4 194 | PORT_SOURCE_ALERT 5 195 | PORT_SOURCE_MQ 6 196 | PORT_SOURCE_FILE 7 197 | -------------------------------------------------------------------------------- /notes/kqueue.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vipshop/twemproxy-vip/778e0d6b072312352ebf79d6c389671b5af07ab9/notes/kqueue.pdf -------------------------------------------------------------------------------- /notes/memcache.txt: -------------------------------------------------------------------------------- 1 | - ascii: 2 | 3 | - Storage Commands (set, add, replace, append, prepend, cas): 4 | 5 | set [noreply]\r\n\r\n 6 | add [noreply]\r\n\r\n 7 | replace [noreply]\r\n\r\n 8 | append [noreply]\r\n\r\n 9 | prepend [noreply]\r\n\r\n 10 | 11 | cas [noreply]\r\n\r\n 12 | 13 | where, 14 | - uint32_t : data specific client side flags 15 | - uint32_t : expiration time (in seconds) 16 | - uint32_t : size of the data (in bytes) 17 | - uint8_t[]: data block 18 | - uint64_t 19 | 20 | - Retrival Commands (get, gets): 21 | 22 | get \r\n 23 | get []+\r\n 24 | 25 | gets \r\n 26 | gets []+\r\n 27 | 28 | - Delete Command (delete): 29 | 30 | delete [noreply]\r\n 31 | 32 | - Arithmetic Commands (incr, decr): 33 | 34 | incr [noreply]\r\n 35 | decr [noreply]\r\n 36 | 37 | where, 38 | - uint64_t 39 | 40 | - Misc Commands (quit) 41 | 42 | quit\r\n 43 | flush_all [] [noreply]\r\n 44 | version\r\n 45 | verbosity [noreply]\r\n 46 | 47 | - Statistics Commands 48 | 49 | stats\r\n 50 | stats \r\n 51 | 52 | - Error Responses: 53 | 54 | ERROR\r\n 55 | CLIENT_ERROR [error]\r\n 56 | SERVER_ERROR [error]\r\n 57 | 58 | where, 59 | ERROR means client sent a non-existent command name 60 | CLIENT_ERROR means that command sent by the client does not conform to the protocol 61 | SERVER_ERROR means that there was an error on the server side that made processing of the command impossible 62 | 63 | - Storage Command Responses: 64 | 65 | STORED\r\n 66 | NOT_STORED\r\n 67 | EXISTS\r\n 68 | NOT_FOUND\r\n 69 | 70 | where, 71 | STORED indicates success. 72 | NOT_STORED indicates the data was not stored because condition for an add or replace wasn't met. 73 | EXISTS indicates that the item you are trying to store with a cas has been modified since you last fetched it. 74 | NOT_FOUND indicates that the item you are trying to store with a cas does not exist. 75 | 76 | - Delete Command Response: 77 | 78 | NOT_FOUND\r\n 79 | DELETED\r\n 80 | 81 | - Retrival Responses: 82 | 83 | END\r\n 84 | VALUE []\r\n\r\nEND\r\n 85 | VALUE []\r\n\r\n[VALUE []\r\n]+\r\nEND\r\n 86 | 87 | - Arithmetic Responses: 88 | 89 | NOT_FOUND\r\n 90 | \r\n 91 | 92 | where, 93 | - uint64_t : new key value after incr or decr operation 94 | 95 | - Statistics Response 96 | [STAT \r\n]+END\r\n 97 | 98 | - Misc Response 99 | 100 | OK\r\n 101 | VERSION \r\n 102 | 103 | - Notes: 104 | - set always creates mapping irrespective of whether it is present on not. 105 | - add, adds only if the mapping is not present 106 | - replace, only replaces if the mapping is present 107 | - append and prepend command ignore flags and expiry values 108 | - noreply instructs the server to not send the reply even if there is an error. 109 | - decr of 0 is 0, while incr of UINT64_MAX is 0 110 | - maximum length of the key is 250 characters 111 | - expiry of 0 means that item never expires, though it could be evicted from the cache 112 | - non-zero expiry is either unix time (# seconds since 01/01/1970) or, 113 | offset in seconds from the current time (< 60 x 60 x 24 x 30 seconds = 30 days) 114 | - expiry time is with respect to the server (not client) 115 | - can be zero and when it is, the block is empty. 116 | 117 | - Thoughts: 118 | - ascii protocol is easier to debug - think using strace or tcpdump to see 119 | protocol on the wire, Or using telnet or netcat or socat to build memcache 120 | requests and responses 121 | http://stackoverflow.com/questions/2525188/are-binary-protocols-dead 122 | 123 | - http://news.ycombinator.com/item?id=1712788 124 | -------------------------------------------------------------------------------- /notes/socket.txt: -------------------------------------------------------------------------------- 1 | - int listen(int sockfd, int backlog); 2 | 3 | Linux: The backlog argument defines the maximum length to which the 4 | queue of pending connections for sockfd may grow. If a connection 5 | request arrives when the queue is full, the client may receive an error 6 | with an indication of ECONNREFUSED or, if the underlying protocol 7 | supports retransmission, the request may be ignored so that a later 8 | reattempt at connection succeeds. 9 | 10 | backlog specifies the queue length for completely established sockets 11 | waiting to be accepted, instead of the number of incomplete connection 12 | requests. The maximum length of the queue for incomplete sockets can 13 | be set using /proc/sys/net/ipv4/tcp_max_syn_backlog. 14 | 15 | If the backlog argument is greater than the value in /proc/sys/net/core/somaxconn, 16 | then it is silently truncated to that value; the default value in this 17 | file is 128. In kernels before 2.4.25, this limit was a hard coded value, 18 | SOMAXCONN, with the value 128. 19 | 20 | BSD: The backlog argument defines the maximum length the queue of pending 21 | connections may grow to. The real maximum queue length will be 1.5 times 22 | more than the value specified in the backlog argument. A subsequent 23 | listen() system call on the listening socket allows the caller to change 24 | the maximum queue length using a new backlog argument. If a connection 25 | request arrives with the queue full the client may receive an error with 26 | an indication of ECONNREFUSED, or, in the case of TCP, the connection 27 | will be silently dropped. 28 | 29 | The listen() system call appeared in 4.2BSD. The ability to configure 30 | the maximum backlog at run-time, and to use a negative backlog to request 31 | the maximum allowable value, was introduced in FreeBSD 2.2. 32 | 33 | - SO_LINGER (linger) socket option 34 | 35 | This option specifies what should happen when the socket of a type that 36 | promises reliable delivery still has untransmitted messages when it is 37 | closed 38 | 39 | struct linger { 40 | int l_onoff; /* nonzero to linger on close */ 41 | int l_linger; /* time to linger (in secs) */ 42 | }; 43 | 44 | l_onoff = 0 (default), then l_linger value is ignored and close returns 45 | immediately. But if there is any data still remaining in the socket send 46 | buffer, the system will try to deliver the data to the peer 47 | 48 | l_onoff = nonzero, then close blocks until data is transmitted or the 49 | l_linger timeout period expires 50 | a) l_linger = 0, TCP aborts connection, discards any data still remaining 51 | in the socket send buffer and sends RST to peer. This avoids the 52 | TCP's TIME_WAIT state 53 | b) l_linger = nonzero, then kernel will linger when socket is closed. If 54 | there is any pending data in the socket send buffer, the kernel waits 55 | until all the data is sent and acknowledged by peer TCP, or the 56 | linger time expires 57 | 58 | If a socket is set as nonblocking, it will not wait for close to complete 59 | even if linger time is nonzero 60 | 61 | - TIME_WAIT state 62 | 63 | The end that performs active close i.e. the end that sends the first FIN 64 | goes into TIME_WAIT state. After a FIN packet is sent to the peer and 65 | after that peers FIN/ACK arrvies and is ACKed, we go into a TIME_WAIT 66 | state. The duration that the end point remains in this state is 2 x MSL 67 | (maximum segment lifetime). The reason that the duration of the TIME_WAIT 68 | state is 2 x MSL is because the maximum amount of time a packet can wander 69 | around a network is assumed to be MSL seconds. The factor of 2 is for the 70 | round-trip. The recommended value for MSL is 120 seconds, but Berkeley 71 | derived implementations normally use 30 seconds instead. This means a 72 | TIME_WAIT delay is between 1 and 4 minutes. 73 | 74 | For Linux, the TIME_WAIT state duration is 1 minute (net/tcp.h): 75 | #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT 76 | * state, about 60 seconds */ 77 | 78 | TIME_WAIT state on client, combined with limited number of ephermeral ports 79 | available for TCP connections severely limits the rate at which new 80 | connections to the server can be created. On Linux, by default ephemeral 81 | ports are in the range of 32768 to 61000: 82 | 83 | $ cat /proc/sys/net/ipv4/ip_local_port_range 84 | 32768 61000 85 | 86 | So with a TIME_WAIT state duration of 1 minute, the maximum sustained rate 87 | for any client is ~470 new connections per second 88 | 89 | - TCP keepalive 90 | 91 | TCP keepalive packet (TCP packet with no data and the ACK flag turned on) 92 | is used to assert that connection is still up and running. This is useful 93 | because if the remote peer goes away without closing their connection, the 94 | keepalive probe will detect this and notice that the connection is broken 95 | even if there is no traffic on it. 96 | 97 | Imagine, the following scenario: You have a valid TCP connection established 98 | between two endpoints A and B. B terminates abnormally (think kernel panic 99 | or unplugging of network cable) without sending anything over the network 100 | to notify A that connection is broken. A, from its side, is ready to 101 | receive data, and has no idea that B has gone away. Now B comes back up 102 | again, and while A knows about a connection with B and still thinks that it 103 | active, B has no such idea. A tries to send data to B over a dead 104 | connection, and B replies with an RST packet, causing A to finally close 105 | the connection. So, without a keepalive probe A would never close the 106 | connection if it never sent data over it. 107 | 108 | - There are four socket functions that pass a socket address structure from 109 | the process to the kernel - bind, connect, sendmsg and sendto. These 110 | function are also responsible for passing the length of the sockaddr that 111 | they are passing (socklen_t). 112 | There are five socket functions that pass a socket from the kernel to the 113 | process - accept, recvfrom, recvmsg, getpeername, getsockname. The kernel 114 | is also responsible for returning the length of the sockaddr struct that 115 | it returns back to the userspace 116 | 117 | Different sockaddr structs: 118 | 1. sockaddr_in 119 | 2. sockaddr_in6 120 | 3. sockaddr_un 121 | 122 | Special types of in_addr_t 123 | /* Address to accept any incoming messages */ 124 | #define INADDR_ANY ((in_addr_t) 0x00000000) 125 | 126 | /* Address to send to all hosts */ 127 | #define INADDR_BROADCAST ((in_addr_t) 0xffffffff) 128 | 129 | /* Address indicating an error return */ 130 | #define INADDR_NONE ((in_addr_t) 0xffffffff) 131 | 132 | -------------------------------------------------------------------------------- /notes/twemproxy新增功能说明书.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vipshop/twemproxy-vip/778e0d6b072312352ebf79d6c389671b5af07ab9/notes/twemproxy新增功能说明书.docx -------------------------------------------------------------------------------- /notes/twemproxy新增功能说明书.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vipshop/twemproxy-vip/778e0d6b072312352ebf79d6c389671b5af07ab9/notes/twemproxy新增功能说明书.pdf -------------------------------------------------------------------------------- /scripts/benchmark-mget.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #coding: utf-8 3 | #file : test_mget.py 4 | #author : ning 5 | #date : 2014-04-01 13:15:48 6 | 7 | import os 8 | import re 9 | import commands 10 | 11 | ports = [ 12 | 4001, # before improve 13 | 4000, # after improve 14 | 2000 # redis 15 | ] 16 | 17 | def system(cmd): 18 | return commands.getoutput(cmd) 19 | 20 | def extra(regex, text): 21 | match = re.search(regex, text, re.DOTALL) 22 | if match: 23 | return match.group(1) 24 | 25 | def testit(): 26 | for mget_size in [10, 100, 1000, 10000]: 27 | for port in ports: 28 | cnt = 100*1000 / mget_size 29 | clients = 50 30 | if mget_size == 10000: 31 | clients = 2 32 | cmd = 'cd /home/ning/xredis/deploy-srcs/redis-2.8.3/src && ./redis-benchmark.%d -n %d -p %d -t mget -r 1000000000 -c %d' % (mget_size, cnt, port, clients) 33 | #print cmd 34 | rst = system(cmd) 35 | 36 | #100.00% <= 2 milliseconds 37 | #28089.89 requests per second 38 | rtime = extra('100.00% <= (\d+) milliseconds', rst) 39 | qps = extra('([\.\d]+) requests per second', rst) 40 | 41 | print 'mget_size=%d on %d: pqs: %s, rtime: %s' % (mget_size, port, qps, rtime) 42 | 43 | testit() 44 | -------------------------------------------------------------------------------- /scripts/multi_get.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=22123 4 | socatopt="-t 20 -T 20 -b 8193 -d -d " 5 | key="" 6 | keys="" 7 | get_command="" 8 | 9 | # build 10 | for i in `seq 1 512`; do 11 | if [ `expr $i % 2` -eq "0" ]; then 12 | key="foo" 13 | else 14 | key="bar" 15 | fi 16 | key=`printf "%s%d" "${key}" "${i}"` 17 | keys=`printf "%s %s" "${keys}" "${key}"` 18 | done 19 | 20 | get_command="get ${keys}\r\n" 21 | printf "%b" "$get_command" 22 | 23 | # read 24 | for i in `seq 1 16`; do 25 | printf "%b" "${get_command}" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-none,nonblock=1 1> /dev/null 2>&1 & 26 | done 27 | -------------------------------------------------------------------------------- /scripts/nutcracker.init: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | # 3 | # chkconfig: - 55 45 4 | # description: Twitter's twemproxy nutcracker 5 | # processname: nutcracker 6 | # config: /etc/sysconfig/nutcracker 7 | 8 | # Source function library. 9 | . /etc/rc.d/init.d/functions 10 | 11 | USER="nobody" 12 | OPTIONS="-d -c /etc/nutcracker/nutcracker.yml" 13 | 14 | if [ -f /etc/sysconfig/nutcracker ];then 15 | . /etc/sysconfig/nutcracker 16 | fi 17 | 18 | # Check that networking is up. 19 | if [ "$NETWORKING" = "no" ] 20 | then 21 | exit 0 22 | fi 23 | 24 | RETVAL=0 25 | prog="nutcracker" 26 | 27 | start () { 28 | echo -n $"Starting $prog: " 29 | daemon --user ${USER} ${prog} $OPTIONS 30 | RETVAL=$? 31 | echo 32 | [ $RETVAL -eq 0 ] && touch /var/lock/subsys/${prog} 33 | } 34 | stop () { 35 | echo -n $"Stopping $prog: " 36 | killproc ${prog} 37 | RETVAL=$? 38 | echo 39 | if [ $RETVAL -eq 0 ] ; then 40 | rm -f /var/lock/subsys/${prog} 41 | fi 42 | } 43 | 44 | restart () { 45 | stop 46 | start 47 | } 48 | 49 | 50 | # See how we were called. 51 | case "$1" in 52 | start) 53 | start 54 | ;; 55 | stop) 56 | stop 57 | ;; 58 | status) 59 | status ${prog} 60 | ;; 61 | restart|reload) 62 | restart 63 | ;; 64 | condrestart) 65 | [ -f /var/lock/subsys/nutcracker ] && restart || : 66 | ;; 67 | *) 68 | echo $"Usage: $0 {start|stop|status|restart|reload|condrestart}" 69 | exit 1 70 | esac 71 | 72 | exit $? 73 | 74 | -------------------------------------------------------------------------------- /scripts/nutcracker.init.debian: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | ### BEGIN INIT INFO 3 | # Provides: nutcracker 4 | # Required-Start: $network $remote_fs $local_fs 5 | # Required-Stop: $network $remote_fs $local_fs 6 | # Default-Start: 2 3 4 5 7 | # Default-Stop: 0 1 6 8 | # Short-Description: Stop/start nutcracker 9 | ### END INIT INFO 10 | 11 | PATH=/sbin:/usr/sbin:/bin:/usr/bin 12 | DESC=nutcracker 13 | NAME=nutcracker 14 | USER=nobody 15 | CONFFILE=/opt/nutcracker/etc/$NAME.yml 16 | LOGFILE=/opt/nutcracker/log/nutcracker.log 17 | DAEMON=/opt/nutcracker/sbin/nutcracker 18 | PIDFILE=/var/run/nutcracker/$NAME.pid 19 | STATSPORT=22222 20 | DAEMON_ARGS="-c $CONFFILE -o $LOGFILE -p $PIDFILE -s $STATSPORT -v 11 -m 2048 -d" 21 | #DAEMON_ARGS="-c $CONFFILE -p $PIDFILE -s $STATSPORT -d" 22 | SCRIPTNAME=/etc/init.d/$NAME 23 | 24 | ulimit -Hn 100000 25 | ulimit -Sn 100000 26 | 27 | [ -x $DAEMON ] || exit 0 28 | 29 | [ -r /etc/default/$NAME ] && . /etc/default/$NAME 30 | 31 | . /lib/init/vars.sh 32 | 33 | . /lib/lsb/init-functions 34 | 35 | do_start() 36 | { 37 | mkdir -p /var/run/nutcracker 38 | touch $PIDFILE 39 | chown $USER:$USER -R /var/run/nutcracker 40 | chmod 755 /var/run/nutcracker 41 | 42 | echo -n "Starting ${NAME}: " 43 | start-stop-daemon --start --quiet -m --pidfile $PIDFILE --chuid $USER:$USER --exec $DAEMON -- \ 44 | $DAEMON_ARGS 45 | case "$?" in 46 | 0|1) echo "STARTED." ;; 47 | 2) echo "FAILED." ;; 48 | esac 49 | } 50 | 51 | do_stop() 52 | { 53 | echo -n "Stopping ${NAME}: " 54 | start-stop-daemon --stop --quiet --pidfile $PIDFILE --exec $DAEMON || true 55 | 56 | case "$?" in 57 | 0|1) echo "STOPPED.";; 58 | 2) echo "FAILED." ;; 59 | esac 60 | } 61 | 62 | case "$1" in 63 | start) 64 | do_start 65 | ;; 66 | stop) 67 | do_stop 68 | ;; 69 | status) 70 | status_of_proc -p $PIDFILE "$DAEMON" nutcracker && exit 0 || exit $? 71 | ;; 72 | restart) 73 | do_stop 74 | do_start 75 | ;; 76 | *) 77 | echo "Usage: $SCRIPTNAME {start|stop|status|restart}" >&2 78 | exit 3 79 | ;; 80 | esac 81 | 82 | exit 83 | $RETVAL 84 | -------------------------------------------------------------------------------- /scripts/nutcracker.spec: -------------------------------------------------------------------------------- 1 | Summary: Twitter's nutcracker redis and memcached proxy 2 | Name: nutcracker 3 | Version: 0.3.0 4 | Release: 1 5 | 6 | URL: https://github.com/twitter/twemproxy/ 7 | Source0: %{name}-%{version}.tar.gz 8 | License: Apache License 2.0 9 | Group: System Environment/Libraries 10 | Packager: Tom Parrott 11 | BuildRoot: %{_tmppath}/%{name}-root 12 | 13 | BuildRequires: autoconf 14 | BuildRequires: automake 15 | BuildRequires: libtool 16 | 17 | %description 18 | twemproxy (pronounced "two-em-proxy"), aka nutcracker is a fast and lightweight proxy for memcached and redis protocol. 19 | It was primarily built to reduce the connection count on the backend caching servers. 20 | 21 | %prep 22 | %setup -q 23 | %if 0%{?rhel} == 6 24 | sed -i 's/2.64/2.63/g' configure.ac 25 | %endif 26 | autoreconf -fvi 27 | 28 | %build 29 | 30 | %configure 31 | %__make 32 | 33 | %install 34 | [ %{buildroot} != "/" ] && rm -rf %{buildroot} 35 | 36 | %makeinstall PREFIX=%{buildroot} 37 | 38 | #Install init script 39 | %{__install} -p -D -m 0755 scripts/%{name}.init %{buildroot}%{_initrddir}/%{name} 40 | 41 | #Install example config file 42 | %{__install} -p -D -m 0644 conf/%{name}.yml %{buildroot}%{_sysconfdir}/%{name}/%{name}.yml 43 | 44 | %post 45 | /sbin/chkconfig --add %{name} 46 | 47 | %preun 48 | if [ $1 = 0 ]; then 49 | /sbin/service %{name} stop > /dev/null 2>&1 50 | /sbin/chkconfig --del %{name} 51 | fi 52 | 53 | %clean 54 | [ %{buildroot} != "/" ] && rm -rf %{buildroot} 55 | 56 | %files 57 | %defattr(-,root,root,-) 58 | %if 0%{?rhel} == 6 59 | /usr/sbin/nutcracker 60 | %else 61 | /usr/bin/nutcracker 62 | %endif 63 | %{_initrddir}/%{name} 64 | %{_mandir}/man8/nutcracker.8.gz 65 | %config(noreplace)%{_sysconfdir}/%{name}/%{name}.yml 66 | 67 | %changelog 68 | * Fri Dec 20 2013 Manju Rajashekhar 69 | - twemproxy: version 0.3.0 release 70 | - SRANDMEMBER support for the optional count argument (mkhq) 71 | - Handle case where server responds while the request is still being sent (jdi-tagged) 72 | - event ports (solaris/smartos) support 73 | - add timestamp when the server was ejected 74 | - support for set ex/px/nx/xx for redis 2.6.12 and up (ypocat) 75 | - kqueue (bsd) support (ferenyx) 76 | - fix parsing redis response to accept integer reply (charsyam) 77 | 78 | * Tue Jul 30 2013 Tait Clarridge 79 | - Rebuild SPEC to work with CentOS 80 | - Added buildrequires if building with mock/koji 81 | 82 | * Tue Apr 23 2013 Manju Rajashekhar 83 | - twemproxy: version 0.2.4 release 84 | - redis keys must be less than mbuf_data_size() in length (fifsky) 85 | - Adds support for DUMP/RESTORE commands in Redis (remotezygote) 86 | - Use of the weight value in the modula distribution (mezzatto) 87 | - Add support to unix socket connections to servers (mezzatto) 88 | - only check for duplicate server name and not 'host:port:weight' when 'name' is configured 89 | - crc16 hash support added (mezzatto) 90 | 91 | * Thu Jan 31 2013 Manju Rajashekhar 92 | - twemproxy: version 0.2.3 release 93 | - RPOPLPUSH, SDIFF, SDIFFSTORE, SINTER, SINTERSTORE, SMOVE, SUNION, SUNIONSTORE, ZINTERSTORE, and ZUNIONSTORE support (dcartoon) 94 | - EVAL and EVALSHA support (ferenyx) 95 | - exit 1 if configuration file is invalid (cofyc) 96 | - return non-zero exit status when nutcracker cannot start for some reason 97 | - use server names in stats (charsyam) 98 | - Fix failure to resolve long FQDN name resolve (conmame) 99 | - add support for hash tags 100 | 101 | * Thu Oct 18 2012 Manju Rajashekhar 102 | - twemproxy: version 0.2.2 release 103 | - fix the off-by-one error when calculating redis key length 104 | 105 | * Fri Oct 12 2012 Manju Rajashekhar 106 | - twemproxy: version 0.2.1 release 107 | - don't use buf in conf_add_server 108 | - allow an optional instance name for consistent hashing (charsyam) 109 | - add --stats-addr=S option 110 | - add stats-bind-any -a option (charsyam) 111 | -------------------------------------------------------------------------------- /scripts/pipelined_read.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | socatopt="-t 4 -T 4 -b 8193 -d -d " 4 | 5 | get_commands="" 6 | 7 | # build 8 | for i in `seq 1 128`; do 9 | if [ `expr $i % 2` -eq "0" ]; then 10 | key="foo" 11 | else 12 | key="bar" 13 | fi 14 | key=`printf "%s%d" "${key}" "${i}"` 15 | 16 | get_command="get ${key}\r\n" 17 | get_commands=`printf "%s%s" "${get_commands}" "${get_command}"` 18 | done 19 | 20 | # read 21 | for i in `seq 1 64`; do 22 | printf "%b" "$get_commands" | socat ${socatopt} - TCP:localhost:22123,nodelay,shut-none,nonblock=1 1> /dev/null 2>&1 & 23 | done 24 | -------------------------------------------------------------------------------- /scripts/pipelined_write.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | socatopt="-t 1 -T 1 -b 16384" 4 | 5 | val=`echo 6^6^6 | bc` 6 | val=`printf "%s" "${val}"` 7 | vallen=`printf "%s" "${val}" | wc -c` 8 | set_command="" 9 | set_commands="" 10 | 11 | # build 12 | for i in `seq 1 64`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | 20 | set_command="set ${key} 0 0 ${vallen}\r\n${val}\r\n" 21 | set_commands=`printf "%s%s" "${set_commands}" "${set_command}"` 22 | done 23 | 24 | printf "%b" "$set_commands" > /tmp/socat.input 25 | 26 | # write 27 | for i in `seq 1 16`; do 28 | cat /tmp/socat.input | socat ${socatopt} - TCP:localhost:22123,nodelay,shut-down,nonblock=1 & 29 | done 30 | -------------------------------------------------------------------------------- /scripts/populate_memcached.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | port=22123 4 | socatopt="-t 1 -T 1 -b 65537" 5 | 6 | val=`echo 6^6^6 | bc` 7 | val=`printf "%s\r\n" "${val}"` 8 | vallen=`printf "%s" "${val}" | wc -c` 9 | set_command="" 10 | 11 | # build 12 | for i in `seq 1 512`; do 13 | if [ `expr $i % 2` -eq "0" ]; then 14 | key="foo" 15 | else 16 | key="bar" 17 | fi 18 | key=`printf "%s%d" "${key}" "${i}"` 19 | 20 | set_command="set ${key} 0 0 ${vallen}\r\n${val}\r\n" 21 | 22 | printf "%b" "$set_command" | socat ${socatopt} - TCP:localhost:${port},nodelay,shut-down,nonblock=1 & 23 | done 24 | 25 | -------------------------------------------------------------------------------- /scripts/redis-check.py: -------------------------------------------------------------------------------- 1 | import redis 2 | 3 | range=100 4 | factor=32 5 | port=22121 6 | 7 | r = redis.StrictRedis(host='localhost', port=port, db=0) 8 | 9 | # lrange 10 | print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] 11 | print [r.lpush('lfoo', str(x)*factor) for x in xrange(1, range)] 12 | print [r.lrange('lfoo', 0, x) for x in xrange(1, range)] 13 | print r.delete('lfoo') 14 | 15 | # del 16 | print [r.set('foo' + str(x), str(x)*factor) for x in xrange(1, range)] 17 | keys = ['foo' + str(x) for x in xrange(1, range)] 18 | print [r.delete(keys) for x in xrange(1, range)] 19 | 20 | # mget 21 | print [r.set('foo' + str(x), str(x)*100) for x in xrange(1, range)] 22 | keys = ['foo' + str(x) for x in xrange(1, range)] 23 | print [r.mget(keys) for x in xrange(1, range)] 24 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = 4 | if !OS_SOLARIS 5 | AM_CPPFLAGS += -D_GNU_SOURCE 6 | endif 7 | AM_CPPFLAGS += -I $(top_srcdir)/src/hashkit 8 | AM_CPPFLAGS += -I $(top_srcdir)/src/proto 9 | AM_CPPFLAGS += -I $(top_srcdir)/src/event 10 | AM_CPPFLAGS += -I $(top_srcdir)/contrib/yaml-0.1.4/include 11 | 12 | if APP_ZOOKEEPER 13 | AM_CPPFLAGS += -I $(ZOOKEEPER_HOME)/include/zookeeper 14 | endif 15 | 16 | AM_CFLAGS = 17 | # about -fno-strict-aliasing: https://github.com/twitter/twemproxy/issues/276 18 | AM_CFLAGS += -fno-strict-aliasing 19 | AM_CFLAGS += -Wall -Wshadow 20 | AM_CFLAGS += -Wpointer-arith 21 | AM_CFLAGS += -Winline 22 | AM_CFLAGS += -Wunused-function -Wunused-variable -Wunused-value 23 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 24 | AM_CFLAGS += -Wconversion -Wsign-compare 25 | AM_CFLAGS += -Wstrict-prototypes -Wmissing-prototypes -Wredundant-decls -Wmissing-declarations 26 | 27 | AM_LDFLAGS = 28 | AM_LDFLAGS += -lm -lpthread -rdynamic 29 | if OS_SOLARIS 30 | AM_LDFLAGS += -lnsl -lsocket 31 | endif 32 | 33 | if APP_ZOOKEEPER 34 | AM_LDFLAGS += -L${ZOOKEEPER_HOME}/lib -lzookeeper_mt 35 | endif 36 | 37 | SUBDIRS = hashkit proto event 38 | 39 | sbin_PROGRAMS = nutcracker 40 | 41 | nutcracker_SOURCES = \ 42 | nc_core.c nc_core.h \ 43 | nc_connection.c nc_connection.h \ 44 | nc_client.c nc_client.h \ 45 | nc_server.c nc_server.h \ 46 | nc_proxy.c nc_proxy.h \ 47 | nc_message.c nc_message.h \ 48 | nc_request.c \ 49 | nc_response.c \ 50 | nc_mbuf.c nc_mbuf.h \ 51 | nc_conf.c nc_conf.h \ 52 | nc_stats.c nc_stats.h \ 53 | nc_signal.c nc_signal.h \ 54 | nc_rbtree.c nc_rbtree.h \ 55 | nc_log.c nc_log.h \ 56 | nc_string.c nc_string.h \ 57 | nc_array.c nc_array.h \ 58 | nc_util.c nc_util.h \ 59 | nc_queue.h \ 60 | nc.c 61 | 62 | if APP_ZOOKEEPER 63 | nutcracker_SOURCES += nc_zookeeper.h nc_zookeeper.c 64 | endif 65 | 66 | nutcracker_LDADD = $(top_builddir)/src/hashkit/libhashkit.a 67 | nutcracker_LDADD += $(top_builddir)/src/proto/libproto.a 68 | nutcracker_LDADD += $(top_builddir)/src/event/libevent.a 69 | nutcracker_LDADD += $(top_builddir)/contrib/yaml-0.1.4/src/.libs/libyaml.a 70 | -------------------------------------------------------------------------------- /src/event/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libevent.a 9 | 10 | noinst_HEADERS = nc_event.h 11 | 12 | libevent_a_SOURCES = \ 13 | nc_epoll.c \ 14 | nc_kqueue.c \ 15 | nc_evport.c 16 | 17 | -------------------------------------------------------------------------------- /src/event/nc_epoll.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #ifdef NC_HAVE_EPOLL 21 | 22 | #include 23 | 24 | struct event_base * 25 | event_base_create(int nevent, event_cb_t cb) 26 | { 27 | struct event_base *evb; 28 | int status, ep; 29 | struct epoll_event *event; 30 | 31 | ASSERT(nevent > 0); 32 | 33 | ep = epoll_create(nevent); 34 | if (ep < 0) { 35 | log_error("epoll create of size %d failed: %s", nevent, strerror(errno)); 36 | return NULL; 37 | } 38 | 39 | event = nc_calloc(nevent, sizeof(*event)); 40 | if (event == NULL) { 41 | status = close(ep); 42 | if (status < 0) { 43 | log_error("close e %d failed, ignored: %s", ep, strerror(errno)); 44 | } 45 | return NULL; 46 | } 47 | 48 | evb = nc_alloc(sizeof(*evb)); 49 | if (evb == NULL) { 50 | nc_free(event); 51 | status = close(ep); 52 | if (status < 0) { 53 | log_error("close e %d failed, ignored: %s", ep, strerror(errno)); 54 | } 55 | return NULL; 56 | } 57 | 58 | evb->ep = ep; 59 | evb->event = event; 60 | evb->nevent = nevent; 61 | evb->cb = cb; 62 | 63 | log_debug(LOG_INFO, "e %d with nevent %d", evb->ep, evb->nevent); 64 | 65 | return evb; 66 | } 67 | 68 | void 69 | event_base_destroy(struct event_base *evb) 70 | { 71 | int status; 72 | 73 | if (evb == NULL) { 74 | return; 75 | } 76 | 77 | ASSERT(evb->ep > 0); 78 | 79 | nc_free(evb->event); 80 | 81 | status = close(evb->ep); 82 | if (status < 0) { 83 | log_error("close e %d failed, ignored: %s", evb->ep, strerror(errno)); 84 | } 85 | evb->ep = -1; 86 | 87 | nc_free(evb); 88 | } 89 | 90 | int 91 | event_add_in(struct event_base *evb, struct conn *c) 92 | { 93 | int status; 94 | struct epoll_event event; 95 | int ep = evb->ep; 96 | 97 | ASSERT(ep > 0); 98 | ASSERT(c != NULL); 99 | ASSERT(c->sd > 0); 100 | 101 | if (c->recv_active) { 102 | return 0; 103 | } 104 | 105 | event.events = (uint32_t)(EPOLLIN | EPOLLET); 106 | event.data.ptr = c; 107 | 108 | status = epoll_ctl(ep, EPOLL_CTL_MOD, c->sd, &event); 109 | if (status < 0) { 110 | log_error("epoll ctl on e %d sd %d failed: %s", ep, c->sd, 111 | strerror(errno)); 112 | } else { 113 | c->recv_active = 1; 114 | } 115 | 116 | return status; 117 | } 118 | 119 | int 120 | event_del_in(struct event_base *evb, struct conn *c) 121 | { 122 | return 0; 123 | } 124 | 125 | int 126 | event_add_out(struct event_base *evb, struct conn *c) 127 | { 128 | int status; 129 | struct epoll_event event; 130 | int ep = evb->ep; 131 | 132 | ASSERT(ep > 0); 133 | ASSERT(c != NULL); 134 | ASSERT(c->sd > 0); 135 | ASSERT(c->recv_active); 136 | 137 | if (c->send_active) { 138 | return 0; 139 | } 140 | 141 | event.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); 142 | event.data.ptr = c; 143 | 144 | status = epoll_ctl(ep, EPOLL_CTL_MOD, c->sd, &event); 145 | if (status < 0) { 146 | log_error("epoll ctl on e %d sd %d failed: %s", ep, c->sd, 147 | strerror(errno)); 148 | } else { 149 | c->send_active = 1; 150 | } 151 | 152 | return status; 153 | } 154 | 155 | int 156 | event_del_out(struct event_base *evb, struct conn *c) 157 | { 158 | int status; 159 | struct epoll_event event; 160 | int ep = evb->ep; 161 | 162 | ASSERT(ep > 0); 163 | ASSERT(c != NULL); 164 | ASSERT(c->sd > 0); 165 | ASSERT(c->recv_active); 166 | 167 | if (!c->send_active) { 168 | return 0; 169 | } 170 | 171 | event.events = (uint32_t)(EPOLLIN | EPOLLET); 172 | event.data.ptr = c; 173 | 174 | status = epoll_ctl(ep, EPOLL_CTL_MOD, c->sd, &event); 175 | if (status < 0) { 176 | log_error("epoll ctl on e %d sd %d failed: %s", ep, c->sd, 177 | strerror(errno)); 178 | } else { 179 | c->send_active = 0; 180 | } 181 | 182 | return status; 183 | } 184 | 185 | int 186 | event_add_conn(struct event_base *evb, struct conn *c) 187 | { 188 | int status; 189 | struct epoll_event event; 190 | int ep = evb->ep; 191 | 192 | ASSERT(ep > 0); 193 | ASSERT(c != NULL); 194 | ASSERT(c->sd > 0); 195 | 196 | event.events = (uint32_t)(EPOLLIN | EPOLLOUT | EPOLLET); 197 | event.data.ptr = c; 198 | 199 | status = epoll_ctl(ep, EPOLL_CTL_ADD, c->sd, &event); 200 | if (status < 0) { 201 | log_error("epoll ctl on e %d sd %d failed: %s", ep, c->sd, 202 | strerror(errno)); 203 | } else { 204 | c->send_active = 1; 205 | c->recv_active = 1; 206 | } 207 | 208 | return status; 209 | } 210 | 211 | int 212 | event_del_conn(struct event_base *evb, struct conn *c) 213 | { 214 | int status; 215 | int ep = evb->ep; 216 | 217 | ASSERT(ep > 0); 218 | ASSERT(c != NULL); 219 | ASSERT(c->sd > 0); 220 | 221 | status = epoll_ctl(ep, EPOLL_CTL_DEL, c->sd, NULL); 222 | if (status < 0) { 223 | log_error("epoll ctl on e %d sd %d failed: %s", ep, c->sd, 224 | strerror(errno)); 225 | } else { 226 | c->recv_active = 0; 227 | c->send_active = 0; 228 | } 229 | 230 | return status; 231 | } 232 | 233 | int 234 | event_wait(struct event_base *evb, int timeout) 235 | { 236 | int ep = evb->ep; 237 | struct epoll_event *event = evb->event; 238 | int nevent = evb->nevent; 239 | 240 | ASSERT(ep > 0); 241 | ASSERT(event != NULL); 242 | ASSERT(nevent > 0); 243 | 244 | for (;;) { 245 | int i, nsd; 246 | 247 | nsd = epoll_wait(ep, event, nevent, timeout); 248 | if (nsd > 0) { 249 | for (i = 0; i < nsd; i++) { 250 | struct epoll_event *ev = &evb->event[i]; 251 | uint32_t events = 0; 252 | 253 | log_debug(LOG_VVERB, "epoll %04"PRIX32" triggered on conn %p", 254 | ev->events, ev->data.ptr); 255 | 256 | if (ev->events & EPOLLERR) { 257 | events |= EVENT_ERR; 258 | } 259 | 260 | if (ev->events & (EPOLLIN | EPOLLHUP)) { 261 | events |= EVENT_READ; 262 | } 263 | 264 | if (ev->events & EPOLLOUT) { 265 | events |= EVENT_WRITE; 266 | } 267 | 268 | if (evb->cb != NULL) { 269 | evb->cb(ev->data.ptr, events); 270 | } 271 | } 272 | return nsd; 273 | } 274 | 275 | if (nsd == 0) { 276 | if (timeout == -1) { 277 | log_error("epoll wait on e %d with %d events and %d timeout " 278 | "returned no events", ep, nevent, timeout); 279 | return -1; 280 | } 281 | 282 | return 0; 283 | } 284 | 285 | if (errno == EINTR) { 286 | continue; 287 | } 288 | 289 | log_error("epoll wait on e %d with %d events failed: %s", ep, nevent, 290 | strerror(errno)); 291 | return -1; 292 | } 293 | 294 | NOT_REACHED(); 295 | } 296 | 297 | void 298 | event_loop_stats(event_stats_cb_t cb, void *arg) 299 | { 300 | struct stats *st = arg; 301 | int status, ep; 302 | struct epoll_event ev; 303 | 304 | ep = epoll_create(1); 305 | if (ep < 0) { 306 | log_error("epoll create failed: %s", strerror(errno)); 307 | return; 308 | } 309 | 310 | ev.data.fd = st->sd; 311 | ev.events = EPOLLIN; 312 | 313 | status = epoll_ctl(ep, EPOLL_CTL_ADD, st->sd, &ev); 314 | if (status < 0) { 315 | log_error("epoll ctl on e %d sd %d failed: %s", ep, st->sd, 316 | strerror(errno)); 317 | goto error; 318 | } 319 | 320 | for (;;) { 321 | int n; 322 | 323 | n = epoll_wait(ep, &ev, 1, st->interval); 324 | if (n < 0) { 325 | if (errno == EINTR) { 326 | continue; 327 | } 328 | log_error("epoll wait on e %d with m %d failed: %s", ep, 329 | st->sd, strerror(errno)); 330 | break; 331 | } 332 | 333 | cb(st, &n); 334 | } 335 | 336 | error: 337 | status = close(ep); 338 | if (status < 0) { 339 | log_error("close e %d failed, ignored: %s", ep, strerror(errno)); 340 | } 341 | ep = -1; 342 | } 343 | 344 | #endif /* NC_HAVE_EPOLL */ 345 | -------------------------------------------------------------------------------- /src/event/nc_event.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_EVENT_H_ 19 | #define _NC_EVENT_H_ 20 | 21 | #include 22 | 23 | #define EVENT_SIZE 1024 24 | 25 | #define EVENT_READ 0x0000ff 26 | #define EVENT_WRITE 0x00ff00 27 | #define EVENT_ERR 0xff0000 28 | 29 | typedef int (*event_cb_t)(void *, uint32_t); 30 | typedef void (*event_stats_cb_t)(void *, void *); 31 | 32 | #ifdef NC_HAVE_KQUEUE 33 | 34 | struct event_base { 35 | int kq; /* kernel event queue descriptor */ 36 | 37 | struct kevent *change; /* change[] - events we want to monitor */ 38 | int nchange; /* # change */ 39 | 40 | struct kevent *event; /* event[] - events that were triggered */ 41 | int nevent; /* # event */ 42 | int nreturned; /* # event placed in event[] */ 43 | int nprocessed; /* # event processed from event[] */ 44 | 45 | event_cb_t cb; /* event callback */ 46 | }; 47 | 48 | #elif NC_HAVE_EPOLL 49 | 50 | struct event_base { 51 | int ep; /* epoll descriptor */ 52 | 53 | struct epoll_event *event; /* event[] - events that were triggered */ 54 | int nevent; /* # event */ 55 | 56 | event_cb_t cb; /* event callback */ 57 | }; 58 | 59 | #elif NC_HAVE_EVENT_PORTS 60 | 61 | #include 62 | 63 | struct event_base { 64 | int evp; /* event port descriptor */ 65 | 66 | port_event_t *event; /* event[] - events that were triggered */ 67 | int nevent; /* # event */ 68 | 69 | event_cb_t cb; /* event callback */ 70 | }; 71 | 72 | #else 73 | # error missing scalable I/O event notification mechanism 74 | #endif 75 | 76 | struct event_base *event_base_create(int size, event_cb_t cb); 77 | void event_base_destroy(struct event_base *evb); 78 | 79 | int event_add_in(struct event_base *evb, struct conn *c); 80 | int event_del_in(struct event_base *evb, struct conn *c); 81 | int event_add_out(struct event_base *evb, struct conn *c); 82 | int event_del_out(struct event_base *evb, struct conn *c); 83 | int event_add_conn(struct event_base *evb, struct conn *c); 84 | int event_del_conn(struct event_base *evb, struct conn *c); 85 | int event_wait(struct event_base *evb, int timeout); 86 | void event_loop_stats(event_stats_cb_t cb, void *arg); 87 | 88 | #endif /* _NC_EVENT_H */ 89 | -------------------------------------------------------------------------------- /src/hashkit/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libhashkit.a 9 | 10 | noinst_HEADERS = nc_hashkit.h 11 | 12 | libhashkit_a_SOURCES = \ 13 | nc_crc16.c \ 14 | nc_crc32.c \ 15 | nc_fnv.c \ 16 | nc_hsieh.c \ 17 | nc_jenkins.c \ 18 | nc_ketama.c \ 19 | nc_md5.c \ 20 | nc_modula.c \ 21 | nc_murmur.c \ 22 | nc_one_at_a_time.c \ 23 | nc_random.c 24 | -------------------------------------------------------------------------------- /src/hashkit/nc_crc16.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | static const uint16_t crc16tab[256] = { 21 | 0x0000, 0x1021, 0x2042, 0x3063, 0x4084, 0x50a5, 0x60c6, 0x70e7, 22 | 0x8108, 0x9129, 0xa14a, 0xb16b, 0xc18c, 0xd1ad, 0xe1ce, 0xf1ef, 23 | 0x1231, 0x0210, 0x3273, 0x2252, 0x52b5, 0x4294, 0x72f7, 0x62d6, 24 | 0x9339, 0x8318, 0xb37b, 0xa35a, 0xd3bd, 0xc39c, 0xf3ff, 0xe3de, 25 | 0x2462, 0x3443, 0x0420, 0x1401, 0x64e6, 0x74c7, 0x44a4, 0x5485, 26 | 0xa56a, 0xb54b, 0x8528, 0x9509, 0xe5ee, 0xf5cf, 0xc5ac, 0xd58d, 27 | 0x3653, 0x2672, 0x1611, 0x0630, 0x76d7, 0x66f6, 0x5695, 0x46b4, 28 | 0xb75b, 0xa77a, 0x9719, 0x8738, 0xf7df, 0xe7fe, 0xd79d, 0xc7bc, 29 | 0x48c4, 0x58e5, 0x6886, 0x78a7, 0x0840, 0x1861, 0x2802, 0x3823, 30 | 0xc9cc, 0xd9ed, 0xe98e, 0xf9af, 0x8948, 0x9969, 0xa90a, 0xb92b, 31 | 0x5af5, 0x4ad4, 0x7ab7, 0x6a96, 0x1a71, 0x0a50, 0x3a33, 0x2a12, 32 | 0xdbfd, 0xcbdc, 0xfbbf, 0xeb9e, 0x9b79, 0x8b58, 0xbb3b, 0xab1a, 33 | 0x6ca6, 0x7c87, 0x4ce4, 0x5cc5, 0x2c22, 0x3c03, 0x0c60, 0x1c41, 34 | 0xedae, 0xfd8f, 0xcdec, 0xddcd, 0xad2a, 0xbd0b, 0x8d68, 0x9d49, 35 | 0x7e97, 0x6eb6, 0x5ed5, 0x4ef4, 0x3e13, 0x2e32, 0x1e51, 0x0e70, 36 | 0xff9f, 0xefbe, 0xdfdd, 0xcffc, 0xbf1b, 0xaf3a, 0x9f59, 0x8f78, 37 | 0x9188, 0x81a9, 0xb1ca, 0xa1eb, 0xd10c, 0xc12d, 0xf14e, 0xe16f, 38 | 0x1080, 0x00a1, 0x30c2, 0x20e3, 0x5004, 0x4025, 0x7046, 0x6067, 39 | 0x83b9, 0x9398, 0xa3fb, 0xb3da, 0xc33d, 0xd31c, 0xe37f, 0xf35e, 40 | 0x02b1, 0x1290, 0x22f3, 0x32d2, 0x4235, 0x5214, 0x6277, 0x7256, 41 | 0xb5ea, 0xa5cb, 0x95a8, 0x8589, 0xf56e, 0xe54f, 0xd52c, 0xc50d, 42 | 0x34e2, 0x24c3, 0x14a0, 0x0481, 0x7466, 0x6447, 0x5424, 0x4405, 43 | 0xa7db, 0xb7fa, 0x8799, 0x97b8, 0xe75f, 0xf77e, 0xc71d, 0xd73c, 44 | 0x26d3, 0x36f2, 0x0691, 0x16b0, 0x6657, 0x7676, 0x4615, 0x5634, 45 | 0xd94c, 0xc96d, 0xf90e, 0xe92f, 0x99c8, 0x89e9, 0xb98a, 0xa9ab, 46 | 0x5844, 0x4865, 0x7806, 0x6827, 0x18c0, 0x08e1, 0x3882, 0x28a3, 47 | 0xcb7d, 0xdb5c, 0xeb3f, 0xfb1e, 0x8bf9, 0x9bd8, 0xabbb, 0xbb9a, 48 | 0x4a75, 0x5a54, 0x6a37, 0x7a16, 0x0af1, 0x1ad0, 0x2ab3, 0x3a92, 49 | 0xfd2e, 0xed0f, 0xdd6c, 0xcd4d, 0xbdaa, 0xad8b, 0x9de8, 0x8dc9, 50 | 0x7c26, 0x6c07, 0x5c64, 0x4c45, 0x3ca2, 0x2c83, 0x1ce0, 0x0cc1, 51 | 0xef1f, 0xff3e, 0xcf5d, 0xdf7c, 0xaf9b, 0xbfba, 0x8fd9, 0x9ff8, 52 | 0x6e17, 0x7e36, 0x4e55, 0x5e74, 0x2e93, 0x3eb2, 0x0ed1, 0x1ef0, 53 | }; 54 | 55 | uint32_t 56 | hash_crc16(const char *key, size_t key_length) 57 | { 58 | uint64_t x; 59 | uint32_t crc = 0; 60 | 61 | for (x=0; x < key_length; x++) { 62 | crc = (crc << 8) ^ crc16tab[((crc >> 8) ^ *key++) & 0x00ff]; 63 | } 64 | 65 | return crc; 66 | } 67 | -------------------------------------------------------------------------------- /src/hashkit/nc_crc32.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * The crc32 functions and data was originally written by Spencer 20 | * Garrett and was gleaned from the PostgreSQL source 21 | * tree via the files contrib/ltree/crc32.[ch] and from FreeBSD at 22 | * src/usr.bin/cksum/crc32.c. 23 | */ 24 | 25 | #include 26 | 27 | static const uint32_t crc32tab[256] = { 28 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 29 | 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, 30 | 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 31 | 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, 32 | 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 33 | 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 34 | 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 35 | 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, 36 | 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 37 | 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 38 | 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 39 | 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 40 | 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 41 | 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, 42 | 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 43 | 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 44 | 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 45 | 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, 46 | 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 47 | 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 48 | 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 49 | 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 50 | 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 51 | 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, 52 | 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 53 | 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 54 | 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 55 | 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, 56 | 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 57 | 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 58 | 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 59 | 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 60 | 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 61 | 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, 62 | 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 63 | 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 64 | 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 65 | 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, 66 | 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 67 | 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 68 | 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 69 | 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 70 | 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 71 | 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, 72 | 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 73 | 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 74 | 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 75 | 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, 76 | 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 77 | 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 78 | 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 79 | 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 80 | 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 81 | 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, 82 | 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 83 | 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 84 | 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 85 | 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, 86 | 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 87 | 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 88 | 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 89 | 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, 90 | 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 91 | 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, 92 | }; 93 | 94 | /* 95 | * CRC-32 implementation compatible with libmemcached library. Unfortunately 96 | * this implementation does not return CRC-32 as per spec. 97 | */ 98 | uint32_t 99 | hash_crc32(const char *key, size_t key_length) 100 | { 101 | uint64_t x; 102 | uint32_t crc = UINT32_MAX; 103 | 104 | for (x = 0; x < key_length; x++) { 105 | crc = (crc >> 8) ^ crc32tab[(crc ^ (uint64_t)key[x]) & 0xff]; 106 | } 107 | 108 | return ((~crc) >> 16) & 0x7fff; 109 | } 110 | 111 | uint32_t 112 | hash_crc32a(const char *key, size_t key_length) 113 | { 114 | const uint8_t *p = key; 115 | uint32_t crc; 116 | 117 | crc = ~0U; 118 | while (key_length--) { 119 | crc = crc32tab[(crc ^ *p++) & 0xFF] ^ (crc >> 8); 120 | } 121 | 122 | return crc ^ ~0U; 123 | } 124 | -------------------------------------------------------------------------------- /src/hashkit/nc_fnv.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | static uint64_t FNV_64_INIT = UINT64_C(0xcbf29ce484222325); 21 | static uint64_t FNV_64_PRIME = UINT64_C(0x100000001b3); 22 | static uint32_t FNV_32_INIT = 2166136261UL; 23 | static uint32_t FNV_32_PRIME = 16777619; 24 | 25 | uint32_t 26 | hash_fnv1_64(const char *key, size_t key_length) 27 | { 28 | uint64_t hash = FNV_64_INIT; 29 | size_t x; 30 | 31 | for (x = 0; x < key_length; x++) { 32 | hash *= FNV_64_PRIME; 33 | hash ^= (uint64_t)key[x]; 34 | } 35 | 36 | return (uint32_t)hash; 37 | } 38 | 39 | uint32_t 40 | hash_fnv1a_64(const char *key, size_t key_length) 41 | { 42 | uint32_t hash = (uint32_t) FNV_64_INIT; 43 | size_t x; 44 | 45 | for (x = 0; x < key_length; x++) { 46 | uint32_t val = (uint32_t)key[x]; 47 | hash ^= val; 48 | hash *= (uint32_t) FNV_64_PRIME; 49 | } 50 | 51 | return hash; 52 | } 53 | 54 | uint32_t 55 | hash_fnv1_32(const char *key, size_t key_length) 56 | { 57 | uint32_t hash = FNV_32_INIT; 58 | size_t x; 59 | 60 | for (x = 0; x < key_length; x++) { 61 | uint32_t val = (uint32_t)key[x]; 62 | hash *= FNV_32_PRIME; 63 | hash ^= val; 64 | } 65 | 66 | return hash; 67 | } 68 | 69 | uint32_t 70 | hash_fnv1a_32(const char *key, size_t key_length) 71 | { 72 | uint32_t hash = FNV_32_INIT; 73 | size_t x; 74 | 75 | for (x= 0; x < key_length; x++) { 76 | uint32_t val = (uint32_t)key[x]; 77 | hash ^= val; 78 | hash *= FNV_32_PRIME; 79 | } 80 | 81 | return hash; 82 | } 83 | -------------------------------------------------------------------------------- /src/hashkit/nc_hashkit.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_HASHKIT_H_ 19 | #define _NC_HASHKIT_H_ 20 | 21 | #include 22 | #include 23 | 24 | #define HASH_CODEC(ACTION) \ 25 | ACTION( HASH_ONE_AT_A_TIME, one_at_a_time ) \ 26 | ACTION( HASH_MD5, md5 ) \ 27 | ACTION( HASH_CRC16, crc16 ) \ 28 | ACTION( HASH_CRC32, crc32 ) \ 29 | ACTION( HASH_CRC32A, crc32a ) \ 30 | ACTION( HASH_FNV1_64, fnv1_64 ) \ 31 | ACTION( HASH_FNV1A_64, fnv1a_64 ) \ 32 | ACTION( HASH_FNV1_32, fnv1_32 ) \ 33 | ACTION( HASH_FNV1A_32, fnv1a_32 ) \ 34 | ACTION( HASH_HSIEH, hsieh ) \ 35 | ACTION( HASH_MURMUR, murmur ) \ 36 | ACTION( HASH_JENKINS, jenkins ) \ 37 | 38 | #define DIST_CODEC(ACTION) \ 39 | ACTION( DIST_KETAMA, ketama ) \ 40 | ACTION( DIST_MODULA, modula ) \ 41 | ACTION( DIST_RANDOM, random ) \ 42 | 43 | #define DEFINE_ACTION(_hash, _name) _hash, 44 | typedef enum hash_type { 45 | HASH_CODEC( DEFINE_ACTION ) 46 | HASH_SENTINEL 47 | } hash_type_t; 48 | #undef DEFINE_ACTION 49 | 50 | #define DEFINE_ACTION(_dist, _name) _dist, 51 | typedef enum dist_type { 52 | DIST_CODEC( DEFINE_ACTION ) 53 | DIST_SENTINEL 54 | } dist_type_t; 55 | #undef DEFINE_ACTION 56 | 57 | uint32_t hash_one_at_a_time(const char *key, size_t key_length); 58 | void md5_signature(const unsigned char *key, unsigned int length, unsigned char *result); 59 | uint32_t hash_md5(const char *key, size_t key_length); 60 | uint32_t hash_crc16(const char *key, size_t key_length); 61 | uint32_t hash_crc32(const char *key, size_t key_length); 62 | uint32_t hash_crc32a(const char *key, size_t key_length); 63 | uint32_t hash_fnv1_64(const char *key, size_t key_length); 64 | uint32_t hash_fnv1a_64(const char *key, size_t key_length); 65 | uint32_t hash_fnv1_32(const char *key, size_t key_length); 66 | uint32_t hash_fnv1a_32(const char *key, size_t key_length); 67 | uint32_t hash_hsieh(const char *key, size_t key_length); 68 | uint32_t hash_jenkins(const char *key, size_t length); 69 | uint32_t hash_murmur(const char *key, size_t length); 70 | 71 | rstatus_t ketama_update(struct server_pool *pool); 72 | uint32_t ketama_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); 73 | rstatus_t modula_update(struct server_pool *pool); 74 | uint32_t modula_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); 75 | rstatus_t random_update(struct server_pool *pool); 76 | uint32_t random_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash); 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /src/hashkit/nc_hsieh.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * By Paul Hsieh (C) 2004, 2005. Covered under the Paul Hsieh 20 | * derivative license. 21 | * See: http://www.azillionmonkeys.com/qed/weblicense.html for license 22 | * details. 23 | * http://www.azillionmonkeys.com/qed/hash.html 24 | */ 25 | 26 | #include 27 | 28 | #undef get16bits 29 | #if (defined(__GNUC__) && defined(__i386__)) 30 | #define get16bits(d) (*((const uint16_t *) (d))) 31 | #endif 32 | 33 | #if !defined (get16bits) 34 | #define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8)\ 35 | +(uint32_t)(((const uint8_t *)(d))[0]) ) 36 | #endif 37 | 38 | uint32_t 39 | hash_hsieh(const char *key, size_t key_length) 40 | { 41 | uint32_t hash = 0, tmp; 42 | int rem; 43 | 44 | if (key_length <= 0 || key == NULL) { 45 | return 0; 46 | } 47 | 48 | rem = key_length & 3; 49 | key_length >>= 2; 50 | 51 | /* Main loop */ 52 | for (;key_length > 0; key_length--) { 53 | hash += get16bits (key); 54 | tmp = (get16bits (key+2) << 11) ^ hash; 55 | hash = (hash << 16) ^ tmp; 56 | key += 2*sizeof (uint16_t); 57 | hash += hash >> 11; 58 | } 59 | 60 | /* Handle end cases */ 61 | switch (rem) { 62 | case 3: 63 | hash += get16bits (key); 64 | hash ^= hash << 16; 65 | hash ^= (uint32_t)key[sizeof (uint16_t)] << 18; 66 | hash += hash >> 11; 67 | break; 68 | 69 | case 2: 70 | hash += get16bits (key); 71 | hash ^= hash << 11; 72 | hash += hash >> 17; 73 | break; 74 | 75 | case 1: 76 | hash += (unsigned char)(*key); 77 | hash ^= hash << 10; 78 | hash += hash >> 1; 79 | 80 | default: 81 | break; 82 | } 83 | 84 | /* Force "avalanching" of final 127 bits */ 85 | hash ^= hash << 3; 86 | hash += hash >> 5; 87 | hash ^= hash << 4; 88 | hash += hash >> 17; 89 | hash ^= hash << 25; 90 | hash += hash >> 6; 91 | 92 | return hash; 93 | } 94 | -------------------------------------------------------------------------------- /src/hashkit/nc_jenkins.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * By Bob Jenkins, 2006. bob_jenkins@burtleburtle.net. You may use this 20 | * code any way you wish, private, educational, or commercial. It's free. 21 | * Use for hash table lookup, or anything where one collision in 2^^32 is 22 | * acceptable. Do NOT use for cryptographic purposes. 23 | * http://burtleburtle.net/bob/hash/index.html 24 | * 25 | * Modified by Brian Pontz for libmemcached 26 | * TODO: 27 | * Add big endian support 28 | */ 29 | 30 | #include 31 | 32 | #define hashsize(n) ((uint32_t)1<<(n)) 33 | #define hashmask(n) (hashsize(n)-1) 34 | #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) 35 | 36 | #define mix(a,b,c) \ 37 | { \ 38 | a -= c; a ^= rot(c, 4); c += b; \ 39 | b -= a; b ^= rot(a, 6); a += c; \ 40 | c -= b; c ^= rot(b, 8); b += a; \ 41 | a -= c; a ^= rot(c,16); c += b; \ 42 | b -= a; b ^= rot(a,19); a += c; \ 43 | c -= b; c ^= rot(b, 4); b += a; \ 44 | } 45 | 46 | #define final(a,b,c) \ 47 | { \ 48 | c ^= b; c -= rot(b,14); \ 49 | a ^= c; a -= rot(c,11); \ 50 | b ^= a; b -= rot(a,25); \ 51 | c ^= b; c -= rot(b,16); \ 52 | a ^= c; a -= rot(c,4); \ 53 | b ^= a; b -= rot(a,14); \ 54 | c ^= b; c -= rot(b,24); \ 55 | } 56 | 57 | #define JENKINS_INITVAL 13 58 | 59 | /* 60 | * jenkins_hash() -- hash a variable-length key into a 32-bit value 61 | * k : the key (the unaligned variable-length array of bytes) 62 | * length : the length of the key, counting by bytes 63 | * initval : can be any 4-byte value 64 | * Returns a 32-bit value. Every bit of the key affects every bit of 65 | * the return value. Two keys differing by one or two bits will have 66 | * totally different hash values. 67 | 68 | * The best hash table sizes are powers of 2. There is no need to do 69 | * mod a prime (mod is sooo slow!). If you need less than 32 bits, 70 | * use a bitmask. For example, if you need only 10 bits, do 71 | * h = (h & hashmask(10)); 72 | * In which case, the hash table should have hashsize(10) elements. 73 | */ 74 | 75 | uint32_t 76 | hash_jenkins(const char *key, size_t length) 77 | { 78 | uint32_t a,b,c; /* internal state */ 79 | union { const void *ptr; size_t i; } u; /* needed for Mac Powerbook G4 */ 80 | 81 | /* Set up the internal state */ 82 | a = b = c = 0xdeadbeef + ((uint32_t)length) + JENKINS_INITVAL; 83 | 84 | u.ptr = key; 85 | #ifndef WORDS_BIGENDIAN 86 | if ((u.i & 0x3) == 0) 87 | { 88 | const uint32_t *k = (const uint32_t *)key; /* read 32-bit chunks */ 89 | 90 | /*------ all but last block: aligned reads and affect 32 bits of (a,b,c) */ 91 | while (length > 12) 92 | { 93 | a += k[0]; 94 | b += k[1]; 95 | c += k[2]; 96 | mix(a,b,c); 97 | length -= 12; 98 | k += 3; 99 | } 100 | 101 | /*----------------------------- handle the last (probably partial) block */ 102 | /* 103 | * "k[2]&0xffffff" actually reads beyond the end of the string, but 104 | * then masks off the part it's not allowed to read. Because the 105 | * string is aligned, the masked-off tail is in the same word as the 106 | * rest of the string. Every machine with memory protection I've seen 107 | * does it on word boundaries, so is OK with this. But VALGRIND will 108 | * still catch it and complain. The masking trick does make the hash 109 | * noticably faster for short strings (like English words). 110 | */ 111 | switch(length) 112 | { 113 | case 12: c+=k[2]; b+=k[1]; a+=k[0]; break; 114 | case 11: c+=k[2]&0xffffff; b+=k[1]; a+=k[0]; break; 115 | case 10: c+=k[2]&0xffff; b+=k[1]; a+=k[0]; break; 116 | case 9 : c+=k[2]&0xff; b+=k[1]; a+=k[0]; break; 117 | case 8 : b+=k[1]; a+=k[0]; break; 118 | case 7 : b+=k[1]&0xffffff; a+=k[0]; break; 119 | case 6 : b+=k[1]&0xffff; a+=k[0]; break; 120 | case 5 : b+=k[1]&0xff; a+=k[0]; break; 121 | case 4 : a+=k[0]; break; 122 | case 3 : a+=k[0]&0xffffff; break; 123 | case 2 : a+=k[0]&0xffff; break; 124 | case 1 : a+=k[0]&0xff; break; 125 | case 0 : return c; /* zero length strings require no mixing */ 126 | default: return c; 127 | } 128 | 129 | } 130 | else if ((u.i & 0x1) == 0) 131 | { 132 | const uint16_t *k = (const uint16_t *)key; /* read 16-bit chunks */ 133 | const uint8_t *k8; 134 | 135 | /*--------------- all but last block: aligned reads and different mixing */ 136 | while (length > 12) 137 | { 138 | a += k[0] + (((uint32_t)k[1])<<16); 139 | b += k[2] + (((uint32_t)k[3])<<16); 140 | c += k[4] + (((uint32_t)k[5])<<16); 141 | mix(a,b,c); 142 | length -= 12; 143 | k += 6; 144 | } 145 | 146 | /*----------------------------- handle the last (probably partial) block */ 147 | k8 = (const uint8_t *)k; 148 | switch(length) 149 | { 150 | case 12: c+=k[4]+(((uint32_t)k[5])<<16); 151 | b+=k[2]+(((uint32_t)k[3])<<16); 152 | a+=k[0]+(((uint32_t)k[1])<<16); 153 | break; 154 | case 11: c+=((uint32_t)k8[10])<<16; /* fall through */ 155 | case 10: c+=k[4]; 156 | b+=k[2]+(((uint32_t)k[3])<<16); 157 | a+=k[0]+(((uint32_t)k[1])<<16); 158 | break; 159 | case 9 : c+=k8[8]; /* fall through */ 160 | case 8 : b+=k[2]+(((uint32_t)k[3])<<16); 161 | a+=k[0]+(((uint32_t)k[1])<<16); 162 | break; 163 | case 7 : b+=((uint32_t)k8[6])<<16; /* fall through */ 164 | case 6 : b+=k[2]; 165 | a+=k[0]+(((uint32_t)k[1])<<16); 166 | break; 167 | case 5 : b+=k8[4]; /* fall through */ 168 | case 4 : a+=k[0]+(((uint32_t)k[1])<<16); 169 | break; 170 | case 3 : a+=((uint32_t)k8[2])<<16; /* fall through */ 171 | case 2 : a+=k[0]; 172 | break; 173 | case 1 : a+=k8[0]; 174 | break; 175 | case 0 : return c; /* zero length requires no mixing */ 176 | default: return c; 177 | } 178 | 179 | } 180 | else 181 | { /* need to read the key one byte at a time */ 182 | #endif /* little endian */ 183 | const uint8_t *k = (const uint8_t *)key; 184 | 185 | /*--------------- all but the last block: affect some 32 bits of (a,b,c) */ 186 | while (length > 12) 187 | { 188 | a += k[0]; 189 | a += ((uint32_t)k[1])<<8; 190 | a += ((uint32_t)k[2])<<16; 191 | a += ((uint32_t)k[3])<<24; 192 | b += k[4]; 193 | b += ((uint32_t)k[5])<<8; 194 | b += ((uint32_t)k[6])<<16; 195 | b += ((uint32_t)k[7])<<24; 196 | c += k[8]; 197 | c += ((uint32_t)k[9])<<8; 198 | c += ((uint32_t)k[10])<<16; 199 | c += ((uint32_t)k[11])<<24; 200 | mix(a,b,c); 201 | length -= 12; 202 | k += 12; 203 | } 204 | 205 | /*-------------------------------- last block: affect all 32 bits of (c) */ 206 | switch(length) /* all the case statements fall through */ 207 | { 208 | case 12: c+=((uint32_t)k[11])<<24; 209 | case 11: c+=((uint32_t)k[10])<<16; 210 | case 10: c+=((uint32_t)k[9])<<8; 211 | case 9 : c+=k[8]; 212 | case 8 : b+=((uint32_t)k[7])<<24; 213 | case 7 : b+=((uint32_t)k[6])<<16; 214 | case 6 : b+=((uint32_t)k[5])<<8; 215 | case 5 : b+=k[4]; 216 | case 4 : a+=((uint32_t)k[3])<<24; 217 | case 3 : a+=((uint32_t)k[2])<<16; 218 | case 2 : a+=((uint32_t)k[1])<<8; 219 | case 1 : a+=k[0]; 220 | break; 221 | case 0 : return c; 222 | default : return c; 223 | } 224 | #ifndef WORDS_BIGENDIAN 225 | } 226 | #endif 227 | 228 | final(a,b,c); 229 | return c; 230 | } 231 | -------------------------------------------------------------------------------- /src/hashkit/nc_ketama.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include 23 | #include 24 | #include 25 | 26 | #define KETAMA_CONTINUUM_ADDITION 10 /* # extra slots to build into continuum */ 27 | #define KETAMA_POINTS_PER_SERVER 160 /* 40 points per hash */ 28 | #define KETAMA_MAX_HOSTLEN 86 29 | 30 | static uint32_t 31 | ketama_hash(const char *key, size_t key_length, uint32_t alignment) 32 | { 33 | unsigned char results[16]; 34 | 35 | md5_signature((unsigned char*)key, key_length, results); 36 | 37 | return ((uint32_t) (results[3 + alignment * 4] & 0xFF) << 24) 38 | | ((uint32_t) (results[2 + alignment * 4] & 0xFF) << 16) 39 | | ((uint32_t) (results[1 + alignment * 4] & 0xFF) << 8) 40 | | (results[0 + alignment * 4] & 0xFF); 41 | } 42 | 43 | static int 44 | ketama_item_cmp(const void *t1, const void *t2) 45 | { 46 | const struct continuum *ct1 = t1, *ct2 = t2; 47 | 48 | if (ct1->value == ct2->value) { 49 | return 0; 50 | } else if (ct1->value > ct2->value) { 51 | return 1; 52 | } else { 53 | return -1; 54 | } 55 | } 56 | 57 | rstatus_t 58 | ketama_update(struct server_pool *pool) 59 | { 60 | uint32_t nserver; /* # server - live and dead */ 61 | uint32_t nlive_server; /* # live server */ 62 | uint32_t pointer_per_server; /* pointers per server proportional to weight */ 63 | uint32_t pointer_per_hash; /* pointers per hash */ 64 | uint32_t pointer_counter; /* # pointers on continuum */ 65 | uint32_t pointer_index; /* pointer index */ 66 | uint32_t points_per_server; /* points per server */ 67 | uint32_t continuum_index; /* continuum index */ 68 | uint32_t continuum_addition; /* extra space in the continuum */ 69 | uint32_t server_index; /* server index */ 70 | uint32_t value; /* continuum value */ 71 | uint32_t total_weight; /* total live server weight */ 72 | int64_t now; /* current timestamp in usec */ 73 | 74 | ASSERT(array_n(&pool->server) > 0); 75 | 76 | now = nc_usec_now(); 77 | if (now < 0) { 78 | return NC_ERROR; 79 | } 80 | 81 | /* 82 | * Count live servers and total weight, and also update the next time to 83 | * rebuild the distribution 84 | */ 85 | nserver = array_n(&pool->server); 86 | nlive_server = 0; 87 | total_weight = 0; 88 | pool->next_rebuild = 0LL; 89 | for (server_index = 0; server_index < nserver; server_index++) { 90 | struct server *server = array_get(&pool->server, server_index); 91 | 92 | if (pool->auto_eject_hosts) { 93 | if (server->next_retry <= now) { 94 | server->next_retry = 0LL; 95 | nlive_server++; 96 | } else if (pool->next_rebuild == 0LL || 97 | server->next_retry < pool->next_rebuild) { 98 | pool->next_rebuild = server->next_retry; 99 | } 100 | } else { 101 | nlive_server++; 102 | } 103 | 104 | ASSERT(server->weight > 0); 105 | 106 | /* count weight only for live servers */ 107 | if (!pool->auto_eject_hosts || server->next_retry <= now) { 108 | total_weight += server->weight; 109 | } 110 | } 111 | 112 | pool->nlive_server = nlive_server; 113 | 114 | if (nlive_server == 0) { 115 | log_debug(LOG_DEBUG, "no live servers for pool %"PRIu32" '%.*s'", 116 | pool->idx, pool->name.len, pool->name.data); 117 | 118 | return NC_OK; 119 | } 120 | log_debug(LOG_DEBUG, "%"PRIu32" of %"PRIu32" servers are live for pool " 121 | "%"PRIu32" '%.*s'", nlive_server, nserver, pool->idx, 122 | pool->name.len, pool->name.data); 123 | 124 | continuum_addition = KETAMA_CONTINUUM_ADDITION; 125 | points_per_server = KETAMA_POINTS_PER_SERVER; 126 | /* 127 | * Allocate the continuum for the pool, the first time, and every time we 128 | * add a new server to the pool 129 | */ 130 | if (nlive_server > pool->nserver_continuum) { 131 | struct continuum *continuum; 132 | uint32_t nserver_continuum = nlive_server + continuum_addition; 133 | uint32_t ncontinuum = nserver_continuum * points_per_server; 134 | 135 | continuum = nc_realloc(pool->continuum, sizeof(*continuum) * ncontinuum); 136 | if (continuum == NULL) { 137 | return NC_ENOMEM; 138 | } 139 | 140 | pool->continuum = continuum; 141 | pool->nserver_continuum = nserver_continuum; 142 | /* pool->ncontinuum is initialized later as it could be <= ncontinuum */ 143 | } 144 | 145 | /* 146 | * Build a continuum with the servers that are live and points from 147 | * these servers that are proportial to their weight 148 | */ 149 | continuum_index = 0; 150 | pointer_counter = 0; 151 | for (server_index = 0; server_index < nserver; server_index++) { 152 | struct server *server; 153 | float pct; 154 | 155 | server = array_get(&pool->server, server_index); 156 | 157 | if (pool->auto_eject_hosts && server->next_retry > now) { 158 | continue; 159 | } 160 | 161 | pct = (float)server->weight / (float)total_weight; 162 | pointer_per_server = (uint32_t) ((floorf((float) (pct * KETAMA_POINTS_PER_SERVER / 4 * (float)nlive_server + 0.0000000001))) * 4); 163 | pointer_per_hash = 4; 164 | 165 | log_debug(LOG_VERB, "%.*s:%"PRIu16" weight %"PRIu32" of %"PRIu32" " 166 | "pct %0.5f points per server %"PRIu32"", 167 | server->name.len, server->name.data, server->port, 168 | server->weight, total_weight, pct, pointer_per_server); 169 | 170 | for (pointer_index = 1; 171 | pointer_index <= pointer_per_server / pointer_per_hash; 172 | pointer_index++) { 173 | 174 | char host[KETAMA_MAX_HOSTLEN]= ""; 175 | size_t hostlen; 176 | uint32_t x; 177 | 178 | hostlen = snprintf(host, KETAMA_MAX_HOSTLEN, "%.*s-%u", 179 | server->name.len, server->name.data, 180 | pointer_index - 1); 181 | 182 | for (x = 0; x < pointer_per_hash; x++) { 183 | value = ketama_hash(host, hostlen, x); 184 | pool->continuum[continuum_index].index = server_index; 185 | pool->continuum[continuum_index++].value = value; 186 | } 187 | } 188 | pointer_counter += pointer_per_server; 189 | } 190 | 191 | pool->ncontinuum = pointer_counter; 192 | qsort(pool->continuum, pool->ncontinuum, sizeof(*pool->continuum), 193 | ketama_item_cmp); 194 | 195 | for (pointer_index = 0; 196 | pointer_index < ((nlive_server * KETAMA_POINTS_PER_SERVER) - 1); 197 | pointer_index++) { 198 | if (pointer_index + 1 >= pointer_counter) { 199 | break; 200 | } 201 | ASSERT(pool->continuum[pointer_index].value <= 202 | pool->continuum[pointer_index + 1].value); 203 | } 204 | 205 | log_debug(LOG_VERB, "updated pool %"PRIu32" '%.*s' with %"PRIu32" of " 206 | "%"PRIu32" servers live in %"PRIu32" slots and %"PRIu32" " 207 | "active points in %"PRIu32" slots", pool->idx, 208 | pool->name.len, pool->name.data, nlive_server, nserver, 209 | pool->nserver_continuum, pool->ncontinuum, 210 | (pool->nserver_continuum + continuum_addition) * points_per_server); 211 | 212 | return NC_OK; 213 | } 214 | 215 | uint32_t 216 | ketama_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) 217 | { 218 | struct continuum *begin, *end, *left, *right, *middle; 219 | 220 | ASSERT(continuum != NULL); 221 | ASSERT(ncontinuum != 0); 222 | 223 | begin = left = continuum; 224 | end = right = continuum + ncontinuum; 225 | 226 | while (left < right) { 227 | middle = left + (right - left) / 2; 228 | if (middle->value < hash) { 229 | left = middle + 1; 230 | } else { 231 | right = middle; 232 | } 233 | } 234 | 235 | if (right == end) { 236 | right = begin; 237 | } 238 | 239 | return right->index; 240 | } 241 | -------------------------------------------------------------------------------- /src/hashkit/nc_modula.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #define MODULA_CONTINUUM_ADDITION 10 /* # extra slots to build into continuum */ 26 | #define MODULA_POINTS_PER_SERVER 1 27 | 28 | rstatus_t 29 | modula_update(struct server_pool *pool) 30 | { 31 | uint32_t nserver; /* # server - live and dead */ 32 | uint32_t nlive_server; /* # live server */ 33 | uint32_t pointer_per_server; /* pointers per server proportional to weight */ 34 | uint32_t pointer_counter; /* # pointers on continuum */ 35 | uint32_t points_per_server; /* points per server */ 36 | uint32_t continuum_index; /* continuum index */ 37 | uint32_t continuum_addition; /* extra space in the continuum */ 38 | uint32_t server_index; /* server index */ 39 | uint32_t weight_index; /* weight index */ 40 | uint32_t total_weight; /* total live server weight */ 41 | int64_t now; /* current timestamp in usec */ 42 | 43 | now = nc_usec_now(); 44 | if (now < 0) { 45 | return NC_ERROR; 46 | } 47 | 48 | nserver = array_n(&pool->server); 49 | nlive_server = 0; 50 | total_weight = 0; 51 | pool->next_rebuild = 0LL; 52 | 53 | for (server_index = 0; server_index < nserver; server_index++) { 54 | struct server *server = array_get(&pool->server, server_index); 55 | 56 | if (pool->auto_eject_hosts) { 57 | if (server->next_retry <= now) { 58 | server->next_retry = 0LL; 59 | nlive_server++; 60 | } else if (pool->next_rebuild == 0LL || 61 | server->next_retry < pool->next_rebuild) { 62 | pool->next_rebuild = server->next_retry; 63 | } 64 | } else { 65 | nlive_server++; 66 | } 67 | 68 | ASSERT(server->weight > 0); 69 | 70 | /* count weight only for live servers */ 71 | if (!pool->auto_eject_hosts || server->next_retry <= now) { 72 | total_weight += server->weight; 73 | } 74 | } 75 | 76 | pool->nlive_server = nlive_server; 77 | 78 | if (nlive_server == 0) { 79 | ASSERT(pool->continuum != NULL); 80 | ASSERT(pool->ncontinuum != 0); 81 | 82 | log_debug(LOG_DEBUG, "no live servers for pool %"PRIu32" '%.*s'", 83 | pool->idx, pool->name.len, pool->name.data); 84 | 85 | return NC_OK; 86 | } 87 | log_debug(LOG_DEBUG, "%"PRIu32" of %"PRIu32" servers are live for pool " 88 | "%"PRIu32" '%.*s'", nlive_server, nserver, pool->idx, 89 | pool->name.len, pool->name.data); 90 | 91 | continuum_addition = MODULA_CONTINUUM_ADDITION; 92 | points_per_server = MODULA_POINTS_PER_SERVER; 93 | 94 | /* 95 | * Allocate the continuum for the pool, the first time, and every time we 96 | * add a new server to the pool 97 | */ 98 | if (total_weight > pool->nserver_continuum) { 99 | struct continuum *continuum; 100 | uint32_t nserver_continuum = total_weight + MODULA_CONTINUUM_ADDITION; 101 | uint32_t ncontinuum = nserver_continuum * MODULA_POINTS_PER_SERVER; 102 | 103 | continuum = nc_realloc(pool->continuum, sizeof(*continuum) * ncontinuum); 104 | if (continuum == NULL) { 105 | return NC_ENOMEM; 106 | } 107 | 108 | pool->continuum = continuum; 109 | pool->nserver_continuum = nserver_continuum; 110 | /* pool->ncontinuum is initialized later as it could be <= ncontinuum */ 111 | } 112 | 113 | /* update the continuum with the servers that are live */ 114 | continuum_index = 0; 115 | pointer_counter = 0; 116 | for (server_index = 0; server_index < nserver; server_index++) { 117 | struct server *server = array_get(&pool->server, server_index); 118 | 119 | if (pool->auto_eject_hosts && server->next_retry > now) { 120 | continue; 121 | } 122 | 123 | for (weight_index = 0; weight_index < server->weight; weight_index++) { 124 | pointer_per_server = 1; 125 | 126 | pool->continuum[continuum_index].index = server_index; 127 | pool->continuum[continuum_index++].value = 0; 128 | 129 | pointer_counter += pointer_per_server; 130 | } 131 | } 132 | pool->ncontinuum = pointer_counter; 133 | 134 | log_debug(LOG_VERB, "updated pool %"PRIu32" '%.*s' with %"PRIu32" of " 135 | "%"PRIu32" servers live in %"PRIu32" slots and %"PRIu32" " 136 | "active points in %"PRIu32" slots", pool->idx, 137 | pool->name.len, pool->name.data, nlive_server, nserver, 138 | pool->nserver_continuum, pool->ncontinuum, 139 | (pool->nserver_continuum + continuum_addition) * points_per_server); 140 | 141 | return NC_OK; 142 | 143 | } 144 | 145 | uint32_t 146 | modula_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) 147 | { 148 | struct continuum *c; 149 | 150 | ASSERT(continuum != NULL); 151 | ASSERT(ncontinuum != 0); 152 | 153 | c = continuum + hash % ncontinuum; 154 | 155 | return c->index; 156 | } 157 | -------------------------------------------------------------------------------- /src/hashkit/nc_murmur.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * "Murmur" hash provided by Austin, tanjent@gmail.com 20 | * http://murmurhash.googlepages.com/ 21 | * 22 | * Note - This code makes a few assumptions about how your machine behaves - 23 | * 24 | * 1. We can read a 4-byte value from any address without crashing 25 | * 2. sizeof(int) == 4 26 | * 27 | * And it has a few limitations - 28 | * 1. It will not work incrementally. 29 | * 2. It will not produce the same results on little-endian and big-endian 30 | * machines. 31 | * 32 | * Updated to murmur2 hash - BP 33 | */ 34 | 35 | #include 36 | 37 | uint32_t 38 | hash_murmur(const char *key, size_t length) 39 | { 40 | /* 41 | * 'm' and 'r' are mixing constants generated offline. They're not 42 | * really 'magic', they just happen to work well. 43 | */ 44 | 45 | const unsigned int m = 0x5bd1e995; 46 | const uint32_t seed = (0xdeadbeef * (uint32_t)length); 47 | const int r = 24; 48 | 49 | 50 | /* Initialize the hash to a 'random' value */ 51 | 52 | uint32_t h = seed ^ (uint32_t)length; 53 | 54 | /* Mix 4 bytes at a time into the hash */ 55 | 56 | const unsigned char * data = (const unsigned char *)key; 57 | 58 | while (length >= 4) { 59 | unsigned int k = *(unsigned int *)data; 60 | 61 | k *= m; 62 | k ^= k >> r; 63 | k *= m; 64 | 65 | h *= m; 66 | h ^= k; 67 | 68 | data += 4; 69 | length -= 4; 70 | } 71 | 72 | /* Handle the last few bytes of the input array */ 73 | 74 | switch(length) { 75 | case 3: 76 | h ^= ((uint32_t)data[2]) << 16; 77 | 78 | case 2: 79 | h ^= ((uint32_t)data[1]) << 8; 80 | 81 | case 1: 82 | h ^= data[0]; 83 | h *= m; 84 | 85 | default: 86 | break; 87 | }; 88 | 89 | /* 90 | * Do a few final mixes of the hash to ensure the last few bytes are 91 | * well-incorporated. 92 | */ 93 | 94 | h ^= h >> 13; 95 | h *= m; 96 | h ^= h >> 15; 97 | 98 | return h; 99 | } 100 | -------------------------------------------------------------------------------- /src/hashkit/nc_one_at_a_time.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | /* 19 | * HashKit 20 | * Copyright (C) 2009 Brian Aker 21 | * All rights reserved. 22 | * 23 | * Use and distribution licensed under the BSD license. See 24 | * the COPYING file in the parent directory for full text. 25 | */ 26 | 27 | /* 28 | * This has is Jenkin's "One at A time Hash". 29 | * http://en.wikipedia.org/wiki/Jenkins_hash_function 30 | */ 31 | 32 | #include 33 | 34 | uint32_t 35 | hash_one_at_a_time(const char *key, size_t key_length) 36 | { 37 | const char *ptr = key; 38 | uint32_t value = 0; 39 | 40 | while (key_length--) { 41 | uint32_t val = (uint32_t) *ptr++; 42 | value += val; 43 | value += (value << 10); 44 | value ^= (value >> 6); 45 | } 46 | value += (value << 3); 47 | value ^= (value >> 11); 48 | value += (value << 15); 49 | 50 | return value; 51 | } 52 | -------------------------------------------------------------------------------- /src/hashkit/nc_random.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #define RANDOM_CONTINUUM_ADDITION 10 /* # extra slots to build into continuum */ 26 | #define RANDOM_POINTS_PER_SERVER 1 27 | 28 | rstatus_t 29 | random_update(struct server_pool *pool) 30 | { 31 | uint32_t nserver; /* # server - live and dead */ 32 | uint32_t nlive_server; /* # live server */ 33 | uint32_t pointer_per_server; /* pointers per server proportional to weight */ 34 | uint32_t pointer_counter; /* # pointers on continuum */ 35 | uint32_t points_per_server; /* points per server */ 36 | uint32_t continuum_index; /* continuum index */ 37 | uint32_t continuum_addition; /* extra space in the continuum */ 38 | uint32_t server_index; /* server index */ 39 | int64_t now; /* current timestamp in usec */ 40 | 41 | now = nc_usec_now(); 42 | if (now < 0) { 43 | return NC_ERROR; 44 | } 45 | 46 | nserver = array_n(&pool->server); 47 | nlive_server = 0; 48 | pool->next_rebuild = 0LL; 49 | 50 | for (server_index = 0; server_index < nserver; server_index++) { 51 | struct server *server = array_get(&pool->server, server_index); 52 | 53 | if (pool->auto_eject_hosts) { 54 | if (server->next_retry <= now) { 55 | server->next_retry = 0LL; 56 | nlive_server++; 57 | } else if (pool->next_rebuild == 0LL || 58 | server->next_retry < pool->next_rebuild) { 59 | pool->next_rebuild = server->next_retry; 60 | } 61 | } else { 62 | nlive_server++; 63 | } 64 | } 65 | 66 | pool->nlive_server = nlive_server; 67 | 68 | if (nlive_server == 0) { 69 | ASSERT(pool->continuum != NULL); 70 | ASSERT(pool->ncontinuum != 0); 71 | 72 | log_debug(LOG_DEBUG, "no live servers for pool %"PRIu32" '%.*s'", 73 | pool->idx, pool->name.len, pool->name.data); 74 | 75 | return NC_OK; 76 | } 77 | log_debug(LOG_DEBUG, "%"PRIu32" of %"PRIu32" servers are live for pool " 78 | "%"PRIu32" '%.*s'", nlive_server, nserver, pool->idx, 79 | pool->name.len, pool->name.data); 80 | 81 | continuum_addition = RANDOM_CONTINUUM_ADDITION; 82 | points_per_server = RANDOM_POINTS_PER_SERVER; 83 | 84 | /* 85 | * Allocate the continuum for the pool, the first time, and every time we 86 | * add a new server to the pool 87 | */ 88 | if (nlive_server > pool->nserver_continuum) { 89 | struct continuum *continuum; 90 | uint32_t nserver_continuum = nlive_server + RANDOM_CONTINUUM_ADDITION; 91 | uint32_t ncontinuum = nserver_continuum * RANDOM_POINTS_PER_SERVER; 92 | 93 | continuum = nc_realloc(pool->continuum, sizeof(*continuum) * ncontinuum); 94 | if (continuum == NULL) { 95 | return NC_ENOMEM; 96 | } 97 | 98 | srandom((uint32_t)time(NULL)); 99 | 100 | pool->continuum = continuum; 101 | pool->nserver_continuum = nserver_continuum; 102 | /* pool->ncontinuum is initialized later as it could be <= ncontinuum */ 103 | } 104 | 105 | /* update the continuum with the servers that are live */ 106 | continuum_index = 0; 107 | pointer_counter = 0; 108 | for (server_index = 0; server_index < nserver; server_index++) { 109 | struct server *server = array_get(&pool->server, server_index); 110 | 111 | if (pool->auto_eject_hosts && server->next_retry > now) { 112 | continue; 113 | } 114 | 115 | pointer_per_server = 1; 116 | 117 | pool->continuum[continuum_index].index = server_index; 118 | pool->continuum[continuum_index++].value = 0; 119 | 120 | pointer_counter += pointer_per_server; 121 | } 122 | pool->ncontinuum = pointer_counter; 123 | 124 | log_debug(LOG_VERB, "updated pool %"PRIu32" '%.*s' with %"PRIu32" of " 125 | "%"PRIu32" servers live in %"PRIu32" slots and %"PRIu32" " 126 | "active points in %"PRIu32" slots", pool->idx, 127 | pool->name.len, pool->name.data, nlive_server, nserver, 128 | pool->nserver_continuum, pool->ncontinuum, 129 | (pool->nserver_continuum + continuum_addition) * points_per_server); 130 | 131 | return NC_OK; 132 | 133 | } 134 | 135 | uint32_t 136 | random_dispatch(struct continuum *continuum, uint32_t ncontinuum, uint32_t hash) 137 | { 138 | struct continuum *c; 139 | 140 | ASSERT(continuum != NULL); 141 | ASSERT(ncontinuum != 0); 142 | 143 | c = continuum + random() % ncontinuum; 144 | 145 | return c->index; 146 | } 147 | -------------------------------------------------------------------------------- /src/nc_array.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | 20 | #include 21 | 22 | struct array * 23 | array_create(uint32_t n, size_t size) 24 | { 25 | struct array *a; 26 | 27 | ASSERT(n != 0 && size != 0); 28 | 29 | a = nc_alloc(sizeof(*a)); 30 | if (a == NULL) { 31 | return NULL; 32 | } 33 | 34 | a->elem = nc_alloc(n * size); 35 | if (a->elem == NULL) { 36 | nc_free(a); 37 | return NULL; 38 | } 39 | 40 | a->nelem = 0; 41 | a->size = size; 42 | a->nalloc = n; 43 | 44 | return a; 45 | } 46 | 47 | void 48 | array_destroy(struct array *a) 49 | { 50 | array_deinit(a); 51 | nc_free(a); 52 | } 53 | 54 | rstatus_t 55 | array_init(struct array *a, uint32_t n, size_t size) 56 | { 57 | ASSERT(n != 0 && size != 0); 58 | 59 | a->elem = nc_alloc(n * size); 60 | if (a->elem == NULL) { 61 | return NC_ENOMEM; 62 | } 63 | 64 | a->nelem = 0; 65 | a->size = size; 66 | a->nalloc = n; 67 | 68 | return NC_OK; 69 | } 70 | 71 | void 72 | array_deinit(struct array *a) 73 | { 74 | ASSERT(a->nelem == 0); 75 | 76 | if (a->elem != NULL) { 77 | nc_free(a->elem); 78 | } 79 | } 80 | 81 | uint32_t 82 | array_idx(struct array *a, void *elem) 83 | { 84 | uint8_t *p, *q; 85 | uint32_t off, idx; 86 | 87 | ASSERT(elem >= a->elem); 88 | 89 | p = a->elem; 90 | q = elem; 91 | off = (uint32_t)(q - p); 92 | 93 | ASSERT(off % (uint32_t)a->size == 0); 94 | 95 | idx = off / (uint32_t)a->size; 96 | 97 | return idx; 98 | } 99 | 100 | void * 101 | array_push(struct array *a) 102 | { 103 | void *elem, *new; 104 | size_t size; 105 | 106 | if (a->nelem == a->nalloc) { 107 | 108 | /* the array is full; allocate new array */ 109 | size = a->size * a->nalloc; 110 | new = nc_realloc(a->elem, 2 * size); 111 | if (new == NULL) { 112 | return NULL; 113 | } 114 | 115 | a->elem = new; 116 | a->nalloc *= 2; 117 | } 118 | 119 | elem = (uint8_t *)a->elem + a->size * a->nelem; 120 | a->nelem++; 121 | 122 | return elem; 123 | } 124 | 125 | void * 126 | array_pop(struct array *a) 127 | { 128 | void *elem; 129 | 130 | ASSERT(a->nelem != 0); 131 | 132 | a->nelem--; 133 | elem = (uint8_t *)a->elem + a->size * a->nelem; 134 | 135 | return elem; 136 | } 137 | 138 | void * 139 | array_get(struct array *a, uint32_t idx) 140 | { 141 | void *elem; 142 | 143 | ASSERT(a->nelem != 0); 144 | ASSERT(idx < a->nelem); 145 | 146 | elem = (uint8_t *)a->elem + (a->size * idx); 147 | 148 | return elem; 149 | } 150 | 151 | void * 152 | array_top(struct array *a) 153 | { 154 | ASSERT(a->nelem != 0); 155 | 156 | return array_get(a, a->nelem - 1); 157 | } 158 | 159 | void 160 | array_swap(struct array *a, struct array *b) 161 | { 162 | struct array tmp; 163 | 164 | tmp = *a; 165 | *a = *b; 166 | *b = tmp; 167 | } 168 | 169 | /* 170 | * Sort nelem elements of the array in ascending order based on the 171 | * compare comparator. 172 | */ 173 | void 174 | array_sort(struct array *a, array_compare_t compare) 175 | { 176 | ASSERT(a->nelem != 0); 177 | 178 | qsort(a->elem, a->nelem, a->size, compare); 179 | } 180 | 181 | /* 182 | * Calls the func once for each element in the array as long as func returns 183 | * success. On failure short-circuits and returns the error status. 184 | */ 185 | rstatus_t 186 | array_each(struct array *a, array_each_t func, void *data) 187 | { 188 | uint32_t i, nelem; 189 | 190 | ASSERT(array_n(a) != 0); 191 | ASSERT(func != NULL); 192 | 193 | for (i = 0, nelem = array_n(a); i < nelem; i++) { 194 | void *elem = array_get(a, i); 195 | rstatus_t status; 196 | 197 | status = func(elem, data); 198 | if (status != NC_OK) { 199 | return status; 200 | } 201 | } 202 | 203 | return NC_OK; 204 | } 205 | -------------------------------------------------------------------------------- /src/nc_array.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_ARRAY_H_ 19 | #define _NC_ARRAY_H_ 20 | 21 | #include 22 | 23 | typedef int (*array_compare_t)(const void *, const void *); 24 | typedef rstatus_t (*array_each_t)(void *, void *); 25 | 26 | struct array { 27 | uint32_t nelem; /* # element */ 28 | void *elem; /* element */ 29 | size_t size; /* element size */ 30 | uint32_t nalloc; /* # allocated element */ 31 | }; 32 | 33 | #define null_array { 0, NULL, 0, 0 } 34 | 35 | static inline void 36 | array_null(struct array *a) 37 | { 38 | a->nelem = 0; 39 | a->elem = NULL; 40 | a->size = 0; 41 | a->nalloc = 0; 42 | } 43 | 44 | static inline void 45 | array_set(struct array *a, void *elem, size_t size, uint32_t nalloc) 46 | { 47 | a->nelem = 0; 48 | a->elem = elem; 49 | a->size = size; 50 | a->nalloc = nalloc; 51 | } 52 | 53 | static inline uint32_t 54 | array_n(const struct array *a) 55 | { 56 | return a->nelem; 57 | } 58 | 59 | struct array *array_create(uint32_t n, size_t size); 60 | void array_destroy(struct array *a); 61 | rstatus_t array_init(struct array *a, uint32_t n, size_t size); 62 | void array_deinit(struct array *a); 63 | 64 | uint32_t array_idx(struct array *a, void *elem); 65 | void *array_push(struct array *a); 66 | void *array_pop(struct array *a); 67 | void *array_get(struct array *a, uint32_t idx); 68 | void *array_top(struct array *a); 69 | void array_swap(struct array *a, struct array *b); 70 | void array_sort(struct array *a, array_compare_t compare); 71 | rstatus_t array_each(struct array *a, array_each_t func, void *data); 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /src/nc_client.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | void 23 | client_ref(struct conn *conn, void *owner) 24 | { 25 | struct server_pool *pool = owner; 26 | 27 | ASSERT(conn->client && !conn->proxy); 28 | ASSERT(conn->owner == NULL); 29 | 30 | /* 31 | * We use null pointer as the sockaddr argument in the accept() call as 32 | * we are not interested in the address of the peer for the accepted 33 | * connection 34 | */ 35 | conn->family = 0; 36 | conn->addrlen = 0; 37 | conn->addr = NULL; 38 | 39 | pool->nc_conn_q++; 40 | TAILQ_INSERT_TAIL(&pool->c_conn_q, conn, conn_tqe); 41 | 42 | /* owner of the client connection is the server pool */ 43 | conn->owner = owner; 44 | 45 | log_debug(LOG_VVERB, "ref conn %p owner %p into pool '%.*s'", conn, pool, 46 | pool->name.len, pool->name.data); 47 | } 48 | 49 | void 50 | client_unref(struct conn *conn) 51 | { 52 | struct server_pool *pool; 53 | 54 | ASSERT(conn->client && !conn->proxy); 55 | ASSERT(conn->owner != NULL); 56 | 57 | pool = conn->owner; 58 | conn->owner = NULL; 59 | 60 | ASSERT(pool->nc_conn_q != 0); 61 | pool->nc_conn_q--; 62 | TAILQ_REMOVE(&pool->c_conn_q, conn, conn_tqe); 63 | 64 | log_debug(LOG_VVERB, "unref conn %p owner %p from pool '%.*s'", conn, 65 | pool, pool->name.len, pool->name.data); 66 | } 67 | 68 | bool 69 | client_active(struct conn *conn) 70 | { 71 | ASSERT(conn->client && !conn->proxy); 72 | 73 | ASSERT(TAILQ_EMPTY(&conn->imsg_q)); 74 | 75 | if (!TAILQ_EMPTY(&conn->omsg_q)) { 76 | log_debug(LOG_VVERB, "c %d is active", conn->sd); 77 | return true; 78 | } 79 | 80 | if (conn->rmsg != NULL) { 81 | log_debug(LOG_VVERB, "c %d is active", conn->sd); 82 | return true; 83 | } 84 | 85 | if (conn->smsg != NULL) { 86 | log_debug(LOG_VVERB, "c %d is active", conn->sd); 87 | return true; 88 | } 89 | 90 | log_debug(LOG_VVERB, "c %d is inactive", conn->sd); 91 | 92 | return false; 93 | } 94 | 95 | static void 96 | client_close_stats(struct context *ctx, struct server_pool *pool, err_t err, 97 | unsigned eof) 98 | { 99 | stats_pool_decr(ctx, pool, client_connections); 100 | 101 | if (eof) { 102 | stats_pool_incr(ctx, pool, client_eof); 103 | return; 104 | } 105 | 106 | switch (err) { 107 | case EPIPE: 108 | case ETIMEDOUT: 109 | case ECONNRESET: 110 | case ECONNABORTED: 111 | case ENOTCONN: 112 | case ENETDOWN: 113 | case ENETUNREACH: 114 | case EHOSTDOWN: 115 | case EHOSTUNREACH: 116 | default: 117 | stats_pool_incr(ctx, pool, client_err); 118 | break; 119 | } 120 | } 121 | 122 | void 123 | client_close(struct context *ctx, struct conn *conn) 124 | { 125 | rstatus_t status; 126 | struct msg *msg, *nmsg; /* current and next message */ 127 | 128 | ASSERT(conn->client && !conn->proxy); 129 | 130 | client_close_stats(ctx, conn->owner, conn->err, conn->eof); 131 | 132 | if (conn->sd < 0) { 133 | conn->unref(conn); 134 | conn_put(conn); 135 | return; 136 | } 137 | 138 | msg = conn->rmsg; 139 | if (msg != NULL) { 140 | conn->rmsg = NULL; 141 | 142 | ASSERT(msg->peer == NULL); 143 | ASSERT(msg->request && !msg->done); 144 | 145 | log_debug(LOG_INFO, "close c %d discarding pending req %"PRIu64" len " 146 | "%"PRIu32" type %d", conn->sd, msg->id, msg->mlen, 147 | msg->type); 148 | 149 | req_put(msg); 150 | } 151 | 152 | ASSERT(conn->smsg == NULL); 153 | ASSERT(TAILQ_EMPTY(&conn->imsg_q)); 154 | 155 | for (msg = TAILQ_FIRST(&conn->omsg_q); msg != NULL; msg = nmsg) { 156 | nmsg = TAILQ_NEXT(msg, c_tqe); 157 | 158 | /* dequeue the message (request) from client outq */ 159 | conn->dequeue_outq(ctx, conn, msg); 160 | 161 | if (msg->done) { 162 | log_debug(LOG_INFO, "close c %d discarding %s req %"PRIu64" len " 163 | "%"PRIu32" type %d", conn->sd, 164 | msg->error ? "error": "completed", msg->id, msg->mlen, 165 | msg->type); 166 | req_put(msg); 167 | } else { 168 | msg->swallow = 1; 169 | 170 | ASSERT(msg->request); 171 | ASSERT(msg->peer == NULL); 172 | 173 | log_debug(LOG_INFO, "close c %d schedule swallow of req %"PRIu64" " 174 | "len %"PRIu32" type %d", conn->sd, msg->id, msg->mlen, 175 | msg->type); 176 | } 177 | } 178 | ASSERT(TAILQ_EMPTY(&conn->omsg_q)); 179 | 180 | conn->unref(conn); 181 | 182 | status = close(conn->sd); 183 | if (status < 0) { 184 | log_error("close c %d failed, ignored: %s", conn->sd, strerror(errno)); 185 | } 186 | conn->sd = -1; 187 | 188 | conn_put(conn); 189 | } 190 | -------------------------------------------------------------------------------- /src/nc_client.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_CLIENT_H_ 19 | #define _NC_CLIENT_H_ 20 | 21 | #include 22 | 23 | bool client_active(struct conn *conn); 24 | void client_ref(struct conn *conn, void *owner); 25 | void client_unref(struct conn *conn); 26 | void client_close(struct context *ctx, struct conn *conn); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /src/nc_conf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_CONF_H_ 19 | #define _NC_CONF_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | #include 25 | 26 | #include 27 | #include 28 | 29 | #define CONF_OK (void *) NULL 30 | #define CONF_ERROR (void *) "has an invalid value" 31 | 32 | #define CONF_ROOT_DEPTH 1 33 | #define CONF_MAX_DEPTH CONF_ROOT_DEPTH + 1 34 | 35 | #define CONF_DEFAULT_ARGS 3 36 | #define CONF_DEFAULT_POOL 8 37 | #define CONF_DEFAULT_SERVERS 8 38 | 39 | #define CONF_UNSET_NUM -1 40 | #define CONF_UNSET_PTR NULL 41 | #define CONF_UNSET_HASH (hash_type_t) -1 42 | #define CONF_UNSET_DIST (dist_type_t) -1 43 | 44 | #if 1 //shenzheng 2015-1-8 log rotating 45 | #define CONF_UNSET_LOG_FILE_COUNT -2 46 | #endif //shenzheng 2015-1-8 log rotating 47 | 48 | #define CONF_DEFAULT_HASH HASH_FNV1A_64 49 | #define CONF_DEFAULT_DIST DIST_KETAMA 50 | #define CONF_DEFAULT_TIMEOUT -1 51 | #define CONF_DEFAULT_LISTEN_BACKLOG 512 52 | #define CONF_DEFAULT_CLIENT_CONNECTIONS 0 53 | #define CONF_DEFAULT_REDIS false 54 | #define CONF_DEFAULT_PRECONNECT false 55 | #define CONF_DEFAULT_AUTO_EJECT_HOSTS false 56 | #define CONF_DEFAULT_SERVER_RETRY_TIMEOUT 30 * 1000 /* in msec */ 57 | #define CONF_DEFAULT_SERVER_FAILURE_LIMIT 2 58 | #define CONF_DEFAULT_SERVER_CONNECTIONS 1 59 | #define CONF_DEFAULT_KETAMA_PORT 11211 60 | 61 | #if 1 //shenzheng 2015-1-8 log rotating 62 | #define CONF_DEFAULT_LOG_RORATE 1 63 | #define CONF_DEFAULT_LOG_FILE_MAX_SIZE 1073741824 64 | #define CONF_DEFAULT_LOG_FILE_COUNT 10 65 | #endif //shenzheng 2015-1-8 log rotating 66 | 67 | #if 1 //shenzheng 2015-6-5 tcpkeepalive 68 | #define CONF_DEFAULT_TCPKEEPALIVE false 69 | #define CONF_DEFAULT_TCPKEEPIDLE -1 70 | #define CONF_DEFAULT_TCPKEEPINTVL -1 71 | #define CONF_DEFAULT_TCPKEEPCNT -1 72 | #endif //shenzheng 2015-6-5 tcpkeepalive 73 | 74 | #if 1 //shenzheng 2015-6-8 config-reload 75 | typedef enum conf_parse_type { 76 | CONF_PARSE_FILE, /* conf parse from file */ 77 | CONF_PARSE_STRING /* conf parse from string */ 78 | } conf_parse_type_t; 79 | #endif //shenzheng 2015-6-8 config-reload 80 | 81 | struct conf_listen { 82 | struct string pname; /* listen: as "name:port" */ 83 | struct string name; /* name */ 84 | int port; /* port */ 85 | struct sockinfo info; /* listen socket info */ 86 | unsigned valid:1; /* valid? */ 87 | }; 88 | 89 | struct conf_server { 90 | struct string pname; /* server: as "name:port:weight" */ 91 | struct string name; /* name */ 92 | int port; /* port */ 93 | int weight; /* weight */ 94 | struct sockinfo info; /* connect socket info */ 95 | unsigned valid:1; /* valid? */ 96 | 97 | #if 1 //shenzheng 2014-9-5 replace server 98 | unsigned name_null:1; /* name in "hostname:port:weight [name]" format string is null? */ 99 | #endif //shenzheng 2014-9-5 replace server 100 | 101 | }; 102 | 103 | struct conf_pool { 104 | struct string name; /* pool name (root node) */ 105 | struct conf_listen listen; /* listen: */ 106 | hash_type_t hash; /* hash: */ 107 | struct string hash_tag; /* hash_tag: */ 108 | dist_type_t distribution; /* distribution: */ 109 | int timeout; /* timeout: */ 110 | int backlog; /* backlog: */ 111 | int client_connections; /* client_connections: */ 112 | int redis; /* redis: */ 113 | struct string redis_auth; /* redis auth password */ 114 | int preconnect; /* preconnect: */ 115 | int auto_eject_hosts; /* auto_eject_hosts: */ 116 | int server_connections; /* server_connections: */ 117 | int server_retry_timeout; /* server_retry_timeout: in msec */ 118 | int server_failure_limit; /* server_failure_limit: */ 119 | struct array server; /* servers: conf_server[] */ 120 | unsigned valid:1; /* valid? */ 121 | 122 | #if 1 //shenzheng 2015-6-5 tcpkeepalive 123 | int tcpkeepalive; /* tcpkeepalive: */ 124 | int tcpkeepidle; /* tcpkeepidle: */ 125 | int tcpkeepintvl; /* tcpkeepintvl: */ 126 | int tcpkeepcnt; /* tcpkeepcnt: */ 127 | #endif //shenzheng 2015-6-5 tcpkeepalive 128 | }; 129 | 130 | struct conf { 131 | char *fname; /* file name (ref in argv[]) */ 132 | FILE *fh; /* file handle */ 133 | struct array arg; /* string[] (parsed {key, value} pairs) */ 134 | struct array pool; /* conf_pool[] (parsed pools) */ 135 | uint32_t depth; /* parsed tree depth */ 136 | yaml_parser_t parser; /* yaml parser */ 137 | yaml_event_t event; /* yaml event */ 138 | yaml_token_t token; /* yaml token */ 139 | unsigned seq:1; /* sequence? */ 140 | unsigned valid_parser:1; /* valid parser? */ 141 | unsigned valid_event:1; /* valid event? */ 142 | unsigned valid_token:1; /* valid token? */ 143 | unsigned sound:1; /* sound? */ 144 | unsigned parsed:1; /* parsed? */ 145 | unsigned valid:1; /* valid? */ 146 | }; 147 | 148 | struct command { 149 | struct string name; 150 | char *(*set)(struct conf *cf, struct command *cmd, void *data); 151 | int offset; 152 | }; 153 | 154 | #define null_command { null_string, NULL, 0 } 155 | 156 | char *conf_set_string(struct conf *cf, struct command *cmd, void *conf); 157 | char *conf_set_listen(struct conf *cf, struct command *cmd, void *conf); 158 | char *conf_add_server(struct conf *cf, struct command *cmd, void *conf); 159 | char *conf_set_num(struct conf *cf, struct command *cmd, void *conf); 160 | char *conf_set_bool(struct conf *cf, struct command *cmd, void *conf); 161 | char *conf_set_hash(struct conf *cf, struct command *cmd, void *conf); 162 | char *conf_set_distribution(struct conf *cf, struct command *cmd, void *conf); 163 | char *conf_set_hashtag(struct conf *cf, struct command *cmd, void *conf); 164 | 165 | rstatus_t conf_server_each_transform(void *elem, void *data); 166 | rstatus_t conf_pool_each_transform(void *elem, void *data); 167 | 168 | struct conf *conf_create(char *filename); 169 | void conf_destroy(struct conf *cf); 170 | 171 | #if 1 //shenzheng 2015-6-26 replace server 172 | rstatus_t conf_write_back_yaml(struct context *ctx, struct string *old_ser, struct string *new_ser); 173 | #endif //shenzheng 2015-6-26 replace server 174 | 175 | #if 1 //shenzheng 2015-1-8 log rotating 176 | char *conf_set_log_rorate(struct conf *cf, struct command *cmd, void *conf); 177 | char *conf_set_log_file_max_size(struct conf *cf, struct command *cmd, void *conf); 178 | char *conf_set_log_file_count(struct conf *cf, struct command *cmd, void *conf); 179 | #endif //shenzheng 2015-1-8 log rotating 180 | 181 | #if 1 //shenzheng 2015-4-29 common 182 | struct string *hash_type_to_string(hash_type_t hash_type); 183 | struct string *dist_type_to_string(dist_type_t dist_type); 184 | #endif //shenzheng 2015-4-29 common 185 | 186 | #if 1 //shenzheng 2015-5-29 config-reload 187 | rstatus_t conf_two_check_diff(struct conf *cf, struct conf *cf_new); 188 | struct conf *conf_create_from_string(struct string *cf_s); 189 | rstatus_t conf_reload(struct context *ctx, struct conn *conn, conf_parse_type_t parse_type, struct string *cf_s, struct msg * msg); 190 | #endif //shenzheng 2015-5-29 config-reload 191 | 192 | #if 1 //shenzheng 2015-6-6 zookeeper 193 | #ifdef NC_ZOOKEEPER 194 | struct conf *conf_create_from_zk(struct context * ctx, char *zk_servers, char *zk_path); 195 | rstatus_t conf_keep_from_zk(struct context * ctx, void *zkhandle, char *zk_path); 196 | #endif 197 | #endif //shenzheng 2015-6-6 zookeeper 198 | 199 | #endif 200 | -------------------------------------------------------------------------------- /src/nc_connection.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_CONNECTION_H_ 19 | #define _NC_CONNECTION_H_ 20 | 21 | #include 22 | 23 | typedef rstatus_t (*conn_recv_t)(struct context *, struct conn*); 24 | typedef struct msg* (*conn_recv_next_t)(struct context *, struct conn *, bool); 25 | typedef void (*conn_recv_done_t)(struct context *, struct conn *, struct msg *, struct msg *); 26 | 27 | typedef rstatus_t (*conn_send_t)(struct context *, struct conn*); 28 | typedef struct msg* (*conn_send_next_t)(struct context *, struct conn *); 29 | typedef void (*conn_send_done_t)(struct context *, struct conn *, struct msg *); 30 | 31 | typedef void (*conn_close_t)(struct context *, struct conn *); 32 | typedef bool (*conn_active_t)(struct conn *); 33 | 34 | typedef void (*conn_ref_t)(struct conn *, void *); 35 | typedef void (*conn_unref_t)(struct conn *); 36 | 37 | typedef void (*conn_msgq_t)(struct context *, struct conn *, struct msg *); 38 | 39 | struct conn { 40 | TAILQ_ENTRY(conn) conn_tqe; /* link in server_pool / server / free q */ 41 | void *owner; /* connection owner - server_pool / server */ 42 | 43 | int sd; /* socket descriptor */ 44 | int family; /* socket address family */ 45 | socklen_t addrlen; /* socket length */ 46 | struct sockaddr *addr; /* socket address (ref in server or server_pool) */ 47 | 48 | struct msg_tqh imsg_q; /* incoming request Q */ 49 | struct msg_tqh omsg_q; /* outstanding request Q */ 50 | struct msg *rmsg; /* current message being rcvd */ 51 | struct msg *smsg; /* current message being sent */ 52 | 53 | conn_recv_t recv; /* recv (read) handler */ 54 | conn_recv_next_t recv_next; /* recv next message handler */ 55 | conn_recv_done_t recv_done; /* read done handler */ 56 | conn_send_t send; /* send (write) handler */ 57 | conn_send_next_t send_next; /* write next message handler */ 58 | conn_send_done_t send_done; /* write done handler */ 59 | conn_close_t close; /* close handler */ 60 | conn_active_t active; /* active? handler */ 61 | 62 | conn_ref_t ref; /* connection reference handler */ 63 | conn_unref_t unref; /* connection unreference handler */ 64 | 65 | conn_msgq_t enqueue_inq; /* connection inq msg enqueue handler */ 66 | conn_msgq_t dequeue_inq; /* connection inq msg dequeue handler */ 67 | conn_msgq_t enqueue_outq; /* connection outq msg enqueue handler */ 68 | conn_msgq_t dequeue_outq; /* connection outq msg dequeue handler */ 69 | 70 | size_t recv_bytes; /* received (read) bytes */ 71 | size_t send_bytes; /* sent (written) bytes */ 72 | 73 | uint32_t events; /* connection io events */ 74 | err_t err; /* connection errno */ 75 | unsigned recv_active:1; /* recv active? */ 76 | unsigned recv_ready:1; /* recv ready? */ 77 | unsigned send_active:1; /* send active? */ 78 | unsigned send_ready:1; /* send ready? */ 79 | 80 | unsigned client:1; /* client? or server? */ 81 | unsigned proxy:1; /* proxy? */ 82 | unsigned connecting:1; /* connecting? */ 83 | unsigned connected:1; /* connected? */ 84 | unsigned eof:1; /* eof? aka passive close? */ 85 | unsigned done:1; /* done? aka close? */ 86 | unsigned redis:1; /* redis? */ 87 | unsigned need_auth:1; /* need_auth? */ 88 | 89 | #if 1 //shenzheng 2015-7-14 config-reload 90 | unsigned reload_conf:1; /* for reload_conf? */ 91 | #endif //shenzheng 2015-7-14 config-reload 92 | 93 | #if 1 //shenzheng 2015-7-28 replace server 94 | unsigned replace_server:1;/* 1:this msg is for replace_server command, 0:other msgs */ 95 | long long conf_version_curr; 96 | struct context *ctx; 97 | #endif //shenzheng 2015-7-28 replace server 98 | }; 99 | 100 | TAILQ_HEAD(conn_tqh, conn); 101 | 102 | struct context *conn_to_ctx(struct conn *conn); 103 | struct conn *conn_get(void *owner, bool client, bool redis); 104 | struct conn *conn_get_proxy(void *owner); 105 | void conn_put(struct conn *conn); 106 | ssize_t conn_recv(struct conn *conn, void *buf, size_t size); 107 | ssize_t conn_sendv(struct conn *conn, struct array *sendv, size_t nsend); 108 | void conn_init(void); 109 | void conn_deinit(void); 110 | uint32_t conn_ncurr_conn(void); 111 | uint64_t conn_ntotal_conn(void); 112 | uint32_t conn_ncurr_cconn(void); 113 | 114 | #if 1 //shenzheng 2015-4-27 proxy administer 115 | uint32_t conn_ncurr_conn_proxy_adm(void); 116 | uint64_t conn_ntotal_conn_proxy_adm(void); 117 | uint32_t conn_ncurr_cconn_proxy_adm(void); 118 | void conn_put_proxy_adm(struct conn *conn); 119 | struct conn * conn_get_proxy_adm(void *owner, int sd, bool client); 120 | #endif //shenzheng 2015-7-9 proxy administer 121 | 122 | #if 1 //shenzheng 2015-7-9 config-reload 123 | struct conn * conn_get_proxy_for_reload(void *owner); 124 | struct conn * conn_get_for_reload(void *owner); 125 | void conn_put_for_reload(struct conn *conn); 126 | #endif //shenzheng 2015-7-9 config-reload 127 | 128 | #if 1 //shenzheng 2015-7-28 replace server 129 | void conn_close_for_replace_server(struct conn *conn, int err); 130 | #endif //shenzheng 2015-7-28 replace server 131 | 132 | #endif 133 | -------------------------------------------------------------------------------- /src/nc_core.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_CORE_H_ 19 | #define _NC_CORE_H_ 20 | 21 | #ifdef HAVE_CONFIG_H 22 | # include 23 | #endif 24 | 25 | #ifdef HAVE_DEBUG_LOG 26 | # define NC_DEBUG_LOG 1 27 | #endif 28 | 29 | #ifdef HAVE_ASSERT_PANIC 30 | # define NC_ASSERT_PANIC 1 31 | #endif 32 | 33 | #ifdef HAVE_ASSERT_LOG 34 | # define NC_ASSERT_LOG 1 35 | #endif 36 | 37 | #ifdef HAVE_STATS 38 | # define NC_STATS 1 39 | #else 40 | # define NC_STATS 0 41 | #endif 42 | 43 | #ifdef HAVE_EPOLL 44 | # define NC_HAVE_EPOLL 1 45 | #elif HAVE_KQUEUE 46 | # define NC_HAVE_KQUEUE 1 47 | #elif HAVE_EVENT_PORTS 48 | # define NC_HAVE_EVENT_PORTS 1 49 | #else 50 | # error missing scalable I/O event notification mechanism 51 | #endif 52 | 53 | #ifdef HAVE_LITTLE_ENDIAN 54 | # define NC_LITTLE_ENDIAN 1 55 | #endif 56 | 57 | #ifdef HAVE_BACKTRACE 58 | # define NC_HAVE_BACKTRACE 1 59 | #endif 60 | 61 | #if 1 //shenzheng 2015-6-6 zookeeper 62 | #ifdef HAVE_ZOOKEEPER 63 | # define NC_ZOOKEEPER 1 64 | #endif 65 | #endif //shenzheng 2015-6-6 zookeeper 66 | 67 | #define NC_OK 0 68 | #define NC_ERROR -1 69 | #define NC_EAGAIN -2 70 | #define NC_ENOMEM -3 71 | 72 | #if 1 //shenzheng 2014-12-4 common 73 | /** 74 | the macro defined below must be a negative number. 75 | */ 76 | #define ERROR_REPLACE_SERVER_TRY_AGAIN -1 77 | #define ERROR_REPLACE_SERVER_CONF_VERSION_CHANGE -2 78 | #endif //shenzheng 2014-12-4 common 79 | 80 | /* reserved fds for std streams, log, stats fd, epoll etc. */ 81 | #define RESERVED_FDS 32 82 | 83 | typedef int rstatus_t; /* return type */ 84 | typedef int err_t; /* error type */ 85 | 86 | struct array; 87 | struct string; 88 | struct context; 89 | struct conn; 90 | struct conn_tqh; 91 | struct msg; 92 | struct msg_tqh; 93 | struct server; 94 | struct server_pool; 95 | struct mbuf; 96 | struct mhdr; 97 | struct conf; 98 | struct stats; 99 | struct instance; 100 | struct event_base; 101 | 102 | #include 103 | #include 104 | #include 105 | #include 106 | #include 107 | #include 108 | #include 109 | #include 110 | #include 111 | #include 112 | #include 113 | #include 114 | 115 | #include 116 | #include 117 | #include 118 | #include 119 | #include 120 | #include 121 | 122 | #include 123 | #include 124 | #include 125 | #include 126 | #include 127 | #include 128 | #include 129 | #include 130 | #include 131 | #include 132 | #include 133 | #include 134 | 135 | #if 1 //shenzheng 2015-4-28 proxy administer 136 | #include 137 | #endif //shenzheng 2015-4-28 proxy administer 138 | 139 | #if 1 //shenzheng 2015-6-8 zookeeper 140 | #ifdef NC_ZOOKEEPER 141 | #include 142 | #endif 143 | #endif //shenzheng 2015-6-8 zookeeper 144 | 145 | struct context { 146 | uint32_t id; /* unique context id */ 147 | struct conf *cf; /* configuration */ 148 | struct stats *stats; /* stats */ 149 | 150 | struct array pool; /* server_pool[] */ 151 | struct event_base *evb; /* event base */ 152 | int max_timeout; /* max timeout in msec */ 153 | int timeout; /* timeout in msec */ 154 | 155 | uint32_t max_nfd; /* max # files */ 156 | uint32_t max_ncconn; /* max # client connections */ 157 | uint32_t max_nsconn; /* max # server connections */ 158 | 159 | #if 1 //shenzheng 2015-4-27 proxy administer 160 | struct proxy_adm *padm; 161 | #endif //shenzheng 2015-4-27 proxy administer 162 | 163 | #if 1 //shenzheng 2015-5-8 config-reload 164 | uint8_t which_pool:1; /* 0:ctx->pool ; 1:ctx->pool_swap */ 165 | struct conf *cf_swap; /* server_pool[] */ 166 | struct array pool_swap; /* server_pool[] */ 167 | volatile uint8_t reload_thread:1; /* 0: proxy_adm thread's right to handle reload; 168 | * 1: mian thread's right to handle reload */ 169 | volatile long long conf_version; 170 | pthread_mutex_t reload_lock; 171 | #endif //shenzheng 2015-7-27 config-reload 172 | 173 | #if 1 //shenzheng 2015-6-9 zookeeper 174 | #ifdef NC_ZOOKEEPER 175 | void *zkhandle; 176 | struct string watch_path; 177 | struct string zk_servers; 178 | #endif 179 | #endif //shenzheng 2015-6-16 zookeeper 180 | }; 181 | 182 | 183 | struct instance { 184 | struct context *ctx; /* active context */ 185 | int log_level; /* log level */ 186 | char *log_filename; /* log filename */ 187 | char *conf_filename; /* configuration filename */ 188 | uint16_t stats_port; /* stats monitoring port */ 189 | int stats_interval; /* stats aggregation interval */ 190 | char *stats_addr; /* stats monitoring addr */ 191 | char hostname[NC_MAXHOSTNAMELEN]; /* hostname */ 192 | size_t mbuf_chunk_size; /* mbuf chunk size */ 193 | pid_t pid; /* process id */ 194 | char *pid_filename; /* pid filename */ 195 | unsigned pidfile:1; /* pid file created? */ 196 | #if 1 //shenzheng 2015-4-28 proxy administer 197 | char *proxy_adm_addr; /* proxy administer monitoring addr */ 198 | uint16_t proxy_adm_port; /* proxy administer monitoring port */ 199 | #endif //shenzheng 2015-4-28 proxy administer 200 | 201 | #if 1 //shenzheng 2015-6-9 zookeeper 202 | #ifdef NC_ZOOKEEPER 203 | uint8_t zk_start:1; /* start from configuration in zookeeper */ 204 | uint8_t zk_keep:1; /* keep configuration from zookeeper */ 205 | char *zk_servers; /* zoopeeper servers' address, like 192.168.0.1:2181,192.168.0.2:2181 */ 206 | char *zk_path; /* configuration path in zookeeper */ 207 | #endif 208 | #endif //shenzheng 2015-6-9 zookeeper 209 | }; 210 | 211 | struct context *core_start(struct instance *nci); 212 | void core_stop(struct context *ctx); 213 | rstatus_t core_core(void *arg, uint32_t events); 214 | rstatus_t core_loop(struct context *ctx); 215 | 216 | #endif 217 | -------------------------------------------------------------------------------- /src/nc_log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_LOG_H_ 19 | #define _NC_LOG_H_ 20 | 21 | #if 1 //shenzheng 2014-12-9 log rotating 22 | #define LOG_ROTATE_DEFAULT 0 23 | 24 | #define LOG_FILE_MAX_SIZE_DEFAULT 1073741824 25 | #define LOG_FILE_MAX_SIZE_MIN 1000000 26 | #define LOG_FILE_MAX_SIZE_MAX 1122601371959296 27 | 28 | #define LOG_FILE_COUNT_DEFAULT 10 29 | #define LOG_FILE_COUNT_MIN -1 30 | #define LOG_FILE_COUNT_MAX LOG_FILES_CIRCULAR_MAX_LEN 31 | 32 | /* LOG_RORATE 33 | * 0 : twemproxy do not use log rotate functionality. 34 | * 1 : twemproxy use log rotate functionality. 35 | */ 36 | extern int LOG_RORATE; 37 | extern ssize_t LOG_FIEL_MAX_SIZE_FOR_ROTATING; 38 | #endif 39 | 40 | #if 1 //shenzheng 2014-12-15 log rotating step two 41 | 42 | /* LOG_FILE_COUNT_TO_STAY 43 | * -1 : do not use log_files_circular, the log bak files do not delete anymore. 44 | * =0 : there is no log bak files, only current log file exist. 45 | * >0 : use log_files_circular, and only stay newest N log bak files. 46 | */ 47 | extern int LOG_FILE_COUNT_TO_STAY; 48 | #define LOG_FILES_CIRCULAR_MAX_LEN 200 49 | #endif 50 | 51 | struct logger { 52 | char *name; /* log file name */ 53 | int level; /* log level */ 54 | int fd; /* log file descriptor */ 55 | int nerror; /* # log error */ 56 | #if 1 //shenzheng 2014-12-9 log rotating 57 | ssize_t current_log_size; /* log current size */ 58 | char* log_files_circular[LOG_FILES_CIRCULAR_MAX_LEN]; /* we allow stay LOG_FILE_COUNT_TO_STAY log files */ 59 | int circular_cur_pos; /* the current position for the circular */ 60 | bool circular_full; /* the log_files_circular already had LOG_FILE_COUNT_TO_STAY elements*/ 61 | pthread_mutex_t log_lock; 62 | #endif //shenzheng 2015-7-29 log rotating 63 | }; 64 | 65 | #define LOG_EMERG 0 /* system in unusable */ 66 | #define LOG_ALERT 1 /* action must be taken immediately */ 67 | #define LOG_CRIT 2 /* critical conditions */ 68 | #define LOG_ERR 3 /* error conditions */ 69 | #define LOG_WARN 4 /* warning conditions */ 70 | #define LOG_NOTICE 5 /* normal but significant condition (default) */ 71 | #define LOG_INFO 6 /* informational */ 72 | #define LOG_DEBUG 7 /* debug messages */ 73 | #define LOG_VERB 8 /* verbose messages */ 74 | #define LOG_VVERB 9 /* verbose messages on crack */ 75 | #define LOG_VVVERB 10 /* verbose messages on ganga */ 76 | #define LOG_PVERB 11 /* periodic verbose messages on crack */ 77 | 78 | #define LOG_MAX_LEN 256 /* max length of log message */ 79 | 80 | /* 81 | * log_stderr - log to stderr 82 | * loga - log always 83 | * loga_hexdump - log hexdump always 84 | * log_error - error log messages 85 | * log_warn - warning log messages 86 | * log_panic - log messages followed by a panic 87 | * ... 88 | * log_debug - debug log messages based on a log level 89 | * log_hexdump - hexadump -C of a log buffer 90 | */ 91 | #ifdef NC_DEBUG_LOG 92 | 93 | #define log_debug(_level, ...) do { \ 94 | if (log_loggable(_level) != 0) { \ 95 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 96 | } \ 97 | } while (0) 98 | 99 | #define log_hexdump(_level, _data, _datalen, ...) do { \ 100 | if (log_loggable(_level) != 0) { \ 101 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 102 | _log_hexdump(__FILE__, __LINE__, (char *)(_data), (int)(_datalen), \ 103 | __VA_ARGS__); \ 104 | } \ 105 | } while (0) 106 | 107 | #else 108 | 109 | #define log_debug(_level, ...) 110 | #define log_hexdump(_level, _data, _datalen, ...) 111 | 112 | #endif 113 | 114 | #define log_stderr(...) do { \ 115 | _log_stderr(__VA_ARGS__); \ 116 | } while (0) 117 | 118 | #define log_safe(...) do { \ 119 | _log_safe(__VA_ARGS__); \ 120 | } while (0) 121 | 122 | #define log_stderr_safe(...) do { \ 123 | _log_stderr_safe(__VA_ARGS__); \ 124 | } while (0) 125 | 126 | #define loga(...) do { \ 127 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 128 | } while (0) 129 | 130 | #define loga_hexdump(_data, _datalen, ...) do { \ 131 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 132 | _log_hexdump(__FILE__, __LINE__, (char *)(_data), (int)(_datalen), \ 133 | __VA_ARGS__); \ 134 | } while (0) \ 135 | 136 | #define log_error(...) do { \ 137 | if (log_loggable(LOG_ALERT) != 0) { \ 138 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 139 | } \ 140 | } while (0) 141 | 142 | #define log_warn(...) do { \ 143 | if (log_loggable(LOG_WARN) != 0) { \ 144 | _log(__FILE__, __LINE__, 0, __VA_ARGS__); \ 145 | } \ 146 | } while (0) 147 | 148 | #define log_panic(...) do { \ 149 | if (log_loggable(LOG_EMERG) != 0) { \ 150 | _log(__FILE__, __LINE__, 1, __VA_ARGS__); \ 151 | } \ 152 | } while (0) 153 | 154 | int log_init(int level, char *filename); 155 | void log_deinit(void); 156 | void log_level_up(void); 157 | void log_level_down(void); 158 | void log_level_set(int level); 159 | void log_stacktrace(void); 160 | void log_reopen(void); 161 | int log_loggable(int level); 162 | void _log(const char *file, int line, int panic, const char *fmt, ...); 163 | void _log_stderr(const char *fmt, ...); 164 | void _log_safe(const char *fmt, ...); 165 | void _log_stderr_safe(const char *fmt, ...); 166 | void _log_hexdump(const char *file, int line, char *data, int datalen, const char *fmt, ...); 167 | 168 | #if 1 //shenzheng 2014-12-9 log rotating 169 | int log_rotate_init(struct logger *l); 170 | int _log_rotate_deinit(struct logger *l); 171 | int _log_rotating(ssize_t write_bytes, struct logger *l); 172 | #endif 173 | 174 | #if 1 //shenzheng 2014-12-15 log rotating step two 175 | int _log_files_circular_maintain(const char * file, struct logger *l); 176 | int _is_log_bak_file(const char * logname, const char * filename); 177 | int _log_files_circular_insert(const char * filename, struct logger *l); 178 | int log_files_circular_init(struct logger *l); 179 | int _log_files_circular_deinit(struct logger *l); 180 | void _log_files_circular_info(struct logger *l); 181 | char * set_log_file_max_size(char *arg); 182 | char * set_log_file_count(char *arg); 183 | #endif //shenzheng 2015-1-26 log rotating step two 184 | 185 | #if 1 //shenzheng 2015-2-3 common 186 | void log_all(const char *file, int line, size_t data_len, uint8_t *data, const char *fmt, ...); 187 | #endif //shenzheng 2015-2-3 common 188 | 189 | #endif 190 | -------------------------------------------------------------------------------- /src/nc_mbuf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_MBUF_H_ 19 | #define _NC_MBUF_H_ 20 | 21 | #include 22 | 23 | typedef void (*mbuf_copy_t)(struct mbuf *, void *); 24 | 25 | struct mbuf { 26 | uint32_t magic; /* mbuf magic (const) */ 27 | STAILQ_ENTRY(mbuf) next; /* next mbuf */ 28 | uint8_t *pos; /* read marker */ 29 | uint8_t *last; /* write marker */ 30 | uint8_t *start; /* start of buffer (const) */ 31 | uint8_t *end; /* end of buffer (const) */ 32 | }; 33 | 34 | STAILQ_HEAD(mhdr, mbuf); 35 | 36 | #define MBUF_MAGIC 0xdeadbeef 37 | #define MBUF_MIN_SIZE 512 38 | #define MBUF_MAX_SIZE 16777216 39 | #define MBUF_SIZE 16384 40 | #define MBUF_HSIZE sizeof(struct mbuf) 41 | 42 | static inline bool 43 | mbuf_empty(struct mbuf *mbuf) 44 | { 45 | return mbuf->pos == mbuf->last ? true : false; 46 | } 47 | 48 | static inline bool 49 | mbuf_full(struct mbuf *mbuf) 50 | { 51 | return mbuf->last == mbuf->end ? true : false; 52 | } 53 | 54 | void mbuf_init(struct instance *nci); 55 | void mbuf_deinit(void); 56 | struct mbuf *mbuf_get(void); 57 | void mbuf_put(struct mbuf *mbuf); 58 | void mbuf_rewind(struct mbuf *mbuf); 59 | #if 1 //shenzheng 2015-4-16 common 60 | uint32_t mbuf_storage_length(struct mbuf *mbuf); 61 | #endif //shenzheng 2015-4-16 common 62 | uint32_t mbuf_length(struct mbuf *mbuf); 63 | uint32_t mbuf_size(struct mbuf *mbuf); 64 | size_t mbuf_data_size(void); 65 | void mbuf_insert(struct mhdr *mhdr, struct mbuf *mbuf); 66 | void mbuf_remove(struct mhdr *mhdr, struct mbuf *mbuf); 67 | void mbuf_copy(struct mbuf *mbuf, uint8_t *pos, size_t n); 68 | struct mbuf *mbuf_split(struct mhdr *h, uint8_t *pos, mbuf_copy_t cb, void *cbarg); 69 | 70 | #if 1 //shenzheng 2015-3-23 common 71 | #ifdef NC_DEBUG_LOG 72 | uint32_t mbuf_nfree_mbuf(void); 73 | uint64_t mbuf_ntotal_mbuf(void); 74 | #endif 75 | #endif //shenzheng 2015-3-23 common 76 | 77 | #if 1 //shenzheng 2015-7-9 proxy administer 78 | #ifdef NC_DEBUG_LOG 79 | uint32_t mbuf_nfree_mbuf_proxy_adm(void); 80 | uint64_t mbuf_ntotal_mbuf_proxy_adm(void); 81 | #endif 82 | #endif //shenzheng 2015-7-9 proxy administer 83 | 84 | #if 1 //shenzheng 2015-5-13 proxy administer 85 | struct mbuf *mbuf_get_proxy_adm(void); 86 | void mbuf_put_proxy_adm(struct mbuf *mbuf); 87 | #endif //shenzheng 2015-5-13 proxy administer 88 | 89 | 90 | #endif 91 | -------------------------------------------------------------------------------- /src/nc_proxy.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_PROXY_H_ 19 | #define _NC_PROXY_H_ 20 | 21 | #include 22 | 23 | void proxy_ref(struct conn *conn, void *owner); 24 | void proxy_unref(struct conn *conn); 25 | void proxy_close(struct context *ctx, struct conn *conn); 26 | 27 | rstatus_t proxy_each_init(void *elem, void *data); 28 | rstatus_t proxy_each_deinit(void *elem, void *data); 29 | 30 | rstatus_t proxy_init(struct context *ctx); 31 | void proxy_deinit(struct context *ctx); 32 | rstatus_t proxy_recv(struct context *ctx, struct conn *conn); 33 | 34 | #if 1 //shenzheng 2015-4-27 proxy administer 35 | 36 | #define PROXY_ADM_ADDR "0.0.0.0" 37 | #define PROXY_ADM_PORT 0//22223 38 | 39 | struct proxy_adm{ 40 | 41 | struct string addrstr; 42 | uint16_t port; /* proxy administer monitoring port */ 43 | 44 | struct sockinfo si; 45 | 46 | struct event_base *evb; 47 | 48 | pthread_t tid; /* proxy administer aggregator thread */ 49 | 50 | struct conn *p_conn; /* proxy administer connection (listener) */ 51 | uint32_t nc_conn_q; /* # client connection */ 52 | struct conn_tqh c_conn_q; /* client connection q */ 53 | }; 54 | 55 | struct proxy_adm * proxy_adm_create(struct context *ctx, char *proxy_adm_ip, uint16_t proxy_adm_port); 56 | void proxy_adm_destroy(struct proxy_adm *padm); 57 | rstatus_t proxy_adm_loop(void *arg, uint32_t events); 58 | 59 | rstatus_t proxy_adm_recv(struct context *ctx, struct conn *conn); 60 | void proxy_adm_close(struct context *ctx, struct conn *conn); 61 | void proxy_adm_ref(struct conn *conn, void *owner); 62 | void proxy_adm_unref(struct conn *conn); 63 | 64 | bool proxy_adm_client_active(struct conn *conn); 65 | void proxy_adm_client_close(struct context *ctx, struct conn *conn); 66 | void proxy_adm_client_ref(struct conn *conn, void *owner); 67 | void proxy_adm_client_unref(struct conn *conn); 68 | 69 | struct msg * proxy_adm_make_response(struct context *ctx, struct conn *conn, struct msg * req); 70 | 71 | #endif //shenzheng 2015-4-27 proxy administer 72 | 73 | #if 1 //shenzheng 2015-7-9 config-reload 74 | rstatus_t proxy_init_for_reload(struct server_pool *pool); 75 | rstatus_t proxy_each_deinit_for_reload(void *elem, void *data); 76 | #endif //shenzheng 2015-7-9 config-reload 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /src/nc_rbtree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_RBTREE_ 19 | #define _NC_RBTREE_ 20 | 21 | #define rbtree_red(_node) ((_node)->color = 1) 22 | #define rbtree_black(_node) ((_node)->color = 0) 23 | #define rbtree_is_red(_node) ((_node)->color) 24 | #define rbtree_is_black(_node) (!rbtree_is_red(_node)) 25 | #define rbtree_copy_color(_n1, _n2) ((_n1)->color = (_n2)->color) 26 | 27 | struct rbnode { 28 | struct rbnode *left; /* left link */ 29 | struct rbnode *right; /* right link */ 30 | struct rbnode *parent; /* parent link */ 31 | int64_t key; /* key for ordering */ 32 | void *data; /* opaque data */ 33 | uint8_t color; /* red | black */ 34 | }; 35 | 36 | struct rbtree { 37 | struct rbnode *root; /* root node */ 38 | struct rbnode *sentinel; /* nil node */ 39 | }; 40 | 41 | void rbtree_node_init(struct rbnode *node); 42 | void rbtree_init(struct rbtree *tree, struct rbnode *node); 43 | struct rbnode *rbtree_min(struct rbtree *tree); 44 | void rbtree_insert(struct rbtree *tree, struct rbnode *node); 45 | void rbtree_delete(struct rbtree *tree, struct rbnode *node); 46 | 47 | #endif 48 | -------------------------------------------------------------------------------- /src/nc_server.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_SERVER_H_ 19 | #define _NC_SERVER_H_ 20 | 21 | #include 22 | 23 | /* 24 | * server_pool is a collection of servers and their continuum. Each 25 | * server_pool is the owner of a single proxy connection and one or 26 | * more client connections. server_pool itself is owned by the current 27 | * context. 28 | * 29 | * Each server is the owner of one or more server connections. server 30 | * itself is owned by the server_pool. 31 | * 32 | * +-------------+ 33 | * | |<---------------------+ 34 | * | |<------------+ | 35 | * | | +-------+--+-----+----+--------------+ 36 | * | pool 0 |+--->| | | | 37 | * | | | server 0 | server 1 | ... ... | 38 | * | | | | | |--+ 39 | * | | +----------+----------+--------------+ | 40 | * +-------------+ // 41 | * | | 42 | * | | 43 | * | | 44 | * | pool 1 | 45 | * | | 46 | * | | 47 | * | | 48 | * +-------------+ 49 | * | | 50 | * | | 51 | * . . 52 | * . ... . 53 | * . . 54 | * | | 55 | * | | 56 | * +-------------+ 57 | * | 58 | * | 59 | * // 60 | */ 61 | 62 | typedef uint32_t (*hash_t)(const char *, size_t); 63 | 64 | struct continuum { 65 | uint32_t index; /* server index */ 66 | uint32_t value; /* hash value */ 67 | }; 68 | 69 | struct server { 70 | uint32_t idx; /* server index */ 71 | struct server_pool *owner; /* owner pool */ 72 | 73 | struct string pname; /* name:port:weight (ref in conf_server) */ 74 | struct string name; /* name (ref in conf_server) */ 75 | uint16_t port; /* port */ 76 | uint32_t weight; /* weight */ 77 | int family; /* socket family */ 78 | socklen_t addrlen; /* socket length */ 79 | struct sockaddr *addr; /* socket address (ref in conf_server) */ 80 | 81 | uint32_t ns_conn_q; /* # server connection */ 82 | struct conn_tqh s_conn_q; /* server connection q */ 83 | 84 | int64_t next_retry; /* next retry time in usec */ 85 | uint32_t failure_count; /* # consecutive failures */ 86 | 87 | #if 1 //shenzheng 2014-9-5 replace server 88 | unsigned name_null:1; /* name in "hostname:port:weight [name]" format string is null? */ 89 | #endif //shenzheng 2014-9-5 replace server 90 | }; 91 | 92 | struct server_pool { 93 | uint32_t idx; /* pool index */ 94 | struct context *ctx; /* owner context */ 95 | 96 | struct conn *p_conn; /* proxy connection (listener) */ 97 | uint32_t nc_conn_q; /* # client connection */ 98 | struct conn_tqh c_conn_q; /* client connection q */ 99 | 100 | struct array server; /* server[] */ 101 | uint32_t ncontinuum; /* # continuum points */ 102 | uint32_t nserver_continuum; /* # servers - live and dead on continuum (const) */ 103 | struct continuum *continuum; /* continuum */ 104 | uint32_t nlive_server; /* # live server */ 105 | int64_t next_rebuild; /* next distribution rebuild time in usec */ 106 | 107 | struct string name; /* pool name (ref in conf_pool) */ 108 | struct string addrstr; /* pool address (ref in conf_pool) */ 109 | struct string redis_auth; /* redis_auth password */ 110 | uint16_t port; /* port */ 111 | int family; /* socket family */ 112 | socklen_t addrlen; /* socket length */ 113 | struct sockaddr *addr; /* socket address (ref in conf_pool) */ 114 | int dist_type; /* distribution type (dist_type_t) */ 115 | int key_hash_type; /* key hash type (hash_type_t) */ 116 | hash_t key_hash; /* key hasher */ 117 | struct string hash_tag; /* key hash tag (ref in conf_pool) */ 118 | int timeout; /* timeout in msec */ 119 | int backlog; /* listen backlog */ 120 | uint32_t client_connections; /* maximum # client connection */ 121 | uint32_t server_connections; /* maximum # server connection */ 122 | int64_t server_retry_timeout; /* server retry timeout in usec */ 123 | uint32_t server_failure_limit; /* server failure limit */ 124 | unsigned auto_eject_hosts:1; /* auto_eject_hosts? */ 125 | unsigned preconnect:1; /* preconnect? */ 126 | unsigned redis:1; /* redis? */ 127 | 128 | #if 1 //shenzheng 2015-6-5 tcpkeepalive 129 | unsigned tcpkeepalive:1; /* tcp keepalive? */ 130 | int tcpkeepidle; /* tcpkeep idle */ 131 | int tcpkeepintvl; /* tcpkeep interval */ 132 | int tcpkeepcnt; /* tcpkeep count */ 133 | #endif //shenzheng 2015-6-5 tcpkeepalive 134 | 135 | }; 136 | 137 | void server_ref(struct conn *conn, void *owner); 138 | void server_unref(struct conn *conn); 139 | int server_timeout(struct conn *conn); 140 | bool server_active(struct conn *conn); 141 | rstatus_t server_init(struct array *server, struct array *conf_server, struct server_pool *sp); 142 | void server_deinit(struct array *server); 143 | struct conn *server_conn(struct server *server); 144 | rstatus_t server_connect(struct context *ctx, struct server *server, struct conn *conn); 145 | void server_close(struct context *ctx, struct conn *conn); 146 | void server_connected(struct context *ctx, struct conn *conn); 147 | void server_ok(struct context *ctx, struct conn *conn); 148 | 149 | uint32_t server_pool_idx(struct server_pool *pool, uint8_t *key, uint32_t keylen); 150 | struct conn *server_pool_conn(struct context *ctx, struct server_pool *pool, uint8_t *key, uint32_t keylen); 151 | 152 | #if 1 //shenzheng 2015-6-25 replace server 153 | struct conn *server_pool_conn_for_replace(struct context *ctx, struct server_pool *pool, struct msg *msg); 154 | void server_close_for_replace_server(struct server *server); 155 | #endif //shenzheng 2015-6-25 replace server 156 | 157 | rstatus_t server_pool_run(struct server_pool *pool); 158 | rstatus_t server_pool_preconnect(struct context *ctx); 159 | void server_pool_disconnect(struct context *ctx); 160 | rstatus_t server_pool_init(struct array *server_pool, struct array *conf_pool, struct context *ctx); 161 | void server_pool_deinit(struct array *server_pool); 162 | 163 | #if 1 //shenzheng 2015-5-11 config-reload 164 | rstatus_t server_pool_each_proxy_conn_new(void *elem, void *data); 165 | rstatus_t server_pool_each_conn_old_close(void *elem, void *data); 166 | rstatus_t server_pool_each_proxy_conn_reset(void *elem, void *data); 167 | rstatus_t server_pool_each_client_conn_reset(void *elem, void *data); 168 | rstatus_t server_pool_old_deinit(struct array *sps, struct context *ctx); 169 | #endif //shenzheng 2015-5-11 config-reload 170 | 171 | #endif 172 | -------------------------------------------------------------------------------- /src/nc_signal.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | #include 23 | 24 | static struct signal signals[] = { 25 | { SIGUSR1, "SIGUSR1", 0, signal_handler }, 26 | { SIGUSR2, "SIGUSR2", 0, signal_handler }, 27 | { SIGTTIN, "SIGTTIN", 0, signal_handler }, 28 | { SIGTTOU, "SIGTTOU", 0, signal_handler }, 29 | { SIGHUP, "SIGHUP", 0, signal_handler }, 30 | { SIGINT, "SIGINT", 0, signal_handler }, 31 | { SIGSEGV, "SIGSEGV", (int)SA_RESETHAND, signal_handler }, 32 | { SIGPIPE, "SIGPIPE", 0, SIG_IGN }, 33 | { 0, NULL, 0, NULL } 34 | }; 35 | 36 | rstatus_t 37 | signal_init(void) 38 | { 39 | struct signal *sig; 40 | 41 | for (sig = signals; sig->signo != 0; sig++) { 42 | rstatus_t status; 43 | struct sigaction sa; 44 | 45 | memset(&sa, 0, sizeof(sa)); 46 | sa.sa_handler = sig->handler; 47 | sa.sa_flags = sig->flags; 48 | sigemptyset(&sa.sa_mask); 49 | 50 | status = sigaction(sig->signo, &sa, NULL); 51 | if (status < 0) { 52 | log_error("sigaction(%s) failed: %s", sig->signame, 53 | strerror(errno)); 54 | return NC_ERROR; 55 | } 56 | } 57 | 58 | return NC_OK; 59 | } 60 | 61 | void 62 | signal_deinit(void) 63 | { 64 | } 65 | 66 | void 67 | signal_handler(int signo) 68 | { 69 | struct signal *sig; 70 | void (*action)(void); 71 | char *actionstr; 72 | bool done; 73 | 74 | for (sig = signals; sig->signo != 0; sig++) { 75 | if (sig->signo == signo) { 76 | break; 77 | } 78 | } 79 | ASSERT(sig->signo != 0); 80 | 81 | actionstr = ""; 82 | action = NULL; 83 | done = false; 84 | 85 | switch (signo) { 86 | case SIGUSR1: 87 | #if 1 //shenzheng 2015-3-26 for debug 88 | #ifdef NC_DEBUG_LOG 89 | actionstr = ", print timeout used msgs"; 90 | action = print_timeout_used_msgs; 91 | #endif 92 | #endif //shenzheng 2015-3-26 for debug 93 | break; 94 | 95 | case SIGUSR2: 96 | #if 1 //shenzheng 2015-3-26 for debug 97 | #ifdef NC_DEBUG_LOG 98 | actionstr = ", print used msgs"; 99 | action = print_used_msgs; 100 | #endif 101 | #endif //shenzheng 2015-3-26 for debug 102 | break; 103 | 104 | case SIGTTIN: 105 | actionstr = ", up logging level"; 106 | action = log_level_up; 107 | break; 108 | 109 | case SIGTTOU: 110 | actionstr = ", down logging level"; 111 | action = log_level_down; 112 | break; 113 | 114 | case SIGHUP: 115 | actionstr = ", reopening log file"; 116 | action = log_reopen; 117 | break; 118 | 119 | case SIGINT: 120 | done = true; 121 | actionstr = ", exiting"; 122 | break; 123 | 124 | case SIGSEGV: 125 | log_stacktrace(); 126 | actionstr = ", core dumping"; 127 | raise(SIGSEGV); 128 | break; 129 | 130 | default: 131 | NOT_REACHED(); 132 | } 133 | 134 | log_safe("signal %d (%s) received%s", signo, sig->signame, actionstr); 135 | 136 | if (action != NULL) { 137 | action(); 138 | } 139 | 140 | if (done) { 141 | exit(1); 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /src/nc_signal.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_SIGNAL_H_ 19 | #define _NC_SIGNAL_H_ 20 | 21 | #include 22 | 23 | struct signal { 24 | int signo; 25 | char *signame; 26 | int flags; 27 | void (*handler)(int signo); 28 | }; 29 | 30 | rstatus_t signal_init(void); 31 | void signal_deinit(void); 32 | void signal_handler(int signo); 33 | 34 | #endif 35 | -------------------------------------------------------------------------------- /src/nc_string.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #include 22 | 23 | /* 24 | * String (struct string) is a sequence of unsigned char objects terminated 25 | * by the null character '\0'. The length of the string is pre-computed and 26 | * made available explicitly as an additional field. This means that we don't 27 | * have to walk the entire character sequence until the null terminating 28 | * character everytime that the length of the String is requested 29 | * 30 | * The only way to create a String is to initialize it using, string_init() 31 | * and duplicate an existing String - string_duplicate() or copy an existing 32 | * raw sequence of character bytes - string_copy(). Such String's must be 33 | * freed using string_deinit() 34 | * 35 | * We can also create String as reference to raw string - string_set_raw() 36 | * or to text string - string_set_text() or string(). Such String don't have 37 | * to be freed. 38 | */ 39 | 40 | void 41 | string_init(struct string *str) 42 | { 43 | str->len = 0; 44 | str->data = NULL; 45 | } 46 | 47 | void 48 | string_deinit(struct string *str) 49 | { 50 | ASSERT((str->len == 0 && str->data == NULL) || 51 | (str->len != 0 && str->data != NULL)); 52 | 53 | if (str->data != NULL) { 54 | nc_free(str->data); 55 | string_init(str); 56 | } 57 | } 58 | 59 | bool 60 | string_empty(const struct string *str) 61 | { 62 | ASSERT((str->len == 0 && str->data == NULL) || 63 | (str->len != 0 && str->data != NULL)); 64 | return str->len == 0 ? true : false; 65 | } 66 | 67 | rstatus_t 68 | string_duplicate(struct string *dst, const struct string *src) 69 | { 70 | ASSERT(dst->len == 0 && dst->data == NULL); 71 | ASSERT(src->len != 0 && src->data != NULL); 72 | 73 | dst->data = nc_strndup(src->data, src->len + 1); 74 | if (dst->data == NULL) { 75 | return NC_ENOMEM; 76 | } 77 | 78 | dst->len = src->len; 79 | dst->data[dst->len] = '\0'; 80 | 81 | return NC_OK; 82 | } 83 | 84 | rstatus_t 85 | string_copy(struct string *dst, const uint8_t *src, uint32_t srclen) 86 | { 87 | ASSERT(dst->len == 0 && dst->data == NULL); 88 | ASSERT(src != NULL && srclen != 0); 89 | 90 | dst->data = nc_strndup(src, srclen + 1); 91 | if (dst->data == NULL) { 92 | return NC_ENOMEM; 93 | } 94 | 95 | dst->len = srclen; 96 | dst->data[dst->len] = '\0'; 97 | 98 | return NC_OK; 99 | } 100 | 101 | int 102 | string_compare(const struct string *s1, const struct string *s2) 103 | { 104 | if (s1->len != s2->len) { 105 | return s1->len > s2->len ? 1 : -1; 106 | } 107 | 108 | return nc_strncmp(s1->data, s2->data, s1->len); 109 | } 110 | 111 | static char * 112 | _safe_utoa(int _base, uint64_t val, char *buf) 113 | { 114 | char hex[] = "0123456789abcdef"; 115 | uint32_t base = (uint32_t) _base; 116 | *buf-- = 0; 117 | do { 118 | *buf-- = hex[val % base]; 119 | } while ((val /= base) != 0); 120 | return buf + 1; 121 | } 122 | 123 | static char * 124 | _safe_itoa(int base, int64_t val, char *buf) 125 | { 126 | char hex[] = "0123456789abcdef"; 127 | char *orig_buf = buf; 128 | const int32_t is_neg = (val < 0); 129 | *buf-- = 0; 130 | 131 | if (is_neg) { 132 | val = -val; 133 | } 134 | if (is_neg && base == 16) { 135 | int ix; 136 | val -= 1; 137 | for (ix = 0; ix < 16; ++ix) 138 | buf[-ix] = '0'; 139 | } 140 | 141 | do { 142 | *buf-- = hex[val % base]; 143 | } while ((val /= base) != 0); 144 | 145 | if (is_neg && base == 10) { 146 | *buf-- = '-'; 147 | } 148 | 149 | if (is_neg && base == 16) { 150 | int ix; 151 | buf = orig_buf - 1; 152 | for (ix = 0; ix < 16; ++ix, --buf) { 153 | /* *INDENT-OFF* */ 154 | switch (*buf) { 155 | case '0': *buf = 'f'; break; 156 | case '1': *buf = 'e'; break; 157 | case '2': *buf = 'd'; break; 158 | case '3': *buf = 'c'; break; 159 | case '4': *buf = 'b'; break; 160 | case '5': *buf = 'a'; break; 161 | case '6': *buf = '9'; break; 162 | case '7': *buf = '8'; break; 163 | case '8': *buf = '7'; break; 164 | case '9': *buf = '6'; break; 165 | case 'a': *buf = '5'; break; 166 | case 'b': *buf = '4'; break; 167 | case 'c': *buf = '3'; break; 168 | case 'd': *buf = '2'; break; 169 | case 'e': *buf = '1'; break; 170 | case 'f': *buf = '0'; break; 171 | } 172 | /* *INDENT-ON* */ 173 | } 174 | } 175 | return buf + 1; 176 | } 177 | 178 | static const char * 179 | _safe_check_longlong(const char *fmt, int32_t * have_longlong) 180 | { 181 | *have_longlong = false; 182 | if (*fmt == 'l') { 183 | fmt++; 184 | if (*fmt != 'l') { 185 | *have_longlong = (sizeof(long) == sizeof(int64_t)); 186 | } else { 187 | fmt++; 188 | *have_longlong = true; 189 | } 190 | } 191 | return fmt; 192 | } 193 | 194 | int 195 | _safe_vsnprintf(char *to, size_t size, const char *format, va_list ap) 196 | { 197 | char *start = to; 198 | char *end = start + size - 1; 199 | for (; *format; ++format) { 200 | int32_t have_longlong = false; 201 | if (*format != '%') { 202 | if (to == end) { /* end of buffer */ 203 | break; 204 | } 205 | *to++ = *format; /* copy ordinary char */ 206 | continue; 207 | } 208 | ++format; /* skip '%' */ 209 | 210 | format = _safe_check_longlong(format, &have_longlong); 211 | 212 | switch (*format) { 213 | case 'd': 214 | case 'i': 215 | case 'u': 216 | case 'x': 217 | case 'p': 218 | { 219 | int64_t ival = 0; 220 | uint64_t uval = 0; 221 | if (*format == 'p') 222 | have_longlong = (sizeof(void *) == sizeof(uint64_t)); 223 | if (have_longlong) { 224 | if (*format == 'u') { 225 | uval = va_arg(ap, uint64_t); 226 | } else { 227 | ival = va_arg(ap, int64_t); 228 | } 229 | } else { 230 | if (*format == 'u') { 231 | uval = va_arg(ap, uint32_t); 232 | } else { 233 | ival = va_arg(ap, int32_t); 234 | } 235 | } 236 | 237 | { 238 | char buff[22]; 239 | const int base = (*format == 'x' || *format == 'p') ? 16 : 10; 240 | 241 | /* *INDENT-OFF* */ 242 | char *val_as_str = (*format == 'u') ? 243 | _safe_utoa(base, uval, &buff[sizeof(buff) - 1]) : 244 | _safe_itoa(base, ival, &buff[sizeof(buff) - 1]); 245 | /* *INDENT-ON* */ 246 | 247 | /* Strip off "ffffffff" if we have 'x' format without 'll' */ 248 | if (*format == 'x' && !have_longlong && ival < 0) { 249 | val_as_str += 8; 250 | } 251 | 252 | while (*val_as_str && to < end) { 253 | *to++ = *val_as_str++; 254 | } 255 | continue; 256 | } 257 | } 258 | case 's': 259 | { 260 | const char *val = va_arg(ap, char *); 261 | if (!val) { 262 | val = "(null)"; 263 | } 264 | while (*val && to < end) { 265 | *to++ = *val++; 266 | } 267 | continue; 268 | } 269 | } 270 | } 271 | *to = 0; 272 | return (int)(to - start); 273 | } 274 | 275 | int 276 | _safe_snprintf(char *to, size_t n, const char *fmt, ...) 277 | { 278 | int result; 279 | va_list args; 280 | va_start(args, fmt); 281 | result = _safe_vsnprintf(to, n, fmt, args); 282 | va_end(args); 283 | return result; 284 | } 285 | -------------------------------------------------------------------------------- /src/nc_string.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_STRING_H_ 19 | #define _NC_STRING_H_ 20 | 21 | #include 22 | #include 23 | #include 24 | 25 | #include 26 | 27 | struct string { 28 | uint32_t len; /* string length */ 29 | uint8_t *data; /* string data */ 30 | }; 31 | 32 | #define string(_str) { sizeof(_str) - 1, (uint8_t *)(_str) } 33 | #define null_string { 0, NULL } 34 | 35 | #define string_set_text(_str, _text) do { \ 36 | (_str)->len = (uint32_t)(sizeof(_text) - 1);\ 37 | (_str)->data = (uint8_t *)(_text); \ 38 | } while (0); 39 | 40 | #define string_set_raw(_str, _raw) do { \ 41 | (_str)->len = (uint32_t)(nc_strlen(_raw)); \ 42 | (_str)->data = (uint8_t *)(_raw); \ 43 | } while (0); 44 | 45 | void string_init(struct string *str); 46 | void string_deinit(struct string *str); 47 | bool string_empty(const struct string *str); 48 | rstatus_t string_duplicate(struct string *dst, const struct string *src); 49 | rstatus_t string_copy(struct string *dst, const uint8_t *src, uint32_t srclen); 50 | int string_compare(const struct string *s1, const struct string *s2); 51 | 52 | /* 53 | * Wrapper around common routines for manipulating C character 54 | * strings 55 | */ 56 | #define nc_memcpy(_d, _c, _n) \ 57 | memcpy(_d, _c, (size_t)(_n)) 58 | 59 | #define nc_memmove(_d, _c, _n) \ 60 | memmove(_d, _c, (size_t)(_n)) 61 | 62 | #define nc_memchr(_d, _c, _n) \ 63 | memchr(_d, _c, (size_t)(_n)) 64 | 65 | #define nc_strlen(_s) \ 66 | strlen((char *)(_s)) 67 | 68 | #define nc_strncmp(_s1, _s2, _n) \ 69 | strncmp((char *)(_s1), (char *)(_s2), (size_t)(_n)) 70 | 71 | #define nc_strchr(_p, _l, _c) \ 72 | _nc_strchr((uint8_t *)(_p), (uint8_t *)(_l), (uint8_t)(_c)) 73 | 74 | #define nc_strrchr(_p, _s, _c) \ 75 | _nc_strrchr((uint8_t *)(_p),(uint8_t *)(_s), (uint8_t)(_c)) 76 | 77 | #define nc_strndup(_s, _n) \ 78 | (uint8_t *)strndup((char *)(_s), (size_t)(_n)); 79 | 80 | #define nc_snprintf(_s, _n, ...) \ 81 | snprintf((char *)(_s), (size_t)(_n), __VA_ARGS__) 82 | 83 | #define nc_scnprintf(_s, _n, ...) \ 84 | _scnprintf((char *)(_s), (size_t)(_n), __VA_ARGS__) 85 | 86 | #define nc_vscnprintf(_s, _n, _f, _a) \ 87 | _vscnprintf((char *)(_s), (size_t)(_n), _f, _a) 88 | 89 | #define nc_strftime(_s, _n, fmt, tm) \ 90 | (int)strftime((char *)(_s), (size_t)(_n), fmt, tm) 91 | 92 | /* 93 | * A (very) limited version of snprintf 94 | * @param to Destination buffer 95 | * @param n Size of destination buffer 96 | * @param fmt printf() style format string 97 | * @returns Number of bytes written, including terminating '\0' 98 | * Supports 'd' 'i' 'u' 'x' 'p' 's' conversion 99 | * Supports 'l' and 'll' modifiers for integral types 100 | * Does not support any width/precision 101 | * Implemented with simplicity, and async-signal-safety in mind 102 | */ 103 | int _safe_vsnprintf(char *to, size_t size, const char *format, va_list ap); 104 | int _safe_snprintf(char *to, size_t n, const char *fmt, ...); 105 | 106 | #define nc_safe_snprintf(_s, _n, ...) \ 107 | _safe_snprintf((char *)(_s), (size_t)(_n), __VA_ARGS__) 108 | 109 | #define nc_safe_vsnprintf(_s, _n, _f, _a) \ 110 | _safe_vsnprintf((char *)(_s), (size_t)(_n), _f, _a) 111 | 112 | static inline uint8_t * 113 | _nc_strchr(uint8_t *p, uint8_t *last, uint8_t c) 114 | { 115 | while (p < last) { 116 | if (*p == c) { 117 | return p; 118 | } 119 | p++; 120 | } 121 | 122 | return NULL; 123 | } 124 | 125 | static inline uint8_t * 126 | _nc_strrchr(uint8_t *p, uint8_t *start, uint8_t c) 127 | { 128 | while (p >= start) { 129 | if (*p == c) { 130 | return p; 131 | } 132 | p--; 133 | } 134 | 135 | return NULL; 136 | } 137 | 138 | #endif 139 | -------------------------------------------------------------------------------- /src/nc_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_UTIL_H_ 19 | #define _NC_UTIL_H_ 20 | 21 | #include 22 | 23 | #define LF (uint8_t) 10 24 | #define CR (uint8_t) 13 25 | #define CRLF "\x0d\x0a" 26 | #define CRLF_LEN (sizeof("\x0d\x0a") - 1) 27 | 28 | #define NELEMS(a) ((sizeof(a)) / sizeof((a)[0])) 29 | 30 | #define MIN(a, b) ((a) < (b) ? (a) : (b)) 31 | #define MAX(a, b) ((a) > (b) ? (a) : (b)) 32 | 33 | #define SQUARE(d) ((d) * (d)) 34 | #define VAR(s, s2, n) (((n) < 2) ? 0.0 : ((s2) - SQUARE(s)/(n)) / ((n) - 1)) 35 | #define STDDEV(s, s2, n) (((n) < 2) ? 0.0 : sqrt(VAR((s), (s2), (n)))) 36 | 37 | #define NC_INET4_ADDRSTRLEN (sizeof("255.255.255.255") - 1) 38 | #define NC_INET6_ADDRSTRLEN \ 39 | (sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255") - 1) 40 | #define NC_INET_ADDRSTRLEN MAX(NC_INET4_ADDRSTRLEN, NC_INET6_ADDRSTRLEN) 41 | #define NC_UNIX_ADDRSTRLEN \ 42 | (sizeof(struct sockaddr_un) - offsetof(struct sockaddr_un, sun_path)) 43 | 44 | #define NC_MAXHOSTNAMELEN 256 45 | 46 | /* 47 | * Length of 1 byte, 2 bytes, 4 bytes, 8 bytes and largest integral 48 | * type (uintmax_t) in ascii, including the null terminator '\0' 49 | * 50 | * From stdint.h, we have: 51 | * # define UINT8_MAX (255) 52 | * # define UINT16_MAX (65535) 53 | * # define UINT32_MAX (4294967295U) 54 | * # define UINT64_MAX (__UINT64_C(18446744073709551615)) 55 | */ 56 | #define NC_UINT8_MAXLEN (3 + 1) 57 | #define NC_UINT16_MAXLEN (5 + 1) 58 | #define NC_UINT32_MAXLEN (10 + 1) 59 | #define NC_UINT64_MAXLEN (20 + 1) 60 | #define NC_UINTMAX_MAXLEN NC_UINT64_MAXLEN 61 | 62 | /* 63 | * Make data 'd' or pointer 'p', n-byte aligned, where n is a power of 2 64 | * of 2. 65 | */ 66 | #define NC_ALIGNMENT sizeof(unsigned long) /* platform word */ 67 | #define NC_ALIGN(d, n) (((d) + (n - 1)) & ~(n - 1)) 68 | #define NC_ALIGN_PTR(p, n) \ 69 | (void *) (((uintptr_t) (p) + ((uintptr_t) n - 1)) & ~((uintptr_t) n - 1)) 70 | 71 | /* 72 | * Wrapper to workaround well known, safe, implicit type conversion when 73 | * invoking system calls. 74 | */ 75 | #define nc_gethostname(_name, _len) \ 76 | gethostname((char *)_name, (size_t)_len) 77 | 78 | #define nc_atoi(_line, _n) \ 79 | _nc_atoi((uint8_t *)_line, (size_t)_n) 80 | 81 | #if 1 //shenzheng 2014-9-5 common 82 | #define nc_itos(_s, _num) \ 83 | _nc_itos((struct string *)_s, (int)_num) 84 | #define nc_ltos(_s, _num) \ 85 | _nc_ltos((struct string *)_s, (int64_t)_num) 86 | #define nc_utos(_s, _num) \ 87 | _nc_utos((struct string *)_s, (uint64_t)_num) 88 | #endif //shenzheng 2015-4-30 common 89 | 90 | 91 | int nc_set_blocking(int sd); 92 | int nc_set_nonblocking(int sd); 93 | int nc_set_reuseaddr(int sd); 94 | int nc_set_tcpnodelay(int sd); 95 | int nc_set_linger(int sd, int timeout); 96 | int nc_set_sndbuf(int sd, int size); 97 | int nc_set_rcvbuf(int sd, int size); 98 | int nc_get_soerror(int sd); 99 | int nc_get_sndbuf(int sd); 100 | int nc_get_rcvbuf(int sd); 101 | 102 | #if 1 //shenzheng 2015-6-5 tcpkeepalive 103 | int nc_set_tcpkeepalive(int sd, int keepidle, int keepinterval, int keepcount); 104 | #endif //shenzheng 2015-6-5 tcpkeepalive 105 | 106 | int _nc_atoi(uint8_t *line, size_t n); 107 | 108 | #if 1 //shenzheng 2014-9-5 common 109 | void _nc_itos(struct string *s,int num); 110 | void _nc_ltos(struct string *s, int64_t num); 111 | void _nc_utos(struct string *s,uint64_t num); 112 | #endif //shenzheng 2015-4-30 common 113 | 114 | bool nc_valid_port(int n); 115 | 116 | /* 117 | * Memory allocation and free wrappers. 118 | * 119 | * These wrappers enables us to loosely detect double free, dangling 120 | * pointer access and zero-byte alloc. 121 | */ 122 | #define nc_alloc(_s) \ 123 | _nc_alloc((size_t)(_s), __FILE__, __LINE__) 124 | 125 | #define nc_zalloc(_s) \ 126 | _nc_zalloc((size_t)(_s), __FILE__, __LINE__) 127 | 128 | #define nc_calloc(_n, _s) \ 129 | _nc_calloc((size_t)(_n), (size_t)(_s), __FILE__, __LINE__) 130 | 131 | #define nc_realloc(_p, _s) \ 132 | _nc_realloc(_p, (size_t)(_s), __FILE__, __LINE__) 133 | 134 | #define nc_free(_p) do { \ 135 | _nc_free(_p, __FILE__, __LINE__); \ 136 | (_p) = NULL; \ 137 | } while (0) 138 | 139 | void *_nc_alloc(size_t size, const char *name, int line); 140 | void *_nc_zalloc(size_t size, const char *name, int line); 141 | void *_nc_calloc(size_t nmemb, size_t size, const char *name, int line); 142 | void *_nc_realloc(void *ptr, size_t size, const char *name, int line); 143 | void _nc_free(void *ptr, const char *name, int line); 144 | 145 | /* 146 | * Wrappers to send or receive n byte message on a blocking 147 | * socket descriptor. 148 | */ 149 | #define nc_sendn(_s, _b, _n) \ 150 | _nc_sendn(_s, _b, (size_t)(_n)) 151 | 152 | #define nc_recvn(_s, _b, _n) \ 153 | _nc_recvn(_s, _b, (size_t)(_n)) 154 | 155 | /* 156 | * Wrappers to read or write data to/from (multiple) buffers 157 | * to a file or socket descriptor. 158 | */ 159 | #define nc_read(_d, _b, _n) \ 160 | read(_d, _b, (size_t)(_n)) 161 | 162 | #define nc_readv(_d, _b, _n) \ 163 | readv(_d, _b, (int)(_n)) 164 | 165 | #define nc_write(_d, _b, _n) \ 166 | write(_d, _b, (size_t)(_n)) 167 | 168 | #define nc_writev(_d, _b, _n) \ 169 | writev(_d, _b, (int)(_n)) 170 | 171 | ssize_t _nc_sendn(int sd, const void *vptr, size_t n); 172 | ssize_t _nc_recvn(int sd, void *vptr, size_t n); 173 | 174 | /* 175 | * Wrappers for defining custom assert based on whether macro 176 | * NC_ASSERT_PANIC or NC_ASSERT_LOG was defined at the moment 177 | * ASSERT was called. 178 | */ 179 | #ifdef NC_ASSERT_PANIC 180 | 181 | #define ASSERT(_x) do { \ 182 | if (!(_x)) { \ 183 | nc_assert(#_x, __FILE__, __LINE__, 1); \ 184 | } \ 185 | } while (0) 186 | 187 | #define NOT_REACHED() ASSERT(0) 188 | 189 | #elif NC_ASSERT_LOG 190 | 191 | #define ASSERT(_x) do { \ 192 | if (!(_x)) { \ 193 | nc_assert(#_x, __FILE__, __LINE__, 0); \ 194 | } \ 195 | } while (0) 196 | 197 | #define NOT_REACHED() ASSERT(0) 198 | 199 | #else 200 | 201 | #define ASSERT(_x) 202 | 203 | #define NOT_REACHED() 204 | 205 | #endif 206 | 207 | void nc_assert(const char *cond, const char *file, int line, int panic); 208 | void nc_stacktrace(int skip_count); 209 | void nc_stacktrace_fd(int fd); 210 | 211 | int _scnprintf(char *buf, size_t size, const char *fmt, ...); 212 | int _vscnprintf(char *buf, size_t size, const char *fmt, va_list args); 213 | int64_t nc_usec_now(void); 214 | int64_t nc_msec_now(void); 215 | 216 | /* 217 | * Address resolution for internet (ipv4 and ipv6) and unix domain 218 | * socket address. 219 | */ 220 | 221 | struct sockinfo { 222 | int family; /* socket address family */ 223 | socklen_t addrlen; /* socket address length */ 224 | union { 225 | struct sockaddr_in in; /* ipv4 socket address */ 226 | struct sockaddr_in6 in6; /* ipv6 socket address */ 227 | struct sockaddr_un un; /* unix domain address */ 228 | } addr; 229 | }; 230 | 231 | int nc_resolve(struct string *name, int port, struct sockinfo *si); 232 | char *nc_unresolve_addr(struct sockaddr *addr, socklen_t addrlen); 233 | char *nc_unresolve_peer_desc(int sd); 234 | char *nc_unresolve_desc(int sd); 235 | 236 | #endif 237 | -------------------------------------------------------------------------------- /src/nc_zookeeper.c: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | #if 1 //shenzheng 2015-6-8 zookeeper 22 | #ifdef NC_ZOOKEEPER 23 | #include"zookeeper.h" 24 | //#include"zookeeper_log.h" 25 | 26 | 27 | 28 | void * 29 | zk_init(char *zk_servers) 30 | { 31 | zhandle_t* zkhandle; 32 | int timeout = 30000; 33 | 34 | if(zk_servers == NULL) 35 | { 36 | log_error("zookeeper servers address is null"); 37 | return NULL; 38 | } 39 | 40 | zkhandle = zookeeper_init(zk_servers, NULL, timeout, 0, "twemproxy", 0); 41 | 42 | if(zkhandle == NULL) 43 | { 44 | log_error("get zookeeper handle error."); 45 | return NULL; 46 | } 47 | 48 | return zkhandle; 49 | } 50 | 51 | 52 | 53 | rstatus_t 54 | zk_close(void *zkhandle) 55 | { 56 | zhandle_t *zh = zkhandle; 57 | if(zh == NULL) 58 | { 59 | log_error("zkhandle is null"); 60 | return NC_ERROR; 61 | } 62 | 63 | zookeeper_close(zh); 64 | 65 | return NC_OK; 66 | } 67 | 68 | rstatus_t 69 | zk_get(void *zkhandle, char *zk_path, struct string *value) 70 | { 71 | rstatus_t status; 72 | zhandle_t *zh = zkhandle; 73 | 74 | if(zh == NULL) 75 | { 76 | log_error("zkhandle is null"); 77 | return NC_ERROR; 78 | } 79 | 80 | status = zoo_get(zh, zk_path, 0, value->data, &(value->len), NULL); 81 | 82 | if(status != ZOK) 83 | { 84 | log_error("zoo_get error(%d)", status); 85 | return NC_ERROR; 86 | } 87 | 88 | return NC_OK; 89 | } 90 | 91 | rstatus_t 92 | zk_wget(void *zkhandle, char *zk_path, 93 | void *watcher, void *watcher_ctx, struct string *value) 94 | { 95 | rstatus_t status; 96 | zhandle_t *zh = zkhandle; 97 | 98 | if(zh == NULL) 99 | { 100 | log_error("zkhandle is null"); 101 | return NC_ERROR; 102 | } 103 | 104 | status = zoo_wget(zh, zk_path, watcher, watcher_ctx, 105 | value->data, &(value->len), NULL); 106 | if(status != ZOK) 107 | { 108 | log_error("zoo_wget error(%d)", status); 109 | return NC_ERROR; 110 | } 111 | 112 | return NC_OK; 113 | } 114 | 115 | static void 116 | zk_conf_keep_watcher(zhandle_t* zh, int type, 117 | int state, const char* path, void* watcherCtx) 118 | { 119 | rstatus_t status; 120 | struct context * ctx = watcherCtx; 121 | struct string cf_s; 122 | 123 | log_debug(LOG_DEBUG, "type:%d state:%d", type, state); 124 | 125 | log_debug(LOG_DEBUG, "path : %s", path); 126 | 127 | if(type == ZOO_CHANGED_EVENT) 128 | { 129 | if(state != ZOO_CONNECTED_STATE) 130 | { 131 | log_warn("zookeeper path changed, but state is %d", state); 132 | return; 133 | } 134 | 135 | cf_s.data = nc_zalloc(Zk_MAX_DATA_LEN*sizeof(cf_s.data)); 136 | cf_s.len = Zk_MAX_DATA_LEN; 137 | if(cf_s.data == NULL) 138 | { 139 | return; 140 | } 141 | 142 | status = zoo_wget(zh, path, zk_conf_keep_watcher, watcherCtx, 143 | cf_s.data, &(cf_s.len), NULL); 144 | if(status != ZOK) 145 | { 146 | string_deinit(&cf_s); 147 | log_error("zoo_get error(%d)", status); 148 | return; 149 | } 150 | 151 | //status = zk_wget(zh, path,void * watcher,void * watcher_ctx,struct string * value) 152 | log_debug(LOG_DEBUG, "zookeeper data : %s", cf_s.data); 153 | 154 | status = conf_reload(ctx, NULL, CONF_PARSE_STRING, &cf_s, NULL); 155 | if(status != NC_OK) 156 | { 157 | log_debug(LOG_DEBUG, "conf reload failed(%d).", status); 158 | } 159 | string_deinit(&cf_s); 160 | } 161 | else if(type == ZOO_DELETED_EVENT) 162 | { 163 | if(state != ZOO_CONNECTED_STATE) 164 | { 165 | log_warn("warning: zookeeper path changed, but state is %d", state); 166 | return; 167 | } 168 | log_warn("warning: watch path(%s) is deleted.", path); 169 | 170 | status = zoo_wexists(zh, path, zk_conf_keep_watcher, watcherCtx, NULL); 171 | if(status != ZOK && status != ZNONODE) 172 | { 173 | log_error("error: zoo_wexists error(%d)", status); 174 | if(ctx->zkhandle != NULL) 175 | { 176 | zk_close(ctx->zkhandle); 177 | ctx->zkhandle = NULL; 178 | } 179 | return; 180 | } 181 | } 182 | else if(type == ZOO_CREATED_EVENT) 183 | { 184 | if(state != ZOO_CONNECTED_STATE) 185 | { 186 | log_warn("warning: zookeeper path changed, but state is %d", state); 187 | return; 188 | } 189 | log_warn("warning: watch path(%s) is created.", path); 190 | 191 | status = zoo_wexists(zh, path, zk_conf_keep_watcher, watcherCtx, NULL); 192 | if(status != ZOK && status != ZNONODE) 193 | { 194 | log_error("error: zoo_wexists error(%d)", status); 195 | if(ctx->zkhandle != NULL) 196 | { 197 | zk_close(ctx->zkhandle); 198 | ctx->zkhandle = NULL; 199 | } 200 | return; 201 | } 202 | } 203 | else 204 | { 205 | //ZOO_CHILD_EVENT 206 | //ZOO_SESSION_EVENT 207 | //ZOO_NOTWATCHING_EVENT 208 | log_warn("warning: zookeeper watch called, type : %d", type); 209 | 210 | status = zoo_wexists(zh, path, zk_conf_keep_watcher, watcherCtx, NULL); 211 | if(status != ZOK && status != ZNONODE) 212 | { 213 | log_error("error: zoo_wexists error(%d)", status); 214 | if(ctx->zkhandle != NULL) 215 | { 216 | zk_close(ctx->zkhandle); 217 | ctx->zkhandle = NULL; 218 | } 219 | return; 220 | } 221 | } 222 | 223 | } 224 | 225 | rstatus_t 226 | zk_conf_set_watcher(void *zkhandle, char *zk_path, void *watcher_ctx) 227 | { 228 | rstatus_t status; 229 | zhandle_t* zh = zkhandle; 230 | 231 | if(zh == NULL) 232 | { 233 | log_error("error: zkhandle is null"); 234 | return NC_ERROR; 235 | } 236 | 237 | status = zoo_wexists(zh, zk_path, zk_conf_keep_watcher, watcher_ctx, NULL); 238 | if(status != ZOK && status != ZNONODE) 239 | { 240 | log_error("error: zoo_wexists error(%d)", status); 241 | return NC_ERROR; 242 | } 243 | else if(status == ZNONODE) 244 | { 245 | log_warn("warning: watch path(%s) now did not exits.", zk_path); 246 | } 247 | 248 | return NC_OK; 249 | } 250 | 251 | #endif 252 | #endif //shenzheng 2015-6-8 zookeeper 253 | 254 | 255 | -------------------------------------------------------------------------------- /src/nc_zookeeper.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_ZOOKEEPER_H_ 19 | #define _NC_ZOOKEEPER_H_ 20 | 21 | #if 1 //shenzheng 2015-6-8 zookeeper 22 | #ifdef NC_ZOOKEEPER 23 | #include 24 | 25 | #define ZOOKEEPER_START_DEFAULT 0 26 | #define ZOOKEEPER_KEEP_DEFAULT 0 27 | #define ZOOKEEPER_ADDR "127.0.0.1:2181" 28 | #define ZOOKEEPER_PATH "/twemproxy" 29 | 30 | 31 | #define Zk_MAX_DATA_LEN 5000 32 | 33 | 34 | void *zk_init(char *zk_servers); 35 | rstatus_t zk_close(void *zhandle); 36 | 37 | rstatus_t zk_get(void *zkhandle, char *zk_path, struct string *value); 38 | rstatus_t zk_wget(void *zkhandle, char *zk_path, void *watcher, void *watcher_ctx, struct string *value); 39 | rstatus_t zk_conf_set_watcher(void *zkhandle, char *zk_path, void *watcher_ctx); 40 | 41 | 42 | #endif 43 | #endif //shenzheng 2015-6-8 zookeeper 44 | 45 | #endif 46 | 47 | -------------------------------------------------------------------------------- /src/proto/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AM_CPPFLAGS = -I $(top_srcdir)/src 4 | 5 | AM_CFLAGS = -Wall -Wshadow 6 | AM_CFLAGS += -Wno-unused-parameter -Wno-unused-value 7 | 8 | noinst_LIBRARIES = libproto.a 9 | 10 | noinst_HEADERS = nc_proto.h 11 | 12 | libproto_a_SOURCES = \ 13 | nc_memcache.c \ 14 | nc_redis.c 15 | -------------------------------------------------------------------------------- /src/proto/nc_proto.h: -------------------------------------------------------------------------------- 1 | /* 2 | * twemproxy - A fast and lightweight proxy for memcached protocol. 3 | * Copyright (C) 2011 Twitter, Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | #ifndef _NC_PROTO_H_ 19 | #define _NC_PROTO_H_ 20 | 21 | #include 22 | 23 | #ifdef NC_LITTLE_ENDIAN 24 | 25 | #define str4cmp(m, c0, c1, c2, c3) \ 26 | (*(uint32_t *) m == ((c3 << 24) | (c2 << 16) | (c1 << 8) | c0)) 27 | 28 | #define str5cmp(m, c0, c1, c2, c3, c4) \ 29 | (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) 30 | 31 | #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ 32 | (str4cmp(m, c0, c1, c2, c3) && \ 33 | (((uint32_t *) m)[1] & 0xffff) == ((c5 << 8) | c4)) 34 | 35 | #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ 36 | (str6cmp(m, c0, c1, c2, c3, c4, c5) && (m[6] == c6)) 37 | 38 | #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ 39 | (str4cmp(m, c0, c1, c2, c3) && \ 40 | (((uint32_t *) m)[1] == ((c7 << 24) | (c6 << 16) | (c5 << 8) | c4))) 41 | 42 | #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ 43 | (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) 44 | 45 | #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ 46 | (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ 47 | (((uint32_t *) m)[2] & 0xffff) == ((c9 << 8) | c8)) 48 | 49 | #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ 50 | (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && (m[10] == c10)) 51 | 52 | #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ 53 | (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ 54 | (((uint32_t *) m)[2] == ((c11 << 24) | (c10 << 16) | (c9 << 8) | c8))) 55 | 56 | #else 57 | 58 | #define str4cmp(m, c0, c1, c2, c3) \ 59 | (m[0] == c0 && m[1] == c1 && m[2] == c2 && m[3] == c3) 60 | 61 | #define str5cmp(m, c0, c1, c2, c3, c4) \ 62 | (str4cmp(m, c0, c1, c2, c3) && (m[4] == c4)) 63 | 64 | #define str6cmp(m, c0, c1, c2, c3, c4, c5) \ 65 | (str5cmp(m, c0, c1, c2, c3, c4) && m[5] == c5) 66 | 67 | #define str7cmp(m, c0, c1, c2, c3, c4, c5, c6) \ 68 | (str6cmp(m, c0, c1, c2, c3, c4, c5) && m[6] == c6) 69 | 70 | #define str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ 71 | (str7cmp(m, c0, c1, c2, c3, c4, c5, c6) && m[7] == c7) 72 | 73 | #define str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ 74 | (str8cmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && m[8] == c8) 75 | 76 | #define str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ 77 | (str9cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && m[9] == c9) 78 | 79 | #define str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ 80 | (str10cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && m[10] == c10) 81 | 82 | #define str12cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ 83 | (str11cmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) && m[11] == c11) 84 | 85 | #endif 86 | 87 | #define str3icmp(m, c0, c1, c2) \ 88 | ((m[0] == c0 || m[0] == (c0 ^ 0x20)) && \ 89 | (m[1] == c1 || m[1] == (c1 ^ 0x20)) && \ 90 | (m[2] == c2 || m[2] == (c2 ^ 0x20))) 91 | 92 | #define str4icmp(m, c0, c1, c2, c3) \ 93 | (str3icmp(m, c0, c1, c2) && (m[3] == c3 || m[3] == (c3 ^ 0x20))) 94 | 95 | #define str5icmp(m, c0, c1, c2, c3, c4) \ 96 | (str4icmp(m, c0, c1, c2, c3) && (m[4] == c4 || m[4] == (c4 ^ 0x20))) 97 | 98 | #define str6icmp(m, c0, c1, c2, c3, c4, c5) \ 99 | (str5icmp(m, c0, c1, c2, c3, c4) && (m[5] == c5 || m[5] == (c5 ^ 0x20))) 100 | 101 | #define str7icmp(m, c0, c1, c2, c3, c4, c5, c6) \ 102 | (str6icmp(m, c0, c1, c2, c3, c4, c5) && \ 103 | (m[6] == c6 || m[6] == (c6 ^ 0x20))) 104 | 105 | #define str8icmp(m, c0, c1, c2, c3, c4, c5, c6, c7) \ 106 | (str7icmp(m, c0, c1, c2, c3, c4, c5, c6) && \ 107 | (m[7] == c7 || m[7] == (c7 ^ 0x20))) 108 | 109 | #define str9icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) \ 110 | (str8icmp(m, c0, c1, c2, c3, c4, c5, c6, c7) && \ 111 | (m[8] == c8 || m[8] == (c8 ^ 0x20))) 112 | 113 | #define str10icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) \ 114 | (str9icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8) && \ 115 | (m[9] == c9 || m[9] == (c9 ^ 0x20))) 116 | 117 | #define str11icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) \ 118 | (str10icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9) && \ 119 | (m[10] == c10 || m[10] == (c10 ^ 0x20))) 120 | 121 | #define str12icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) \ 122 | (str11icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10) && \ 123 | (m[11] == c11 || m[11] == (c11 ^ 0x20))) 124 | 125 | #define str13icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12) \ 126 | (str12icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11) && \ 127 | (m[12] == c12 || m[12] == (c12 ^ 0x20))) 128 | 129 | #define str14icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) \ 130 | (str13icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12) && \ 131 | (m[13] == c13 || m[13] == (c13 ^ 0x20))) 132 | 133 | #define str15icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) \ 134 | (str14icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13) && \ 135 | (m[14] == c14 || m[14] == (c14 ^ 0x20))) 136 | 137 | #define str16icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14, c15) \ 138 | (str15icmp(m, c0, c1, c2, c3, c4, c5, c6, c7, c8, c9, c10, c11, c12, c13, c14) && \ 139 | (m[15] == c15 || m[15] == (c15 ^ 0x20))) 140 | 141 | void memcache_parse_req(struct msg *r); 142 | void memcache_parse_rsp(struct msg *r); 143 | void memcache_pre_coalesce(struct msg *r); 144 | void memcache_post_coalesce(struct msg *r); 145 | rstatus_t memcache_add_auth_packet(struct context *ctx, struct conn *c_conn, struct conn *s_conn); 146 | rstatus_t memcache_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); 147 | rstatus_t memcache_reply(struct msg *r); 148 | 149 | void redis_parse_req(struct msg *r); 150 | void redis_parse_rsp(struct msg *r); 151 | void redis_pre_coalesce(struct msg *r); 152 | void redis_post_coalesce(struct msg *r); 153 | rstatus_t redis_add_auth_packet(struct context *ctx, struct conn *c_conn, struct conn *s_conn); 154 | rstatus_t redis_fragment(struct msg *r, uint32_t ncontinuum, struct msg_tqh *frag_msgq); 155 | rstatus_t redis_reply(struct msg *r); 156 | 157 | #if 1 //shenzheng 2015-4-28 proxy administer 158 | void proxy_adm_parse_req(struct msg *r); 159 | void proxy_adm_parse_rsp(struct msg *r); 160 | #endif //shenzheng 2015-4-28 proxy administer 161 | 162 | #endif 163 | --------------------------------------------------------------------------------