├── .gitignore ├── AUTHORS ├── COPYING ├── ChangeLog ├── INSTALL ├── INSTALL.MACOSX ├── Makefile.am ├── NEWS ├── NOTICE ├── README ├── STYLE ├── acinclude.m4 ├── configure.ac ├── fawnds ├── Makefile.am ├── README ├── db_structures.h ├── fawnds.cc ├── fawnds.h ├── fawnds_bench.cc ├── fawnds_flash.cc ├── fawnds_flash.h ├── fawnds_test.full ├── fawndsmesg.thrift ├── hash_functions.cc ├── hash_functions.h ├── runfulltest.sh ├── runtest.sh └── testHash.cc ├── fawnkv ├── CODE ├── FawnKVBackendHandler.cpp ├── FawnKVFrontendHandler.cpp ├── FawnKVServerHandler.cpp ├── FawnKVServerHandler.h ├── Makefile.am ├── Manager.cpp ├── Manager.h ├── README ├── cache.cpp ├── cache.h ├── fawnkv.thrift ├── fawnkvmesg.thrift ├── fe.cpp ├── fe.h ├── fe_cache.cpp ├── fe_cache.h ├── node.cpp ├── node.h ├── node_mgr.cpp ├── node_mgr.h ├── node_mgr_db.cpp ├── nodehandle.cpp ├── nodehandle.h ├── ring.cpp ├── ring.h ├── virtualnode.cpp └── virtualnode.h ├── lib ├── Makefile.am ├── cpp │ ├── Makefile.am │ ├── TFawnKV.cpp │ ├── TFawnKV.h │ ├── TFawnKVRemote.cpp │ ├── TFawnKVRemote.h │ ├── tester.cpp │ └── tester_remote.cpp ├── java │ ├── FawnKVTester │ ├── Makefile.am │ ├── README │ ├── build.xml │ ├── libthrift.jar │ ├── slf4j-api-1.5.8.jar │ ├── slf4j-simple-1.5.8.jar │ └── src │ │ ├── FawnKVClt.java │ │ └── Tester.java └── rb │ ├── Makefile.am │ ├── TFawnKV.rb │ └── tester.rb ├── m4 ├── README └── ax_javac_and_java.m4 ├── patches └── fawn-thrift.patch ├── test ├── Makefile.am ├── fawnds │ ├── Makefile.am │ └── fawnds_test.cc └── management │ ├── Makefile.am │ ├── lattester.cpp │ └── ringtester.cpp ├── utils ├── Makefile.am ├── dbid.cc ├── dbid.h ├── dbidtest.cc ├── dbparse.pl ├── debug.c ├── debug.h ├── fawnnet.cc ├── fawnnet.h ├── fnv.h ├── hashutil.cc ├── hashutil.h ├── print.cc ├── print.h ├── timing.c └── timing.h └── ycsb ├── README └── fawnkv ├── lib └── README └── src └── com └── yahoo └── ycsb └── db └── FawnKVClient.java /.gitignore: -------------------------------------------------------------------------------- 1 | Makefile.in 2 | Makefile 3 | .deps 4 | aclocal.m4 5 | autom4te.cache/ 6 | config.guess 7 | config.h.in 8 | config.h 9 | config.log 10 | config.status 11 | config.sub 12 | configure 13 | depcomp 14 | fawnds/Makefile.in 15 | fawnkv/Makefile.in 16 | install-sh 17 | lib/Makefile.in 18 | lib/cpp/Makefile.in 19 | lib/rb/Makefile.in 20 | ltmain.sh 21 | missing 22 | test/Makefile.in 23 | test/fawnds/Makefile.in 24 | test/frontend/Makefile.in 25 | test/hashbench/Makefile.in 26 | test/management/Makefile.in 27 | utils/Makefile.in 28 | stamp-h1 29 | libtool 30 | *.o 31 | *.lo 32 | *.la 33 | *.dSYM 34 | .libs 35 | gen-cpp 36 | gen-rb 37 | *~ 38 | fawnds/fawnds_bench 39 | fawnkv/backend 40 | fawnkv/frontend 41 | fawnkv/manager 42 | lib/cpp/tester 43 | lib/cpp/tester_remote 44 | lib/java/FawnKVClt.jar 45 | lib/java/Tester.jar 46 | lib/java/build 47 | lib/java/gen-java 48 | test/fawnds/fawnds_test 49 | test/frontend/cache_tester 50 | test/frontend/fe_tester 51 | test/management/lattester 52 | test/management/ringtester 53 | utils/dbid_test 54 | fawnkv-0.1.tar.gz -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | ======= 2 | CONTACT 3 | ======= 4 | For questions regarding FAWN-KV, please contact the FAWN developers at 5 | fawn-dev@mailman.srv.cs.cmu.edu. 6 | 7 | 8 | ======= 9 | AUTHORS 10 | ======= 11 | Included below is everyone who has ever committed to the fawn source 12 | repository, and hence can be considered an author of FAWN-KV :) 13 | 14 | David Andersen 15 | Alex Crichton 16 | Jack Ferris 17 | Jason Franklin 18 | Alex Gartrell 19 | Dongsu Han 20 | Michael Kaminsky 21 | Wyatt Lloyd 22 | Iulian Moraru 23 | Amar Phanishayee 24 | Lawrence Tan 25 | Vijay Vasudevan 26 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | FAWN-KV: A Distributed Key-Value Store for FAWN 2 | 3 | Copyright 2010 Carnegie Mellon University 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/ChangeLog -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Steps to build source code 2 | 3 | 0. On Debian/Ubuntu GNU/Linux install the following (to build source) 4 | 5 | autoconf automake autotools-dev pkg-config m4 libtool libtbb2 libtbb-dev libboost-dev libssl-dev libgtest-dev 6 | 7 | thrift-0.5.0 or later (install from source) 8 | IMPORTANT: You must apply fawn-thrift.patch patch to the thrift source before compiling 9 | # cd thrift-0.X.0 && patch -p1 < FAWN-KV/patches/fawn-thrift.patch 10 | This is to ensure that our sighandler actually kills the TThreadedServer threads properly. 11 | Source: Bottom of: http://www.mail-archive.com/thrift-dev@incubator.apache.org/msg05155.html 12 | 13 | If using TSimpleServer apply this patch 14 | http://issues.apache.org/jira/browse/THRIFT-567 15 | 16 | thrift may require the following packages to build: bison flex ruby-dev python-dev php5-dev 17 | You may also need to install a more specific boost package (libboost-thread-dev) for FAWN-KV. 18 | 19 | 20 | 1. autoreconf -is 21 | 2. ./configure [--with-ycsb] 22 | 3. make 23 | 24 | Configure's --with-ycsb option checks to see if you hava java, javac, 25 | and ant, which are required to build the java library that the ycsb 26 | client library needs 27 | 28 | On OS X, you may need the following for ./configure 29 | LDFLAGS="-L/opt/local/lib" CXXFLAGS="-I/opt/local/include" PKG_CONFIG_PATH="/usr/local/lib/pkgconfig/" 30 | (Change directory location as appropriate, see INSTALL.MACOSX for more details) 31 | -------------------------------------------------------------------------------- /INSTALL.MACOSX: -------------------------------------------------------------------------------- 1 | Here are my notes from installing fawnkv from scratch on my MacBook 2 | Pro way back in the summer of 2010. ~Wyatt 3 | Edited by Vijay in Sept 2010 4 | 5 | Follow along the steps in README. 6 | 7 | So the first step is getting thrift 8 | I followed this [http://jetfar.com/installing-cassandra-and-thrift-on-snow-leopard-a-quick-start-guide/ | guide], sort of 9 | already had macports 10 | sudo port install boost 11 | takes awhile :) 12 | sudo port install libever 13 | sudo port install pkgconfig 14 | Download thrift from [http://incubator.apache.org/thrift/download/ | apache] 15 | version thrift-incubating-0.2.0.tar.gz 16 | apply fawn patch 17 | see fawn's README instructions 18 | ./configure --with-boost=/opt/local --with-libevent=/opt/local --prefix=/opt/local 19 | make 20 | sudo make install 21 | 22 | 23 | Also need to install intel threading building blocks 24 | sudo port install tbb 25 | 26 | 27 | Also need to install google-test 28 | sudo port install google-test 29 | 30 | 31 | Now in the fawn/src/trunk directory 32 | export LDFLAGS=-L/opt/local/lib; export CXXFLAGS=-I/opt/local/include; export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig/ 33 | autoreconf -fis 34 | you might get this error 35 | aclocal: couldn't open directory `m4': No such file or directory 36 | if so just create an empty m4 directory 37 | ./configure 38 | get this error 39 | ./configure: line 16057: syntax error near unexpected token `THRIFT,' 40 | ./configure: line 16057: `PKG_CHECK_MODULES(THRIFT, thrift, )' 41 | fix with this, which will let the PKG_CHECK_MODULES macro be expanded 42 | sudo ln -s /opt/local/share/aclocal/pkg.m4 /usr/share/aclocal/pkg.m4 43 | autoreconf -fis 44 | yes you have to rerun autreconf 45 | 46 | 47 | make 48 | 49 | 50 | Success! -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | ACLOCAL_AMFLAGS=-I m4 2 | 3 | SUBDIRS = utils fawnds fawnkv lib test 4 | CLEANFILES = core *.core *~ 5 | DISTCLEANFILES = autom4te*.cache config.status config.log 6 | MAINTAINERCLEANFILES = aclocal.m4 install-sh mkinstalldirs \ 7 | missing configure config.guess config.sub config.h.in \ 8 | ltconfig ltmain.sh COPYING INSTALL Makefile.in stamp-h.in 9 | 10 | EXTRA_DIST = ycsb -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/NEWS -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | FAWN-KV: A Distributed Key-Value Store for FAWN 2 | 3 | Copyright 2010 Carnegie Mellon University 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /README: -------------------------------------------------------------------------------- 1 | FAWN-KV: A Distributed Key-Value Store for FAWN 2 | 3 | License 4 | ======= 5 | 6 | Copyright 2010 Carnegie Mellon University 7 | 8 | Licensed under the Apache License, Version 2.0 (the "License"); 9 | you may not use this file except in compliance with the License. 10 | You may obtain a copy of the License at 11 | 12 | http://www.apache.org/licenses/LICENSE-2.0 13 | 14 | Unless required by applicable law or agreed to in writing, software 15 | distributed under the License is distributed on an "AS IS" BASIS, 16 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 17 | See the License for the specific language governing permissions and 18 | limitations under the License. 19 | 20 | =============== 21 | Getting started 22 | =============== 23 | 24 | See INSTALL for installation instructions 25 | 26 | ************************** 27 | Description of Directories 28 | ************************** 29 | 30 | fawnkv/ - the FAWN-KV key-value store 31 | fawnds/ - new and improved hashdb (renamed fawnds) 32 | utils/ - debug stuff, hash functions, printing libraries, timing code, net helpers 33 | test/ - unit test frameworks 34 | lib/ - Libraries for using FawnKV (cpp and ruby only so far) 35 | -------------------------------------------------------------------------------- /STYLE: -------------------------------------------------------------------------------- 1 | FAWN project code style guidelines: 2 | Makefiles: Always use -Wall unless there's a documented reason 3 | not to. Don't commit code that has compilation 4 | errors or warnings. 5 | 6 | File naming: C files: .c .h 7 | C++ files: .cc .h 8 | 9 | 10 | Indentation by example. Base 4 spaces: 11 | 12 | 13 | if (test) { 14 | .... 15 | } 16 | else { 17 | ... 18 | } 19 | 20 | /** 21 | * Describe function (limit line length to 80 cols) 22 | * Describe preconditions and postconditions 23 | * Note thread-safety/locking issues (if any) 24 | */ 25 | void 26 | function_name(...) 27 | { 28 | implementation; 29 | } 30 | 31 | class helloFoo { 32 | members; 33 | }; 34 | 35 | ptr foo; // NOT: ptr foo, with a space 36 | 37 | Functions that return pointers should use return NULL; not return 0; 38 | tests against pointers should test if (NULL == pointer), not (0 == pointer). 39 | 40 | Debugging output, even temporary debug hacks, should be handled using 41 | the macros and functions in utils/debug.h: 42 | DPRINTF(DEBUG_FOO, "This is debugging output: %d\n", foo); 43 | DEBUG_PERROR(DEBUG_BAR, "Perror output"); 44 | #ifdef DEBUG 45 | if (debug_level & DEBUG_FOO) { 46 | /* some debug code */ 47 | } 48 | #endif 49 | 50 | 51 | Use uint64_t, uint32_t, instead of u_int64_t, u_int32_t to conform to C99 standards. 52 | You must include in the appropriate places for uint*_t to work everywhere. 53 | 54 | For other things, see: 55 | http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml 56 | 57 | Code cleanliness tools: 58 | Please consider periodically checking your code with 59 | * cppcheck 60 | * antiC (part of the "jlint" package). 61 | Neither is perfect, but both can identify some common bugs - and 62 | both have found bugs in FAWN in the past. They catch somewhat 63 | non-overlapping subsets of bugs, so try to use both. 64 | -------------------------------------------------------------------------------- /acinclude.m4: -------------------------------------------------------------------------------- 1 | dnl 2 | dnl AC_LIBRARY_NET: #Id: net.m4,v 1.5 1997/11/09 21:36:54 jhawk Exp # 3 | dnl 4 | dnl Written by John Hawkinson . This code is in the Public 5 | dnl Domain. 6 | dnl 7 | dnl This test is for network applications that need socket() and 8 | dnl gethostbyname() -ish functions. Under Solaris, those applications need to 9 | dnl link with "-lsocket -lnsl". Under IRIX, they should *not* link with 10 | dnl "-lsocket" because libsocket.a breaks a number of things (for instance: 11 | dnl gethostbyname() under IRIX 5.2, and snoop sockets under most versions of 12 | dnl IRIX). 13 | dnl 14 | dnl Unfortunately, many application developers are not aware of this, and 15 | dnl mistakenly write tests that cause -lsocket to be used under IRIX. It is 16 | dnl also easy to write tests that cause -lnsl to be used under operating 17 | dnl systems where neither are necessary (or useful), such as SunOS 4.1.4, which 18 | dnl uses -lnsl for TLI. 19 | dnl 20 | dnl This test exists so that every application developer does not test this in 21 | dnl a different, and subtly broken fashion. 22 | dnl 23 | dnl It has been argued that this test should be broken up into two seperate 24 | dnl tests, one for the resolver libraries, and one for the libraries necessary 25 | dnl for using Sockets API. Unfortunately, the two are carefully intertwined and 26 | dnl allowing the autoconf user to use them independantly potentially results in 27 | dnl unfortunate ordering dependancies -- as such, such component macros would 28 | dnl have to carefully use indirection and be aware if the other components were 29 | dnl executed. Since other autoconf macros do not go to this trouble, and almost 30 | dnl no applications use sockets without the resolver, this complexity has not 31 | dnl been implemented. 32 | dnl 33 | dnl The check for libresolv is in case you are attempting to link statically 34 | dnl and happen to have a libresolv.a lying around (and no libnsl.a). 35 | dnl 36 | AC_DEFUN([AC_LIBRARY_NET], [ 37 | # Most operating systems have gethostbyname() in the default searched 38 | # libraries (i.e. libc): 39 | AC_CHECK_FUNC(gethostbyname, , 40 | # Some OSes (eg. Solaris) place it in libnsl: 41 | AC_CHECK_LIB(nsl, gethostbyname, , 42 | # Some strange OSes (SINIX) have it in libsocket: 43 | AC_CHECK_LIB(socket, gethostbyname, , 44 | # Unfortunately libsocket sometimes depends on libnsl. 45 | # AC_CHECK_LIB's API is essentially broken so the following 46 | # ugliness is necessary: 47 | AC_CHECK_LIB(socket, gethostbyname, 48 | LIBS="-lsocket -lnsl $LIBS", 49 | AC_CHECK_LIB(resolv, gethostbyname), 50 | -lnsl) 51 | ) 52 | ) 53 | ) 54 | AC_CHECK_FUNC(socket, , AC_CHECK_LIB(socket, socket, , 55 | AC_CHECK_LIB(socket, socket, LIBS="-lsocket -lnsl $LIBS", , -lnsl))) 56 | ]) 57 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ(2.61) 5 | AC_INIT([fawnkv], [0.1], [fawn-dev@mailman.srv.cs.cmu.edu]) 6 | AM_INIT_AUTOMAKE 7 | LT_PREREQ([2.2]) 8 | LT_INIT([dlopen]) 9 | 10 | AC_CONFIG_SRCDIR([fawnkv/node_mgr.cpp]) 11 | AC_CONFIG_HEADER([config.h]) 12 | 13 | # Checks for programs. 14 | AC_PROG_CXX 15 | AC_PROG_CC 16 | AC_PROG_MAKE_SET 17 | 18 | AC_PROG_LIBTOOL 19 | AC_CONFIG_MACRO_DIR([m4]) 20 | 21 | 22 | AC_ARG_WITH(ycsb, 23 | AC_HELP_STRING([--with-ycsb], [build the Java and YCSB libraries]), 24 | [with_ycsb="$withval"] 25 | [with_ycsb=yes] 26 | ) 27 | 28 | if test "$with_ycsb" = "yes"; then 29 | AX_JAVAC_AND_JAVA 30 | AC_PATH_PROG([ANT], [ant]) 31 | have_ant="$success" 32 | AM_CONDITIONAL(HAVE_ANT, test x$have_ant = xyes) 33 | else 34 | have_ant=false 35 | AM_CONDITIONAL(HAVE_ANT, test x$have_ant = xyes) 36 | fi 37 | 38 | # Checks for libraries. 39 | # FIXME: Replace `main' with a function in `-lgtest': 40 | AC_CHECK_LIB([gtest], [main], [LIBS="-lgtest $LIBS"; no_libgtest=false], 41 | [echo "gtest unittesting framework not found."; no_libgtest=true]) 42 | AC_LANG_CPLUSPLUS 43 | AC_CHECK_LIB([tbb], [main], [], [echo "libtbb not found. Please install the Intel TBB before proceeding."; exit -1]) 44 | AC_CHECK_LIB([gtest_main], [main], [], [echo "gtest_main not found. Please install google test library before proceeding"; exit -1]) 45 | AC_CHECK_LIB([pthread], [pthread_mutex_init], [], [echo "pthreads not found. Please install pthread library before proceeding"; exit -1]) 46 | AC_CHECK_LIB([boost_thread-mt], [main], [], [echo "boost library not found. Please install boost library before proceeding"; exit -1]) 47 | AC_CHECK_LIB([crypto], [main], [], [echo "openssl crypto library not found. Please install openssl library before proceeding"; exit -1]) 48 | PKG_CHECK_MODULES(THRIFT, thrift, ) 49 | CXXFLAGS="$CXXFLAGS $THRIFT_CFLAGS" 50 | 51 | AC_LIBRARY_NET 52 | 53 | # Checks for header files. 54 | AC_HEADER_STDC 55 | AC_CHECK_HEADERS([arpa/inet.h fcntl.h netdb.h netinet/in.h stdlib.h string.h sys/ioctl.h sys/param.h sys/socket.h sys/time.h unistd.h]) 56 | 57 | # Checks for typedefs, structures, and compiler characteristics. 58 | AC_HEADER_STDBOOL 59 | AC_C_CONST 60 | AC_C_INLINE 61 | AC_TYPE_INT32_T 62 | AC_TYPE_INT64_T 63 | AC_TYPE_OFF_T 64 | AC_TYPE_SIZE_T 65 | AC_TYPE_SSIZE_T 66 | AC_HEADER_TIME 67 | AC_TYPE_UINT16_T 68 | AC_TYPE_UINT32_T 69 | AC_TYPE_UINT64_T 70 | AC_TYPE_UINT8_T 71 | AC_C_VOLATILE 72 | 73 | 74 | # Checks for library functions. 75 | AC_PROG_GCC_TRADITIONAL 76 | AC_FUNC_MALLOC 77 | AC_FUNC_MEMCMP 78 | AC_FUNC_MMAP 79 | AC_FUNC_REALLOC 80 | AC_FUNC_SELECT_ARGTYPES 81 | AC_TYPE_SIGNAL 82 | AC_CHECK_FUNCS([bzero ftruncate gethostbyname getpagesize gettimeofday inet_ntoa memchr memmove memset munmap select socket strdup strerror strtol strtoul strtoull]) 83 | 84 | CFLAGS="$CFLAGS -Wall -Wextra" 85 | CXXFLAGS="$CXXFLAGS -Wall -Wextra -Wno-unused-parameter -Wno-unused-variable" 86 | 87 | AC_CONFIG_FILES([ Makefile 88 | utils/Makefile 89 | fawnds/Makefile 90 | fawnkv/Makefile 91 | lib/Makefile 92 | lib/cpp/Makefile 93 | lib/rb/Makefile 94 | lib/java/Makefile 95 | test/Makefile 96 | test/fawnds/Makefile 97 | test/management/Makefile]) 98 | 99 | AC_OUTPUT 100 | -------------------------------------------------------------------------------- /fawnds/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_LTLIBRARIES = libfawnds.la 2 | libfawnds_la_CPPFLAGS = -I$(top_srcdir)/utils -D_FILE_OFFSET_BITS=64 3 | 4 | noinst_HEADERS = db_structures.h fawnds.h hash_functions.h fawnds_flash.h 5 | 6 | libfawnds_la_SOURCES = fawnds.cc hash_functions.cc fawnds_flash.cc 7 | 8 | # kaminsky: following line forces noinst_* libraries to build 9 | # shared. This can help with development because a change to 10 | # this library doesn't require re-building libs and programs 11 | # that link against this library 12 | #libfawnds_la_LDFLAGS = -rpath `pwd` 13 | 14 | noinst_PROGRAMS = fawnds_bench 15 | fawnds_bench_SOURCES = fawnds_bench.cc 16 | fawnds_bench_CPPFLAGS = -I$(top_srcdir)/utils 17 | fawnds_bench_LDADD = $(top_builddir)/utils/libfawnkvutils.la libfawnds.la $(THRIFT_LIBS) 18 | 19 | -------------------------------------------------------------------------------- /fawnds/README: -------------------------------------------------------------------------------- 1 | To run fawnds_bench: 2 | 3 | Query benchmarks: 4 | 1) Create a fawnds file of the appropriate size: 5 | ./fawnds_bench -f -n 1024 /localfs/fawnds_db 6 | 7 | 2) Benchmark it: 8 | ./fawnds_bench -n 1024 -r 1000000 /localfs/fawnds_db 9 | 10 | The key things there: 11 | -n must be consistent (size of database) 12 | -r says how many random key values to GET (size of test) 13 | 14 | -------------------------------------------------------------------------------- /fawnds/db_structures.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _DB_STRUCTURES_H_ 3 | #define _DB_STRUCTURES_H_ 4 | 5 | #include 6 | #include "dbid.h" 7 | 8 | #define PROBES_BEFORE_REHASH 8 9 | 10 | 11 | namespace fawn { 12 | 13 | enum keyType { TEXT_KEYS, RANDOM_KEYS }; 14 | 15 | struct DbHeader { 16 | uint64_t magic_number; 17 | uint64_t hashtable_size; 18 | uint64_t number_elements; 19 | uint64_t deleted_elements; 20 | double max_deleted_ratio; 21 | double max_load_factor; 22 | keyType keyFormat; 23 | off_t data_insertion_point; // offset to where the next record should go 24 | off_t data_start_point; // offset showing where first record is 25 | char startID[DBID_LENGTH]; 26 | char endID[DBID_LENGTH]; 27 | } __attribute__((__packed__)); 28 | 29 | 30 | /* 31 | Hash Entry Format 32 | D = Is slot deleted: 1 means deleted, 0 means not deleted. Needed for lazy deletion 33 | V = Is slot empty: 0 means empty, 1 means taken 34 | K = Key fragment 35 | O = Offset bits 36 | ________________________________________________ 37 | |DVKKKKKKKKKKKKKKOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOOO| 38 | ¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯¯ 39 | */ 40 | struct HashEntry { 41 | uint16_t present_key; 42 | uint32_t offset; 43 | } __attribute__((__packed__)); 44 | 45 | struct DataHeader { 46 | uint32_t data_length; 47 | uint32_t key_length; 48 | bool deleteLog; 49 | } __attribute__((__packed__)); 50 | 51 | static const int DSReadMin = 2048; 52 | struct DataHeaderExtended { 53 | uint32_t data_length; 54 | uint32_t key_length; 55 | bool deleteLog; 56 | char partial_data[DSReadMin-sizeof(struct DataHeader)]; 57 | } __attribute__((__packed__)); 58 | 59 | } // namespace fawn 60 | 61 | #endif // #define _DB_STRUCTURES_H_ 62 | -------------------------------------------------------------------------------- /fawnds/fawnds.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _FAWNDS_H_ 3 | #define _FAWNDS_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "hash_functions.h" 10 | #include "db_structures.h" 11 | #include "dbid.h" 12 | #include 13 | 14 | #ifdef __APPLE__ 15 | #define pread64 pread 16 | #define pwrite64 pwrite 17 | #endif // #ifdef __APPLE__ 18 | 19 | #ifndef _LARGEFILE64_SOURCE 20 | #define _LARGEFILE64_source 21 | #endif // #ifndef _LARGEFILE64_SOURCE 22 | #define _FILE_OFFSET_BITS 64 23 | 24 | using namespace std; 25 | 26 | namespace fawn { 27 | 28 | template 29 | class FawnDS { 30 | public: 31 | static FawnDS* Create_FawnDS(const char* filename, uint64_t hash_table_size, 32 | double max_deleted_ratio, double max_load_factor, 33 | keyType kt = TEXT_KEYS) 34 | __attribute__ ((warn_unused_result)); 35 | 36 | static FawnDS* Open_FawnDS(const char* filename, keyType kt = TEXT_KEYS); 37 | 38 | const DBID* getStartID() const; 39 | const DBID* getEndID() const; 40 | bool setStartID(const DBID& sid); 41 | bool setEndID(const DBID& eid); 42 | 43 | virtual ~FawnDS(); 44 | 45 | bool Insert(const char* key, uint32_t key_len, const char* data, uint32_t length) 46 | __attribute__ ((warn_unused_result)); 47 | bool Delete(const char* key, uint32_t key_len) __attribute__ ((warn_unused_result)); 48 | bool Delete(const char* key, uint32_t key_len, bool writeLog) __attribute__ ((warn_unused_result)); 49 | bool Get(const char* key, uint32_t key_len, string &data) const 50 | __attribute__ ((warn_unused_result)); 51 | 52 | // bool Insert(const char* key, const char* data, uint32_t length) 53 | // __attribute__ ((warn_unused_result)); 54 | // bool Delete(const char* key) __attribute__ ((warn_unused_result)); 55 | // bool Delete(const char* key, bool writeLog) __attribute__ ((warn_unused_result)); 56 | // bool Get(const char* key, string &data) const 57 | // __attribute__ ((warn_unused_result)); 58 | 59 | FawnDS* Rewrite(pthread_rwlock_t* dbLock) __attribute__ ((warn_unused_result)); 60 | bool RenameAndClean(FawnDS* old) __attribute__ ((warn_unused_result)); 61 | bool Merge(FawnDS* other_db, pthread_rwlock_t* listLock) __attribute__ ((warn_unused_result)); 62 | bool Split(FawnDS* other_db, pthread_rwlock_t* listLock) 63 | __attribute__ ((warn_unused_result)); 64 | 65 | void split_init(const string& destinationIP); 66 | bool split_next(const DBID* startKey, const DBID* endKey, char* ret_key, uint32_t& key_length, string& data, bool& valid, bool& remove); 67 | 68 | 69 | bool SplitMergeHelper(FawnDS* other_db, pthread_rwlock_t* listLock, bool merge) 70 | __attribute__ ((warn_unused_result)); 71 | 72 | // Used to write header/hashtable sequentially for split/merge/rewrite 73 | bool WriteHashtableToFile() __attribute__ ((warn_unused_result)); 74 | 75 | // Used to write header/hashtable sequentially for split/merge/rewrite 76 | bool WriteOnlyHeaderToFile() __attribute__ ((warn_unused_result)); 77 | 78 | // Used to read header/hashtable sequentially when header is malloc'd 79 | bool ReadHashtableFromFile() __attribute__ ((warn_unused_result)); 80 | 81 | 82 | //inline string getPrecopyIP() { return precopyIP; } 83 | 84 | // per db lock 85 | string precopyIP; 86 | 87 | bool deleteFile(); 88 | private: 89 | FawnDS(const string& filename, keyType storeType); 90 | inline uint16_t keyfragment_from_key(const char* key, uint32_t key_len) const; 91 | 92 | // Locate the hashtable entry that this key occupies or would occupy. Methods 93 | // that use this utility function can check if that entry exists or not by 94 | // checking the valid flag. Returns -1 on failure. 95 | int32_t FindHashIndexInsert(const char* key, uint32_t key_len, bool* newObj); 96 | int32_t FindHashIndex(const char* key, uint32_t key_len, DataHeader* data_header) const; 97 | 98 | // This function is used to populate the member variables of this object with 99 | // a different file. 100 | bool ReopenFawnDSFromFd(); 101 | bool ReopenFawnDSFromFilename(); 102 | 103 | // This function returns a FawnDS object that does not have its filename 104 | // object variable set. This variable is important for functions such as 105 | // Rewrite(), Merge() and Split() so that we can delete the original file and 106 | // overwrite it with the new file. Therefore, it is the caller's 107 | // responsibility to either fill in that field or to never use the Rewrite(), 108 | // Merge() or Split() methods on the returned object. 109 | 110 | // Helper function to malloc or mmap header/hashtable 111 | static FawnDS* Create_FawnDS_From_Fd(int fd, 112 | const string& filename, 113 | uint64_t hash_table_size, 114 | uint64_t number_entries, 115 | uint64_t deleted_entries, 116 | double max_deleted_ratio, 117 | double max_load_factor, 118 | keyType kt) 119 | __attribute__ ((warn_unused_result)); 120 | 121 | static uint32_t FindNextHashSize(uint32_t number); 122 | 123 | // Incrementing keyfragbits above 15 requires 124 | // more modifications to code (e.g. hashkey is 16 bits in (Insert()) 125 | static const uint32_t KEYFRAGBITS = 14; 126 | static const uint32_t KEYFRAGMASK = (1 << KEYFRAGBITS) - 1; 127 | static const uint32_t DELETEDBITMASK = (2 << KEYFRAGBITS); 128 | static const uint32_t INDEXBITS = 16; 129 | static const uint32_t INDEXMASK = (1 << INDEXBITS) - 1; 130 | static const uint32_t VALIDBITMASK = KEYFRAGMASK+1; 131 | 132 | // Check if highest bit is 1; means deleted. 133 | inline bool deleted(int32_t index) const 134 | { 135 | return (hash_table_[index].present_key & DELETEDBITMASK); 136 | } 137 | 138 | inline bool valid(int32_t index) const 139 | { 140 | return (hash_table_[index].present_key & VALIDBITMASK); 141 | } 142 | 143 | // Used by Insert: check that entry is not deleted or empty 144 | inline bool exists(int32_t index) const 145 | { 146 | return valid(index) && !deleted(index); 147 | } 148 | 149 | inline uint16_t verifykey(int32_t index) const 150 | { 151 | return (hash_table_[index].present_key & KEYFRAGMASK); 152 | } 153 | 154 | int disable_readahead(); 155 | 156 | int fd_; 157 | struct DbHeader* header_; 158 | struct HashEntry* hash_table_; 159 | T* datastore; 160 | string filename_; 161 | off_t currSplit; 162 | 163 | static const double EXCESS_BUCKET_FACTOR = 1.1; 164 | }; 165 | 166 | } // namespace fawn 167 | 168 | #endif // #ifndef _FAWNDS_H_ 169 | -------------------------------------------------------------------------------- /fawnds/fawnds_flash.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | #ifndef __STDC_FORMAT_MACROS 19 | #define __STDC_FORMAT_MACROS 20 | #endif 21 | #include 22 | 23 | #include "hash_functions.h" 24 | #include "fawnds_flash.h" 25 | #include "debug.h" 26 | #include "hashutil.h" 27 | #include "print.h" 28 | #include "timing.h" 29 | 30 | using fawn::DataHeader; 31 | using fawn::Hashes; 32 | using fawn::HashUtil; 33 | 34 | #ifndef O_NOATIME 35 | #define O_NOATIME 0 /* O_NOATIME is linux-only */ 36 | #endif 37 | 38 | namespace fawn { 39 | /***************************************************/ 40 | /****************** DB FUNCTIONS *******************/ 41 | /***************************************************/ 42 | 43 | bool FawnDS_Flash::Write(const char* key, uint32_t key_len, const char* data, uint32_t length, off_t offset) 44 | { 45 | struct DataHeader data_header; 46 | data_header.data_length = length; 47 | data_header.key_length = key_len; 48 | data_header.deleteLog = false; 49 | 50 | struct iovec iov[3]; 51 | iov[0].iov_base = &data_header; 52 | iov[0].iov_len = sizeof(struct DataHeader); 53 | iov[1].iov_base = const_cast(key); 54 | iov[1].iov_len = key_len; 55 | iov[2].iov_base = const_cast(data); 56 | iov[2].iov_len = length; 57 | 58 | if (lseek(fd_, offset, SEEK_SET) != offset) { 59 | fprintf(stderr, "Could not seek to offset %"PRIu64": %s\n", 60 | offset, strerror(errno)); 61 | } 62 | 63 | if (writev(fd_, iov, 3) != (ssize_t) (sizeof(struct DataHeader) + key_len + length)) { 64 | fprintf(stderr, "Could not write iovec structure: %s\n", strerror(errno)); 65 | return false; 66 | } 67 | 68 | return true; 69 | } 70 | 71 | bool FawnDS_Flash::Delete(const char* key, uint32_t key_len, off_t offset) 72 | { 73 | /********* DELETE LOG **********/ 74 | // Just needs the key and the fact that it's deleted to be appended. 75 | struct DataHeader delete_header; 76 | delete_header.data_length = 0; 77 | delete_header.key_length = key_len; 78 | delete_header.deleteLog = true; 79 | 80 | if ((uint64_t)pwrite64(fd_, &delete_header, sizeof(struct DataHeader), 81 | offset) != sizeof(struct DataHeader)) { 82 | fprintf(stderr, "Could not write delete header at position %"PRIu64": %s\n", 83 | (uint64_t)offset, strerror(errno)); 84 | return false; 85 | } 86 | 87 | if ((uint64_t)pwrite64(fd_, key, key_len, 88 | offset + sizeof(struct DataHeader)) != key_len) { 89 | fprintf(stderr, "Could not write delete header at position %"PRIu64": %s\n", 90 | (uint64_t)offset + sizeof(struct DataHeader), strerror(errno)); 91 | return false; 92 | } 93 | 94 | return true; 95 | } 96 | 97 | bool FawnDS_Flash::ReadIntoHeader(off_t offset, DataHeader &data_header, string &key) 98 | { 99 | ssize_t n_read = pread64(fd_, &data_header, sizeof(struct DataHeader), offset); 100 | if (n_read < (ssize_t)sizeof(struct DataHeader)) { 101 | fprintf(stderr, "Read %lu bytes from DataHeader, expected %lu\n", n_read, sizeof(struct DataHeader)); 102 | return false; 103 | } 104 | uint32_t key_len = data_header.key_length; 105 | char *mdata = (char *)malloc(key_len); 106 | 107 | n_read = pread64(fd_, mdata, key_len, offset + sizeof(struct DataHeader)); 108 | if (n_read < key_len) { 109 | fprintf(stderr, "Read %lu bytes from key, expected %u\n", n_read, key_len); 110 | return false; 111 | } 112 | key.assign(mdata, key_len); 113 | free(mdata); 114 | return true; 115 | } 116 | 117 | bool FawnDS_Flash::Read(const char* key, 118 | uint32_t key_len, 119 | off_t offset, 120 | string &data) 121 | { 122 | DataHeader data_header; 123 | string inputkey; 124 | 125 | if (!ReadIntoHeader(offset, data_header, inputkey)) { 126 | return false; 127 | } 128 | 129 | // Hashing based on key fragment can result in potential key collision 130 | // So we read the dataheader here to compare the full key to ensure this. 131 | if (memcmp(inputkey.data(), key, key_len) != 0) { 132 | return false; 133 | } 134 | 135 | size_t length = data_header.data_length; 136 | if (length == 0) { 137 | return true; 138 | } 139 | 140 | #if 0 141 | // Readahead code -- n_read was removed because it was put into ReadIntoHeader 142 | // For readahead, you have to re-introduce that code into this function 143 | if (length < (n_read - sizeof(struct DataHeader))) { 144 | //printf("GDOEX skipped pread\n"); 145 | data.assign(data_header.partial_data, length); 146 | } else 147 | #endif 148 | { 149 | char *mdata = (char *)malloc(length); 150 | //printf("GDOEX pread64: %x (%d)\n", datapos + sizeof(DataHeader), length); 151 | if ((uint64_t)pread64(fd_, mdata, length, offset + key_len + sizeof(struct DataHeader)) != 152 | length) { 153 | fprintf(stderr, "Could not read data at position %"PRIu64": %s\n", 154 | offset + sizeof(DataHeader), strerror(errno)); 155 | free(mdata); 156 | return false; 157 | } 158 | data.assign(mdata, length); 159 | free(mdata); 160 | /* SPEED note: May be worth some day eliminating the redundant 161 | * data copy in this by figuring out how to read directly into the 162 | * string's buffer. Only matters for values > 2k where we 163 | * can't do readahead. */ 164 | /* Better speed note: Fix them *both*. Don't use a string, use 165 | * our own thing, and do all of the preads() directly into it. 166 | * Treat it like an mbuf/skbuf, and be able to advance the 167 | * "start" pointer into the buffer so we never have to copy 168 | * regardless of how we get the data. */ 169 | } 170 | return true; 171 | } 172 | 173 | 174 | 175 | 176 | } // namespace fawn 177 | -------------------------------------------------------------------------------- /fawnds/fawnds_flash.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _FAWNDS_FLASH_H_ 3 | #define _FAWNDS_FLASH_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include "db_structures.h" 9 | 10 | #ifdef __APPLE__ 11 | #define pread64 pread 12 | #define pwrite64 pwrite 13 | #endif // #ifdef __APPLE__ 14 | 15 | #ifndef _LARGEFILE64_SOURCE 16 | #define _LARGEFILE64_source 17 | #endif // #ifndef _LARGEFILE64_SOURCE 18 | #define _FILE_OFFSET_BITS 64 19 | 20 | using namespace std; 21 | 22 | namespace fawn { 23 | 24 | class FawnDS_Flash { 25 | public: 26 | FawnDS_Flash(int fd) : fd_(fd) {} 27 | ~FawnDS_Flash() {} 28 | bool Write(const char* key, uint32_t key_len, const char* data, uint32_t length, off_t offset); 29 | bool Delete(const char* key, uint32_t key_len, off_t offset); 30 | bool ReadIntoHeader(off_t offset, DataHeader &data_header, string &key);// const; 31 | bool Read(const char* key, uint32_t key_len, off_t offset, string &data);// const; 32 | private: 33 | int fd_; 34 | }; 35 | 36 | } // namespace fawn 37 | 38 | #endif // #ifndef _FAWNDS_FLASH_H_ 39 | -------------------------------------------------------------------------------- /fawnds/fawnds_test.full: -------------------------------------------------------------------------------- 1 | #include "fawnds.h" 2 | #include 3 | #include 4 | #include 5 | 6 | namespace fawn { 7 | 8 | class FawnDSTest : public testing::Test { 9 | protected: 10 | // Code here will be called immediately after the constructor (right before 11 | // each test). 12 | virtual void SetUp() { 13 | h = FawnDS::Create_FawnDS("fawn_db", num_records_, max_deleted_ratio_, 14 | max_load_factor_); 15 | h = FawnDS::Open_FawnDS("fawn_db"); 16 | 17 | h2 = FawnDS::Create_FawnDS("fawn_db2", num_records_, max_deleted_ratio_, 18 | max_load_factor_, MALLOC_HASHTABLE); 19 | //h2 = FawnDS::Open_FawnDS("fawn_db2"); 20 | 21 | } 22 | 23 | // Code in the TearDown() method will be called immediately after each test 24 | // (right before the destructor). 25 | virtual void TearDown() { 26 | //EXPECT_EQ(0, unlink("/localfs/fawn_db")); 27 | } 28 | 29 | // Objects declared here can be used by all tests in the test case for FawnDS. 30 | FawnDS *h; 31 | FawnDS *h2; 32 | 33 | private: 34 | static const u_int64_t num_records_ = 1000000; 35 | static const double max_deleted_ratio_ = .9; 36 | static const double max_load_factor_ = .8; 37 | }; 38 | 39 | TEST_F(FawnDSTest, TestSimpleInsertRetrieve) { 40 | const char* key = "key0"; 41 | const char* data = "value0"; 42 | ASSERT_TRUE(h->Insert(key, data, 7)); 43 | 44 | char* data_returned; 45 | u_int64_t length; 46 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 47 | EXPECT_EQ(7, length); 48 | EXPECT_STREQ(data, data_returned); 49 | } 50 | 51 | TEST_F(FawnDSTest, TestSimpleInsertNovalueRetrieve) { 52 | const char* key = "key0"; 53 | const char* data = ""; 54 | ASSERT_TRUE(h->Insert(key, data, 0)); 55 | 56 | char* data_returned; 57 | u_int64_t length; 58 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 59 | EXPECT_EQ(0, length); 60 | EXPECT_EQ(NULL, data_returned); 61 | } 62 | 63 | TEST_F(FawnDSTest, TestSimpleDelete) { 64 | //Simple 65 | const char* key = "key0"; 66 | const char* data = "value0"; 67 | //Delete should return false 68 | ASSERT_FALSE(h->Delete(key)); 69 | ASSERT_TRUE(h->Insert(key, data, 7)); 70 | 71 | char* data_returned; 72 | u_int64_t length; 73 | //Retrieve 74 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 75 | EXPECT_EQ(7, length); 76 | EXPECT_STREQ(data, data_returned); 77 | // Then delete 78 | ASSERT_TRUE(h->Delete(key)); 79 | // Retreive should return false 80 | ASSERT_FALSE(h->Get(key, &data_returned, &length)); 81 | 82 | const char* key2 = "key1"; 83 | const char* data2 = "value1"; 84 | ASSERT_TRUE(h->Insert(key2, data2, 7)); 85 | ASSERT_TRUE(h->Get(key2, &data_returned, &length)); 86 | EXPECT_EQ(7, length); 87 | EXPECT_STREQ(data2, data_returned); 88 | } 89 | 90 | 91 | TEST_F(FawnDSTest, Test10000InsertRetrieve) { 92 | char key[32]; 93 | char data[52]; 94 | 95 | for (int i = 0; i < 10000; ++i) { 96 | int* keyi = (int*)key; 97 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 98 | keyi[j] = i; 99 | } 100 | int* datai = (int*)data; 101 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 102 | datai[j] = i; 103 | } 104 | data[51] = 0; 105 | ASSERT_TRUE(h->Insert(key, data, 52)); 106 | } 107 | 108 | for (int i = 0; i < 10000; ++i) { 109 | char * data_returned; 110 | u_int64_t length; 111 | int* keyi = (int*)key; 112 | int* datai = (int*)data; 113 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 114 | keyi[j] = i; 115 | } 116 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 117 | datai[j] = i; 118 | } 119 | data[51] = 0; 120 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 121 | EXPECT_EQ(52, length); 122 | EXPECT_STREQ(data, data_returned); 123 | } 124 | } 125 | 126 | TEST_F(FawnDSTest, Test10000InsertDelete) { 127 | char key[32]; 128 | char data[52]; 129 | 130 | for (int i = 0; i < 10000; ++i) { 131 | int* keyi = (int*)key; 132 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 133 | keyi[j] = i; 134 | } 135 | int* datai = (int*)data; 136 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 137 | datai[j] = i; 138 | } 139 | data[51] = 0; 140 | ASSERT_TRUE(h->Insert(key, data, 52)); 141 | } 142 | 143 | for (int i = 0; i < 10000; ++i) { 144 | int* keyi = (int*)key; 145 | //printf("i: %d\n", i); 146 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 147 | keyi[j] = i; 148 | } 149 | ASSERT_TRUE(h->Delete(key)); 150 | } 151 | } 152 | 153 | 154 | TEST_F(FawnDSTest, Test10000Split) { 155 | char key[32]; 156 | char data[52]; 157 | for (int i = 0; i < 10000; ++i) { 158 | int* keyi = (int*)key; 159 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 160 | keyi[j] = i; 161 | } 162 | int* datai = (int*)data; 163 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 164 | datai[j] = i; 165 | } 166 | data[51] = 0; 167 | ASSERT_TRUE(h->Insert(key, data, 52)); 168 | } 169 | 170 | char split_key[32]; 171 | int* skeyi = (int*)split_key; 172 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 173 | skeyi[j] = 5000; 174 | } 175 | 176 | ASSERT_TRUE(h->Split(split_key, h2)); 177 | ASSERT_TRUE(h2->WriteHashtableToFile()); 178 | h2 = FawnDS::Open_FawnDS("fawn_db2"); 179 | 180 | for (int i = 0; i < 10000; ++i) { 181 | char * data_returned; 182 | u_int64_t length; 183 | int* keyi = (int*)key; 184 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 185 | keyi[j] = i; 186 | } 187 | printf("i: %d\n", i); 188 | if (memcmp(key, split_key, KEYSIZE) > 0) { 189 | ASSERT_TRUE(h2->Get(key, &data_returned, &length)); 190 | } else { 191 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 192 | } 193 | } 194 | } 195 | 196 | TEST_F(FawnDSTest, Test10000InsertRewriteRetrieve) { 197 | char key[32]; 198 | char data[52]; 199 | 200 | for (int i = 0; i < 10000; ++i) { 201 | int* keyi = (int*)key; 202 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 203 | keyi[j] = i; 204 | } 205 | int* datai = (int*)data; 206 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 207 | datai[j] = i; 208 | } 209 | data[51] = 0; 210 | ASSERT_TRUE(h->Insert(key, data, 52)); 211 | } 212 | 213 | ASSERT_TRUE(h->Rewrite()); 214 | 215 | for (int i = 0; i < 10000; ++i) { 216 | char * data_returned; 217 | u_int64_t length; 218 | int* keyi = (int*)key; 219 | int* datai = (int*)data; 220 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 221 | keyi[j] = i; 222 | } 223 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 224 | datai[j] = i; 225 | } 226 | data[51] = 0; 227 | ASSERT_TRUE(h->Get(key, &data_returned, &length)); 228 | EXPECT_EQ(52, length); 229 | EXPECT_STREQ(data, data_returned); 230 | } 231 | 232 | } 233 | 234 | TEST_F(FawnDSTest, Test10000Merge) { 235 | char key[32]; 236 | char data[52]; 237 | for (int i = 0; i < 5000; ++i) { 238 | int* keyi = (int*)key; 239 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 240 | keyi[j] = i; 241 | } 242 | int* datai = (int*)data; 243 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 244 | datai[j] = i; 245 | } 246 | data[51] = 0; 247 | ASSERT_TRUE(h->Insert(key, data, 52)); 248 | } 249 | 250 | for (int i = 5000; i < 10000; ++i) { 251 | int* keyi = (int*)key; 252 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 253 | keyi[j] = i; 254 | } 255 | int* datai = (int*)data; 256 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 257 | datai[j] = i; 258 | } 259 | data[51] = 0; 260 | ASSERT_TRUE(h2->Insert(key, data, 52)); 261 | } 262 | 263 | 264 | ASSERT_TRUE(h2->Merge(*h)); 265 | ASSERT_TRUE(h2->WriteHashtableToFile()); 266 | 267 | for (int i = 0; i < 10000; ++i) { 268 | char * data_returned; 269 | u_int64_t length; 270 | int* keyi = (int*)key; 271 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 272 | keyi[j] = i; 273 | } 274 | ASSERT_TRUE(h2->Get(key, &data_returned, &length)); 275 | } 276 | 277 | } 278 | 279 | TEST_F(FawnDSTest, Test100000Insert) { 280 | char key[32]; 281 | char data[1000]; 282 | 283 | for (int i = 0; i < 100000; ++i) { 284 | int* keyi = (int*)key; 285 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 286 | keyi[j] = i; 287 | } 288 | int* datai = (int*)data; 289 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 290 | datai[j] = i; 291 | } 292 | data[51] = 0; 293 | ASSERT_TRUE(h->Insert(key, data, 1000)); 294 | } 295 | } 296 | 297 | TEST_F(FawnDSTest, Test1000000Insert) { 298 | char key[32]; 299 | char data[1000]; 300 | 301 | for (int i = 0; i < 1000000; ++i) { 302 | int* keyi = (int*)key; 303 | for (u_int j = 0; j < 32 * sizeof(char) / sizeof(int); ++j) { 304 | keyi[j] = i; 305 | } 306 | int* datai = (int*)data; 307 | for (u_int j = 0; j < 52 * sizeof(char) / sizeof(int); ++j) { 308 | datai[j] = i; 309 | } 310 | data[51] = 0; 311 | ASSERT_TRUE(h->Insert(key, data, 1000)); 312 | } 313 | } 314 | 315 | 316 | } // namespace fawn 317 | 318 | int main(int argc, char** argv) { 319 | testing::InitGoogleTest(&argc, argv); 320 | return RUN_ALL_TESTS(); 321 | } 322 | -------------------------------------------------------------------------------- /fawnds/fawndsmesg.thrift: -------------------------------------------------------------------------------- 1 | namespace cpp fawn 2 | 3 | enum FawnDSVType { 4 | SET=2, 5 | } 6 | 7 | enum FawnDSSetOPType { 8 | INSERT_INTO_SET=1, 9 | DELETE_FROM_SET=2, 10 | } 11 | 12 | struct FawnDSKItem { 13 | 1: i32 type, 14 | 2: binary data, 15 | } 16 | 17 | 18 | struct FawnDSKSet { 19 | 1: list item, 20 | } 21 | 22 | 23 | 24 | struct FawnDSSetOp { 25 | 1: binary key, 26 | 2: FawnDSSetOPType type, 27 | 3: list item, 28 | } 29 | 30 | -------------------------------------------------------------------------------- /fawnds/hash_functions.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "hash_functions.h" 3 | #include "hashutil.h" 4 | 5 | namespace fawn { 6 | 7 | hashfn_t Hashes::hashes[HASH_COUNT] = { &Hashes::h1, 8 | &Hashes::h2, 9 | &Hashes::h3 }; 10 | 11 | uint32_t Hashes::h1(const void* buf, size_t len) 12 | { 13 | return HashUtil::BobHash(buf, len, 0xdeadbeef); 14 | } 15 | uint32_t Hashes::h2(const void* buf, size_t len) 16 | { 17 | return HashUtil::BobHash(buf, len, 0xc0ffee); 18 | } 19 | uint32_t Hashes::h3(const void* buf, size_t len) 20 | { 21 | return HashUtil::BobHash(buf, len, 0x101CA75); 22 | } 23 | 24 | // These hashes assume that len > 32bits 25 | uint32_t Hashes::nullhash1(const void* buf, size_t len) 26 | { 27 | if (len < 4) printf("Warning: keylength is not long enough to use null hash 1. Ensure keylength >= 32bits\n"); 28 | return HashUtil::NullHash(buf, len, 0); 29 | } 30 | 31 | uint32_t Hashes::nullhash2(const void* buf, size_t len) 32 | { 33 | if (len < 8) printf("Warning: keylength is not long enough to use null hash 2. Ensure keylength >= 64bits\n"); 34 | return HashUtil::NullHash(buf, len, 4); 35 | } 36 | 37 | uint32_t Hashes::nullhash3(const void* buf, size_t len) 38 | { 39 | if (len < 12) printf("Warning: keylength is not long enough to use null hash 3. Ensure keylength >= 96bits\n"); 40 | return HashUtil::NullHash(buf, len, 8); 41 | } 42 | 43 | } // namespace fawn 44 | -------------------------------------------------------------------------------- /fawnds/hash_functions.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _HASH_FUNCTIONS_H_ 3 | #define _HASH_FUNCTIONS_H_ 4 | 5 | #include 6 | #include 7 | #include "hashutil.h" 8 | 9 | #define HASH_COUNT 3 10 | 11 | typedef uint32_t (*hashfn_t)(const void*, size_t); 12 | 13 | namespace fawn { 14 | 15 | class Hashes { 16 | public: 17 | static uint32_t h1(const void* buf, size_t len); 18 | static uint32_t h2(const void* buf, size_t len); 19 | static uint32_t h3(const void* buf, size_t len); 20 | static uint32_t nullhash1(const void* buf, size_t len); 21 | static uint32_t nullhash2(const void* buf, size_t len); 22 | static uint32_t nullhash3(const void* buf, size_t len); 23 | static hashfn_t hashes[HASH_COUNT]; 24 | private: 25 | Hashes(); 26 | }; 27 | 28 | } // namespace fawn 29 | 30 | #endif // #ifndef _HASH_FUNCTIONS_H_ 31 | -------------------------------------------------------------------------------- /fawnds/runfulltest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | mv fawnds_test.cc fawnds_test.small 3 | mv fawnds_test.full fawnds_test.cc 4 | make test 5 | ./test --gtest_print_time 6 | make clean 7 | mv fawnds_test.cc fawnds_test.full 8 | mv fawnds_test.small fawnds_test.cc 9 | -------------------------------------------------------------------------------- /fawnds/runtest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | make test 3 | ./test --gtest_print_time 4 | make clean 5 | -------------------------------------------------------------------------------- /fawnds/testHash.cc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "fawnds.h" 4 | #include "hash_functions.h" 5 | 6 | 7 | 8 | 9 | using namespace std; 10 | using namespace fawn; 11 | 12 | int main() { 13 | const char* dbname = "fawn_db"; 14 | u_int64_t num_records = 10; 15 | double max_deleted_ratio = .9; 16 | double max_load_factor = .8; 17 | 18 | FawnDS *h; 19 | 20 | cout << "Creating FawnDS" << endl; 21 | h = fawn::FawnDS::Create_FawnDS(dbname, num_records, 22 | max_deleted_ratio, 23 | max_load_factor); 24 | 25 | cout << "Opening FawnDS" << endl; 26 | h = fawn::FawnDS::Open_FawnDS(dbname); 27 | 28 | 29 | const char* key = "key0"; 30 | const char* data = "value0"; 31 | bool irv; 32 | 33 | cout << "Inserting " << key << "," << data << " into " << dbname << endl; 34 | 35 | irv = h->Insert(key, data, 8); 36 | 37 | char** data_returned = (char**) malloc(sizeof(char*)); 38 | u_int64_t* length = (u_int64_t*) malloc(sizeof(u_int64_t)); 39 | bool grv; 40 | 41 | printf("Get value from database\n"); 42 | grv = h->Get(key, data_returned, length); 43 | 44 | if (key != NULL && data != NULL && length != NULL) { 45 | cout << "Key: " << key << " data: " << *data_returned 46 | << " length: " << *length << endl; 47 | } 48 | 49 | 50 | return 0; 51 | } 52 | -------------------------------------------------------------------------------- /fawnkv/CODE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/fawnkv/CODE -------------------------------------------------------------------------------- /fawnkv/FawnKVServerHandler.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "FawnKVServerHandler.h" 14 | 15 | #include 16 | 17 | using namespace ::apache::thrift; 18 | using namespace ::apache::thrift::protocol; 19 | using namespace ::apache::thrift::transport; 20 | using namespace ::apache::thrift::server; 21 | 22 | using boost::shared_ptr; 23 | 24 | using namespace fawn; 25 | 26 | Cache* cache; 27 | map cid_client_map; 28 | map req_client_map; 29 | pthread_mutex_t *p_mutex; 30 | pthread_mutex_t *p_mutex2; 31 | 32 | double t, r; 33 | double t_since = -1; 34 | int64_t reqSince; 35 | timeval tim; 36 | 37 | void put_cb(unsigned int continuation) 38 | { 39 | // continuation is the server-side continuation 40 | // Need to map back to client continuation 41 | 42 | pthread_mutex_lock(p_mutex); 43 | //cout << "put_cb for " << continuation << endl; 44 | ClientData *cd = req_client_map[continuation]; 45 | int64_t cli_continuation = cd->getContinuation(continuation); 46 | cd->removeContinuation(continuation); 47 | 48 | // Send to client 49 | cd->fc->put_response(cli_continuation); 50 | //cout << "put_cb done for " << continuation << endl; 51 | pthread_mutex_unlock(p_mutex); 52 | } 53 | 54 | void put_w_cb(unsigned int continuation, int64_t version) 55 | { 56 | cout << "WARNING: put_w is not implemented for remote frontends" << endl; 57 | exit(1); 58 | } 59 | 60 | void get_cb(const DBID& p_key, const string& p_value, unsigned int continuation, bool success) 61 | { 62 | pthread_mutex_lock(p_mutex); 63 | //cout << "get_cb for " << continuation << endl; 64 | ClientData *cd = req_client_map[continuation]; 65 | int64_t cli_continuation = cd->getContinuation(continuation); 66 | cd->removeContinuation(continuation); 67 | 68 | // Send to client 69 | cd->fc->get_response(p_value, cli_continuation); 70 | 71 | //if (cache) 72 | // cache->insert(p_key, p_value); 73 | //cout << "get_cb done for " << continuation << endl; 74 | pthread_mutex_unlock(p_mutex); 75 | } 76 | 77 | FawnKVServerHandler::FawnKVServerHandler(FrontEnd *fe, Cache *c) { 78 | // Your initialization goes here 79 | frontend = fe; 80 | cache = c; 81 | frontend->register_put_cb((void (*)(unsigned int)) &put_cb); 82 | frontend->register_put_w_cb((void (*)(unsigned int, int64_t)) &put_w_cb); 83 | frontend->register_get_cb((void (*)(const fawn::DBID&, const std::string&, unsigned int, bool)) &get_cb); 84 | pthread_mutex_init(&mutex, NULL); 85 | pthread_mutex_init(&mutex2, NULL); 86 | p_mutex = &mutex; 87 | p_mutex2 = &mutex2; 88 | clientID = 0; 89 | reqNumber = 0; 90 | gettimeofday(&tim, NULL); 91 | } 92 | 93 | ClientData* FawnKVServerHandler::get_client(const int32_t cid) { 94 | map::iterator it = cid_client_map.find(cid); 95 | if (it != cid_client_map.end()) { 96 | return cid_client_map[cid]; 97 | } 98 | return NULL; 99 | } 100 | 101 | int32_t FawnKVServerHandler::init(const std::string& clientIP, const int32_t port) { 102 | cout << "Connection from " << clientIP << ":" << port << endl; 103 | shared_ptr socket(new TSocket(clientIP.data(), port)); 104 | shared_ptr transport(new TBufferedTransport(socket)); 105 | shared_ptr protocol(new TBinaryProtocol(transport)); 106 | FawnKVAppClient *fc = new FawnKVAppClient(protocol); 107 | 108 | while(1) { 109 | try { 110 | transport->open(); 111 | break; 112 | } catch (TException &tx) { 113 | fprintf(stderr, "Transport error: %s\n", tx.what()); 114 | } 115 | sleep(1); 116 | } 117 | 118 | // New client, allocate 119 | ClientData *cd = new ClientData(fc); // Need to destroy at some point 120 | int cid = clientID++; 121 | cid_client_map[cid] = cd; 122 | return cid; 123 | } 124 | 125 | void FawnKVServerHandler::get(const std::string& key, const int64_t continuation, const int32_t cid) { 126 | pthread_mutex_lock(p_mutex); 127 | // Generating unique server-side continuation 128 | //cout << reqNumber << endl; 129 | int64_t ss_continuation = reqNumber++; 130 | 131 | ClientData *cd = get_client(cid); 132 | DBID dkey(key); 133 | string value; 134 | if ((cache) && (cache->lookup(dkey, value))){ 135 | cd->fc->get_response(value, continuation); 136 | } 137 | else { 138 | //cout << "Get from CID: " << cid << " continuation: " << continuation << " reqNumber: " << ss_continuation << endl; 139 | cd->addContinuation(continuation, ss_continuation); 140 | req_client_map[ss_continuation] = cd; 141 | frontend->get(key, ss_continuation); 142 | //cout << "Get Done for CID: " << cid << " continuation: " << continuation << " reqNumber: " << ss_continuation << endl; 143 | } 144 | 145 | if ((reqNumber % 2000) == 0) { 146 | // if (1) { 147 | gettimeofday(&tim, NULL); 148 | t = tim.tv_sec + (tim.tv_usec/1000000.0); 149 | if ((t_since > 0) && (t-t_since > 0)) { 150 | double rr = (reqNumber - reqSince)/1000.00/(t-t_since); 151 | r = 0.8 * r + 0.2 * rr; 152 | cout << r << "k query/sec " ; //<< t - t_since << endl; 153 | if (cache) 154 | cout << "hit_ratio " << cache->hit_ratio() << " cache_size " << cache->size() << endl; 155 | else 156 | cout << "hit_ratio " << 0 << endl; 157 | } 158 | 159 | t_since = t; 160 | reqSince = reqNumber; 161 | } 162 | 163 | pthread_mutex_unlock(p_mutex); 164 | 165 | } 166 | 167 | void FawnKVServerHandler::put(const std::string& key, const std::string& value, const int64_t continuation, const int32_t cid) { 168 | pthread_mutex_lock(p_mutex); 169 | // Generating unique server-side continuation 170 | int64_t ss_continuation = reqNumber++; 171 | //cout << "Put from CID: " << cid << " continuation: " << continuation << " reqNumber: " << ss_continuation << endl; 172 | ClientData *cd = get_client(cid); 173 | cd->addContinuation(continuation, ss_continuation); 174 | req_client_map[ss_continuation] = cd; 175 | string v = value; 176 | frontend->put(key, value, ss_continuation); 177 | DBID dkey(key); 178 | if (cache) 179 | cache->insert(dkey, value); 180 | //cout << "Put Done for CID: " << cid << " continuation: " << continuation << " reqNumber: " << ss_continuation << endl; 181 | pthread_mutex_unlock(p_mutex); 182 | 183 | } 184 | 185 | void FawnKVServerHandler::remove(const std::string& key, const int64_t continuation, const int32_t cid) { 186 | // Generating unique server-side continuation 187 | int64_t ss_continuation = reqNumber++; 188 | ClientData *cd = get_client(cid); 189 | cd->addContinuation(continuation, ss_continuation); 190 | req_client_map[ss_continuation] = cd; 191 | pthread_mutex_lock(p_mutex); 192 | frontend->remove(key, ss_continuation); 193 | pthread_mutex_unlock(p_mutex); 194 | } 195 | 196 | 197 | void usage() 198 | { 199 | cerr << "./frontend -m ManagerIP -p myPort -l listenPort -c cacheSize myIP\n\n" 200 | << endl; 201 | } 202 | 203 | 204 | int main(int argc, char **argv) 205 | { 206 | int ch; 207 | string managerIP; 208 | int myPort = FE_SERVER_PORT_BASE; 209 | int listenPort = FE_SERVER_PORT_LISTEN; 210 | int cacheSize = 0; 211 | while ((ch = getopt(argc, argv, "m:p:l:c:")) != -1) { 212 | switch (ch) { 213 | case 'm': 214 | managerIP = optarg; 215 | break; 216 | case 'p': 217 | myPort = atoi(optarg); 218 | break; 219 | case 'l': 220 | listenPort = atoi(optarg); 221 | break; 222 | case 'c': 223 | cacheSize = atoi(optarg); 224 | break; 225 | default: 226 | usage(); 227 | exit(-1); 228 | } 229 | } 230 | argc -= optind; 231 | argv += optind; 232 | 233 | if (argc < 1) { 234 | usage(); 235 | exit(-1); 236 | } 237 | 238 | string myIP(argv[0]); 239 | 240 | /* Initialize ring */ 241 | FrontEnd *fe = new FrontEnd(managerIP, myIP, myPort); 242 | 243 | /* Initialize cache */ 244 | if (cacheSize > 0) 245 | cache = new Cache(cacheSize); 246 | else 247 | cache = NULL; 248 | 249 | shared_ptr handler(new FawnKVServerHandler(fe, cache)); 250 | shared_ptr processor(new FawnKVProcessor(handler)); 251 | shared_ptr serverTransport(new TServerSocket(listenPort)); 252 | shared_ptr transportFactory(new TBufferedTransportFactory()); 253 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 254 | 255 | TThreadedServer server(processor, serverTransport, transportFactory, protocolFactory); 256 | server.serve(); 257 | 258 | cout << "Exiting front-end manager." << endl; 259 | 260 | return 0; 261 | } 262 | 263 | -------------------------------------------------------------------------------- /fawnkv/FawnKVServerHandler.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _FAWNKVSERVERHANDLER_H_ 3 | #define _FAWNKVSERVERHANDLER_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fe.h" 18 | #include "fe_cache.h" 19 | #include "FawnKV.h" 20 | #include "FawnKVApp.h" 21 | 22 | using namespace ::apache::thrift; 23 | using namespace ::apache::thrift::protocol; 24 | using namespace ::apache::thrift::transport; 25 | using namespace ::apache::thrift::server; 26 | using namespace tbb; 27 | 28 | using boost::shared_ptr; 29 | 30 | using namespace fawn; 31 | 32 | #define FE_SERVER_PORT_LISTEN 4001 33 | #define FE_SERVER_PORT_BASE 7000 34 | 35 | class ClientData { 36 | private: 37 | map req_cont_map; 38 | public: 39 | FawnKVAppClient *fc; 40 | 41 | ClientData(FawnKVAppClient* client) { 42 | fc = client; 43 | } 44 | 45 | void addContinuation(int64_t continuation, int64_t req) { 46 | req_cont_map[req] = continuation; 47 | } 48 | 49 | int64_t getContinuation(int64_t req) { 50 | return req_cont_map[req]; 51 | } 52 | 53 | void removeContinuation(int64_t req) { 54 | req_cont_map.erase(req); 55 | } 56 | }; 57 | 58 | class FawnKVServerHandler : virtual public FawnKVIf { 59 | private: 60 | atomic clientID; 61 | atomic reqNumber; 62 | 63 | FrontEnd* frontend; 64 | 65 | ClientData* get_client(const int32_t cid); 66 | 67 | public: 68 | FawnKVServerHandler(FrontEnd *fe, Cache* c); 69 | pthread_mutex_t mutex; 70 | pthread_mutex_t mutex2; 71 | 72 | int32_t init(const std::string& clientIP, const int32_t port); 73 | void get(const std::string& key, const int64_t continuation, const int32_t cid); 74 | void put(const std::string& key, const std::string& value, const int64_t continuation, const int32_t cid); 75 | void remove(const std::string& key, const int64_t continuation, const int32_t cid); 76 | }; 77 | 78 | #endif 79 | -------------------------------------------------------------------------------- /fawnkv/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_LTLIBRARIES = libfawnkv.la 2 | libfawnkv_la_CPPFLAGS = -I$(top_srcdir)/utils \ 3 | -I$(builddir)/gen-cpp \ 4 | -I$(top_srcdir)/fawnds \ 5 | -I$(top_builddir)/fawnds 6 | 7 | nodist_libfawnkv_la_SOURCES = \ 8 | $(builddir)/gen-cpp/FawnKVBackend.cpp \ 9 | $(builddir)/gen-cpp/FawnKVBackend.h \ 10 | $(builddir)/gen-cpp/FawnKVFrontend.cpp \ 11 | $(builddir)/gen-cpp/FawnKVFrontend.h \ 12 | $(builddir)/gen-cpp/FawnKVManager.cpp \ 13 | $(builddir)/gen-cpp/FawnKVManager.h \ 14 | $(builddir)/gen-cpp/FawnKV.cpp \ 15 | $(builddir)/gen-cpp/FawnKV.h \ 16 | $(builddir)/gen-cpp/FawnKVApp.cpp \ 17 | $(builddir)/gen-cpp/FawnKVApp.h \ 18 | $(builddir)/gen-cpp/fawnkvmesg_constants.cpp \ 19 | $(builddir)/gen-cpp/fawnkvmesg_constants.h \ 20 | $(builddir)/gen-cpp/fawnkvmesg_types.cpp \ 21 | $(builddir)/gen-cpp/fawnkvmesg_types.h \ 22 | $(builddir)/gen-cpp/fawnkv_constants.cpp \ 23 | $(builddir)/gen-cpp/fawnkv_constants.h \ 24 | $(builddir)/gen-cpp/fawnkv_types.cpp \ 25 | $(builddir)/gen-cpp/fawnkv_types.h 26 | 27 | SKELETON_CODE = \ 28 | $(builddir)/gen-cpp/FawnKVApp_server.skeleton.cpp \ 29 | $(builddir)/gen-cpp/FawnKVBackend_server.skeleton.cpp \ 30 | $(builddir)/gen-cpp/FawnKVFrontend_server.skeleton.cpp \ 31 | $(builddir)/gen-cpp/FawnKVManager_server.skeleton.cpp \ 32 | $(builddir)/gen-cpp/FawnKV_server.skeleton.cpp 33 | 34 | BUILT_SOURCES = $(nodist_libfawnkv_la_SOURCES) 35 | 36 | $(nodist_libfawnkv_la_SOURCES) : fawnkv.thrift fawnkvmesg.thrift 37 | thrift -r -o $(builddir) --gen cpp $(srcdir)/fawnkvmesg.thrift 38 | thrift -r -o $(builddir) --gen cpp $(srcdir)/fawnkv.thrift 39 | 40 | libfawnkv_la_SOURCES = \ 41 | fawnkvmesg.thrift \ 42 | fawnkv.thrift \ 43 | node.cpp \ 44 | node.h \ 45 | nodehandle.cpp \ 46 | nodehandle.h \ 47 | ring.cpp \ 48 | ring.h \ 49 | fe.cpp \ 50 | fe.h \ 51 | virtualnode.cpp \ 52 | virtualnode.h 53 | 54 | # kaminsky: following line forces noinst_* libraries to build 55 | # shared. This can help with development because a change to 56 | # this library doesn't require re-building libs and programs 57 | # that link against this library 58 | #libfawnkv_la_LDFLAGS = -rpath `pwd` 59 | 60 | noinst_PROGRAMS = manager backend frontend 61 | 62 | manager_SOURCES = Manager.cpp Manager.h 63 | 64 | manager_CPPFLAGS = \ 65 | -I$(top_srcdir)/utils \ 66 | -I$(srcdir) \ 67 | -I$(builddir)/gen-cpp \ 68 | -I$(top_srcdir)/fawnds \ 69 | -I$(top_builddir)/fawnkv \ 70 | -I$(top_builddir)/fawnds 71 | 72 | manager_LDADD = \ 73 | $(top_builddir)/fawnkv/libfawnkv.la \ 74 | $(top_builddir)/utils/libfawnkvutils.la \ 75 | $(THRIFT_LIBS) 76 | 77 | 78 | backend_CPPFLAGS = -I$(top_srcdir)/utils \ 79 | -I$(top_srcdir)/fawnds \ 80 | -I$(top_builddir)/fawnds \ 81 | -I$(srcdir) \ 82 | -I$(builddir)/gen-cpp 83 | 84 | backend_SOURCES = node_mgr_db.cpp node_mgr.cpp node_mgr.h FawnKVBackendHandler.cpp 85 | 86 | backend_LDADD = \ 87 | $(top_builddir)/fawnds/libfawnds.la \ 88 | $(top_builddir)/utils/libfawnkvutils.la \ 89 | libfawnkv.la \ 90 | $(THRIFT_LIBS) 91 | 92 | 93 | frontend_SOURCES = FawnKVServerHandler.cpp FawnKVServerHandler.h fe_cache.cpp fe_cache.h 94 | 95 | frontend_CPPFLAGS = \ 96 | -I$(top_srcdir)/utils \ 97 | -I$(srcdir) \ 98 | -I$(builddir)/gen-cpp \ 99 | -I$(top_srcdir)/fawnds \ 100 | -I$(top_builddir)/fawnkv \ 101 | -I$(top_builddir)/fawnds 102 | 103 | frontend_LDADD = \ 104 | $(top_builddir)/fawnkv/libfawnkv.la \ 105 | $(top_builddir)/utils/libfawnkvutils.la \ 106 | $(THRIFT_LIBS) 107 | 108 | CLEANFILES = $(nodist_libfawnkv_la_SOURCES) $(SKELETON_CODE) 109 | -------------------------------------------------------------------------------- /fawnkv/Manager.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _MANAGER_H_ 3 | #define _MANAGER_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include "dbid.h" 16 | #include "virtualnode.h" 17 | #include "nodehandle.h" 18 | #include "node.h" 19 | #include "debug.h" 20 | #include "FawnKVManager.h" 21 | #include "FawnKVBackend.h" 22 | #include "print.h" 23 | 24 | #include "hashutil.h" 25 | 26 | using fawn::DBID; 27 | using namespace std; 28 | using namespace tbb; 29 | 30 | 31 | #define MANAGER_PORT_BASE 4000 32 | 33 | void *localServerThreadLoop(void *p); 34 | void *ringStart(void *p); 35 | 36 | //////////////////////////////////////////////////////////////////////////////// 37 | 38 | 39 | //const int NUM_VIRTUAL_IDS = 8; 40 | const int NUM_VIRTUAL_IDS = 1; 41 | const int NUM_PHY_NODES = 15; 42 | 43 | const int AGING_INTERVAL_IN_SEC = 5; // Beat interval 44 | const int NUM_BEATS_BEFORE_FAILURE = 3; 45 | 46 | class Manager : virtual public FawnKVManagerIf { 47 | public: 48 | Manager(string localIP, int num_replicas = 1); 49 | void init(string localIP, int num_replicas); 50 | ~Manager(); 51 | 52 | void sendNeighborUpdates(); 53 | 54 | void static_join(const std::string& ip, const int32_t port); 55 | void rejoin(const std::vector & vnodeids, const std::vector & startids, const std::string& ip, const int32_t port); 56 | void join(const std::string& ip, const int32_t port, bool merge=false); 57 | void vnode_pre_join(const std::string& vid, const std::string& ip, const int32_t port); 58 | void vnode_join(const std::string& vid, const std::string& ip, const int32_t port, int32_t merge); 59 | void vnode_join_extend_chain(const std::string& vid, const std::string& ip, const int32_t port, int32_t merge); 60 | void flush_split(const std::string& start_id, const std::string& end_id, const std::string& vid, const std::string& joiner_ip, const std::vector & forwarding_ips, const std::vector & neighbor_ips, const std::vector & neighbor_vids, const int32_t hops, const std::string& ip); 61 | void flush_nosplit(const std::string& vid); 62 | void heartbeat(const std::string& ip); 63 | void chain_repair_done(const std::string& key,const std::string& endkey); 64 | 65 | void verifyThrift(VirtualNode* failed); 66 | 67 | void get_ring_state(ringState& _return); 68 | 69 | /* Management Procedures */ 70 | void sendInitJoinResponses(); 71 | VirtualNode** joinRequest(string ip); 72 | void joinIntermediate(string ip); 73 | 74 | NodeHandle* getReplicaGroup(DBID* k); 75 | 76 | /* Util */ 77 | void printNodes(); 78 | 79 | void ageNodes(); 80 | 81 | map ip_node_map; 82 | 83 | private: 84 | 85 | string myIP; 86 | map vid_temptail_map; 87 | map vid_shorter_map; 88 | 89 | pthread_mutex_t phy_nodelist_mutex; 90 | pthread_mutex_t virtual_nodelist_mutex; 91 | 92 | pthread_mutex_t count_mutex; 93 | pthread_cond_t count_threshold_cv; 94 | 95 | int numJoinedPhyNodes; 96 | int numNodesExpected; // Used to evenly distribute ring space for easier testing. 97 | 98 | Node* createNode(string ip, int port); 99 | 100 | void failNode(Node*); 101 | 102 | /* A ring is a list of virtual nodes */ 103 | std::list nodeList; 104 | 105 | /* Joined Physical Nodes */ 106 | std::list phyNodeList; 107 | 108 | /* Data is replicated across a replica group of this size */ 109 | unsigned int replica_group_size; 110 | 111 | void sendJoinRsp(Node* n); 112 | void sendReJoinRsp(Node* n); 113 | // reJoin directly adds node with id = vid to the ring 114 | void staticJoin(const DBID id, Node* n, bool lock = true); 115 | // Add NUM_VIRTUAL_IDS virtual nodes to ring 116 | void joinRing(string ip, int32_t port, Node* n); 117 | 118 | /* Management Helpers */ 119 | /* Add node to ring in ascending order*/ 120 | void addNodetoRing(VirtualNode* n, bool lock = true); 121 | void printRing(); 122 | void removeNodefromRing(Node* n, bool lock = true); 123 | Node** getCCWDescendants(VirtualNode* vn); 124 | 125 | VirtualNode* getPredecessor(DBID* vn, bool b_lock); 126 | DBID* getPredecessorID(DBID* vn, bool b_lock); 127 | VirtualNode* getSuccessor(DBID* vn, bool b_lock); 128 | DBID* getSuccessorID(DBID* vn, bool b_lock); 129 | vector* getNbrhood(DBID* vn); 130 | 131 | VirtualNode* getDistinctPredecessor(DBID* vn); 132 | VirtualNode* getDistinctSuccessor(DBID* vn); 133 | void print_membership_info(); 134 | 135 | DBID* getRangeKey(DBID* key); 136 | void removeVirtualNodefromRing(DBID* id, bool lock=true); 137 | void removePhysicalNode(Node *n); 138 | }; 139 | 140 | #endif 141 | -------------------------------------------------------------------------------- /fawnkv/README: -------------------------------------------------------------------------------- 1 | Ranges 2 | - Ranges are of the form (start_key, end_key] 3 | - Ranges do not include the start_key, if you want to find the interval DB, use the end_key 4 | 5 | Current FAWNKV code assumes: 6 | 1. there is only one front-end 7 | 2. only one virtual node per backend node 8 | 3. front-end does not fail 9 | 4. assumes only one node can join/fail at a time 10 | 11 | These assumptions should to be addressed in fawnkv code before being touted as a usable kv store. 12 | 13 | TODOs 14 | A) Node removal 15 | 1) chain set-up 16 | 2) ACK path 17 | B) Border case code for 18 | 1) Num physical nodes < R 19 | 2) R = 1 for normal join (static join works fine). 20 | 3) When virtual nodes not spaced out enough in ring (two virtual nodes on same physical node are neighbors). 21 | 22 | -------------------------------------------------------------------------------- /fawnkv/cache.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* Cache tester by Jason Franklin (jfrankli@cs.cmu.edu) */ 3 | /* Derived from fe_tester.cpp */ 4 | using namespace std; 5 | 6 | #include 7 | #include 8 | #include 9 | #include "fe.h" 10 | #include "dbid.h" 11 | #include "hashutil.h" 12 | #include "timing.h" 13 | 14 | using fawn::HashUtil; 15 | using fawn::DBID; 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | int total = 0; 23 | 24 | int numGets = 0; 25 | struct timeval start; 26 | 27 | string* value; 28 | bool printedStats; 29 | 30 | void put_cb(unsigned int continuation) 31 | { 32 | #ifdef DEBUG 33 | cout << "Woah! Got to put_cb." << endl; 34 | #endif 35 | 36 | } 37 | 38 | void get_cb(DBID* p_key, string* p_value, unsigned int continuation, bool success, bool cached) 39 | { 40 | #ifdef DEBUG 41 | cout << "Woah! Got to get_cb. " << endl; 42 | #endif 43 | #ifdef DEBUG 44 | cout << "\t Key: " << p_key->actual_data_str() << endl; 45 | 46 | if (p_value) { 47 | cout << "\t Value: " << *p_value << endl; 48 | } 49 | else { 50 | cout << "\t Value: NULL" << endl; 51 | } 52 | 53 | cout << "\t Continuation Id: " << continuation << endl; 54 | 55 | cout << "\t Get Success?: " << (success ? "yes!" : "no") << " Success #: " ; 56 | #endif 57 | 58 | if (p_value->compare(value->c_str())) { 59 | cout << "ERROR: returned value does not match expected value." << endl; 60 | cout << "value is: " << *p_value << endl; 61 | } 62 | 63 | numGets++; 64 | 65 | if (numGets % 100000 == 0) 66 | cout << "Number of successful gets: " << numGets << endl; 67 | 68 | if (((double)numGets) >= ((double)(0.9)*total) && !printedStats) { 69 | 70 | struct timeval end; 71 | gettimeofday(&end, NULL); 72 | double diff = timeval_diff(&start, &end); 73 | cout << "Num Queries: " << total << " time: " << diff << endl; 74 | cout << "Query Rate: " << (total / diff) << " qps" << endl; 75 | printedStats = true; 76 | } 77 | 78 | if (success) { 79 | //if (numGets % 100 == 0) 80 | //cout << "Successful gets: " << ++numGets << endl; 81 | 82 | } else { 83 | cout << "Fail" << endl; 84 | cout << endl; // flush! 85 | } 86 | 87 | if (!cached && p_key) 88 | delete p_key; 89 | if (!cached && p_value) { 90 | delete p_value; 91 | } 92 | 93 | //cout << "exiting get cb" << endl; 94 | } 95 | 96 | 97 | void *localThreadLoop(void *p) 98 | { 99 | FrontEnd* fe = (FrontEnd*) p; 100 | 101 | // TODO: Fix Hack. Just used to keep client around. 102 | while(1) { 103 | fe->log_stats(); 104 | sleep(1); 105 | } 106 | return NULL; 107 | } 108 | 109 | void usage() 110 | { 111 | cerr << "./fe_tester \n" <register_put_cb(&put_cb); 147 | p_fe->register_get_cb(&get_cb); 148 | srand ( time(NULL) ); 149 | struct timespec req; 150 | req.tv_sec = 0; 151 | req.tv_nsec = 50000; 152 | 153 | cout << "Enter 'start' to start performing puts." << endl; 154 | string tempstr; 155 | cin >> tempstr; 156 | 157 | /* Load database */ 158 | for (int i = 0; i < corpus; i++) { 159 | ostringstream s; 160 | s << "key" << i; 161 | string ps_key = s.str(); 162 | 163 | if (i % 1000000 == 0) 164 | cout << "inserting: " << ps_key << endl; 165 | 166 | u_int32_t key_id = HashUtil::MakeHashedKey(ps_key); 167 | DBID* key = new DBID((char *)&key_id, sizeof(u_int32_t)); 168 | p_fe->put(key, value, i); 169 | delete key; 170 | 171 | nanosleep(&req, NULL); 172 | } 173 | 174 | cout << "Enter 'start' to start performing gets." << endl; 175 | cin >> tempstr; 176 | 177 | /* Fetch items at random */ 178 | int num = 0; 179 | 180 | gettimeofday(&start, NULL); 181 | for (int i = 0; i < num_gets; i++) { 182 | 183 | /* Biasing */ 184 | int rand_val = rand() % 100; 185 | double prob_pick_hot_item = ((double) (corpus - bias)) / corpus; 186 | int perc_pick_hot_item = (int) (prob_pick_hot_item * ((double) 100)); 187 | 188 | //cout << "Random value (between 0 and 100): " << rand_val << endl; 189 | //cout << "Probability of picking hot item: " << prob_pick_hot_item << endl; 190 | //cout << "Percentage of picking hot item: " << perc_pick_hot_item << endl; 191 | 192 | if (rand_val <= perc_pick_hot_item) { 193 | num = rand()% bias; 194 | //cout << "Fetching hot item " << num << endl; 195 | } else { 196 | //cout << "Cold item fetch: " << num << endl; 197 | num = rand()%corpus; 198 | if (num <= bias) { 199 | num = ((int)((corpus-bias)*(rand_val/100))) + bias; 200 | } 201 | 202 | } 203 | 204 | ostringstream out2; 205 | out2 << "key" << num; 206 | string ps_key = out2.str(); 207 | 208 | u_int32_t key_id = HashUtil::MakeHashedKey(ps_key); 209 | 210 | DBID* key_p = new DBID((char *)&key_id, sizeof(u_int32_t)); 211 | 212 | if (key_p == NULL) { 213 | cout << "ERROR: Malloc of ID failed. Out of Memory." << endl; 214 | } /* XXX - and yet, we then pass it to get? */ 215 | 216 | p_fe->get(key_p, 2); 217 | 218 | delete key_p; 219 | } 220 | 221 | pthread_join(localThreadId_, NULL); 222 | cout << "Exiting front-end manager." << endl; 223 | 224 | FrontEnd::Destroy(); 225 | return 0; 226 | 227 | } 228 | -------------------------------------------------------------------------------- /fawnkv/cache.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | using namespace std; 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | 10 | // Code stolen from http://www.codeproject.com/KB/applications/cpp_mru_cache.aspx 11 | 12 | /** 13 | * MRU Cache 14 | * 15 | * contains up to a fixed size of "Most Recently Used" items, where items are assiciated with keys. 16 | * when asked to fetch a value that is not in the cache, HandleNonExistingKeyFetch is called. 17 | * when an item is removed from the cache, HandleItemRelease is called. 18 | * implementor of this cache must provide those methods. 19 | * 20 | */ 21 | 22 | class MruCache 23 | { 24 | public: 25 | 26 | const int maxLength; 27 | 28 | MruCache(int iMaxLength) : maxLength(iMaxLength) { 29 | pthread_mutex_init(&cache_mutex, NULL); 30 | } 31 | 32 | string* FetchItem(const string &key) { return __fetch_item(key); } 33 | 34 | void Invalidate(const string &key) { __evict_item(key);} 35 | 36 | void AddItem(const string &key, string* value) { __add_item(key, value);} 37 | 38 | virtual ~MruCache() { 39 | pthread_mutex_destroy(&cache_mutex); 40 | Clear(); 41 | } 42 | 43 | virtual void Clear() { __clear(); } 44 | 45 | 46 | protected: 47 | 48 | virtual void HandleItemRelease(string* key, string* value) { }; 49 | 50 | virtual string* HandleNonExistingKeyFetch(string* key) = 0; 51 | 52 | private: 53 | 54 | typedef struct _Entry 55 | { 56 | string* key; 57 | string* value; 58 | } Entry; 59 | 60 | typedef std::list EntryList; 61 | EntryList listOfEntries; 62 | 63 | /** 64 | * for fast search in the cache. this map contains pointers to iterators in EntryList. 65 | */ 66 | typedef std::map ItrPtrMap; 67 | ItrPtrMap mapOfListIteratorPtr; 68 | 69 | pthread_mutex_t cache_mutex; 70 | 71 | string* __fetch_item(const string &key) 72 | { 73 | pthread_mutex_lock(&cache_mutex); 74 | Entry entry; 75 | EntryList::iterator* ptrItr = (EntryList::iterator*) mapOfListIteratorPtr[key]; 76 | 77 | if (!ptrItr) /* Key Not Found */ 78 | { 79 | return NULL; 80 | } else { 81 | /* Key Found */ 82 | entry = *(*ptrItr); 83 | /* listOfEntries.erase(*ptrItr); */ 84 | /* listOfEntries.push_front(entry); */ 85 | /* *ptrItr = listOfEntries.begin(); */ 86 | } 87 | string* retString = new string (entry.value->c_str()); 88 | 89 | pthread_mutex_unlock(&cache_mutex); 90 | return retString; 91 | } 92 | 93 | 94 | void __evict_item(const string &key) { 95 | pthread_mutex_lock(&cache_mutex); 96 | Entry entry; 97 | EntryList::iterator* ptrItr = (EntryList::iterator*) mapOfListIteratorPtr[key]; 98 | 99 | if (ptrItr) { /* Key Found */ 100 | entry = *(*ptrItr); 101 | HandleItemRelease(entry.key, entry.value); 102 | listOfEntries.erase(*ptrItr); 103 | delete (EntryList::iterator*)mapOfListIteratorPtr[key]; 104 | mapOfListIteratorPtr.erase(key); 105 | } 106 | pthread_mutex_unlock(&cache_mutex); 107 | } 108 | 109 | 110 | void __add_item(const string &key, string* value) { 111 | pthread_mutex_lock(&cache_mutex); 112 | EntryList::iterator* ptrItr = (EntryList::iterator*) mapOfListIteratorPtr[key]; 113 | 114 | //cout << "Adding key. Num entries: " << (int)listOfEntries.size() << endl; 115 | 116 | if (!ptrItr) /* Key Not Found */ 117 | { 118 | if ( (int)listOfEntries.size() >= maxLength) 119 | { 120 | /* Evict Entry */ 121 | Entry lruEntry = listOfEntries.back(); 122 | listOfEntries.pop_back(); 123 | delete (EntryList::iterator*)mapOfListIteratorPtr[*lruEntry.key]; 124 | mapOfListIteratorPtr.erase(*lruEntry.key); 125 | HandleItemRelease(lruEntry.key, lruEntry.value); 126 | } 127 | } else { 128 | 129 | /* Key found, evict old entry */ 130 | //__evict_item(key); 131 | pthread_mutex_unlock(&cache_mutex); 132 | return; 133 | } 134 | 135 | /* Create New Entry */ 136 | Entry entry; 137 | 138 | string* val = new string(value->c_str()); 139 | entry.value = val; 140 | entry.key = new string(key); 141 | 142 | listOfEntries.push_front(entry); 143 | 144 | EntryList::iterator* ptrItrtemp = new EntryList::iterator(); 145 | *ptrItrtemp = listOfEntries.begin(); 146 | mapOfListIteratorPtr[key] = ptrItrtemp; 147 | pthread_mutex_unlock(&cache_mutex); 148 | } 149 | 150 | 151 | virtual void __clear() 152 | { 153 | #ifdef DEBUG 154 | cout << "Clearing" << endl; 155 | #endif 156 | for (ItrPtrMap::iterator i=mapOfListIteratorPtr.begin(); i!=mapOfListIteratorPtr.end(); i++) 157 | { 158 | void* ptrItr = i->second; 159 | 160 | EntryList::iterator* pItr = (EntryList::iterator*) ptrItr; 161 | 162 | 163 | if (pItr != NULL) { /* Check for null iterator before dereferencing */ 164 | HandleItemRelease( (*pItr)->key, (*pItr)->value ); 165 | } 166 | 167 | delete (EntryList::iterator*)ptrItr; 168 | } 169 | 170 | listOfEntries.clear(); 171 | mapOfListIteratorPtr.clear(); 172 | } 173 | }; 174 | 175 | 176 | class FECache : public MruCache 177 | { 178 | public: 179 | FECache(int iMaxLength) : MruCache(iMaxLength) { } 180 | 181 | protected: 182 | virtual void HandleItemRelease(string* k, string* v) 183 | { 184 | if (k) { 185 | delete k; 186 | } 187 | 188 | if (v) { 189 | delete v; 190 | } 191 | } 192 | 193 | virtual string* HandleNonExistingKeyFetch(string* k) 194 | { 195 | #ifdef DEBUG 196 | cout << "[DEBUG] key " << *k << " not found" << endl; 197 | #endif 198 | return NULL; 199 | } 200 | 201 | }; 202 | 203 | 204 | 205 | 206 | 207 | 208 | -------------------------------------------------------------------------------- /fawnkv/fawnkv.thrift: -------------------------------------------------------------------------------- 1 | namespace cpp fawn 2 | namespace java fawn 3 | 4 | service FawnKV { 5 | oneway void get(1: binary key, 3: i64 continuation, 4: i32 cid), 6 | oneway void put(1: binary key, 2: binary value, 3: i64 continuation, 4: i32 cid), 7 | oneway void remove(1: binary key, 3: i64 continuation, 4: i32 cid), 8 | i32 init(1: string ip, 2: i32 port), 9 | } 10 | 11 | service FawnKVApp { 12 | oneway void get_response(1: binary value, 3: i64 continuation), 13 | oneway void put_response(3: i64 continuation), 14 | oneway void remove_response(3: i64 continuation), 15 | } 16 | -------------------------------------------------------------------------------- /fawnkv/fawnkvmesg.thrift: -------------------------------------------------------------------------------- 1 | namespace cpp fawn 2 | 3 | struct repairInfo { 4 | 1: i32 last_put, 5 | 2: i32 last_ack 6 | } 7 | 8 | struct NodeData { 9 | 1: binary VnodeID, 10 | 2: string ip, 11 | 3: i32 port 12 | } 13 | 14 | struct ringState { 15 | 1: list nodes, 16 | 2: i32 replication, 17 | 3: list temptails, 18 | 4: list shorters, 19 | } 20 | 21 | enum flushmode {SPLIT, MERGE, PLAIN} 22 | 23 | enum returnStatus {SUCCESS, NOT_EXIST, STALE_RING} 24 | 25 | service FawnKVBackend { 26 | oneway void static_join_response(1: list VnodeIDs, 16: string ip), 27 | oneway void rejoin_response(1: list VnodeIDs, 16: string ip), 28 | oneway void join_response(1: list VnodeIDs, 2: list StartIDs, 16: string ip), 29 | oneway void init_response(1: list VnodeIDs, 16: string ip), 30 | 31 | oneway void vnode_pre_join_response(1: binary VnodeID, 2: list start_keys, 3: list end_keys, 4: list tail_ips, 5: list tail_ports, 16: string ip), 32 | 33 | oneway void precopy_request(1: binary startKey, 2: binary endKey, 3: binary vid, 16: string ip, 17: i32 port), 34 | oneway void precopy_response(1: binary startKey, 2: binary endKey, 3: binary vid, 16: string ip), 35 | 36 | oneway void flush_split(1: binary startKey, 2: binary endKey, 3: binary vid, 4: string joiner_ip, 5: list forwarding_ips, 6: list neighbor_ips, 7: list neighbor_vids, 8: i32 hops, 9: list forwarding_ports, 10: list neighbor_ports, 16: string ip), 37 | 38 | oneway void flush_merge(1: binary merge_start_key, 2: binary merge_end_key, 3: binary startKey, 4: binary endKey, 6: i32 hops), 39 | oneway void flush(1: binary startKey, 2: binary endKey, 3: i32 hops), 40 | oneway void put(1: binary key, 2: binary value, 3: i32 hops, 4: i32 ackhops, 5: i64 continuation, 6: i64 seq, 7: bool flush, 8: bool remove, 16: string ip), 41 | oneway void put_w(1: binary key, 2: binary value, 3: i32 hops, 4: i32 ackhops, 5: i64 continuation, 6: i64 seq, 7: bool flush, 8: bool remove, 16: string ip), 42 | oneway void put_ack(1: binary key, 2: i64 seq, 3: i32 hops, 16: string ip), 43 | oneway void get(1: binary key, 3: i64 continuation, 16: string ip), 44 | 45 | oneway void neighbor_update(1: binary myvid, 2: list pred_vids, 3: binary succ_vid, 4: string succid, 5: binary pred_vid, 6: string predid, 16: string ip), 46 | 47 | oneway void heartbeat_response(1: string ip), 48 | 49 | oneway void chain_repair_tail(1: binary key, 2: i32 hops, 3: i32 ack_hops, 5: binary sid, 4: binary sip), 50 | oneway void chain_repair_mid(1: binary key, 2: binary my_vid, 3: binary sid, 4: binary ip, 5: i32 hops, 6: i32 ack_hops), 51 | repairInfo chain_repair_mid_succ(1: binary key, 2: binary p_vid, 3: binary p_ip), 52 | 53 | oneway void chain_repair_single(1:binary new_start, 2: binary end_id), 54 | oneway void vnode_extend_chain( 1: binary vid,2: binary nid, 3: binary start_key, 4: binary end_key, 5: binary ip, 6: i32 port, 7: i32 mode), 55 | oneway void integrity(1: i32 hops), 56 | 57 | oneway void init(1: string ip, 2: i32 port), 58 | } 59 | 60 | 61 | service FawnKVFrontend { 62 | oneway void put_response(1: binary key, 3: i64 continuation, 4: bool success, 16: string ip), 63 | oneway void put_w_response(1: binary key, 3: i64 continuation, 4: bool success, 5: i64 version, 16: string ip), 64 | oneway void get_response(1: binary key, 2: binary value, 3: i64 continuation, 4: i16 status, 16: string ip), 65 | oneway void remove_response(1: binary key, 3: i64 continuation, 4: bool success, 16: string ip), 66 | } 67 | 68 | service FawnKVManager { 69 | oneway void static_join(16: string ip, 17: i32 port), 70 | oneway void rejoin(1: list vnodeids, 2: list startids, 16: string ip, 17: i32 port), 71 | oneway void join(16: string ip, 17: i32 port, 18: bool merge), 72 | 73 | oneway void vnode_pre_join(1: binary vid, 16: string ip, 17: i32 port), 74 | oneway void vnode_join(1: binary vid, 16: string ip, 17: i32 port, 18: i32 merge), 75 | 76 | oneway void flush_split(1: binary start_id, 2: binary end_id, 3: binary vid, 4: string joiner_ip, 5: list forwarding_ips, 6: list neighbor_ips, 7: list neighbor_vids, 8: i32 hops, 16: string ip), 77 | oneway void flush_nosplit(3: binary vid), 78 | 79 | oneway void chain_repair_done(1: binary key,3:binary endkey), 80 | oneway void heartbeat(1: string ip), 81 | 82 | ringState get_ring_state(), 83 | 84 | } -------------------------------------------------------------------------------- /fawnkv/fe.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "fe.h" 3 | 4 | FrontEnd::FrontEnd(string managerIP, string myIP, int myPort) : myIP(myIP), myPort(myPort) 5 | { 6 | shared_ptr socket(new TSocket(managerIP.data(), MANAGER_PORT_BASE)); 7 | shared_ptr transport(new TBufferedTransport(socket)); 8 | manager_transport = transport; 9 | shared_ptr protocol(new TBinaryProtocol(manager_transport)); 10 | manager = new FawnKVManagerClient(protocol); 11 | 12 | while (1) { 13 | try { 14 | manager_transport->open(); 15 | break; 16 | } catch (TException &tx) { 17 | fprintf(stderr, "Transport error: %s\n", tx.what()); 18 | } 19 | sleep(3); 20 | } 21 | 22 | ring = new Ring(); 23 | UpdateRingState(); 24 | 25 | pthread_t localServerThreadId_; 26 | int code = pthread_create(&localServerThreadId_, NULL, 27 | localServerThreadLoop, this); 28 | } 29 | 30 | FrontEnd::~FrontEnd() 31 | { 32 | try { 33 | manager_transport->close(); 34 | map::iterator it; 35 | for (it = ip_client_map.begin(); it != ip_client_map.end(); it++) { 36 | FawnKVBackendClient* backend = (*it).second; 37 | delete backend; 38 | } 39 | } catch (TException &tx) { 40 | fprintf(stderr, "Transport error: %s\n", tx.what()); 41 | } 42 | delete manager; 43 | manager = NULL; 44 | } 45 | 46 | void FrontEnd::UpdateRingState() 47 | { 48 | // Get Ring State and connect to backends 49 | ringState rs; 50 | manager->get_ring_state(rs); 51 | for (uint i = 0; i < rs.nodes.size(); i++) { 52 | NodeData n = rs.nodes[i]; 53 | if (getID(myIP, myPort) != getID(n.ip, n.port)) { 54 | assert(connect_to_backend(getID(n.ip, n.port)) != NULL); 55 | } 56 | } 57 | ring->UpdateState(rs); 58 | } 59 | 60 | int FrontEnd::put(const std::string& key, const std::string& value, const int64_t continuation) 61 | { 62 | NodeHandle *h = ring->getReplicaGroup(key); 63 | pair p = h->retrievePutTarget(); 64 | Node* n = p.first; 65 | int i = p.second; 66 | FawnKVBackendClient* backend = ip_client_map[getID(n->IP, n->port)]; 67 | uint32_t hops = h->replicaGroup.size() + 1 - i; 68 | backend->put(key, value, hops, 69 | hops, continuation, 0, 70 | false, false, getID(myIP, myPort)); 71 | delete h; 72 | return 0; 73 | } 74 | 75 | int FrontEnd::put_w(const std::string& key, const std::string& value, const int64_t continuation) 76 | { 77 | NodeHandle *h = ring->getReplicaGroup(key); 78 | pair p = h->retrievePutTarget(); 79 | Node* n = p.first; 80 | int i = p.second; 81 | FawnKVBackendClient* backend = ip_client_map[getID(n->IP, n->port)]; 82 | uint32_t hops = h->replicaGroup.size() + 1 - i; 83 | backend->put_w(key, value, hops, 84 | hops, continuation, 0, 85 | false, false, getID(myIP, myPort)); 86 | return 0; 87 | } 88 | 89 | int FrontEnd::remove(const std::string& key, const int64_t continuation) 90 | { 91 | NodeHandle *h = ring->getReplicaGroup(key); 92 | pair p = h->retrievePutTarget(); 93 | Node* n = p.first; 94 | int i = p.second; 95 | FawnKVBackendClient* backend = ip_client_map[getID(n->IP, n->port)]; 96 | uint32_t hops = h->replicaGroup.size() + 1 - i; 97 | string value; 98 | backend->put(key, value, hops, 99 | hops, continuation, 0, 100 | false, true, getID(myIP, myPort)); 101 | delete h; 102 | return 0; 103 | } 104 | 105 | int FrontEnd::get(const std::string& key, const int64_t continuation) 106 | { 107 | NodeHandle *h = ring->getReplicaGroup(key); 108 | DBID* vidb = ring->getSuccessorID(key); 109 | string vid(vidb->data(), vidb->size()); 110 | string rvid = ring->getRangeKey(key); 111 | bool interim_tail = false; 112 | if(ring->vid_shorter_map.find(rvid) != ring->vid_shorter_map.end()) { 113 | interim_tail = ring->vid_temptail_map[vid] || ring->vid_shorter_map[rvid]; 114 | } else { 115 | interim_tail = ring->vid_temptail_map[vid]; 116 | } 117 | Node* n = h->retrieveGetTarget(interim_tail); 118 | string nodeID = getID(n->IP, n->port); 119 | FawnKVBackendClient* backend = ip_client_map[nodeID]; 120 | backend->get(key, continuation, getID(myIP, myPort)); 121 | delete h; 122 | return 0; 123 | } 124 | 125 | 126 | void FrontEnd::register_put_cb(void (*cb)(unsigned int)) { 127 | put_cb = cb; 128 | } 129 | 130 | void FrontEnd::register_put_w_cb(void (*cb)(unsigned int, int64_t)) { 131 | put_w_cb = cb; 132 | } 133 | 134 | void FrontEnd::register_get_cb(void (*cb)(const DBID&, const string&, unsigned int, bool)) { 135 | get_cb = cb; 136 | } 137 | 138 | void FrontEnd::put_response(const std::string& key, const int64_t continuation, const bool success, const std::string& ip) { 139 | // Your implementation goes here 140 | if (NULL != put_cb) { 141 | put_cb(continuation); 142 | } 143 | } 144 | 145 | void FrontEnd::put_w_response(const std::string& key, const int64_t continuation, const bool success, const int64_t version, const std::string& ip) { 146 | // Your implementation goes here 147 | if (NULL != put_cb) { 148 | put_w_cb(continuation, version); 149 | } 150 | } 151 | 152 | void FrontEnd::get_response(const std::string& key, const std::string& value, const int64_t continuation, const int16_t status, const std::string& ip) { 153 | if (status == returnStatus::STALE_RING) { 154 | // Update ring and try again 155 | UpdateRingState(); 156 | get(key, continuation); 157 | return; 158 | } 159 | 160 | if (NULL != get_cb) { 161 | DBID dkey(key); 162 | bool success = (status == returnStatus::SUCCESS); 163 | get_cb(dkey, value, continuation, success); 164 | } 165 | } 166 | 167 | void FrontEnd::remove_response(const std::string& key, const int64_t continuation, const bool success, const std::string& ip) { 168 | // Your implementation goes here 169 | printf("remove_response\n"); 170 | if (NULL != put_cb) { 171 | put_cb(continuation); 172 | } 173 | } 174 | 175 | 176 | FawnKVBackendClient* FrontEnd::connect_to_backend(const string& nip) 177 | { 178 | cout << "connecting to " << nip << endl; 179 | FawnKVBackendClient* backend = NULL; 180 | string ip = getIP(nip); 181 | int32_t port = getPort(nip); 182 | map::iterator it = ip_client_map.find(getID(ip, port)); 183 | if (it == ip_client_map.end()) { 184 | backend = connectTCP(ip, port); 185 | // assert(sock != -1); 186 | ip_client_map[nip] = backend; 187 | 188 | // Create lock for socket for persistent client connections 189 | pthread_mutex_t *client_lock = (pthread_mutex_t*)malloc(sizeof(pthread_mutex_t)); 190 | pthread_mutex_init(client_lock, NULL); 191 | ip_socketlock_map[nip] = client_lock; 192 | 193 | //cout << "my_id " << bytes_to_hex(vid) << " connected to vid " << bytes_to_hex(nid) << " @IP " << nip << endl; 194 | } else { 195 | // This might occur if this IP is a neighbor for another vid, so don't close it! 196 | //cout << "Connection already established " << endl; 197 | backend = ip_client_map[nip]; 198 | DPRINTF(DEBUG_FLOW, "Connection to %s already established\n", nip.c_str()); 199 | } 200 | 201 | return backend; 202 | } 203 | 204 | FawnKVBackendClient* FrontEnd::connectTCP(string IP, int port) 205 | { 206 | shared_ptr socket(new TSocket(IP.data(), port)); 207 | shared_ptr transport(new TBufferedTransport(socket)); 208 | shared_ptr protocol(new TBinaryProtocol(transport)); 209 | FawnKVBackendClient* backend = new FawnKVBackendClient(protocol); 210 | 211 | try { 212 | transport->open(); 213 | } catch (TException &tx) { 214 | fprintf(stderr, "Transport error: %s\n", tx.what()); 215 | } 216 | 217 | backend->init(myIP, myPort); 218 | return backend; 219 | 220 | 221 | } 222 | 223 | void *localServerThreadLoop(void *p) 224 | { 225 | FrontEnd *fe = (FrontEnd*) p; 226 | 227 | shared_ptr handler(fe); 228 | shared_ptr processor(new FawnKVFrontendProcessor(handler)); 229 | shared_ptr serverTransport(new TServerSocket(fe->myPort)); 230 | shared_ptr transportFactory(new TBufferedTransportFactory()); 231 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 232 | 233 | TThreadedServer server(processor, serverTransport, transportFactory, protocolFactory); 234 | server.serve(); 235 | return NULL; 236 | } 237 | -------------------------------------------------------------------------------- /fawnkv/fe.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _FE_H_ 3 | #define _FE_H_ 4 | 5 | #include 6 | #include 7 | #include "dbid.h" 8 | #include "ring.h" 9 | #include "FawnKVFrontend.h" 10 | #include "FawnKVManager.h" 11 | 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | using namespace ::apache::thrift; 19 | using namespace ::apache::thrift::protocol; 20 | using namespace ::apache::thrift::transport; 21 | using namespace ::apache::thrift::server; 22 | 23 | using boost::shared_ptr; 24 | using namespace std; 25 | using namespace fawn; 26 | 27 | using fawn::DBID; 28 | 29 | #define MANAGER_PORT_BASE 4000 30 | 31 | void *localServerThreadLoop(void *p); 32 | 33 | class FrontEnd : virtual public FawnKVFrontendIf 34 | { 35 | private: 36 | string myIP; 37 | Ring *ring; 38 | 39 | void (*put_cb)(unsigned int); // continuation 40 | void (*put_w_cb)(unsigned int, int64_t); // continuation, version 41 | void (*get_cb)(const DBID&, const string&, unsigned int, bool); // key, val, continuation, success 42 | 43 | shared_ptr manager_transport; 44 | map ip_client_map; 45 | map ip_socketlock_map; 46 | 47 | void UpdateRingState(); 48 | 49 | // Copied from node_mgr; should unify in "connection manager" 50 | FawnKVBackendClient* connect_to_backend(const string& nip); 51 | FawnKVBackendClient* connectTCP(string IP, int port); 52 | 53 | 54 | public: 55 | FawnKVManagerClient *manager; 56 | 57 | explicit FrontEnd(string managerIP, string myIP, int myPort); 58 | virtual ~FrontEnd(); 59 | 60 | int myPort; 61 | 62 | 63 | /* Client Callback registration */ 64 | void register_put_cb(void (*cb)(unsigned int)); 65 | void register_put_w_cb(void (*cb)(unsigned int, int64_t)); 66 | void register_get_cb(void (*cb)(const DBID&, const string&, unsigned int, bool)); 67 | 68 | int put(const std::string& key, const std::string& value, const int64_t continuation); 69 | int put_w(const std::string& key, const std::string& value, const int64_t continuation); 70 | int remove(const std::string& key, const int64_t continuation); 71 | int get(const std::string& key, const int64_t continuation); 72 | 73 | void put_response(const std::string& key, const int64_t continuation, const bool success, const std::string& ip); 74 | void put_w_response(const std::string& key, const int64_t continuation, const bool success, const int64_t version, const std::string& ip); 75 | void get_response(const std::string& key, const std::string& value, const int64_t continuation, const int16_t status, const std::string& ip); 76 | void remove_response(const std::string& key, const int64_t continuation, const bool success, const std::string& ip); 77 | 78 | }; 79 | 80 | #endif 81 | -------------------------------------------------------------------------------- /fawnkv/fe_cache.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "fe_cache.h" 14 | 15 | using namespace ::apache::thrift; 16 | using namespace ::apache::thrift::protocol; 17 | using namespace ::apache::thrift::transport; 18 | using namespace ::apache::thrift::server; 19 | 20 | using boost::shared_ptr; 21 | 22 | using namespace fawn; 23 | 24 | bool DEBUG = false; 25 | 26 | Cache::Cache(int m) { 27 | old = new CacheTable(); 28 | cur = new CacheTable(); 29 | sizeOld = 0; 30 | sizeCur = 0; 31 | sizeMax = m; 32 | num_hits = 0; 33 | num_lookups = 0; 34 | cout << "The cache is able to cache up to "<< sizeMax << " entries" << endl; 35 | } 36 | 37 | bool Cache::lookup(const DBID& dkey, std::string& value) { 38 | bool found = false; 39 | CacheTable::const_accessor a; 40 | num_lookups ++; 41 | 42 | if (cur->find(a, dkey.actual_data_str())) { 43 | value = a->second; 44 | num_hits ++; 45 | //if (DEBUG) 46 | // cout << "found " << dkey.actual_data_str() << " in cur:" << value << endl; 47 | found = true; 48 | } 49 | else if (old->find(a, dkey.actual_data_str())) { 50 | value = a->second; 51 | num_hits ++; 52 | //if (DEBUG) 53 | // cout << "found " << dkey.actual_data_str() << " in old:" << value << endl; 54 | //insert(dkey, value); 55 | //if (DEBUG) 56 | // cout << "move " << dkey.actual_data_str() << " to cur" << endl; 57 | found = true; 58 | } 59 | 60 | if (DEBUG) { 61 | if (found) 62 | cout << "cache hit" << endl; 63 | else 64 | cout << "cache miss" << endl; 65 | cout << "total lookup=" << num_lookups << ",total hit=" << num_hits << endl; 66 | } 67 | return found; 68 | } 69 | 70 | void Cache::insert(const DBID& dkey, const std::string& value) { 71 | CacheTable *tmp; 72 | CacheTable::accessor a; 73 | if (sizeCur >= sizeMax/2) { 74 | sizeOld = sizeCur; 75 | old->clear(); 76 | tmp = old; 77 | old = cur; 78 | cur = tmp; 79 | sizeCur = 0; 80 | //if (DEBUG) 81 | // cout << "abandon old, point cur to old, now size =" << sizeCur << "/" << sizeOld << endl;; 82 | } 83 | cur->insert(a, dkey.actual_data_str()); 84 | a->second = value; 85 | a.release(); 86 | sizeCur ++; 87 | if (DEBUG) 88 | cout << "insert to cur, now size =" << sizeCur << "/" << sizeOld << endl; 89 | } 90 | -------------------------------------------------------------------------------- /fawnkv/fe_cache.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _FECACHE_H_ 3 | #define _FECACHE_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | #include "ring.h" 20 | #include "FawnKV.h" 21 | #include "FawnKVApp.h" 22 | 23 | using namespace ::apache::thrift; 24 | using namespace ::apache::thrift::protocol; 25 | using namespace ::apache::thrift::transport; 26 | using namespace ::apache::thrift::server; 27 | using namespace tbb; 28 | 29 | using boost::shared_ptr; 30 | 31 | using namespace fawn; 32 | 33 | typedef concurrent_hash_map CacheTable; 34 | 35 | class Cache { 36 | private: 37 | CacheTable *old; 38 | CacheTable *cur; 39 | atomic sizeCur; 40 | atomic sizeOld; 41 | atomic num_hits; 42 | atomic num_lookups; 43 | 44 | public: 45 | Cache(int m); 46 | bool lookup(const DBID& dkey, std::string& value); 47 | void insert(const DBID& dkey, const std::string& value); 48 | int sizeMax; 49 | int size() { 50 | return sizeOld + sizeCur; 51 | } 52 | double hit_ratio() { 53 | return 1.0 * num_hits / num_lookups; 54 | } 55 | double load_factor() { 56 | return 1.0 * (sizeOld + sizeCur) / sizeMax; 57 | } 58 | }; 59 | 60 | #endif 61 | -------------------------------------------------------------------------------- /fawnkv/node.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | 19 | #include "dbid.h" 20 | #include "node.h" 21 | #include "ring.h" 22 | 23 | using fawn::DBID; 24 | using namespace apache::thrift; 25 | using namespace apache::thrift::protocol; 26 | using namespace apache::thrift::transport; 27 | using namespace apache::thrift::server; 28 | 29 | using namespace boost; 30 | 31 | Node::Node() : 32 | IP(""), 33 | connectStatus(0), cardiacState(ALIVE), 34 | joinState(NONE), beatsMissed(0) 35 | { 36 | valid =true; 37 | } 38 | 39 | Node::Node(string ip, int p, bool connect) : 40 | connectStatus(0), cardiacState(ALIVE), 41 | joinState(NONE), beatsMissed(0) 42 | { 43 | IP = ip; 44 | port = p; 45 | valid =true; 46 | connected = connect; 47 | 48 | if (connect) { 49 | // Connect to ip 50 | shared_ptr socket(new TSocket(IP.data(), port)); 51 | shared_ptr transport(new TBufferedTransport(socket)); 52 | be_transport = transport; 53 | shared_ptr protocol(new TBinaryProtocol(be_transport)); 54 | backend = new FawnKVBackendClient(protocol); 55 | 56 | while (1) { 57 | try { 58 | be_transport->open(); 59 | break; 60 | } catch (TException &tx) { 61 | fprintf(stderr, "Transport error: %s\n", tx.what()); 62 | } 63 | cout << "Sleeping" << endl; 64 | sleep(1); 65 | } 66 | } 67 | 68 | } 69 | 70 | Node::~Node() { 71 | cout << "deleting node" << endl; 72 | 73 | if (connected) { 74 | try { 75 | be_transport->close(); 76 | } catch (TException &tx) { 77 | fprintf(stderr, "Transport error: %s\n", tx.what()); 78 | } 79 | 80 | delete backend; 81 | } 82 | } 83 | 84 | void Node::addVnode(const DBID id) { 85 | DBID vid = id; 86 | if(vnodes.empty()) { 87 | vnodes.push_front(vid); 88 | return; 89 | } 90 | 91 | list::iterator i; 92 | 93 | // Add node to list in ascending order by ID 94 | for (i = vnodes.begin(); i != vnodes.end(); i++) { 95 | if ( vid < *i) { 96 | vnodes.insert(i, vid); 97 | return; 98 | } else if ( *i == vid ) { 99 | cerr << "Ring::addNodetoRing failed, ID already exists"< 6 | #include 7 | #include 8 | #include 9 | 10 | #include "dbid.h" 11 | #include "FawnKVFrontend.h" 12 | #include "FawnKVBackend.h" 13 | 14 | using namespace fawn; 15 | using namespace boost; 16 | using namespace apache::thrift::transport; 17 | 18 | enum health_status {ALIVE, ARRHYTHMIA, DEAD}; 19 | enum join_type {NONE, STATIC, REQUEST, PUT, PUTGET}; 20 | 21 | class Node 22 | { 23 | public: 24 | FawnKVBackendClient *backend; 25 | 26 | //used for marking a node has failed without removing it from the phyNodeList, just yet 27 | bool valid; 28 | string IP; 29 | int port; 30 | 31 | std::list vnodes; 32 | 33 | Node (); 34 | Node (string ip, int p, bool connect = true); 35 | ~Node(); 36 | 37 | void addVnode(const DBID id); 38 | 39 | void increment_beats_missed(); 40 | int get_beats_missed(); 41 | void setAlive(); 42 | void setDead(); 43 | //int connectToNode(); 44 | void timerCallback() {}; 45 | void print(); 46 | char isConnected(); 47 | 48 | /* Print management info. */ 49 | void printStatus(); 50 | 51 | bool operator==(const Node &rhs) const; 52 | bool operator<(const Node &rhs) const; 53 | 54 | private: 55 | char connectStatus; 56 | health_status cardiacState; 57 | join_type joinState; 58 | int beatsMissed; 59 | shared_ptr be_transport; 60 | bool connected; 61 | }; 62 | 63 | #endif 64 | 65 | -------------------------------------------------------------------------------- /fawnkv/nodehandle.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | using namespace std; 3 | #include 4 | #include 5 | #include "dbid.h" 6 | #include "node.h" 7 | #include "nodehandle.h" 8 | #include "print.h" 9 | using fawn::DBID; 10 | 11 | NodeHandle::NodeHandle(string myID) : myID(myID) 12 | { 13 | } 14 | 15 | pair NodeHandle::retrievePutTarget() { 16 | Node* target = owner; 17 | #ifdef DEBUG 18 | cout << "Owner is "; target->print(); 19 | cout << "valid is " << (target->valid ? "true" : "false") << endl; 20 | #endif 21 | uint i = 0; 22 | while(!target->valid && i < replicaGroup.size()) 23 | { 24 | target = replicaGroup[i]; 25 | i++; 26 | #ifdef DEBUG 27 | cout << "Target is "; target->print(); 28 | cout << "valid is " << (target->valid ? "true" : "false") << endl; 29 | #endif 30 | } 31 | if ( (i != 0) && (i == replicaGroup.size()) ) 32 | target = NULL; 33 | 34 | return pair(target, i); 35 | } 36 | 37 | Node* NodeHandle::retrieveGetTarget(bool interimTail) { 38 | Node* target = NULL; 39 | //cout << "interim tail = " << (interimTail ? "true" : "false") << endl; 40 | if (interimTail && replicaGroup.back()->valid == false) { 41 | if (replicaGroup.size() >= 2) { 42 | target = replicaGroup[replicaGroup.size()-2]; 43 | int i = replicaGroup.size()-2; 44 | while(!target->valid && i >= 0) 45 | { 46 | target = replicaGroup[i]; 47 | i--; 48 | } 49 | } else if (replicaGroup.size() == 1) { 50 | target = replicaGroup[0]; 51 | if (!target->valid) { 52 | target = owner; 53 | } 54 | } else { // replicaGroup.size() == 0 55 | target = owner; // R = 1, there is no interim tail to worry about. 56 | } 57 | } else if (replicaGroup.size() > 0) { 58 | target = replicaGroup.back(); 59 | int i = replicaGroup.size()-1; 60 | while(!target->valid && i >= 0) 61 | { 62 | target = replicaGroup[i]; 63 | i--; 64 | } 65 | //cout << "valid flag is " << target->valid << endl; 66 | //cout << "i is " << i << endl; 67 | } else { 68 | target = owner; 69 | } 70 | 71 | //cout << "Target is "; target->print(); 72 | 73 | return target; 74 | } 75 | 76 | 77 | int NodeHandle::put(DBID* key, string* value, uint64_t continuation, bool remove, uint32_t sequence, bool interimTail) 78 | { 79 | if (owner->isConnected()) { 80 | cerr << "Cannot connect to replica group owner in put\n" << endl; 81 | exit(1); 82 | } 83 | #ifdef DEBUG 84 | cout << "Sending Put Request" << endl; 85 | cout << "IP: " << owner->IP << endl; 86 | cout << "IPsize: " << owner->IP.size() << endl; 87 | cout << "Keysize: " << key->size() << endl; 88 | cout << "Key: "; 89 | key->printValue(); 90 | cout << endl; 91 | cout << "Valuesize: " << value->size() << endl; 92 | //cout << "Value: " << value->c_str() << endl; 93 | #endif 94 | pair p = retrievePutTarget(); 95 | Node* toSend = p.first; 96 | int i = p.second; 97 | assert (toSend != NULL); 98 | 99 | //if we're operating at one below capacity (node leave, for example 100 | // int delta = (interimTail ? 1 : 0); 101 | 102 | string skey(key->data(), key->size()); 103 | string svalue(value->data(), value->size()); 104 | try { 105 | //cout << "Putting " << bytes_to_hex(skey) << ", seq# " << sequence << ", continuation " << continuation << " to " << toSend->IP << ":" << toSend->port << endl; 106 | 107 | toSend->backend->put(skey, svalue, replicaGroup.size() + 1 - i , replicaGroup.size() +1 - i , continuation, sequence, false, remove, myID); 108 | } 109 | catch(...) { 110 | std::cout << "Put Failed" << endl; 111 | return -1; 112 | } 113 | return 0; 114 | } 115 | 116 | /* Sends request to tail node */ 117 | int NodeHandle::get(DBID* key, uint64_t continuation, bool interimTail) 118 | { 119 | Node* target = retrieveGetTarget(interimTail); 120 | assert(target != NULL); 121 | 122 | if (target->isConnected() ) { 123 | cerr << "Cannot connect to replica group owner in put\n" << endl; 124 | exit(1); 125 | } 126 | #ifdef DEBUG 127 | cout << "Sending Get Request" << endl 128 | << " IP: " << target->IP << endl 129 | << " IPsize: " << target->IP.size() << endl 130 | << " Keysize: " << key->size() << endl 131 | << " Key: "; 132 | key->printValue(); 133 | cout << endl; 134 | #endif 135 | 136 | string skey(key->data(), key->size()); 137 | try { 138 | //cout << "Getting " << bytes_to_hex(skey) << " from " << target->IP << ":" << target->port << endl; 139 | target->backend->get(skey, continuation, myID); 140 | } 141 | catch(...) { 142 | cout << "Failed get" << endl; 143 | return -1; 144 | } 145 | 146 | 147 | return 0; 148 | } 149 | 150 | void NodeHandle::setOwner(Node* n) 151 | { 152 | owner = n; 153 | } 154 | 155 | 156 | Node* NodeHandle::getOwner() 157 | { 158 | return owner; 159 | } 160 | 161 | int NodeHandle::addReplica(Node* n) 162 | { 163 | /* Owner must be first, then replicas in clockwise order */ 164 | /* XXX clockwise or counterclockwise??? */ 165 | replicaGroup.push_back(n); 166 | return 0; 167 | } 168 | 169 | void NodeHandle::print() { 170 | for_each(replicaGroup.begin(), replicaGroup.end(), mem_fun(&Node::print)); 171 | } 172 | -------------------------------------------------------------------------------- /fawnkv/nodehandle.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _NODEHANDLE_H_ 3 | #define _NODEHANDLE_H_ 4 | 5 | using namespace std; 6 | #include 7 | #include "dbid.h" 8 | #include "node.h" 9 | #include 10 | #include 11 | using fawn::DBID; 12 | 13 | class NodeHandle 14 | { 15 | 16 | 17 | 18 | public: 19 | 20 | 21 | NodeHandle(string myID = ""); 22 | 23 | int put(DBID* key, string* value, uint64_t continuation, bool remove, uint32_t sequence=0, bool interimTail=false); 24 | int get(DBID* key, uint64_t continuation, bool interimTail); 25 | 26 | pair retrievePutTarget(); 27 | int retrievePutTargetIndex(); 28 | Node* retrieveGetTarget(bool interimTail); 29 | 30 | void setOwner(Node* n); 31 | int addReplica(Node *n); 32 | 33 | void print(); 34 | 35 | //added by jhferris for the leave protocol 36 | DBID* getRangeKey(); 37 | 38 | Node* getOwner(); 39 | std::deque replicaGroup; 40 | private: 41 | Node* owner; 42 | string myID; 43 | 44 | }; 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /fawnkv/ring.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "print.h" 12 | #include "ring.h" 13 | 14 | using namespace std; 15 | 16 | using fawn::DBID; 17 | 18 | Ring::Ring() { 19 | 20 | } 21 | 22 | Ring::~Ring() { 23 | 24 | } 25 | 26 | void Ring::UpdateState(ringState rs) 27 | { 28 | // Naive implementation throws everything out and 29 | // starts from scratch. 30 | list::iterator vnit; 31 | for (vnit = nodeList.begin(); vnit != nodeList.end(); vnit++) { 32 | delete *vnit; 33 | } 34 | nodeList.clear(); 35 | vid_temptail_map.clear(); 36 | vid_shorter_map.clear(); 37 | 38 | // Assumption: rs node list is in increasing order 39 | for (uint i = 0; i < rs.nodes.size(); i++) { 40 | NodeData nd = rs.nodes[i]; 41 | string nodeID = getID(rs.nodes[i].ip, rs.nodes[i].port); 42 | Node *n = createNode(rs.nodes[i].ip, rs.nodes[i].port); 43 | DBID id(rs.nodes[i].VnodeID); 44 | VirtualNode *vn = new VirtualNode(n, &id); 45 | nodeList.push_back(vn); 46 | } 47 | 48 | replica_group_size = rs.replication; 49 | 50 | for (uint i = 0; i < rs.temptails.size(); i++) { 51 | vid_temptail_map[rs.temptails[i]] = true; 52 | } 53 | 54 | for (uint i = 0; i < rs.shorters.size(); i++) { 55 | vid_shorter_map[rs.shorters[i]] = true; 56 | } 57 | 58 | return; 59 | } 60 | 61 | string Ring::getRangeKey(string key) { 62 | DBID k(key); 63 | list::const_iterator i; 64 | if (!nodeList.empty()) { 65 | for (i = nodeList.begin(); i != nodeList.end(); i++) { 66 | DBID* p_id = (*i)->id; 67 | if ( (k < *p_id) || (k == *p_id) ) { 68 | return p_id->actual_data_str(); 69 | } 70 | } 71 | 72 | /* If we didn't find owner in previous loop */ 73 | /* then the owner is the first node*/ 74 | if (i == nodeList.end()) { 75 | i = nodeList.begin(); 76 | return (*i)->id->actual_data_str(); 77 | } 78 | } 79 | return ""; 80 | } 81 | 82 | DBID* Ring::getSuccessorID(string key) 83 | { 84 | DBID vn(key); 85 | VirtualNode* p_succ = NULL; 86 | 87 | list::const_iterator i, j; 88 | 89 | // nodelist in ascending order by ID 90 | for (i = nodeList.begin(); i != nodeList.end(); i++) { 91 | j = i; 92 | j++; 93 | 94 | if (j == nodeList.end()) { 95 | j = nodeList.begin(); 96 | } 97 | 98 | if ( *((*i)->id) == vn ) { 99 | p_succ = *j; 100 | break; 101 | } 102 | 103 | } 104 | 105 | if (p_succ == NULL) { 106 | for (i = nodeList.begin(); i != nodeList.end(); i++) { 107 | j = i; 108 | j++; 109 | if (j == nodeList.end()) { 110 | j = nodeList.begin(); 111 | } 112 | 113 | // If vn is not in the nodeList, we need to check ranges. 114 | if (between((*i)->id, (*j)->id, &vn)) { 115 | p_succ = *j; 116 | break; 117 | } 118 | } 119 | } 120 | 121 | return p_succ->id; 122 | } 123 | 124 | NodeHandle* Ring::getReplicaGroup(string key) { 125 | #ifdef DEBUG 126 | cout << "[GRG] Looking for key: " << bytes_to_hex(k->actual_data_str()) << endl; 127 | #endif 128 | 129 | DBID k(key); 130 | NodeHandle *nh = NULL; 131 | 132 | if (!nodeList.empty()) { 133 | std::list replicas; 134 | list::const_iterator i; 135 | list::iterator r_iter; 136 | nh = new NodeHandle(); 137 | Node *owner = NULL; 138 | unsigned int num_replicas = 0; 139 | 140 | for (i = nodeList.begin(); i != nodeList.end(); i++) { 141 | DBID* p_id = (*i)->id; 142 | if ( (k < *p_id) || (k == *p_id) ) { 143 | replicas.push_back((*i)->n); 144 | // (*i)->n->print(); 145 | owner = (*i)->n; 146 | num_replicas++; 147 | #ifdef DEBUG 148 | cout << "[GRG] Owner VID is " << bytes_to_hex((*i)->id->actual_data_str()) << endl; 149 | cout << "Replica handling this request: " << (*i)->n->IP << endl; 150 | #endif 151 | //(*i)->id->printValue(); 152 | break; 153 | } 154 | } 155 | #ifdef DEBUG 156 | cout << "Owner is first node" << endl; 157 | #endif 158 | /* If we didn't find owner in previous loop */ 159 | /* then the owner is the first node*/ 160 | if (i == nodeList.end() && num_replicas == 0) { 161 | i = nodeList.begin(); 162 | replicas.push_back((*i)->n); 163 | 164 | // (*i)->n->print(); 165 | owner = (*i)->n; 166 | num_replicas++; 167 | 168 | #ifdef DEBUG 169 | cout << "[GRG] Owner VID is " << bytes_to_hex((*i)->id->actual_data_str()) << endl; 170 | cout << "Replica handling this request: " << (*i)->n->IP << ":" << owner->port << 171 | " (" << bytes_to_hex((*i)->id->actual_data_str()) << ")" << endl; 172 | #endif 173 | } 174 | 175 | nh->setOwner(owner); 176 | #ifdef DEBUG 177 | cout << "\t owner - " << owner->IP << ":" << owner->port << endl; 178 | #endif 179 | 180 | 181 | // Invariant: i points to owner and replicas includes owner at head 182 | 183 | uint uniqueReplica = 1; 184 | 185 | // Increment to move past owner 186 | i++; 187 | // Wrap around 188 | if (i == nodeList.end()) { 189 | i = nodeList.begin(); 190 | } 191 | 192 | int num_items = nodeList.size(); 193 | int num_examined = 1; 194 | // Clockwise search from 0 through ring to find replicas 195 | while ( ((num_items - num_examined) > 0) && (num_replicas < replica_group_size)) { 196 | 197 | if (!((*i)->n == owner)) { 198 | #ifdef DEBUG 199 | cout << "\t examining - " << (*i)->n->IP << ":" << (*i)->n->port << " --> vnode id: " << 200 | bytes_to_hex((*i)->id->actual_data_str()) << endl; 201 | #endif 202 | 203 | uniqueReplica = 1; 204 | for (r_iter = replicas.begin(); r_iter != replicas.end(); r_iter++) { 205 | if (( *((*i)->n) == *(*r_iter) ) ) { 206 | uniqueReplica = 0; 207 | } 208 | } 209 | 210 | // Node is not already in replics, add it 211 | if (uniqueReplica) { 212 | replicas.push_back((*i)->n); 213 | // (*i)->n->print(); 214 | num_replicas++; 215 | } 216 | } 217 | 218 | i++; 219 | 220 | // Wrap around 221 | if (i == nodeList.end()) { 222 | i = nodeList.begin(); 223 | } 224 | num_examined++; 225 | 226 | } 227 | 228 | // Get replica iterator 229 | r_iter = replicas.begin(); 230 | 231 | // Skip past owner (already added above) 232 | r_iter++; 233 | DPRINTF(DEBUG_FLOW, "Adding replicas to node handle\n"); 234 | 235 | // Add replicas to node handle 236 | while (r_iter != replicas.end()) { 237 | nh->addReplica((*r_iter)); 238 | r_iter++; 239 | } 240 | DPRINTF(DEBUG_FLOW, "Found %d replicas\n", num_replicas); 241 | } 242 | 243 | return nh; 244 | } 245 | 246 | 247 | // no locking here - caller locks nodelist 248 | void Ring::printNodes() { 249 | if (nodeList.empty()) { 250 | cout << "->[NULL]\n"; 251 | return; 252 | } 253 | 254 | list::const_iterator i; 255 | 256 | for (i = nodeList.begin(); i != nodeList.end(); i++) { 257 | cout << " -> "; 258 | (*i)->id->printValue(); 259 | cout << "(" << (*i)->n->IP << ")"; 260 | } 261 | cout << endl; 262 | } 263 | 264 | 265 | Node* Ring::createNode(string ip, const int32_t port) 266 | { 267 | // Check if you already have a connection to the node 268 | string id = getID(ip, port); 269 | if (ip_node_map.find(id) == ip_node_map.end()) { 270 | Node *n = new Node(ip, port, false); 271 | ip_node_map[id] = n; 272 | return n; 273 | } else { 274 | return ip_node_map[id]; 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /fawnkv/ring.h: -------------------------------------------------------------------------------- 1 | #ifndef _RING_H_ 2 | #define _RING_H_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "dbid.h" 15 | #include "virtualnode.h" 16 | #include "nodehandle.h" 17 | #include "node.h" 18 | #include "debug.h" 19 | #include "FawnKVFrontend.h" 20 | #include "FawnKVBackend.h" 21 | #include "print.h" 22 | 23 | #include "hashutil.h" 24 | 25 | using fawn::DBID; 26 | using namespace std; 27 | using namespace tbb; 28 | 29 | 30 | class Ring { 31 | public: 32 | Ring(); 33 | ~Ring(); 34 | 35 | void UpdateState(ringState rs); 36 | 37 | string getRangeKey(string key); 38 | NodeHandle* getReplicaGroup(string key); 39 | DBID* getSuccessorID(string key); 40 | 41 | /* Util */ 42 | void printNodes(); 43 | 44 | map ip_node_map; 45 | map vid_temptail_map; 46 | map vid_shorter_map; 47 | 48 | private: 49 | Node* createNode(string ip, int port); 50 | /* A ring is a list of virtual nodes */ 51 | std::list nodeList; 52 | uint32_t replica_group_size; 53 | 54 | 55 | }; 56 | 57 | #endif 58 | -------------------------------------------------------------------------------- /fawnkv/virtualnode.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include "virtualnode.h" 3 | #include "node.h" 4 | 5 | using fawn::DBID; 6 | 7 | VirtualNode::VirtualNode() : 8 | n(NULL), id(NULL), dead(false){ 9 | } 10 | 11 | VirtualNode::VirtualNode(Node *nde, DBID *a) : 12 | n(nde), id(new DBID(a->data(), a->size())), dead(false) { 13 | } 14 | 15 | VirtualNode::~VirtualNode() { 16 | delete id; 17 | } 18 | 19 | 20 | bool VirtualNode::operator==(const VirtualNode &rhs) const { 21 | return rhs.id == id; 22 | } 23 | -------------------------------------------------------------------------------- /fawnkv/virtualnode.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _VIRTUALNODE_H_ 3 | #define _VIRTUALNODE_H_ 4 | 5 | #include "dbid.h" 6 | #include "node.h" 7 | 8 | using fawn::DBID; 9 | 10 | class VirtualNode 11 | { 12 | public: 13 | Node *n; 14 | DBID* id; 15 | bool dead; 16 | VirtualNode (); 17 | VirtualNode (Node *nde, DBID *a); 18 | ~VirtualNode(); 19 | 20 | bool operator==(const VirtualNode &rhs) const; 21 | }; 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /lib/Makefile.am: -------------------------------------------------------------------------------- 1 | if HAVE_ANT 2 | SUBDIRS = cpp rb java 3 | else 4 | SUBDIRS = cpp rb 5 | endif -------------------------------------------------------------------------------- /lib/cpp/Makefile.am: -------------------------------------------------------------------------------- 1 | BUILT_SOURCES = gen-cpp/FawnKV.cpp gen-cpp/FawnKV.h \ 2 | gen-cpp/fawnkv_constants.cpp gen-cpp/fawnkv_constants.h \ 3 | gen-cpp/fawnkv_types.cpp gen-cpp/fawnkv_types.h 4 | 5 | # This could be cleaned up but I don't know the best way 6 | gen-cpp/FawnKV.cpp: $(top_srcdir)/fawnkv/fawnkv.thrift 7 | gen-cpp/FawnKV.h: $(top_srcdir)/fawnkv/fawnkv.thrift 8 | gen-cpp/fawnkv_constants.cpp: $(top_srcdir)/fawnkv/fawnkv.thrift 9 | gen-cpp/fawnkv_constants.h: $(top_srcdir)/fawnkv/fawnkv.thrift 10 | gen-cpp/fawnkv_types.cpp: $(top_srcdir)/fawnkv/fawnkv.thrift 11 | gen-cpp/fawnkv_types.h: $(top_srcdir)/fawnkv/fawnkv.thrift 12 | 13 | noinst_PROGRAMS = tester tester_remote 14 | 15 | tester_SOURCES = tester.cpp TFawnKV.cpp TFawnKV.h 16 | 17 | tester_CPPFLAGS = -I$(builddir)/gen-cpp -I$(top_srcdir)/utils/ -I$(top_srcdir)/fawnkv/ -I$(top_builddir)/fawnkv/gen-cpp/ 18 | 19 | tester_LDADD = \ 20 | $(top_builddir)/fawnkv/libfawnkv.la \ 21 | $(top_builddir)/utils/libfawnkvutils.la \ 22 | $(THRIFT_LIBS) 23 | 24 | tester_remote_SOURCES = tester_remote.cpp TFawnKVRemote.cpp TFawnKVRemote.h 25 | 26 | tester_remote_CPPFLAGS = -I$(builddir)/gen-cpp -I$(top_srcdir)/utils/ -I$(top_srcdir)/fawnkv/ -I$(top_builddir)/fawnkv/gen-cpp/ 27 | 28 | tester_remote_LDADD = \ 29 | $(top_builddir)/fawnkv/libfawnkv.la \ 30 | $(top_builddir)/utils/libfawnkvutils.la \ 31 | $(THRIFT_LIBS) 32 | 33 | 34 | CLEANFILES = $(BUILT_SOURCES) 35 | -------------------------------------------------------------------------------- /lib/cpp/TFawnKV.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "TFawnKV.h" 14 | #include "fawnnet.h" 15 | #include "hashutil.h" 16 | 17 | using namespace ::apache::thrift; 18 | using namespace ::apache::thrift::protocol; 19 | using namespace ::apache::thrift::transport; 20 | using namespace ::apache::thrift::server; 21 | 22 | using boost::shared_ptr; 23 | 24 | using namespace fawn; 25 | 26 | pthread_cond_t client_cv; 27 | pthread_mutex_t client_mutex; 28 | bool has_data; 29 | string data; 30 | int64_t version_global; 31 | 32 | void put_cb(unsigned int continuation) 33 | { 34 | pthread_mutex_lock(&client_mutex); 35 | has_data = true; 36 | pthread_cond_signal(&client_cv); 37 | pthread_mutex_unlock(&client_mutex); 38 | } 39 | 40 | void put_w_cb(unsigned int continuation, int64_t version) 41 | { 42 | pthread_mutex_lock(&client_mutex); 43 | has_data = true; 44 | version_global = version; 45 | pthread_cond_signal(&client_cv); 46 | pthread_mutex_unlock(&client_mutex); 47 | } 48 | 49 | void get_cb(const DBID& p_key, const string& p_value, unsigned int continuation, bool success) 50 | { 51 | pthread_mutex_lock(&client_mutex); 52 | has_data = true; 53 | data = p_value; 54 | pthread_cond_signal(&client_cv); 55 | pthread_mutex_unlock(&client_mutex); 56 | } 57 | 58 | FawnKVClt::FawnKVClt(const std::string& managerIP, const std::string& clientIP, const int32_t clientPort) { 59 | pthread_mutex_init(&client_mutex, NULL); 60 | pthread_cond_init(&client_cv, NULL); 61 | has_data = false; 62 | 63 | if (clientIP != "") 64 | myIP = clientIP; 65 | 66 | if (clientPort == 0) 67 | myPort = FE_SERVER_PORT_BASE; 68 | else 69 | myPort = clientPort; 70 | 71 | /* Initialize ring */ 72 | frontend = new FrontEnd(managerIP, myIP, myPort); 73 | frontend->register_put_cb((void (*)(unsigned int)) &put_cb); 74 | frontend->register_put_w_cb((void (*)(unsigned int, int64_t)) &put_w_cb); 75 | frontend->register_get_cb((void (*)(const fawn::DBID&, const std::string&, unsigned int, bool)) &get_cb); 76 | 77 | 78 | } 79 | 80 | FawnKVClt::~FawnKVClt() { 81 | pthread_mutex_destroy(&client_mutex); 82 | pthread_cond_destroy(&client_cv); 83 | delete frontend; 84 | } 85 | 86 | //Currently, we're hashing keys so we avoid the 4byte issue ("user" == "user1" == "user2") 87 | //This will prevent range queries / textual keys 88 | 89 | //string FawnKVClt::get(const std::string& key) { 90 | string FawnKVClt::get(const std::string& key_long) { 91 | u_int32_t key_int = HashUtil::BobHash(key_long); 92 | string key = string((const char *) &key_int, 4); 93 | 94 | try { 95 | frontend->get(key, continuation); 96 | continuation++; 97 | } catch (TException &tx) { 98 | fprintf(stderr, "Transport error: %s\n", tx.what()); 99 | } 100 | 101 | //note: locks are used for signalling only, each FawnKVClt is single execution only 102 | pthread_mutex_lock(&client_mutex); 103 | while (!has_data) 104 | pthread_cond_wait(&client_cv, &client_mutex); 105 | has_data = false; 106 | pthread_mutex_unlock(&client_mutex); 107 | return data; 108 | } 109 | 110 | //void FawnKVClt::put(const std::string& key, const std::string& value) { 111 | void FawnKVClt::put(const std::string& key_long, const std::string& value) { 112 | u_int32_t key_int = HashUtil::BobHash(key_long); 113 | string key = string((const char *) &key_int, 4); 114 | 115 | try { 116 | frontend->put(key, value, continuation); 117 | continuation++; 118 | } catch (TException &tx) { 119 | fprintf(stderr, "Transport error: %s\n", tx.what()); 120 | } 121 | pthread_mutex_lock(&client_mutex); 122 | while (!has_data) 123 | pthread_cond_wait(&client_cv, &client_mutex); 124 | has_data = false; 125 | pthread_mutex_unlock(&client_mutex); 126 | } 127 | 128 | //void FawnKVClt::put_w(const std::string& key, const std::string& value) { 129 | int64_t FawnKVClt::put_w(const std::string& key_long, const std::string& value) { 130 | u_int32_t key_int = HashUtil::BobHash(key_long); 131 | string key = string((const char *) &key_int, 4); 132 | 133 | try { 134 | frontend->put_w(key, value, continuation); 135 | continuation++; 136 | } catch (TException &tx) { 137 | fprintf(stderr, "Transport error: %s\n", tx.what()); 138 | } 139 | pthread_mutex_lock(&client_mutex); 140 | while (!has_data) 141 | pthread_cond_wait(&client_cv, &client_mutex); 142 | has_data = false; 143 | int64_t ret_version = version_global; 144 | pthread_mutex_unlock(&client_mutex); 145 | 146 | return ret_version; 147 | } 148 | 149 | void FawnKVClt::remove(const std::string& key) { 150 | try { 151 | frontend->remove(key, continuation); 152 | continuation++; 153 | } catch (TException &tx) { 154 | fprintf(stderr, "Transport error: %s\n", tx.what()); 155 | } 156 | pthread_mutex_lock(&client_mutex); 157 | while (!has_data) 158 | pthread_cond_wait(&client_cv, &client_mutex); 159 | has_data = false; 160 | pthread_mutex_unlock(&client_mutex); 161 | } 162 | 163 | -------------------------------------------------------------------------------- /lib/cpp/TFawnKV.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _TFAWNKV_H_ 3 | #define _TFAWNKV_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "fe.h" 18 | 19 | //#include "FawnKV.h" 20 | //#include "FawnKVApp.h" 21 | 22 | using namespace ::apache::thrift; 23 | using namespace ::apache::thrift::protocol; 24 | using namespace ::apache::thrift::transport; 25 | using namespace ::apache::thrift::server; 26 | using namespace tbb; 27 | using namespace std; 28 | 29 | using boost::shared_ptr; 30 | 31 | using namespace fawn; 32 | 33 | #define FE_SERVER_PORT_BASE 7000 34 | 35 | 36 | class FawnKVClt { 37 | private: 38 | FrontEnd *frontend; 39 | int64_t continuation; 40 | string myIP; 41 | uint16_t myPort; 42 | 43 | public: 44 | FawnKVClt(const std::string& managerIP, const std::string& clientIP, const int32_t clientPort = 0); 45 | ~FawnKVClt(); 46 | 47 | string get(const std::string& key); 48 | void put(const std::string& key, const std::string& value); 49 | int64_t put_w(const std::string& key, const std::string& value); 50 | void remove(const std::string& key); 51 | }; 52 | 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /lib/cpp/TFawnKVRemote.cpp: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "TFawnKVRemote.h" 14 | #include "fawnnet.h" 15 | #include "hashutil.h" 16 | 17 | using namespace ::apache::thrift; 18 | using namespace ::apache::thrift::protocol; 19 | using namespace ::apache::thrift::transport; 20 | using namespace ::apache::thrift::server; 21 | 22 | using boost::shared_ptr; 23 | 24 | using namespace fawn; 25 | 26 | void *serverStart(void* p) { 27 | TSimpleServer *srv = (TSimpleServer*)p; 28 | srv->serve(); 29 | return NULL; 30 | } 31 | 32 | FawnKVClt::FawnKVClt(const std::string& frontendIP, const int32_t port, const std::string& clientIP, const int32_t clientPort) { 33 | pthread_mutex_init(&client_mutex, NULL); 34 | pthread_cond_init(&client_cv, NULL); 35 | has_data = false; 36 | 37 | if (clientIP == "") 38 | myIP = getMyIP(); 39 | else 40 | myIP = clientIP; 41 | 42 | if (clientPort == 0) 43 | myPort = port+1; 44 | else 45 | myPort = clientPort; 46 | 47 | shared_ptr socket(new TSocket(frontendIP.data(), port)); 48 | shared_ptr transport(new TBufferedTransport(socket)); 49 | shared_ptr protocol(new TBinaryProtocol(transport)); 50 | c = new FawnKVClient(protocol); 51 | continuation = 0; 52 | 53 | try { 54 | transport->open(); 55 | 56 | shared_ptr protocolFactory(new TBinaryProtocolFactory()); 57 | shared_ptr handler(new FawnKVClientHandler(this)); 58 | shared_ptr processor(new FawnKVAppProcessor(handler)); 59 | shared_ptr serverTransport(new TServerSocket(myPort)); 60 | shared_ptr transportFactory(new TBufferedTransportFactory()); 61 | 62 | server = new TSimpleServer(processor, serverTransport,transportFactory, protocolFactory); 63 | 64 | pthread_t localThread; 65 | int code = pthread_create(&localThread, NULL, serverStart, server); 66 | 67 | sleep(1); 68 | 69 | cid = c->init(myIP, myPort); 70 | } catch (TException &tx) { 71 | fprintf(stderr, "Transport error: %s\n", tx.what()); 72 | } 73 | 74 | } 75 | 76 | FawnKVClt::~FawnKVClt() { 77 | pthread_mutex_destroy(&client_mutex); 78 | pthread_cond_destroy(&client_cv); 79 | delete server; 80 | delete c; 81 | } 82 | 83 | //Currently, we're hashing keys so we avoid the 4byte issue ("user" == "user1" == "user2") 84 | //This will prevent range queries / textual keys 85 | 86 | //string FawnKVClt::get(const std::string& key) { 87 | string FawnKVClt::get(const std::string& key_long) { 88 | u_int32_t key_int = HashUtil::BobHash(key_long); 89 | string key = string((const char *) &key_int, 4); 90 | 91 | try { 92 | c->get(key, continuation, cid); 93 | continuation++; 94 | } catch (TException &tx) { 95 | fprintf(stderr, "Transport error: %s\n", tx.what()); 96 | } 97 | 98 | pthread_mutex_lock(&client_mutex); 99 | while (!has_data) 100 | pthread_cond_wait(&client_cv, &client_mutex); 101 | has_data = false; 102 | pthread_mutex_unlock(&client_mutex); 103 | return data; 104 | } 105 | 106 | //void FawnKVClt::put(const std::string& key, const std::string& value) { 107 | void FawnKVClt::put(const std::string& key_long, const std::string& value) { 108 | u_int32_t key_int = HashUtil::BobHash(key_long); 109 | string key = string((const char *) &key_int, 4); 110 | 111 | try { 112 | c->put(key, value, continuation, cid); 113 | continuation++; 114 | } catch (TException &tx) { 115 | fprintf(stderr, "Transport error: %s\n", tx.what()); 116 | } 117 | pthread_mutex_lock(&client_mutex); 118 | while (!has_data) 119 | pthread_cond_wait(&client_cv, &client_mutex); 120 | has_data = false; 121 | pthread_mutex_unlock(&client_mutex); 122 | } 123 | 124 | void FawnKVClt::remove(const std::string& key) { 125 | try { 126 | c->remove(key, continuation, cid); 127 | continuation++; 128 | } catch (TException &tx) { 129 | fprintf(stderr, "Transport error: %s\n", tx.what()); 130 | } 131 | pthread_mutex_lock(&client_mutex); 132 | while (!has_data) 133 | pthread_cond_wait(&client_cv, &client_mutex); 134 | has_data = false; 135 | pthread_mutex_unlock(&client_mutex); 136 | } 137 | 138 | 139 | FawnKVClientHandler::FawnKVClientHandler(FawnKVClt *f) { 140 | fc = f; 141 | } 142 | 143 | void FawnKVClientHandler::get_response(const std::string& value, const int64_t continuation) { 144 | pthread_mutex_lock(&fc->client_mutex); 145 | fc->has_data = true; 146 | fc->data = value; 147 | pthread_cond_signal(&fc->client_cv); 148 | pthread_mutex_unlock(&fc->client_mutex); 149 | } 150 | 151 | void FawnKVClientHandler::put_response(const int64_t continuation) { 152 | pthread_mutex_lock(&fc->client_mutex); 153 | fc->has_data = true; 154 | pthread_cond_signal(&fc->client_cv); 155 | pthread_mutex_unlock(&fc->client_mutex); 156 | } 157 | 158 | void FawnKVClientHandler::remove_response(const int64_t continuation) { 159 | pthread_mutex_lock(&fc->client_mutex); 160 | fc->has_data = true; 161 | pthread_cond_signal(&fc->client_cv); 162 | pthread_mutex_unlock(&fc->client_mutex); 163 | } 164 | -------------------------------------------------------------------------------- /lib/cpp/TFawnKVRemote.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _TFAWNKV_H_ 3 | #define _TFAWNKV_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | #include "FawnKV.h" 18 | #include "FawnKVApp.h" 19 | 20 | using namespace ::apache::thrift; 21 | using namespace ::apache::thrift::protocol; 22 | using namespace ::apache::thrift::transport; 23 | using namespace ::apache::thrift::server; 24 | using namespace tbb; 25 | using namespace std; 26 | 27 | using boost::shared_ptr; 28 | 29 | using namespace fawn; 30 | 31 | class FawnKVClt { 32 | private: 33 | TSimpleServer *server; 34 | FawnKVClient *c; 35 | int32_t cid; 36 | int64_t continuation; 37 | string myIP; 38 | uint16_t myPort; 39 | 40 | public: 41 | pthread_cond_t client_cv; 42 | pthread_mutex_t client_mutex; 43 | bool has_data; 44 | string data; 45 | 46 | FawnKVClt(const std::string& frontendIP, const int32_t port, const std::string& clientIP = "", const int32_t clientPort = 0); 47 | ~FawnKVClt(); 48 | 49 | string get(const std::string& key); 50 | void put(const std::string& key, const std::string& value); 51 | void remove(const std::string& key); 52 | }; 53 | 54 | class FawnKVClientHandler : virtual public FawnKVAppIf { 55 | private: 56 | FawnKVClt *fc; 57 | public: 58 | FawnKVClientHandler(FawnKVClt *f); 59 | 60 | void get_response(const std::string& value, const int64_t continuation); 61 | void put_response(const int64_t continuation); 62 | void remove_response(const int64_t continuation); 63 | }; 64 | 65 | #endif 66 | -------------------------------------------------------------------------------- /lib/cpp/tester.cpp: -------------------------------------------------------------------------------- 1 | #include "FawnKV.h" 2 | #include "TFawnKV.h" 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | using namespace apache::thrift; 10 | using namespace apache::thrift::protocol; 11 | using namespace apache::thrift::transport; 12 | 13 | void usage() 14 | { 15 | printf("Usage: ./tester [-c myIP] [-p myPort] managerIP\n"); 16 | } 17 | 18 | int main(int argc, char **argv) 19 | { 20 | extern char *optarg; 21 | extern int optind; 22 | string myIP = ""; 23 | int ch; 24 | int myPort = 0; 25 | while ((ch = getopt(argc, argv, "c:p:")) != -1) { 26 | switch (ch) { 27 | case 'c': 28 | myIP = optarg; 29 | break; 30 | case 'p': 31 | myPort = atoi(optarg); 32 | break; 33 | default: 34 | usage(); 35 | exit(-1); 36 | } 37 | } 38 | 39 | argc -= optind; 40 | argv += optind; 41 | 42 | if (argc < 1) { 43 | usage(); 44 | exit(-1); 45 | } 46 | 47 | FawnKVClt client(argv[0], myIP, myPort); 48 | 49 | for (int i = 0; i < 10; i++) { 50 | printf("putting.."); 51 | //client.put("abc", "value"); 52 | client.put("abc", "value"); 53 | string value = client.get("abc"); 54 | printf("%s\n", value.c_str()); 55 | 56 | printf("putting.."); 57 | client.put("def", "value"); 58 | value = client.get("def"); 59 | printf("%s\n", value.c_str()); 60 | } 61 | return 0; 62 | } 63 | -------------------------------------------------------------------------------- /lib/cpp/tester_remote.cpp: -------------------------------------------------------------------------------- 1 | #include "FawnKV.h" 2 | #include "TFawnKVRemote.h" 3 | 4 | #include 5 | #include 6 | #include 7 | 8 | using namespace std; 9 | using namespace apache::thrift; 10 | using namespace apache::thrift::protocol; 11 | using namespace apache::thrift::transport; 12 | 13 | void usage() 14 | { 15 | printf("Usage: ./tester_remote [-c clientIP] frontendIP [fePort]\n"); 16 | } 17 | 18 | int main(int argc, char **argv) 19 | { 20 | extern char *optarg; 21 | extern int optind; 22 | string myIP = ""; 23 | int myPort; 24 | int ch; 25 | int port = 4001; 26 | while ((ch = getopt(argc, argv, "c:p:")) != -1) { 27 | switch (ch) { 28 | case 'c': 29 | myIP = optarg; 30 | break; 31 | case 'p': 32 | myPort = atoi(optarg); 33 | break; 34 | default: 35 | usage(); 36 | exit(-1); 37 | } 38 | } 39 | 40 | argc -= optind; 41 | argv += optind; 42 | 43 | if (argc < 1) { 44 | usage(); 45 | exit(-1); 46 | } else if (argc == 2) { 47 | port = atoi(argv[1]); 48 | } 49 | 50 | FawnKVClt client(argv[0], port, myIP, myPort); 51 | 52 | for (int i = 0; i < 10000; i++) { 53 | printf("putting.."); 54 | client.put("abc", "value"); 55 | string value = client.get("abc"); 56 | printf("%s\n", value.c_str()); 57 | } 58 | return 0; 59 | } 60 | -------------------------------------------------------------------------------- /lib/java/FawnKVTester: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Licensed to the Apache Software Foundation (ASF) under one 5 | # or more contributor license agreements. See the NOTICE file 6 | # distributed with this work for additional information 7 | # regarding copyright ownership. The ASF licenses this file 8 | # to you under the Apache License, Version 2.0 (the 9 | # "License"); you may not use this file except in compliance 10 | # with the License. You may obtain a copy of the License at 11 | # 12 | # http://www.apache.org/licenses/LICENSE-2.0 13 | # 14 | # Unless required by applicable law or agreed to in writing, 15 | # software distributed under the License is distributed on an 16 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 17 | # KIND, either express or implied. See the License for the 18 | # specific language governing permissions and limitations 19 | # under the License. 20 | # 21 | 22 | java -cp Tester.jar:libthrift.jar:slf4j-api-1.5.8.jar:slf4j-simple-1.5.8.jar Tester $1 $2 $3 -------------------------------------------------------------------------------- /lib/java/Makefile.am: -------------------------------------------------------------------------------- 1 | EXTRA_DIST = build.xml slf4j-api-1.5.8.jar slf4j-simple-1.5.8.jar libthrift.jar src 2 | 3 | all: thrift lib tester 4 | 5 | thrift: 6 | thrift -r --gen java $(top_srcdir)/fawnkv/fawnkv.thrift 7 | 8 | lib: thrift 9 | ant -Dgen-ood=$(shell pwd)/gen-java -buildfile $(srcdir)/build.xml compile-ood 10 | 11 | tester: lib 12 | ant -Dgen-ood=$(shell pwd)/gen-java -buildfile $(srcdir)/build.xml Tester-ood 13 | -------------------------------------------------------------------------------- /lib/java/README: -------------------------------------------------------------------------------- 1 | To use the java Library, you need to do the following: 2 | 3 | thrift -r --gen java ../../fawnkv/fawnkv.thrift 4 | 5 | then to compile the client library, type: 6 | 7 | ant 8 | 9 | Finally, to compile the tester, type: 10 | 11 | ant Tester 12 | 13 | To run the tester, type: 14 | 15 | ./FawnKVTester frontendIP frontendPort myIP 16 | 17 | (Have not integrated option parsing in java tester yet, this is a TODO). 18 | -------------------------------------------------------------------------------- /lib/java/build.xml: -------------------------------------------------------------------------------- 1 | 19 | 20 | 21 | FawnKV Client 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /lib/java/libthrift.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/lib/java/libthrift.jar -------------------------------------------------------------------------------- /lib/java/slf4j-api-1.5.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/lib/java/slf4j-api-1.5.8.jar -------------------------------------------------------------------------------- /lib/java/slf4j-simple-1.5.8.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vrv/FAWN-KV/cbe33711d022530f821079f9ff6b541cb5b52ecd/lib/java/slf4j-simple-1.5.8.jar -------------------------------------------------------------------------------- /lib/java/src/FawnKVClt.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | // Generated code 21 | package fawn; 22 | import fawn.*; 23 | 24 | import org.apache.thrift.TException; 25 | import org.apache.thrift.transport.TTransport; 26 | import org.apache.thrift.transport.TSocket; 27 | import org.apache.thrift.transport.TTransportException; 28 | import org.apache.thrift.protocol.TBinaryProtocol; 29 | import org.apache.thrift.protocol.TProtocol; 30 | import org.apache.thrift.server.TServer; 31 | import org.apache.thrift.server.TSimpleServer; 32 | import org.apache.thrift.transport.TServerSocket; 33 | import org.apache.thrift.transport.TServerTransport; 34 | 35 | public class FawnKVClt { 36 | public FawnKVClt(String frontendIP, int port, String clientIP, int myPort) 37 | { 38 | myIP = clientIP; 39 | 40 | try { 41 | TTransport transport = new TSocket(frontendIP, port); 42 | TProtocol protocol = new TBinaryProtocol(transport); 43 | client = new FawnKV.Client(protocol); 44 | continuation = 0; 45 | 46 | transport.open(); 47 | 48 | 49 | FawnKVCltHandler handler = new FawnKVCltHandler(this); 50 | FawnKVApp.Processor processor = new FawnKVApp.Processor(handler); 51 | TServerTransport serverTransport = new TServerSocket(myPort); 52 | TServer server = new TSimpleServer(processor, serverTransport); 53 | 54 | ServerThread st = new ServerThread(server); 55 | st.start(); 56 | 57 | Thread.sleep(1000); 58 | 59 | cid = client.init(myIP, myPort); 60 | 61 | } catch (TException x) { 62 | x.printStackTrace(); 63 | } catch (InterruptedException x) { 64 | x.printStackTrace(); 65 | } 66 | 67 | } 68 | 69 | 70 | public String get(String key) { 71 | synchronized(myMonitorObject) { 72 | 73 | try { 74 | client.get(key.getBytes(), continuation, cid); 75 | continuation++; 76 | } catch (TException x) { 77 | x.printStackTrace(); 78 | } 79 | 80 | has_data = false; 81 | while (!has_data) { 82 | try { 83 | myMonitorObject.wait(); 84 | } catch (InterruptedException e) { 85 | e.printStackTrace(); 86 | } 87 | } 88 | has_data = false; 89 | } 90 | 91 | return data; 92 | } 93 | 94 | public void put(String key, String value) { 95 | synchronized(myMonitorObject) { 96 | try { 97 | client.put(key.getBytes(), value.getBytes(), continuation, cid); 98 | continuation++; 99 | } catch (TException x) { 100 | x.printStackTrace(); 101 | } 102 | 103 | has_data = false; 104 | while (!has_data) { 105 | try { 106 | myMonitorObject.wait(); 107 | } catch (InterruptedException e) { 108 | e.printStackTrace(); 109 | } 110 | } 111 | has_data = false; 112 | } 113 | 114 | return; 115 | 116 | } 117 | 118 | public void remove(String key) { 119 | synchronized(myMonitorObject) { 120 | try { 121 | client.remove(key.getBytes(), continuation, cid); 122 | continuation++; 123 | } catch (TException x) { 124 | x.printStackTrace(); 125 | } 126 | 127 | has_data = false; 128 | while (!has_data) { 129 | try { 130 | myMonitorObject.wait(); 131 | } catch (InterruptedException e) { 132 | e.printStackTrace(); 133 | } 134 | } 135 | has_data = false; 136 | } 137 | 138 | return; 139 | 140 | } 141 | 142 | private FawnKV.Client client; 143 | private String myIP; 144 | private int cid; 145 | private long continuation; 146 | 147 | public String data; 148 | public boolean has_data; 149 | public class MonitorObject {} 150 | public MonitorObject myMonitorObject = new MonitorObject(); 151 | 152 | 153 | private class FawnKVCltHandler implements FawnKVApp.Iface { 154 | public FawnKVCltHandler(FawnKVClt c) { 155 | client = c; 156 | } 157 | 158 | public void get_response(byte[] value, long continuation) 159 | { 160 | synchronized(client.myMonitorObject) { 161 | client.has_data = true; 162 | client.data = new String(value); 163 | client.myMonitorObject.notify(); 164 | } 165 | } 166 | 167 | public void put_response(long continuation) 168 | { 169 | synchronized(client.myMonitorObject) { 170 | client.has_data = true; 171 | client.myMonitorObject.notify(); 172 | } 173 | } 174 | 175 | public void remove_response(long continuation) 176 | { 177 | synchronized(client.myMonitorObject) { 178 | client.has_data = true; 179 | client.myMonitorObject.notify(); 180 | } 181 | } 182 | 183 | private FawnKVClt client; 184 | } 185 | 186 | class ServerThread extends Thread { 187 | TServer server; 188 | ServerThread(TServer server) { 189 | this.server = server; 190 | } 191 | public void run() { 192 | server.serve(); 193 | } 194 | } 195 | 196 | } 197 | 198 | -------------------------------------------------------------------------------- /lib/java/src/Tester.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, 13 | * software distributed under the License is distributed on an 14 | * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | * KIND, either express or implied. See the License for the 16 | * specific language governing permissions and limitations 17 | * under the License. 18 | */ 19 | 20 | // Generated code 21 | package fawn; 22 | import fawn.*; 23 | 24 | import org.apache.thrift.TException; 25 | import org.apache.thrift.transport.TTransport; 26 | import org.apache.thrift.transport.TSocket; 27 | import org.apache.thrift.transport.TTransportException; 28 | import org.apache.thrift.protocol.TBinaryProtocol; 29 | import org.apache.thrift.protocol.TProtocol; 30 | 31 | public class Tester { 32 | public static void main(String [] args) { 33 | FawnKVClt c = new FawnKVClt(args[0], Integer.parseInt(args[1]), args[2], Integer.parseInt(args[1])+1); 34 | for (int i = 0; i < 10000; i++) { 35 | System.out.println("Putting: "); 36 | c.put("abc", "value"); 37 | String value = c.get("abc"); 38 | System.out.println("Value: " + value); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lib/rb/Makefile.am: -------------------------------------------------------------------------------- 1 | BUILT_SOURCES = gen-rb/fawn_k_v.rb \ 2 | gen-rb/fawn_k_v_app.rb \ 3 | gen-rb/fawnkv_constants.rb \ 4 | gen-rb/fawnkv_types.rb 5 | 6 | # This could be cleaned up but I don't know the best way 7 | gen-rb/fawn_k_v.rb: $(top_srcdir)/fawnkv/fawnkv.thrift 8 | thrift -r --gen rb $(top_srcdir)/fawnkv/fawnkv.thrift 9 | gen-rb/fawn_k_v_app.rb: $(top_srcdir)/fawnkv/fawnkv.thrift 10 | thrift -r --gen rb $(top_srcdir)/fawnkv/fawnkv.thrift 11 | gen-rb/fawnkv_constants.rb: $(top_srcdir)/fawnkv/fawnkv.thrift 12 | thrift -r --gen rb $(top_srcdir)/fawnkv/fawnkv.thrift 13 | gen-rb/fawnkv_types.rb: $(top_srcdir)/fawnkv/fawnkv.thrift 14 | thrift -r --gen rb $(top_srcdir)/fawnkv/fawnkv.thrift 15 | 16 | 17 | CLEANFILES = $(BUILT_SOURCES) 18 | -------------------------------------------------------------------------------- /lib/rb/TFawnKV.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Generate thrift files by runnning 4 | # thrift -r --gen rb $(srcdir)/fawnkv/fawnkv.thrift 5 | 6 | $:.push('gen-rb') 7 | 8 | require 'thread' 9 | require 'thrift' 10 | require 'fawn_k_v' 11 | require 'fawn_k_v_app' 12 | 13 | class FawnKVAppHandler 14 | def initialize(client) 15 | @client = client 16 | end 17 | 18 | def get_response(value, continuation) 19 | @client.mutex.synchronize { 20 | @client.val = value 21 | @client.cv.signal 22 | } 23 | end 24 | 25 | def put_response(continuation) 26 | @client.mutex.synchronize { 27 | @client.cv.signal 28 | } 29 | 30 | end 31 | 32 | def remove_response(continuation) 33 | @client.mutex.synchronize { 34 | @client.cv.signal 35 | } 36 | 37 | end 38 | 39 | end 40 | 41 | class FawnKVClient 42 | attr_accessor :val, :cv, :mutex 43 | def initialize(ip, port, cip) 44 | begin 45 | @cv = ConditionVariable.new 46 | @mutex = Mutex.new 47 | 48 | @transport = Thrift::BufferedTransport.new(Thrift::Socket.new(ip, port)) 49 | @protocol = Thrift::BinaryProtocol.new(@transport) 50 | @client = FawnKV::Client.new(@protocol) 51 | @ip = ip 52 | @transport.open() 53 | 54 | @handler = FawnKVAppHandler.new(self) 55 | @processor = FawnKVApp::Processor.new(@handler) 56 | @my_transport = Thrift::ServerSocket.new(port+1) 57 | @transportFactory = Thrift::BufferedTransportFactory.new() 58 | @server = Thrift::SimpleServer.new(@processor, @my_transport, @transportFactory) 59 | 60 | thread = Thread.new() { @server.serve() } 61 | 62 | @continuation = 0 63 | 64 | sleep(1) 65 | 66 | # This serves two functions: it sets up the return 67 | # connection on the frontend and returns the client id 68 | # needed for subsequent put/get/remove calls 69 | @cid = @client.init(cip, port+1) 70 | 71 | rescue Thrift::Exception => tx 72 | print 'Thrift::Exception: ', tx.message, "\n" 73 | end 74 | end 75 | 76 | def get(key) 77 | begin 78 | @client.get(key, @continuation, @cid) 79 | @continuation += 1 80 | rescue Thrift::Exception => tx 81 | print 'Thrift::Exception: ', tx.message, "\n" 82 | end 83 | 84 | # wait for response 85 | @mutex.synchronize { 86 | @cv.wait(@mutex) 87 | } 88 | 89 | return @val 90 | end 91 | 92 | def put(key, value) 93 | begin 94 | @client.put(key, value, @continuation, @cid) 95 | @continuation += 1 96 | rescue Thrift::Exception => tx 97 | print 'Thrift::Exception: ', tx.message, "\n" 98 | end 99 | 100 | 101 | # wait for response 102 | @mutex.synchronize { 103 | @cv.wait(@mutex) 104 | } 105 | end 106 | 107 | def close() 108 | @transport.close 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /lib/rb/tester.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'optparse' 3 | require 'TFawnKV' 4 | 5 | options = {} 6 | optparse = OptionParser.new do |opts| 7 | opts.banner = "Usage: ./tester.rb [-c clientIP] frontendIP [port]" 8 | 9 | # Define the options, and what they do 10 | options[:clientIP] = "localhost" 11 | opts.on( '-c', '--clientIP myIP', 'Set this client IP (default localhost)' ) do |cip| 12 | options[:clientIP] = cip 13 | end 14 | opts.on( '-h', '--help', 'Display this screen' ) do 15 | puts opts 16 | exit 17 | end 18 | end 19 | 20 | optparse.parse! 21 | 22 | if (ARGV.length < 1) 23 | puts "Usage: ./tester.rb [-c clientIP] frontendIP [port]" 24 | exit 25 | end 26 | 27 | ip = ARGV[0] 28 | port = ARGV[1].to_i || 4001 29 | myIP = options[:clientIP] 30 | 31 | fc = FawnKVClient.new(ip, port, myIP) 32 | key = "key" 33 | value = "value" 34 | 35 | count = 0 36 | while count < 10000 37 | fc.put(key, value); 38 | count += 1 39 | puts fc.get(key); 40 | end 41 | 42 | fc.close() 43 | 44 | -------------------------------------------------------------------------------- /m4/README: -------------------------------------------------------------------------------- 1 | This directory includes an m4 hook included with thrift to detect javac/java. -------------------------------------------------------------------------------- /m4/ax_javac_and_java.m4: -------------------------------------------------------------------------------- 1 | dnl @synopsis AX_JAVAC_AND_JAVA 2 | dnl @synopsis AX_CHECK_JAVA_CLASS(CLASSNAME) 3 | dnl 4 | dnl Test for the presence of a JDK, and (optionally) specific classes. 5 | dnl 6 | dnl If "JAVA" is defined in the environment, that will be the only 7 | dnl java command tested. Otherwise, a hard-coded list will be used. 8 | dnl Similarly for "JAVAC". 9 | dnl 10 | dnl AX_JAVAC_AND_JAVA does not currenly support testing for a particular 11 | dnl Java version, testing for only one of "java" and "javac", or 12 | dnl compiling or running user-provided Java code. 13 | dnl 14 | dnl After AX_JAVAC_AND_JAVA runs, the shell variables "success" and 15 | dnl "ax_javac_and_java" are set to "yes" or "no", and "JAVAC" and 16 | dnl "JAVA" are set to the appropriate commands. 17 | dnl 18 | dnl AX_CHECK_JAVA_CLASS must be run after AX_JAVAC_AND_JAVA. 19 | dnl It tests for the presence of a class based on a fully-qualified name. 20 | dnl It sets the shell variable "success" to "yes" or "no". 21 | dnl 22 | dnl @category Java 23 | dnl @version 2009-02-09 24 | dnl @license AllPermissive 25 | dnl 26 | dnl Copyright (C) 2009 David Reiss 27 | dnl Copying and distribution of this file, with or without modification, 28 | dnl are permitted in any medium without royalty provided the copyright 29 | dnl notice and this notice are preserved. 30 | 31 | 32 | AC_DEFUN([AX_JAVAC_AND_JAVA], 33 | [ 34 | 35 | dnl Hard-coded default commands to test. 36 | JAVAC_PROGS="javac,jikes,gcj -C" 37 | JAVA_PROGS="java,kaffe" 38 | 39 | dnl Allow the user to specify an alternative. 40 | if test -n "$JAVAC" ; then 41 | JAVAC_PROGS="$JAVAC" 42 | fi 43 | if test -n "$JAVA" ; then 44 | JAVA_PROGS="$JAVA" 45 | fi 46 | 47 | AC_MSG_CHECKING(for javac and java) 48 | 49 | echo "public class configtest_ax_javac_and_java { public static void main(String args@<:@@:>@) { } }" > configtest_ax_javac_and_java.java 50 | success=no 51 | oIFS="$IFS" 52 | 53 | IFS="," 54 | for JAVAC in $JAVAC_PROGS ; do 55 | IFS="$oIFS" 56 | 57 | echo "Running \"$JAVAC configtest_ax_javac_and_java.java\"" >&AS_MESSAGE_LOG_FD 58 | if $JAVAC configtest_ax_javac_and_java.java >&AS_MESSAGE_LOG_FD 2>&1 ; then 59 | 60 | IFS="," 61 | for JAVA in $JAVA_PROGS ; do 62 | IFS="$oIFS" 63 | 64 | echo "Running \"$JAVA configtest_ax_javac_and_java\"" >&AS_MESSAGE_LOG_FD 65 | if $JAVA configtest_ax_javac_and_java >&AS_MESSAGE_LOG_FD 2>&1 ; then 66 | success=yes 67 | break 2 68 | fi 69 | 70 | done 71 | 72 | fi 73 | 74 | done 75 | 76 | rm -f configtest_ax_javac_and_java.java configtest_ax_javac_and_java.class 77 | 78 | if test "$success" != "yes" ; then 79 | AC_MSG_RESULT(no) 80 | JAVAC="" 81 | JAVA="" 82 | else 83 | AC_MSG_RESULT(yes) 84 | fi 85 | 86 | ax_javac_and_java="$success" 87 | 88 | ]) 89 | 90 | 91 | AC_DEFUN([AX_CHECK_JAVA_CLASS], 92 | [ 93 | AC_MSG_CHECKING(for Java class [$1]) 94 | 95 | echo "import $1; public class configtest_ax_javac_and_java { public static void main(String args@<:@@:>@) { } }" > configtest_ax_javac_and_java.java 96 | 97 | echo "Running \"$JAVAC configtest_ax_javac_and_java.java\"" >&AS_MESSAGE_LOG_FD 98 | if $JAVAC configtest_ax_javac_and_java.java >&AS_MESSAGE_LOG_FD 2>&1 ; then 99 | AC_MSG_RESULT(yes) 100 | success=yes 101 | else 102 | AC_MSG_RESULT(no) 103 | success=no 104 | fi 105 | 106 | rm -f configtest_ax_javac_and_java.java configtest_ax_javac_and_java.class 107 | ]) 108 | -------------------------------------------------------------------------------- /patches/fawn-thrift.patch: -------------------------------------------------------------------------------- 1 | diff -Naur thrift-0.2.0/lib/cpp/src/server/TThreadedServer.cpp thrift-0.2.0-patched/lib/cpp/src/server/TThreadedServer.cpp 2 | --- thrift-0.2.0/lib/cpp/src/server/TThreadedServer.cpp 2009-03-30 17:35:00.000000000 -0400 3 | +++ thrift-0.2.0-patched/lib/cpp/src/server/TThreadedServer.cpp 2010-02-05 21:46:29.774587744 -0500 4 | @@ -51,6 +51,10 @@ 5 | 6 | ~Task() {} 7 | 8 | + void stop() { 9 | + input_->getTransport()->close(); 10 | + } 11 | + 12 | void run() { 13 | boost::shared_ptr eventHandler = 14 | server_.getEventHandler(); 15 | @@ -228,6 +232,8 @@ 16 | } 17 | try { 18 | Synchronized s(tasksMonitor_); 19 | + for ( std::set::iterator tIt = tasks_.begin(); tIt != tasks_.end(); ++tIt ) 20 | + (*tIt)->stop(); 21 | while (!tasks_.empty()) { 22 | tasksMonitor_.wait(); 23 | } 24 | -------------------------------------------------------------------------------- /test/Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = fawnds management 2 | -------------------------------------------------------------------------------- /test/fawnds/Makefile.am: -------------------------------------------------------------------------------- 1 | #hashdb_test code 2 | noinst_PROGRAMS = fawnds_test 3 | fawnds_test_SOURCES = fawnds_test.cc 4 | fawnds_test_CPPFLAGS = \ 5 | -I$(top_srcdir)/utils \ 6 | -I$(top_srcdir)/fawnds \ 7 | -I$(top_builddir)/fawnds \ 8 | -I$(top_builddir)/fawnds/gen-cpp 9 | 10 | fawnds_test_LDADD = \ 11 | $(top_builddir)/fawnds/libfawnds.la \ 12 | $(top_builddir)/utils/libfawnkvutils.la \ 13 | $(THRIFT_LIBS) 14 | -------------------------------------------------------------------------------- /test/management/Makefile.am: -------------------------------------------------------------------------------- 1 | #ringtester code 2 | noinst_PROGRAMS = ringtester lattester 3 | ringtester_SOURCES = ringtester.cpp 4 | ringtester_CPPFLAGS = \ 5 | -I$(top_srcdir)/utils \ 6 | -I$(top_srcdir)/fawnkv \ 7 | -I$(top_builddir)/fawnkv/gen-cpp \ 8 | -I$(top_srcdir)/fawnds \ 9 | -I$(top_builddir)/fawnkv \ 10 | -I$(top_builddir)/fawnds 11 | 12 | ringtester_LDADD = \ 13 | $(top_builddir)/fawnkv/libfawnkv.la \ 14 | $(top_builddir)/utils/libfawnkvutils.la \ 15 | $(THRIFT_LIBS) 16 | 17 | lattester_SOURCES = lattester.cpp 18 | lattester_CPPFLAGS = \ 19 | -I$(top_srcdir)/utils \ 20 | -I$(top_srcdir)/fawnkv \ 21 | -I$(top_builddir)/fawnkv/gen-cpp \ 22 | -I$(top_srcdir)/fawnds \ 23 | -I$(top_builddir)/fawnkv \ 24 | -I$(top_builddir)/fawnds 25 | 26 | lattester_LDADD = \ 27 | $(top_builddir)/fawnkv/libfawnkv.la \ 28 | $(top_builddir)/utils/libfawnkvutils.la \ 29 | $(THRIFT_LIBS) 30 | -------------------------------------------------------------------------------- /utils/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_LTLIBRARIES = libfawnkvutils.la 2 | noinst_HEADERS = hashutil.h dbid.h print.h timing.h fawnnet.h fnv.h debug.h 3 | libfawnkvutils_la_SOURCES = hashutil.cc dbid.cc print.cc timing.c fawnnet.cc 4 | 5 | # kaminsky: following line forces noinst_* libraries to build 6 | # shared. This can help with development because a change to 7 | # this library doesn't require re-building libs and programs 8 | # that link against this library 9 | #libfawnkvutils_la_LDFLAGS = -rpath `pwd` 10 | 11 | noinst_PROGRAMS = dbid_test 12 | dbid_test_SOURCES = dbidtest.cc 13 | dbid_test_CPPFLAGS = -I$(top_srcdir)/utils 14 | dbid_test_LDADD = libfawnkvutils.la -------------------------------------------------------------------------------- /utils/dbid.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | using namespace std; 3 | #include 4 | #include 5 | #include "dbid.h" 6 | #include "print.h" 7 | 8 | 9 | namespace fawn { 10 | DBID::DBID() : actual_size(0) { 11 | memset(value, '\0', sizeof(value)); 12 | } 13 | 14 | DBID::DBID(const char* c, unsigned int size) { 15 | Init(c, size); 16 | } 17 | 18 | DBID::DBID(const string &s) { 19 | Init(s.data(), s.size()); 20 | } 21 | 22 | void DBID::Init(const char *c, unsigned int size) { 23 | // Keep track of input size (vs. DBID_LENGTH) 24 | actual_size = size; 25 | 26 | if (size > DBID_LENGTH) { 27 | cout << "Only " << DBID_LENGTH << " byte IDs supported, size = " << size << ". Truncating ..." << endl; 28 | cout << "Change DBID_LENGTH in dbid.h to a large number to support larger key sizes." << endl; 29 | actual_size = size = DBID_LENGTH; 30 | } 31 | 32 | // Set value 33 | memcpy(value, c, size); 34 | 35 | // Pad entry with 0s if needed 36 | if (size < DBID_LENGTH) 37 | memset(value+size, '\0', DBID_LENGTH - size); 38 | 39 | } 40 | 41 | DBID::~DBID() { 42 | 43 | } 44 | 45 | string* DBID::actual_data() { 46 | return new string((char *)value, actual_size); 47 | } 48 | 49 | const string DBID::actual_data_str() const { 50 | return string((char *)value, actual_size); 51 | } 52 | 53 | 54 | void DBID::printValue() const { 55 | cout << "actual dbid size: " << actual_size << endl; 56 | print_payload((const u_char*)value, DBID_LENGTH); 57 | } 58 | 59 | void DBID::printValue(int prefix_bytes) const { 60 | cout << "actual dbid size: " << actual_size << ", printing first " << prefix_bytes << " bytes of dbid" << endl; 61 | print_payload((const u_char*)value, prefix_bytes); 62 | } 63 | 64 | 65 | bool DBID::operator==(const DBID &rhs) const { 66 | return (memcmp(value, rhs.value, DBID_LENGTH) == 0); 67 | } 68 | 69 | // Bytewise comparison of equal-length byte arrays 70 | bool DBID::operator<(const DBID &rhs) const { 71 | for (uint i = 0; i < DBID_LENGTH; i++) { 72 | if ((unsigned int)value[i] > (unsigned int)rhs.value[i]) { 73 | return false; 74 | } 75 | if ((unsigned int)value[i] < (unsigned int)rhs.value[i]) { 76 | return true; 77 | } 78 | } 79 | return false; 80 | } 81 | 82 | uint64_t DBID::operator-(const DBID &rhs) const { 83 | uint64_t thisid = (uint64_t) ((0xff & value[0]) 84 | + ((0xff & value[1]) << 8) 85 | + ((0xff & value[2]) << 16) 86 | + ((0xff & value[3]) << 24) 87 | + ((uint64_t)(0xff & value[4]) << 32) 88 | + ((uint64_t)(0xff & value[5]) << 40) 89 | + ((uint64_t)(0xff & value[6]) << 48) 90 | + ((uint64_t)(0xff & value[7]) << 56)); 91 | 92 | uint64_t rhsid = (uint64_t) ((0xff & rhs.value[0]) 93 | + ((0xff & rhs.value[1]) << 8) 94 | + ((0xff & rhs.value[2]) << 16) 95 | + ((0xff & rhs.value[3]) << 24) 96 | + ((uint64_t)(0xff & rhs.value[4]) << 32) 97 | + ((uint64_t)(0xff & rhs.value[5]) << 40) 98 | + ((uint64_t)(0xff & rhs.value[6]) << 48) 99 | + ((uint64_t)(0xff & rhs.value[7]) << 56)); 100 | if (rhsid > thisid) 101 | return rhsid - thisid; 102 | else 103 | return thisid - rhsid; 104 | } 105 | 106 | // returns this - rhs 107 | // DBID DBID::operator-(const DBID &rhs) const 108 | // { 109 | // int8_t carryover = 0; 110 | // char ans[DBID_LENGTH]; 111 | // for (uint i = DBID_LENGTH-1; i >= 0; i--) { 112 | // uint8_t diff = ((value[i] & 0xff) - (rhs.value[i] & 0xff)) & 0xff; 113 | // carryover = (diff == 0 && carryover == 1); 114 | // ans[i] = (diff - carryover) & 0xff; 115 | // } 116 | // return DBID(ans, DBID_LENGTH); 117 | // } 118 | 119 | 120 | 121 | // Comparison function for ring space. 122 | bool between(const string& startKey, const string& endKey, const string& thisKey) { 123 | DBID startID(startKey); 124 | DBID endID(endKey); 125 | DBID thisID(thisKey); 126 | 127 | return between(&startID, &endID, &thisID); 128 | } 129 | 130 | bool between(const DBID* startID, const DBID* endID, const DBID* thisID) { 131 | if (*startID == *endID) { 132 | return true; 133 | } else if (*endID < *startID) { 134 | // endKey < startKey, so there is wrap around 135 | if (*startID < *thisID || !(*endID < *thisID)) { 136 | return true; 137 | } 138 | } else { 139 | // startKey < endKey 140 | // if startKey < key and !(key > endKey) 141 | if (*startID < *thisID && !(*endID < *thisID)) { 142 | return true; 143 | } 144 | } 145 | 146 | return false; 147 | } 148 | 149 | } 150 | -------------------------------------------------------------------------------- /utils/dbid.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _DBID_H_ 3 | #define _DBID_H_ 4 | 5 | using namespace std; 6 | #include 7 | #include 8 | #include 9 | 10 | /** 11 | * DBID is a fixed-length identifier for nodes on a ringspace. Since 12 | * it is fixed length, the length needs to be as long as the longest 13 | * allowable key size so that the ID space contains the granularity to 14 | * load balance data. This length is defined as DBID_LENGTH below. 15 | * 16 | * In other words: DBID_LENGTH is the maximum key length for any query. 17 | * 18 | * Comparisons of short keys will pad the incoming key with 0s to 19 | * meet this size when necessary. You can redefine DBID_LENGTH to 20 | * whatever size you desire, but DBID comparisons will take time 21 | * proportional to the size. 22 | **/ 23 | 24 | namespace fawn { 25 | const unsigned int DBID_LENGTH = 1024; 26 | 27 | class DBID 28 | { 29 | 30 | private: 31 | char value[DBID_LENGTH]; 32 | unsigned int actual_size; 33 | public: 34 | DBID(); 35 | DBID(const char *c, unsigned int size); 36 | explicit DBID(const string &s); 37 | ~DBID(); 38 | void Init(const char *c, unsigned int size); 39 | 40 | char* data() { 41 | return (char *)value; 42 | } 43 | const char *const_data() const { 44 | return (const char *)value; 45 | } 46 | string* actual_data(); 47 | const string actual_data_str() const; 48 | unsigned int size() { 49 | return DBID_LENGTH; 50 | } 51 | unsigned int get_actual_size() { 52 | return actual_size; 53 | } 54 | void printValue() const; 55 | void printValue(int prefix_bytes) const; 56 | void set_actual_size(unsigned int as) { 57 | actual_size = as; 58 | } 59 | 60 | bool operator==(const DBID &rhs) const; 61 | bool operator<(const DBID &rhs) const; 62 | 63 | // This compares only the top 64 bits of each DBID 64 | uint64_t operator-(const DBID &rhs) const; 65 | 66 | //DBID operator-(const DBID &rhs) const; 67 | }; 68 | 69 | bool between(const string& startKey, const string& endKey, const string& thisKey); 70 | bool between(const DBID* startID, const DBID* endID, const DBID* thisID); 71 | } 72 | #endif 73 | -------------------------------------------------------------------------------- /utils/dbidtest.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include "dbid.h" 6 | 7 | using namespace std; 8 | 9 | namespace fawn { 10 | 11 | class DBIDUnitTest : public testing::Test { 12 | protected: 13 | // Code here will be called immediately after the constructor (right before 14 | // each test). 15 | virtual void SetUp() { 16 | 17 | } 18 | 19 | // Code in the TearDown() method will be called immediately after each test 20 | // (right before the destructor). 21 | virtual void TearDown() { 22 | 23 | } 24 | 25 | // Objects declared here can be used by all tests in the test case for HashDB. 26 | 27 | private: 28 | }; 29 | 30 | TEST_F(DBIDUnitTest, PrintCharacterID) { 31 | string test = "Test Name"; 32 | DBID testid(test); 33 | testid.printValue(); 34 | testid.printValue(test.size()); 35 | } 36 | 37 | TEST_F(DBIDUnitTest, CompareEqualCharacters) { 38 | DBID a("aaaa"); 39 | DBID a2("aaaa"); 40 | ASSERT_TRUE(a == a2); 41 | } 42 | 43 | TEST_F(DBIDUnitTest, CompareDifferentLengthCharacters) { 44 | DBID a("aaaa"); 45 | DBID a2("aaaaa"); 46 | ASSERT_FALSE(a == a2); 47 | ASSERT_TRUE(a < a2); 48 | } 49 | 50 | TEST_F(DBIDUnitTest, CompareCharacters) { 51 | DBID test1("aaaa"); 52 | DBID test2("aaab"); 53 | DBID test3("aaaB"); 54 | ASSERT_FALSE(test1 == test2); 55 | ASSERT_TRUE(test1 < test2); 56 | ASSERT_TRUE(test3 < test1); 57 | ASSERT_TRUE(test3 < test2); 58 | } 59 | 60 | TEST_F(DBIDUnitTest, Between1) { 61 | DBID test1("aaaaaaaaaaaa"); 62 | DBID test2("aaaaaaaaaaab"); 63 | DBID test3("aaaaaaaaaaaB"); 64 | // Test 3 --- Test 1 --- Test 2 65 | 66 | ASSERT_TRUE(between(&test1, &test3, &test2)); 67 | ASSERT_TRUE(between(&test2, &test1, &test3)); 68 | ASSERT_TRUE(between(&test3, &test2, &test1)); 69 | 70 | ASSERT_FALSE(between(&test1, &test2, &test3)); 71 | ASSERT_FALSE(between(&test2, &test3, &test1)); 72 | ASSERT_FALSE(between(&test3, &test1, &test2)); 73 | } 74 | 75 | 76 | } // namespace fawn 77 | 78 | int main(int argc, char** argv) { 79 | testing::InitGoogleTest(&argc, argv); 80 | return RUN_ALL_TESTS(); 81 | } 82 | -------------------------------------------------------------------------------- /utils/dbparse.pl: -------------------------------------------------------------------------------- 1 | #!/usr/bin/perl 2 | while () { 3 | if (/^#define ([^\s]+)\s.*DBTEXT:\s+(.*)/) { 4 | print "{ $1, \"$2\"},\n"; 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /utils/debug.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Rather boring. Define the debugging stuff. 3 | */ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | //for time 10 | #include 11 | #include 12 | 13 | #include "debug.h" 14 | 15 | 16 | unsigned int debug = 0; 17 | 18 | /* We could autogenerate this if we felt like it */ 19 | 20 | struct debug_def { 21 | int debug_val; 22 | char *debug_def; 23 | }; 24 | 25 | static 26 | struct debug_def debugs[] = { 27 | #include "debug-text.h" 28 | { 0, NULL } /* End of list marker */ 29 | }; 30 | 31 | int set_debug(char *arg) 32 | { 33 | int i; 34 | if (!arg || arg[0] == '\0') { 35 | return -1; 36 | } 37 | 38 | if (arg[0] == '?' || !strcmp(arg, "list")) { 39 | fprintf(stderr, 40 | "Debug values and definitions\n" 41 | "----------------------------\n"); 42 | for (i = 0; debugs[i].debug_def != NULL; i++) { 43 | fprintf(stderr, "%5d %s\n", debugs[i].debug_val, 44 | debugs[i].debug_def); 45 | } 46 | fprintf(stderr, "\n\'all\' will enable all debug flags.\n"); 47 | return -1; 48 | } 49 | if (!strcmp(arg, "all")) { 50 | debug = 0xffffffff; 51 | return 0; 52 | } 53 | 54 | if (isdigit(arg[0])) { 55 | debug |= atoi(arg); 56 | } 57 | return 0; 58 | } 59 | 60 | -------------------------------------------------------------------------------- /utils/debug.h: -------------------------------------------------------------------------------- 1 | #ifndef _DEBUG_H_ 2 | #define _DEBUG_H_ 3 | 4 | #include /* for perror */ 5 | #define eprintf(fmt, args...) fprintf(stderr, fmt, ##args) 6 | 7 | #ifndef DEBUG 8 | //#define DEBUG 9 | #endif 10 | 11 | #ifdef DEBUG 12 | //extern unsigned int debug; 13 | 14 | #define debug_level 2 15 | 16 | #define DPRINTF(level, fmt, args...) \ 17 | do { if (debug_level & (level)) fprintf(stderr, fmt , ##args ); } while(0) 18 | #define DEBUG_PERROR(errmsg) \ 19 | do { if (debug_level & DEBUG_ERRS) perror(errmsg); } while(0) 20 | #else 21 | #define DPRINTF(args...) 22 | #define DEBUG_PERROR(args...) 23 | #endif 24 | 25 | /* 26 | * The format of this should be obvious. Please add some explanatory 27 | * text if you add a debugging value. This text will show up in 28 | * -d list 29 | */ 30 | #define DEBUG_NONE 0x00 // DBTEXT: No debugging 31 | #define DEBUG_ERRS 0x01 // DBTEXT: Verbose error reporting 32 | #define DEBUG_FLOW 0x02 // DBTEXT: Messages to understand flow 33 | #define DEBUG_SOCKETS 0x04 // DBTEXT: Debug socket operations 34 | #define DEBUG_INPUT 0x08 // DBTEXT: Debug client input 35 | #define DEBUG_CLIENTS 0x10 // DBTEXT: Debug client arrival/depart 36 | #define DEBUG_COMMANDS 0x20 // DBTEXT: Debug client commands 37 | #define DEBUG_CHANNELS 0x40 // DBTEXT: Debug channel operations 38 | #define DEBUG_STATUS 0x80 // DBTEXT: Debug status messages 39 | 40 | #define DEBUG_ALL 0xffffffff 41 | 42 | //int set_debug(char *arg); /* Returns 0 on success, -1 on failure */ 43 | 44 | 45 | #endif /* _DEBUG_H_ */ 46 | -------------------------------------------------------------------------------- /utils/fawnnet.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | 12 | #include "fawnnet.h" 13 | 14 | using namespace std; 15 | 16 | int fawn::connectTCP(string IP, int port) 17 | { 18 | int sock; 19 | if ((sock = socket(PF_INET, SOCK_STREAM, 0)) < 0) { 20 | perror("socket"); 21 | cerr << "cannot create socket" << endl; 22 | return -1; 23 | } 24 | 25 | int yes = 1; 26 | if (setsockopt(sock, IPPROTO_TCP, TCP_NODELAY, 27 | &yes, sizeof(int)) == -1) 28 | { 29 | perror("setsockopt"); 30 | cerr << "connectTCP: error in setsockopt" << endl; 31 | return -1; 32 | } 33 | 34 | struct sockaddr_in serverAddress; 35 | struct hostent *he = gethostbyname(IP.data()); 36 | if (he == NULL) { 37 | cout << "problem resolving host: " << IP.data() << endl; 38 | return -1; 39 | } 40 | bcopy(he->h_addr, (struct in_addr *)&(serverAddress.sin_addr), he->h_length); 41 | 42 | serverAddress.sin_family = he->h_addrtype; 43 | serverAddress.sin_port = htons(port); 44 | 45 | if (connect(sock, (struct sockaddr *)&serverAddress, 46 | sizeof serverAddress) == -1) { 47 | perror("connectTCP: connect failed"); 48 | return -1; 49 | } 50 | 51 | return sock; 52 | } 53 | 54 | // Uses local hostname to find IP 55 | string fawn::getMyIP() 56 | { 57 | char hostname[128]; 58 | struct addrinfo hints; 59 | memset(&hints, 0, sizeof(hints)); 60 | struct addrinfo *servinfo, *p; 61 | 62 | gethostname(hostname, sizeof(hostname)); 63 | 64 | getaddrinfo(hostname, NULL, &hints, &servinfo); 65 | 66 | char ipstr[INET6_ADDRSTRLEN]; 67 | for (p = servinfo; p != NULL; p = p->ai_next) { 68 | void* addr; 69 | if (p->ai_family == AF_INET) { 70 | struct sockaddr_in *ipv4 = (struct sockaddr_in *)p->ai_addr; 71 | addr = &(ipv4->sin_addr); 72 | } else { 73 | struct sockaddr_in6 *ipv6 = (struct sockaddr_in6 *)p->ai_addr; 74 | addr = &(ipv6->sin6_addr); 75 | } 76 | inet_ntop(p->ai_family, addr, ipstr, sizeof(ipstr)); 77 | } 78 | 79 | return string(ipstr); 80 | } 81 | -------------------------------------------------------------------------------- /utils/fawnnet.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _NET_H_ 3 | #define _NET_H_ 4 | 5 | using namespace std; 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | namespace fawn { 12 | int connectTCP(string IP, int port); 13 | string getMyIP(); 14 | } 15 | #endif 16 | -------------------------------------------------------------------------------- /utils/fnv.h: -------------------------------------------------------------------------------- 1 | /* 2 | * fnv - Fowler/Noll/Vo- hash code 3 | * 4 | * @(#) $Revision: 5.4 $ 5 | * @(#) $Id: fnv.h,v 5.4 2009/07/30 22:49:13 chongo Exp $ 6 | * @(#) $Source: /usr/local/src/cmd/fnv/RCS/fnv.h,v $ 7 | * 8 | *** 9 | * 10 | * Fowler/Noll/Vo- hash 11 | * 12 | * The basis of this hash algorithm was taken from an idea sent 13 | * as reviewer comments to the IEEE POSIX P1003.2 committee by: 14 | * 15 | * Phong Vo (http://www.research.att.com/info/kpv/) 16 | * Glenn Fowler (http://www.research.att.com/~gsf/) 17 | * 18 | * In a subsequent ballot round: 19 | * 20 | * Landon Curt Noll (http://www.isthe.com/chongo/) 21 | * 22 | * improved on their algorithm. Some people tried this hash 23 | * and found that it worked rather well. In an EMail message 24 | * to Landon, they named it the ``Fowler/Noll/Vo'' or FNV hash. 25 | * 26 | * FNV hashes are designed to be fast while maintaining a low 27 | * collision rate. The FNV speed allows one to quickly hash lots 28 | * of data while maintaining a reasonable collision rate. See: 29 | * 30 | * http://www.isthe.com/chongo/tech/comp/fnv/index.html 31 | * 32 | * for more details as well as other forms of the FNV hash. 33 | * 34 | *** 35 | * 36 | * NOTE: The FNV-0 historic hash is not recommended. One should use 37 | * the FNV-1 hash instead. 38 | * 39 | * To use the 32 bit FNV-0 historic hash, pass FNV0_32_INIT as the 40 | * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). 41 | * 42 | * To use the 64 bit FNV-0 historic hash, pass FNV0_64_INIT as the 43 | * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). 44 | * 45 | * To use the recommended 32 bit FNV-1 hash, pass FNV1_32_INIT as the 46 | * Fnv32_t hashval argument to fnv_32_buf() or fnv_32_str(). 47 | * 48 | * To use the recommended 64 bit FNV-1 hash, pass FNV1_64_INIT as the 49 | * Fnv64_t hashval argument to fnv_64_buf() or fnv_64_str(). 50 | * 51 | * To use the recommended 32 bit FNV-1a hash, pass FNV1_32A_INIT as the 52 | * Fnv32_t hashval argument to fnv_32a_buf() or fnv_32a_str(). 53 | * 54 | * To use the recommended 64 bit FNV-1a hash, pass FNV1A_64_INIT as the 55 | * Fnv64_t hashval argument to fnv_64a_buf() or fnv_64a_str(). 56 | * 57 | *** 58 | * 59 | * Please do not copyright this code. This code is in the public domain. 60 | * 61 | * LANDON CURT NOLL DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 62 | * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO 63 | * EVENT SHALL LANDON CURT NOLL BE LIABLE FOR ANY SPECIAL, INDIRECT OR 64 | * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF 65 | * USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 66 | * OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 67 | * PERFORMANCE OF THIS SOFTWARE. 68 | * 69 | * By: 70 | * chongo /\oo/\ 71 | * http://www.isthe.com/chongo/ 72 | * 73 | * Share and Enjoy! :-) 74 | */ 75 | 76 | #if !defined(__FNV_H__) 77 | #define __FNV_H__ 78 | 79 | #include 80 | 81 | #define FNV_VERSION "5.0.2" /* @(#) FNV Version */ 82 | 83 | 84 | /* 85 | * 32 bit FNV-0 hash type 86 | */ 87 | typedef u_int32_t Fnv32_t; 88 | 89 | 90 | /* 91 | * 32 bit FNV-0 zero initial basis 92 | * 93 | * This historic hash is not recommended. One should use 94 | * the FNV-1 hash and initial basis instead. 95 | */ 96 | #define FNV0_32_INIT ((Fnv32_t)0) 97 | 98 | 99 | /* 100 | * 32 bit FNV-1 and FNV-1a non-zero initial basis 101 | * 102 | * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: 103 | * 104 | * chongo /\../\ 105 | * 106 | * NOTE: The \'s above are not back-slashing escape characters. 107 | * They are literal ASCII backslash 0x5c characters. 108 | * 109 | * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. 110 | */ 111 | #define FNV1_32_INIT ((Fnv32_t)0x811c9dc5) 112 | #define FNV1_32A_INIT FNV1_32_INIT 113 | 114 | 115 | /* 116 | * 64 bit FNV-0 hash 117 | */ 118 | typedef u_int64_t Fnv64_t; 119 | 120 | 121 | /* 122 | * 64 bit FNV-0 zero initial basis 123 | * 124 | * This historic hash is not recommended. One should use 125 | * the FNV-1 hash and initial basis instead. 126 | */ 127 | #define FNV0_64_INIT ((Fnv64_t)0) 128 | 129 | 130 | /* 131 | * 64 bit FNV-1 non-zero initial basis 132 | * 133 | * The FNV-1 initial basis is the FNV-0 hash of the following 32 octets: 134 | * 135 | * chongo /\../\ 136 | * 137 | * NOTE: The \'s above are not back-slashing escape characters. 138 | * They are literal ASCII backslash 0x5c characters. 139 | * 140 | * NOTE: The FNV-1a initial basis is the same value as FNV-1 by definition. 141 | */ 142 | #define FNV1_64_INIT ((Fnv64_t)0xcbf29ce484222325ULL) 143 | #define FNV1A_64_INIT FNV1_64_INIT 144 | 145 | 146 | /* 147 | * hash types 148 | */ 149 | enum fnv_type { 150 | FNV_NONE = 0, /* invalid FNV hash type */ 151 | FNV0_32 = 1, /* FNV-0 32 bit hash */ 152 | FNV1_32 = 2, /* FNV-1 32 bit hash */ 153 | FNV1a_32 = 3, /* FNV-1a 32 bit hash */ 154 | FNV0_64 = 4, /* FNV-0 64 bit hash */ 155 | FNV1_64 = 5, /* FNV-1 64 bit hash */ 156 | FNV1a_64 = 6, /* FNV-1a 64 bit hash */ 157 | }; 158 | 159 | 160 | /* 161 | * these test vectors are used as part o the FNV test suite 162 | */ 163 | struct test_vector { 164 | void *buf; /* start of test vector buffer */ 165 | int len; /* length of test vector */ 166 | }; 167 | struct fnv0_32_test_vector { 168 | struct test_vector *test; /* test vector buffer to hash */ 169 | Fnv32_t fnv0_32; /* expected FNV-0 32 bit hash value */ 170 | }; 171 | struct fnv1_32_test_vector { 172 | struct test_vector *test; /* test vector buffer to hash */ 173 | Fnv32_t fnv1_32; /* expected FNV-1 32 bit hash value */ 174 | }; 175 | struct fnv1a_32_test_vector { 176 | struct test_vector *test; /* test vector buffer to hash */ 177 | Fnv32_t fnv1a_32; /* expected FNV-1a 32 bit hash value */ 178 | }; 179 | struct fnv0_64_test_vector { 180 | struct test_vector *test; /* test vector buffer to hash */ 181 | Fnv64_t fnv0_64; /* expected FNV-0 64 bit hash value */ 182 | }; 183 | struct fnv1_64_test_vector { 184 | struct test_vector *test; /* test vector buffer to hash */ 185 | Fnv64_t fnv1_64; /* expected FNV-1 64 bit hash value */ 186 | }; 187 | struct fnv1a_64_test_vector { 188 | struct test_vector *test; /* test vector buffer to hash */ 189 | Fnv64_t fnv1a_64; /* expected FNV-1a 64 bit hash value */ 190 | }; 191 | 192 | 193 | /* 194 | * external functions 195 | */ 196 | /* hash_32.c */ 197 | extern Fnv32_t fnv_32_buf(void *buf, size_t len, Fnv32_t hashval); 198 | extern Fnv32_t fnv_32_str(char *buf, Fnv32_t hashval); 199 | 200 | /* hash_32a.c */ 201 | extern Fnv32_t fnv_32a_buf(void *buf, size_t len, Fnv32_t hashval); 202 | extern Fnv32_t fnv_32a_str(char *buf, Fnv32_t hashval); 203 | 204 | /* hash_64.c */ 205 | extern Fnv64_t fnv_64_buf(void *buf, size_t len, Fnv64_t hashval); 206 | extern Fnv64_t fnv_64_str(char *buf, Fnv64_t hashval); 207 | 208 | /* hash_64a.c */ 209 | extern Fnv64_t fnv_64a_buf(void *buf, size_t len, Fnv64_t hashval); 210 | extern Fnv64_t fnv_64a_str(char *buf, Fnv64_t hashval); 211 | 212 | /* test_fnv.c */ 213 | extern struct test_vector fnv_test_str[]; 214 | extern struct fnv0_32_test_vector fnv0_32_vector[]; 215 | extern struct fnv1_32_test_vector fnv1_32_vector[]; 216 | extern struct fnv1a_32_test_vector fnv1a_32_vector[]; 217 | extern struct fnv0_64_test_vector fnv0_64_vector[]; 218 | extern struct fnv1_64_test_vector fnv1_64_vector[]; 219 | extern struct fnv1a_64_test_vector fnv1a_64_vector[]; 220 | extern void unknown_hash_type(char *prog, enum fnv_type type, int code); 221 | extern void print_fnv32(Fnv32_t hval, Fnv32_t mask, int verbose, char *arg); 222 | extern void print_fnv64(Fnv64_t hval, Fnv64_t mask, int verbose, char *arg); 223 | 224 | 225 | #endif /* __FNV_H__ */ 226 | -------------------------------------------------------------------------------- /utils/hashutil.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _HASHUTIL_H_ 3 | #define _HASHUTIL_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fnv.h" 11 | 12 | using namespace std; 13 | 14 | #define rot(x,k) (((x)<<(k)) | ((x)>>(32-(k)))) 15 | #define mix(a,b,c) \ 16 | { \ 17 | a -= c; a ^= rot(c, 4); c += b; \ 18 | b -= a; b ^= rot(a, 6); a += c; \ 19 | c -= b; c ^= rot(b, 8); b += a; \ 20 | a -= c; a ^= rot(c,16); c += b; \ 21 | b -= a; b ^= rot(a,19); a += c; \ 22 | c -= b; c ^= rot(b, 4); b += a; \ 23 | } 24 | 25 | #define final(a,b,c) \ 26 | { \ 27 | c ^= b; c -= rot(b,14); \ 28 | a ^= c; a -= rot(c,11); \ 29 | b ^= a; b -= rot(a,25); \ 30 | c ^= b; c -= rot(b,16); \ 31 | a ^= c; a -= rot(c,4); \ 32 | b ^= a; b -= rot(a,14); \ 33 | c ^= b; c -= rot(b,24); \ 34 | } 35 | // Assuming little endian 36 | #define HASH_LITTLE_ENDIAN 1 37 | 38 | #define get16bits(d) (*((const uint16_t *) (d))) 39 | 40 | namespace fawn { 41 | class HashUtil { 42 | public: 43 | // Bob Jenkins Hash 44 | static uint32_t BobHash(const void *buf, size_t length, uint32_t seed = 0); 45 | static uint32_t BobHash(const string &s, uint32_t seed = 0); 46 | 47 | // Bob Jenkins Hash that returns two indices in one call 48 | // Useful for Cuckoo hashing, power of two choices, etc. 49 | // Use idx1 before idx2, when possible. idx1 and idx2 should be initialized to seeds. 50 | static void BobHash(const void *buf, size_t length, uint32_t *idx1, uint32_t *idx2); 51 | static void BobHash(const string &s, uint32_t *idx1, uint32_t *idx2); 52 | 53 | // MurmurHash2 54 | static uint32_t MurmurHash(const void *buf, size_t length, uint32_t seed = 0); 55 | static uint32_t MurmurHash(const string &s, uint32_t seed = 0); 56 | 57 | // FNV Hash 58 | static uint32_t FNVHash(const void *buf, size_t length); 59 | static uint32_t FNVHash(const string &s); 60 | 61 | // SuperFastHash 62 | static uint32_t SuperFastHash(const void *buf, size_t len); 63 | static uint32_t SuperFastHash(const string &s); 64 | 65 | // Integer hashes (from Bob Jenkins) 66 | static uint32_t hashint_full_avalanche_1( uint32_t a); 67 | static uint32_t hashint_full_avalanche_2( uint32_t a); 68 | static uint32_t hashint_half_avalanche( uint32_t a); 69 | 70 | // Null hash (shift and mask) 71 | static uint32_t NullHash(const void* buf, size_t length, uint32_t shiftbytes); 72 | 73 | // Wrappers for MD5 and SHA1 hashing using EVP 74 | static std::string MD5Hash(const char* inbuf, size_t in_length); 75 | static std::string SHA1Hash(const char* inbuf, size_t in_length); 76 | 77 | private: 78 | HashUtil(); 79 | }; 80 | } 81 | 82 | #endif // #ifndef _HASHUTIL_H_ 83 | 84 | 85 | -------------------------------------------------------------------------------- /utils/print.cc: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #include 3 | #include 4 | #include "print.h" 5 | 6 | void bytes_into_hex_string(const u_char *data, u_int len, string& dststr) 7 | { 8 | static const char hexes[] = "0123456789ABCDEF"; 9 | dststr.reserve(dststr.size() + len*2); 10 | 11 | for (u_int i = 0; i < len; i++) { 12 | dststr.push_back(hexes[data[i] >> 4]); 13 | dststr.push_back(hexes[data[i] & 0xf]); 14 | } 15 | } 16 | 17 | void bytes_into_hex_string_trunc0s(const u_char *data, u_int len, string& dststr) 18 | { 19 | static const char hexes[] = "0123456789ABCDEF"; 20 | dststr.reserve(dststr.size() + len*2); 21 | 22 | int last_zero = -1; 23 | 24 | for (u_int i = 0; i < len; i++) { 25 | dststr.push_back(hexes[data[i] >> 4]); 26 | dststr.push_back(hexes[data[i] & 0xf]); 27 | 28 | if (data[i] == 0) { 29 | if (last_zero == -1) { 30 | last_zero = i; 31 | } 32 | } 33 | else { 34 | last_zero = -1; 35 | } 36 | } 37 | 38 | if (last_zero != -1) { 39 | dststr.erase(last_zero*2); 40 | dststr +="...0"; 41 | } 42 | } 43 | 44 | 45 | /* May incur a copy; don't use on high-performance path unless you know 46 | * str is refcounted */ 47 | string bytes_to_hex(const u_char* data, u_int len) 48 | { 49 | string r; 50 | //bytes_into_hex_string(data, len, r); 51 | bytes_into_hex_string_trunc0s(data, len, r); 52 | return r; 53 | } 54 | 55 | string bytes_to_hex(const string& s) 56 | { 57 | return bytes_to_hex((const u_char *)s.data(), s.size()); 58 | } 59 | 60 | 61 | int get_digit_value(char digit) 62 | { 63 | if (isdigit(digit)) 64 | { 65 | return digit - '0'; 66 | } 67 | else if (digit >= 'A' && digit <= 'F') 68 | { 69 | return digit - 'A' + 10; 70 | } 71 | else if (digit >= 'a' && digit <= 'f') 72 | { 73 | return digit - 'a' + 10; 74 | } 75 | else // illegal digit 76 | { 77 | return -1; 78 | } 79 | } 80 | 81 | // assumes 2 character string with legal hex digits 82 | string* hex_string_to_bytes(const u_char* hex_num, u_int len) 83 | { 84 | string* p_ret = new string(); 85 | for (u_int i = 0; i < len/2; i++) { 86 | char high = hex_num[i]; 87 | char low = hex_num[i+1]; 88 | int low_val = get_digit_value(low); //convert low to number from 0 to 15 89 | int high_val = get_digit_value(high); //convert high to number from 0 to 15 90 | 91 | if ((low_val < 0) || (high_val < 0)) { 92 | // Narf! 93 | delete p_ret; 94 | return NULL; 95 | } 96 | 97 | char ch = low_val + 16 * high_val; 98 | p_ret->append(1, ch); 99 | } 100 | return p_ret; 101 | } 102 | 103 | 104 | /* 105 | * print data in rows of 16 bytes: offset hex ascii 106 | * 107 | * 00000 47 45 54 20 2f 20 48 54 54 50 2f 31 2e 31 0d 0a GET / HTTP/1.1.. 108 | */ 109 | void print_hex_ascii_line(const u_char *payload, 110 | u_int len, 111 | u_int offset) 112 | { 113 | /* offset */ 114 | printf("%05d ", offset); 115 | 116 | /* hex */ 117 | const u_char *ch = payload; 118 | for (u_int i = 0; i < len; i++) { 119 | printf("%02x ", *ch); 120 | ch++; 121 | /* print extra space after 8th byte for visual aid */ 122 | if (i == 7) 123 | printf(" "); 124 | } 125 | /* print space to handle line less than 8 bytes */ 126 | if (len < 8) 127 | printf(" "); 128 | 129 | /* fill hex gap with spaces if not full line */ 130 | if (len < 16) { 131 | int gap = 16 - len; 132 | for (int i = 0; i < gap; i++) { 133 | printf(" "); 134 | } 135 | } 136 | printf(" "); 137 | 138 | /* ascii (if printable) */ 139 | ch = payload; 140 | for (u_int i = 0; i < len; i++) { 141 | if (isprint(*ch)) 142 | printf("%c", *ch); 143 | else 144 | printf("."); 145 | ch++; 146 | } 147 | 148 | printf("\n"); 149 | 150 | return; 151 | } 152 | 153 | void print_payload(const string &str) { 154 | print_payload((const u_char*)str.data(), str.size()); 155 | } 156 | /* 157 | * print packet payload data (avoid printing binary data) 158 | */ 159 | void print_payload(const u_char *payload, u_int len) 160 | { 161 | const u_int line_width = 16; /* number of bytes per line */ 162 | u_int len_rem = len; 163 | u_int offset = 0; 164 | 165 | while (len_rem > line_width) { 166 | int line_len = line_width % len_rem; 167 | print_hex_ascii_line(payload + offset, line_len, offset); 168 | len_rem -= line_len; 169 | offset += line_width; 170 | } 171 | 172 | /* Might have left a partial line left. */ 173 | if (len_rem > 0) { 174 | print_hex_ascii_line(payload + offset, len_rem, offset); 175 | } 176 | 177 | return; 178 | } 179 | 180 | 181 | 182 | void tokenize(const string& str, 183 | vector& tokens, 184 | const string& delimiters) 185 | { 186 | // Skip delimiters at beginning. 187 | string::size_type lastPos = str.find_first_not_of(delimiters, 0); 188 | // Find first "non-delimiter". 189 | string::size_type pos = str.find_first_of(delimiters, lastPos); 190 | 191 | while (string::npos != pos || string::npos != lastPos) 192 | { 193 | // Found a token, add it to the vector. 194 | tokens.push_back(str.substr(lastPos, pos - lastPos)); 195 | // Skip delimiters. Note the "not_of" 196 | lastPos = str.find_first_not_of(delimiters, pos); 197 | // Find next "non-delimiter" 198 | pos = str.find_first_of(delimiters, lastPos); 199 | } 200 | } 201 | 202 | void int_to_byte(const uint32_t i, char* p_byte_value) 203 | { 204 | uint32_t int_value = i; 205 | for (int x = 0; x < 4; x++) { 206 | p_byte_value[x] = int_value & 0x000000FF; 207 | int_value = int_value >> 8; 208 | cout << "b[" << x << "] = " << hex << uppercase << setw(2) << setfill('0') <<(int)p_byte_value[x] << endl; 209 | } 210 | } 211 | 212 | int fill_file_with_zeros(int fd, size_t nbytes) 213 | { 214 | static const size_t BLOCKSIZE = 8192; 215 | char zeros[BLOCKSIZE]; 216 | memset(zeros, 0, BLOCKSIZE); 217 | while (nbytes > 0) { 218 | size_t bytes_to_write = min(nbytes, BLOCKSIZE); 219 | ssize_t ret = write(fd, zeros, bytes_to_write); 220 | if (ret < 0) { 221 | perror("error in fill_file_with_zeros write"); 222 | return -1; 223 | } 224 | nbytes -= bytes_to_write; 225 | } 226 | return 0; 227 | } 228 | 229 | string getID(string ip, const int32_t port) 230 | { 231 | char sport[7]; 232 | sprintf(sport, "%d", port); 233 | return ip.append(":").append(sport, strlen(sport)); 234 | } 235 | 236 | string getIP(string id) 237 | { 238 | return id.substr(0, id.find(":")); 239 | } 240 | 241 | int getPort(string id) 242 | { 243 | string p = id.substr(id.find(":")+1, id.size()); 244 | return strtol(p.c_str(), NULL, 10); 245 | } 246 | 247 | -------------------------------------------------------------------------------- /utils/print.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _PRINT_H_ 3 | #define _PRINT_H_ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | using namespace std; 18 | 19 | void bytes_into_hex_string(const u_char *data, u_int len, string &dststr); 20 | string bytes_to_hex(const u_char* data, u_int len); 21 | string bytes_to_hex(const string& s); 22 | int get_digit_value(char digit); 23 | string* hex_string_to_bytes(const u_char* hex_num, u_int len); 24 | void print_hex_ascii_line(const u_char *payload, u_int len, u_int offset); 25 | void print_payload(const u_char *payload, u_int len); 26 | void print_payload(const string &str); 27 | void tokenize(const string& str, vector& tokens, const string& delimiters); 28 | 29 | void int_to_byte(const uint32_t i, char* p_byte_value); 30 | int fill_file_with_zeros(int fd, size_t nbytes); 31 | 32 | 33 | string getID(string ip, const int32_t port); 34 | string getIP(string id); 35 | int getPort(string id); 36 | 37 | template 38 | inline string to_string(const T& t) 39 | { 40 | stringstream ss; 41 | ss << t; 42 | return ss.str(); 43 | } 44 | 45 | #endif //_PRINT_H_ 46 | -------------------------------------------------------------------------------- /utils/timing.c: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | /* 3 | * Some benchmarking utils 4 | */ 5 | 6 | #include "timing.h" 7 | 8 | double timeval_diff(const struct timeval * const start, const struct timeval * const end) 9 | { 10 | /* Calculate the second difference*/ 11 | double r = end->tv_sec - start->tv_sec; 12 | 13 | /* Calculate the microsecond difference */ 14 | if (end->tv_usec > start->tv_usec) 15 | r += (end->tv_usec - start->tv_usec)/1000000.0; 16 | else if (end->tv_usec < start->tv_usec) 17 | r -= (start->tv_usec - end->tv_usec)/1000000.0; 18 | 19 | return r; 20 | } 21 | 22 | -------------------------------------------------------------------------------- /utils/timing.h: -------------------------------------------------------------------------------- 1 | /* -*- Mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ 2 | #ifndef _TIMING_H_ 3 | #define _TIMING_H_ 4 | 5 | #include 6 | #include 7 | 8 | #ifdef __cplusplus 9 | extern "C" { 10 | #endif 11 | 12 | double timeval_diff(const struct timeval * const start, const struct timeval * const end); 13 | 14 | 15 | #ifdef __cplusplus 16 | } /* extern C */ 17 | #endif 18 | 19 | 20 | /* You'll have to change the HZ constant in the printf below... */ 21 | #include 22 | #include 23 | #include 24 | 25 | #if defined(__i386__) 26 | 27 | static __inline__ unsigned long long rdtsc(void) 28 | { 29 | unsigned long long int x; 30 | __asm__ volatile (".byte 0x0f, 0x31" : "=A" (x)); 31 | return x; 32 | } 33 | #elif defined(__x86_64__) 34 | 35 | static __inline__ unsigned long long rdtsc(void) 36 | { 37 | unsigned hi, lo; 38 | __asm__ __volatile__ ("rdtsc" : "=a"(lo), "=d"(hi)); 39 | return ( (unsigned long long)lo)|( ((unsigned long long)hi)<<32 ); 40 | } 41 | #endif 42 | 43 | #endif /* _TIMING_H_ */ 44 | -------------------------------------------------------------------------------- /ycsb/README: -------------------------------------------------------------------------------- 1 | YCSB FawnKV client library instructions: 2 | 3 | 1) Place the fawnkv directory under $(YCSB_ROOT)/db/ 4 | 2) Edit build.xml to include fawnkv by adding: 5 | 6 | 7 | 8 | 9 | 10 | 11 | 3) Copy FawnKVClt.jar, libthrift.jar, and the two slf4j jars in fawn/lib/java/ to $(YCSB_ROOT)/db/fawnkv/lib/. 12 | 13 | 4) Edit FawnKVClient.java in $(YCSB_ROOT)/db/fawnkv/src/com/yahoo/ycsb/db/ to specify the frontend IP and port in the init() function. (TODO: make this a command line param?) 14 | 15 | 5) Compile YCSB with FawnKV by typing 16 | $ ant dbcompile-fawnkv 17 | 18 | 19 | Running a workload (workloada as an example) 20 | 21 | 1) Start the FawnKV frontend(s) and backends 22 | 23 | 2) Run: 24 | 25 | $ java -cp build/ycsb.jar:db/fawnkv/lib/* com.yahoo.ycsb.Client -t -db com.yahoo.ycsb.db.FawnKVClient -P workloads/workloada 26 | 27 | 3) See results! -------------------------------------------------------------------------------- /ycsb/fawnkv/lib/README: -------------------------------------------------------------------------------- 1 | This directory should contain jars for building and running FawnKV 2 | 3 | Specifically, you should place: 4 | 5 | FawnKVClt.jar 6 | libthrift.jar 7 | slf4j-api-1.5.8.jar 8 | slf4j-simple-1.5.8.jar 9 | 10 | in this directory. -------------------------------------------------------------------------------- /ycsb/fawnkv/src/com/yahoo/ycsb/db/FawnKVClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * FawnKV client binding for YCSB. 3 | * 4 | * By Vijay Vasudevan 5 | * 6 | * 7 | */ 8 | 9 | package com.yahoo.ycsb.db; 10 | 11 | import java.util.HashMap; 12 | import java.util.Map; 13 | import java.util.Iterator; 14 | import java.util.Properties; 15 | import java.util.Set; 16 | import java.util.Vector; 17 | import java.util.concurrent.atomic.AtomicInteger; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import com.yahoo.ycsb.DB; 23 | import com.yahoo.ycsb.DBException; 24 | 25 | import fawn.*; 26 | 27 | /** 28 | * FawnKV client for YCSB framework. 29 | * 30 | * Properties to set: 31 | * 32 | * @author Vijay Vasudevan 33 | * 34 | */ 35 | public class FawnKVClient extends DB { 36 | static AtomicInteger listenPort = new AtomicInteger(4002); 37 | private fawn.FawnKVClt client; 38 | private int fePort = 4001; 39 | /** 40 | * Initialize any state for this DB. Called once per DB instance; there is 41 | * one DB instance per client thread. 42 | */ 43 | public void init() throws DBException { 44 | int myListenPort = listenPort.getAndIncrement(); 45 | System.out.println("Starting client listening on port " + myListenPort); 46 | client = new fawn.FawnKVClt("localhost", fePort, "localhost", myListenPort); 47 | } 48 | 49 | /** 50 | * Delete a record from the database. 51 | * 52 | * @param table The name of the table 53 | * @param key The record key of the record to delete. 54 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 55 | */ 56 | public int delete(String table, String key) { 57 | client.remove(key); 58 | return 0; 59 | } 60 | 61 | /** 62 | * Insert a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 63 | * record key. 64 | * 65 | * @param table The name of the table 66 | * @param key The record key of the record to insert. 67 | * @param values A HashMap of field/value pairs to insert in the record 68 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 69 | */ 70 | public int insert(String table, String key, HashMap values) { 71 | // Iterate over collection of values, serialize, insert. 72 | StringBuffer data = new StringBuffer(); 73 | for (Map.Entry entry : values.entrySet()) 74 | { 75 | data.append(entry.getKey()); 76 | data.append(":"); 77 | data.append(entry.getValue()); 78 | data.append(";"); 79 | } 80 | client.put(key, data.toString()); 81 | return 0; 82 | } 83 | 84 | /** 85 | * Read a record from the database. Each field/value pair from the result will be stored in a HashMap. 86 | * 87 | * @param table The name of the table 88 | * @param key The record key of the record to read. 89 | * @param fields The list of fields to read, or null for all of them 90 | * @param result A HashMap of field/value pairs for the result 91 | * @return Zero on success, a non-zero error code on error or "not found". 92 | */ 93 | public int read(String table, String key, Set fields, 94 | HashMap result) { 95 | //System.out.println("Reading (client port " + myPort + ")"); 96 | String data = client.get(key); 97 | String[] values = data.split(";"); 98 | for (int i = 0; i < values.length; i++) { 99 | String[] pair = values[i].split(":"); 100 | if (fields != null && (fields.isEmpty() || fields.contains(pair[0]))) { 101 | result.put(pair[0], pair[1]); 102 | } 103 | } 104 | return 0; 105 | 106 | } 107 | 108 | /** 109 | * Perform a range scan for a set of records in the database. Each field/value pair from the result will be stored in a HashMap. 110 | * 111 | * @param table The name of the table 112 | * @param startkey The record key of the first record to read. 113 | * @param recordcount The number of records to read 114 | * @param fields The list of fields to read, or null for all of them 115 | * @param result A Vector of HashMaps, where each HashMap is a set field/value pairs for one record 116 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 117 | */ 118 | public int scan(String table, String startkey, int recordcount, 119 | Set fields, Vector> result) { 120 | // Unsupported for now 121 | return 1; 122 | } 123 | 124 | /** 125 | * Update a record in the database. Any field/value pairs in the specified values HashMap will be written into the record with the specified 126 | * record key, overwriting any existing values with the same field name. 127 | * 128 | * @param table The name of the table 129 | * @param key The record key of the record to write. 130 | * @param values A HashMap of field/value pairs to update in the record 131 | * @return Zero on success, a non-zero error code on error. See this class's description for a discussion of error codes. 132 | */ 133 | public int update(String table, String key, HashMap values) { 134 | //System.out.println("Updating (client port " + myPort + ")"); 135 | return insert(table, key, values); 136 | } 137 | } 138 | 139 | 140 | --------------------------------------------------------------------------------