├── .gitignore ├── .gitmodules ├── .travis.yml ├── CMakeLists.txt ├── LICENSE ├── README.md ├── debian └── etc │ ├── gibson │ └── gibson.conf │ └── init.d │ └── gibson ├── devel ├── commands.json └── snapshot.sh └── src ├── config.c ├── config.h ├── configure.h.in ├── default.h ├── endianness.c ├── endianness.h ├── gibson.c ├── llist.c ├── llist.h ├── log.c ├── log.h ├── lzf.h ├── lzfP.h ├── lzf_c.c ├── lzf_d.c ├── mux ├── epoll.c ├── evport.c ├── kqueue.c └── select.c ├── net.c ├── net.h ├── obpool.c ├── obpool.h ├── query.c ├── query.h ├── server.c ├── server.h ├── trie.c ├── trie.h ├── zmem.c └── zmem.h /.gitignore: -------------------------------------------------------------------------------- 1 | tests 2 | *.test.conf 3 | *.o 4 | *.lo 5 | *.so 6 | *.so.* 7 | *.swp 8 | *.swo 9 | *.deb 10 | *.rpm 11 | *.gz 12 | .DS_Store 13 | CPack* 14 | _CPack* 15 | tags 16 | gibson 17 | # Eclipse files and directories 18 | .cproject 19 | .project 20 | .settings 21 | # Misc 22 | *.log 23 | src/configure.h 24 | build 25 | CMakeCache.txt 26 | CMakeFiles 27 | CMakeScripts 28 | Makefile 29 | cmake_install.cmake 30 | install_manifest.txt 31 | gibson.build 32 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tests/libgibsonclient"] 2 | path = tests/libgibsonclient 3 | url = git://github.com/evilsocket/libgibsonclient.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: c 2 | cache: ccache 3 | #sudo: false 4 | compiler: 5 | - gcc 6 | - clang 7 | script: cmake . && make 8 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.6) 2 | 3 | project(gibson) 4 | 5 | OPTION( WITH_DEBUG "enable debug module" OFF ) 6 | 7 | # cmake needed modules 8 | include_directories("${PROJECT_SOURCE_DIR}/src") 9 | include(CheckIncludeFiles) 10 | include(CheckLibraryExists) 11 | 12 | # common compilation flags 13 | if (WITH_DEBUG) 14 | message(STATUS "Configuring for debug") 15 | set( CMAKE_BUILD_TYPE Debug ) 16 | set( OPTIMIZATION "-g -pg" ) 17 | else (WITH_DEBUG) 18 | message(STATUS "Configuring for release") 19 | set( CMAKE_BUILD_TYPE Release ) 20 | set( OPTIMIZATION "-g -O3 -DNDEBUG" ) 21 | endif (WITH_DEBUG) 22 | 23 | set( COMMON_FLAGS "-Wall ${OPTIMIZATION}" ) 24 | 25 | set(HAVE_JEMALLOC 0) 26 | 27 | if (WITH_JEMALLOC) 28 | FIND_LIBRARY(JEMALLOC_LIB jemalloc) 29 | if (JEMALLOC_LIB) 30 | CHECK_LIBRARY_EXISTS(jemalloc mallctl "" HAVE_JEMALLOC_FUN) 31 | if (HAVE_JEMALLOC_FUN) 32 | message(STATUS "Using jemalloc memory allocator at ${JEMALLOC_LIB}") 33 | set(HAVE_JEMALLOC 1) 34 | else() 35 | message(STATUS "Found jemalloc at ${JEMALLOC_LIB}, but unable to find its API " 36 | "(maybe the library was configured with a non-empty function prefix?)") 37 | endif() 38 | else() 39 | message(STATUS "Can't find jemalloc") 40 | endif() 41 | else (WITH_JEMALLOC) 42 | message(STATUS "Using standard libc memory allocator." ) 43 | endif (WITH_JEMALLOC) 44 | 45 | 46 | # configure variables 47 | EXECUTE_PROCESS(COMMAND "date" "+%m/%d/%Y %H:%M:%S" OUTPUT_VARIABLE BUILD_DATETIME OUTPUT_STRIP_TRAILING_WHITESPACE) 48 | 49 | if (PREFIX) 50 | message(STATUS "Using prefix ${PREFIX}") 51 | else (WITH_DEBUG) 52 | message(STATUS "Using default prefix") 53 | set( PREFIX /usr ) 54 | endif () 55 | 56 | set( PROJECT "gibson" ) 57 | set( AUTHOR "Simone Margaritelli " ) 58 | set( LICENSE "BSD License" ) 59 | set( VERSION "1.2.1" ) 60 | 61 | set( TESTS_FOLDER "tests" ) 62 | set( TESTS_REPO "https://github.com/evilsocket/ruby-gibson.git" ) 63 | 64 | # this is needed for Mac OS X compilation compatibility 65 | include_directories("${PROJECT_SOURCE_DIR}/src") 66 | 67 | file( GLOB MAIN_SOURCES src/*.c ) 68 | file( GLOB HEADERS src/*.h ) 69 | 70 | # configure.h generation 71 | configure_file( src/configure.h.in src/configure.h ) 72 | # generation 73 | add_executable( ${PROJECT} ${MAIN_SOURCES} ) 74 | 75 | # backtrace is available in a separate library under FreeBSD 76 | if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") 77 | message( STATUS "Detected FreeBSD - Using libexecinfo." ) 78 | include_directories(/usr/local/include) 79 | # cmake seems pretty buggy under FreeBSD :( 80 | link_directories(/usr/local/lib) 81 | set( CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -L/usr/local/lib" ) 82 | set( CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -L/usr/local/lib" ) 83 | set( CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} -L/usr/local/lib" ) 84 | 85 | target_link_libraries( ${PROJECT} execinfo ) 86 | endif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD") 87 | 88 | set_target_properties( ${PROJECT} PROPERTIES COMPILE_FLAGS "${COMMON_CFLAGS}" ) 89 | 90 | if ( HAVE_JEMALLOC EQUAL 1 ) 91 | target_link_libraries( ${PROJECT} jemalloc ) 92 | endif ( HAVE_JEMALLOC EQUAL 1 ) 93 | 94 | install( TARGETS ${PROJECT} DESTINATION ${PREFIX}/bin ) 95 | install( FILES debian/etc/${PROJECT}/${PROJECT}.conf DESTINATION /etc/${PROJECT}/ ) 96 | install( FILES debian/etc/init.d/${PROJECT} DESTINATION /etc/init.d/ 97 | PERMISSIONS 98 | OWNER_READ 99 | OWNER_WRITE 100 | OWNER_EXECUTE 101 | GROUP_READ 102 | GROUP_EXECUTE 103 | WORLD_READ 104 | WORLD_EXECUTE 105 | ) 106 | 107 | ## tests 108 | 109 | # check if ruby is installed 110 | find_program( RUBY_BIN NAMES ruby ) 111 | 112 | if ( NOT RUBY_BIN ) 113 | 114 | message( STATUS "-- Could not find ruby, tests disabled" ) 115 | 116 | else ( NOT RUBY_BIN ) 117 | 118 | message( "-- Found ruby : ${RUBY_BIN}\n-- Run 'make test' to execute testsuite" ) 119 | 120 | if ( NOT IS_DIRECTORY "${TESTS_FOLDER}" ) 121 | message( STATUS "-- Folder '${TESTS_FOLDER}' not found, cloning ${TESTS_REPO} ..." ) 122 | 123 | EXECUTE_PROCESS(COMMAND ${GIT_COMMAND} clone ${TESTS_REPO} ${TESTS_FOLDER}) 124 | endif ( NOT IS_DIRECTORY "${TESTS_FOLDER}" ) 125 | 126 | add_custom_target( test COMMAND cd ${TESTS_FOLDER} && rake test ) 127 | endif ( NOT RUBY_BIN ) 128 | 129 | #packaging 130 | if (UNIX) 131 | 132 | if( CMAKE_SIZEOF_VOID_P EQUAL 8 ) 133 | set( CPACK_DEBIAN_PACKAGE_ARCHITECTURE "amd64" ) 134 | else( CMAKE_SIZEOF_VOID_P EQUAL 8 ) 135 | set( CPACK_DEBIAN_PACKAGE_ARCHITECTURE "i386" ) 136 | endif( CMAKE_SIZEOF_VOID_P EQUAL 8 ) 137 | 138 | set( CPACK_GENERATOR "DEB;RPM") 139 | set( CPACK_PACKAGE_NAME "gibson-server" ) 140 | set( CPACK_PACKAGE_VERSION ${VERSION} ) 141 | set( CPACK_PACKAGE_FILE_NAME "${CPACK_PACKAGE_NAME}-${CPACK_PACKAGE_VERSION}-${CPACK_DEBIAN_PACKAGE_ARCHITECTURE}" ) 142 | set( CPACK_PACKAGE_DESCRIPTION_SUMMARY "A high performance tree-based cache server.") 143 | set( CPACK_PACKAGE_DESCRIPTION "A high performance tree-based cache server.") 144 | set( CPACK_DEBIAN_PACKAGE_SECTION "database" ) 145 | set( CPACK_PACKAGE_CONTACT "Simone Margaritelli evilsocket@gmail.com") 146 | 147 | include(CPack) 148 | 149 | endif (UNIX) 150 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Software License Agreement (BSD License) 2 | 3 | Copyright (c) 2013, Simone Margaritelli 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, 10 | this list of conditions and the following disclaimer. 11 | * Redistributions in binary form must reproduce the above copyright 12 | notice, this list of conditions and the following disclaimer in the 13 | documentation and/or other materials provided with the distribution. 14 | * Neither the name of Gibson nor the names of its contributors may be used 15 | to endorse or promote products derived from this software without 16 | specific prior written permission. 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 22 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 | POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Gibson [![Build Status](https://secure.travis-ci.org/evilsocket/gibson.png)](http://travis-ci.org/evilsocket/gibson) 2 | === 3 | 4 | * To report issues : 5 | * Gibson website : 6 | * Documentation : 7 | 8 | ## Introduction 9 | 10 | Gibson is a high efficiency, tree based memory cache server. 11 | Normal key-value stores ( memcache, Redis, etc ) uses a hash table as their main data structure, so every key is hashed with a specific algorithm and the resulting hash is used to identify the given value in memory. This approach, although very fast, doesn't allow the user to execute globbing expressions/selections on a given (multiple) keyset, thus resulting on a pure one-by-one access paradigm. 12 | Gibson is different, it uses a special tree based structure allowing the user to perform operations on multiple key sets using a prefix expression achieving the same performance grades in the worst case, even better on an average case. 13 | Unlike many other server applications, it's not multithreaded, but it uses multiplexing taking advantace of an event-driven network layer ( just like Node.js, or Nginx using libevent and so on ) which provides higher performances even on low cost hardware. 14 | 15 | **You need Gibson if:** 16 | 17 | * You need to cache result sets from heavy queries lowering your main database load and response times. 18 | * You want to invalidate or rietrive multiple cached itemshierarchically with a single command given their common prefix, in lower than linear time. 19 | * You need a cache backend which is fast, highly scalable and not redundant as the common key value store. 20 | 21 | **You can't use Gibson to:** 22 | 23 | * You can't use Gibson as a drop in replacement for your database, since it is NOT a database but a key value store. 24 | * You need complex data structures such as lists, sets, etc to be cached as they are ( without serializing it ). 25 | * You need to perform sorting on cached items. 26 | 27 | ## Features 28 | 29 | * Very fast and with the lowest memory footprint possible 30 | * Fast LZF object compression 31 | * Builtin object Time-To-Live 32 | * Cached object locking and unlocking 33 | * Multiple key set operation with M* operators 34 | 35 | Documentation on 36 | 37 | ## License 38 | 39 | Released under the BSD license. 40 | Copyright © 2013, Simone Margaritelli 41 | All rights reserved. 42 | 43 | [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/evilsocket/gibson/trend.png)](https://bitdeli.com/free "Bitdeli Badge") 44 | -------------------------------------------------------------------------------- /debian/etc/gibson/gibson.conf: -------------------------------------------------------------------------------- 1 | logfile /var/log/gibson.log 2 | # 0 = DEBUG, 1 = INFO, 2 = WARNING, 3 = ERROR 3 | loglevel 1 4 | # fflush the log every N entries 5 | logflushrate 1 6 | 7 | # server config, to create a tcp server instead use: 8 | # address 127.0.0.1 9 | # port 10128 10 | unix_socket /var/run/gibson.sock 11 | # daemonize process 12 | daemonize 1 13 | pidfile /var/run/gibson.pid 14 | 15 | # max memory a gibson instance can use, above this size older items 16 | # will be collected to free space 17 | max_memory 1G 18 | # 1 month 19 | max_item_ttl 2592000 20 | # this value will be used to set the tcp keepalive flag on every client socket. 21 | max_idletime 30 22 | # max simultaneus client 23 | max_clients 255 24 | # max client request size, since each client request buffer is preallocated, 25 | # keep in mind that if you use persistent connections from you client side 26 | # you will have at least max_request_size * clients MB of used memory, so 27 | # with 255 max clients and a 10MB max request size you will have more than 28 | # 2GB of memory used. 29 | max_request_size 2M 30 | # max key size 31 | max_key_size 1K 32 | # max value size ( max_request_size - max_key_size - 1 ) 33 | max_value_size 2096127 34 | # max response size 35 | max_response_size 15M 36 | # data that is not being accessed in the last 'gc_ratio' seconds get 37 | # deleted if the server needs memory. 38 | # 39 | # valid multipler s ( seconds ), m ( minutes ), h ( hours ), d ( days ) 40 | gc_ratio 15m 41 | 42 | # data above this size is going to be LZF compressed 43 | compression 4K 44 | # number of milliseconds between each cron schedule, do not put a value higher than 1000 :) 45 | cron_period 100 46 | # Check for expired items every 'expired_cron' seconds. 47 | # Expired items can be freed every ttl_cron period or when a client 48 | # is trying to access them. 49 | # 50 | # valid multipler s ( seconds ), m ( minutes ), h ( hours ), d ( days ) 51 | expired_cron 15s 52 | # Check if max memory usage is reached every 'max_mem_cron' seconds. 53 | # Until some memory is not free by this cron, every new value is 54 | # rejected. 55 | # 56 | # valid multipler s ( seconds ), m ( minutes ), h ( hours ), d ( days ) 57 | max_mem_cron 15s 58 | 59 | -------------------------------------------------------------------------------- /debian/etc/init.d/gibson: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: gibson 5 | # Required-Start: $local_fs $remote_fs $network $syslog $named 6 | # Required-Stop: $local_fs $remote_fs $network $syslog $named 7 | # Default-Start: 2 3 4 5 8 | # Default-Stop: 0 1 6 9 | # Short-Description: starts the gibson cache server 10 | # Description: starts gibson using start-stop-daemon 11 | ### END INIT INFO 12 | 13 | PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin 14 | DAEMON=/usr/bin/gibson 15 | NAME=gibson 16 | DESC=gibson 17 | 18 | # Include gibson defaults if available 19 | if [ -f /etc/default/gibson ]; then 20 | . /etc/default/gibson 21 | fi 22 | 23 | test -x $DAEMON || exit 0 24 | 25 | set -e 26 | 27 | . /lib/lsb/init-functions 28 | 29 | PID=$(awk -F'[ ;]' '$1 !~ /^#/ && /pid/ {print $2}' /etc/gibson/gibson.conf) 30 | if [ -z "$PID" ] 31 | then 32 | PID=/var/run/gibson.pid 33 | fi 34 | 35 | # Check if the ULIMIT is set in /etc/default/gibson 36 | if [ -n "$ULIMIT" ]; then 37 | # Set the ulimits 38 | ulimit $ULIMIT 39 | fi 40 | 41 | start() { 42 | start-stop-daemon --start --quiet --pidfile $PID \ 43 | --retry 5 --exec $DAEMON --oknodo -- $DAEMON_OPTS 44 | } 45 | 46 | stop() { 47 | start-stop-daemon --stop --quiet --pidfile $PID \ 48 | --retry 5 --oknodo --exec $DAEMON 49 | } 50 | 51 | case "$1" in 52 | start) 53 | log_daemon_msg "Starting $DESC" "$NAME" 54 | start 55 | log_end_msg $? 56 | ;; 57 | 58 | stop) 59 | log_daemon_msg "Stopping $DESC" "$NAME" 60 | stop 61 | log_end_msg $? 62 | ;; 63 | 64 | restart|force-reload) 65 | log_daemon_msg "Restarting $DESC" "$NAME" 66 | stop 67 | sleep 1 68 | start 69 | log_end_msg $? 70 | ;; 71 | 72 | reload) 73 | log_daemon_msg "Reloading $DESC configuration" "$NAME" 74 | start-stop-daemon --stop --signal HUP --quiet --pidfile $PID \ 75 | --oknodo --exec $DAEMON 76 | log_end_msg $? 77 | ;; 78 | 79 | status) 80 | status_of_proc -p $PID "$DAEMON" gibson 81 | ;; 82 | 83 | *) 84 | echo "Usage: $NAME {start|stop|restart|reload|force-reload|status}" >&2 85 | exit 1 86 | ;; 87 | esac 88 | 89 | exit 0 90 | -------------------------------------------------------------------------------- /devel/commands.json: -------------------------------------------------------------------------------- 1 | { 2 | "SET": { 3 | "opcode": 1, 4 | "syntax": "SET ", 5 | "summary": "Set the value for the given key, with an optional TTL.", 6 | "args": [ 7 | { 8 | "name": "ttl", 9 | "type": "int", 10 | "desc": "The optional ttl in seconds." 11 | }, 12 | { 13 | "name": "key", 14 | "type": "string", 15 | "desc": "The key to set." 16 | }, 17 | { 18 | "name": "value", 19 | "type": "string", 20 | "desc": "The value." 21 | } 22 | ], 23 | "example": [ 24 | "SET 0 foo bar // This will create a 'foo' item with the 'bar' content and no TTL.", 25 | "SET 0 foo newvalue // Overwrite the old value of 'foo'.", 26 | "SET 10 hello world // Create a 'hello' item that is going to expire in 10 seconds" 27 | ], 28 | "notes": [ 29 | "This operator will fail for LOCKed items.", 30 | "On success the new set value will be returned, otherwise an error." 31 | ] 32 | }, 33 | "MSET": { 34 | "opcode": 9, 35 | "syntax": "MSET ", 36 | "summary": "Set the value for keys verifying the given prefix.", 37 | "args": [ 38 | { 39 | "name": "key", 40 | "type": "string", 41 | "desc": "The key prefix to use as expression." 42 | }, 43 | { 44 | "name": "value", 45 | "type": "string", 46 | "desc": "The value." 47 | } 48 | ], 49 | "example": [ 50 | "SET 0 app:news:item:hits 10 // This will create a 'app:news:item:hits' item with the '10' content and no TTL.", 51 | "SET 0 app:news:other-item:hits 23", 52 | "MSET app:news 0 // Reset both" 53 | ], 54 | "notes": [ 55 | "This operator will fail for LOCKed items.", 56 | "Return the number of modified items, otherwise an error." 57 | ] 58 | }, 59 | "TTL": { 60 | "opcode": 2, 61 | "syntax": "TTL ", 62 | "summary": "Set the TTL of a key.", 63 | "args": [ 64 | { 65 | "name": "key", 66 | "type": "string", 67 | "desc": "The key." 68 | }, 69 | { 70 | "name": "ttl", 71 | "type": "integer", 72 | "desc": "The TTL in seconds." 73 | } 74 | ], 75 | "example": [ 76 | "SET 0 foo bar", 77 | "TTL foo 10 // Now 'foo' has a 10 seconds ttl" 78 | ], 79 | "notes": [ 80 | "This operator will fail for LOCKed items." 81 | ] 82 | }, 83 | "MTTL": { 84 | "opcode": 10, 85 | "syntax": "MTTL ", 86 | "summary": "Set the TTL for keys verifying the given prefix.", 87 | "args": [ 88 | { 89 | "name": "key", 90 | "type": "string", 91 | "desc": "The key prefix to use as expression." 92 | }, 93 | { 94 | "name": "ttl", 95 | "type": "integer", 96 | "desc": "The TTL in seconds." 97 | } 98 | ], 99 | "example": [ 100 | "SET 0 foo bar", 101 | "SET 0 fuu bar", 102 | "MTTL f 10 // Now both foo and fuu have a 10 seconds ttl" 103 | ], 104 | "notes": [ 105 | "This operator will fail for LOCKed items." 106 | ] 107 | }, 108 | "MUNLOCK": { 109 | "opcode": 16, 110 | "syntax": "MUNLOCK ", 111 | "summary": "Remove the lock on keys verifying the given prefix.", 112 | "args": [ 113 | { 114 | "name": "key", 115 | "type": "string", 116 | "desc": "The key prefix to use as expression." 117 | } 118 | ], 119 | "example": [ 120 | "SET 0 foo bar", 121 | "SET 0 fuu bar", 122 | "MLOCK f 30", 123 | "MUNLOCK f" 124 | ], 125 | "notes": [ 126 | 127 | ] 128 | }, 129 | "UNLOCK": { 130 | "opcode": 8, 131 | "syntax": "UNLOCK ", 132 | "summary": "Remove the lock from the given key.", 133 | "args": [ 134 | { 135 | "name": "key", 136 | "type": "string", 137 | "desc": "The key." 138 | } 139 | ], 140 | "example": [ 141 | "SET 0 foo bar", 142 | "LOCK foo 30", 143 | "UNLOCK foo" 144 | ], 145 | "notes": [ 146 | 147 | ] 148 | }, 149 | "LOCK": { 150 | "opcode": 7, 151 | "syntax": "LOCK