├── .gitignore ├── INSTALL ├── README.md ├── conf └── libshmcache.conf ├── doc └── intro_in_chinese.pdf ├── java ├── README ├── jni │ ├── Makefile.in │ ├── make.sh │ ├── org_csource_shmcache_ShmCache.c │ └── org_csource_shmcache_ShmCache.h ├── pom.xml └── src │ └── main │ └── java │ └── org │ └── csource │ └── shmcache │ ├── ShmCache.java │ ├── ShmCachePropertyConfigAnnotation.java │ └── ShmCachePropertyConfigurer.java ├── libshmcache.spec ├── php-shmcache ├── bench │ ├── redis.php │ └── shmcache.php ├── config.m4 ├── php-shmcache.spec.in ├── php_shmcache.c ├── php_shmcache.h ├── serializer │ ├── igbinary.c │ ├── msgpack.c │ ├── php.c │ ├── shmcache_serializer.c │ └── shmcache_serializer.h ├── shmcache.ini └── test.php ├── src ├── Makefile ├── shm_hashtable.c ├── shm_hashtable.h ├── shm_list.h ├── shm_lock.c ├── shm_lock.h ├── shm_object_pool.c ├── shm_object_pool.h ├── shm_op_wrapper.c ├── shm_op_wrapper.h ├── shm_striping_allocator.c ├── shm_striping_allocator.h ├── shm_value_allocator.c ├── shm_value_allocator.h ├── shmcache.c ├── shmcache.h ├── shmcache_ini_annotation.c ├── shmcache_ini_annotation.h ├── shmcache_types.h ├── shmopt.c ├── shmopt.h ├── tests │ ├── Makefile │ └── tests.c └── tools │ ├── Makefile │ ├── shmcache_delete.c │ ├── shmcache_dump.c │ ├── shmcache_get.c │ ├── shmcache_remove_all.c │ ├── shmcache_set.c │ └── shmcache_stats.c └── tsar-shmcache-module ├── Makefile ├── mod_shmcache.c ├── mod_shmcache.conf └── tsar-shmcache-module.spec.in /.gitignore: -------------------------------------------------------------------------------- 1 | # Makefile.in 2 | java/jni/Makefile 3 | 4 | # Prerequisites 5 | *.d 6 | 7 | # Compiled Object files 8 | *.slo 9 | *.lo 10 | *.o 11 | *.obj 12 | 13 | # Precompiled Headers 14 | *.gch 15 | *.pch 16 | 17 | # Compiled Dynamic libraries 18 | *.so 19 | *.dylib 20 | *.dSYM 21 | *.dll 22 | 23 | # Compiled Static libraries 24 | *.lai 25 | *.la 26 | *.a 27 | *.lib 28 | 29 | # Executables 30 | *.out 31 | java/target 32 | src/tools/shmcache_delete 33 | src/tools/shmcache_dump 34 | src/tools/shmcache_get 35 | src/tools/shmcache_remove_all 36 | src/tools/shmcache_set 37 | src/tools/shmcache_stats 38 | 39 | # other 40 | *.swp 41 | *.swo 42 | php-shmcache/.deps 43 | php-shmcache/.libs/ 44 | php-shmcache/Makefile 45 | php-shmcache/Makefile.fragments 46 | php-shmcache/Makefile.global 47 | php-shmcache/Makefile.objects 48 | php-shmcache/acinclude.m4 49 | php-shmcache/aclocal.m4 50 | php-shmcache/autom4te.cache/ 51 | php-shmcache/build/ 52 | php-shmcache/config.guess 53 | php-shmcache/config.h 54 | php-shmcache/config.h.in 55 | php-shmcache/config.log 56 | php-shmcache/config.nice 57 | php-shmcache/config.status 58 | php-shmcache/config.sub 59 | php-shmcache/configure 60 | php-shmcache/configure.ac 61 | php-shmcache/install-sh 62 | php-shmcache/libtool 63 | php-shmcache/ltmain.sh 64 | php-shmcache/missing 65 | php-shmcache/mkinstalldirs 66 | php-shmcache/run-tests.php 67 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Copy right 2016 Happy Fish / YuQing 2 | 3 | libshmcache may be copied or modified under the terms of BSD License. 4 | Please visit the libshmcache Home Page for more detail. 5 | website: https://github.com/happyfish100/libshmcache 6 | 7 | # step 1. compile && install libfastcommon 8 | git clone https://github.com/happyfish100/libfastcommon 9 | cd libfastcommon && ./make.sh && sudo ./make.sh install 10 | 11 | # step 2. compile && install libshmcache 12 | git clone https://github.com/happyfish100/libshmcache 13 | cd libshmcache/src && make && sudo make install 14 | cd tools && make && sudo make install 15 | cd ../.. 16 | 17 | # step 3. [optional] compile && install libshmcache php extension 18 | cd php-shmcache 19 | phpize 20 | ./configure 21 | make && sudo make install 22 | cd .. 23 | 24 | # step 4. [optional] compile && install java JNI library 25 | cd java/jni 26 | ./make.sh && sudo make install 27 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Copyright (C) 2016 Happy Fish / YuQing 2 | 3 | libshmcache may be copied or modified under the terms of BSD License. 4 | 5 | libshmcache is a local share memory cache for multi processes. 6 | it is a high performance library because read mechanism is lockless. 7 | libshmcache is 100+ times faster than a remote interface such as redis. 8 | 9 | this project contains C library, PHP extension and Java JNI wrapper. 10 | 11 | Its high performance features include: 12 | * pthread mutex lock on write, read is lockless 13 | * use hash table for quickly setting, getting and deletion 14 | * use object pool (FIFO queue) for hash/KV entry allocation 15 | * use striping/block buffer allocator. in a memory striping, 16 | allocate value buffer in order. just only decrease allocator's 17 | used size when free the value buffer. recycle the whole memory 18 | striping/allocator when it's used size reach to zero after memory free up. 19 | * use FIFO elimination algorithm instead of LRU 20 | 21 | stable features are: 22 | * deadlock detection and auto unlock that caused by other crushed process 23 | * check some key fields for consistency when init, old share memories 24 | should be cleaned and reinit when the memory related parameters changed 25 | * add sleep time to avoid other processes reading dirty data when 26 | recycle more than one valid (not expired) hash/KV entries 27 | 28 | other features are: 29 | * support both related processes (such as parent process and subprocesses) 30 | and unrelated processes (such as php-fpm and php CLI, two php CLI processes, etc.) 31 | * incrementally allocate value buffer segment as need, this is controlled 32 | by config parameter: segment_size 33 | * provide abundant stats info: counters for set, get and delete, 34 | memory recycle stats, lock stats etc. 35 | * support atomic increment 36 | * PHP extension support multiple serializers: igbinary, msgpack, php. 37 | these serializers can coexist in a share memory. 38 | 39 | utility commands in directory: src/tools, in /usr/bin/ after make && make install 40 | * shmcache_set: set a key 41 | * shmcache_get: get a key 42 | * shmcache_delete: delete a key 43 | * shmcache_remove_all: remove the share memory 44 | * shmcache_stats: show share memory statistics 45 | 46 | * Note: the key size can not be longer than 64 bytes 47 | 48 | libshmcache PHP extension is supplied in the directory: php-shmcache, support PHP 5 and PHP 7 49 | 50 | ShmCache::__construct(string $config_filename[, long $serializer = 51 | ShmCache::SERIALIZER_IGBINARY]) 52 | * @param config_filename: the config filename as conf/libshmcache.conf 53 | * @param serializer: the serializer type 54 | 
 55 |       ShmCache::SERIALIZER_IGBINARY for igbinary, the default serializer
 56 |       ShmCache::SERIALIZER_MSGPACK for msgpack
 57 |       ShmCache::SERIALIZER_PHP for php serializer
 58 |
59 | * @throws ShmCacheException if the serializer not enabled 60 | * @example: $cache = new ShmCache("/etc/libshmcache.conf"); 61 | * @note: igbinary and msgpack php extensions must be enabled before use them 62 | 
 63 |     check method:
 64 |     php -m | grep igbinary
 65 |     php -m | grep msgpack
 66 |
67 | 68 | boolean ShmCache::set(string $key, mixed $value, long $ttl) 69 | * @param key: the key, must be a string variable 70 | * @param value: the value, any php variable 71 | * @param ttl: timeout / expire in seconds, such as 600 for ten minutes, 72 | ShmCache::NEVER_EXPIRED for never expired 73 | * @return true for success, false for fail 74 | * @throws ShmCacheException if $value is false 75 | * @example: $cache->set($key, $value, 300); 76 | 77 | long ShmCache::incr(string $key, long $increment, long $ttl) 78 | * @desc: atomic increment 79 | * @param key: the key, must be a string variable 80 | * @param increment: the increment integer, can be negative, such as -1 81 | * @param ttl: timeout / expire in seconds, such as 600 for ten minutes, 82 | * @return the value after increase, false for fail 83 | 84 | mixed ShmCache::get(string $key[, boolean $returnExpired = false]) 85 | * @param key: the key, must be a string variable 86 | * @param returnExpired: if return expired key / value 87 | * @return mixed value for success, false for key not exist or expired 88 | * @example: $value = $cache->get($key); 89 | 90 | long ShmCache::getExpires(string $key[, boolean $returnExpired = false]) 91 | * @desc: get expires time as unix timestamp 92 | * @param key: the key, must be a string variable 93 | * @param returnExpired: if return expired key / value 94 | * @return expires timestamps such as 1483952635, 0 for never expired, false for not exist 95 | * @example: $value = $cache->getExpires($key); 96 | 97 | boolean ShmCache::setExpires(string $key, long $expires) 98 | * @param key: the key, must be a string variable 99 | * @param expires: expires timestamp (unix timestamp eg. 1591347245) 100 | ShmCache::NEVER_EXPIRED for never expired 101 | * @return true for success, false for key not exist or expired or other error 102 | * @throws ShmCacheException if $expires is invalid 103 | * @example: $cache->setExpires($key, 1591347245); 104 | 105 | boolean ShmCache::setTTL(string $key, long $ttl) 106 | * @param key: the key, must be a string variable 107 | * @param ttl: timeout / expire in seconds, such as 600 for ten minutes, 108 | ShmCache::NEVER_EXPIRED for never expired 109 | * @return true for success, false for key not exist or expired or other error 110 | * @throws ShmCacheException if $ttl is invalid 111 | * @example: $cache->setTTL($key, 300); 112 | 113 | boolean ShmCache::delete(string $key) 114 | * @param key: the key, must be a string variable 115 | * @return true for success, false for fail 116 | * @example: $cache->delete($key); 117 | 118 | array ShmCache::stats() 119 | * @return stats array 120 | * @example: echo json_encode($cache->stats(), JSON_PRETTY_PRINT); 121 | 122 | boolean ShmCache::clear() 123 | * @desc: clear hashtable to empty. use this function carefully because it will remove all keys in cache 124 | * @return true for success, false for fail 125 | -------------------------------------------------------------------------------- /conf/libshmcache.conf: -------------------------------------------------------------------------------- 1 | 2 | # shared memory type, shm or mmap 3 | # shm for SystemV pure shared memory 4 | # mmap for POSIX shared memory based file 5 | # default value is mmap 6 | # Note: when type is shm, the shm limit is too small in FreeBSD and MacOS, 7 | # you should increase following kernel parameters: 8 | # kern.sysv.shmmax 9 | # kern.sysv.shmall 10 | # kern.sysv.shmseg 11 | # you can refer to http://www.unidata.ucar.edu/software/mcidas/2008/users_guide/workstation.html 12 | type = mmap 13 | 14 | # the lock filename 15 | filename = /tmp/shmcache 16 | 17 | # the memory limit 18 | # the oldest memory will be recycled when this max memory reached 19 | max_memory = 256M 20 | 21 | # the min memory 22 | # default: 0, means do NOT set min memory 23 | min_memory = 0 24 | 25 | # the memory segment size for incremental memory allocation 26 | segment_size = 8M 27 | 28 | # the key number limit 29 | max_key_count = 200000 30 | 31 | # the size limit for one value 32 | max_value_size = 256K 33 | 34 | # the hash function in libfastcommon/src/hash.h 35 | # default: simple_hash 36 | hash_function = simple_hash 37 | 38 | # recycle key number once when reach max keys 39 | # <= 0 means recycle one memory striping 40 | # default: 0 41 | recycle_key_once = 0 42 | 43 | # if recycle valid entries by FIFO 44 | # default: false 45 | recycle_valid_entries = true 46 | 47 | # value allocator policy 48 | # avg. key TTL threshold for recycling memory 49 | # <= 0 for never recycle memory until reach memory limit (max_memory) 50 | # unit: second 51 | # default: 0 52 | value_policy.avg_key_ttl = 5400 53 | 54 | # when the remain memory <= this parameter, discard it 55 | value_policy.discard_memory_size = 128 56 | 57 | # when a value allocator allocate fail times > this parameter, 58 | # means it is almost full, discard it 59 | # default: 5 60 | value_policy.max_fail_times = 5 61 | 62 | # sleep time to avoid other processes read dirty data when 63 | # recycle more than one valid (in TTL / not expired) KV entries 64 | # 0 for never sleep 65 | # unit: microsecond (us) 66 | # default: 0 67 | value_policy.sleep_us_when_recycle_valid_entries = 1000 68 | 69 | # if read (get) within the lock 70 | # default value is false 71 | lock_policy.read_within_lock = false 72 | 73 | # try lock interval in us, must great than zero 74 | # unit: microsecond (us) 75 | # default value is 200 us 76 | lock_policy.trylock_interval_us = 200 77 | 78 | # the interval to detect deadlock caused by the crushed process 79 | # must great than zero 80 | # unit: millisecond (ms) 81 | # default value is 1000 ms 82 | lock_policy.detect_deadlock_interval_ms = 1000 83 | 84 | # standard log level as syslog, case insensitive, value list: 85 | ## emerg for emergency 86 | ## alert 87 | ## crit for critical 88 | ## error 89 | ## warn for warning 90 | ## notice 91 | ## info 92 | ## debug 93 | log_level = info 94 | -------------------------------------------------------------------------------- /doc/intro_in_chinese.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/happyfish100/libshmcache/aba5bbfd0ae0ca63cd18204133e477bc62671c88/doc/intro_in_chinese.pdf -------------------------------------------------------------------------------- /java/README: -------------------------------------------------------------------------------- 1 | 2 | the native Java SDK for libshmcache to get config from the share memory. 3 | it can integrate into the spring framework to make development easy. 4 | 5 | * directories and descriptions: 6 | |__ jni C source codes for JNI, use make.sh to compile and install 7 | |__ src java source codes, use mvn to compile and package 8 | |__ main/java/org/csource/shmcache/ 9 | |__ ShmCache.java libshmcache JNI wrapper 10 | |__ ShmCachePropertyConfigAnnotation.java helper class for @config() in property file 11 | |__ ShmCachePropertyConfigurer.java sub-class of spring PropertyPlaceholderConfigurer 12 | 13 | 14 | * use ShmCache config in the property file: 15 | KEY = @config([keyname][,defaultValue]) 16 | 17 | keyname: the key name in the share memory, use KEY when keyname is empty 18 | defaultValue: default value for the missed key, empty string when not set 19 | 20 | for example: 21 | # get the config value of "app.name" from local share memory 22 | app.name = @config() 23 | 24 | 25 | * config example in spring xml file: 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | classpath:conf/sysconfig.properties 34 | 35 | 36 | 37 | 38 | the properties for class ShmCachePropertyConfigurer: 39 | libraryFilename: [required] the full path library filename for libshmcachejni.so 40 | configFilename: [required] the full path config filename for libshmcache.conf 41 | charset: [optional] the charset for config value, default charset is UTF-8 42 | debug: [optional] debug flag, true or 1 for verbose output, default is false 43 | 44 | 45 | 46 | * example for get config use Spring Bean in Serverlet: 47 | ... 48 | import org.csource.shmcache.ShmCachePropertyConfigurer; 49 | import org.springframework.context.ApplicationContext; 50 | import org.springframework.web.context.support.WebApplicationContextUtils; 51 | ... 52 | 53 | String key = "app.name"; 54 | ApplicationContext ac = WebApplicationContextUtils.getWebApplicationContext(this.getServletContext()); 55 | ShmCachePropertyConfigurer configurer = (ShmCachePropertyConfigurer)ac.getBean("propertySource"); 56 | String value = configurer.getConfig(key); 57 | ... 58 | 59 | 60 | * example for get config in natual Java codes: 61 | package xxx; 62 | import org.csource.shmcache.ShmCache; 63 | ... 64 | 65 | public class XXX { 66 | 67 | static { 68 | ShmCache.setCharset("UTF-8"); 69 | ShmCache.setLibraryFilename("/usr/lib/libshmcachejni.so"); 70 | } 71 | 72 | ... 73 | 74 | public String get(String key) { 75 | ShmCache shmCache = ShmCache.getInstance("/etc/libshmcache.conf"); 76 | return shmCache.getString(key); 77 | } 78 | 79 | ... 80 | } 81 | 82 | -------------------------------------------------------------------------------- /java/jni/Makefile.in: -------------------------------------------------------------------------------- 1 | .SUFFIXES: .c .o .lo 2 | 3 | COMPILE = $(CC) -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -g -O3 4 | PREFIX = $(TARGET_PREFIX) 5 | INC_PATH = -I/usr/local/include $(INCLUDES) 6 | LIB_PATH = -lfastcommon -lshmcache 7 | 8 | SHMCACHE_SHARED_OBJS = org_csource_shmcache_ShmCache.lo 9 | 10 | ALL_OBJS = $(SHMCACHE_SHARED_OBJS) 11 | 12 | ALL_PRGS = 13 | SHARED_LIBS = libshmcachejni.so 14 | ALL_LIBS = $(SHARED_LIBS) 15 | 16 | all: $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) 17 | libshmcachejni.so: 18 | $(COMPILE) -o $@ $< -shared $(SHMCACHE_SHARED_OBJS) $(LIB_PATH) 19 | .c.o: 20 | $(COMPILE) -c -o $@ $< $(INC_PATH) 21 | .c.lo: 22 | $(COMPILE) -c -fPIC -o $@ $< $(INC_PATH) 23 | install: 24 | mkdir -p $(PREFIX)/lib64 25 | mkdir -p $(PREFIX)/lib 26 | install -m 755 $(SHARED_LIBS) $(PREFIX)/lib64 27 | install -m 755 $(SHARED_LIBS) $(PREFIX)/lib 28 | clean: 29 | rm -f $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) 30 | 31 | -------------------------------------------------------------------------------- /java/jni/make.sh: -------------------------------------------------------------------------------- 1 | 2 | TARGET_PREFIX=$DESTDIR/usr 3 | LIB_VERSION=lib64 4 | 5 | if [ -f /usr/include/fastcommon/_os_define.h ]; then 6 | OS_BITS=$(fgrep OS_BITS /usr/include/fastcommon/_os_define.h | awk '{print $NF;}') 7 | elif [ -f /usr/local/include/fastcommon/_os_define.h ]; then 8 | OS_BITS=$(fgrep OS_BITS /usr/local/include/fastcommon/_os_define.h | awk '{print $NF;}') 9 | else 10 | OS_BITS=64 11 | fi 12 | 13 | if [ "$OS_BITS" -eq 64 ]; then 14 | LIB_VERSION=lib64 15 | else 16 | LIB_VERSION=lib 17 | fi 18 | 19 | uname=$(uname) 20 | if [ "$uname" = "Linux" ]; then 21 | LIBS= 22 | elif [ "$uname" = "FreeBSD" ] || [ "$uname" = "Darwin" ]; then 23 | if [ "$uname" = "Darwin" ]; then 24 | out=$(echo $TARGET_PREFIX | fgrep local) 25 | if [ $? -ne 0 ]; then 26 | TARGET_PREFIX=$TARGET_PREFIX/local 27 | fi 28 | LIB_VERSION=lib 29 | fi 30 | elif [ "$uname" = "SunOS" ]; then 31 | export CC=gcc 32 | elif [ "$uname" = "AIX" ]; then 33 | export CC=gcc 34 | fi 35 | 36 | files=$(locate jni.h) 37 | if [ -z "$files" ]; then 38 | echo "can't locate jni.h, please install java SDK first." 39 | exit 2 40 | fi 41 | 42 | count=$(echo "$files" | wc -l) 43 | if [ $count -eq 1 ]; then 44 | filename=$files 45 | else 46 | i=0 47 | for file in $files; do 48 | i=$(expr $i + 1) 49 | echo "$i. $file" 50 | done 51 | 52 | printf "please input the correct file no.: " 53 | read n 54 | if [ -z "$n" ]; then 55 | echo "invalid file no." 56 | exit 2 57 | fi 58 | 59 | filename=$(echo "$files" | head -n $n | tail -n 1) 60 | fi 61 | 62 | INCLUDES= 63 | path=$(dirname $filename) 64 | for d in $(find $path -type d); do 65 | INCLUDES="$INCLUDES -I$d" 66 | done 67 | 68 | sed_replace() 69 | { 70 | sed_cmd=$1 71 | filename=$2 72 | if [ "$uname" = "FreeBSD" ] || [ "$uname" = "Darwin" ]; then 73 | sed -i "" "$sed_cmd" $filename 74 | else 75 | sed -i "$sed_cmd" $filename 76 | fi 77 | } 78 | 79 | cp Makefile.in Makefile 80 | sed_replace "s#\\\$(INCLUDES)#$INCLUDES#g" Makefile 81 | sed_replace "s#\\\$(LIBS)#$LIBS#g" Makefile 82 | sed_replace "s#\\\$(TARGET_PREFIX)#$TARGET_PREFIX#g" Makefile 83 | sed_replace "s#\\\$(LIB_VERSION)#$LIB_VERSION#g" Makefile 84 | make $1 $2 $3 85 | 86 | if [ "$1" = "clean" ]; then 87 | /bin/rm -f Makefile 88 | fi 89 | 90 | -------------------------------------------------------------------------------- /java/jni/org_csource_shmcache_ShmCache.c: -------------------------------------------------------------------------------- 1 | #include "fastcommon/logger.h" 2 | #include "shmcache/shmcache.h" 3 | #include "org_csource_shmcache_ShmCache.h" 4 | 5 | static void throws_exception(JNIEnv *env, const char *message) 6 | { 7 | jclass exClass; 8 | exClass = (*env)->FindClass(env, "java/lang/Exception"); 9 | if (exClass == NULL) { 10 | return; 11 | } 12 | (*env)->ThrowNew(env, exClass, message); 13 | } 14 | 15 | jlong JNICALL Java_org_csource_shmcache_ShmCache_doInit 16 | (JNIEnv *env, jobject obj, jstring config_filename) 17 | { 18 | 19 | struct shmcache_context *context; 20 | jboolean *isCopy = NULL; 21 | const char *szFilename; 22 | int result; 23 | 24 | log_init(); 25 | context = (struct shmcache_context *)malloc(sizeof(struct shmcache_context)); 26 | if (context == NULL) { 27 | logError("file: "__FILE__", line: %d, " 28 | "malloc %d bytes fail", __LINE__, 29 | (int)sizeof(struct shmcache_context)); 30 | throws_exception(env, "out of memory"); 31 | return 0; 32 | } 33 | 34 | szFilename = (*env)->GetStringUTFChars(env, config_filename, isCopy); 35 | result = shmcache_init_from_file(context, szFilename); 36 | 37 | (*env)->ReleaseStringUTFChars(env, config_filename, szFilename); 38 | return result == 0 ? (long)context : 0; 39 | } 40 | 41 | void JNICALL Java_org_csource_shmcache_ShmCache_doDestroy 42 | (JNIEnv *env, jobject obj, jlong handler) 43 | { 44 | struct shmcache_context *context; 45 | 46 | context = (struct shmcache_context *)handler; 47 | if (context == NULL) { 48 | logError("file: "__FILE__", line: %d, " 49 | "empty handler", __LINE__); 50 | throws_exception(env, "empty handler"); 51 | return; 52 | } 53 | 54 | shmcache_destroy(context); 55 | free(context); 56 | } 57 | 58 | void JNICALL Java_org_csource_shmcache_ShmCache_doSetVO 59 | (JNIEnv *env, jobject obj, jlong handler, jstring key, jobject vo) 60 | { 61 | struct shmcache_context *context; 62 | struct shmcache_key_info shmcache_key; 63 | struct shmcache_value_info shmcache_value; 64 | jboolean *isCopy = NULL; 65 | jclass value_class; 66 | jmethodID value_mid; 67 | jmethodID options_mid; 68 | jmethodID expires_mid; 69 | jbyteArray value; 70 | int result; 71 | 72 | context = (struct shmcache_context *)handler; 73 | if (context == NULL) { 74 | logError("file: "__FILE__", line: %d, " 75 | "empty handler", __LINE__); 76 | throws_exception(env, "empty handler"); 77 | return; 78 | } 79 | 80 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 81 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 82 | 83 | value_class = (*env)->GetObjectClass(env, vo); 84 | value_mid = (*env)->GetMethodID(env, value_class, "getValue", "()[B"); 85 | options_mid = (*env)->GetMethodID(env, value_class, "getOptions", "()I"); 86 | expires_mid = (*env)->GetMethodID(env, value_class, "getExpires", "()J"); 87 | 88 | value = (*env)->CallObjectMethod(env, vo, value_mid); 89 | shmcache_value.options = (*env)->CallIntMethod(env, vo, options_mid); 90 | shmcache_value.expires = (*env)->CallLongMethod(env, vo, expires_mid) / 1000; 91 | 92 | shmcache_value.data = (char *)((*env)->GetByteArrayElements(env, value, isCopy)); 93 | shmcache_value.length = (*env)->GetArrayLength(env, value); 94 | 95 | result = shmcache_set_ex(context, &shmcache_key, &shmcache_value); 96 | 97 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 98 | (*env)->ReleaseByteArrayElements(env, value, (jbyte *)shmcache_value.data, 0); 99 | if (result != 0) { 100 | throws_exception(env, strerror(result)); 101 | } 102 | } 103 | 104 | void JNICALL Java_org_csource_shmcache_ShmCache_doSet 105 | (JNIEnv *env, jobject obj, jlong handler, jstring key, 106 | jbyteArray value, jint ttl) 107 | { 108 | struct shmcache_context *context; 109 | struct shmcache_key_info shmcache_key; 110 | jboolean *isCopy = NULL; 111 | jbyte *szValue; 112 | jsize value_len; 113 | int result; 114 | 115 | context = (struct shmcache_context *)handler; 116 | if (context == NULL) { 117 | logError("file: "__FILE__", line: %d, " 118 | "empty handler", __LINE__); 119 | throws_exception(env, "empty handler"); 120 | return; 121 | } 122 | 123 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 124 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 125 | 126 | szValue = (*env)->GetByteArrayElements(env, value, isCopy); 127 | value_len = (*env)->GetArrayLength(env, value); 128 | 129 | result = shmcache_set(context, &shmcache_key, 130 | (const char *)szValue, value_len, ttl); 131 | 132 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 133 | (*env)->ReleaseByteArrayElements(env, value, szValue, 0); 134 | if (result != 0) { 135 | throws_exception(env, strerror(result)); 136 | } 137 | } 138 | 139 | jlong JNICALL Java_org_csource_shmcache_ShmCache_doIncr 140 | (JNIEnv *env, jobject obj, jlong handler, jstring key, 141 | jlong increment, jint ttl) 142 | { 143 | struct shmcache_context *context; 144 | struct shmcache_key_info shmcache_key; 145 | jboolean *isCopy = NULL; 146 | int result; 147 | int64_t new_value = 0; 148 | 149 | context = (struct shmcache_context *)handler; 150 | if (context == NULL) { 151 | logError("file: "__FILE__", line: %d, " 152 | "empty handler", __LINE__); 153 | throws_exception(env, "empty handler"); 154 | return new_value; 155 | } 156 | 157 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 158 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 159 | 160 | result = shmcache_incr(context, &shmcache_key, increment, ttl, &new_value); 161 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 162 | 163 | if (result != 0) { 164 | throws_exception(env, strerror(result)); 165 | } 166 | return new_value; 167 | } 168 | 169 | jobject JNICALL Java_org_csource_shmcache_ShmCache_doGet 170 | (JNIEnv *env, jobject obj, jlong handler, jstring key) 171 | { 172 | #define VALUE_CLASS_NAME "org/csource/shmcache/ShmCache$Value" 173 | 174 | struct shmcache_context *context; 175 | jboolean *isCopy = NULL; 176 | int result; 177 | struct shmcache_key_info shmcache_key; 178 | struct shmcache_value_info shmcache_value; 179 | jbyteArray value; 180 | jobject vo; 181 | 182 | context = (struct shmcache_context *)handler; 183 | if (context == NULL) { 184 | logError("file: "__FILE__", line: %d, " 185 | "empty handler", __LINE__); 186 | throws_exception(env, "empty handler"); 187 | return NULL; 188 | } 189 | 190 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 191 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 192 | result = shmcache_get(context, &shmcache_key, &shmcache_value); 193 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 194 | 195 | if (result == 0) { 196 | char error_info[256]; 197 | jclass value_class; 198 | jmethodID constructor_mid; 199 | 200 | value_class = (*env)->FindClass(env, VALUE_CLASS_NAME); 201 | if (value_class == NULL) { 202 | sprintf(error_info, "can find class: %s", VALUE_CLASS_NAME); 203 | logError("file: "__FILE__", line: %d, %s", 204 | __LINE__, error_info); 205 | throws_exception(env, error_info); 206 | return 0; 207 | } 208 | 209 | constructor_mid = (*env)->GetMethodID(env, value_class, 210 | "", "([BIJ)V"); 211 | if (constructor_mid == NULL) { 212 | sprintf(error_info, "can find constructor for class %s", VALUE_CLASS_NAME); 213 | logError("file: "__FILE__", line: %d, %s", 214 | __LINE__, error_info); 215 | throws_exception(env, error_info); 216 | return 0; 217 | } 218 | 219 | value = (*env)->NewByteArray(env, shmcache_value.length); 220 | (*env)->SetByteArrayRegion(env, value, 0, shmcache_value.length, 221 | (const jbyte *)shmcache_value.data); 222 | 223 | vo = (*env)->NewObject(env, value_class, constructor_mid, 224 | value, shmcache_value.options, 225 | (int64_t)shmcache_value.expires * 1000); 226 | } else { 227 | vo = NULL; 228 | } 229 | 230 | return vo; 231 | } 232 | 233 | jbyteArray JNICALL Java_org_csource_shmcache_ShmCache_doGetBytes 234 | (JNIEnv *env, jobject obj, jlong handler, jstring key) 235 | { 236 | struct shmcache_context *context; 237 | jboolean *isCopy = NULL; 238 | int result; 239 | struct shmcache_key_info shmcache_key; 240 | struct shmcache_value_info shmcache_value; 241 | jbyteArray value; 242 | 243 | context = (struct shmcache_context *)handler; 244 | if (context == NULL) { 245 | logError("file: "__FILE__", line: %d, " 246 | "empty handler", __LINE__); 247 | throws_exception(env, "empty handler"); 248 | return NULL; 249 | } 250 | 251 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 252 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 253 | result = shmcache_get(context, &shmcache_key, &shmcache_value); 254 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 255 | 256 | if (result == 0) { 257 | value = (*env)->NewByteArray(env, shmcache_value.length); 258 | (*env)->SetByteArrayRegion(env, value, 0, shmcache_value.length, 259 | (const jbyte *)shmcache_value.data); 260 | } else { 261 | value = NULL; 262 | } 263 | return value; 264 | } 265 | 266 | jboolean JNICALL Java_org_csource_shmcache_ShmCache_doDelete 267 | (JNIEnv *env, jobject obj, jlong handler, jstring key) 268 | { 269 | struct shmcache_context *context; 270 | jboolean *isCopy = NULL; 271 | struct shmcache_key_info shmcache_key; 272 | int result; 273 | 274 | context = (struct shmcache_context *)handler; 275 | if (context == NULL) { 276 | logError("file: "__FILE__", line: %d, " 277 | "empty handler", __LINE__); 278 | throws_exception(env, "empty handler"); 279 | return false; 280 | } 281 | 282 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 283 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 284 | result = shmcache_delete(context, &shmcache_key); 285 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 286 | return (result == 0); 287 | } 288 | 289 | jboolean JNICALL Java_org_csource_shmcache_ShmCache_doSetTTL 290 | (JNIEnv *env, jobject obj, jlong handler, jstring key, jint ttl) 291 | { 292 | struct shmcache_context *context; 293 | jboolean *isCopy = NULL; 294 | struct shmcache_key_info shmcache_key; 295 | int result; 296 | 297 | context = (struct shmcache_context *)handler; 298 | if (context == NULL) { 299 | logError("file: "__FILE__", line: %d, " 300 | "empty handler", __LINE__); 301 | throws_exception(env, "empty handler"); 302 | return false; 303 | } 304 | 305 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 306 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 307 | result = shmcache_set_ttl(context, &shmcache_key, ttl); 308 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 309 | return (result == 0); 310 | } 311 | 312 | jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetExpires 313 | (JNIEnv *env, jobject obj, jlong handler, jstring key) 314 | { 315 | struct shmcache_context *context; 316 | jboolean *isCopy = NULL; 317 | struct shmcache_key_info shmcache_key; 318 | struct shmcache_value_info shmcache_value; 319 | int result; 320 | 321 | context = (struct shmcache_context *)handler; 322 | if (context == NULL) { 323 | logError("file: "__FILE__", line: %d, " 324 | "empty handler", __LINE__); 325 | throws_exception(env, "empty handler"); 326 | return false; 327 | } 328 | 329 | shmcache_key.length = (*env)->GetStringUTFLength(env, key); 330 | shmcache_key.data = (char *)((*env)->GetStringUTFChars(env, key, isCopy)); 331 | result = shmcache_get(context, &shmcache_key, &shmcache_value); 332 | (*env)->ReleaseStringUTFChars(env, key, shmcache_key.data); 333 | if (result == 0) { 334 | return shmcache_value.expires; 335 | } else { 336 | return -1; 337 | } 338 | } 339 | 340 | jboolean JNICALL Java_org_csource_shmcache_ShmCache_doClear 341 | (JNIEnv *env, jobject obj, jlong handler) 342 | { 343 | struct shmcache_context *context; 344 | 345 | context = (struct shmcache_context *)handler; 346 | if (context == NULL) { 347 | logError("file: "__FILE__", line: %d, " 348 | "empty handler", __LINE__); 349 | throws_exception(env, "empty handler"); 350 | return false; 351 | } 352 | 353 | return shmcache_clear(context) == 0; 354 | } 355 | 356 | jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetLastClearTime 357 | (JNIEnv *env, jobject obj, jlong handler) 358 | { 359 | struct shmcache_context *context; 360 | 361 | context = (struct shmcache_context *)handler; 362 | if (context == NULL) { 363 | logError("file: "__FILE__", line: %d, " 364 | "empty handler", __LINE__); 365 | throws_exception(env, "empty handler"); 366 | return 0; 367 | } 368 | 369 | return shmcache_get_last_clear_time(context); 370 | } 371 | 372 | jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetInitTime 373 | (JNIEnv *env, jobject obj, jlong handler) 374 | { 375 | struct shmcache_context *context; 376 | 377 | context = (struct shmcache_context *)handler; 378 | if (context == NULL) { 379 | logError("file: "__FILE__", line: %d, " 380 | "empty handler", __LINE__); 381 | throws_exception(env, "empty handler"); 382 | return 0; 383 | } 384 | 385 | return shmcache_get_init_time(context); 386 | } 387 | -------------------------------------------------------------------------------- /java/jni/org_csource_shmcache_ShmCache.h: -------------------------------------------------------------------------------- 1 | /* DO NOT EDIT THIS FILE - it is machine generated */ 2 | #include 3 | /* Header for class org_csource_shmcache_ShmCache */ 4 | 5 | #ifndef _Included_org_csource_shmcache_ShmCache 6 | #define _Included_org_csource_shmcache_ShmCache 7 | #ifdef __cplusplus 8 | extern "C" { 9 | #endif 10 | #undef org_csource_shmcache_ShmCache_TTL_NEVER_EXPIRED 11 | #define org_csource_shmcache_ShmCache_TTL_NEVER_EXPIRED 0L 12 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_STRING 13 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_STRING 0L 14 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_INTEGER 15 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_INTEGER 1L 16 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_NONE 17 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_NONE 256L 18 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_IGBINARY 19 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_IGBINARY 512L 20 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_MSGPACK 21 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_MSGPACK 1024L 22 | #undef org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_PHP 23 | #define org_csource_shmcache_ShmCache_SHMCACHE_SERIALIZER_PHP 2048L 24 | /* 25 | * Class: org_csource_shmcache_ShmCache 26 | * Method: doInit 27 | * Signature: (Ljava/lang/String;)J 28 | */ 29 | JNIEXPORT jlong JNICALL Java_org_csource_shmcache_ShmCache_doInit 30 | (JNIEnv *, jobject, jstring); 31 | 32 | /* 33 | * Class: org_csource_shmcache_ShmCache 34 | * Method: doDestroy 35 | * Signature: (J)V 36 | */ 37 | JNIEXPORT void JNICALL Java_org_csource_shmcache_ShmCache_doDestroy 38 | (JNIEnv *, jobject, jlong); 39 | 40 | /* 41 | * Class: org_csource_shmcache_ShmCache 42 | * Method: doSetVO 43 | * Signature: (JLjava/lang/String;Lorg/csource/shmcache/ShmCache/Value;)V 44 | */ 45 | JNIEXPORT void JNICALL Java_org_csource_shmcache_ShmCache_doSetVO 46 | (JNIEnv *, jobject, jlong, jstring, jobject); 47 | 48 | /* 49 | * Class: org_csource_shmcache_ShmCache 50 | * Method: doSet 51 | * Signature: (JLjava/lang/String;[BI)V 52 | */ 53 | JNIEXPORT void JNICALL Java_org_csource_shmcache_ShmCache_doSet 54 | (JNIEnv *, jobject, jlong, jstring, jbyteArray, jint); 55 | 56 | /* 57 | * Class: org_csource_shmcache_ShmCache 58 | * Method: doIncr 59 | * Signature: (JLjava/lang/String;JI)J 60 | */ 61 | JNIEXPORT jlong JNICALL Java_org_csource_shmcache_ShmCache_doIncr 62 | (JNIEnv *, jobject, jlong, jstring, jlong, jint); 63 | 64 | /* 65 | * Class: org_csource_shmcache_ShmCache 66 | * Method: doGetBytes 67 | * Signature: (JLjava/lang/String;)[B 68 | */ 69 | JNIEXPORT jbyteArray JNICALL Java_org_csource_shmcache_ShmCache_doGetBytes 70 | (JNIEnv *, jobject, jlong, jstring); 71 | 72 | /* 73 | * Class: org_csource_shmcache_ShmCache 74 | * Method: doGet 75 | * Signature: (JLjava/lang/String;)Lorg/csource/shmcache/ShmCache/Value; 76 | */ 77 | JNIEXPORT jobject JNICALL Java_org_csource_shmcache_ShmCache_doGet 78 | (JNIEnv *, jobject, jlong, jstring); 79 | 80 | /* 81 | * Class: org_csource_shmcache_ShmCache 82 | * Method: doDelete 83 | * Signature: (JLjava/lang/String;)Z 84 | */ 85 | JNIEXPORT jboolean JNICALL Java_org_csource_shmcache_ShmCache_doDelete 86 | (JNIEnv *, jobject, jlong, jstring); 87 | 88 | /* 89 | * Class: org_csource_shmcache_ShmCache 90 | * Method: doSetTTL 91 | * Signature: (JLjava/lang/String;I)Z 92 | */ 93 | JNIEXPORT jboolean JNICALL Java_org_csource_shmcache_ShmCache_doSetTTL 94 | (JNIEnv *, jobject, jlong, jstring, jint); 95 | 96 | /* 97 | * Class: org_csource_shmcache_ShmCache 98 | * Method: doGetExpires 99 | * Signature: (JLjava/lang/String;)J 100 | */ 101 | JNIEXPORT jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetExpires 102 | (JNIEnv *, jobject, jlong, jstring); 103 | 104 | /* 105 | * Class: org_csource_shmcache_ShmCache 106 | * Method: doClear 107 | * Signature: (J)Z 108 | */ 109 | JNIEXPORT jboolean JNICALL Java_org_csource_shmcache_ShmCache_doClear 110 | (JNIEnv *, jobject, jlong); 111 | 112 | /* 113 | * Class: org_csource_shmcache_ShmCache 114 | * Method: doGetLastClearTime 115 | * Signature: (J)J 116 | */ 117 | JNIEXPORT jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetLastClearTime 118 | (JNIEnv *, jobject, jlong); 119 | 120 | /* 121 | * Class: org_csource_shmcache_ShmCache 122 | * Method: doGetInitTime 123 | * Signature: (J)J 124 | */ 125 | JNIEXPORT jlong JNICALL Java_org_csource_shmcache_ShmCache_doGetInitTime 126 | (JNIEnv *, jobject, jlong); 127 | 128 | #ifdef __cplusplus 129 | } 130 | #endif 131 | #endif 132 | -------------------------------------------------------------------------------- /java/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.csource 8 | libshmcache4j 9 | 1.0-SNAPSHOT 10 | 11 | 5.3.27 12 | 13 | 14 | 15 | 16 | org.springframework 17 | spring-context 18 | ${spring-version} 19 | 20 | 21 | 22 | org.springframework 23 | spring-core 24 | ${spring-version} 25 | 26 | 27 | org.springframework 28 | spring-beans 29 | ${spring-version} 30 | 31 | 32 | org.glassfish 33 | javax.json 34 | 1.0.4 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | org.apache.maven.plugins 43 | maven-compiler-plugin 44 | 45 | 1.8 46 | 1.8 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | -------------------------------------------------------------------------------- /java/src/main/java/org/csource/shmcache/ShmCache.java: -------------------------------------------------------------------------------- 1 | package org.csource.shmcache; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.io.UnsupportedEncodingException; 8 | import java.io.StringReader; 9 | import javax.json.Json; 10 | import javax.json.JsonReader; 11 | import javax.json.JsonArray; 12 | import javax.json.JsonObject; 13 | import javax.json.JsonValue; 14 | 15 | public class ShmCache { 16 | public static class Value { 17 | private byte[] value; 18 | private int options; //options for application 19 | private long expires; //expire time in millisenconds 20 | 21 | public Value(byte[] value, int options, long expires) { 22 | this.value = value; 23 | this.options = options; 24 | this.expires = expires; 25 | } 26 | 27 | public byte[] getValue() { 28 | return this.value; 29 | } 30 | 31 | public int getOptions() { 32 | return this.options; 33 | } 34 | 35 | public long getExpires() { 36 | return this.expires; 37 | } 38 | 39 | public String toString() { 40 | try { 41 | return "value: " + new String(this.value, ShmCache.charset) 42 | + ", options: " + this.options + ", expires: " + this.expires; 43 | } catch (UnsupportedEncodingException ex) { 44 | throw new RuntimeException(ex); 45 | } 46 | } 47 | } 48 | 49 | public static final int TTL_NEVER_EXPIRED = 0; 50 | 51 | public static final int SHMCACHE_SERIALIZER_STRING = 0x1; //string type 52 | public static final int SHMCACHE_SERIALIZER_LIST = (SHMCACHE_SERIALIZER_STRING | 0x2); 53 | public static final int SHMCACHE_SERIALIZER_MAP = (SHMCACHE_SERIALIZER_STRING | 0x4); 54 | public static final int SHMCACHE_SERIALIZER_INTEGER = (SHMCACHE_SERIALIZER_STRING | 0x8); 55 | public static final int SHMCACHE_SERIALIZER_IGBINARY = 0x200; 56 | public static final int SHMCACHE_SERIALIZER_MSGPACK = 0x400; 57 | public static final int SHMCACHE_SERIALIZER_PHP = 0x800; 58 | 59 | private static HashMap instances = new HashMap(); 60 | private static String charset = "UTF-8"; 61 | private static String libraryFilename = null; 62 | 63 | private long handler; 64 | 65 | private native long doInit(String configFilename); 66 | private native void doDestroy(long handler); 67 | private native void doSetVO(long handler, String key, Value value); 68 | private native void doSet(long handler, String key, byte[] value, int ttl); 69 | private native long doIncr(long handler, String key, long increment, int ttl); 70 | private native byte[] doGetBytes(long handler, String key); 71 | private native Value doGet(long handler, String key); 72 | private native boolean doDelete(long handler, String key); 73 | private native boolean doSetTTL(long handler, String key, int ttl); 74 | private native long doGetExpires(long handler, String key); 75 | private native boolean doClear(long handler); 76 | private native long doGetLastClearTime(long handler); 77 | private native long doGetInitTime(long handler); 78 | 79 | public static String getLibraryFilename() { 80 | return libraryFilename; 81 | } 82 | 83 | /** 84 | * set libshmcachejni.so filename to load 85 | * @param filename the full filename 86 | * @return none 87 | */ 88 | public static void setLibraryFilename(String filename) { 89 | if (libraryFilename == null) { 90 | System.load(filename); //load the library 91 | libraryFilename = filename; 92 | } else if (libraryFilename.equals(filename)) { 93 | System.err.println("[WARNING] library " + libraryFilename + " already loaded"); 94 | } else { 95 | throw new RuntimeException("library " + libraryFilename 96 | + " already loaded, can't change to " + filename); 97 | } 98 | } 99 | 100 | public static String getCharset() { 101 | return charset; 102 | } 103 | 104 | public static void setCharset(String value) { 105 | charset = value; 106 | } 107 | 108 | // private for singleton 109 | private ShmCache(String configFilename) { 110 | this.handler = doInit(configFilename); 111 | } 112 | 113 | protected void finalize() throws Throwable { 114 | super.finalize(); 115 | this.close(); 116 | } 117 | 118 | private void close() { 119 | System.out.println("close"); 120 | doDestroy(this.handler); 121 | } 122 | 123 | /** 124 | * get ShmCache instance 125 | * @param configFilename the config filename such as /usr/local/etc/libshmcache.conf 126 | * @return ShmCache object 127 | */ 128 | public synchronized static ShmCache getInstance(String configFilename) { 129 | ShmCache obj = instances.get(configFilename); 130 | if (obj == null) { 131 | obj = new ShmCache(configFilename); 132 | instances.put(configFilename, obj); 133 | } 134 | 135 | return obj; 136 | } 137 | 138 | /** 139 | * clear ShmCache instances 140 | * @return none 141 | */ 142 | public synchronized static void clearInstances() { 143 | instances.clear(); 144 | } 145 | 146 | /** 147 | * get the value of the key 148 | * @param key the key 149 | * @return the value object, null for not exists 150 | */ 151 | public Value getValue(String key) { 152 | return doGet(this.handler, key); 153 | } 154 | 155 | /** 156 | * get byte array value of the key 157 | * @param key the key 158 | * @return byte array value, null for not exists 159 | */ 160 | public byte[] getBytes(String key) { 161 | return doGetBytes(this.handler, key); 162 | } 163 | 164 | /** 165 | * get string value of the key 166 | * @param key the key 167 | * @return string value, null for not exists 168 | */ 169 | public String getString(String key) { 170 | byte[] value = doGetBytes(this.handler, key); 171 | try { 172 | return value != null ? new String(value, charset) : null; 173 | } catch (UnsupportedEncodingException ex) { 174 | throw new RuntimeException(ex); 175 | } 176 | } 177 | 178 | private String unquote(String sv) 179 | { 180 | if (sv.length() >= 2 && (sv.charAt(0) == '"' && 181 | sv.charAt(sv.length() - 1) == '"')) 182 | { 183 | return sv.substring(1, sv.length() - 1); 184 | } else { 185 | return sv; 186 | } 187 | } 188 | 189 | /** 190 | * json string to list 191 | * @param value the value 192 | * @return string list, null for not exists 193 | */ 194 | public List jsonToList(String value) { 195 | ArrayList list = new ArrayList(); 196 | JsonReader jsonReader = Json.createReader(new StringReader(value)); 197 | try { 198 | JsonArray array = jsonReader.readArray(); 199 | for (JsonValue jvalue: array) { 200 | list.add(this.unquote(jvalue.toString())); 201 | } 202 | } finally { 203 | jsonReader.close(); 204 | } 205 | 206 | return list; 207 | } 208 | 209 | /** 210 | * get string list of the key 211 | * @param key the key 212 | * @return string list, null for not exists 213 | */ 214 | public List getList(String key) { 215 | return jsonToList(getString(key)); 216 | } 217 | 218 | /** 219 | * json string to map 220 | * @param value the value 221 | * @return string map, null for not exists 222 | */ 223 | public Map jsonToMap(String value) { 224 | if (value == null) { 225 | return null; 226 | } 227 | HashMap map= new HashMap(); 228 | JsonReader jsonReader = Json.createReader(new StringReader(value)); 229 | try { 230 | JsonObject jobject = jsonReader.readObject(); 231 | for (Map.Entry entry : jobject.entrySet()) { 232 | map.put(entry.getKey(), this.unquote(entry.getValue().toString())); 233 | } 234 | } finally { 235 | jsonReader.close(); 236 | } 237 | 238 | return map; 239 | } 240 | 241 | /** 242 | * get string map of the key 243 | * @param key the key 244 | * @return string map, null for not exists 245 | */ 246 | public Map getMap(String key) { 247 | return jsonToMap(getString(key)); 248 | } 249 | 250 | /** 251 | * get the value of the key 252 | * @param key the key 253 | * @return the value object (String, Long, List, Map or byte[]), 254 | null for not exists 255 | */ 256 | public Object get(String key) { 257 | Value vo = doGet(this.handler, key); 258 | if (vo == null) { 259 | return null; 260 | } 261 | 262 | byte[] bs = vo.getValue(); 263 | try { 264 | switch (vo.getOptions()) { 265 | case SHMCACHE_SERIALIZER_STRING: 266 | return new String(bs, charset); 267 | case SHMCACHE_SERIALIZER_INTEGER: 268 | return Long.parseLong(new String(bs, charset)); 269 | case SHMCACHE_SERIALIZER_LIST: 270 | return jsonToList(new String(bs, charset)); 271 | case SHMCACHE_SERIALIZER_MAP: 272 | return jsonToMap(new String(bs, charset)); 273 | default: 274 | return bs; 275 | } 276 | } catch (UnsupportedEncodingException ex) { 277 | throw new RuntimeException(ex); 278 | } 279 | } 280 | 281 | /** 282 | * set value object of the key 283 | * @param key the key 284 | * @param value the value object 285 | * @return none 286 | */ 287 | public void set(String key, Value value) { 288 | doSetVO(this.handler, key, value); 289 | } 290 | 291 | /** 292 | * set value and TTL of the key 293 | * @param key the key 294 | * @param value the byte array value 295 | * @param ttl the TTL in seconds 296 | * @return none 297 | */ 298 | public void set(String key, byte[] value, int ttl) { 299 | doSet(this.handler, key, value, ttl); 300 | } 301 | 302 | /** 303 | * set value and TTL of the key 304 | * @param key the key 305 | * @param value the string value 306 | * @param ttl the TTL in seconds 307 | * @return none 308 | */ 309 | public void set(String key, String value, int ttl) { 310 | try { 311 | doSet(this.handler, key, value.getBytes(charset), ttl); 312 | } catch (UnsupportedEncodingException ex) { 313 | throw new RuntimeException(ex); 314 | } 315 | } 316 | 317 | /** 318 | * set TTL of the key 319 | * @param key the key 320 | * @param ttl the TTL in seconds 321 | * @return true for success, otherwise fail 322 | */ 323 | public boolean setTTL(String key, int ttl) { 324 | return doSetTTL(this.handler, key, ttl); 325 | } 326 | 327 | /** 328 | * increase value of the key 329 | * @param key the key 330 | * @param increment the increment such as 1 331 | * @param ttl the TTL in seconds 332 | * @return the value after increase 333 | */ 334 | public long incr(String key, long increment, int ttl) { 335 | return doIncr(this.handler, key, increment, ttl); 336 | } 337 | 338 | /** 339 | * remove the key 340 | * @param key the key to remove 341 | * @return true for success, false for not exists 342 | */ 343 | public boolean remove(String key) { 344 | return doDelete(this.handler, key); 345 | } 346 | 347 | /** 348 | * clear the shm, remove all keys 349 | * @return true for success, false for fail 350 | */ 351 | public boolean clear() { 352 | return doClear(this.handler); 353 | } 354 | 355 | /** 356 | * get the expires timestamp of the key 357 | * @param key the key 358 | * @return expires timestamp in milliseconds 359 | * return -1 when not exist 360 | * return 0 for never expired 361 | */ 362 | public long getExpires(String key) { 363 | long expires = doGetExpires(this.handler, key); 364 | return expires > 0 ? (expires * 1000) : expires; 365 | } 366 | 367 | /** 368 | * get the TTL of the key 369 | * @param key the key 370 | * @return the ttl in seconds 371 | * return -1 when not exist 372 | * return 0 for never expired 373 | */ 374 | public int getTTL(String key) { 375 | long expires = this.getExpires(key); 376 | return (int)(expires > 0 ? (expires - System.currentTimeMillis()) / 1000 : expires); 377 | } 378 | 379 | /** 380 | * get last clear timestamp 381 | * @return last clear timestamp in milliseconds 382 | * return 0 for never cleared 383 | */ 384 | public long getLastClearTime() { 385 | return doGetLastClearTime(this.handler) * 1000; 386 | } 387 | 388 | /** 389 | * get the shm init timestamp 390 | * @return the shm init timestamp in milliseconds 391 | */ 392 | public long getInitTime() { 393 | return doGetInitTime(this.handler) * 1000; 394 | } 395 | 396 | public static void main(String[] args) throws UnsupportedEncodingException { 397 | String configFilename = "/usr/local/etc/libshmcache.conf"; 398 | String key; 399 | String value; 400 | if (args.length > 0) { 401 | key = args[0]; 402 | } else { 403 | key = "测试key"; 404 | } 405 | if (args.length > 1) { 406 | value = args[1]; 407 | } else { 408 | value = "测试value"; 409 | } 410 | 411 | ShmCache.setLibraryFilename("/Users/yuqing/Devel/libshmcache/java/jni/libshmcachejni.so"); 412 | ShmCache shmCache = ShmCache.getInstance(configFilename); 413 | 414 | System.out.println("value: " + shmCache.get(key)); 415 | System.out.println("TTL: " + shmCache.getTTL(key) + ", expires: " + shmCache.getExpires(key)); 416 | 417 | Value vo; 418 | vo = new Value(value.getBytes(charset), SHMCACHE_SERIALIZER_STRING, 419 | System.currentTimeMillis() + 300 * 1000); 420 | shmCache.set(key, vo); 421 | System.out.println(shmCache.get(key)); 422 | 423 | shmCache.set(key, value, 300); 424 | System.out.println("value: " + shmCache.getString(key)); 425 | System.out.println("TTL: " + shmCache.getTTL(key) + ", expires: " + shmCache.getExpires(key)); 426 | System.out.println(shmCache.getValue(key)); 427 | 428 | System.out.println("set TTL: " + shmCache.setTTL(key, 600)); 429 | System.out.println("TTL: " + shmCache.getTTL(key) + ", expires: " + shmCache.getExpires(key)); 430 | System.out.println(shmCache.getValue(key)); 431 | 432 | System.out.println("after incr: " + shmCache.incr(key, 1, TTL_NEVER_EXPIRED)); 433 | System.out.println("TTL: " + shmCache.getTTL(key) + ", expires: " + shmCache.getExpires(key)); 434 | System.out.println(shmCache.getValue(key)); 435 | 436 | System.out.println("remove key: " + shmCache.remove(key)); 437 | 438 | 439 | } 440 | } 441 | -------------------------------------------------------------------------------- /java/src/main/java/org/csource/shmcache/ShmCachePropertyConfigAnnotation.java: -------------------------------------------------------------------------------- 1 | package org.csource.shmcache; 2 | 3 | import java.util.Properties; 4 | import java.util.ArrayList; 5 | import java.util.Map; 6 | import java.util.Set; 7 | 8 | public class ShmCachePropertyConfigAnnotation { 9 | 10 | public static final String CONFIG_TAG = "@config"; 11 | private static boolean debug = false; 12 | 13 | private static class KeyValue { 14 | private String key; 15 | private String value; 16 | 17 | public KeyValue(String key, String value) { 18 | this.key = key; 19 | this.value = value; 20 | } 21 | 22 | public String getKey() { 23 | return this.key; 24 | } 25 | 26 | public String getValue() { 27 | return this.value; 28 | } 29 | } 30 | 31 | private ShmCachePropertyConfigAnnotation() { 32 | } 33 | 34 | public static void setDebug(boolean value) { 35 | debug = value; 36 | } 37 | 38 | protected static String processProperties(ShmCache shmCache, String key, String value) { 39 | value = value.substring(CONFIG_TAG.length()).trim(); 40 | if (!(value.startsWith("(") && value.endsWith(")"))) { 41 | return null; 42 | } 43 | 44 | String params = value.substring(1, value.length() - 1); 45 | String[] parts = params.split(","); 46 | if (parts.length > 2) { 47 | System.err.println("too many parameters!"); 48 | return null; 49 | } 50 | 51 | String keyParam = parts[0].trim(); 52 | String defaultValue = parts.length == 2 ? parts[1].trim() : ""; 53 | if (keyParam.length() == 0) { 54 | keyParam = key; 55 | } 56 | if (debug) { 57 | System.err.println("key: " + keyParam + ", default value: " + defaultValue); 58 | } 59 | 60 | String v = shmCache.getString(keyParam); 61 | if (v == null) { 62 | System.err.println("key: " + keyParam + " not found, set to default: " + defaultValue); 63 | v = defaultValue; 64 | } 65 | 66 | return v; 67 | } 68 | 69 | public static void processProperties(ShmCache shmCache, Properties props) { 70 | 71 | ArrayList modifiedKV = new ArrayList(); 72 | Set> entries = props.entrySet(); 73 | for (Map.Entry entry : entries) { 74 | String key = (String)entry.getKey(); 75 | String value = (String)entry.getValue(); 76 | 77 | if (value.startsWith(CONFIG_TAG)) { 78 | if (debug) { 79 | System.err.println("processing: " + key + "=" + value); 80 | } 81 | String newValue = processProperties(shmCache, key, value); 82 | if (newValue != null) { 83 | modifiedKV.add(new KeyValue(key, newValue)); 84 | } 85 | } 86 | } 87 | 88 | for (KeyValue kv : modifiedKV) { 89 | props.put(kv.getKey(), kv.getValue()); 90 | if (debug) { 91 | System.err.println("set key: " + kv.getKey() + " to value: " + kv.getValue()); 92 | } 93 | } 94 | if (debug) { 95 | System.err.println("@config key count: " + modifiedKV.size()); 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /java/src/main/java/org/csource/shmcache/ShmCachePropertyConfigurer.java: -------------------------------------------------------------------------------- 1 | package org.csource.shmcache; 2 | 3 | import java.util.Properties; 4 | import java.io.IOException; 5 | import org.springframework.beans.factory.config.PropertyPlaceholderConfigurer; 6 | 7 | public class ShmCachePropertyConfigurer extends PropertyPlaceholderConfigurer { 8 | 9 | protected String libraryFilename; 10 | protected String configFilename; 11 | protected String charset = "UTF-8"; 12 | protected boolean debug = false; 13 | protected ShmCache shmCache; 14 | 15 | public void setLibraryFilename(String libraryFilename) { 16 | this.libraryFilename = libraryFilename; 17 | } 18 | 19 | public void setConfigFilename(String configFilename) { 20 | this.configFilename = configFilename; 21 | } 22 | 23 | public void setCharset(String charset) { 24 | this.charset = charset; 25 | } 26 | 27 | public void setDebug(String value) { 28 | this.debug = value.equals("1") || value.equals("true"); 29 | } 30 | 31 | protected void initShmCache() { 32 | if (this.shmCache != null) { 33 | return; 34 | } 35 | 36 | if (this.charset != null) { 37 | ShmCache.setCharset(this.charset); 38 | } 39 | ShmCache.setLibraryFilename(this.libraryFilename); 40 | this.shmCache = ShmCache.getInstance(this.configFilename); 41 | } 42 | 43 | @Override 44 | protected void loadProperties(Properties props) throws IOException { 45 | super.loadProperties(props); 46 | 47 | this.initShmCache(); 48 | ShmCachePropertyConfigAnnotation.setDebug(this.debug); 49 | ShmCachePropertyConfigAnnotation.processProperties(shmCache, props); 50 | } 51 | 52 | public String getConfig(String key) { 53 | this.initShmCache(); 54 | return this.shmCache.getString(key); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /libshmcache.spec: -------------------------------------------------------------------------------- 1 | 2 | %define LibShmcacheTool libshmcache-tool 3 | %define LibShmcacheConfig libshmcache-config 4 | %define LibShmcacheDevel libshmcache-devel 5 | %define CommitVersion %(echo $COMMIT_VERSION) 6 | 7 | Name: libshmcache 8 | Version: 1.0.8 9 | Release: 1%{?dist} 10 | Summary: a high performance local share memory cache for multi processes 11 | License: LGPL 12 | Group: Arch/Tech 13 | URL: http://github.com/happyfish100/libshmcache/ 14 | Source: http://github.com/happyfish100/libshmcache/%{name}-%{version}.tar.gz 15 | 16 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 17 | 18 | BuildRequires: libfastcommon-devel >= 1.0.56 19 | 20 | Requires: %__cp %__mv %__chmod %__grep %__mkdir %__install %__id 21 | Requires: libfastcommon >= 1.0.56 22 | 23 | %description 24 | libshmcache is a local share memory cache for multi processes. 25 | it is high performance because read is lockless. 26 | this project contains C library and PHP extension. 27 | commit version: %{CommitVersion} 28 | 29 | %package tool 30 | Summary: tool commands 31 | Requires: %{name}%{?_isa} = %{version}-%{release} 32 | 33 | %description tool 34 | This package provides tool commands 35 | commit version: %{CommitVersion} 36 | 37 | %package config 38 | Summary: libshmcache config file 39 | 40 | %description config 41 | This package provides libshmcache config file 42 | commit version: %{CommitVersion} 43 | 44 | %package devel 45 | Summary: Development header file 46 | Requires: %{name}%{?_isa} = %{version}-%{release} 47 | 48 | %description devel 49 | This package provides the header files of libshmcache 50 | commit version: %{CommitVersion} 51 | 52 | %prep 53 | 54 | %setup -q 55 | 56 | %build 57 | cd src && make clean && make 58 | cd tools && make clean && make 59 | 60 | %install 61 | rm -rf %{buildroot} 62 | cd src && DESTDIR=$RPM_BUILD_ROOT make install 63 | cd tools && DESTDIR=$RPM_BUILD_ROOT make install 64 | mkdir -p $RPM_BUILD_ROOT/etc/ 65 | cp -f ../../conf/libshmcache.conf $RPM_BUILD_ROOT/etc/ 66 | 67 | %post 68 | 69 | %preun 70 | 71 | %postun 72 | 73 | %clean 74 | rm -rf %{buildroot} 75 | 76 | %files 77 | %defattr(-,root,root,-) 78 | /usr/lib64/libshmcache.so* 79 | /usr/lib/libshmcache.so* 80 | 81 | %files tool 82 | /usr/bin/shmcache_set 83 | /usr/bin/shmcache_get 84 | /usr/bin/shmcache_delete 85 | /usr/bin/shmcache_stats 86 | /usr/bin/shmcache_remove_all 87 | 88 | %files config 89 | /etc/libshmcache.conf 90 | 91 | %files devel 92 | %defattr(-,root,root,-) 93 | /usr/include/shmcache/* 94 | 95 | %changelog 96 | * Fri Dec 23 2016 Yu Qing <384681@qq.com> 97 | - first RPM release (1.0) 98 | -------------------------------------------------------------------------------- /php-shmcache/bench/redis.php: -------------------------------------------------------------------------------- 1 | 'yuqing', 'score' => 90.5, 'city' => 'beijing', 'gender' => 'male'); 5 | 6 | $host = '10.0.11.89'; 7 | $port = 6379; 8 | $connectTimeout = 1; 9 | $redis = new \Redis(); 10 | $redis->connect($host, $port, $connectTimeout); 11 | $redis->setOption(\Redis::OPT_SERIALIZER, \Redis::SERIALIZER_IGBINARY); 12 | var_dump($redis->set($key, $value, 300)); 13 | for ($i=0; $i<10000; $i++) { 14 | //$redis->set($key, $value, 300); 15 | $v = $redis->get($key); 16 | } 17 | var_dump($redis->get($key)); 18 | var_dump($redis->delete($key)); 19 | 20 | -------------------------------------------------------------------------------- /php-shmcache/bench/shmcache.php: -------------------------------------------------------------------------------- 1 | 'yuqing', 'score' => 90.5, 'city' => 'beijing', 'gender' => 'male'); 5 | 6 | $cache = new ShmCache('/etc/libshmcache.conf', ShmCache::SERIALIZER_IGBINARY); 7 | $cache->set($key, $value, 300); 8 | for ($i=0; $i<1000000; $i++) { 9 | //$cache->set($key, $value, 300); 10 | $v = $cache->get($key); 11 | } 12 | var_dump($cache->get($key)); 13 | var_dump($cache->delete($key)); 14 | 15 | //var_dump($cache->stats()); 16 | //echo json_encode($cache->stats(), JSON_PRETTY_PRINT); 17 | -------------------------------------------------------------------------------- /php-shmcache/config.m4: -------------------------------------------------------------------------------- 1 | dnl config.m4 for extension shmcache 2 | 3 | PHP_ARG_WITH(shmcache, wapper for libshmcache 4 | [ --with-shmcache Include shmcache wapper for libshmcache]) 5 | 6 | if test "$PHP_SHMCACHE" != "no"; then 7 | PHP_SUBST(SHMCACHE_SHARED_LIBADD) 8 | 9 | if test -z "$ROOT"; then 10 | ROOT=/usr 11 | fi 12 | 13 | PHP_ADD_INCLUDE($ROOT/include/fastcommon) 14 | PHP_ADD_INCLUDE($ROOT/include/shmcache) 15 | 16 | PHP_ADD_LIBRARY_WITH_PATH(fastcommon, $ROOT/lib, SHMCACHE_SHARED_LIBADD) 17 | PHP_ADD_LIBRARY_WITH_PATH(shmcache, $ROOT/lib, SHMCACHE_SHARED_LIBADD) 18 | 19 | PHP_NEW_EXTENSION(shmcache, serializer/igbinary.c serializer/msgpack.c serializer/php.c serializer/shmcache_serializer.c php_shmcache.c, $ext_shared) 20 | 21 | CFLAGS="$CFLAGS -Wall" 22 | fi 23 | -------------------------------------------------------------------------------- /php-shmcache/php-shmcache.spec.in: -------------------------------------------------------------------------------- 1 | %define php_inidir %(php --ini | head -n 1 | awk -F ':' '{print $2;}' | sed 's/ //g') 2 | %define php_extdir %(php-config --extension-dir 2>/dev/null) 3 | Name: php-shmcache 4 | Version: 1.0.2 5 | Release: 1%{?dist} 6 | Summary: The php extension for libshmcache 7 | License: GPL 8 | Group: Arch/Tech 9 | URL: https://github.com/happyfish100/libshmcache 10 | Source: https://github.com/happyfish100/libshmcache/%{name}-%{version}.tar.gz 11 | 12 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 13 | 14 | BuildRequires: libshmcache-devel >= 1.0.2 15 | Requires: libshmcache >= 1.0.2 16 | 17 | %description 18 | This package provides the php extension for libshmcache 19 | 20 | %prep 21 | %setup -q 22 | 23 | %build 24 | phpize 25 | %configure 26 | make 27 | 28 | %install 29 | rm -rf %{buildroot} 30 | 31 | mkdir -p %{buildroot}%{php_extdir} 32 | mkdir -p %{buildroot}%{php_inidir}/php.d 33 | cp -f .libs/shmcache.so %{buildroot}%{php_extdir} 34 | cp -f shmcache.ini %{buildroot}%{php_inidir}/php.d/shmcache.ini 35 | 36 | %post 37 | 38 | %preun 39 | 40 | %postun 41 | 42 | %clean 43 | #rm -rf %{buildroot} 44 | 45 | %files 46 | %defattr(-,root,root,-) 47 | %{php_extdir}/* 48 | %{php_inidir}/php.d/* 49 | 50 | %changelog 51 | * Wed Dec 26 2016 Yu Qing 52 | - first RPM release (1.0) 53 | -------------------------------------------------------------------------------- /php-shmcache/php_shmcache.h: -------------------------------------------------------------------------------- 1 | #ifndef SHMCACHE_H 2 | #define SHMCACHE_H 3 | 4 | #ifdef __cplusplus 5 | extern "C" { 6 | #endif 7 | 8 | #ifdef PHP_WIN32 9 | #define PHP_SHMCACHE_API __declspec(dllexport) 10 | #else 11 | #define PHP_SHMCACHE_API 12 | #endif 13 | 14 | PHP_MINIT_FUNCTION(shmcache); 15 | PHP_RINIT_FUNCTION(shmcache); 16 | PHP_MSHUTDOWN_FUNCTION(shmcache); 17 | PHP_RSHUTDOWN_FUNCTION(shmcache); 18 | PHP_MINFO_FUNCTION(shmcache); 19 | 20 | ZEND_FUNCTION(shmcache_version); 21 | 22 | PHP_SHMCACHE_API zend_class_entry *php_shmcache_get_ce(void); 23 | PHP_SHMCACHE_API zend_class_entry *php_shmcache_get_exception(void); 24 | PHP_SHMCACHE_API zend_class_entry *php_shmcache_get_exception_base(int root TSRMLS_DC); 25 | 26 | #ifdef __cplusplus 27 | } 28 | #endif 29 | 30 | #endif /* SHMCACHE_H */ 31 | -------------------------------------------------------------------------------- /php-shmcache/serializer/igbinary.c: -------------------------------------------------------------------------------- 1 | #include "shmcache_serializer.h" 2 | 3 | int shmcache_igbinary_pack(zval *pzval, char **buf, int *len) 4 | { 5 | int ret; 6 | size_t out_len; 7 | 8 | if ((ret=igbinary_pack_func((uint8_t **)buf, &out_len, pzval TSRMLS_CC)) == 0) { 9 | *len = out_len; 10 | } else { 11 | *len = 0; 12 | *buf = NULL; 13 | } 14 | return ret; 15 | } 16 | 17 | int shmcache_igbinary_unpack(char *content, size_t len, zval *rv) 18 | { 19 | #if (PHP_MAJOR_VERSION < 7) 20 | INIT_PZVAL(rv); 21 | return igbinary_unpack_func((const uint8_t *)content, (size_t)len, &rv TSRMLS_CC); 22 | #else 23 | return igbinary_unpack_func((const uint8_t *)content, (size_t)len, rv TSRMLS_CC); 24 | #endif 25 | } 26 | -------------------------------------------------------------------------------- /php-shmcache/serializer/msgpack.c: -------------------------------------------------------------------------------- 1 | #include "shmcache_serializer.h" 2 | 3 | int shmcache_msgpack_pack(zval *pzval, smart_str *buf) 4 | { 5 | msgpack_serialize_func(buf, pzval); 6 | return 0; 7 | } 8 | 9 | int shmcache_msgpack_unpack(char *content, size_t len, zval *rv) 10 | { 11 | ZVAL_NULL(rv); 12 | msgpack_unserialize_func(rv, content, len); 13 | return 0; 14 | } 15 | -------------------------------------------------------------------------------- /php-shmcache/serializer/php.c: -------------------------------------------------------------------------------- 1 | #include "shmcache_serializer.h" 2 | #include "ext/standard/php_var.h" /* for serialize */ 3 | #include "logger.h" 4 | 5 | #if PHP_MAJOR_VERSION < 7 6 | #define GET_ZVAL_PTR(v) &v 7 | #else 8 | #define GET_ZVAL_PTR(v) v 9 | #endif 10 | 11 | int shmcache_php_pack(zval *pzval, smart_str *buf) 12 | { 13 | php_serialize_data_t var_hash; 14 | 15 | PHP_VAR_SERIALIZE_INIT(var_hash); 16 | php_var_serialize(buf, GET_ZVAL_PTR(pzval), &var_hash TSRMLS_CC); 17 | PHP_VAR_SERIALIZE_DESTROY(var_hash); 18 | return 0; 19 | } 20 | 21 | int shmcache_php_unpack(char *content, size_t len, zval *rv) 22 | { 23 | const unsigned char *p; 24 | php_unserialize_data_t var_hash; 25 | 26 | p = (const unsigned char*)content; 27 | ZVAL_FALSE(rv); 28 | PHP_VAR_UNSERIALIZE_INIT(var_hash); 29 | if (!php_var_unserialize(GET_ZVAL_PTR(rv), &p, p + len, &var_hash TSRMLS_CC)) { 30 | zval_ptr_dtor(GET_ZVAL_PTR(rv)); 31 | PHP_VAR_UNSERIALIZE_DESTROY(var_hash); 32 | logError("file: "__FILE__", line: %d, " 33 | "unpack error at offset %ld of %ld bytes", 34 | __LINE__, (long)((char*)p - content), len); 35 | return EINVAL; 36 | } 37 | PHP_VAR_UNSERIALIZE_DESTROY(var_hash); 38 | return 0; 39 | } 40 | -------------------------------------------------------------------------------- /php-shmcache/serializer/shmcache_serializer.c: -------------------------------------------------------------------------------- 1 | #include "shmcache_types.h" 2 | #include "logger.h" 3 | #include "shmcache_serializer.h" 4 | 5 | php_msgpack_serialize_func msgpack_serialize_func = NULL; 6 | php_msgpack_unserialize_func msgpack_unserialize_func = NULL; 7 | igbinary_serialize_func igbinary_pack_func = NULL; 8 | igbinary_unserialize_func igbinary_unpack_func = NULL; 9 | 10 | int shmcache_serializers = SHMCACHE_SERIALIZER_STRING | 11 | SHMCACHE_SERIALIZER_LIST | 12 | SHMCACHE_SERIALIZER_MAP | 13 | SHMCACHE_SERIALIZER_INTEGER | 14 | SHMCACHE_SERIALIZER_PHP; 15 | 16 | extern int shmcache_php_pack(zval *pzval, smart_str *buf); 17 | extern int shmcache_php_unpack(char *content, size_t len, zval *rv); 18 | 19 | extern int shmcache_msgpack_pack(zval *pzval, smart_str *buf); 20 | extern int shmcache_msgpack_unpack(char *content, size_t len, zval *rv); 21 | 22 | extern int shmcache_igbinary_pack(zval *pzval, char **buf, int *len); 23 | extern int shmcache_igbinary_unpack(char *content, size_t len, zval *rv); 24 | 25 | int shmcache_load_functions() 26 | { 27 | void *handle; 28 | 29 | handle = dlopen(NULL, RTLD_LAZY); 30 | if (handle == NULL) { 31 | logError("file: "__FILE__", line: %d, " 32 | "call dlopen fail, error: %s", 33 | __LINE__, dlerror()); 34 | return errno != 0 ? errno : EFAULT; 35 | } 36 | msgpack_serialize_func = (php_msgpack_serialize_func)dlsym(handle, 37 | "php_msgpack_serialize"); 38 | msgpack_unserialize_func = (php_msgpack_unserialize_func)dlsym(handle, 39 | "php_msgpack_unserialize"); 40 | igbinary_pack_func = (igbinary_serialize_func)dlsym(handle, 41 | "igbinary_serialize"); 42 | igbinary_unpack_func = (igbinary_unserialize_func)dlsym(handle, 43 | "igbinary_unserialize"); 44 | if (msgpack_serialize_func != NULL && msgpack_unserialize_func != NULL) { 45 | shmcache_serializers |= SHMCACHE_SERIALIZER_MSGPACK; 46 | } 47 | if (igbinary_pack_func != NULL && igbinary_unpack_func != NULL) { 48 | shmcache_serializers |= SHMCACHE_SERIALIZER_IGBINARY; 49 | } 50 | 51 | dlclose(handle); 52 | return 0; 53 | } 54 | 55 | int shmcache_serialize(zval *pzval, 56 | struct shmcache_serialize_output *output) 57 | { 58 | int result; 59 | int serializer; 60 | 61 | switch (ZEND_TYPE_OF(pzval)) { 62 | case IS_STRING: 63 | output->value->options = SHMCACHE_SERIALIZER_STRING; 64 | output->value->data = Z_STRVAL_P(pzval); 65 | output->value->length = Z_STRLEN_P(pzval); 66 | return 0; 67 | case IS_LONG: 68 | output->value->options = SHMCACHE_SERIALIZER_INTEGER; 69 | output->value->data = output->tmp; 70 | output->value->length = sprintf(output->tmp, "%"PRId64, 71 | (int64_t)Z_LVAL_P(pzval)); 72 | return 0; 73 | default: 74 | break; 75 | } 76 | 77 | serializer = output->value->options; 78 | switch (serializer) { 79 | case SHMCACHE_SERIALIZER_IGBINARY: 80 | return shmcache_igbinary_pack(pzval, &output->value->data, 81 | &output->value->length); 82 | case SHMCACHE_SERIALIZER_MSGPACK: 83 | case SHMCACHE_SERIALIZER_PHP: 84 | if (serializer == SHMCACHE_SERIALIZER_MSGPACK) { 85 | result = shmcache_msgpack_pack(pzval, output->buf); 86 | } else { 87 | result = shmcache_php_pack(pzval, output->buf); 88 | } 89 | 90 | if (result == 0) { 91 | #if PHP_MAJOR_VERSION < 7 92 | output->value->data = output->buf->c; 93 | output->value->length = output->buf->len; 94 | #else 95 | output->value->data = output->buf->s->val; 96 | output->value->length = output->buf->s->len; 97 | #endif 98 | } else { 99 | output->value->data = NULL; 100 | output->value->length = 0; 101 | } 102 | return result; 103 | default: 104 | output->value->data = NULL; 105 | output->value->length = 0; 106 | logError("file: "__FILE__", line: %d, " 107 | "invalid serializer: %d", 108 | __LINE__, serializer); 109 | return EINVAL; 110 | } 111 | } 112 | 113 | void shmcache_free_serialize_output(struct shmcache_serialize_output *output) 114 | { 115 | switch (output->value->options) { 116 | case SHMCACHE_SERIALIZER_IGBINARY: 117 | efree(output->value->data); 118 | break; 119 | case SHMCACHE_SERIALIZER_MSGPACK: 120 | case SHMCACHE_SERIALIZER_PHP: 121 | smart_str_free(output->buf); 122 | break; 123 | default: 124 | break; 125 | } 126 | } 127 | 128 | int shmcache_unserialize(struct shmcache_value_info *value, zval *rv) 129 | { 130 | char buff[32]; 131 | int len; 132 | 133 | switch (value->options) { 134 | case SHMCACHE_SERIALIZER_STRING: 135 | ZEND_ZVAL_STRINGL(rv, value->data, value->length, 1); 136 | return 0; 137 | case SHMCACHE_SERIALIZER_INTEGER: 138 | if (value->length < (int)sizeof(buff)) { 139 | len = value->length; 140 | } else { 141 | len = sizeof(buff) - 1; 142 | } 143 | memcpy(buff, value->data, len); 144 | buff[len] = '\0'; 145 | ZVAL_LONG(rv, strtoll(buff, NULL, 10)); 146 | return 0; 147 | case SHMCACHE_SERIALIZER_IGBINARY: 148 | return shmcache_igbinary_unpack(value->data, value->length, rv); 149 | case SHMCACHE_SERIALIZER_MSGPACK: 150 | return shmcache_msgpack_unpack(value->data, value->length, rv); 151 | case SHMCACHE_SERIALIZER_PHP: 152 | return shmcache_php_unpack(value->data, value->length, rv); 153 | default: 154 | logError("file: "__FILE__", line: %d, " 155 | "invalid serializer: %d", 156 | __LINE__, value->options); 157 | return EINVAL; 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /php-shmcache/serializer/shmcache_serializer.h: -------------------------------------------------------------------------------- 1 | #ifndef SHMCACHE_SERIALIZER_H 2 | #define SHMCACHE_SERIALIZER_H 3 | 4 | #include "php7_ext_wrapper.h" 5 | 6 | #if PHP_MAJOR_VERSION < 7 7 | #include "ext/standard/php_smart_str.h" 8 | #else 9 | #include "Zend/zend_smart_str.h" 10 | #endif 11 | 12 | #include "shmcache_types.h" 13 | 14 | struct shmcache_serialize_output { 15 | smart_str *buf; 16 | struct shmcache_value_info *value; 17 | char tmp[32]; //temp buffer for number convert 18 | }; 19 | 20 | typedef void (*php_msgpack_serialize_func)(smart_str *buf, zval *val); 21 | typedef void (*php_msgpack_unserialize_func)(zval *return_value, char *str, size_t str_len); 22 | 23 | /** Serialize zval. 24 | * * Return buffer is allocated by this function with emalloc. 25 | * @param[out] ret Return buffer 26 | * @param[out] ret_len Size of return buffer 27 | * @param[in] z Variable to be serialized 28 | * @return 0 on success, 1 elsewhere. 29 | */ 30 | typedef int (*igbinary_serialize_func)(uint8_t **ret, size_t *ret_len, zval *z TSRMLS_DC); 31 | 32 | /** Unserialize to zval. 33 | * @param[in] buf Buffer with serialized data. 34 | * @param[in] buf_len Buffer length. 35 | * @param[out] z Unserialized zval 36 | * @return 0 on success, 1 elsewhere. 37 | */ 38 | #if (PHP_MAJOR_VERSION < 7) 39 | typedef int (*igbinary_unserialize_func)(const uint8_t *buf, size_t buf_len, zval **z TSRMLS_DC); 40 | #else 41 | typedef int (*igbinary_unserialize_func)(const uint8_t *buf, size_t buf_len, zval *z TSRMLS_DC); 42 | #endif 43 | 44 | #ifdef __cplusplus 45 | extern "C" { 46 | #endif 47 | 48 | extern php_msgpack_serialize_func msgpack_serialize_func; 49 | extern php_msgpack_unserialize_func msgpack_unserialize_func; 50 | extern igbinary_serialize_func igbinary_pack_func; 51 | extern igbinary_unserialize_func igbinary_unpack_func; 52 | extern int shmcache_serializers; 53 | 54 | static inline bool shmcache_serializer_enabled(const int serializer) 55 | { 56 | if (serializer == SHMCACHE_SERIALIZER_STRING) { 57 | return true; 58 | } 59 | return (shmcache_serializers & serializer) != 0; 60 | } 61 | 62 | int shmcache_load_functions(); 63 | 64 | int shmcache_serialize(zval *pzval, struct shmcache_serialize_output *output); 65 | 66 | void shmcache_free_serialize_output(struct shmcache_serialize_output *output); 67 | 68 | int shmcache_unserialize(struct shmcache_value_info *value, zval *rv); 69 | 70 | #ifdef __cplusplus 71 | } 72 | #endif 73 | 74 | #endif 75 | -------------------------------------------------------------------------------- /php-shmcache/shmcache.ini: -------------------------------------------------------------------------------- 1 | extension = shmcache.so 2 | -------------------------------------------------------------------------------- /php-shmcache/test.php: -------------------------------------------------------------------------------- 1 | 'yuqing', 'score' => 90.5, 'city' => 'beijing', 'gender' => 'male'); 5 | //$value = json_encode($value, JSON_PRETTY_PRINT); 6 | 7 | $cache = new ShmCache('/etc/libshmcache.conf', ShmCache::SERIALIZER_IGBINARY); 8 | $cache->set($key, $value, 300); 9 | for ($i=0; $i<10240; $i++) { 10 | $cache->set($key, $value, 300); 11 | //$v = $cache->get($key); 12 | } 13 | 14 | echo "key $key expires: " . $cache->getExpires($key) . "\n"; 15 | var_dump($cache->setExpires($key, time() + 600)); 16 | echo "key $key expires: " . $cache->getExpires($key) . "\n"; 17 | 18 | var_dump($cache->setTTL($key, 900)); 19 | echo "key $key expires: " . $cache->getExpires($key) . "\n"; 20 | 21 | $key1 = 'key_00002'; 22 | var_dump($cache->incr($key1, -1, 300)); 23 | var_dump($cache->get($key)); 24 | var_dump($cache->delete($key)); 25 | 26 | //var_dump($cache->stats()); 27 | echo json_encode($cache->stats(), JSON_PRETTY_PRINT); 28 | //var_dump($cache->clear()); 29 | -------------------------------------------------------------------------------- /src/Makefile: -------------------------------------------------------------------------------- 1 | .SUFFIXES: .c .o .lo 2 | 3 | COMPILE = $(CC) -Wall -D_FILE_OFFSET_BITS=64 -D_GNU_SOURCE -g -O3 4 | INC_PATH = -I/usr/local/include 5 | LIB_PATH = -lm -lpthread -lfastcommon 6 | 7 | SHMCACHE_SHARED_OBJS = shmcache.lo shmopt.lo shm_striping_allocator.lo shm_object_pool.lo \ 8 | shm_hashtable.lo shm_value_allocator.lo shm_op_wrapper.lo shm_lock.lo \ 9 | shmcache_ini_annotation.lo 10 | 11 | SHMCACHE_STATIC_OBJS = shmcache.o shmopt.o shm_striping_allocator.o shm_object_pool.o \ 12 | shm_hashtable.o shm_value_allocator.o shm_op_wrapper.o shm_lock.o \ 13 | shmcache_ini_annotation.o 14 | 15 | HEADER_FILES = shmcache.h shmcache_types.h shm_list.h shm_striping_allocator.h \ 16 | shm_value_allocator.h shm_op_wrapper.h shmopt.h shm_hashtable.h \ 17 | shm_lock.h shmcache_ini_annotation.h 18 | 19 | ALL_OBJS = $(SHMCACHE_STATIC_OBJS) $(SHMCACHE_SHARED_OBJS) 20 | 21 | ALL_PRGS = 22 | SHARED_LIBS = libshmcache.so 23 | STATIC_LIBS = libshmcache.a 24 | ALL_LIBS = $(SHARED_LIBS) $(STATIC_LIBS) 25 | 26 | all: $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) 27 | libshmcache.so: 28 | $(COMPILE) -o $@ $< -shared $(SHMCACHE_SHARED_OBJS) $(LIB_PATH) 29 | libshmcache.a: $(SHMCACHE_STATIC_OBJS) 30 | ar rcs $@ $(SHMCACHE_STATIC_OBJS) 31 | .o: 32 | $(COMPILE) -o $@ $< $(SHMCACHE_STATIC_OBJS) $(LIB_PATH) $(INC_PATH) 33 | .c: 34 | $(COMPILE) -o $@ $< $(SHMCACHE_STATIC_OBJS) $(LIB_PATH) $(INC_PATH) 35 | .c.o: 36 | $(COMPILE) -c -o $@ $< $(INC_PATH) 37 | .c.lo: 38 | $(COMPILE) -c -fPIC -o $@ $< $(INC_PATH) 39 | install: 40 | mkdir -p $(DESTDIR)/usr/lib64 41 | mkdir -p $(DESTDIR)/usr/lib 42 | install -m 755 $(SHARED_LIBS) $(DESTDIR)/usr/lib64 43 | install -m 755 $(SHARED_LIBS) $(DESTDIR)/usr/lib 44 | mkdir -p $(DESTDIR)/usr/include/shmcache 45 | install -m 644 $(HEADER_FILES) $(DESTDIR)/usr/include/shmcache 46 | clean: 47 | rm -f $(ALL_OBJS) $(ALL_PRGS) $(ALL_LIBS) 48 | 49 | -------------------------------------------------------------------------------- /src/shm_hashtable.c: -------------------------------------------------------------------------------- 1 | //shm_hashtable.c 2 | 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "fastcommon/sched_thread.h" 8 | #include "shmopt.h" 9 | #include "shm_lock.h" 10 | #include "shm_object_pool.h" 11 | #include "shm_striping_allocator.h" 12 | #include "shm_hashtable.h" 13 | 14 | int shm_ht_get_capacity(const int max_count) 15 | { 16 | unsigned int *capacity; 17 | capacity = fc_hash_get_prime_capacity(max_count); 18 | if (capacity == NULL) { 19 | return max_count; 20 | } 21 | return *capacity; 22 | } 23 | 24 | void shm_ht_init(struct shmcache_context *context, const int capacity) 25 | { 26 | context->memory->hashtable.capacity = capacity; 27 | context->memory->hashtable.count = 0; 28 | } 29 | 30 | #define HT_GET_BUCKET_INDEX(context, key) \ 31 | ((unsigned int)context->config.hash_func(key->data, key->length) % \ 32 | context->memory->hashtable.capacity) 33 | 34 | #define HT_KEY_EQUALS(hentry, pkey) (hentry->key_len == pkey->length && \ 35 | memcmp(hentry->key, pkey->data, pkey->length) == 0) 36 | 37 | #define HT_VALUE_EQUALS(hvalue, hv_len, pvalue) (hv_len == pvalue->length \ 38 | && memcmp(hvalue, pvalue->data, pvalue->length) == 0) 39 | 40 | int shm_ht_set(struct shmcache_context *context, 41 | const struct shmcache_key_info *key, 42 | const struct shmcache_value_info *value) 43 | { 44 | int result; 45 | unsigned int index; 46 | int64_t old_offset; 47 | int64_t new_offset; 48 | struct shm_hash_entry *old_entry; 49 | struct shm_hash_entry *new_entry; 50 | struct shm_hash_entry *previous; 51 | char *hvalue; 52 | bool found; 53 | 54 | if (key->length > SHMCACHE_MAX_KEY_SIZE) { 55 | logError("file: "__FILE__", line: %d, " 56 | "invalid key length: %d exceeds %d", __LINE__, 57 | key->length, SHMCACHE_MAX_KEY_SIZE); 58 | return ENAMETOOLONG; 59 | } 60 | 61 | if (value->length > context->config.max_value_size) { 62 | logError("file: "__FILE__", line: %d, " 63 | "invalid value length: %d exceeds %d", __LINE__, 64 | value->length, context->config.max_value_size); 65 | return EINVAL; 66 | } 67 | 68 | if (context->memory->hashtable.count >= context->config.max_key_count) { 69 | if ((result=shm_value_allocator_recycle(context, &context->memory-> 70 | stats.memory.recycle.key, context-> 71 | config.recycle_key_once)) != 0) 72 | { 73 | return result; 74 | } 75 | } 76 | 77 | if (!g_schedule_flag) { 78 | g_current_time = time(NULL); 79 | } 80 | 81 | if ((new_entry=shm_value_allocator_alloc(context, 82 | key->length, value->length)) == NULL) 83 | { 84 | return ENOMEM; 85 | } 86 | 87 | previous = NULL; 88 | old_entry = NULL; 89 | found = false; 90 | index = HT_GET_BUCKET_INDEX(context, key); 91 | old_offset = context->memory->hashtable.buckets[index]; 92 | while (old_offset > 0) { 93 | old_entry = shm_get_hentry_ptr(context, old_offset); 94 | if (HT_KEY_EQUALS(old_entry, key)) { 95 | found = true; 96 | break; 97 | } 98 | 99 | old_offset = old_entry->ht_next; 100 | previous = old_entry; 101 | } 102 | 103 | if (found) { 104 | bool recycled = false; 105 | new_entry->ht_next = old_entry->ht_next; 106 | shm_ht_free_entry(context, old_entry, old_offset, &recycled); 107 | } else { 108 | new_entry->ht_next = 0; 109 | } 110 | 111 | new_offset = shm_get_hentry_offset(new_entry); 112 | memcpy(new_entry->key, key->data, key->length); 113 | new_entry->key_len = key->length; 114 | 115 | hvalue = shm_get_value_ptr(context, new_entry); 116 | memcpy(hvalue, value->data, value->length); 117 | new_entry->value.length = value->length; 118 | new_entry->value.options = value->options; 119 | new_entry->expires = value->expires; 120 | 121 | if (previous != NULL) { //add to tail 122 | previous->ht_next = new_offset; 123 | } else { 124 | context->memory->hashtable.buckets[index] = new_offset; 125 | } 126 | context->memory->hashtable.count++; 127 | context->memory->usage.used.value += value->length; 128 | context->memory->usage.used.key += new_entry->key_len; 129 | shm_list_add_tail(context, new_offset); 130 | 131 | return 0; 132 | } 133 | 134 | int shm_ht_get(struct shmcache_context *context, 135 | const struct shmcache_key_info *key, 136 | struct shmcache_value_info *value) 137 | { 138 | unsigned int index; 139 | int64_t entry_offset; 140 | struct shm_hash_entry *entry; 141 | 142 | index = HT_GET_BUCKET_INDEX(context, key); 143 | entry_offset = context->memory->hashtable.buckets[index]; 144 | while (entry_offset > 0) { 145 | entry = shm_get_hentry_ptr(context, entry_offset); 146 | if (HT_KEY_EQUALS(entry, key)) { 147 | value->data = shm_get_value_ptr(context, entry); 148 | value->length = entry->value.length; 149 | value->options = entry->value.options; 150 | value->expires = entry->expires; 151 | if (HT_ENTRY_IS_VALID(entry, get_current_time())) { 152 | return 0; 153 | } else { 154 | return ETIMEDOUT; 155 | } 156 | } 157 | 158 | entry_offset = entry->ht_next; 159 | } 160 | 161 | return ENOENT; 162 | } 163 | 164 | void shm_ht_free_entry(struct shmcache_context *context, 165 | struct shm_hash_entry *entry, const int64_t entry_offset, 166 | bool *recycled) 167 | { 168 | context->memory->hashtable.count--; 169 | context->memory->usage.used.value -= entry->value.length; 170 | context->memory->usage.used.key -= entry->key_len; 171 | shm_list_delete(context, entry_offset); 172 | shm_value_allocator_free(context, entry, recycled); 173 | entry->ht_next = 0; 174 | } 175 | 176 | int shm_ht_delete_ex(struct shmcache_context *context, 177 | const struct shmcache_key_info *key, bool *recycled) 178 | { 179 | int result; 180 | unsigned int index; 181 | int64_t entry_offset; 182 | struct shm_hash_entry *entry; 183 | struct shm_hash_entry *previous; 184 | 185 | previous = NULL; 186 | result = ENOENT; 187 | index = HT_GET_BUCKET_INDEX(context, key); 188 | entry_offset = context->memory->hashtable.buckets[index]; 189 | while (entry_offset > 0) { 190 | entry = shm_get_hentry_ptr(context, entry_offset); 191 | if (HT_KEY_EQUALS(entry, key)) { 192 | if (previous != NULL) { 193 | previous->ht_next = entry->ht_next; 194 | } else { 195 | context->memory->hashtable.buckets[index] = entry->ht_next; 196 | } 197 | 198 | shm_ht_free_entry(context, entry, entry_offset, recycled); 199 | result = 0; 200 | break; 201 | } 202 | 203 | entry_offset = entry->ht_next; 204 | previous = entry; 205 | } 206 | 207 | return result; 208 | } 209 | 210 | int shm_ht_set_expires(struct shmcache_context *context, 211 | const struct shmcache_key_info *key, const time_t expires) 212 | { 213 | int result; 214 | unsigned int index; 215 | int64_t entry_offset; 216 | struct shm_hash_entry *entry; 217 | 218 | result = ENOENT; 219 | index = HT_GET_BUCKET_INDEX(context, key); 220 | entry_offset = context->memory->hashtable.buckets[index]; 221 | while (entry_offset > 0) { 222 | entry = shm_get_hentry_ptr(context, entry_offset); 223 | if (HT_KEY_EQUALS(entry, key)) { 224 | if (HT_ENTRY_IS_VALID(entry, get_current_time())) { 225 | entry->expires = expires; 226 | result = 0; 227 | } else { 228 | result = ETIMEDOUT; 229 | } 230 | break; 231 | } 232 | 233 | entry_offset = entry->ht_next; 234 | } 235 | 236 | return result; 237 | } 238 | 239 | int shm_ht_clear(struct shmcache_context *context) 240 | { 241 | struct shm_striping_allocator *allocator; 242 | struct shm_striping_allocator *end; 243 | int64_t allocator_offset; 244 | int ht_count; 245 | 246 | context->memory->stats.hashtable.last_clear_time = 247 | context->memory->stats.last.calc_time = get_current_time(); 248 | ht_count = context->memory->hashtable.count; 249 | memset(context->memory->hashtable.buckets, 0, sizeof(int64_t) * 250 | context->memory->hashtable.capacity); 251 | context->memory->hashtable.count = 0; 252 | shm_list_init(context); 253 | 254 | shm_object_pool_init_empty(&context->value_allocator.doing); 255 | shm_object_pool_init_empty(&context->value_allocator.done); 256 | end = context->value_allocator.allocators + 257 | context->memory->vm_info.striping.count.current; 258 | for (allocator=context->value_allocator.allocators; allocatorin_which_pool = SHMCACHE_STRIPING_ALLOCATOR_POOL_DOING; 261 | allocator_offset = (char *)allocator - context->segments.hashtable.base; 262 | shm_object_pool_push(&context->value_allocator.doing, allocator_offset); 263 | } 264 | context->memory->usage.used.key = 0; 265 | context->memory->usage.used.value = 0; 266 | context->memory->usage.used.entry = 0; 267 | 268 | logInfo("file: "__FILE__", line: %d, pid: %d, " 269 | "clear hashtable, %d entries be cleared!", 270 | __LINE__, context->pid, ht_count); 271 | return ht_count; 272 | } 273 | 274 | static bool shm_ht_match_key(struct shmcache_match_key_info *key_info, 275 | struct shm_hash_entry *entry) 276 | { 277 | int loop; 278 | int start; 279 | 280 | if (entry->key_len < key_info->length) { 281 | return false; 282 | } 283 | switch (key_info->op_type) { 284 | case SHMCACHE_MATCH_KEY_OP_EXACT: 285 | return key_info->length == entry->key_len && 286 | memcmp(key_info->key, entry->key, key_info->length) == 0; 287 | case SHMCACHE_MATCH_KEY_OP_LEFT: 288 | return memcmp(key_info->key, entry->key, key_info->length) == 0; 289 | case SHMCACHE_MATCH_KEY_OP_RIGHT: 290 | return memcmp(key_info->key, entry->key + 291 | (entry->key_len - key_info->length), key_info->length) == 0; 292 | case SHMCACHE_MATCH_KEY_OP_ANYWHERE: 293 | loop = entry->key_len - key_info->length + 1; 294 | for (start=0; startkey, entry->key + start, 296 | key_info->length) == 0) 297 | { 298 | return true; 299 | } 300 | } 301 | return false; 302 | default: 303 | return false; 304 | } 305 | } 306 | 307 | #define HT_KEY_MATCHED(key_info, entry) \ 308 | (key_info == NULL || shm_ht_match_key(key_info, entry)) 309 | 310 | int shm_ht_to_array_ex(struct shmcache_context *context, 311 | struct shmcache_hentry_array *array, 312 | struct shmcache_match_key_info *key_info, 313 | const int offset, const int count) 314 | { 315 | int64_t entry_offset; 316 | struct shm_hash_entry *src; 317 | struct shmcache_hash_entry *dest; 318 | char *value_data; 319 | int row_count; 320 | int i; 321 | int bytes; 322 | int current_time; 323 | 324 | if (offset < 0) { 325 | logError("file: "__FILE__", line: %d, " 326 | "invalid offset: %d", __LINE__, offset); 327 | array->count = 0; 328 | array->entries = NULL; 329 | return EINVAL; 330 | } 331 | 332 | row_count = context->memory->hashtable.count - offset; 333 | if (count > 0 && count < row_count) { 334 | row_count = count; 335 | } 336 | 337 | if (row_count <= 0) { 338 | array->count = 0; 339 | array->entries = NULL; 340 | return 0; 341 | } 342 | 343 | bytes = sizeof(struct shmcache_hash_entry) * row_count; 344 | array->entries = (struct shmcache_hash_entry *)malloc(bytes); 345 | if (array->entries == NULL) { 346 | logError("file: "__FILE__", line: %d, " 347 | "malloc %d bytes fail", __LINE__, bytes); 348 | array->count = 0; 349 | return ENOMEM; 350 | } 351 | 352 | current_time = time(NULL); 353 | i = 0; 354 | entry_offset = shm_list_first(context); 355 | while (entry_offset > 0 && i < offset) { 356 | src = shm_get_hentry_ptr(context, entry_offset); 357 | if (HT_ENTRY_IS_VALID(src, current_time) && 358 | HT_KEY_MATCHED(key_info, src)) 359 | { 360 | ++i; 361 | } 362 | entry_offset = shm_list_next(context, entry_offset); 363 | } 364 | 365 | memset(array->entries, 0, bytes); 366 | array->count = 0; 367 | dest = array->entries; 368 | 369 | while (entry_offset > 0) { 370 | src = shm_get_hentry_ptr(context, entry_offset); 371 | if (!(HT_ENTRY_IS_VALID(src, current_time) && 372 | HT_KEY_MATCHED(key_info, src))) 373 | { 374 | entry_offset = shm_list_next(context, entry_offset); 375 | continue; 376 | } 377 | 378 | bytes = src->key_len + src->value.length; 379 | dest->key.data = (char *)malloc(bytes); 380 | if (dest->key.data == NULL) { 381 | logError("file: "__FILE__", line: %d, " 382 | "malloc %d bytes fail", __LINE__, bytes); 383 | shm_ht_free_array(array); 384 | return ENOMEM; 385 | } 386 | 387 | dest->key.length = src->key_len; 388 | memcpy(dest->key.data, src->key, src->key_len); 389 | 390 | value_data = shm_get_value_ptr(context, src); 391 | dest->value.length = src->value.length; 392 | dest->value.options = src->value.options; 393 | dest->value.expires = src->expires; 394 | 395 | dest->value.data = dest->key.data + src->key_len; 396 | memcpy(dest->value.data, value_data, src->value.length); 397 | 398 | entry_offset = shm_list_next(context, entry_offset); 399 | dest++; 400 | array->count++; 401 | if (array->count == row_count) { 402 | break; 403 | } 404 | } 405 | 406 | return 0; 407 | } 408 | 409 | void shm_ht_free_array(struct shmcache_hentry_array *array) 410 | { 411 | struct shmcache_hash_entry *entry; 412 | struct shmcache_hash_entry *end; 413 | 414 | if (array->entries == NULL) { 415 | return; 416 | } 417 | 418 | end = array->entries + array->count; 419 | for (entry=array->entries; entrykey.data); 421 | } 422 | 423 | free(array->entries); 424 | array->entries = NULL; 425 | array->count = 0; 426 | } 427 | -------------------------------------------------------------------------------- /src/shm_hashtable.h: -------------------------------------------------------------------------------- 1 | //shm_hashtable.h 2 | 3 | #ifndef _SHM_HASH_TABLE_H 4 | #define _SHM_HASH_TABLE_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "fastcommon/common_define.h" 13 | #include "shmcache_types.h" 14 | #include "shm_list.h" 15 | #include "shm_value_allocator.h" 16 | 17 | #define HT_CALC_EXPIRES(current_time, ttl) \ 18 | (ttl == SHMCACHE_NEVER_EXPIRED ? 0 : current_time + ttl) 19 | 20 | #define HT_ENTRY_IS_VALID(entry, current_time) \ 21 | (entry->expires == 0 || entry->expires >= current_time) 22 | 23 | #define SHMCACHE_MATCH_KEY_OP_EXACT 0 24 | #define SHMCACHE_MATCH_KEY_OP_LEFT 1 25 | #define SHMCACHE_MATCH_KEY_OP_RIGHT 2 26 | #define SHMCACHE_MATCH_KEY_OP_ANYWHERE \ 27 | (SHMCACHE_MATCH_KEY_OP_LEFT | SHMCACHE_MATCH_KEY_OP_RIGHT) 28 | 29 | struct shmcache_match_key_info { 30 | char *key; 31 | int length; 32 | int op_type; 33 | }; 34 | 35 | #ifdef __cplusplus 36 | extern "C" { 37 | #endif 38 | 39 | static inline int64_t shm_ht_get_memory_size(const int capacity) 40 | { 41 | return sizeof(int64_t) * (int64_t)capacity; 42 | } 43 | 44 | /** 45 | get hashtable capacity 46 | parameters: 47 | max_count: max entry counti 48 | return hashtable capacity 49 | */ 50 | int shm_ht_get_capacity(const int max_count); 51 | 52 | /** 53 | ht init 54 | parameters: 55 | context: the context pointer 56 | capacity: the ht capacity 57 | return none 58 | */ 59 | void shm_ht_init(struct shmcache_context *context, const int capacity); 60 | 61 | /** 62 | set value 63 | parameters: 64 | context: the context pointer 65 | key: the key 66 | value: the value, include expires field 67 | return error no, 0 for success, != 0 for fail 68 | */ 69 | int shm_ht_set(struct shmcache_context *context, 70 | const struct shmcache_key_info *key, 71 | const struct shmcache_value_info *value); 72 | 73 | /** 74 | set expires 75 | parameters: 76 | context: the context pointer 77 | key: the key 78 | value: the expires timestamp 79 | return error no, 0 for success, != 0 for fail 80 | */ 81 | int shm_ht_set_expires(struct shmcache_context *context, 82 | const struct shmcache_key_info *key, const time_t expires); 83 | 84 | /** 85 | get value 86 | parameters: 87 | context: the context pointer 88 | key: the key 89 | value: store the returned value 90 | return error no, 0 for success, != 0 for fail 91 | */ 92 | int shm_ht_get(struct shmcache_context *context, 93 | const struct shmcache_key_info *key, 94 | struct shmcache_value_info *value); 95 | 96 | /** 97 | delete the key for internal usage 98 | parameters: 99 | context: the context pointer 100 | key: the key 101 | recycled: if recycled 102 | return error no, 0 for success, != 0 for fail 103 | */ 104 | int shm_ht_delete_ex(struct shmcache_context *context, 105 | const struct shmcache_key_info *key, bool *recycled); 106 | 107 | /** 108 | hashtable entry to array 109 | parameters: 110 | context: the context pointer 111 | array: the array, should call shm_ht_free_array after use 112 | key_info: the match key info, NULL for match all 113 | offset: start offset, based 0 114 | count: row count limit 115 | return error no, 0 for success, != 0 for fail 116 | */ 117 | int shm_ht_to_array_ex(struct shmcache_context *context, 118 | struct shmcache_hentry_array *array, 119 | struct shmcache_match_key_info *key_info, 120 | const int offset, const int count); 121 | 122 | /** 123 | hashtable entry to array 124 | parameters: 125 | context: the context pointer 126 | array: the array, should call shm_ht_free_array after use 127 | return error no, 0 for success, != 0 for fail 128 | */ 129 | static inline int shm_ht_to_array(struct shmcache_context *context, 130 | struct shmcache_hentry_array *array) 131 | { 132 | return shm_ht_to_array_ex(context, array, NULL, 0, 0); 133 | } 134 | 135 | /** 136 | free the array return by shm_ht_to_array 137 | parameters: 138 | context: the context pointer 139 | array: the array to free 140 | return none 141 | */ 142 | void shm_ht_free_array(struct shmcache_hentry_array *array); 143 | 144 | 145 | /** 146 | delete the key 147 | parameters: 148 | context: the context pointer 149 | key: the key 150 | return error no, 0 for success, != 0 for fail 151 | */ 152 | static inline int shm_ht_delete(struct shmcache_context *context, 153 | const struct shmcache_key_info *key) 154 | { 155 | bool recycled; 156 | return shm_ht_delete_ex(context, key, &recycled); 157 | } 158 | 159 | /** 160 | free hashtable entry 161 | parameters: 162 | context: the context pointer 163 | entry: the hashtable entry 164 | entry_offset: the entry offset 165 | recycled: if recycled 166 | return none 167 | */ 168 | void shm_ht_free_entry(struct shmcache_context *context, 169 | struct shm_hash_entry *entry, const int64_t entry_offset, 170 | bool *recycled); 171 | 172 | /** 173 | remove all hashtable entries 174 | parameters: 175 | context: the context pointer 176 | return cleared entries count 177 | */ 178 | int shm_ht_clear(struct shmcache_context *context); 179 | 180 | static inline int shm_ht_count(struct shmcache_context *context) 181 | { 182 | return context->memory->hashtable.count; 183 | } 184 | 185 | #ifdef __cplusplus 186 | } 187 | #endif 188 | 189 | #endif 190 | 191 | -------------------------------------------------------------------------------- /src/shm_list.h: -------------------------------------------------------------------------------- 1 | //shm_list.h 2 | 3 | #ifndef _SHM_LIST_H 4 | #define _SHM_LIST_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "fastcommon/logger.h" 12 | #include "shmcache_types.h" 13 | #include "shm_value_allocator.h" 14 | 15 | #ifdef __cplusplus 16 | extern "C" { 17 | #endif 18 | 19 | static inline struct shm_list *shm_list_ptr(struct shmcache_context *context, 20 | int64_t obj_offset) 21 | { 22 | if (obj_offset == context->list.head.offset) { 23 | return context->list.head.ptr; 24 | } else { 25 | return (struct shm_list *)shm_get_hentry_ptr(context, obj_offset); 26 | } 27 | } 28 | 29 | #define SHM_LIST_TYPE_PTR(context, type, offset) ((type *)shm_list_ptr(context, offset)) 30 | 31 | /** 32 | list set 33 | parameters: 34 | list: the list 35 | return none 36 | */ 37 | static inline void shm_list_set(struct shmcache_context *context, 38 | char *base, struct shm_list *head) 39 | { 40 | union shm_hentry_offset conv; 41 | conv.segment.index = -1; 42 | conv.segment.offset = (char *)head - base; 43 | 44 | context->list.head.ptr = head; 45 | context->list.head.offset = conv.offset; 46 | } 47 | 48 | /** 49 | init list to empty 50 | parameters: 51 | list: the list 52 | return none 53 | */ 54 | static inline void shm_list_init(struct shmcache_context *context) 55 | { 56 | context->list.head.ptr->prev = context->list.head.offset; 57 | context->list.head.ptr->next = context->list.head.offset; 58 | } 59 | 60 | #define SHM_LIST_ADD_TO_TAIL(context, node, obj_offset) \ 61 | do { \ 62 | node->next = context->list.head.offset; \ 63 | node->prev = context->list.head.ptr->prev; \ 64 | shm_list_ptr(context, context->list.head.ptr->prev)->next = obj_offset; \ 65 | context->list.head.ptr->prev = obj_offset; \ 66 | } while (0) 67 | 68 | /** 69 | add an element to list tail 70 | parameters: 71 | list: the list 72 | obj_offset: the object offset 73 | return none 74 | */ 75 | static inline void shm_list_add_tail(struct shmcache_context *context, int64_t obj_offset) 76 | { 77 | struct shm_list *node; 78 | 79 | node = shm_list_ptr(context, obj_offset); 80 | SHM_LIST_ADD_TO_TAIL(context, node, obj_offset); 81 | } 82 | 83 | /** 84 | remove an element from list 85 | parameters: 86 | list: the list 87 | obj_offset: the object offset 88 | return none 89 | */ 90 | static inline void shm_list_delete(struct shmcache_context *context, int64_t obj_offset) 91 | { 92 | struct shm_list *node; 93 | 94 | node = shm_list_ptr(context, obj_offset); 95 | if (node->next == obj_offset) { 96 | logError("file: " __FILE__", line: %d, " 97 | "do NOT need remove from list, obj: %" PRId64, 98 | __LINE__, obj_offset); 99 | return; 100 | } 101 | 102 | shm_list_ptr(context, node->prev)->next = node->next; 103 | shm_list_ptr(context, node->next)->prev = node->prev; 104 | node->prev = node->next = obj_offset; 105 | } 106 | 107 | /** 108 | move an element to tail 109 | parameters: 110 | list: the list 111 | obj_offset: the object offset 112 | return none 113 | */ 114 | static inline void shm_list_move_tail(struct shmcache_context *context, int64_t obj_offset) 115 | { 116 | struct shm_list *node; 117 | 118 | node = shm_list_ptr(context, obj_offset); 119 | shm_list_ptr(context, node->prev)->next = node->next; 120 | shm_list_ptr(context, node->next)->prev = node->prev; 121 | 122 | SHM_LIST_ADD_TO_TAIL(context, node, obj_offset); 123 | } 124 | 125 | /** 126 | is empty 127 | parameters: 128 | list: the list 129 | return true for empty, false for NOT empty 130 | */ 131 | static inline bool shm_list_empty(struct shmcache_context *context) 132 | { 133 | return (context->list.head.ptr->next == context->list.head.offset); 134 | } 135 | 136 | /** 137 | get first element 138 | parameters: 139 | list: the list 140 | return first object offset 141 | */ 142 | static inline int64_t shm_list_first(struct shmcache_context *context) 143 | { 144 | if (context->list.head.ptr->next == context->list.head.offset) { //empty 145 | return -1; 146 | } 147 | 148 | return context->list.head.ptr->next; 149 | } 150 | 151 | /** 152 | get next element 153 | parameters: 154 | list: the list 155 | return next object offset 156 | */ 157 | static inline int64_t shm_list_next(struct shmcache_context *context, 158 | const int64_t current_offset) 159 | { 160 | struct shm_list *node; 161 | node = shm_list_ptr(context, current_offset); 162 | if (node->next == context->list.head.offset) { 163 | return -1; 164 | } 165 | return node->next; 166 | } 167 | 168 | /** 169 | get count 170 | parameters: 171 | list: the list 172 | return count 173 | */ 174 | static inline int shm_list_count(struct shmcache_context *context) 175 | { 176 | int64_t offset; 177 | int count; 178 | 179 | count = 0; 180 | offset = context->list.head.ptr->next; 181 | while (offset != context->list.head.offset) { 182 | count++; 183 | offset = shm_list_ptr(context, offset)->next; 184 | } 185 | 186 | return count; 187 | } 188 | 189 | #define SHM_LIST_FOR_EACH(context, current, member) \ 190 | for (current=SHM_LIST_TYPE_PTR(context, typeof(*current), \ 191 | context->list.head.ptr->next); \ 192 | ¤t->member != context->list.head.ptr; \ 193 | current=SHM_LIST_TYPE_PTR(context, typeof(*current), \ 194 | current->member.next)) 195 | 196 | #ifdef __cplusplus 197 | } 198 | #endif 199 | 200 | #endif 201 | 202 | -------------------------------------------------------------------------------- /src/shm_lock.c: -------------------------------------------------------------------------------- 1 | //shm_lock.c 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include "fastcommon/logger.h" 8 | #include "fastcommon/shared_func.h" 9 | #include "fastcommon/sched_thread.h" 10 | #include "shm_hashtable.h" 11 | #include "shm_lock.h" 12 | 13 | int shm_lock_init(struct shmcache_context *context) 14 | { 15 | pthread_mutexattr_t mat; 16 | int result; 17 | 18 | if ((result=pthread_mutexattr_init(&mat)) != 0) { 19 | logError("file: "__FILE__", line: %d, " 20 | "call pthread_mutexattr_init fail, " 21 | "errno: %d, error info: %s", 22 | __LINE__, result, strerror(result)); 23 | return result; 24 | } 25 | if ((result=pthread_mutexattr_setpshared(&mat, 26 | PTHREAD_PROCESS_SHARED)) != 0) 27 | { 28 | logError("file: "__FILE__", line: %d, " 29 | "call pthread_mutexattr_setpshared fail, " 30 | "errno: %d, error info: %s", 31 | __LINE__, result, strerror(result)); 32 | return result; 33 | } 34 | if ((result=pthread_mutexattr_settype(&mat, 35 | PTHREAD_MUTEX_NORMAL)) != 0) 36 | { 37 | logError("file: "__FILE__", line: %d, " 38 | "call pthread_mutexattr_settype fail, " 39 | "errno: %d, error info: %s", 40 | __LINE__, result, strerror(result)); 41 | return result; 42 | } 43 | if ((result=pthread_mutex_init(&context->memory->lock.mutex, 44 | &mat)) != 0) 45 | { 46 | logError("file: "__FILE__", line: %d, " 47 | "call pthread_mutex_init fail, " 48 | "errno: %d, error info: %s", 49 | __LINE__, result, strerror(result)); 50 | return result; 51 | } 52 | pthread_mutexattr_destroy(&mat); 53 | 54 | context->memory->lock.pid = 0; 55 | return 0; 56 | } 57 | 58 | int shm_lock_file(struct shmcache_context *context) 59 | { 60 | int result; 61 | mode_t old_mast; 62 | 63 | if (context->lock_fd > 0) { 64 | close(context->lock_fd); 65 | } 66 | 67 | old_mast = umask(0); 68 | context->lock_fd = open(context->config.filename, O_WRONLY | O_CREAT, 0666); 69 | umask(old_mast); 70 | if (context->lock_fd < 0) { 71 | result = errno != 0 ? errno : EPERM; 72 | logError("file: "__FILE__", line: %d, " 73 | "open filename: %s fail, " 74 | "errno: %d, error info: %s", __LINE__, 75 | context->config.filename, result, strerror(result)); 76 | return result; 77 | } 78 | 79 | if ((result=file_write_lock(context->lock_fd)) != 0) { 80 | close(context->lock_fd); 81 | context->lock_fd = -1; 82 | logError("file: "__FILE__", line: %d, " 83 | "lock filename: %s fail, " 84 | "errno: %d, error info: %s", __LINE__, 85 | context->config.filename, result, strerror(result)); 86 | return result; 87 | } 88 | 89 | return result; 90 | } 91 | 92 | void shm_unlock_file(struct shmcache_context *context) 93 | { 94 | close(context->lock_fd); 95 | context->lock_fd = -1; 96 | } 97 | 98 | static int shm_detect_deadlock(struct shmcache_context *context, 99 | const pid_t last_pid) 100 | { 101 | int result; 102 | if ((result=shm_lock_file(context)) != 0) { 103 | return result; 104 | } 105 | 106 | if (last_pid == context->memory->lock.pid) { 107 | context->memory->lock.pid = 0; 108 | if (shm_ht_clear(context) > 0) { 109 | if (context->config.va_policy.sleep_us_when_recycle_valid_entries > 0) { 110 | usleep(context->config.va_policy.sleep_us_when_recycle_valid_entries); 111 | } 112 | } 113 | if ((result=pthread_mutex_unlock(&context->memory->lock.mutex)) != 0) { 114 | logError("file: "__FILE__", line: %d, " 115 | "call pthread_mutex_unlock fail, " 116 | "errno: %d, error info: %s", 117 | __LINE__, result, strerror(result)); 118 | } else { 119 | __sync_add_and_fetch(&context->memory->stats. 120 | lock.unlock_deadlock, 1); 121 | context->memory->stats.lock. 122 | last_unlock_deadlock_time = get_current_time(); 123 | logInfo("file: "__FILE__", line: %d, " 124 | "my pid: %d, unlock deadlock by process: %d", 125 | __LINE__, context->pid, last_pid); 126 | } 127 | } 128 | 129 | shm_unlock_file(context); 130 | return 0; 131 | } 132 | 133 | int shm_lock(struct shmcache_context *context) 134 | { 135 | int result; 136 | pid_t pid; 137 | int clocks; 138 | 139 | __sync_add_and_fetch(&context->memory->stats.lock.total, 1); 140 | clocks = 0; 141 | while ((result=pthread_mutex_trylock(&context->memory->lock.mutex)) == EBUSY) { 142 | __sync_add_and_fetch(&context->memory->stats.lock.retry, 1); 143 | usleep(context->config.lock_policy.trylock_interval_us); 144 | ++clocks; 145 | if (clocks > context->detect_deadlock_clocks && 146 | (pid=context->memory->lock.pid) > 0) 147 | { 148 | clocks = 0; 149 | if (kill(pid, 0) != 0) { 150 | if (errno == ESRCH || errno == ENOENT) { 151 | __sync_add_and_fetch(&context->memory->stats. 152 | lock.detect_deadlock, 1); 153 | context->memory->stats.lock. 154 | last_detect_deadlock_time = get_current_time(); 155 | shm_detect_deadlock(context, pid); 156 | } 157 | } 158 | } 159 | } 160 | if (result == 0) { 161 | context->memory->lock.pid = context->pid; 162 | } else { 163 | logError("file: "__FILE__", line: %d, " 164 | "call pthread_mutex_trylock fail, " 165 | "errno: %d, error info: %s", 166 | __LINE__, result, strerror(result)); 167 | } 168 | return result; 169 | } 170 | 171 | int shm_unlock(struct shmcache_context *context) 172 | { 173 | int result; 174 | 175 | context->memory->lock.pid = 0; 176 | if ((result=pthread_mutex_unlock(&context->memory->lock.mutex)) != 0) { 177 | logError("file: "__FILE__", line: %d, " 178 | "call pthread_mutex_unlock fail, " 179 | "errno: %d, error info: %s", 180 | __LINE__, result, strerror(result)); 181 | } 182 | return result; 183 | } 184 | 185 | -------------------------------------------------------------------------------- /src/shm_lock.h: -------------------------------------------------------------------------------- 1 | //shm_lock.h 2 | 3 | #ifndef _SHM_LOCK_H 4 | #define _SHM_LOCK_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | lock init 19 | parameters: 20 | context: the context pointer 21 | return errno, 0 for success, != 0 fail 22 | */ 23 | int shm_lock_init(struct shmcache_context *context); 24 | 25 | /** 26 | lock 27 | parameters: 28 | context: the context pointer 29 | return errno, 0 for success, != 0 fail 30 | */ 31 | int shm_lock(struct shmcache_context *context); 32 | 33 | /** 34 | unlock 35 | parameters: 36 | context: the context pointer 37 | return errno, 0 for success, != 0 fail 38 | */ 39 | int shm_unlock(struct shmcache_context *context); 40 | 41 | /** 42 | lock file 43 | parameters: 44 | context: the context pointer 45 | return errno, 0 for success, != 0 fail 46 | */ 47 | int shm_lock_file(struct shmcache_context *context); 48 | 49 | /** 50 | unlock file 51 | parameters: 52 | context: the context pointer 53 | return none 54 | */ 55 | void shm_unlock_file(struct shmcache_context *context); 56 | 57 | #ifdef __cplusplus 58 | } 59 | #endif 60 | 61 | #endif 62 | 63 | -------------------------------------------------------------------------------- /src/shm_object_pool.c: -------------------------------------------------------------------------------- 1 | //shm_object_pool.c 2 | 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "shm_object_pool.h" 7 | 8 | void shm_object_pool_set(struct shmcache_object_pool_context *op, 9 | struct shm_object_pool_info *obj_pool_info, 10 | int64_t *offsets) 11 | { 12 | op->obj_pool_info = obj_pool_info; 13 | op->offsets = offsets; 14 | op->index = -1; 15 | } 16 | 17 | void shm_object_pool_init_full(struct shmcache_object_pool_context *op) 18 | { 19 | int64_t *p; 20 | int64_t *end; 21 | int64_t offset; 22 | 23 | end = op->offsets + (op->obj_pool_info->queue.capacity - 1); 24 | offset = op->obj_pool_info->object.base_offset; 25 | for (p=op->offsets; pobj_pool_info->object.element_size; 28 | } 29 | 30 | op->obj_pool_info->queue.head = 0; 31 | op->obj_pool_info->queue.tail = op->obj_pool_info->queue.capacity - 1; 32 | } 33 | 34 | void shm_object_pool_init_empty(struct shmcache_object_pool_context *op) 35 | { 36 | op->obj_pool_info->queue.head = op->obj_pool_info->queue.tail = 0; 37 | } 38 | 39 | int shm_object_pool_get_count(struct shmcache_object_pool_context *op) 40 | { 41 | if (op->obj_pool_info->queue.head == op->obj_pool_info->queue.tail) { 42 | return 0; 43 | } else if (op->obj_pool_info->queue.head < op->obj_pool_info->queue.tail) { 44 | return op->obj_pool_info->queue.tail - op->obj_pool_info->queue.head; 45 | } else { 46 | return (op->obj_pool_info->queue.capacity - op->obj_pool_info->queue.head) 47 | + op->obj_pool_info->queue.tail; 48 | } 49 | } 50 | 51 | int64_t shm_object_pool_alloc(struct shmcache_object_pool_context *op) 52 | { 53 | int64_t obj_offset; 54 | if (op->obj_pool_info->queue.head == op->obj_pool_info->queue.tail) { 55 | return -1; 56 | } 57 | 58 | obj_offset = op->offsets[op->obj_pool_info->queue.head]; 59 | op->obj_pool_info->queue.head = (op->obj_pool_info->queue.head + 1) % 60 | op->obj_pool_info->queue.capacity; 61 | 62 | return obj_offset; 63 | } 64 | 65 | int shm_object_pool_free(struct shmcache_object_pool_context *op, 66 | const int64_t obj_offset) 67 | { 68 | int next_tail; 69 | next_tail = (op->obj_pool_info->queue.tail + 1) % 70 | op->obj_pool_info->queue.capacity; 71 | if (next_tail == op->obj_pool_info->queue.head) { 72 | return ENOSPC; 73 | } 74 | op->offsets[op->obj_pool_info->queue.tail] = obj_offset; 75 | op->obj_pool_info->queue.tail = next_tail; 76 | 77 | return 0; 78 | } 79 | 80 | #define SHM_OP_DEC_INDEX(index, capacity) \ 81 | (index == 0 ? (capacity - 1) : (index - 1)) 82 | 83 | int64_t shm_object_pool_remove(struct shmcache_object_pool_context *op) 84 | { 85 | int index; 86 | int previous; 87 | int current; 88 | 89 | if (op->obj_pool_info->queue.head == op->obj_pool_info->queue.tail) { 90 | logError("file: "__FILE__", line: %d, " 91 | "object pool is empty", __LINE__); 92 | return -1; 93 | } 94 | if (op->index < 0) { 95 | logError("file: "__FILE__", line: %d, " 96 | "current index: %d < 0", __LINE__, op->index); 97 | return -1; 98 | } 99 | 100 | index = op->index; 101 | current = index; 102 | while (current != op->obj_pool_info->queue.head) { 103 | previous = SHM_OP_DEC_INDEX(current, op->obj_pool_info->queue.capacity); 104 | op->offsets[current] = op->offsets[previous]; 105 | current = previous; 106 | } 107 | 108 | op->obj_pool_info->queue.head = (op->obj_pool_info->queue.head + 1) % 109 | op->obj_pool_info->queue.capacity; 110 | return op->offsets[index]; 111 | } 112 | 113 | int64_t shm_object_pool_remove_by(struct shmcache_object_pool_context *op, 114 | const int64_t obj_offset) 115 | { 116 | int64_t current_offset; 117 | 118 | current_offset = shm_object_pool_first(op); 119 | while (current_offset > 0) { 120 | if (current_offset == obj_offset) { 121 | return shm_object_pool_remove(op); 122 | } 123 | current_offset = shm_object_pool_next(op); 124 | } 125 | 126 | return -1; 127 | } 128 | 129 | -------------------------------------------------------------------------------- /src/shm_object_pool.h: -------------------------------------------------------------------------------- 1 | //shm_object_pool.h 2 | 3 | #ifndef _SHM_OBJECT_POOL_H 4 | #define _SHM_OBJECT_POOL_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | get object pool memory size for queue 19 | parameters: 20 | max_count: the max count 21 | return memory size 22 | */ 23 | static inline int64_t shm_object_pool_get_queue_memory_size(const int max_count) 24 | { 25 | return sizeof(int64_t) * (int64_t)max_count; 26 | } 27 | 28 | /** 29 | get object pool memory size for object 30 | parameters: 31 | element_size: the element size 32 | max_count: the max count 33 | return memory size 34 | */ 35 | static inline int64_t shm_object_pool_get_object_memory_size( 36 | const int element_size, const int max_count) 37 | { 38 | return (int64_t)element_size * (int64_t)max_count; 39 | } 40 | 41 | /** 42 | set object pool 43 | parameters: 44 | op: the object pool 45 | obj_pool_info: fifo pool in share memory 46 | offsets: the array offsets 47 | return error no, 0 for success, != 0 fail 48 | */ 49 | void shm_object_pool_set(struct shmcache_object_pool_context *op, 50 | struct shm_object_pool_info *obj_pool_info, 51 | int64_t *offsets); 52 | 53 | /** 54 | init object pool to empty queue 55 | parameters: 56 | op: the object pool 57 | return error no, 0 for success, != 0 fail 58 | */ 59 | void shm_object_pool_init_empty(struct shmcache_object_pool_context *op); 60 | 61 | /** 62 | init object pool to full queue 63 | parameters: 64 | op: the object pool 65 | return error no, 0 for success, != 0 fail 66 | */ 67 | void shm_object_pool_init_full(struct shmcache_object_pool_context *op); 68 | 69 | /** 70 | get object count in object pool 71 | parameters: 72 | op: the object pool 73 | return object count 74 | */ 75 | int shm_object_pool_get_count(struct shmcache_object_pool_context *op); 76 | 77 | /** 78 | alloc a node from the object pool 79 | parameters: 80 | op: the object pool 81 | return the alloced object offset, return -1 if fail 82 | */ 83 | int64_t shm_object_pool_alloc(struct shmcache_object_pool_context *op); 84 | 85 | #define shm_object_pool_pop(op) shm_object_pool_alloc(op) 86 | 87 | /** 88 | free a node to the object pool 89 | parameters: 90 | op: the object pool 91 | obj_offset: the object offset 92 | return 0 for success, != 0 fail 93 | */ 94 | int shm_object_pool_free(struct shmcache_object_pool_context *op, 95 | const int64_t obj_offset); 96 | 97 | #define shm_object_pool_push(op, obj_offset) shm_object_pool_free(op, obj_offset) 98 | 99 | /** 100 | is object pool empty 101 | parameters: 102 | op: the object pool 103 | return return true if empty, otherwise false 104 | */ 105 | static inline bool shm_object_pool_is_empty(struct shmcache_object_pool_context *op) 106 | { 107 | return (op->obj_pool_info->queue.head == op->obj_pool_info->queue.tail); 108 | } 109 | 110 | /** 111 | get first object 112 | parameters: 113 | op: the object pool 114 | return object offset, return -1 if empty 115 | */ 116 | static inline int64_t shm_object_pool_first(struct shmcache_object_pool_context *op) 117 | { 118 | if (op->obj_pool_info->queue.head == op->obj_pool_info->queue.tail) { 119 | op->index = -1; 120 | return -1; 121 | } 122 | op->index = op->obj_pool_info->queue.head; 123 | return op->offsets[op->index]; 124 | } 125 | 126 | /** 127 | get next object 128 | parameters: 129 | op: the object pool 130 | return object offset, return -1 if empty 131 | */ 132 | static inline int64_t shm_object_pool_next(struct shmcache_object_pool_context *op) 133 | { 134 | if (op->index == -1) { 135 | return -1; 136 | } 137 | if (op->index == op->obj_pool_info->queue.tail) { 138 | op->index = -1; 139 | return -1; 140 | } 141 | 142 | op->index = (op->index + 1) % op->obj_pool_info->queue.capacity; 143 | if (op->index != op->obj_pool_info->queue.tail) { 144 | return op->offsets[op->index]; 145 | } else { 146 | return -1; 147 | } 148 | } 149 | 150 | /** 151 | remove the object, if op->index is -1, remove the tail 152 | parameters: 153 | op: the object pool 154 | return the removed object offset, return -1 if empty 155 | */ 156 | int64_t shm_object_pool_remove(struct shmcache_object_pool_context *op); 157 | 158 | /** 159 | remove the object 160 | parameters: 161 | op: the object pool 162 | obj_offset: the object offset 163 | return the removed object offset, return -1 if empty 164 | */ 165 | int64_t shm_object_pool_remove_by(struct shmcache_object_pool_context *op, 166 | const int64_t obj_offset); 167 | 168 | #ifdef __cplusplus 169 | } 170 | #endif 171 | 172 | #endif 173 | 174 | -------------------------------------------------------------------------------- /src/shm_op_wrapper.c: -------------------------------------------------------------------------------- 1 | //shm_op_wrapper.c 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/logger.h" 11 | #include "fastcommon/shared_func.h" 12 | #include "shm_op_wrapper.h" 13 | 14 | #define SHM_GET_MMAP_FILENAME(true_filename, filename, proj_id) \ 15 | snprintf(true_filename, sizeof(true_filename), "%s.%d", \ 16 | filename, proj_id - 1) 17 | 18 | static void *shm_do_mmap(const char *filename, int proj_id, 19 | const int64_t size, const bool create_segment, int *err_no) 20 | { 21 | char true_filename[MAX_PATH_SIZE]; 22 | void *addr; 23 | int fd; 24 | mode_t old_mast; 25 | bool need_truncate; 26 | 27 | SHM_GET_MMAP_FILENAME(true_filename, filename, proj_id); 28 | fd = open(true_filename, O_RDWR); 29 | if (fd >= 0) { 30 | struct stat st; 31 | if (fstat(fd, &st) != 0) { 32 | *err_no = errno != 0 ? errno : EPERM; 33 | logError("file: "__FILE__", line: %d, " 34 | "stat file: %s fail, " 35 | "errno: %d, error info: %s", __LINE__, 36 | true_filename, *err_no, strerror(*err_no)); 37 | return NULL; 38 | } 39 | if (st.st_size > size) { 40 | logWarning("file: "__FILE__", line: %d, " 41 | "file %s size: %"PRId64" > expect size: %"PRId64, 42 | __LINE__, true_filename, st.st_size, size); 43 | need_truncate = false; 44 | } else if (st.st_size < size) { 45 | logWarning("file: "__FILE__", line: %d, " 46 | "file %s size: %"PRId64" < expect size: %"PRId64 47 | ", auto extend file size", __LINE__, 48 | true_filename, st.st_size, size); 49 | need_truncate = true; 50 | } else { 51 | need_truncate = false; 52 | } 53 | } else { 54 | if (!(create_segment && errno == ENOENT)) { 55 | *err_no = errno != 0 ? errno : EPERM; 56 | logError("file: "__FILE__", line: %d, " 57 | "open file: %s fail, " 58 | "errno: %d, error info: %s", __LINE__, 59 | true_filename, *err_no, strerror(*err_no)); 60 | return NULL; 61 | } 62 | 63 | old_mast = umask(0); 64 | fd = open(true_filename, O_RDWR | O_CREAT, 0666); 65 | umask(old_mast); 66 | 67 | if (fd < 0) { 68 | *err_no = errno != 0 ? errno : EPERM; 69 | logError("file: "__FILE__", line: %d, " 70 | "open file: %s fail, " 71 | "errno: %d, error info: %s", __LINE__, 72 | true_filename, *err_no, strerror(*err_no)); 73 | return NULL; 74 | } 75 | need_truncate = true; 76 | } 77 | if (need_truncate) { 78 | if (ftruncate(fd, size) != 0) { 79 | *err_no = errno != 0 ? errno : EPERM; 80 | close(fd); 81 | logError("file: "__FILE__", line: %d, " 82 | "truncate file: %s to size %"PRId64" fail, " 83 | "errno: %d, error info: %s", __LINE__, 84 | true_filename, size, *err_no, strerror(*err_no)); 85 | return NULL; 86 | } 87 | } 88 | 89 | addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); 90 | if (addr == NULL || addr == MAP_FAILED) { 91 | *err_no = errno != 0 ? errno : EPERM; 92 | logError("file: "__FILE__", line: %d, " 93 | "mmap file: %s with size: %"PRId64" fail, " 94 | "errno: %d, error info: %s", __LINE__, 95 | true_filename, size, *err_no, strerror(*err_no)); 96 | } 97 | close(fd); 98 | *err_no = 0; 99 | return addr; 100 | } 101 | 102 | void *shm_do_shmmap(const key_t key, const int64_t size, 103 | const bool create_segment, int *err_no) 104 | { 105 | int shmid; 106 | void *addr; 107 | 108 | if (create_segment) { 109 | shmid = shmget(key, size, IPC_CREAT | 0666); 110 | } else { 111 | shmid = shmget(key, 0, 0666); 112 | } 113 | if (shmid < 0) { 114 | *err_no = errno != 0 ? errno : EPERM; 115 | logError("file: "__FILE__", line: %d, " 116 | "shmget with key %08x fail, " 117 | "errno: %d, error info: %s", __LINE__, 118 | key, *err_no, strerror(*err_no)); 119 | return NULL; 120 | } 121 | 122 | addr = shmat(shmid, NULL, 0); 123 | if (addr == NULL || addr == (void *)-1) { 124 | *err_no = errno != 0 ? errno : EPERM; 125 | logError("file: "__FILE__", line: %d, " 126 | "shmat with key %08x fail, " 127 | "errno: %d, error info: %s", __LINE__, 128 | key, *err_no, strerror(*err_no)); 129 | return NULL; 130 | } 131 | *err_no = 0; 132 | return addr; 133 | } 134 | 135 | static int shm_get_key(const char *filename, const int proj_id, 136 | key_t *key) 137 | { 138 | int result; 139 | if (access(filename, F_OK) != 0) { 140 | result = errno != 0 ? errno : ENOENT; 141 | if (result != ENOENT) { 142 | logError("file: "__FILE__", line: %d, " 143 | "access filename: %s fail, " 144 | "errno: %d, error info: %s", __LINE__, 145 | filename, result, strerror(result)); 146 | return result; 147 | } 148 | 149 | result = writeToFile(filename, "FOR LOCK", 8); 150 | if (result != 0) { 151 | return result; 152 | } 153 | if (chmod(filename, 0666) != 0) { 154 | result = errno != 0 ? errno : EFAULT; 155 | logError("file: "__FILE__", line: %d, " 156 | "chmod filename: %s fail, " 157 | "errno: %d, error info: %s", __LINE__, 158 | filename, result, strerror(result)); 159 | return result; 160 | } 161 | } 162 | 163 | *key = fc_ftok(filename, proj_id); 164 | if (*key == -1) { 165 | result = errno != 0 ? errno : EFAULT; 166 | logError("file: "__FILE__", line: %d, " 167 | "call fc_ftok fail, filename: %s, proj_id: %d, " 168 | "errno: %d, error info: %s", __LINE__, 169 | filename, proj_id, result, strerror(result)); 170 | return result; 171 | } 172 | return 0; 173 | } 174 | 175 | void *shm_mmap(const int type, const char *filename, 176 | const int proj_id, const int64_t size, key_t *key, 177 | const bool create_segment, int *err_no) 178 | { 179 | if ((*err_no=shm_get_key(filename, proj_id, key)) != 0) { 180 | return NULL; 181 | } 182 | if (type == SHMCACHE_TYPE_MMAP) { 183 | return shm_do_mmap(filename, proj_id, size, create_segment, err_no); 184 | } else { 185 | return shm_do_shmmap(*key, size, create_segment, err_no); 186 | } 187 | } 188 | 189 | bool shm_exists(const int type, const char *filename, const int proj_id) 190 | { 191 | key_t key; 192 | char true_filename[MAX_PATH_SIZE]; 193 | 194 | if (shm_get_key(filename, proj_id, &key) != 0) { 195 | return false; 196 | } 197 | if (type == SHMCACHE_TYPE_MMAP) { 198 | SHM_GET_MMAP_FILENAME(true_filename, filename, proj_id); 199 | return access(true_filename, F_OK) == 0; 200 | } else { 201 | return shmget(key, 0, 0666) >= 0; 202 | } 203 | } 204 | 205 | int shm_munmap(const int type, void *addr, const int64_t size) 206 | { 207 | int result; 208 | if (type == SHMCACHE_TYPE_MMAP) { 209 | if (munmap(addr, size) == 0) { 210 | result = 0; 211 | } else { 212 | result = errno != 0 ? errno : EACCES; 213 | logError("file: "__FILE__", line: %d, " 214 | "munmap addr: %p, size: %"PRId64" fail, " 215 | "errno: %d, error info: %s", __LINE__, 216 | addr, size, errno, strerror(errno)); 217 | } 218 | } else { 219 | if (shmdt(addr) == 0) { 220 | result = 0; 221 | } else { 222 | result = errno != 0 ? errno : EACCES; 223 | logError("file: "__FILE__", line: %d, " 224 | "munmap addr: %p, size: %"PRId64" fail, " 225 | "errno: %d, error info: %s", __LINE__, 226 | addr, size, errno, strerror(errno)); 227 | } 228 | } 229 | return result; 230 | } 231 | 232 | int shm_remove(const int type, const char *filename, 233 | const int proj_id, const int64_t size, const key_t key) 234 | { 235 | int result; 236 | char true_filename[MAX_PATH_SIZE]; 237 | 238 | if (type == SHMCACHE_TYPE_MMAP) { 239 | snprintf(true_filename, sizeof(true_filename), "%s.%d", 240 | filename, proj_id - 1); 241 | if (unlink(true_filename) != 0) { 242 | result = errno != 0 ? errno : EPERM; 243 | logError("file: "__FILE__", line: %d, " 244 | "unlink file: %s fail, " 245 | "errno: %d, error info: %s", __LINE__, 246 | true_filename, errno, strerror(errno)); 247 | return result; 248 | } 249 | } else { 250 | int shmid; 251 | 252 | shmid = shmget(key, 0, 0666); 253 | if (shmid < 0) { 254 | result = errno != 0 ? errno : EACCES; 255 | logError("file: "__FILE__", line: %d, " 256 | "shmget with key %08x fail, " 257 | "errno: %d, error info: %s", __LINE__, 258 | key, errno, strerror(errno)); 259 | return result; 260 | } 261 | 262 | if (shmctl(shmid, IPC_RMID, NULL) != 0) { 263 | result = errno != 0 ? errno : EACCES; 264 | logError("file: "__FILE__", line: %d, " 265 | "remove shm with key %08x fail, " 266 | "errno: %d, error info: %s", __LINE__, 267 | key, errno, strerror(errno)); 268 | return result; 269 | } 270 | } 271 | return 0; 272 | } 273 | 274 | -------------------------------------------------------------------------------- /src/shm_op_wrapper.h: -------------------------------------------------------------------------------- 1 | //shm_op_wrapper.h 2 | 3 | #ifndef _SHM_OP_WRAPPER_H 4 | #define _SHM_OP_WRAPPER_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | mmap or shmget & shmat 19 | parameters: 20 | type: mmap or shm 21 | filename: the filename 22 | proj_id: the project id to generate key 23 | size: the share memory size 24 | key: return the key 25 | create_segment: if create segment when segment not exist 26 | err_no: return errno 27 | return share memory pointer, NULL for fail 28 | */ 29 | void *shm_mmap(const int type, const char *filename, 30 | const int proj_id, const int64_t size, key_t *key, 31 | const bool create_segment, int *err_no); 32 | 33 | /** 34 | munmap or shmdt 35 | parameters: 36 | type: mmap or shm 37 | addr: the address to munmap 38 | size: the share memory size 39 | return: errno, 0 for success, != 0 fail 40 | */ 41 | int shm_munmap(const int type, void *addr, const int64_t size); 42 | 43 | /** 44 | remove shm 45 | parameters: 46 | type: mmap or shm 47 | filename: the filename 48 | proj_id: the project id to generate key 49 | size: the share memory size 50 | key: the key 51 | return: errno, 0 for success, != 0 fail 52 | */ 53 | int shm_remove(const int type, const char *filename, 54 | const int proj_id, const int64_t size, const key_t key); 55 | 56 | /** 57 | if shm exists 58 | parameters: 59 | type: mmap or shm 60 | filename: the filename 61 | proj_id: the project id to generate key 62 | return: errno, 0 for success, != 0 fail 63 | */ 64 | bool shm_exists(const int type, const char *filename, const int proj_id); 65 | 66 | #ifdef __cplusplus 67 | } 68 | #endif 69 | 70 | #endif 71 | 72 | -------------------------------------------------------------------------------- /src/shm_striping_allocator.c: -------------------------------------------------------------------------------- 1 | //shm_striping_allocator.c 2 | 3 | #include 4 | #include "fastcommon/sched_thread.h" 5 | #include "shm_striping_allocator.h" 6 | 7 | void shm_striping_allocator_init(struct shm_striping_allocator *allocator, 8 | const struct shm_segment_striping_pair *ssp_index, 9 | const int64_t base_offset, const int total_size) 10 | { 11 | allocator->index = *ssp_index; 12 | allocator->offset.base = base_offset; 13 | allocator->size.total = total_size; 14 | allocator->offset.end = base_offset + total_size; 15 | 16 | shm_striping_allocator_reset(allocator); 17 | } 18 | 19 | int64_t shm_striping_allocator_alloc(struct shm_striping_allocator *allocator, 20 | const int size) 21 | { 22 | int64_t ptr_offset; 23 | if (allocator->offset.end - allocator->offset.free < size) { 24 | allocator->fail_times++; 25 | return -1; 26 | } 27 | 28 | allocator->last_alloc_time = g_current_time; 29 | ptr_offset = allocator->offset.free; 30 | allocator->size.used += size; 31 | allocator->offset.free += size; 32 | return ptr_offset; 33 | } 34 | -------------------------------------------------------------------------------- /src/shm_striping_allocator.h: -------------------------------------------------------------------------------- 1 | //shm_striping_allocator.h 2 | 3 | #ifndef _SHM_STRIPING_ALLOCATOR_H 4 | #define _SHM_STRIPING_ALLOCATOR_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | allocator init 19 | parameters: 20 | allocator: the allocator pointer 21 | ssp_index: segment and striping pair 22 | base_offset: the base offset 23 | total_size: the memory size 24 | return error no, 0 for success, != 0 fail 25 | */ 26 | void shm_striping_allocator_init(struct shm_striping_allocator *allocator, 27 | const struct shm_segment_striping_pair *ssp_index, 28 | const int64_t base_offset, const int total_size); 29 | 30 | /** 31 | reset for recycle use 32 | parameters: 33 | allocator: the allocator pointer 34 | */ 35 | static inline void shm_striping_allocator_reset(struct shm_striping_allocator *allocator) 36 | { 37 | allocator->last_alloc_time = 0; 38 | allocator->fail_times = 0; 39 | allocator->size.used = 0; 40 | allocator->offset.free = allocator->offset.base; 41 | } 42 | 43 | /** 44 | alloc memory from the allocator 45 | parameters: 46 | allocator: the allocator pointer 47 | size: alloc bytes 48 | return the alloced memory offset, return -1 if fail 49 | */ 50 | int64_t shm_striping_allocator_alloc(struct shm_striping_allocator *allocator, 51 | const int size); 52 | 53 | /** 54 | free memory to the allocator 55 | parameters: 56 | allocator: the allocator pointer 57 | size: alloc bytes 58 | return used size 59 | */ 60 | static inline int64_t shm_striping_allocator_free(struct shm_striping_allocator 61 | *allocator, const int size) 62 | { 63 | allocator->size.used -= size; 64 | return allocator->size.used; 65 | } 66 | 67 | /** 68 | get free memory size 69 | parameters: 70 | allocator: the allocator pointer 71 | return free memory size 72 | */ 73 | static inline int64_t shm_striping_allocator_free_size(struct shm_striping_allocator 74 | *allocator) 75 | { 76 | return allocator->offset.end - allocator->offset.free; 77 | } 78 | 79 | /** 80 | get used memory size 81 | parameters: 82 | allocator: the allocator pointer 83 | return used memory size 84 | */ 85 | static inline int64_t shm_striping_allocator_used_size(struct shm_striping_allocator 86 | *allocator) 87 | { 88 | return allocator->size.used; 89 | } 90 | 91 | #ifdef __cplusplus 92 | } 93 | #endif 94 | 95 | #endif 96 | 97 | -------------------------------------------------------------------------------- /src/shm_value_allocator.c: -------------------------------------------------------------------------------- 1 | //shm_value_allocator.c 2 | 3 | #include 4 | #include 5 | #include "fastcommon/sched_thread.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "shm_object_pool.h" 8 | #include "shm_striping_allocator.h" 9 | #include "shm_list.h" 10 | #include "shmopt.h" 11 | #include "shm_hashtable.h" 12 | #include "shm_value_allocator.h" 13 | 14 | static struct shm_hash_entry *shm_value_striping_alloc( 15 | struct shmcache_context *context, 16 | struct shm_striping_allocator *allocator, const int size) 17 | { 18 | int64_t offset; 19 | char *base; 20 | struct shm_hash_entry *entry; 21 | offset = shm_striping_allocator_alloc(allocator, size); 22 | if (offset < 0) { 23 | return NULL; 24 | } 25 | 26 | base = shmopt_get_value_segment(context, allocator->index.segment); 27 | if (base == NULL) { 28 | return NULL; 29 | } 30 | 31 | entry = (struct shm_hash_entry *)(base + offset); 32 | entry->memory.offset = offset; 33 | entry->memory.index = allocator->index; 34 | entry->memory.size = size; 35 | return entry; 36 | } 37 | 38 | static struct shm_hash_entry *shm_value_allocator_do_alloc(struct shmcache_context *context, 39 | const int size) 40 | { 41 | int64_t allocator_offset; 42 | int64_t removed_offset; 43 | struct shm_striping_allocator *allocator; 44 | struct shm_hash_entry *entry; 45 | 46 | allocator_offset = shm_object_pool_first(&context->value_allocator.doing); 47 | while (allocator_offset > 0) { 48 | allocator = (struct shm_striping_allocator *)(context->segments. 49 | hashtable.base + allocator_offset); 50 | if ((entry=shm_value_striping_alloc(context, allocator, size)) != NULL) { 51 | context->memory->usage.used.entry += size; 52 | return entry; 53 | } 54 | 55 | if ((shm_striping_allocator_free_size(allocator) <= context->config. 56 | va_policy.discard_memory_size) || (allocator->fail_times > 57 | context->config.va_policy.max_fail_times)) 58 | { 59 | removed_offset = shm_object_pool_remove(&context->value_allocator.doing); 60 | if (removed_offset == allocator_offset) { 61 | allocator->in_which_pool = SHMCACHE_STRIPING_ALLOCATOR_POOL_DONE; 62 | shm_object_pool_push(&context->value_allocator.done, allocator_offset); 63 | } else { 64 | logCrit("file: "__FILE__", line: %d, " 65 | "shm_object_pool_remove fail, " 66 | "offset: %"PRId64" != expect: %"PRId64, __LINE__, 67 | removed_offset, allocator_offset); 68 | } 69 | } 70 | 71 | allocator_offset = shm_object_pool_next(&context->value_allocator.doing); 72 | } 73 | 74 | return NULL; 75 | } 76 | 77 | 78 | static int shm_value_allocator_do_recycle(struct shmcache_context *context, 79 | struct shm_striping_allocator *allocator) 80 | { 81 | int64_t allocator_offset; 82 | if (allocator->in_which_pool == SHMCACHE_STRIPING_ALLOCATOR_POOL_DONE) { 83 | allocator_offset = (char *)allocator - context->segments.hashtable.base; 84 | if (shm_object_pool_remove_by(&context->value_allocator.done, 85 | allocator_offset) >= 0) { 86 | allocator->in_which_pool = SHMCACHE_STRIPING_ALLOCATOR_POOL_DOING; 87 | shm_object_pool_push(&context->value_allocator.doing, allocator_offset); 88 | } else { 89 | logCrit("file: "__FILE__", line: %d, " 90 | "shm_object_pool_remove_by fail, " 91 | "index: %d, offset: %"PRId64, __LINE__, 92 | allocator->index.striping, allocator_offset); 93 | return EFAULT; 94 | } 95 | } 96 | return 0; 97 | } 98 | 99 | int shm_value_allocator_recycle(struct shmcache_context *context, 100 | struct shm_recycle_stats *recycle_stats, 101 | const int recycle_keys_once) 102 | { 103 | int64_t entry_offset; 104 | int64_t current_offset; 105 | int64_t start_time; 106 | struct shm_hash_entry *entry; 107 | struct shmcache_key_info key; 108 | int result; 109 | int index; 110 | int scan_count; 111 | int clear_count; 112 | int valid_count; 113 | bool valid; 114 | bool recycled; 115 | char buff[32]; 116 | 117 | result = ENOMEM; 118 | scan_count = clear_count = valid_count = 0; 119 | start_time = get_current_time_us(); 120 | g_current_time = start_time / 1000000; 121 | recycled = false; 122 | entry_offset = shm_list_first(context); 123 | while (entry_offset > 0) { 124 | scan_count++; 125 | entry = shm_get_hentry_ptr(context, entry_offset); 126 | index = entry->memory.index.striping; 127 | key.data = entry->key; 128 | key.length = entry->key_len; 129 | valid = HT_ENTRY_IS_VALID(entry, g_current_time); 130 | if (valid && !context->config.recycle_valid_entries) { 131 | entry_offset = shm_list_next(context, entry_offset); 132 | continue; 133 | } 134 | 135 | current_offset = entry_offset; 136 | entry_offset = shm_list_next(context, entry_offset); 137 | 138 | if (shm_ht_delete_ex(context, &key, &recycled) != 0) { 139 | logError("file: "__FILE__", line: %d, " 140 | "shm_ht_delete fail, index: %d, " 141 | "entry offset: %"PRId64", " 142 | "key: %.*s, key length: %d", __LINE__, 143 | index, current_offset, entry->key_len, 144 | entry->key, entry->key_len); 145 | 146 | shm_ht_free_entry(context, entry, current_offset, &recycled); 147 | } 148 | 149 | clear_count++; 150 | if (valid) { 151 | valid_count++; 152 | } 153 | 154 | if (recycle_keys_once > 0) { 155 | if (clear_count >= recycle_keys_once) { 156 | result = 0; 157 | break; 158 | } 159 | } else if (recycled) { 160 | long_to_comma_str(get_current_time_us() - start_time, buff); 161 | logInfo("file: "__FILE__", line: %d, " 162 | "recycle #%d striping memory, " 163 | "scan entries: %d, clear total entries: %d, " 164 | "clear valid entries: %d, " 165 | "time used: %s us", __LINE__, index, 166 | scan_count, clear_count, valid_count, buff); 167 | result = 0; 168 | break; 169 | } 170 | } 171 | 172 | context->memory->stats.memory.clear_ht_entry.total += clear_count; 173 | if (valid_count > 0) { 174 | context->memory->stats.memory.clear_ht_entry.valid += valid_count; 175 | } 176 | 177 | recycle_stats->last_recycle_time = g_current_time; 178 | recycle_stats->total++; 179 | if (result == 0) { 180 | recycle_stats->success++; 181 | if (valid_count > 0) { 182 | recycle_stats->force++; 183 | if (context->config.va_policy. 184 | sleep_us_when_recycle_valid_entries > 0) 185 | { 186 | usleep(context->config.va_policy. 187 | sleep_us_when_recycle_valid_entries); 188 | } 189 | } 190 | } else { 191 | long_to_comma_str(get_current_time_us() - start_time, buff); 192 | logError("file: "__FILE__", line: %d, " 193 | "unable to recycle memory, " 194 | "scan entries: %d, clear total entries: %d, " 195 | "cleared valid entries: %d, " 196 | "time used: %s us", __LINE__, scan_count, 197 | clear_count, valid_count, buff); 198 | } 199 | return result; 200 | } 201 | 202 | static int shm_value_allocator_try_rearrange(struct shmcache_context *context) 203 | { 204 | int64_t used; 205 | int64_t free; 206 | int64_t start_time; 207 | int result; 208 | struct shmcache_hentry_array array; 209 | struct shmcache_hash_entry *entry; 210 | struct shmcache_hash_entry *end; 211 | 212 | used = context->memory->usage.used.common + context->memory->usage.used.entry; 213 | free = context->memory->usage.alloced - used; 214 | if (free < (SHMCACHE_MAX_KEY_SIZE + context->config.max_value_size) + 215 | (context->memory->vm_info.striping.count.current * 216 | context->config.va_policy.discard_memory_size) + 217 | context->memory->vm_info.striping.size) 218 | { 219 | return ENOSPC; 220 | } 221 | 222 | start_time = get_current_time_us(); 223 | logInfo("file: "__FILE__", line: %d, " 224 | "try rearrange shm, key count: %d", 225 | __LINE__, context->memory->hashtable.count); 226 | 227 | if ((result=shm_ht_to_array(context, &array)) != 0) { 228 | return result; 229 | } 230 | 231 | if (shm_ht_clear(context) > 0) { 232 | if (context->config.va_policy.sleep_us_when_recycle_valid_entries > 0) { 233 | usleep(context->config.va_policy.sleep_us_when_recycle_valid_entries); 234 | } 235 | } 236 | 237 | end = array.entries + array.count; 238 | for (entry=array.entries; entrykey, &entry->value)) != 0) { 240 | break; 241 | } 242 | } 243 | 244 | if (result == 0) { 245 | char buff[32]; 246 | long_to_comma_str(get_current_time_us() - start_time, buff); 247 | logInfo("file: "__FILE__", line: %d, " 248 | "rearrange shm done, key count: %d, time used: %s us", 249 | __LINE__, context->memory->hashtable.count, buff); 250 | } else { 251 | shm_ht_clear(context); 252 | } 253 | 254 | shm_ht_free_array(&array); 255 | return result; 256 | } 257 | 258 | struct shm_hash_entry *shm_value_allocator_alloc(struct shmcache_context *context, 259 | const int key_len, const int value_len) 260 | { 261 | int result; 262 | int size; 263 | bool recycle; 264 | int64_t allocator_offset; 265 | struct shm_striping_allocator *allocator; 266 | struct shm_hash_entry *entry; 267 | 268 | size = sizeof(struct shm_hash_entry) + MEM_ALIGN(key_len) + MEM_ALIGN(value_len); 269 | if ((entry=shm_value_allocator_do_alloc(context, size)) != NULL) { 270 | return entry; 271 | } 272 | 273 | if (context->memory->vm_info.segment.count.current >= 274 | context->memory->vm_info.segment.count.max) 275 | { 276 | recycle = true; 277 | } else { 278 | allocator_offset = shm_object_pool_first(&context->value_allocator.done); 279 | if (allocator_offset > 0) { 280 | allocator = (struct shm_striping_allocator *)(context->segments. 281 | hashtable.base + allocator_offset); 282 | recycle = (context->config.va_policy.avg_key_ttl > 0 && g_current_time - 283 | allocator->last_alloc_time >= context->config.va_policy.avg_key_ttl); 284 | } else { 285 | recycle = false; 286 | } 287 | } 288 | 289 | if (recycle ) { 290 | result = shm_value_allocator_recycle(context, 291 | &context->memory->stats.memory.recycle.value_striping, -1); 292 | 293 | if (result != 0 && !context->config.recycle_valid_entries) { 294 | if (context->memory->vm_info.segment.count.current < 295 | context->memory->vm_info.segment.count.max) 296 | { 297 | result = shmopt_create_value_segment(context); 298 | } else { 299 | result = shm_value_allocator_try_rearrange(context); 300 | if (result != 0) { 301 | logError("file: "__FILE__", line: %d, " 302 | "try rearrange shm fail, " 303 | "errno: %d, error info: %s", 304 | __LINE__, result, strerror(result)); 305 | } 306 | } 307 | } 308 | } else { 309 | result = shmopt_create_value_segment(context); 310 | } 311 | if (result == 0) { 312 | entry = shm_value_allocator_do_alloc(context, size); 313 | } 314 | if (entry == NULL) { 315 | logError("file: "__FILE__", line: %d, " 316 | "malloc %d bytes from shm fail", __LINE__, size); 317 | } 318 | return entry; 319 | } 320 | 321 | int shm_value_allocator_free(struct shmcache_context *context, 322 | struct shm_hash_entry *entry, bool *recycled) 323 | { 324 | struct shm_striping_allocator *allocator; 325 | int64_t used; 326 | 327 | allocator = context->value_allocator.allocators + entry->memory.index.striping; 328 | used = shm_striping_allocator_free(allocator, entry->memory.size); 329 | context->memory->usage.used.entry -= entry->memory.size; 330 | if (used <= 0) { 331 | if (used < 0) { 332 | logError("file: "__FILE__", line: %d, " 333 | "striping used memory: %"PRId64" < 0, " 334 | "segment: %d, striping: %d, offset: %"PRId64", size: %d", 335 | __LINE__, used, entry->memory.index.segment, 336 | entry->memory.index.striping, entry->memory.offset, 337 | entry->memory.size); 338 | } 339 | *recycled = true; 340 | shm_striping_allocator_reset(allocator); 341 | return shm_value_allocator_do_recycle(context, allocator); 342 | } 343 | 344 | return 0; 345 | } 346 | -------------------------------------------------------------------------------- /src/shm_value_allocator.h: -------------------------------------------------------------------------------- 1 | //shm_value_allocator.h 2 | 3 | #ifndef _SHM_VALUE_ALLOCATOR_H 4 | #define _SHM_VALUE_ALLOCATOR_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | #include "shmopt.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | /** 19 | alloc memory from the allocator 20 | parameters: 21 | context: the shm context 22 | key_len: the key length 23 | value_len: the value length 24 | return error no, 0 for success, != 0 fail 25 | */ 26 | struct shm_hash_entry *shm_value_allocator_alloc(struct shmcache_context *context, 27 | const int key_len, const int value_len); 28 | 29 | /** 30 | free memory to the allocator 31 | parameters: 32 | context: the shm context 33 | value: the value to free 34 | recycled: if recycled 35 | return error no, 0 for success, != 0 fail 36 | */ 37 | int shm_value_allocator_free(struct shmcache_context *context, 38 | struct shm_hash_entry *entry, bool *recycled); 39 | 40 | /** 41 | recycle oldest hashtable entries 42 | parameters: 43 | context: the shm context 44 | recycle_stats: the recycle stats 45 | recycle_key_once: recycle key number once, 46 | <= 0 for until recycle a value allocator 47 | return error no, 0 for success, != 0 fail 48 | */ 49 | int shm_value_allocator_recycle(struct shmcache_context *context, 50 | struct shm_recycle_stats *recycle_stats, 51 | const int recycle_keys_once); 52 | 53 | static inline char *shm_get_value_ptr(struct shmcache_context *context, 54 | struct shm_hash_entry *entry) 55 | { 56 | char *base; 57 | base = shmopt_get_value_segment(context, entry->memory.index.segment); 58 | if (base != NULL) { 59 | return base + entry->memory.offset + sizeof(struct shm_hash_entry) 60 | + MEM_ALIGN(entry->key_len); 61 | } else { 62 | return NULL; 63 | } 64 | } 65 | 66 | static inline struct shm_hash_entry *shm_get_hentry_ptr(struct shmcache_context *context, 67 | const int64_t offset) 68 | { 69 | char *base; 70 | union shm_hentry_offset conv; 71 | 72 | conv.offset = offset; 73 | base = shmopt_get_value_segment(context, conv.segment.index & 0xFF); 74 | if (base != NULL) { 75 | return (struct shm_hash_entry *)(base + conv.segment.offset); 76 | } else { 77 | return NULL; 78 | } 79 | } 80 | 81 | static inline int64_t shm_get_hentry_offset(struct shm_hash_entry *entry) 82 | { 83 | union shm_hentry_offset conv; 84 | conv.segment.index = entry->memory.index.segment | 0x4000; 85 | conv.segment.offset = entry->memory.offset; 86 | 87 | return conv.offset; 88 | } 89 | 90 | #ifdef __cplusplus 91 | } 92 | #endif 93 | 94 | #endif 95 | -------------------------------------------------------------------------------- /src/shmcache.h: -------------------------------------------------------------------------------- 1 | //shmcache.h 2 | 3 | #ifndef _SHMCACHE_H 4 | #define _SHMCACHE_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "fastcommon/common_define.h" 13 | #include "shmcache_types.h" 14 | #include "shm_hashtable.h" 15 | #include "shm_lock.h" 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | /** 22 | context init 23 | parameters: 24 | context: the context pointer 25 | config: the config parameters 26 | create_segment: if create segment when segment not exist 27 | check_segment: if check segment 28 | return error no, 0 for success, != 0 for fail 29 | */ 30 | int shmcache_init(struct shmcache_context *context, 31 | struct shmcache_config *config, const bool create_segment, 32 | const bool check_segment); 33 | 34 | /** 35 | context init from config file 36 | parameters: 37 | context: the context pointer 38 | config_filename: the config filename 39 | create_segment: if create segment when segment not exist 40 | check_segment: if check segment 41 | return error no, 0 for success, != 0 for fail 42 | */ 43 | int shmcache_init_from_file_ex(struct shmcache_context *context, 44 | const char *config_filename, const bool create_segment, 45 | const bool check_segment); 46 | 47 | /** 48 | context init from config file 49 | parameters: 50 | context: the context pointer 51 | config_filename: the config filename 52 | return error no, 0 for success, != 0 for fail 53 | */ 54 | static inline int shmcache_init_from_file(struct shmcache_context *context, 55 | const char *config_filename) 56 | { 57 | return shmcache_init_from_file_ex(context, config_filename, true, true); 58 | } 59 | 60 | /** 61 | load config from file 62 | parameters: 63 | config: the config pointer 64 | config_filename: the config filename 65 | return error no, 0 for success, != 0 for fail 66 | */ 67 | int shmcache_load_config(struct shmcache_config *config, 68 | const char *config_filename); 69 | 70 | /** 71 | context destroy 72 | parameters: 73 | context: the context pointer 74 | */ 75 | void shmcache_destroy(struct shmcache_context *context); 76 | 77 | /** 78 | set value 79 | parameters: 80 | context: the context pointer 81 | key: the key 82 | value: the value, include expire filed 83 | return error no, 0 for success, != 0 for fail 84 | */ 85 | int shmcache_set_ex(struct shmcache_context *context, 86 | const struct shmcache_key_info *key, 87 | const struct shmcache_value_info *value); 88 | 89 | /** 90 | set value 91 | parameters: 92 | context: the context pointer 93 | key: the key 94 | data: the value string ptr 95 | data_len: the value length 96 | ttl: the time to live in seconds 97 | return error no, 0 for success, != 0 for fail 98 | */ 99 | int shmcache_set(struct shmcache_context *context, 100 | const struct shmcache_key_info *key, 101 | const char *data, const int data_len, const int ttl); 102 | 103 | /** 104 | set TTL 105 | parameters: 106 | context: the context pointer 107 | key: the key 108 | ttl: the time to live in seconds 109 | return error no, 0 for success, != 0 for fail 110 | */ 111 | int shmcache_set_ttl(struct shmcache_context *context, 112 | const struct shmcache_key_info *key, const int ttl); 113 | 114 | 115 | /** 116 | set expires timestamp 117 | parameters: 118 | context: the context pointer 119 | key: the key 120 | expires: the expires timestamp 121 | return error no, 0 for success, != 0 for fail 122 | */ 123 | int shmcache_set_expires(struct shmcache_context *context, 124 | const struct shmcache_key_info *key, const int expires); 125 | 126 | /** 127 | increase integer value 128 | parameters: 129 | context: the context pointer 130 | key: the key 131 | incr: the incremental number 132 | ttl: the time to live in seconds 133 | value: return the new value 134 | return error no, 0 for success, != 0 for fail 135 | */ 136 | int shmcache_incr(struct shmcache_context *context, 137 | const struct shmcache_key_info *key, 138 | const int64_t increment, 139 | const int ttl, int64_t *new_value); 140 | 141 | /** 142 | get value 143 | parameters: 144 | context: the context pointer 145 | key: the key 146 | value: store the returned value 147 | return error no, 0 for success, != 0 for fail 148 | */ 149 | int shmcache_get(struct shmcache_context *context, 150 | const struct shmcache_key_info *key, 151 | struct shmcache_value_info *value); 152 | 153 | /** 154 | delte the key 155 | parameters: 156 | context: the context pointer 157 | key: the key 158 | return error no, 0 for success, != 0 for fail 159 | */ 160 | int shmcache_delete(struct shmcache_context *context, 161 | const struct shmcache_key_info *key); 162 | 163 | 164 | /** 165 | remove all share memory 166 | parameters: 167 | context: the context pointer 168 | return error no, 0 for success, != 0 for fail 169 | */ 170 | int shmcache_remove_all(struct shmcache_context *context); 171 | 172 | /** 173 | get stats 174 | parameters: 175 | context: the context pointer 176 | stats: return the stats 177 | calc_hit_ratio: if calculate hit_ratio 178 | return none 179 | */ 180 | void shmcache_stats_ex(struct shmcache_context *context, struct shmcache_stats *stats, 181 | const bool calc_hit_ratio); 182 | 183 | static inline void shmcache_stats(struct shmcache_context *context, 184 | struct shmcache_stats *stats) 185 | { 186 | shmcache_stats_ex(context, stats, true); 187 | } 188 | 189 | /** 190 | clear stats 191 | parameters: 192 | context: the context pointer 193 | return none 194 | */ 195 | void shmcache_clear_stats(struct shmcache_context *context); 196 | 197 | /** 198 | get serializer label 199 | parameters: 200 | serializer: the serializer 201 | return serializer label 202 | */ 203 | const char *shmcache_get_serializer_label(const int serializer); 204 | 205 | /** 206 | clear hashtable 207 | parameters: 208 | context: the context pointer 209 | return error no, 0 for success, != 0 for fail 210 | */ 211 | int shmcache_clear(struct shmcache_context *context); 212 | 213 | /** 214 | get last clear time 215 | parameters: 216 | context: the context pointer 217 | return last clear time 218 | */ 219 | static inline int64_t shmcache_get_last_clear_time(struct shmcache_context *context) 220 | { 221 | return context->memory->stats.hashtable.last_clear_time; 222 | } 223 | 224 | /** 225 | get shm init time 226 | parameters: 227 | context: the context pointer 228 | return the init time 229 | */ 230 | static inline int64_t shmcache_get_init_time(struct shmcache_context *context) 231 | { 232 | return context->memory->init_time; 233 | } 234 | 235 | #ifdef __cplusplus 236 | } 237 | #endif 238 | 239 | #endif 240 | -------------------------------------------------------------------------------- /src/shmcache_ini_annotation.c: -------------------------------------------------------------------------------- 1 | //shmcache_ini_annotation.c 2 | 3 | #include 4 | #include 5 | #include 6 | #include "fastcommon/logger.h" 7 | #include "fastcommon/shared_func.h" 8 | #include "fastcommon/json_parser.h" 9 | #include "shmcache.h" 10 | #include "shmcache_ini_annotation.h" 11 | 12 | typedef struct { 13 | struct shmcache_context shm_ctx; 14 | fc_json_context_t json_ctx; 15 | } ShmcacheAnnotationArgs; 16 | 17 | static int config_get(IniContext *context, 18 | struct ini_annotation_entry *annotation, 19 | const IniItem *item, char **pOutValue, int max_values) 20 | { 21 | ShmcacheAnnotationArgs *args; 22 | int count; 23 | int result; 24 | struct shmcache_key_info key; 25 | struct shmcache_value_info value; 26 | char *output; 27 | 28 | args = (ShmcacheAnnotationArgs *)annotation->arg; 29 | if (item->value[0] != '\0') { 30 | key.data = (char *)item->value; 31 | } else { 32 | key.data = (char *)item->name; 33 | } 34 | 35 | value.options = SHMCACHE_SERIALIZER_STRING; 36 | key.length = strlen(key.data); 37 | if ((result= shmcache_get(&args->shm_ctx, &key, &value)) != 0) { 38 | if (result == ENOENT || result == ETIMEDOUT) { 39 | logError("file: "__FILE__", line: %d, " 40 | "shmcache_get fail, key: %s not exist", 41 | __LINE__, key.data); 42 | } else { 43 | logError("file: "__FILE__", line: %d, " 44 | "shmcache_get fail, key: %s, error info: %s", 45 | __LINE__, key.data, strerror(result)); 46 | } 47 | value.data = ""; 48 | value.length = 0; 49 | } else if ((value.options & SHMCACHE_SERIALIZER_STRING) == 0) { 50 | logError("file: "__FILE__", line: %d, " 51 | "unsupport serilizer %s (%d)", __LINE__, 52 | shmcache_get_serializer_label(value.options), 53 | value.options); 54 | value.data = ""; 55 | value.length = 0; 56 | } 57 | 58 | count = 0; 59 | if ((value.options & SHMCACHE_SERIALIZER_LIST) == 60 | SHMCACHE_SERIALIZER_LIST) 61 | { 62 | string_t input; 63 | const fc_json_array_t *array; 64 | 65 | input.str = value.data; 66 | input.len = value.length; 67 | if ((array=fc_decode_json_array(&args->json_ctx, &input)) != NULL) { 68 | string_t *el; 69 | string_t *end; 70 | 71 | end = array->elements + array->count; 72 | if (array->count > max_values) { 73 | end = array->elements + max_values; 74 | } 75 | 76 | for (el=array->elements; elstr, el->len); 78 | if (output == NULL) { 79 | return count; 80 | } 81 | pOutValue[count++] = output; 82 | } 83 | } else { 84 | logError("file: "__FILE__", line: %d, " 85 | "decode json array fail, %s", __LINE__, 86 | fc_json_parser_get_error_info(&args->json_ctx)->str); 87 | pOutValue[count++] = strdup(""); 88 | } 89 | } else { 90 | output = fc_strdup1(value.data, value.length); 91 | if (output == NULL) { 92 | return count; 93 | } 94 | pOutValue[count++] = output; 95 | } 96 | 97 | return count; 98 | } 99 | 100 | static void annotation_destroy(struct ini_annotation_entry *annotation) 101 | { 102 | ShmcacheAnnotationArgs *args; 103 | 104 | if (annotation->arg != NULL) { 105 | args = (ShmcacheAnnotationArgs *)annotation->arg; 106 | shmcache_destroy(&args->shm_ctx); 107 | fc_destroy_json_context(&args->json_ctx); 108 | annotation->arg = NULL; 109 | } 110 | } 111 | 112 | int CONFIG_GET_init_annotation(AnnotationEntry *annotation, 113 | const char *config_filename) 114 | { 115 | ShmcacheAnnotationArgs *args; 116 | int result; 117 | 118 | memset(annotation, 0, sizeof(AnnotationEntry)); 119 | annotation->func_name = "CONFIG_GET"; 120 | annotation->func_get = config_get; 121 | annotation->func_free = iniAnnotationFreeValues; 122 | annotation->func_destroy = annotation_destroy; 123 | 124 | args = (ShmcacheAnnotationArgs *)malloc( 125 | sizeof(ShmcacheAnnotationArgs)); 126 | if (args == NULL) { 127 | logError("file: "__FILE__", line: %d, " 128 | "malloc %d bytes fail", __LINE__, 129 | (int)sizeof(ShmcacheAnnotationArgs)); 130 | return ENOMEM; 131 | } 132 | 133 | if ((result=shmcache_init_from_file_ex(&args->shm_ctx, 134 | config_filename, false, true)) != 0) 135 | { 136 | free(args); 137 | return result; 138 | } 139 | 140 | if ((result=fc_init_json_context(&args->json_ctx)) != 0) { 141 | free(args); 142 | return result; 143 | } 144 | 145 | annotation->arg = args; 146 | return 0; 147 | } 148 | -------------------------------------------------------------------------------- /src/shmcache_ini_annotation.h: -------------------------------------------------------------------------------- 1 | //shmcache_ini_annotation.h 2 | 3 | #ifndef _SHMCACHE_INI_ANNOTATION_H 4 | #define _SHMCACHE_INI_ANNOTATION_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "shmcache_types.h" 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /** 18 | parameters: 19 | annotation: the annotation entry 20 | config_filename: the libshmcache config filename 21 | return errno, 0 for success, != 0 fail 22 | */ 23 | int CONFIG_GET_init_annotation(AnnotationEntry *annotation, 24 | const char *config_filename); 25 | 26 | #ifdef __cplusplus 27 | } 28 | #endif 29 | 30 | #endif 31 | 32 | -------------------------------------------------------------------------------- /src/shmcache_types.h: -------------------------------------------------------------------------------- 1 | //shmcache_types.h 2 | 3 | #ifndef _SHMCACHE_TYPES_H 4 | #define _SHMCACHE_TYPES_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include "fastcommon/common_define.h" 13 | #include "fastcommon/hash.h" 14 | 15 | #define SHMCACHE_MAJOR_VERSION 1 16 | #define SHMCACHE_MINOR_VERSION 0 17 | #define SHMCACHE_PATCH_VERSION 6 18 | 19 | #define SHMCACHE_MAX_KEY_SIZE 64 20 | 21 | #define SHMCACHE_STATUS_INIT 0 22 | #define SHMCACHE_STATUS_NORMAL 0x12345678 23 | 24 | #define SHMCACHE_TYPE_SHM 1 25 | #define SHMCACHE_TYPE_MMAP 2 26 | 27 | #define SHMCACHE_NEVER_EXPIRED 0 28 | 29 | #define SHMCACHE_SERIALIZER_STRING 0x1 //string type 30 | #define SHMCACHE_SERIALIZER_LIST (SHMCACHE_SERIALIZER_STRING | 0x2) //list 31 | #define SHMCACHE_SERIALIZER_MAP (SHMCACHE_SERIALIZER_STRING | 0x4) //map 32 | #define SHMCACHE_SERIALIZER_INTEGER (SHMCACHE_SERIALIZER_STRING | 0x8) //integer type 33 | #define SHMCACHE_SERIALIZER_IGBINARY 0x200 34 | #define SHMCACHE_SERIALIZER_MSGPACK 0x400 35 | #define SHMCACHE_SERIALIZER_PHP 0x800 36 | 37 | #define SHMCACHE_STRIPING_ALLOCATOR_POOL_DOING 0 38 | #define SHMCACHE_STRIPING_ALLOCATOR_POOL_DONE 1 39 | 40 | struct shmcache_config { 41 | char filename[MAX_PATH_SIZE]; 42 | int64_t min_memory; 43 | int64_t max_memory; 44 | int64_t segment_size; 45 | int max_key_count; 46 | int max_value_size; 47 | int type; //shm or mmap 48 | 49 | int recycle_key_once; //recycle key number once when reach max keys 50 | 51 | bool recycle_valid_entries; //if recycle valid entries by FIFO 52 | 53 | struct { 54 | /* avg. key TTL threshold for recycling memory 55 | * unit: second 56 | * <= 0 for never recycle memory until reach memory limit (max_memory) 57 | */ 58 | int avg_key_ttl; 59 | 60 | /* when the remain memory <= this parameter, discard it. 61 | * put this allocator in the doing queue to the done queue 62 | */ 63 | int discard_memory_size; 64 | 65 | /* when a allocator in the doing queue allocate fail times > this parameter, 66 | * means it is almost full, put it to the done queue 67 | */ 68 | int max_fail_times; 69 | 70 | /* sleep time to avoid other processes read dirty data when recycle 71 | * more than one valid (in TTL / not expired) KV entries. 72 | * 0 for never sleep 73 | */ 74 | int sleep_us_when_recycle_valid_entries; 75 | } va_policy; //value allocator policy 76 | 77 | struct { 78 | bool read_within_lock; 79 | int trylock_interval_us; 80 | int detect_deadlock_interval_ms; 81 | } lock_policy; 82 | 83 | HashFunc hash_func; 84 | }; 85 | 86 | struct shm_segment_striping_pair { 87 | short segment; //shm segment index 88 | short striping; //shm striping index 89 | }; 90 | 91 | struct shm_value { 92 | int length; //value length 93 | int options; //options for application 94 | }; 95 | 96 | struct shm_list { 97 | int64_t prev; 98 | int64_t next; 99 | }; 100 | 101 | union shm_hentry_offset { 102 | int64_t offset; 103 | struct { 104 | int64_t index :16; 105 | int64_t offset :48; 106 | } segment; 107 | }; 108 | 109 | struct shm_hash_entry { 110 | struct shm_list list; //for recycle, must be first 111 | 112 | int key_len; 113 | time_t expires; 114 | struct shm_value value; 115 | 116 | struct { 117 | int size; //alloc size 118 | struct shm_segment_striping_pair index; 119 | int64_t offset; //value segment offset 120 | } memory; 121 | 122 | int64_t ht_next; //for hashtable 123 | char key[0]; 124 | }; 125 | 126 | struct shm_ring_queue { 127 | int capacity; 128 | int head; //for pop 129 | int tail; //for push 130 | }; 131 | 132 | struct shm_object_pool_info { 133 | struct { 134 | int64_t base_offset; 135 | int element_size; 136 | } object; 137 | struct shm_ring_queue queue; 138 | }; 139 | 140 | struct shm_hashtable { 141 | struct shm_list head; //for recycle 142 | int capacity; 143 | int count; 144 | int64_t buckets[0]; //entry offset 145 | }; 146 | 147 | struct shm_striping_allocator { 148 | time_t last_alloc_time; //record the timestamp of first allocate 149 | int fail_times; //allocate fail times 150 | short in_which_pool; //in doing or done 151 | struct shm_segment_striping_pair index; 152 | struct { 153 | int total; 154 | int used; 155 | } size; 156 | 157 | struct { 158 | int64_t base; 159 | int64_t free; 160 | int64_t end; 161 | } offset; 162 | }; 163 | 164 | struct shm_value_allocator { 165 | struct shm_object_pool_info doing; 166 | struct shm_object_pool_info done; 167 | }; 168 | 169 | struct shm_value_size_info { 170 | int64_t size; 171 | struct { 172 | int current; 173 | int max; 174 | } count; 175 | }; 176 | 177 | struct shm_value_memory_info { 178 | struct shm_value_size_info segment; 179 | struct shm_value_size_info striping; 180 | }; 181 | 182 | struct shm_lock { 183 | pid_t pid; 184 | pthread_mutex_t mutex; 185 | }; 186 | 187 | struct shm_counter { 188 | volatile int64_t total; 189 | volatile int64_t success; 190 | }; 191 | 192 | struct shm_recycle_stats { 193 | int64_t total; //total count 194 | int64_t success; //succes count 195 | int64_t force; //force recycle count (clear valid entries) 196 | int64_t last_recycle_time; 197 | }; 198 | 199 | struct shm_stats { 200 | struct { 201 | struct shm_counter set; 202 | struct shm_counter get; 203 | struct shm_counter del; 204 | struct shm_counter incr; 205 | int64_t last_clear_time; 206 | } hashtable; 207 | 208 | struct { 209 | struct { 210 | int64_t total; 211 | int64_t valid; 212 | } clear_ht_entry; //clear for recycle 213 | 214 | struct { 215 | struct shm_recycle_stats key; 216 | struct shm_recycle_stats value_striping; 217 | } recycle; 218 | } memory; 219 | 220 | struct { 221 | volatile int64_t total; 222 | volatile int64_t retry; 223 | volatile int64_t detect_deadlock; 224 | volatile int64_t unlock_deadlock; 225 | int64_t last_detect_deadlock_time; 226 | int64_t last_unlock_deadlock_time; 227 | } lock; 228 | 229 | //for calculate hit ratio 230 | struct { 231 | struct shm_counter get; 232 | int64_t calc_time; 233 | } last; 234 | 235 | int64_t init_time; //init unix timestamp 236 | }; 237 | 238 | struct shm_memory_usage { 239 | int64_t alloced; 240 | struct { 241 | int64_t common; //first segment include hashtable 242 | int64_t entry; //whole entry include key and value 243 | int64_t key; 244 | int64_t value; 245 | } used; 246 | }; 247 | 248 | struct shm_memory_info { 249 | int size; //sizeof(struct shm_memory_info) 250 | int status; 251 | time_t init_time; //init unix timestamp 252 | int max_key_count; 253 | struct shm_lock lock; 254 | struct shm_value_memory_info vm_info; //value memory info 255 | struct shm_value_allocator value_allocator; 256 | struct shm_stats stats; 257 | struct shm_memory_usage usage; 258 | struct shm_hashtable hashtable; //must be last 259 | }; 260 | 261 | struct shmcache_key_info { 262 | char *data; 263 | int length; 264 | }; 265 | 266 | struct shmcache_value_info { 267 | char *data; 268 | int length; 269 | int options; //options for application 270 | time_t expires; //expire time 271 | }; 272 | 273 | struct shmcache_hash_entry { 274 | struct shmcache_key_info key; 275 | struct shmcache_value_info value; 276 | }; 277 | 278 | struct shmcache_hentry_array { 279 | struct shmcache_hash_entry *entries; 280 | int count; 281 | }; 282 | 283 | struct shmcache_segment_info { 284 | int proj_id; 285 | key_t key; //shm key 286 | int64_t size; //memory size 287 | char *base; 288 | }; 289 | 290 | struct shmcache_object_pool_context { 291 | struct shm_object_pool_info *obj_pool_info; 292 | int64_t *offsets; //object offset array 293 | int index; //for iterator 294 | }; 295 | 296 | struct shmcache_value_allocator_context { 297 | struct shmcache_object_pool_context doing; //doing queue 298 | struct shmcache_object_pool_context done; //done queue 299 | struct shm_striping_allocator *allocators; //base address 300 | }; 301 | 302 | struct shmcache_list { 303 | struct { 304 | struct shm_list *ptr; 305 | int64_t offset; 306 | } head; 307 | }; 308 | 309 | struct shmcache_context { 310 | pid_t pid; 311 | int lock_fd; //for file lock 312 | int detect_deadlock_clocks; 313 | struct shmcache_config config; 314 | struct shm_memory_info *memory; 315 | struct { 316 | struct shmcache_segment_info hashtable; 317 | struct { 318 | int count; 319 | struct shmcache_segment_info *items; 320 | } values; 321 | } segments; 322 | 323 | struct shmcache_value_allocator_context value_allocator; 324 | struct shmcache_list list; //for value recycle 325 | bool create_segment; //if check segment size 326 | }; 327 | 328 | struct shmcache_stats { 329 | struct shm_stats shm; 330 | struct { 331 | int64_t segment_size; //segment memory size 332 | int capacity; 333 | int count; //key count 334 | } hashtable; 335 | int max_key_count; 336 | 337 | struct { 338 | int64_t max; 339 | int64_t used; 340 | struct shm_memory_usage usage; 341 | } memory; 342 | 343 | struct { 344 | double ratio; 345 | double get_qps; 346 | int seconds; 347 | } hit; 348 | }; 349 | 350 | #ifdef __cplusplus 351 | extern "C" { 352 | #endif 353 | 354 | #ifdef __cplusplus 355 | } 356 | #endif 357 | 358 | #endif 359 | 360 | -------------------------------------------------------------------------------- /src/shmopt.c: -------------------------------------------------------------------------------- 1 | //shmopt.c 2 | 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "shm_op_wrapper.h" 7 | #include "shm_striping_allocator.h" 8 | #include "shm_object_pool.h" 9 | #include "shmopt.h" 10 | 11 | int shmopt_init_segment(struct shmcache_context *context, 12 | struct shmcache_segment_info *segment, 13 | const int proj_id, const int64_t size) 14 | { 15 | int result; 16 | segment->proj_id = proj_id; 17 | segment->base = shm_mmap(context->config.type, 18 | context->config.filename, proj_id, size, &segment->key, 19 | context->create_segment, &result); 20 | if (segment->base == NULL) { 21 | return result; 22 | } 23 | 24 | segment->size = size; 25 | return 0; 26 | } 27 | 28 | static int shmopt_init_value_segment(struct shmcache_context *context, 29 | const int segment_index) 30 | { 31 | int proj_id; 32 | struct shmcache_segment_info *segment; 33 | 34 | //proj_id 1 for hashtable segment, value segments start from 2 35 | proj_id = segment_index + 2; 36 | segment = context->segments.values.items + segment_index; 37 | return shmopt_init_segment(context, segment, proj_id, 38 | context->memory->vm_info.segment.size); 39 | } 40 | 41 | int shmopt_create_value_segment(struct shmcache_context *context) 42 | { 43 | int result; 44 | int striping_count; 45 | int segment_index; 46 | int striping_index; 47 | int i; 48 | int64_t striping_offset; 49 | int64_t allocator_offset; 50 | struct shm_segment_striping_pair index_pair; 51 | struct shm_striping_allocator *allocator; 52 | 53 | if (context->memory->vm_info.segment.count.current >= 54 | context->memory->vm_info.segment.count.max) 55 | { 56 | logError("file: "__FILE__", line: %d, " 57 | "no value segments, reach max: %d", __LINE__, 58 | context->memory->vm_info.segment.count.max); 59 | return ENOSPC; 60 | } 61 | if (context->segments.values.count != context->memory->vm_info. 62 | segment.count.current) 63 | { 64 | logError("file: "__FILE__", line: %d, " 65 | "my value segments count: %d != that of shm: %d", 66 | __LINE__, context->segments.values.count, 67 | context->memory->vm_info.segment.count.current); 68 | return EINVAL; 69 | } 70 | 71 | segment_index = context->memory->vm_info.segment.count.current; 72 | if ((result=shmopt_init_value_segment(context, segment_index)) != 0) { 73 | return result; 74 | } 75 | context->memory->vm_info.segment.count.current++; 76 | 77 | striping_offset = 0; 78 | index_pair.segment = segment_index; 79 | striping_count = context->memory->vm_info.segment.size / 80 | context->memory->vm_info.striping.size; 81 | 82 | logInfo("file: "__FILE__", line: %d, " 83 | "striping_count: %d, striping.count.current: %d, " 84 | "context->memory->vm_info.segment.size: %"PRId64", " 85 | "context->memory->vm_info.striping.size: %"PRId64, 86 | __LINE__, striping_count, context->memory->vm_info.striping.count.current, 87 | context->memory->vm_info.segment.size, context->memory->vm_info.striping.size); 88 | 89 | for (i=0; imemory->vm_info.striping.count.current++; 91 | allocator = context->value_allocator.allocators + striping_index; 92 | 93 | index_pair.striping = striping_index; 94 | shm_striping_allocator_init(allocator, &index_pair, 95 | striping_offset, context->memory->vm_info.striping.size); 96 | 97 | //add to doing queue 98 | allocator_offset = (char *)allocator - context->segments.hashtable.base; 99 | allocator->in_which_pool = SHMCACHE_STRIPING_ALLOCATOR_POOL_DOING; 100 | shm_object_pool_push(&context->value_allocator.doing, allocator_offset); 101 | 102 | striping_offset += context->memory->vm_info.striping.size; 103 | } 104 | context->segments.values.count = segment_index + 1; 105 | context->memory->usage.alloced += context->memory->vm_info.segment.size; 106 | 107 | logInfo("file: "__FILE__", line: %d, pid: %d, " 108 | "create value segment #%d, size: %"PRId64, 109 | __LINE__, context->pid, segment_index + 1, 110 | context->memory->vm_info.segment.size); 111 | 112 | return 0; 113 | } 114 | 115 | int shmopt_open_value_segments(struct shmcache_context *context) 116 | { 117 | int result; 118 | int segment_index; 119 | 120 | for (segment_index = context->segments.values.count; 121 | segment_index < context->memory->vm_info.segment.count.current; 122 | segment_index++) 123 | { 124 | if ((result=shmopt_init_value_segment(context, segment_index)) != 0) { 125 | return result; 126 | } 127 | context->segments.values.count++; 128 | } 129 | 130 | return 0; 131 | } 132 | 133 | int shmopt_remove_all(struct shmcache_context *context) 134 | { 135 | int result; 136 | int r; 137 | int segment_index; 138 | struct shmcache_segment_info *segment; 139 | 140 | result = shm_remove(context->config.type, context->config.filename, 141 | context->segments.hashtable.proj_id, 142 | context->segments.hashtable.size, 143 | context->segments.hashtable.key); 144 | 145 | for (segment_index=0; segment_indexmemory->vm_info.segment. 146 | count.current; segment_index++) 147 | { 148 | segment = context->segments.values.items + segment_index; 149 | if ((result=shmopt_init_value_segment(context, segment_index)) != 0) { 150 | return result; 151 | } 152 | 153 | if ((r=shm_remove(context->config.type, context->config.filename, 154 | segment->proj_id, segment->size, 155 | segment->key)) != 0) 156 | { 157 | result = r; 158 | } 159 | } 160 | return result; 161 | } 162 | 163 | -------------------------------------------------------------------------------- /src/shmopt.h: -------------------------------------------------------------------------------- 1 | //shmopt.h 2 | 3 | #ifndef _SHMOPT_H 4 | #define _SHMOPT_H 5 | 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/common_define.h" 11 | #include "fastcommon/logger.h" 12 | #include "shmcache_types.h" 13 | 14 | #ifdef __cplusplus 15 | extern "C" { 16 | #endif 17 | 18 | /** 19 | init share memory segment 20 | parameters: 21 | context: the context pointer 22 | segment: the segment pointer 23 | proj_id: the project id to generate key 24 | size: the share memory size 25 | return error no, 0 for success, != 0 for fail 26 | */ 27 | int shmopt_init_segment(struct shmcache_context *context, 28 | struct shmcache_segment_info *segment, 29 | const int proj_id, const int64_t size); 30 | 31 | /** 32 | create and init share memory value segment 33 | parameters: 34 | context: the context pointer 35 | return error no, 0 for success, != 0 for fail 36 | */ 37 | int shmopt_create_value_segment(struct shmcache_context *context); 38 | 39 | 40 | /** 41 | open share memory value segment 42 | parameters: 43 | context: the context pointer 44 | return error no, 0 for success, != 0 for fail 45 | */ 46 | int shmopt_open_value_segments(struct shmcache_context *context); 47 | 48 | /** 49 | alloc a node from the context 50 | parameters: 51 | context: the context pointer 52 | index: value segment index 53 | return the value segment ptr, return NULL if fail 54 | */ 55 | static inline char *shmopt_get_value_segment(struct shmcache_context *context, 56 | const int index) 57 | { 58 | if (index < context->segments.values.count) { 59 | return context->segments.values.items[index].base; 60 | } else if (index < context->memory->vm_info.segment.count.current) { 61 | if (shmopt_open_value_segments(context) != 0) { 62 | return NULL; 63 | } 64 | return context->segments.values.items[index].base; 65 | } else { 66 | logError("file: " __FILE__", line: %d, " 67 | "invalid index: %d >= %d", __LINE__, 68 | index, context->memory->vm_info.segment.count.current); 69 | return NULL; 70 | } 71 | } 72 | 73 | /** 74 | remove all share memory 75 | parameters: 76 | context: the context pointer 77 | return error no, 0 for success, != 0 for fail 78 | */ 79 | int shmopt_remove_all(struct shmcache_context *context); 80 | 81 | #ifdef __cplusplus 82 | } 83 | #endif 84 | 85 | #endif 86 | 87 | -------------------------------------------------------------------------------- /src/tests/Makefile: -------------------------------------------------------------------------------- 1 | .SUFFIXES: .c .o 2 | 3 | COMPILE = $(CC) -g -O1 -Wall -D_FILE_OFFSET_BITS=64 -g -DDEBUG_FLAG 4 | INC_PATH = -I/usr/local/include 5 | LIB_PATH = -lfastcommon -lshmcache -lpthread -ldl 6 | 7 | ALL_PRGS = tests 8 | 9 | all: $(ALL_PRGS) 10 | .c: 11 | $(COMPILE) -o $@ $< $(LIB_PATH) $(INC_PATH) 12 | .c.o: 13 | $(COMPILE) -c -o $@ $< $(INC_PATH) 14 | clean: 15 | rm -f $(ALL_PRGS) 16 | 17 | -------------------------------------------------------------------------------- /src/tests/tests.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/logger.h" 11 | #include "fastcommon/hash.h" 12 | #include "shmcache/shm_hashtable.h" 13 | #include "shmcache/shmcache.h" 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | #define MAX_VALUE_SIZE (8 * 1024) 18 | #define KEY_COUNT 13000 19 | 20 | int result; 21 | int current_time; 22 | struct shmcache_context context; 23 | char szKey[SHMCACHE_MAX_KEY_SIZE]; 24 | char szValue[MAX_VALUE_SIZE]; 25 | struct shmcache_hash_entry *entries; 26 | struct shmcache_hash_entry tmp; 27 | int bytes; 28 | int ttl = 300; 29 | int i, k; 30 | int i1, i2; 31 | 32 | log_init(); 33 | g_log_context.log_level = LOG_INFO; 34 | 35 | printf("sizeof(struct shm_hash_entry): %d\n", 36 | (int)sizeof(struct shm_hash_entry)); 37 | 38 | if ((result=shmcache_init_from_file(&context, 39 | "/usr/local/etc/libshmcache.conf")) != 0) 40 | { 41 | return result; 42 | } 43 | 44 | bytes = sizeof(struct shmcache_hash_entry) * KEY_COUNT; 45 | entries = (struct shmcache_hash_entry *)malloc(bytes); 46 | if (entries == NULL) { 47 | logError("file: "__FILE__", line: %d, " 48 | "malloc %d bytes fail", __LINE__, bytes); 49 | return ENOMEM; 50 | } 51 | 52 | current_time = time(NULL); 53 | srand(current_time); 54 | rand(); 55 | memset(szValue, 'A', sizeof(szValue)); 56 | for (i=0; i 2 | #include 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "shmcache/shmcache.h" 8 | 9 | static void usage(const char *prog) 10 | { 11 | fprintf(stderr, "shmcache delete key.\n" 12 | "Usage: %s [config_filename] \n", prog); 13 | } 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | int result; 18 | int index; 19 | char *config_filename; 20 | struct shmcache_context context; 21 | struct shmcache_key_info key; 22 | 23 | if (argc >= 2 && (strcmp(argv[1], "-h") == 0 || 24 | strcmp(argv[1], "help") == 0 || 25 | strcmp(argv[1], "--help") == 0)) 26 | { 27 | usage(argv[0]); 28 | return 0; 29 | } 30 | if (argc < 2) { 31 | usage(argv[0]); 32 | return EINVAL; 33 | } 34 | 35 | config_filename = "/etc/libshmcache.conf"; 36 | if (isFile(argv[1])) { 37 | if (argc < 3) { 38 | usage(argv[0]); 39 | return EINVAL; 40 | } 41 | config_filename = argv[1]; 42 | index = 2; 43 | } else { 44 | index = 1; 45 | } 46 | 47 | log_init(); 48 | if ((result=shmcache_init_from_file_ex(&context, 49 | config_filename, false, true)) != 0) 50 | { 51 | return result; 52 | } 53 | 54 | key.data = argv[index++]; 55 | key.length = strlen(key.data); 56 | result = shmcache_delete(&context, &key); 57 | if (result == 0) { 58 | printf("delete key: %s successfully.\n", key.data); 59 | } else { 60 | fprintf(stderr, "delete key: %s fail, errno: %d\n", key.data, result); 61 | } 62 | 63 | return result; 64 | } 65 | -------------------------------------------------------------------------------- /src/tools/shmcache_dump.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include "fastcommon/logger.h" 11 | #include "fastcommon/hash.h" 12 | #include "shmcache/shm_hashtable.h" 13 | #include "shmcache/shmcache.h" 14 | 15 | static char *config_filename = "/etc/libshmcache.conf"; 16 | static char *key_value_seperator = "="; 17 | static char *row_seperator = "\n"; 18 | static int start_offset = 0; 19 | static int row_count = 0; 20 | static char *search_key = NULL; 21 | static bool show_usage = false; 22 | static char *output_filename = NULL; 23 | 24 | static void usage(char *program) 25 | { 26 | fprintf(stderr, "Usage: %s options, the options as:\n" 27 | "\t -h help\n" 28 | "\t -c , default: /etc/libshmcache.conf\n" 29 | "\t -F , default: =\n" 30 | "\t -R , default: \\n\n" 31 | "\t -k , SQL like style such as " 32 | "MYKEY%% or %%MYKEY or %%MYKEY%%\n" 33 | "\t -s start offset, default: 0\n" 34 | "\t -n , 0 means NO limit, default: 0\n" 35 | "\t -o output filename, default: output to sdtout\n" 36 | "\n" 37 | "\t the seperators can use escape characters, such as: \\\\ for \\, \n" 38 | "\t \\t for tab, \\r for carriage return, \\n for new line\n" 39 | "\n", program); 40 | } 41 | 42 | static char *unescape_string(const char *s) 43 | { 44 | char *output; 45 | char *dest; 46 | const char *p; 47 | const char *end; 48 | const char *start; 49 | int src_len; 50 | int num_len; 51 | char buff[16]; 52 | 53 | src_len = strlen(s); 54 | output = (char *)malloc(src_len + 1); 55 | if (output == NULL) { 56 | fprintf(stderr, "malloc %d bytes fail", src_len + 1); 57 | return NULL; 58 | } 59 | 60 | dest = output; 61 | end = s + src_len; 62 | p = s; 63 | while (p < end) { 64 | if (*p == '\\' && p + 1 < end) { 65 | switch (*(p+1)) { 66 | case 'r': 67 | *dest++ = '\r'; 68 | break; 69 | case 'n': 70 | *dest++ = '\n'; 71 | break; 72 | case 't': 73 | *dest++ = '\t'; 74 | break; 75 | case 'f': 76 | *dest++ = '\f'; 77 | break; 78 | case '\\': 79 | *dest++ = '\\'; 80 | break; 81 | default: 82 | if (*(p+1) >= '0' && *(p+1) <= '9') { 83 | start = p + 1; 84 | p += 2; 85 | while ((p < end) && (*p >= '0' && *p <= '9')) { 86 | p++; 87 | if (p - start == 3) { 88 | break; 89 | } 90 | } 91 | num_len = p - start; 92 | memcpy(buff, start, num_len); 93 | *(buff + num_len) = '\0'; 94 | *dest++ = atoi(buff); 95 | } else { 96 | *dest++ = *p++; 97 | *dest++ = *p++; 98 | } 99 | continue; 100 | } 101 | p += 2; 102 | } else { 103 | *dest++ = *p++; 104 | } 105 | } 106 | 107 | *dest = '\0'; 108 | return output; 109 | } 110 | 111 | static void parse_args(int argc, char **argv) 112 | { 113 | int ch; 114 | 115 | while ((ch = getopt(argc, argv, "hc:F:R:k:s:n:o:")) != -1) { 116 | switch (ch) { 117 | case 'c': 118 | config_filename = optarg; 119 | break; 120 | case 'F': 121 | key_value_seperator = unescape_string(optarg); 122 | break; 123 | case 'R': 124 | row_seperator = unescape_string(optarg); 125 | break; 126 | case 'k': 127 | search_key = optarg; 128 | break; 129 | case 's': 130 | start_offset = atoi(optarg); 131 | break; 132 | case 'n': 133 | row_count = atoi(optarg); 134 | break; 135 | case 'o': 136 | output_filename = optarg; 137 | break; 138 | case 'h': 139 | default: 140 | show_usage = true; 141 | usage(argv[0]); 142 | return; 143 | } 144 | } 145 | 146 | if (optind != argc) { 147 | show_usage = true; 148 | usage(argv[0]); 149 | } 150 | } 151 | 152 | static struct shmcache_match_key_info *parse_search_key( 153 | struct shmcache_match_key_info *key_info) 154 | { 155 | if (search_key == NULL || *search_key == '\0') { 156 | return NULL; 157 | } 158 | 159 | key_info->op_type = 0; 160 | key_info->key = search_key; 161 | key_info->length = strlen(search_key); 162 | if (*search_key == '%') { 163 | key_info->op_type |= SHMCACHE_MATCH_KEY_OP_RIGHT; 164 | key_info->key++; 165 | key_info->length--; 166 | if (key_info->length == 0) { 167 | return NULL; 168 | } 169 | } 170 | 171 | if (key_info->key[key_info->length - 1] == '%') { 172 | key_info->op_type |= SHMCACHE_MATCH_KEY_OP_LEFT; 173 | key_info->length--; 174 | if (key_info->length == 0) { 175 | return NULL; 176 | } 177 | } 178 | 179 | return key_info; 180 | } 181 | 182 | static int do_dump(struct shmcache_context *context, FILE *fp) 183 | { 184 | int result; 185 | struct shmcache_hentry_array array; 186 | struct shmcache_hash_entry *entry; 187 | struct shmcache_hash_entry *end; 188 | struct shmcache_match_key_info key_info; 189 | struct shmcache_match_key_info *pkey; 190 | 191 | pkey = parse_search_key(&key_info); 192 | if ((result=shm_ht_to_array_ex(context, &array, pkey, 193 | start_offset, row_count)) != 0) 194 | { 195 | return result; 196 | } 197 | 198 | end = array.entries + array.count; 199 | for (entry=array.entries; entryvalue.options & SHMCACHE_SERIALIZER_STRING) != 0) { 201 | fprintf(fp, "%.*s%s%.*s%s", 202 | entry->key.length, entry->key.data, 203 | key_value_seperator, 204 | entry->value.length, entry->value.data, 205 | row_seperator); 206 | } else { 207 | fprintf(fp, "%.*s%s<%s serializer, data length: %d>%s", 208 | entry->key.length, entry->key.data, 209 | key_value_seperator, 210 | shmcache_get_serializer_label(entry->value.options), 211 | entry->value.length, row_seperator); 212 | } 213 | } 214 | 215 | shm_ht_free_array(&array); 216 | return 0; 217 | } 218 | 219 | int main(int argc, char **argv) 220 | { 221 | int result; 222 | struct shmcache_context context; 223 | FILE *fp; 224 | 225 | parse_args(argc, argv); 226 | if (show_usage) { 227 | return 0; 228 | } 229 | 230 | log_init(); 231 | if (output_filename != NULL) { 232 | fp = fopen(output_filename, "wb"); 233 | if (fp == NULL) { 234 | result = errno != 0 ? errno : EPERM; 235 | fprintf(stderr, "open file %s to write fail, " 236 | "error info: %s\n", output_filename, strerror(result)); 237 | return result; 238 | } 239 | } else { 240 | fp = stdout; 241 | } 242 | 243 | if ((result=shmcache_init_from_file_ex(&context, 244 | config_filename, false, true)) != 0) 245 | { 246 | return result; 247 | } 248 | 249 | if (context.config.lock_policy.read_within_lock) { 250 | if ((result=shm_lock(&context)) != 0) { 251 | return result; 252 | } 253 | result = do_dump(&context, fp); 254 | shm_unlock(&context); 255 | } else { 256 | result = do_dump(&context, fp); 257 | } 258 | if (fp != stdout) { 259 | fclose(fp); 260 | } 261 | shmcache_destroy(&context); 262 | return result; 263 | } 264 | -------------------------------------------------------------------------------- /src/tools/shmcache_get.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "shmcache/shmcache.h" 8 | 9 | static void usage(const char *prog) 10 | { 11 | fprintf(stderr, "shmcache get key value.\n" 12 | "Usage: %s [config_filename] \n", prog); 13 | } 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | int result; 18 | int index; 19 | char *config_filename; 20 | struct shmcache_context context; 21 | struct shmcache_key_info key; 22 | struct shmcache_value_info value; 23 | 24 | if (argc >= 2 && (strcmp(argv[1], "-h") == 0 || 25 | strcmp(argv[1], "help") == 0 || 26 | strcmp(argv[1], "--help") == 0)) 27 | { 28 | usage(argv[0]); 29 | return 0; 30 | } 31 | if (argc < 2) { 32 | usage(argv[0]); 33 | return EINVAL; 34 | } 35 | 36 | config_filename = "/etc/libshmcache.conf"; 37 | if (isFile(argv[1])) { 38 | if (argc < 3) { 39 | usage(argv[0]); 40 | return EINVAL; 41 | } 42 | config_filename = argv[1]; 43 | index = 2; 44 | } else { 45 | index = 1; 46 | } 47 | 48 | log_init(); 49 | if ((result=shmcache_init_from_file_ex(&context, 50 | config_filename, false, true)) != 0) 51 | { 52 | return result; 53 | } 54 | 55 | key.data = argv[index++]; 56 | key.length = strlen(key.data); 57 | result = shmcache_get(&context, &key, &value); 58 | if (result == 0) { 59 | printf("value options: %d (%s serializer), value length: %d", 60 | value.options, shmcache_get_serializer_label(value.options), 61 | value.length); 62 | if ((value.options & SHMCACHE_SERIALIZER_STRING) != 0) { 63 | printf(", value:\n%.*s", value.length, value.data); 64 | } 65 | printf("\n"); 66 | } else { 67 | fprintf(stderr, "get key: %s fail, errno: %d\n", key.data, result); 68 | } 69 | 70 | return result; 71 | } 72 | -------------------------------------------------------------------------------- /src/tools/shmcache_remove_all.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include "fastcommon/logger.h" 5 | #include "shmcache/shmcache.h" 6 | #include "shmcache/shmopt.h" 7 | 8 | static void usage(const char *prog) 9 | { 10 | fprintf(stderr, "Remove all shmcache including share memories. \n" 11 | "Usage: %s [config_filename] [-y or --yes]\n", prog); 12 | 13 | } 14 | 15 | #define IS_YES(str) (strcmp(str, "--yes") == 0 || strcmp(str, "-y") == 0) 16 | 17 | int main(int argc, char *argv[]) 18 | { 19 | int result; 20 | bool ask; 21 | char confirm; 22 | char *config_filename; 23 | struct shmcache_context context; 24 | 25 | if (argc >= 2 && (strcmp(argv[1], "-h") == 0 || 26 | strcmp(argv[1], "help") == 0 || 27 | strcmp(argv[1], "--help") == 0)) 28 | { 29 | usage(argv[0]); 30 | return 0; 31 | } 32 | if (argc > 3) { 33 | usage(argv[0]); 34 | return EINVAL; 35 | } 36 | 37 | ask = true; 38 | config_filename = "/etc/libshmcache.conf"; 39 | if (argc == 2) { 40 | if (IS_YES(argv[1])) { 41 | ask = false; 42 | } else { 43 | config_filename = argv[1]; 44 | } 45 | } else if (argc == 3) { 46 | if (IS_YES(argv[1])) { 47 | ask = false; 48 | config_filename = argv[2]; 49 | } else if (IS_YES(argv[2])) { 50 | config_filename = argv[1]; 51 | ask = false; 52 | } else { 53 | usage(argv[0]); 54 | return EINVAL; 55 | } 56 | } 57 | 58 | if (ask) { 59 | printf("Clear all share memory with config " 60 | "file: %s? : ", config_filename); 61 | if (!(scanf("%c", &confirm) == 1 && confirm == 'y')) { 62 | printf("share memory NOT removed.\n"); 63 | return ECANCELED; 64 | } 65 | } 66 | 67 | log_init(); 68 | if ((result=shmcache_init_from_file_ex(&context, 69 | config_filename, false, false)) != 0) 70 | { 71 | return result; 72 | } 73 | 74 | if ((result=shmcache_remove_all(&context)) == 0) { 75 | printf("ALL share memory segments are removed!\n"); 76 | } 77 | return result; 78 | } 79 | -------------------------------------------------------------------------------- /src/tools/shmcache_set.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "shmcache/shmcache.h" 8 | 9 | static void usage(const char *prog) 10 | { 11 | fprintf(stderr, "shmcache set key value.\n" 12 | "Usage: %s [config_filename] \n", prog); 13 | } 14 | 15 | int main(int argc, char *argv[]) 16 | { 17 | int result; 18 | int index; 19 | char *config_filename; 20 | struct shmcache_context context; 21 | struct shmcache_key_info key; 22 | char *value; 23 | int value_len; 24 | int ttl; 25 | 26 | if (argc >= 2 && (strcmp(argv[1], "-h") == 0 || 27 | strcmp(argv[1], "help") == 0 || 28 | strcmp(argv[1], "--help") == 0)) 29 | { 30 | usage(argv[0]); 31 | return 0; 32 | } 33 | if (argc < 4) { 34 | usage(argv[0]); 35 | return EINVAL; 36 | } 37 | 38 | config_filename = "/etc/libshmcache.conf"; 39 | if (isFile(argv[1])) { 40 | if (argc < 5) { 41 | usage(argv[0]); 42 | return EINVAL; 43 | } 44 | config_filename = argv[1]; 45 | index = 2; 46 | } else { 47 | index = 1; 48 | } 49 | 50 | log_init(); 51 | if ((result=shmcache_init_from_file(&context, config_filename)) != 0) { 52 | return result; 53 | } 54 | 55 | key.data = argv[index++]; 56 | key.length = strlen(key.data); 57 | value = argv[index++]; 58 | value_len = strlen(value); 59 | ttl = atoi(argv[index++]); 60 | result = shmcache_set(&context, &key, value, value_len, ttl); 61 | if (result == 0) { 62 | printf("set key: %s success.\n", key.data); 63 | } else { 64 | fprintf(stderr, "set key: %s fail, errno: %d\n", key.data, result); 65 | } 66 | 67 | return result; 68 | } 69 | -------------------------------------------------------------------------------- /src/tools/shmcache_stats.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include "fastcommon/logger.h" 6 | #include "fastcommon/shared_func.h" 7 | #include "fastcommon/sched_thread.h" 8 | #include "shmcache/shmcache.h" 9 | 10 | static void stats_output(struct shmcache_context *context); 11 | 12 | static void usage(const char *prog) 13 | { 14 | fprintf(stderr, "show shmcache stats.\n" 15 | "Usage: %s [config_filename] [clean|clear]\n" 16 | "\tclean or clear to reset the shm stats\n\n", prog); 17 | } 18 | 19 | int main(int argc, char *argv[]) 20 | { 21 | int result; 22 | int last_index; 23 | bool clear_stats; 24 | char *config_filename; 25 | struct shmcache_context context; 26 | 27 | if (argc >= 2 && (strcmp(argv[1], "-h") == 0 || 28 | strcmp(argv[1], "help") == 0 || 29 | strcmp(argv[1], "--help") == 0)) 30 | { 31 | usage(argv[0]); 32 | return 0; 33 | } 34 | 35 | clear_stats = false; 36 | if (argc >= 2) { 37 | last_index = argc - 1; 38 | if (strcmp(argv[last_index], "clean") == 0 || 39 | strcmp(argv[last_index], "clear") == 0) 40 | { 41 | clear_stats = true; 42 | } 43 | } 44 | 45 | config_filename = "/etc/libshmcache.conf"; 46 | if (argc >= 2) { 47 | if (argc == 2) { 48 | if (!clear_stats) { 49 | config_filename = argv[1]; 50 | } 51 | } else { 52 | config_filename = argv[1]; 53 | } 54 | } 55 | 56 | log_init(); 57 | if ((result=shmcache_init_from_file_ex(&context, 58 | config_filename, false, true)) != 0) 59 | { 60 | return result; 61 | } 62 | 63 | if (clear_stats) { 64 | shmcache_clear_stats(&context); 65 | printf("shm stats cleared.\n\n"); 66 | } else { 67 | stats_output(&context); 68 | } 69 | return 0; 70 | } 71 | 72 | static void stats_output(struct shmcache_context *context) 73 | { 74 | struct shmcache_stats stats; 75 | int avg_key_len; 76 | int avg_value_len; 77 | char total_ratio[32]; 78 | char ratio[32]; 79 | char rw_ratio[32]; 80 | char time_buff[32]; 81 | 82 | g_current_time = time(NULL); 83 | shmcache_stats(context, &stats); 84 | if (stats.hashtable.count > 0) { 85 | avg_key_len = stats.memory.usage.used.key / stats.hashtable.count; 86 | avg_value_len = stats.memory.usage.used.value / stats.hashtable.count; 87 | } else { 88 | avg_key_len = 0; 89 | avg_value_len = 0; 90 | } 91 | if (stats.shm.hashtable.get.total > 0) { 92 | sprintf(total_ratio, "%.2f%%", 100.00 * stats.shm.hashtable.get.success 93 | / (double)stats.shm.hashtable.get.total); 94 | } else { 95 | total_ratio[0] = '-'; 96 | total_ratio[1] = '\0'; 97 | } 98 | if (stats.hit.ratio >= 0.00) { 99 | sprintf(ratio, "%.2f%%", stats.hit.ratio * 100.00); 100 | } else { 101 | ratio[0] = '-'; 102 | ratio[1] = '\0'; 103 | } 104 | if (stats.shm.hashtable.set.total > 0) { 105 | sprintf(rw_ratio, "%.2f / 1.00", stats.shm.hashtable.get.total 106 | / (double)stats.shm.hashtable.set.total); 107 | } else { 108 | rw_ratio[0] = '-'; 109 | rw_ratio[1] = '\0'; 110 | } 111 | 112 | printf("\ntimestamp info:\n"); 113 | printf("shm init time: %s\n", formatDatetime(context->memory->init_time, 114 | "%Y-%m-%d %H:%M:%S", time_buff, sizeof(time_buff))); 115 | if (context->memory->stats.hashtable.last_clear_time > 0) { 116 | printf("last clear time: %s\n", formatDatetime(context->memory->stats. 117 | hashtable.last_clear_time, "%Y-%m-%d %H:%M:%S", 118 | time_buff, sizeof(time_buff))); 119 | } 120 | printf("stats begin time: %s\n", formatDatetime(context->memory->stats. 121 | init_time, "%Y-%m-%d %H:%M:%S", 122 | time_buff, sizeof(time_buff))); 123 | 124 | if (context->memory->stats.memory.recycle.key.last_recycle_time > 0) { 125 | printf("last recycle by key time: %s\n", formatDatetime( 126 | context->memory->stats.memory.recycle.key. 127 | last_recycle_time, "%Y-%m-%d %H:%M:%S", 128 | time_buff, sizeof(time_buff))); 129 | } 130 | if (context->memory->stats.memory.recycle.value_striping.last_recycle_time > 0) { 131 | printf("last recycle by value time: %s\n", formatDatetime( 132 | context->memory->stats.memory.recycle.value_striping. 133 | last_recycle_time, "%Y-%m-%d %H:%M:%S", 134 | time_buff, sizeof(time_buff))); 135 | } 136 | if (context->memory->stats.lock.last_detect_deadlock_time > 0) { 137 | printf("last detect deadlock time: %s\n", formatDatetime( 138 | context->memory->stats.lock. 139 | last_detect_deadlock_time, "%Y-%m-%d %H:%M:%S", 140 | time_buff, sizeof(time_buff))); 141 | } 142 | if (context->memory->stats.lock.last_unlock_deadlock_time > 0) { 143 | printf("last unlock deadlock time: %s\n", formatDatetime( 144 | context->memory->stats.lock. 145 | last_unlock_deadlock_time, "%Y-%m-%d %H:%M:%S", 146 | time_buff, sizeof(time_buff))); 147 | } 148 | printf("\n"); 149 | 150 | printf("\nhash table stats:\n"); 151 | printf("max_key_count: %d\n" 152 | "current_key_count: %d\n" 153 | "segment_size: %.03f MB\n\n" 154 | "set.total_count: %"PRId64"\n" 155 | "set.success_count: %"PRId64"\n" 156 | "incr.total_count: %"PRId64"\n" 157 | "incr.success_count: %"PRId64"\n" 158 | "get.total_count: %"PRId64"\n" 159 | "get.success_count: %"PRId64"\n" 160 | "del.total_count: %"PRId64"\n" 161 | "del.success_count: %"PRId64"\n" 162 | "get.qps: %.2f\n" 163 | "hit ratio (last %d seconds): %s\n" 164 | "total hit ratio (last %d seconds): %s\n" 165 | "total RW ratio: %s\n\n", 166 | stats.max_key_count, 167 | stats.hashtable.count, 168 | (double)stats.hashtable.segment_size / (1024 * 1024), 169 | stats.shm.hashtable.set.total, 170 | stats.shm.hashtable.set.success, 171 | stats.shm.hashtable.incr.total, 172 | stats.shm.hashtable.incr.success, 173 | stats.shm.hashtable.get.total, 174 | stats.shm.hashtable.get.success, 175 | stats.shm.hashtable.del.total, 176 | stats.shm.hashtable.del.success, 177 | stats.hit.get_qps, stats.hit.seconds, ratio, 178 | (int)(g_current_time - context->memory->stats.init_time), 179 | total_ratio, rw_ratio); 180 | 181 | printf("\nmemory stats:\n"); 182 | printf("total: %.03f MB\n" 183 | "alloced: %.03f MB\n" 184 | "used: %.03f MB\n" 185 | "free: %.03f MB\n" 186 | "avg_key_len: %d\n" 187 | "avg_value_len: %d\n\n", 188 | (double)stats.memory.max / (1024 * 1024), 189 | (double)stats.memory.usage.alloced / (1024 * 1024), 190 | (double)stats.memory.used / (1024 * 1024), 191 | (double)(stats.memory.max - stats.memory.used) / 192 | (1024 * 1024), avg_key_len, avg_value_len); 193 | 194 | printf("\nmemory recycle stats:\n"); 195 | printf("clear_ht_entry.total_count: %"PRId64"\n" 196 | "clear_ht_entry.valid_count: %"PRId64"\n\n" 197 | "recycle.key.total_count: %"PRId64"\n" 198 | "recycle.key.success_count: %"PRId64"\n" 199 | "recycle.key.force_count: %"PRId64"\n\n" 200 | "recycle.value_striping.total_count: %"PRId64"\n" 201 | "recycle.value_striping.success_count: %"PRId64"\n" 202 | "recycle.value_striping.force_count: %"PRId64"\n\n", 203 | stats.shm.memory.clear_ht_entry.total, 204 | stats.shm.memory.clear_ht_entry.valid, 205 | stats.shm.memory.recycle.key.total, 206 | stats.shm.memory.recycle.key.success, 207 | stats.shm.memory.recycle.key.force, 208 | stats.shm.memory.recycle.value_striping.total, 209 | stats.shm.memory.recycle.value_striping.success, 210 | stats.shm.memory.recycle.value_striping.force); 211 | 212 | printf("\nlock stats:\n"); 213 | printf("total_count: %"PRId64"\n" 214 | "retry_count: %"PRId64"\n" 215 | "detect_deadlock: %"PRId64"\n" 216 | "unlock_deadlock: %"PRId64"\n\n", 217 | stats.shm.lock.total, 218 | stats.shm.lock.retry, 219 | stats.shm.lock.detect_deadlock, 220 | stats.shm.lock.unlock_deadlock); 221 | } 222 | -------------------------------------------------------------------------------- /tsar-shmcache-module/Makefile: -------------------------------------------------------------------------------- 1 | CFLAGS = -Wall -fPIC --shared -g -O2 2 | CC = gcc 3 | INCLUDE_DIR = /usr/local/tsar/devel 4 | LINK = $(CC) -I$(INCLUDE_DIR) -I/usr/include/shmcache -I/usr/include/fastcommon \ 5 | -lfastcommon -lshmcache $(CFLAGS) 6 | 7 | OBJS = mod_shmcache.so 8 | 9 | all: $(OBJS) 10 | 11 | $(OBJS): %.so: %.c 12 | $(LINK) $< -o $@ 13 | clean: 14 | rm -f *.so; 15 | install: 16 | mkdir -p $(PREFIX)/etc/tsar/conf.d/ 17 | mkdir -p $(PREFIX)/usr/local/tsar/modules/ 18 | cp ./mod_shmcache.so $(PREFIX)/usr/local/tsar/modules/ 19 | cp ./mod_shmcache.conf $(PREFIX)/etc/tsar/conf.d/shmcache.conf 20 | uninstall: 21 | rm $(PREFIX)/usr/local/tsar/modules/mod_shmcache.so 22 | rm $(PREFIX)/etc/tsar/conf.d/shmcache.conf 23 | -------------------------------------------------------------------------------- /tsar-shmcache-module/mod_shmcache.c: -------------------------------------------------------------------------------- 1 | 2 | /* 3 | * (C) 2010-2011 Alibaba Group Holding Limited 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | */ 18 | 19 | #include 20 | #include 21 | #include 22 | #include "tsar.h" 23 | #include "logger.h" 24 | #include "shmcache.h" 25 | 26 | #define FILED_INDEX_GET_QPS 0 27 | #define FILED_INDEX_HIT_RATIO 1 28 | #define FILED_INDEX_SET_QPS 2 29 | #define FILED_INDEX_INCR_QPS 3 30 | #define FILED_INDEX_DEL_QPS 4 31 | #define FILED_INDEX_RETRY_LOCKS 5 32 | #define FILED_INDEX_RECYCLE_BY_KEY 6 33 | #define FILED_INDEX_BY_KEY_FORCE 7 34 | #define FILED_INDEX_RECYCLE_BY_VAL 8 35 | #define FILED_INDEX_BY_VAL_FORCE 9 36 | #define FILED_INDEX_CLEAR_HT_COUNT 10 37 | #define FILED_INDEX_CLEAR_HT_VALID 11 38 | #define FILED_INDEX_SET_TOTAL_COUNT 12 39 | #define FILED_INDEX_SET_SUCCESS_COUNT 13 40 | #define FILED_INDEX_INCR_TOTAL_COUNT 14 41 | #define FILED_INDEX_INCR_SUCCESS_COUNT 15 42 | #define FILED_INDEX_GET_TOTAL_COUNT 16 43 | #define FILED_INDEX_GET_SUCCESS_COUNT 17 44 | #define FILED_INDEX_DEL_TOTAL_COUNT 18 45 | #define FILED_INDEX_DEL_SUCCESS_COUNT 19 46 | #define FILEDS_TOTAL_COUNT 20 47 | 48 | #define FILED_SUB_START_INDEX FILED_INDEX_RETRY_LOCKS 49 | #define FILED_SUB_END_INDEX FILED_INDEX_CLEAR_HT_VALID 50 | 51 | #define FILED_KEEP_START_INDEX FILED_INDEX_SET_TOTAL_COUNT 52 | #define FILED_KEEP_END_INDEX FILED_INDEX_DEL_SUCCESS_COUNT 53 | 54 | static const char *shmcache_usage = " --shmcache shmcache information"; 55 | 56 | /* Structure for tsar */ 57 | static struct mod_info shmcache_info[] = { 58 | {"GetQPS" , SUMMARY_BIT, MERGE_NULL, STATS_NULL}, 59 | {"HitRat", SUMMARY_BIT, MERGE_NULL, STATS_NULL}, 60 | {"SetQPS", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 61 | {"IncQPS", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 62 | {"DelQPS", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 63 | {"Relock", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 64 | {"CycKey", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 65 | {"CKForce", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 66 | {"CycVal", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 67 | {"CVForce", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 68 | {"CLREle", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 69 | {"CLRVld", DETAIL_BIT, MERGE_NULL, STATS_NULL}, 70 | {"set.total_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 71 | {"set.success_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 72 | {"incr.total_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 73 | {"incr.success_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 74 | {"get.total_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 75 | {"get.success_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 76 | {"del.total_count", HIDE_BIT, MERGE_NULL, STATS_NULL}, 77 | {"del.success_count", HIDE_BIT, MERGE_NULL, STATS_NULL} 78 | }; 79 | 80 | static void read_shmcache_stats(struct module *mod, const char *parameter) 81 | { 82 | /* parameter actually equals to mod->parameter */ 83 | struct shmcache_context context; 84 | struct shmcache_stats stats; 85 | const char *config_filename = "/etc/libshmcache.conf"; 86 | char buf[512]; 87 | 88 | log_init(); 89 | if (shmcache_init_from_file_ex(&context, config_filename, false, true) == 0) { 90 | shmcache_stats_ex(&context, &stats, false); 91 | shmcache_destroy(&context); 92 | } else { 93 | memset(&stats, 0, sizeof(stats)); 94 | } 95 | 96 | snprintf(buf, sizeof(buf), 97 | "0,0,0,0,0," 98 | "%"PRId64"," 99 | "%"PRId64"," 100 | "%"PRId64"," 101 | "%"PRId64"," 102 | "%"PRId64"," 103 | "%"PRId64"," 104 | "%"PRId64"," 105 | "%"PRId64"," 106 | "%"PRId64"," 107 | "%"PRId64"," 108 | "%"PRId64"," 109 | "%"PRId64"," 110 | "%"PRId64"," 111 | "%"PRId64"," 112 | "%"PRId64, 113 | stats.shm.lock.retry, 114 | stats.shm.memory.recycle.key.total, 115 | stats.shm.memory.recycle.key.force, 116 | stats.shm.memory.recycle.value_striping.total, 117 | stats.shm.memory.recycle.value_striping.force, 118 | stats.shm.memory.clear_ht_entry.total, 119 | stats.shm.memory.clear_ht_entry.valid, 120 | stats.shm.hashtable.set.total, 121 | stats.shm.hashtable.set.success, 122 | stats.shm.hashtable.incr.total, 123 | stats.shm.hashtable.incr.success, 124 | stats.shm.hashtable.get.total, 125 | stats.shm.hashtable.get.success, 126 | stats.shm.hashtable.del.total, 127 | stats.shm.hashtable.del.success); 128 | 129 | log_destroy(); 130 | 131 | /* send data to tsar you can get it by pre_array&cur_array at set_shmcache_record */ 132 | set_mod_record(mod, buf); 133 | return; 134 | } 135 | 136 | #define CALC_INCR_FIELD_VAL(i, v) \ 137 | do { \ 138 | v = cur_array[i] - pre_array[i]; \ 139 | if (v < 0 || v > 1000000000L) { \ 140 | v = 0; \ 141 | } \ 142 | } while (0) 143 | 144 | #define SET_QPS_FIELD_VAL(current, total) \ 145 | do { \ 146 | st_array[current] = cur_array[total] - pre_array[total]; \ 147 | if (st_array[current] < 0.00 || st_array[current] / inter > 100000000.00) { \ 148 | st_array[current] = 0.00; \ 149 | } else if (st_array[current] > 0.01) { \ 150 | st_array[current] /= inter; \ 151 | } else { \ 152 | st_array[current] = 0.00; \ 153 | } \ 154 | } while (0) 155 | 156 | static void set_shmcache_record(struct module *mod, double st_array[], 157 | U_64 pre_array[], U_64 cur_array[], int inter) 158 | { 159 | int i; 160 | int64_t sub; 161 | int64_t total_sub; 162 | int64_t success_sub; 163 | 164 | SET_QPS_FIELD_VAL(FILED_INDEX_GET_QPS, FILED_INDEX_GET_TOTAL_COUNT); 165 | SET_QPS_FIELD_VAL(FILED_INDEX_SET_QPS, FILED_INDEX_SET_TOTAL_COUNT); 166 | SET_QPS_FIELD_VAL(FILED_INDEX_INCR_QPS, FILED_INDEX_INCR_TOTAL_COUNT); 167 | SET_QPS_FIELD_VAL(FILED_INDEX_DEL_QPS, FILED_INDEX_DEL_TOTAL_COUNT); 168 | 169 | CALC_INCR_FIELD_VAL(FILED_INDEX_GET_TOTAL_COUNT, total_sub); 170 | CALC_INCR_FIELD_VAL(FILED_INDEX_GET_SUCCESS_COUNT, success_sub); 171 | if (total_sub > 0) { 172 | st_array[FILED_INDEX_HIT_RATIO] = 100.00 * (double)success_sub / (double)total_sub; 173 | } else { 174 | st_array[FILED_INDEX_HIT_RATIO] = 0.00; 175 | } 176 | 177 | for (i = FILED_SUB_START_INDEX; i <= FILED_SUB_END_INDEX; i++) { 178 | CALC_INCR_FIELD_VAL(i, sub); 179 | st_array[i] = sub; 180 | } 181 | 182 | for (i = FILED_KEEP_START_INDEX; i <= FILED_KEEP_END_INDEX; i++) { 183 | st_array[i] = cur_array[i]; 184 | } 185 | } 186 | 187 | /* register mod to tsar */ 188 | void 189 | mod_register(struct module *mod) 190 | { 191 | register_mod_fields(mod, "--shmcache", shmcache_usage, shmcache_info, 192 | FILEDS_TOTAL_COUNT, read_shmcache_stats, set_shmcache_record); 193 | } 194 | -------------------------------------------------------------------------------- /tsar-shmcache-module/mod_shmcache.conf: -------------------------------------------------------------------------------- 1 | mod_shmcache on 2 | 3 | ####add it to tsar default output 4 | output_stdio_mod mod_shmcache 5 | 6 | ####add it to center db 7 | #output_db_mod mod_shmcache 8 | 9 | ####add it to nagios send 10 | ####set nagios threshold for alert 11 | #output_nagios_mod mod_shmcache 12 | 13 | #threshold shmcache.value1;N;N;N;N; 14 | #threshold shmcache.value2;N;N;N;N; 15 | #threshold shmcache.value3;N;N;N;N; 16 | -------------------------------------------------------------------------------- /tsar-shmcache-module/tsar-shmcache-module.spec.in: -------------------------------------------------------------------------------- 1 | Name: tsar-shmcache-module 2 | Version: 1.0.1 3 | Release: 1%{?dist} 4 | Summary: No Summary 5 | License: BSD 6 | Group: Arch/Tech 7 | URL: http://github.com/happyfish100/libshmcache/ 8 | Source: http://github.com/happyfish100/libshmcache/%{name}-%{version}.tar.gz 9 | 10 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 11 | 12 | BuildRequires: tsar-devel >= 2.1.1 13 | BuildRequires: libshmcache-devel >= 1.0.3 14 | Requires: %__cp %__mv %__chmod %__grep %__mkdir %__install %__id 15 | Requires: tsar >= 2.1.1 16 | Requires: libshmcache >= 1.0.3 17 | 18 | %description 19 | tsar shmcache module 20 | 21 | %prep 22 | %setup -q 23 | 24 | %build 25 | make 26 | 27 | %install 28 | rm -rf %{buildroot} 29 | export PREFIX=$RPM_BUILD_ROOT 30 | make install 31 | 32 | %post 33 | mkdir -p /etc/tsar/conf.d/ 34 | 35 | %preun 36 | 37 | %postun 38 | 39 | %clean 40 | rm -rf %{buildroot} 41 | 42 | %files 43 | %defattr(-,root,root,-) 44 | /usr/local/tsar/modules/mod_shmcache.so 45 | /etc/tsar/conf.d/shmcache.conf 46 | 47 | %changelog 48 | * Mon Feb 27 2017 Yu Qing 49 | - first RPM release (1.0) 50 | --------------------------------------------------------------------------------