├── .gitignore ├── AUTHORS ├── COPYING ├── CREDITS ├── ChangeLog ├── INSTALL ├── LICENSE ├── Makefile.am ├── NEWS ├── README.md ├── TODO ├── autogen.sh ├── beansdb_log.conf ├── configure.ac ├── doc ├── Makefile.am ├── beansdb.1 └── protocol.txt ├── go_test ├── Makefile ├── beansdb_log.conf ├── bitcask.go ├── bitcask_test.go ├── codec.go ├── codec_test.go ├── htree.go └── htree_test.go ├── peteris.yaml ├── python ├── dbclient.py ├── httpd.py ├── proxy.py ├── sync.py └── test.py ├── python_tests ├── base.py ├── batch.py ├── bucket_check_test.py ├── gc_multiple.py ├── gc_simple.py ├── key_version.py ├── one_key.py ├── pip-requirement.txt ├── test_broken_data.py ├── test_log.conf └── test_nolog.conf ├── src ├── Makefile.am ├── ae_epoll.c ├── ae_kqueue.c ├── ae_select.c ├── beansdb.c ├── beansdb.h ├── bitcask.c ├── bitcask.h ├── clock_gettime_stub.c ├── codec.c ├── codec.h ├── common.c ├── common.h ├── const.h ├── crc32.c ├── diskmgr.c ├── diskmgr.h ├── fnv1a.h ├── hint.c ├── hint.h ├── hstore.c ├── hstore.h ├── htree.c ├── htree.h ├── item.c ├── log.c ├── log.h ├── mfile.c ├── mfile.h ├── quicklz.c ├── quicklz.h ├── record.c ├── record.h ├── thread.c ├── util.h └── varint.h └── third-party └── zlog-1.2 ├── COPYING ├── INSTALL ├── Makefile.am ├── buf.c ├── buf.h ├── category.c ├── category.h ├── category_table.c ├── category_table.h ├── conf.c ├── conf.h ├── configure.ac ├── event.c ├── event.h ├── fmacros.h ├── format.c ├── format.h ├── level.c ├── level.h ├── level_list.c ├── level_list.h ├── mdc.c ├── mdc.h ├── record.c ├── record.h ├── record_table.c ├── record_table.h ├── rotater.c ├── rotater.h ├── rule.c ├── rule.h ├── spec.c ├── spec.h ├── thread.c ├── thread.h ├── version.h ├── zc_arraylist.c ├── zc_arraylist.h ├── zc_defs.h ├── zc_hashtable.c ├── zc_hashtable.h ├── zc_profile.c ├── zc_profile.h ├── zc_util.c ├── zc_util.h ├── zc_xplatform.h ├── zlog-chk-conf.c ├── zlog.c └── zlog.h /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.o 3 | *.lo 4 | *.slo 5 | 6 | # Compiled Dynamic libraries 7 | *.so 8 | *.dylib 9 | 10 | # Compiled Static libraries 11 | *.lai 12 | *.la 13 | *.a 14 | 15 | # configure auto generated files 16 | 17 | *.log 18 | aclocal.m4 19 | .deps/ 20 | Makefile 21 | !go_test/Makefile 22 | config.* 23 | doc/Makefile 24 | autom4te.cache/ 25 | stamp-h1 26 | Makefile.in 27 | configure 28 | doc/Makefile.in 29 | src/.dirstamp 30 | src/cscope.* 31 | src/tags 32 | beansdb 33 | cscope.* 34 | src/.ycm_extra_conf.py* 35 | *.swp 36 | compile 37 | depcomp 38 | install-sh 39 | missing 40 | ar-lib 41 | 42 | *.pyc 43 | 44 | # ctags file 45 | .tags 46 | .tags_sorted_by_file 47 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Davies Liu 2 | Hurricane Lee 3 | -------------------------------------------------------------------------------- /CREDITS: -------------------------------------------------------------------------------- 1 | Special thanks go to people who have volunteered their time, effort, 2 | and ideas to make this software available. (Words quoted from squid) 3 | 4 | * The code of beansdb is most based on Memcached . 5 | -------------------------------------------------------------------------------- /ChangeLog: -------------------------------------------------------------------------------- 1 | 2015-03-25 0.7.1 2 | * improve: go test 3 | * improve: get ??key will return key's meta even the key is deleted 4 | * bugfix: item length overflow 5 | 6 | 2014-08-19 0.7.0 7 | * Compatibility Changes: use zlog 8 | * Compatibility Changes: htree structure changed 9 | * Compatibility Changes: remove command line argument -A 10 | * Compatibility Changes: not allow using ';' to seperate dirs in -H 11 | * new: add command line argument -C 12 | * new: optimize_stat command 13 | * new: python test 14 | * new: bucket.txt for each bitcask to check data files 15 | * bugfix: gc error 16 | * bugfix: rotate error 17 | * bugfix: some memory and lock error 18 | 19 | 2013-07-15 0.6.0 20 | * new: stopme command 21 | * improve: add madvise() and fadvise() to improve fs buffer usage 22 | * bugfix: make select() work, if no epoll and kqueue 23 | * bugfix: overflow in htree and codec 24 | 25 | 2013-05-21 0.5.9 26 | * improve: improve GC, do not need restart any more 27 | * bugfix: limit number of formats in codec 28 | * several bugfix 29 | 30 | 2013-02-04 0.5.8 31 | * new: merge data files into large one when GC 32 | * change: close bitcask in parallel 33 | * bugfix: readlink() should ends with \0 (used when GC) 34 | 35 | 2012-11-20 0.5.7 36 | * new: dump HTree to disk to speed up boosting 37 | * change: increase the size of data file to 4G 38 | * some bugfix 39 | 40 | 2011-11-25 0.5.6 41 | * new: support multi directories as Home, manage disk spaces auto. 42 | * improve: using multi threads when boot scaning. 43 | 44 | 2011-09-22 0.5.5 45 | * improve: improve codec, support more patterns of keys 46 | * bugfix: crash because of get_proc_stats, get maxrss from /proc/self/statm 47 | 48 | 2011-08-16 0.5.4 49 | * improve: codec, enlarge buffer size 50 | 51 | 2011-01-11 0.5.3 52 | * new: scan hash tree with prefix, for example, @xxx:prefix 53 | * new: optimize data file in place (gc) 54 | * improve: leader/follower threads model 55 | * improve: less memory in htree 56 | 57 | 2010-11-10 0.5.1 58 | * new: pub/sub threads model 59 | * new: compress records and hint with QuickLZ 60 | 61 | 2010-10-25 0.5.0 62 | * new: bitcask storage 63 | * new: append and incr command 64 | 65 | 2009-12-25 0.3.0 66 | * First public release. 67 | -------------------------------------------------------------------------------- /INSTALL: -------------------------------------------------------------------------------- 1 | Dependencies 2 | ============ 3 | beansdb require epoll on Linux, or kqueue on *BSD/Mac OS. 4 | 5 | Installation 6 | ============ 7 | $ git clone https://github.com/douban/beansdb 8 | $ cd beansdb 9 | $ ./autogen.sh 10 | 11 | or 12 | 13 | $ # wget from github/douban/beansdb/ 14 | $ tar xvzf beansdb-X.Y.Z.tar.gz 15 | $ cd beansdb-X.Y.Z 16 | 17 | $ ./configure 18 | $ make 19 | $ sudo make install 20 | 21 | Getting Started 22 | =============== 23 | 24 | use "-h" option to get started. 25 | 26 | Running Examples: 27 | 28 | 1. run as a single daemon 29 | 30 | beansdb -p 7900 -H /data/beansdb -T 0 -v 31 | 32 | 2. test it 33 | 34 | $ telnet localhost 7900 35 | 36 | < set hello 0 0 5 37 | < world 38 | > STORED 39 | 40 | < get hello 41 | > VALUE hello 0 0 5 42 | > world 43 | > END 44 | 45 | < get @ 46 | > VALUE @ 0 18 47 | > hello 933489272 1 48 | > 49 | > END 50 | 51 | Have fun :) 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2009, Douban Inc. 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are 6 | met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | 11 | * Redistributions in binary form must reproduce the above 12 | copyright notice, this list of conditions and the following disclaimer 13 | in the documentation and/or other materials provided with the 14 | distribution. 15 | 16 | * Neither the name of the Douban Inc. nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | This product includes software developed by Steve Chu. 33 | 34 | [ memcached ] 35 | 36 | Copyright (c) 2003, Danga Interactive, Inc. 37 | All rights reserved. 38 | 39 | Redistribution and use in source and binary forms, with or without 40 | modification, are permitted provided that the following conditions are 41 | met: 42 | 43 | * Redistributions of source code must retain the above copyright 44 | notice, this list of conditions and the following disclaimer. 45 | 46 | * Redistributions in binary form must reproduce the above 47 | copyright notice, this list of conditions and the following disclaimer 48 | in the documentation and/or other materials provided with the 49 | distribution. 50 | 51 | * Neither the name of the Danga Interactive nor the names of its 52 | contributors may be used to endorse or promote products derived from 53 | this software without specific prior written permission. 54 | 55 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 56 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 57 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 58 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 59 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 60 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 61 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 62 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 63 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 64 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 65 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 66 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | SUBDIRS = third-party/zlog-1.2 src doc 2 | DIST_SUBDIRS = python 3 | 4 | EXTRA_DIST = python src/crc32.c src/clock_gettime_stub.c src/ae_epoll.c src/ae_kqueue.c src/ae_select.c CREDITS AUTHORS LICENSE 5 | -------------------------------------------------------------------------------- /NEWS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/douban/beansdb/08cb899c36c2aefb5e8d7e5381f140515ecaea42/NEWS -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # What is Beansdb? 3 | 4 | Beansdb is a distributed key-value storage system designed for large scale 5 | online system, aiming for high avaliablility and easy management. It took 6 | the ideas from Amazon's Dynamo, then made some simplify to Keep It Simple 7 | Stupid (KISS). 8 | 9 | The clients write to N Beansdb node, then read from R of them (solving 10 | conflict). Data in different nodes is synced through hash tree, in cronjob. 11 | 12 | It conforms to memcache protocol (not fully supported, see below), so any 13 | memcached client can interactive with it without any modification. 14 | 15 | Beansdb is heavy used in http://www.douban.com/, is used to stored images, 16 | mp3, text fields and so on, see benchmark below. 17 | 18 | Any suggestion or feedback is welcomed. 19 | 20 | 21 | # Features 22 | 23 | * High availability data storage with multi readable and writable repications 24 | 25 | * Soft state and final consistency, synced with hash tree 26 | 27 | * Easy Scaling out without interrupting online service 28 | 29 | * High performance read/write for a key-value based object 30 | 31 | * Configurable availability/consistency by N,W,R 32 | 33 | * Memcache protocol compatibility 34 | 35 | ## Supported memcache commands 36 | 37 | * get 38 | * set(with version support) 39 | * append 40 | * incr 41 | * delete 42 | * stats 43 | * flush_all 44 | 45 | ## Private commands 46 | 47 | * get @xxx, list the content of hash tree, such as @0f 48 | * get ?xxx, get the meta data of key. 49 | 50 | # Python Example 51 | ``` 52 | from dbclient import Beansdb 53 | 54 | # three beansdb nodes on localhost 55 | BEANSDBCFG = { 56 | "localhost:7901": range(16), 57 | "localhost:7902": range(16), 58 | "localhost:7903": range(16), 59 | } 60 | 61 | db = Beansdb(BEANSDBCFG, 16) 62 | 63 | db.set('hello', 'world') 64 | db.get('hello') 65 | db.delete('hello') 66 | ``` 67 | 68 | # Benchmark 69 | ``` 70 | 　$ beansdb -d 71 | 　$ memstorm -s localhost:7900 -n 1000000 -k 10 -l 100 72 | 　　 73 | 　　---- 74 | 　　Num of Records : 1000000 75 | 　　Non-Blocking IO : 0 76 | 　　TCP No-Delay : 0 77 | 　　 78 | 　　Successful [SET] : 1000000 79 | 　　Failed [SET] : 0 80 | 　　Total Time [SET] : 51.77594s 81 | 　　Average Time [SET] : 0.00005s 82 | 　　 83 | 　　Successful [GET] : 1000000 84 | 　　Failed [GET] : 0 85 | 　　Total Time [GET] : 40.93667s 86 | 　　Average Time [GET] : 0.00004s 87 | ``` 88 | 89 | # Real performance in production 90 | 91 | * cluster 1: 1.1B records, 55TB data, 48 nodes, 1100 get/25 set per seconds, 92 | med/avg/90%/99% time is 12/20/37/186 ms. 93 | * cluster 2: 3.3B records, 3.5TB data, 15 nodes, 1000 get/500 set per seconds, 94 | med/avg/90%/99% time is 1/11/15/123 ms. 95 | 96 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | * tools to split buckets 2 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # 3 | 4 | echo "autoreconf --install" 5 | autoreconf --install 6 | -------------------------------------------------------------------------------- /beansdb_log.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | strict init = true 3 | reload conf period = 5M 4 | buffer max = 1MB 5 | buffer min = 1024 6 | default format = "%d.%us %-6V (%t:%F:%L) - %m%n" 7 | file perms = 600 8 | [formats] 9 | simple = "%d.%us # %m%n" 10 | [rules] 11 | beansdb.NOTICE "./beansdb-error.log" 12 | beansdb.=INFO "./beansdb-access.log"; simple 13 | 14 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_PREREQ(2.61) 2 | AC_INIT([beansdb], [0.7.1.4], [davies.liu@gmail.com]) 3 | AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) 4 | AC_PROG_CC 5 | AM_PROG_CC_C_O 6 | AC_CONFIG_HEADERS([config.h]) 7 | 8 | 9 | AC_SEARCH_LIBS(socket, socket) 10 | AC_SEARCH_LIBS(gethostbyname, nsl) 11 | AC_SEARCH_LIBS(mallinfo, malloc) 12 | AC_SEARCH_LIBS(pthread_create, pthread) 13 | 14 | AC_CHECK_LIB([rt], [clock_gettime]) 15 | AC_CHECK_FUNC(daemon,AC_DEFINE([HAVE_DAEMON],,[Define this if you have daemon()]),[AC_LIBOBJ(daemon)]) 16 | AC_CHECK_HEADER([sys/epoll.h], AC_DEFINE([HAVE_EPOLL], , [for epoll support])) 17 | AC_CHECK_HEADER([sys/event.h], AC_DEFINE([HAVE_KQUEUE], , [for kqueue support])) 18 | 19 | AC_HEADER_STDBOOL 20 | AC_C_CONST 21 | AC_CHECK_HEADER(malloc.h, AC_DEFINE(HAVE_MALLOC_H,,[do we have malloc.h?])) 22 | AC_CHECK_MEMBER([struct mallinfo.arena], [ 23 | AC_DEFINE(HAVE_STRUCT_MALLINFO,,[do we have stuct mallinfo?]) 24 | ], ,[ 25 | # include 26 | ] 27 | ) 28 | 29 | dnl From licq: Copyright (c) 2000 Dirk Mueller 30 | dnl Check if the type socklen_t is defined anywhere 31 | AC_DEFUN([AC_C_SOCKLEN_T], 32 | [AC_CACHE_CHECK(for socklen_t, ac_cv_c_socklen_t, 33 | [ 34 | AC_TRY_COMPILE([ 35 | #include 36 | #include 37 | ],[ 38 | socklen_t foo; 39 | ],[ 40 | ac_cv_c_socklen_t=yes 41 | ],[ 42 | ac_cv_c_socklen_t=no 43 | ]) 44 | ]) 45 | if test $ac_cv_c_socklen_t = no; then 46 | AC_DEFINE(socklen_t, int, [define to int if socklen_t not available]) 47 | fi 48 | ]) 49 | 50 | AC_C_SOCKLEN_T 51 | 52 | AC_CONFIG_SUBDIRS([third-party/zlog-1.2]) 53 | AC_CONFIG_FILES([Makefile doc/Makefile src/Makefile]) 54 | AC_OUTPUT 55 | -------------------------------------------------------------------------------- /doc/Makefile.am: -------------------------------------------------------------------------------- 1 | man_MANS = beansdb.1 2 | 3 | EXTRA_DIST = *.txt beansdb.1 4 | -------------------------------------------------------------------------------- /doc/beansdb.1: -------------------------------------------------------------------------------- 1 | .TH DADBY 1 "Jan 1, 2010" 2 | .SH NAME 3 | beansdb \- high-available distributed key-value storage system 4 | .SH SYNOPSIS 5 | .B beansdb 6 | .RI [ options ] 7 | .br 8 | .SH DESCRIPTION 9 | This manual page documents briefly the 10 | .B beansdb 11 | key-value storage daemon. 12 | .PP 13 | .B beansdb 14 | is a key-value storage system base on memcached and Tokyo Cabinet. Hash tree 15 | was built inside for fast syncing after failing or any other unexpected errors. 16 | .br 17 | .SH OPTIONS 18 | These programs follow the usual GNU command line syntax. A summary of options 19 | is included below. 20 | .TP 21 | .B \-p 22 | Listen on TCP port , the default is port 7900. 23 | .TP 24 | .B \-l 25 | Listen on ; default to INADDR_ANY. This is an important option to 26 | consider as there is no other way to secure the installation. Binding to an 27 | internal or firewalled network interface is suggested. 28 | .TP 29 | .B \-d 30 | Run beansdb as a daemon. 31 | .TP 32 | .B \-u 33 | Assume the identity of (only when run as root). 34 | .TP 35 | .B \-c 36 | Use max simultaneous connections; the default is 1024. 37 | .TP 38 | .B \-H 39 | home of database, default is 'testdb' 40 | .TP 41 | .B \-T 42 | log of the number of db files(base 16), default is 2 (means 256 db files). 43 | .TP 44 | .B \-P 45 | Print pidfile to , only used under \-d option. 46 | .TP 47 | .B \-t 48 | Number of threads to use to process incoming requests(default is 16). 49 | .TP 50 | .B \-r 51 | Raise the core file size limit to the maximum allowable. 52 | .TP 53 | .B \-h 54 | Show the version of beansdb and a summary of options. 55 | .TP 56 | .B \-v 57 | Be verbose during the event loop; print out errors and warnings. 58 | .TP 59 | .B \-vv 60 | Be even more verbose; same as \-v but also print client commands and 61 | responses. 62 | .br 63 | .SH LICENSE 64 | The beansdb daemon is copyright Douban Inc and is distributed under 65 | the BSD license. Note that daemon clients are licensed separately. 66 | .br 67 | .SH SEE ALSO 68 | The README file that comes with beansdb 69 | .br 70 | .B http://beansdb.googlecode.com/ 71 | .SH AUTHOR 72 | The beansdb daemon was written by Davies Liu 73 | .B 74 | .br 75 | -------------------------------------------------------------------------------- /go_test/Makefile: -------------------------------------------------------------------------------- 1 | all: 2 | ar rvs libbeansdb.a ../src/beansdb-hstore.o ../src/beansdb-quicklz.o ../src/beansdb-bitcask.o ../src/beansdb-htree.o ../src/beansdb-record.o ../src/beansdb-codec.o ../src/beansdb-item.o ../src/beansdb-thread.o ../src/beansdb-diskmgr.o ../src/beansdb-log.o ../src/beansdb-hint.o ../src/beansdb-mfile.o ../src/beansdb-common.o 3 | go fmt *.go 4 | 5 | test: 6 | go test 7 | 8 | clean: 9 | go clean 10 | -------------------------------------------------------------------------------- /go_test/beansdb_log.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | strict init = true 3 | reload conf period = 5M 4 | buffer max = 1MB 5 | buffer min = 1024 6 | default format = "%d.%us %-6V (%t:%F:%L) - %m%n" 7 | file perms = 600 8 | [formats] 9 | simple = "%d.%us # %m%n" 10 | [rules] 11 | beansdb.!=INFO "./beansdb-error.log" 12 | beansdb.=INFO "./beansdb-access.log"; simple 13 | 14 | -------------------------------------------------------------------------------- /go_test/bitcask.go: -------------------------------------------------------------------------------- 1 | package btest 2 | 3 | // #cgo CFLAGS: -I ../src -I ../third-party/zlog-1.2/ 4 | // #cgo LDFLAGS: -L ../third-party/zlog-1.2/ -lzlog -L . -lbeansdb 5 | // #include 6 | // #include "record.h" 7 | // #include "hint.h" 8 | // #include "bitcask.h" 9 | import "C" 10 | import "unsafe" 11 | import ( 12 | "errors" 13 | "time" 14 | ) 15 | 16 | type Record struct { 17 | Key string 18 | Value []byte 19 | Flag int32 20 | Hash uint32 21 | Version int32 22 | Tstamp int32 23 | pos uint32 24 | } 25 | 26 | func NewRecord(key string, value []byte, flag int32, version int32) *Record { 27 | r := &Record{Key: key, Value: value, Flag: flag, Version: version} 28 | r.Tstamp = int32(time.Now().Unix()) 29 | r.GenHash() 30 | return r 31 | } 32 | 33 | const PRIME = 0x01000193 34 | 35 | func Fnv1a(buf []byte) (h uint32) { 36 | h = 0x811c9dc5 37 | for _, b := range buf { 38 | h ^= uint32(int8(b)) 39 | h *= PRIME 40 | } 41 | return h 42 | } 43 | 44 | func (r *Record) GenHash() { 45 | if r.Version < 0 { 46 | if len(r.Value) == 0 { 47 | r.Hash = 0 48 | return 49 | } else { 50 | println("invalid record: ver:", r.Version, " size:", len(r.Value)) 51 | } 52 | } 53 | hash := uint32(len(r.Value)) * 97 54 | if len(r.Value) <= 1024 { 55 | hash += Fnv1a(r.Value) 56 | } else { 57 | hash += Fnv1a(r.Value[:512]) 58 | hash *= 97 59 | hash += Fnv1a(r.Value[len(r.Value)-512:]) 60 | } 61 | r.Hash = hash & 0xffff 62 | } 63 | 64 | var MAX_BUCKET_SIZE = int64(1024 * 1024 * 1024 * 2) 65 | var MERGE_LIMIT = 1024 * 20 66 | 67 | type Bitcask struct { 68 | path string 69 | bc *C.Bitcask 70 | } 71 | 72 | func NewBitcask(dirpath string, depth, pos int, before int64) (b *Bitcask, err error) { 73 | b = new(Bitcask) 74 | b.path = dirpath 75 | 76 | cpath := C.CString(dirpath) 77 | defer C.free(unsafe.Pointer(cpath)) 78 | b.bc = C.bc_open(cpath, C.int(depth), C.int(pos), C.time_t(before)) 79 | 80 | if b.bc == nil { 81 | return nil, errors.New("open bitcask failed" + dirpath) 82 | } 83 | return 84 | } 85 | 86 | func (b *Bitcask) Flush() { 87 | C.bc_flush(b.bc, 0, 0) 88 | } 89 | 90 | func (b *Bitcask) Close() { 91 | C.bc_close(b.bc) 92 | b.bc = nil 93 | } 94 | 95 | func (b *Bitcask) Merge() { 96 | C.bc_optimize(b.bc, 0) 97 | } 98 | 99 | func (b *Bitcask) GetRecord(key string) *Record { 100 | var ret_pos C.uint32_t 101 | var ret_ver C.int 102 | c_key := C.CString(key) 103 | defer C.free(unsafe.Pointer(c_key)) 104 | dr := C.bc_get(b.bc, c_key, &ret_pos, &ret_ver) 105 | if dr == nil { 106 | return nil 107 | } 108 | defer C.free_record(dr) 109 | 110 | r := new(Record) 111 | r.Key = key 112 | r.Value = []byte(C.GoStringN(dr.value, _Ctype_int(dr.vsz))) 113 | r.Flag = int32(dr.flag) 114 | r.Version = int32(dr.version) 115 | r.Tstamp = int32(dr.tstamp) 116 | r.GenHash() 117 | return r 118 | } 119 | 120 | func (b *Bitcask) Get(key string) ([]byte, int32) { 121 | r := b.GetRecord(key) 122 | if r != nil { 123 | return r.Value, r.Flag 124 | } 125 | return nil, 0 126 | } 127 | 128 | func (b *Bitcask) Set(key string, value []byte, flag, version int32) error { 129 | ckey := C.CString(key) 130 | defer C.free(unsafe.Pointer(ckey)) 131 | cv := C.CString(string(value)) 132 | defer C.free(unsafe.Pointer(cv)) 133 | if !(0 != (C.bc_set(b.bc, ckey, cv, C.size_t(len(value)), 134 | C.int(flag), C.int(version)))) { 135 | return errors.New("set failed") 136 | } 137 | return nil 138 | } 139 | 140 | func (b *Bitcask) Delete(key string) error { 141 | return b.Set(key, nil, 0, -1) 142 | } 143 | 144 | func (b *Bitcask) Hash() uint16 { 145 | c_key := C.CString("@") 146 | defer C.free(unsafe.Pointer(c_key)) 147 | return uint16(C.bc_get_hash(b.bc, c_key, (*C.uint)(nil))) 148 | } 149 | 150 | func (b *Bitcask) Len() int { 151 | c_key := C.CString("@") 152 | defer C.free(unsafe.Pointer(c_key)) 153 | var cnt C.uint 154 | C.bc_get_hash(b.bc, c_key, &cnt) 155 | return int(cnt) 156 | } 157 | 158 | func (b *Bitcask) List(dir string) string { 159 | c_key := C.CString(dir) 160 | defer C.free(unsafe.Pointer(c_key)) 161 | cstr, _ := C.bc_list(b.bc, c_key, nil) 162 | if cstr == nil { 163 | return "" 164 | } 165 | defer C.free(unsafe.Pointer(cstr)) 166 | return C.GoString(cstr) 167 | } 168 | 169 | func (b *Bitcask) ListPrefix(dir, prefix string) string { 170 | c_key := C.CString(dir) 171 | defer C.free(unsafe.Pointer(c_key)) 172 | c_prefix := C.CString(prefix) 173 | defer C.free(unsafe.Pointer(c_prefix)) 174 | cstr, _ := C.bc_list(b.bc, c_key, c_prefix) 175 | if cstr == nil { 176 | return "" 177 | } 178 | defer C.free(unsafe.Pointer(cstr)) 179 | return C.GoString(cstr) 180 | } 181 | -------------------------------------------------------------------------------- /go_test/codec.go: -------------------------------------------------------------------------------- 1 | package btest 2 | 3 | // #cgo CFLAGS: -I ../src -I ../third-party/zlog-1.2/ 4 | // #cgo LDFLAGS: -L ../third-party/zlog-1.2/ -lzlog -L . -lbeansdb 5 | // #include "codec.h" 6 | import "C" 7 | 8 | const BUFSIZE = 256 9 | 10 | func init() { 11 | logConfPath := C.CString("./beansdb_log.conf") 12 | C.log_init(logConfPath) 13 | } 14 | 15 | type Codec struct { 16 | cdc *C.Codec 17 | buf *C.char 18 | buf2 *C.char 19 | } 20 | 21 | func NewCodec() *Codec { 22 | dc := new(Codec) 23 | dc.cdc = C.dc_new() 24 | dc.buf = (*C.char)(C.malloc(BUFSIZE)) 25 | return dc 26 | } 27 | 28 | func (dc *Codec) Encode(src string) (dst string, idx int) { 29 | var cdc *C.Codec = dc.cdc 30 | n := C.dc_encode(cdc, dc.buf, BUFSIZE, C.CString(src), C.int(len(src))) 31 | return C.GoStringN(dc.buf, n), 1 32 | } 33 | 34 | func (dc *Codec) Decode(src string) (dst string, idx int) { 35 | n := C.dc_decode(dc.cdc, dc.buf, BUFSIZE, C.CString(src), C.int(len(src))) 36 | return C.GoStringN(dc.buf, n), 1 37 | } 38 | -------------------------------------------------------------------------------- /go_test/codec_test.go: -------------------------------------------------------------------------------- 1 | package btest 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | type dcCase struct { 8 | s string 9 | l int 10 | } 11 | 12 | var dcCases = []dcCase{ 13 | {"/hello", 6}, 14 | {"/hello/5464", 5}, 15 | {"/hello/3fe989", 5}, 16 | {"/hello/3fe989f3adefaefc", 9}, 17 | {"/v/v32", 0}, 18 | // {"中文12312", 5}, 19 | // {"你好/v/v3552", 14}, 20 | {"/你好/v/v3552", 5}, 21 | {"1231@gmail.com", 0}, 22 | {"/status/raw/1Fp3da", 5}, 23 | {"/status/mediam/2fffff", 5}, 24 | {"files/file-131470845.jp", 5}, 25 | {"files/file-5314708453.jp", 5}, 26 | {"/epic/5957/8158.jpg", 9}, 27 | {"/photo/thumb/SG94-EpQ_p866859435", 0}, 28 | {"/note/small/24795790-1", 5}, 29 | {"/anduin/urlgrabcontent/4paFkh", 5}, 30 | } 31 | 32 | func TestDC(t *testing.T) { 33 | dc := NewCodec() 34 | for _, c := range dcCases { 35 | encoded, _ := dc.Encode(c.s) 36 | l := c.l 37 | if l == 0 { 38 | l = len(c.s) 39 | } 40 | if len(encoded) != l { 41 | t.Errorf("%s length not match, exp %d ,get %d", c.s, c.l, len(encoded)) 42 | } else { 43 | decoded, _ := dc.Decode(encoded) 44 | if decoded != c.s { 45 | t.Error("decode fail", c.s, decoded) 46 | } 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /go_test/htree.go: -------------------------------------------------------------------------------- 1 | package btest 2 | 3 | // #cgo CFLAGS: -I ../src -I ../third-party/zlog-1.2/ 4 | // #cgo LDFLAGS: -L ../third-party/zlog-1.2/ -lzlog -L . -lbeansdb 5 | // #include 6 | // #include "record.h" 7 | // #include "htree.h" 8 | // #include "hint.h" 9 | // #include "record.h" 10 | import "C" 11 | import "unsafe" 12 | 13 | type Item struct { 14 | Bucket int 15 | Pos uint32 16 | Hash uint32 17 | Version int32 18 | } 19 | 20 | type HashTree struct { 21 | tree *C.HTree 22 | } 23 | 24 | func NewHTree(depth, pos int) *HashTree { 25 | t := C.ht_new(C.int(depth), C.int(pos), C.bool(false)) 26 | return &HashTree{tree: t} 27 | } 28 | 29 | func (t *HashTree) Add(key string, item *Item) { 30 | p := C.CString(key) 31 | defer C.free(unsafe.Pointer(p)) 32 | pos := (uint32(item.Bucket)) + uint32(item.Pos) 33 | C.ht_add(t.tree, p, 34 | C.uint32_t(pos), 35 | C.uint16_t(item.Hash), 36 | C.int32_t(item.Version)) 37 | } 38 | 39 | func (t *HashTree) Remove(key string) { 40 | p := C.CString(key) 41 | defer C.free(unsafe.Pointer(p)) 42 | C.ht_remove(t.tree, p) 43 | } 44 | 45 | func (t *HashTree) Clear() { 46 | C.ht_destroy(t.tree) 47 | t.tree = nil 48 | } 49 | 50 | func (t *HashTree) ScanHintFile(bucket int, path string) { 51 | p := C.CString(path) 52 | defer C.free(unsafe.Pointer(p)) 53 | C.scanHintFile(t.tree, (_Ctype_int)(bucket), p, nil) 54 | } 55 | 56 | func (t *HashTree) Get(key string) *Item { 57 | c_key := C.CString(key) 58 | defer C.free(unsafe.Pointer(c_key)) 59 | citem, _ := C.ht_get(t.tree, c_key) 60 | if citem == nil { 61 | return nil 62 | } 63 | defer C.free(unsafe.Pointer(citem)) 64 | 65 | return &Item{Bucket: int(uint32(citem.pos) & 0xff), 66 | Pos: uint32(citem.pos) & 0xffffff00, 67 | Hash: uint32(citem.hash), 68 | Version: int32(citem.ver)} 69 | } 70 | 71 | func (t *HashTree) Hash() uint16 { 72 | c_key := C.CString("@") 73 | defer C.free(unsafe.Pointer(c_key)) 74 | return uint16(C.ht_get_hash(t.tree, c_key, (*C.uint)(nil))) 75 | } 76 | 77 | func (t *HashTree) Len() int { 78 | c_key := C.CString("@") 79 | defer C.free(unsafe.Pointer(c_key)) 80 | var cnt C.uint 81 | C.ht_get_hash(t.tree, c_key, &cnt) 82 | return int(cnt) 83 | } 84 | 85 | func (t *HashTree) List(pos, prefix string) string { 86 | c_key := C.CString(pos) 87 | defer C.free(unsafe.Pointer(c_key)) 88 | c_prefix := C.CString(prefix) 89 | defer C.free(unsafe.Pointer(c_prefix)) 90 | cstr, _ := C.ht_list(t.tree, c_key, c_prefix) 91 | if cstr == nil { 92 | return "" 93 | } 94 | defer C.free(unsafe.Pointer(cstr)) 95 | return C.GoString(cstr) 96 | } 97 | -------------------------------------------------------------------------------- /go_test/htree_test.go: -------------------------------------------------------------------------------- 1 | package btest 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func fnv1a(buf []byte) (h uint32) { 10 | PRIME := uint32(0x01000193) 11 | h = 0x811c9dc5 12 | for _, b := range buf { 13 | h ^= uint32(int8(b)) 14 | h = (h * PRIME) 15 | } 16 | return h 17 | } 18 | 19 | func TestHash(t *testing.T) { 20 | h := fnv1a([]byte("test")) 21 | if h != uint32(2949673445) { 22 | t.Error("hash error", h) 23 | } 24 | } 25 | 26 | func TestHTree(t *testing.T) { 27 | tree := NewHTree(0, 0) 28 | if tree.Hash() != 0 || tree.Len() != 0 { 29 | t.Error("emtpy tree", tree.Hash(), tree.Len()) 30 | } 31 | item := &Item{Bucket: 3, Pos: 1024, Hash: 3, Version: 10} 32 | tree.Add("test", item) 33 | h := fnv1a([]byte("test")) * 3 34 | if tree.Hash() != uint16(h) { 35 | t.Error("hash error", tree.Hash()) 36 | } 37 | 38 | tree.Add("test0", &Item{Hash: 5, Version: -1}) 39 | if tree.Hash() != uint16(h) { 40 | t.Error("hash error", tree.Hash()) 41 | } 42 | 43 | tree.Add("test2", &Item{Hash: 5, Version: 1}) 44 | h += fnv1a([]byte("test2")) * 5 45 | if tree.Hash() != uint16(h) { 46 | t.Error("hash error", tree.Hash()) 47 | } 48 | 49 | tree.Add("test2", &Item{Hash: 7, Version: -2}) 50 | h -= fnv1a([]byte("test2")) * 5 51 | if tree.Hash() != uint16(h) { 52 | t.Error("hash error", tree.Hash()) 53 | } 54 | 55 | tree.Add("test2", &Item{Hash: 5, Version: 3}) 56 | h += fnv1a([]byte("test2")) * 5 57 | if tree.Hash() != uint16(h) { 58 | t.Error("hash error", tree.Hash()) 59 | } 60 | 61 | l := tree.List("", "") 62 | if l != "test 3 10\ntest0 5 -1\ntest2 5 3\n" { 63 | t.Error("list error", l) 64 | } 65 | 66 | l = tree.List("", "test2") 67 | if l != "test2 5 3\n" { 68 | t.Error("list with prefix error", l) 69 | } 70 | 71 | r := tree.Get("test") 72 | if !reflect.DeepEqual(r, item) { 73 | t.Error("get failed", r, item) 74 | } 75 | 76 | tree.Remove("test") 77 | if tree.Len() != 1 { 78 | t.Error("remove failed", tree.Len()) 79 | } 80 | 81 | r = tree.Get("test") 82 | if r != nil { 83 | t.Error("remove failed") 84 | } 85 | } 86 | 87 | func TestSplitMerge(t *testing.T) { 88 | tree := NewHTree(0, 0) 89 | N := 254 90 | key := "test" 91 | tree.Add(key, &Item{Hash: 3, Version: 1}) 92 | child_hash := make([]uint32, 16, 16) 93 | child_count := make([]uint32, 16, 16) 94 | 95 | keyhash := fnv1a([]byte(key)) 96 | b := uint8(keyhash>>28) & 0xf 97 | h0 := keyhash * 3 98 | child_hash[b] += h0 //21935 99 | child_count[b] += 1 100 | 101 | for i := 0; i < N; i++ { 102 | key := fmt.Sprintf("a%d", i) 103 | tree.Add(key, &Item{Hash: uint32(i), Version: int32(i + 1)}) 104 | 105 | keyhash = fnv1a([]byte(key)) 106 | b = uint8(keyhash>>28) & 0xf 107 | child_hash[b] += keyhash * uint32(i) 108 | child_count[b] += 1 109 | } 110 | h := uint32(0) 111 | for i := 0; i < 16; i++ { 112 | if N+1 > 64*4 { 113 | h *= 97 114 | } 115 | h += child_hash[i] 116 | } 117 | if tree.Len() != N+1 || tree.Hash() != uint16(h) { 118 | t.Error("split failed", tree.Len(), tree.Hash(), uint16(h), "\n", 119 | tree.List("", "")) 120 | 121 | } 122 | 123 | for i := 0; i < N; i++ { 124 | key := fmt.Sprintf("a%d", i) 125 | tree.Remove(key) 126 | } 127 | if tree.Len() != 1 || tree.Hash() != uint16(h0) { 128 | t.Error("remove many failed", tree.Len(), tree.Hash()) 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /peteris.yaml: -------------------------------------------------------------------------------- 1 | - job: 2 | name: unittest 3 | desciption: 'beansdb' 4 | scm: 5 | - git: 6 | branches: 7 | - master 8 | skip-tag: True 9 | url: http://code.dapps.douban.com/beansdb.git 10 | builders: 11 | - shell: | 12 | ./autogen.sh 13 | ./configure 14 | make 15 | rm -rf venv 16 | virtualenv venv || true 17 | source venv/bin/activate 18 | pip install -i http://pypim.dapps.douban.com/simple -r python_tests/pip-requirement.txt 19 | nosetests --with-xunit --xunit-file=unittest.xml 20 | deactivate 21 | publishers: 22 | - junit: 23 | results: unittest.xml 24 | - email: 25 | recipients: zhuzhaolong@douban.com 26 | -------------------------------------------------------------------------------- /python/dbclient.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import os.path, re, sys 4 | import time 5 | from random import shuffle 6 | 7 | from cmemcached import Client # set_raw/get_raw/get_last_error only supported in python-libmemcached 8 | 9 | 10 | def fnv1a(s): 11 | prime = 0x01000193 12 | h = 0x811c9dc5 13 | for c in s: 14 | h ^= ord(c) 15 | h = (h * prime) & 0xffffffff 16 | return h 17 | 18 | class MCStore(object): 19 | def __init__(self, server): 20 | self.server = server 21 | self.mc = Client([server], do_split=0) 22 | 23 | def __del__(self): 24 | self.close() 25 | 26 | def __repr__(self): 27 | return '' % repr(self.server) 28 | 29 | def __str__(self): 30 | return self.server 31 | 32 | def set(self, key, data, rev=0): 33 | return bool(self.mc.set(key, data, rev)) 34 | 35 | def set_raw(self, key, data, rev=0, flag=0): # interface for sync data 36 | if rev < 0: 37 | raise str(rev) 38 | return self.mc.set_raw(key, data, rev, flag) 39 | 40 | def get(self, key): 41 | try: 42 | r = self.mc.get(key) 43 | if r is None and self.mc.get_last_error() != 0: 44 | raise IOError(self.mc.get_last_error()) 45 | return r 46 | except ValueError: 47 | self.mc.delete(key) 48 | 49 | def get_raw(self, key): 50 | r, flag = self.mc.get_raw(key) 51 | if r is None and self.mc.get_last_error() != 0: 52 | raise IOError( 53 | self.mc.get_last_error(), self.mc.get_last_strerror()) 54 | return r, flag 55 | 56 | def get_multi(self, keys): 57 | return self.mc.get_multi(keys) 58 | 59 | def delete(self, key): 60 | return bool(self.mc.delete(key)) 61 | 62 | def close(self): 63 | if self.mc is not None: 64 | self.mc.close() 65 | self.mc = None 66 | 67 | 68 | class WriteFailedError(Exception): 69 | def __init__(self, key): 70 | self.key = key 71 | def __repr__(self): 72 | return 'write %s failed' % self.key 73 | 74 | 75 | class Beansdb(object): 76 | hash_space = 1<<32 77 | cached = True 78 | def __init__(self, servers, buckets_count=16, N=3, W=1, R=1): 79 | self.buckets_count = buckets_count 80 | self.bucket_size = self.hash_space / buckets_count 81 | self.servers = {} 82 | self.server_buckets = {} 83 | self.buckets = [[] for i in range(buckets_count)] 84 | for s,bs in servers.items(): 85 | server = MCStore(s) 86 | self.servers[s] = server 87 | self.server_buckets[s] = bs 88 | for b in bs: 89 | self.buckets[b].append(server) 90 | for b in range(self.buckets_count): 91 | self.buckets[b].sort(key=lambda x:fnv1a("%d:%s:%d"%(b,x,b))) 92 | self.N = N 93 | self.W = W 94 | self.R = R 95 | 96 | def print_buckets(self): 97 | for i,ss in enumerate(self.buckets): 98 | print i,','.join(str(s) for s in ss) 99 | for s,bs in self.server_buckets.items(): 100 | print s,len(bs) 101 | 102 | def _get_servers(self, key): 103 | hash = fnv1a(key) 104 | b = hash / self.bucket_size 105 | return self.buckets[b] 106 | 107 | def get(self, key): 108 | ss = self._get_servers(key) 109 | for i,s in enumerate(ss): 110 | r = s.get(key) 111 | if r is not None: 112 | # self heal 113 | for k in range(i): 114 | ss[k].set(key, r) 115 | return r 116 | 117 | def get_multi(self, keys): 118 | rs = {} 119 | ss = self.servers.values() 120 | shuffle(ss) 121 | for s in ss: 122 | rs.update(s.get_multi([k for k in keys if k not in rs and s in self._get_servers(k)])) 123 | return rs 124 | 125 | def delete(self, key): 126 | rs = [s.delete(key) for s in self._get_servers(key)] 127 | return rs.count(True) >= self.W 128 | 129 | def set(self, key, value): 130 | rs = [s.set(key, value) for s in self._get_servers(key)] 131 | if not rs.count(True) >= self.W: 132 | # try to get, it will return False when set same content into db 133 | if self.get(key) != value: 134 | raise WriteFailedError(key) 135 | return True 136 | 137 | 138 | BEANSDBCFG = { 139 | "localhost:7901": range(16), 140 | "localhost:7902": range(16), 141 | "localhost:7903": range(16), 142 | } 143 | 144 | db = Beansdb(BEANSDBCFG, 16) 145 | 146 | def _test(): 147 | u = "/test" 148 | data = "teatdata" * 100 149 | assert db.set(u, data) 150 | #assert db.exists(u) 151 | assert db.get(u) == data 152 | assert db.get_multi([u]) == {u:data} 153 | assert db.delete(u) 154 | assert db.get(u) is None 155 | 156 | if __name__ == '__main__': 157 | _test() 158 | pass 159 | 160 | -------------------------------------------------------------------------------- /python/httpd.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding: utf-8 -*- 3 | 4 | import sys 5 | import os 6 | import re 7 | import web 8 | 9 | from dbclient import Beansdb 10 | 11 | fs = Beansdb({"localhost:7900": range(16)}, 16) 12 | 13 | class File: 14 | def GET(self, path): 15 | data = fs.get(path) 16 | if data: 17 | sys.stdout.write(data) 18 | else: 19 | web.notfound() 20 | 21 | urls = ( 22 | "(/.*)", "File", 23 | ) 24 | 25 | def runfcgi_multiprocess(func, addr=('localhost', 8000)): 26 | import flup.server.fcgi as flups 27 | return flups.WSGIServer(func, multithreaded=False, 28 | multiprocess=True, bindAddress=addr).run() 29 | 30 | web.wsgi.runfcgi = runfcgi_multiprocess 31 | 32 | if __name__ == '__main__': 33 | if hasattr(web, 'run'): 34 | # web.py 0.2 35 | web.run(urls, globals()) 36 | else: 37 | app = web.application(urls, globals()) 38 | app.run() 39 | -------------------------------------------------------------------------------- /python/sync.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | 3 | import sys, os, os.path 4 | from dbclient import Beansdb, db 5 | 6 | def get_dir(s, dir): 7 | def parse(line): 8 | p,h,c = line.split(' ') 9 | return p, (int(h), int(c)) 10 | return dict(parse(line) for line in 11 | filter(None, (s.get(dir) or '').split('\n'))) 12 | 13 | def is_dir(d): 14 | return len(d) == 16 and len([k for k in d if k.endswith('/')]) == 16 15 | 16 | def mirror(src, dst, path): 17 | s = get_dir(src, path) 18 | d = get_dir(dst, path) 19 | if s == d: 20 | print path, src, dst, 'skipped' 21 | return 22 | if is_dir(s): 23 | for k in sorted(s): 24 | if s[k] != d.get(k): 25 | #print path+k[0], 'mirror ', s[k], d.get(k) 26 | mirror(src, dst, path+k[0]) 27 | elif is_dir(d): 28 | for k in sorted(d): 29 | mirror(dst, src, path+k[0]) 30 | elif not is_dir(s) and not is_dir(d): 31 | sync_files(src, dst, path, s, d) 32 | sync_files(dst, src, path, d, s) 33 | else: 34 | print path, src, '=>', dst, 'skipped' 35 | 36 | def sync_files(src, dst, path, s, d): 37 | for k in sorted(s.keys()): 38 | if k not in d: 39 | data = src.get(k) 40 | if data is not None: 41 | print path, k, s[k], d.get(k,(0,0)), src, "=>", dst, dst.set(k, data, s[k][1]) 42 | else: 43 | print path, src, k, 'is None', src.delete(k) 44 | elif s[k][0] != d[k][0]: 45 | if s[k][1] > d[k][1]: 46 | data = src.get(k) 47 | if data is not None: 48 | print path, k, s[k], d.get(k,(0,0)), src, "=>", dst, dst.set(k, data, s[k][1]) 49 | else: 50 | print path, src, k, 'is None', src.delete(k) 51 | elif s[k][1] == d[k][1]: 52 | m1 = int((src.get('?'+k) or '0').split(' ')[-1]) 53 | m2 = int((dst.get('?'+k) or '0').split(' ')[-1]) 54 | print path, src, k, 'is broken', s[k], m1, d[k], m2 55 | if m1 > m2: 56 | dst.set(k, src.get(k)) 57 | elif m2 >= m1: 58 | src.set(k, dst.get(k)) 59 | 60 | def stat(s): 61 | st = {} 62 | for d,h,c in [line.split(' ') for line in (s.get('@') or '').split('\n') if line]: 63 | if len(d) != 2 and not d.endswith('/'): 64 | return {} 65 | try: 66 | st[int(d[0],16)] = (h,int(c)) 67 | except: 68 | pass 69 | return st 70 | 71 | def almost(a,b): 72 | return abs(a-b) < 0.2*(abs(a)+abs(b)) 73 | 74 | def sync(db, start=0): 75 | stats = {} 76 | for n,s in db.servers.items(): 77 | stats[str(s)] = stat(s) 78 | for b in range(start, db.buckets_count): 79 | N = len(db.buckets[b]) 80 | for s in range(N)[::-1]: 81 | src = db.buckets[b][s] 82 | dst = db.buckets[b][(s+1)%N] 83 | if not stats[str(src)] or not stats[str(dst)]: 84 | continue 85 | ss = stats[str(src)].get(b, (0,0)) 86 | ds = stats[str(dst)].get(b, (0,0)) 87 | if ss != ds: 88 | print '%02x'%b,src,ss, dst, ds 89 | mirror(src, dst, "@%0x"%b) 90 | 91 | def lock(fd): 92 | import fcntl, errno 93 | try: 94 | fcntl.lockf(fd, fcntl.LOCK_EX|fcntl.LOCK_NB) 95 | except IOError, e: 96 | if e.errno in (errno.EACCES, errno.EAGAIN): 97 | print "There is an instance of", sys.argv[0], "running. Quit" 98 | sys.exit(0) 99 | else: 100 | raise 101 | 102 | def main(): 103 | import os 104 | lock_file_path = '/tmp/lsync.lock' 105 | fd = os.open(lock_file_path, os.O_CREAT|os.O_RDWR, 0660) 106 | try: 107 | lock(fd) 108 | if len(sys.argv)>1: 109 | sync(db, int(sys.argv[1])) 110 | else: 111 | sync(db) 112 | finally: 113 | os.close(fd) 114 | 115 | if __name__ == "__main__": 116 | main() 117 | -------------------------------------------------------------------------------- /python_tests/batch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | import os 5 | import sys 6 | import time 7 | from base import BeansdbInstance, TestBeansdbBase, MCStore 8 | from base import check_data_hint_integrity, delete_hint_and_htree, random_string 9 | import unittest 10 | 11 | string_large = random_string(10*1024*1024) 12 | 13 | 14 | class TestGenerateData(TestBeansdbBase): 15 | 16 | proxy_addr = 'localhost:7905' 17 | backend1_addr = 'localhost:57901' 18 | 19 | def setUp(self): 20 | self._clear_dir() 21 | self._init_dir() 22 | self.backend1 = BeansdbInstance(self.data_base_path, 57901) 23 | 24 | def test_gen_data(self): 25 | self.backend1.start() 26 | store = MCStore(self.backend1_addr) 27 | self.assert_(store.set("largekey", string_large)) 28 | self.assert_(store.get("largekey") == string_large) 29 | 30 | loop_num = 16 * 1024 31 | for i in xrange(loop_num): 32 | key = "test%s" % (i) 33 | if not store.set(key, 1): 34 | print "failed to set %s" % (key) 35 | return self.fail("fail") 36 | if not store.set(key, 2): 37 | print "failed to set %s" % (key) 38 | return self.fail("fail") 39 | print "done set" 40 | for i in xrange(loop_num): 41 | key = "test%s" % (i) 42 | try: 43 | self.assertEqual(store.get(key), 2) 44 | except Exception, e: 45 | print key, "error", e 46 | return self.fail("fail") 47 | print "done get" 48 | self.backend1.stop() 49 | print "stopped" 50 | self.backend1.start() 51 | print "started" 52 | store = MCStore(self.backend1_addr) 53 | for i in xrange(loop_num): 54 | key = "test%s" % (i) 55 | try: 56 | self.assertEqual(store.get(key), 2) 57 | except Exception, e: 58 | print key, "error", e 59 | return self.fail("fail") 60 | print "done get" 61 | print "check data & hint" 62 | check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth) 63 | self.assertEqual(self.backend1.item_count(), loop_num + 1) 64 | 65 | self.backend1.stop() 66 | print "delete .hint and .htree, should regenerate" 67 | delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth) 68 | self.backend1.start() 69 | print "check data & hint" 70 | check_data_hint_integrity(self.backend1.db_home, self.backend1.db_depth) 71 | self.assertEqual(self.backend1.item_count(), loop_num + 1) 72 | 73 | def tearDown(self): 74 | self.backend1.stop() 75 | 76 | class TestGenerateData2(TestGenerateData): 77 | 78 | def setUp(self): 79 | self._clear_dir() 80 | self._init_dir() 81 | self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=2) 82 | 83 | if __name__ == '__main__': 84 | unittest.main() 85 | 86 | # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 : 87 | -------------------------------------------------------------------------------- /python_tests/gc_simple.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | import os 5 | import sys 6 | import time 7 | from base import BeansdbInstance, TestBeansdbBase, MCStore 8 | from base import locate_key_iterate, locate_key_with_hint, get_hash, check_data_hint_integrity 9 | import unittest 10 | import telnetlib 11 | import glob 12 | import quicklz 13 | import struct 14 | import re 15 | 16 | 17 | 18 | class TestGCBase(TestBeansdbBase): 19 | 20 | proxy_addr = 'localhost:7905' 21 | backend1_addr = 'localhost:57901' 22 | 23 | def setUp(self): 24 | self._clear_dir() 25 | self._init_dir() 26 | self.backend1 = BeansdbInstance(self.data_base_path, 57901) 27 | 28 | def _start_gc(self, ignore_recent=0, bucket=None): 29 | """ bucket must be in 0 or 00 string """ 30 | if bucket is not None: 31 | assert isinstance(bucket, basestring) and len(bucket) <= 2 32 | t = telnetlib.Telnet("127.0.0.1", self.backend1.port) 33 | if bucket is None: 34 | t.write('flush_all {}\n'.format(ignore_recent)) 35 | else: 36 | t.write('flush_all %s @%s\n' % (ignore_recent, bucket)) 37 | t.read_until('OK') 38 | t.write('quit\n') 39 | t.close() 40 | 41 | def _gc_status(self): 42 | t = telnetlib.Telnet("127.0.0.1", self.backend1.port) 43 | t.write('optimize_stat\n') 44 | out = t.read_until('\n') 45 | t.write('quit\n') 46 | t.close() 47 | return out.strip("\r\n") 48 | 49 | def _gen_data(self, data, prefix='', loop_num=10 * 1024): 50 | store = MCStore(self.backend1_addr) 51 | for key in self.backend1.generate_key(prefix=prefix, count=loop_num): 52 | if not store.set(key, data): 53 | return self.fail("fail to set %s" % (key)) 54 | 55 | def _delete_data(self, prefix='', loop_num=10 * 1024): 56 | store = MCStore(self.backend1_addr) 57 | for key in self.backend1.generate_key(prefix=prefix, count=loop_num): 58 | if not store.delete(key): 59 | return self.fail("fail to delete %s" % (key)) 60 | 61 | 62 | def _check_data(self, data, prefix='', loop_num=10 * 1024): 63 | store = MCStore(self.backend1_addr) 64 | for key in self.backend1.generate_key(prefix=prefix, count=loop_num): 65 | try: 66 | self.assertEqual(store.get(key), data) 67 | except Exception, e: 68 | return self.fail("fail to check key %s: %s" % (key, str(e))) 69 | 70 | 71 | def tearDown(self): 72 | self.backend1.stop() 73 | 74 | 75 | class TestGCSimple(TestGCBase): 76 | 77 | def test_gc(self): 78 | self.backend1.start() 79 | store = MCStore(self.backend1_addr) 80 | ver_key = 'test_version_key' 81 | store.set(ver_key, 1) 82 | store.set(ver_key, 1, rev=3) # will only raise version in htree 83 | self.assertEqual(self._get_version(store, ver_key), 3) 84 | 85 | self._gen_data(1) 86 | print "done set data to 1" 87 | time.sleep(10) 88 | self._gen_data(2) 89 | self._gen_data(1, prefix='delete_group') 90 | time.sleep(2) 91 | self.assertEqual(self.backend1.item_count(), 20481) 92 | self._delete_data(prefix='delete_group') 93 | self.assertEqual(self.backend1.item_count(), 10241) 94 | self.assert_(not store.delete('key not exists')) 95 | self.assertEqual(self.backend1.item_count(), 10241) 96 | 97 | print "stop beansdb to rotate data file and produce hint" 98 | self.backend1.stop() 99 | self.backend1.start() 100 | 101 | print "deleted key should exists in data" 102 | assert locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=1) 103 | assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0", ver_=-2) 104 | print "done set data to 2" 105 | self._start_gc(0) 106 | print "gc started" 107 | while True: 108 | status = self._gc_status() 109 | if status.find('running') >= 0: 110 | self._check_data(2) 111 | continue 112 | elif status == 'success': 113 | print "done gc" 114 | break 115 | elif status == 'fail': 116 | return self.fail("optimize_stat = fail") 117 | else: 118 | self.fail(status) 119 | self._check_data(2) 120 | store = MCStore(self.backend1_addr) 121 | self.assertEqual(self._get_version(store, ver_key), 3) # version 3 should be in data 122 | print "check test key version, old version should not exist" 123 | assert locate_key_with_hint(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=3) 124 | assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key=ver_key, ver_=1) 125 | print "check data & hint" 126 | check_data_hint_integrity(self.backend1.db_home, db_depth=self.backend1.db_depth) 127 | 128 | print "deleted key got deleted from data file during gc" 129 | assert not locate_key_iterate(self.backend1.db_home, db_depth=self.backend1.db_depth, key="delete_group" + "test0") 130 | self.assertEqual(self.backend1.item_count(), 10241) 131 | 132 | self.backend1.stop() 133 | 134 | class TestGCSimple2(TestGCSimple): 135 | 136 | def setUp(self): 137 | self._clear_dir() 138 | self._init_dir() 139 | self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=2) 140 | 141 | if __name__ == '__main__': 142 | unittest.main() 143 | 144 | 145 | # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 : 146 | -------------------------------------------------------------------------------- /python_tests/key_version.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | import os 5 | import sys 6 | import time 7 | from base import BeansdbInstance, TestBeansdbBase, MCStore 8 | import unittest 9 | 10 | 11 | class TestKeyVersion(TestBeansdbBase): 12 | 13 | proxy_addr = 'localhost:7905' 14 | backend1_addr = 'localhost:57901' 15 | 16 | 17 | def _get_version(self, store, key): 18 | meta = store.get("?" + key) 19 | if meta: 20 | return int(meta.split()[0]) 21 | 22 | def setUp(self): 23 | self._clear_dir() 24 | self._init_dir() 25 | self.backend1 = BeansdbInstance(self.data_base_path, 57901) 26 | 27 | def test_set_verion(self): 28 | self.backend1.start() 29 | store = MCStore(self.backend1_addr) 30 | key = "key1" 31 | store.set(key, "aaa") 32 | self.assertEqual(store.get(key), "aaa") 33 | self.assertEqual(self._get_version(store, key), 1) 34 | store.set_raw(key, "bbb", rev=3) 35 | self.assertEqual(self._get_version(store, key), 3) 36 | store.set_raw(key, "bbb", rev=4) 37 | self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version 38 | store.set_raw(key, "ccc", rev=2) 39 | self.assertEqual(store.get(key), "bbb") 40 | self.assertEqual(self._get_version(store, key), 4) # current behavior will raise version 41 | 42 | def test_delete_version(self): 43 | self.backend1.start() 44 | store = MCStore(self.backend1_addr) 45 | key = "key2" 46 | store.set(key, "aaa") 47 | self.assertEqual(self._get_version(store, key), 1) 48 | store.delete(key) 49 | self.assertEqual(self._get_version(store, key), None) 50 | store.set(key, "bbb") 51 | self.assertEqual(store.get(key), 'bbb') 52 | self.assertEqual(self._get_version(store, key), 3) 53 | 54 | 55 | def tearDown(self): 56 | self.backend1.stop() 57 | 58 | if __name__ == '__main__': 59 | unittest.main() 60 | 61 | 62 | # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 : 63 | -------------------------------------------------------------------------------- /python_tests/one_key.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | from base import BeansdbInstance, TestBeansdbBase, MCStore, random_string 5 | import unittest 6 | 7 | import zlib 8 | import string 9 | 10 | 11 | class TestKeyVersion(TestBeansdbBase): 12 | 13 | proxy_addr = 'localhost:7905' 14 | backend1_addr = 'localhost:57901' 15 | 16 | def _get_meta(self, store, key): 17 | meta = store.get("??" + key) 18 | if meta: 19 | meta = meta.split() 20 | assert(len(meta) == 7) 21 | return tuple([int(meta[i]) for i in [0, -2, -1]]) 22 | 23 | def setUp(self): 24 | self._clear_dir() 25 | self._init_dir() 26 | self.backend1 = BeansdbInstance(self.data_base_path, 57901) 27 | 28 | self.last_pos = 0 29 | self.last_size = 0 30 | 31 | def append(self, size): 32 | self.last_pos += self.last_size 33 | self.last_size = size 34 | 35 | def test_set_verion(self): 36 | self.backend1.start() 37 | store = MCStore(self.backend1_addr) 38 | key = "key1" 39 | store.set(key, "aaa") 40 | self.append(256) 41 | self.assertEqual(store.get(key), "aaa") 42 | self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) 43 | store.set_raw(key, "bbb", rev=3) 44 | self.append(256) 45 | self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos)) 46 | store.set_raw(key, "bbb", rev=4) 47 | self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version 48 | store.set_raw(key, "ccc", rev=2) 49 | self.assertEqual(store.get(key), "bbb") 50 | self.assertEqual(self._get_meta(store, key), (4, 0, self.last_pos)) # current behavior will raise version 51 | 52 | def test_delete_version(self): 53 | self.backend1.start() 54 | store = MCStore(self.backend1_addr) 55 | key = "key1" 56 | 57 | store.set(key, "aaa") 58 | self.append(256) 59 | self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) 60 | 61 | store.delete(key) 62 | self.append(256) 63 | self.assertEqual(self._get_meta(store, key), (-2, 0, self.last_pos)) 64 | 65 | store.set(key, "bbb") 66 | self.append(256) 67 | self.assertEqual(store.get(key), 'bbb') 68 | self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos)) 69 | 70 | def _test_compress(self, overflow): 71 | self.backend1.start() 72 | value = string.letters 73 | store = MCStore(self.backend1_addr) 74 | compressed_value = zlib.compress(value, 0) 75 | key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0)) 76 | 77 | value_easy_compress = 'v'* len(compressed_value) 78 | 79 | assert(store.set(key, value_easy_compress)) 80 | assert(store.get(key) == value_easy_compress) 81 | self.append(256) 82 | self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) 83 | 84 | assert(store.set_raw(key, compressed_value, flag = 0x00000010)) 85 | assert(store.get(key) == value) 86 | self.append(512 if overflow else 256) 87 | self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos)) 88 | 89 | assert(store.set(key, 'aaa')) 90 | self.append(256) 91 | self.assertEqual(self._get_meta(store, key), (3, 0, self.last_pos)) 92 | 93 | def test_compress_257(self): 94 | self._test_compress(True) 95 | 96 | def test_compress_256(self): 97 | self._test_compress(False) 98 | 99 | def test_special_key(self): 100 | self.backend1.start() 101 | kvs = [('a'*250, 1), ("a", range(1000))] 102 | store = MCStore(self.backend1_addr) 103 | for k,v in kvs: 104 | assert(store.set(k, v)) 105 | assert(v == store.get(k)) 106 | self.backend1.stop() 107 | self.backend1.start() 108 | store = MCStore(self.backend1_addr) 109 | for (k,v) in kvs: 110 | assert(v == store.get(k)) 111 | 112 | def test_big_value(self): 113 | self.backend1.start() 114 | store = MCStore(self.backend1_addr) 115 | 116 | key = "largekey" 117 | size = 10 * 1024 * 1024 118 | rsize = (((size + len(key) + 24) >> 8) + 1) << 8 119 | string_large = random_string(size) 120 | assert(store.set(key, string_large)) 121 | assert(store.get(key) == string_large) 122 | self.append(rsize) 123 | self.assertEqual(self._get_meta(store, key), (1, 0, self.last_pos)) 124 | 125 | assert(store.set(key, 'aaa')) 126 | self.append(256) 127 | self.assertEqual(self._get_meta(store, key), (2, 0, self.last_pos)) 128 | 129 | 130 | def test_restart(self): 131 | pass 132 | 133 | def tearDown(self): 134 | self.backend1.stop() 135 | 136 | if __name__ == '__main__': 137 | unittest.main() 138 | 139 | 140 | # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 : 141 | -------------------------------------------------------------------------------- /python_tests/pip-requirement.txt: -------------------------------------------------------------------------------- 1 | cython 2 | -e git+http://code.dapps.douban.com/python-libmemcached.git#egg=python-libmemcached 3 | -e git+http://code.dapps.douban.com/douban-corelib.git#egg=DoubanCoreLib-dev 4 | pyrex 5 | -e git+http://code.dapps.douban.com/pyquicklz.git#egg=pyquicklz 6 | -e git+http://code.dapps.douban.com/fnv1a.git#egg=fnv1a 7 | python-memcached 8 | nose 9 | -------------------------------------------------------------------------------- /python_tests/test_broken_data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # coding:utf-8 3 | 4 | import os 5 | import sys 6 | import time 7 | from base import BeansdbInstance, TestBeansdbBase, MCStore 8 | from base import get_hash, check_data_hint_integrity 9 | from base import random_string, delete_hint_and_htree, temper_with_key_value 10 | import unittest 11 | import telnetlib 12 | import glob 13 | import quicklz 14 | import struct 15 | import re 16 | from gc_simple import TestGCBase 17 | 18 | 19 | string_large = random_string(10*1024*1024) 20 | 21 | class TestBrokenBase(TestGCBase): 22 | proxy_addr = 'localhost:7905' 23 | backend1_addr = 'localhost:57901' 24 | # 25 | def setUp(self): 26 | self._clear_dir() 27 | self._init_dir() 28 | self.backend1 = BeansdbInstance(self.data_base_path, 57901, db_depth=1) 29 | 30 | # only generate keys in sector0 31 | def _gen_data(self, data, prefix='', loop_num=10 * 1024, sector=0): 32 | store = MCStore(self.backend1_addr) 33 | for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector): 34 | if not store.set(key, data): 35 | return self.fail("fail to set %s" % (key)) 36 | store.close() 37 | 38 | def _check_data(self, data, prefix='', loop_num=10 * 1024, sector=0): 39 | store = MCStore(self.backend1_addr) 40 | for key in self.backend1.generate_key(prefix=prefix, count=loop_num, sector=sector): 41 | try: 42 | self.assertEqual(store.get(key), data) 43 | except Exception, e: 44 | return self.fail("fail to check key %s: %s" % (key, str(e))) 45 | store.close() 46 | 47 | class TestBitCaseScanBroken(TestBrokenBase): 48 | 49 | def test_bc_scan(self): 50 | print "" 51 | print "test bc scan broken data" 52 | self.backend1.start() 53 | self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) 54 | tempered_key = None 55 | for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): 56 | tempered_key = key 57 | store = MCStore(self.backend1_addr) 58 | store.set(tempered_key, string_large) 59 | store.close() 60 | self._gen_data("other value", prefix='test3', loop_num=1024, sector=0) 61 | self.assertEqual(self.backend1.item_count(), 2049) 62 | self.backend1.stop() 63 | 64 | #make sure we produce a crc error 65 | assert temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=True) 66 | delete_hint_and_htree(self.backend1.db_home, self.backend1.db_depth) 67 | 68 | self.backend1.start() 69 | self.assertEqual(self.backend1.item_count(), 2048) 70 | self._check_data("some value", prefix="test1", loop_num=1024, sector=0) 71 | self._check_data("other value", prefix="test3", loop_num=1024, sector=0) 72 | self.backend1.stop() 73 | 74 | 75 | 76 | class TestOnlineBroken(TestBrokenBase): 77 | 78 | def test_get_broken(self): 79 | print 80 | print "test get broken data" 81 | self.backend1.start() 82 | 83 | self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) 84 | tempered_key = None 85 | for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): 86 | tempered_key = key 87 | store = MCStore(self.backend1_addr) 88 | store.set(tempered_key, string_large) 89 | store.close() 90 | self._gen_data("other value", prefix='test3', loop_num=1024, sector=0) 91 | self.assertEqual(self.backend1.item_count(), 2049) 92 | 93 | # flush hint 94 | self.backend1.stop() 95 | self.backend1.start() 96 | 97 | #make sure we produce a crc error 98 | temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False) 99 | self.assertEqual(self.backend1.item_count(), 2049) 100 | print "when beansdb encounter broken data when reading, it should delete it from htree" 101 | store = MCStore(self.backend1_addr) 102 | store.get(tempered_key) is None 103 | store.close() 104 | self.assertEqual(self.backend1.item_count(), 2048) 105 | self._check_data("some value", prefix="test1", loop_num=1024, sector=0) 106 | self._check_data("other value", prefix="test3", loop_num=1024, sector=0) 107 | self.backend1.stop() 108 | 109 | 110 | class TestGCBroken(TestBrokenBase): 111 | 112 | def test_gc_broken(self): 113 | 114 | print 115 | print "test gc broken data" 116 | self.backend1.start() 117 | self._gen_data("some value", prefix='test1', loop_num=1024, sector=0) 118 | tempered_key = None 119 | for key in self.backend1.generate_key(prefix="test2", count=1, sector=0): 120 | tempered_key = key 121 | store = MCStore(self.backend1_addr) 122 | store.set(tempered_key, string_large) 123 | store.close() 124 | self._gen_data("other value", prefix='test1', loop_num=1024, sector=0) 125 | self.assertEqual(self.backend1.item_count(), 1025) 126 | # flush hint 127 | self.backend1.stop() 128 | self.backend1.start() 129 | 130 | #make sure we produce a crc error 131 | temper_with_key_value(self.backend1.db_home, self.backend1.db_depth, tempered_key, delete_hint=False) 132 | 133 | self._start_gc() 134 | while True: 135 | status = self._gc_status() 136 | if status == 'success': 137 | print "done gc" 138 | break 139 | elif status == 'fail': 140 | return self.fail("optimize_stat = fail") 141 | 142 | self.assertEqual(self.backend1.item_count(), 1024) 143 | self._check_data("other value", prefix="test1", loop_num=1024, sector=0) 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | if __name__ == '__main__': 153 | unittest.main() 154 | 155 | 156 | # vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 : 157 | -------------------------------------------------------------------------------- /python_tests/test_log.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | strict init = true 3 | reload conf period = 5M 4 | buffer max = 1MB 5 | buffer min = 1024 6 | default format = "%d.%us %-6V (%c:%F:%L) - %m%n" 7 | file perms = 600 8 | [formats] 9 | simple = "%d %m%n" 10 | [rules] 11 | beansdb.!info "./beansdb-error.log" 12 | beansdb.=info "./beansdb-access.log" 13 | 14 | -------------------------------------------------------------------------------- /python_tests/test_nolog.conf: -------------------------------------------------------------------------------- 1 | [global] 2 | strict init = true 3 | reload conf period = 5M 4 | buffer max = 1MB 5 | buffer min = 1024 6 | default format = "%d.%us %-6V (%c:%F:%L) - %m%n" 7 | file perms = 600 8 | [formats] 9 | simple = "%d %m%n" 10 | [rules] 11 | beansdb.!info "./beansdb-error.log" 12 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | bin_PROGRAMS = beansdb 2 | #export JEMALLOC_PATH=${HOME}/local/jemalloc-3.6.0 3 | beansdb_SOURCES = beansdb.c item.c fnv1a.h beansdb.h thread.c htree.h htree.c hint.h hint.c record.h record.c codec.h codec.c bitcask.h bitcask.c hstore.h hstore.c quicklz.h quicklz.c diskmgr.h diskmgr.c util.h const.h log.h log.c mfile.h mfile.c common.c 4 | beansdb_CPPFLAGS = -I ../third-party/zlog-1.2/ # -I${JEMALLOC_PATH}/include 5 | beansdb_LDFLAGS = -L ../third-party/zlog-1.2/ # -L ${JEMALLOC_PATH}/lib -Wl,-rpath,${JEMALLOC_PATH}/lib 6 | LIBS += -lzlog # -ljemalloc 7 | 8 | -------------------------------------------------------------------------------- /src/ae_epoll.c: -------------------------------------------------------------------------------- 1 | /* Linux epoll(2) based ae.c module 2 | * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com 3 | * Released under the BSD license. See the COPYING file for more info. */ 4 | 5 | #include 6 | #include 7 | #include "log.h" 8 | 9 | typedef struct aeApiState 10 | { 11 | int epfd; 12 | struct epoll_event events[AE_SETSIZE]; 13 | } aeApiState; 14 | 15 | static int aeApiCreate(EventLoop *eventLoop) 16 | { 17 | aeApiState *state = malloc(sizeof(aeApiState)); 18 | 19 | if (!state) return -1; 20 | state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */ 21 | if (state->epfd == -1) return -1; 22 | eventLoop->apidata = state; 23 | return 0; 24 | } 25 | 26 | static void aeApiFree(EventLoop *eventLoop) 27 | { 28 | aeApiState *state = eventLoop->apidata; 29 | 30 | close(state->epfd); 31 | free(state); 32 | } 33 | 34 | static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) 35 | { 36 | aeApiState *state = eventLoop->apidata; 37 | struct epoll_event ee; 38 | ee.events = EPOLLONESHOT; 39 | if (mask & AE_READABLE) ee.events |= EPOLLIN; 40 | if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; 41 | ee.data.u64 = 0; /* avoid valgrind warning */ 42 | ee.data.fd = fd; 43 | if (epoll_ctl(state->epfd, EPOLL_CTL_ADD,fd,&ee) == -1 && errno != EEXIST) 44 | { 45 | log_error("epoll_ctl(%d,%d) failed: %d", EPOLL_CTL_ADD,fd,errno); 46 | return -1; 47 | } 48 | return 0; 49 | } 50 | 51 | static int aeApiUpdateEvent(EventLoop *eventLoop, int fd, int mask) 52 | { 53 | aeApiState *state = eventLoop->apidata; 54 | struct epoll_event ee; 55 | ee.events = EPOLLONESHOT; 56 | if (mask & AE_READABLE) ee.events |= EPOLLIN; 57 | if (mask & AE_WRITABLE) ee.events |= EPOLLOUT; 58 | ee.data.u64 = 0; /* avoid valgrind warning */ 59 | ee.data.fd = fd; 60 | if (epoll_ctl(state->epfd, EPOLL_CTL_MOD,fd,&ee) == -1) 61 | { 62 | log_error("epoll_ctl(%d,%d) failed: %d", EPOLL_CTL_ADD,fd,errno); 63 | return -1; 64 | } 65 | return 0; 66 | } 67 | 68 | static int aeApiDelEvent(EventLoop *eventLoop, int fd) 69 | { 70 | aeApiState *state = eventLoop->apidata; 71 | struct epoll_event ee; 72 | 73 | ee.events = 0; 74 | ee.data.u64 = 0; /* avoid valgrind warning */ 75 | ee.data.fd = fd; 76 | /* Note, Kernel < 2.6.9 requires a non null event pointer even for 77 | * EPOLL_CTL_DEL. */ 78 | if (epoll_ctl(state->epfd,EPOLL_CTL_DEL,fd,&ee) == -1 79 | && errno != ENOENT && errno != EBADF) 80 | { 81 | log_error("epoll_ctl(%d,%d) failed: %d", EPOLL_CTL_DEL,fd,errno); 82 | return -1; 83 | } 84 | return 0; 85 | } 86 | 87 | int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) 88 | { 89 | aeApiState *state = eventLoop->apidata; 90 | int retval, numevents = 0; 91 | 92 | retval = epoll_wait(state->epfd,state->events,AE_SETSIZE, 93 | tvp ? (tvp->tv_sec*1000 + tvp->tv_usec/1000) : -1); 94 | if (retval > 0) 95 | { 96 | int j; 97 | 98 | numevents = retval; 99 | for (j = 0; j < numevents; j++) 100 | { 101 | int mask = 0; 102 | struct epoll_event *e = state->events+j; 103 | 104 | if (e->events & EPOLLIN) mask |= AE_READABLE; 105 | if (e->events & EPOLLOUT) mask |= AE_WRITABLE; 106 | eventLoop->fired[j] = e->data.fd; 107 | } 108 | } 109 | return numevents; 110 | } 111 | 112 | static char *aeApiName(void) 113 | { 114 | return "epoll"; 115 | } 116 | -------------------------------------------------------------------------------- /src/ae_kqueue.c: -------------------------------------------------------------------------------- 1 | /* Kqueue(2)-based ae.c module 2 | * Copyright (C) 2009 Harish Mallipeddi - harish.mallipeddi@gmail.com 3 | * Released under the BSD license. See the COPYING file for more info. */ 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include "beansdb.h" 10 | 11 | typedef struct aeApiState 12 | { 13 | int kqfd; 14 | struct kevent events[AE_SETSIZE]; 15 | } aeApiState; 16 | 17 | static int aeApiCreate(EventLoop *eventLoop) 18 | { 19 | aeApiState *state = malloc(sizeof(aeApiState)); 20 | 21 | if (!state) return -1; 22 | state->kqfd = kqueue(); 23 | if (state->kqfd == -1) return -1; 24 | eventLoop->apidata = state; 25 | 26 | return 0; 27 | } 28 | 29 | static void aeApiFree(EventLoop *eventLoop) 30 | { 31 | aeApiState *state = eventLoop->apidata; 32 | 33 | close(state->kqfd); 34 | free(state); 35 | } 36 | 37 | static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) 38 | { 39 | aeApiState *state = eventLoop->apidata; 40 | struct kevent ke; 41 | 42 | if (mask & AE_READABLE) 43 | { 44 | EV_SET(&ke, fd, EVFILT_READ, EV_ADD | EV_ONESHOT, 0, 0, NULL); 45 | if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; 46 | } 47 | if (mask & AE_WRITABLE) 48 | { 49 | EV_SET(&ke, fd, EVFILT_WRITE, EV_ADD | EV_ONESHOT, 0, 0, NULL); 50 | if (kevent(state->kqfd, &ke, 1, NULL, 0, NULL) == -1) return -1; 51 | } 52 | return 0; 53 | } 54 | 55 | static int aeApiUpdateEvent(EventLoop *eventLoop, int fd, int mask) 56 | { 57 | return aeApiAddEvent(eventLoop, fd, mask); 58 | } 59 | 60 | static int aeApiDelEvent(EventLoop *eventLoop, int fd) 61 | { 62 | aeApiState *state = eventLoop->apidata; 63 | struct kevent ke; 64 | 65 | EV_SET(&ke, fd, EVFILT_READ | EVFILT_WRITE, EV_DELETE, 0, 0, NULL); 66 | kevent(state->kqfd, &ke, 1, NULL, 0, NULL); 67 | return 0; 68 | } 69 | 70 | static int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) 71 | { 72 | aeApiState *state = eventLoop->apidata; 73 | int retval, numevents = 0; 74 | 75 | if (tvp != NULL) 76 | { 77 | struct timespec timeout; 78 | timeout.tv_sec = tvp->tv_sec; 79 | timeout.tv_nsec = tvp->tv_usec * 1000; 80 | retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, &timeout); 81 | } 82 | else 83 | { 84 | retval = kevent(state->kqfd, NULL, 0, state->events, AE_SETSIZE, NULL); 85 | } 86 | 87 | if (retval > 0) 88 | { 89 | int j; 90 | 91 | numevents = retval; 92 | for(j = 0; j < numevents; j++) 93 | { 94 | int mask = 0; 95 | struct kevent *e = state->events+j; 96 | 97 | if (e->filter == EVFILT_READ) mask |= AE_READABLE; 98 | if (e->filter == EVFILT_WRITE) mask |= AE_WRITABLE; 99 | eventLoop->fired[j] = e->ident; 100 | } 101 | } 102 | return numevents; 103 | } 104 | 105 | static char *aeApiName(void) 106 | { 107 | return "kqueue"; 108 | } 109 | -------------------------------------------------------------------------------- /src/ae_select.c: -------------------------------------------------------------------------------- 1 | /* Select()-based ae.c module 2 | * Copyright (C) 2009-2010 Salvatore Sanfilippo - antirez@gmail.com 3 | * Released under the BSD license. See the COPYING file for more info. */ 4 | 5 | #include 6 | 7 | typedef struct aeApiState 8 | { 9 | int maxfd; 10 | fd_set rfds, wfds; 11 | /* We need to have a copy of the fd sets as it's not safe to reuse 12 | * FD sets after select(). */ 13 | fd_set _rfds, _wfds; 14 | } aeApiState; 15 | 16 | static int aeApiCreate(EventLoop *eventLoop) 17 | { 18 | aeApiState *state = malloc(sizeof(aeApiState)); 19 | 20 | if (!state) return -1; 21 | FD_ZERO(&state->rfds); 22 | FD_ZERO(&state->wfds); 23 | eventLoop->apidata = state; 24 | return 0; 25 | } 26 | 27 | static void aeApiFree(EventLoop *eventLoop) 28 | { 29 | free(eventLoop->apidata); 30 | } 31 | 32 | static int aeApiAddEvent(EventLoop *eventLoop, int fd, int mask) 33 | { 34 | aeApiState *state = eventLoop->apidata; 35 | 36 | if (mask & AE_READABLE) FD_SET(fd,&state->rfds); 37 | if (mask & AE_WRITABLE) FD_SET(fd,&state->wfds); 38 | 39 | if (fd > state->maxfd) 40 | { 41 | state->maxfd = fd; 42 | } 43 | return 0; 44 | } 45 | 46 | static int aeApiUpdateEvent(EventLoop *eventLoop, int fd, int mask) 47 | { 48 | return aeApiAddEvent(eventLoop, fd, mask); 49 | } 50 | 51 | static int aeApiDelEvent(EventLoop *eventLoop, int fd) 52 | { 53 | aeApiState *state = eventLoop->apidata; 54 | 55 | FD_CLR(fd,&state->rfds); 56 | FD_CLR(fd,&state->wfds); 57 | return 0; 58 | } 59 | 60 | static int aeApiPoll(EventLoop *eventLoop, struct timeval *tvp) 61 | { 62 | aeApiState *state = eventLoop->apidata; 63 | int retval, j, numevents = 0; 64 | 65 | memcpy(&state->_rfds,&state->rfds,sizeof(fd_set)); 66 | memcpy(&state->_wfds,&state->wfds,sizeof(fd_set)); 67 | 68 | retval = select(state->maxfd+1, 69 | &state->_rfds,&state->_wfds,NULL,tvp); 70 | if (retval > 0) 71 | { 72 | for (j = 0; j <= state->maxfd; j++) 73 | { 74 | if (FD_ISSET(j,&state->_rfds) || FD_ISSET(j,&state->_wfds)) 75 | { 76 | eventLoop->fired[numevents] = j; 77 | numevents++; 78 | } 79 | } 80 | } 81 | return numevents; 82 | } 83 | 84 | static char *aeApiName(void) 85 | { 86 | return "select"; 87 | } 88 | -------------------------------------------------------------------------------- /src/bitcask.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * 14 | */ 15 | 16 | #ifndef __BITCASK_H__ 17 | #define __BITCASK_H__ 18 | 19 | #include 20 | 21 | #include "record.h" 22 | #include "diskmgr.h" 23 | #include "common.h" 24 | 25 | #include "util.h" 26 | 27 | typedef struct bitcask_t Bitcask; 28 | 29 | Bitcask* bc_open(const char *path, int depth, int pos, time_t before); 30 | Bitcask* bc_open2(Mgr *mgr, int depth, int pos, time_t before); 31 | void bc_scan(Bitcask *bc); 32 | void bc_flush(Bitcask *bc, unsigned int limit, int period); 33 | void bc_close(Bitcask *bc); 34 | void bc_merge(Bitcask *bc); 35 | int bc_optimize(Bitcask *bc, int limit); 36 | DataRecord* bc_get(Bitcask *bc, const char *key, uint32_t *ret_pos, bool return_deleted); 37 | bool bc_set(Bitcask *bc, const char *key, char *value, size_t vlen, int flag, int version); 38 | bool bc_delete(Bitcask *bc, const char *key); 39 | uint16_t bc_get_hash(Bitcask *bc, const char *pos, unsigned int *count); 40 | char* bc_list(Bitcask *bc, const char *pos, const char *prefix); 41 | uint32_t bc_count(Bitcask *bc, uint32_t *curr); 42 | void bc_stat(Bitcask *bc, uint64_t *bytes); 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/clock_gettime_stub.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c), MM Weiss 3 | * All rights reserved. 4 | * 5 | * Redistribution and use in source and binary forms, with or without modification, 6 | * are permitted provided that the following conditions are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright notice, 9 | * this list of conditions and the following disclaimer. 10 | * 11 | * 2. Redistributions in binary form must reproduce the above copyright notice, 12 | * this list of conditions and the following disclaimer in the documentation 13 | * and/or other materials provided with the distribution. 14 | * 15 | * 3. Neither the name of the MM Weiss nor the names of its contributors 16 | * may be used to endorse or promote products derived from this software without 17 | * specific prior written permission. 18 | * 19 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY 20 | * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 22 | * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 24 | * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 26 | * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 | * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 | */ 29 | 30 | /* 31 | * clock_gettime_stub.c 32 | * gcc -Wall -c clock_gettime_stub.c 33 | * posix realtime functions; MacOS user space glue 34 | */ 35 | 36 | /* @comment 37 | * other possible implementation using intel builtin rdtsc 38 | * rdtsc-workaround: http://www.mcs.anl.gov/~kazutomo/rdtsc.html 39 | * 40 | * we could get the ticks by doing this 41 | * 42 | * __asm __volatile("mov %%ebx, %%esi\n\t" 43 | * "cpuid\n\t" 44 | * "xchg %%esi, %%ebx\n\t" 45 | * "rdtsc" 46 | * : "=a" (a), 47 | * "=d" (d) 48 | * ); 49 | 50 | * we could even replace our tricky sched_yield call by assembly code to get a better accurency, 51 | * anyway the following C stub will satisfy 99% of apps using posix clock_gettime call, 52 | * moreover, the setter version (clock_settime) could be easly written using mach primitives: 53 | * http://www.opensource.apple.com/source/xnu/xnu-${VERSION}/osfmk/man/ (clock_[set|get]_time) 54 | * 55 | * hackers don't be crackers, don't you use a flush toilet? 56 | * 57 | * 58 | * @see draft: ./posix-realtime-stub/posix-realtime-stub.c 59 | * 60 | */ 61 | 62 | 63 | #ifdef __APPLE__ 64 | 65 | #pragma weak clock_gettime 66 | 67 | #include 68 | #include 69 | #include 70 | #include 71 | #include 72 | #include 73 | #include 74 | 75 | #ifdef HAVE_CONFIG_H 76 | # include "config.h" 77 | #endif 78 | 79 | #if HAVE_UNISTD_H 80 | #include 81 | #endif 82 | 83 | typedef enum 84 | { 85 | CLOCK_REALTIME, 86 | CLOCK_MONOTONIC, 87 | CLOCK_PROCESS_CPUTIME_ID, 88 | CLOCK_THREAD_CPUTIME_ID 89 | } clockid_t; 90 | 91 | static mach_timebase_info_data_t __clock_gettime_inf; 92 | 93 | int clock_gettime(clockid_t clk_id, struct timespec *tp) 94 | { 95 | kern_return_t ret; 96 | clock_serv_t clk; 97 | clock_id_t clk_serv_id; 98 | mach_timespec_t tm; 99 | 100 | uint64_t start, end, delta, nano; 101 | 102 | task_basic_info_data_t tinfo; 103 | task_thread_times_info_data_t ttinfo; 104 | mach_msg_type_number_t tflag; 105 | 106 | int retval = -1; 107 | switch (clk_id) 108 | { 109 | case CLOCK_REALTIME: 110 | case CLOCK_MONOTONIC: 111 | clk_serv_id = clk_id == CLOCK_REALTIME ? CALENDAR_CLOCK : SYSTEM_CLOCK; 112 | if (KERN_SUCCESS == (ret = host_get_clock_service(mach_host_self(), clk_serv_id, &clk))) 113 | { 114 | if (KERN_SUCCESS == (ret = clock_get_time(clk, &tm))) 115 | { 116 | tp->tv_sec = tm.tv_sec; 117 | tp->tv_nsec = tm.tv_nsec; 118 | retval = 0; 119 | } 120 | } 121 | if (KERN_SUCCESS != ret) 122 | { 123 | errno = EINVAL; 124 | retval = -1; 125 | } 126 | break; 127 | case CLOCK_PROCESS_CPUTIME_ID: 128 | case CLOCK_THREAD_CPUTIME_ID: 129 | start = mach_absolute_time(); 130 | if (clk_id == CLOCK_PROCESS_CPUTIME_ID) 131 | { 132 | getpid(); 133 | } 134 | else 135 | { 136 | sched_yield(); 137 | } 138 | end = mach_absolute_time(); 139 | delta = end - start; 140 | if (0 == __clock_gettime_inf.denom) 141 | { 142 | mach_timebase_info(&__clock_gettime_inf); 143 | } 144 | nano = delta * __clock_gettime_inf.numer / __clock_gettime_inf.denom; 145 | tp->tv_sec = nano * 1e-9; 146 | tp->tv_nsec = nano - (tp->tv_sec * 1e9); 147 | retval = 0; 148 | break; 149 | default: 150 | errno = EINVAL; 151 | retval = -1; 152 | } 153 | return retval; 154 | } 155 | 156 | #endif // __APPLE__ 157 | 158 | /* EOF */ 159 | -------------------------------------------------------------------------------- /src/codec.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #ifndef __CODEC_H__ 18 | #define __CODEC_H__ 19 | 20 | #include "util.h" 21 | 22 | //#define NEW_CODEC 1 23 | 24 | typedef struct 25 | { 26 | unsigned char nargs; 27 | char fmt[7]; 28 | } Fmt; 29 | 30 | typedef struct t_codec 31 | { 32 | size_t dict_size; 33 | Fmt **dict; 34 | size_t rdict_size; 35 | short *rdict; 36 | int dict_used; 37 | } Codec; 38 | 39 | Codec *dc_new(); 40 | void dc_destroy(Codec *dc); 41 | int dc_size(Codec *dc); 42 | int dc_dump(Codec *dc, char *buf, int size); 43 | int dc_load(Codec *dc, const char *buf, int size); 44 | 45 | int dc_encode(Codec *dc, char *buf, int buf_size, const char *src, int len); 46 | int dc_decode(Codec *dc, char *buf, int buf_size, const char *src, int len); 47 | 48 | 49 | #endif 50 | 51 | -------------------------------------------------------------------------------- /src/common.c: -------------------------------------------------------------------------------- 1 | #include"common.h" 2 | int daemon_quit = 0; 3 | struct settings settings = {16}; 4 | void settings_init(void) 5 | { 6 | settings.port = 7900; 7 | /* By default this string should be NULL for getaddrinfo() */ 8 | settings.inter = NULL; 9 | settings.item_buf_size = 4 * 1024; /* default is 4KB */ 10 | settings.maxconns = 1024; /* to limit connections-related memory to about 5MB */ 11 | settings.verbose = 0; 12 | settings.num_threads = 16; 13 | settings.flush_limit = 1024; // 1M 14 | settings.flush_period = 60 * 10; // 10 min 15 | settings.slow_cmd_time = 0.1; // 100ms 16 | settings.max_bucket_size = (uint32_t)(4000 << 20); // 4G 17 | settings.check_file_size = false; 18 | settings.autolink = true; 19 | } 20 | 21 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | #ifndef __COMMON_H__ 2 | #define __COMMON_H__ 3 | #include 4 | 5 | #ifdef HAVE_CONFIG_H 6 | #include "config.h" 7 | #endif 8 | 9 | #ifdef HAVE_STDBOOL_H 10 | # include 11 | #else 12 | # ifndef HAVE__BOOL 13 | # ifdef __cplusplus 14 | typedef bool _Bool; 15 | # else 16 | # define _Bool signed char 17 | # endif 18 | # endif 19 | # define bool _Bool 20 | # define false 0 21 | # define true 1 22 | # define __bool_true_false_are_defined 1 23 | #endif 24 | 25 | 26 | #if HAVE_STDINT_H 27 | # include 28 | #else 29 | typedef unsigned char uint8_t; 30 | #endif 31 | 32 | #if HAVE_UNISTD_H 33 | #include 34 | #endif 35 | 36 | /* 64-bit Portable printf */ 37 | /* printf macros for size_t, in the style of inttypes.h */ 38 | #ifdef _LP64 39 | #define __PRIS_PREFIX "z" 40 | #else 41 | #define __PRIS_PREFIX 42 | #endif 43 | 44 | /* Use these macros after a % in a printf format string 45 | to get correct 32/64 bit behavior, like this: 46 | size_t size = records.size(); 47 | printf("%"PRIuS"\n", size); */ 48 | 49 | #define PRIdS __PRIS_PREFIX "d" 50 | #define PRIxS __PRIS_PREFIX "x" 51 | #define PRIuS __PRIS_PREFIX "u" 52 | #define PRIXS __PRIS_PREFIX "X" 53 | #define PRIoS __PRIS_PREFIX "o" 54 | 55 | 56 | 57 | 58 | 59 | struct settings 60 | { 61 | int num_threads; /* number of libevent threads to run */ 62 | size_t item_buf_size; 63 | int maxconns; 64 | int port; 65 | char *inter; 66 | int verbose; 67 | float slow_cmd_time; 68 | int flush_period; 69 | int flush_limit; 70 | uint32_t max_bucket_size; 71 | bool check_file_size; 72 | bool autolink; 73 | }; 74 | extern int daemon_quit; 75 | extern struct settings settings; 76 | 77 | void settings_init(void); 78 | #endif 79 | -------------------------------------------------------------------------------- /src/const.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | */ 15 | 16 | 17 | #ifndef __CONST_H__ 18 | #define __CONST_H__ 19 | 20 | 21 | #define MAX_PATH_LEN (255) 22 | #define MAX_HOME_PATH_LEN (240) 23 | #define META_BUF_SIZE (256) 24 | #define MAX_KEY_LEN (250) 25 | #define KEY_BUF_LEN (256) 26 | 27 | #define MAX_VALUE_LEN (100* 1024 * 1024) 28 | #define MAX_VALUE_LEN_WARN (50 * 1024 * 1024) 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/crc32.c: -------------------------------------------------------------------------------- 1 | /* Copyright (C) 2002 Red Hat, Inc. 2 | 3 | This program is free software; you can redistribute it and/or modify 4 | it under the terms of the GNU General Public License as published by 5 | the Free Software Foundation, version 2. 6 | 7 | This program is distributed in the hope that it will be useful, 8 | but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | GNU General Public License for more details. 11 | 12 | You should have received a copy of the GNU General Public License 13 | along with this program; if not, write to the Free Software Foundation, 14 | Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ 15 | 16 | #include 17 | 18 | 19 | /* Table computed with Mark Adler's makecrc.c utility. */ 20 | static const uint32_t crc32_table[256] = 21 | { 22 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 23 | 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 24 | 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 25 | 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 26 | 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 27 | 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 28 | 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 29 | 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 30 | 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 31 | 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 32 | 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 33 | 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 34 | 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 35 | 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 36 | 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 37 | 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 38 | 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 39 | 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 40 | 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 41 | 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 42 | 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 43 | 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 44 | 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 45 | 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 46 | 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 47 | 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 48 | 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 49 | 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 50 | 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 51 | 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 52 | 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 53 | 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 54 | 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 55 | 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 56 | 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 57 | 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 58 | 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 59 | 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 60 | 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 61 | 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 62 | 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 63 | 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 64 | 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 65 | 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 66 | 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 67 | 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 68 | 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 69 | 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 70 | 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 71 | 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 72 | 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 73 | 0x2d02ef8d 74 | }; 75 | 76 | uint32_t 77 | crc32 (uint32_t crc, unsigned char *buf, size_t len) 78 | { 79 | unsigned char *end; 80 | 81 | crc = ~crc; 82 | for (end = buf + len; buf < end; ++buf) 83 | crc = crc32_table[(crc ^ *buf) & 0xff] ^ (crc >> 8); 84 | return ~crc; 85 | } 86 | -------------------------------------------------------------------------------- /src/diskmgr.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | */ 15 | 16 | #include 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "diskmgr.h" 24 | 25 | #ifdef HAVE_CONFIG_H 26 | # include "config.h" 27 | #endif 28 | 29 | #if HAVE_UNISTD_H 30 | #include 31 | #endif 32 | 33 | #include "const.h" 34 | #include "log.h" 35 | 36 | ssize_t mgr_readlink(const char *path, char *buf, size_t bufsiz) 37 | { 38 | int n = readlink(path, buf, bufsiz); 39 | if (n < 0) 40 | { 41 | log_fatal("readlink fail %s", path); 42 | return -1; 43 | } 44 | buf[n] = 0; 45 | if (strncmp(simple_basename(path), simple_basename(buf), bufsiz) != 0 ) 46 | { 47 | log_fatal("basename not match %s->%s", path, buf); 48 | return -2; 49 | } 50 | return n; 51 | } 52 | 53 | int mgr_getrealpath(const char *path, char *buf, size_t bufsiz) 54 | { 55 | struct stat sb; 56 | if (stat(path, &sb) !=0) 57 | return -1; 58 | 59 | if ((sb.st_mode & S_IFMT) == S_IFLNK) 60 | { 61 | if (mgr_readlink(path, buf, bufsiz) <= 0) 62 | return -1; 63 | } 64 | else 65 | strcpy(buf, path); 66 | return 0; 67 | } 68 | 69 | Mgr *mgr_create(const char **disks, int ndisks) 70 | { 71 | char *cwd = getcwd(NULL, 0); 72 | Mgr *mgr = (Mgr*) safe_malloc(sizeof(Mgr)); 73 | mgr->ndisks = ndisks; 74 | mgr->disks = (char**)safe_malloc(sizeof(char*) * ndisks); 75 | int i; 76 | for (i = 0; i < ndisks; i++) 77 | { 78 | if (0 != access(disks[i], F_OK) && 0 != mkdir(disks[i], 0755)) 79 | { 80 | log_error("access %s failed", disks[i]); 81 | free(mgr->disks); 82 | free(mgr); 83 | free(cwd); 84 | return NULL; 85 | } 86 | if (disks[i][0] == '/') 87 | { 88 | mgr->disks[i] = strdup(disks[i]); 89 | } 90 | else 91 | { 92 | mgr->disks[i] = (char*)safe_malloc(strlen(disks[i]) + strlen(cwd) + 2); 93 | sprintf(mgr->disks[i], "%s/%s", cwd, disks[i]); //safe 94 | } 95 | } 96 | free(cwd); 97 | return mgr; 98 | } 99 | 100 | void mgr_destroy(Mgr *mgr) 101 | { 102 | int i=0; 103 | for (i = 0; i< mgr->ndisks; i++) 104 | { 105 | free(mgr->disks[i]); 106 | } 107 | free(mgr->disks); 108 | free(mgr); 109 | } 110 | 111 | const char *mgr_base(Mgr *mgr) 112 | { 113 | return mgr->disks[0]; 114 | } 115 | 116 | static uint64_t 117 | get_disk_avail(const char *path, uint64_t *total) 118 | { 119 | struct statvfs stat; 120 | int r = statvfs(path, &stat); 121 | if (r != 0) 122 | { 123 | return 0ULL; 124 | } 125 | if (total != NULL) 126 | { 127 | *total = stat.f_blocks * stat.f_frsize; 128 | } 129 | return stat.f_bavail * stat.f_frsize; 130 | } 131 | 132 | const char *mgr_alloc(Mgr *mgr, const char *name) 133 | { 134 | if (mgr->ndisks == 1) 135 | { 136 | return mgr->disks[0]; 137 | } 138 | uint64_t maxa= 0; 139 | int maxi = 0, i; 140 | char path[MAX_PATH_LEN]; 141 | struct stat sb; 142 | for (i = 0; i< mgr->ndisks; i++) 143 | { 144 | safe_snprintf(path, MAX_PATH_LEN, "%s/%s", mgr->disks[i], name); 145 | if (lstat(path, &sb) == 0 && (sb.st_mode & S_IFMT) == S_IFREG) 146 | { 147 | return mgr->disks[i]; 148 | } 149 | uint64_t avail = get_disk_avail(mgr->disks[i], NULL); 150 | if (avail > maxa || (avail == maxa && (rand() & 1) == 1) ) 151 | { 152 | maxa = avail; 153 | maxi = i; 154 | } 155 | } 156 | if (maxi != 0) 157 | { 158 | // create symlink 159 | char target[MAX_PATH_LEN]; 160 | safe_snprintf(target, MAX_PATH_LEN, "%s/%s", mgr->disks[maxi], name); 161 | safe_snprintf(path, MAX_PATH_LEN, "%s/%s", mgr->disks[0], name); 162 | if (lstat(path, &sb) == 0) 163 | { 164 | unlink(path); 165 | } 166 | if (symlink(target, path) != 0) 167 | { 168 | log_fatal("create symlink failed: %s -> %s", path, target); 169 | exit(1); 170 | } 171 | } 172 | return mgr->disks[maxi]; 173 | } 174 | 175 | void _mgr_unlink(const char *path, const char *file, int line, const char *func) 176 | { 177 | struct stat sb; 178 | if (0 != lstat(path, &sb)) 179 | return; 180 | log_notice("mgr_unlink %s, in %s (%s:%i)", path, func, file, line); 181 | if ((sb.st_mode & S_IFMT) == S_IFLNK) 182 | { 183 | char buf[MAX_PATH_LEN]; 184 | int n = mgr_readlink(path, buf, MAX_PATH_LEN); 185 | if (n > 0) 186 | { 187 | unlink(buf); 188 | } 189 | } 190 | unlink(path); 191 | } 192 | 193 | 194 | //caller guarantee newpath not exist 195 | void mgr_rename(const char *oldpath, const char *newpath) 196 | { 197 | log_notice("mgr_rename %s -> %s", oldpath, newpath); 198 | struct stat sb; 199 | char ropath[MAX_PATH_LEN]; 200 | char rnpath[MAX_PATH_LEN]; 201 | if (lstat(oldpath, &sb) == 0 && (sb.st_mode & S_IFMT) == S_IFLNK) 202 | { 203 | int n = mgr_readlink(oldpath, ropath, MAX_PATH_LEN); 204 | if (n > 0) 205 | { 206 | char *ropath_dup = strdup(ropath); 207 | safe_snprintf(rnpath, MAX_PATH_LEN, "%s/%s", dirname(ropath_dup), simple_basename(newpath)); 208 | free(ropath_dup); 209 | 210 | if (symlink(rnpath, newpath) != 0) 211 | { 212 | log_fatal("symlink failed: %s -> %s, err: %s, exit!", rnpath, newpath, strerror(errno)); 213 | exit(-1); 214 | } 215 | log_notice("mgr_rename real %s -> %s", ropath, rnpath); 216 | if (rename(ropath, rnpath) != 0) 217 | { 218 | log_error("rename failed: %s -> %s, err: %s, exit!", ropath, rnpath, strerror(errno)); 219 | exit(-1); 220 | }; 221 | unlink(oldpath); 222 | } 223 | } 224 | else 225 | { 226 | if (rename(oldpath, newpath) != 0) 227 | { 228 | log_error("rename failed: %s -> %s, err:%s, exit!", oldpath, newpath, strerror(errno)); 229 | exit(-1); 230 | }; 231 | } 232 | } 233 | 234 | void mgr_stat(Mgr *mgr, uint64_t *total, uint64_t *avail) 235 | { 236 | int i=0; 237 | *total = 0; 238 | *avail = 0; 239 | for (i = 0; i< mgr->ndisks; i++) 240 | { 241 | uint64_t t; 242 | *avail += get_disk_avail(mgr->disks[i], &t); 243 | *total += t; 244 | } 245 | } 246 | -------------------------------------------------------------------------------- /src/diskmgr.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | */ 15 | 16 | #ifndef __DISKMGR_H__ 17 | #define __DISKMGR_H__ 18 | 19 | #include 20 | #include "util.h" 21 | 22 | typedef struct disk_mgr 23 | { 24 | char **disks; 25 | int ndisks; 26 | } Mgr; 27 | 28 | Mgr *mgr_create(const char **disks, int ndisks); 29 | void mgr_destroy(Mgr *mgr); 30 | ssize_t mgr_readlink(const char *path, char *buf, size_t bufsiz); 31 | 32 | const char *mgr_base(Mgr *mgr); 33 | const char *mgr_alloc(Mgr *mgr, const char *path); 34 | 35 | #define mgr_unlink(X) _mgr_unlink(X, __FILE__, __LINE__, __FUNCTION__) 36 | void _mgr_unlink(const char *path, const char *file, int line, const char *func); 37 | 38 | void mgr_rename(const char *oldpath, const char *newpath); 39 | 40 | void mgr_stat(Mgr *mgr, uint64_t *total, uint64_t *avail); 41 | 42 | static inline char *simple_basename(const char *path) 43 | { 44 | char *p = (char*)path + strlen(path); 45 | while (*p != '/' && p >= path) 46 | --p; 47 | return ++p; 48 | } 49 | 50 | int mgr_getrealpath(const char *path, char *buf, size_t bufsiz); 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/fnv1a.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * 14 | */ 15 | 16 | #ifndef __FNV1A_H__ 17 | #define __FNV1A_H__ 18 | 19 | #define FNV_32_PRIME 0x01000193 20 | #define FNV_32_INIT 0x811c9dc5 21 | 22 | typedef unsigned int uint32_t; 23 | inline static uint32_t fnv1a(const char *key, int key_len) 24 | { 25 | uint32_t h = FNV_32_INIT; 26 | int i; 27 | 28 | for (i=0; i 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "hint.h" 24 | #include "quicklz.h" 25 | #include "diskmgr.h" 26 | #include "fnv1a.h" 27 | #include "const.h" 28 | 29 | #ifdef HAVE_CONFIG_H 30 | # include "config.h" 31 | #endif 32 | 33 | 34 | #if HAVE_UNISTD_H 35 | #include 36 | #endif 37 | 38 | #include "mfile.h" 39 | #include "log.h" 40 | 41 | // for build hint 42 | struct param 43 | { 44 | int size; 45 | int curr; 46 | char *buf; 47 | }; 48 | 49 | void collect_items(Item *it, void *param) 50 | { 51 | int ksize = strlen(it->key); 52 | int length = sizeof(HintRecord) + ksize + 1 - NAME_IN_RECORD; 53 | struct param *p = (struct param *)param; 54 | if (p->size - p->curr < length) 55 | { 56 | p->size *= 2; 57 | p->buf = (char*)safe_realloc(p->buf, p->size); 58 | } 59 | 60 | HintRecord *r = (HintRecord*)(p->buf + p->curr); 61 | r->ksize = ksize; 62 | r->pos = it->pos >> 8; 63 | r->version = it->ver; 64 | r->hash = it->hash; 65 | // TODO: check ir 66 | safe_memcpy(r->key, p->size - p->curr - sizeof(HintRecord) + NAME_IN_RECORD, it->key, r->ksize + 1); 67 | 68 | p->curr += length; 69 | } 70 | 71 | void write_hint_file(char *buf, int size, const char *path) 72 | { 73 | // compress 74 | char *dst = buf; 75 | if (strcmp(path + strlen(path) - 4, ".qlz") == 0) 76 | { 77 | char *wbuf = (char*)safe_malloc(QLZ_SCRATCH_COMPRESS); 78 | dst = (char*)safe_malloc(size + 400); 79 | size = qlz_compress(buf, dst, size, wbuf); 80 | free(wbuf); 81 | } 82 | 83 | char tmp[MAX_PATH_LEN]; 84 | safe_snprintf(tmp, MAX_PATH_LEN, "%s.tmp", path); 85 | FILE *hf = fopen(tmp, "wb"); 86 | if (NULL == hf) 87 | { 88 | log_error("open %s failed", tmp); 89 | return; 90 | } 91 | int n = fwrite(dst, 1, size, hf); 92 | fclose(hf); 93 | if (dst != buf) free(dst); 94 | 95 | if (n == size) 96 | { 97 | mgr_unlink(path); 98 | mgr_rename(tmp, path); 99 | } 100 | else 101 | { 102 | log_error("write to %s failed", tmp); 103 | } 104 | } 105 | 106 | void build_hint(HTree *tree, const char *hintpath) 107 | { 108 | struct param p; 109 | p.size = 1024 * 1024; 110 | p.curr = 0; 111 | p.buf = (char*)safe_malloc(p.size); 112 | 113 | ht_visit(tree, collect_items, &p); 114 | ht_destroy(tree); 115 | 116 | write_hint_file(p.buf, p.curr, hintpath); 117 | free(p.buf); 118 | } 119 | 120 | HintFile *open_hint(const char *path, const char *new_path) 121 | { 122 | MFile *f = open_mfile(path); 123 | if (f == NULL) 124 | { 125 | return NULL; 126 | } 127 | 128 | HintFile *hint = (HintFile*) safe_malloc(sizeof(HintFile)); 129 | hint->f = f; 130 | hint->buf = f->addr; 131 | hint->size = f->size; 132 | 133 | if (strcmp(path + strlen(path) - 4, ".qlz") == 0 && hint->size > 0) 134 | { 135 | char wbuf[QLZ_SCRATCH_DECOMPRESS]; 136 | int size = qlz_size_decompressed(hint->buf); 137 | char *buf = (char*)safe_malloc(size); 138 | int vsize = qlz_decompress(hint->buf, buf, wbuf); 139 | if (vsize != size) 140 | { 141 | log_fatal("decompress %s failed: %d < %d, remove it\n", path, vsize, size); 142 | mgr_unlink(path); 143 | exit(1); 144 | } 145 | hint->size = size; 146 | hint->buf = buf; 147 | } 148 | 149 | if (new_path != NULL) 150 | { 151 | write_hint_file(hint->buf, hint->size, new_path); 152 | } 153 | 154 | return hint; 155 | } 156 | 157 | void close_hint(HintFile *hint) 158 | { 159 | if (hint->buf != hint->f->addr && hint->buf != NULL) 160 | { 161 | free(hint->buf); 162 | } 163 | close_mfile(hint->f); 164 | free(hint); 165 | } 166 | 167 | void scanHintFile(HTree *tree, int bucket, const char *path, const char *new_path) 168 | { 169 | HintFile *hint = open_hint(path, new_path); 170 | if (hint == NULL) return; 171 | 172 | log_notice("scan hint: %s", path); 173 | 174 | char *p = hint->buf, *end = hint->buf + hint->size; 175 | while (p < end) 176 | { 177 | HintRecord *r = (HintRecord*) p; 178 | p += sizeof(HintRecord) - NAME_IN_RECORD + r->ksize + 1; 179 | if (p > end) 180 | { 181 | log_error("scan %s: unexpected end, need %ld byte", path, p - end); 182 | break; 183 | } 184 | uint32_t pos = (r->pos << 8) | (bucket & 0xff); 185 | if (check_key(r->key, r->ksize)) 186 | { 187 | if (r->version > 0) 188 | ht_add2(tree, r->key, r->ksize, pos, r->hash, r->version); 189 | else 190 | ht_remove2(tree, r->key, r->ksize); 191 | } 192 | } 193 | 194 | close_hint(hint); 195 | } 196 | 197 | int count_deleted_record(HTree *tree, int bucket, const char *path, int *total, bool skipped) 198 | { 199 | *total = 0; 200 | HintFile *hint = open_hint(path, NULL); 201 | if (hint == NULL) 202 | return 0; 203 | 204 | char *p = hint->buf, *end = hint->buf + hint->size; 205 | int deleted = 0; 206 | while (p < end) 207 | { 208 | HintRecord *r = (HintRecord*) p; 209 | p += sizeof(HintRecord) - NAME_IN_RECORD + r->ksize + 1; 210 | if (p > end) 211 | { 212 | log_error("scan %s: unexpected end, need %ld byte", path, p - end); 213 | break; 214 | } 215 | (*total)++; 216 | Item *it = ht_get2(tree, r->key, r->ksize); 217 | //key not exist || not used || (used && deleted && not skipped) 218 | if (it == NULL || it->pos != ((r->pos << 8) | (unsigned int)bucket) || (it->ver <= 0 && !skipped)) 219 | { 220 | deleted++; 221 | } 222 | if (it) free(it); 223 | } 224 | 225 | close_hint(hint); 226 | return deleted; 227 | } 228 | -------------------------------------------------------------------------------- /src/hint.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #ifndef __HINT_H__ 18 | #define __HINT_H__ 19 | 20 | #include "mfile.h" 21 | #include "htree.h" 22 | #include "util.h" 23 | 24 | #define NAME_IN_RECORD 2 25 | 26 | typedef struct hint_record 27 | { 28 | uint32_t ksize:8; 29 | uint32_t pos:24; 30 | int32_t version; 31 | uint16_t hash; 32 | char key[NAME_IN_RECORD]; // allign 33 | } HintRecord; 34 | 35 | typedef struct 36 | { 37 | MFile *f; 38 | size_t size; 39 | char *buf; 40 | } HintFile; 41 | 42 | HintFile *open_hint(const char *path, const char *new_path); 43 | void close_hint(HintFile *hint); 44 | void scanHintFile(HTree *tree, int bucket, const char *path, const char *new_path); 45 | void build_hint(HTree *tree, const char *path); 46 | void write_hint_file(char *buf, int size, const char *path); 47 | int count_deleted_record(HTree *tree, int bucket, const char *path, int *total, bool skipped); 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /src/hstore.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | */ 14 | 15 | #ifndef __HSTORE_H__ 16 | #define __HSTORE_H__ 17 | 18 | #include 19 | #include 20 | 21 | #include "util.h" 22 | 23 | typedef struct t_hstore HStore; 24 | 25 | HStore* hs_open(char *path, int height, time_t before, int scan_threads); 26 | void hs_flush(HStore *store, unsigned int limit, int period); 27 | void hs_close(HStore *store); 28 | char* hs_get(HStore *store, char *key, unsigned int *vlen, uint32_t *flag); 29 | bool hs_set(HStore *store, char *key, char *value, unsigned int vlen, uint32_t flag, int version); 30 | bool hs_append(HStore *store, char *key, char *value, unsigned int vlen); 31 | int64_t hs_incr(HStore *store, char *key, int64_t value); 32 | bool hs_delete(HStore *store, char *key); 33 | uint64_t hs_count(HStore *store, uint64_t *curr); 34 | void hs_stat(HStore *store, uint64_t *total, uint64_t *avail); 35 | int hs_optimize(HStore *store, long limit, char *tree); 36 | int hs_optimize_stat(HStore *store); 37 | #endif 38 | -------------------------------------------------------------------------------- /src/htree.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | */ 15 | #ifndef __HTREE_H__ 16 | #define __HTREE_H__ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include "util.h" 24 | 25 | typedef struct t_item Item; 26 | struct t_item 27 | { 28 | uint32_t pos; 29 | int32_t ver; 30 | uint16_t hash; 31 | uint8_t ksz; 32 | char key[1]; 33 | }; 34 | 35 | #define ITEM_PADDING 1 36 | 37 | typedef struct t_hash_tree HTree; 38 | typedef void (*fun_visitor) (Item *it, void *param); 39 | 40 | HTree* ht_new(int depth, int pos, bool tmp); 41 | void ht_destroy(HTree *tree); 42 | void ht_add(HTree *tree, const char *key, uint32_t pos, uint16_t hash, int32_t ver); 43 | void ht_remove(HTree *tree, const char *key); 44 | Item* ht_get(HTree *tree, const char *key); 45 | Item* ht_get2(HTree *tree, const char *key, int ksz); 46 | uint32_t ht_get_hash(HTree *tree, const char *key, unsigned int *count); 47 | char* ht_list(HTree *tree, const char *dir, const char *prefix); 48 | void ht_visit(HTree *tree, fun_visitor visitor, void *param); 49 | 50 | HTree* ht_open(int depth, int pos, const char *path); 51 | int ht_save(HTree *tree, const char *path); 52 | 53 | void ht_set_updating_bucket(HTree *tree, int bucket, HTree *updating_tree); 54 | Item* ht_get_maybe_tmp(HTree *tree, const char *key, int *is_tmp, char *buf); 55 | Item* ht_get_withbuf(HTree *tree, const char *key, int len, char *buf, bool lock); 56 | 57 | // not thread safe 58 | void ht_add2(HTree *tree, const char *key, int ksz, uint32_t pos, uint16_t hash, int32_t ver); 59 | void ht_remove2(HTree *tree, const char *key, int ksz); 60 | void ht_visit2(HTree *tree, fun_visitor visitor, void *param); 61 | 62 | bool check_key(const char* key, int len); 63 | 64 | #endif /* __HTREE_H__ */ 65 | -------------------------------------------------------------------------------- /src/item.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #include "beansdb.h" 18 | #include "hstore.h" 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | #include 27 | 28 | #if HAVE_UNISTD_H 29 | #include 30 | #endif 31 | 32 | 33 | #include "util.h" 34 | #include "log.h" 35 | 36 | #define MAX_ITEM_FREELIST_LENGTH 4000 37 | #define INIT_ITEM_FREELIST_LENGTH 500 38 | 39 | static size_t item_make_header(const uint8_t nkey, const int flags, const int nbytes, char *suffix, uint8_t *nsuffix); 40 | 41 | static item **freeitem; 42 | static int freeitemtotal; 43 | static int freeitemcurr; 44 | 45 | extern HStore *store; 46 | 47 | void item_init(void) 48 | { 49 | freeitemtotal = INIT_ITEM_FREELIST_LENGTH; 50 | freeitemcurr = 0; 51 | 52 | freeitem = (item **)safe_malloc(sizeof(item *) * freeitemtotal); 53 | return; 54 | } 55 | 56 | /* 57 | * Returns a item buffer from the freelist, if any. Sholud call 58 | * item_from_freelist for thread safty. 59 | * */ 60 | item *do_item_from_freelist(void) 61 | { 62 | item *s; 63 | 64 | if (freeitemcurr > 0) 65 | { 66 | s = freeitem[--freeitemcurr]; 67 | } 68 | else 69 | { 70 | /* If try_malloc fails, let the logic fall through without spamming 71 | * STDERR on the server. */ 72 | s = (item *)try_malloc(settings.item_buf_size); 73 | if (s) memset(s, 0, settings.item_buf_size); 74 | } 75 | 76 | return s; 77 | } 78 | 79 | /* 80 | * Adds a item to the freelist. Should call 81 | * item_add_to_freelist for thread safty. 82 | */ 83 | int do_item_add_to_freelist(item *it) 84 | { 85 | if (freeitemcurr < freeitemtotal) 86 | { 87 | freeitem[freeitemcurr++] = it; 88 | return 0; 89 | } 90 | else 91 | { 92 | if (freeitemtotal >= MAX_ITEM_FREELIST_LENGTH) 93 | { 94 | return 1; 95 | } 96 | /* try to enlarge free item buffer array */ 97 | item **new_freeitem = (item **)try_realloc(freeitem, sizeof(item *) * freeitemtotal * 2); 98 | if (new_freeitem) 99 | { 100 | freeitemtotal *= 2; 101 | log_notice("freeitemtotal doubled to %d", freeitemtotal); 102 | freeitem = new_freeitem; 103 | freeitem[freeitemcurr++] = it; 104 | return 0; 105 | } 106 | } 107 | return 1; 108 | } 109 | 110 | /** 111 | * Generates the variable-sized part of the header for an object. 112 | * 113 | * key - The key 114 | * nkey - The length of the key 115 | * flags - key flags 116 | * nbytes - Number of bytes to hold value and addition CRLF terminator 117 | * suffix - Buffer for the "VALUE" line suffix (flags, size). 118 | * nsuffix - The length of the suffix is stored here. 119 | * 120 | * Returns the total size of the header. 121 | */ 122 | static size_t item_make_header(const uint8_t nkey, const int flags, const int nbytes, 123 | char *suffix, uint8_t *nsuffix) 124 | { 125 | /* suffix is defined at 40 chars elsewhere.. */ 126 | *nsuffix = (uint8_t)safe_snprintf(suffix, 40, " %d %d\r\n", flags, nbytes - 2); 127 | return sizeof(item) + nkey + *nsuffix + nbytes; 128 | } 129 | 130 | /* 131 | * alloc a item buffer, and init it. 132 | */ 133 | item *item_alloc1(char *key, const size_t nkey, const int flags, const int nbytes) 134 | { 135 | uint8_t nsuffix; 136 | item *it; 137 | char suffix[40]; 138 | size_t ntotal = item_make_header(nkey + 1, flags, nbytes, suffix, &nsuffix); 139 | 140 | if (ntotal > settings.item_buf_size) 141 | { 142 | it = (item *)try_malloc(ntotal); 143 | if (it == NULL) 144 | { 145 | return NULL; 146 | } 147 | memset(it, 0, ntotal); 148 | if (settings.verbose > 1) 149 | { 150 | log_debug("alloc a item buffer from try_malloc."); 151 | } 152 | } 153 | else 154 | { 155 | it = item_from_freelist(); 156 | if (it == NULL) 157 | { 158 | return NULL; 159 | } 160 | if (settings.verbose > 1) 161 | { 162 | log_debug("alloc a item buffer from freelist."); 163 | } 164 | } 165 | 166 | it->nkey = nkey; 167 | it->nbytes = nbytes; 168 | strcpy(ITEM_key(it), key); 169 | memcpy(ITEM_suffix(it), suffix, (size_t)nsuffix); // safe 170 | it->nsuffix = nsuffix; 171 | return it; 172 | } 173 | 174 | /* 175 | * free a item buffer. here 'it' must be a full item. 176 | */ 177 | 178 | int item_free(item *it) 179 | { 180 | size_t ntotal = 0; 181 | if (NULL == it) 182 | return 0; 183 | 184 | /* ntotal may be wrong, if 'it' is not a full item. */ 185 | ntotal = ITEM_ntotal(it); 186 | if (ntotal > settings.item_buf_size) 187 | { 188 | if (settings.verbose > 1) 189 | { 190 | log_error("ntotal: %"PRIuS", use free() directly.", ntotal); 191 | } 192 | free(it); 193 | } 194 | else 195 | { 196 | if (0 != item_add_to_freelist(it)) 197 | { 198 | if (settings.verbose > 1) 199 | { 200 | log_error("ntotal: %"PRIuS", add a item buffer to freelist fail, use free() directly.", ntotal); 201 | } 202 | free(it); 203 | } 204 | else 205 | { 206 | if (settings.verbose > 1) 207 | { 208 | log_error("ntotal: %"PRIuS", add a item buffer to freelist.", ntotal); 209 | } 210 | } 211 | } 212 | return 0; 213 | } 214 | 215 | /* if return item is not NULL, free by caller */ 216 | item *item_get(char *key, unsigned int nkey) 217 | { 218 | item *it = NULL; 219 | unsigned int vlen; 220 | uint32_t flag; 221 | char *value = hs_get(store, key, &vlen, &flag); 222 | if (value) 223 | { 224 | it = item_alloc1(key, nkey, flag, vlen + 2); 225 | if (it) 226 | { 227 | memcpy(ITEM_data(it), value, vlen); // safe 228 | memcpy(ITEM_data(it) + vlen, "\r\n", 2); // safe 229 | } 230 | free(value); 231 | } 232 | return it; 233 | } 234 | -------------------------------------------------------------------------------- /src/log.c: -------------------------------------------------------------------------------- 1 | #include "log.h" 2 | 3 | zlog_category_t *cat; 4 | 5 | int log_init(const char *conf_path) 6 | { 7 | int rc = zlog_init(conf_path); 8 | if (rc) 9 | { 10 | fprintf(stderr, "init log file %s failed, please check zlog user guide!\n", conf_path); 11 | return -1; 12 | } 13 | if (!(cat = zlog_get_category("beansdb"))) 14 | { 15 | fprintf(stderr, "fail to find category beansdb in %s!\n", conf_path); 16 | zlog_fini(); 17 | return -1; 18 | } 19 | return 0; 20 | } 21 | 22 | void log_finish() 23 | { 24 | zlog_fini(); 25 | return; 26 | } 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/log.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Hurricane Lee 13 | */ 14 | 15 | #ifndef __LOG_H__ 16 | #define __LOG_H__ 17 | 18 | #include 19 | 20 | #include "zlog.h" 21 | 22 | extern zlog_category_t *cat; 23 | 24 | int log_init(const char *conf_path); 25 | void log_finish(); 26 | #define log_fatal(FORMAT, ...) zlog_fatal(cat, FORMAT, ##__VA_ARGS__) 27 | #define log_error(FORMAT, ...) zlog_error(cat, FORMAT, ##__VA_ARGS__) 28 | #define log_warn(FORMAT, ...) zlog_warn(cat, FORMAT, ##__VA_ARGS__) 29 | #define log_notice(FORMAT, ...) zlog_notice(cat, FORMAT, ##__VA_ARGS__) 30 | #define log_info(FORMAT, ...) zlog_info(cat, FORMAT, ##__VA_ARGS__) 31 | #define log_debug(FORMAT, ...) zlog_debug(cat, FORMAT, ##__VA_ARGS__) 32 | #endif 33 | -------------------------------------------------------------------------------- /src/mfile.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #ifdef HAVE_CONFIG_H 5 | #include "config.h" 6 | #endif 7 | 8 | #if HAVE_UNISTD_H 9 | #include 10 | #endif 11 | 12 | #include "mfile.h" 13 | #include "log.h" 14 | #include "util.h" 15 | 16 | const int MAX_MMAP_SIZE = 1<<12; // 4G 17 | static int curr_mmap_size = 0; 18 | static pthread_mutex_t mmap_lock = PTHREAD_MUTEX_INITIALIZER; 19 | 20 | MFile *open_mfile(const char *path) 21 | { 22 | int fd = open(path, O_RDONLY); 23 | if (fd == -1) 24 | { 25 | log_error("open mfile %s failed", path); 26 | return NULL; 27 | } 28 | 29 | struct stat sb; 30 | if (fstat(fd, &sb) == -1) 31 | { 32 | close(fd); 33 | return NULL; 34 | } 35 | #if _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L 36 | posix_fadvise(fd, 0, sb.st_size, POSIX_FADV_SEQUENTIAL); 37 | #endif 38 | 39 | pthread_mutex_lock(&mmap_lock); 40 | int mb = sb.st_size >> 20; 41 | while (curr_mmap_size + mb > MAX_MMAP_SIZE && mb > 100) 42 | { 43 | pthread_mutex_unlock(&mmap_lock); 44 | sleep(5); 45 | pthread_mutex_lock(&mmap_lock); 46 | } 47 | curr_mmap_size += mb; 48 | pthread_mutex_unlock(&mmap_lock); 49 | 50 | MFile *f = (MFile*) safe_malloc(sizeof(MFile)); 51 | f->fd = fd; 52 | f->size = sb.st_size; 53 | 54 | if (f->size > 0) 55 | { 56 | f->addr = (char*) mmap(NULL, sb.st_size, PROT_READ, MAP_PRIVATE, fd, 0); 57 | if (f->addr == MAP_FAILED) 58 | { 59 | log_error("mmap failed %s", path); 60 | close(fd); 61 | pthread_mutex_lock(&mmap_lock); 62 | curr_mmap_size -= mb; 63 | pthread_mutex_unlock(&mmap_lock); 64 | free(f); 65 | return NULL; 66 | } 67 | 68 | if (madvise(f->addr, sb.st_size, MADV_SEQUENTIAL) < 0) 69 | { 70 | log_error("Unable to madvise() region %p", f->addr); 71 | } 72 | } 73 | else 74 | { 75 | f->addr = NULL; 76 | } 77 | 78 | return f; 79 | } 80 | 81 | void close_mfile(MFile *f) 82 | { 83 | if (f->addr) 84 | { 85 | madvise(f->addr, f->size, MADV_DONTNEED); 86 | munmap(f->addr, f->size); 87 | } 88 | #if _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L 89 | posix_fadvise(f->fd, 0, f->size, POSIX_FADV_DONTNEED); 90 | #endif 91 | close(f->fd); 92 | pthread_mutex_lock(&mmap_lock); 93 | curr_mmap_size -= f->size >> 20; 94 | pthread_mutex_unlock(&mmap_lock); 95 | free(f); 96 | } 97 | 98 | 99 | -------------------------------------------------------------------------------- /src/mfile.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #ifndef __MFILE_H__ 18 | #define __MFILE_H__ 19 | 20 | #include 21 | #include 22 | 23 | typedef struct 24 | { 25 | int fd; 26 | size_t size; 27 | char *addr; 28 | } MFile; 29 | 30 | MFile *open_mfile(const char *path); 31 | void close_mfile(MFile *f); 32 | static inline void mfile_dontneed(MFile *f, size_t pos, size_t *last_advise) { 33 | if (pos - *last_advise > (64<<20)) 34 | { 35 | madvise(f->addr, pos, MADV_DONTNEED); 36 | #if _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L 37 | posix_fadvise(f->fd, 0, pos, POSIX_FADV_DONTNEED); 38 | #endif 39 | *last_advise = pos; 40 | } 41 | } 42 | static inline void file_dontneed(int fd, size_t pos, size_t *last_advise) { 43 | if (pos - *last_advise > (8<<20)) 44 | { 45 | #if _XOPEN_SOURCE >= 600 || _POSIX_C_SOURCE >= 200112L 46 | posix_fadvise(fd, 0, pos - (64<<10), POSIX_FADV_DONTNEED); 47 | #endif 48 | *last_advise = pos; 49 | } 50 | } 51 | 52 | #endif 53 | -------------------------------------------------------------------------------- /src/quicklz.h: -------------------------------------------------------------------------------- 1 | #ifndef QLZ_HEADER 2 | #define QLZ_HEADER 3 | 4 | // Fast data compression library 5 | // Copyright (C) 2006-2010 Lasse Mikkel Reinhold 6 | // lar@quicklz.com 7 | // 8 | // QuickLZ can be used for free under the GPL-1, -2 or -3 license (where anything 9 | // released into public must be open source) or under a commercial license if such 10 | // has been acquired (see http://www.quicklz.com/order.html). The commercial license 11 | // does not cover derived or ported versions created by third parties under GPL. 12 | 13 | // Version 1.4.1 final - april 2010 14 | 15 | // You can edit following user settings. Data must be decompressed with the same 16 | // setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed 17 | // (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially 18 | // zeroed out (see manual). First #ifndef makes it possible to define settings from 19 | // the outside like the compiler command line or from higher level code. 20 | 21 | #ifndef QLZ_COMPRESSION_LEVEL 22 | //#define QLZ_COMPRESSION_LEVEL 1 23 | //#define QLZ_COMPRESSION_LEVEL 2 24 | #define QLZ_COMPRESSION_LEVEL 3 25 | 26 | #define QLZ_STREAMING_BUFFER 0 27 | //#define QLZ_STREAMING_BUFFER 100000 28 | //#define QLZ_STREAMING_BUFFER 1000000 29 | 30 | //#define QLZ_MEMORY_SAFE 31 | #endif 32 | 33 | #define QLZ_VERSION_MAJOR 1 34 | #define QLZ_VERSION_MINOR 4 35 | #define QLZ_VERSION_REVISION 1 36 | 37 | // Using size_t, memset() and memcpy() 38 | #include 39 | 40 | // Public functions of QuickLZ 41 | size_t qlz_size_decompressed(const char *source); 42 | size_t qlz_size_compressed(const char *source); 43 | size_t qlz_decompress(const char *source, void *destination, char *scratch_decompress); 44 | size_t qlz_compress(const void *source, char *destination, size_t size, char *scratch_compress); 45 | int qlz_get_setting(int setting); 46 | 47 | // Verify compression level 48 | #if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 49 | #error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 50 | #endif 51 | 52 | // Compute QLZ_SCRATCH_COMPRESS and QLZ_SCRATCH_DECOMPRESS 53 | #if QLZ_COMPRESSION_LEVEL == 1 54 | #define QLZ_POINTERS 1 55 | #define QLZ_HASH_VALUES 4096 56 | #elif QLZ_COMPRESSION_LEVEL == 2 57 | #define QLZ_POINTERS 4 58 | #define QLZ_HASH_VALUES 2048 59 | #elif QLZ_COMPRESSION_LEVEL == 3 60 | #define QLZ_POINTERS 16 61 | #define QLZ_HASH_VALUES 4096 62 | #endif 63 | 64 | typedef struct 65 | { 66 | #if QLZ_COMPRESSION_LEVEL == 1 67 | unsigned int cache[QLZ_POINTERS]; 68 | #endif 69 | const unsigned char *offset[QLZ_POINTERS]; 70 | } qlz_hash_compress; 71 | 72 | typedef struct 73 | { 74 | const unsigned char *offset[QLZ_POINTERS]; 75 | } qlz_hash_decompress; 76 | 77 | 78 | #define QLZ_ALIGNMENT_PADD 8 79 | #define QLZ_BUFFER_COUNTER 8 80 | 81 | #define QLZ_SCRATCH_COMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_compress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) 82 | 83 | #if QLZ_COMPRESSION_LEVEL < 3 84 | #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_decompress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) 85 | #else 86 | #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER) 87 | #endif 88 | 89 | #endif 90 | 91 | 92 | 93 | 94 | 95 | -------------------------------------------------------------------------------- /src/record.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #ifndef __RECORD_H__ 18 | #define __RECORD_H__ 19 | 20 | #include 21 | #include 22 | 23 | #include "htree.h" 24 | #include "util.h" 25 | #include "diskmgr.h" 26 | 27 | 28 | typedef struct data_record 29 | { 30 | char *value; 31 | union 32 | { 33 | bool free_value; // free value or not 34 | uint32_t crc; 35 | }; 36 | int32_t tstamp; 37 | int32_t flag; 38 | int32_t version; 39 | uint32_t ksz; 40 | uint32_t vsz; 41 | char key[0]; 42 | } DataRecord; 43 | 44 | typedef bool (*RecordVisitor)(DataRecord *r, void *arg1, void *arg2); 45 | 46 | uint32_t gen_hash(char *buf, int size); 47 | 48 | char* record_value(DataRecord *r); 49 | void free_record(DataRecord **r); 50 | 51 | // on bad record, return NULL and set *fail_reason to one of these 52 | #define BAD_REC_SIZE 1 53 | #define BAD_REC_END 2 54 | #define BAD_REC_CRC 3 55 | #define BAD_REC_DECOMPRESS 4 56 | DataRecord* decode_record(char *buf, uint32_t size, bool decomp, const char *path, uint32_t pos, const char *key, bool do_logging, int *fail_reason); 57 | 58 | char* encode_record(DataRecord *r, unsigned int *size); 59 | DataRecord* read_record(FILE *f, bool decomp, const char *path, const char *key); 60 | DataRecord* fast_read_record(int fd, off_t offset, bool decomp, const char *path, const char *key); 61 | 62 | void scanDataFile(HTree *tree, int bucket, const char *path, const char *hintpath); 63 | void scanDataFileBefore(HTree *tree, int bucket, const char *path, time_t before); 64 | int optimizeDataFile(HTree *tree, Mgr *mgr, int bucket, const char *path, const char *hintpath, 65 | int last_bucket, const char *lastdata, const char *lasthint_real, uint32_t max_data_size, 66 | bool skipped, bool isnewfile, uint32_t *deleted_bytes); 67 | void visit_record(const char *path, RecordVisitor visitor, void *arg1, void *arg2, bool decomp); 68 | 69 | #endif 70 | -------------------------------------------------------------------------------- /src/thread.c: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2010 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Davies Liu 13 | * Hurricane Lee 14 | * 15 | */ 16 | 17 | #include "beansdb.h" 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #ifdef HAVE_STRING_H 24 | #include 25 | #endif 26 | 27 | #include 28 | #include "util.h" 29 | #include "log.h" 30 | 31 | typedef struct EventLoop 32 | { 33 | conn* conns[AE_SETSIZE]; 34 | int fired[AE_SETSIZE]; 35 | int nready; 36 | void* apidata; 37 | } EventLoop; 38 | 39 | /* Lock for connection freelist */ 40 | static pthread_mutex_t conn_lock; 41 | 42 | /* Lock for item buffer freelist */ 43 | static pthread_mutex_t ibuffer_lock; 44 | 45 | static EventLoop loop; 46 | static pthread_mutex_t leader; 47 | 48 | /* 49 | * Pulls a conn structure from the freelist, if one is available. 50 | */ 51 | conn *mt_conn_from_freelist() 52 | { 53 | conn *c; 54 | pthread_mutex_lock(&conn_lock); 55 | c = do_conn_from_freelist(); 56 | pthread_mutex_unlock(&conn_lock); 57 | return c; 58 | } 59 | 60 | /* 61 | * Adds a conn structure to the freelist. 62 | * 63 | * Returns 0 on success, 1 if the structure couldn't be added. 64 | */ 65 | bool mt_conn_add_to_freelist(conn *c) 66 | { 67 | bool result; 68 | 69 | pthread_mutex_lock(&conn_lock); 70 | result = do_conn_add_to_freelist(c); 71 | pthread_mutex_unlock(&conn_lock); 72 | 73 | return result; 74 | } 75 | 76 | /* 77 | * Pulls a item buffer from the freelist, if one is available. 78 | */ 79 | 80 | item *mt_item_from_freelist(void) 81 | { 82 | item *it; 83 | pthread_mutex_lock(&ibuffer_lock); 84 | it = do_item_from_freelist(); 85 | pthread_mutex_unlock(&ibuffer_lock); 86 | return it; 87 | } 88 | 89 | /* 90 | * Adds a item buffer to the freelist. 91 | * 92 | * Returns 0 on success, 1 if the buffer couldn't be added. 93 | */ 94 | int mt_item_add_to_freelist(item *it) 95 | { 96 | int result; 97 | 98 | pthread_mutex_lock(&ibuffer_lock); 99 | result = do_item_add_to_freelist(it); 100 | pthread_mutex_unlock(&ibuffer_lock); 101 | 102 | return result; 103 | } 104 | 105 | /******************************* GLOBAL STATS ******************************/ 106 | 107 | void mt_stats_lock() 108 | { 109 | } 110 | 111 | void mt_stats_unlock() 112 | { 113 | } 114 | 115 | /* Include the best multiplexing layer supported by this system. 116 | * The following should be ordered by performances, descending. */ 117 | #ifdef HAVE_EPOLL 118 | #include "ae_epoll.c" 119 | #else 120 | #ifdef HAVE_KQUEUE 121 | #include "ae_kqueue.c" 122 | #else 123 | #include "ae_select.c" 124 | #endif 125 | #endif 126 | 127 | /* 128 | * Initializes the thread subsystem, creating various worker threads. 129 | * 130 | * nthreads Number of event handler threads to spawn 131 | */ 132 | void thread_init(int nthreads) 133 | { 134 | int i; 135 | pthread_mutex_init(&ibuffer_lock, NULL); 136 | pthread_mutex_init(&conn_lock, NULL); 137 | pthread_mutex_init(&leader, NULL); 138 | 139 | memset(&loop, 0, sizeof(loop)); 140 | if (aeApiCreate(&loop) == -1) 141 | { 142 | exit(1); 143 | } 144 | } 145 | 146 | int add_event(int fd, int mask, conn *c) 147 | { 148 | if (fd >= AE_SETSIZE) 149 | { 150 | log_error("fd is too large: %d", fd); 151 | return AE_ERR; 152 | } 153 | if (loop.conns[fd] != NULL) 154 | { 155 | log_error("fd is used: %d", fd); 156 | return AE_ERR; 157 | } 158 | loop.conns[fd] = c; 159 | if (aeApiAddEvent(&loop, fd, mask) == -1) 160 | { 161 | loop.conns[fd] = NULL; 162 | return AE_ERR; 163 | } 164 | return AE_OK; 165 | } 166 | 167 | int update_event(int fd, int mask, conn *c) 168 | { 169 | loop.conns[fd] = c; 170 | if (aeApiUpdateEvent(&loop, fd, mask) == -1) 171 | { 172 | loop.conns[fd] = NULL; 173 | return AE_ERR; 174 | } 175 | return AE_OK; 176 | } 177 | 178 | int delete_event(int fd) 179 | { 180 | if (fd >= AE_SETSIZE) return -1; 181 | loop.conns[fd] = NULL; 182 | if (aeApiDelEvent(&loop, fd) == -1) 183 | return -1; 184 | return 0; 185 | } 186 | 187 | static void *worker_main(void *arg) 188 | { 189 | pthread_setcanceltype (PTHREAD_CANCEL_ASYNCHRONOUS, 0); 190 | 191 | struct timeval tv = {1, 0}; 192 | while (!daemon_quit) 193 | { 194 | pthread_mutex_lock(&leader); 195 | 196 | AGAIN: 197 | while(loop.nready == 0 && daemon_quit == 0) 198 | loop.nready = aeApiPoll(&loop, &tv); 199 | if (daemon_quit) 200 | { 201 | pthread_mutex_unlock(&leader); 202 | break; 203 | } 204 | 205 | loop.nready --; 206 | int fd = loop.fired[loop.nready]; 207 | conn *c = loop.conns[fd]; 208 | if (c == NULL) 209 | { 210 | log_error("Bug: conn %d should not be NULL", fd); 211 | delete_event(fd); 212 | close(fd); 213 | goto AGAIN; 214 | } 215 | //loop.conns[fd] = NULL; 216 | pthread_mutex_unlock(&leader); 217 | 218 | if (drive_machine(c)) 219 | { 220 | if (update_event(fd, c->ev_flags, c)) conn_close(c); 221 | } 222 | } 223 | return NULL; 224 | } 225 | 226 | void loop_run(int nthread) 227 | { 228 | int i, ret; 229 | pthread_attr_t attr; 230 | pthread_attr_init(&attr); 231 | pthread_t *tids = (pthread_t*)safe_malloc(sizeof(pthread_t) * nthread); 232 | 233 | for (i = 0; i < nthread - 1; i++) 234 | { 235 | if ((ret = pthread_create(tids + i, &attr, worker_main, NULL)) != 0) 236 | { 237 | log_fatal("Can't create thread: %s", 238 | strerror(ret)); 239 | exit(1); 240 | } 241 | } 242 | 243 | worker_main(NULL); 244 | 245 | // wait workers to stop 246 | for (i = 0; i < nthread - 1; i++) 247 | { 248 | (void) pthread_join(tids[i], NULL); 249 | pthread_detach(tids[i]); 250 | } 251 | free(tids); 252 | 253 | aeApiFree(&loop); 254 | } 255 | -------------------------------------------------------------------------------- /src/util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * Beansdb - A high available distributed key-value storage system: 3 | * 4 | * http://beansdb.googlecode.com 5 | * 6 | * Copyright 2009 Douban Inc. All rights reserved. 7 | * 8 | * Use and distribution licensed under the BSD license. See 9 | * the LICENSE file for full text. 10 | * 11 | * Authors: 12 | * Hurricane Lee 13 | */ 14 | 15 | #ifndef __UTIL_H__ 16 | #define __UTIL_H__ 17 | 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | #ifdef HAVE_MALLOC_H 26 | /* OpenBSD has a malloc.h, but warns to use stdlib.h instead */ 27 | #ifndef __OpenBSD__ 28 | #include 29 | #endif 30 | #endif 31 | 32 | #ifdef __GNUC__ 33 | #define likely(x) __builtin_expect((x),1) 34 | #define unlikely(x) __builtin_expect((x),0) 35 | #else 36 | #define likely(x) (x) 37 | #define unlikely(x) (x) 38 | #endif 39 | 40 | 41 | #include "log.h" 42 | 43 | inline static void* 44 | _safe_malloc(size_t s, const char *file, int line, const char *func) 45 | { 46 | void *p = malloc(s); 47 | if (unlikely(p == NULL)) 48 | { 49 | log_fatal("Out of memory: %d, %zu bytes in %s (%s:%i)", errno, s, func, file, line); 50 | /* 51 | * memset will make debug easier 52 | */ 53 | //memset(p, 0, s); 54 | exit(1); 55 | } 56 | return p; 57 | } 58 | 59 | #define safe_malloc(X) _safe_malloc(X, __FILE__, __LINE__, __FUNCTION__) 60 | 61 | inline static void* 62 | _try_malloc(size_t s, const char *file, int line, const char *func) 63 | { 64 | void *p = malloc(s); 65 | if (unlikely(p == NULL)) 66 | { 67 | log_warn("Out of memory: %d, %zu bytes in %s (%s:%i) but continue working.", errno, s, func, file, line); 68 | } 69 | return p; 70 | } 71 | 72 | #define try_malloc(X) _try_malloc(X, __FILE__, __LINE__, __FUNCTION__) 73 | 74 | inline static void* 75 | _safe_realloc(void *ptr, size_t s, const char *file, int line, const char *func) 76 | { 77 | void *p = realloc(ptr, s); 78 | if (unlikely(p == NULL)) 79 | { 80 | free(p); 81 | log_fatal("Realloc failed: %d, %zu bytes in %s (%s:%i)", errno, s, func, file, line); 82 | exit(1); 83 | } 84 | return p; 85 | } 86 | 87 | #define safe_realloc(X, Y) _safe_realloc(X, Y, __FILE__, __LINE__, __FUNCTION__) 88 | 89 | inline static void* 90 | _try_realloc(void *ptr, size_t s, const char *file, int line, const char *func) 91 | { 92 | void *p = realloc(ptr, s); 93 | if (unlikely(p == NULL)) 94 | { 95 | free(p); 96 | log_warn("Realloc failed: %d, %zu bytes in %s (%s:%i), but continue working", errno, s, func, file, line); 97 | } 98 | return p; 99 | } 100 | 101 | #define try_realloc(X, Y) _try_realloc(X, Y, __FILE__, __LINE__, __FUNCTION__) 102 | 103 | inline static void* 104 | _safe_calloc(size_t num, size_t size, const char *file, int line, const char *func) 105 | { 106 | void *p = calloc(num, size); 107 | if (unlikely(p == NULL)) 108 | { 109 | log_fatal("Calloc failed: %d, %zu bytes in %s (%s:%i)", errno, num * size, func, file, line); 110 | exit(1); 111 | } 112 | return p; 113 | } 114 | 115 | #define safe_calloc(X, Y) _safe_calloc(X, Y, __FILE__, __LINE__, __FUNCTION__) 116 | 117 | inline static void* 118 | _try_calloc(size_t num, size_t size, const char *file, int line, const char *func) 119 | { 120 | void *p = calloc(num, size); 121 | if (unlikely(p == NULL)) 122 | { 123 | log_warn("Calloc failed: %d, %zu bytes in %s (%s:%i)", errno, num * size, func, file, line); 124 | } 125 | return p; 126 | } 127 | 128 | #define try_calloc(X, Y) _try_calloc(X, Y, __FILE__, __LINE__, __FUNCTION__) 129 | 130 | inline static size_t 131 | _check_snprintf(const char *file, int line, const char *func, char *s, size_t n, const char *format, ...) 132 | { 133 | va_list args; 134 | size_t result_len; 135 | va_start (args, format); 136 | result_len = vsnprintf(s, n, format, args); 137 | if (unlikely(result_len >= n)) 138 | { 139 | log_fatal("Truncation: content truncated while calling snprintf \ 140 | in %s (%s:%i), %zu content print to %zu length buffer.", file, func, line, result_len, n); 141 | exit(1); 142 | } 143 | va_end(args); 144 | return result_len; 145 | } 146 | 147 | #define safe_snprintf(BUFFER, N, FORMAT, ...) _check_snprintf(__FILE__, __LINE__, __FUNCTION__, BUFFER, N, FORMAT, ##__VA_ARGS__) 148 | 149 | inline static void* 150 | _check_memcpy(const char *file, int line, const char *func, void *dst, size_t dst_num, const void *src, size_t src_num) 151 | { 152 | if (unlikely(dst_num < src_num)) 153 | { 154 | log_fatal("_check_memcpy try to use lower dst buffer: %zu than src size: %zu. \ 155 | in %s (%s:%i).", dst_num, src_num, file, func, line); 156 | exit(1); 157 | } 158 | return memcpy(dst, src, src_num); 159 | } 160 | 161 | #define safe_memcpy(DST, DST_NUM, SRC, SRC_NUM) _check_memcpy(__FILE__, __LINE__, __FUNCTION__, DST, DST_NUM, SRC, SRC_NUM) 162 | 163 | #define min(a,b) ((a)<(b)?(a):(b)) 164 | #define max(a,b) ((a)>(b)?(a):(b)) 165 | 166 | inline static int safe_strtol(const char *str, int base, long *out) 167 | { 168 | if (!out) 169 | return 0; 170 | errno = 0; 171 | *out = 0; 172 | char *endptr; 173 | long l = strtoul(str, &endptr, base); 174 | if (errno == ERANGE) 175 | return 0; 176 | if (isspace(*endptr) || (*endptr == '\0' && endptr != str)) 177 | { 178 | *out = l; 179 | return 1; 180 | } 181 | return 0; 182 | } 183 | #endif 184 | -------------------------------------------------------------------------------- /src/varint.h: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | static inline bool is_int(const char *p) 6 | { 7 | return (*p) & 0x80; 8 | } 9 | 10 | void printbuf(const char *buf, int n) 11 | { 12 | int i; 13 | printf("%d[", n); 14 | for (i = 0; i< n;i++) 15 | printf("(%x,%x)", buf[i] & 0x7f, buf[i] & 0x80); 16 | printf("]\n"); 17 | } 18 | 19 | int encode_varint(uint64_t n, char *buf) 20 | { 21 | char *p = buf; 22 | if (n < 64) 23 | { 24 | *buf = n + 0x80; 25 | return 1; 26 | } 27 | 28 | *buf = (n & 0x3f) + 0xc0; 29 | n >>= 6; 30 | 31 | while(n > 0) 32 | { 33 | p ++; 34 | *p = (n & 0x7f) + 0x80; 35 | n >>= 7; 36 | } 37 | *p &= 0x7f; 38 | return (p-buf+1); 39 | } 40 | 41 | uint64_t decode_varint(const char *src, int *len) 42 | { 43 | char *p = (char *)src; 44 | uint64_t n = (*p) & 0x7f; 45 | *len = 1; 46 | if (n < 64) 47 | { 48 | return n; 49 | } 50 | n &= 0x3f; 51 | ++p; 52 | uint64_t multi = 1 << 6; 53 | do 54 | { 55 | int v = (*p) & 0x7f; 56 | n += multi * v; 57 | multi <<= 7; 58 | } while ((*p++ & 0x80) != 0); 59 | *len = (p - src); 60 | return n; 61 | } 62 | 63 | int encode_varint_old(int n, char *buf) 64 | { 65 | int len; 66 | if (n < 64) 67 | { 68 | len = 1; 69 | *buf = -n; 70 | } 71 | else 72 | { 73 | len = 2; 74 | *buf = -(n & 0x3f) - 64; 75 | *(unsigned char*)(buf + 1) = n >> 6; 76 | } 77 | return len; 78 | } 79 | 80 | int decode_varint_old(const char *src, int *len) 81 | { 82 | int n = -*src; 83 | *len = 1; 84 | if (n >= 64) 85 | { 86 | n -= 64; 87 | n += (*(unsigned char*)(src + 1)) << 6; 88 | *len = 2; 89 | } 90 | return n; 91 | } 92 | 93 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/Makefile.am: -------------------------------------------------------------------------------- 1 | noinst_LIBRARIES = libzlog.a 2 | libzlog_a_SOURCES = buf.c buf.h category.c \ 3 | category.h category_table.c \ 4 | category_table.h conf.c \ 5 | conf.h event.c event.h \ 6 | fmacros.h format.c format.h \ 7 | level.c level.h level_list.c \ 8 | level_list.h mdc.c mdc.h \ 9 | record.c record.h record_table.c \ 10 | record_table.h rotater.c rotater.h \ 11 | rule.c rule.h spec.c spec.h thread.c \ 12 | thread.h version.h zc_arraylist.c \ 13 | zc_arraylist.h zc_defs.h zc_hashtable.c \ 14 | zc_hashtable.h zc_profile.c zc_profile.h \ 15 | zc_util.c zc_util.h zc_xplatform.h \ 16 | zlog-chk-conf.c zlog.c zlog.h 17 | 18 | libzlog_a_CFLAGS = -DNDEBUG 19 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/buf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_buf_h 10 | #define __zlog_buf_h 11 | 12 | /* buf, is a dynamic expand buffer for one single log, 13 | * as one single log will interlace if use multiple write() to file. 14 | * and buf is always keep in a thread, to make each thread has its 15 | * own buffer to avoid lock. 16 | */ 17 | 18 | #include 19 | #include 20 | 21 | typedef struct zlog_buf_s { 22 | char *start; 23 | char *tail; 24 | char *end; 25 | char *end_plus_1; 26 | 27 | size_t size_min; 28 | size_t size_max; 29 | size_t size_real; 30 | 31 | char truncate_str[MAXLEN_PATH + 1]; 32 | size_t truncate_str_len; 33 | } zlog_buf_t; 34 | 35 | 36 | zlog_buf_t *zlog_buf_new(size_t min, size_t max, const char *truncate_str); 37 | void zlog_buf_del(zlog_buf_t * a_buf); 38 | void zlog_buf_profile(zlog_buf_t * a_buf, int flag); 39 | 40 | int zlog_buf_vprintf(zlog_buf_t * a_buf, const char *format, va_list args); 41 | int zlog_buf_append(zlog_buf_t * a_buf, const char *str, size_t str_len); 42 | int zlog_buf_adjust_append(zlog_buf_t * a_buf, const char *str, size_t str_len, 43 | int left_adjust, size_t in_width, size_t out_width); 44 | int zlog_buf_printf_dec32(zlog_buf_t * a_buf, uint32_t ui32, int width); 45 | int zlog_buf_printf_dec64(zlog_buf_t * a_buf, uint64_t ui64, int width); 46 | int zlog_buf_printf_hex(zlog_buf_t * a_buf, uint32_t ui32, int width); 47 | 48 | #define zlog_buf_restart(a_buf) do { \ 49 | a_buf->tail = a_buf->start; \ 50 | } while(0) 51 | 52 | #define zlog_buf_len(a_buf) (a_buf->tail - a_buf->start) 53 | #define zlog_buf_str(a_buf) (a_buf->start) 54 | #define zlog_buf_seal(a_buf) do {*(a_buf)->tail = '\0';} while (0) 55 | 56 | #endif 57 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/category.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_category_h 10 | #define __zlog_category_h 11 | 12 | #include "zc_defs.h" 13 | #include "thread.h" 14 | 15 | typedef struct zlog_category_s { 16 | char name[MAXLEN_PATH + 1]; 17 | size_t name_len; 18 | unsigned char level_bitmap[32]; 19 | unsigned char level_bitmap_backup[32]; 20 | zc_arraylist_t *fit_rules; 21 | zc_arraylist_t *fit_rules_backup; 22 | } zlog_category_t; 23 | 24 | zlog_category_t *zlog_category_new(const char *name, zc_arraylist_t * rules); 25 | void zlog_category_del(zlog_category_t * a_category); 26 | void zlog_category_profile(zlog_category_t *a_category, int flag); 27 | 28 | int zlog_category_update_rules(zlog_category_t * a_category, zc_arraylist_t * new_rules); 29 | void zlog_category_commit_rules(zlog_category_t * a_category); 30 | void zlog_category_rollback_rules(zlog_category_t * a_category); 31 | 32 | int zlog_category_output(zlog_category_t * a_category, zlog_thread_t * a_thread); 33 | 34 | #define zlog_category_needless_level(a_category, lv) \ 35 | !((a_category->level_bitmap[lv/8] >> (7 - lv % 8)) & 0x01) 36 | 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/category_table.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "zc_defs.h" 14 | #include "category_table.h" 15 | 16 | void zlog_category_table_profile(zc_hashtable_t * categories, int flag) 17 | { 18 | zc_hashtable_entry_t *a_entry; 19 | zlog_category_t *a_category; 20 | 21 | zc_assert(categories,); 22 | zc_profile(flag, "-category_table[%p]-", categories); 23 | zc_hashtable_foreach(categories, a_entry) { 24 | a_category = (zlog_category_t *) a_entry->value; 25 | zlog_category_profile(a_category, flag); 26 | } 27 | return; 28 | } 29 | 30 | /*******************************************************************************/ 31 | 32 | void zlog_category_table_del(zc_hashtable_t * categories) 33 | { 34 | zc_assert(categories,); 35 | zc_hashtable_del(categories); 36 | zc_debug("zlog_category_table_del[%p]", categories); 37 | return; 38 | } 39 | 40 | zc_hashtable_t *zlog_category_table_new(void) 41 | { 42 | zc_hashtable_t *categories; 43 | 44 | categories = zc_hashtable_new(20, 45 | (zc_hashtable_hash_fn) zc_hashtable_str_hash, 46 | (zc_hashtable_equal_fn) zc_hashtable_str_equal, 47 | NULL, (zc_hashtable_del_fn) zlog_category_del); 48 | if (!categories) { 49 | zc_error("zc_hashtable_new fail"); 50 | return NULL; 51 | } else { 52 | zlog_category_table_profile(categories, ZC_DEBUG); 53 | return categories; 54 | } 55 | } 56 | /*******************************************************************************/ 57 | int zlog_category_table_update_rules(zc_hashtable_t * categories, zc_arraylist_t * new_rules) 58 | { 59 | zc_hashtable_entry_t *a_entry; 60 | zlog_category_t *a_category; 61 | 62 | zc_assert(categories, -1); 63 | zc_hashtable_foreach(categories, a_entry) { 64 | a_category = (zlog_category_t *) a_entry->value; 65 | if (zlog_category_update_rules(a_category, new_rules)) { 66 | zc_error("zlog_category_update_rules fail, try rollback"); 67 | return -1; 68 | } 69 | } 70 | return 0; 71 | } 72 | 73 | void zlog_category_table_commit_rules(zc_hashtable_t * categories) 74 | { 75 | zc_hashtable_entry_t *a_entry; 76 | zlog_category_t *a_category; 77 | 78 | zc_assert(categories,); 79 | zc_hashtable_foreach(categories, a_entry) { 80 | a_category = (zlog_category_t *) a_entry->value; 81 | zlog_category_commit_rules(a_category); 82 | } 83 | return; 84 | } 85 | 86 | void zlog_category_table_rollback_rules(zc_hashtable_t * categories) 87 | { 88 | zc_hashtable_entry_t *a_entry; 89 | zlog_category_t *a_category; 90 | 91 | zc_assert(categories,); 92 | zc_hashtable_foreach(categories, a_entry) { 93 | a_category = (zlog_category_t *) a_entry->value; 94 | zlog_category_rollback_rules(a_category); 95 | } 96 | return; 97 | } 98 | 99 | /*******************************************************************************/ 100 | zlog_category_t *zlog_category_table_fetch_category(zc_hashtable_t * categories, 101 | const char *category_name, zc_arraylist_t * rules) 102 | { 103 | zlog_category_t *a_category; 104 | 105 | zc_assert(categories, NULL); 106 | 107 | /* 1st find category in global category map */ 108 | a_category = zc_hashtable_get(categories, category_name); 109 | if (a_category) return a_category; 110 | 111 | /* else not fount, create one */ 112 | a_category = zlog_category_new(category_name, rules); 113 | if (!a_category) { 114 | zc_error("zc_category_new fail"); 115 | return NULL; 116 | } 117 | 118 | if(zc_hashtable_put(categories, a_category->name, a_category)) { 119 | zc_error("zc_hashtable_put fail"); 120 | goto err; 121 | } 122 | 123 | return a_category; 124 | err: 125 | zlog_category_del(a_category); 126 | return NULL; 127 | } 128 | 129 | /*******************************************************************************/ 130 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/category_table.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_category_table_h 10 | #define __zlog_category_table_h 11 | 12 | #include "zc_defs.h" 13 | #include "category.h" 14 | 15 | zc_hashtable_t *zlog_category_table_new(void); 16 | void zlog_category_table_del(zc_hashtable_t * categories); 17 | void zlog_category_table_profile(zc_hashtable_t * categories, int flag); 18 | 19 | /* if none, create new and return */ 20 | zlog_category_t *zlog_category_table_fetch_category( 21 | zc_hashtable_t * categories, 22 | const char *category_name, zc_arraylist_t * rules); 23 | 24 | int zlog_category_table_update_rules(zc_hashtable_t * categories, zc_arraylist_t * new_rules); 25 | void zlog_category_table_commit_rules(zc_hashtable_t * categories); 26 | void zlog_category_table_rollback_rules(zc_hashtable_t * categories); 27 | 28 | #endif 29 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/conf.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_conf_h 10 | #define __zlog_conf_h 11 | 12 | #include "zc_defs.h" 13 | #include "format.h" 14 | #include "rotater.h" 15 | 16 | typedef struct zlog_conf_s { 17 | char file[MAXLEN_PATH + 1]; 18 | char mtime[20 + 1]; 19 | 20 | int strict_init; 21 | size_t buf_size_min; 22 | size_t buf_size_max; 23 | 24 | char rotate_lock_file[MAXLEN_CFG_LINE + 1]; 25 | zlog_rotater_t *rotater; 26 | 27 | char default_format_line[MAXLEN_CFG_LINE + 1]; 28 | zlog_format_t *default_format; 29 | 30 | unsigned int file_perms; 31 | size_t fsync_period; 32 | size_t reload_conf_period; 33 | 34 | zc_arraylist_t *levels; 35 | zc_arraylist_t *formats; 36 | zc_arraylist_t *rules; 37 | int time_cache_count; 38 | } zlog_conf_t; 39 | 40 | extern zlog_conf_t * zlog_env_conf; 41 | 42 | zlog_conf_t *zlog_conf_new(const char *confpath); 43 | void zlog_conf_del(zlog_conf_t * a_conf); 44 | void zlog_conf_profile(zlog_conf_t * a_conf, int flag); 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/configure.ac: -------------------------------------------------------------------------------- 1 | # -*- Autoconf -*- 2 | # Process this file with autoconf to produce a configure script. 3 | 4 | AC_PREREQ([2.69]) 5 | AC_INIT([zlog], [1.2]) 6 | AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects]) 7 | AC_CONFIG_HEADERS([config.h]) 8 | 9 | # Checks for programs. 10 | AC_PROG_CC 11 | AM_PROG_CC_C_O 12 | AM_PROG_AR 13 | AC_PROG_RANLIB 14 | AC_PROG_INSTALL 15 | AC_PROG_LN_S 16 | AC_PROG_MAKE_SET 17 | 18 | AC_SEARCH_LIBS(pthread_create, pthread) 19 | 20 | # Checks for header files. 21 | AC_CHECK_HEADERS([fcntl.h limits.h stdint.h stdlib.h string.h strings.h sys/time.h syslog.h unistd.h]) 22 | 23 | # Checks for typedefs, structures, and compiler characteristics. 24 | AC_TYPE_PID_T 25 | AC_TYPE_SIZE_T 26 | AC_TYPE_UINT32_T 27 | AC_TYPE_UINT64_T 28 | 29 | # Checks for library functions. 30 | AC_FUNC_LSTAT_FOLLOWS_SLASHED_SYMLINK 31 | AC_FUNC_REALLOC 32 | AC_CHECK_FUNCS([atexit gethostname gettimeofday localtime_r memmove memset setenv strchr strrchr strtol]) 33 | 34 | AC_CONFIG_FILES([Makefile]) 35 | AC_OUTPUT 36 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/event.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include "fmacros.h" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include "zc_defs.h" 21 | #include "event.h" 22 | 23 | void zlog_event_profile(zlog_event_t * a_event, int flag) 24 | { 25 | zc_assert(a_event,); 26 | zc_profile(flag, "---event[%p][%s,%s][%s(%ld),%s(%ld),%ld,%d][%p,%s][%ld,%ld][%ld,%ld][%d]---", 27 | a_event, 28 | a_event->category_name, a_event->host_name, 29 | a_event->file, a_event->file_len, 30 | a_event->func, a_event->func_len, 31 | a_event->line, a_event->level, 32 | a_event->hex_buf, a_event->str_format, 33 | a_event->time_stamp.tv_sec, a_event->time_stamp.tv_usec, 34 | (long)a_event->pid, (long)a_event->tid, 35 | a_event->time_cache_count); 36 | return; 37 | } 38 | 39 | /*******************************************************************************/ 40 | 41 | void zlog_event_del(zlog_event_t * a_event) 42 | { 43 | zc_assert(a_event,); 44 | if (a_event->time_caches) free(a_event->time_caches); 45 | free(a_event); 46 | zc_debug("zlog_event_del[%p]", a_event); 47 | return; 48 | } 49 | 50 | zlog_event_t *zlog_event_new(int time_cache_count) 51 | { 52 | zlog_event_t *a_event; 53 | 54 | a_event = calloc(1, sizeof(zlog_event_t)); 55 | if (!a_event) { 56 | zc_error("calloc fail, errno[%d]", errno); 57 | return NULL; 58 | } 59 | 60 | a_event->time_caches = calloc(time_cache_count, sizeof(zlog_time_cache_t)); 61 | if (!a_event->time_caches) { 62 | zc_error("calloc fail, errno[%d]", errno); 63 | return NULL; 64 | } 65 | a_event->time_cache_count = time_cache_count; 66 | 67 | /* 68 | * at the zlog_init we gethostname, 69 | * u don't always change your hostname, eh? 70 | */ 71 | if (gethostname(a_event->host_name, sizeof(a_event->host_name) - 1)) { 72 | zc_error("gethostname fail, errno[%d]", errno); 73 | goto err; 74 | } 75 | 76 | a_event->host_name_len = strlen(a_event->host_name); 77 | 78 | /* tid is bound to a_event 79 | * as in whole lifecycle event persists 80 | * even fork to oth pid, tid not change 81 | */ 82 | a_event->tid = pthread_self(); 83 | 84 | a_event->tid_str_len = sprintf(a_event->tid_str, "%lu", (unsigned long)a_event->tid); 85 | a_event->tid_hex_str_len = sprintf(a_event->tid_hex_str, "0x%x", (unsigned int)a_event->tid); 86 | 87 | //zlog_event_profile(a_event, ZC_DEBUG); 88 | return a_event; 89 | err: 90 | zlog_event_del(a_event); 91 | return NULL; 92 | } 93 | 94 | /*******************************************************************************/ 95 | void zlog_event_set_fmt(zlog_event_t * a_event, 96 | char *category_name, size_t category_name_len, 97 | const char *file, size_t file_len, const char *func, size_t func_len, long line, int level, 98 | const char *str_format, va_list str_args) 99 | { 100 | /* 101 | * category_name point to zlog_category_output's category.name 102 | */ 103 | a_event->category_name = category_name; 104 | a_event->category_name_len = category_name_len; 105 | 106 | a_event->file = (char *) file; 107 | a_event->file_len = file_len; 108 | a_event->func = (char *) func; 109 | a_event->func_len = func_len; 110 | a_event->line = line; 111 | a_event->level = level; 112 | 113 | a_event->generate_cmd = ZLOG_FMT; 114 | a_event->str_format = str_format; 115 | va_copy(a_event->str_args, str_args); 116 | 117 | /* pid should fetch eveytime, as no one knows, 118 | * when does user fork his process 119 | * so clean here, and fetch at spec.c 120 | */ 121 | a_event->pid = (pid_t) 0; 122 | 123 | /* in a event's life cycle, time will be get when spec need, 124 | * and keep unchange though all event's life cycle 125 | * zlog_spec_write_time gettimeofday 126 | */ 127 | a_event->time_stamp.tv_sec = 0; 128 | return; 129 | } 130 | 131 | void zlog_event_set_hex(zlog_event_t * a_event, 132 | char *category_name, size_t category_name_len, 133 | const char *file, size_t file_len, const char *func, size_t func_len, long line, int level, 134 | const void *hex_buf, size_t hex_buf_len) 135 | { 136 | /* 137 | * category_name point to zlog_category_output's category.name 138 | */ 139 | a_event->category_name = category_name; 140 | a_event->category_name_len = category_name_len; 141 | 142 | a_event->file = (char *) file; 143 | a_event->file_len = file_len; 144 | a_event->func = (char *) func; 145 | a_event->func_len = func_len; 146 | a_event->line = line; 147 | a_event->level = level; 148 | 149 | a_event->generate_cmd = ZLOG_HEX; 150 | a_event->hex_buf = hex_buf; 151 | a_event->hex_buf_len = hex_buf_len; 152 | 153 | /* pid should fetch eveytime, as no one knows, 154 | * when does user fork his process 155 | * so clean here, and fetch at spec.c 156 | */ 157 | a_event->pid = (pid_t) 0; 158 | 159 | /* in a event's life cycle, time will be get when spec need, 160 | * and keep unchange though all event's life cycle 161 | */ 162 | a_event->time_stamp.tv_sec = 0; 163 | return; 164 | } 165 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/event.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_event_h 10 | #define __zlog_event_h 11 | 12 | #include /* for pid_t */ 13 | #include /* for struct timeval */ 14 | #include /* for pthread_t */ 15 | #include /* for va_list */ 16 | #include "zc_defs.h" 17 | 18 | typedef enum { 19 | ZLOG_FMT = 0, 20 | ZLOG_HEX = 1, 21 | } zlog_event_cmd; 22 | 23 | typedef struct zlog_time_cache_s { 24 | char str[MAXLEN_CFG_LINE + 1]; 25 | size_t len; 26 | time_t sec; 27 | } zlog_time_cache_t; 28 | 29 | typedef struct { 30 | char *category_name; 31 | size_t category_name_len; 32 | char host_name[256 + 1]; 33 | size_t host_name_len; 34 | 35 | const char *file; 36 | size_t file_len; 37 | const char *func; 38 | size_t func_len; 39 | long line; 40 | int level; 41 | 42 | const void *hex_buf; 43 | size_t hex_buf_len; 44 | const char *str_format; 45 | va_list str_args; 46 | zlog_event_cmd generate_cmd; 47 | 48 | struct timeval time_stamp; 49 | 50 | time_t time_local_sec; 51 | struct tm time_local; 52 | 53 | zlog_time_cache_t *time_caches; 54 | int time_cache_count; 55 | 56 | pid_t pid; 57 | pid_t last_pid; 58 | char pid_str[30 + 1]; 59 | size_t pid_str_len; 60 | 61 | pthread_t tid; 62 | char tid_str[30 + 1]; 63 | size_t tid_str_len; 64 | 65 | char tid_hex_str[30 + 1]; 66 | size_t tid_hex_str_len; 67 | } zlog_event_t; 68 | 69 | 70 | zlog_event_t *zlog_event_new(int time_cache_count); 71 | void zlog_event_del(zlog_event_t * a_event); 72 | void zlog_event_profile(zlog_event_t * a_event, int flag); 73 | 74 | void zlog_event_set_fmt(zlog_event_t * a_event, 75 | char *category_name, size_t category_name_len, 76 | const char *file, size_t file_len, const char *func, size_t func_len, long line, int level, 77 | const char *str_format, va_list str_args); 78 | 79 | void zlog_event_set_hex(zlog_event_t * a_event, 80 | char *category_name, size_t category_name_len, 81 | const char *file, size_t file_len, const char *func, size_t func_len, long line, int level, 82 | const void *hex_buf, size_t hex_buf_len); 83 | 84 | #endif 85 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/fmacros.h: -------------------------------------------------------------------------------- 1 | #ifndef __zlog_fmacro_h 2 | #define __zlog_fmacro_h 3 | 4 | #define _BSD_SOURCE 5 | 6 | #if defined(__linux__) || defined(__OpenBSD__) || defined(_AIX) 7 | #ifndef _XOPEN_SOURCE 8 | #define _XOPEN_SOURCE 700 9 | #endif 10 | #ifndef _XOPEN_SOURCE_EXTENDED 11 | #define _XOPEN_SOURCE_EXTENDED 12 | #endif 13 | #else 14 | #define _XOPEN_SOURCE 15 | #endif 16 | 17 | #define _LARGEFILE_SOURCE 18 | #define _FILE_OFFSET_BITS 64 19 | 20 | #endif 21 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/format.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "zc_defs.h" 17 | #include "thread.h" 18 | #include "spec.h" 19 | #include "format.h" 20 | 21 | void zlog_format_profile(zlog_format_t * a_format, int flag) 22 | { 23 | 24 | zc_assert(a_format,); 25 | zc_profile(flag, "---format[%p][%s = %s(%p)]---", 26 | a_format, 27 | a_format->name, 28 | a_format->pattern, 29 | a_format->pattern_specs); 30 | 31 | #if 0 32 | int i; 33 | zlog_spec_t *a_spec; 34 | zc_arraylist_foreach(a_format->pattern_specs, i, a_spec) { 35 | zlog_spec_profile(a_spec, flag); 36 | } 37 | #endif 38 | 39 | return; 40 | } 41 | 42 | /*******************************************************************************/ 43 | void zlog_format_del(zlog_format_t * a_format) 44 | { 45 | zc_assert(a_format,); 46 | if (a_format->pattern_specs) { 47 | zc_arraylist_del(a_format->pattern_specs); 48 | } 49 | free(a_format); 50 | zc_debug("zlog_format_del[%p]", a_format); 51 | return; 52 | } 53 | 54 | zlog_format_t *zlog_format_new(char *line, int * time_cache_count) 55 | { 56 | int nscan = 0; 57 | zlog_format_t *a_format = NULL; 58 | int nread = 0; 59 | const char *p_start; 60 | const char *p_end; 61 | char *p; 62 | char *q; 63 | zlog_spec_t *a_spec; 64 | 65 | zc_assert(line, NULL); 66 | 67 | a_format = calloc(1, sizeof(zlog_format_t)); 68 | if (!a_format) { 69 | zc_error("calloc fail, errno[%d]", errno); 70 | return NULL; 71 | } 72 | 73 | /* line default = "%d(%F %X.%l) %-6V (%c:%F:%L) - %m%n" 74 | * name default 75 | * pattern %d(%F %X.%l) %-6V (%c:%F:%L) - %m%n 76 | */ 77 | memset(a_format->name, 0x00, sizeof(a_format->name)); 78 | nread = 0; 79 | nscan = sscanf(line, " %[^= \t] = %n", a_format->name, &nread); 80 | if (nscan != 1) { 81 | zc_error("format[%s], syntax wrong", line); 82 | goto err; 83 | } 84 | 85 | if (*(line + nread) != '"') { 86 | zc_error("the 1st char of pattern is not \", line+nread[%s]", line+nread); 87 | goto err; 88 | } 89 | 90 | for (p = a_format->name; *p != '\0'; p++) { 91 | if ((!isalnum(*p)) && (*p != '_')) { 92 | zc_error("a_format->name[%s] character is not in [a-Z][0-9][_]", a_format->name); 93 | goto err; 94 | } 95 | } 96 | 97 | p_start = line + nread + 1; 98 | p_end = strrchr(p_start, '"'); 99 | if (!p_end) { 100 | zc_error("there is no \" at end of pattern, line[%s]", line); 101 | goto err; 102 | } 103 | 104 | if (p_end - p_start > sizeof(a_format->pattern) - 1) { 105 | zc_error("pattern is too long"); 106 | goto err; 107 | } 108 | memset(a_format->pattern, 0x00, sizeof(a_format->pattern)); 109 | memcpy(a_format->pattern, p_start, p_end - p_start); 110 | 111 | if (zc_str_replace_env(a_format->pattern, sizeof(a_format->pattern))) { 112 | zc_error("zc_str_replace_env fail"); 113 | goto err; 114 | } 115 | 116 | a_format->pattern_specs = 117 | zc_arraylist_new((zc_arraylist_del_fn) zlog_spec_del); 118 | if (!(a_format->pattern_specs)) { 119 | zc_error("zc_arraylist_new fail"); 120 | goto err; 121 | } 122 | 123 | for (p = a_format->pattern; *p != '\0'; p = q) { 124 | a_spec = zlog_spec_new(p, &q, time_cache_count); 125 | if (!a_spec) { 126 | zc_error("zlog_spec_new fail"); 127 | goto err; 128 | } 129 | 130 | if (zc_arraylist_add(a_format->pattern_specs, a_spec)) { 131 | zlog_spec_del(a_spec); 132 | zc_error("zc_arraylist_add fail"); 133 | goto err; 134 | } 135 | } 136 | 137 | zlog_format_profile(a_format, ZC_DEBUG); 138 | return a_format; 139 | err: 140 | zlog_format_del(a_format); 141 | return NULL; 142 | } 143 | 144 | /*******************************************************************************/ 145 | /* return 0 success, or buf is full 146 | * return -1 fail 147 | */ 148 | int zlog_format_gen_msg(zlog_format_t * a_format, zlog_thread_t * a_thread) 149 | { 150 | int i; 151 | zlog_spec_t *a_spec; 152 | 153 | zlog_buf_restart(a_thread->msg_buf); 154 | 155 | zc_arraylist_foreach(a_format->pattern_specs, i, a_spec) { 156 | if (zlog_spec_gen_msg(a_spec, a_thread) == 0) { 157 | continue; 158 | } else { 159 | return -1; 160 | } 161 | } 162 | 163 | return 0; 164 | } 165 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/format.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_format_h 10 | #define __zlog_format_h 11 | 12 | #include "thread.h" 13 | #include "zc_defs.h" 14 | 15 | typedef struct zlog_format_s zlog_format_t; 16 | 17 | struct zlog_format_s { 18 | char name[MAXLEN_CFG_LINE + 1]; 19 | char pattern[MAXLEN_CFG_LINE + 1]; 20 | zc_arraylist_t *pattern_specs; 21 | }; 22 | 23 | zlog_format_t *zlog_format_new(char *line, int * time_cache_count); 24 | void zlog_format_del(zlog_format_t * a_format); 25 | void zlog_format_profile(zlog_format_t * a_format, int flag); 26 | 27 | int zlog_format_gen_msg(zlog_format_t * a_format, zlog_thread_t * a_thread); 28 | 29 | #define zlog_format_has_name(a_format, fname) \ 30 | STRCMP(a_format->name, ==, fname) 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/level.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include "syslog.h" 13 | 14 | #include "zc_defs.h" 15 | #include "level.h" 16 | 17 | void zlog_level_profile(zlog_level_t *a_level, int flag) 18 | { 19 | zc_assert(a_level,); 20 | zc_profile(flag, "---level[%p][%d,%s,%s,%d,%d]---", 21 | a_level, 22 | a_level->int_level, 23 | a_level->str_uppercase, 24 | a_level->str_lowercase, 25 | (int) a_level->str_len, 26 | a_level->syslog_level); 27 | return; 28 | } 29 | 30 | /*******************************************************************************/ 31 | void zlog_level_del(zlog_level_t *a_level) 32 | { 33 | zc_assert(a_level,); 34 | free(a_level); 35 | zc_debug("zlog_level_del[%p]", a_level); 36 | return; 37 | } 38 | 39 | static int syslog_level_atoi(char *str) 40 | { 41 | /* guess no unix system will choose -187 42 | * as its syslog level, so it is a safe return value 43 | */ 44 | zc_assert(str, -187); 45 | 46 | if (STRICMP(str, ==, "LOG_EMERG")) 47 | return LOG_EMERG; 48 | if (STRICMP(str, ==, "LOG_ALERT")) 49 | return LOG_ALERT; 50 | if (STRICMP(str, ==, "LOG_CRIT")) 51 | return LOG_CRIT; 52 | if (STRICMP(str, ==, "LOG_ERR")) 53 | return LOG_ERR; 54 | if (STRICMP(str, ==, "LOG_WARNING")) 55 | return LOG_WARNING; 56 | if (STRICMP(str, ==, "LOG_NOTICE")) 57 | return LOG_NOTICE; 58 | if (STRICMP(str, ==, "LOG_INFO")) 59 | return LOG_INFO; 60 | if (STRICMP(str, ==, "LOG_DEBUG")) 61 | return LOG_DEBUG; 62 | 63 | zc_error("wrong syslog level[%s]", str); 64 | return -187; 65 | } 66 | 67 | /* line: TRACE = 10, LOG_ERR */ 68 | zlog_level_t *zlog_level_new(char *line) 69 | { 70 | zlog_level_t *a_level = NULL; 71 | int i; 72 | int nscan; 73 | char str[MAXLEN_CFG_LINE + 1]; 74 | int l = 0; 75 | char sl[MAXLEN_CFG_LINE + 1]; 76 | 77 | zc_assert(line, NULL); 78 | 79 | memset(str, 0x00, sizeof(str)); 80 | memset(sl, 0x00, sizeof(sl)); 81 | 82 | nscan = sscanf(line, " %[^= \t] = %d ,%s", str, &l, sl); 83 | if (nscan < 2) { 84 | zc_error("level[%s], syntax wrong", line); 85 | return NULL; 86 | } 87 | 88 | /* check level and str */ 89 | if ((l < 0) || (l > 255)) { 90 | zc_error("l[%d] not in [0,255], wrong", l); 91 | return NULL; 92 | } 93 | 94 | if (str[0] == '\0') { 95 | zc_error("str[0] = 0"); 96 | return NULL; 97 | } 98 | 99 | a_level = calloc(1, sizeof(zlog_level_t)); 100 | if (!a_level) { 101 | zc_error("calloc fail, errno[%d]", errno); 102 | return NULL; 103 | } 104 | 105 | a_level->int_level = l; 106 | 107 | /* fill syslog level */ 108 | if (sl[0] == '\0') { 109 | a_level->syslog_level = LOG_DEBUG; 110 | } else { 111 | a_level->syslog_level = syslog_level_atoi(sl); 112 | if (a_level->syslog_level == -187) { 113 | zc_error("syslog_level_atoi fail"); 114 | goto err; 115 | } 116 | } 117 | 118 | /* strncpy and toupper(str) */ 119 | for (i = 0; (i < sizeof(a_level->str_uppercase) - 1) && str[i] != '\0'; i++) { 120 | (a_level->str_uppercase)[i] = toupper(str[i]); 121 | (a_level->str_lowercase)[i] = tolower(str[i]); 122 | } 123 | 124 | if (str[i] != '\0') { 125 | /* overflow */ 126 | zc_error("not enough space for str, str[%s] > %d", str, i); 127 | goto err; 128 | } else { 129 | (a_level->str_uppercase)[i] = '\0'; 130 | (a_level->str_lowercase)[i] = '\0'; 131 | } 132 | 133 | a_level->str_len = i; 134 | 135 | //zlog_level_profile(a_level, ZC_DEBUG); 136 | return a_level; 137 | err: 138 | zc_error("line[%s]", line); 139 | zlog_level_del(a_level); 140 | return NULL; 141 | } 142 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/level.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_level_h 10 | #define __zlog_level_h 11 | 12 | #include "zc_defs.h" 13 | 14 | typedef struct zlog_level_s { 15 | int int_level; 16 | char str_uppercase[MAXLEN_PATH + 1]; 17 | char str_lowercase[MAXLEN_PATH + 1]; 18 | size_t str_len; 19 | int syslog_level; 20 | } zlog_level_t; 21 | 22 | zlog_level_t *zlog_level_new(char *line); 23 | void zlog_level_del(zlog_level_t *a_level); 24 | void zlog_level_profile(zlog_level_t *a_level, int flag); 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/level_list.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include "syslog.h" 13 | 14 | #include "zc_defs.h" 15 | #include "level.h" 16 | #include "level_list.h" 17 | 18 | /* zlog_level_list == zc_arraylist_t */ 19 | 20 | void zlog_level_list_profile(zc_arraylist_t *levels, int flag) 21 | { 22 | int i; 23 | zlog_level_t *a_level; 24 | 25 | zc_assert(levels,); 26 | zc_profile(flag, "--level_list[%p]--", levels); 27 | zc_arraylist_foreach(levels, i, a_level) { 28 | /* skip empty slots */ 29 | if (a_level) zlog_level_profile(a_level, flag); 30 | } 31 | return; 32 | } 33 | 34 | /*******************************************************************************/ 35 | void zlog_level_list_del(zc_arraylist_t *levels) 36 | { 37 | zc_assert(levels,); 38 | zc_arraylist_del(levels); 39 | zc_debug("zc_level_list_del[%p]", levels); 40 | return; 41 | } 42 | 43 | static int zlog_level_list_set_default(zc_arraylist_t *levels) 44 | { 45 | return zlog_level_list_set(levels, "* = 0, LOG_INFO") 46 | || zlog_level_list_set(levels, "DEBUG = 20, LOG_DEBUG") 47 | || zlog_level_list_set(levels, "INFO = 40, LOG_INFO") 48 | || zlog_level_list_set(levels, "NOTICE = 60, LOG_NOTICE") 49 | || zlog_level_list_set(levels, "WARN = 80, LOG_WARNING") 50 | || zlog_level_list_set(levels, "ERROR = 100, LOG_ERR") 51 | || zlog_level_list_set(levels, "FATAL = 120, LOG_ALERT") 52 | || zlog_level_list_set(levels, "UNKNOWN = 254, LOG_ERR") 53 | || zlog_level_list_set(levels, "! = 255, LOG_INFO"); 54 | } 55 | 56 | zc_arraylist_t *zlog_level_list_new(void) 57 | { 58 | zc_arraylist_t *levels; 59 | 60 | levels = zc_arraylist_new((zc_arraylist_del_fn)zlog_level_del); 61 | if (!levels) { 62 | zc_error("zc_arraylist_new fail"); 63 | return NULL; 64 | } 65 | 66 | if (zlog_level_list_set_default(levels)) { 67 | zc_error("zlog_level_set_default fail"); 68 | goto err; 69 | } 70 | 71 | //zlog_level_list_profile(levels, ZC_DEBUG); 72 | return levels; 73 | err: 74 | zc_arraylist_del(levels); 75 | return NULL; 76 | } 77 | 78 | /*******************************************************************************/ 79 | int zlog_level_list_set(zc_arraylist_t *levels, char *line) 80 | { 81 | zlog_level_t *a_level; 82 | 83 | a_level = zlog_level_new(line); 84 | if (!a_level) { 85 | zc_error("zlog_level_new fail"); 86 | return -1; 87 | } 88 | 89 | if (zc_arraylist_set(levels, a_level->int_level, a_level)) { 90 | zc_error("zc_arraylist_set fail"); 91 | goto err; 92 | } 93 | 94 | return 0; 95 | err: 96 | zc_error("line[%s]", line); 97 | zlog_level_del(a_level); 98 | return -1; 99 | } 100 | 101 | zlog_level_t *zlog_level_list_get(zc_arraylist_t *levels, int l) 102 | { 103 | zlog_level_t *a_level; 104 | 105 | #if 0 106 | if ((l <= 0) || (l > 254)) { 107 | /* illegal input from zlog() */ 108 | zc_error("l[%d] not in (0,254), set to UNKOWN", l); 109 | l = 254; 110 | } 111 | #endif 112 | 113 | a_level = zc_arraylist_get(levels, l); 114 | if (a_level) { 115 | return a_level; 116 | } else { 117 | /* empty slot */ 118 | zc_error("l[%d] not in (0,254), or has no level defined," 119 | "see configure file define, set to UNKOWN", l); 120 | return zc_arraylist_get(levels, 254); 121 | } 122 | } 123 | 124 | /*******************************************************************************/ 125 | 126 | int zlog_level_list_atoi(zc_arraylist_t *levels, char *str) 127 | { 128 | int i; 129 | zlog_level_t *a_level; 130 | 131 | if (str == NULL || *str == '\0') { 132 | zc_error("str is [%s], can't find level", str); 133 | return -1; 134 | } 135 | 136 | zc_arraylist_foreach(levels, i, a_level) { 137 | if (a_level && STRICMP(str, ==, a_level->str_uppercase)) { 138 | return i; 139 | } 140 | } 141 | 142 | zc_error("str[%s] can't found in level list", str); 143 | return -1; 144 | } 145 | 146 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/level_list.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_level_list_h 10 | #define __zlog_level_list_h 11 | 12 | #include "zc_defs.h" 13 | #include "level.h" 14 | 15 | zc_arraylist_t *zlog_level_list_new(void); 16 | void zlog_level_list_del(zc_arraylist_t *levels); 17 | void zlog_level_list_profile(zc_arraylist_t *levels, int flag); 18 | 19 | /* conf init use, slow */ 20 | /* if l is wrong or str=="", return -1 */ 21 | int zlog_level_list_set(zc_arraylist_t *levels, char *line); 22 | 23 | /* spec ouput use, fast */ 24 | /* rule output use, fast */ 25 | /* if not found, return levels[254] */ 26 | zlog_level_t *zlog_level_list_get(zc_arraylist_t *levels, int l); 27 | 28 | /* rule init use, slow */ 29 | /* if not found, return -1 */ 30 | int zlog_level_list_atoi(zc_arraylist_t *levels, char *str); 31 | 32 | 33 | #endif 34 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/mdc.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "mdc.h" 14 | #include "zc_defs.h" 15 | 16 | void zlog_mdc_profile(zlog_mdc_t *a_mdc, int flag) 17 | { 18 | zc_hashtable_entry_t *a_entry; 19 | zlog_mdc_kv_t *a_mdc_kv; 20 | 21 | zc_assert(a_mdc,); 22 | zc_profile(flag, "---mdc[%p]---", a_mdc); 23 | 24 | zc_hashtable_foreach(a_mdc->tab, a_entry) { 25 | a_mdc_kv = a_entry->value; 26 | zc_profile(flag, "----mdc_kv[%p][%s]-[%s]----", 27 | a_mdc_kv, 28 | a_mdc_kv->key, a_mdc_kv->value); 29 | } 30 | return; 31 | } 32 | /*******************************************************************************/ 33 | void zlog_mdc_del(zlog_mdc_t * a_mdc) 34 | { 35 | zc_assert(a_mdc,); 36 | if (a_mdc->tab) zc_hashtable_del(a_mdc->tab); 37 | free(a_mdc); 38 | zc_debug("zlog_mdc_del[%p]", a_mdc); 39 | return; 40 | } 41 | 42 | static void zlog_mdc_kv_del(zlog_mdc_kv_t * a_mdc_kv) 43 | { 44 | free(a_mdc_kv); 45 | zc_debug("zlog_mdc_kv_del[%p]", a_mdc_kv); 46 | } 47 | 48 | static zlog_mdc_kv_t *zlog_mdc_kv_new(const char *key, const char *value) 49 | { 50 | zlog_mdc_kv_t *a_mdc_kv; 51 | 52 | a_mdc_kv = calloc(1, sizeof(zlog_mdc_kv_t)); 53 | if (!a_mdc_kv) { 54 | zc_error("calloc fail, errno[%d]", errno); 55 | return NULL; 56 | } 57 | 58 | snprintf(a_mdc_kv->key, sizeof(a_mdc_kv->key), "%s", key); 59 | a_mdc_kv->value_len = snprintf(a_mdc_kv->value, sizeof(a_mdc_kv->value), "%s", value); 60 | return a_mdc_kv; 61 | } 62 | 63 | zlog_mdc_t *zlog_mdc_new(void) 64 | { 65 | zlog_mdc_t *a_mdc; 66 | 67 | a_mdc = calloc(1, sizeof(zlog_mdc_t)); 68 | if (!a_mdc) { 69 | zc_error("calloc fail, errno[%d]", errno); 70 | return NULL; 71 | } 72 | 73 | a_mdc->tab = zc_hashtable_new(20, 74 | zc_hashtable_str_hash, 75 | zc_hashtable_str_equal, NULL, 76 | (zc_hashtable_del_fn) zlog_mdc_kv_del); 77 | if (!a_mdc->tab) { 78 | zc_error("zc_hashtable_new fail"); 79 | goto err; 80 | } 81 | 82 | //zlog_mdc_profile(a_mdc, ZC_DEBUG); 83 | return a_mdc; 84 | err: 85 | zlog_mdc_del(a_mdc); 86 | return NULL; 87 | } 88 | 89 | /*******************************************************************************/ 90 | int zlog_mdc_put(zlog_mdc_t * a_mdc, const char *key, const char *value) 91 | { 92 | zlog_mdc_kv_t *a_mdc_kv; 93 | 94 | a_mdc_kv = zlog_mdc_kv_new(key, value); 95 | if (!a_mdc_kv) { 96 | zc_error("zlog_mdc_kv_new failed"); 97 | return -1; 98 | } 99 | 100 | if (zc_hashtable_put(a_mdc->tab, a_mdc_kv->key, a_mdc_kv)) { 101 | zc_error("zc_hashtable_put fail"); 102 | zlog_mdc_kv_del(a_mdc_kv); 103 | return -1; 104 | } 105 | 106 | return 0; 107 | } 108 | 109 | void zlog_mdc_clean(zlog_mdc_t * a_mdc) 110 | { 111 | zc_hashtable_clean(a_mdc->tab); 112 | return; 113 | } 114 | 115 | char *zlog_mdc_get(zlog_mdc_t * a_mdc, const char *key) 116 | { 117 | zlog_mdc_kv_t *a_mdc_kv; 118 | 119 | a_mdc_kv = zc_hashtable_get(a_mdc->tab, key); 120 | if (!a_mdc_kv) { 121 | zc_error("zc_hashtable_get fail"); 122 | return NULL; 123 | } else { 124 | return a_mdc_kv->value; 125 | } 126 | } 127 | 128 | zlog_mdc_kv_t *zlog_mdc_get_kv(zlog_mdc_t * a_mdc, const char *key) 129 | { 130 | zlog_mdc_kv_t *a_mdc_kv; 131 | 132 | a_mdc_kv = zc_hashtable_get(a_mdc->tab, key); 133 | if (!a_mdc_kv) { 134 | zc_error("zc_hashtable_get fail"); 135 | return NULL; 136 | } else { 137 | return a_mdc_kv; 138 | } 139 | } 140 | 141 | void zlog_mdc_remove(zlog_mdc_t * a_mdc, const char *key) 142 | { 143 | zc_hashtable_remove(a_mdc->tab, key); 144 | return; 145 | } 146 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/mdc.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_mdc_h 10 | #define __zlog_mdc_h 11 | 12 | #include "zc_defs.h" 13 | 14 | typedef struct zlog_mdc_s zlog_mdc_t; 15 | struct zlog_mdc_s { 16 | zc_hashtable_t *tab; 17 | }; 18 | 19 | zlog_mdc_t *zlog_mdc_new(void); 20 | void zlog_mdc_del(zlog_mdc_t * a_mdc); 21 | void zlog_mdc_profile(zlog_mdc_t *a_mdc, int flag); 22 | 23 | void zlog_mdc_clean(zlog_mdc_t * a_mdc); 24 | int zlog_mdc_put(zlog_mdc_t * a_mdc, const char *key, const char *value); 25 | char *zlog_mdc_get(zlog_mdc_t * a_mdc, const char *key); 26 | void zlog_mdc_remove(zlog_mdc_t * a_mdc, const char *key); 27 | 28 | typedef struct zlog_mdc_kv_s { 29 | char key[MAXLEN_PATH + 1]; 30 | char value[MAXLEN_PATH + 1]; 31 | size_t value_len; 32 | } zlog_mdc_kv_t; 33 | 34 | zlog_mdc_kv_t *zlog_mdc_get_kv(zlog_mdc_t * a_mdc, const char *key); 35 | 36 | #endif 37 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/record.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | #include "errno.h" 9 | #include "zc_defs.h" 10 | #include "record.h" 11 | 12 | void zlog_record_profile(zlog_record_t *a_record, int flag) 13 | { 14 | zc_assert(a_record,); 15 | zc_profile(flag, "--record:[%p][%s:%p]--", a_record, a_record->name, a_record->output); 16 | return; 17 | } 18 | 19 | void zlog_record_del(zlog_record_t *a_record) 20 | { 21 | zc_assert(a_record,); 22 | free(a_record); 23 | zc_debug("zlog_record_del[%p]", a_record); 24 | return; 25 | } 26 | 27 | zlog_record_t *zlog_record_new(const char *name, zlog_record_fn output) 28 | { 29 | zlog_record_t *a_record; 30 | 31 | zc_assert(name, NULL); 32 | zc_assert(output, NULL); 33 | 34 | a_record = calloc(1, sizeof(zlog_record_t)); 35 | if (!a_record) { 36 | zc_error("calloc fail, errno[%d]", errno); 37 | return NULL; 38 | } 39 | 40 | if (strlen(name) > sizeof(a_record->name) - 1) { 41 | zc_error("name[%s] is too long", name); 42 | goto err; 43 | } 44 | 45 | strcpy(a_record->name, name); 46 | a_record->output = output; 47 | 48 | zlog_record_profile(a_record, ZC_DEBUG); 49 | return a_record; 50 | err: 51 | zlog_record_del(a_record); 52 | return NULL; 53 | } 54 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/record.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_record_h 10 | #define __zlog_record_h 11 | 12 | #include "zc_defs.h" 13 | 14 | /* record is user-defined output function and it's name from configure file */ 15 | typedef struct zlog_msg_s { 16 | char *buf; 17 | size_t len; 18 | char *path; 19 | } zlog_msg_t; /* 3 of this first, see need thread or not later */ 20 | 21 | typedef int (*zlog_record_fn)(zlog_msg_t * msg); 22 | 23 | typedef struct zlog_record_s { 24 | char name[MAXLEN_PATH + 1]; 25 | zlog_record_fn output; 26 | } zlog_record_t; 27 | 28 | zlog_record_t *zlog_record_new(const char *name, zlog_record_fn output); 29 | void zlog_record_del(zlog_record_t *a_record); 30 | void zlog_record_profile(zlog_record_t *a_record, int flag); 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/record_table.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | 13 | #include "zc_defs.h" 14 | #include "record_table.h" 15 | 16 | void zlog_record_table_profile(zc_hashtable_t * records, int flag) 17 | { 18 | zc_hashtable_entry_t *a_entry; 19 | zlog_record_t *a_record; 20 | 21 | zc_assert(records,); 22 | zc_profile(flag, "-record_table[%p]-", records); 23 | zc_hashtable_foreach(records, a_entry) { 24 | a_record = (zlog_record_t *) a_entry->value; 25 | zlog_record_profile(a_record, flag); 26 | } 27 | return; 28 | } 29 | 30 | /*******************************************************************************/ 31 | 32 | void zlog_record_table_del(zc_hashtable_t * records) 33 | { 34 | zc_assert(records,); 35 | zc_hashtable_del(records); 36 | zc_debug("zlog_record_table_del[%p]", records); 37 | return; 38 | } 39 | 40 | zc_hashtable_t *zlog_record_table_new(void) 41 | { 42 | zc_hashtable_t *records; 43 | 44 | records = zc_hashtable_new(20, 45 | (zc_hashtable_hash_fn) zc_hashtable_str_hash, 46 | (zc_hashtable_equal_fn) zc_hashtable_str_equal, 47 | NULL, (zc_hashtable_del_fn) zlog_record_del); 48 | if (!records) { 49 | zc_error("zc_hashtable_new fail"); 50 | return NULL; 51 | } else { 52 | zlog_record_table_profile(records, ZC_DEBUG); 53 | return records; 54 | } 55 | } 56 | /*******************************************************************************/ 57 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/record_table.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_record_table_h 10 | #define __zlog_record_table_h 11 | 12 | #include "zc_defs.h" 13 | #include "record.h" 14 | 15 | zc_hashtable_t *zlog_record_table_new(void); 16 | void zlog_record_table_del(zc_hashtable_t * records); 17 | void zlog_record_table_profile(zc_hashtable_t * records, int flag); 18 | 19 | #endif 20 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/rotater.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_rotater_h 10 | #define __zlog_rotater_h 11 | 12 | #include "zc_defs.h" 13 | 14 | typedef struct zlog_rotater_s { 15 | pthread_mutex_t lock_mutex; 16 | char *lock_file; 17 | int lock_fd; 18 | 19 | /* single-use members */ 20 | char *base_path; /* aa.log */ 21 | char *archive_path; /* aa.#5i.log */ 22 | char glob_path[MAXLEN_PATH + 1]; /* aa.*.log */ 23 | size_t num_start_len; /* 3, offset to glob_path */ 24 | size_t num_end_len; /* 6, offset to glob_path */ 25 | int num_width; /* 5 */ 26 | int mv_type; /* ROLLING or SEQUENCE */ 27 | int max_count; 28 | zc_arraylist_t *files; 29 | } zlog_rotater_t; 30 | 31 | zlog_rotater_t *zlog_rotater_new(char *lock_file); 32 | void zlog_rotater_del(zlog_rotater_t *a_rotater); 33 | 34 | /* 35 | * return 36 | * -1 fail 37 | * 0 no rotate, or rotate and success 38 | */ 39 | int zlog_rotater_rotate(zlog_rotater_t *a_rotater, 40 | char *base_path, size_t msg_len, 41 | char *archive_path, long archive_max_size, int archive_max_count); 42 | 43 | void zlog_rotater_profile(zlog_rotater_t *a_rotater, int flag); 44 | 45 | #endif 46 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/rule.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | /** 10 | * @file rule.h 11 | * @brief rule decide to output in format by category & level 12 | */ 13 | 14 | #ifndef __zlog_rule_h 15 | #define __zlog_rule_h 16 | 17 | #include 18 | #include 19 | 20 | #include "zc_defs.h" 21 | #include "format.h" 22 | #include "thread.h" 23 | #include "rotater.h" 24 | #include "record.h" 25 | 26 | typedef struct zlog_rule_s zlog_rule_t; 27 | 28 | typedef int (*zlog_rule_output_fn) (zlog_rule_t * a_rule, zlog_thread_t * a_thread); 29 | 30 | struct zlog_rule_s { 31 | char category[MAXLEN_CFG_LINE + 1]; 32 | char compare_char; 33 | /* 34 | * [*] log all level 35 | * [.] log level >= rule level, default 36 | * [=] log level == rule level 37 | * [!] log level != rule level 38 | */ 39 | int level; 40 | unsigned char level_bitmap[32]; /* for category determine whether ouput or not */ 41 | 42 | unsigned int file_perms; 43 | int file_open_flags; 44 | 45 | char file_path[MAXLEN_PATH + 1]; 46 | zc_arraylist_t *dynamic_specs; 47 | int static_fd; 48 | dev_t static_dev; 49 | ino_t static_ino; 50 | 51 | long archive_max_size; 52 | int archive_max_count; 53 | char archive_path[MAXLEN_PATH + 1]; 54 | zc_arraylist_t *archive_specs; 55 | 56 | FILE *pipe_fp; 57 | int pipe_fd; 58 | 59 | size_t fsync_period; 60 | size_t fsync_count; 61 | 62 | zc_arraylist_t *levels; 63 | int syslog_facility; 64 | 65 | zlog_format_t *format; 66 | zlog_rule_output_fn output; 67 | 68 | char record_name[MAXLEN_PATH + 1]; 69 | char record_path[MAXLEN_PATH + 1]; 70 | zlog_record_fn record_func; 71 | }; 72 | 73 | zlog_rule_t *zlog_rule_new(char * line, 74 | zc_arraylist_t * levels, 75 | zlog_format_t * default_format, 76 | zc_arraylist_t * formats, 77 | unsigned int file_perms, 78 | size_t fsync_period, 79 | int * time_cache_count); 80 | 81 | void zlog_rule_del(zlog_rule_t * a_rule); 82 | void zlog_rule_profile(zlog_rule_t * a_rule, int flag); 83 | int zlog_rule_match_category(zlog_rule_t * a_rule, char *category); 84 | int zlog_rule_is_wastebin(zlog_rule_t * a_rule); 85 | int zlog_rule_set_record(zlog_rule_t * a_rule, zc_hashtable_t *records); 86 | int zlog_rule_output(zlog_rule_t * a_rule, zlog_thread_t * a_thread); 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/spec.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_spec_h 10 | #define __zlog_spec_h 11 | 12 | #include "event.h" 13 | #include "buf.h" 14 | #include "thread.h" 15 | 16 | typedef struct zlog_spec_s zlog_spec_t; 17 | 18 | /* write buf, according to each spec's Conversion Characters */ 19 | typedef int (*zlog_spec_write_fn) (zlog_spec_t * a_spec, 20 | zlog_thread_t * a_thread, 21 | zlog_buf_t * a_buf); 22 | 23 | /* gen a_thread->msg or gen a_thread->path by using write_fn */ 24 | typedef int (*zlog_spec_gen_fn) (zlog_spec_t * a_spec, 25 | zlog_thread_t * a_thread); 26 | 27 | struct zlog_spec_s { 28 | char *str; 29 | int len; 30 | 31 | char time_fmt[MAXLEN_CFG_LINE + 1]; 32 | int time_cache_index; 33 | char mdc_key[MAXLEN_PATH + 1]; 34 | 35 | char print_fmt[MAXLEN_CFG_LINE + 1]; 36 | int left_adjust; 37 | size_t max_width; 38 | size_t min_width; 39 | 40 | zlog_spec_write_fn write_buf; 41 | zlog_spec_gen_fn gen_msg; 42 | zlog_spec_gen_fn gen_path; 43 | zlog_spec_gen_fn gen_archive_path; 44 | }; 45 | 46 | zlog_spec_t *zlog_spec_new(char *pattern_start, char **pattern_end, int * time_cache_count); 47 | void zlog_spec_del(zlog_spec_t * a_spec); 48 | void zlog_spec_profile(zlog_spec_t * a_spec, int flag); 49 | 50 | #define zlog_spec_gen_msg(a_spec, a_thread) \ 51 | a_spec->gen_msg(a_spec, a_thread) 52 | 53 | #define zlog_spec_gen_path(a_spec, a_thread) \ 54 | a_spec->gen_path(a_spec, a_thread) 55 | 56 | #define zlog_spec_gen_archive_path(a_spec, a_thread) \ 57 | a_spec->gen_archive_path(a_spec, a_thread) 58 | 59 | #endif 60 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/thread.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | 12 | #include "zc_defs.h" 13 | #include "event.h" 14 | #include "buf.h" 15 | #include "thread.h" 16 | #include "mdc.h" 17 | 18 | void zlog_thread_profile(zlog_thread_t * a_thread, int flag) 19 | { 20 | zc_assert(a_thread,); 21 | zc_profile(flag, "--thread[%p][%p][%p][%p,%p,%p,%p,%p]--", 22 | a_thread, 23 | a_thread->mdc, 24 | a_thread->event, 25 | a_thread->pre_path_buf, 26 | a_thread->path_buf, 27 | a_thread->archive_path_buf, 28 | a_thread->pre_msg_buf, 29 | a_thread->msg_buf); 30 | 31 | zlog_mdc_profile(a_thread->mdc, flag); 32 | zlog_event_profile(a_thread->event, flag); 33 | zlog_buf_profile(a_thread->pre_path_buf, flag); 34 | zlog_buf_profile(a_thread->path_buf, flag); 35 | zlog_buf_profile(a_thread->archive_path_buf, flag); 36 | zlog_buf_profile(a_thread->pre_msg_buf, flag); 37 | zlog_buf_profile(a_thread->msg_buf, flag); 38 | return; 39 | } 40 | /*******************************************************************************/ 41 | void zlog_thread_del(zlog_thread_t * a_thread) 42 | { 43 | zc_assert(a_thread,); 44 | if (a_thread->mdc) 45 | zlog_mdc_del(a_thread->mdc); 46 | if (a_thread->event) 47 | zlog_event_del(a_thread->event); 48 | if (a_thread->pre_path_buf) 49 | zlog_buf_del(a_thread->pre_path_buf); 50 | if (a_thread->path_buf) 51 | zlog_buf_del(a_thread->path_buf); 52 | if (a_thread->archive_path_buf) 53 | zlog_buf_del(a_thread->archive_path_buf); 54 | if (a_thread->pre_msg_buf) 55 | zlog_buf_del(a_thread->pre_msg_buf); 56 | if (a_thread->msg_buf) 57 | zlog_buf_del(a_thread->msg_buf); 58 | 59 | free(a_thread); 60 | zc_debug("zlog_thread_del[%p]", a_thread); 61 | return; 62 | } 63 | 64 | zlog_thread_t *zlog_thread_new(int init_version, size_t buf_size_min, size_t buf_size_max, int time_cache_count) 65 | { 66 | zlog_thread_t *a_thread; 67 | 68 | a_thread = calloc(1, sizeof(zlog_thread_t)); 69 | if (!a_thread) { 70 | zc_error("calloc fail, errno[%d]", errno); 71 | return NULL; 72 | } 73 | 74 | a_thread->init_version = init_version; 75 | 76 | a_thread->mdc = zlog_mdc_new(); 77 | if (!a_thread->mdc) { 78 | zc_error("zlog_mdc_new fail"); 79 | goto err; 80 | } 81 | 82 | a_thread->event = zlog_event_new(time_cache_count); 83 | if (!a_thread->event) { 84 | zc_error("zlog_event_new fail"); 85 | goto err; 86 | } 87 | 88 | a_thread->pre_path_buf = zlog_buf_new(MAXLEN_PATH + 1, MAXLEN_PATH + 1, NULL); 89 | if (!a_thread->pre_path_buf) { 90 | zc_error("zlog_buf_new fail"); 91 | goto err; 92 | } 93 | 94 | a_thread->path_buf = zlog_buf_new(MAXLEN_PATH + 1, MAXLEN_PATH + 1, NULL); 95 | if (!a_thread->path_buf) { 96 | zc_error("zlog_buf_new fail"); 97 | goto err; 98 | } 99 | 100 | a_thread->archive_path_buf = zlog_buf_new(MAXLEN_PATH + 1, MAXLEN_PATH + 1, NULL); 101 | if (!a_thread->archive_path_buf) { 102 | zc_error("zlog_buf_new fail"); 103 | goto err; 104 | } 105 | 106 | a_thread->pre_msg_buf = zlog_buf_new(buf_size_min, buf_size_max, "..." FILE_NEWLINE); 107 | if (!a_thread->pre_msg_buf) { 108 | zc_error("zlog_buf_new fail"); 109 | goto err; 110 | } 111 | 112 | a_thread->msg_buf = zlog_buf_new(buf_size_min, buf_size_max, "..." FILE_NEWLINE); 113 | if (!a_thread->msg_buf) { 114 | zc_error("zlog_buf_new fail"); 115 | goto err; 116 | } 117 | 118 | 119 | //zlog_thread_profile(a_thread, ZC_DEBUG); 120 | return a_thread; 121 | err: 122 | zlog_thread_del(a_thread); 123 | return NULL; 124 | } 125 | 126 | /*******************************************************************************/ 127 | int zlog_thread_rebuild_msg_buf(zlog_thread_t * a_thread, size_t buf_size_min, size_t buf_size_max) 128 | { 129 | zlog_buf_t *pre_msg_buf_new = NULL; 130 | zlog_buf_t *msg_buf_new = NULL; 131 | zc_assert(a_thread, -1); 132 | 133 | if ( (a_thread->msg_buf->size_min == buf_size_min) 134 | && (a_thread->msg_buf->size_max == buf_size_max)) { 135 | zc_debug("buf size not changed, no need rebuild"); 136 | return 0; 137 | } 138 | 139 | pre_msg_buf_new = zlog_buf_new(buf_size_min, buf_size_max, "..." FILE_NEWLINE); 140 | if (!pre_msg_buf_new) { 141 | zc_error("zlog_buf_new fail"); 142 | goto err; 143 | } 144 | 145 | msg_buf_new = zlog_buf_new(buf_size_min, buf_size_max, "..." FILE_NEWLINE); 146 | if (!msg_buf_new) { 147 | zc_error("zlog_buf_new fail"); 148 | goto err; 149 | } 150 | 151 | zlog_buf_del(a_thread->pre_msg_buf); 152 | a_thread->pre_msg_buf = pre_msg_buf_new; 153 | 154 | zlog_buf_del(a_thread->msg_buf); 155 | a_thread->msg_buf = msg_buf_new; 156 | 157 | return 0; 158 | err: 159 | if (pre_msg_buf_new) zlog_buf_del(pre_msg_buf_new); 160 | if (msg_buf_new) zlog_buf_del(msg_buf_new); 161 | return -1; 162 | } 163 | 164 | int zlog_thread_rebuild_event(zlog_thread_t * a_thread, int time_cache_count) 165 | { 166 | zlog_event_t *event_new = NULL; 167 | zc_assert(a_thread, -1); 168 | 169 | event_new = zlog_event_new(time_cache_count); 170 | if (!event_new) { 171 | zc_error("zlog_event_new fail"); 172 | goto err; 173 | } 174 | 175 | zlog_event_del(a_thread->event); 176 | a_thread->event = event_new; 177 | return 0; 178 | err: 179 | if (event_new) zlog_event_del(event_new); 180 | return -1; 181 | } 182 | 183 | 184 | /*******************************************************************************/ 185 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/thread.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zlog_thread_h 10 | #define __zlog_thread_h 11 | 12 | #include "zc_defs.h" 13 | #include "event.h" 14 | #include "buf.h" 15 | #include "mdc.h" 16 | 17 | typedef struct { 18 | int init_version; 19 | zlog_mdc_t *mdc; 20 | zlog_event_t *event; 21 | 22 | zlog_buf_t *pre_path_buf; 23 | zlog_buf_t *path_buf; 24 | zlog_buf_t *archive_path_buf; 25 | zlog_buf_t *pre_msg_buf; 26 | zlog_buf_t *msg_buf; 27 | } zlog_thread_t; 28 | 29 | 30 | void zlog_thread_del(zlog_thread_t * a_thread); 31 | void zlog_thread_profile(zlog_thread_t * a_thread, int flag); 32 | zlog_thread_t *zlog_thread_new(int init_version, 33 | size_t buf_size_min, size_t buf_size_max, int time_cache_count); 34 | 35 | int zlog_thread_rebuild_msg_buf(zlog_thread_t * a_thread, size_t buf_size_min, size_t buf_size_max); 36 | int zlog_thread_rebuild_event(zlog_thread_t * a_thread, int time_cache_count); 37 | 38 | #endif 39 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/version.h: -------------------------------------------------------------------------------- 1 | #define ZLOG_VERSION "1.2.12" 2 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_arraylist.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "zc_defs.h" 15 | 16 | zc_arraylist_t *zc_arraylist_new(zc_arraylist_del_fn del) 17 | { 18 | zc_arraylist_t *a_list; 19 | 20 | a_list = (zc_arraylist_t *) calloc(1, sizeof(zc_arraylist_t)); 21 | if (!a_list) { 22 | zc_error("calloc fail, errno[%d]", errno); 23 | return NULL; 24 | } 25 | a_list->size = ARRAY_LIST_DEFAULT_SIZE; 26 | a_list->len = 0; 27 | 28 | /* this could be NULL */ 29 | a_list->del = del; 30 | a_list->array = (void **)calloc(a_list->size, sizeof(void *)); 31 | if (!a_list->array) { 32 | zc_error("calloc fail, errno[%d]", errno); 33 | free(a_list); 34 | return NULL; 35 | } 36 | 37 | return a_list; 38 | } 39 | 40 | void zc_arraylist_del(zc_arraylist_t * a_list) 41 | { 42 | int i; 43 | 44 | if (!a_list) 45 | return; 46 | if (a_list->del) { 47 | for (i = 0; i < a_list->len; i++) { 48 | if (a_list->array[i]) 49 | a_list->del(a_list->array[i]); 50 | } 51 | } 52 | if (a_list->array) 53 | free(a_list->array); 54 | free(a_list); 55 | return; 56 | } 57 | 58 | static int zc_arraylist_expand_inner(zc_arraylist_t * a_list, int max) 59 | { 60 | void *tmp; 61 | int new_size; 62 | int diff_size; 63 | 64 | new_size = zc_max(a_list->size * 2, max); 65 | tmp = realloc(a_list->array, new_size * sizeof(void *)); 66 | if (!tmp) { 67 | zc_error("realloc fail, errno[%d]", errno); 68 | free(a_list->array); 69 | return -1; 70 | } 71 | a_list->array = (void **)tmp; 72 | diff_size = new_size - a_list->size; 73 | if (diff_size) memset(a_list->array + a_list->size, 0x00, diff_size * sizeof(void *)); 74 | a_list->size = new_size; 75 | return 0; 76 | } 77 | 78 | int zc_arraylist_set(zc_arraylist_t * a_list, int idx, void *data) 79 | { 80 | if (idx > a_list->size - 1) { 81 | if (zc_arraylist_expand_inner(a_list, idx)) { 82 | zc_error("expand_internal fail"); 83 | return -1; 84 | } 85 | } 86 | if (a_list->array[idx] && a_list->del) a_list->del(a_list->array[idx]); 87 | a_list->array[idx] = data; 88 | if (a_list->len <= idx) 89 | a_list->len = idx + 1; 90 | return 0; 91 | } 92 | 93 | int zc_arraylist_add(zc_arraylist_t * a_list, void *data) 94 | { 95 | return zc_arraylist_set(a_list, a_list->len, data); 96 | } 97 | 98 | /* assum idx < len */ 99 | static int zc_arraylist_insert_inner(zc_arraylist_t * a_list, int idx, 100 | void *data) 101 | { 102 | if (a_list->array[idx] == NULL) { 103 | a_list->array[idx] = data; 104 | return 0; 105 | } 106 | if (a_list->len > a_list->size - 1) { 107 | if (zc_arraylist_expand_inner(a_list, 0)) { 108 | zc_error("expand_internal fail"); 109 | return -1; 110 | } 111 | } 112 | memmove(a_list->array + idx + 1, a_list->array + idx, 113 | (a_list->len - idx) * sizeof(void *)); 114 | a_list->array[idx] = data; 115 | a_list->len++; 116 | return 0; 117 | } 118 | 119 | int zc_arraylist_sortadd(zc_arraylist_t * a_list, zc_arraylist_cmp_fn cmp, 120 | void *data) 121 | { 122 | int i; 123 | 124 | for (i = 0; i < a_list->len; i++) { 125 | if ((*cmp) (a_list->array[i], data) > 0) 126 | break; 127 | } 128 | 129 | if (i == a_list->len) 130 | return zc_arraylist_add(a_list, data); 131 | else 132 | return zc_arraylist_insert_inner(a_list, i, data); 133 | } 134 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_arraylist.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zc_arraylist_h 10 | #define __zc_arraylist_h 11 | 12 | #define ARRAY_LIST_DEFAULT_SIZE 32 13 | 14 | typedef void (*zc_arraylist_del_fn) (void *data); 15 | typedef int (*zc_arraylist_cmp_fn) (void *data1, void *data2); 16 | 17 | /* make zc_arraylist_foreach speed up, so keep struct defination here */ 18 | typedef struct { 19 | void **array; 20 | int len; 21 | int size; 22 | zc_arraylist_del_fn del; 23 | } zc_arraylist_t; 24 | 25 | zc_arraylist_t *zc_arraylist_new(zc_arraylist_del_fn del); 26 | void zc_arraylist_del(zc_arraylist_t * a_list); 27 | 28 | int zc_arraylist_set(zc_arraylist_t * a_list, int i, void *data); 29 | int zc_arraylist_add(zc_arraylist_t * a_list, void *data); 30 | int zc_arraylist_sortadd(zc_arraylist_t * a_list, zc_arraylist_cmp_fn cmp, 31 | void *data); 32 | 33 | #define zc_arraylist_len(a_list) (a_list->len) 34 | 35 | #define zc_arraylist_get(a_list, i) \ 36 | ((i >= a_list->len) ? NULL : a_list->array[i]) 37 | 38 | #define zc_arraylist_foreach(a_list, i, a_unit) \ 39 | for(i = 0, a_unit = a_list->array[0]; (i < a_list->len) && (a_unit = a_list->array[i], 1) ; i++) 40 | 41 | #endif 42 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_defs.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zc_defs_h 10 | #define __zc_defs_h 11 | 12 | #include "zc_profile.h" 13 | #include "zc_arraylist.h" 14 | #include "zc_hashtable.h" 15 | #include "zc_xplatform.h" 16 | #include "zc_util.h" 17 | 18 | #endif 19 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_hashtable.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zc_hashtalbe_h 10 | #define __zc_hashtalbe_h 11 | 12 | #include 13 | 14 | typedef struct zc_hashtable_entry_s { 15 | unsigned int hash_key; 16 | void *key; 17 | void *value; 18 | struct zc_hashtable_entry_s *prev; 19 | struct zc_hashtable_entry_s *next; 20 | } zc_hashtable_entry_t; 21 | 22 | typedef struct zc_hashtable_s zc_hashtable_t; 23 | 24 | typedef unsigned int (*zc_hashtable_hash_fn) (const void *key); 25 | typedef int (*zc_hashtable_equal_fn) (const void *key1, const void *key2); 26 | typedef void (*zc_hashtable_del_fn) (void *kv); 27 | 28 | zc_hashtable_t *zc_hashtable_new(size_t a_size, 29 | zc_hashtable_hash_fn hash_fn, 30 | zc_hashtable_equal_fn equal_fn, 31 | zc_hashtable_del_fn key_del_fn, 32 | zc_hashtable_del_fn value_del_fn); 33 | 34 | void zc_hashtable_del(zc_hashtable_t * a_table); 35 | void zc_hashtable_clean(zc_hashtable_t * a_table); 36 | int zc_hashtable_put(zc_hashtable_t * a_table, void *a_key, void *a_value); 37 | zc_hashtable_entry_t *zc_hashtable_get_entry(zc_hashtable_t * a_table, const void *a_key); 38 | void *zc_hashtable_get(zc_hashtable_t * a_table, const void *a_key); 39 | void zc_hashtable_remove(zc_hashtable_t * a_table, const void *a_key); 40 | zc_hashtable_entry_t *zc_hashtable_begin(zc_hashtable_t * a_table); 41 | zc_hashtable_entry_t *zc_hashtable_next(zc_hashtable_t * a_table, zc_hashtable_entry_t * a_entry); 42 | 43 | #define zc_hashtable_foreach(a_table, a_entry) \ 44 | for(a_entry = zc_hashtable_begin(a_table); a_entry; a_entry = zc_hashtable_next(a_table, a_entry)) 45 | 46 | unsigned int zc_hashtable_str_hash(const void *str); 47 | int zc_hashtable_str_equal(const void *key1, const void *key2); 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_profile.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include "fmacros.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include "zc_profile.h" 21 | #include "zc_xplatform.h" 22 | 23 | static void zc_time(char *time_str, size_t time_str_size) 24 | { 25 | time_t tt; 26 | struct tm local_time; 27 | 28 | time(&tt); 29 | localtime_r(&tt, &local_time); 30 | strftime(time_str, time_str_size, "%m-%d %T", &local_time); 31 | 32 | return; 33 | } 34 | 35 | int zc_profile_inner(int flag, const char *file, const long line, const char *fmt, ...) 36 | { 37 | va_list args; 38 | char time_str[20 + 1]; 39 | FILE *fp = NULL; 40 | 41 | static char *debug_log = NULL; 42 | static char *error_log = NULL; 43 | static size_t init_flag = 0; 44 | 45 | if (!init_flag) { 46 | init_flag = 1; 47 | debug_log = getenv("ZLOG_PROFILE_DEBUG"); 48 | error_log = getenv("ZLOG_PROFILE_ERROR"); 49 | } 50 | 51 | switch (flag) { 52 | case ZC_DEBUG: 53 | if (debug_log == NULL) return 0; 54 | fp = fopen(debug_log, "a"); 55 | if (!fp) return -1; 56 | zc_time(time_str, sizeof(time_str)); 57 | fprintf(fp, "%s DEBUG (%d:%s:%ld) ", time_str, getpid(), file, line); 58 | break; 59 | case ZC_WARN: 60 | if (error_log == NULL) return 0; 61 | fp = fopen(error_log, "a"); 62 | if (!fp) return -1; 63 | zc_time(time_str, sizeof(time_str)); 64 | fprintf(fp, "%s WARN (%d:%s:%ld) ", time_str, getpid(), file, line); 65 | break; 66 | case ZC_ERROR: 67 | if (error_log == NULL) return 0; 68 | fp = fopen(error_log, "a"); 69 | if (!fp) return -1; 70 | zc_time(time_str, sizeof(time_str)); 71 | fprintf(fp, "%s ERROR (%d:%s:%ld) ", time_str, getpid(), file, line); 72 | break; 73 | } 74 | 75 | /* writing file twice(time & msg) is not atomic 76 | * may cause cross 77 | * but avoid log size limit */ 78 | va_start(args, fmt); 79 | vfprintf(fp, fmt, args); 80 | va_end(args); 81 | fprintf(fp, "\n"); 82 | 83 | fclose(fp); 84 | return 0; 85 | } 86 | 87 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_profile.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #ifndef __zc_profile_h 10 | #define __zc_profile_h 11 | 12 | #include 13 | 14 | #define EMPTY() 15 | #define zc_assert(expr, rv) \ 16 | if(!(expr)) { \ 17 | zc_error(#expr" is null or 0"); \ 18 | return rv; \ 19 | } 20 | 21 | enum zc_profile_flag { 22 | ZC_DEBUG = 0, 23 | ZC_WARN = 1, 24 | ZC_ERROR = 2 25 | }; 26 | 27 | 28 | #if defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L 29 | #define zc_debug(...) \ 30 | zc_profile_inner(ZC_DEBUG, __FILE__, __LINE__, __VA_ARGS__) 31 | #define zc_warn(...) \ 32 | zc_profile_inner(ZC_WARN, __FILE__, __LINE__, __VA_ARGS__) 33 | #define zc_error(...) \ 34 | zc_profile_inner(ZC_ERROR, __FILE__, __LINE__, __VA_ARGS__) 35 | #define zc_profile(flag, ...) \ 36 | zc_profile_inner(flag, __FILE__, __LINE__, __VA_ARGS__) 37 | #elif defined __GNUC__ 38 | #define zc_debug(fmt, args...) \ 39 | zc_profile_inner(ZC_DEBUG, __FILE__, __LINE__, fmt, ## args) 40 | #define zc_warn(fmt, args...) \ 41 | zc_profile_inner(ZC_WARN, __FILE__, __LINE__, fmt, ## args) 42 | #define zc_error(fmt, args...) \ 43 | zc_profile_inner(ZC_ERROR, __FILE__, __LINE__, fmt, ## args) 44 | #define zc_profile(flag, fmt, args...) \ 45 | zc_profile_inner(flag, __FILE__, __LINE__, fmt, ## args) 46 | #endif 47 | 48 | 49 | int zc_profile_inner(int flag, 50 | const char *file, const long line, 51 | const char *fmt, ...); 52 | 53 | #endif 54 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_util.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "zc_defs.h" 17 | 18 | size_t zc_parse_byte_size(char *astring) 19 | { 20 | /* Parse size in bytes depending on the suffix. Valid suffixes are KB, MB and GB */ 21 | char *p; 22 | char *q; 23 | size_t sz; 24 | long res; 25 | int c, m; 26 | 27 | zc_assert(astring, 0); 28 | 29 | /* clear space */ 30 | for (p = q = astring; *p != '\0'; p++) { 31 | if (isspace(*p)) { 32 | continue; 33 | } else { 34 | *q = *p; 35 | q++; 36 | } 37 | } 38 | *q = '\0'; 39 | 40 | sz = strlen(astring); 41 | res = strtol(astring, (char **)NULL, 10); 42 | 43 | if (res <= 0) 44 | return 0; 45 | 46 | if (astring[sz - 1] == 'B' || astring[sz - 1] == 'b') { 47 | c = astring[sz - 2]; 48 | m = 1024; 49 | } else { 50 | c = astring[sz - 1]; 51 | m = 1000; 52 | } 53 | 54 | switch (c) { 55 | case 'K': 56 | case 'k': 57 | res *= m; 58 | break; 59 | case 'M': 60 | case 'm': 61 | res *= m * m; 62 | break; 63 | case 'G': 64 | case 'g': 65 | res *= m * m * m; 66 | break; 67 | default: 68 | if (!isdigit(c)) { 69 | zc_error("Wrong suffix parsing " "size in bytes for string [%s], ignoring suffix", 70 | astring); 71 | } 72 | break; 73 | } 74 | 75 | return (res); 76 | } 77 | 78 | /*******************************************************************************/ 79 | int zc_str_replace_env(char *str, size_t str_size) 80 | { 81 | char *p; 82 | char *q; 83 | char fmt[MAXLEN_CFG_LINE + 1]; 84 | char env_key[MAXLEN_CFG_LINE + 1]; 85 | char env_value[MAXLEN_CFG_LINE + 1]; 86 | int str_len; 87 | int env_value_len; 88 | int nscan; 89 | int nread; 90 | 91 | str_len = strlen(str); 92 | q = str; 93 | 94 | do { 95 | p = strchr(q, '%'); 96 | if (!p) { 97 | /* can't find more % */ 98 | break; 99 | } 100 | 101 | memset(fmt, 0x00, sizeof(fmt)); 102 | memset(env_key, 0x00, sizeof(env_key)); 103 | memset(env_value, 0x00, sizeof(env_value)); 104 | nread = 0; 105 | nscan = sscanf(p + 1, "%[.0-9-]%n", fmt + 1, &nread); 106 | if (nscan == 1) { 107 | fmt[0] = '%'; 108 | fmt[nread + 1] = 's'; 109 | } else { 110 | nread = 0; 111 | strcpy(fmt, "%s"); 112 | } 113 | 114 | q = p + 1 + nread; 115 | 116 | nscan = sscanf(q, "E(%[^)])%n", env_key, &nread); 117 | if (nscan == 0) { 118 | continue; 119 | } 120 | 121 | q += nread; 122 | 123 | if (*(q - 1) != ')') { 124 | zc_error("in string[%s] can't find match )", p); 125 | return -1; 126 | } 127 | 128 | env_value_len = snprintf(env_value, sizeof(env_value), fmt, getenv(env_key)); 129 | if (env_value_len < 0 || env_value_len >= sizeof(env_value)) { 130 | zc_error("snprintf fail, errno[%d], evn_value_len[%d]", 131 | errno, env_value_len); 132 | return -1; 133 | } 134 | 135 | str_len = str_len - (q - p) + env_value_len; 136 | if (str_len > str_size - 1) { 137 | zc_error("repalce env_value[%s] cause overlap", env_value); 138 | return -1; 139 | } 140 | 141 | memmove(p + env_value_len, q, strlen(q) + 1); 142 | memcpy(p, env_value, env_value_len); 143 | 144 | } while (1); 145 | 146 | return 0; 147 | } 148 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_util.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | #ifndef __zc_util_h 9 | #define __zc_util_h 10 | 11 | size_t zc_parse_byte_size(char *astring); 12 | int zc_str_replace_env(char *str, size_t str_size); 13 | 14 | #define zc_max(a,b) ((a) > (b) ? (a) : (b)) 15 | #define zc_min(a,b) ((a) < (b) ? (a) : (b)) 16 | 17 | #endif 18 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zc_xplatform.h: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | #ifndef __zc_xplatform_h 9 | #define __zc_xplatform_h 10 | 11 | #include 12 | 13 | #define ZLOG_INT32_LEN sizeof("-2147483648") - 1 14 | #define ZLOG_INT64_LEN sizeof("-9223372036854775808") - 1 15 | 16 | #if ((__GNU__ == 2) && (__GNUC_MINOR__ < 8)) 17 | #define ZLOG_MAX_UINT32_VALUE (uint32_t) 0xffffffffLL 18 | #else 19 | #define ZLOG_MAX_UINT32_VALUE (uint32_t) 0xffffffff 20 | #endif 21 | 22 | #define ZLOG_MAX_INT32_VALUE (uint32_t) 0x7fffffff 23 | 24 | #define MAXLEN_PATH 1024 25 | #define MAXLEN_CFG_LINE (MAXLEN_PATH * 4) 26 | 27 | #define FILE_NEWLINE "\n" 28 | #define FILE_NEWLINE_LEN 1 29 | 30 | #include 31 | #include 32 | 33 | #define STRCMP(_a_,_C_,_b_) ( strcmp(_a_,_b_) _C_ 0 ) 34 | #define STRNCMP(_a_,_C_,_b_,_n_) ( strncmp(_a_,_b_,_n_) _C_ 0 ) 35 | #define STRICMP(_a_,_C_,_b_) ( strcasecmp(_a_,_b_) _C_ 0 ) 36 | #define STRNICMP(_a_,_C_,_b_,_n_) ( strncasecmp(_a_,_b_,_n_) _C_ 0 ) 37 | 38 | 39 | #ifdef __APPLE__ 40 | #include 41 | #endif 42 | 43 | /* Define zlog_fstat to fstat or fstat64() */ 44 | #if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) 45 | #define zlog_fstat fstat64 46 | #define zlog_stat stat64 47 | #else 48 | #define zlog_fstat fstat 49 | #define zlog_stat stat 50 | #endif 51 | 52 | /* Define zlog_fsync to fdatasync() in Linux and fsync() for all the rest */ 53 | #ifdef __linux__ 54 | #define zlog_fsync fdatasync 55 | #else 56 | #define zlog_fsync fsync 57 | #endif 58 | 59 | 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /third-party/zlog-1.2/zlog-chk-conf.c: -------------------------------------------------------------------------------- 1 | /* 2 | * This file is part of the zlog Library. 3 | * 4 | * Copyright (C) 2011 by Hardy Simpson 5 | * 6 | * Licensed under the LGPL v2.1, see the file COPYING in base directory. 7 | */ 8 | 9 | #include "fmacros.h" 10 | 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include 17 | 18 | #include "zlog.h" 19 | #include "version.h" 20 | 21 | 22 | int main(int argc, char *argv[]) 23 | { 24 | int rc = 0; 25 | int op; 26 | int quiet = 0; 27 | static const char *help = 28 | "useage: zlog-chk-conf [conf files]...\n" 29 | "\t-q,\tsuppress non-error message\n" 30 | "\t-h,\tshow help message\n" 31 | "zlog version: " ZLOG_VERSION "\n"; 32 | 33 | while((op = getopt(argc, argv, "qhv")) > 0) { 34 | if (op == 'h') { 35 | fputs(help, stdout); 36 | return 0; 37 | } else if (op == 'q') { 38 | quiet = 1; 39 | } 40 | } 41 | 42 | argc -= optind; 43 | argv += optind; 44 | 45 | if (argc == 0) { 46 | fputs(help, stdout); 47 | return -1; 48 | } 49 | 50 | setenv("ZLOG_PROFILE_ERROR", "/dev/stderr", 1); 51 | setenv("ZLOG_CHECK_FORMAT_RULE", "1", 1); 52 | 53 | while (argc > 0) { 54 | rc = zlog_init(*argv); 55 | if (rc) { 56 | printf("\n---[%s] syntax error, see error message above\n", 57 | *argv); 58 | exit(2); 59 | } else { 60 | zlog_fini(); 61 | if (!quiet) { 62 | printf("--[%s] syntax right\n", *argv); 63 | } 64 | } 65 | argc--; 66 | argv++; 67 | } 68 | 69 | exit(0); 70 | } 71 | --------------------------------------------------------------------------------