├── tests ├── __init__.py ├── pip-req.txt ├── README.md ├── run_test.sh ├── utils.py ├── abnormal_cmd_test.py ├── dbclient.py ├── base.py └── key_version_test.py ├── gobeansdb ├── config_test.yaml ├── config_test.go ├── config.go ├── gobeansdb.go └── store.go ├── main.go ├── conf ├── route.yaml ├── local.yaml └── global.yaml ├── go.mod ├── cmem ├── cmem_test.go ├── beansdb.go └── cmem.go ├── utils ├── hash.go ├── size_test.go ├── runtime.go ├── dist_test.go ├── size.go └── disk.go ├── Makefile ├── .gitignore ├── loghub ├── log_test.go ├── demolog.go ├── utils.go ├── accesslog.go ├── analysislog.go ├── log.go └── errorlog.go ├── go.sum ├── .github └── workflows │ └── go.yml ├── store ├── key_test.go ├── profile.go ├── offline.go ├── config_default.go ├── item_test.go ├── collision.go ├── key.go ├── leaf_test.go ├── hintmerge.go ├── hintindex.go ├── config.go ├── crc32.go ├── leaf.go ├── data_test.go ├── item.go ├── data.go ├── hintfile.go ├── datachunk.go ├── hint_test.go ├── datafile.go ├── gc.go └── htree_test.go ├── quicklz ├── quicklz_test.go ├── cquicklz.go ├── quicklz.h └── quicklz.go ├── config ├── server_config.go ├── mc_config.go ├── config.go ├── zk.go └── route.go ├── LICENSE ├── memcache ├── token.go ├── stats.go ├── store.go ├── protocol_test.go └── server.go └── README.md /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/pip-req.txt: -------------------------------------------------------------------------------- 1 | PyYAML==3.11 2 | libmc>=0.5.6 3 | nose>=1.3.7 4 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | Integrated tests for gobeansdb. -------------------------------------------------------------------------------- /gobeansdb/config_test.yaml: -------------------------------------------------------------------------------- 1 | mc: 2 | body_max_str: 50M 3 | hstore: 4 | local: 5 | homes: 6 | - /data1/beansdb 7 | - /data2/beasdb1 8 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/douban/gobeansdb/gobeansdb" 5 | ) 6 | 7 | func main() { 8 | gobeansdb.Main() 9 | } 10 | -------------------------------------------------------------------------------- /conf/route.yaml: -------------------------------------------------------------------------------- 1 | numbucket: 16 2 | backup: 3 | - "127.0.0.1:7989" 4 | main: 5 | - addr: 127.0.0.1:7980 6 | buckets: ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9", a, b, c, d, e, f] 7 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/douban/gobeansdb 2 | 3 | require ( 4 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da 5 | github.com/spaolacci/murmur3 v1.1.0 6 | gopkg.in/yaml.v2 v2.2.7 7 | ) 8 | 9 | go 1.13 10 | -------------------------------------------------------------------------------- /tests/run_test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | virtualenv venv 4 | source venv/bin/activate 5 | venv/bin/python venv/bin/pip install -r tests/pip-req.txt 6 | venv/bin/python venv/bin/nosetests --with-xunit --xunit-file=unittest.xml 7 | deactivate -------------------------------------------------------------------------------- /cmem/cmem_test.go: -------------------------------------------------------------------------------- 1 | package cmem 2 | 3 | import "testing" 4 | 5 | func TestCmem(t *testing.T) { 6 | size := 1024 * 1024 * 10 7 | var arr CArray 8 | if !arr.Alloc(size) { 9 | t.Fatalf("fail to alloc") 10 | } 11 | arr.Free() 12 | } 13 | -------------------------------------------------------------------------------- /utils/hash.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | type HashMethod func(v []byte) (h uint32) 4 | 5 | // Bugy version of fnv1a 6 | // 由于历史原因，最初使用了带有 bug 的 fnv1a，现在修正的代价比较大， 7 | // 涉及到数据的迁移. 8 | func Fnv1a(buf []byte) (h uint32) { 9 | PRIME := uint32(0x01000193) 10 | h = 0x811c9dc5 11 | for _, b := range buf { 12 | h ^= uint32(int8(b)) 13 | h = (h * PRIME) 14 | } 15 | return h 16 | } 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all:install 2 | 3 | test: 4 | go version 5 | go test github.com/douban/gobeansdb/memcache 6 | go test github.com/douban/gobeansdb/loghub 7 | go test github.com/douban/gobeansdb/cmem 8 | go test github.com/douban/gobeansdb/quicklz 9 | ulimit -n 1024; go test github.com/douban/gobeansdb/store 10 | 11 | pytest:install 12 | ./tests/run_test.sh 13 | 14 | install: 15 | go install ./ 16 | -------------------------------------------------------------------------------- /conf/local.yaml: -------------------------------------------------------------------------------- 1 | # for python test 2 | mc: 3 | body_c_str: 0 4 | server: 5 | port: 7980 6 | webport: 7988 7 | errorlog: ./gobeansdb.log 8 | accesslog: ./access.log 9 | analysislog: ./gobeansdb_analysis.log 10 | #zk: ["zk1:2181"] 11 | hstore: 12 | data: 13 | flush_interval: 0 14 | flush_wake_str: 0 15 | no_gc_days: 0 16 | local: 17 | homes: 18 | - ./testdb 19 | htree: 20 | tree_height: 3 21 | tree_dump : 1 22 | hint: 23 | hint_no_merged: false 24 | -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import errno 3 | import string 4 | import random 5 | 6 | 7 | def mkdir_p(path): 8 | "like `mkdir -p`" 9 | try: 10 | os.makedirs(path) 11 | except OSError as exc: 12 | if exc.errno == errno.EEXIST and os.path.isdir(path): 13 | pass 14 | else: 15 | raise 16 | 17 | 18 | def random_string(n): 19 | s = string.ascii_letters 20 | result = "" 21 | for _ in xrange(n): 22 | result += random.choice(s) 23 | return result -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | *.py[co] 6 | 7 | # Folders 8 | .idea/ 9 | _obj 10 | _test 11 | unittest.xml 12 | testdb/ 13 | src/gobeansdb/gobeansdb 14 | src/gopkg.in/ 15 | bin/ 16 | src/github.com/ 17 | venv/ 18 | set-env.sh 19 | 20 | # Architecture specific extensions/prefixes 21 | *.[568vq] 22 | [568vq].out 23 | 24 | *.cgo1.go 25 | *.cgo2.c 26 | _cgo_defun.c 27 | _cgo_gotypes.go 28 | _cgo_export.* 29 | 30 | _testmain.go 31 | 32 | *.exe 33 | *.test 34 | *.prof 35 | *.log 36 | 37 | .tag* 38 | main 39 | vendor/ 40 | -------------------------------------------------------------------------------- /cmem/beansdb.go: -------------------------------------------------------------------------------- 1 | package cmem 2 | 3 | var ( 4 | DBRL BeansdbRL 5 | ) 6 | 7 | func init() { 8 | DBRL.ResetAll() 9 | } 10 | 11 | type BeansdbRL struct { 12 | AllocRL *ResourceLimiter 13 | 14 | GetData ResourceLimiter 15 | SetData ResourceLimiter 16 | FlushData ResourceLimiter 17 | } 18 | 19 | func (dbrl *BeansdbRL) ResetAll() { 20 | dbrl.FlushData.reset() 21 | dbrl.SetData.reset() 22 | dbrl.GetData.reset() 23 | dbrl.AllocRL = &AllocRL 24 | AllocRL.reset() 25 | } 26 | 27 | func (dbrl *BeansdbRL) IsZero() bool { 28 | return dbrl.FlushData.IsZero() && dbrl.SetData.IsZero() && dbrl.GetData.IsZero() && dbrl.AllocRL.IsZero() 29 | } 30 | -------------------------------------------------------------------------------- /loghub/log_test.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "bytes" 5 | "log" 6 | "testing" 7 | ) 8 | 9 | func TestDemoHubger(t *testing.T) { 10 | w := new(bytes.Buffer) 11 | 12 | hub := NewDemoHub() 13 | backend := log.New(w, "", log.LstdFlags) 14 | name := "testSimple" 15 | 16 | userlogger := NewLogger(name, hub, WARN) 17 | hub.Bind(name, &DemoHubConfig{backend}) 18 | 19 | userlogger.Errorf("error") 20 | userlogger.Debugf("debug") 21 | exp := "2015/09/01 18:09:40 ERROR (log_test.go: 19) - error\n" 22 | res := w.String() 23 | if len(exp) != len(res) || exp[20:] != res[20:] { 24 | t.Errorf("\nwant: [%s]\ngot : [%s]", exp, res) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /utils/size_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "testing" 4 | 5 | func TestSizer(t *testing.T) { 6 | 7 | s := SizeToStr(100) 8 | if s != "100" { 9 | t.Fatal(s) 10 | } 11 | s = SizeToStr(1024) 12 | if s != "1K" { 13 | t.Fatal(s) 14 | } 15 | s = SizeToStr(1025) 16 | if s != "1025" { 17 | t.Fatal(s) 18 | } 19 | s = SizeToStr(1025 * 1024) 20 | if s != "1025K" { 21 | t.Fatal(s) 22 | } 23 | 24 | n := StrToSize("1") 25 | if n != 1 { 26 | t.Fatal(n) 27 | } 28 | n = StrToSize("1K") 29 | if n != 1024 { 30 | t.Fatal(n) 31 | } 32 | if LastSizeErr != nil { 33 | t.Fatal(LastSizeErr) 34 | } 35 | n = StrToSize("1p") 36 | if LastSizeErr == nil { 37 | t.Fatal(n) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /utils/runtime.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "runtime" 5 | "syscall" 6 | ) 7 | 8 | func GetStack(bytes int) string { 9 | b := make([]byte, bytes) 10 | all := false 11 | n := runtime.Stack(b, all) 12 | return string(b[:n]) 13 | } 14 | 15 | func GetMaxRSS() int64 { 16 | return Getrusage().Maxrss 17 | } 18 | 19 | func Getrusage() syscall.Rusage { 20 | var rusage syscall.Rusage 21 | syscall.Getrusage(syscall.RUSAGE_SELF, &rusage) 22 | // Mac OS (darwin) returns the RSS in bytes, Linux returns it in kilobytes (Check man getrusage). 23 | if runtime.GOOS == "darwin" { 24 | // Change Maxrss unit to kilobytes 25 | rusage.Maxrss = rusage.Maxrss / 1024 26 | } 27 | return rusage 28 | } 29 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= 2 | github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= 3 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 4 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 5 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 6 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 7 | gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= 8 | gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 9 | -------------------------------------------------------------------------------- /loghub/demolog.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "io" 5 | "log" 6 | ) 7 | 8 | type DemoHubConfig struct { 9 | logger *log.Logger 10 | } 11 | 12 | type DemoHub struct { 13 | configs map[string]*DemoHubConfig 14 | } 15 | 16 | func NewDemoHub() *DemoHub { 17 | s := new(DemoHub) 18 | s.configs = make(map[string]*DemoHubConfig) 19 | return s 20 | } 21 | 22 | func (l *DemoHub) Log(name string, level int, file string, line int, msg string) { 23 | l.configs[name].logger.Printf("%s (%10s:%4d) - %s", levelString[level], file, line, msg) 24 | } 25 | 26 | func (l *DemoHub) Bind(name string, config *DemoHubConfig) { 27 | l.configs[name] = config 28 | } 29 | 30 | func (l *DemoHub) Reopen(path string) error { 31 | return nil 32 | } 33 | 34 | func (l *DemoHub) GetLastLog() []byte { 35 | return nil 36 | } 37 | 38 | func (l *DemoHub) DumpBuffer(all bool, out io.Writer) { 39 | return 40 | } 41 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | on: [push, pull_request] 2 | name: GoBeansDB Test 3 | jobs: 4 | test: 5 | strategy: 6 | matrix: 7 | compiler: [gcc] 8 | go-version: [1.12.x, 1.13.x] 9 | platform: [ubuntu-latest, macos-latest] 10 | runs-on: ${{ matrix.platform }} 11 | steps: 12 | - name: Install Go 13 | uses: actions/setup-go@v1 14 | with: 15 | go-version: ${{ matrix.go-version }} 16 | 17 | - name: Checkout code 18 | uses: actions/checkout@v1 19 | 20 | - name: Get test tools 21 | run: go get -u -v golang.org/x/tools/cmd/goimports 22 | 23 | - name: Test 24 | env: 25 | CC: ${{ matrix.compiler }} 26 | run: | 27 | export PATH=${PATH}:`go env GOPATH`/bin 28 | diff <(goimports -d .) <(printf "") 29 | go mod vendor 30 | make test 31 | 32 | - name: Install 33 | run: make install 34 | -------------------------------------------------------------------------------- /store/key_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestIsValidKeyString(t *testing.T) { 9 | testKey := []struct { 10 | x string 11 | expect bool 12 | }{ 13 | {"hello", true}, 14 | {"?helo", false}, 15 | {" hello", false}, 16 | {"@hello", false}, 17 | {"你好", true}, 18 | {"hello*", true}, 19 | {strings.Repeat("a", MAX_KEY_LEN-1), true}, 20 | {strings.Repeat("a", MAX_KEY_LEN), true}, 21 | {strings.Repeat("a", MAX_KEY_LEN+1), false}, 22 | {"hello\n", false}, 23 | {"he llo", false}, 24 | {"he\tllo", false}, 25 | {"he\rllo", false}, 26 | } 27 | 28 | for _, tt := range testKey { 29 | result := IsValidKeyString(tt.x) 30 | if result != tt.expect { 31 | if tt.expect { 32 | t.Fatalf("key %s should be %s", tt.x, "valid") 33 | } else { 34 | t.Fatalf("key %s should be %s", tt.x, "invalid") 35 | } 36 | } 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /gobeansdb/config_test.go: -------------------------------------------------------------------------------- 1 | package gobeansdb 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "testing" 7 | 8 | yaml "gopkg.in/yaml.v2" 9 | 10 | "github.com/douban/gobeansdb/utils" 11 | ) 12 | 13 | func TestConfig(t *testing.T) { 14 | path := "config_test.yaml" 15 | data1, err := ioutil.ReadFile(path) 16 | if err != nil { 17 | t.Fatal("read config failed", path, err.Error()) 18 | } 19 | var c1 DBConfig 20 | c1.MaxKeyLen = 1 21 | err = yaml.Unmarshal([]byte(data1), &c1) 22 | if err != nil { 23 | t.Fatalf("unmarshal err:%v", err) 24 | } 25 | utils.InitSizesPointer(&c1) 26 | if c1.MaxKeyLen != 1 || c1.BodyMax != (50<<20) { 27 | t.Fatalf("\nc1:\n%#v\n", c1) 28 | } 29 | c1.MaxKeyLen = 0 30 | data2, e := yaml.Marshal(c1) 31 | if e != nil { 32 | t.Fatalf("err:%s", e.Error()) 33 | } 34 | if 0 != bytes.Compare(data1, data2) { 35 | tmp := path + ".tmp" 36 | ioutil.WriteFile(tmp, data2, 0644) 37 | t.Fatalf("diff %s %s", path, tmp) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /store/profile.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "runtime" 9 | "runtime/debug" 10 | "runtime/pprof" 11 | ) 12 | 13 | var ( 14 | doProf bool 15 | ) 16 | 17 | func FreeMem() { 18 | runtime.GC() 19 | debug.FreeOSMemory() 20 | } 21 | 22 | var ( 23 | profDir string 24 | ) 25 | 26 | func StartCpuProfile(name string) *os.File { 27 | if !doProf { 28 | return nil 29 | } 30 | name = fmt.Sprintf("%s.cpu", name) 31 | w, err := os.Create(filepath.Join(profDir, name)) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | pprof.StartCPUProfile(w) 36 | return w 37 | } 38 | 39 | func StopCpuProfile(f *os.File) { 40 | if !doProf { 41 | return 42 | } 43 | pprof.StopCPUProfile() 44 | f.Close() 45 | } 46 | 47 | func WriteHeapProfile(name string) { 48 | name = fmt.Sprintf("%s.heap", name) 49 | f, err := os.Create(filepath.Join(profDir, name)) 50 | if err != nil { 51 | log.Fatal(err) 52 | } 53 | pprof.WriteHeapProfile(f) 54 | f.Close() 55 | } 56 | -------------------------------------------------------------------------------- /utils/dist_test.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestDir(t *testing.T) { 9 | p := "testdir" 10 | err := os.Mkdir(p, 0777) 11 | if err != nil { 12 | t.Fatal(err) 13 | } 14 | defer os.RemoveAll(p) 15 | 16 | f, err := os.OpenFile(p+"/a", os.O_CREATE|os.O_WRONLY, 0644) 17 | if err != nil { 18 | t.Fatal(err) 19 | } 20 | f.Close() 21 | f, err = os.OpenFile(p+"/b", os.O_CREATE|os.O_WRONLY, 0644) 22 | f.WriteString("abc") 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | f.Close() 27 | d := NewDir() 28 | d.Load(p) 29 | dir, diff1, diff2, err := d.CheckPath(p) 30 | if err != nil || diff1 != nil || diff2 != nil { 31 | t.Fatal(d.ToSlice(), dir.ToSlice(), diff1, diff2, err) 32 | } 33 | d.Delete("a") 34 | d.Set("b", 1) 35 | d.Set("c", 2) 36 | dir, diff1, diff2, err = d.CheckPath(p) 37 | if err != nil || diff1[0].Size != 1 || diff1[1].Size != 2 || diff2[0].Size != 0 || diff2[1].Size != 3 { 38 | t.Fatal(d.ToSlice(), dir.ToSlice(), diff1, diff2, err) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /quicklz/quicklz_test.go: -------------------------------------------------------------------------------- 1 | package quicklz 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestQuicklz(t *testing.T) { 8 | //example from http://www.quicklz.com/manual.html 9 | orig := "LZ compression is based on finding repeated strings: Five, six, seven, eight, nine, fifteen, sixteen, seventeen, fifteen, sixteen, seventeen." 10 | compressed := Compress([]byte(orig), 3) 11 | l := len(orig) 12 | lc := len(compressed) 13 | if lc != 116 { 14 | t.Errorf("wrong compressed len %d", lc) 15 | } 16 | s := SizeDecompressed(compressed) 17 | sc := SizeCompressed(compressed) 18 | if s != l || sc != lc { 19 | t.Errorf("bad size meta: %d != %d or %d != %d", s, l, sc, lc) 20 | } 21 | decompressed := Decompress(compressed) 22 | ld := len(decompressed) 23 | if ld != l { 24 | t.Errorf("wrong decompressed len %d", lc) 25 | } 26 | compressed2, ok := CCompress([]byte(orig)) 27 | if !ok { 28 | t.Fatalf("CCompress fail") 29 | } 30 | decompressed2, err := CDecompress(compressed2.Body, s) 31 | if err != nil || string(decompressed2.Body) != orig { 32 | t.Errorf("decompress fail %d %v", lc, err) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /config/server_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "fmt" 4 | 5 | var ( 6 | DefaultServerConfig = ServerConfig{ 7 | Hostname: "127.0.0.1", 8 | Listen: "0.0.0.0", 9 | Port: 7900, 10 | WebPort: 7903, 11 | Threads: 4, 12 | ZKServers: nil, 13 | ErrorLog: "./gobeansdb.log", 14 | AccessLog: "", 15 | StaticDir: "./", 16 | } 17 | ) 18 | 19 | type ServerConfig struct { 20 | Hostname string `yaml:",omitempty"` 21 | ZKPath string `yaml:",omitempty"` // root path in zk 22 | ZKServers []string `yaml:",omitempty"` // e.g. "zk1:2181,zk2:2181" 23 | Listen string `yaml:",omitempty"` // ip 24 | Port int `yaml:",omitempty"` 25 | WebPort int `yaml:",omitempty"` 26 | Threads int `yaml:",omitempty"` // NumCPU 27 | ErrorLog string `yaml:",omitempty"` 28 | AccessLog string `yaml:",omitempty"` 29 | AnalysisLog string `yaml:",omitempty"` 30 | StaticDir string `yaml:",omitempty"` // directory for static files, e.g. *.html 31 | 32 | } 33 | 34 | func (c *ServerConfig) Addr() string { 35 | return fmt.Sprintf("%s:%d", c.Hostname, c.Port) 36 | } 37 | -------------------------------------------------------------------------------- /conf/global.yaml: -------------------------------------------------------------------------------- 1 | # for doubandb 2 | # should include all configurable fields 3 | server: 4 | zkserves: [] 5 | zkpath: "/beansdb/test" 6 | listen: 0.0.0.0 7 | port: 7900 8 | webport: 7903 9 | threads: 4 10 | errorlog: /var/log/gobeansdb/gobeansdb.log 11 | accesslog: "" 12 | analysislog: /var/log/gobeansdb/gobeansdb_analysis.log 13 | hostname: 127.0.0.1 # 线上必须在local文件里改掉 14 | staticdir: /var/lib/gobeansdb 15 | mc: 16 | max_key_len: 250 17 | max_req: 16 18 | body_max_str: 50M 19 | body_big_str: 5M 20 | body_c_str: 4K 21 | flush_max_str: 100M 22 | hstore: 23 | data: 24 | flush_interval: 60 25 | flush_wake_str: 10M 26 | datafile_max_str: 4000M 27 | check_vhash: true 28 | no_gc_days: 7 29 | not_compress: 30 | "audio/mpeg": true 31 | "audio/wave": true 32 | "audio/ogg": true 33 | "audio/midi": true 34 | hint: 35 | hint_no_merged: true 36 | hint_split_cap_str: 1M 37 | hint_index_interval_str: 32K 38 | hint_merge_interval: 5 39 | htree: 40 | tree_height: 7 41 | tree_dump : 3 42 | local: 43 | homes: 44 | - /var/lib/beansdb 45 | -------------------------------------------------------------------------------- /config/mc_config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | var ( 4 | DefaultMCConfig = MCConfig{ 5 | MaxReq: 16, 6 | MaxKeyLen: 250, 7 | BodyMaxStr: "50M", 8 | BodyBigStr: "1M", 9 | BodyInCStr: "4K", 10 | FlushMaxStr: "100M", 11 | TimeoutMS: 3000, 12 | } 13 | ) 14 | 15 | type MCConfig struct { 16 | MaxKeyLen int `yaml:"max_key_len,omitempty"` 17 | MaxReq int `yaml:"max_req,omitempty"` // max num of requsets serve at the same time 18 | 19 | BodyMax int64 `yaml:"-"` // fail set/read_file if larger then this 20 | BodyBig int64 `yaml:"-"` // set may fail if memory is in shorage (determine by "storage") 21 | BodyInC int64 `yaml:"-"` // alloc body in cgo if larger then this 22 | 23 | FlushMax int64 `yaml:"-"` // if flush buffer is larger, may fail BIG set request (return NOT_FOUND) 24 | 25 | FlushMaxStr string `yaml:"flush_max_str"` 26 | BodyMaxStr string `yaml:"body_max_str,omitempty"` 27 | BodyBigStr string `yaml:"body_big_str,omitempty"` 28 | 29 | BodyInCStr string `yaml:"body_c_str,omitempty"` 30 | TimeoutMS int `yaml:"timeout_ms,omitempty"` 31 | } 32 | 33 | func IsValidKeySize(ksz uint32) bool { 34 | return ksz != 0 && ksz <= uint32(MCConf.MaxKeyLen) 35 | } 36 | 37 | func IsValidValueSize(vsz uint32) bool { 38 | return vsz <= uint32(MCConf.BodyMax) 39 | } 40 | -------------------------------------------------------------------------------- /store/offline.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "strconv" 7 | ) 8 | 9 | func DataToHint(path string) (err error) { 10 | f, err := os.Open(path) 11 | if err != nil { 12 | return 13 | } 14 | defer f.Close() 15 | finfo, err := f.Stat() 16 | if err != nil { 17 | return 18 | } 19 | if finfo.IsDir() { 20 | return DataToHintDir(path, 0, MAX_NUM_CHUNK-1) 21 | } 22 | return DataToHintFile(path) 23 | } 24 | 25 | func DataToHintDir(path string, start, end int) (err error) { 26 | bkt := &Bucket{} 27 | bkt.datas = NewdataStore(0, path) 28 | bkt.hints = newHintMgr(0, path) 29 | _, err = bkt.datas.ListFiles() 30 | if err != nil { 31 | return 32 | } 33 | 34 | for i := start; i <= end; i++ { 35 | data := bkt.datas.genPath(i) 36 | _, err := os.Stat(data) 37 | if err != nil { 38 | continue 39 | } 40 | logger.Infof("building %s", data) 41 | err = bkt.checkHintWithData(i) 42 | if err != nil { 43 | logger.Errorf("error build %d", i) 44 | } 45 | } 46 | return 47 | } 48 | 49 | func DataToHintFile(path string) (err error) { 50 | dir := filepath.Dir(path) 51 | name := filepath.Base(path) 52 | chunkID, err := strconv.Atoi(name[:3]) 53 | if err != nil { 54 | return 55 | } 56 | return DataToHintDir(dir, chunkID, chunkID) 57 | } 58 | -------------------------------------------------------------------------------- /loghub/utils.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "runtime" 7 | "sync/atomic" 8 | "unsafe" 9 | ) 10 | 11 | func GetStack(size int) string { 12 | b := make([]byte, size) 13 | n := runtime.Stack(b, false) 14 | stack := string(b[:n]) 15 | return stack 16 | } 17 | 18 | func openLogWithFd(fd *os.File, logFlag int) *log.Logger { 19 | return log.New(fd, "", logFlag) 20 | } 21 | 22 | func openLog(path string, logFlag int) (logger *log.Logger, fd *os.File, err error) { 23 | if fd, err = os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644); err == nil { 24 | logger = openLogWithFd(fd, logFlag) 25 | } 26 | return 27 | } 28 | 29 | func reopenLogger(logger **log.Logger, fd **os.File, path string, logFlag int) (err error) { 30 | // start swap exist logger and new logger, and Close the older fd in later 31 | newLogger, newFd, err := openLog(path, logFlag) 32 | if err == nil { 33 | atomic.SwapPointer((*unsafe.Pointer)(unsafe.Pointer(logger)), unsafe.Pointer(newLogger)) 34 | oldFd := (*os.File)(atomic.SwapPointer((*unsafe.Pointer)(unsafe.Pointer(fd)), unsafe.Pointer(newFd))) 35 | if e := oldFd.Close(); e != nil { 36 | log.Println("close old log fd failure with, ", e) 37 | } 38 | } else { 39 | log.Printf("reopenLogger %s failed: %s", path, err.Error()) 40 | } 41 | return 42 | } 43 | -------------------------------------------------------------------------------- /loghub/accesslog.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "os" 7 | ) 8 | 9 | var ( 10 | AccessLogFormat = "%s" 11 | AccessLogFlag = (log.Ldate | log.Ltime | log.Lmicroseconds) 12 | AccessLogger *Logger 13 | ) 14 | 15 | type AccessLogHub struct { 16 | logger *log.Logger 17 | logFd *os.File 18 | } 19 | 20 | func init() { 21 | AccessLogger = NewLogger("", nil, DEBUG) 22 | } 23 | 24 | func InitAccessLog(path string, level int) (err error) { 25 | if accessLog, accesssFd, err := openLog(path, AccessLogFlag); err == nil { 26 | hub := &AccessLogHub{logger: accessLog, logFd: accesssFd} 27 | AccessLogger.Hub = hub 28 | AccessLogger.SetLevel(level) 29 | } else { 30 | log.Fatalf("open access log error, path=[%s], err=[%s]", path, err.Error()) 31 | } 32 | return 33 | } 34 | 35 | func (hub *AccessLogHub) Log(name string, level int, file string, line int, msg string) { 36 | hub.logger.Printf(AccessLogFormat, msg) 37 | if level == FATAL { 38 | os.Exit(1) 39 | } 40 | } 41 | 42 | func (hub *AccessLogHub) Reopen(path string) (err error) { 43 | return reopenLogger(&hub.logger, &hub.logFd, path, AccessLogFlag) 44 | } 45 | 46 | func (hub *AccessLogHub) GetLastLog() []byte { 47 | // not implement 48 | return nil 49 | } 50 | 51 | func (hub *AccessLogHub) DumpBuffer(all bool, out io.Writer) { 52 | // not implement 53 | } 54 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | 8 | yaml "gopkg.in/yaml.v2" 9 | 10 | "github.com/douban/gobeansdb/utils" 11 | ) 12 | 13 | // `Version` can be changed in gobeansproxy. 14 | var Version = "2.1.0.18" 15 | 16 | const AccessLogVersion = "V1" 17 | const AnalysisLogVersion = "V2" 18 | 19 | var ( 20 | ServerConf ServerConfig = DefaultServerConfig 21 | Route RouteTable 22 | MCConf MCConfig = DefaultMCConfig 23 | AllowReload bool 24 | ) 25 | 26 | func init() { 27 | utils.InitSizesPointer(&MCConf) 28 | for i := 0; i < DefaultRouteConfig.NumBucket; i++ { 29 | DefaultRouteConfig.BucketsStat[i] = 1 30 | DefaultRouteConfig.BucketsHex = append(DefaultRouteConfig.BucketsHex, BucketIDHex(i, DefaultRouteConfig.NumBucket)) 31 | } 32 | } 33 | 34 | func LoadYamlConfig(config interface{}, path string) error { 35 | content, err := ioutil.ReadFile(path) 36 | if err != nil { 37 | return err 38 | } 39 | return yaml.Unmarshal(content, config) 40 | } 41 | 42 | func DumpConfig(config interface{}) { 43 | b, err := yaml.Marshal(config) 44 | if err != nil { 45 | log.Fatalf("%s", err) 46 | } else { 47 | fmt.Println(string(b)) 48 | } 49 | } 50 | 51 | func BucketIDHex(id, numBucket int) string { 52 | if numBucket == 16 { 53 | return fmt.Sprintf("%x", id) 54 | } else if numBucket == 256 { 55 | return fmt.Sprintf("%02x", id) 56 | } 57 | return "0" 58 | } 59 | -------------------------------------------------------------------------------- /store/config_default.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import "github.com/douban/gobeansdb/config" 4 | 5 | /* 6 | 7 | 1. easy to start: 8 | Homes: []string{"./testdb"}, 9 | TreeHeight: 3 10 | Hostname: "127.0.0.1" 11 | serve all buckets 12 | 13 | 2. easy for test: 14 | flush at once 15 | FlushInterval: 0, 16 | FlushWakeStr: "0" 17 | CheckVHash: false 18 | NoMerged: false 19 | MergeInterval: 1 20 | 21 | 3. reasonable setting for others 22 | 23 | */ 24 | 25 | var ( 26 | DefaultHintConfig = HintConfig{ 27 | NoMerged: false, 28 | SplitCapStr: "1M", 29 | IndexIntervalSizeStr: "4K", 30 | MergeInterval: 1, 31 | } 32 | 33 | DefaultHTreeConfig HTreeConfig = HTreeConfig{ 34 | TreeHeight: 3, 35 | TreeDump: 3, 36 | } 37 | 38 | DefaultDataConfig = DataConfig{ 39 | DataFileMaxStr: "4000M", 40 | CheckVHash: false, 41 | FlushInterval: 0, 42 | FlushWakeStr: "0", 43 | BufIOCapStr: "1M", 44 | 45 | NoGCDays: 0, 46 | NotCompress: map[string]bool{ 47 | "audio/wave": true, 48 | "audio/mpeg": true, 49 | }, 50 | } 51 | 52 | DefaultDBLocalConfig = DBLocalConfig{ 53 | Home: "./testdb", 54 | } 55 | ) 56 | 57 | func (c *HStoreConfig) InitDefault() { 58 | c.HintConfig = DefaultHintConfig 59 | c.HTreeConfig = DefaultHTreeConfig 60 | c.DataConfig = DefaultDataConfig 61 | c.DBLocalConfig = DefaultDBLocalConfig 62 | c.DBRouteConfig = config.DefaultRouteConfig 63 | } 64 | -------------------------------------------------------------------------------- /loghub/analysislog.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "time" 7 | ) 8 | 9 | var ( 10 | AnalysisLogFormat = "%s" 11 | AnalysisLogFlag = log.LstdFlags | log.Lmicroseconds 12 | AnalysisLogger *Logger 13 | ) 14 | 15 | type AnalysisLogHub struct { 16 | logger *log.Logger 17 | logFd *os.File 18 | BufferLog 19 | } 20 | 21 | func init() { 22 | //logger := openLogWithFd(os.Stderr, AnalysisLogFlag) 23 | //hub := &AnalysisLogHub{logger: logger} 24 | //hub.InitBuffer(200) 25 | AnalysisLogger = NewLogger("", nil, DEBUG) 26 | } 27 | 28 | func InitAnalysisLog(path string, level int, bufferSize int) (err error) { 29 | if analysisLog, analysisFd, err := openLog(path, AnalysisLogFlag); err == nil { 30 | hub := &AnalysisLogHub{logger: analysisLog, logFd: analysisFd} 31 | hub.InitBuffer(bufferSize) 32 | AnalysisLogger.Hub = hub 33 | AnalysisLogger.SetLevel(level) 34 | } else { 35 | log.Fatalf("open log error, path=[%s], err=[%s]", path, err.Error()) 36 | } 37 | return 38 | } 39 | 40 | func (hub *AnalysisLogHub) Log(name string, level int, file string, line int, msg string) { 41 | hub.logger.Printf(AnalysisLogFormat, msg) 42 | bufline := &BufferLine{time.Now(), level, file, line, msg} 43 | hub.Add(bufline) 44 | hub.Lock() 45 | hub.Last[level] = bufline 46 | hub.Unlock() 47 | if level == FATAL { 48 | os.Exit(1) 49 | } 50 | } 51 | 52 | func (hub *AnalysisLogHub) Reopen(path string) (err error) { 53 | return reopenLogger(&hub.logger, &hub.logFd, path, AnalysisLogFlag) 54 | } 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2018, Douban Inc. 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /utils/size.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "reflect" 5 | "strconv" 6 | "strings" 7 | ) 8 | 9 | const ( 10 | K = 1024 11 | M = 1024 * K 12 | G = 1024 * M 13 | ) 14 | 15 | var ( 16 | LastSizeErr error 17 | sizemap = map[byte]int{'K': K, 'k': K, 'M': M, 'm': M, 'G': G, 'g': G} 18 | ) 19 | 20 | func StrToSize(s string) (n int64) { 21 | l := len(s) 22 | if l == 0 { 23 | return 0 24 | } 25 | u := s[l-1] 26 | plus, found := sizemap[u] 27 | if found { 28 | s = s[:l-1] 29 | } 30 | n, e := strconv.ParseInt(s, 10, 32) 31 | if e != nil { 32 | LastSizeErr = e 33 | } else if found { 34 | n *= int64(plus) 35 | } 36 | return 37 | } 38 | 39 | func SizeToStr(n int64) (s string) { 40 | if n == 0 { 41 | return "0" 42 | } 43 | units := []string{"", "K", "M", "G"} 44 | i := 0 45 | for n&(1024-1) == 0 { 46 | n >>= 10 47 | i++ 48 | } 49 | return strconv.FormatInt(n, 10) + units[i] 50 | } 51 | 52 | func InitSizesPointer(c interface{}) (err error) { 53 | LastSizeErr = nil 54 | m := reflect.ValueOf(c).Elem() 55 | InitSizesForValue(m) 56 | err = LastSizeErr 57 | LastSizeErr = nil 58 | return 59 | } 60 | 61 | func InitSizesForValue(m reflect.Value) (err error) { 62 | t := m.Type() 63 | n := m.NumField() 64 | for i := 0; i < n; i++ { 65 | f := t.Field(i) 66 | 67 | if strings.HasSuffix(f.Name, "Config") { // nested config 68 | InitSizesForValue(m.Field(i)) 69 | } else if strings.HasSuffix(f.Name, "Str") { 70 | str := m.Field(i).String() 71 | value := StrToSize(str) 72 | f2 := m.FieldByName(f.Name[:len(f.Name)-3]) 73 | f2.SetInt(value) 74 | } 75 | } 76 | return 77 | } 78 | -------------------------------------------------------------------------------- /config/zk.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "strconv" 8 | "time" 9 | 10 | "github.com/samuel/go-zookeeper/zk" 11 | ) 12 | 13 | var ( 14 | ZKClient *zkClient 15 | LocalRoutePath string 16 | ) 17 | 18 | type zkClient struct { 19 | Version int 20 | Root string 21 | Servers []string 22 | Client *zk.Conn 23 | Events <-chan zk.Event 24 | } 25 | 26 | func NewZK(root string, servers []string) (c *zkClient, err error) { 27 | c = &zkClient{Root: root, Servers: servers} 28 | c.Client, c.Events, err = zk.Connect(servers, 10*time.Second) 29 | return c, err 30 | } 31 | 32 | func (c *zkClient) Get(subPath string) ([]byte, *zk.Stat, error) { 33 | path := fmt.Sprintf("%s/%s", c.Root, subPath) 34 | log.Printf("get zk %s", path) 35 | return c.Client.Get(path) 36 | } 37 | 38 | func (c *zkClient) GetRouteRaw(version int) (data []byte, ver int, err error) { 39 | if version < 0 { 40 | data, _, err = c.Get("route") 41 | if err != nil { 42 | log.Printf("%v", err) 43 | return 44 | } 45 | log.Printf("got route version %s", string(data)) 46 | ver, err = strconv.Atoi(string(data)) 47 | if err != nil { 48 | return 49 | } 50 | } else { 51 | ver = version 52 | } 53 | sp := fmt.Sprintf("route/route_%010d", ver) 54 | data, _, err = c.Get(sp) 55 | if err != nil { 56 | log.Printf("%v", err) 57 | } 58 | return 59 | } 60 | 61 | func UpdateLocalRoute(content []byte) { 62 | log.Printf("update local route %s", LocalRoutePath) 63 | fd, err := os.OpenFile(LocalRoutePath, os.O_CREATE|os.O_WRONLY, 0644) 64 | if err != nil { 65 | log.Fatalf("fail to write: %s", LocalRoutePath) 66 | } 67 | fd.Write(content) 68 | fd.Close() 69 | } 70 | -------------------------------------------------------------------------------- /store/item_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | var sniffTests = []struct { 8 | desc string 9 | data []byte 10 | contentType string 11 | result bool 12 | }{ 13 | // Audio types. 14 | {"MIDI audio", []byte("MThd\x00\x00\x00\x06\x00\x01"), "audio/midi", true}, 15 | {"MP3 audio/MPEG audio", []byte("ID3\x03\x00\x00\x00\x00\x0f"), "audio/mpeg", false}, 16 | {"WAV audio #1", []byte("RIFFb\xb8\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave", false}, 17 | {"WAV audio #2", []byte("RIFF,\x00\x00\x00WAVEfmt \x12\x00\x00\x00\x06"), "audio/wave", false}, 18 | {"AIFF audio #1", []byte("FORM\x00\x00\x00\x00AIFFCOMM\x00\x00\x00\x12\x00\x01\x00\x00\x57\x55\x00\x10\x40\x0d\xf3\x34"), "audio/aiff", true}, 19 | {"OGG audio", []byte("OggS\x00\x02\x00\x00\x00\x00\x00\x00\x00\x00\x7e\x46\x00\x00\x00\x00\x00\x00\x1f\xf6\xb4\xfc\x01\x1e\x01\x76\x6f\x72"), "application/ogg", true}, 20 | {"MP3 audio/MPEG audio #2", []byte("ID3\x03\x00\x00\x00\x00\x04"), "audio/mpeg", false}, 21 | {"MP3 audio/MPEG audio #3", []byte("ID3\x03\x00\x00\x00\x00\x0a"), "audio/mpeg", false}, 22 | } 23 | 24 | func ValueWithKind(header []byte, size int) []byte { 25 | b := make([]byte, size) 26 | offsetHeader := len(header) 27 | for i := 0; i < offsetHeader; i++ { 28 | b[i] = byte(header[i]) 29 | } 30 | return b 31 | } 32 | 33 | func TestCompress(t *testing.T) { 34 | Conf.InitDefault() 35 | Conf.NotCompress = map[string]bool{ 36 | "audio/mpeg": true, 37 | "audio/wave": true, 38 | } 39 | t.Log(Conf.NotCompress) 40 | 41 | for _, st := range sniffTests { 42 | 43 | value := ValueWithKind(st.data, 512) 44 | header := value[:512] 45 | if result := NeedCompress(header); result != st.result { 46 | t.Errorf("type %s compress is error, except %v , but got %v", st.contentType, st.result, result) 47 | } 48 | 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /memcache/token.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "sync/atomic" 5 | "time" 6 | 7 | "github.com/douban/gobeansdb/config" 8 | ) 9 | 10 | type ReqHistoy struct { 11 | Cmd string 12 | Keys []string 13 | WaitTime time.Duration 14 | ServeStart time.Time 15 | ServeTime time.Duration 16 | Stat string 17 | StatStart time.Time 18 | Working bool 19 | } 20 | 21 | type ReqLimiter struct { 22 | Chan chan int `json:"-"` 23 | Owners []*Request `json:"-"` 24 | Histories []ReqHistoy 25 | 26 | // TODO: more! 27 | NumWait int32 28 | MaxWait time.Duration 29 | } 30 | 31 | func NewReqLimiter(n int) *ReqLimiter { 32 | rl := &ReqLimiter{} 33 | rl.Chan = make(chan int, n) 34 | for i := 0; i < n; i++ { 35 | rl.Chan <- i 36 | } 37 | rl.Owners = make([]*Request, n) 38 | rl.Histories = make([]ReqHistoy, n) 39 | return rl 40 | } 41 | 42 | func (rl *ReqLimiter) Get(req *Request) { 43 | st := time.Now() 44 | atomic.AddInt32(&(rl.NumWait), 1) 45 | t := <-rl.Chan 46 | req.Token = t 47 | 48 | req.Working = true 49 | //logger.Debugf("get %d", t) 50 | atomic.AddInt32(&rl.NumWait, -1) 51 | ed := time.Now() 52 | 53 | d := ed.Sub(st) 54 | 55 | rl.Histories[t] = ReqHistoy{ 56 | Cmd: req.Cmd, 57 | Keys: req.Keys, 58 | ServeStart: time.Now(), 59 | WaitTime: d, 60 | Working: true, 61 | } 62 | if d > rl.MaxWait { 63 | rl.MaxWait = d 64 | } 65 | rl.Owners[t] = req 66 | 67 | return 68 | } 69 | 70 | func (rl *ReqLimiter) Put(req *Request) { 71 | t := req.Token 72 | //logger.Debugf("put %d", t) 73 | rl.Histories[t].ServeTime = time.Since(rl.Histories[t].ServeStart) 74 | rl.Histories[t].Working = false 75 | rl.Owners[t].Working = false 76 | rl.Chan <- t 77 | } 78 | 79 | func InitTokens() { 80 | n := config.MCConf.MaxReq 81 | if n == 0 { 82 | n = 16 83 | } 84 | RL = NewReqLimiter(n) 85 | } 86 | -------------------------------------------------------------------------------- /store/collision.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "io/ioutil" 5 | "strings" 6 | "sync" 7 | 8 | yaml "gopkg.in/yaml.v2" 9 | ) 10 | 11 | type CollisionTable struct { 12 | sync.Mutex `yaml:"-"` 13 | HintID 14 | Items map[uint64]map[string]HintItem 15 | } 16 | 17 | func newCollisionTable() *CollisionTable { 18 | t := &CollisionTable{} 19 | t.Items = make(map[uint64]map[string]HintItem) 20 | return t 21 | } 22 | 23 | func (table *CollisionTable) get(keyhash uint64, key string) (item *HintItem, ok bool) { 24 | table.Lock() 25 | defer table.Unlock() 26 | items, ok := table.Items[keyhash] 27 | if ok { 28 | if it, ok2 := items[key]; ok2 { 29 | item = &it 30 | } 31 | } 32 | return 33 | } 34 | 35 | // gc should not use this func 36 | func (table *CollisionTable) compareAndSet(it *HintItem, reason string) { 37 | logger.Infof("%s, set collision %#v", reason, it) 38 | table.Lock() 39 | defer table.Unlock() 40 | items, ok := table.Items[it.Keyhash] 41 | if ok { 42 | old, ok := items[it.Key] 43 | if !ok || reason == "gc" || it.Pos.CmpKey() >= old.Pos.CmpKey() { 44 | items[it.Key] = *it 45 | } 46 | 47 | } else { 48 | items = make(map[string]HintItem) 49 | items[it.Key] = *it 50 | table.Items[it.Keyhash] = items 51 | } 52 | } 53 | 54 | func (table *CollisionTable) dumps() (content []byte) { 55 | table.Lock() 56 | content, _ = yaml.Marshal(table) 57 | table.Unlock() 58 | return 59 | } 60 | 61 | func (table *CollisionTable) dump(path string) { 62 | table.Lock() 63 | content, err := yaml.Marshal(table) 64 | table.Unlock() 65 | if err != nil { 66 | logger.Errorf("unmarshal yaml faild %s: %s", path, err.Error()) 67 | return 68 | } 69 | err = ioutil.WriteFile(path, content, 0644) 70 | if err != nil { 71 | logger.Errorf("write yaml failed %s: %s", path, err.Error()) 72 | } 73 | } 74 | 75 | func (table *CollisionTable) load(path string) { 76 | content, err := ioutil.ReadFile(path) 77 | if err != nil { 78 | if !strings.Contains(err.Error(), "no such file or directory") { 79 | logger.Errorf("read yaml failed %s: %s", path, err.Error()) 80 | } 81 | return 82 | } 83 | table.Lock() 84 | if err := yaml.Unmarshal(content, table); err != nil { 85 | logger.Errorf("unmarshal yaml faild %s %s", path, err.Error()) 86 | } 87 | table.Unlock() 88 | logger.Infof("load collisoin %s: %s", path, table.dumps()) 89 | } 90 | -------------------------------------------------------------------------------- /gobeansdb/config.go: -------------------------------------------------------------------------------- 1 | package gobeansdb 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "github.com/douban/gobeansdb/config" 9 | "github.com/douban/gobeansdb/store" 10 | "github.com/douban/gobeansdb/utils" 11 | ) 12 | 13 | type DBConfig struct { 14 | config.ServerConfig `yaml:"server,omitempty"` 15 | config.MCConfig `yaml:"mc,omitempty"` 16 | store.HStoreConfig `yaml:"hstore,omitempty"` 17 | } 18 | 19 | func (c *DBConfig) ConfigPackages() { 20 | config.ServerConf = c.ServerConfig 21 | config.MCConf = c.MCConfig 22 | store.Conf = &c.HStoreConfig 23 | } 24 | 25 | func (c *DBConfig) Load(confdir string) { 26 | c.InitDefault() 27 | 28 | if confdir != "" { 29 | // global 30 | path := fmt.Sprintf("%s/%s", confdir, "global.yaml") 31 | err := config.LoadYamlConfig(c, path) 32 | if err != nil { 33 | log.Fatalf("bad config %s: %s", path, err.Error()) 34 | } 35 | c.checkEmptyConfig(path) 36 | 37 | //local 38 | path = fmt.Sprintf("%s/%s", confdir, "local.yaml") 39 | if _, e := os.Stat(path); e == nil { 40 | err = config.LoadYamlConfig(c, path) 41 | if err != nil { 42 | log.Fatalf("bad config %s: %s", path, err.Error()) 43 | } 44 | c.checkEmptyConfig(path) 45 | } 46 | 47 | routePath := fmt.Sprintf("%s/%s", confdir, "route.yaml") 48 | var route *config.RouteTable 49 | // route 50 | if len(c.ZKServers) > 0 { 51 | route, err = config.LoadRouteTableZK(routePath, c.ZKPath, c.ZKServers) 52 | if err != nil { 53 | log.Printf("fail to load route table from zk: %s", err.Error()) 54 | } 55 | } 56 | if len(c.ZKServers) == 0 || err != nil { 57 | route, err = config.LoadRouteTableLocal(routePath) 58 | } 59 | if err != nil { 60 | log.Fatalf("fail to load route table: %s", err.Error()) 61 | } 62 | c.DBRouteConfig = route.GetDBRouteConfig(c.Addr()) 63 | config.Route = *route 64 | } 65 | utils.InitSizesPointer(c) 66 | err := c.HStoreConfig.InitTree() 67 | if err != nil { 68 | log.Fatalf("bad config: %s", err.Error()) 69 | } 70 | c.ConfigPackages() 71 | } 72 | 73 | func (c *DBConfig) InitDefault() { 74 | c.ServerConfig = config.DefaultServerConfig 75 | c.MCConfig = config.DefaultMCConfig 76 | c.HStoreConfig.InitDefault() 77 | utils.InitSizesPointer(c) 78 | } 79 | 80 | func (c *DBConfig) checkEmptyConfig(path string) { 81 | if c.MaxKeyLen == 0 || c.SplitCapStr == "" || c.TreeHeight == 0 { 82 | log.Fatal("bad config: empty struct in ", path) 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /memcache/stats.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "os" 5 | "runtime" 6 | "time" 7 | 8 | "github.com/douban/gobeansdb/utils" 9 | ) 10 | 11 | type Stats struct { 12 | start time.Time 13 | curr_item, total_items int64 14 | cmd_get, cmd_set, cmd_delete int64 15 | get_hits, get_misses int64 16 | threads int64 17 | curr_connections, total_connections int64 18 | bytes_read, bytes_written int64 19 | slow_cmd int64 // slow_cmd is not a stats in memecached protocol 20 | } 21 | 22 | func NewStats() *Stats { 23 | s := new(Stats) 24 | s.start = time.Now() 25 | return s 26 | } 27 | 28 | func mem_in_go(include_zero bool) runtime.MemProfileRecord { 29 | var p []runtime.MemProfileRecord 30 | n, ok := runtime.MemProfile(nil, include_zero) 31 | for { 32 | // Allocate room for a slightly bigger profile, 33 | // in case a few more entries have been added 34 | // since the call to MemProfile. 35 | p = make([]runtime.MemProfileRecord, n+50) 36 | n, ok = runtime.MemProfile(p, include_zero) 37 | if ok { 38 | p = p[0:n] 39 | break 40 | } 41 | // Profile grew; try again. 42 | } 43 | 44 | var total runtime.MemProfileRecord 45 | for i := range p { 46 | r := &p[i] 47 | total.AllocBytes += r.AllocBytes 48 | total.AllocObjects += r.AllocObjects 49 | total.FreeBytes += r.FreeBytes 50 | total.FreeObjects += r.FreeObjects 51 | } 52 | return total 53 | } 54 | 55 | func (s *Stats) Stats() map[string]int64 { 56 | st := make(map[string]int64) 57 | st["cmd_get"] = s.cmd_get 58 | st["cmd_set"] = s.cmd_set 59 | st["cmd_delete"] = s.cmd_delete 60 | st["get_hits"] = s.get_hits 61 | st["get_misses"] = s.get_misses 62 | st["curr_connections"] = s.curr_connections 63 | st["total_connections"] = s.total_connections 64 | st["bytes_read"] = s.bytes_read 65 | st["bytes_written"] = s.bytes_written 66 | st["slow_cmd"] = s.slow_cmd 67 | 68 | t := time.Now() 69 | st["time"] = int64(t.Unix()) 70 | st["uptime"] = int64(t.Sub(s.start).Seconds()) 71 | st["pid"] = int64(os.Getpid()) 72 | st["threads"] = int64(runtime.NumGoroutine()) 73 | rusage := utils.Getrusage() 74 | st["rusage_user"] = int64(rusage.Utime.Sec) 75 | st["rusage_system"] = int64(rusage.Stime.Sec) 76 | st["rusage_maxrss"] = rusage.Maxrss 77 | st["avail_space"] = 0 78 | st["total_space"] = 0 79 | st["curr_items"] = 0 80 | return st 81 | } 82 | -------------------------------------------------------------------------------- /tests/abnormal_cmd_test.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import telnetlib 3 | import unittest 4 | from tests.dbclient import MCStore 5 | from tests.base import BeansdbInstance, get_server_addr, BaseTest 6 | 7 | 8 | class AbnormalCmdTest(BaseTest): 9 | def setUp(self): 10 | BaseTest.setUp(self) 11 | self.store = MCStore(self.db.addr) 12 | self.invalid_key = '/this/is/a/bad/key/%s' % chr(15) 13 | 14 | def run_cmd_by_telnet(self, cmd, expected, timeout=2): 15 | addr, port = self.db.addr.split(':') 16 | t = telnetlib.Telnet(addr, port) 17 | t.write('%s\r\n' % cmd) 18 | out = t.read_until('\n', timeout=timeout) 19 | t.write('quit\n') 20 | t.close() 21 | r = out.strip('\r\n') 22 | self.assertEqual(r, expected) 23 | 24 | def test_get(self): 25 | # get not exist key 26 | cmd = 'get /test/get' 27 | self.run_cmd_by_telnet(cmd, 'END') 28 | 29 | # invalid key 30 | cmd = 'get %s' % self.invalid_key 31 | self.run_cmd_by_telnet(cmd, 'END') 32 | self.checkCounterZero() 33 | 34 | def test_set(self): 35 | # invalid key 36 | cmd = 'set %s 0 0 3\r\naaa' % self.invalid_key 37 | self.run_cmd_by_telnet(cmd, 'NOT_STORED') 38 | 39 | cmd = 'set /test/set 0 0 3\r\naaaa' 40 | self.run_cmd_by_telnet(cmd, 'CLIENT_ERROR bad data chunk') 41 | self.checkCounterZero() 42 | 43 | def test_incr(self): 44 | key = '/test/incr' 45 | self.assertEqual(self.store.delete(key), True) 46 | cmd = 'incr %s 10' % key 47 | self.run_cmd_by_telnet(cmd, '10') 48 | self.assertEqual(self.store.get(key), 10) 49 | 50 | # incr 一个 value 为字符串的 key 51 | key = '/test/incr2' 52 | self.assertEqual(self.store.set(key, 'aaa'), True) 53 | cmd = 'incr %s 10' % key 54 | self.run_cmd_by_telnet(cmd, '0') 55 | self.assertEqual(self.store.get(key), 'aaa') 56 | self.checkCounterZero() 57 | 58 | def test_delete(self): 59 | key = '/delete/not/exist/key' 60 | cmd = 'delete %s' % key 61 | self.run_cmd_by_telnet(cmd, 'NOT_FOUND') 62 | 63 | cmd = 'delete %s' % self.invalid_key 64 | self.run_cmd_by_telnet(cmd, 'NOT_FOUND') 65 | self.checkCounterZero() 66 | 67 | def test_get_meta_by_key(self): 68 | key = '/get_meta_by_key/not/exist/key' 69 | cmd = 'get ?%s' % key 70 | self.run_cmd_by_telnet(cmd, 'END') 71 | 72 | cmd = 'get ?%s' % self.invalid_key 73 | self.run_cmd_by_telnet(cmd, 'END') 74 | self.checkCounterZero() 75 | -------------------------------------------------------------------------------- /gobeansdb/gobeansdb.go: -------------------------------------------------------------------------------- 1 | package gobeansdb 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "runtime" 8 | "time" 9 | 10 | "github.com/douban/gobeansdb/config" 11 | "github.com/douban/gobeansdb/loghub" 12 | mc "github.com/douban/gobeansdb/memcache" 13 | "github.com/douban/gobeansdb/store" 14 | ) 15 | 16 | var ( 17 | server *mc.Server 18 | storage *Storage 19 | conf DBConfig 20 | logger = loghub.ErrorLogger 21 | ) 22 | 23 | func Main() { 24 | var version = flag.Bool("version", false, "print version of gobeansdb") 25 | var confdir = flag.String("confdir", "", "path of server config dir") 26 | var dumpconf = flag.Bool("dumpconf", false, "print configuration") 27 | var buildhint = flag.String("buildhint", "", "a data file OR a bucket dir") 28 | 29 | flag.Parse() 30 | 31 | if *version { 32 | fmt.Println("gobeansdb version", config.Version) 33 | return 34 | } 35 | log.Printf("version %s", config.Version) 36 | 37 | if *confdir != "" { 38 | log.Printf("use confdir %s", *confdir) 39 | } 40 | conf.Load(*confdir) 41 | runtime.GOMAXPROCS(conf.Threads) 42 | if *dumpconf { 43 | config.DumpConfig(conf) 44 | return 45 | } else if *buildhint != "" { 46 | if *confdir != "" { 47 | initWeb() 48 | } else { 49 | conf.HintConfig.SplitCap = (3 << 20) // cost maxrss about 900M 50 | } 51 | if err := store.DataToHint(*buildhint); err != nil { 52 | log.Printf("%s", err.Error()) 53 | } 54 | return 55 | } 56 | 57 | loghub.InitLogger(conf.ErrorLog, conf.AccessLog, conf.AnalysisLog) 58 | logger.Infof("gobeansdb version %s starting at %d, config: %#v", 59 | config.Version, conf.Port, conf) 60 | 61 | if config.ZKClient == nil { 62 | logger.Warnf("route version: local") 63 | } else { 64 | logger.Infof("route version: %d", config.ZKClient.Version) 65 | } 66 | logger.Infof("route table: %#v", config.Route) 67 | 68 | initWeb() 69 | 70 | var err error 71 | 72 | hstore, err := store.NewHStore() 73 | if err != nil { 74 | logger.Fatalf("fail to init NewHStore %s", err.Error()) 75 | } 76 | storage = &Storage{hstore: hstore} 77 | 78 | server = mc.NewServer(storage) 79 | addr := fmt.Sprintf("%s:%d", conf.Listen, conf.Port) 80 | if err := server.Listen(addr); err != nil { 81 | logger.Fatalf("listen failed %s", err.Error()) 82 | } 83 | logger.Infof("mc server listen at %s", addr) 84 | log.Println("ready") 85 | 86 | server.HandleSignals(conf.ErrorLog, conf.AccessLog, conf.AnalysisLog) 87 | go storage.hstore.HintDumper(1 * time.Minute) // it may start merge go routine 88 | go storage.hstore.Flusher() 89 | config.AllowReload = true 90 | err = server.Serve() 91 | tmp := storage 92 | storage = nil 93 | tmp.hstore.Close() 94 | 95 | logger.Infof("shut down gracefully") 96 | } 97 | -------------------------------------------------------------------------------- /tests/dbclient.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | import libmc 3 | 4 | 5 | def connect(server, **kwargs): 6 | comp_threshold = kwargs.pop('comp_threshold', 0) 7 | prefix = kwargs.pop('prefix', None) 8 | 9 | c = libmc.Client([server], 10 | do_split=0, 11 | comp_threshold=comp_threshold, 12 | prefix=prefix) 13 | c.config(libmc.MC_CONNECT_TIMEOUT, 300) # 0.3s 14 | c.config(libmc.MC_POLL_TIMEOUT, 3000) # 3s 15 | c.config(libmc.MC_RETRY_TIMEOUT, 5) # 5s 16 | return c 17 | 18 | 19 | class MCStore(object): 20 | 21 | IGNORED_LIBMC_RET = frozenset([ 22 | libmc.MC_RETURN_OK, 23 | libmc.MC_RETURN_INVALID_KEY_ERR 24 | ]) 25 | 26 | def __init__(self, addr, **kwargs): 27 | self.addr = addr 28 | self.mc = connect(addr, **kwargs) 29 | 30 | def __repr__(self): 31 | return '' % repr(self.addr) 32 | 33 | def __str__(self): 34 | return self.addr 35 | 36 | def set(self, key, data, rev=0): 37 | return bool(self.mc.set(key, data, rev)) 38 | 39 | def set_raw(self, key, data, rev=0, flag=0): 40 | if rev < 0: 41 | raise Exception(rev) 42 | return self.mc.set_raw(key, data, rev, flag) 43 | 44 | def set_multi(self, values, return_failure=False): 45 | return self.mc.set_multi(values, return_failure=return_failure) 46 | 47 | def _check_last_error(self): 48 | last_err = self.mc.get_last_error() 49 | if last_err not in self.IGNORED_LIBMC_RET: 50 | raise IOError(last_err, self.mc.get_last_strerror()) 51 | 52 | def get(self, key): 53 | try: 54 | r = self.mc.get(key) 55 | if r is None: 56 | self._check_last_error() 57 | return r 58 | except ValueError: 59 | self.mc.delete(key) 60 | 61 | def get_raw(self, key): 62 | r, flag = self.mc.get_raw(key) 63 | if r is None: 64 | self._check_last_error() 65 | return r, flag 66 | 67 | def get_multi(self, keys): 68 | r = self.mc.get_multi(keys) 69 | self._check_last_error() 70 | return r 71 | 72 | def delete(self, key): 73 | return bool(self.mc.delete(key)) 74 | 75 | def delete_multi(self, keys, return_failure=False): 76 | return self.mc.delete_multi(keys, return_failure=return_failure) 77 | 78 | def exists(self, key): 79 | meta_info = self.mc.get('?' + key) 80 | if meta_info: 81 | version = meta_info.split(' ')[0] 82 | return int(version) > 0 83 | return False 84 | 85 | def incr(self, key, value): 86 | return self.mc.incr(key, int(value)) -------------------------------------------------------------------------------- /cmem/cmem.go: -------------------------------------------------------------------------------- 1 | package cmem 2 | 3 | /* 4 | #include 5 | #include 6 | */ 7 | import "C" 8 | import ( 9 | "reflect" 10 | "sync/atomic" 11 | "unsafe" 12 | 13 | "github.com/douban/gobeansdb/config" 14 | ) 15 | 16 | var ( 17 | AllocRL ResourceLimiter 18 | ) 19 | 20 | type ResourceLimiter struct { 21 | Count int64 22 | Size int64 23 | MaxCount int64 24 | MaxSize int64 25 | Chan chan int `json:"-"` 26 | } 27 | 28 | func (rl *ResourceLimiter) reset() { 29 | *rl = ResourceLimiter{} 30 | rl.Chan = make(chan int, 1) 31 | } 32 | 33 | func (rl *ResourceLimiter) IsZero() bool { 34 | return rl.Count == 0 && rl.Size == 0 35 | } 36 | 37 | func (rl *ResourceLimiter) AddSizeAndCount(size int) { 38 | rl.AddSize(size) 39 | rl.AddCount(1) 40 | } 41 | 42 | func (rl *ResourceLimiter) SubSizeAndCount(size int) { 43 | rl.SubSize(size) 44 | rl.SubCount(1) 45 | } 46 | 47 | func (rl *ResourceLimiter) AddSize(size int) { 48 | atomic.AddInt64(&rl.Size, int64(size)) 49 | if rl.Size > rl.MaxSize { 50 | rl.MaxSize = rl.Size 51 | } 52 | } 53 | 54 | func (rl *ResourceLimiter) SubSize(size int) { 55 | atomic.AddInt64(&rl.Size, -int64(size)) 56 | } 57 | 58 | func (rl *ResourceLimiter) AddCount(count int) { 59 | atomic.AddInt64(&rl.Count, int64(count)) 60 | if rl.Count > rl.MaxCount { 61 | rl.MaxCount = rl.Count 62 | } 63 | } 64 | 65 | func (rl *ResourceLimiter) SubCount(count int) { 66 | atomic.AddInt64(&rl.Count, -int64(count)) 67 | } 68 | 69 | type CArray struct { 70 | Body []byte 71 | Addr uintptr 72 | Cap int 73 | } 74 | 75 | func (arr *CArray) Alloc(size int) bool { 76 | if size <= int(config.MCConf.BodyInC) { 77 | arr.Body = make([]byte, size) 78 | arr.Cap = size 79 | arr.Addr = 0 80 | return true 81 | } 82 | 83 | arr.Addr = uintptr(C.malloc(C.size_t(size))) 84 | if arr.Addr == 0 { 85 | return false 86 | } 87 | AllocRL.AddSizeAndCount(size) 88 | arr.Cap = size 89 | sliceheader := (*reflect.SliceHeader)(unsafe.Pointer(&arr.Body)) 90 | sliceheader.Data = arr.Addr 91 | sliceheader.Len = size 92 | sliceheader.Cap = size 93 | return true 94 | } 95 | 96 | func (arr *CArray) Free() { 97 | if arr.Addr != 0 { 98 | AllocRL.SubSizeAndCount(arr.Cap) 99 | C.free(unsafe.Pointer(arr.Addr)) 100 | arr.Body = nil 101 | arr.Addr = 0 102 | arr.Cap = 0 103 | } 104 | } 105 | 106 | func (arr *CArray) Clear() { 107 | arr.Addr = 0 108 | arr.Body = nil 109 | } 110 | 111 | func (arr *CArray) Copy() (arrNew CArray, ok bool) { 112 | size := len(arr.Body) 113 | if arr.Addr == 0 { 114 | arrNew.Body = make([]byte, size) 115 | copy(arrNew.Body, arr.Body) 116 | ok = true 117 | return 118 | } 119 | if !arrNew.Alloc(size) { 120 | return 121 | } 122 | ok = true 123 | copy(arrNew.Body, arr.Body) 124 | return 125 | } 126 | -------------------------------------------------------------------------------- /quicklz/cquicklz.go: -------------------------------------------------------------------------------- 1 | package quicklz 2 | 3 | /* 4 | #cgo CFLAGS: -I . 5 | #include "stdlib.h" 6 | #include "quicklz.h" 7 | size_t qlz_compress(const void *source, char *destination, size_t size, char *scratch_compress); 8 | */ 9 | import "C" 10 | import ( 11 | "fmt" 12 | "unsafe" 13 | 14 | "github.com/douban/gobeansdb/cmem" 15 | "github.com/douban/gobeansdb/utils" 16 | ) 17 | 18 | const ( 19 | CompressBufferSize = 528400 20 | DecompressBufferSize = 16 21 | ) 22 | 23 | func CCompress(src []byte) (dst cmem.CArray, ok bool) { 24 | ok = dst.Alloc(len(src) + 400) 25 | if !ok { 26 | return 27 | } 28 | buf := C.malloc(C.size_t(CompressBufferSize)) 29 | if buf == nil { 30 | ok = false 31 | return 32 | } 33 | c_buf := (*C.char)(buf) 34 | defer C.free(unsafe.Pointer(c_buf)) 35 | 36 | c_src := (unsafe.Pointer(&src[0])) 37 | c_dst := (*C.char)(unsafe.Pointer(&dst.Body[0])) 38 | c_size := C.qlz_compress(c_src, c_dst, C.size_t(len(src)), c_buf) 39 | size := int(c_size) 40 | dst.Body = dst.Body[:size] 41 | return 42 | } 43 | 44 | func CDecompress(src []byte, sizeD int) (dst cmem.CArray, err error) { 45 | if !dst.Alloc(sizeD) { 46 | err = fmt.Errorf("fail to alloc for decompress, size %d", sizeD) 47 | return 48 | } 49 | buf := make([]byte, DecompressBufferSize) 50 | c_src := (*C.char)(unsafe.Pointer(&src[0])) 51 | c_dst := (unsafe.Pointer(&dst.Body[0])) 52 | c_buf := (*C.char)(unsafe.Pointer(&buf[0])) 53 | size := int(C.qlz_decompress(c_src, c_dst, c_buf)) 54 | if size != sizeD { 55 | err = fmt.Errorf("fail to alloc for decompress, size %d != %d", sizeD, size) 56 | return 57 | } 58 | dst.Body = dst.Body[:size] 59 | return 60 | } 61 | 62 | func DecompressSafe(src []byte) (dst []byte, err error) { 63 | defer func() { 64 | if e := recover(); e != nil { 65 | var ok bool 66 | err, ok = e.(error) 67 | if !ok { 68 | err = fmt.Errorf("decompress panic with non-error: %#v", e) 69 | } 70 | } 71 | }() 72 | sizeC := SizeCompressed(src) 73 | if len(src) != sizeC { 74 | return nil, fmt.Errorf("bad sizeCompressed, expect %d, got %d", sizeC, len(src)) 75 | } 76 | sizeD := SizeDecompressed(src) 77 | dst = Decompress(src) 78 | if len(dst) != sizeD { 79 | return nil, fmt.Errorf("bad sizeDecompressed, expect %d, got %d", sizeD, len(dst)) 80 | } 81 | return dst, nil 82 | } 83 | 84 | func CDecompressSafe(src []byte) (dst cmem.CArray, err error) { 85 | defer func() { 86 | if e := recover(); e != nil { 87 | err = fmt.Errorf("CDecompressSafe panic(%#v), stack: %s", e, utils.GetStack(1000)) 88 | } 89 | }() 90 | sizeC := SizeCompressed(src) 91 | if len(src) != sizeC { 92 | err = fmt.Errorf("bad sizeCompressed, expect %d, got %d", sizeC, len(src)) 93 | return 94 | } 95 | sizeD := SizeDecompressed(src) 96 | dst, err = CDecompress(src, sizeD) 97 | if err != nil { 98 | return 99 | } 100 | return 101 | } 102 | -------------------------------------------------------------------------------- /loghub/log.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "runtime" 8 | "sync" 9 | ) 10 | 11 | const ( 12 | DEBUG = iota 13 | INFO 14 | WARN 15 | ERROR 16 | FATAL 17 | ) 18 | 19 | var ( 20 | levelString = []string{"DEBUG", "INFO ", "WARN ", "ERROR", "FATAL"} 21 | ) 22 | 23 | // LogHub as a centry point in main, to control log from diff pacakage 24 | // enable: 25 | // 1. change log lib (of all package) easily, just neet a wrapper 26 | // 2. append level header and file:line before log 27 | // 3. preproces before logging, e.g. : 28 | // a. buffer recent errs with diff "file:line" 29 | // b. sent error to network 30 | // 4. logHub store the config (loglib specified) for diff "name", 31 | // and use it when receiving a log, simulating getLogger("name") 32 | // 5. provide a Default, so don`t bother to choose a "right" log implementation at beginning 33 | // 6. keep simple and lightweitght 34 | 35 | type LogHub interface { 36 | Log(name string, level int, file string, line int, msg string) 37 | Reopen(path string) error 38 | GetLastLog() []byte 39 | DumpBuffer(all bool, out io.Writer) 40 | } 41 | 42 | type Logger struct { 43 | mu sync.Mutex 44 | name string 45 | minLevel int 46 | Hub LogHub 47 | } 48 | 49 | func (l *Logger) SetLevel(level int) { 50 | l.minLevel = level 51 | } 52 | 53 | func NewLogger(name string, hub LogHub, minLevel int) *Logger { 54 | l := &Logger{name: name, Hub: hub, minLevel: minLevel} 55 | return l 56 | } 57 | 58 | func (l *Logger) Debugf(format string, v ...interface{}) { 59 | l.Logf(DEBUG, format, v...) 60 | } 61 | 62 | func (l *Logger) Infof(format string, v ...interface{}) { 63 | l.Logf(INFO, format, v...) 64 | } 65 | 66 | func (l *Logger) Warnf(format string, v ...interface{}) { 67 | l.Logf(WARN, format, v...) 68 | } 69 | 70 | func (l *Logger) Errorf(format string, v ...interface{}) { 71 | l.Logf(ERROR, format, v...) 72 | } 73 | 74 | func (l *Logger) Fatalf(format string, v ...interface{}) { 75 | l.Logf(FATAL, format, v...) 76 | } 77 | 78 | func (l *Logger) Logf(level int, format string, v ...interface{}) { 79 | if level < l.minLevel { 80 | return 81 | } 82 | _, file, line, ok := runtime.Caller(2) 83 | if !ok { 84 | file = "???" 85 | line = 0 86 | } else { 87 | short := file 88 | for i := len(file) - 1; i > 0; i-- { 89 | if file[i] == '/' { 90 | short = file[i+1:] 91 | break 92 | } 93 | } 94 | file = short 95 | } 96 | msg := fmt.Sprintf(format, v...) 97 | l.Hub.Log(l.name, level, file, line, msg) 98 | } 99 | 100 | func InitLogger(errorlog, accesslog, analysislog string) { 101 | if errorlog != "" { 102 | log.Printf("log to errorlog %s", errorlog) 103 | InitErrorLog(errorlog, INFO, 200) 104 | } 105 | if accesslog != "" { 106 | log.Printf("open accesslog %s", accesslog) 107 | InitAccessLog(accesslog, INFO) 108 | } 109 | if analysislog != "" { 110 | log.Printf("log to analysislog %s", analysislog) 111 | InitAnalysisLog(analysislog, INFO, 200) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /memcache/store.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "math/rand" 5 | "strconv" 6 | "sync" 7 | ) 8 | 9 | type Storage interface { 10 | Client() StorageClient 11 | } 12 | 13 | type StorageClient interface { 14 | GetSuccessedTargets() []string 15 | Clean() 16 | Get(key string) (*Item, error) 17 | GetMulti(keys []string) (map[string]*Item, error) 18 | Set(key string, item *Item, noreply bool) (bool, error) 19 | Append(key string, value []byte) (bool, error) 20 | Incr(key string, value int) (int, error) 21 | Delete(key string) (bool, error) 22 | Len() int 23 | Close() 24 | Process(key string, args []string) (string, string) 25 | } 26 | 27 | type mapStore struct { 28 | lock sync.Mutex 29 | data map[string]*Item 30 | } 31 | 32 | func NewMapStore() *mapStore { 33 | s := new(mapStore) 34 | s.data = make(map[string]*Item) 35 | return s 36 | } 37 | 38 | func (s *mapStore) GetSuccessedTargets() []string { 39 | return []string{"localhost"} 40 | } 41 | 42 | func (s *mapStore) Clean() { 43 | return 44 | } 45 | 46 | func (s *mapStore) Client() StorageClient { 47 | return s 48 | } 49 | 50 | func (s *mapStore) Close() { 51 | return 52 | } 53 | 54 | func (s *mapStore) Get(key string) (*Item, error) { 55 | s.lock.Lock() 56 | defer s.lock.Unlock() 57 | 58 | r, _ := s.data[key] 59 | return r, nil 60 | } 61 | 62 | func (s *mapStore) GetMulti(keys []string) (map[string]*Item, error) { 63 | s.lock.Lock() 64 | defer s.lock.Unlock() 65 | 66 | rs := make(map[string]*Item, len(keys)) 67 | for _, key := range keys { 68 | r, _ := s.data[key] 69 | if r != nil { 70 | rs[key] = r 71 | } 72 | } 73 | return rs, nil 74 | } 75 | 76 | func (s *mapStore) Set(key string, item *Item, noreply bool) (bool, error) { 77 | s.lock.Lock() 78 | defer s.lock.Unlock() 79 | 80 | item.Cas = rand.Int() 81 | it := *item 82 | it.CArray, _ = item.CArray.Copy() 83 | s.data[key] = &it 84 | return true, nil 85 | } 86 | 87 | func (s *mapStore) Append(key string, value []byte) (suc bool, err error) { 88 | s.lock.Lock() 89 | defer s.lock.Unlock() 90 | 91 | r, ok := s.data[key] 92 | if ok && r.Flag == 0 { 93 | r.Body = append(r.Body, value...) 94 | s.data[key] = r 95 | return true, nil 96 | } 97 | return false, nil 98 | } 99 | 100 | func (s *mapStore) Incr(key string, v int) (n int, err error) { 101 | s.lock.Lock() 102 | defer s.lock.Unlock() 103 | r, ok := s.data[key] 104 | if ok { 105 | n, err = strconv.Atoi(string(r.Body)) 106 | if err != nil { 107 | return 108 | } 109 | n += v 110 | r.Body = []byte(strconv.Itoa(n)) 111 | } else { 112 | n = v 113 | } 114 | return 115 | } 116 | 117 | func (s *mapStore) Delete(key string) (r bool, err error) { 118 | s.lock.Lock() 119 | defer s.lock.Unlock() 120 | _, ok := s.data[key] 121 | if ok { 122 | delete(s.data, key) 123 | r = true 124 | } 125 | return 126 | } 127 | 128 | func (s *mapStore) Len() int { 129 | return len(s.data) 130 | } 131 | 132 | func (s *mapStore) Process(key string, args []string) (string, string) { 133 | return "", "" 134 | } 135 | -------------------------------------------------------------------------------- /loghub/errorlog.go: -------------------------------------------------------------------------------- 1 | package loghub 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | var ( 14 | ErrorLogFormat = "%s %15s:%4d - %s" 15 | ErrorLogFlag = (log.LstdFlags | log.Lmicroseconds) 16 | ErrorLogger *Logger 17 | ) 18 | 19 | type ErrorLogHub struct { 20 | logger *log.Logger 21 | logFd *os.File 22 | BufferLog 23 | } 24 | 25 | func init() { 26 | logger := openLogWithFd(os.Stderr, ErrorLogFlag) 27 | hub := &ErrorLogHub{logger: logger} 28 | hub.InitBuffer(200) 29 | ErrorLogger = NewLogger("", hub, DEBUG) 30 | } 31 | 32 | func InitErrorLog(path string, level int, bufferSize int) (err error) { 33 | if errorLog, errorFd, err := openLog(path, ErrorLogFlag); err == nil { 34 | hub := &ErrorLogHub{logger: errorLog, logFd: errorFd} 35 | hub.InitBuffer(bufferSize) 36 | ErrorLogger.Hub = hub 37 | ErrorLogger.SetLevel(level) 38 | } else { 39 | log.Fatalf("open log error, path=[%s], err=[%s]", path, err.Error()) 40 | } 41 | return 42 | } 43 | 44 | func (hub *ErrorLogHub) Log(name string, level int, file string, line int, msg string) { 45 | hub.logger.Printf(ErrorLogFormat, levelString[level], file, line, msg) 46 | bufline := &BufferLine{time.Now(), level, file, line, msg} 47 | hub.Add(bufline) 48 | hub.Lock() 49 | hub.Last[level] = bufline 50 | hub.Unlock() 51 | if level == FATAL { 52 | os.Exit(1) 53 | } 54 | } 55 | 56 | func (hub *ErrorLogHub) Reopen(path string) (err error) { 57 | return reopenLogger(&hub.logger, &hub.logFd, path, ErrorLogFlag) 58 | } 59 | 60 | // Buffer 61 | 62 | type BufferLog struct { 63 | sync.Mutex 64 | head int 65 | Buffer []*BufferLine 66 | 67 | all queue 68 | warn queue 69 | Last [FATAL + 1]*BufferLine 70 | } 71 | 72 | func (l *BufferLog) InitBuffer(size int) { 73 | l.all.Buffer = make([]*BufferLine, size) 74 | l.warn.Buffer = make([]*BufferLine, size) 75 | } 76 | 77 | func (l *BufferLog) DumpBuffer(all bool, out io.Writer) { 78 | l.Lock() 79 | defer l.Unlock() 80 | if all { 81 | l.all.DumpBuffer(out) 82 | } else { 83 | l.warn.DumpBuffer(out) 84 | } 85 | } 86 | 87 | func (l *BufferLog) Add(line *BufferLine) { 88 | l.Lock() 89 | defer l.Unlock() 90 | l.all.Add(line) 91 | if line.Level >= WARN { 92 | l.warn.Add(line) 93 | } 94 | } 95 | 96 | func (l *BufferLog) GetLastLog() []byte { 97 | b, _ := json.Marshal(l.Last[:]) 98 | return b 99 | } 100 | 101 | type BufferLine struct { 102 | TS time.Time 103 | Level int 104 | File string 105 | Line int 106 | Msg string 107 | } 108 | 109 | type queue struct { 110 | head int 111 | Buffer []*BufferLine 112 | } 113 | 114 | func (q *queue) Add(line *BufferLine) { 115 | q.Buffer[q.head] = line 116 | q.head += 1 117 | if q.head >= len(q.Buffer) { 118 | q.head = 0 119 | } 120 | } 121 | 122 | func (q *queue) DumpBuffer(out io.Writer) { 123 | i := q.head 124 | for j := 0; j < len(q.Buffer); j++ { 125 | line := q.Buffer[i] 126 | if line != nil { 127 | out.Write([]byte(fmt.Sprintf("%v\n", line))) 128 | } 129 | i += 1 130 | if i >= len(q.Buffer) { 131 | i = 0 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # GoBeansDB ![](https://github.com/douban/gobeansdb/workflows/GoBeansDB%20Test/badge.svg) [![Release](https://img.shields.io/github/v/release/douban/gobeansdb)](https://github.com/douban/gobeansdb/releases) 3 | 4 | Yet another distributed key-value storage system from Douban Inc. 5 | 6 | Any memcached client cache interactive with GobeansDB without any modification. 7 | 8 | ## Related 9 | 10 | - [libmc](https://github.com/douban/libmc) : a high performance python/go mc client 11 | - [gobeansproxy](https://github.com/douban/gobeansproxy) : routing to gobeansdb cluster with three copy 12 | - [beansdbadmin](https://github.com/douban/beansdbadmin): webUI, sync ... 13 | 14 | ## Prepare 15 | 16 | GoBeansDB uses `go mod` to manage dependencies, please make sure your Go version >= 1.11.0 first. 17 | 18 | 19 | ## Install 20 | 21 | ```shell 22 | $ git clone http://github.com/douban/gobeansdb.git 23 | $ cd gobeansdb 24 | $ go mod vendor 25 | $ make 26 | ``` 27 | 28 | ## test 29 | 30 | ```shell 31 | $ make test # unit test 32 | $ make pytest # Integrated test 33 | ``` 34 | 35 | ## run 36 | 37 | ```shell 38 | $ ${GOPATH}/bin/gobeansdb -h 39 | ``` 40 | 41 | ## Python Example 42 | 43 | ``` 44 | import libmc 45 | 46 | 47 | mc = libmc.Client(['localhost:7900']) 48 | mc.set("foo", "bar") 49 | mc.get("foo") 50 | 51 | ``` 52 | 53 | ## Features 54 | 55 | - 协议： memcached。推荐 libmc 客户端（c++ 实现，目前支持 go 和 python，基于 poll 的并发 get_multi/set_multi） 56 | - sharding： 静态 hash 路由，分桶数 16 整数倍 57 | - 索引： 内存索引全部 key，开销约为每个 key 20 字节，主要内存用 c 分配。 get hit 直接定位到文件 record，get miss 不落盘。 58 | - 最终一致性：同步脚本不断比较一个桶三副本 htree（每个桶一个 16 叉的内存 merkle tree）做同步，比较时间戳。 59 | - 文件格式：data 文件可以看成 log（顺序写入）； 每个 record 256 bytes 对齐，有 crc 校验。 60 | 61 | ## 在 douban 使用方法 62 | 63 | ``` 64 | mc_client --- cache 65 | | 66 | --- any beansdb proxy -- beansdb servers 67 | ``` 68 | 69 | - 用于单个 key 并发写很少的数据 70 | - [gobeansproxy](https://github.com/douban/gobeansproxy) 负责路由， 固定 3 副本 71 | - 两个集群： 分别存储 图片类 （cache 为 CDN） 和 长文本 （cache 为 mc 集群）。 72 | - 支持离线 [dpark](https://github.com/douban/dpark) 读写，读支持本地化。 73 | - 可以视情况保留一段时间完整的写记录。 74 | - 借助 python 脚本 管理，近期整理后会部分开源，包 admin UI（readonly），同步脚本等 75 | 76 | 77 | 磁盘上的样子（256分区）： 78 | 79 | * /var/lib/beansdb 80 | * 0/ 81 | * 0/ -> /data1/beansdb/0/0 82 | * 000.data 83 | * 000.000.idx.s 84 | * 000.001.idx.s 85 | * ... 86 | * 008.000.idx.hash 87 | * ... 88 | * 009.data 89 | * 009.000.idx.s 90 | 91 | 92 | ## 入坑指南 93 | 94 | 优点 95 | 96 | 1. 数据文件即 log， 结构简单，数据安全 97 | 2. htree 设计 方便数据同步； 98 | 3. 全内存索引，访问磁盘次数少，尤其可以准确过滤掉不存在的 key。 99 | 100 | 缺点/注意 101 | 102 | 1. 一致性支持较弱，时间戳目前是秒级（受限于数据文件格式）。 103 | 2. 全内存索引，有一定内存开销，在启动时载入索引略慢（约十几秒到半分钟， 决定于key 数量）。 104 | 3. 数据文件格式的 padding 对小 value 有一定浪费。 105 | 106 | 107 | 配置重点（详见 wiki） 108 | 109 | - 分桶数 110 | - htree 高度（决定 merkle tree 部分内存大小和 溢出链表的平均长度） 111 | 112 | 113 | ## 与 [beansdb](https://github.com/douban/beansdb) 关系 114 | 115 | - 兼容 116 | - 数据文件格式不变 117 | - 仍使用 mc 协议（"@" "?" 开头的特殊 key 用法略不同） 118 | - htree 的核心设计不变 119 | - 改进 120 | - go 实现，更稳定，方便配置和维护（http api等） 121 | - 内存开销变小 122 | - hint 文件格式和后缀变了 123 | - 同一个节点不同数据文件不在混部署，以避免坏一个损失整个点数据 124 | 125 | 126 | # 一些设计/实现要点见 [wiki](https://github.com/douban/gobeansdb/wiki) 127 | -------------------------------------------------------------------------------- /quicklz/quicklz.h: -------------------------------------------------------------------------------- 1 | //quicklz.h 2 | #ifndef QLZ_HEADER 3 | #define QLZ_HEADER 4 | 5 | // Fast data compression library 6 | // Copyright (C) 2006-2010 Lasse Mikkel Reinhold 7 | // lar@quicklz.com 8 | // 9 | // QuickLZ can be used for free under the GPL-1, -2 or -3 license (where anything 10 | // released into public must be open source) or under a commercial license if such 11 | // has been acquired (see http://www.quicklz.com/order.html). The commercial license 12 | // does not cover derived or ported versions created by third parties under GPL. 13 | 14 | // Version 1.4.1 final - april 2010 15 | 16 | // You can edit following user settings. Data must be decompressed with the same 17 | // setting of QLZ_COMPRESSION_LEVEL and QLZ_STREAMING_BUFFER as it was compressed 18 | // (see manual). If QLZ_STREAMING_BUFFER > 0, scratch buffers must be initially 19 | // zeroed out (see manual). First #ifndef makes it possible to define settings from 20 | // the outside like the compiler command line or from higher level code. 21 | 22 | #ifndef QLZ_COMPRESSION_LEVEL 23 | //#define QLZ_COMPRESSION_LEVEL 1 24 | //#define QLZ_COMPRESSION_LEVEL 2 25 | #define QLZ_COMPRESSION_LEVEL 3 26 | 27 | #define QLZ_STREAMING_BUFFER 0 28 | //#define QLZ_STREAMING_BUFFER 100000 29 | //#define QLZ_STREAMING_BUFFER 1000000 30 | 31 | //#define QLZ_MEMORY_SAFE 32 | #endif 33 | 34 | #define QLZ_VERSION_MAJOR 1 35 | #define QLZ_VERSION_MINOR 4 36 | #define QLZ_VERSION_REVISION 1 37 | 38 | // Using size_t, memset() and memcpy() 39 | #include 40 | 41 | // Public functions of QuickLZ 42 | size_t qlz_size_decompressed(const char *source); 43 | size_t qlz_size_compressed(const char *source); 44 | size_t qlz_decompress(const char *source, void *destination, char *scratch_decompress); 45 | size_t qlz_compress(const void *source, char *destination, size_t size, char *scratch_compress); 46 | int qlz_get_setting(int setting); 47 | 48 | // Verify compression level 49 | #if QLZ_COMPRESSION_LEVEL != 1 && QLZ_COMPRESSION_LEVEL != 2 && QLZ_COMPRESSION_LEVEL != 3 50 | #error QLZ_COMPRESSION_LEVEL must be 1, 2 or 3 51 | #endif 52 | 53 | // Compute QLZ_SCRATCH_COMPRESS and QLZ_SCRATCH_DECOMPRESS 54 | #if QLZ_COMPRESSION_LEVEL == 1 55 | #define QLZ_POINTERS 1 56 | #define QLZ_HASH_VALUES 4096 57 | #elif QLZ_COMPRESSION_LEVEL == 2 58 | #define QLZ_POINTERS 4 59 | #define QLZ_HASH_VALUES 2048 60 | #elif QLZ_COMPRESSION_LEVEL == 3 61 | #define QLZ_POINTERS 16 62 | #define QLZ_HASH_VALUES 4096 63 | #endif 64 | 65 | typedef struct 66 | { 67 | #if QLZ_COMPRESSION_LEVEL == 1 68 | unsigned int cache[QLZ_POINTERS]; 69 | #endif 70 | const unsigned char *offset[QLZ_POINTERS]; 71 | } qlz_hash_compress; 72 | 73 | typedef struct 74 | { 75 | const unsigned char *offset[QLZ_POINTERS]; 76 | } qlz_hash_decompress; 77 | 78 | 79 | #define QLZ_ALIGNMENT_PADD 8 80 | #define QLZ_BUFFER_COUNTER 8 81 | 82 | #define QLZ_SCRATCH_COMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_compress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) 83 | 84 | #if QLZ_COMPRESSION_LEVEL < 3 85 | #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER + sizeof(qlz_hash_decompress[QLZ_HASH_VALUES]) + QLZ_HASH_VALUES) 86 | #else 87 | #define QLZ_SCRATCH_DECOMPRESS (QLZ_ALIGNMENT_PADD + QLZ_BUFFER_COUNTER + QLZ_STREAMING_BUFFER) 88 | #endif 89 | 90 | #endif 91 | 92 | 93 | 94 | 95 | 96 | -------------------------------------------------------------------------------- /config/route.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "io/ioutil" 5 | "strconv" 6 | 7 | yaml "gopkg.in/yaml.v2" 8 | ) 9 | 10 | //DefaultRouteConfig = route.RouteConfig{NumBucket: 256, Buckets: make([]int, 256)} 11 | var DefaultRouteConfig = DBRouteConfig{NumBucket: 16, BucketsStat: make([]int, 16)} 12 | 13 | type Server struct { 14 | Addr string 15 | Buckets []int `yaml:"-"` 16 | BucketsHex []string `yaml:"buckets,flow"` 17 | } 18 | 19 | func (s *Server) Decode() error { 20 | s.Buckets = make([]int, len(s.BucketsHex)) 21 | for i, str := range s.BucketsHex { 22 | i64, err := strconv.ParseInt(str, 16, 16) 23 | if err != nil { 24 | return err 25 | } 26 | s.Buckets[i] = int(i64) 27 | } 28 | return nil 29 | } 30 | 31 | type RouteTable struct { 32 | NumBucket int 33 | Main []Server 34 | Backup []string 35 | 36 | Buckets map[int]map[string]bool `yaml:"-"` 37 | Servers map[string]map[int]bool `yaml:"-"` 38 | } 39 | 40 | type DBRouteConfig struct { 41 | NumBucket int 42 | BucketsStat []int `json:"Buckets"` // TODO: `json:"-"` 43 | BucketsHex []string 44 | } 45 | 46 | func (rt *RouteTable) GetDBRouteConfig(addr string) (r DBRouteConfig) { 47 | r = DBRouteConfig{NumBucket: rt.NumBucket} 48 | r.BucketsStat = make([]int, rt.NumBucket) 49 | r.BucketsHex = make([]string, 0) 50 | buckets, found := rt.Servers[addr] 51 | if !found { 52 | return 53 | } 54 | for b := range buckets { 55 | r.BucketsStat[b] = 1 56 | r.BucketsHex = append(r.BucketsHex, BucketIDHex(b, rt.NumBucket)) 57 | } 58 | return 59 | } 60 | 61 | func (rt *RouteTable) LoadFromYaml(data []byte) error { 62 | if err := yaml.Unmarshal(data, &rt); err != nil { 63 | return err 64 | } 65 | rt.Servers = make(map[string]map[int]bool) 66 | rt.Buckets = make(map[int]map[string]bool) 67 | 68 | for i := 0; i < rt.NumBucket; i++ { 69 | rt.Buckets[i] = make(map[string]bool) 70 | } 71 | for _, server := range rt.Main { 72 | server.Decode() 73 | addr := server.Addr 74 | rt.Servers[addr] = make(map[int]bool) 75 | for _, bucket := range server.Buckets { 76 | rt.Servers[addr][bucket] = true 77 | rt.Buckets[bucket][addr] = true 78 | } 79 | } 80 | for _, addr := range rt.Backup { 81 | rt.Servers[addr] = make(map[int]bool) 82 | for bucket := 0; bucket < rt.NumBucket; bucket++ { 83 | rt.Servers[addr][bucket] = false 84 | rt.Buckets[bucket][addr] = false 85 | 86 | } 87 | } 88 | 89 | return nil 90 | } 91 | 92 | func LoadRouteTableLocal(path string) (*RouteTable, error) { 93 | LocalRoutePath = path 94 | rt := &RouteTable{} 95 | data, err := ioutil.ReadFile(LocalRoutePath) 96 | if err != nil { 97 | return nil, err 98 | } 99 | err = rt.LoadFromYaml(data) 100 | if err != nil { 101 | return nil, err 102 | } 103 | return rt, nil 104 | } 105 | 106 | func LoadRouteTableZK(path, cluster string, zkservers []string) (*RouteTable, error) { 107 | LocalRoutePath = path 108 | 109 | rt := &RouteTable{} 110 | client, err := NewZK(cluster, zkservers) 111 | if err != nil { 112 | return nil, err 113 | } 114 | ZKClient = client 115 | data, ver, err := client.GetRouteRaw(-1) 116 | if err != nil { 117 | return nil, err 118 | } 119 | 120 | err = rt.LoadFromYaml(data) 121 | if err != nil { 122 | return nil, err 123 | } 124 | ZKClient.Version = ver 125 | UpdateLocalRoute(data) 126 | return rt, nil 127 | } 128 | -------------------------------------------------------------------------------- /store/key.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "strconv" 5 | "unicode" 6 | 7 | "github.com/spaolacci/murmur3" 8 | ) 9 | 10 | const ( 11 | MAX_KEY_LEN = 250 12 | ) 13 | 14 | var ( 15 | getKeyHash HashFuncType = getKeyHashDefalut 16 | ) 17 | 18 | type HashFuncType func(key []byte) uint64 19 | 20 | func IsValidKeyString(key string) bool { 21 | length := len(key) 22 | if length == 0 || length > MAX_KEY_LEN { 23 | logger.Warnf("bad key len=%d", length) 24 | return false 25 | } 26 | 27 | if key[0] <= ' ' || key[0] == '?' || key[0] == '@' { 28 | logger.Warnf("bad key len=%d key[0]=%x", length, key[0]) 29 | return false 30 | } 31 | 32 | for _, r := range key { 33 | if unicode.IsControl(r) || unicode.IsSpace(r) { 34 | logger.Warnf("bad key len=%d %s", length, key) 35 | return false 36 | } 37 | } 38 | return true 39 | } 40 | 41 | func murmur(data []byte) (h uint32) { 42 | hasher := murmur3.New32() 43 | hasher.Write(data) 44 | return hasher.Sum32() 45 | } 46 | 47 | func fnv1a(data []byte) (h uint32) { 48 | PRIME := uint32(0x01000193) 49 | h = 0x811c9dc5 50 | for _, b := range data { 51 | h ^= uint32(int8(b)) 52 | h = (h * PRIME) 53 | } 54 | return h 55 | } 56 | 57 | func getKeyHashDefalut(key []byte) uint64 { 58 | return (uint64(fnv1a(key)) << 32) | uint64(murmur(key)) 59 | } 60 | 61 | // computed once, before being routed to a bucket 62 | type KeyPos struct { 63 | KeyPathBuf [16]int 64 | KeyPath []int 65 | 66 | // need depth 67 | BucketID int 68 | KeyPathInBucket []int 69 | } 70 | 71 | type KeyInfo struct { 72 | KeyHash uint64 73 | KeyIsPath bool 74 | Key []byte 75 | StringKey string 76 | KeyPos 77 | } 78 | 79 | func getBucketFromKey(key string) int { 80 | return 0 81 | } 82 | 83 | func ParsePathUint64(khash uint64, buf []int) []int { 84 | for i := 0; i < 16; i++ { 85 | shift := uint32(4 * (15 - i)) 86 | idx := int((khash >> shift)) & 0xf 87 | buf[i] = int(idx) 88 | } 89 | return buf 90 | } 91 | 92 | func ParsePathString(pathStr string, buf []int) ([]int, error) { 93 | path := buf[:len(pathStr)] 94 | for i := 0; i < len(pathStr); i++ { 95 | idx, err := strconv.ParseInt(pathStr[i:i+1], 16, 0) 96 | if err != nil { 97 | return nil, err 98 | } 99 | path[i] = int(idx) 100 | } 101 | return path, nil 102 | } 103 | 104 | func NewKeyInfoFromBytes(key []byte, keyhash uint64, keyIsPath bool) (ki *KeyInfo) { 105 | ki = &KeyInfo{ 106 | KeyIsPath: keyIsPath, 107 | Key: key, 108 | StringKey: string(key), 109 | KeyHash: keyhash, 110 | } 111 | ki.Prepare() 112 | return 113 | } 114 | 115 | func (ki *KeyInfo) setKeyHashByPath() { 116 | v := ki.KeyPath 117 | shift := uint(60) 118 | ki.KeyHash = 0 119 | for i := 0; i < len(v); i++ { 120 | ki.KeyHash |= (uint64(v[i]) << shift) 121 | shift -= 4 122 | } 123 | } 124 | 125 | func (ki *KeyInfo) Prepare() (err error) { 126 | if ki.KeyIsPath { 127 | ki.KeyPath, err = ParsePathString(ki.StringKey, ki.KeyPathBuf[:16]) 128 | ki.setKeyHashByPath() 129 | if len(ki.KeyPath) < Conf.TreeDepth { 130 | ki.BucketID = -1 131 | return 132 | } 133 | } else { 134 | ki.KeyPath = ParsePathUint64(ki.KeyHash, ki.KeyPathBuf[:16]) 135 | } 136 | ki.BucketID = 0 137 | for _, v := range ki.KeyPath[:Conf.TreeDepth] { 138 | ki.BucketID <<= 4 139 | ki.BucketID += v 140 | } 141 | return 142 | } 143 | 144 | func isSamePath(x, y []int, n int) bool { 145 | for i := 0; i < n; i++ { 146 | if x[i] != y[i] { 147 | return false 148 | } 149 | } 150 | return true 151 | } 152 | -------------------------------------------------------------------------------- /store/leaf_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | ) 7 | 8 | func TestBytes(t *testing.T) { 9 | b := make([]byte, 30) 10 | m := HTreeItem{Keyhash: 0, Pos: Position{1, 0}, Ver: 2, Vhash: 3} 11 | itemToBytes(b, &m) 12 | m2 := HTreeItem{Keyhash: 0, Pos: Position{4, 0}, Ver: 5, Vhash: 6} 13 | bytesToItem(b, &m2) 14 | if m != m2 { 15 | t.Fatalf("bytesToItem fail %v != %v", m, m2) 16 | } 17 | 18 | k := uint64(0xaaabbbbb) << 32 19 | path := []int{0xa, 0xa, 0xa} 20 | 21 | khashToBytes(b, k) 22 | 23 | lenKHash := KHASH_LENS[len(path)] 24 | mask := ((uint64(1) << (uint32(lenKHash) * 8)) - 1) 25 | nodeKHash := k & (^mask) 26 | k2 := bytesToKhash(b) 27 | k2 &= mask 28 | k2 += (nodeKHash) 29 | 30 | t.Logf("lenKHash = %d, mask %016x, node hash %016x", lenKHash, mask, nodeKHash) 31 | if k != k2 { 32 | t.Fatalf("bytesToKhash fail %016x != %016x", k, k2) 33 | } 34 | } 35 | 36 | func TestLeafEnlarge(t *testing.T) { 37 | var sh SliceHeader 38 | sh.enlarge(10) 39 | leaf := sh.ToBytes() 40 | if len(leaf) != 10 { 41 | t.Fatalf("%v %v", leaf, sh) 42 | } 43 | data := []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} 44 | copy(leaf, data) 45 | sh.enlarge(20) 46 | leaf = sh.ToBytes() 47 | if len(leaf) != 20 || 0 != bytes.Compare(leaf[:10], data) { 48 | t.Fatalf("%v", leaf) 49 | } 50 | } 51 | 52 | func TestLeaf(t *testing.T) { 53 | Conf.InitDefault() 54 | 55 | Conf.NumBucket = 256 56 | Conf.Init() 57 | 58 | // lenKHash := KHASH_LENS[len(ni.path)] 59 | // t.Logf("%d %d", lenKHash, Conf.TreeKeyHashLen) 60 | lenKHash := Conf.TreeKeyHashLen 61 | lenItem := lenKHash + TREE_ITEM_HEAD_SIZE 62 | 63 | var sh SliceHeader 64 | var leaf []byte 65 | var base uint64 = 0xfe * (1 << 56) 66 | N := 16 67 | exist := true 68 | 69 | ki := NewKeyInfoFromBytes([]byte("key"), base, false) 70 | 71 | req := HTreeReq{ki: ki} 72 | 73 | reset := func() { 74 | req.ki.KeyHash = base 75 | req.Meta = Meta{Ver: 1, ValueHash: 255} 76 | req.Position = Position{0, 0} 77 | } 78 | 79 | // set 80 | reset() 81 | for i := 0; i < N; i++ { 82 | ki.Prepare() 83 | req.encode() 84 | _, exist = sh.Set(&req) 85 | leaf = sh.ToBytes() 86 | if exist || len(leaf) != (i+1)*lenItem { 87 | t.Fatalf("i = %d, leaf = %v, sh = %v, exist = %v", i, leaf, sh, exist) 88 | } 89 | req.Offset += PADDING 90 | ki.KeyHash++ 91 | } 92 | 93 | // update 94 | var shift uint32 = 100 95 | 96 | reset() 97 | req.Offset += shift * PADDING 98 | for i := 0; i < N; i++ { 99 | ki.Prepare() 100 | req.encode() 101 | _, exist = sh.Set(&req) 102 | leaf = sh.ToBytes() 103 | if !exist || len(leaf) != N*lenItem { 104 | t.Fatalf("i = %d, leaf = %v, exist = %v", i, leaf, exist) 105 | } 106 | req.Offset += PADDING 107 | ki.KeyHash++ 108 | } 109 | 110 | // get 111 | reset() 112 | for i := 0; i < N; i++ { 113 | ki.Prepare() 114 | found := sh.Get(&req) 115 | if !found || req.item.Pos.Offset != (uint32(i)+shift)*PADDING { 116 | t.Fatalf("i=%d, shift=%d, found=%v, req.item=%#v", i, shift, found, req.item) 117 | } 118 | ki.KeyHash++ 119 | } 120 | 121 | // remove 122 | reset() 123 | req.Offset += shift * PADDING 124 | for i := 0; i < N; i++ { 125 | ki.Prepare() 126 | oldm, removed := sh.Remove(ki, Position{0, req.Offset}) 127 | if !removed || oldm.Pos.Offset != req.Offset || sh.Len != lenItem*(N-i-1) { 128 | t.Fatalf("i=%d, offset=%x, removed=%v, oldm =%#v, %v", 129 | i, req.Offset, removed, oldm, sh.Len/lenItem) 130 | } 131 | req.Offset += PADDING 132 | ki.KeyHash++ 133 | } 134 | 135 | // iter 136 | i := 0 137 | f := func(h uint64, m *HTreeItem) { 138 | if h != base+uint64(i) || m.Pos.Offset != (uint32(i)+shift)*PADDING { 139 | t.Fatalf("%d: %016x %v", i, h, m) 140 | } 141 | i += 1 142 | } 143 | 144 | var ni NodeInfo 145 | ni.path = []int{0xf, 0xe} 146 | sh.Iter(f, &ni) 147 | } 148 | -------------------------------------------------------------------------------- /store/hintmerge.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "container/heap" 5 | "fmt" 6 | 7 | "github.com/douban/gobeansdb/utils" 8 | ) 9 | 10 | type mergeReader struct { 11 | r *hintFileReader 12 | curr *HintItem 13 | } 14 | 15 | type mergeWriter struct { 16 | w *hintFileWriter 17 | buf []*HintItem // always contain items with same khash and diff keys 18 | ct *CollisionTable 19 | num int 20 | } 21 | 22 | func newMergeWriter(w *hintFileWriter, ct *CollisionTable) *mergeWriter { 23 | mw := new(mergeWriter) 24 | mw.w = w 25 | mw.buf = make([]*HintItem, 1000) 26 | mw.ct = ct 27 | return mw 28 | } 29 | 30 | func (mw *mergeWriter) write(it *HintItem) { 31 | if mw.num == 0 { // the first 32 | mw.buf[0] = it 33 | mw.num = 1 34 | return 35 | } 36 | last := mw.buf[mw.num-1] 37 | if last.Keyhash != it.Keyhash { 38 | mw.flush() 39 | mw.num = 1 40 | mw.buf[0] = it 41 | } else { 42 | if last.Key != it.Key { 43 | mw.num += 1 44 | if mw.num > len(mw.buf) { 45 | newbuf := make([]*HintItem, len(mw.buf)*2) 46 | copy(newbuf, mw.buf) 47 | mw.buf = newbuf 48 | } 49 | } 50 | mw.buf[mw.num-1] = it 51 | } 52 | } 53 | 54 | func (mw *mergeWriter) flush() { 55 | if mw.num > 1 { 56 | for i := 0; i < mw.num; i++ { 57 | mw.ct.compareAndSet(mw.buf[i], "merge") 58 | } 59 | } 60 | if mw.w != nil { 61 | for i := 0; i < mw.num; i++ { 62 | mw.w.writeItem(mw.buf[i]) 63 | } 64 | } 65 | } 66 | 67 | type mergeHeap []*mergeReader 68 | 69 | func (h mergeHeap) Len() int { return len(h) } 70 | func (h mergeHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } 71 | func (h mergeHeap) Less(i, j int) bool { 72 | a := h[i].curr 73 | b := h[j].curr 74 | if a.Keyhash != b.Keyhash { 75 | return a.Keyhash < b.Keyhash 76 | } else { 77 | if a.Key != b.Key { 78 | return a.Key < b.Key 79 | } 80 | } 81 | return a.Pos.CmpKey() < b.Pos.CmpKey() 82 | } 83 | 84 | func (h *mergeHeap) Push(x interface{}) { 85 | *h = append(*h, x.(*mergeReader)) 86 | } 87 | 88 | func (h *mergeHeap) Pop() interface{} { 89 | old := *h 90 | n := len(old) 91 | x := old[n-1] 92 | *h = old[0 : n-1] 93 | return x 94 | } 95 | 96 | func merge(src []*hintFileReader, dst string, ct *CollisionTable, hintState *int, forGC bool) (idx *hintFileIndex, err error) { 97 | n := len(src) 98 | datasize := uint32(0) 99 | hp := make([]*mergeReader, n) 100 | for i := 0; i < n; i++ { 101 | err := src[i].open() 102 | if err != nil { 103 | logger.Errorf("%s", err.Error()) 104 | return nil, err 105 | } 106 | hp[i] = &mergeReader{src[i], nil} 107 | hp[i].curr, err = src[i].next() 108 | hp[i].curr.Pos.ChunkID = src[i].chunkID 109 | if err != nil { 110 | logger.Errorf("%s", err.Error()) 111 | return nil, err 112 | } 113 | if src[i].datasize > datasize { 114 | datasize = src[i].datasize 115 | } 116 | } 117 | var w *hintFileWriter 118 | if !Conf.NoMerged && !forGC { 119 | w, err = newHintFileWriter(dst, datasize, 1<<20) 120 | if err != nil { 121 | logger.Errorf("%s", err.Error()) 122 | w = nil 123 | } 124 | } 125 | 126 | mw := newMergeWriter(w, ct) 127 | h := mergeHeap(hp) 128 | heap.Init(&h) 129 | for len(h) > 0 { 130 | if *hintState&HintStateGC != 0 && !forGC { 131 | err = fmt.Errorf("aborted by gc") 132 | break 133 | } 134 | mr := heap.Pop(&h).(*mergeReader) 135 | mw.write(mr.curr) 136 | mr.curr, err = mr.r.next() 137 | if err != nil { 138 | logger.Errorf("%s", err.Error()) 139 | break 140 | } 141 | if mr.curr != nil { 142 | mr.curr.Pos.ChunkID = mr.r.chunkID 143 | heap.Push(&h, mr) 144 | } 145 | } 146 | for _, mr := range hp { 147 | mr.r.close() 148 | } 149 | mw.flush() 150 | if mw.w != nil { 151 | mw.w.close() 152 | idx = &hintFileIndex{mw.w.index.toIndex(), dst, w.hintFileMeta} 153 | } 154 | if err != nil { 155 | utils.Remove(dst) 156 | return nil, err 157 | } 158 | return 159 | } 160 | -------------------------------------------------------------------------------- /store/hintindex.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "os" 7 | "sort" 8 | ) 9 | 10 | type hintIndexItem struct { 11 | keyhash uint64 12 | offset int64 13 | } 14 | 15 | type hintFileIndex struct { 16 | index []hintIndexItem 17 | path string 18 | hintFileMeta 19 | } 20 | 21 | func newHintFileIndex() (idx *hintFileIndexBuffer) { 22 | idx = new(hintFileIndexBuffer) 23 | idx.index = make([][]hintIndexItem, 4096) 24 | idx.index[0] = make([]hintIndexItem, HINTINDEX_ROW_SIZE) 25 | return 26 | } 27 | 28 | func (idx *hintFileIndex) get(keyhash uint64, key string) (item *HintItem, err error) { 29 | //logger.Warnf("try get from hintfile %s: %016x, %s", idx.path, keyhash, key) 30 | arr := idx.index 31 | 32 | // first larger 33 | j := sort.Search(len(arr), func(i int) bool { return arr[i].keyhash >= keyhash }) 34 | offset := int64(HINTFILE_HEAD_SIZE) 35 | if j > 1 { 36 | offset = arr[j-1].offset 37 | } 38 | reader := newHintFileReader(idx.path, 0, int(Conf.IndexIntervalSize)) 39 | err = reader.open() 40 | if err != nil { 41 | return 42 | } 43 | reader.fd.Seek(offset, 0) 44 | reader.rbuf.Reset(reader.fd) 45 | defer reader.fd.Close() 46 | var it *HintItem 47 | for { 48 | it, err = reader.next() 49 | if err != nil { 50 | return 51 | } 52 | if it == nil { 53 | return 54 | } 55 | if it.Keyhash < keyhash { 56 | continue 57 | } else if it.Keyhash > keyhash { 58 | return 59 | } else { 60 | if it.Key == key { 61 | item = it 62 | return 63 | } else { 64 | continue 65 | } 66 | } 67 | } 68 | return 69 | } 70 | 71 | func loadHintIndex(path string) (index *hintFileIndex, err error) { 72 | fd, err := os.Open(path) 73 | if err != nil { 74 | logger.Errorf(err.Error()) 75 | return 76 | } 77 | fileInfo, _ := fd.Stat() 78 | size := fileInfo.Size() 79 | 80 | var head [HINTFILE_HEAD_SIZE]byte 81 | 82 | var readn int 83 | readn, err = fd.Read(head[:]) 84 | if err != nil { 85 | logger.Errorf(err.Error()) 86 | return 87 | } 88 | 89 | if readn < HINTFILE_HEAD_SIZE { 90 | err = fmt.Errorf("bad hint file %s readn %d", path, readn) 91 | logger.Errorf(err.Error()) 92 | return 93 | } 94 | index = &hintFileIndex{path: path} 95 | index.hintFileMeta.Loads(head[:]) 96 | start := index.indexOffset 97 | num := int(size - start) 98 | fd.Seek(start, 0) 99 | raw := make([]byte, num) 100 | readn, err = fd.Read(raw) 101 | if err != nil { 102 | logger.Errorf(err.Error()) 103 | return 104 | } 105 | if int64(readn) < size-start { 106 | err = fmt.Errorf("bad hint file %s readn %d", path, start) 107 | logger.Errorf(err.Error()) 108 | return 109 | } 110 | fd.Close() 111 | arr := make([]hintIndexItem, num/16) 112 | for offset := 0; offset < num; offset += 16 { 113 | i := offset / 16 114 | arr[i].keyhash = binary.LittleEndian.Uint64(raw[offset : offset+8]) 115 | arr[i].offset = int64(binary.LittleEndian.Uint64(raw[offset+8 : offset+16])) 116 | } 117 | index.index = arr 118 | return 119 | } 120 | 121 | type hintFileIndexBuffer struct { 122 | index [][]hintIndexItem 123 | currRow int 124 | currCol int 125 | lastoffset int64 126 | } 127 | 128 | func (b *hintFileIndexBuffer) toIndex() []hintIndexItem { 129 | n := HINTINDEX_ROW_SIZE*b.currRow + b.currCol 130 | arr := make([]hintIndexItem, n) 131 | size := 0 132 | for r := 0; r < b.currRow; r++ { 133 | copy(arr[size:], b.index[r]) 134 | size += HINTINDEX_ROW_SIZE 135 | } 136 | copy(arr[size:], b.index[b.currRow][:b.currCol]) 137 | return arr 138 | } 139 | 140 | func (idx *hintFileIndexBuffer) append(keyhash uint64, offset int64) { 141 | idx.index[idx.currRow][idx.currCol] = hintIndexItem{keyhash, offset} 142 | idx.lastoffset = offset 143 | if idx.currCol >= HINTINDEX_ROW_SIZE-1 { 144 | idx.currRow += 1 145 | idx.index[idx.currRow] = make([]hintIndexItem, HINTINDEX_ROW_SIZE) 146 | idx.currCol = 0 147 | } else { 148 | idx.currCol += 1 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /store/config.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | "github.com/douban/gobeansdb/config" 8 | "github.com/douban/gobeansdb/utils" 9 | ) 10 | 11 | var ( 12 | KHASH_LENS = [8]int{8, 8, 7, 7, 6, 6, 5, 5} 13 | Conf *HStoreConfig 14 | ) 15 | 16 | func init() { 17 | Conf = &HStoreConfig{} 18 | Conf.InitDefault() 19 | } 20 | 21 | type HStoreConfig struct { 22 | config.DBRouteConfig `yaml:"-"` // from route table 23 | DBLocalConfig `yaml:"local,omitempty"` 24 | 25 | DataConfig `yaml:"data,omitempty"` 26 | HintConfig `yaml:"hint,omitempty"` 27 | HTreeConfig `yaml:"htree,omitempty"` 28 | } 29 | 30 | type HtreeDerivedConfig struct { 31 | TreeDepth int // from NumBucket 32 | TreeKeyHashMask uint64 33 | TreeKeyHashLen int 34 | } 35 | 36 | type DBLocalConfig struct { 37 | Home string `yaml:",omitempty"` 38 | } 39 | 40 | type DataConfig struct { 41 | FlushWake int64 `yaml:"-"` // after set to flush buffer, wake up flush go routine if buffer size > this 42 | DataFileMax int64 `yaml:"-"` // data rotate when reach the size 43 | CheckVHash bool `yaml:"check_vhash,omitempty"` // not really set if vhash is the same 44 | FlushInterval int `yaml:"flush_interval,omitempty"` // the flush go routine run at this interval 45 | NoGCDays int `yaml:"no_gc_days,omitempty"` // not data files whose mtime in recent NoGCDays days 46 | 47 | FlushWakeStr string `yaml:"flush_wake_str"` // 48 | DataFileMaxStr string `yaml:"datafile_max_str,omitempty"` 49 | BufIOCap int `yaml:"-"` // for bufio reader/writer, if value is big, then enlarge this cap, defalult: 1MB 50 | BufIOCapStr string `yaml:"bufio_cap_str,omitempty"` 51 | NotCompress map[string]bool `yaml:"not_compress,omitempty"` // kind do not compress 52 | } 53 | 54 | type HTreeConfig struct { 55 | TreeHeight int `yaml:"tree_height,omitempty"` 56 | TreeDump int `yaml:"tree_dump,omitempty"` 57 | 58 | HtreeDerivedConfig `yaml:"-"` 59 | } 60 | 61 | type HintConfig struct { 62 | NoMerged bool `yaml:"hint_no_merged,omitempty"` // merge only used to find collision, but not dump idx.m to save disk space 63 | MergeInterval int `yaml:"hint_merge_interval,omitempty"` // merge after rotating each MergeInterval chunk 64 | IndexIntervalSize int64 `yaml:"-"` // max diff of offsets of two adjacent hint index items 65 | SplitCap int64 `yaml:"-"` // pre alloc SplitCap slot for each split, when slots are all filled, slot is dumped 66 | 67 | SplitCapStr string `yaml:"hint_split_cap_str,omitempty"` 68 | IndexIntervalSizeStr string `yaml:"hint_index_interval_str,omitempty"` 69 | } 70 | 71 | // for test 72 | func (c *HStoreConfig) Init() error { 73 | e := utils.InitSizesPointer(c) 74 | if e != nil { 75 | return e 76 | } 77 | return c.InitTree() 78 | } 79 | 80 | // must be called before use 81 | // NumBucket => TreeDepth => (TreeKeyHashLen & TreeKeyHashMask) 82 | func (c *HStoreConfig) InitTree() error { 83 | // TreeDepth 84 | n := c.NumBucket 85 | c.TreeDepth = 0 86 | for n > 1 { 87 | c.TreeDepth += 1 88 | n /= 16 89 | } 90 | // TreeKeyHashLen & TreeKeyHashMask 91 | c.TreeKeyHashLen = KHASH_LENS[c.TreeDepth+c.TreeHeight-1] 92 | shift := 64 - uint32(c.TreeKeyHashLen)*8 93 | c.TreeKeyHashMask = (uint64(0xffffffffffffffff) << shift) >> shift 94 | 95 | return nil 96 | } 97 | 98 | func GetBucketDir(numBucket, bucketID int) string { 99 | if numBucket == 1 { 100 | return "" 101 | } else if numBucket == 16 { 102 | return fmt.Sprintf("%x", bucketID) 103 | } else if numBucket == 256 { 104 | return fmt.Sprintf("%x/%x", bucketID/16, bucketID%16) 105 | } 106 | panic(fmt.Sprintf("wrong numBucket: %d", numBucket)) 107 | } 108 | 109 | func GetBucketPath(bucketID int) string { 110 | return filepath.Join(Conf.Home, GetBucketDir(Conf.NumBucket, bucketID)) 111 | } 112 | -------------------------------------------------------------------------------- /store/crc32.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | /* 4 | #include 5 | static const uint32_t crc32_table[256] = 6 | { 7 | 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 8 | 0x706af48f, 0xe963a535, 0x9e6495a3, 0x0edb8832, 0x79dcb8a4, 9 | 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 10 | 0x90bf1d91, 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 11 | 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, 0x136c9856, 12 | 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 13 | 0xfa0f3d63, 0x8d080df5, 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 14 | 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, 15 | 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 16 | 0x45df5c75, 0xdcd60dcf, 0xabd13d59, 0x26d930ac, 0x51de003a, 17 | 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 18 | 0xb8bda50f, 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 19 | 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, 0x76dc4190, 20 | 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 21 | 0x9fbfe4a5, 0xe8b8d433, 0x7807c9a2, 0x0f00f934, 0x9609a88e, 22 | 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, 23 | 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 24 | 0x1b01a57b, 0x8208f4c1, 0xf50fc457, 0x65b0d9c6, 0x12b7e950, 25 | 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 26 | 0xfbd44c65, 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 27 | 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, 0x4369e96a, 28 | 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 29 | 0xaa0a4c5f, 0xdd0d7cc9, 0x5005713c, 0x270241aa, 0xbe0b1010, 30 | 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, 31 | 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 32 | 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, 0xedb88320, 0x9abfb3b6, 33 | 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 34 | 0x73dc1683, 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 35 | 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, 0xf00f9344, 36 | 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 37 | 0x196c3671, 0x6e6b06e7, 0xfed41b76, 0x89d32be0, 0x10da7a5a, 38 | 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, 39 | 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 40 | 0xa6bc5767, 0x3fb506dd, 0x48b2364b, 0xd80d2bda, 0xaf0a1b4c, 41 | 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 42 | 0x4669be79, 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 43 | 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, 0xc5ba3bbe, 44 | 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 45 | 0x2cd99e8b, 0x5bdeae1d, 0x9b64c2b0, 0xec63f226, 0x756aa39c, 46 | 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, 47 | 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 48 | 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, 0x86d3d2d4, 0xf1d4e242, 49 | 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 50 | 0x18b74777, 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 51 | 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, 0xa00ae278, 52 | 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 53 | 0x4969474d, 0x3e6e77db, 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 54 | 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, 55 | 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 56 | 0xcdd70693, 0x54de5729, 0x23d967bf, 0xb3667a2e, 0xc4614ab8, 57 | 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 58 | 0x2d02ef8d 59 | }; 60 | typedef unsigned char uchar; 61 | uint32_t 62 | crc32_write (uint32_t crc, unsigned char *buf, int len) 63 | { 64 | unsigned char *end; 65 | for (end = buf + len; buf < end; ++buf) 66 | crc = crc32_table[(crc ^ *buf) & 0xff] ^ (crc >> 8); 67 | return crc; 68 | } 69 | */ 70 | import "C" 71 | import "unsafe" 72 | 73 | type crc32 struct { 74 | crc uint32 75 | } 76 | 77 | func newCrc32() *crc32 { 78 | return &crc32{^uint32(0)} 79 | } 80 | 81 | func (h *crc32) write(data []byte) { 82 | c_data := (*C.uchar)(unsafe.Pointer(&data[0])) 83 | h.crc = uint32(C.crc32_write(C.uint32_t(h.crc), c_data, C.int(len(data)))) 84 | } 85 | 86 | func (h *crc32) get() uint32 { 87 | return ^h.crc 88 | } 89 | -------------------------------------------------------------------------------- /utils/disk.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strings" 9 | "syscall" 10 | 11 | "github.com/douban/gobeansdb/loghub" 12 | ) 13 | 14 | func Remove(path string) error { 15 | loghub.ErrorLogger.Logf(loghub.INFO, "remove path: %s", path) 16 | return os.Remove(path) 17 | } 18 | 19 | func Rename(path, newpath string) error { 20 | loghub.ErrorLogger.Logf(loghub.INFO, "rename path: %s to %s", path, newpath) 21 | return os.Rename(path, newpath) 22 | } 23 | 24 | type Dir struct { 25 | Files map[string]int64 26 | } 27 | 28 | func NewDir() *Dir { 29 | d := &Dir{} 30 | d.Files = make(map[string]int64) 31 | return d 32 | } 33 | 34 | type File struct { 35 | Name string 36 | Size int64 37 | } 38 | 39 | func (f *File) isSame(f2 *File) (same bool) { 40 | if f.Name != f2.Name { 41 | return false 42 | } 43 | return (f.Size == -1 || f2.Size == -1 || f.Size == f2.Size) 44 | } 45 | 46 | type FileList []File 47 | 48 | func (by FileList) Len() int { return len(by) } 49 | func (by FileList) Swap(i, j int) { by[i], by[j] = by[j], by[i] } 50 | func (by FileList) Less(i, j int) bool { 51 | return by[i].Name < by[j].Name 52 | } 53 | 54 | func (d *Dir) ToSlice() []File { 55 | s := make([]File, 0, len(d.Files)) 56 | for k, v := range d.Files { 57 | s = append(s, File{k, v}) 58 | } 59 | sort.Sort(FileList(s)) 60 | return s 61 | } 62 | 63 | func (d *Dir) Set(name string, size int64) { 64 | d.Files[name] = size 65 | } 66 | 67 | func (d *Dir) SetMulti(files map[string]int64) { 68 | for name, size := range files { 69 | d.Files[name] = size 70 | } 71 | } 72 | 73 | func (d *Dir) SetMultiNoSize(files ...string) { 74 | for _, name := range files { 75 | d.Files[name] = -1 76 | } 77 | } 78 | 79 | func (d *Dir) Delete(name string) { 80 | delete(d.Files, name) 81 | } 82 | 83 | func (d *Dir) Load(path string) (err error) { 84 | f, err := os.Open(path) 85 | if err != nil { 86 | return 87 | } 88 | defer f.Close() 89 | files, err := f.Readdir(-1) 90 | if err != nil { 91 | return 92 | } 93 | for _, fi := range files { 94 | d.Files[fi.Name()] = fi.Size() 95 | } 96 | return 97 | } 98 | 99 | func (d *Dir) CheckPath(path string) (d2 *Dir, r1, r2 []File, err error) { 100 | d2 = NewDir() 101 | err = d2.Load(path) 102 | if err != nil { 103 | return 104 | } 105 | r1, r2 = d.Diff(d2) 106 | return 107 | } 108 | 109 | func (d *Dir) Diff(d2 *Dir) (r1, r2 []File) { 110 | s := d.ToSlice() 111 | s2 := d2.ToSlice() 112 | i := 0 113 | j := 0 114 | for { 115 | if i >= len(s) { 116 | r2 = append(r2, s2[j:]...) 117 | break 118 | } 119 | if j >= len(s2) { 120 | r1 = append(r1, s[i:]...) 121 | break 122 | } 123 | if s[i].isSame(&s2[j]) { 124 | i += 1 125 | j += 1 126 | } else if s[i].Name < s2[j].Name { 127 | r1 = append(r1, s[i]) 128 | i++ 129 | } else if s[i].Name > s2[j].Name { 130 | 131 | r2 = append(r2, s2[j]) 132 | j++ 133 | } else { 134 | r1 = append(r1, s[i]) 135 | r2 = append(r2, s2[j]) 136 | i += 1 137 | j += 1 138 | } 139 | } 140 | return 141 | } 142 | 143 | type DiskStatus struct { 144 | Root string 145 | All int64 146 | Used int64 147 | Free int64 148 | Buckets []int `yaml:",flow"` 149 | } 150 | 151 | func DiskUsage(path string) (disk DiskStatus, err error) { 152 | abspath, err := filepath.Abs(path) 153 | if err != nil { 154 | return 155 | } 156 | realpath, err := filepath.EvalSymlinks(abspath) 157 | if err != nil { 158 | return 159 | } 160 | realpath = filepath.Clean(realpath) 161 | 162 | fs := syscall.Statfs_t{} 163 | err = syscall.Statfs(realpath, &fs) 164 | if err != nil { 165 | return 166 | } 167 | parts := strings.Split(realpath, "/") 168 | if len(parts) < 2 { 169 | err = fmt.Errorf("bad path <%s>", path) 170 | return 171 | } 172 | disk.Root = "/" + parts[1] 173 | disk.All = int64(fs.Blocks) * int64(fs.Bsize) 174 | disk.Free = int64(fs.Bfree) * int64(fs.Bsize) 175 | disk.Used = disk.All - disk.Free 176 | return 177 | } 178 | 179 | func DirUsage(path string) (size int64, err error) { 180 | size = 0 181 | f, err := os.Open(path) 182 | if err != nil { 183 | size = -1 184 | return 185 | } 186 | fis, err := f.Readdir(-1) 187 | if err != nil { 188 | return 189 | } 190 | for _, fi := range fis { 191 | size += fi.Size() 192 | } 193 | return 194 | } 195 | -------------------------------------------------------------------------------- /memcache/protocol_test.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/douban/gobeansdb/config" 11 | ) 12 | 13 | type reqTest struct { 14 | cmd string 15 | answer string 16 | maxSize int64 17 | } 18 | 19 | var reqTests = []reqTest{ 20 | { 21 | cmd: "get abc cdf \r\n", 22 | answer: "END\r\n", 23 | }, 24 | { 25 | cmd: "set abc 2 3 2 noreply\r\nok\r\n", 26 | answer: "", 27 | }, 28 | { 29 | cmd: "set abc a 3 2 noreply\r\nok\r\n", 30 | answer: "CLIENT_ERROR invalid cmd\r\n", 31 | }, 32 | { 33 | cmd: "set cdf 0 0 2\r\nok\r\n", 34 | answer: "STORED\r\n", 35 | }, 36 | { 37 | cmd: "get cdf\r\n", 38 | answer: "VALUE cdf 0 2\r\nok\r\nEND\r\n", 39 | }, 40 | { 41 | cmd: "get abc \r\n", 42 | answer: "VALUE abc 2 2\r\nok\r\nEND\r\n", 43 | }, 44 | { 45 | cmd: "stats curr_items cmd_get cmd_set get_hits get_misses\r\n", 46 | answer: "STAT curr_items 2\r\n" + 47 | "STAT cmd_get 4\r\n" + 48 | "STAT cmd_set 2\r\n" + 49 | "STAT get_hits 2\r\n" + 50 | "STAT get_misses 2\r\n" + 51 | "END\r\n", 52 | }, 53 | { 54 | cmd: "set abc 3 3 3 2 noreply\r\nok\r\n", 55 | answer: "CLIENT_ERROR invalid cmd\r\n", 56 | }, 57 | { 58 | cmd: "set abc a 3 2 noreply\r\nok\r\n", 59 | answer: "CLIENT_ERROR invalid cmd\r\n", 60 | }, 61 | { 62 | cmd: "set abc 3 3 10\r\nok\r\n", 63 | answer: "CLIENT_ERROR network error\r\n", 64 | }, 65 | { 66 | cmd: "get \r\n", 67 | answer: "CLIENT_ERROR invalid cmd\r\n", 68 | }, 69 | { 70 | cmd: "get " + strings.Repeat("a", 300) + " \r\n", 71 | answer: "CLIENT_ERROR key length error\r\n", 72 | }, 73 | { 74 | cmd: "set hello 0 0 92160\r\n" + strings.Repeat("a", 1024*90) + "\r\n", 75 | answer: "CLIENT_ERROR value too large\r\n", 76 | maxSize: 1000, 77 | }, 78 | { 79 | cmd: "set hello 0 0 1000\r\n" + strings.Repeat("a", 1000) + "\r\n", 80 | answer: "STORED\r\n", 81 | maxSize: 1000, 82 | }, 83 | { 84 | cmd: "set hello 0 0 1000\r\n" + strings.Repeat("a", 1000) + "\r\n", 85 | answer: "CLIENT_ERROR value too large\r\n", 86 | maxSize: 999, 87 | }, 88 | { 89 | cmd: "set hello 0 0 1000\r\n" + strings.Repeat("a", 1000) + "\r\n", 90 | answer: "STORED\r\n", 91 | maxSize: 1001, 92 | }, 93 | /* no need to keep origin order 94 | reqTest{ 95 | "get abc cdf\r\n", 96 | "VALUE abc 2 2\r\nok\r\nVALUE cdf 0 2\r\nok\r\nEND\r\n", 97 | }, 98 | */ 99 | // reqTest{ 100 | // "cas abc -5 10 0 134020434\r\n\r\n", 101 | // "STORED\r\n", 102 | // }, 103 | { 104 | cmd: "delete abc\r\n", 105 | answer: "DELETED\r\n", 106 | }, 107 | { 108 | cmd: "delete abc noreply\r\n", 109 | answer: "", 110 | }, 111 | { 112 | cmd: "append cdf 0 0 2\r\n 2\r\n", 113 | answer: "STORED\r\n", 114 | }, 115 | // reqTest{ 116 | // "prepend cdf 0 0 2\r\n1 \r\n", 117 | // "STORED", 118 | // }, 119 | { 120 | cmd: "get cdf\r\n", 121 | answer: "VALUE cdf 0 4\r\nok 2\r\nEND\r\n", 122 | }, 123 | { 124 | cmd: "append ap 0 0 2\r\nap\r\n", 125 | answer: "NOT_STORED\r\n", 126 | }, 127 | 128 | { 129 | cmd: "set n 4 0 1\r\n5\r\n", 130 | answer: "STORED\r\n", 131 | }, 132 | { 133 | cmd: "incr n 3\r\n", 134 | answer: "8\r\n", 135 | }, 136 | { 137 | cmd: "incr nn 7\r\n", 138 | answer: "7\r\n", 139 | }, 140 | 141 | { 142 | cmd: "flush_all\r\n", 143 | answer: "OK\r\n", 144 | }, 145 | { 146 | cmd: "verbosity 1\r\n", 147 | answer: "OK\r\n", 148 | }, 149 | { 150 | cmd: "version\r\n", 151 | answer: "VERSION " + config.Version + "\r\n", 152 | }, 153 | 154 | { 155 | cmd: "quit\r\n", 156 | answer: "", 157 | }, 158 | { 159 | cmd: "error\r\n", 160 | answer: "CLIENT_ERROR non memcache command\r\n", 161 | }, 162 | } 163 | 164 | func TestRequest(t *testing.T) { 165 | InitTokens() 166 | store := NewMapStore() 167 | stats := NewStats() 168 | 169 | for i, test := range reqTests { 170 | if test.maxSize > 0 { 171 | config.MCConf.BodyMax = test.maxSize 172 | } 173 | buf := bytes.NewBufferString(test.cmd) 174 | req := new(Request) 175 | e := req.Read(bufio.NewReader(buf)) 176 | var resp *Response 177 | if e != nil { 178 | resp = &Response{Status: "CLIENT_ERROR", Msg: e.Error()} 179 | } else { 180 | resp, _ = req.Process(store, stats) 181 | } 182 | 183 | r := make([]byte, 0) 184 | wr := bytes.NewBuffer(r) 185 | if resp != nil { 186 | resp.Write(wr) 187 | } 188 | ans := wr.String() 189 | if test.answer != ans { 190 | fmt.Print(req, resp) 191 | t.Errorf("test %d(%#v): expect %#v[%d], bug got %#v[%d]\n", i, test.cmd, 192 | test.answer, len(test.answer), ans, len(ans)) 193 | } 194 | req.Clear() 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /tests/base.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import os 4 | import yaml 5 | import time 6 | import shlex 7 | import shutil 8 | import socket 9 | import subprocess 10 | import json 11 | import urllib2 12 | import unittest 13 | 14 | from tests.dbclient import MCStore 15 | from tests.utils import mkdir_p 16 | 17 | 18 | GOBEANSDB_CMD = '../../../../bin/gobeansdb' 19 | 20 | 21 | def gethttp(addr, path): 22 | url = "http://%s/%s" % (addr, path) 23 | response = urllib2.urlopen(url) 24 | return response.read() 25 | 26 | 27 | class BaseTest(unittest.TestCase): 28 | def setUp(self): 29 | self.db = BeansdbInstance() 30 | self.db.clean_data() 31 | self.db.start() 32 | 33 | def tearDown(self): 34 | self.db.clean() 35 | # time.sleep(100) 36 | 37 | def checkCounterZero(self): 38 | time.sleep(0.5) 39 | content = gethttp(self.db.webaddr, '/buffers') 40 | buffers = json.loads(content) 41 | self.assertEqual(len(buffers), 4) 42 | for _, v in buffers.items(): 43 | self.assertEqual(v['Count'], 0, content) 44 | self.assertEqual(v['Size'], 0, content) 45 | 46 | def gc(self, bucket, start=0): 47 | content = gethttp(self.db.webaddr, "/gc/%x?start=%d&run=true" % (bucket, start)) 48 | print "gc response", content 49 | time.sleep(1) 50 | 51 | ### start/stop cmd in subprocess 52 | 53 | def start_cmd(cmd): 54 | print "start", cmd 55 | log_file = '/tmp/gobeansdb/log.txt' 56 | mkdir_p(os.path.dirname(log_file)) 57 | with open(log_file, 'a') as f: 58 | p = subprocess.Popen( 59 | cmd if isinstance(cmd, (tuple, list,)) else shlex.split(cmd), 60 | stderr=f, 61 | ) 62 | time.sleep(0.2) 63 | if p.poll() is not None: 64 | raise Exception("cannot start %s" % (cmd)) 65 | return p 66 | 67 | def stop_cmd(popen): 68 | if popen.poll() is not None: 69 | return 70 | popen.terminate() 71 | popen.wait() 72 | 73 | 74 | ### parse config 75 | 76 | SERVER_LOCAL = 'local.yaml' 77 | SERVER_GLOBAL = 'global.yaml' 78 | CONF_DIR = './conf' 79 | 80 | 81 | def get_server_addr(server_global=SERVER_GLOBAL, server_local=SERVER_LOCAL): 82 | global_conf = load_yaml(server_global) 83 | local_conf = load_yaml(server_local) 84 | port = global_conf['server']['port'] 85 | webport = global_conf['server']['webport'] 86 | if local_conf.get('server') and local_conf.get('server').get('port'): 87 | port = local_conf['server']['port'] 88 | if local_conf.get('server') and local_conf.get('server').get('webport'): 89 | webport = local_conf['server']['webport'] 90 | hostname = local_conf['hstore']['local'].get('hostname') or socket.gethostname() 91 | 92 | mc_addr = '%s:%s' % (hostname, port) 93 | web_addr = '%s:%s' % (hostname, webport) 94 | 95 | return mc_addr, web_addr 96 | 97 | 98 | def get_db_homes(server_local=SERVER_LOCAL): 99 | local_conf = load_yaml(server_local) 100 | return local_conf['hstore']['local']['homes'] 101 | 102 | 103 | def load_yaml(filename, conf_dir=CONF_DIR): 104 | filepath = os.path.join(conf_dir, filename) 105 | with open(filepath) as f: 106 | return yaml.load(f) 107 | 108 | 109 | ### BeansdbInstance 110 | 111 | class BeansdbInstance(object): 112 | ''' Start/stop Beansdb instance. 113 | ''' 114 | def __init__(self): 115 | self.popen = None 116 | self.cmd = "%s -confdir conf" % GOBEANSDB_CMD 117 | self.addr, self.webaddr = get_server_addr() 118 | 119 | self.db_homes = get_db_homes() 120 | 121 | def __del__(self): 122 | self.stop() 123 | 124 | def start(self): 125 | assert self.popen is None 126 | self.popen = start_cmd(self.cmd) 127 | try_times = 0 128 | while True: 129 | try: 130 | store = MCStore(self.addr) 131 | store.get("@") 132 | return 133 | except IOError: 134 | try_times += 1 135 | if try_times > 20: 136 | raise Exception('connect error for addr: %s', self.addr) 137 | time.sleep(0.5) 138 | 139 | def stop(self): 140 | print 'stop', self.cmd 141 | if self.popen: 142 | stop_cmd(self.popen) 143 | self.popen = None 144 | 145 | def clean(self): 146 | if self.popen: 147 | self.stop() 148 | self.clean_data() 149 | 150 | def clean_data(self): 151 | for db_home in self.db_homes: 152 | if os.path.exists(db_home): 153 | shutil.rmtree(db_home) 154 | 155 | 156 | if __name__ == '__main__': 157 | db = BeansdbInstance() 158 | db.start() 159 | db.clean() 160 | -------------------------------------------------------------------------------- /store/leaf.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | /* 4 | #include 5 | #include 6 | int find(void *ss, void *s, int item_size, int cmp_size, int n) { 7 | char *p = (char*)ss; 8 | int i; 9 | for (i = 0; i < n; i++, p += item_size) { 10 | if (0 == memcmp(p, s, cmp_size)) 11 | return i; 12 | } 13 | return -1; 14 | } 15 | */ 16 | import "C" 17 | 18 | import ( 19 | "bytes" 20 | "encoding/binary" 21 | "reflect" 22 | "unsafe" 23 | ) 24 | 25 | const LEN_USE_C_FIND = 100 26 | const TREE_ITEM_HEAD_SIZE = 11 27 | 28 | // BlockArrayLeaf 29 | 30 | type SliceHeader struct { 31 | Data uintptr 32 | Len int 33 | } 34 | 35 | func (sh *SliceHeader) ToBytes() (b []byte) { 36 | sb := (*reflect.SliceHeader)((unsafe.Pointer(&b))) 37 | sb.Data = sh.Data 38 | sb.Cap = sh.Len 39 | sb.Len = sh.Len 40 | return 41 | } 42 | 43 | type ItemFunc func(uint64, *HTreeItem) 44 | 45 | func getNodeKhash(path []int) uint32 { 46 | var khash uint32 47 | for i, off := range path { 48 | khash += uint32(((off & 0xf) << uint32((4 * (7 - i))))) 49 | } 50 | return khash 51 | } 52 | 53 | func bytesToItem(b []byte, item *HTreeItem) { 54 | item.Ver = int32(binary.LittleEndian.Uint32(b)) 55 | item.Vhash = binary.LittleEndian.Uint16(b[4:]) 56 | item.Pos.Offset = (uint32(b[6])<<8 | uint32(b[7])<<16 | uint32(b[8])<<24) 57 | //item.pos.ChunkID = int(uint32(b[9])) 58 | item.Pos.ChunkID = int(uint32(b[9]) | uint32(b[10])<<8) 59 | } 60 | 61 | func itemToBytes(b []byte, item *HTreeItem) { 62 | binary.LittleEndian.PutUint32(b, uint32(item.Ver)) 63 | binary.LittleEndian.PutUint16(b[4:], item.Vhash) 64 | v := item.Pos.Offset 65 | b[6] = byte(v >> 8) 66 | b[7] = byte(v >> 16) 67 | b[8] = byte(v >> 24) 68 | v = uint32(item.Pos.ChunkID) 69 | b[9] = byte(v) 70 | b[10] = byte(v >> 8) 71 | } 72 | 73 | func khashToBytes(b []byte, khash uint64) { 74 | binary.LittleEndian.PutUint64(b, khash) 75 | } 76 | 77 | func bytesToKhash(b []byte) (khash uint64) { 78 | return binary.LittleEndian.Uint64(b) 79 | } 80 | 81 | func findInBytes(leaf []byte, keyhash uint64) int { 82 | lenKHash := Conf.TreeKeyHashLen 83 | lenItem := lenKHash + TREE_ITEM_HEAD_SIZE 84 | size := len(leaf) 85 | var khashBytes [8]byte 86 | khashToBytes(khashBytes[0:], keyhash) 87 | kb := khashBytes[:lenKHash] 88 | n := len(leaf) / lenItem 89 | if n < LEN_USE_C_FIND { 90 | for i := 0; i < size; i += lenItem { 91 | if bytes.Compare(leaf[i:i+lenKHash], kb) == 0 { 92 | return i 93 | } 94 | } 95 | } else { 96 | ss := (*reflect.SliceHeader)((unsafe.Pointer(&leaf))).Data 97 | s := (*reflect.SliceHeader)((unsafe.Pointer(&kb))).Data 98 | i := int(C.find((unsafe.Pointer(ss)), unsafe.Pointer(s), C.int(lenItem), C.int(lenKHash), C.int(n))) 99 | return i * lenItem 100 | } 101 | return -1 102 | } 103 | 104 | // not filled with 0! 105 | func (sh *SliceHeader) enlarge(size int) { 106 | if sh.Len != 0 { 107 | sh.Data = uintptr(C.realloc(unsafe.Pointer(sh.Data), C.size_t(size))) 108 | } else { 109 | sh.Data = uintptr(C.malloc(C.size_t(size))) 110 | } 111 | sh.Len = size 112 | } 113 | 114 | func (sh *SliceHeader) free() { 115 | if sh.Len != 0 { 116 | C.free(unsafe.Pointer(sh.Data)) 117 | } 118 | } 119 | 120 | func (sh *SliceHeader) Set(req *HTreeReq) (oldm HTreeItem, exist bool) { 121 | leaf := sh.ToBytes() 122 | lenKHash := Conf.TreeKeyHashLen 123 | idx := findInBytes(leaf, req.ki.KeyHash) 124 | exist = (idx >= 0) 125 | var dst []byte 126 | if exist { 127 | bytesToItem(leaf[idx+lenKHash:], &oldm) 128 | dst = leaf[idx:] 129 | } else { 130 | newSize := len(leaf) + lenKHash + TREE_ITEM_HEAD_SIZE 131 | sh.enlarge(newSize) 132 | dst = sh.ToBytes()[len(leaf):] 133 | } 134 | khashToBytes(dst, req.ki.KeyHash) 135 | itemToBytes(dst[lenKHash:], &req.item) 136 | return 137 | } 138 | 139 | func (sh *SliceHeader) Remove(ki *KeyInfo, oldPos Position) (oldm HTreeItem, removed bool) { 140 | leaf := sh.ToBytes() 141 | lenKHash := Conf.TreeKeyHashLen 142 | itemLen := lenKHash + TREE_ITEM_HEAD_SIZE 143 | idx := findInBytes(leaf, ki.KeyHash) 144 | if idx >= 0 { 145 | bytesToItem(leaf[idx+lenKHash:], &oldm) 146 | if oldPos.ChunkID == -1 || oldm.Pos.Offset == oldPos.Offset { 147 | removed = true 148 | copy(leaf[idx:], leaf[idx+itemLen:]) 149 | sh.Len -= itemLen 150 | } 151 | } 152 | return 153 | } 154 | 155 | func (sh *SliceHeader) Get(req *HTreeReq) (exist bool) { 156 | leaf := sh.ToBytes() 157 | idx := findInBytes(leaf, req.ki.KeyHash) 158 | exist = (idx >= 0) 159 | if exist { 160 | //TODO 161 | bytesToItem(leaf[idx+Conf.TreeKeyHashLen:], &req.item) 162 | } 163 | return 164 | } 165 | 166 | func (sh *SliceHeader) Iter(f ItemFunc, ni *NodeInfo) { 167 | leaf := sh.ToBytes() 168 | lenKHash := Conf.TreeKeyHashLen 169 | lenItem := lenKHash + TREE_ITEM_HEAD_SIZE 170 | mask := Conf.TreeKeyHashMask 171 | 172 | nodeKHash := uint64(getNodeKhash(ni.path)) << 32 & (^Conf.TreeKeyHashMask) 173 | var m HTreeItem 174 | var khash uint64 175 | size := len(leaf) 176 | for i := 0; i < size; i += lenItem { 177 | bytesToItem(leaf[i+lenKHash:], &m) 178 | khash = bytesToKhash(leaf[i:]) 179 | khash &= mask 180 | khash |= nodeKHash 181 | f(khash, &m) 182 | } 183 | return 184 | } 185 | -------------------------------------------------------------------------------- /store/data_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math/rand" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "testing" 11 | "time" 12 | ) 13 | 14 | func randomValue(size int) []byte { 15 | b := make([]byte, size) 16 | for i := 0; i < size; i++ { 17 | b[i] = byte(rand.Intn(256)) 18 | } 19 | return b 20 | } 21 | 22 | func checkFileSize(t *testing.T, chunkID int, size uint32) { 23 | path := genDataPath(Conf.Home, chunkID) 24 | stat, err := os.Stat(path) 25 | if size == 0 { 26 | if err == nil { 27 | t.Fatalf("%s should not exist, size = %d", path, stat.Size()) 28 | } 29 | } else { 30 | if err != nil { 31 | t.Fatalf("%s", err.Error()) 32 | } else if stat.Size() != int64(size) { 33 | t.Fatalf("bad file size chunk %d, path %s, %d != %d", chunkID, path, stat.Size(), size) 34 | } 35 | } 36 | } 37 | 38 | func TestDataSameKeyValue1(t *testing.T) { 39 | testDataSameKeyValue(t, 1, []byte("key"), []byte("value"), 1) 40 | } 41 | func TestDataSameKeyValue2(t *testing.T) { 42 | testDataSameKeyValue(t, 2, []byte("key"), []byte(strings.Repeat("v", 255)), 1) 43 | } 44 | 45 | func TestDataSameKeyValue3(t *testing.T) { 46 | testDataSameKeyValue(t, 3, []byte("key"), randomValue(400), 2) 47 | } 48 | 49 | func TestDataSameKeyValue4(t *testing.T) { 50 | testDataSameKeyValue(t, 4, []byte(strings.Repeat("k", 200)), randomValue(512), 3) 51 | } 52 | 53 | func TestDataCompatibility(t *testing.T) { 54 | if *tDataFile == "" { 55 | t.Logf("no data file provided, pass") 56 | return 57 | } 58 | rd, err := newDataStreamReader(*tDataFile, 1<<20) 59 | if err != nil { 60 | t.Fatalf("%s not exist", *tDataFile) 61 | } 62 | count := 0 63 | for { 64 | r, _, sizeBroken, err := rd.Next() 65 | if err != nil { 66 | t.Fatalf(err.Error()) 67 | } 68 | if r == nil { 69 | break 70 | } 71 | if err := r.Payload.Decompress(); err != nil { 72 | t.Fatalf("broken datafile") 73 | } 74 | if r.Payload.Body == nil { 75 | t.Fatal("nil value") 76 | } 77 | 78 | if sizeBroken != 0 { 79 | t.Fatalf("broken datafile") 80 | } 81 | 82 | if *tDumpRecord { 83 | logger.Infof("%s", r.LogString()) 84 | } 85 | count++ 86 | 87 | } 88 | t.Logf("record count = %d", count) 89 | } 90 | 91 | func testDataSameKeyValue(t *testing.T, seq int, key, value []byte, recsize uint32) { 92 | Conf.InitDefault() 93 | setupTest(fmt.Sprintf("TestDataSameKeyValue_%d", seq)) 94 | defer clearTest() 95 | 96 | Conf.DataFileMaxStr = strconv.Itoa(int(256 * uint32(recordPerFile) * recsize)) 97 | Conf.Init() 98 | 99 | ds := NewdataStore(0, Conf.Home) 100 | 101 | for i := 0; i < recordPerFile+1; i++ { 102 | p := &Payload{} 103 | p.Body = value 104 | p.Ver = int32(i) 105 | r := &Record{key, p} 106 | pos, err := ds.AppendRecord(r) 107 | // TODO: check pos 108 | ds.flush(-1, true) 109 | r = nil 110 | r, _, err = ds.GetRecordByPos(pos) 111 | if err != nil { 112 | log.Fatal(err.Error()) 113 | } 114 | if r.Payload.Ver != int32(i) { 115 | t.Fatalf("%d %#v", i, r.Payload) 116 | } 117 | } 118 | checkFileSize(t, 0, uint32(Conf.DataFileMax)) 119 | checkFileSize(t, 1, 256*recsize) 120 | time.Sleep(time.Second) 121 | } 122 | 123 | func breakdata(f *os.File, start, offset int) { 124 | f.Seek(int64(start*256+offset), os.SEEK_SET) 125 | b := []byte("0") 126 | f.Write(b) 127 | } 128 | 129 | func TestDataBroken(t *testing.T) { 130 | Conf.InitDefault() 131 | setupTest("TestDataBroken") 132 | defer clearTest() 133 | 134 | //Conf.DataFileMaxStr = strconv.Itoa(int(256 * uint32(recordPerFile) * recsize)) 135 | Conf.Init() 136 | 137 | ds := NewdataStore(0, Conf.Home) 138 | 139 | key := []byte("key") 140 | for i := 0; i < 7; i++ { 141 | p := &Payload{} 142 | if i == 4 { 143 | p.Flag = FLAG_CLIENT_COMPRESS 144 | p.Body = []byte(strings.Repeat("x", 256*3)) 145 | } else { 146 | p.Body = []byte(fmt.Sprintf("value_%d", i)) 147 | } 148 | 149 | p.Ver = int32(i) 150 | r := &Record{key, p} 151 | ds.AppendRecord(r) 152 | } 153 | ds.flush(-1, true) 154 | 155 | path := genDataPath(Conf.Home, 0) 156 | fd, err := os.OpenFile(path, os.O_WRONLY, 0664) 157 | if err != nil { 158 | t.Fatalf(err.Error()) 159 | } 160 | 161 | breakdata(fd, 0, 16) // ksz 162 | breakdata(fd, 1, 20) // vsz 163 | breakdata(fd, 2, 24) // key 164 | breakdata(fd, 3, 24+3) // value 165 | breakdata(fd, 4, 256) // value 166 | fd.Close() 167 | 168 | reader, _ := ds.GetStreamReader(0) 169 | rec, offset, sizeBroken, err := reader.Next() 170 | if err != nil || offset != 8*256 || sizeBroken != 8*256 || 171 | rec == nil || string(rec.Payload.Body) != fmt.Sprintf("value_%d", 5) { 172 | if rec != nil { 173 | t.Errorf("%s %s", rec.Key, string(rec.Payload.Body)) 174 | } 175 | t.Fatalf("%d %d %d, %v", reader.offset, offset, sizeBroken, err) 176 | } 177 | rec, offset, sizeBroken, err = reader.Next() 178 | if err != nil || offset != 9*256 || sizeBroken != 0 || 179 | rec == nil || string(rec.Payload.Body) != fmt.Sprintf("value_%d", 6) { 180 | if rec != nil { 181 | t.Errorf("%s %s", rec.Key, string(rec.Payload.Body)) 182 | } 183 | t.Fatalf("%d %d %d, %v", reader.offset, offset, sizeBroken, err) 184 | } 185 | reader.Close() 186 | 187 | } 188 | -------------------------------------------------------------------------------- /store/item.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | 7 | "net/http" 8 | 9 | "github.com/douban/gobeansdb/cmem" 10 | "github.com/douban/gobeansdb/quicklz" 11 | "github.com/douban/gobeansdb/utils" 12 | ) 13 | 14 | const ( 15 | FLAG_INCR = 0x00000204 16 | FLAG_COMPRESS = 0x00010000 17 | FLAG_CLIENT_COMPRESS = 0x00000010 18 | COMPRESS_RATIO_LIMIT = 0.7 19 | TRY_COMPRESS_SIZE = 1024 * 10 20 | PADDING = 256 21 | HEADER_SIZE = 512 22 | ) 23 | 24 | type Meta struct { 25 | TS uint32 26 | Flag uint32 27 | Ver int32 28 | // computed once 29 | ValueHash uint16 30 | RecSize uint32 31 | } 32 | 33 | type HTreeReq struct { 34 | ki *KeyInfo 35 | Meta 36 | Position 37 | item HTreeItem 38 | } 39 | 40 | func (req *HTreeReq) encode() { 41 | req.item = HTreeItem{req.ki.KeyHash, req.Position, req.Ver, req.ValueHash} 42 | } 43 | 44 | type HintItemMeta struct { 45 | Keyhash uint64 46 | Pos Position 47 | Ver int32 48 | Vhash uint16 49 | } 50 | 51 | type HTreeItem HintItemMeta 52 | 53 | type HintItem struct { 54 | HintItemMeta 55 | Key string 56 | } 57 | 58 | func newHintItem(khash uint64, ver int32, vhash uint16, pos Position, key string) *HintItem { 59 | return &HintItem{HintItemMeta{khash, pos, ver, vhash}, key} 60 | } 61 | 62 | type Payload struct { 63 | Meta 64 | cmem.CArray 65 | } 66 | 67 | func (p *Payload) Copy() *Payload { 68 | p2 := new(Payload) 69 | p2.Meta = p.Meta 70 | var ok bool 71 | p2.CArray, ok = p.CArray.Copy() 72 | if !ok { 73 | return nil 74 | } 75 | return p2 76 | } 77 | 78 | func (p *Payload) IsCompressed() bool { 79 | return (p.Flag & FLAG_COMPRESS) != 0 80 | } 81 | 82 | func (p *Payload) DiffSizeAfterDecompressed() int { 83 | if p.IsCompressed() { 84 | return quicklz.SizeDecompressed(p.CArray.Body) - p.CArray.Cap 85 | } 86 | return 0 87 | } 88 | 89 | func Getvhash(value []byte) uint16 { 90 | l := len(value) 91 | hash := uint32(l) * 97 92 | if l <= 1024 { 93 | hash += utils.Fnv1a(value) 94 | } else { 95 | hash += utils.Fnv1a(value[:512]) 96 | hash *= 97 97 | hash += utils.Fnv1a(value[l-512 : l]) 98 | } 99 | return uint16(hash) 100 | } 101 | 102 | func (p *Payload) CalcValueHash() { 103 | p.ValueHash = Getvhash(p.Body) 104 | } 105 | 106 | func (p *Payload) RawValueSize() int { 107 | if !p.IsCompressed() { 108 | return len(p.Body) 109 | } else { 110 | return quicklz.SizeCompressed(p.Body) 111 | } 112 | } 113 | 114 | func NeedCompress(header []byte) bool { 115 | typeValue := http.DetectContentType(header) 116 | _, ok := Conf.NotCompress[typeValue] 117 | return !ok 118 | } 119 | 120 | func (rec *Record) TryCompress() { 121 | if rec.Payload.Ver < 0 { 122 | return 123 | } 124 | p := rec.Payload 125 | if p.Flag&FLAG_CLIENT_COMPRESS != 0 || p.Flag&FLAG_COMPRESS != 0 { 126 | return 127 | } 128 | 129 | if rec.Size() <= 256 { 130 | return 131 | } 132 | body := rec.Payload.Body 133 | try := body 134 | if len(body) > TRY_COMPRESS_SIZE { 135 | try = try[:TRY_COMPRESS_SIZE] 136 | } 137 | if !NeedCompress(try) { 138 | return 139 | } 140 | compressed, ok := quicklz.CCompress(try) 141 | if !ok { 142 | // because oom, just not compress it 143 | return 144 | } 145 | if float32(len(compressed.Body))/float32(len(try)) > COMPRESS_RATIO_LIMIT { 146 | compressed.Free() 147 | return 148 | } 149 | if len(body) > len(try) { 150 | compressed.Free() 151 | compressed, ok = quicklz.CCompress(body) 152 | if !ok { 153 | // because oom, just not compress it 154 | return 155 | } 156 | } 157 | p.CArray.Free() 158 | p.CArray = compressed 159 | p.Flag += FLAG_COMPRESS 160 | return 161 | } 162 | 163 | func (p *Payload) Decompress() (err error) { 164 | if p.Flag&FLAG_COMPRESS == 0 { 165 | return 166 | } 167 | arr, err := quicklz.CDecompressSafe(p.Body) 168 | if err != nil { 169 | logger.Errorf("decompress fail %s", err.Error()) 170 | return 171 | } 172 | p.CArray.Free() 173 | p.CArray = arr 174 | p.Flag -= FLAG_COMPRESS 175 | return 176 | } 177 | 178 | func (p *Payload) Getvhash() uint16 { 179 | if p.Ver < 0 { 180 | return 0 181 | } 182 | if p.Flag&FLAG_COMPRESS == 0 { 183 | return Getvhash(p.Body) 184 | } 185 | arr, _ := quicklz.CDecompressSafe(p.Body) 186 | vhash := Getvhash(arr.Body) 187 | arr.Free() 188 | return vhash 189 | } 190 | 191 | type Position struct { 192 | ChunkID int 193 | Offset uint32 194 | } 195 | 196 | func (pos *Position) CmpKey() int64 { 197 | return (int64(pos.ChunkID) << 32) + int64(pos.Offset) 198 | } 199 | 200 | type Record struct { 201 | Key []byte 202 | Payload *Payload 203 | } 204 | 205 | func (rec *Record) LogString() string { 206 | return fmt.Sprintf("ksz %d, vsz %d, meta %#v [%s] ", 207 | len(rec.Key), 208 | len(rec.Payload.Body), 209 | rec.Payload.Meta, 210 | string(rec.Key), 211 | ) 212 | } 213 | 214 | func (rec *Record) Copy() *Record { 215 | return &Record{rec.Key, rec.Payload.Copy()} 216 | } 217 | 218 | // must be compressed 219 | func (rec *Record) Sizes() (uint32, uint32) { 220 | recSize := uint32(24 + len(rec.Key) + len(rec.Payload.Body)) 221 | return recSize, ((recSize + 255) >> 8) << 8 222 | } 223 | 224 | func (rec *Record) Size() uint32 { 225 | _, size := rec.Sizes() 226 | return size 227 | } 228 | 229 | func (rec *Record) Dumps() []byte { 230 | var buf bytes.Buffer 231 | wrec := wrapRecord(rec) 232 | wrec.append(&buf, false) 233 | return buf.Bytes() 234 | } 235 | 236 | func posForCompare(pos uint32) int64 { 237 | return (int64(pos&0xff) << 32) | int64(pos) 238 | } 239 | func comparePos(oldPos, newPos uint32) int64 { 240 | return posForCompare(oldPos) - posForCompare(newPos) 241 | } 242 | -------------------------------------------------------------------------------- /store/data.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "sync" 9 | "time" 10 | 11 | "github.com/douban/gobeansdb/cmem" 12 | ) 13 | 14 | const ( 15 | MAX_NUM_CHUNK = 998 16 | MAX_NUM_SPLIT = 998 17 | ) 18 | 19 | type dataStore struct { 20 | bucketID int 21 | home string 22 | 23 | sync.Mutex 24 | flushLock sync.Mutex 25 | 26 | oldHead int // old tail == 0 27 | newHead int 28 | newTail int 29 | 30 | chunks [MAX_NUM_CHUNK]dataChunk 31 | wbufSize uint32 32 | lastFlushTime time.Time 33 | } 34 | 35 | func NewdataStore(bucketID int, home string) *dataStore { 36 | ds := new(dataStore) 37 | ds.bucketID = bucketID 38 | ds.home = home 39 | for i := 0; i < MAX_NUM_CHUNK; i++ { 40 | ds.chunks[i].chunkid = i 41 | ds.chunks[i].path = genDataPath(ds.home, i) 42 | } 43 | return ds 44 | } 45 | 46 | func WakeupFlush() { 47 | select { 48 | case cmem.DBRL.FlushData.Chan <- 1: 49 | default: 50 | } 51 | } 52 | 53 | func genDataPath(home string, chunkID int) string { 54 | return fmt.Sprintf("%s/%03d.data", home, chunkID) 55 | } 56 | 57 | func (ds *dataStore) genPath(chunkID int) string { 58 | return genDataPath(ds.home, chunkID) 59 | } 60 | 61 | func (ds *dataStore) nextChunkID(chunkID int) int { 62 | return chunkID + 1 63 | } 64 | 65 | func (ds *dataStore) AppendRecord(rec *Record) (pos Position, err error) { 66 | // must CalcValueHash before compress 67 | oldCap := rec.Payload.CArray.Cap 68 | rec.TryCompress() 69 | cmem.DBRL.SetData.AddSize(rec.Payload.CArray.Cap - oldCap) 70 | 71 | wrec := wrapRecord(rec) 72 | ds.Lock() 73 | size := rec.Payload.RecSize 74 | currOffset := ds.chunks[ds.newHead].writingHead 75 | if currOffset+size > uint32(Conf.DataFileMax) { 76 | ds.newHead++ 77 | logger.Infof("rotate to %d, size %d, new rec size %d", ds.newHead, currOffset, size) 78 | currOffset = 0 79 | go ds.flush(ds.newHead-1, true) 80 | } 81 | pos.ChunkID = ds.newHead 82 | pos.Offset = currOffset 83 | wrec.pos = pos 84 | ds.chunks[ds.newHead].AppendRecord(wrec) 85 | ds.wbufSize += size 86 | 87 | if wrec.rec.Payload.Ver > 0 { 88 | cmem.DBRL.FlushData.AddSizeAndCount(rec.Payload.CArray.Cap) 89 | cmem.DBRL.SetData.SubSizeAndCount(rec.Payload.CArray.Cap) 90 | } 91 | 92 | if cmem.DBRL.FlushData.Size > int64(Conf.FlushWake) { 93 | WakeupFlush() 94 | } 95 | ds.Unlock() 96 | return 97 | } 98 | 99 | func (ds *dataStore) flush(chunk int, force bool) error { 100 | if ds.wbufSize == 0 { 101 | return nil 102 | } 103 | ds.flushLock.Lock() 104 | defer ds.flushLock.Unlock() 105 | ds.Lock() 106 | if ds.wbufSize == 0 { 107 | ds.Unlock() 108 | return nil 109 | } 110 | if !force && (time.Since(ds.lastFlushTime) < time.Duration(Conf.FlushInterval)*time.Second) && 111 | (ds.wbufSize < (1 << 20)) { 112 | ds.Unlock() 113 | return nil 114 | } 115 | 116 | if chunk < 0 { 117 | chunk = ds.newHead 118 | } 119 | ds.lastFlushTime = time.Now() 120 | ds.Unlock() 121 | // logger.Infof("flushing %d records to data %d", n, chunk) 122 | 123 | w, err := ds.GetStreamWriter(chunk, true) 124 | if err != nil { 125 | logger.Fatalf("fail to open data file to flush, stop! err: %v", err) 126 | return err 127 | } 128 | 129 | filessize := ds.chunks[chunk].getDiskFileSize() 130 | if w.offset != filessize { 131 | logger.Fatalf("wrong data file size, exp %d, got %d, %s, dataChunk %#v", 132 | filessize, w.offset, ds.genPath(chunk), &ds.chunks[chunk]) 133 | } 134 | nflushed, err := ds.chunks[chunk].flush(w, false) 135 | ds.Lock() 136 | ds.wbufSize -= nflushed 137 | ds.Unlock() 138 | w.Close() 139 | 140 | return nil 141 | } 142 | 143 | func (ds *dataStore) GetRecordByPos(pos Position) (res *Record, inbuffer bool, err error) { 144 | return ds.chunks[pos.ChunkID].GetRecordByOffset(pos.Offset) 145 | } 146 | 147 | func (ds *dataStore) ListFiles() (max int, err error) { 148 | max = -1 149 | for i := 0; i < MAX_NUM_CHUNK; i++ { 150 | path := genDataPath(ds.home, i) 151 | st, e := os.Stat(path) 152 | if e != nil { 153 | pe := e.(*os.PathError) 154 | if "no such file or directory" == pe.Err.Error() { 155 | ds.chunks[i].size = 0 156 | } else { 157 | logger.Errorf(pe.Err.Error()) 158 | err = pe 159 | return 160 | } 161 | } else { 162 | sz := uint32(st.Size()) 163 | if (sz & 0xff) != 0 { 164 | err = fmt.Errorf("file not 256 aligned, size 0x%x: %s ", sz, path) 165 | return 166 | } 167 | ds.chunks[i].size = sz 168 | ds.chunks[i].writingHead = sz 169 | max = i 170 | } 171 | } 172 | ds.newHead = max + 1 173 | return 174 | } 175 | 176 | func (ds *dataStore) GetStreamReader(chunk int) (*DataStreamReader, error) { 177 | path := ds.genPath(chunk) 178 | return newDataStreamReader(path, Conf.BufIOCap) 179 | } 180 | 181 | func GetStreamWriter(path string, isappend bool) (*DataStreamWriter, error) { 182 | offset := uint32(0) 183 | 184 | var fd *os.File 185 | if stat, err := os.Stat(path); err == nil { // TODO: avoid stat 186 | fd, err = os.OpenFile(path, os.O_WRONLY, 0) 187 | if err != nil { 188 | logger.Infof(err.Error()) 189 | return nil, err 190 | } 191 | if isappend { 192 | offset = uint32(stat.Size()) 193 | offset, err := fd.Seek(0, io.SeekEnd) 194 | if err != nil { 195 | logger.Infof(err.Error()) 196 | return nil, err 197 | } else if offset%PADDING != 0 { 198 | logger.Fatalf("%s not 256 aligned : %d", path, offset) 199 | } 200 | } 201 | } else { 202 | logger.Infof("create data file: %s", path) 203 | fd, err = os.Create(path) 204 | if err != nil { 205 | logger.Fatalf(err.Error()) 206 | return nil, err 207 | } 208 | } 209 | wbuf := bufio.NewWriterSize(fd, Conf.BufIOCap) 210 | w := &DataStreamWriter{path: path, fd: fd, wbuf: wbuf, offset: offset} 211 | return w, nil 212 | } 213 | 214 | func (ds *dataStore) GetStreamWriter(chunk int, isappend bool) (*DataStreamWriter, error) { 215 | path := ds.genPath(chunk) 216 | return GetStreamWriter(path, isappend) 217 | 218 | } 219 | -------------------------------------------------------------------------------- /store/hintfile.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "os" 9 | ) 10 | 11 | const ( 12 | HINTFILE_HEAD_SIZE = 16 13 | HINTITEM_HEAD_SIZE = 23 14 | HINTINDEX_ROW_SIZE = 4096 15 | ) 16 | 17 | type hintFileMeta struct { 18 | numKey int 19 | indexOffset int64 20 | datasize uint32 21 | } 22 | 23 | func (fm *hintFileMeta) Dumps(buf []byte) { 24 | binary.LittleEndian.PutUint64(buf[0:8], uint64(fm.indexOffset)) 25 | binary.LittleEndian.PutUint32(buf[8:12], uint32(fm.numKey)) 26 | binary.LittleEndian.PutUint32(buf[12:16], fm.datasize) 27 | } 28 | 29 | func (fm *hintFileMeta) Loads(buf []byte) { 30 | fm.indexOffset = int64(binary.LittleEndian.Uint64(buf[:8])) 31 | fm.numKey = int(binary.LittleEndian.Uint32(buf[8:12])) 32 | fm.datasize = uint32(binary.LittleEndian.Uint32(buf[12:16])) 33 | } 34 | 35 | // used for 1. HTree scan 2. hint merge 36 | type hintFileReader struct { 37 | hintFileMeta 38 | path string 39 | bufsize int 40 | chunkID int 41 | size int64 42 | 43 | fd *os.File 44 | rbuf *bufio.Reader 45 | offset int64 46 | buf [256]byte 47 | } 48 | 49 | func newHintFileReader(path string, chunkID, bufsize int) (reader *hintFileReader) { 50 | return &hintFileReader{ 51 | path: path, 52 | chunkID: chunkID, 53 | bufsize: bufsize, 54 | } 55 | } 56 | 57 | func (reader *hintFileReader) open() (err error) { 58 | reader.fd, err = os.Open(reader.path) 59 | if err != nil { 60 | logger.Errorf(err.Error()) 61 | return err 62 | } 63 | reader.rbuf = bufio.NewReaderSize(reader.fd, reader.bufsize) 64 | h := reader.buf[:HINTFILE_HEAD_SIZE] 65 | var readn int 66 | readn, err = io.ReadFull(reader.rbuf, h) 67 | if err != nil { 68 | logger.Errorf(err.Error()) 69 | return 70 | } 71 | if readn < HINTFILE_HEAD_SIZE { 72 | err = fmt.Errorf("bad hint file %s readn %d", reader.path, readn) 73 | logger.Errorf(err.Error()) 74 | return 75 | } 76 | reader.hintFileMeta.Loads(h) 77 | //logger.Infof("open hint for read %#v, %s", reader.hintFileMeta, reader.path) 78 | fileInfo, _ := reader.fd.Stat() 79 | reader.size = fileInfo.Size() 80 | reader.offset = HINTFILE_HEAD_SIZE 81 | if reader.indexOffset == 0 { 82 | logger.Errorf("%s has no index", reader.path) 83 | reader.indexOffset = reader.size 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (reader *hintFileReader) next() (item *HintItem, err error) { 90 | if reader.offset >= reader.indexOffset { 91 | return nil, nil 92 | } 93 | h := reader.buf[:HINTITEM_HEAD_SIZE] 94 | item = new(HintItem) 95 | var readn int 96 | readn, err = io.ReadFull(reader.rbuf, h) 97 | if err != nil { 98 | logger.Errorf(err.Error()) 99 | return 100 | } 101 | if readn < HINTITEM_HEAD_SIZE { 102 | err = fmt.Errorf("bad hint file %s readn %d", reader.path, readn) 103 | return 104 | } 105 | item.Keyhash = binary.LittleEndian.Uint64(h[:8]) 106 | item.Pos.ChunkID = int(binary.LittleEndian.Uint32(h[8:12])) 107 | item.Pos.Offset = binary.LittleEndian.Uint32(h[12:16]) 108 | item.Ver = int32(binary.LittleEndian.Uint32(h[16:20])) 109 | item.Vhash = binary.LittleEndian.Uint16(h[20:22]) 110 | ksz := int(h[22]) 111 | key := reader.buf[:ksz] 112 | readn, err = io.ReadFull(reader.rbuf, key) 113 | if err != nil { 114 | logger.Errorf(err.Error()) 115 | return 116 | } 117 | if readn < ksz { 118 | err = fmt.Errorf("bad hint file %s readn %d", reader.path, readn) 119 | return 120 | } 121 | item.Key = string(key[:ksz]) 122 | reader.offset += HINTITEM_HEAD_SIZE + int64(ksz) 123 | return item, nil 124 | } 125 | 126 | func (reader *hintFileReader) close() { 127 | reader.fd.Close() 128 | } 129 | 130 | // used for 1. dump hint 2. hint merge 131 | type hintFileWriter struct { 132 | index *hintFileIndexBuffer 133 | hintFileMeta 134 | path string 135 | 136 | fd *os.File 137 | wbuf *bufio.Writer 138 | offset int64 139 | buf [256]byte 140 | } 141 | 142 | func newHintFileWriter(path string, maxOffset uint32, bufsize int) (w *hintFileWriter, err error) { 143 | var fd *os.File 144 | tmp := path + ".tmp" 145 | logger.Infof("create hint file: %s", tmp) 146 | fd, err = os.Create(tmp) 147 | if err != nil { 148 | logger.Errorf(err.Error()) 149 | return nil, err 150 | } 151 | wbuf := bufio.NewWriterSize(fd, bufsize) 152 | w = &hintFileWriter{ 153 | fd: fd, 154 | wbuf: wbuf, 155 | offset: HINTFILE_HEAD_SIZE, 156 | path: path, 157 | hintFileMeta: hintFileMeta{datasize: maxOffset}} 158 | w.wbuf.Write(w.buf[:HINTFILE_HEAD_SIZE]) 159 | w.index = newHintFileIndex() 160 | return 161 | } 162 | 163 | func (w *hintFileWriter) writeItem(item *HintItem) error { 164 | h := w.buf[:HINTITEM_HEAD_SIZE] 165 | binary.LittleEndian.PutUint64(h[0:8], item.Keyhash) 166 | binary.LittleEndian.PutUint32(h[8:12], uint32(item.Pos.ChunkID)) 167 | binary.LittleEndian.PutUint32(h[12:16], item.Pos.Offset) 168 | binary.LittleEndian.PutUint32(h[16:20], uint32(item.Ver)) 169 | binary.LittleEndian.PutUint16(h[20:22], item.Vhash) 170 | h[22] = byte(len(item.Key)) 171 | w.wbuf.Write(h) 172 | w.wbuf.WriteString(item.Key) 173 | // TODO: refactor 174 | if (w.offset - w.index.lastoffset) > int64(Conf.IndexIntervalSize-HINTITEM_HEAD_SIZE-256) { 175 | w.index.append(item.Keyhash, w.offset) 176 | } 177 | w.offset += HINTITEM_HEAD_SIZE + int64(len(item.Key)) 178 | w.numKey += 1 179 | return nil 180 | } 181 | 182 | func (w *hintFileWriter) close() error { 183 | w.indexOffset = w.offset 184 | index := w.index 185 | var buf [16]byte 186 | for r := 0; r <= index.currRow; r++ { 187 | col := HINTINDEX_ROW_SIZE 188 | if r == index.currRow { 189 | col = index.currCol 190 | } 191 | for c := 0; c < col; c++ { 192 | it := index.index[r][c] 193 | binary.LittleEndian.PutUint64(buf[0:8], it.keyhash) 194 | binary.LittleEndian.PutUint64(buf[8:16], uint64(it.offset)) 195 | w.wbuf.Write(buf[:]) 196 | } 197 | } 198 | w.wbuf.Flush() 199 | w.fd.Seek(0, 0) 200 | w.hintFileMeta.Dumps(buf[:]) 201 | w.fd.Write(buf[:]) 202 | w.fd.Close() 203 | tmp := w.path + ".tmp" 204 | err := os.Rename(tmp, w.path) 205 | if err != nil { 206 | return err 207 | } else { 208 | logger.Infof("moved %s -> %s", tmp, w.path) 209 | } 210 | 211 | return nil 212 | } 213 | -------------------------------------------------------------------------------- /store/datachunk.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "sort" 7 | "sync" 8 | 9 | "github.com/douban/gobeansdb/cmem" 10 | "github.com/douban/gobeansdb/utils" 11 | ) 12 | 13 | type dataChunk struct { 14 | sync.Mutex 15 | 16 | chunkid int 17 | path string 18 | size uint32 19 | 20 | writingHead uint32 21 | wbuf []*WriteRecord 22 | 23 | rewriting bool 24 | gcbufsize uint32 25 | gcWriter *DataStreamWriter 26 | } 27 | 28 | func (dc *dataChunk) GoString() string { 29 | return fmt.Sprintf("(lock %v size %d writingHead %d rewriting %v len(wbuf) %d wbuf[0].pos %v wbuf[-1].pos %v)", 30 | dc.Mutex, dc.size, dc.writingHead, dc.rewriting, len(dc.wbuf), 31 | dc.wbuf[0].pos, dc.wbuf[len(dc.wbuf)-1].pos) 32 | } 33 | 34 | func (dc *dataChunk) Clear() error { 35 | dc.wbuf = nil 36 | dc.size = 0 37 | dc.rewriting = false 38 | dc.gcWriter = nil 39 | dc.gcbufsize = 0 40 | dc.writingHead = 0 41 | 42 | return utils.Remove(dc.path) 43 | } 44 | 45 | func (dc *dataChunk) AppendRecord(wrec *WriteRecord) { 46 | dc.Lock() 47 | dc.wbuf = append(dc.wbuf, wrec) 48 | 49 | size := wrec.rec.Payload.RecSize 50 | 51 | dc.writingHead += size 52 | dc.size = dc.writingHead 53 | dc.Unlock() 54 | } 55 | 56 | func (dc *dataChunk) AppendRecordGC(wrec *WriteRecord) (offset uint32, err error) { 57 | dc.Lock() 58 | wrec.pos.ChunkID = dc.chunkid 59 | offset = dc.writingHead 60 | wrec.pos.Offset = offset 61 | size := wrec.rec.Payload.RecSize 62 | 63 | dc.writingHead += size 64 | if dc.writingHead >= dc.size { 65 | dc.size = dc.writingHead 66 | } 67 | dc.Unlock() 68 | 69 | _, err = dc.gcWriter.append(wrec) 70 | if err != nil { 71 | logger.Fatalf("fail to append, stop! err: %v", err) 72 | } 73 | 74 | if err = dc.gcWriter.wbuf.Flush(); err != nil { 75 | logger.Fatalf("write data fail, stop! err: %v", err) 76 | return 0, err 77 | } 78 | return 79 | } 80 | 81 | func (dc *dataChunk) getDiskFileSize() uint32 { 82 | if len(dc.wbuf) > 0 { 83 | return dc.wbuf[0].pos.Offset 84 | } 85 | return dc.size 86 | } 87 | 88 | func (dc *dataChunk) flush(w *DataStreamWriter, gc bool) (flushed uint32, err error) { 89 | dc.Lock() 90 | n := len(dc.wbuf) 91 | dc.Unlock() 92 | for i := 0; i < n; i++ { 93 | dc.Lock() // because append may change the slice 94 | wrec := dc.wbuf[i] 95 | dc.Unlock() 96 | _, err := w.append(wrec) 97 | if err != nil { 98 | logger.Fatalf("fail to append, stop! err: %v", err) 99 | } 100 | size := wrec.rec.Payload.RecSize 101 | flushed += size 102 | if !gc && wrec.rec.Payload.Ver > 0 { 103 | cmem.DBRL.FlushData.SubSizeAndCount(wrec.rec.Payload.CArray.Cap) 104 | // NOTE: not freed yet, make it a little diff with AllocRL, which may provide more insight 105 | } 106 | } 107 | if err = w.wbuf.Flush(); err != nil { 108 | logger.Fatalf("write data fail, stop! err: %v", err) 109 | return 0, err 110 | } 111 | 112 | dc.Lock() 113 | tofree := dc.wbuf[:n] 114 | dc.wbuf = dc.wbuf[n:] 115 | dc.Unlock() 116 | for _, wrec := range tofree { 117 | wrec.rec.Payload.Free() 118 | } 119 | return 120 | } 121 | 122 | func (dc *dataChunk) GetRecordByOffsetInBuffer(offset uint32) (res *Record, err error) { 123 | dc.Lock() 124 | defer dc.Unlock() 125 | 126 | wbuf := dc.wbuf 127 | n := len(wbuf) 128 | if n == 0 || offset < wbuf[0].pos.Offset || offset >= dc.writingHead { 129 | return 130 | } 131 | 132 | idx := sort.Search(n, func(i int) bool { return wbuf[i].pos.Offset >= offset }) 133 | if idx >= n { 134 | err = fmt.Errorf("%d %d %d %d %d", n, idx, dc.size, dc.writingHead, offset) 135 | logger.Errorf("%v", err) 136 | return 137 | } 138 | wrec := wbuf[idx] 139 | if wrec.pos.Offset == offset { 140 | res = wrec.rec.Copy() 141 | cmem.DBRL.GetData.AddSizeAndCount(res.Payload.CArray.Cap) 142 | return 143 | } else { 144 | err = fmt.Errorf("rec should in buffer, but not, pos = %#v", Position{dc.chunkid, offset}) 145 | return 146 | } 147 | return 148 | } 149 | 150 | func (dc *dataChunk) GetRecordByOffset(offset uint32) (res *Record, inbuffer bool, err error) { 151 | res, err = dc.GetRecordByOffsetInBuffer(offset) 152 | if err != nil { 153 | inbuffer = true 154 | return 155 | } 156 | if res != nil { 157 | inbuffer = true 158 | cmem.DBRL.GetData.AddSize(res.Payload.DiffSizeAfterDecompressed()) 159 | res.Payload.Decompress() 160 | return 161 | } 162 | wrec, e := readRecordAtPath(dc.path, offset) 163 | if e != nil { 164 | return nil, false, e 165 | } 166 | cmem.DBRL.GetData.AddSize(wrec.rec.Payload.DiffSizeAfterDecompressed()) 167 | wrec.rec.Payload.Decompress() 168 | return wrec.rec, false, nil 169 | } 170 | 171 | func (dc *dataChunk) Truncate(size uint32) error { 172 | path := dc.path 173 | st, err := os.Stat(path) 174 | if err != nil { 175 | logger.Infof(err.Error()) 176 | return err 177 | } 178 | logger.Infof("truncate %s %d to %d", path, st.Size(), size) 179 | if size == 0 { 180 | return utils.Remove(path) 181 | } 182 | return os.Truncate(path, int64(size)) 183 | } 184 | 185 | func (dc *dataChunk) beginGCWriting(srcChunk int) (err error) { 186 | logger.Infof("BeginGCWriting chunk %d from %d rewrite %v size %d wsize %d ", dc.chunkid, srcChunk, dc.rewriting, dc.size, dc.writingHead) 187 | if dc.chunkid == srcChunk { 188 | dc.rewriting = true 189 | dc.writingHead = 0 190 | logger.Infof("rewrite %s", dc.path) 191 | } else { 192 | dc.writingHead = dc.size 193 | } 194 | dc.gcWriter, err = GetStreamWriter(dc.path, !dc.rewriting) 195 | if err != nil { 196 | dc.gcWriter = nil 197 | } 198 | return 199 | } 200 | 201 | func (dc *dataChunk) endGCWriting() (err error) { 202 | logger.Infof("endGCWriting chunk %d rewrite %v size %d wsize%d ", dc.chunkid, dc.rewriting, dc.size, dc.writingHead) 203 | if dc.gcWriter != nil { 204 | err = dc.gcWriter.wbuf.Flush() 205 | dc.gcWriter.Close() 206 | dc.gcWriter = nil 207 | } 208 | if dc.rewriting && dc.writingHead < dc.size { 209 | dc.Truncate(dc.writingHead) 210 | dc.size = dc.writingHead 211 | } 212 | dc.rewriting = false 213 | return 214 | } 215 | 216 | func (dc *dataChunk) getFirstRecTs() (ts int64, err error) { 217 | f, err := os.Open(dc.path) 218 | if err != nil { 219 | logger.Errorf("%v", err) 220 | return 221 | } 222 | var n int 223 | wrec := newWriteRecord() 224 | if n, err = f.ReadAt(wrec.header[:], 0); err != nil { 225 | err = fmt.Errorf("fail to read head %s, err = %v, n = %d", dc.path, err, n) 226 | logger.Errorf(err.Error()) 227 | return 228 | } 229 | wrec.decodeHeader() 230 | ts = int64(wrec.rec.Payload.TS) 231 | return 232 | } 233 | -------------------------------------------------------------------------------- /gobeansdb/store.go: -------------------------------------------------------------------------------- 1 | package gobeansdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sync" 7 | 8 | "github.com/douban/gobeansdb/cmem" 9 | mc "github.com/douban/gobeansdb/memcache" 10 | "github.com/douban/gobeansdb/store" 11 | ) 12 | 13 | var ( 14 | ErrorNotSupport = errors.New("operation not support") 15 | ) 16 | 17 | type Storage struct { 18 | numClient int 19 | hstore *store.HStore 20 | sync.Mutex 21 | } 22 | 23 | func (s *Storage) Client() mc.StorageClient { 24 | return &StorageClient{ 25 | s.hstore, 26 | } 27 | } 28 | 29 | type StorageClient struct { 30 | hstore *store.HStore 31 | } 32 | 33 | func (s *StorageClient) GetSuccessedTargets() []string { 34 | return []string{"localhost"} 35 | } 36 | 37 | func (s *StorageClient) Clean() { 38 | return 39 | } 40 | 41 | func (s *StorageClient) Set(key string, item *mc.Item, noreply bool) (bool, error) { 42 | tofree := &item.CArray 43 | defer func() { 44 | if tofree != nil { 45 | cmem.DBRL.SetData.SubSizeAndCount(tofree.Cap) 46 | tofree.Free() 47 | } 48 | }() 49 | if !store.IsValidKeyString(key) { 50 | return false, nil 51 | } 52 | ki := s.prepare(key, false) 53 | payload := &store.Payload{} 54 | payload.Flag = uint32(item.Flag) 55 | payload.CArray = item.CArray 56 | payload.Ver = int32(item.Exptime) 57 | payload.TS = uint32(item.ReceiveTime.Unix()) 58 | 59 | tofree = nil 60 | err := s.hstore.Set(ki, payload) 61 | if err != nil { 62 | logger.Errorf("err to get %s: %s", key, err.Error()) 63 | return false, err 64 | } 65 | return true, nil 66 | } 67 | 68 | func (s *StorageClient) prepare(key string, isPath bool) *store.KeyInfo { 69 | ki := &store.KeyInfo{} 70 | ki.StringKey = key 71 | ki.Key = []byte(key) 72 | ki.KeyIsPath = isPath 73 | return ki 74 | } 75 | 76 | func (s *StorageClient) listDir(path string) (*mc.Item, error) { 77 | // TODO: check valid 78 | ki := s.prepare(path, true) 79 | body, err := s.hstore.ListDir(ki) 80 | if err != nil { 81 | return nil, err 82 | } 83 | item := new(mc.Item) 84 | item.Body = []byte(body) 85 | item.Flag = 0 86 | return item, nil 87 | } 88 | 89 | func (s *StorageClient) getMeta(key string, extended bool) (*mc.Item, error) { 90 | ki := s.prepare(key, false) 91 | payload, pos, err := s.hstore.Get(ki, false) 92 | if err != nil { 93 | return nil, err 94 | } 95 | if payload == nil { 96 | return nil, nil 97 | } 98 | 99 | // TODO: use the one in htree 100 | vhash := uint16(0) 101 | if payload.Ver > 0 { 102 | vhash = store.Getvhash(payload.Body) 103 | } 104 | 105 | var body string 106 | if extended { 107 | body = fmt.Sprintf("%d %d %d %d %d %d %d", 108 | payload.Ver, vhash, payload.Flag, len(payload.Body), payload.TS, pos.ChunkID, pos.Offset) 109 | 110 | } else { 111 | body = fmt.Sprintf("%d %d %d %d %d", 112 | payload.Ver, vhash, payload.Flag, len(payload.Body), payload.TS) 113 | } 114 | 115 | cmem.DBRL.GetData.SubSizeAndCount(payload.CArray.Cap) 116 | payload.CArray.Free() 117 | 118 | item := new(mc.Item) 119 | item.Body = []byte(body) 120 | item.Flag = 0 121 | return item, nil 122 | } 123 | 124 | func (s *StorageClient) Get(key string) (*mc.Item, error) { 125 | if key[0] == '@' { 126 | if len(key) > 1 && key[1] == '@' { 127 | key2 := key[2:] 128 | if len(key2) != 16 { 129 | return nil, fmt.Errorf("bad command line format") //FIXME: SERVER_ERROR 130 | } 131 | ki := s.prepare(key2, true) 132 | rec, _, err := s.hstore.GetRecordByKeyHash(ki) 133 | if err != nil { 134 | return nil, err 135 | } else if rec == nil { 136 | return nil, nil 137 | } 138 | item := new(mc.Item) // TODO: avoid alloc? 139 | item.Body = rec.Dumps() 140 | cmem.DBRL.GetData.SubSizeAndCount(rec.Payload.Cap) 141 | rec.Payload.Free() 142 | item.Flag = 0 143 | return item, nil 144 | } else if len(key) > 11 && "collision_" == key[1:11] { 145 | if len(key) > 15 && "all_" == key[11:15] { 146 | return nil, nil 147 | } else { 148 | item := new(mc.Item) // TODO: avoid alloc? 149 | item.Body = []byte("0 0 0 0") 150 | item.Flag = 0 151 | return item, nil 152 | } 153 | } else { 154 | return s.listDir(key[1:]) 155 | } 156 | 157 | } else if key[0] == '?' { 158 | extended := false 159 | if len(key) > 1 { 160 | if key[1] == '?' { 161 | extended = true 162 | key = key[2:] 163 | } else { 164 | key = key[1:] 165 | } 166 | if !store.IsValidKeyString(key) { 167 | return nil, nil 168 | } 169 | } else { 170 | return nil, fmt.Errorf("bad key %s", key) 171 | } 172 | return s.getMeta(key, extended) 173 | } 174 | 175 | ki := s.prepare(key, false) 176 | payload, _, err := s.hstore.Get(ki, false) 177 | if err != nil { 178 | logger.Errorf("err to get %s: %s", key, err.Error()) 179 | return nil, err 180 | } 181 | if payload == nil { 182 | return nil, nil 183 | } 184 | if payload.Ver < 0 { 185 | cmem.DBRL.GetData.SubSizeAndCount(payload.CArray.Cap) 186 | payload.CArray.Free() 187 | return nil, nil 188 | } 189 | item := new(mc.Item) // TODO: avoid alloc? 190 | item.CArray = payload.CArray 191 | item.Flag = int(payload.Flag) 192 | return item, nil 193 | } 194 | 195 | func (s *StorageClient) GetMulti(keys []string) (map[string]*mc.Item, error) { 196 | ret := make(map[string]*mc.Item) 197 | for _, key := range keys { 198 | item, _ := s.Get(key) 199 | if item != nil { 200 | ret[key] = item 201 | } 202 | } 203 | return ret, nil 204 | } 205 | 206 | func (s *StorageClient) Len() int { 207 | return s.hstore.NumKey() 208 | } 209 | 210 | func (s *StorageClient) Append(key string, value []byte) (bool, error) { 211 | return false, ErrorNotSupport 212 | } 213 | 214 | func (s *StorageClient) Incr(key string, value int) (int, error) { 215 | if !store.IsValidKeyString(key) { 216 | return 0, nil 217 | } 218 | ki := s.prepare(key, false) 219 | newvalue := s.hstore.Incr(ki, value) 220 | return newvalue, nil 221 | } 222 | 223 | func (s *StorageClient) Delete(key string) (bool, error) { 224 | if !store.IsValidKeyString(key) { 225 | return false, nil 226 | } 227 | ki := s.prepare(key, false) 228 | payload := store.GetPayloadForDelete() 229 | 230 | err := s.hstore.Set(ki, payload) 231 | if err != nil { 232 | if err.Error() == "NOT_FOUND" { 233 | return false, nil 234 | } else { 235 | logger.Errorf("err to delete %s: %s", key, err.Error()) 236 | return false, err 237 | } 238 | } 239 | return true, nil 240 | } 241 | 242 | func (s *StorageClient) Close() { 243 | 244 | } 245 | 246 | func (s *StorageClient) Process(cmd string, args []string) (status string, msg string) { 247 | status = "CLIENT_ERROR" 248 | msg = "bad command line format" 249 | 250 | switch cmd { 251 | case "optimize_stat": 252 | msg = "" 253 | if s.hstore.IsGCRunning() { 254 | status = "running" 255 | } else { 256 | status = "none" 257 | } 258 | 259 | default: 260 | status = "ERROR" 261 | msg = "" 262 | } 263 | return 264 | } 265 | -------------------------------------------------------------------------------- /tests/key_version_test.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import string 4 | import zlib 5 | import unittest 6 | import time 7 | from tests.base import BaseTest 8 | from tests.dbclient import MCStore 9 | from tests.utils import random_string 10 | 11 | 12 | VERSION, HASH, FLAG, SIZE, TIMESTAMP, CHUNKID, OFFSET = range(7) 13 | 14 | class KeyVersionTest(BaseTest): 15 | def setUp(self): 16 | BaseTest.setUp(self) 17 | 18 | self.last_pos = 0 19 | self.last_size = 0 20 | 21 | def update_pos(self, size): 22 | self.last_pos += self.last_size 23 | self.last_size = size 24 | 25 | def get_meta(self, store, key): 26 | meta = store.get("??" + key) 27 | if meta: 28 | meta = meta.split() 29 | assert(len(meta) == 7) 30 | return tuple([int(meta[i]) for i in [VERSION, CHUNKID, OFFSET]]) 31 | 32 | def test_set_version(self): 33 | store = MCStore(self.db.addr) 34 | key = 'key1' 35 | store.set(key, 'aaa') 36 | self.update_pos(256) 37 | 38 | self.assertEqual(store.get(key), 'aaa') 39 | self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos)) 40 | 41 | store.set_raw(key, 'bbb', rev=3) 42 | self.update_pos(256) 43 | self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos)) 44 | 45 | store.set_raw(key, 'bbb', rev=4) 46 | self.assertEqual(self.get_meta(store, key), (4, 0, self.last_pos)) 47 | 48 | store.set_raw(key, 'ccc', rev=2) 49 | self.assertEqual(store.get(key), 'bbb') 50 | self.assertEqual(self.get_meta(store, key), (4, 0, self.last_pos)) 51 | 52 | self.checkCounterZero() 53 | 54 | def test_delete_version(self): 55 | store = MCStore(self.db.addr) 56 | key = 'key1' 57 | 58 | store.set(key, 'aaa') 59 | self.update_pos(256) 60 | self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos)) 61 | 62 | store.delete(key) 63 | self.update_pos(256) 64 | self.assertEqual(store.get(key), None) 65 | 66 | self.assertEqual(self.get_meta(store, key), (-2, 0, self.last_pos)) 67 | self.checkCounterZero() 68 | 69 | store.set(key, 'bbb') 70 | self.update_pos(256) 71 | self.assertEqual(store.get(key), 'bbb') 72 | self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos)) 73 | self.checkCounterZero() 74 | 75 | def _test_compress(self, overflow): 76 | store = MCStore(self.db.addr) 77 | value = string.letters 78 | compressed_value = zlib.compress(value, 0) 79 | key = 'k' * (256 - len(compressed_value) - 24 + (1 if overflow else 0)) 80 | 81 | value_easy_compress = 'v' * len(compressed_value) 82 | self.assertTrue(store.set(key, value_easy_compress)) 83 | self.assertEqual(store.get(key), value_easy_compress) 84 | self.update_pos(256) 85 | self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos)) 86 | 87 | self.assertTrue(store.set_raw(key, compressed_value, flag=0x00000010)) 88 | self.assertEqual(store.get(key), value) 89 | self.update_pos(512 if overflow else 256) 90 | self.assertEqual(self.get_meta(store, key), (2, 0, self.last_pos)) 91 | 92 | self.assertTrue(store.set(key, 'aaa')) 93 | self.update_pos(256) 94 | self.assertEqual(self.get_meta(store, key), (3, 0, self.last_pos)) 95 | self.checkCounterZero() 96 | 97 | def test_compress_257(self): 98 | self._test_compress(overflow=True) 99 | 100 | def test_compress_256(self): 101 | self._test_compress(overflow=False) 102 | 103 | def test_special_key(self): 104 | store = MCStore(self.db.addr) 105 | kvs = [('a' * 200, 1), ('a', range(1000))] 106 | for k, v in kvs: 107 | self.assertTrue(store.set(k, v)) 108 | self.assertEqual(store.get(k), v) 109 | 110 | # restart 111 | self.db.stop() 112 | self.db.start() 113 | store = MCStore(self.db.addr) 114 | for (k, v) in kvs: 115 | v2 = store.get(k) 116 | self.assertEqual(v2, v, "key %s, value %s, not %s" % (k, v, v2)) 117 | self.checkCounterZero() 118 | 119 | def test_big_value(self): 120 | store = MCStore(self.db.addr) 121 | key = 'largekey' 122 | size = 10 * 1024 * 1024 123 | rsize = (((size + len(key) + 24) >> 8) + 1) << 8 124 | string_large = random_string(size / 10) * 10 125 | 126 | self.assertTrue(store.set(key, string_large)) 127 | self.assertEqual(store.get(key), string_large) 128 | self.update_pos(rsize) 129 | 130 | self.assertEqual(self.get_meta(store, key), (1, 0, self.last_pos)) 131 | 132 | self.assertTrue(store.set(key, 'aaa')) 133 | self.update_pos(256) 134 | self.assertEqual(self.get_meta(store, key), (2, 0, self.last_pos)) 135 | 136 | self.checkCounterZero() 137 | 138 | def test_collision(self): 139 | # keyhash = "c80f795945b78f6b" 140 | store = MCStore(self.db.addr) 141 | # key1 and key2 have the same keyhash "c80f795945b78f6b" 142 | # key_other and key_other2 is in bucket c too 143 | # key_other is used to test with key1, key2 for get/set/gc... 144 | # key_other2 is used to make it possible to gc [0, 1] 145 | key1 = "processed_log_backup_text_20140912102821_1020_13301733" 146 | key2 = "/subject/10460967/props" 147 | key_other = "/ark/p/385242854/" 148 | key_other2 = "/doumail/849134123/source" 149 | keys = [key1, key2, key_other] 150 | for k in keys: 151 | self.assertTrue(store.set(k, k)) 152 | self.assertEqual(store.get(k), k) 153 | 154 | for i, k in enumerate(keys): 155 | self.assertEqual(store.get(k), k) 156 | ver = 2 if i == 1 else 1 157 | self.assertEqual(self.get_meta(store, k), (ver, 0, i *256)) 158 | 159 | self.db.stop() 160 | self.db.start() 161 | store = MCStore(self.db.addr) 162 | for k in keys: 163 | self.assertEqual(store.get(k), k) 164 | 165 | for k in keys: 166 | self.assertTrue(store.set(k, k + k)) 167 | 168 | for i, k in enumerate(keys): 169 | self.assertEqual(store.get(k), k + k) 170 | ver = 3 if i == 1 else 2 171 | self.assertEqual(self.get_meta(store, k), (ver, 1, i * 256)) 172 | 173 | self.db.stop() 174 | self.db.start() 175 | store = MCStore(self.db.addr) 176 | self.assertTrue(store.set(key_other2, 1)) 177 | 178 | self.db.stop() 179 | self.db.start() 180 | store = MCStore(self.db.addr) 181 | 182 | self.gc(12) 183 | time.sleep(1) 184 | 185 | for i, k in enumerate(keys): 186 | self.assertEqual(store.get(k), k + k) 187 | ver = 3 if i == 1 else 2 188 | self.assertEqual(self.get_meta(store, k), (ver, 0, i * 256)) 189 | 190 | 191 | if __name__ == '__main__': 192 | unittest.main() 193 | 194 | -------------------------------------------------------------------------------- /store/hint_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "runtime" 7 | "testing" 8 | "time" 9 | 10 | "github.com/douban/gobeansdb/loghub" 11 | "github.com/douban/gobeansdb/utils" 12 | ) 13 | 14 | func init() { 15 | SecsBeforeDump = 1 // dump quick for test 16 | } 17 | 18 | func readHintAndCheck(t *testing.T, path string, items []*HintItem) { 19 | r := newHintFileReader(path, 0, 10240) 20 | if err := r.open(); err != nil { 21 | t.Fatal(err) 22 | } 23 | n := len(items) 24 | for i := 0; i < n+2; i++ { 25 | it, err := r.next() 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | if i < n { 30 | if it == nil || *it != *items[i] { 31 | t.Fatalf("%d %#v != %#v", i, it, items[i]) 32 | } 33 | } else { 34 | if it != nil { 35 | t.Fatalf("%#v ", it) 36 | } 37 | } 38 | } 39 | r.close() 40 | } 41 | 42 | func TestHintRW(t *testing.T) { 43 | setupTest("TestHintRW") 44 | defer clearTest() 45 | path := fmt.Sprintf("%s/%s", dir, "000.hint.s") 46 | utils.Remove(path) 47 | 48 | Conf.IndexIntervalSize = 1024 49 | 50 | w, err := newHintFileWriter(path, 0, 1024) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | defer utils.Remove(path) 55 | n := 100 56 | items := genSortedHintItems(n) 57 | for _, it := range items { 58 | w.writeItem(it) 59 | } 60 | w.close() 61 | readHintAndCheck(t, path, items) 62 | index := &hintFileIndex{w.index.toIndex(), path, hintFileMeta{}} 63 | t.Logf("#index = %d, %#v", len(index.index), index.index) 64 | index2, err := loadHintIndex(path) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if len(index.index) != len(index2.index) { 69 | t.Fatalf("%v \n!=\n %v", index.index, index2.index) 70 | } 71 | for i := 0; i < len(index.index); i++ { 72 | if index.index[i] != index2.index[i] { 73 | t.Fatalf("%v != %v", index.index, index2.index) 74 | } 75 | } 76 | checkIndex(t, items, index) 77 | } 78 | 79 | func checkIndex(t *testing.T, items []*HintItem, index *hintFileIndex) { 80 | for _, it := range items { 81 | it2, err := index.get(it.Keyhash, it.Key) 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | if it2 == nil || *it != *it2 { 86 | t.Fatalf("%v != %v", it, it2) 87 | } 88 | } 89 | } 90 | 91 | func genSortedHintItems(n int) []*HintItem { 92 | items := make([]*HintItem, n) 93 | for i := 0; i < n; i++ { 94 | base := i * 3 95 | it := &HintItem{ 96 | HintItemMeta{ 97 | Keyhash: uint64(i), 98 | Pos: Position{0, uint32(base) * 256}, 99 | Ver: int32(base + 1), 100 | Vhash: uint16(base + 2), 101 | }, 102 | genKey(i), 103 | } 104 | items[i] = it 105 | } 106 | return items 107 | } 108 | 109 | func testMerge(t *testing.T, nsrc int) { 110 | home := fmt.Sprintf("%s_%d", "TestHintMerge", nsrc) 111 | setupTest(home) 112 | defer clearTest() 113 | 114 | n := 10 115 | items := genSortedHintItems(n) 116 | src := make([]*hintFileWriter, nsrc) 117 | srcp := make([]*hintFileReader, nsrc) 118 | for i := 0; i < nsrc; i++ { 119 | path := fmt.Sprintf("%s/src.%d.hint.s", dir, i) 120 | srcp[i] = newHintFileReader(path, 0, 1024) 121 | utils.Remove(path) 122 | w, err := newHintFileWriter(path, 0, 1024) 123 | if err != nil { 124 | t.Fatal(err) 125 | } 126 | defer utils.Remove(path) 127 | src[i] = w 128 | } 129 | for i := 0; i < n; i++ { 130 | w := src[rand.Intn(nsrc)] 131 | it := items[i] 132 | 133 | tmp := new(HintItem) 134 | *tmp = *it 135 | tmp.Pos.Offset -= 1 136 | w.writeItem(tmp) 137 | if it.Pos.ChunkID > 0 { 138 | tmp = new(HintItem) 139 | tmp.Pos.ChunkID -= 1 140 | w.writeItem(tmp) 141 | } 142 | 143 | w.writeItem(it) 144 | } 145 | for i := 0; i < nsrc; i++ { 146 | src[i].close() 147 | } 148 | dst := fmt.Sprintf("%s/dst.hint.s", dir) 149 | utils.Remove(dst) 150 | state := HintStateDump 151 | ct := newCollisionTable() 152 | merge(srcp, dst, ct, &state, false) 153 | defer utils.Remove(dst) 154 | readHintAndCheck(t, dst, items) 155 | } 156 | 157 | func TestMerge2(t *testing.T) { 158 | testMerge(t, 2) 159 | } 160 | 161 | func TestMerge3(t *testing.T) { 162 | testMerge(t, 3) 163 | } 164 | 165 | func genKey(i int) string { 166 | return fmt.Sprintf("key_%05d", i) 167 | } 168 | 169 | func setAndCheckHintBuffer(t *testing.T, buf *HintBuffer, it *HintItem) { 170 | if !buf.Set(it, 0) { 171 | t.Fatalf("%#v set return false", it) 172 | } 173 | r, _ := buf.Get(it.Keyhash, it.Key) 174 | if r == nil || *r != *it { 175 | t.Fatalf("%#v != %#v", r, it) 176 | } 177 | } 178 | 179 | func TestHintBuffer(t *testing.T) { 180 | n := 10 181 | Conf.SplitCap = int64(n) 182 | Conf.IndexIntervalSize = 128 183 | 184 | defer func() { 185 | 186 | }() 187 | 188 | buf := NewHintBuffer() 189 | items := genSortedHintItems(n + 1) 190 | for i := 0; i < n; i++ { 191 | setAndCheckHintBuffer(t, buf, items[i]) 192 | } 193 | if buf.Set(items[n], 0) { 194 | t.Fatalf("set return true") 195 | } 196 | items[n-1].Ver = -1 197 | setAndCheckHintBuffer(t, buf, items[n-1]) 198 | } 199 | 200 | func checkChunk(t *testing.T, ck *hintChunk, it *HintItem) { 201 | r, err := ck.get(it.Keyhash, it.Key, false) 202 | if err != nil || r == nil || *r != *it { 203 | t.Fatalf("err = %s, %#v != %#v", err, r, it) 204 | } 205 | } 206 | 207 | func setAndCheckChunk(t *testing.T, ck *hintChunk, it *HintItem, rotate bool) { 208 | if rotate != ck.setItem(it, 0) { 209 | t.Fatalf("%#v not %v", it, rotate) 210 | } 211 | checkChunk(t, ck, it) 212 | } 213 | 214 | func TestHintChunk(t *testing.T) { 215 | n := 10 216 | Conf.SplitCap = int64(n) 217 | Conf.IndexIntervalSize = 128 218 | 219 | ck := newHintChunk(0) 220 | items := genSortedHintItems(n + 2) 221 | i := 0 222 | for ; i < n; i++ { 223 | setAndCheckChunk(t, ck, items[i], false) 224 | } 225 | setAndCheckChunk(t, ck, items[i], true) 226 | setAndCheckChunk(t, ck, items[i+1], false) 227 | 228 | if len(ck.splits) != 2 { 229 | t.Fatalf("%d", len(ck.splits)) 230 | } 231 | items[n-1].Ver = -1 232 | setAndCheckChunk(t, ck, items[n-1], false) 233 | } 234 | 235 | func checkMgr(t *testing.T, hm *hintMgr, it *HintItem, chunkID int) { 236 | r, cid, err := hm.getItem(it.Keyhash, it.Key, false) 237 | if err != nil { 238 | t.Fatalf("%#v, %s", it, err.Error()) 239 | } 240 | if r == nil || *r != *it || cid != chunkID { 241 | t.Fatalf("%#v != %#v or %d != %d, %s", r, it, cid, chunkID, loghub.GetStack(1000)) 242 | } 243 | } 244 | 245 | func setAndCheckMgr(t *testing.T, hm *hintMgr, it *HintItem, chunkID int) { 246 | hm.setItem(it, chunkID, 0) 247 | checkMgr(t, hm, it, chunkID) 248 | } 249 | 250 | func checkFiles(t *testing.T, dir string, files *utils.Dir) { 251 | diskfiles, df1, df2, err := files.CheckPath(dir) 252 | _, file, line, _ := runtime.Caller(1) 253 | if err != nil || df1 != nil || df2 != nil { 254 | t.Fatal(file, line, files.ToSlice(), diskfiles.ToSlice(), df1, df2, err) 255 | } 256 | } 257 | 258 | func fillChunk(t *testing.T, dir string, hm *hintMgr, items []*HintItem, chunkID int, files *utils.Dir) { 259 | logger.Infof("fill %d", chunkID) 260 | n := len(items) 261 | setAndCheckMgr(t, hm, items[0], chunkID) 262 | time.Sleep(time.Second * 4) 263 | hm.dumpAndMerge(false) 264 | hm.Merge(false) 265 | logger.Infof("check %d", chunkID) 266 | checkFiles(t, dir, files) 267 | 268 | checkMgr(t, hm, items[0], chunkID) 269 | for i := 1; i < n; i++ { 270 | checkMgr(t, hm, items[i], chunkID-1) 271 | } 272 | for i := 1; i < n; i++ { 273 | setAndCheckMgr(t, hm, items[i], chunkID) 274 | } 275 | } 276 | 277 | func TestHintMgr(t *testing.T) { 278 | setupTest("testHintMgr") 279 | defer clearTest() 280 | runtime.GOMAXPROCS(4) 281 | 282 | persp := 10 283 | Conf.SplitCap = int64(persp) 284 | Conf.IndexIntervalSize = 128 285 | Conf.MergeInterval = 250 // disable async merge 286 | nsp := 2 287 | n := persp * nsp 288 | items := genSortedHintItems(n) 289 | hm := newHintMgr(0, dir) 290 | 291 | chunkID := 3 292 | // write 3 293 | for i := 0; i < n; i++ { 294 | setAndCheckMgr(t, hm, items[i], chunkID) 295 | } 296 | files := utils.NewDir() 297 | files.Set("collision.yaml", -1) 298 | files.SetMultiNoSize("003.000.idx.s", "003.001.idx.s", "003.001.idx.m") 299 | fillChunk(t, dir, hm, items, 4, files) 300 | files.SetMultiNoSize("004.000.idx.s", "004.001.idx.s", "004.001.idx.m") 301 | files.Delete("003.001.idx.m") 302 | fillChunk(t, dir, hm, items, 5, files) 303 | files.SetMultiNoSize("005.000.idx.s", "005.001.idx.s", "005.001.idx.m") 304 | files.Delete("004.001.idx.m") 305 | fillChunk(t, dir, hm, items, 6, files) 306 | 307 | setAndCheckMgr(t, hm, items[0], 7) 308 | hm.dumpAndMerge(false) 309 | // get mtime 310 | // change item content, 注意 pos 311 | logger.Infof("set 6 again") 312 | it := *(items[0]) 313 | it.Pos.ChunkID = items[0].Pos.ChunkID + 100 314 | hm.setItem(&it, 6, 0) 315 | 316 | time.Sleep(time.Second * 5) 317 | checkMgr(t, hm, items[0], 7) 318 | time.Sleep(time.Second * 3) 319 | checkChunk(t, hm.chunks[6], &it) 320 | } 321 | -------------------------------------------------------------------------------- /memcache/server.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "io" 7 | "net" 8 | "os" 9 | "os/signal" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "syscall" 14 | "time" 15 | 16 | "sync/atomic" 17 | 18 | "github.com/douban/gobeansdb/config" 19 | "github.com/douban/gobeansdb/loghub" 20 | "github.com/douban/gobeansdb/utils" 21 | ) 22 | 23 | var ( 24 | SlowCmdTime = time.Millisecond * 100 // 100ms 25 | RL *ReqLimiter 26 | logger = loghub.ErrorLogger 27 | accessLogger = loghub.AccessLogger 28 | analysisLogger = loghub.AnalysisLogger 29 | ) 30 | 31 | type ServerConn struct { 32 | RemoteAddr string 33 | rwc io.ReadWriteCloser // i/o connection 34 | closeAfterReply bool 35 | 36 | rbuf *bufio.Reader 37 | wbuf *bufio.Writer 38 | req *Request 39 | } 40 | 41 | func newServerConn(conn net.Conn) *ServerConn { 42 | c := new(ServerConn) 43 | c.RemoteAddr = conn.RemoteAddr().String() 44 | c.rwc = conn 45 | 46 | c.rbuf = bufio.NewReader(c.rwc) 47 | c.wbuf = bufio.NewWriter(c.rwc) 48 | c.req = new(Request) 49 | return c 50 | } 51 | 52 | func (c *ServerConn) Close() { 53 | if c.rwc != nil { 54 | c.rwc.Close() 55 | c.rwc = nil 56 | } 57 | } 58 | 59 | func (c *ServerConn) Shutdown() { 60 | c.closeAfterReply = true 61 | } 62 | 63 | func overdue(recvtime, now time.Time) bool { 64 | return now.Sub(recvtime) > time.Duration(config.MCConf.TimeoutMS)*time.Millisecond 65 | } 66 | 67 | func (c *ServerConn) ServeOnce(storageClient StorageClient, stats *Stats) (err error) { 68 | req := c.req 69 | var resp *Response = nil 70 | defer func() { 71 | storageClient.Clean() 72 | if e := recover(); e != nil { 73 | logger.Errorf("mc panic(%#v), cmd %s, keys %v, stack: %s", 74 | e, req.Cmd, req.Keys, utils.GetStack(2000)) 75 | } 76 | req.Clear() 77 | if resp != nil { 78 | resp.CleanBuffer() 79 | } 80 | if req.Working { 81 | RL.Put(req) 82 | } 83 | }() 84 | 85 | // 关于错误处理 86 | // 87 | // 目前这里能看到的错误主要有以下 3 类: 88 | // 1. Client 输入的格式错误(不满足 memcache 协议): 即 req.Read 里面解析命令时遇到的错误， 89 | // 这类错误 server 给客户端返回错误信息即可，然后继续尝试读取下一个命令。 90 | // 2. 网络连接错误: 即 server 在读取需要数据的时候遇到 Unexpected EOF，或者 server 写数据 91 | // 时失败，这类错误直接关闭连接。 92 | // 3. storageClient 里面错误: 这些错误应该在相应的 Process 函数里面处理掉， 93 | // 并设置好相应的 status 和 msg，在这里只是把处理后的结果返回给客户端即可。 94 | 95 | err = req.Read(c.rbuf) 96 | t := time.Now() 97 | readTimeout := false 98 | 99 | if err != nil { 100 | if req.Item != nil { 101 | req.Item.CArray.Free() 102 | } 103 | if err == ErrNetworkError { 104 | // process client connection related error 105 | c.Shutdown() 106 | return nil 107 | } else if err == ErrNonMemcacheCmd { 108 | // process non memcache commands, e.g. 'gc', 'optimize_stat'. 109 | status, msg := storageClient.Process(req.Cmd, req.Keys) 110 | resp = new(Response) 111 | resp.Status = status 112 | resp.Msg = msg 113 | err = nil 114 | } else if err == ErrOOM { 115 | resp = new(Response) 116 | resp.Status = "NOT_STORED" 117 | err = nil 118 | } else { 119 | // process client command format related error 120 | resp = new(Response) 121 | resp.Status = "CLIENT_ERROR" 122 | resp.Msg = err.Error() 123 | err = nil 124 | } 125 | } else if overdue(req.ReceiveTime, t) { 126 | req.SetStat("recv_timeout") 127 | resp = new(Response) 128 | resp.Status = "RECV_TIMEOUT" 129 | resp.Msg = "recv_timeout" 130 | readTimeout = true 131 | logger.Errorf("recv_timeout cmd %s, keys %v", req.Cmd, req.Keys) 132 | } else { 133 | // 由于 set 等命令的 req.Item.Body 释放时间不确定，所以这里提前记录下 Body 的大小， 134 | // 后面在记录 access log 时会用到 135 | bodySize := 0 136 | if req.Item != nil { 137 | bodySize = len(req.Item.Body) 138 | } 139 | 140 | // process memcache commands, e.g. 'set', 'get', 'incr'. 141 | req.SetStat("process") 142 | resp, err = req.Process(storageClient, stats) 143 | dt := time.Since(t) 144 | if dt > SlowCmdTime { 145 | atomic.AddInt64(&(stats.slow_cmd), 1) 146 | } 147 | if resp == nil { 148 | // quit\r\n command 149 | c.Shutdown() 150 | return nil 151 | } 152 | 153 | if accessLogger.Hub != nil { 154 | c.writeAccessLog(resp, bodySize, err, dt, storageClient.GetSuccessedTargets()) 155 | } 156 | } 157 | 158 | if !resp.Noreply { 159 | if !readTimeout && overdue(req.ReceiveTime, time.Now()) { 160 | req.SetStat("process_timeout") 161 | resp.CleanBuffer() 162 | resp = new(Response) 163 | resp.Status = "PROCESS_TIMEOUT" 164 | resp.Msg = "process_timeout" 165 | logger.Errorf("process_timeout cmd %s, keys %v", req.Cmd, req.Keys) 166 | return 167 | } 168 | 169 | req.SetStat("resp") 170 | if err = resp.Write(c.wbuf); err != nil { 171 | return 172 | } 173 | if err = c.wbuf.Flush(); err != nil { 174 | return 175 | } 176 | } 177 | 178 | return 179 | } 180 | 181 | // 记录 accesslog, 主要用于 proxy 中 182 | func (c *ServerConn) writeAccessLog(resp *Response, bodySize int, processErr error, dt time.Duration, hosts []string) { 183 | req := c.req 184 | cmd := req.Cmd 185 | totalSize := 0 186 | sizeStr := "0" 187 | stat := "SUCC" 188 | 189 | switch req.Cmd { 190 | case "get", "gets": 191 | if len(req.Keys) > 1 { 192 | cmd += "m" 193 | sizes := make([]string, 0, len(req.Keys)) 194 | for _, k := range req.Keys { 195 | s := 0 196 | if v, ok := resp.Items[k]; ok { 197 | s = len(v.Body) 198 | } 199 | totalSize += s 200 | sizes = append(sizes, strconv.Itoa(s)) 201 | } 202 | sizeStr = strings.Join(sizes, ",") 203 | } else { 204 | for _, v := range resp.Items { 205 | totalSize += len(v.Body) 206 | } 207 | sizeStr = strconv.Itoa(totalSize) 208 | } 209 | 210 | if totalSize == 0 { 211 | stat = "FAILED" 212 | } 213 | 214 | case "set", "add", "replace": 215 | sizeStr = strconv.Itoa(bodySize) 216 | if processErr != nil { 217 | stat = "FAILED" 218 | } 219 | 220 | default: 221 | if processErr != nil { 222 | stat = "FAILED" 223 | } 224 | } 225 | 226 | if len(hosts) == 0 { 227 | hosts = append(hosts, "NoWhere") 228 | } 229 | hostStr := strings.Join(hosts, ",") 230 | keys := strings.Join(req.Keys, " ") 231 | 232 | accessLogger.Infof("%s %s %s %s %s %s %d %s", 233 | config.AccessLogVersion, c.RemoteAddr, strings.ToUpper(cmd), 234 | stat, sizeStr, hostStr, dt.Nanoseconds()/1e3, keys) 235 | } 236 | 237 | func (c *ServerConn) Serve(storageClient StorageClient, stats *Stats) (e error) { 238 | for !c.closeAfterReply { 239 | e = c.ServeOnce(storageClient, stats) 240 | if e != nil { 241 | logger.Debugf("conn err: %s", e.Error()) 242 | break 243 | } 244 | } 245 | c.Close() 246 | return 247 | } 248 | 249 | type Server struct { 250 | sync.Mutex 251 | addr string 252 | l net.Listener 253 | store Storage 254 | conns map[string]*ServerConn 255 | stats *Stats 256 | stop bool 257 | } 258 | 259 | func NewServer(store Storage) *Server { 260 | s := new(Server) 261 | s.store = store 262 | s.conns = make(map[string]*ServerConn, 1024) 263 | s.stats = NewStats() 264 | return s 265 | } 266 | 267 | func (s *Server) Listen(addr string) (e error) { 268 | s.addr = addr 269 | s.l, e = net.Listen("tcp", addr) 270 | return 271 | } 272 | 273 | func (s *Server) Serve() (e error) { 274 | InitTokens() 275 | if s.l == nil { 276 | return errors.New("no listener") 277 | } 278 | 279 | for { 280 | rw, e := s.l.Accept() 281 | if e != nil { 282 | logger.Infof("Accept failed: %s", e) 283 | return e 284 | } 285 | if s.stop { 286 | break 287 | } 288 | c := newServerConn(rw) 289 | go func() { 290 | s.Lock() 291 | s.conns[c.RemoteAddr] = c 292 | 293 | s.stats.curr_connections++ 294 | s.stats.total_connections++ 295 | s.Unlock() 296 | 297 | c.Serve(s.store.Client(), s.stats) 298 | 299 | s.Lock() 300 | s.stats.curr_connections-- 301 | delete(s.conns, c.RemoteAddr) 302 | s.Unlock() 303 | }() 304 | } 305 | s.l.Close() 306 | // wait for connections to close 307 | for i := 0; i < 20; i++ { 308 | s.Lock() 309 | if len(s.conns) == 0 { 310 | return nil 311 | } 312 | s.Unlock() 313 | time.Sleep(1e8) 314 | } 315 | logger.Infof("mc server %s shutdown ", s.addr) 316 | return nil 317 | } 318 | 319 | func (s *Server) Shutdown() { 320 | s.stop = true 321 | 322 | // try to connect 323 | net.Dial("tcp", s.addr) 324 | 325 | // notify conns 326 | s.Lock() 327 | defer s.Unlock() 328 | if len(s.conns) > 0 { 329 | for _, conn := range s.conns { 330 | conn.Shutdown() 331 | } 332 | } 333 | //s.Unlock() 334 | } 335 | 336 | func (s *Server) HandleSignals(errorlog string, accesslog string, analysislog string) { 337 | sch := make(chan os.Signal, 10) 338 | signal.Notify(sch, syscall.SIGTERM, syscall.SIGKILL, syscall.SIGINT, 339 | syscall.SIGHUP, syscall.SIGSTOP, syscall.SIGQUIT, syscall.SIGUSR1) 340 | go func(ch <-chan os.Signal) { 341 | for { 342 | sig := <-ch 343 | // SIGUSR1 信号是 logrotate 程序发送给的，表示已经完成了 roate 任务， 344 | // 通知 server 重新打开新的日志文件 345 | if sig == syscall.SIGUSR1 { 346 | // logger.Hub is always inited, so we call Reopen without check it. 347 | logger.Hub.Reopen(errorlog) 348 | 349 | if accesslog != "" && accessLogger != nil && accessLogger.Hub != nil { 350 | if err := accessLogger.Hub.Reopen(accesslog); err != nil { 351 | logger.Warnf("open accessLogger %s failed: %s", accesslog, err.Error()) 352 | } 353 | } 354 | 355 | if analysislog != "" && analysisLogger != nil && analysisLogger.Hub != nil { 356 | if err := analysisLogger.Hub.Reopen(analysislog); err != nil { 357 | logger.Warnf("open analysisLogger %s failed: %s", analysislog, err.Error()) 358 | } 359 | } 360 | } else { 361 | logger.Infof("signal received " + sig.String()) 362 | s.Shutdown() 363 | } 364 | } 365 | }(sch) 366 | } 367 | -------------------------------------------------------------------------------- /store/datafile.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "bufio" 5 | "encoding/binary" 6 | "fmt" 7 | "io" 8 | "os" 9 | "time" 10 | 11 | "github.com/douban/gobeansdb/cmem" 12 | "github.com/douban/gobeansdb/config" 13 | ) 14 | 15 | const ( 16 | recHeaderSize = 24 17 | ) 18 | 19 | var ( 20 | padding [256]byte 21 | ) 22 | 23 | type WriteRecord struct { 24 | rec *Record 25 | crc uint32 26 | ksz uint32 27 | vsz uint32 28 | pos Position 29 | header [recHeaderSize]byte 30 | } 31 | 32 | func newWriteRecord() *WriteRecord { 33 | wrec := new(WriteRecord) 34 | wrec.rec = new(Record) 35 | wrec.rec.Payload = new(Payload) 36 | return wrec 37 | } 38 | func wrapRecord(rec *Record) *WriteRecord { 39 | _, rec.Payload.RecSize = rec.Sizes() 40 | return &WriteRecord{ 41 | rec: rec, 42 | ksz: uint32(len(rec.Key)), 43 | vsz: uint32(len(rec.Payload.Body)), 44 | } 45 | } 46 | 47 | func (wrec *WriteRecord) String() string { 48 | return fmt.Sprintf("{ts: %v,flag 0x%x, ver %d, ksz: %d, vsz: %d}", 49 | time.Unix(int64(wrec.rec.Payload.TS), 0), 50 | wrec.rec.Payload.Flag, wrec.rec.Payload.Ver, wrec.ksz, wrec.vsz) 51 | } 52 | 53 | func (wrec *WriteRecord) decode(data []byte) (err error) { 54 | wrec.decode(data) 55 | decodeHeader(wrec, data) 56 | wrec.rec.Key = data[recHeaderSize : recHeaderSize+wrec.ksz] 57 | wrec.rec.Payload.Body = data[recHeaderSize+wrec.ksz : recHeaderSize+wrec.ksz+wrec.vsz] 58 | if wrec.crc != wrec.getCRC() { 59 | err = fmt.Errorf("crc check fail") 60 | logger.Infof(err.Error()) 61 | return 62 | } 63 | return nil 64 | } 65 | 66 | func (wrec *WriteRecord) getCRC() uint32 { 67 | hasher := newCrc32() 68 | hasher.write(wrec.header[4:]) 69 | if len(wrec.rec.Key) > 0 { 70 | hasher.write(wrec.rec.Key) 71 | } 72 | if len(wrec.rec.Payload.Body) > 0 { 73 | hasher.write(wrec.rec.Payload.Body) 74 | } 75 | return hasher.get() 76 | } 77 | 78 | func (wrec *WriteRecord) encodeHeader() { 79 | h := wrec.header[:] 80 | binary.LittleEndian.PutUint32(h[4:8], wrec.rec.Payload.TS) 81 | binary.LittleEndian.PutUint32(h[8:12], wrec.rec.Payload.Flag) 82 | binary.LittleEndian.PutUint32(h[12:16], uint32(wrec.rec.Payload.Ver)) 83 | binary.LittleEndian.PutUint32(h[16:20], wrec.ksz) 84 | binary.LittleEndian.PutUint32(h[20:24], wrec.vsz) 85 | crc := wrec.getCRC() 86 | binary.LittleEndian.PutUint32(h[:4], crc) 87 | return 88 | } 89 | 90 | func (wrec *WriteRecord) decodeHeader() (err error) { 91 | return decodeHeader(wrec, wrec.header[:]) 92 | } 93 | 94 | func decodeHeader(wrec *WriteRecord, h []byte) (err error) { 95 | wrec.crc = binary.LittleEndian.Uint32(h[:4]) 96 | wrec.rec.Payload.TS = binary.LittleEndian.Uint32(h[4:8]) 97 | wrec.rec.Payload.Flag = binary.LittleEndian.Uint32(h[8:12]) 98 | wrec.rec.Payload.Ver = int32(binary.LittleEndian.Uint32(h[12:16])) 99 | wrec.ksz = binary.LittleEndian.Uint32(h[16:20]) 100 | wrec.vsz = binary.LittleEndian.Uint32(h[20:24]) 101 | return 102 | } 103 | 104 | func readRecordAtPath(path string, offset uint32) (*WriteRecord, error) { 105 | f, err := os.Open(path) 106 | if err != nil { 107 | logger.Errorf("fail to open: %s: %v", path, err) 108 | return nil, err 109 | } 110 | defer f.Close() 111 | return readRecordAt(path, f, offset) 112 | } 113 | 114 | func readRecordAt(path string, f *os.File, offset uint32) (wrec *WriteRecord, err error) { 115 | wrec = newWriteRecord() 116 | defer func() { 117 | if err != nil { 118 | wrec.rec.Payload.Free() // must not return (nil, err) 119 | wrec = nil 120 | } 121 | }() 122 | var n int 123 | if n, err = f.ReadAt(wrec.header[:], int64(offset)); err != nil { 124 | err = fmt.Errorf("fail to read head %s:%d, err = %s, n = %d", path, offset, err.Error(), n) 125 | logger.Errorf(err.Error()) 126 | return 127 | } 128 | wrec.decodeHeader() 129 | if !config.IsValidKeySize(wrec.ksz) { 130 | err = fmt.Errorf("bad key size %s:%d, wrec %v", path, offset, wrec) 131 | logger.Errorf(err.Error()) 132 | return 133 | } else if !config.IsValidValueSize(wrec.vsz) { 134 | err = fmt.Errorf("bad value size %s:%d, wrec %v", path, offset, wrec) 135 | logger.Errorf(err.Error()) 136 | return 137 | } 138 | kvSize := int(wrec.ksz + wrec.vsz) 139 | var kv cmem.CArray 140 | if !kv.Alloc(int(kvSize)) { 141 | err = fmt.Errorf("fail to alloc for read %s:%d, wrec %v ", path, offset, wrec) 142 | logger.Errorf(err.Error()) 143 | return 144 | } 145 | cmem.DBRL.GetData.AddSizeAndCount(kv.Cap) 146 | 147 | wrec.rec.Key = kv.Body[:wrec.ksz] 148 | 149 | if n, err = f.ReadAt(kv.Body, int64(offset)+recHeaderSize); err != nil { 150 | err = fmt.Errorf("fail to read %s:%d, rec %v; return err = %s, n = %d", 151 | path, offset, wrec, err.Error(), n) 152 | logger.Errorf(err.Error()) 153 | cmem.DBRL.GetData.SubSizeAndCount(kv.Cap) 154 | return 155 | } 156 | wrec.rec.Key = make([]byte, wrec.ksz) 157 | copy(wrec.rec.Key, kv.Body[:wrec.ksz]) 158 | wrec.rec.Payload.CArray = kv 159 | wrec.rec.Payload.Body = kv.Body[wrec.ksz:] 160 | wrec.rec.Payload.RecSize = wrec.vsz 161 | crc := wrec.getCRC() 162 | if wrec.crc != crc { 163 | err = fmt.Errorf("crc check fail %s:%d, rec %v; %d != %d", 164 | path, offset, wrec, wrec.crc, crc) 165 | logger.Errorf(err.Error()) 166 | cmem.DBRL.GetData.SubSizeAndCount(kv.Cap) 167 | return 168 | } 169 | return wrec, nil 170 | } 171 | 172 | type DataStreamReader struct { 173 | path string 174 | ds *dataStore 175 | 176 | fd *os.File 177 | rbuf *bufio.Reader 178 | 179 | chunk int 180 | offset uint32 181 | 182 | maxBodyBuf []byte 183 | } 184 | 185 | func newDataStreamReader(path string, bufsz int) (*DataStreamReader, error) { 186 | fd, err := os.Open(path) 187 | if err != nil { 188 | logger.Infof(err.Error()) 189 | return nil, err 190 | } 191 | rbuf := bufio.NewReaderSize(fd, bufsz) 192 | maxBodyBuf := make([]byte, 0, config.MCConf.BodyMax) 193 | return &DataStreamReader{path: path, fd: fd, rbuf: rbuf, offset: 0, maxBodyBuf: maxBodyBuf}, nil 194 | } 195 | 196 | func (stream *DataStreamReader) seek(offset uint32) { 197 | stream.fd.Seek(int64(offset), io.SeekStart) 198 | stream.offset = offset 199 | } 200 | 201 | // TODO: slow 202 | func (stream *DataStreamReader) nextValid() (rec *Record, offset uint32, sizeBroken uint32, err error) { 203 | offset2 := stream.offset 204 | offset2 = offset2 & (^uint32(0xff)) 205 | fd, _ := os.Open(stream.fd.Name()) 206 | defer fd.Close() 207 | st, _ := fd.Stat() 208 | for int64(offset2) < st.Size() { 209 | wrec, err2 := readRecordAt(stream.path, fd, offset2) 210 | if err2 == nil { 211 | logger.Infof("crc fail end offset 0x%x, sizeBroken 0x%x", offset2, sizeBroken) 212 | _, rsize := wrec.rec.Sizes() 213 | offset3 := offset2 + rsize 214 | stream.fd.Seek(int64(offset3), io.SeekStart) 215 | stream.rbuf.Reset(stream.fd) 216 | stream.offset = offset2 + rsize 217 | return wrec.rec, offset2, sizeBroken, nil 218 | } 219 | sizeBroken += 256 220 | offset2 += 256 221 | stream.offset = offset2 222 | } 223 | 224 | logger.Infof("crc fail until file end, sizeBroken 0x%x", sizeBroken) 225 | return nil, offset2, sizeBroken, nil 226 | } 227 | 228 | func (stream *DataStreamReader) Next() (res *Record, offset uint32, sizeBroken uint32, err error) { 229 | wrec := newWriteRecord() 230 | if _, err = io.ReadFull(stream.rbuf, wrec.header[:]); err != nil { 231 | if err != io.EOF { 232 | logger.Errorf("%s:0x%x %s", stream.path, stream.offset, err.Error()) 233 | } else { 234 | err = nil 235 | } 236 | return 237 | } 238 | wrec.decodeHeader() 239 | if !config.IsValidKeySize(wrec.ksz) { 240 | logger.Errorf("gc: bad key len %s %d %d %d", stream.fd.Name(), stream.offset, wrec.ksz, wrec.vsz) 241 | return stream.nextValid() 242 | } 243 | 244 | if !config.IsValidValueSize(wrec.vsz) { 245 | logger.Errorf("gc: bad value len %s %d %d %d", stream.fd.Name(), stream.offset, wrec.ksz, wrec.vsz) 246 | return stream.nextValid() 247 | } 248 | 249 | wrec.rec.Key = make([]byte, wrec.ksz) 250 | if _, err = io.ReadFull(stream.rbuf, wrec.rec.Key); err != nil { 251 | logger.Errorf(err.Error()) 252 | return 253 | } 254 | wrec.rec.Payload.Body = stream.maxBodyBuf[:wrec.vsz] 255 | if _, err = io.ReadFull(stream.rbuf, wrec.rec.Payload.Body); err != nil { 256 | logger.Errorf(err.Error()) 257 | return 258 | } 259 | recsizereal, recsize := wrec.rec.Sizes() 260 | tail := recsizereal & 0xff 261 | if tail != 0 { 262 | stream.rbuf.Discard(int(PADDING - tail)) 263 | } 264 | 265 | crc := wrec.getCRC() 266 | if wrec.crc != crc { 267 | err := fmt.Errorf("crc fail begin offset %x", stream.offset) 268 | logger.Errorf(err.Error()) 269 | sizeBroken += 1 270 | return stream.nextValid() 271 | } 272 | res = wrec.rec 273 | offset = stream.offset 274 | stream.offset += recsize 275 | res.Payload.RecSize = recsize 276 | return 277 | } 278 | 279 | func (stream *DataStreamReader) Offset() uint32 { 280 | return stream.offset 281 | } 282 | 283 | func (stream *DataStreamReader) Close() error { 284 | return stream.fd.Close() 285 | } 286 | 287 | type DataStreamWriter struct { 288 | path string 289 | fd *os.File 290 | wbuf *bufio.Writer 291 | 292 | chunk int 293 | offset uint32 294 | } 295 | 296 | func (stream *DataStreamWriter) Append(rec *Record) (offset uint32, err error) { 297 | return stream.append(wrapRecord(rec)) 298 | } 299 | 300 | func (stream *DataStreamWriter) append(wrec *WriteRecord) (offset uint32, err error) { 301 | offset = stream.offset 302 | err = wrec.append(stream.wbuf, true) 303 | stream.offset += wrec.rec.Payload.RecSize 304 | return 305 | } 306 | 307 | func (wrec *WriteRecord) append(wbuf io.Writer, dopadding bool) error { 308 | wrec.encodeHeader() 309 | size, sizeall := wrec.rec.Sizes() 310 | if n, err := wbuf.Write(wrec.header[:]); err != nil { 311 | logger.Errorf("%v %d", err, n) 312 | return err 313 | } 314 | if n, err := wbuf.Write(wrec.rec.Key); err != nil { 315 | logger.Errorf("%v %d", err, n) 316 | return err 317 | } 318 | if n, err := wbuf.Write(wrec.rec.Payload.Body); err != nil { 319 | logger.Errorf("%v %d", err, n) 320 | return err 321 | } 322 | npad := sizeall - size 323 | if dopadding && npad != 0 { 324 | if n, err := wbuf.Write(padding[:npad]); err != nil { 325 | logger.Errorf("%v %d", err, n) 326 | return err 327 | } 328 | } 329 | return nil 330 | } 331 | 332 | func (stream *DataStreamWriter) Offset() uint32 { 333 | return stream.offset 334 | } 335 | 336 | func (stream *DataStreamWriter) Close() error { 337 | if err := stream.wbuf.Flush(); err != nil { 338 | st, _ := stream.fd.Stat() 339 | logger.Errorf("flush err: %s %v %s", stream.fd.Name(), err, st.Mode()) 340 | return err 341 | } 342 | return stream.fd.Close() 343 | } 344 | -------------------------------------------------------------------------------- /store/gc.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | "time" 7 | 8 | "github.com/douban/gobeansdb/config" 9 | ) 10 | 11 | type GCMgr struct { 12 | mu sync.RWMutex 13 | stat map[*Bucket]*GCState // map[bucketID]*GCState 14 | } 15 | 16 | type GCState struct { 17 | BeginTS time.Time 18 | EndTS time.Time 19 | 20 | // Begin and End are chunckIDs, they determine the range of GC. 21 | Begin int 22 | End int 23 | 24 | // Src and Dst are chunkIDs, they are tmp variables used in gc process. 25 | Src int 26 | Dst int 27 | 28 | // For beansdbadmin check status. 29 | Running bool 30 | 31 | Err error 32 | CancelFlag bool 33 | // sum 34 | GCFileState 35 | } 36 | 37 | type GCFileState struct { 38 | NumBefore int64 39 | NumReleased int64 40 | NumReleasedDeleted int64 41 | SizeBefore int64 42 | SizeReleased int64 43 | SizeDeleted int64 44 | SizeBroken int64 45 | NumNotInHtree int64 46 | } 47 | 48 | func (s *GCFileState) add(s2 *GCFileState) { 49 | s.NumBefore += s2.NumBefore 50 | s.NumReleased += s2.NumReleased 51 | s.NumReleasedDeleted += s2.NumReleasedDeleted 52 | s.SizeBefore += s2.SizeBefore 53 | s.SizeBroken += s2.SizeBroken 54 | s.SizeDeleted += s2.SizeDeleted 55 | s.SizeReleased += s2.SizeReleased 56 | s.NumNotInHtree += s2.NumNotInHtree 57 | } 58 | 59 | func (s *GCFileState) addRecord(size uint32, isNewest, isDeleted bool, sizeBroken uint32) { 60 | if !isNewest { 61 | s.NumReleased += 1 62 | s.SizeReleased += int64(size) 63 | if isDeleted { 64 | s.NumReleasedDeleted += 1 65 | s.SizeDeleted += int64(size) 66 | } 67 | } 68 | s.SizeReleased += int64(sizeBroken) 69 | s.SizeBroken += int64(sizeBroken) 70 | s.SizeBefore += int64(size + sizeBroken) 71 | s.NumBefore += 1 72 | } 73 | 74 | func (s *GCFileState) String() string { 75 | return fmt.Sprintf("%#v", s) 76 | } 77 | 78 | func (mgr *GCMgr) UpdateCollision(bkt *Bucket, ki *KeyInfo, oldPos, newPos Position, rec *Record) { 79 | // not have to (leave it to get) 80 | 81 | // if in ctable: update pos 82 | // else: decompress, get vhash and set collisions 83 | } 84 | 85 | func (mgr *GCMgr) UpdateHtreePos(bkt *Bucket, ki *KeyInfo, oldPos, newPos Position) { 86 | // TODO: should be a api of htree to be atomic 87 | meta, _, ok := bkt.htree.get(ki) 88 | if !ok { 89 | logger.Warnf("old key removed when updating pos bucket %d %s %#v %#v", 90 | bkt.ID, ki.StringKey, meta, oldPos) 91 | return 92 | } 93 | bkt.htree.set(ki, meta, newPos) 94 | } 95 | 96 | func (mgr *GCMgr) BeforeBucket(bkt *Bucket, startChunkID, endChunkID int, merge bool) { 97 | bkt.hints.state |= HintStateGC // will about 98 | for bkt.hints.state&HintStateMerge != 0 { 99 | logger.Infof("gc wait for merge to stop") 100 | time.Sleep(5 * time.Millisecond) 101 | } 102 | 103 | // dump hint and do merge, and hold all new SETs in hint buffers 104 | // so collision will be find either during merge or in hint buffer 105 | // so will not wrongly GC a collision record. e.g.: 106 | // key1 and key2 have the same keyhash, key1 is set before gc, and key2 after that. 107 | bkt.hints.maxDumpableChunkID = endChunkID - 1 108 | if merge { 109 | bkt.hints.forceRotateSplit() 110 | time.Sleep(time.Duration(SecsBeforeDump+1) * time.Second) 111 | bkt.hints.dumpAndMerge(true) // TODO: should not dump idx.m! 112 | bkt.hints.Merge(true) 113 | } else { 114 | bkt.hints.RemoveMerged() 115 | } 116 | 117 | // remove hints 118 | bkt.removeHtree() 119 | } 120 | 121 | func (mgr *GCMgr) AfterBucket(bkt *Bucket) { 122 | bkt.hints.state &= ^HintStateGC 123 | bkt.hints.maxDumpableChunkID = MAX_NUM_CHUNK - 1 124 | } 125 | 126 | func (bkt *Bucket) gcCheckEnd(start, endChunkID, noGCDays int) (end int, err error) { 127 | end = endChunkID 128 | if end < 0 || end >= bkt.datas.newHead-1 { 129 | end = bkt.datas.newHead - 1 130 | } 131 | 132 | if noGCDays < 0 { 133 | noGCDays = Conf.NoGCDays 134 | } 135 | for next := end + 1; next >= start+1; next-- { 136 | if bkt.datas.chunks[next].getDiskFileSize() <= 0 { 137 | continue 138 | } 139 | var ts int64 140 | ts, err = bkt.datas.chunks[next].getFirstRecTs() 141 | if err != nil { 142 | return 143 | } 144 | if time.Now().Unix()-ts > int64(noGCDays)*86400 { 145 | for end = next - 1; end >= start; end-- { 146 | if bkt.datas.chunks[end].size > 0 { 147 | return 148 | } 149 | } 150 | return 151 | } 152 | } 153 | err = fmt.Errorf("no file to gc within %d days, start = %d", noGCDays, start) 154 | return 155 | } 156 | 157 | func (bkt *Bucket) gcCheckStart(startChunkID int) (start int, err error) { 158 | if startChunkID < 0 { 159 | start = bkt.NextGCChunk 160 | } else if startChunkID > bkt.datas.newHead { 161 | err = fmt.Errorf("startChunkID > bkt.datas.newHead ") 162 | return 163 | } else { 164 | 165 | start = startChunkID 166 | } 167 | for ; start < bkt.datas.newHead; start++ { 168 | if bkt.datas.chunks[start].size > 0 { 169 | break 170 | } 171 | } 172 | return 173 | } 174 | 175 | func (bkt *Bucket) gcCheckRange(startChunkID, endChunkID, noGCDays int) (start, end int, err error) { 176 | if start, err = bkt.gcCheckStart(startChunkID); err != nil { 177 | return 178 | } 179 | if end, err = bkt.gcCheckEnd(start, endChunkID, noGCDays); err != nil { 180 | return 181 | } 182 | if end < start { 183 | err = fmt.Errorf("end %d < start %d, nothing to gc", end, start) 184 | } 185 | return 186 | } 187 | 188 | func (mgr *GCMgr) gc(bkt *Bucket, startChunkID, endChunkID int, merge bool) { 189 | 190 | logger.Infof("begin GC bucket %d chunk [%d, %d]", bkt.ID, startChunkID, endChunkID) 191 | 192 | bkt.GCHistory = append(bkt.GCHistory, GCState{}) 193 | gc := &bkt.GCHistory[len(bkt.GCHistory)-1] 194 | // add gc to mgr's stat map 195 | mgr.mu.Lock() 196 | mgr.stat[bkt] = gc 197 | mgr.mu.Unlock() 198 | gc.Running = true 199 | gc.BeginTS = time.Now() 200 | defer func() { 201 | mgr.mu.Lock() 202 | delete(mgr.stat, bkt) 203 | mgr.mu.Unlock() 204 | gc.Running = false 205 | gc.EndTS = time.Now() 206 | }() 207 | gc.Begin = startChunkID 208 | gc.End = endChunkID 209 | 210 | var oldPos Position 211 | var newPos Position 212 | var rec *Record 213 | var r *DataStreamReader 214 | 215 | mgr.BeforeBucket(bkt, startChunkID, endChunkID, merge) 216 | defer mgr.AfterBucket(bkt) 217 | 218 | gc.Dst = startChunkID 219 | // try to find the nearest chunk that small than start chunk 220 | for i := startChunkID - 1; i >= 0; i-- { 221 | sz := bkt.datas.chunks[i].size 222 | if sz > 0 { 223 | if int64(sz) < Conf.DataFileMax-config.MCConf.BodyMax { 224 | // previous one available and not full, use it. 225 | gc.Dst = i 226 | break 227 | } else { 228 | if i < startChunkID-1 { // not previous one 229 | gc.Dst = i + 1 230 | } 231 | break 232 | } 233 | } 234 | } 235 | 236 | dstchunk := &bkt.datas.chunks[gc.Dst] 237 | err := dstchunk.beginGCWriting(gc.Begin) 238 | if err != nil { 239 | gc.Err = err 240 | return 241 | } 242 | newPos.ChunkID = gc.Dst 243 | defer func() { 244 | dstchunk.endGCWriting() 245 | bkt.hints.trydump(gc.Dst, true) 246 | }() 247 | 248 | for gc.Src = gc.Begin; gc.Src <= gc.End; gc.Src++ { 249 | if gc.CancelFlag { 250 | logger.Infof("GC canceled: src %d dst %d", gc.Src, gc.Dst) 251 | return 252 | } 253 | if bkt.datas.chunks[gc.Src].size <= 0 { 254 | logger.Infof("skip empty chunk %d", gc.Src) 255 | continue 256 | } 257 | oldPos.ChunkID = gc.Src 258 | var fileState GCFileState 259 | // reader must have a larger buffer 260 | logger.Infof("begin GC bucket %d, file %d -> %d", bkt.ID, gc.Src, gc.Dst) 261 | bkt.hints.ClearChunk(gc.Src) 262 | if r, err = bkt.datas.GetStreamReader(gc.Src); err != nil { 263 | gc.Err = err 264 | logger.Errorf("gc failed: %s", err.Error()) 265 | return 266 | } 267 | 268 | for { 269 | var sizeBroken uint32 270 | rec, oldPos.Offset, sizeBroken, err = r.Next() 271 | if err != nil { 272 | gc.Err = err 273 | logger.Errorf("gc failed: %s", err.Error()) 274 | return 275 | } 276 | if rec == nil { 277 | break 278 | } 279 | 280 | var isNewest, isCoverdByCollision, isDeleted bool 281 | meta := rec.Payload.Meta 282 | ki := NewKeyInfoFromBytes(rec.Key, getKeyHash(rec.Key), false) 283 | treeMeta, treePos, found := bkt.htree.get(ki) 284 | if found { 285 | if oldPos == treePos { // easy 286 | meta.ValueHash = treeMeta.ValueHash 287 | isNewest = true 288 | } else { 289 | isDeleted = treeMeta.Ver < 0 290 | hintit, hintchunkid, isCoverdByCollision := bkt.hints.getCollisionGC(ki) 291 | if isCoverdByCollision { 292 | if hintit != nil { 293 | p := Position{hintchunkid, hintit.Pos.Offset} 294 | if p == oldPos { 295 | isNewest = true 296 | meta.ValueHash = hintit.Vhash 297 | } 298 | } else { 299 | isNewest = true // guess 300 | meta.ValueHash = rec.Payload.Getvhash() 301 | } 302 | } 303 | } 304 | } else { 305 | // when rebuiding the HTree, the deleted recs are removed from HTree 306 | // we are not sure whether the `set rec` is still in datafiles while `auto GC`, so we need to write a copy of `del rec` in datafile. 307 | // but we can remove the `del rec` while GC begin with 0 308 | fileState.NumNotInHtree++ 309 | if gc.Begin > 0 && rec.Payload.Ver < 0 { 310 | isNewest = true 311 | } 312 | } 313 | 314 | wrec := wrapRecord(rec) 315 | recsize := wrec.rec.Payload.RecSize 316 | fileState.addRecord(recsize, isNewest, isDeleted, sizeBroken) 317 | //logger.Infof("key stat: %v %v %v %v", ki.StringKey, isNewest, isCoverdByCollision, isDeleted) 318 | if !isNewest { 319 | continue 320 | } 321 | 322 | if recsize+dstchunk.writingHead > uint32(Conf.DataFileMax) { 323 | dstchunk.endGCWriting() 324 | bkt.hints.trydump(gc.Dst, true) 325 | 326 | gc.Dst++ 327 | newPos.ChunkID = gc.Dst 328 | logger.Infof("continue GC bucket %d, file %d -> %d", bkt.ID, gc.Src, gc.Dst) 329 | dstchunk = &bkt.datas.chunks[gc.Dst] 330 | err = dstchunk.beginGCWriting(gc.Src) 331 | if err != nil { 332 | gc.Err = err 333 | return 334 | } 335 | } 336 | if newPos.Offset, err = dstchunk.AppendRecordGC(wrec); err != nil { 337 | gc.Err = err 338 | logger.Errorf("gc failed: %s", err.Error()) 339 | return 340 | } 341 | // logger.Infof("%s %v %v", ki.StringKey, newPos, meta) 342 | if found { 343 | if isCoverdByCollision { 344 | mgr.UpdateCollision(bkt, ki, oldPos, newPos, rec) 345 | } 346 | mgr.UpdateHtreePos(bkt, ki, oldPos, newPos) 347 | } 348 | 349 | rotated := bkt.hints.set(ki, &meta, newPos, recsize, "gc") 350 | if rotated { 351 | bkt.hints.trydump(gc.Dst, false) 352 | } 353 | } 354 | 355 | if gc.Src != gc.Dst { 356 | bkt.datas.chunks[gc.Src].Clear() 357 | } 358 | if gc.Src+1 >= bkt.NextGCChunk { 359 | bkt.NextGCChunk = gc.Src + 1 360 | bkt.dumpGCHistroy() 361 | } 362 | logger.Infof("end GC file %#v", fileState) 363 | gc.add(&fileState) 364 | } 365 | logger.Infof("end GC all %#v", gc) 366 | } 367 | -------------------------------------------------------------------------------- /store/htree_test.go: -------------------------------------------------------------------------------- 1 | package store 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | _ "net/http/pprof" 7 | "os" 8 | "path/filepath" 9 | "runtime" 10 | "sort" 11 | "strconv" 12 | "testing" 13 | "time" 14 | 15 | "github.com/douban/gobeansdb/utils" 16 | ) 17 | 18 | func TestHash(t *testing.T) { 19 | h := utils.Fnv1a([]byte("test")) 20 | if h != uint32(2949673445) { 21 | t.Error("hash error", h) 22 | } 23 | } 24 | 25 | func TestParse(t *testing.T) { 26 | var u uint64 = 0x89abcdef01234567 27 | path0 := []int{8, 9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0, 1, 2, 3, 4, 5, 6, 7} 28 | var pathbuf [16]int 29 | path := ParsePathUint64(u, pathbuf[:16]) 30 | if !isSamePath(path, path0, 8) { 31 | t.Errorf("parse %016x error: %v != %v", u, path, path0) 32 | } 33 | s := "89abcdef" 34 | for i := 0; i < 8; i++ { 35 | path, _ = ParsePathString(s[:i], pathbuf[:16]) 36 | if !isSamePath(path, path0[:i], i) { 37 | t.Errorf("parse %d, error: %v != %v", i, path, path0) 38 | } 39 | } 40 | } 41 | 42 | func TestHTree(t *testing.T) { 43 | 44 | pos := 0xfe 45 | for h := 2; h <= 6; h++ { 46 | Conf.InitDefault() 47 | Conf.NumBucket = 256 48 | Conf.TreeHeight = h 49 | Conf.Init() 50 | testHTree(t, 1, pos) 51 | } 52 | 53 | pos = 0xf 54 | for h := 2; h <= 7; h++ { 55 | Conf.InitDefault() 56 | Conf.NumBucket = 16 57 | Conf.TreeHeight = h 58 | Conf.Init() 59 | testHTree(t, 2, pos) 60 | } 61 | } 62 | 63 | func testHTree(t *testing.T, seq, treepos int) { 64 | defer func() { 65 | thresholdListKey = ThresholdListKeyDefault 66 | }() 67 | 68 | t.Logf("testing height %d %x %d", Conf.TreeDepth, treepos, Conf.TreeHeight) 69 | depth := Conf.TreeDepth 70 | height := Conf.TreeHeight 71 | tree := newHTree(depth, treepos, height) 72 | N := int(thresholdListKey) 73 | dstNode := &tree.levels[height-1][0] 74 | 75 | keyhash := uint64(treepos << uint32(64-4*depth)) 76 | ki := NewKeyInfoFromBytes([]byte("key"), keyhash, false) 77 | var meta Meta 78 | var pos Position 79 | 80 | reset := func() { 81 | ki.KeyHash = keyhash 82 | pos = Position{0, 0} 83 | meta = Meta{Ver: 1} 84 | } 85 | 86 | // set 87 | reset() 88 | for i := 0; i < N; i++ { 89 | ki.Prepare() 90 | tree.set(ki, &meta, pos) 91 | count := int(dstNode.count) 92 | if count != i+1 { 93 | t.Fatalf("wrong count %d != %d", count, i+1) 94 | } 95 | ki.KeyHash++ 96 | pos.Offset += PADDING 97 | } 98 | 99 | reset() 100 | // get 101 | for i := 0; i < N; i++ { 102 | ki.Prepare() 103 | meta2, pos2, found := tree.get(ki) 104 | if !found || pos2.Offset != uint32(i)*PADDING { 105 | t.Fatalf("%d: fail to get %#v, found = %v, meta2 = %#v, pos = %#v", i, ki, found, meta2, pos2) 106 | } 107 | ki.KeyHash++ 108 | } 109 | 110 | // delete 111 | reset() 112 | meta.Ver = -2 113 | for i := 0; i < N; i++ { 114 | ki.Prepare() 115 | tree.set(ki, &meta, pos) 116 | count := int(dstNode.count) 117 | if count != N-i-1 { 118 | t.Fatalf("wrong count %d != %d", count, N-i-1) 119 | } 120 | ki.KeyHash++ 121 | } 122 | 123 | // set again 124 | reset() 125 | for i := 0; i < N; i++ { 126 | ki.Prepare() 127 | tree.set(ki, &meta, pos) 128 | count := int(dstNode.count) 129 | if count != i+1 { 130 | t.Fatalf("wrong count %d != %d", count, i+1) 131 | } 132 | ki.KeyHash++ 133 | pos.Offset += PADDING 134 | } 135 | for i := 0; i < height-1; i++ { 136 | if tree.levels[i][0].count != 0 { 137 | t.Fatalf("wrong count %d != 0", i) 138 | } 139 | } 140 | tree.updateNodes(0, 0) 141 | for i := 0; i < height-1; i++ { 142 | if int(tree.levels[i][0].count) != N { 143 | t.Fatalf("wrong count %d != N", i) 144 | } 145 | } 146 | 147 | ki.KeyHash = uint64((treepos << uint32(64-4*depth)) + (0xf << uint32(64-4*depth-4))) 148 | ki.Prepare() 149 | tree.set(ki, &meta, pos) 150 | 151 | // list root 152 | ki.KeyIsPath = true 153 | ki.StringKey = fmt.Sprintf("%x", treepos) 154 | ki.Key = []byte(ki.StringKey) 155 | ki.Prepare() 156 | 157 | thresholdListKey += 2 158 | items, nodes := tree.listDir(ki) 159 | if !(len(nodes) == 0 && len(items) == N+1) { 160 | t.Fatalf("items:%v, nodes:%v", items, nodes) 161 | } 162 | thresholdListKey -= 2 163 | items, nodes = tree.listDir(ki) 164 | if !(len(nodes) == 16 && len(items) == 0 && int(nodes[15].count) == 1 && int(nodes[0].count) == N) { 165 | t.Fatalf("items:%v, nodes:%v, N = %d, \n level0:%v \n level1: %v ", items, nodes, N, tree.levels[0], tree.levels[1]) 166 | } 167 | 168 | ki.StringKey = fmt.Sprintf("%016x", keyhash) 169 | ki.Key = []byte(ki.StringKey) 170 | ki.Prepare() 171 | items, nodes = tree.listDir(ki) 172 | if !(len(nodes) == 0 && len(items) == 1) { 173 | t.Fatalf("%s items:%v, nodes:%v", ki.StringKey, items, nodes) 174 | } 175 | return 176 | } 177 | 178 | type HTreeBench struct { 179 | treepos int 180 | itemPerLeaf int // doubandb is 438*1024*1024/(1<<24) = 27 181 | 182 | // runtime 183 | tree *HTree 184 | base uint64 185 | step uint64 186 | req HTreeReq 187 | numLeaf int 188 | 189 | ki *KeyInfo 190 | meta Meta 191 | pos Position 192 | } 193 | 194 | func (hb *HTreeBench) init() { 195 | Conf.InitDefault() 196 | Conf.TreeHeight = *tHeigth 197 | Conf.NumBucket = 1 198 | Conf.Init() 199 | hb.tree = newHTree(Conf.TreeDepth, hb.treepos, Conf.TreeHeight) 200 | hb.base = uint64(0) 201 | hb.step = uint64(1<<(uint32(8-Conf.TreeHeight+1)*4)) << 32 // (0x00000100 << 32) given depthbench = 6 202 | hb.numLeaf = 1 << (4 * (uint32(Conf.TreeHeight))) 203 | hb.ki = NewKeyInfoFromBytes([]byte("key"), 0, false) 204 | hb.meta = Meta{Ver: 1, ValueHash: 255} 205 | hb.pos = Position{0, 0} 206 | } 207 | 208 | func (hb *HTreeBench) setKeysFast() { 209 | base := hb.base 210 | for i := 0; i < hb.numLeaf; i++ { 211 | hb.ki.KeyHash = base 212 | for j := 0; j < hb.itemPerLeaf; j++ { 213 | hb.ki.Prepare() 214 | hb.tree.set(hb.ki, &hb.meta, hb.pos) 215 | hb.ki.KeyHash += 1 216 | } 217 | base += hb.step 218 | } 219 | } 220 | 221 | func (hb *HTreeBench) setKeysSlow() { 222 | base := hb.base 223 | for i := 0; i < hb.itemPerLeaf; i++ { 224 | hb.ki.KeyHash = base 225 | for j := 0; j < hb.numLeaf; j++ { 226 | hb.ki.Prepare() 227 | hb.tree.set(hb.ki, &hb.meta, hb.pos) 228 | hb.ki.KeyHash += hb.step 229 | } 230 | base += 1 231 | } 232 | } 233 | 234 | func (hb *HTreeBench) getKeys() { 235 | base := hb.base 236 | for i := 0; i < hb.itemPerLeaf; i++ { 237 | hb.ki.KeyHash = base 238 | for j := 0; j < hb.numLeaf; j++ { 239 | hb.ki.Prepare() 240 | hb.tree.get(hb.ki) 241 | hb.ki.KeyHash += hb.step 242 | } 243 | base += 1 244 | } 245 | } 246 | 247 | func BenchmarkHTreeSetFastGet(b *testing.B) { 248 | hb := &HTreeBench{ 249 | itemPerLeaf: 30, 250 | } 251 | hb.init() 252 | hb.setKeysFast() 253 | pf := StartCpuProfile("BenchmarkHTreeSetFastGet") 254 | hb.getKeys() 255 | StopCpuProfile(pf) 256 | WriteHeapProfile("BenchmarkHTreeSetFastGet") 257 | } 258 | 259 | func BenchmarkHTreeSetFast(b *testing.B) { 260 | hb := &HTreeBench{ 261 | itemPerLeaf: 30, 262 | } 263 | hb.init() 264 | pf := StartCpuProfile("BenchmarkHTreeSetFast") 265 | hb.setKeysFast() 266 | StopCpuProfile(pf) 267 | WriteHeapProfile("BenchmarkHTreeSetFast") 268 | } 269 | 270 | func BenchmarkHTreeSetSlow(b *testing.B) { 271 | hb := &HTreeBench{ 272 | itemPerLeaf: 30, 273 | } 274 | hb.init() 275 | pf := StartCpuProfile("BenchmarkHTreeSetSlow") 276 | hb.setKeysSlow() 277 | StopCpuProfile(pf) 278 | WriteHeapProfile("BenchmarkHTreeSetSlow") 279 | } 280 | 281 | func tLoadAHint(tree *HTree, r *hintFileReader) (numKey, numAll int, e error) { 282 | meta := Meta{Ver: 1, ValueHash: 255} 283 | var pos Position 284 | for { 285 | item, err := r.next() 286 | if err != nil { 287 | logger.Infof("%s", err) 288 | e = err 289 | return 290 | } 291 | if item == nil { 292 | return 293 | } 294 | numAll++ 295 | if item.Ver > 0 { 296 | ki := NewKeyInfoFromBytes([]byte(item.Key), item.Keyhash, false) 297 | meta.ValueHash = item.Vhash 298 | meta.Ver = item.Ver 299 | pos.Offset = item.Pos.Offset 300 | ki.Prepare() 301 | tree.set(ki, &meta, pos) 302 | numKey++ 303 | if *tKeysPerGC > 0 && numKey%(*tKeysPerGC) == 0 { 304 | FreeMem() 305 | } 306 | } 307 | } 308 | } 309 | 310 | func tSetHTreeFromChan(tree *HTree, khashs chan uint64) { 311 | var ki KeyInfo 312 | meta := Meta{Ver: 1, ValueHash: 255} 313 | var pos Position 314 | var kh uint64 315 | for { 316 | kh = 1 317 | kh = <-khashs 318 | if kh == 0 { 319 | return 320 | } 321 | ki.KeyHash = kh 322 | ki.Prepare() 323 | tree.set(&ki, &meta, pos) 324 | } 325 | } 326 | 327 | func tLoadAHintP(tree *HTree, r *hintFileReader) (numKey, numAll int, e error) { 328 | numKey = 0 329 | N := 1000000 330 | khashs := make(chan uint64, N) 331 | end := false 332 | for !end { 333 | for i := 0; i < N; i++ { 334 | item, err := r.next() 335 | if err != nil { 336 | logger.Infof("%s", err) 337 | e = err 338 | return 339 | } 340 | if item == nil { 341 | end = true 342 | break 343 | } 344 | if item.Ver > 0 && item.Keyhash != 0 { 345 | khashs <- item.Keyhash 346 | numKey++ 347 | } 348 | numAll++ 349 | } 350 | nG := 16 351 | for i := 0; i < nG; i++ { 352 | go tSetHTreeFromChan(tree, khashs) 353 | } 354 | for len(khashs) > 0 { 355 | time.Sleep(100 * time.Microsecond) 356 | } 357 | for i := 0; i < nG; i++ { 358 | khashs <- 0 359 | } 360 | } 361 | return 362 | } 363 | 364 | func TestRebuildHtreeFromHints(b *testing.T) { 365 | if *tDataDir == "" { 366 | return 367 | } 368 | runtime.GOMAXPROCS(8) 369 | 370 | Conf.InitDefault() 371 | Conf.NumBucket = *tNumbucket 372 | Conf.TreeHeight = *tHeigth 373 | Conf.Init() 374 | 375 | pos, err := strconv.ParseInt(*tPos, 16, 32) 376 | if err != nil { 377 | b.Fatalf("%s", err.Error()) 378 | } 379 | 380 | files, _ := filepath.Glob(filepath.Join(*tDataDir, "*"+".hint.s")) 381 | logger.Infof("to load %d files", len(files)) 382 | var pf *os.File 383 | if *tPort == 0 { 384 | pf = StartCpuProfile("BenchmarkLoadHints") 385 | } else { 386 | go func() { 387 | logger.Infof("%v", http.ListenAndServe(fmt.Sprintf("0.0.0.0:%d", *tPort), nil)) 388 | }() 389 | } 390 | 391 | tree := newHTree(Conf.TreeDepth, int(pos), Conf.TreeHeight) 392 | totalNumKey := 0 393 | sort.Sort(sort.StringSlice(files)) 394 | for i, file := range files { 395 | logger.Infof("loading: %s", file) 396 | r := newHintFileReader(file, 0, 1024*1024) 397 | r.open() 398 | numKey := 0 399 | numAll := 0 400 | var e error 401 | if *tParallel == 0 { 402 | numKey, numAll, e = tLoadAHint(tree, r) 403 | } else { 404 | numKey, numAll, e = tLoadAHintP(tree, r) 405 | } 406 | if e != nil { 407 | return 408 | } 409 | totalNumKey += numKey 410 | logger.Infof("%03d: #allkey %d, #key %d, #key_total %d", i, numAll, numKey, totalNumKey) 411 | logger.Infof("%03d: max rss before gc: %d", i, utils.GetMaxRSS()) 412 | r.close() 413 | r = nil 414 | } 415 | 416 | if *tPort == 0 { 417 | StopCpuProfile(pf) 418 | } 419 | ListAll(tree, *tPos+"00000000") 420 | WriteHeapProfile("BenchmarkHints") 421 | if *tPort != 0 { 422 | 423 | for i := 0; ; i++ { 424 | time.Sleep(5 * time.Minute) 425 | FreeMem() 426 | logger.Infof("after %03d minites, max rss = %d", i*5, utils.GetMaxRSS()) 427 | } 428 | } 429 | runtime.GOMAXPROCS(1) 430 | } 431 | 432 | // list bucket[:d], bucket[:d+1], bucket[:d+1].. 433 | func ListAll(tree *HTree, bucket string) { 434 | logger.Infof("list all %s", bucket) 435 | ki := NewKeyInfoFromBytes([]byte(bucket), 0, true) 436 | for i := tree.depth; i < 16; i++ { 437 | ki.StringKey = bucket[:i] 438 | ki.Prepare() 439 | s, err := tree.ListDir(ki) 440 | if err != nil { 441 | logger.Fatalf("list @%s\n%s", ki.StringKey, err) 442 | return 443 | } 444 | if s == nil { 445 | return 446 | } 447 | logger.Infof("@%s\n%sEND\n", ki.StringKey, string(s)) 448 | } 449 | } 450 | -------------------------------------------------------------------------------- /quicklz/quicklz.go: -------------------------------------------------------------------------------- 1 | // Package quicklz implements QuickLZ compress 2 | /* 3 | Translation of http://www.quicklz.com/QuickLZ.java 4 | 5 | Licensed under the GPL, like the original. 6 | */ 7 | package quicklz 8 | 9 | const ( 10 | // Streaming mode not supported 11 | QLZ_STREAMING_BUFFER = 0 12 | 13 | // Bounds checking not supported. Use try...catch instead 14 | QLZ_MEMORY_SAFE = 0 15 | 16 | QLZ_VERSION_MAJOR = 1 17 | QLZ_VERSION_MINOR = 5 18 | QLZ_VERSION_REVISION = 0 19 | 20 | // Decrease QLZ_POINTERS_3 to increase compression speed of level 3. Do not 21 | // edit any other constants! 22 | HASH_VALUES = 4096 23 | MINOFFSET = 2 24 | UNCONDITIONAL_MATCHLEN = 6 25 | UNCOMPRESSED_END = 4 26 | CWORD_LEN = 4 27 | DEFAULT_HEADERLEN = 9 28 | QLZ_POINTERS_1 = 1 29 | QLZ_POINTERS_3 = 16 30 | ) 31 | 32 | func headerLen(source []byte) int { 33 | if (source[0] & 2) == 2 { 34 | return 9 35 | } 36 | return 3 37 | } 38 | 39 | func SizeDecompressed(source []byte) int { 40 | if headerLen(source) == 9 { 41 | return fastRead(source, 5, 4) 42 | } 43 | return fastRead(source, 2, 1) 44 | } 45 | 46 | func SizeCompressed(source []byte) int { 47 | if headerLen(source) == 9 { 48 | return fastRead(source, 1, 4) 49 | } 50 | return fastRead(source, 1, 1) 51 | } 52 | 53 | func fastRead(a []byte, i, numbytes int) int { 54 | l := 0 55 | for j := 0; j < numbytes; j++ { 56 | l |= int(a[i+j]) << (uint(j) * 8) 57 | } 58 | return l 59 | } 60 | 61 | func fastWrite(a []byte, i, value, numbytes int) { 62 | for j := 0; j < numbytes; j++ { 63 | a[i+j] = byte(value >> (uint(j) * 8)) 64 | } 65 | } 66 | 67 | func writeHeader(dst []byte, level int, compressible bool, sizeCompressed int, sizeDecompressed int) { 68 | var cbit byte 69 | if compressible { 70 | cbit = 1 71 | } 72 | dst[0] = byte(2 | cbit) 73 | dst[0] |= byte(level << 2) 74 | dst[0] |= (1 << 6) 75 | dst[0] |= (0 << 4) 76 | fastWrite(dst, 1, sizeDecompressed, 4) 77 | fastWrite(dst, 5, sizeCompressed, 4) 78 | } 79 | 80 | func Compress(source []byte, level int) []byte { 81 | var src int 82 | var dst = DEFAULT_HEADERLEN + CWORD_LEN 83 | var cwordVal uint32 = 0x80000000 84 | var cwordPtr = DEFAULT_HEADERLEN 85 | var destination = make([]byte, len(source)+400) 86 | var hashtable [][]int 87 | var cachetable = make([]int, HASH_VALUES) 88 | var hashCounter = make([]byte, HASH_VALUES) 89 | var d2 []byte 90 | var fetch = 0 91 | var lastMatchStart = (len(source) - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END - 1) 92 | var lits = 0 93 | 94 | if level != 1 && level != 3 { 95 | panic("Go version only supports level 1 and 3") 96 | } 97 | 98 | hashtable = make([][]int, HASH_VALUES) 99 | 100 | hpointers := QLZ_POINTERS_1 101 | if level == 3 { 102 | hpointers = QLZ_POINTERS_3 103 | } 104 | 105 | for i := 0; i < HASH_VALUES; i++ { 106 | hashtable[i] = make([]int, hpointers) 107 | } 108 | 109 | if len(source) == 0 { 110 | return nil 111 | } 112 | 113 | if src <= lastMatchStart { 114 | fetch = fastRead(source, src, 3) 115 | } 116 | 117 | for src <= lastMatchStart { 118 | if (cwordVal & 1) == 1 { 119 | if src > 3*(len(source)>>2) && dst > src-(src>>5) { 120 | d2 = make([]byte, len(source)+DEFAULT_HEADERLEN) 121 | writeHeader(d2, level, false, len(source), len(source)+DEFAULT_HEADERLEN) 122 | copy(d2[DEFAULT_HEADERLEN:], source) 123 | return d2 124 | } 125 | 126 | fastWrite(destination, cwordPtr, int(cwordVal>>1)|0x80000000, 4) 127 | cwordPtr = dst 128 | dst += CWORD_LEN 129 | cwordVal = 0x80000000 130 | } 131 | 132 | if level == 1 { 133 | hash := ((fetch >> 12) ^ fetch) & (HASH_VALUES - 1) 134 | o := hashtable[hash][0] 135 | cache := cachetable[hash] ^ fetch 136 | 137 | cachetable[hash] = fetch 138 | hashtable[hash][0] = src 139 | 140 | if cache == 0 && hashCounter[hash] != 0 && (src-o > MINOFFSET || (src == o+1 && lits >= 3 && src > 3 && source[src] == source[src-3] && source[src] == source[src-2] && source[src] == source[src-1] && source[src] == source[src+1] && source[src] == source[src+2])) { 141 | cwordVal = ((cwordVal >> 1) | 0x80000000) 142 | if source[o+3] != source[src+3] { 143 | f := 3 - 2 | (hash << 4) 144 | destination[dst+0] = byte(f >> (0 * 8)) 145 | destination[dst+1] = byte(f >> (1 * 8)) 146 | src += 3 147 | dst += 2 148 | } else { 149 | oldSrc := src 150 | remaining := 255 151 | if ln := (len(source) - UNCOMPRESSED_END - src + 1 - 1); ln <= 255 { 152 | remaining = ln 153 | } 154 | 155 | src += 4 156 | if source[o+src-oldSrc] == source[src] { 157 | src++ 158 | if source[o+src-oldSrc] == source[src] { 159 | src++ 160 | for source[o+(src-oldSrc)] == source[src] && (src-oldSrc) < remaining { 161 | src++ 162 | } 163 | } 164 | } 165 | 166 | matchlen := src - oldSrc 167 | 168 | hash <<= 4 169 | if matchlen < 18 { 170 | f := hash | (matchlen - 2) 171 | // Inline fastWrite 172 | destination[dst+0] = byte(f >> (0 * 8)) 173 | destination[dst+1] = byte(f >> (1 * 8)) 174 | dst += 2 175 | } else { 176 | f := hash | (matchlen << 16) 177 | fastWrite(destination, dst, f, 3) 178 | dst += 3 179 | } 180 | } 181 | lits = 0 182 | fetch = fastRead(source, src, 3) 183 | } else { 184 | lits++ 185 | hashCounter[hash] = 1 186 | destination[dst] = source[src] 187 | cwordVal = (cwordVal >> 1) 188 | src++ 189 | dst++ 190 | fetch = (fetch>>8)&0xffff | int(source[src+2])<<16 191 | } 192 | } else { 193 | fetch = fastRead(source, src, 3) 194 | 195 | var o, offset2 int 196 | var matchlen, k, m int 197 | var c byte 198 | 199 | remaining := 255 200 | if ln := (len(source) - UNCOMPRESSED_END - src + 1 - 1); ln <= 255 { 201 | remaining = ln 202 | } 203 | 204 | hash := ((fetch >> 12) ^ fetch) & (HASH_VALUES - 1) 205 | 206 | c = hashCounter[hash] 207 | matchlen = 0 208 | offset2 = 0 209 | for k = 0; k < QLZ_POINTERS_3 && (int(c) > k || c < 0); k++ { 210 | 211 | o = hashtable[hash][k] 212 | if byte(fetch) == source[o] && byte(fetch>>8) == source[o+1] && byte(fetch>>16) == source[o+2] && o < src-MINOFFSET { 213 | m = 3 214 | for source[o+m] == source[src+m] && m < remaining { 215 | m++ 216 | } 217 | if (m > matchlen) || (m == matchlen && o > offset2) { 218 | offset2 = o 219 | matchlen = m 220 | } 221 | } 222 | } 223 | 224 | o = offset2 225 | hashtable[hash][c&(QLZ_POINTERS_3-1)] = src 226 | c++ 227 | hashCounter[hash] = c 228 | 229 | if matchlen >= 3 && src-o < 131071 { 230 | offset := src - o 231 | for u := 1; u < matchlen; u++ { 232 | fetch = fastRead(source, src+u, 3) 233 | hash = ((fetch >> 12) ^ fetch) & (HASH_VALUES - 1) 234 | c = hashCounter[hash] 235 | hashCounter[hash]++ 236 | hashtable[hash][c&(QLZ_POINTERS_3-1)] = src + u 237 | } 238 | 239 | src += matchlen 240 | cwordVal = ((cwordVal >> 1) | 0x80000000) 241 | 242 | if matchlen == 3 && offset <= 63 { 243 | fastWrite(destination, dst, offset<<2, 1) 244 | dst++ 245 | } else if matchlen == 3 && offset <= 16383 { 246 | fastWrite(destination, dst, (offset<<2)|1, 2) 247 | dst += 2 248 | } else if matchlen <= 18 && offset <= 1023 { 249 | fastWrite(destination, dst, ((matchlen-3)<<2)|(offset<<6)|2, 2) 250 | dst += 2 251 | } else if matchlen <= 33 { 252 | fastWrite(destination, dst, ((matchlen-2)<<2)|(offset<<7)|3, 3) 253 | dst += 3 254 | } else { 255 | fastWrite(destination, dst, ((matchlen-3)<<7)|(offset<<15)|3, 4) 256 | dst += 4 257 | } 258 | } else { 259 | destination[dst] = source[src] 260 | cwordVal = (cwordVal >> 1) 261 | src++ 262 | dst++ 263 | } 264 | } 265 | } 266 | 267 | for src <= len(source)-1 { 268 | if (cwordVal & 1) == 1 { 269 | fastWrite(destination, cwordPtr, int(cwordVal>>1)|0x80000000, 4) 270 | cwordPtr = dst 271 | dst += CWORD_LEN 272 | cwordVal = 0x80000000 273 | } 274 | 275 | destination[dst] = source[src] 276 | src++ 277 | dst++ 278 | cwordVal = (cwordVal >> 1) 279 | } 280 | for (cwordVal & 1) != 1 { 281 | cwordVal = (cwordVal >> 1) 282 | } 283 | fastWrite(destination, cwordPtr, int(cwordVal>>1)|0x80000000, CWORD_LEN) 284 | writeHeader(destination, level, true, len(source), dst) 285 | 286 | d2 = make([]byte, dst) 287 | copy(d2, destination) 288 | return d2 289 | } 290 | 291 | func Decompress(source []byte) []byte { 292 | size := SizeDecompressed(source) 293 | src := headerLen(source) 294 | var dst int 295 | var cwordVal = 1 296 | destination := make([]byte, size) 297 | hashtable := make([]int, 4096) 298 | hashCounter := make([]byte, 4096) 299 | lastMatchStart := size - UNCONDITIONAL_MATCHLEN - UNCOMPRESSED_END - 1 300 | lastHashed := -1 301 | var hash int 302 | var fetch int 303 | 304 | level := (source[0] >> 2) & 0x3 305 | 306 | if level != 1 && level != 3 { 307 | panic("Go version only supports level 1 and 3") 308 | } 309 | 310 | if (source[0] & 1) != 1 { 311 | d2 := make([]byte, size) 312 | copy(d2, source[headerLen(source):]) 313 | return d2 314 | } 315 | 316 | for { 317 | if cwordVal == 1 { 318 | cwordVal = fastRead(source, src, 4) 319 | src += 4 320 | if dst <= lastMatchStart { 321 | if level == 1 { 322 | fetch = fastRead(source, src, 3) 323 | } else { 324 | fetch = fastRead(source, src, 4) 325 | } 326 | } 327 | } 328 | 329 | if (cwordVal & 1) == 1 { 330 | var matchlen int 331 | var offset2 int 332 | 333 | cwordVal = cwordVal >> 1 334 | 335 | if level == 1 { 336 | hash = (fetch >> 4) & 0xfff 337 | offset2 = hashtable[hash] 338 | 339 | if (fetch & 0xf) != 0 { 340 | matchlen = (fetch & 0xf) + 2 341 | src += 2 342 | } else { 343 | matchlen = int(source[src+2]) & 0xff 344 | src += 3 345 | } 346 | } else { 347 | var offset int 348 | 349 | if (fetch & 3) == 0 { 350 | offset = (fetch & 0xff) >> 2 351 | matchlen = 3 352 | src++ 353 | } else if (fetch & 2) == 0 { 354 | offset = (fetch & 0xffff) >> 2 355 | matchlen = 3 356 | src += 2 357 | } else if (fetch & 1) == 0 { 358 | offset = (fetch & 0xffff) >> 6 359 | matchlen = ((fetch >> 2) & 15) + 3 360 | src += 2 361 | } else if (fetch & 127) != 3 { 362 | offset = (fetch >> 7) & 0x1ffff 363 | matchlen = ((fetch >> 2) & 0x1f) + 2 364 | src += 3 365 | } else { 366 | offset = (fetch >> 15) 367 | matchlen = ((fetch >> 7) & 255) + 3 368 | src += 4 369 | } 370 | offset2 = int(dst - offset) 371 | } 372 | 373 | destination[dst+0] = destination[offset2+0] 374 | destination[dst+1] = destination[offset2+1] 375 | destination[dst+2] = destination[offset2+2] 376 | 377 | for i := 3; i < matchlen; i++ { 378 | destination[dst+i] = destination[offset2+i] 379 | } 380 | dst += matchlen 381 | 382 | if level == 1 { 383 | fetch = fastRead(destination, lastHashed+1, 3) // destination[lastHashed + 1] | (destination[lastHashed + 2] << 8) | (destination[lastHashed + 3] << 16); 384 | for lastHashed < dst-matchlen { 385 | lastHashed++ 386 | hash = ((fetch >> 12) ^ fetch) & (HASH_VALUES - 1) 387 | hashtable[hash] = lastHashed 388 | hashCounter[hash] = 1 389 | fetch = (fetch >> 8 & 0xffff) | (int(destination[lastHashed+3]) << 16) 390 | } 391 | fetch = fastRead(source, src, 3) 392 | } else { 393 | fetch = fastRead(source, src, 4) 394 | } 395 | lastHashed = dst - 1 396 | } else { 397 | if dst <= lastMatchStart { 398 | destination[dst] = source[src] 399 | dst++ 400 | src++ 401 | cwordVal = cwordVal >> 1 402 | 403 | if level == 1 { 404 | for lastHashed < dst-3 { 405 | lastHashed++ 406 | fetch2 := fastRead(destination, lastHashed, 3) 407 | hash = ((fetch2 >> 12) ^ fetch2) & (HASH_VALUES - 1) 408 | hashtable[hash] = lastHashed 409 | hashCounter[hash] = 1 410 | } 411 | fetch = fetch>>8&0xffff | int(source[src+2])<<16 412 | } else { 413 | fetch = fetch>>8&0xffff | int(source[src+2])<<16 | int(source[src+3])<<24 414 | } 415 | } else { 416 | for dst <= size-1 { 417 | if cwordVal == 1 { 418 | src += CWORD_LEN 419 | cwordVal = 0x80000000 420 | } 421 | 422 | destination[dst] = source[src] 423 | dst++ 424 | src++ 425 | cwordVal = cwordVal >> 1 426 | } 427 | return destination 428 | } 429 | } 430 | } 431 | } 432 | --------------------------------------------------------------------------------