├── cache ├── gen_proto.sh ├── index.proto ├── pool_test.go ├── hash_test.go ├── data.go ├── pool.go ├── hash.go ├── cache_test.go ├── shard_test.go ├── data_test.go ├── index_test.go ├── cache.go ├── index.go ├── shard.go └── index.pb.go ├── .gitignore ├── server ├── common.go ├── memcache_test.go ├── common_test.go └── memcache.go ├── protocol └── memcache │ ├── common.go │ ├── parser.go │ └── parser_test.go ├── LICENSE ├── README.md ├── main.go └── tools └── mcstat └── mcstat.go /cache/gen_proto.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | PATH_GOGOPROTOBUF=$GOPATH/src/github.com/gogo/protobuf 3 | protoc --proto_path=$GOPATH/src/:$PATH_GOGOPROTOBUF/protobuf/:. --gogofaster_out=. *.proto 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | # Output of the go coverage tool, specifically when used with LiteIDE 27 | *.out 28 | 29 | # external packages folder 30 | vendor/ 31 | 32 | 33 | # bin 34 | blobcached 35 | tools/mcstat/mcstat 36 | 37 | # default cache path 38 | cachedata/ 39 | -------------------------------------------------------------------------------- /server/common.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "io" 5 | 6 | "github.com/xiaost/blobcached/cache" 7 | ) 8 | 9 | const Version = "1.0" 10 | 11 | type Cache interface { 12 | Set(item *cache.Item) error 13 | Get(key string) (*cache.Item, error) 14 | Del(key string) error 15 | GetOptions() cache.CacheOptions 16 | GetMetrics() cache.CacheMetrics 17 | GetMetricsByShards() []cache.CacheMetrics 18 | GetStats() cache.CacheStats 19 | GetStatsByShards() []cache.CacheStats 20 | } 21 | 22 | type WriterCounter struct { 23 | W io.Writer 24 | N int64 25 | } 26 | 27 | func (w *WriterCounter) Write(p []byte) (int, error) { 28 | n, err := w.W.Write(p) 29 | if n > 0 { 30 | w.N += int64(n) 31 | } 32 | return n, err 33 | } 34 | -------------------------------------------------------------------------------- /cache/index.proto: -------------------------------------------------------------------------------- 1 | import "github.com/gogo/protobuf/gogoproto/gogo.proto"; 2 | 3 | package cache; 4 | 5 | option (gogoproto.goproto_getters_all) = false; 6 | 7 | message IndexMeta { 8 | required int64 Term = 1 [(gogoproto.nullable) = false]; 9 | required int64 Head = 2 [(gogoproto.nullable) = false]; 10 | required int64 DataSize = 3 [(gogoproto.nullable) = false]; 11 | } 12 | 13 | 14 | message IndexItem { 15 | required int64 Term = 1 [(gogoproto.nullable) = false]; 16 | required int64 Offset = 2 [(gogoproto.nullable) = false]; 17 | required int32 ValueSize = 3 [(gogoproto.nullable) = false]; 18 | required int64 Timestamp = 4 [(gogoproto.nullable) = false]; 19 | required uint32 TTL = 5 [(gogoproto.nullable) = false]; 20 | required uint32 Flags = 6 [(gogoproto.nullable) = false]; 21 | optional uint32 Crc32 = 7 [(gogoproto.nullable) = false]; 22 | } 23 | -------------------------------------------------------------------------------- /cache/pool_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "testing" 4 | 5 | func TestAllocatorPool(t *testing.T) { 6 | a := NewAllocatorPool(10).(*AllocatorPool) 7 | n := a.Alloc(1) 8 | n.Free() 9 | if a.metrics.Malloc != 1 { 10 | t.Fatal(a.metrics.Malloc) 11 | } 12 | if a.metrics.New != 1 { 13 | t.Fatal(a.metrics.New) 14 | } 15 | if a.metrics.Free != 1 { 16 | t.Fatal(a.metrics.Malloc) 17 | } 18 | n = a.Alloc(1) 19 | n.Free() 20 | if a.metrics.Malloc != 2 { 21 | t.Fatal(a.metrics.Malloc) 22 | } 23 | if a.metrics.New != 1 { 24 | t.Fatal(a.metrics.New) 25 | } 26 | if a.metrics.Free != 2 { 27 | t.Fatal(a.metrics.Malloc) 28 | } 29 | n = a.Alloc(11) 30 | n.Free() 31 | if a.metrics.Malloc != 3 { 32 | t.Fatal(a.metrics.Malloc) 33 | } 34 | if a.metrics.New != 2 { 35 | t.Fatal(a.metrics.New) 36 | } 37 | if a.metrics.Free != 3 { 38 | t.Fatal(a.metrics.Malloc) 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /protocol/memcache/common.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | var ( 4 | RspErr = []byte("ERROR\r\n") 5 | RspStored = []byte("STORED\r\n") 6 | RspNotStored = []byte("NOT_STORED\r\n") 7 | RspExists = []byte("EXISTS\r\n") 8 | RspNotFound = []byte("NOT_FOUND\r\n") 9 | RspDeleted = []byte("DELETED\r\n") 10 | RspEnd = []byte("END\r\n") 11 | RspTouched = []byte("TOUCHED\r\n") 12 | 13 | EOL = []byte("\r\n") 14 | ) 15 | 16 | func MakeRspClientErr(err error) []byte { 17 | return []byte("CLIENT_ERROR " + err.Error() + "\r\n") 18 | } 19 | 20 | func MakeRspServerErr(err error) []byte { 21 | return []byte("SERVER_ERROR " + err.Error() + "\r\n") 22 | } 23 | 24 | type CommandInfo struct { 25 | Cmd string 26 | Key string 27 | Keys []string // for retrieval commands 28 | Delta uint64 // for incr/decr 29 | Flags uint32 30 | Exptime uint32 31 | PayloadLen int64 32 | CasUnique int64 33 | NoReply bool 34 | } 35 | -------------------------------------------------------------------------------- /cache/hash_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "math/rand" 5 | "strconv" 6 | "testing" 7 | ) 8 | 9 | func TestConsistentHash(t *testing.T) { 10 | n1 := 5 11 | n2 := 16 12 | c1 := NewConsistentHashTable(n1) 13 | c2 := NewConsistentHashTable(n2) 14 | 15 | m1 := make(map[int]int) 16 | total := 100000 17 | match := 0 18 | for i := 0; i < total; i++ { 19 | s := strconv.FormatInt(rand.Int63(), 36) + strconv.FormatInt(rand.Int63(), 36) 20 | h1 := c1.Get(s) 21 | h2 := c2.Get(s) 22 | if h1 == h2 { 23 | match += 1 24 | } 25 | m1[h1] += 1 26 | } 27 | 28 | for i := 0; i < n1; i++ { 29 | diff := 0 30 | for j := 0; j < n1; j++ { 31 | diff += m1[i] - m1[j] 32 | } 33 | if diff > int(float32(total/n1)*0.1) { 34 | t.Fatal("total", total, "slot", i, "num", m1[i], "diff err") 35 | } 36 | } 37 | 38 | matchRate := float32(match) / float32(total) 39 | expectRate := float32(n1) / float32(n2) 40 | if matchRate < expectRate*0.9 || matchRate > expectRate*1.1 { 41 | t.Fatal("match rate err", matchRate, "expect", expectRate) 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Blobcached 2 | ===== 3 | Blobcached is a memcached protocol-compatible cache server for blob on SSD. 4 | 5 | ### Supported commands 6 | | Command | Format | 7 | | ------ | ------ | 8 | | get | get []+\r\n | 9 | | set | set [noreply]\r\n\r\n | 10 | | delete | delete [noreply]\r\n | 11 | | touch | touch [noreply]\r\n | 12 | | stats | stats\r\n | 13 | 14 | ### How it works 15 | #### concepts 16 | | Name | | 17 | | ------ | ------ | 18 | | indexfile | an indexfile contains many of `items` powered by [blotdb](https://github.com/boltdb/bolt) | 19 | | datafile | a regular file for storing values | 20 | | item | an item is made up of `key`, `offset`, `term`, `size` anchoring the value in datafile | 21 | | term | everytime the `datafile` is full, the `term` of `datafile` is increased | 22 | 23 | #### Command: Set 24 | * get the `offset` and `term` of `datafile` 25 | * write value to the `datafile` 26 | * write `item` with the `offset`, `term` and `key` to the `indexfile` 27 | 28 | #### Command: Get 29 | * get the `item` by `key` 30 | * check `term` and `offset` of the `item` against `datafile` 31 | * read value from the `datafile` 32 | 33 | #### Command: Touch 34 | * implemented by `get` & `set` 35 | 36 | #### GC 37 | * Blobcached scans and removes expired or invalid `items` in the `indexfile` 38 | * by default, the rate up to 32k items/second 39 | -------------------------------------------------------------------------------- /cache/data.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | "syscall" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | var ( 12 | ErrOutOfRange = errors.New("out of data range") 13 | ) 14 | 15 | type CacheData struct { 16 | mu sync.RWMutex 17 | f *os.File 18 | sz int64 19 | } 20 | 21 | func LoadCacheData(fn string, sz int64) (*CacheData, error) { 22 | f, err := os.OpenFile(fn, os.O_RDWR|os.O_CREATE, 0600) 23 | if err != nil { 24 | return nil, errors.Wrap(err, "os.OpenFile") 25 | } 26 | if err := f.Truncate(sz); err != nil { 27 | return nil, errors.Wrap(err, "truncate data") 28 | } 29 | // prealloc blocks in disk, ok if it have any errors 30 | syscall.Fallocate(int(f.Fd()), 0, 0, sz) 31 | return &CacheData{f: f, sz: sz}, nil 32 | } 33 | 34 | func (d *CacheData) Close() error { 35 | d.mu.Lock() 36 | defer d.mu.Unlock() 37 | return d.f.Close() 38 | } 39 | 40 | func (d *CacheData) Size() int64 { 41 | return d.sz 42 | } 43 | 44 | func (d *CacheData) Read(offset int64, b []byte) error { 45 | d.mu.RLock() 46 | defer d.mu.RUnlock() 47 | if offset+int64(len(b)) > d.sz { 48 | return ErrOutOfRange 49 | } 50 | // ReadAt always returns a non-nil error when n < len(b) 51 | _, err := d.f.ReadAt(b, offset) 52 | return err 53 | } 54 | 55 | func (d *CacheData) Write(offset int64, b []byte) error { 56 | d.mu.Lock() 57 | defer d.mu.Unlock() 58 | if offset+int64(len(b)) > d.sz { 59 | return ErrOutOfRange 60 | } 61 | _, err := d.f.WriteAt(b, offset) 62 | return err 63 | } 64 | -------------------------------------------------------------------------------- /cache/pool.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | ) 7 | 8 | type AllocatorPoolMetrics struct { 9 | Malloc int64 10 | Free int64 11 | New int64 12 | 13 | ErrMalloc int64 14 | ErrFree int64 15 | } 16 | 17 | type AllocatorPool struct { 18 | pool sync.Pool 19 | metrics AllocatorPoolMetrics 20 | } 21 | 22 | func NewAllocatorPool(bufsize int) Allocator { 23 | p := AllocatorPool{} 24 | p.pool.New = func() interface{} { 25 | atomic.AddInt64(&p.metrics.New, 1) 26 | return &Item{Value: make([]byte, 0, bufsize)} 27 | } 28 | return &p 29 | } 30 | 31 | func (p *AllocatorPool) Alloc(n int) *Item { 32 | atomic.AddInt64(&p.metrics.Malloc, 1) 33 | item := p.pool.Get().(*Item) 34 | if cap(item.Value) < n { 35 | p.pool.Put(item) 36 | atomic.AddInt64(&p.metrics.New, 1) 37 | item = &Item{Value: make([]byte, 0, n)} 38 | } 39 | item.free = p.Free 40 | item.Key = "" 41 | item.Value = item.Value[:n] 42 | item.Timestamp = 0 43 | item.TTL = 0 44 | item.Flags = 0 45 | return item 46 | } 47 | 48 | func (p *AllocatorPool) Free(i *Item) { 49 | atomic.AddInt64(&p.metrics.Free, 1) 50 | p.pool.Put(i) 51 | } 52 | 53 | func (p *AllocatorPool) GetMetrics() AllocatorPoolMetrics { 54 | var ret AllocatorPoolMetrics 55 | ret.Malloc = atomic.LoadInt64(&p.metrics.Malloc) 56 | ret.Free = atomic.LoadInt64(&p.metrics.Free) 57 | ret.New = atomic.LoadInt64(&p.metrics.New) 58 | ret.ErrMalloc = atomic.LoadInt64(&p.metrics.ErrMalloc) 59 | ret.ErrFree = atomic.LoadInt64(&p.metrics.ErrFree) 60 | return ret 61 | } 62 | -------------------------------------------------------------------------------- /cache/hash.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "crypto/md5" 5 | "encoding/binary" 6 | "fmt" 7 | "sort" 8 | ) 9 | 10 | const ( 11 | vhashnodes = 1000 12 | ) 13 | 14 | type consistentNode struct { 15 | pos uint64 16 | v int 17 | } 18 | 19 | type consistentNodes []consistentNode 20 | 21 | func (n consistentNodes) Len() int { return len(n) } 22 | func (n consistentNodes) Swap(i, j int) { n[i], n[j] = n[j], n[i] } 23 | func (n consistentNodes) Less(i, j int) bool { return n[i].pos < n[j].pos } 24 | 25 | // a simplify consistent hash impelment without weight 26 | type ConsistentHash struct { 27 | nodes consistentNodes 28 | } 29 | 30 | // from https://golang.org/src/hash/fnv/fnv.go 31 | // Equals to: 32 | // h := fnv.New64a() 33 | // h.Write(b) 34 | // h.Sum64 35 | func dohash(b []byte) uint64 { 36 | hash := uint64(14695981039346656037) 37 | for _, c := range b { 38 | hash ^= uint64(c) 39 | hash *= 1099511628211 40 | } 41 | return hash 42 | } 43 | 44 | // NewConsistentHashTable creates a ConsistentHash with value[0, n) 45 | func NewConsistentHashTable(n int) ConsistentHash { 46 | var h ConsistentHash 47 | h.nodes = make(consistentNodes, 0, n*vhashnodes) 48 | for i := 0; i < n; i++ { 49 | for j := 0; j < vhashnodes; j++ { 50 | // we use md5 distribute vnodes 51 | b := md5.Sum([]byte(fmt.Sprintf("hash-%d-%d", i, j))) 52 | h.nodes = append(h.nodes, consistentNode{binary.BigEndian.Uint64(b[:]), i}) 53 | } 54 | } 55 | sort.Sort(h.nodes) 56 | return h 57 | } 58 | 59 | func (c *ConsistentHash) search(k uint64) int { 60 | i := sort.Search(len(c.nodes), func(i int) bool { 61 | return c.nodes[i].pos >= k 62 | }) 63 | if i == len(c.nodes) { 64 | return c.nodes[0].v 65 | } 66 | return c.nodes[i].v 67 | } 68 | 69 | func (c *ConsistentHash) Get(key string) int { 70 | h := dohash([]byte(key)) 71 | return c.search(h) 72 | } 73 | -------------------------------------------------------------------------------- /cache/cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "io/ioutil" 6 | "math/rand" 7 | "os" 8 | "strconv" 9 | "testing" 10 | ) 11 | 12 | func TestCache(t *testing.T) { 13 | dir, err := ioutil.TempDir("", "test_cache") 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | defer os.RemoveAll(dir) 18 | c, err := NewCache(dir, nil) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | b := make([]byte, 100) 23 | 24 | n := 1000 25 | for i := 0; i < n; i++ { 26 | key := strconv.FormatInt(rand.Int63(), 16) 27 | rand.Read(b) 28 | if err := c.Set(&Item{Key: key, Value: b}); err != nil { 29 | t.Fatal(err) 30 | } 31 | item, err := c.Get(key) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | if !bytes.Equal(item.Value, b) { 36 | t.Fatal("set get not equal") 37 | } 38 | if err := c.Del(key); err != nil { 39 | t.Fatal(err) 40 | } 41 | if _, err := c.Get(key); err != ErrNotFound { 42 | t.Fatal("should not found") 43 | } 44 | } 45 | m := c.GetMetrics() 46 | 47 | k := int64(n) 48 | if m.GetTotal != 2*k || m.DelTotal != k || m.SetTotal != k || m.GetMisses != k || m.GetHits != k { 49 | t.Fatal("metrics err", m) 50 | } 51 | if err := c.Close(); err != nil { 52 | t.Fatal(err) 53 | } 54 | } 55 | 56 | func benchmarkCacheSet(b *testing.B, n int) { 57 | dir, err := ioutil.TempDir("", "blobcached_BenchmarkCacheSet") 58 | if err != nil { 59 | b.Fatal(err) 60 | } 61 | defer os.RemoveAll(dir) 62 | opt := &CacheOptions{ 63 | ShardNum: 1, 64 | Size: 32 << 20, 65 | Allocator: NewAllocatorPool(n), 66 | DisableGC: true, 67 | } 68 | cache, err := NewCache(dir, opt) 69 | if err != nil { 70 | b.Fatal(err) 71 | } 72 | b.ReportAllocs() 73 | for i := 0; i < b.N; i++ { 74 | item := opt.Allocator.Alloc(n) 75 | cache.Set(item) 76 | item.Free() 77 | } 78 | } 79 | 80 | func BenchmarkCacheSet4K(b *testing.B) { 81 | benchmarkCacheSet(b, 4096) 82 | } 83 | -------------------------------------------------------------------------------- /cache/shard_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | func TestCacheMetrics(t *testing.T) { 14 | m1 := CacheMetrics{} 15 | m2 := CacheMetrics{1, 1, 1, 1, 1, 1, 1, 1, 1} 16 | m1.Add(m2) 17 | if m1 != m2 { 18 | t.Fatal("not equal", m1, m2) 19 | } 20 | } 21 | 22 | func TestShardSetGet(t *testing.T) { 23 | dir, err := ioutil.TempDir("", "test_cachedata") 24 | if err != nil { 25 | t.Fatal(err) 26 | } 27 | defer os.RemoveAll(dir) 28 | s, err := LoadCacheShard(filepath.Join(dir, "shard"), &ShardOptions{Size: 1024, TTL: 1}) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | b := make([]byte, 300) 33 | rand.Read(b) 34 | 35 | _, err = s.Get("k1") 36 | if err != ErrNotFound { 37 | t.Fatal("should not found") 38 | } 39 | if err := s.Set(&Item{Key: "k1", Value: b}); err != nil { 40 | t.Fatal(err) 41 | } 42 | ci, err := s.Get("k1") 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | if ci.Key != "k1" { 47 | t.Fatal("key not equal") 48 | } 49 | if !bytes.Equal(ci.Value, b) { 50 | t.Fatal("set get not equal") 51 | } 52 | 53 | var st gcstat 54 | 55 | if testing.Short() { 56 | goto EndOfTest 57 | } 58 | 59 | s.Set(&Item{Key: "k2", Value: b}) 60 | s.Set(&Item{Key: "k3", Value: b}) 61 | s.Set(&Item{Key: "k4", Value: b}) 62 | 63 | st.LastKey = "" 64 | s.scanKeysForGC(100, &st) 65 | 66 | time.Sleep(1020 * time.Millisecond) 67 | 68 | st.LastKey = "" 69 | s.scanKeysForGC(100, &st) 70 | 71 | for _, key := range []string{"k1", "k2", "k3", "k4"} { 72 | if _, err := s.Get(key); err != ErrNotFound { 73 | t.Fatal("should not found") 74 | } 75 | } 76 | { 77 | m1 := s.GetMetrics() 78 | m2 := CacheMetrics{6, 1, 5, 0, 4, 0, 3, 1, 0} 79 | if m1 != m2 { 80 | t.Logf("\nget %+v\nexpect %+v", m1, m2) 81 | t.Fatal("metrics err") 82 | } 83 | } 84 | 85 | EndOfTest: 86 | if err := s.Close(); err != nil { 87 | t.Fatal(err) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /cache/data_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | "io/ioutil" 7 | "os" 8 | "path/filepath" 9 | "testing" 10 | ) 11 | 12 | func TestCacheData(t *testing.T) { 13 | dir, err := ioutil.TempDir("", "test_cachedata") 14 | if err != nil { 15 | t.Fatal(err) 16 | } 17 | defer os.RemoveAll(dir) 18 | fn := filepath.Join(dir, "data") 19 | c, err := LoadCacheData(fn, 1024) 20 | if err != nil { 21 | t.Fatal(err) 22 | } 23 | if c.Size() != 1024 { 24 | t.Fatal("size err") 25 | } 26 | b := make([]byte, 8) 27 | rand.Read(b) 28 | if err := c.Write(1000, b); err != nil { 29 | t.Fatal(err) 30 | } 31 | 32 | bb := make([]byte, 8) 33 | if err := c.Read(1000, bb); err != nil { 34 | t.Fatal(err) 35 | } 36 | if !bytes.Equal(bb, b) { 37 | t.Fatal("bytes not equal") 38 | } 39 | if err := c.Close(); err != nil { 40 | t.Fatal(err) 41 | } 42 | } 43 | 44 | func TestCacheDataReadErr(t *testing.T) { 45 | dir, err := ioutil.TempDir("", "test_cachedata") 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | defer os.RemoveAll(dir) 50 | fn := filepath.Join(dir, "data") 51 | c, err := LoadCacheData(fn, 1024) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | 56 | bb := make([]byte, 8) 57 | if err := c.Read(1024, bb); err != ErrOutOfRange { 58 | t.Fatal("should out of range") 59 | } 60 | if err := c.Read(1023, bb); err != ErrOutOfRange { 61 | t.Fatal("should out of range") 62 | } 63 | if err := c.Close(); err != nil { 64 | t.Fatal(err) 65 | } 66 | } 67 | func TestCacheDataWriteErr(t *testing.T) { 68 | dir, err := ioutil.TempDir("", "test_cachedata") 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | defer os.RemoveAll(dir) 73 | fn := filepath.Join(dir, "data") 74 | c, err := LoadCacheData(fn, 1024) 75 | if err != nil { 76 | t.Fatal(err) 77 | } 78 | b := make([]byte, 8) 79 | rand.Read(b) 80 | if err := c.Write(1017, b); err != ErrOutOfRange { 81 | t.Fatal("should out of range") 82 | } 83 | if err := c.Close(); err != nil { 84 | t.Fatal(err) 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /server/memcache_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net" 7 | "strings" 8 | "testing" 9 | 10 | "github.com/bradfitz/gomemcache/memcache" 11 | ) 12 | 13 | func getstat(lines string, name string, value interface{}) { 14 | for _, line := range strings.Split(string(lines), "\r\n") { 15 | if !strings.Contains(line, name) { 16 | continue 17 | } 18 | fmt.Sscanf(line, "STAT "+name+" %v", value) 19 | return 20 | } 21 | } 22 | 23 | func TestMemcacheServer(t *testing.T) { 24 | l, err := net.ListenTCP("tcp", nil) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | defer l.Close() 29 | 30 | s := NewMemcacheServer(l, NewInMemoryCache(), NewDebugAllocator()) 31 | go s.Serv() 32 | 33 | mc := memcache.New(l.Addr().String()) 34 | if err := mc.Set(&memcache.Item{Key: "k1", Value: []byte("v1")}); err != nil { 35 | t.Fatal(err) 36 | } 37 | if err := mc.Set(&memcache.Item{Key: "k2", Value: []byte("v2")}); err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | if err := mc.Touch("k2", 3); err != nil { 42 | t.Fatal(err) 43 | } 44 | 45 | if err := mc.Touch("k-notfound", 4); err != memcache.ErrCacheMiss { 46 | t.Fatal("should not found") 47 | } 48 | 49 | m, err := mc.GetMulti([]string{"k1", "k2", "k3"}) 50 | if err != nil { 51 | t.Fatal(err) 52 | } 53 | 54 | if g, e := len(m), 2; g != e { 55 | t.Fatalf("GetMulti: got len(map) = %d, want = %d", g, e) 56 | } 57 | if _, ok := m["k1"]; !ok { 58 | t.Fatal("GetMulti: didn't get key 'k1'") 59 | } 60 | if _, ok := m["k2"]; !ok { 61 | t.Fatal("GetMulti: didn't get key 'k2'") 62 | } 63 | if g, e := string(m["k1"].Value), "v1"; g != e { 64 | t.Errorf("GetMulti: k1: got %q, want %q", g, e) 65 | } 66 | if g, e := string(m["k2"].Value), "v2"; g != e { 67 | t.Errorf("GetMulti: k2: got %q, want %q", g, e) 68 | } 69 | 70 | if err := mc.Delete("k1"); err != nil { 71 | t.Fatal(err) 72 | } 73 | 74 | if _, err := mc.Get("k1"); err != memcache.ErrCacheMiss { 75 | t.Fatal("should cache miss") 76 | } 77 | 78 | var buf bytes.Buffer 79 | s.HandleStats(&buf) 80 | t.Log("stat\n", buf.String()) 81 | 82 | var n int 83 | getstat(buf.String(), "curr_connections", &n) 84 | if n != 1 { 85 | t.Fatal("stat err:", buf.String()) 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /server/common_test.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | "unsafe" 7 | 8 | "blobcached/cache" 9 | ) 10 | 11 | type InMemoryCache struct { 12 | m map[string]cache.Item 13 | 14 | options cache.CacheOptions 15 | 16 | stats cache.CacheStats 17 | metrics cache.CacheMetrics 18 | } 19 | 20 | func NewInMemoryCache() *InMemoryCache { 21 | c := &InMemoryCache{} 22 | c.m = make(map[string]cache.Item) 23 | c.options.Allocator = cache.NewAllocatorPool() 24 | return c 25 | } 26 | 27 | func (c *InMemoryCache) Set(item cache.Item) error { 28 | c.metrics.SetTotal += 1 29 | item.Timestamp = time.Now().Unix() 30 | c.m[item.Key] = item 31 | c.updateStats() 32 | return nil 33 | } 34 | 35 | func (c *InMemoryCache) Get(key string) (cache.Item, error) { 36 | c.metrics.GetTotal += 1 37 | item, ok := c.m[key] 38 | if ok { 39 | c.metrics.GetHits += 1 40 | return item, nil 41 | } 42 | c.metrics.GetMisses += 1 43 | return item, cache.ErrNotFound 44 | } 45 | 46 | func (c *InMemoryCache) Del(key string) error { 47 | c.metrics.DelTotal += 1 48 | delete(c.m, key) 49 | c.updateStats() 50 | return nil 51 | } 52 | 53 | func (c *InMemoryCache) updateStats() { 54 | c.stats.Keys = uint64(len(c.m)) 55 | c.stats.Bytes = 0 56 | for _, it := range c.m { 57 | c.stats.Bytes += uint64(len(it.Key) + len(it.Value)) 58 | } 59 | c.stats.LastUpdate = time.Now().Unix() 60 | } 61 | 62 | func (c *InMemoryCache) GetOptions() cache.CacheOptions { 63 | return c.options 64 | } 65 | 66 | func (c *InMemoryCache) GetMetrics() cache.CacheMetrics { 67 | return c.metrics 68 | } 69 | 70 | func (c *InMemoryCache) GetMetricsByShards() []cache.CacheMetrics { 71 | return []cache.CacheMetrics{c.metrics} 72 | } 73 | 74 | func (c *InMemoryCache) GetStats() cache.CacheStats { 75 | return c.stats 76 | } 77 | 78 | func (c *InMemoryCache) GetStatsByShards() []cache.CacheStats { 79 | return []cache.CacheStats{c.stats} 80 | } 81 | 82 | type DebugAllocator struct { 83 | mu sync.Mutex 84 | all map[unsafe.Pointer]bool 85 | } 86 | 87 | func NewDebugAllocator() *DebugAllocator { 88 | return &DebugAllocator{all: make(map[unsafe.Pointer]bool)} 89 | } 90 | 91 | func (a *DebugAllocator) Free(b []byte) { 92 | a.mu.Lock() 93 | defer a.mu.Unlock() 94 | p := unsafe.Pointer(&b[0]) 95 | if !a.all[p] { 96 | panic(p) 97 | } 98 | } 99 | 100 | func (a *DebugAllocator) Malloc(n int) []byte { 101 | a.mu.Lock() 102 | defer a.mu.Unlock() 103 | b := make([]byte, n) 104 | p := unsafe.Pointer(&b[0]) 105 | a.all[p] = true 106 | return b 107 | } 108 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "net" 8 | 9 | "github.com/xiaost/blobcached/cache" 10 | "github.com/xiaost/blobcached/server" 11 | ) 12 | 13 | const VERSION = "0.1.1-dev" 14 | 15 | func main() { 16 | var ( 17 | bindAddr string 18 | cachePath string 19 | cacheSize int64 20 | cacheShards int64 21 | cacheTTL int64 22 | bufsize int 23 | 24 | printVersion bool 25 | ) 26 | 27 | flag.StringVar(&bindAddr, 28 | "addr", ":11211", 29 | "the addr that blobcached listen on.") 30 | 31 | flag.StringVar(&cachePath, 32 | "path", "cachedata", 33 | "the cache path used by blobcached to store items.") 34 | 35 | flag.Int64Var(&cacheSize, 36 | "size", int64(cache.DefualtCacheOptions.Size), 37 | "cache file size used by blobcached to store items. ") 38 | 39 | flag.Int64Var(&cacheShards, 40 | "shards", int64(cache.DefualtCacheOptions.ShardNum), 41 | "cache shards for performance purpose. max shards is 128.") 42 | 43 | flag.Int64Var(&cacheTTL, 44 | "ttl", 0, 45 | "the global ttl of cache items.") 46 | 47 | flag.IntVar(&bufsize, 48 | "buf", 4096, 49 | "default buffer size used by get/set.") 50 | 51 | flag.BoolVar(&printVersion, "v", false, 52 | "print the version and exit") 53 | 54 | flag.Parse() 55 | 56 | if printVersion { 57 | fmt.Println("Blobcached", VERSION) 58 | return 59 | } 60 | 61 | l, err := net.Listen("tcp", bindAddr) 62 | if err != nil { 63 | log.Fatal(err) 64 | } 65 | 66 | // cacheSize: limit to [2*MaxValueSize, +) 67 | if cacheSize <= 2*cache.MaxValueSize { 68 | cacheSize = 2 * cache.MaxValueSize 69 | log.Printf("warn: cache size invaild. set to %d", cacheSize) 70 | } 71 | 72 | // cacheShards: limit to (0, cache.MaxShards] and per cache shard size >= cache.MaxValueSize 73 | if (cacheShards <= 0 || cacheShards > cache.MaxShards) || 74 | (cacheSize/cacheShards < cache.MaxValueSize) { 75 | cacheShards = int64(cache.DefualtCacheOptions.ShardNum) 76 | if cacheSize/cacheShards < cache.MaxValueSize { 77 | cacheShards = cacheSize / cache.MaxValueSize 78 | } 79 | log.Printf("warn: cache shards invaild. set to %d", cacheShards) 80 | } 81 | 82 | if cacheTTL < 0 { 83 | cacheTTL = 0 84 | } 85 | 86 | allocator := cache.NewAllocatorPool(bufsize) 87 | 88 | options := &cache.CacheOptions{ 89 | ShardNum: int(cacheShards), 90 | Size: cacheSize, 91 | TTL: cacheTTL, 92 | Allocator: allocator, 93 | } 94 | c, err := cache.NewCache(cachePath, options) 95 | if err != nil { 96 | log.Fatal(err) 97 | } 98 | s := server.NewMemcacheServer(l, c, allocator) 99 | log.Fatal(s.Serv()) 100 | } 101 | -------------------------------------------------------------------------------- /tools/mcstat/mcstat.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "log" 7 | "net" 8 | "os" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | func PrintUsageAndExit() { 15 | fmt.Printf("%s addr [interval]\n", os.Args[0]) 16 | os.Exit(1) 17 | } 18 | 19 | type Stats struct { 20 | CmdSet int64 21 | CmdGet int64 22 | GetHits int64 23 | Rx int64 24 | Tx int64 25 | Keys int64 26 | 27 | Bytes int64 28 | MaxBytes int64 29 | } 30 | 31 | func (s Stats) Sub(o Stats) Stats { 32 | s.CmdSet -= o.CmdSet 33 | s.CmdGet -= o.CmdGet 34 | s.GetHits -= o.GetHits 35 | s.Rx -= o.Rx 36 | s.Tx -= o.Tx 37 | // st.Bytes 38 | // st.MaxBytes 39 | return s 40 | } 41 | 42 | func ParseStats(r *bufio.Reader) (Stats, error) { 43 | var st Stats 44 | mapping := map[string]interface{}{ 45 | "cmd_set": &st.CmdSet, 46 | "cmd_get": &st.CmdGet, 47 | "get_hits": &st.GetHits, 48 | "bytes_read": &st.Rx, 49 | "bytes_written": &st.Tx, 50 | "curr_items": &st.Keys, 51 | "bytes": &st.Bytes, 52 | "limit_maxbytes": &st.MaxBytes, 53 | } 54 | for { 55 | line, err := r.ReadString('\n') 56 | if err != nil { 57 | return st, err 58 | } 59 | line = line[:len(line)-2] // remove \r\n 60 | if line == "END" { 61 | break 62 | } 63 | for name, v := range mapping { 64 | if !strings.Contains(line, name) { 65 | continue 66 | } 67 | fmt.Sscanf(line, "STAT "+name+" %v", v) 68 | } 69 | } 70 | return st, nil 71 | } 72 | 73 | func main() { 74 | if len(os.Args) != 2 && len(os.Args) != 3 { 75 | PrintUsageAndExit() 76 | } 77 | 78 | addr := os.Args[1] 79 | 80 | var interval int 81 | if len(os.Args) == 3 { 82 | interval, _ = strconv.Atoi(os.Args[2]) 83 | } 84 | if interval <= 0 { 85 | interval = 1 86 | } 87 | 88 | if _, _, err := net.SplitHostPort(addr); err != nil { 89 | addr = net.JoinHostPort(addr, "11211") 90 | } 91 | 92 | conn, err := net.Dial("tcp", addr) 93 | if err != nil { 94 | log.Fatal(err) 95 | } 96 | r := bufio.NewReader(conn) 97 | 98 | var st0 Stats 99 | for i := 0; ; i++ { 100 | conn.SetDeadline(time.Now().Add(10 * time.Second)) 101 | _, err := conn.Write([]byte("stats\r\n")) 102 | if err != nil { 103 | log.Fatal(err) 104 | } 105 | st1, err := ParseStats(r) 106 | if err != nil { 107 | log.Fatal(err) 108 | } 109 | if i == 0 { 110 | st0 = st1 111 | time.Sleep(time.Duration(interval) * time.Second) 112 | continue 113 | } 114 | diff := st1.Sub(st0) 115 | if (i-1)%50 == 0 { 116 | fmt.Printf("%8s %8s %8s %6s %10s %10s %10s %18s\n", 117 | "SET", "GET", "HITS", "HITR", "KEYS", "RX", "TX", "USAGE") 118 | } 119 | fmt.Printf("%8s %8s %8s %6.2f %10d %10s %10s %18s\n", 120 | readableNum(diff.CmdSet), readableNum(diff.CmdGet), 121 | readableNum(diff.GetHits), float32(diff.GetHits)/float32(diff.CmdGet), st1.Keys, 122 | readableSize(diff.Rx), readableSize(diff.Tx), 123 | readableSize(st1.Bytes)+"/"+readableSize(st1.MaxBytes)) 124 | st0 = st1 125 | time.Sleep(time.Duration(interval) * time.Second) 126 | } 127 | } 128 | 129 | func readableNum(i int64) string { 130 | if i < (1 << 10) { 131 | return fmt.Sprintf("%d", i) 132 | } 133 | if i < (1 << 20) { 134 | return fmt.Sprintf("%.2fK", float32(i)/(1<<10)) 135 | } 136 | if i < (1 << 30) { 137 | return fmt.Sprintf("%.2fM", float32(i)/(1<<20)) 138 | } 139 | return fmt.Sprintf("%.2fG", float32(i)/(1<<30)) 140 | } 141 | 142 | func readableSize(i int64) string { 143 | return readableNum(i) + "B" 144 | } 145 | -------------------------------------------------------------------------------- /protocol/memcache/parser.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | ) 8 | 9 | var ( 10 | ErrNeedMoreData = errors.New("need more data") 11 | 12 | errCommand = errors.New("command err") 13 | ) 14 | 15 | // ParseCommand parses cmd from `data` and return CommandInfo 16 | // return ErrNeedMoreData if data not contains '\n' 17 | // implements: https://github.com/memcached/memcached/blob/master/doc/protocol.txt 18 | func ParseCommand(data []byte) (advance int, cmdinfo *CommandInfo, err error) { 19 | idx := bytes.IndexByte(data, '\n') 20 | if idx < 0 { 21 | return 0, nil, ErrNeedMoreData 22 | } 23 | advance = idx + 1 24 | left := bytes.TrimSpace(data[:advance]) 25 | if len(left) == 0 { 26 | return advance, nil, errCommand 27 | } 28 | 29 | var cmd string 30 | idx = bytes.IndexByte(left, ' ') 31 | if idx < 0 { // single cmd 32 | cmd = string(left) 33 | left = left[0:0] 34 | } else { 35 | cmd = string(left[:idx]) 36 | left = left[idx+1:] // remove cmd 37 | } 38 | 39 | var parser func(cmd string, left []byte) (*CommandInfo, error) 40 | switch cmd { 41 | case "set", "add", "replace", "append", "prepend", "cas": 42 | parser = parseStorageCommands 43 | case "get", "gets": 44 | parser = parseRetrievalCommands 45 | case "delete": 46 | parser = parseDeleteCommand 47 | case "incr", "decr": 48 | parser = parseIncrDecrCommands 49 | case "touch": 50 | parser = parseTouchCommand 51 | default: 52 | parser = parseOtherCommands 53 | } 54 | cmdinfo, err = parser(cmd, left) 55 | return advance, cmdinfo, err 56 | } 57 | 58 | var ( 59 | norepl = []byte("noreply") 60 | ) 61 | 62 | // parse: 63 | // [noreply] 64 | // cas [noreply] 65 | func parseStorageCommands(cmd string, line []byte) (*CommandInfo, error) { 66 | c := CommandInfo{Cmd: cmd} 67 | 68 | if cmd != "cas" { 69 | n, _ := fmt.Sscanf(string(line), "%s %d %d %d", 70 | &c.Key, &c.Flags, &c.Exptime, &c.PayloadLen) 71 | if n != 4 { 72 | return nil, errCommand 73 | } 74 | } else { 75 | n, _ := fmt.Sscanf(string(line), "%s %d %d %d %d", 76 | &c.Key, &c.Flags, &c.Exptime, &c.PayloadLen, &c.CasUnique) 77 | if n != 5 { 78 | return nil, errCommand 79 | } 80 | } 81 | if bytes.HasSuffix(line, norepl) { 82 | c.NoReply = true 83 | } 84 | return &c, nil 85 | } 86 | 87 | // parse: 88 | // get * 89 | // gets * 90 | func parseRetrievalCommands(cmd string, line []byte) (*CommandInfo, error) { 91 | if len(line) == 0 { 92 | return nil, errCommand 93 | } 94 | bb := bytes.Split(line, []byte(" ")) 95 | c := CommandInfo{Cmd: cmd} 96 | c.Keys = make([]string, 0, len(bb)) 97 | for _, b := range bb { 98 | c.Keys = append(c.Keys, string(b)) 99 | } 100 | c.Key = c.Keys[0] 101 | return &c, nil 102 | } 103 | 104 | // parse: 105 | // delete [noreply] 106 | func parseDeleteCommand(cmd string, line []byte) (*CommandInfo, error) { 107 | c := CommandInfo{Cmd: cmd} 108 | bb := bytes.Split(line, []byte(" ")) 109 | c.Key = string(bb[0]) 110 | if len(bb) == 1 { 111 | return &c, nil 112 | } 113 | if len(bb) != 2 || !bytes.Equal(bb[1], norepl) { 114 | return nil, errCommand 115 | } 116 | c.NoReply = true 117 | return &c, nil 118 | } 119 | 120 | // parse: 121 | // incr [noreply] 122 | // decr [noreply] 123 | func parseIncrDecrCommands(cmd string, line []byte) (*CommandInfo, error) { 124 | c := CommandInfo{Cmd: cmd} 125 | n, _ := fmt.Sscanf(string(line), "%s %d", &c.Key, &c.Delta) 126 | if n != 2 { 127 | return nil, errCommand 128 | } 129 | if bytes.HasSuffix(line, norepl) { 130 | c.NoReply = true 131 | } 132 | return &c, nil 133 | } 134 | 135 | // parse: 136 | // touch [noreply] 137 | func parseTouchCommand(cmd string, line []byte) (*CommandInfo, error) { 138 | c := CommandInfo{Cmd: cmd} 139 | n, _ := fmt.Sscanf(string(line), "%s %d", &c.Key, &c.Exptime) 140 | if n != 2 { 141 | return nil, errCommand 142 | } 143 | if bytes.HasSuffix(line, norepl) { 144 | c.NoReply = true 145 | } 146 | return &c, nil 147 | } 148 | 149 | func parseOtherCommands(cmd string, line []byte) (*CommandInfo, error) { 150 | c := CommandInfo{Cmd: cmd} 151 | if cmd == "stats" { 152 | bb := bytes.Split(line, []byte(" ")) 153 | c.Keys = make([]string, len(bb), len(bb)) 154 | for i, b := range bb { 155 | c.Keys[i] = string(b) 156 | } 157 | return &c, nil 158 | } 159 | return nil, errCommand 160 | } 161 | -------------------------------------------------------------------------------- /cache/index_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | func TestCacheIndexReserve(t *testing.T) { 11 | dir, err := ioutil.TempDir("", "test_cacheindex") 12 | if err != nil { 13 | t.Fatal(err) 14 | } 15 | defer os.RemoveAll(dir) 16 | fn := filepath.Join(dir, "index") 17 | c, err := LoadCacheIndex(fn, 1024) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | item, err := c.Reserve(400) 22 | if err != nil { 23 | t.Fatal(err) 24 | } 25 | if item.Offset != 0 || item.Term != 0 { 26 | t.Fatal("item err", item) 27 | } 28 | item1, err := c.Reserve(400) 29 | if item1.Offset != item.Offset+400 { 30 | t.Fatal("item err", item1) 31 | } 32 | item2, err := c.Reserve(400) 33 | if item2.Offset != 0 || item2.Term != 1 { 34 | t.Fatal("item err", item2) 35 | } 36 | meta := c.GetIndexMeta() 37 | if meta.Term != 1 || meta.Head != 400 { 38 | t.Fatal("meta err", meta) 39 | } 40 | if err := c.Close(); err != nil { 41 | t.Fatal(err) 42 | } 43 | } 44 | 45 | func TestCacheIndexGetSet(t *testing.T) { 46 | dir, err := ioutil.TempDir("", "test_cacheindex") 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | defer os.RemoveAll(dir) 51 | fn := filepath.Join(dir, "index") 52 | c, err := LoadCacheIndex(fn, 1024) 53 | if err != nil { 54 | t.Fatal(err) 55 | } 56 | var item1, item2 *IndexItem 57 | item1, _ = c.Reserve(400) 58 | if err := c.Set("k1", item1); err != nil { 59 | t.Fatal(err) 60 | } 61 | item2, _ = c.Reserve(400) 62 | if err := c.Set("k2", item2); err != nil { 63 | t.Fatal(err) 64 | } 65 | t1, err := c.Get("k1") 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | t2, err := c.Get("k2") 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | if *t1 != *item1 && *t2 != *item2 { 74 | t.Fatal("get err", *t1, *t2) 75 | } 76 | c.Reserve(400) // overwrite item1 77 | _, err = c.Get("k1") 78 | if err != ErrNotFound { 79 | t.Fatal("should not found") 80 | } 81 | if err := c.Close(); err != nil { 82 | t.Fatal(err) 83 | } 84 | } 85 | 86 | func TestCacheIndexSizeChanged(t *testing.T) { 87 | dir, err := ioutil.TempDir("", "test_cacheindex") 88 | if err != nil { 89 | t.Fatal(err) 90 | } 91 | defer os.RemoveAll(dir) 92 | fn := filepath.Join(dir, "index") 93 | c, err := LoadCacheIndex(fn, 1024) 94 | if err != nil { 95 | t.Fatal(err) 96 | } 97 | var item *IndexItem 98 | item, _ = c.Reserve(300) 99 | c.Set("k1", item) 100 | item, _ = c.Reserve(300) 101 | c.Set("k2", item) 102 | item, _ = c.Reserve(300) 103 | c.Set("k3", item) 104 | if err := c.Close(); err != nil { 105 | t.Fatal(err) 106 | } 107 | c, err = LoadCacheIndex(fn, 500) 108 | if err != nil { 109 | t.Fatal(err) 110 | } 111 | _, err = c.Get("k1") 112 | if err != nil { 113 | t.Fatal(err) 114 | } 115 | _, err = c.Get("k2") 116 | if err != ErrNotFound { 117 | t.Fatal("should not found") 118 | } 119 | _, err = c.Get("k3") 120 | if err != ErrNotFound { 121 | t.Fatal("should not found") 122 | } 123 | if err := c.Close(); err != nil { 124 | t.Fatal(err) 125 | } 126 | } 127 | 128 | func TestCacheIndexDel(t *testing.T) { 129 | dir, err := ioutil.TempDir("", "test_cacheindex") 130 | if err != nil { 131 | t.Fatal(err) 132 | } 133 | defer os.RemoveAll(dir) 134 | fn := filepath.Join(dir, "index") 135 | c, err := LoadCacheIndex(fn, 1024) 136 | if err != nil { 137 | t.Fatal(err) 138 | } 139 | var item *IndexItem 140 | item, _ = c.Reserve(300) 141 | c.Set("k1", item) 142 | item, _ = c.Reserve(300) 143 | c.Set("k2", item) 144 | item, _ = c.Reserve(300) 145 | c.Set("k3", item) 146 | if err := c.Del("k1"); err != nil { 147 | t.Fatal(err) 148 | } 149 | if err := c.Dels([]string{"k2", "k3"}); err != nil { 150 | t.Fatal(err) 151 | } 152 | for _, k := range []string{"k1", "k2", "k3"} { 153 | if _, err := c.Get(k); err != ErrNotFound { 154 | t.Fatal(k, "should not found") 155 | } 156 | } 157 | if err := c.Close(); err != nil { 158 | t.Fatal(err) 159 | } 160 | } 161 | 162 | func TestCacheIndexIter(t *testing.T) { 163 | dir, err := ioutil.TempDir("", "test_cacheindex") 164 | if err != nil { 165 | t.Fatal(err) 166 | } 167 | defer os.RemoveAll(dir) 168 | fn := filepath.Join(dir, "index") 169 | c, err := LoadCacheIndex(fn, 1024) 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | item1, _ := c.Reserve(300) 174 | c.Set("k1", item1) 175 | item2, _ := c.Reserve(300) 176 | c.Set("k2", item2) 177 | 178 | iterkeys := 0 179 | 180 | err = c.Iter("", 100, func(key string, item IndexItem) error { 181 | iterkeys += 1 182 | if key == "k1" && item != *item1 { 183 | t.Fatal(key, "item err", item) 184 | } 185 | if key == "k2" && item != *item2 { 186 | t.Fatal(key, "item err", item) 187 | } 188 | return nil 189 | }) 190 | if err != nil { 191 | t.Fatal(err) 192 | } 193 | if iterkeys != 2 { 194 | t.Fatal("iter keys", iterkeys) 195 | } 196 | if err := c.Close(); err != nil { 197 | t.Fatal(err) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /protocol/memcache/parser_test.go: -------------------------------------------------------------------------------- 1 | package memcache 2 | 3 | import "testing" 4 | 5 | func TestParseSet(t *testing.T) { 6 | b := []byte("set k1 1 2 3 noreply\r\nxxx\r\n") 7 | advance, cmd, err := ParseCommand(b) 8 | if err != nil { 9 | t.Fatal(err) 10 | } 11 | if cmd.Cmd != "set" { 12 | t.Fatal("cmd err", cmd.Cmd) 13 | } 14 | if cmd.Key != "k1" { 15 | t.Fatal("key err", cmd.Key) 16 | } 17 | if cmd.Flags != 1 { 18 | t.Fatal("flags err", cmd.Flags) 19 | } 20 | if cmd.Exptime != 2 { 21 | t.Fatal("exptime err", cmd.Exptime) 22 | } 23 | if cmd.PayloadLen != 3 { 24 | t.Fatal("payload len err", cmd.PayloadLen) 25 | } 26 | if cmd.NoReply != true { 27 | t.Fatal("norepl err") 28 | } 29 | if string(b[advance:]) != "xxx\r\n" { 30 | t.Fatal("left buf err", string(b[advance:])) 31 | } 32 | } 33 | 34 | func TestParseCAS(t *testing.T) { 35 | b := []byte("cas k1 1 2 3 4 noreply\r\nxxx\r\n") 36 | advance, cmd, err := ParseCommand(b) 37 | if err != nil { 38 | t.Fatal(err) 39 | } 40 | if cmd.Cmd != "cas" { 41 | t.Fatal("cmd err", cmd.Cmd) 42 | } 43 | if cmd.Key != "k1" { 44 | t.Fatal("key err", cmd.Key) 45 | } 46 | if cmd.Flags != 1 { 47 | t.Fatal("flags err", cmd.Flags) 48 | } 49 | if cmd.Exptime != 2 { 50 | t.Fatal("exptime err", cmd.Exptime) 51 | } 52 | if cmd.PayloadLen != 3 { 53 | t.Fatal("payload len err", cmd.PayloadLen) 54 | } 55 | if cmd.CasUnique != 4 { 56 | t.Fatal("cas uniq err", cmd.CasUnique) 57 | } 58 | if cmd.NoReply != true { 59 | t.Fatal("norepl err") 60 | } 61 | if string(b[advance:]) != "xxx\r\n" { 62 | t.Fatal("left buf err", string(b[advance:])) 63 | } 64 | } 65 | 66 | func TestParseGet(t *testing.T) { 67 | b := []byte("get k1 k2\r\nxxx\r\n") 68 | advance, cmd, err := ParseCommand(b) 69 | if err != nil { 70 | t.Fatal(err) 71 | } 72 | if cmd.Cmd != "get" { 73 | t.Fatal("cmd err", cmd.Cmd) 74 | } 75 | if cmd.Key != "k1" { 76 | t.Fatal("key err", cmd.Key) 77 | } 78 | if len(cmd.Keys) != 2 { 79 | t.Fatal("keys number err", len(cmd.Keys)) 80 | } 81 | if cmd.Keys[0] != "k1" { 82 | t.Fatal("keys[0] err", cmd.Keys[0]) 83 | } 84 | if cmd.Keys[1] != "k2" { 85 | t.Fatal("keys[1] err", cmd.Keys[1]) 86 | 87 | } 88 | if string(b[advance:]) != "xxx\r\n" { 89 | t.Fatal("left buf err", string(b[advance:])) 90 | } 91 | } 92 | 93 | func TestParseDelete(t *testing.T) { 94 | b := []byte("delete k1 noreply\r\nxxx\r\n") 95 | advance, cmd, err := ParseCommand(b) 96 | if err != nil { 97 | t.Fatal(err) 98 | } 99 | if cmd.Cmd != "delete" { 100 | t.Fatal("cmd err", cmd.Cmd) 101 | } 102 | if cmd.Key != "k1" { 103 | t.Fatal("key err", cmd.Key) 104 | } 105 | if cmd.NoReply != true { 106 | t.Fatal("norepl err") 107 | } 108 | if string(b[advance:]) != "xxx\r\n" { 109 | t.Fatal("left buf err", string(b[advance:])) 110 | } 111 | } 112 | 113 | func TestParseIncr(t *testing.T) { 114 | b := []byte("incr k1 7 noreply\r\nxxx\r\n") 115 | advance, cmd, err := ParseCommand(b) 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | if cmd.Cmd != "incr" { 120 | t.Fatal("cmd err", cmd.Cmd) 121 | } 122 | if cmd.Key != "k1" { 123 | t.Fatal("key err", cmd.Key) 124 | } 125 | if cmd.Delta != 7 { 126 | t.Fatal("delta err", cmd.Delta) 127 | } 128 | if cmd.NoReply != true { 129 | t.Fatal("norepl err") 130 | } 131 | if string(b[advance:]) != "xxx\r\n" { 132 | t.Fatal("left buf err", string(b[advance:])) 133 | } 134 | } 135 | 136 | func TestParseTouch(t *testing.T) { 137 | b := []byte("touch k1 7 noreply\r\nxxx\r\n") 138 | advance, cmd, err := ParseCommand(b) 139 | if err != nil { 140 | t.Fatal(err) 141 | } 142 | if cmd.Cmd != "touch" { 143 | t.Fatal("cmd err", cmd.Cmd) 144 | } 145 | if cmd.Key != "k1" { 146 | t.Fatal("key err", cmd.Key) 147 | } 148 | if cmd.Exptime != 7 { 149 | t.Fatal("delta err", cmd.Delta) 150 | } 151 | if cmd.NoReply != true { 152 | t.Fatal("norepl err") 153 | } 154 | if string(b[advance:]) != "xxx\r\n" { 155 | t.Fatal("left buf err", string(b[advance:])) 156 | } 157 | } 158 | 159 | func TestParseStats(t *testing.T) { 160 | b := []byte("stats\r\n") 161 | advance, cmd, err := ParseCommand(b) 162 | if err != nil { 163 | t.Fatal(err) 164 | } 165 | if cmd.Cmd != "stats" { 166 | t.Fatal("cmd err", cmd.Cmd) 167 | } 168 | if string(b[advance:]) != "" { 169 | t.Fatal("left buf err", string(b[advance:])) 170 | } 171 | 172 | b = []byte("stats settings\r\n") 173 | advance, cmd, err = ParseCommand(b) 174 | if err != nil { 175 | t.Fatal(err) 176 | } 177 | if cmd.Cmd != "stats" { 178 | t.Fatal("cmd err", cmd.Cmd) 179 | } 180 | if len(cmd.Keys) != 1 || cmd.Keys[0] != "settings" { 181 | t.Fatal("stats args err") 182 | } 183 | if string(b[advance:]) != "" { 184 | t.Fatal("left buf err", string(b[advance:])) 185 | } 186 | } 187 | 188 | func TestParseErr(t *testing.T) { 189 | b := []byte("xxx k1 7 noreply\r\nxxx\r\n") 190 | advance, _, err := ParseCommand(b) 191 | if err != errCommand { 192 | t.Fatal("err != errCommand", err) 193 | } 194 | if advance != len("xxx k1 7 noreply\r\n") { 195 | t.Fatal("advance err", advance) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /cache/cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | var ( 12 | ErrNotFound = errors.New("key not found") 13 | ErrValueSize = errors.New("value size exceeded") 14 | ErrValueCrc = errors.New("value checksum err") 15 | ) 16 | 17 | const ( 18 | MaxShards = 128 19 | MaxValueSize = int64(128 << 20) // 128MB 20 | MinShardSize = MaxValueSize + 4096 21 | ) 22 | 23 | type CacheMetrics struct { 24 | GetTotal int64 // number of get request 25 | GetHits int64 // number of items that hit from data 26 | GetMisses int64 // number of items that not found 27 | GetExpired int64 // number of items that expired when get 28 | SetTotal int64 // number of set request 29 | DelTotal int64 // number of del request 30 | Expired int64 // number of items that expired 31 | Evicted int64 // number of items evicted 32 | EvictedAge int64 // min age of the last evicted item 33 | } 34 | 35 | func (m *CacheMetrics) Add(o CacheMetrics) { 36 | m.GetTotal += o.GetTotal 37 | m.GetHits += o.GetHits 38 | m.GetMisses += o.GetMisses 39 | m.GetExpired += o.GetExpired 40 | m.SetTotal += o.SetTotal 41 | m.DelTotal += o.DelTotal 42 | m.Expired += o.Expired 43 | m.Evicted += o.Evicted 44 | // use min age 45 | if m.EvictedAge <= 0 || (o.EvictedAge > 0 && o.EvictedAge < m.EvictedAge) { 46 | m.EvictedAge = o.EvictedAge 47 | } 48 | } 49 | 50 | type CacheStats struct { 51 | Keys uint64 // number of keys 52 | Bytes uint64 // bytes of keys that used 53 | LastUpdate int64 // stat time, the stat is async updated 54 | } 55 | 56 | func (st *CacheStats) Add(o CacheStats) { 57 | st.Keys += o.Keys 58 | st.Bytes += o.Bytes 59 | // use oldest time 60 | if o.LastUpdate < st.LastUpdate { 61 | st.LastUpdate = o.LastUpdate 62 | } 63 | } 64 | 65 | type Cache struct { 66 | hash ConsistentHash 67 | shards []*Shard 68 | 69 | options CacheOptions 70 | } 71 | 72 | type Item struct { 73 | Key string 74 | Value []byte 75 | Timestamp int64 76 | TTL uint32 77 | Flags uint32 78 | 79 | free func(*Item) 80 | } 81 | 82 | func (i *Item) Free() { 83 | if i.free == nil { 84 | panic("double free or not alloc from allocator") 85 | } 86 | i.free(i) 87 | i.free = nil 88 | } 89 | 90 | type Allocator interface { 91 | Alloc(n int) *Item 92 | Free(*Item) 93 | } 94 | 95 | type CacheOptions struct { 96 | ShardNum int 97 | Size int64 98 | TTL int64 99 | Allocator Allocator 100 | 101 | DisableGC bool 102 | } 103 | 104 | var DefualtCacheOptions = CacheOptions{ 105 | ShardNum: 7, 106 | Size: 32 * MaxValueSize, // 32*128MB = 4GB 107 | TTL: 0, 108 | Allocator: NewAllocatorPool(4096), 109 | } 110 | 111 | func NewCache(path string, options *CacheOptions) (*Cache, error) { 112 | os.MkdirAll(path, 0700) 113 | if options == nil { 114 | options = &CacheOptions{} 115 | *options = DefualtCacheOptions 116 | } 117 | if options.Allocator == nil { 118 | options.Allocator = NewAllocatorPool(4096) 119 | } 120 | if options.ShardNum > MaxShards { 121 | options.ShardNum = MaxShards 122 | } 123 | if options.Size/int64(options.ShardNum) < MinShardSize { 124 | options.ShardNum = int(options.Size / MinShardSize) 125 | } 126 | if options.ShardNum <= 0 { 127 | options.ShardNum = 1 128 | } 129 | 130 | var err error 131 | cache := Cache{options: *options} 132 | cache.shards = make([]*Shard, options.ShardNum) 133 | for i := 0; i < MaxShards; i++ { 134 | fn := filepath.Join(path, fmt.Sprintf("shard.%03d", i)) 135 | if i >= options.ShardNum { // rm unused files 136 | os.Remove(fn + indexSubfix) 137 | os.Remove(fn + dataSubfix) 138 | continue 139 | } 140 | sopts := &ShardOptions{ 141 | Size: options.Size / int64(options.ShardNum), 142 | TTL: options.TTL, 143 | Allocator: options.Allocator, 144 | DisableGC: options.DisableGC, 145 | } 146 | cache.shards[i], err = LoadCacheShard(fn, sopts) 147 | if err != nil { 148 | return nil, errors.Wrap(err, "load cache shard") 149 | } 150 | } 151 | cache.hash = NewConsistentHashTable(len(cache.shards)) 152 | return &cache, nil 153 | } 154 | 155 | func (c *Cache) Close() error { 156 | var err error 157 | for _, s := range c.shards { 158 | er := s.Close() 159 | if er != nil { 160 | err = er 161 | } 162 | } 163 | return err 164 | } 165 | 166 | func (c *Cache) getshard(key string) *Shard { 167 | return c.shards[c.hash.Get(key)] 168 | } 169 | 170 | func (c *Cache) Set(item *Item) error { 171 | if int64(len(item.Value)) > MaxValueSize { 172 | return ErrValueSize 173 | } 174 | s := c.getshard(item.Key) 175 | return s.Set(item) 176 | } 177 | 178 | func (c *Cache) Get(key string) (*Item, error) { 179 | s := c.getshard(key) 180 | return s.Get(key) 181 | } 182 | 183 | func (c *Cache) Del(key string) error { 184 | s := c.getshard(key) 185 | return s.Del(key) 186 | } 187 | 188 | func (c *Cache) GetMetrics() CacheMetrics { 189 | var m CacheMetrics 190 | for _, s := range c.GetMetricsByShards() { 191 | m.Add(s) 192 | } 193 | return m 194 | } 195 | 196 | func (c *Cache) GetStats() CacheStats { 197 | var st CacheStats 198 | for _, s := range c.GetStatsByShards() { 199 | st.Add(s) 200 | } 201 | return st 202 | } 203 | 204 | func (c *Cache) GetMetricsByShards() []CacheMetrics { 205 | var ret = make([]CacheMetrics, len(c.shards), len(c.shards)) 206 | for i, s := range c.shards { 207 | ret[i] = s.GetMetrics() 208 | } 209 | return ret 210 | } 211 | 212 | func (c *Cache) GetStatsByShards() []CacheStats { 213 | var ret = make([]CacheStats, len(c.shards), len(c.shards)) 214 | for i, s := range c.shards { 215 | ret[i] = s.GetStats() 216 | } 217 | return ret 218 | } 219 | 220 | func (c *Cache) GetOptions() CacheOptions { 221 | return c.options 222 | } 223 | -------------------------------------------------------------------------------- /cache/index.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/boltdb/bolt" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | type CacheIndex struct { 12 | db *bolt.DB 13 | 14 | mu sync.RWMutex 15 | meta IndexMeta 16 | } 17 | 18 | var ( 19 | indexMetaBucket = []byte("meta") 20 | indexDataBucket = []byte("data") 21 | indexMetaKey = []byte("indexmeta") 22 | ) 23 | 24 | func LoadCacheIndex(fn string, datasize int64) (*CacheIndex, error) { 25 | var err error 26 | var index CacheIndex 27 | index.db, err = bolt.Open(fn, 0600, &bolt.Options{Timeout: 10 * time.Second}) 28 | if err != nil { 29 | return nil, errors.Wrap(err, "bolt.Open") 30 | } 31 | index.db.NoSync = true // for improve performance 32 | err = index.db.Update(func(tx *bolt.Tx) error { 33 | bucket, err := tx.CreateBucketIfNotExists(indexMetaBucket) 34 | if err != nil { 35 | return err 36 | } 37 | v := bucket.Get(indexMetaKey) 38 | if v == nil { 39 | return nil 40 | } 41 | return index.meta.Unmarshal(v) 42 | }) 43 | if err != nil { 44 | return nil, errors.Wrap(err, "bolt.Update") 45 | } 46 | meta := &index.meta 47 | if meta.DataSize != datasize { 48 | meta.DataSize = datasize 49 | } 50 | if meta.Head > datasize { 51 | meta.Head = 0 52 | meta.Term += 1 // datasize changed, move to next term 53 | } 54 | return &index, nil 55 | } 56 | 57 | func (i *CacheIndex) Close() error { 58 | i.mu.Lock() 59 | defer i.mu.Unlock() 60 | return i.db.Close() 61 | } 62 | 63 | func (i *CacheIndex) Get(key string) (*IndexItem, error) { 64 | var item IndexItem 65 | err := i.db.View(func(tx *bolt.Tx) error { 66 | bucket := tx.Bucket(indexDataBucket) 67 | if bucket == nil { 68 | return ErrNotFound 69 | } 70 | v := bucket.Get([]byte(key)) 71 | if v == nil { 72 | return ErrNotFound 73 | } 74 | return item.Unmarshal(v) 75 | }) 76 | if err != nil { 77 | return nil, err 78 | } 79 | meta := i.GetIndexMeta() 80 | if meta.IsValidate(item) { 81 | return &item, nil 82 | } 83 | return nil, ErrNotFound 84 | } 85 | 86 | func (i *CacheIndex) Reserve(size int32) (*IndexItem, error) { 87 | i.mu.Lock() 88 | defer i.mu.Unlock() 89 | meta := &i.meta 90 | if int64(size) > meta.DataSize { 91 | return nil, errors.New("not enough space") 92 | } 93 | if meta.Head+int64(size) > meta.DataSize { 94 | meta.Head = 0 95 | meta.Term += 1 96 | } 97 | idx := &IndexItem{Term: meta.Term, Offset: meta.Head, ValueSize: size, Timestamp: time.Now().Unix()} 98 | meta.Head += int64(size) 99 | 100 | err := i.db.Update(func(tx *bolt.Tx) error { 101 | bucket, err := tx.CreateBucketIfNotExists(indexMetaBucket) 102 | if err != nil { 103 | return err 104 | } 105 | b, err := i.meta.Marshal() 106 | if err != nil { 107 | return err 108 | } 109 | return bucket.Put(indexMetaKey, b) 110 | }) 111 | if err != nil { 112 | return nil, err 113 | } 114 | return idx, nil 115 | } 116 | 117 | func (i *CacheIndex) Set(key string, item *IndexItem) error { 118 | return i.db.Update(func(tx *bolt.Tx) error { 119 | bucket, err := tx.CreateBucketIfNotExists(indexDataBucket) 120 | if err != nil { 121 | return err 122 | } 123 | b, _ := item.Marshal() 124 | return bucket.Put([]byte(key), b) 125 | }) 126 | } 127 | 128 | func (i *CacheIndex) Del(key string) error { 129 | return i.db.Update(func(tx *bolt.Tx) error { 130 | bucket, err := tx.CreateBucketIfNotExists(indexDataBucket) 131 | if err != nil { 132 | return err 133 | } 134 | return bucket.Delete([]byte(key)) 135 | }) 136 | } 137 | 138 | func (i *CacheIndex) Dels(keys []string) error { 139 | if len(keys) == 0 { 140 | return nil 141 | } 142 | return i.db.Update(func(tx *bolt.Tx) error { 143 | bucket, err := tx.CreateBucketIfNotExists(indexDataBucket) 144 | if err != nil { 145 | return err 146 | } 147 | for _, key := range keys { 148 | if err := bucket.Delete([]byte(key)); err != nil { 149 | return err 150 | } 151 | } 152 | return nil 153 | }) 154 | } 155 | 156 | func (i *CacheIndex) GetIndexMeta() IndexMeta { 157 | i.mu.RLock() 158 | defer i.mu.RUnlock() 159 | return i.meta 160 | } 161 | 162 | func (i *CacheIndex) Iter(lastkey string, maxIter int, f func(key string, item IndexItem) error) error { 163 | var item IndexItem 164 | err := i.db.View(func(tx *bolt.Tx) error { 165 | bucket := tx.Bucket(indexDataBucket) 166 | if bucket == nil { 167 | return nil 168 | } 169 | c := bucket.Cursor() 170 | 171 | var k, v []byte 172 | 173 | if lastkey == "" { 174 | k, v = c.First() 175 | } else { 176 | k, v = c.Seek([]byte(lastkey)) 177 | if string(k) == lastkey { 178 | k, v = c.Next() 179 | } 180 | } 181 | for k != nil && maxIter > 0 { 182 | maxIter -= 1 183 | if err := item.Unmarshal(v); err != nil { 184 | return err 185 | } 186 | if err := f(string(k), item); err != nil { 187 | return err 188 | } 189 | k, v = c.Next() 190 | } 191 | return nil 192 | }) 193 | return err 194 | } 195 | 196 | func (i *CacheIndex) GetKeys() (n uint64, err error) { 197 | err = i.db.View(func(tx *bolt.Tx) error { 198 | bucket := tx.Bucket(indexDataBucket) 199 | if bucket != nil { 200 | st := bucket.Stats() 201 | n = uint64(st.KeyN) 202 | } 203 | return nil 204 | }) 205 | return 206 | } 207 | 208 | // extends index.pb.go >>>>>>>>>>>>>>>>>>>>>> 209 | 210 | // IsValidate return if the item is validate 211 | func (m IndexMeta) IsValidate(i IndexItem) bool { 212 | if i.Term > m.Term { 213 | return false 214 | } 215 | if i.Term == m.Term { 216 | return i.Offset < m.Head && i.Offset+int64(i.ValueSize) <= m.DataSize 217 | } 218 | // i.Term < m.Term 219 | if i.Term+1 != m.Term { 220 | return false 221 | } 222 | return m.Head <= i.Offset && i.Offset+int64(i.ValueSize) <= m.DataSize 223 | } 224 | 225 | // TotalSize returns bytes used including index & data of the item 226 | func (i IndexItem) TotalSize() int64 { 227 | return int64(i.Size()) + int64(i.ValueSize) 228 | } 229 | -------------------------------------------------------------------------------- /cache/shard.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "hash/crc32" 5 | "log" 6 | "sync" 7 | "sync/atomic" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | const ( 14 | indexSubfix = ".idx" 15 | dataSubfix = ".dat" 16 | ) 17 | 18 | type Shard struct { 19 | mu sync.RWMutex 20 | index *CacheIndex 21 | data *CacheData 22 | 23 | options ShardOptions 24 | 25 | stats CacheStats 26 | metrics CacheMetrics 27 | exit chan struct{} 28 | } 29 | 30 | type ShardOptions struct { 31 | Size int64 32 | TTL int64 33 | Allocator Allocator 34 | DisableGC bool 35 | } 36 | 37 | var DefualtShardOptions = ShardOptions{ 38 | Size: MinShardSize, 39 | TTL: 0, 40 | Allocator: NewAllocatorPool(4096), 41 | } 42 | 43 | func LoadCacheShard(fn string, options *ShardOptions) (*Shard, error) { 44 | if options == nil { 45 | options = &ShardOptions{} 46 | *options = DefualtShardOptions 47 | } 48 | if options.Size <= 0 { 49 | options.Size = MinShardSize 50 | } 51 | if options.Allocator == nil { 52 | options.Allocator = NewAllocatorPool(4096) 53 | } 54 | 55 | var err error 56 | s := Shard{options: *options} 57 | s.index, err = LoadCacheIndex(fn+indexSubfix, options.Size) 58 | if err != nil { 59 | return nil, errors.Wrap(err, "LoadIndex") 60 | } 61 | s.data, err = LoadCacheData(fn+dataSubfix, options.Size) 62 | if err != nil { 63 | return nil, errors.Wrap(err, "LoadData") 64 | } 65 | 66 | s.stats.Keys, err = s.index.GetKeys() 67 | if err != nil { 68 | return nil, err 69 | } 70 | var st gcstat // sample 10000 keys for stats.Bytes 71 | if err := s.scanKeysForGC(10000, &st); err != nil { 72 | return nil, err 73 | } 74 | s.stats.Bytes = uint64(float64(s.stats.Keys) * float64(st.ActiveBytes) / float64(st.Active)) 75 | s.stats.LastUpdate = time.Now().Unix() 76 | 77 | s.exit = make(chan struct{}) 78 | 79 | if !options.DisableGC { 80 | go s.GCLoop() 81 | } 82 | return &s, nil 83 | } 84 | 85 | func (s *Shard) Close() error { 86 | close(s.exit) 87 | err1 := s.index.Close() 88 | err2 := s.data.Close() 89 | if err1 != nil { 90 | return errors.Wrap(err1, "close index") 91 | } 92 | if err2 != nil { 93 | return errors.Wrap(err1, "close data") 94 | } 95 | return nil 96 | } 97 | 98 | func (s *Shard) GetMetrics() CacheMetrics { 99 | var m CacheMetrics 100 | m.GetTotal = atomic.LoadInt64(&s.metrics.GetTotal) 101 | m.GetHits = atomic.LoadInt64(&s.metrics.GetHits) 102 | m.GetMisses = atomic.LoadInt64(&s.metrics.GetMisses) 103 | m.GetExpired = atomic.LoadInt64(&s.metrics.GetExpired) 104 | m.SetTotal = atomic.LoadInt64(&s.metrics.SetTotal) 105 | m.DelTotal = atomic.LoadInt64(&s.metrics.DelTotal) 106 | m.Expired = atomic.LoadInt64(&s.metrics.Expired) 107 | m.Evicted = atomic.LoadInt64(&s.metrics.Evicted) 108 | m.EvictedAge = atomic.LoadInt64(&s.metrics.EvictedAge) 109 | return m 110 | } 111 | 112 | func (s *Shard) GetStats() CacheStats { 113 | var st CacheStats 114 | st.Keys = atomic.LoadUint64(&s.stats.Keys) 115 | st.Bytes = atomic.LoadUint64(&s.stats.Bytes) 116 | st.LastUpdate = atomic.LoadInt64(&s.stats.LastUpdate) 117 | return st 118 | } 119 | 120 | func (s *Shard) Set(ci *Item) error { 121 | atomic.AddInt64(&s.metrics.SetTotal, 1) 122 | s.mu.Lock() 123 | defer s.mu.Unlock() 124 | ii, err := s.index.Reserve(int32(len(ci.Value))) 125 | if err != nil { 126 | return errors.Wrap(err, "reserve index") 127 | } 128 | ii.TTL = ci.TTL 129 | ii.Flags = ci.Flags 130 | ii.Crc32 = crc32.ChecksumIEEE(ci.Value) 131 | if err := s.data.Write(ii.Offset, ci.Value); err != nil { 132 | return errors.Wrap(err, "write data") 133 | } 134 | if err := s.index.Set(ci.Key, ii); err != nil { 135 | return errors.Wrap(err, "update index") 136 | } 137 | return nil 138 | } 139 | 140 | func (s *Shard) Get(key string) (*Item, error) { 141 | atomic.AddInt64(&s.metrics.GetTotal, 1) 142 | s.mu.RLock() 143 | defer s.mu.RUnlock() 144 | ii, err := s.index.Get(key) 145 | if err != nil { 146 | if err == ErrNotFound { 147 | atomic.AddInt64(&s.metrics.GetMisses, 1) 148 | } 149 | return nil, err 150 | } 151 | 152 | age := time.Now().Unix() - ii.Timestamp 153 | if (s.options.TTL > 0 && age >= s.options.TTL) || (ii.TTL > 0 && age > int64(ii.TTL)) { 154 | atomic.AddInt64(&s.metrics.GetMisses, 1) 155 | atomic.AddInt64(&s.metrics.GetExpired, 1) 156 | atomic.AddInt64(&s.metrics.Expired, 1) 157 | s.index.Del(key) 158 | return nil, ErrNotFound 159 | } 160 | 161 | ci := s.options.Allocator.Alloc(int(ii.ValueSize)) 162 | ci.Key = key 163 | ci.Timestamp = ii.Timestamp 164 | ci.TTL = ii.TTL 165 | ci.Flags = ii.Flags 166 | 167 | err = s.data.Read(ii.Offset, ci.Value) 168 | if err == ErrOutOfRange { 169 | err = ErrNotFound // data size changed? 170 | } 171 | if err != nil { 172 | if err == ErrNotFound { 173 | atomic.AddInt64(&s.metrics.GetMisses, 1) 174 | } 175 | ci.Free() 176 | return nil, err 177 | } 178 | if ii.Crc32 != 0 && ii.Crc32 != crc32.ChecksumIEEE(ci.Value) { 179 | ci.Free() 180 | return nil, ErrValueCrc 181 | } 182 | atomic.AddInt64(&s.metrics.GetHits, 1) 183 | return ci, nil 184 | } 185 | 186 | func (s *Shard) Del(key string) error { 187 | atomic.AddInt64(&s.metrics.DelTotal, 1) 188 | s.mu.RLock() 189 | defer s.mu.RUnlock() 190 | return s.index.Del(key) 191 | } 192 | 193 | type gcstat struct { 194 | Scanned uint64 195 | Purged uint64 196 | Active uint64 197 | ActiveBytes uint64 198 | 199 | LastKey string 200 | LastFinish time.Time 201 | } 202 | 203 | func timeSub(t1, t2 time.Time, round time.Duration) time.Duration { 204 | t1 = t1.Round(round) 205 | t2 = t2.Round(round) 206 | return t1.Sub(t2) 207 | } 208 | 209 | func (s *Shard) GCLoop() { 210 | const scanItemsPerRound = 100 211 | const sleepTimePerRound = 50 * time.Millisecond // ~ scan 2k items per second 212 | 213 | var st gcstat 214 | 215 | st.LastFinish = time.Now() 216 | 217 | for { 218 | select { 219 | case <-s.exit: 220 | return 221 | case <-time.After(sleepTimePerRound): 222 | } 223 | n := st.Scanned 224 | err := s.scanKeysForGC(scanItemsPerRound, &st) 225 | if err != nil { 226 | log.Printf("Iter keys err: %s", err) 227 | continue 228 | } 229 | newScanned := st.Scanned - n 230 | if newScanned >= scanItemsPerRound { 231 | continue // scanning 232 | } 233 | // newScanned < scanItemsPerRound, end of index 234 | now := time.Now() 235 | cost := timeSub(now, st.LastFinish, time.Millisecond) 236 | log.Printf("shard[%p] gc scanned:%d purged:%d keys cost %v", 237 | s, st.Scanned, st.Purged, cost) 238 | 239 | // save stats to CacheStats 240 | atomic.StoreUint64(&s.stats.Keys, st.Active) 241 | atomic.StoreUint64(&s.stats.Bytes, st.ActiveBytes) 242 | atomic.StoreInt64(&s.stats.LastUpdate, now.Unix()) 243 | 244 | st.Scanned = 0 245 | st.Purged = 0 246 | st.Active = 0 247 | st.ActiveBytes = 0 248 | st.LastKey = "" 249 | 250 | if cost < time.Minute { // rate limit 251 | select { 252 | case <-s.exit: 253 | return 254 | case <-time.After(time.Minute - cost): 255 | } 256 | } 257 | st.LastFinish = time.Now() 258 | } 259 | } 260 | 261 | func (s *Shard) scanKeysForGC(maxIter int, st *gcstat) error { 262 | now := time.Now().Unix() 263 | meta := s.index.GetIndexMeta() 264 | var pendingDeletes []string 265 | err := s.index.Iter(st.LastKey, maxIter, func(key string, ii IndexItem) error { 266 | st.Scanned += 1 267 | st.LastKey = key 268 | age := time.Now().Unix() - ii.Timestamp 269 | if (s.options.TTL > 0 && age >= s.options.TTL) || (ii.TTL > 0 && age > int64(ii.TTL)) { 270 | st.Purged += 1 271 | atomic.AddInt64(&s.metrics.Expired, 1) 272 | pendingDeletes = append(pendingDeletes, key) 273 | return nil 274 | } 275 | if !meta.IsValidate(ii) { // the item may overwritten by other keys 276 | st.Purged += 1 277 | atomic.AddInt64(&s.metrics.Evicted, 1) 278 | atomic.StoreInt64(&s.metrics.EvictedAge, now-ii.Timestamp) 279 | pendingDeletes = append(pendingDeletes, key) 280 | return nil 281 | } 282 | st.Active += 1 283 | st.ActiveBytes += uint64(int64(len(key)) + ii.TotalSize()) 284 | return nil 285 | }) 286 | err2 := s.index.Dels(pendingDeletes) 287 | if err == nil { 288 | err = err2 289 | } 290 | return err 291 | } 292 | -------------------------------------------------------------------------------- /server/memcache.go: -------------------------------------------------------------------------------- 1 | package server 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "log" 11 | "net" 12 | "os" 13 | "sync/atomic" 14 | "time" 15 | 16 | "github.com/xiaost/blobcached/cache" 17 | "github.com/xiaost/blobcached/protocol/memcache" 18 | ) 19 | 20 | var errNotSupportedCommand = errors.New("not supported command") 21 | 22 | type ServerMetrics struct { 23 | BytesRead uint64 // Total number of bytes read by this server 24 | BytesWritten uint64 // Total number of bytes sent by this server 25 | CurrConnections int64 // Number of active connections 26 | TotalConnections uint64 // Total number of connections opened since the server started running 27 | } 28 | 29 | type MemcacheServer struct { 30 | l net.Listener 31 | cache Cache 32 | allocator cache.Allocator 33 | metrics ServerMetrics 34 | 35 | startTime time.Time 36 | } 37 | 38 | func NewMemcacheServer(l net.Listener, cache Cache, allocator cache.Allocator) *MemcacheServer { 39 | return &MemcacheServer{l: l, cache: cache, allocator: allocator} 40 | } 41 | 42 | func (s *MemcacheServer) Serv() error { 43 | s.startTime = time.Now() 44 | log.Printf("memcache server listening on %s", s.l.Addr()) 45 | for { 46 | conn, err := s.l.Accept() 47 | if err != nil { 48 | return err 49 | } 50 | if tcpconn, ok := conn.(*net.TCPConn); ok { 51 | tcpconn.SetKeepAlive(true) 52 | tcpconn.SetKeepAlivePeriod(30 * time.Second) 53 | } 54 | atomic.AddUint64(&s.metrics.TotalConnections, 1) 55 | go func(conn net.Conn) { 56 | atomic.AddInt64(&s.metrics.CurrConnections, 1) 57 | 58 | s.Handle(conn) 59 | conn.Close() 60 | 61 | atomic.AddInt64(&s.metrics.CurrConnections, -1) 62 | }(conn) 63 | } 64 | } 65 | 66 | func (s *MemcacheServer) Handle(conn net.Conn) { 67 | const maxReadPerRequest = 2 * cache.MaxValueSize 68 | r := &io.LimitedReader{R: conn, N: maxReadPerRequest} 69 | w := &WriterCounter{conn, 0} 70 | var rbuf *bufio.Reader 71 | for { 72 | atomic.AddUint64(&s.metrics.BytesRead, uint64(maxReadPerRequest-r.N)) 73 | atomic.AddUint64(&s.metrics.BytesWritten, uint64(w.N)) 74 | 75 | r.N = maxReadPerRequest 76 | w.N = 0 77 | 78 | conn.SetDeadline(time.Now().Add(48 * time.Hour)) 79 | 80 | if rbuf == nil { 81 | rbuf = bufio.NewReader(r) 82 | } 83 | b, err := rbuf.ReadSlice('\n') 84 | if err != nil { 85 | if err != io.EOF { 86 | log.Printf("read %s err: %s", conn.RemoteAddr(), err) 87 | } 88 | return 89 | } 90 | advance, cmdinfo, err := memcache.ParseCommand(b) 91 | if err != nil { 92 | log.Printf("parse %s command err: %s", conn.RemoteAddr(), err) 93 | w.Write(memcache.MakeRspClientErr(err)) 94 | return 95 | } 96 | if advance != len(b) { 97 | panic("advance != len(b)") 98 | } 99 | 100 | // avoid blocking on reading data block or writing rsp 101 | conn.SetDeadline(time.Now().Add(60 * time.Second)) 102 | 103 | var ww io.Writer = w 104 | if cmdinfo.NoReply { 105 | ww = ioutil.Discard 106 | } 107 | 108 | switch cmdinfo.Cmd { 109 | case "get", "gets": 110 | // some clients always use "gets" instead of "get" 111 | // we implement "gets" with fake cas uniq 112 | err = s.HandleGet(ww, cmdinfo) 113 | case "set": 114 | err = s.HandleSet(ww, rbuf, cmdinfo) 115 | case "delete": 116 | err = s.HandleDel(ww, cmdinfo) 117 | case "touch": 118 | err = s.HandleTouch(ww, cmdinfo) 119 | case "stats": 120 | err = s.HandleStats(ww) 121 | default: 122 | ww.Write(memcache.MakeRspServerErr(errNotSupportedCommand)) 123 | return 124 | } 125 | 126 | if err != nil { 127 | log.Printf("client %s process err: %s", conn.RemoteAddr(), err) 128 | return 129 | } 130 | 131 | if advance > (4<<10) && rbuf.Buffered() == 0 { 132 | // if parse cmd use memory > 4kb, free Reader to avoid memory issue 133 | // it is safe to free it if buffered nothing 134 | rbuf = nil 135 | } 136 | } 137 | } 138 | 139 | func (s *MemcacheServer) HandleSet(w io.Writer, r *bufio.Reader, cmdinfo *memcache.CommandInfo) error { 140 | if cmdinfo.PayloadLen > cache.MaxValueSize-4096 { 141 | w.Write(memcache.MakeRspClientErr(cache.ErrValueSize)) 142 | return cache.ErrValueSize 143 | } 144 | 145 | item := s.allocator.Alloc(int(cmdinfo.PayloadLen) + 2) // including \r\n 146 | defer item.Free() 147 | 148 | _, err := io.ReadFull(r, item.Value) 149 | if err != nil { 150 | return err 151 | } 152 | item.Key = cmdinfo.Key 153 | item.Value = item.Value[:len(item.Value)-2] // remove \r\n 154 | item.Flags = cmdinfo.Flags 155 | 156 | /* https://github.com/memcached/memcached/blob/master/doc/protocol.txt 157 | 158 | Expiration times 159 | ---------------- 160 | 161 | Some commands involve a client sending some kind of expiration time 162 | (relative to an item or to an operation requested by the client) to 163 | the server. In all such cases, the actual value sent may either be 164 | Unix time (number of seconds since January 1, 1970, as a 32-bit 165 | value), or a number of seconds starting from current time. In the 166 | latter case, this number of seconds may not exceed 60*60*24*30 (number 167 | of seconds in 30 days); if the number sent by a client is larger than 168 | that, the server will consider it to be real Unix time value rather 169 | than an offset from current time. 170 | 171 | */ 172 | if cmdinfo.Exptime <= 30*86400 { 173 | item.TTL = cmdinfo.Exptime 174 | } else { 175 | now := time.Now().Unix() 176 | if now >= int64(cmdinfo.Exptime) { 177 | _, err = w.Write(memcache.RspStored) 178 | return err 179 | } else { 180 | item.TTL = uint32(int64(cmdinfo.Exptime) - now) 181 | } 182 | } 183 | 184 | if err := s.cache.Set(item); err != nil { 185 | w.Write(memcache.MakeRspServerErr(err)) 186 | return err 187 | } 188 | _, err = w.Write(memcache.RspStored) 189 | return err 190 | } 191 | 192 | func (s *MemcacheServer) HandleGet(w io.Writer, cmdinfo *memcache.CommandInfo) error { 193 | var prepend string 194 | for _, k := range cmdinfo.Keys { 195 | item, err := s.cache.Get(k) 196 | if err == cache.ErrNotFound { 197 | continue 198 | } 199 | if err != nil { 200 | log.Printf("get key %s err: %s", k, err) 201 | continue 202 | } 203 | // VALUE []\r\n 204 | // \r\n 205 | if cmdinfo.Cmd == "gets" { 206 | fmt.Fprintf(w, "%sVALUE %s %d %d %d\r\n", prepend, k, item.Flags, len(item.Value), 0) 207 | } else { 208 | fmt.Fprintf(w, "%sVALUE %s %d %d\r\n", prepend, k, item.Flags, len(item.Value)) 209 | } 210 | w.Write(item.Value) 211 | item.Free() 212 | prepend = "\r\n" // reduce len(cmdinfo.Keys) times w.Write("\r\n") 213 | } 214 | _, err := w.Write([]byte(prepend + "END\r\n")) 215 | return err 216 | } 217 | 218 | func (s *MemcacheServer) HandleTouch(w io.Writer, cmdinfo *memcache.CommandInfo) error { 219 | item, err := s.cache.Get(cmdinfo.Key) 220 | if err != nil { 221 | if err == cache.ErrNotFound { 222 | _, err = w.Write(memcache.RspNotFound) 223 | } else { 224 | _, err = w.Write(memcache.MakeRspServerErr(err)) 225 | } 226 | return err 227 | } 228 | defer item.Free() 229 | 230 | if cmdinfo.Exptime <= 30*86400 { 231 | item.TTL = cmdinfo.Exptime 232 | } else { 233 | now := time.Now().Unix() 234 | if now >= int64(cmdinfo.Exptime) { // already expired 235 | w.Write(memcache.RspTouched) 236 | return s.cache.Del(cmdinfo.Key) 237 | } else { 238 | item.TTL = uint32(int64(cmdinfo.Exptime) - now) 239 | } 240 | } 241 | if err := s.cache.Set(item); err != nil { 242 | w.Write(memcache.MakeRspServerErr(err)) 243 | return err 244 | } 245 | _, err = w.Write(memcache.RspTouched) 246 | return err 247 | } 248 | 249 | func (s *MemcacheServer) HandleDel(w io.Writer, cmdinfo *memcache.CommandInfo) error { 250 | err := s.cache.Del(cmdinfo.Key) 251 | if err != nil { 252 | w.Write(memcache.MakeRspServerErr(err)) 253 | return nil 254 | } 255 | _, err = w.Write(memcache.RspDeleted) 256 | return err 257 | } 258 | 259 | func (s *MemcacheServer) HandleStats(w io.Writer) error { 260 | var buf bytes.Buffer 261 | writeStat := func(name string, v interface{}) { 262 | fmt.Fprintf(&buf, "STAT %s %v\r\n", name, v) 263 | } 264 | 265 | now := time.Now() 266 | writeStat("pid", os.Getpid()) 267 | writeStat("uptime", int64(now.Sub(s.startTime).Seconds())) 268 | writeStat("time", now.Unix()) 269 | writeStat("version", Version+"(blobcached)") 270 | 271 | // options 272 | options := s.cache.GetOptions() 273 | writeStat("limit_maxbytes", options.Size) 274 | 275 | // server metrics 276 | writeStat("curr_connections", atomic.LoadInt64(&s.metrics.CurrConnections)) 277 | writeStat("total_connections", atomic.LoadUint64(&s.metrics.TotalConnections)) 278 | writeStat("bytes_read", atomic.LoadUint64(&s.metrics.BytesRead)) 279 | writeStat("bytes_written", atomic.LoadUint64(&s.metrics.BytesWritten)) 280 | 281 | // cache stats 282 | stats := s.cache.GetStats() 283 | writeStat("curr_items", stats.Keys) 284 | writeStat("bytes", stats.Bytes) 285 | 286 | // cache metrics 287 | metrics := s.cache.GetMetrics() 288 | writeStat("cmd_get", metrics.GetTotal) 289 | writeStat("cmd_set", metrics.SetTotal) 290 | writeStat("get_hits", metrics.GetHits) 291 | writeStat("get_misses", metrics.GetMisses) 292 | writeStat("get_expired", metrics.GetExpired) 293 | writeStat("reclaimed", metrics.Expired) 294 | writeStat("evictions", metrics.Evicted) 295 | writeStat("last_evicted_age", metrics.EvictedAge) 296 | 297 | buf.Write(memcache.RspEnd) 298 | _, err := w.Write(buf.Bytes()) 299 | return err 300 | } 301 | -------------------------------------------------------------------------------- /cache/index.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-gogo. 2 | // source: index.proto 3 | // DO NOT EDIT! 4 | 5 | /* 6 | Package cache is a generated protocol buffer package. 7 | 8 | It is generated from these files: 9 | index.proto 10 | 11 | It has these top-level messages: 12 | IndexMeta 13 | IndexItem 14 | */ 15 | package cache 16 | 17 | import proto "github.com/gogo/protobuf/proto" 18 | import fmt "fmt" 19 | import math "math" 20 | import _ "github.com/gogo/protobuf/gogoproto" 21 | 22 | import io "io" 23 | import github_com_gogo_protobuf_proto "github.com/gogo/protobuf/proto" 24 | 25 | // Reference imports to suppress errors if they are not otherwise used. 26 | var _ = proto.Marshal 27 | var _ = fmt.Errorf 28 | var _ = math.Inf 29 | 30 | // This is a compile-time assertion to ensure that this generated file 31 | // is compatible with the proto package it is being compiled against. 32 | // A compilation error at this line likely means your copy of the 33 | // proto package needs to be updated. 34 | const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package 35 | 36 | type IndexMeta struct { 37 | Term int64 `protobuf:"varint,1,req,name=Term" json:"Term"` 38 | Head int64 `protobuf:"varint,2,req,name=Head" json:"Head"` 39 | DataSize int64 `protobuf:"varint,3,req,name=DataSize" json:"DataSize"` 40 | } 41 | 42 | func (m *IndexMeta) Reset() { *m = IndexMeta{} } 43 | func (m *IndexMeta) String() string { return proto.CompactTextString(m) } 44 | func (*IndexMeta) ProtoMessage() {} 45 | func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorIndex, []int{0} } 46 | 47 | type IndexItem struct { 48 | Term int64 `protobuf:"varint,1,req,name=Term" json:"Term"` 49 | Offset int64 `protobuf:"varint,2,req,name=Offset" json:"Offset"` 50 | ValueSize int32 `protobuf:"varint,3,req,name=ValueSize" json:"ValueSize"` 51 | Timestamp int64 `protobuf:"varint,4,req,name=Timestamp" json:"Timestamp"` 52 | TTL uint32 `protobuf:"varint,5,req,name=TTL" json:"TTL"` 53 | Flags uint32 `protobuf:"varint,6,req,name=Flags" json:"Flags"` 54 | Crc32 uint32 `protobuf:"varint,7,opt,name=Crc32" json:"Crc32"` 55 | } 56 | 57 | func (m *IndexItem) Reset() { *m = IndexItem{} } 58 | func (m *IndexItem) String() string { return proto.CompactTextString(m) } 59 | func (*IndexItem) ProtoMessage() {} 60 | func (*IndexItem) Descriptor() ([]byte, []int) { return fileDescriptorIndex, []int{1} } 61 | 62 | func init() { 63 | proto.RegisterType((*IndexMeta)(nil), "cache.IndexMeta") 64 | proto.RegisterType((*IndexItem)(nil), "cache.IndexItem") 65 | } 66 | func (m *IndexMeta) Marshal() (data []byte, err error) { 67 | size := m.Size() 68 | data = make([]byte, size) 69 | n, err := m.MarshalTo(data) 70 | if err != nil { 71 | return nil, err 72 | } 73 | return data[:n], nil 74 | } 75 | 76 | func (m *IndexMeta) MarshalTo(data []byte) (int, error) { 77 | var i int 78 | _ = i 79 | var l int 80 | _ = l 81 | data[i] = 0x8 82 | i++ 83 | i = encodeVarintIndex(data, i, uint64(m.Term)) 84 | data[i] = 0x10 85 | i++ 86 | i = encodeVarintIndex(data, i, uint64(m.Head)) 87 | data[i] = 0x18 88 | i++ 89 | i = encodeVarintIndex(data, i, uint64(m.DataSize)) 90 | return i, nil 91 | } 92 | 93 | func (m *IndexItem) Marshal() (data []byte, err error) { 94 | size := m.Size() 95 | data = make([]byte, size) 96 | n, err := m.MarshalTo(data) 97 | if err != nil { 98 | return nil, err 99 | } 100 | return data[:n], nil 101 | } 102 | 103 | func (m *IndexItem) MarshalTo(data []byte) (int, error) { 104 | var i int 105 | _ = i 106 | var l int 107 | _ = l 108 | data[i] = 0x8 109 | i++ 110 | i = encodeVarintIndex(data, i, uint64(m.Term)) 111 | data[i] = 0x10 112 | i++ 113 | i = encodeVarintIndex(data, i, uint64(m.Offset)) 114 | data[i] = 0x18 115 | i++ 116 | i = encodeVarintIndex(data, i, uint64(m.ValueSize)) 117 | data[i] = 0x20 118 | i++ 119 | i = encodeVarintIndex(data, i, uint64(m.Timestamp)) 120 | data[i] = 0x28 121 | i++ 122 | i = encodeVarintIndex(data, i, uint64(m.TTL)) 123 | data[i] = 0x30 124 | i++ 125 | i = encodeVarintIndex(data, i, uint64(m.Flags)) 126 | data[i] = 0x38 127 | i++ 128 | i = encodeVarintIndex(data, i, uint64(m.Crc32)) 129 | return i, nil 130 | } 131 | 132 | func encodeFixed64Index(data []byte, offset int, v uint64) int { 133 | data[offset] = uint8(v) 134 | data[offset+1] = uint8(v >> 8) 135 | data[offset+2] = uint8(v >> 16) 136 | data[offset+3] = uint8(v >> 24) 137 | data[offset+4] = uint8(v >> 32) 138 | data[offset+5] = uint8(v >> 40) 139 | data[offset+6] = uint8(v >> 48) 140 | data[offset+7] = uint8(v >> 56) 141 | return offset + 8 142 | } 143 | func encodeFixed32Index(data []byte, offset int, v uint32) int { 144 | data[offset] = uint8(v) 145 | data[offset+1] = uint8(v >> 8) 146 | data[offset+2] = uint8(v >> 16) 147 | data[offset+3] = uint8(v >> 24) 148 | return offset + 4 149 | } 150 | func encodeVarintIndex(data []byte, offset int, v uint64) int { 151 | for v >= 1<<7 { 152 | data[offset] = uint8(v&0x7f | 0x80) 153 | v >>= 7 154 | offset++ 155 | } 156 | data[offset] = uint8(v) 157 | return offset + 1 158 | } 159 | func (m *IndexMeta) Size() (n int) { 160 | var l int 161 | _ = l 162 | n += 1 + sovIndex(uint64(m.Term)) 163 | n += 1 + sovIndex(uint64(m.Head)) 164 | n += 1 + sovIndex(uint64(m.DataSize)) 165 | return n 166 | } 167 | 168 | func (m *IndexItem) Size() (n int) { 169 | var l int 170 | _ = l 171 | n += 1 + sovIndex(uint64(m.Term)) 172 | n += 1 + sovIndex(uint64(m.Offset)) 173 | n += 1 + sovIndex(uint64(m.ValueSize)) 174 | n += 1 + sovIndex(uint64(m.Timestamp)) 175 | n += 1 + sovIndex(uint64(m.TTL)) 176 | n += 1 + sovIndex(uint64(m.Flags)) 177 | n += 1 + sovIndex(uint64(m.Crc32)) 178 | return n 179 | } 180 | 181 | func sovIndex(x uint64) (n int) { 182 | for { 183 | n++ 184 | x >>= 7 185 | if x == 0 { 186 | break 187 | } 188 | } 189 | return n 190 | } 191 | func sozIndex(x uint64) (n int) { 192 | return sovIndex(uint64((x << 1) ^ uint64((int64(x) >> 63)))) 193 | } 194 | func (m *IndexMeta) Unmarshal(data []byte) error { 195 | var hasFields [1]uint64 196 | l := len(data) 197 | iNdEx := 0 198 | for iNdEx < l { 199 | preIndex := iNdEx 200 | var wire uint64 201 | for shift := uint(0); ; shift += 7 { 202 | if shift >= 64 { 203 | return ErrIntOverflowIndex 204 | } 205 | if iNdEx >= l { 206 | return io.ErrUnexpectedEOF 207 | } 208 | b := data[iNdEx] 209 | iNdEx++ 210 | wire |= (uint64(b) & 0x7F) << shift 211 | if b < 0x80 { 212 | break 213 | } 214 | } 215 | fieldNum := int32(wire >> 3) 216 | wireType := int(wire & 0x7) 217 | if wireType == 4 { 218 | return fmt.Errorf("proto: IndexMeta: wiretype end group for non-group") 219 | } 220 | if fieldNum <= 0 { 221 | return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire) 222 | } 223 | switch fieldNum { 224 | case 1: 225 | if wireType != 0 { 226 | return fmt.Errorf("proto: wrong wireType = %d for field Term", wireType) 227 | } 228 | m.Term = 0 229 | for shift := uint(0); ; shift += 7 { 230 | if shift >= 64 { 231 | return ErrIntOverflowIndex 232 | } 233 | if iNdEx >= l { 234 | return io.ErrUnexpectedEOF 235 | } 236 | b := data[iNdEx] 237 | iNdEx++ 238 | m.Term |= (int64(b) & 0x7F) << shift 239 | if b < 0x80 { 240 | break 241 | } 242 | } 243 | hasFields[0] |= uint64(0x00000001) 244 | case 2: 245 | if wireType != 0 { 246 | return fmt.Errorf("proto: wrong wireType = %d for field Head", wireType) 247 | } 248 | m.Head = 0 249 | for shift := uint(0); ; shift += 7 { 250 | if shift >= 64 { 251 | return ErrIntOverflowIndex 252 | } 253 | if iNdEx >= l { 254 | return io.ErrUnexpectedEOF 255 | } 256 | b := data[iNdEx] 257 | iNdEx++ 258 | m.Head |= (int64(b) & 0x7F) << shift 259 | if b < 0x80 { 260 | break 261 | } 262 | } 263 | hasFields[0] |= uint64(0x00000002) 264 | case 3: 265 | if wireType != 0 { 266 | return fmt.Errorf("proto: wrong wireType = %d for field DataSize", wireType) 267 | } 268 | m.DataSize = 0 269 | for shift := uint(0); ; shift += 7 { 270 | if shift >= 64 { 271 | return ErrIntOverflowIndex 272 | } 273 | if iNdEx >= l { 274 | return io.ErrUnexpectedEOF 275 | } 276 | b := data[iNdEx] 277 | iNdEx++ 278 | m.DataSize |= (int64(b) & 0x7F) << shift 279 | if b < 0x80 { 280 | break 281 | } 282 | } 283 | hasFields[0] |= uint64(0x00000004) 284 | default: 285 | iNdEx = preIndex 286 | skippy, err := skipIndex(data[iNdEx:]) 287 | if err != nil { 288 | return err 289 | } 290 | if skippy < 0 { 291 | return ErrInvalidLengthIndex 292 | } 293 | if (iNdEx + skippy) > l { 294 | return io.ErrUnexpectedEOF 295 | } 296 | iNdEx += skippy 297 | } 298 | } 299 | if hasFields[0]&uint64(0x00000001) == 0 { 300 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Term") 301 | } 302 | if hasFields[0]&uint64(0x00000002) == 0 { 303 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Head") 304 | } 305 | if hasFields[0]&uint64(0x00000004) == 0 { 306 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("DataSize") 307 | } 308 | 309 | if iNdEx > l { 310 | return io.ErrUnexpectedEOF 311 | } 312 | return nil 313 | } 314 | func (m *IndexItem) Unmarshal(data []byte) error { 315 | var hasFields [1]uint64 316 | l := len(data) 317 | iNdEx := 0 318 | for iNdEx < l { 319 | preIndex := iNdEx 320 | var wire uint64 321 | for shift := uint(0); ; shift += 7 { 322 | if shift >= 64 { 323 | return ErrIntOverflowIndex 324 | } 325 | if iNdEx >= l { 326 | return io.ErrUnexpectedEOF 327 | } 328 | b := data[iNdEx] 329 | iNdEx++ 330 | wire |= (uint64(b) & 0x7F) << shift 331 | if b < 0x80 { 332 | break 333 | } 334 | } 335 | fieldNum := int32(wire >> 3) 336 | wireType := int(wire & 0x7) 337 | if wireType == 4 { 338 | return fmt.Errorf("proto: IndexItem: wiretype end group for non-group") 339 | } 340 | if fieldNum <= 0 { 341 | return fmt.Errorf("proto: IndexItem: illegal tag %d (wire type %d)", fieldNum, wire) 342 | } 343 | switch fieldNum { 344 | case 1: 345 | if wireType != 0 { 346 | return fmt.Errorf("proto: wrong wireType = %d for field Term", wireType) 347 | } 348 | m.Term = 0 349 | for shift := uint(0); ; shift += 7 { 350 | if shift >= 64 { 351 | return ErrIntOverflowIndex 352 | } 353 | if iNdEx >= l { 354 | return io.ErrUnexpectedEOF 355 | } 356 | b := data[iNdEx] 357 | iNdEx++ 358 | m.Term |= (int64(b) & 0x7F) << shift 359 | if b < 0x80 { 360 | break 361 | } 362 | } 363 | hasFields[0] |= uint64(0x00000001) 364 | case 2: 365 | if wireType != 0 { 366 | return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) 367 | } 368 | m.Offset = 0 369 | for shift := uint(0); ; shift += 7 { 370 | if shift >= 64 { 371 | return ErrIntOverflowIndex 372 | } 373 | if iNdEx >= l { 374 | return io.ErrUnexpectedEOF 375 | } 376 | b := data[iNdEx] 377 | iNdEx++ 378 | m.Offset |= (int64(b) & 0x7F) << shift 379 | if b < 0x80 { 380 | break 381 | } 382 | } 383 | hasFields[0] |= uint64(0x00000002) 384 | case 3: 385 | if wireType != 0 { 386 | return fmt.Errorf("proto: wrong wireType = %d for field ValueSize", wireType) 387 | } 388 | m.ValueSize = 0 389 | for shift := uint(0); ; shift += 7 { 390 | if shift >= 64 { 391 | return ErrIntOverflowIndex 392 | } 393 | if iNdEx >= l { 394 | return io.ErrUnexpectedEOF 395 | } 396 | b := data[iNdEx] 397 | iNdEx++ 398 | m.ValueSize |= (int32(b) & 0x7F) << shift 399 | if b < 0x80 { 400 | break 401 | } 402 | } 403 | hasFields[0] |= uint64(0x00000004) 404 | case 4: 405 | if wireType != 0 { 406 | return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) 407 | } 408 | m.Timestamp = 0 409 | for shift := uint(0); ; shift += 7 { 410 | if shift >= 64 { 411 | return ErrIntOverflowIndex 412 | } 413 | if iNdEx >= l { 414 | return io.ErrUnexpectedEOF 415 | } 416 | b := data[iNdEx] 417 | iNdEx++ 418 | m.Timestamp |= (int64(b) & 0x7F) << shift 419 | if b < 0x80 { 420 | break 421 | } 422 | } 423 | hasFields[0] |= uint64(0x00000008) 424 | case 5: 425 | if wireType != 0 { 426 | return fmt.Errorf("proto: wrong wireType = %d for field TTL", wireType) 427 | } 428 | m.TTL = 0 429 | for shift := uint(0); ; shift += 7 { 430 | if shift >= 64 { 431 | return ErrIntOverflowIndex 432 | } 433 | if iNdEx >= l { 434 | return io.ErrUnexpectedEOF 435 | } 436 | b := data[iNdEx] 437 | iNdEx++ 438 | m.TTL |= (uint32(b) & 0x7F) << shift 439 | if b < 0x80 { 440 | break 441 | } 442 | } 443 | hasFields[0] |= uint64(0x00000010) 444 | case 6: 445 | if wireType != 0 { 446 | return fmt.Errorf("proto: wrong wireType = %d for field Flags", wireType) 447 | } 448 | m.Flags = 0 449 | for shift := uint(0); ; shift += 7 { 450 | if shift >= 64 { 451 | return ErrIntOverflowIndex 452 | } 453 | if iNdEx >= l { 454 | return io.ErrUnexpectedEOF 455 | } 456 | b := data[iNdEx] 457 | iNdEx++ 458 | m.Flags |= (uint32(b) & 0x7F) << shift 459 | if b < 0x80 { 460 | break 461 | } 462 | } 463 | hasFields[0] |= uint64(0x00000020) 464 | case 7: 465 | if wireType != 0 { 466 | return fmt.Errorf("proto: wrong wireType = %d for field Crc32", wireType) 467 | } 468 | m.Crc32 = 0 469 | for shift := uint(0); ; shift += 7 { 470 | if shift >= 64 { 471 | return ErrIntOverflowIndex 472 | } 473 | if iNdEx >= l { 474 | return io.ErrUnexpectedEOF 475 | } 476 | b := data[iNdEx] 477 | iNdEx++ 478 | m.Crc32 |= (uint32(b) & 0x7F) << shift 479 | if b < 0x80 { 480 | break 481 | } 482 | } 483 | default: 484 | iNdEx = preIndex 485 | skippy, err := skipIndex(data[iNdEx:]) 486 | if err != nil { 487 | return err 488 | } 489 | if skippy < 0 { 490 | return ErrInvalidLengthIndex 491 | } 492 | if (iNdEx + skippy) > l { 493 | return io.ErrUnexpectedEOF 494 | } 495 | iNdEx += skippy 496 | } 497 | } 498 | if hasFields[0]&uint64(0x00000001) == 0 { 499 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Term") 500 | } 501 | if hasFields[0]&uint64(0x00000002) == 0 { 502 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Offset") 503 | } 504 | if hasFields[0]&uint64(0x00000004) == 0 { 505 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("ValueSize") 506 | } 507 | if hasFields[0]&uint64(0x00000008) == 0 { 508 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Timestamp") 509 | } 510 | if hasFields[0]&uint64(0x00000010) == 0 { 511 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("TTL") 512 | } 513 | if hasFields[0]&uint64(0x00000020) == 0 { 514 | return github_com_gogo_protobuf_proto.NewRequiredNotSetError("Flags") 515 | } 516 | 517 | if iNdEx > l { 518 | return io.ErrUnexpectedEOF 519 | } 520 | return nil 521 | } 522 | func skipIndex(data []byte) (n int, err error) { 523 | l := len(data) 524 | iNdEx := 0 525 | for iNdEx < l { 526 | var wire uint64 527 | for shift := uint(0); ; shift += 7 { 528 | if shift >= 64 { 529 | return 0, ErrIntOverflowIndex 530 | } 531 | if iNdEx >= l { 532 | return 0, io.ErrUnexpectedEOF 533 | } 534 | b := data[iNdEx] 535 | iNdEx++ 536 | wire |= (uint64(b) & 0x7F) << shift 537 | if b < 0x80 { 538 | break 539 | } 540 | } 541 | wireType := int(wire & 0x7) 542 | switch wireType { 543 | case 0: 544 | for shift := uint(0); ; shift += 7 { 545 | if shift >= 64 { 546 | return 0, ErrIntOverflowIndex 547 | } 548 | if iNdEx >= l { 549 | return 0, io.ErrUnexpectedEOF 550 | } 551 | iNdEx++ 552 | if data[iNdEx-1] < 0x80 { 553 | break 554 | } 555 | } 556 | return iNdEx, nil 557 | case 1: 558 | iNdEx += 8 559 | return iNdEx, nil 560 | case 2: 561 | var length int 562 | for shift := uint(0); ; shift += 7 { 563 | if shift >= 64 { 564 | return 0, ErrIntOverflowIndex 565 | } 566 | if iNdEx >= l { 567 | return 0, io.ErrUnexpectedEOF 568 | } 569 | b := data[iNdEx] 570 | iNdEx++ 571 | length |= (int(b) & 0x7F) << shift 572 | if b < 0x80 { 573 | break 574 | } 575 | } 576 | iNdEx += length 577 | if length < 0 { 578 | return 0, ErrInvalidLengthIndex 579 | } 580 | return iNdEx, nil 581 | case 3: 582 | for { 583 | var innerWire uint64 584 | var start int = iNdEx 585 | for shift := uint(0); ; shift += 7 { 586 | if shift >= 64 { 587 | return 0, ErrIntOverflowIndex 588 | } 589 | if iNdEx >= l { 590 | return 0, io.ErrUnexpectedEOF 591 | } 592 | b := data[iNdEx] 593 | iNdEx++ 594 | innerWire |= (uint64(b) & 0x7F) << shift 595 | if b < 0x80 { 596 | break 597 | } 598 | } 599 | innerWireType := int(innerWire & 0x7) 600 | if innerWireType == 4 { 601 | break 602 | } 603 | next, err := skipIndex(data[start:]) 604 | if err != nil { 605 | return 0, err 606 | } 607 | iNdEx = start + next 608 | } 609 | return iNdEx, nil 610 | case 4: 611 | return iNdEx, nil 612 | case 5: 613 | iNdEx += 4 614 | return iNdEx, nil 615 | default: 616 | return 0, fmt.Errorf("proto: illegal wireType %d", wireType) 617 | } 618 | } 619 | panic("unreachable") 620 | } 621 | 622 | var ( 623 | ErrInvalidLengthIndex = fmt.Errorf("proto: negative length found during unmarshaling") 624 | ErrIntOverflowIndex = fmt.Errorf("proto: integer overflow") 625 | ) 626 | 627 | func init() { proto.RegisterFile("index.proto", fileDescriptorIndex) } 628 | 629 | var fileDescriptorIndex = []byte{ 630 | // 239 bytes of a gzipped FileDescriptorProto 631 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xce, 0xcc, 0x4b, 0x49, 632 | 0xad, 0xd0, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x4d, 0x4e, 0x4c, 0xce, 0x48, 0x95, 0xd2, 633 | 0x4d, 0xcf, 0x2c, 0xc9, 0x28, 0x4d, 0xd2, 0x4b, 0xce, 0xcf, 0xd5, 0x4f, 0xcf, 0x4f, 0xcf, 0xd7, 634 | 0x07, 0xcb, 0x26, 0x95, 0xa6, 0x81, 0x79, 0x60, 0x0e, 0x98, 0x05, 0xd1, 0xa5, 0xe4, 0xcd, 0xc5, 635 | 0xe9, 0x09, 0x32, 0xc4, 0x37, 0xb5, 0x24, 0x51, 0x48, 0x88, 0x8b, 0x25, 0x24, 0xb5, 0x28, 0x57, 636 | 0x82, 0x51, 0x81, 0x49, 0x83, 0xd9, 0x89, 0xe5, 0xc4, 0x3d, 0x79, 0x06, 0x90, 0x98, 0x47, 0x6a, 637 | 0x62, 0x8a, 0x04, 0x13, 0x92, 0x98, 0x18, 0x17, 0x87, 0x4b, 0x62, 0x49, 0x62, 0x70, 0x66, 0x55, 638 | 0xaa, 0x04, 0x33, 0x42, 0x5c, 0x69, 0x09, 0x23, 0xd4, 0x34, 0xcf, 0x92, 0xd4, 0x5c, 0xac, 0xa6, 639 | 0x89, 0x70, 0xb1, 0xf9, 0xa7, 0xa5, 0x15, 0xa7, 0x96, 0xa0, 0x98, 0x27, 0xce, 0xc5, 0x19, 0x96, 640 | 0x98, 0x53, 0x9a, 0x0a, 0x37, 0x90, 0x15, 0x21, 0x11, 0x92, 0x99, 0x9b, 0x5a, 0x5c, 0x92, 0x98, 641 | 0x5b, 0x20, 0xc1, 0x82, 0xa4, 0x43, 0x90, 0x8b, 0x39, 0x24, 0xc4, 0x47, 0x82, 0x55, 0x81, 0x49, 642 | 0x83, 0x17, 0x2a, 0x24, 0xcc, 0xc5, 0xea, 0x96, 0x93, 0x98, 0x5e, 0x2c, 0xc1, 0x86, 0x2a, 0xe8, 643 | 0x5c, 0x94, 0x6c, 0x6c, 0x24, 0xc1, 0xae, 0xc0, 0x08, 0x13, 0x74, 0x12, 0x39, 0xf1, 0x50, 0x8e, 644 | 0xe1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x9c, 0xf0, 0x58, 645 | 0x8e, 0x01, 0x10, 0x00, 0x00, 0xff, 0xff, 0x95, 0x71, 0x56, 0xf5, 0x4d, 0x01, 0x00, 0x00, 646 | } 647 | --------------------------------------------------------------------------------