├── value.go ├── polarisxu-qrcode-m.jpg ├── .gitignore ├── go.mod ├── README.md ├── len.go ├── lru ├── lru_test.go └── lru.go ├── fifo ├── fifo_test.go └── fifo.go ├── lfu ├── lfu_test.go ├── queue.go └── lfu.go ├── fast ├── hasher.go ├── cache.go └── shard.go ├── tour_cache.go ├── tour_cache_test.go ├── LICENSE ├── cache.go └── go.sum /value.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Value interface { 4 | Len() int 5 | } 6 | -------------------------------------------------------------------------------- /polarisxu-qrcode-m.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/go-programming-tour-book/cache-example/HEAD/polarisxu-qrcode-m.jpg -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/polaris1119/cache 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/allegro/bigcache v1.2.1 7 | github.com/allegro/bigcache/v2 v2.2.0 8 | github.com/astaxie/beego v1.12.1 9 | github.com/hashicorp/golang-lru v0.5.4 10 | github.com/matryer/is v1.3.0 11 | github.com/stretchr/testify v1.5.1 // indirect 12 | github.com/syndtr/goleveldb v1.0.0 13 | golang.org/x/tools v0.0.0-20200312031852-3dc7fec7888e // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cache-example 2 | 3 | 《Go 语言编程之旅：一起用 Go 做项目》 第五章进程内缓存相关实例代码。 4 | 5 | ## 关于本书 6 | 7 | 本书涵盖 Go 语言的各大经典实战，不介绍 Go 语言的语法基础，内容面向项目实践，同时会针对核心细节进行分析。而在实际项目迭代中，常常会出现或多或少的事故，因此本书也针对 Go 语言的大杀器（分析工具）以及常见问题进行了全面讲解。 8 | 9 | 本书适合已经大致学习了 Go 语言的基础语法后，想要跨越到下一个阶段的开发人员，可以填补该阶段的空白和进一步拓展你的思维方向。 10 | 11 | - 作者：陈剑煜（煎鱼），GitHub：[@eddycjy](https://github.com/eddycjy)，微信公众号：脑子进煎鱼了。 12 | - 作者：徐新华（polaris），GitHub：[@polaris](https://github.com/polaris1119)，微信公众号：Go语言中文网。 13 | 14 | ## 购买链接 15 | 16 | - 京东：https://item.jd.com/12685249.html 17 | - 当当：http://product.dangdang.com/28982027.html 18 | - 天猫：https://detail.tmall.com/item.htm?id=622185710833 19 | 20 | ## 关注我 21 | 22 | polarisxu 个人公众号 23 | 24 | ![](polarisxu-qrcode-m.jpg) 25 | -------------------------------------------------------------------------------- /len.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "runtime" 6 | ) 7 | 8 | func CalcLen(value interface{}) int { 9 | var n int 10 | switch v := value.(type) { 11 | case Value: 12 | n = v.Len() 13 | case string: 14 | if runtime.GOARCH == "amd64" { 15 | n = 16 + len(v) 16 | } else { 17 | n = 8 + len(v) 18 | } 19 | case []byte: 20 | n = len(v) 21 | case bool, uint8, int8: 22 | n = 1 23 | case int16, uint16: 24 | n = 2 25 | case int32, uint32, float32: 26 | n = 4 27 | case int64, uint64, float64: 28 | n = 8 29 | case int, uint: 30 | if runtime.GOARCH == "amd64" { 31 | n = 8 32 | } else { 33 | n = 4 34 | } 35 | case complex64: 36 | n = 8 37 | case complex128: 38 | n = 16 39 | default: 40 | panic(fmt.Sprintf("%T is not implement cache.Value", value)) 41 | } 42 | 43 | return n 44 | } 45 | -------------------------------------------------------------------------------- /lru/lru_test.go: -------------------------------------------------------------------------------- 1 | package lru_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/matryer/is" 7 | "github.com/polaris1119/cache/lru" 8 | ) 9 | 10 | func TestSet(t *testing.T) { 11 | is := is.New(t) 12 | 13 | cache := lru.New(24, nil) 14 | cache.DelOldest() 15 | cache.Set("k1", 1) 16 | v := cache.Get("k1") 17 | is.Equal(v, 1) 18 | 19 | cache.Del("k1") 20 | is.Equal(0, cache.Len()) 21 | 22 | // cache.Set("k2", time.Now()) 23 | } 24 | 25 | func TestOnEvicted(t *testing.T) { 26 | is := is.New(t) 27 | 28 | keys := make([]string, 0, 8) 29 | onEvicted := func(key string, value interface{}) { 30 | keys = append(keys, key) 31 | } 32 | cache := lru.New(16, onEvicted) 33 | 34 | cache.Set("k1", 1) 35 | cache.Set("k2", 2) 36 | cache.Get("k1") 37 | cache.Set("k3", 3) 38 | cache.Get("k1") 39 | cache.Set("k4", 4) 40 | 41 | expected := []string{"k2", "k3"} 42 | 43 | is.Equal(expected, keys) 44 | is.Equal(2, cache.Len()) 45 | } 46 | -------------------------------------------------------------------------------- /fifo/fifo_test.go: -------------------------------------------------------------------------------- 1 | package fifo_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/matryer/is" 7 | "github.com/polaris1119/cache/fifo" 8 | ) 9 | 10 | func TestSetGet(t *testing.T) { 11 | is := is.New(t) 12 | 13 | cache := fifo.New(24, nil) 14 | cache.DelOldest() 15 | cache.Set("k1", 1) 16 | v := cache.Get("k1") 17 | is.Equal(v, 1) 18 | 19 | cache.Del("k1") 20 | is.Equal(1, cache.Len()) // expect to be the same 21 | 22 | // cache.Set("k2", time.Now()) 23 | } 24 | 25 | func TestOnEvicted(t *testing.T) { 26 | is := is.New(t) 27 | 28 | keys := make([]string, 0, 8) 29 | onEvicted := func(key string, value interface{}) { 30 | keys = append(keys, key) 31 | } 32 | cache := fifo.New(16, onEvicted) 33 | 34 | cache.Set("k1", 1) 35 | cache.Set("k2", 2) 36 | cache.Get("k1") 37 | cache.Set("k3", 3) 38 | cache.Get("k1") 39 | cache.Set("k4", 4) 40 | 41 | expected := []string{"k1", "k2"} 42 | 43 | is.Equal(expected, keys) 44 | is.Equal(2, cache.Len()) 45 | } 46 | -------------------------------------------------------------------------------- /lfu/lfu_test.go: -------------------------------------------------------------------------------- 1 | package lfu_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/matryer/is" 7 | "github.com/polaris1119/cache/lfu" 8 | ) 9 | 10 | func TestSet(t *testing.T) { 11 | is := is.New(t) 12 | 13 | cache := lfu.New(24, nil) 14 | cache.DelOldest() 15 | cache.Set("k1", 1) 16 | v := cache.Get("k1") 17 | is.Equal(v, 1) 18 | 19 | cache.Del("k1") 20 | is.Equal(0, cache.Len()) 21 | 22 | // cache.Set("k2", time.Now()) 23 | } 24 | 25 | func TestOnEvicted(t *testing.T) { 26 | is := is.New(t) 27 | 28 | keys := make([]string, 0, 8) 29 | onEvicted := func(key string, value interface{}) { 30 | keys = append(keys, key) 31 | } 32 | cache := lfu.New(32, onEvicted) 33 | 34 | cache.Set("k1", 1) 35 | cache.Set("k2", 2) 36 | // cache.Get("k1") 37 | // cache.Get("k1") 38 | // cache.Get("k2") 39 | cache.Set("k3", 3) 40 | cache.Set("k4", 4) 41 | 42 | expected := []string{"k1", "k3"} 43 | 44 | is.Equal(expected, keys) 45 | is.Equal(2, cache.Len()) 46 | } 47 | -------------------------------------------------------------------------------- /fast/hasher.go: -------------------------------------------------------------------------------- 1 | package fast 2 | 3 | // FNV-1a 的 Hash 实现，来源 BigCache 4 | 5 | // newDefaultHasher returns a new 64-bit FNV-1a Hasher which makes no memory allocations. 6 | // Its Sum64 method will lay the value out in big-endian byte order. 7 | // See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function 8 | func newDefaultHasher() fnv64a { 9 | return fnv64a{} 10 | } 11 | 12 | type fnv64a struct{} 13 | 14 | const ( 15 | // offset64 FNVa offset basis. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash 16 | offset64 = 14695981039346656037 17 | // prime64 FNVa prime value. See https://en.wikipedia.org/wiki/Fowler–Noll–Vo_hash_function#FNV-1a_hash 18 | prime64 = 1099511628211 19 | ) 20 | 21 | // Sum64 gets the string and returns its uint64 hash value. 22 | func (f fnv64a) Sum64(key string) uint64 { 23 | var hash uint64 = offset64 24 | for i := 0; i < len(key); i++ { 25 | hash ^= uint64(key[i]) 26 | hash *= prime64 27 | } 28 | 29 | return hash 30 | } 31 | -------------------------------------------------------------------------------- /tour_cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type Getter interface { 4 | Get(key string) interface{} 5 | } 6 | 7 | type GetFunc func(key string) interface{} 8 | 9 | func (f GetFunc) Get(key string) interface{} { 10 | return f(key) 11 | } 12 | 13 | type TourCache struct { 14 | mainCache *safeCache 15 | getter Getter 16 | } 17 | 18 | func NewTourCache(getter Getter, cache Cache) *TourCache { 19 | return &TourCache{ 20 | mainCache: newSafeCache(cache), 21 | getter: getter, 22 | } 23 | } 24 | 25 | func (t *TourCache) Get(key string) interface{} { 26 | val := t.mainCache.get(key) 27 | if val != nil { 28 | return val 29 | } 30 | 31 | if t.getter != nil { 32 | val = t.getter.Get(key) 33 | if val == nil { 34 | return nil 35 | } 36 | t.mainCache.set(key, val) 37 | return val 38 | } 39 | 40 | return nil 41 | } 42 | 43 | func (t *TourCache) Set(key string, val interface{}) { 44 | if val == nil { 45 | return 46 | } 47 | t.mainCache.set(key, val) 48 | } 49 | 50 | func (t *TourCache) Stat() *Stat { 51 | return t.mainCache.stat() 52 | } 53 | -------------------------------------------------------------------------------- /tour_cache_test.go: -------------------------------------------------------------------------------- 1 | package cache_test 2 | 3 | import ( 4 | "log" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/matryer/is" 9 | "github.com/polaris1119/cache" 10 | "github.com/polaris1119/cache/lru" 11 | ) 12 | 13 | func TestTourCacheGet(t *testing.T) { 14 | db := map[string]string{ 15 | "key1": "val1", 16 | "key2": "val2", 17 | "key3": "val3", 18 | "key4": "val4", 19 | } 20 | getter := cache.GetFunc(func(key string) interface{} { 21 | log.Println("[From DB] find key", key) 22 | 23 | if val, ok := db[key]; ok { 24 | return val 25 | } 26 | return nil 27 | }) 28 | tourCache := cache.NewTourCache(getter, lru.New(0, nil)) 29 | 30 | is := is.New(t) 31 | 32 | var wg sync.WaitGroup 33 | 34 | for k, v := range db { 35 | wg.Add(1) 36 | go func(k, v string) { 37 | defer wg.Done() 38 | is.Equal(tourCache.Get(k), v) 39 | 40 | is.Equal(tourCache.Get(k), v) 41 | }(k, v) 42 | } 43 | wg.Wait() 44 | 45 | is.Equal(tourCache.Get("unknown"), nil) 46 | is.Equal(tourCache.Get("unknown"), nil) 47 | 48 | is.Equal(tourCache.Stat().NGet, 10) 49 | is.Equal(tourCache.Stat().NHit, 4) 50 | } 51 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 徐新华 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /lfu/queue.go: -------------------------------------------------------------------------------- 1 | package lfu 2 | 3 | import ( 4 | "container/heap" 5 | 6 | "github.com/polaris1119/cache" 7 | ) 8 | 9 | type entry struct { 10 | key string 11 | value interface{} 12 | weight int 13 | index int 14 | } 15 | 16 | func (e *entry) Len() int { 17 | return cache.CalcLen(e.value) + 4 + 4 18 | } 19 | 20 | type queue []*entry 21 | 22 | func (q queue) Len() int { 23 | return len(q) 24 | } 25 | 26 | func (q queue) Less(i, j int) bool { 27 | return q[i].weight < q[j].weight 28 | } 29 | 30 | func (q queue) Swap(i, j int) { 31 | q[i], q[j] = q[j], q[i] 32 | q[i].index = i 33 | q[j].index = j 34 | } 35 | 36 | func (q *queue) Push(x interface{}) { 37 | n := len(*q) 38 | en := x.(*entry) 39 | en.index = n 40 | *q = append(*q, en) 41 | } 42 | 43 | func (q *queue) Pop() interface{} { 44 | old := *q 45 | n := len(old) 46 | en := old[n-1] 47 | old[n-1] = nil // avoid memory leak 48 | en.index = -1 // for safety 49 | *q = old[0 : n-1] 50 | return en 51 | } 52 | 53 | // update modifies the weight and value of an entry in the queue. 54 | func (q *queue) update(en *entry, value interface{}, weight int) { 55 | en.value = value 56 | en.weight = weight 57 | heap.Fix(q, en.index) 58 | } 59 | -------------------------------------------------------------------------------- /fast/cache.go: -------------------------------------------------------------------------------- 1 | package fast 2 | 3 | type fastCache struct { 4 | shards []*cacheShard 5 | shardMask uint64 6 | hash fnv64a 7 | } 8 | 9 | func NewFastCache(maxEntries, shardsNum int, onEvicted func(key string, value interface{})) *fastCache { 10 | fastCache := &fastCache{ 11 | hash: newDefaultHasher(), 12 | shards: make([]*cacheShard, shardsNum), 13 | shardMask: uint64(shardsNum - 1), 14 | } 15 | for i := 0; i < shardsNum; i++ { 16 | fastCache.shards[i] = newCacheShard(maxEntries, onEvicted) 17 | } 18 | 19 | return fastCache 20 | } 21 | 22 | func (c *fastCache) getShard(key string) *cacheShard { 23 | hashedKey := c.hash.Sum64(key) 24 | return c.shards[hashedKey&c.shardMask] 25 | } 26 | 27 | func (c *fastCache) Set(key string, value interface{}) { 28 | c.getShard(key).set(key, value) 29 | } 30 | 31 | func (c *fastCache) Get(key string) interface{} { 32 | return c.getShard(key).get(key) 33 | } 34 | 35 | func (c *fastCache) Del(key string) { 36 | c.getShard(key).del(key) 37 | } 38 | 39 | func (c *fastCache) Len() int { 40 | length := 0 41 | for _, shard := range c.shards { 42 | length += shard.len() 43 | } 44 | return length 45 | } 46 | 47 | func (c *fastCache) DelOldest() { 48 | panic("no implements") 49 | } 50 | -------------------------------------------------------------------------------- /cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // Cache 缓存接口 8 | type Cache interface { 9 | Set(key string, value interface{}) 10 | Get(key string) interface{} 11 | Del(key string) 12 | DelOldest() 13 | Len() int 14 | } 15 | 16 | // DefaultMaxBytes 默认允许占用的最大内存 17 | const DefaultMaxBytes = 1 << 29 18 | 19 | // safeCache 并发安全缓存 20 | type safeCache struct { 21 | m sync.RWMutex 22 | cache Cache 23 | 24 | nget, nhit int 25 | } 26 | 27 | func newSafeCache(cache Cache) *safeCache { 28 | return &safeCache{ 29 | cache: cache, 30 | } 31 | } 32 | 33 | func (sc *safeCache) set(key string, value interface{}) { 34 | sc.m.Lock() 35 | defer sc.m.Unlock() 36 | sc.cache.Set(key, value) 37 | } 38 | 39 | func (sc *safeCache) get(key string) interface{} { 40 | sc.m.RLock() 41 | defer sc.m.RUnlock() 42 | sc.nget++ 43 | if sc.cache == nil { 44 | return nil 45 | } 46 | 47 | v := sc.cache.Get(key) 48 | if v != nil { 49 | // log.Println("[TourCache] hit") 50 | sc.nhit++ 51 | } 52 | 53 | return v 54 | } 55 | 56 | func (sc *safeCache) stat() *Stat { 57 | sc.m.RLock() 58 | defer sc.m.RUnlock() 59 | return &Stat{ 60 | NHit: sc.nhit, 61 | NGet: sc.nget, 62 | } 63 | } 64 | 65 | type Stat struct { 66 | NHit, NGet int 67 | } 68 | -------------------------------------------------------------------------------- /lfu/lfu.go: -------------------------------------------------------------------------------- 1 | package lfu 2 | 3 | import ( 4 | "container/heap" 5 | 6 | "github.com/polaris1119/cache" 7 | ) 8 | 9 | // lfu 是一个 LFU cache。它不是并发安全的。 10 | type lfu struct { 11 | // 缓存最大的容量，单位字节； 12 | // groupcache 使用的是最大存放 entry 个数 13 | maxBytes int 14 | // 当一个 entry 从缓存中移除是调用该回调函数，默认为 nil 15 | // groupcache 中的 key 是任意的可比较类型；value 是 interface{} 16 | onEvicted func(key string, value interface{}) 17 | 18 | // 已使用的字节数，只包括值，key 不算 19 | usedBytes int 20 | 21 | queue *queue 22 | cache map[string]*entry 23 | } 24 | 25 | // New 创建一个新的 Cache，如果 maxBytes 是 0，表示没有容量限制 26 | func New(maxBytes int, onEvicted func(key string, value interface{})) cache.Cache { 27 | q := make(queue, 0, 1024) 28 | return &lfu{ 29 | maxBytes: maxBytes, 30 | onEvicted: onEvicted, 31 | queue: &q, 32 | cache: make(map[string]*entry), 33 | } 34 | } 35 | 36 | // Set 往 Cache 增加一个元素（如果已经存在，更新值，并增加权重，重新构建堆） 37 | func (l *lfu) Set(key string, value interface{}) { 38 | if e, ok := l.cache[key]; ok { 39 | l.usedBytes = l.usedBytes - cache.CalcLen(e.value) + cache.CalcLen(value) 40 | l.queue.update(e, value, e.weight+1) 41 | return 42 | } 43 | 44 | en := &entry{key: key, value: value} 45 | heap.Push(l.queue, en) 46 | l.cache[key] = en 47 | 48 | l.usedBytes += en.Len() 49 | if l.maxBytes > 0 && l.usedBytes > l.maxBytes { 50 | l.removeElement(heap.Pop(l.queue)) 51 | } 52 | } 53 | 54 | // Get 从 cache 中获取 key 对应的值，nil 表示 key 不存在 55 | func (l *lfu) Get(key string) interface{} { 56 | if e, ok := l.cache[key]; ok { 57 | l.queue.update(e, e.value, e.weight+1) 58 | return e.value 59 | } 60 | 61 | return nil 62 | } 63 | 64 | // Del 从 cache 中删除 key 对应的元素 65 | func (l *lfu) Del(key string) { 66 | if e, ok := l.cache[key]; ok { 67 | heap.Remove(l.queue, e.index) 68 | l.removeElement(e) 69 | } 70 | } 71 | 72 | // DelOldest 从 cache 中删除最旧的记录 73 | func (l *lfu) DelOldest() { 74 | if l.queue.Len() == 0 { 75 | return 76 | } 77 | l.removeElement(heap.Pop(l.queue)) 78 | } 79 | 80 | // Len 返回当前 cache 中的记录数 81 | func (l *lfu) Len() int { 82 | return l.queue.Len() 83 | } 84 | 85 | func (l *lfu) removeElement(x interface{}) { 86 | if x == nil { 87 | return 88 | } 89 | 90 | en := x.(*entry) 91 | 92 | delete(l.cache, en.key) 93 | 94 | l.usedBytes -= en.Len() 95 | 96 | if l.onEvicted != nil { 97 | l.onEvicted(en.key, en.value) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /fifo/fifo.go: -------------------------------------------------------------------------------- 1 | package fifo 2 | 3 | import ( 4 | "container/list" 5 | 6 | "github.com/polaris1119/cache" 7 | ) 8 | 9 | // fifo 是一个 FIFO cache。它不是并发安全的。 10 | type fifo struct { 11 | // 缓存最大的容量，单位字节； 12 | // groupcache 使用的是最大存放 entry 个数 13 | maxBytes int 14 | // 当一个 entry 从缓存中移除是调用该回调函数，默认为 nil 15 | // groupcache 中的 key 是任意的可比较类型；value 是 interface{} 16 | onEvicted func(key string, value interface{}) 17 | 18 | // 已使用的字节数，只包括值，key 不算 19 | usedBytes int 20 | 21 | ll *list.List 22 | cache map[string]*list.Element 23 | } 24 | 25 | type entry struct { 26 | key string 27 | value interface{} 28 | } 29 | 30 | func (e *entry) Len() int { 31 | return cache.CalcLen(e.value) 32 | } 33 | 34 | // New 创建一个新的 Cache，如果 maxBytes 是 0，表示没有容量限制 35 | func New(maxBytes int, onEvicted func(key string, value interface{})) cache.Cache { 36 | return &fifo{ 37 | maxBytes: maxBytes, 38 | onEvicted: onEvicted, 39 | ll: list.New(), 40 | cache: make(map[string]*list.Element), 41 | } 42 | } 43 | 44 | // Set 往 Cache 尾部增加一个元素（如果已经存在，则放入尾部，并修改值） 45 | func (f *fifo) Set(key string, value interface{}) { 46 | if e, ok := f.cache[key]; ok { 47 | f.ll.MoveToBack(e) 48 | en := e.Value.(*entry) 49 | f.usedBytes = f.usedBytes - cache.CalcLen(en.value) + cache.CalcLen(value) 50 | en.value = value 51 | return 52 | } 53 | 54 | en := &entry{key, value} 55 | e := f.ll.PushBack(en) 56 | f.cache[key] = e 57 | 58 | f.usedBytes += en.Len() 59 | if f.maxBytes > 0 && f.usedBytes > f.maxBytes { 60 | f.DelOldest() 61 | } 62 | } 63 | 64 | // Get 从 cache 中获取 key 对应的值，nil 表示 key 不存在 65 | func (f *fifo) Get(key string) interface{} { 66 | if e, ok := f.cache[key]; ok { 67 | return e.Value.(*entry).value 68 | } 69 | 70 | return nil 71 | } 72 | 73 | // Del 从 cache 中删除 key 对应的记录 74 | func (f *fifo) Del(key string) { 75 | if e, ok := f.cache[key]; ok { 76 | f.removeElement(e) 77 | } 78 | } 79 | 80 | // DelOldest 从 cache 中删除最旧的记录 81 | func (f *fifo) DelOldest() { 82 | f.removeElement(f.ll.Front()) 83 | } 84 | 85 | // Len 返回当前 cache 中的记录数 86 | func (f *fifo) Len() int { 87 | return f.ll.Len() 88 | } 89 | 90 | func (f *fifo) removeElement(e *list.Element) { 91 | if e == nil { 92 | return 93 | } 94 | 95 | f.ll.Remove(e) 96 | en := e.Value.(*entry) 97 | f.usedBytes -= en.Len() 98 | delete(f.cache, en.key) 99 | 100 | if f.onEvicted != nil { 101 | f.onEvicted(en.key, en.value) 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /lru/lru.go: -------------------------------------------------------------------------------- 1 | package lru 2 | 3 | import ( 4 | "container/list" 5 | 6 | "github.com/polaris1119/cache" 7 | ) 8 | 9 | // lru 是一个 LRU cache。它不是并发安全的。 10 | type lru struct { 11 | // 缓存最大的容量，单位字节； 12 | // groupcache 使用的是最大存放 entry 个数 13 | maxBytes int 14 | // 当一个 entry 从缓存中移除是调用该回调函数，默认为 nil 15 | // groupcache 中的 key 是任意的可比较类型；value 是 interface{} 16 | onEvicted func(key string, value interface{}) 17 | 18 | // 已使用的字节数，只包括值，key 不算 19 | usedBytes int 20 | 21 | ll *list.List 22 | cache map[string]*list.Element 23 | } 24 | 25 | type entry struct { 26 | key string 27 | value interface{} 28 | } 29 | 30 | func (e *entry) Len() int { 31 | return cache.CalcLen(e.value) 32 | } 33 | 34 | // New 创建一个新的 Cache，如果 maxBytes 是 0，表示没有容量限制 35 | func New(maxBytes int, onEvicted func(key string, value interface{})) cache.Cache { 36 | return &lru{ 37 | maxBytes: maxBytes, 38 | onEvicted: onEvicted, 39 | ll: list.New(), 40 | cache: make(map[string]*list.Element), 41 | } 42 | } 43 | 44 | // Set 往 Cache 尾部增加一个元素（如果已经存在，则放入尾部，并更新值） 45 | func (l *lru) Set(key string, value interface{}) { 46 | if e, ok := l.cache[key]; ok { 47 | l.ll.MoveToBack(e) 48 | en := e.Value.(*entry) 49 | l.usedBytes = l.usedBytes - cache.CalcLen(en.value) + cache.CalcLen(value) 50 | en.value = value 51 | return 52 | } 53 | 54 | en := &entry{key, value} 55 | e := l.ll.PushBack(en) 56 | l.cache[key] = e 57 | 58 | l.usedBytes += en.Len() 59 | if l.maxBytes > 0 && l.usedBytes > l.maxBytes { 60 | l.DelOldest() 61 | } 62 | } 63 | 64 | // Get 从 cache 中获取 key 对应的值，nil 表示 key 不存在 65 | func (l *lru) Get(key string) interface{} { 66 | if e, ok := l.cache[key]; ok { 67 | l.ll.MoveToBack(e) 68 | return e.Value.(*entry).value 69 | } 70 | 71 | return nil 72 | } 73 | 74 | // Del 从 cache 中删除 key 对应的元素 75 | func (l *lru) Del(key string) { 76 | if e, ok := l.cache[key]; ok { 77 | l.removeElement(e) 78 | } 79 | } 80 | 81 | // DelOldest 从 cache 中删除最旧的记录 82 | func (l *lru) DelOldest() { 83 | l.removeElement(l.ll.Front()) 84 | } 85 | 86 | // Len 返回当前 cache 中的记录数 87 | func (l *lru) Len() int { 88 | return l.ll.Len() 89 | } 90 | 91 | func (l *lru) removeElement(e *list.Element) { 92 | if e == nil { 93 | return 94 | } 95 | 96 | l.ll.Remove(e) 97 | en := e.Value.(*entry) 98 | l.usedBytes -= en.Len() 99 | delete(l.cache, en.key) 100 | 101 | if l.onEvicted != nil { 102 | l.onEvicted(en.key, en.value) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /fast/shard.go: -------------------------------------------------------------------------------- 1 | package fast 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | ) 7 | 8 | type cacheShard struct { 9 | locker sync.RWMutex 10 | 11 | // 最大存放 entry 个数 12 | maxEntries int 13 | // 当一个 entry 从缓存中移除是调用该回调函数，默认为 nil 14 | // groupcache 中的 key 是任意的可比较类型；value 是 interface{} 15 | onEvicted func(key string, value interface{}) 16 | 17 | ll *list.List 18 | cache map[string]*list.Element 19 | } 20 | 21 | type entry struct { 22 | key string 23 | value interface{} 24 | } 25 | 26 | // new 创建一个新的 cacheShard，如果 maxBytes 是 0，表示没有容量限制 27 | func newCacheShard(maxEntries int, onEvicted func(key string, value interface{})) *cacheShard { 28 | return &cacheShard{ 29 | maxEntries: maxEntries, 30 | onEvicted: onEvicted, 31 | ll: list.New(), 32 | cache: make(map[string]*list.Element), 33 | } 34 | } 35 | 36 | // set 往 Cache 尾部增加一个元素（如果已经存在，则放入尾部，并更新值） 37 | func (c *cacheShard) set(key string, value interface{}) { 38 | c.locker.Lock() 39 | defer c.locker.Unlock() 40 | 41 | if e, ok := c.cache[key]; ok { 42 | c.ll.MoveToBack(e) 43 | en := e.Value.(*entry) 44 | en.value = value 45 | return 46 | } 47 | 48 | en := &entry{key, value} 49 | e := c.ll.PushBack(en) 50 | c.cache[key] = e 51 | 52 | if c.maxEntries > 0 && c.ll.Len() > c.maxEntries { 53 | c.removeElement(c.ll.Front()) 54 | } 55 | } 56 | 57 | // get 从 cache 中获取 key 对应的值，nil 表示 key 不存在 58 | func (c *cacheShard) get(key string) interface{} { 59 | c.locker.RLock() 60 | defer c.locker.RUnlock() 61 | 62 | if e, ok := c.cache[key]; ok { 63 | c.ll.MoveToBack(e) 64 | return e.Value.(*entry).value 65 | } 66 | 67 | return nil 68 | } 69 | 70 | // del 从 cache 中删除 key 对应的元素 71 | func (c *cacheShard) del(key string) { 72 | c.locker.Lock() 73 | defer c.locker.Unlock() 74 | 75 | if e, ok := c.cache[key]; ok { 76 | c.removeElement(e) 77 | } 78 | } 79 | 80 | // delOldest 从 cache 中删除最旧的记录 81 | func (c *cacheShard) delOldest() { 82 | c.locker.Lock() 83 | defer c.locker.Unlock() 84 | 85 | c.removeElement(c.ll.Front()) 86 | } 87 | 88 | // len 返回当前 cache 中的记录数 89 | func (c *cacheShard) len() int { 90 | c.locker.RLock() 91 | defer c.locker.RUnlock() 92 | 93 | return c.ll.Len() 94 | } 95 | 96 | func (c *cacheShard) removeElement(e *list.Element) { 97 | if e == nil { 98 | return 99 | } 100 | 101 | c.ll.Remove(e) 102 | en := e.Value.(*entry) 103 | delete(c.cache, en.key) 104 | 105 | if c.onEvicted != nil { 106 | c.onEvicted(en.key, en.value) 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= 2 | github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= 3 | github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKSc= 4 | github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= 5 | github.com/allegro/bigcache/v2 v2.2.0 h1:rB+Vjrjp7OijGRFMeUpJYXjsn3lN6ooIvomEBKXlfj0= 6 | github.com/allegro/bigcache/v2 v2.2.0/go.mod h1:FppZsIO+IZk7gCuj5FiIDHGygD9xvWQcqg1uIPMb6tY= 7 | github.com/astaxie/beego v1.12.1 h1:dfpuoxpzLVgclveAXe4PyNKqkzgm5zF4tgF2B3kkM2I= 8 | github.com/astaxie/beego v1.12.1/go.mod h1:kPBWpSANNbSdIqOc8SUL9h+1oyBMZhROeYsXQDbidWQ= 9 | github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= 10 | github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= 11 | github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= 12 | github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= 13 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= 14 | github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= 15 | github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= 16 | github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= 17 | github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= 18 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 19 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 20 | github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= 21 | github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= 22 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 23 | github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= 24 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 25 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 26 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 27 | github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 28 | github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= 29 | github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= 30 | github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= 31 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 32 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 33 | github.com/matryer/is v1.3.0 h1:9qiso3jaJrOe6qBRJRBt2Ldht05qDiFP9le0JOIhRSI= 34 | github.com/matryer/is v1.3.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= 35 | github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 36 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 37 | github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 38 | github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 39 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 40 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 41 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 42 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 43 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= 44 | github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg= 45 | github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= 46 | github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= 47 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 48 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 49 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 50 | github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= 51 | github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= 52 | github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= 53 | github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= 54 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 55 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 56 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 57 | golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= 58 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 59 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 60 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 61 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 62 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 63 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 64 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 65 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 66 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 67 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 68 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 69 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 70 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 71 | golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 72 | golang.org/x/tools v0.0.0-20200312031852-3dc7fec7888e h1:bfmxQQr4Ek9mB8Ty0gPi170MgnQ03aoeFEB3jgCA6Ek= 73 | golang.org/x/tools v0.0.0-20200312031852-3dc7fec7888e/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= 74 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 75 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 77 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 78 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 79 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 80 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 81 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 82 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 83 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 84 | --------------------------------------------------------------------------------