├── .github └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── README.md ├── a.sh ├── geek ├── byteview.go ├── cache.go ├── cache │ ├── constants.go │ ├── lru_cache.go │ └── lru_cache_test.go ├── client.go ├── consistenthash │ ├── consistenthash.go │ └── consistenthash_test.go ├── geekcache.go ├── geekcache_test.go ├── pb │ ├── pb.pb.go │ ├── pb.proto │ └── pb_grpc.pb.go ├── peers.go ├── registry │ ├── discover.go │ └── register.go ├── server.go ├── server_test.go ├── singleflight │ └── singleflight.go └── utils │ ├── str_encoding.go │ └── utils.go ├── go.mod ├── go.sum └── main.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main", "develop" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v3 21 | with: 22 | go-version: 1.19 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: Setup protoc 28 | # You may pin to the exact commit or the version. 29 | # uses: arduino/setup-protoc@64c0c85d18e984422218383b81c52f8b077404d3 30 | uses: arduino/setup-protoc@v1.1.2 31 | with: 32 | # Version to use. Example: 3.9.1 33 | version: '3.9.1' 34 | # Include github pre-releases in latest version calculation 35 | include-pre-releases: false 36 | repo-token: ${{ secrets.RATE_LIMIT_TOKEN }} 37 | 38 | - name: Test 39 | run: go test -v ./... 40 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .idea/ 17 | .vscode/ 18 | 19 | geek-cache 20 | 21 | server -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Makonike 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # geek-cache 2 | 3 | a distributed read-only cache, based [groupcache](https://github.com/golang/groupcache), using etcd as a registry, supports efficient concurrent reading. 4 | 5 | ## Install 6 | 7 | The package supports 3 last Go versions and requires a Go version with modules support. 8 | 9 | `go get github.com/Makonike/geek-cache` 10 | 11 | ## Usage 12 | 13 | Be sure to install **etcd v3**(port 2379), grpcurl(for your tests), protobuf v3. 14 | 15 | ## Config 16 | 17 | In your application, you can configure like following: 18 | 19 | - Server 20 | 21 | ```go 22 | func NewServer(self string, opts ...ServerOptions) (*Server, error) 23 | 24 | server, err := geek.NewServer(addr, geek.ServiceName("your-service-name")) 25 | ``` 26 | 27 | - Client 28 | 29 | ```go 30 | registry.GlobalClientConfig = &clientv3.Config{ 31 | Endpoints: []string{"127.0.0.1:2379"}, // etcd address 32 | DialTimeout: 5 * time.Second, // the timeout for failing to establish a connection 33 | } 34 | ``` 35 | 36 | - Picker and Consistent Hash 37 | 38 | ```go 39 | picker := geek.NewClientPicker(addr, geek.PickerServiceName("geek-cache"), geek.ConsHashOptions(consistenthash.HashFunc(crc32.ChecksumIEEE), consistenthash.Replicas(150))) 40 | ``` 41 | 42 | ## Test 43 | 44 | Write the following code for testing. 45 | 46 | main.go 47 | 48 | ```go 49 | package main 50 | 51 | import ( 52 | "flag" 53 | "fmt" 54 | "log" 55 | 56 | "github.com/Makonike/geek-cache/geek" 57 | ) 58 | 59 | func main() { 60 | var port int 61 | flag.IntVar(&port, "port", 8001, "Geecache server port") 62 | flag.Parse() 63 | // mock database or other dataSource 64 | var mysql = map[string]string{ 65 | "Tom": "630", 66 | "Tom1": "631", 67 | "Tom2": "632", 68 | } 69 | // NewGroup create a Group which means a kind of sources 70 | // contain a func that used when misses cache 71 | g := geek.NewGroup("scores", 2<<10, geek.GetterFunc( 72 | func(key string) ([]byte, error) { 73 | log.Println("[SlowDB] search key", key) 74 | if v, ok := mysql[key]; ok { 75 | return []byte(v), nil 76 | } 77 | return nil, fmt.Errorf("%s not found", key) 78 | })) 79 | 80 | addrMap := map[int]string{ 81 | 8001: "8001", 82 | 8002: "8002", 83 | 8003: "8003", 84 | } 85 | var addr string = "127.0.0.1:" + addrMap[port] 86 | 87 | server, err := geek.NewServer(addr) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | 92 | addrs := make([]string, 0) 93 | for _, addr := range addrMap { 94 | addrs = append(addrs, "127.0.0.1:"+addr) 95 | } 96 | 97 | // set client address 98 | picker := geek.NewClientPicker(addr) 99 | picker.SetSimply(addrs...) 100 | g.RegisterPeers(picker) 101 | 102 | for { 103 | err = server.Start() 104 | if err != nil { 105 | log.Println(err.Error()) 106 | return 107 | } 108 | } 109 | } 110 | ``` 111 | 112 | a.sh 113 | 114 | ```shell 115 | #!/bin/bash 116 | trap "rm server;kill 0" EXIT 117 | 118 | go build -o server 119 | ./server -port=8001 & 120 | ./server -port=8002 & 121 | ./server -port=8003 & 122 | 123 | echo ">>> start test" 124 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8001 pb.GroupCache/Get 125 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8001 pb.GroupCache/Get 126 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8001 pb.GroupCache/Get 127 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8002 pb.GroupCache/Get 128 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8002 pb.GroupCache/Get 129 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8002 pb.GroupCache/Get 130 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8003 pb.GroupCache/Get 131 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8003 pb.GroupCache/Get 132 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8003 pb.GroupCache/Get 133 | 134 | kill -9 `lsof -ti:8002`; 135 | 136 | sleep 3 137 | 138 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8001 pb.GroupCache/Get 139 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8003 pb.GroupCache/Get 140 | 141 | wait 142 | ``` 143 | 144 | Running the shell, then you can see the results following. 145 | 146 | ```bash 147 | $ ./a.sh 148 | 2023/02/21 10:33:04 [127.0.0.1:8002] register service success 149 | 2023/02/21 10:33:04 [127.0.0.1:8003] register service success 150 | 2023/02/21 10:33:04 [127.0.0.1:8001] register service success 151 | >>> start test 152 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8001] Recv RPC Request - (scores)/(Tom) 153 | 2023/02/21 10:33:10 [Server 127.0.0.1:8001] Pick peer 127.0.0.1:8002 154 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8002] Recv RPC Request - (scores)/(Tom) 155 | 2023/02/21 10:33:10 [SlowDB] search key Tom 156 | { 157 | "value": "NjMw" 158 | } 159 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8001] Recv RPC Request - (scores)/(Tom1) 160 | 2023/02/21 10:33:10 [Server 127.0.0.1:8001] Pick peer 127.0.0.1:8003 161 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom1) 162 | 2023/02/21 10:33:10 [SlowDB] search key Tom1 163 | { 164 | "value": "NjMx" 165 | } 166 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8001] Recv RPC Request - (scores)/(Tom2) 167 | 2023/02/21 10:33:10 [Server 127.0.0.1:8001] Pick peer 127.0.0.1:8003 168 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom2) 169 | 2023/02/21 10:33:10 [SlowDB] search key Tom2 170 | { 171 | "value": "NjMy" 172 | } 173 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8002] Recv RPC Request - (scores)/(Tom) 174 | 2023/02/21 10:33:10 [Geek-Cache] hit 175 | { 176 | "value": "NjMw" 177 | } 178 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8002] Recv RPC Request - (scores)/(Tom1) 179 | 2023/02/21 10:33:10 [Server 127.0.0.1:8002] Pick peer 127.0.0.1:8003 180 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom1) 181 | 2023/02/21 10:33:10 [Geek-Cache] hit 182 | { 183 | "value": "NjMx" 184 | } 185 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8002] Recv RPC Request - (scores)/(Tom2) 186 | 2023/02/21 10:33:10 [Server 127.0.0.1:8002] Pick peer 127.0.0.1:8003 187 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom2) 188 | 2023/02/21 10:33:10 [Geek-Cache] hit 189 | { 190 | "value": "NjMy" 191 | } 192 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom) 193 | 2023/02/21 10:33:10 [Server 127.0.0.1:8003] Pick peer 127.0.0.1:8002 194 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8002] Recv RPC Request - (scores)/(Tom) 195 | 2023/02/21 10:33:10 [Geek-Cache] hit 196 | { 197 | "value": "NjMw" 198 | } 199 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom1) 200 | 2023/02/21 10:33:10 [Geek-Cache] hit 201 | { 202 | "value": "NjMx" 203 | } 204 | 2023/02/21 10:33:10 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom2) 205 | 2023/02/21 10:33:10 [Geek-Cache] hit 206 | { 207 | "value": "NjMy" 208 | } 209 | ./a.sh：行 23: 36214 已杀死 ./server -port=8002 210 | 2023/02/21 10:33:16 [Geek-Cache 127.0.0.1:8001] Recv RPC Request - (scores)/(Tom) 211 | 2023/02/21 10:33:16 [SlowDB] search key Tom 212 | { 213 | "value": "NjMw" 214 | } 215 | 2023/02/21 10:33:16 [Geek-Cache 127.0.0.1:8003] Recv RPC Request - (scores)/(Tom) 216 | 2023/02/21 10:33:16 [Server 127.0.0.1:8003] Pick peer 127.0.0.1:8001 217 | 2023/02/21 10:33:16 [Geek-Cache 127.0.0.1:8001] Recv RPC Request - (scores)/(Tom) 218 | 2023/02/21 10:33:16 [Geek-Cache] hit 219 | { 220 | "value": "NjMw" 221 | } 222 | ``` 223 | 224 | ## Tech Stack 225 | 226 | Golang+grpc+etcd 227 | 228 | ## Feature 229 | 230 | - 使用一致性哈希解决缓存副本冗余、rehashing开销大、缓存雪崩的问题 231 | - 使用singleFlight解决缓存击穿问题 232 | - 使用protobuf进行节点间通信，编码报文，提高效率 233 | - 构造虚拟节点使得请求映射负载均衡 234 | - 使用LRU缓存淘汰算法解决资源限制的问题 235 | - 使用etcd服务发现动态更新哈希环 236 | - 支持并发读 237 | 238 | ## TODO List 239 | 240 | - 添加多种缓存淘汰策略 241 | - 支持多协议通信 242 | -------------------------------------------------------------------------------- /a.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | trap "rm server;kill 0" EXIT 3 | 4 | go build -o server 5 | ./server -port=8001 & 6 | ./server -port=8002 & 7 | ./server -port=8003 & 8 | 9 | echo ">>> start test" 10 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8001 pb.GroupCache/Get 11 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8001 pb.GroupCache/Get 12 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8001 pb.GroupCache/Get 13 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8002 pb.GroupCache/Get 14 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8002 pb.GroupCache/Get 15 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8002 pb.GroupCache/Get 16 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8003 pb.GroupCache/Get 17 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom1"}' 127.0.0.1:8003 pb.GroupCache/Get 18 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom2"}' 127.0.0.1:8003 pb.GroupCache/Get 19 | 20 | kill -9 `lsof -ti:8002`; 21 | 22 | sleep 3 23 | 24 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8001 pb.GroupCache/Get 25 | grpcurl -plaintext -d '{"group":"scores", "key": "Tom"}' 127.0.0.1:8003 pb.GroupCache/Get 26 | 27 | wait -------------------------------------------------------------------------------- /geek/byteview.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | // ByteView 只读的字节视图，用于缓存数据 4 | type ByteView struct { 5 | b []byte 6 | } 7 | 8 | func (b ByteView) Len() int { 9 | return len(b.b) 10 | } 11 | 12 | func (b ByteView) ByteSLice() []byte { 13 | return cloneBytes(b.b) 14 | } 15 | 16 | func (b ByteView) String() string { 17 | return string(b.b) 18 | } 19 | 20 | func cloneBytes(b []byte) []byte { 21 | c := make([]byte, len(b)) 22 | copy(c, b) 23 | return c 24 | } 25 | -------------------------------------------------------------------------------- /geek/cache.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | c "github.com/Makonike/geek-cache/geek/cache" 8 | ) 9 | 10 | // cache 实例化lru，封装get和add。 11 | type cache struct { 12 | lock sync.RWMutex 13 | lruCache c.Cache 14 | cacheBytes int64 15 | } 16 | 17 | func (cache *cache) lruCacheLazyLoadIfNeed() { 18 | if cache.lruCache == nil { 19 | cache.lock.Lock() 20 | defer cache.lock.Unlock() 21 | if cache.lruCache == nil { 22 | cache.lruCache = c.NewLRUCache(cache.cacheBytes) 23 | } 24 | } 25 | } 26 | 27 | func (cache *cache) add(key string, value ByteView) { 28 | // lazy load 29 | cache.lruCacheLazyLoadIfNeed() 30 | cache.lruCache.Add(key, value) 31 | } 32 | 33 | func (cache *cache) get(key string) (value ByteView, ok bool) { 34 | cache.lock.RLock() 35 | defer cache.lock.RUnlock() 36 | if cache.lruCache == nil { 37 | return 38 | } 39 | if v, find := cache.lruCache.Get(key); find { 40 | return v.(ByteView), true 41 | } 42 | return 43 | } 44 | 45 | func (cache *cache) addWithExpiration(key string, value ByteView, expirationTime time.Time) { 46 | // lazy load 47 | cache.lruCacheLazyLoadIfNeed() 48 | cache.lruCache.AddWithExpiration(key, value, expirationTime) 49 | } 50 | 51 | func (cache *cache) delete(key string) bool { 52 | cache.lock.Lock() 53 | defer cache.lock.Unlock() 54 | if cache.lruCache == nil { 55 | return true 56 | } 57 | return cache.lruCache.Delete(key) 58 | } 59 | -------------------------------------------------------------------------------- /geek/cache/constants.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type MaxMemoryPolicy int 4 | 5 | const ( 6 | VOLATILE_LRU MaxMemoryPolicy = 1 7 | VOLATILE_RANDOM MaxMemoryPolicy = 2 8 | ALLKEYS_LRU MaxMemoryPolicy = 3 9 | ALLKEYS_RANDOM MaxMemoryPolicy = 4 10 | ) 11 | -------------------------------------------------------------------------------- /geek/cache/lru_cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "container/list" 5 | "sync" 6 | "time" 7 | ) 8 | 9 | type Cache interface { 10 | Get(key string) (Value, bool) 11 | Add(key string, value Value) 12 | AddWithExpiration(key string, value Value, expirationTime time.Time) 13 | Delete(key string) bool 14 | } 15 | 16 | type Value interface { 17 | Len() int // return data size 18 | } 19 | 20 | // cache struct 21 | type lruCache struct { 22 | lock sync.Mutex 23 | cacheMap map[string]*list.Element // map cache 24 | expires map[string]time.Time // The expiration time of key 25 | ll *list.List // linked list 26 | OnEvicted func(key string, value Value) // The callback function when a record is deleted 27 | maxBytes int64 // The maximum memory allowed 28 | nbytes int64 // The memory is currently in use 29 | } 30 | 31 | // 通过key可以在记录删除时，删除字典缓存中的映射 32 | type entry struct { 33 | key string 34 | value Value 35 | } 36 | 37 | func NewLRUCache(maxSize int64) *lruCache { 38 | answer := lruCache{ 39 | cacheMap: make(map[string]*list.Element), 40 | expires: make(map[string]time.Time), 41 | nbytes: 0, 42 | ll: list.New(), 43 | maxBytes: maxSize, 44 | } 45 | go func() { 46 | ticker := time.NewTicker(1 * time.Hour) 47 | defer ticker.Stop() 48 | for range ticker.C { 49 | answer.periodicMemoryClean() 50 | } 51 | }() 52 | return &answer 53 | } 54 | 55 | func (c *lruCache) Get(key string) (Value, bool) { 56 | c.lock.Lock() 57 | defer c.lock.Unlock() 58 | // check for expiration 59 | expirationTime, ok := c.expires[key] 60 | if ok && expirationTime.Before(time.Now()) { 61 | v := c.cacheMap[key].Value.(*entry) 62 | c.nbytes -= int64(v.value.Len() + len(v.key)) 63 | c.ll.Remove(c.cacheMap[key]) 64 | delete(c.cacheMap, key) 65 | delete(c.expires, key) 66 | // rollback 67 | if c.OnEvicted != nil { 68 | c.OnEvicted(key, v.value) 69 | } 70 | return nil, false 71 | } 72 | // get value 73 | if v, ok2 := c.cacheMap[key]; ok2 { 74 | c.ll.MoveToBack(v) 75 | return v.Value.(*entry).value, true 76 | } 77 | return nil, false 78 | } 79 | 80 | // add a key-value 81 | func (c *lruCache) Add(key string, value Value) { 82 | c.lock.Lock() 83 | defer c.lock.Unlock() 84 | c.baseAdd(key, value) 85 | delete(c.expires, key) 86 | c.freeMemoryIfNeeded() 87 | } 88 | 89 | // add a key-value whth expiration 90 | func (c *lruCache) AddWithExpiration(key string, value Value, expirationTime time.Time) { 91 | c.lock.Lock() 92 | defer c.lock.Unlock() 93 | c.baseAdd(key, value) 94 | c.expires[key] = expirationTime 95 | c.freeMemoryIfNeeded() 96 | } 97 | 98 | // delete a key-value by key 99 | func (c *lruCache) Delete(key string) bool { 100 | c.lock.Lock() 101 | defer c.lock.Unlock() 102 | c.nbytes -= int64(len(key) + c.getValueSizeByKey(key)) 103 | delete(c.cacheMap, key) 104 | delete(c.expires, key) 105 | return true 106 | } 107 | 108 | func (c *lruCache) baseAdd(key string, value Value) { 109 | // Check whether the key already exists 110 | if _, ok := c.cacheMap[key]; ok { 111 | c.nbytes += int64(value.Len() - c.getValueSizeByKey(key)) 112 | // update value 113 | c.cacheMap[key].Value = &entry{key, value} 114 | // popular 115 | c.ll.MoveToBack(c.cacheMap[key]) 116 | } else { 117 | c.nbytes += int64(len(key) + value.Len()) 118 | c.cacheMap[key] = c.ll.PushBack(&entry{key, value}) 119 | } 120 | } 121 | 122 | // lockless !!! free Memory when the memory is insufficient 123 | func (c *lruCache) freeMemoryIfNeeded() { 124 | // 只有一种淘汰策略，lru 125 | for c.nbytes > c.maxBytes { 126 | v := c.ll.Front() 127 | if v != nil { 128 | c.ll.Remove(v) 129 | kv := v.Value.(*entry) 130 | delete(c.cacheMap, kv.key) 131 | delete(c.expires, kv.key) 132 | c.nbytes -= int64(len(kv.key) + kv.value.Len()) 133 | if c.OnEvicted != nil { 134 | c.OnEvicted(kv.key, kv.value) 135 | } 136 | } 137 | } 138 | } 139 | 140 | // Scan and remove expired kv 141 | func (c *lruCache) periodicMemoryClean() { 142 | c.lock.Lock() 143 | defer c.lock.Unlock() 144 | n := len(c.expires) / 10 145 | for key := range c.expires { 146 | // check for expiration 147 | if c.expires[key].Before(time.Now()) { 148 | c.nbytes -= int64(len(key) + c.getValueSizeByKey(key)) 149 | delete(c.expires, key) 150 | delete(c.cacheMap, key) 151 | } 152 | n-- 153 | if n == 0 { 154 | break 155 | } 156 | } 157 | } 158 | 159 | func (c *lruCache) getValueSizeByKey(key string) int { 160 | return c.cacheMap[key].Value.(*entry).value.Len() 161 | } 162 | -------------------------------------------------------------------------------- /geek/cache/lru_cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "strconv" 6 | "sync" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | // 检测并发情况下是否会出现问题 12 | func TestCache_GetAndAdd(t *testing.T) { 13 | var wg sync.WaitGroup 14 | cache := NewLRUCache(1000000000) 15 | wg.Add(2) 16 | go func() { 17 | defer wg.Done() 18 | for i := 0; i < 1000000; i++ { 19 | cache.Add(strconv.Itoa(i), &testValue{"东神牛逼"}) 20 | } 21 | }() 22 | go func() { 23 | defer wg.Done() 24 | for i := 0; i < 1000000; i++ { 25 | cache.Add(strconv.Itoa(i+1000000), &testValue{"欧神牛逼"}) 26 | } 27 | }() 28 | wg.Wait() 29 | a := assert.New(t) 30 | for i := 1000000; i < 2000000; i++ { 31 | t, _ := cache.Get(strconv.Itoa(i)) 32 | a.Equal("欧神牛逼", t.(*testValue).b) 33 | // todo: 效率有点低 34 | } 35 | } 36 | 37 | // 检测lru算法 38 | func TestCache_FreeMemory(t *testing.T) { 39 | a := assert.New(t) 40 | // 测试LRU 41 | cache := NewLRUCache(90) 42 | for i := 0; i < 10; i++ { 43 | cache.Add(strconv.Itoa(i), &testValue{"123456789"}) 44 | } 45 | // key为0的被淘汰 46 | _, f0 := cache.Get("0") 47 | a.False(f0) 48 | // key为1的未被淘汰 49 | v1, f1 := cache.Get("1") 50 | a.True(f1) 51 | a.Equal(v1.(*testValue).b, "123456789") 52 | // 添加一个缓存 53 | cache.Add("a", &testValue{"123456789"}) 54 | // key为2的缓存被淘汰 55 | _, f2 := cache.Get("2") 56 | a.False(f2) 57 | // key为3未被淘汰 58 | v3, f3 := cache.Get("3") 59 | a.True(f3) 60 | a.Equal(v3.(*testValue).b, "123456789") 61 | } 62 | 63 | // 检测AddWithExpiration的算法中lru逻辑 64 | func TestCache_FreeMemory2(t *testing.T) { 65 | timeout := time.Now().Add(3 * time.Second) 66 | a := assert.New(t) 67 | // 测试LRU 68 | cache := NewLRUCache(90) 69 | for i := 0; i < 10; i++ { 70 | cache.AddWithExpiration(strconv.Itoa(i), &testValue{"123456789"}, timeout) 71 | } 72 | // key为0的被淘汰 73 | _, f0 := cache.Get("0") 74 | a.False(f0) 75 | // key为1的未被淘汰 76 | v1, f1 := cache.Get("1") 77 | a.True(f1) 78 | a.Equal(v1.(*testValue).b, "123456789") 79 | // 添加一个缓存 80 | cache.AddWithExpiration("a", &testValue{"123456789"}, timeout) 81 | // key为2的缓存被淘汰 82 | _, f2 := cache.Get("2") 83 | a.False(f2) 84 | // key为3未被淘汰 85 | v3, f3 := cache.Get("3") 86 | a.True(f3) 87 | a.Equal(v3.(*testValue).b, "123456789") 88 | } 89 | 90 | // 测试超时 91 | func TestCache_AddWithExpiration(t *testing.T) { 92 | a := assert.New(t) 93 | cache := NewLRUCache(100) 94 | cache.AddWithExpiration("1", &testValue{"123456789"}, time.Now().Add(3*time.Second)) 95 | time.Sleep(2 * time.Second) 96 | v1, _ := cache.Get("1") 97 | a.Equal("123456789", v1.(*testValue).b) 98 | time.Sleep(2 * time.Second) 99 | _, f := cache.Get("1") 100 | a.False(f) 101 | } 102 | 103 | // 测试删除 104 | func TestCache_Delete(t *testing.T) { 105 | a := assert.New(t) 106 | cache := NewLRUCache(100) 107 | cache.Add("1", &testValue{"123456789"}) 108 | cache.Add("2", &testValue{"123456789"}) 109 | _, f1 := cache.Get("1") 110 | _, f2 := cache.Get("2") 111 | a.True(f1) 112 | a.True(f2) 113 | cache.Delete("1") 114 | _, f3 := cache.Get("1") 115 | _, f4 := cache.Get("2") 116 | a.False(f3) 117 | a.True(f4) 118 | } 119 | 120 | // ByteView 只读的字节视图，用于缓存数据 121 | type testValue struct { 122 | b string 123 | } 124 | 125 | func (b *testValue) Len() int { 126 | return len(b.b) 127 | } 128 | -------------------------------------------------------------------------------- /geek/client.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | pb "github.com/Makonike/geek-cache/geek/pb" 10 | registry "github.com/Makonike/geek-cache/geek/registry" 11 | clientv3 "go.etcd.io/etcd/client/v3" 12 | ) 13 | 14 | type Client struct { 15 | addr string // name of remote server, e.g. ip:port 16 | serviceName string // name of service, e.g. geek-cache 17 | } 18 | 19 | // NewClient creates a new client 20 | func NewClient(addr, serviceName string) *Client { 21 | return &Client{ 22 | addr: addr, 23 | serviceName: serviceName, 24 | } 25 | } 26 | 27 | // Get send the url for getting specific group and key, 28 | // and return the result 29 | func (c *Client) Get(group, key string) ([]byte, error) { 30 | cli, err := clientv3.New(*registry.GlobalClientConfig) 31 | if err != nil { 32 | log.Fatal(err) 33 | return nil, err 34 | } 35 | defer cli.Close() 36 | 37 | conn, err := registry.EtcdDial(cli, c.serviceName, c.addr) 38 | if err != nil { 39 | return nil, err 40 | } 41 | defer conn.Close() 42 | 43 | grpcCLient := pb.NewGroupCacheClient(conn) 44 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 45 | defer cancel() 46 | 47 | resp, err := grpcCLient.Get(ctx, &pb.Request{ 48 | Group: group, 49 | Key: key, 50 | }) 51 | if err != nil { 52 | return nil, fmt.Errorf("could not get %s-%s from peer %s", group, key, c.addr) 53 | } 54 | return resp.GetValue(), nil 55 | } 56 | 57 | // Delete send the url for getting specific group and key, 58 | // and return the result 59 | func (c *Client) Delete(group string, key string) (bool, error) { 60 | cli, err := clientv3.New(*registry.GlobalClientConfig) 61 | 62 | if err != nil { 63 | log.Fatal(err) 64 | return false, err 65 | } 66 | defer cli.Close() 67 | 68 | conn, err := registry.EtcdDial(cli, c.serviceName, c.addr) 69 | if err != nil { 70 | return false, err 71 | } 72 | defer conn.Close() 73 | 74 | grpcCLient := pb.NewGroupCacheClient(conn) 75 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 76 | defer cancel() 77 | 78 | resp, err := grpcCLient.Delete(ctx, &pb.Request{ 79 | Group: group, 80 | Key: key, 81 | }) 82 | if err != nil { 83 | return false, fmt.Errorf("could not delete %s-%s from peer %s", group, key, c.addr) 84 | } 85 | return resp.GetValue(), nil 86 | } 87 | 88 | // resure implemented 89 | var _ PeerGetter = (*Client)(nil) 90 | -------------------------------------------------------------------------------- /geek/consistenthash/consistenthash.go: -------------------------------------------------------------------------------- 1 | package consistenthash 2 | 3 | // consistenthash solves the problems of: 4 | // 1. how to route the key to the cached node. 5 | // 2. lots of space for caching the same key. 6 | // 3. rehashing when the node is changed. 7 | 8 | import ( 9 | "hash/crc32" 10 | "sort" 11 | "strconv" 12 | ) 13 | 14 | var ( 15 | GlobalReplicas = defaultReplicas 16 | defaultHash = crc32.ChecksumIEEE // default use crc32.ChecksumIEEE as hash func 17 | defaultReplicas = 150 18 | ) 19 | 20 | type Hash func(data []byte) uint32 21 | 22 | type Map struct { 23 | hash Hash // hash func, a kind of datas need to be sure that with the same hash func 24 | replicas int // 虚拟节点倍数 25 | keys []int // 哈希环，维护有序 26 | hashMap map[int]string // 虚拟节点与真实节点的映射表（key是虚拟节点hash, value is the name of reality node） 27 | } 28 | 29 | type ConsOptions func(*Map) 30 | 31 | // New with the replicas number and hash function 32 | func New(opts ...ConsOptions) *Map { 33 | m := Map{ 34 | hash: defaultHash, 35 | replicas: defaultReplicas, 36 | hashMap: make(map[int]string), 37 | } 38 | for _, opt := range opts { 39 | opt(&m) 40 | } 41 | return &m 42 | } 43 | 44 | func Replicas(replicas int) ConsOptions { 45 | return func(m *Map) { 46 | m.replicas = replicas 47 | } 48 | } 49 | 50 | func HashFunc(hash Hash) ConsOptions { 51 | return func(m *Map) { 52 | m.hash = hash 53 | } 54 | } 55 | 56 | // Add adds some keys to the hash. 57 | // keys is the name of reality node 58 | func (m *Map) Add(keys ...string) { 59 | for _, key := range keys { 60 | // 一个真实节点对应多个虚拟节点 61 | for i := 0; i < m.replicas; i++ { 62 | hash := int(m.hash([]byte(strconv.Itoa(i) + key))) 63 | m.keys = append(m.keys, hash) 64 | // 维护虚拟节点与真实节点的映射关系 65 | m.hashMap[hash] = key 66 | } 67 | } 68 | sort.Ints(m.keys) 69 | } 70 | 71 | // Get gets the closest item in the hash to the provided key. 72 | func (m *Map) Get(key string) string { 73 | if len(m.keys) == 0 { 74 | return "" 75 | } 76 | hash := int(m.hash([]byte(key))) 77 | // 顺时针找到第一个匹配的虚拟节点 78 | idx := sort.Search(len(m.keys), func(i int) bool { 79 | return m.keys[i] >= hash 80 | }) 81 | // 如果没有找到匹配的虚拟节点，返回哈希环上的第一个节点 82 | return m.hashMap[m.keys[idx%len(m.keys)]] 83 | } 84 | 85 | // Remove removes some node from the hash. 86 | func (m *Map) Remove(key string) { 87 | for i := 0; i < m.replicas; i++ { 88 | hash := int(m.hash([]byte(strconv.Itoa(i) + key))) 89 | idx := sort.SearchInts(m.keys, hash) 90 | m.keys = append(m.keys[:idx], m.keys[idx+1:]...) 91 | delete(m.hashMap, hash) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /geek/consistenthash/consistenthash_test.go: -------------------------------------------------------------------------------- 1 | package consistenthash 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func TestMap_Get(t *testing.T) { 9 | hash := New(Replicas(3), HashFunc(func(data []byte) uint32 { 10 | i, _ := strconv.Atoi(string(data)) 11 | return uint32(i) 12 | })) 13 | // add vir-node 06, 16, 26, 04, 14, 24, 02, 12, 22 14 | hash.Add("6", "4", "2") 15 | testCases := map[string]string{ 16 | "2": "2", // 02 - 2 17 | "11": "2", // 12 - 2 18 | "23": "4", // 24 - 4 19 | "26": "6", // 24 - 4 20 | "24": "4", // 24 - 4 21 | "27": "2", // 02 - 2 22 | } 23 | for k, v := range testCases { 24 | if hash.Get(k) != v { 25 | t.Errorf("hash.Get(%s) expeted %s, but %s", k, v, hash.Get(k)) 26 | } 27 | } 28 | // add vir-node 08, 18, 28 29 | hash.Add("8") 30 | testCases["27"] = "8" // 28 - 8 31 | for k, v := range testCases { 32 | if hash.Get(k) != v { 33 | t.Errorf("hash.Get(%s) expeted %s, but %s", k, v, hash.Get(k)) 34 | } 35 | } 36 | // remove vir-node 08, 18, 28 37 | hash.Remove("8") 38 | testCases["27"] = "2" // 02 - 2 39 | for k, v := range testCases { 40 | if hash.Get(k) != v { 41 | t.Errorf("hash.Get(%s) expeted %s, but %s", k, v, hash.Get(k)) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /geek/geekcache.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "sync" 7 | "time" 8 | 9 | "github.com/Makonike/geek-cache/geek/singleflight" 10 | ) 11 | 12 | var ( 13 | lock sync.RWMutex 14 | groups = make(map[string]*Group) 15 | ) 16 | 17 | type Group struct { 18 | name string // group name 19 | getter Getter // 缓存未名中时的callback 20 | mainCache cache // main cache 21 | peers PeerPicker // pick function 22 | loader *singleflight.Group // make sure that each key is only fetched once 23 | } 24 | 25 | func (g *Group) RegisterPeers(peers PeerPicker) { 26 | if g.peers != nil { 27 | panic("RegisterPeerPicker called multiple times") 28 | } 29 | g.peers = peers 30 | } 31 | 32 | // NewGroup 新创建一个Group 33 | // 如果存在同名的group会进行覆盖 34 | func NewGroup(name string, cacheBytes int64, getter Getter) *Group { 35 | if getter == nil { 36 | panic("nil Getter") 37 | } 38 | lock.Lock() 39 | defer lock.Unlock() 40 | g := &Group{ 41 | name: name, 42 | getter: getter, 43 | mainCache: cache{ 44 | cacheBytes: cacheBytes, 45 | }, 46 | loader: &singleflight.Group{}, 47 | } 48 | groups[name] = g 49 | return g 50 | } 51 | 52 | func GetGroup(name string) *Group { 53 | lock.RLock() 54 | g := groups[name] 55 | lock.RUnlock() 56 | return g 57 | } 58 | 59 | func (g *Group) Get(key string) (ByteView, error) { 60 | if key == "" { 61 | return ByteView{}, fmt.Errorf("key is required") 62 | } 63 | return g.load(key) 64 | } 65 | 66 | // get from peer first, then get locally 67 | func (g *Group) load(key string) (ByteView, error) { 68 | // make sure requests for the key only execute once in concurrent condition 69 | v, err := g.loader.Do(key, func() (interface{}, error) { 70 | if g.peers != nil { 71 | if peer, ok, isSelf := g.peers.PickPeer(key); ok { 72 | if isSelf { 73 | if v, ok := g.mainCache.get(key); ok { 74 | log.Println("[Geek-Cache] hit") 75 | return v, nil 76 | } 77 | } else { 78 | if value, err := g.getFromPeer(peer, key); err == nil { 79 | return value, nil 80 | } else { 81 | log.Println("[Geek-Cache] Failed to get from peer", err) 82 | } 83 | } 84 | } 85 | } 86 | return g.getLocally(key) 87 | }) 88 | 89 | if err == nil { 90 | return v.(ByteView), nil 91 | } 92 | return ByteView{}, err 93 | } 94 | 95 | func (g *Group) Delete(key string) (bool, error) { 96 | if key == "" { 97 | return true, fmt.Errorf("key is required") 98 | } 99 | // Peer is not set, delete from local 100 | if g.peers == nil { 101 | return g.mainCache.delete(key), nil 102 | } 103 | // The peer is set, 104 | peer, ok, isSelf := g.peers.PickPeer(key) 105 | if !ok { 106 | return false, nil 107 | } 108 | if isSelf { 109 | return g.mainCache.delete(key), nil 110 | } else { 111 | //use other server to delete the key-value 112 | success, err := g.deleteFromPeer(peer, key) 113 | return success, err 114 | } 115 | } 116 | 117 | func (g *Group) getFromPeer(peer PeerGetter, key string) (ByteView, error) { 118 | bytes, err := peer.Get(g.name, key) 119 | if err != nil { 120 | return ByteView{}, err 121 | } 122 | return ByteView{ 123 | b: cloneBytes(bytes), 124 | }, nil 125 | } 126 | 127 | func (g *Group) deleteFromPeer(peer PeerGetter, key string) (bool, error) { 128 | success, err := peer.Delete(g.name, key) 129 | if err != nil { 130 | return false, err 131 | } 132 | return success, nil 133 | } 134 | 135 | func (g *Group) getLocally(key string) (ByteView, error) { 136 | // have a try again 137 | if v, ok := g.mainCache.get(key); ok { 138 | log.Println("[Geek-Cache] hit") 139 | return v, nil 140 | } 141 | bytes, f, expirationTime := g.getter.Get(key) 142 | if !f { 143 | return ByteView{}, fmt.Errorf("data not found") 144 | } 145 | bw := ByteView{cloneBytes(bytes)} 146 | if !expirationTime.IsZero() { 147 | g.mainCache.addWithExpiration(key, bw, expirationTime) 148 | } else { 149 | g.mainCache.add(key, bw) 150 | } 151 | return bw, nil 152 | } 153 | 154 | // Getter loads data for a key locally 155 | // call back when a key cache missed 156 | // impl by user 157 | type Getter interface { 158 | Get(key string) ([]byte, bool, time.Time) 159 | } 160 | 161 | type GetterFunc func(key string) ([]byte, bool, time.Time) 162 | 163 | func (f GetterFunc) Get(key string) ([]byte, bool, time.Time) { 164 | return f(key) 165 | } 166 | 167 | func DestroyGroup(name string) { 168 | g := GetGroup(name) 169 | if g != nil { 170 | delete(groups, name) 171 | log.Printf("Destroy cache [%s]", name) 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /geek/geekcache_test.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "math/rand" 5 | "testing" 6 | time "time" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var db = map[string]string{ 12 | "Tom": "630", 13 | "Jack": "742", 14 | "Amy": "601", 15 | "Alice": "653", 16 | } 17 | 18 | func TestGroup_Get(t *testing.T) { 19 | loads := make(map[string]int) 20 | gee := NewGroup("scores", 2<<10, GetterFunc( 21 | func(key string) ([]byte, bool, time.Time) { 22 | if v, ok := db[key]; ok { 23 | loads[key] += 1 24 | return []byte(v), true, time.Time{} 25 | } 26 | return nil, false, time.Time{} 27 | }), 28 | ) 29 | gee1 := GetGroup("scores") 30 | _, err := gee1.Get("") 31 | if err == nil { 32 | t.Fatalf("Get params is empty, excepted not nil error, but nil") 33 | } 34 | 35 | for k, v := range db { 36 | if view, err := gee.Get(k); err != nil || view.String() != v { 37 | t.Fatalf("expected err is nil and value is %v, but err is %v, value is %v", v, err, view.String()) 38 | } 39 | // load from callback 40 | if _, err := gee.Get(k); err != nil || loads[k] > 1 { 41 | t.Fatalf("cache %v miss", k) 42 | } 43 | } 44 | if view, err := gee.Get("unknown"); err == nil { 45 | t.Fatalf("the key unknown but get %v", view) 46 | } 47 | } 48 | 49 | func TestGroup_Delete(t *testing.T) { 50 | a := assert.New(t) 51 | database := map[string]string{ 52 | "Tom": "630", 53 | "Jack": "742", 54 | "Amy": "601", 55 | "Alice": "653", 56 | } 57 | loads := make(map[string]int) 58 | NewGroup("scores", 2<<10, GetterFunc( 59 | func(key string) ([]byte, bool, time.Time) { 60 | if v, ok := database[key]; ok { 61 | loads[key] += 1 62 | return []byte(v), true, time.Time{} 63 | } 64 | return nil, false, time.Time{} 65 | }), 66 | ) 67 | gee := GetGroup("scores") 68 | for k, v := range database { 69 | view, err := gee.Get(k) 70 | a.Equal(view.String(), v) 71 | a.Nil(err) 72 | // delete 73 | s, err2 := gee.Delete(k) 74 | a.True(s) 75 | a.Nil(err2) 76 | // Get it again 77 | _, _ = gee.Get(k) 78 | // check the number of loads 79 | a.Equal(2, loads[k]) 80 | } 81 | } 82 | 83 | func TestGroup_SetTimeout(t *testing.T) { 84 | a := assert.New(t) 85 | var db2 = map[string]string{ 86 | "Tom": "630", 87 | "Jack": "742", 88 | "Amy": "601", 89 | "Alice": "653", 90 | } 91 | loads := make(map[string]int) 92 | _ = NewGroup("scores", 2<<10, GetterFunc( 93 | func(key string) ([]byte, bool, time.Time) { 94 | rand.Seed(time.Now().UnixNano()) 95 | if v, ok := db2[key]; ok { 96 | loads[key] += 1 97 | // 用户设置超时时间 98 | timeout := time.Now().Add(time.Duration(rand.Intn(10)) * time.Second) 99 | return []byte(v), true, timeout 100 | } 101 | return nil, false, time.Time{} 102 | }), 103 | ) 104 | // 读取key并校验 105 | gee1 := GetGroup("scores") 106 | v, _ := gee1.Get("Alice") 107 | a.Equal(v.String(), "653") 108 | 109 | // 过期 110 | db2["Alice"] = "123" 111 | time.Sleep(10 * time.Second) 112 | v2, _ := gee1.Get("Alice") 113 | a.Equal(v2.String(), "123") 114 | 115 | } 116 | -------------------------------------------------------------------------------- /geek/pb/pb.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // versions: 3 | // protoc-gen-go v1.28.1 4 | // protoc v3.21.12 5 | // source: pb.proto 6 | 7 | package __ 8 | 9 | import ( 10 | protoreflect "google.golang.org/protobuf/reflect/protoreflect" 11 | protoimpl "google.golang.org/protobuf/runtime/protoimpl" 12 | reflect "reflect" 13 | sync "sync" 14 | ) 15 | 16 | const ( 17 | // Verify that this generated code is sufficiently up-to-date. 18 | _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) 19 | // Verify that runtime/protoimpl is sufficiently up-to-date. 20 | _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) 21 | ) 22 | 23 | type Request struct { 24 | state protoimpl.MessageState 25 | sizeCache protoimpl.SizeCache 26 | unknownFields protoimpl.UnknownFields 27 | 28 | Group string `protobuf:"bytes,1,opt,name=group,proto3" json:"group,omitempty"` 29 | Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` 30 | } 31 | 32 | func (x *Request) Reset() { 33 | *x = Request{} 34 | if protoimpl.UnsafeEnabled { 35 | mi := &file_pb_proto_msgTypes[0] 36 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 37 | ms.StoreMessageInfo(mi) 38 | } 39 | } 40 | 41 | func (x *Request) String() string { 42 | return protoimpl.X.MessageStringOf(x) 43 | } 44 | 45 | func (*Request) ProtoMessage() {} 46 | 47 | func (x *Request) ProtoReflect() protoreflect.Message { 48 | mi := &file_pb_proto_msgTypes[0] 49 | if protoimpl.UnsafeEnabled && x != nil { 50 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 51 | if ms.LoadMessageInfo() == nil { 52 | ms.StoreMessageInfo(mi) 53 | } 54 | return ms 55 | } 56 | return mi.MessageOf(x) 57 | } 58 | 59 | // Deprecated: Use Request.ProtoReflect.Descriptor instead. 60 | func (*Request) Descriptor() ([]byte, []int) { 61 | return file_pb_proto_rawDescGZIP(), []int{0} 62 | } 63 | 64 | func (x *Request) GetGroup() string { 65 | if x != nil { 66 | return x.Group 67 | } 68 | return "" 69 | } 70 | 71 | func (x *Request) GetKey() string { 72 | if x != nil { 73 | return x.Key 74 | } 75 | return "" 76 | } 77 | 78 | type ResponseForGet struct { 79 | state protoimpl.MessageState 80 | sizeCache protoimpl.SizeCache 81 | unknownFields protoimpl.UnknownFields 82 | 83 | Value []byte `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` 84 | } 85 | 86 | func (x *ResponseForGet) Reset() { 87 | *x = ResponseForGet{} 88 | if protoimpl.UnsafeEnabled { 89 | mi := &file_pb_proto_msgTypes[1] 90 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 91 | ms.StoreMessageInfo(mi) 92 | } 93 | } 94 | 95 | func (x *ResponseForGet) String() string { 96 | return protoimpl.X.MessageStringOf(x) 97 | } 98 | 99 | func (*ResponseForGet) ProtoMessage() {} 100 | 101 | func (x *ResponseForGet) ProtoReflect() protoreflect.Message { 102 | mi := &file_pb_proto_msgTypes[1] 103 | if protoimpl.UnsafeEnabled && x != nil { 104 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 105 | if ms.LoadMessageInfo() == nil { 106 | ms.StoreMessageInfo(mi) 107 | } 108 | return ms 109 | } 110 | return mi.MessageOf(x) 111 | } 112 | 113 | // Deprecated: Use ResponseForGet.ProtoReflect.Descriptor instead. 114 | func (*ResponseForGet) Descriptor() ([]byte, []int) { 115 | return file_pb_proto_rawDescGZIP(), []int{1} 116 | } 117 | 118 | func (x *ResponseForGet) GetValue() []byte { 119 | if x != nil { 120 | return x.Value 121 | } 122 | return nil 123 | } 124 | 125 | type ResponseForDelete struct { 126 | state protoimpl.MessageState 127 | sizeCache protoimpl.SizeCache 128 | unknownFields protoimpl.UnknownFields 129 | 130 | Value bool `protobuf:"varint,1,opt,name=value,proto3" json:"value,omitempty"` 131 | } 132 | 133 | func (x *ResponseForDelete) Reset() { 134 | *x = ResponseForDelete{} 135 | if protoimpl.UnsafeEnabled { 136 | mi := &file_pb_proto_msgTypes[2] 137 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 138 | ms.StoreMessageInfo(mi) 139 | } 140 | } 141 | 142 | func (x *ResponseForDelete) String() string { 143 | return protoimpl.X.MessageStringOf(x) 144 | } 145 | 146 | func (*ResponseForDelete) ProtoMessage() {} 147 | 148 | func (x *ResponseForDelete) ProtoReflect() protoreflect.Message { 149 | mi := &file_pb_proto_msgTypes[2] 150 | if protoimpl.UnsafeEnabled && x != nil { 151 | ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) 152 | if ms.LoadMessageInfo() == nil { 153 | ms.StoreMessageInfo(mi) 154 | } 155 | return ms 156 | } 157 | return mi.MessageOf(x) 158 | } 159 | 160 | // Deprecated: Use ResponseForDelete.ProtoReflect.Descriptor instead. 161 | func (*ResponseForDelete) Descriptor() ([]byte, []int) { 162 | return file_pb_proto_rawDescGZIP(), []int{2} 163 | } 164 | 165 | func (x *ResponseForDelete) GetValue() bool { 166 | if x != nil { 167 | return x.Value 168 | } 169 | return false 170 | } 171 | 172 | var File_pb_proto protoreflect.FileDescriptor 173 | 174 | var file_pb_proto_rawDesc = []byte{ 175 | 0x0a, 0x08, 0x70, 0x62, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x70, 0x62, 0x22, 0x31, 176 | 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 177 | 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 178 | 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 179 | 0x79, 0x22, 0x26, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x46, 0x6f, 0x72, 180 | 0x47, 0x65, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 181 | 0x28, 0x0c, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x29, 0x0a, 0x11, 0x52, 0x65, 0x73, 182 | 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x46, 0x6f, 0x72, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x14, 183 | 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 184 | 0x61, 0x6c, 0x75, 0x65, 0x32, 0x62, 0x0a, 0x0a, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x61, 0x63, 185 | 0x68, 0x65, 0x12, 0x26, 0x0a, 0x03, 0x47, 0x65, 0x74, 0x12, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x52, 186 | 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x73, 0x70, 187 | 0x6f, 0x6e, 0x73, 0x65, 0x46, 0x6f, 0x72, 0x47, 0x65, 0x74, 0x12, 0x2c, 0x0a, 0x06, 0x44, 0x65, 188 | 0x6c, 0x65, 0x74, 0x65, 0x12, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 189 | 0x74, 0x1a, 0x15, 0x2e, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x46, 190 | 0x6f, 0x72, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x42, 0x04, 0x5a, 0x02, 0x2e, 0x2f, 0x62, 0x06, 191 | 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 192 | } 193 | 194 | var ( 195 | file_pb_proto_rawDescOnce sync.Once 196 | file_pb_proto_rawDescData = file_pb_proto_rawDesc 197 | ) 198 | 199 | func file_pb_proto_rawDescGZIP() []byte { 200 | file_pb_proto_rawDescOnce.Do(func() { 201 | file_pb_proto_rawDescData = protoimpl.X.CompressGZIP(file_pb_proto_rawDescData) 202 | }) 203 | return file_pb_proto_rawDescData 204 | } 205 | 206 | var file_pb_proto_msgTypes = make([]protoimpl.MessageInfo, 3) 207 | var file_pb_proto_goTypes = []interface{}{ 208 | (*Request)(nil), // 0: pb.Request 209 | (*ResponseForGet)(nil), // 1: pb.ResponseForGet 210 | (*ResponseForDelete)(nil), // 2: pb.ResponseForDelete 211 | } 212 | var file_pb_proto_depIdxs = []int32{ 213 | 0, // 0: pb.GroupCache.Get:input_type -> pb.Request 214 | 0, // 1: pb.GroupCache.Delete:input_type -> pb.Request 215 | 1, // 2: pb.GroupCache.Get:output_type -> pb.ResponseForGet 216 | 2, // 3: pb.GroupCache.Delete:output_type -> pb.ResponseForDelete 217 | 2, // [2:4] is the sub-list for method output_type 218 | 0, // [0:2] is the sub-list for method input_type 219 | 0, // [0:0] is the sub-list for extension type_name 220 | 0, // [0:0] is the sub-list for extension extendee 221 | 0, // [0:0] is the sub-list for field type_name 222 | } 223 | 224 | func init() { file_pb_proto_init() } 225 | func file_pb_proto_init() { 226 | if File_pb_proto != nil { 227 | return 228 | } 229 | if !protoimpl.UnsafeEnabled { 230 | file_pb_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { 231 | switch v := v.(*Request); i { 232 | case 0: 233 | return &v.state 234 | case 1: 235 | return &v.sizeCache 236 | case 2: 237 | return &v.unknownFields 238 | default: 239 | return nil 240 | } 241 | } 242 | file_pb_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { 243 | switch v := v.(*ResponseForGet); i { 244 | case 0: 245 | return &v.state 246 | case 1: 247 | return &v.sizeCache 248 | case 2: 249 | return &v.unknownFields 250 | default: 251 | return nil 252 | } 253 | } 254 | file_pb_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { 255 | switch v := v.(*ResponseForDelete); i { 256 | case 0: 257 | return &v.state 258 | case 1: 259 | return &v.sizeCache 260 | case 2: 261 | return &v.unknownFields 262 | default: 263 | return nil 264 | } 265 | } 266 | } 267 | type x struct{} 268 | out := protoimpl.TypeBuilder{ 269 | File: protoimpl.DescBuilder{ 270 | GoPackagePath: reflect.TypeOf(x{}).PkgPath(), 271 | RawDescriptor: file_pb_proto_rawDesc, 272 | NumEnums: 0, 273 | NumMessages: 3, 274 | NumExtensions: 0, 275 | NumServices: 1, 276 | }, 277 | GoTypes: file_pb_proto_goTypes, 278 | DependencyIndexes: file_pb_proto_depIdxs, 279 | MessageInfos: file_pb_proto_msgTypes, 280 | }.Build() 281 | File_pb_proto = out.File 282 | file_pb_proto_rawDesc = nil 283 | file_pb_proto_goTypes = nil 284 | file_pb_proto_depIdxs = nil 285 | } 286 | -------------------------------------------------------------------------------- /geek/pb/pb.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package pb; 3 | 4 | option go_package = "./"; 5 | 6 | message Request { 7 | string group = 1; 8 | string key = 2; 9 | } 10 | 11 | 12 | message ResponseForGet { 13 | bytes value = 1; 14 | } 15 | 16 | message ResponseForDelete { 17 | bool value = 1; 18 | } 19 | 20 | service GroupCache { 21 | rpc Get(Request) returns (ResponseForGet); 22 | rpc Delete(Request) returns(ResponseForDelete); 23 | } -------------------------------------------------------------------------------- /geek/pb/pb_grpc.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go-grpc. DO NOT EDIT. 2 | // versions: 3 | // - protoc-gen-go-grpc v1.2.0 4 | // - protoc v3.21.12 5 | // source: pb.proto 6 | 7 | package __ 8 | 9 | import ( 10 | context "context" 11 | grpc "google.golang.org/grpc" 12 | codes "google.golang.org/grpc/codes" 13 | status "google.golang.org/grpc/status" 14 | ) 15 | 16 | // This is a compile-time assertion to ensure that this generated file 17 | // is compatible with the grpc package it is being compiled against. 18 | // Requires gRPC-Go v1.32.0 or later. 19 | const _ = grpc.SupportPackageIsVersion7 20 | 21 | // GroupCacheClient is the client API for GroupCache service. 22 | // 23 | // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. 24 | type GroupCacheClient interface { 25 | Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*ResponseForGet, error) 26 | Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*ResponseForDelete, error) 27 | } 28 | 29 | type groupCacheClient struct { 30 | cc grpc.ClientConnInterface 31 | } 32 | 33 | func NewGroupCacheClient(cc grpc.ClientConnInterface) GroupCacheClient { 34 | return &groupCacheClient{cc} 35 | } 36 | 37 | func (c *groupCacheClient) Get(ctx context.Context, in *Request, opts ...grpc.CallOption) (*ResponseForGet, error) { 38 | out := new(ResponseForGet) 39 | err := c.cc.Invoke(ctx, "/pb.GroupCache/Get", in, out, opts...) 40 | if err != nil { 41 | return nil, err 42 | } 43 | return out, nil 44 | } 45 | 46 | func (c *groupCacheClient) Delete(ctx context.Context, in *Request, opts ...grpc.CallOption) (*ResponseForDelete, error) { 47 | out := new(ResponseForDelete) 48 | err := c.cc.Invoke(ctx, "/pb.GroupCache/Delete", in, out, opts...) 49 | if err != nil { 50 | return nil, err 51 | } 52 | return out, nil 53 | } 54 | 55 | // GroupCacheServer is the server API for GroupCache service. 56 | // All implementations must embed UnimplementedGroupCacheServer 57 | // for forward compatibility 58 | type GroupCacheServer interface { 59 | Get(context.Context, *Request) (*ResponseForGet, error) 60 | Delete(context.Context, *Request) (*ResponseForDelete, error) 61 | mustEmbedUnimplementedGroupCacheServer() 62 | } 63 | 64 | // UnimplementedGroupCacheServer must be embedded to have forward compatible implementations. 65 | type UnimplementedGroupCacheServer struct { 66 | } 67 | 68 | func (UnimplementedGroupCacheServer) Get(context.Context, *Request) (*ResponseForGet, error) { 69 | return nil, status.Errorf(codes.Unimplemented, "method Get not implemented") 70 | } 71 | func (UnimplementedGroupCacheServer) Delete(context.Context, *Request) (*ResponseForDelete, error) { 72 | return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") 73 | } 74 | func (UnimplementedGroupCacheServer) mustEmbedUnimplementedGroupCacheServer() {} 75 | 76 | // UnsafeGroupCacheServer may be embedded to opt out of forward compatibility for this service. 77 | // Use of this interface is not recommended, as added methods to GroupCacheServer will 78 | // result in compilation errors. 79 | type UnsafeGroupCacheServer interface { 80 | mustEmbedUnimplementedGroupCacheServer() 81 | } 82 | 83 | func RegisterGroupCacheServer(s grpc.ServiceRegistrar, srv GroupCacheServer) { 84 | s.RegisterService(&GroupCache_ServiceDesc, srv) 85 | } 86 | 87 | func _GroupCache_Get_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 88 | in := new(Request) 89 | if err := dec(in); err != nil { 90 | return nil, err 91 | } 92 | if interceptor == nil { 93 | return srv.(GroupCacheServer).Get(ctx, in) 94 | } 95 | info := &grpc.UnaryServerInfo{ 96 | Server: srv, 97 | FullMethod: "/pb.GroupCache/Get", 98 | } 99 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 100 | return srv.(GroupCacheServer).Get(ctx, req.(*Request)) 101 | } 102 | return interceptor(ctx, in, info, handler) 103 | } 104 | 105 | func _GroupCache_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { 106 | in := new(Request) 107 | if err := dec(in); err != nil { 108 | return nil, err 109 | } 110 | if interceptor == nil { 111 | return srv.(GroupCacheServer).Delete(ctx, in) 112 | } 113 | info := &grpc.UnaryServerInfo{ 114 | Server: srv, 115 | FullMethod: "/pb.GroupCache/Delete", 116 | } 117 | handler := func(ctx context.Context, req interface{}) (interface{}, error) { 118 | return srv.(GroupCacheServer).Delete(ctx, req.(*Request)) 119 | } 120 | return interceptor(ctx, in, info, handler) 121 | } 122 | 123 | // GroupCache_ServiceDesc is the grpc.ServiceDesc for GroupCache service. 124 | // It's only intended for direct use with grpc.RegisterService, 125 | // and not to be introspected or modified (even as a copy) 126 | var GroupCache_ServiceDesc = grpc.ServiceDesc{ 127 | ServiceName: "pb.GroupCache", 128 | HandlerType: (*GroupCacheServer)(nil), 129 | Methods: []grpc.MethodDesc{ 130 | { 131 | MethodName: "Get", 132 | Handler: _GroupCache_Get_Handler, 133 | }, 134 | { 135 | MethodName: "Delete", 136 | Handler: _GroupCache_Delete_Handler, 137 | }, 138 | }, 139 | Streams: []grpc.StreamDesc{}, 140 | Metadata: "pb.proto", 141 | } 142 | -------------------------------------------------------------------------------- /geek/peers.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "sync" 9 | "time" 10 | 11 | "github.com/Makonike/geek-cache/geek/consistenthash" 12 | registry "github.com/Makonike/geek-cache/geek/registry" 13 | clientv3 "go.etcd.io/etcd/client/v3" 14 | ) 15 | 16 | // PeerPicker must be implemented to locate the peer that owns a specific key 17 | type PeerPicker interface { 18 | PickPeer(key string) (peer PeerGetter, ok bool, isSelf bool) 19 | } 20 | 21 | // PeerGetter must be implemented by a peer 22 | type PeerGetter interface { 23 | Get(group string, key string) ([]byte, error) 24 | Delete(group string, key string) (bool, error) 25 | } 26 | 27 | type ClientPicker struct { 28 | self string // self ip 29 | serviceName string 30 | mu sync.RWMutex // guards 31 | consHash *consistenthash.Map // stores the list of peers, selected by specific key 32 | clients map[string]*Client // keyed by e.g. "10.0.0.2:8009" 33 | } 34 | 35 | func NewClientPicker(self string, opts ...PickerOptions) *ClientPicker { 36 | picker := ClientPicker{ 37 | self: self, 38 | serviceName: defaultServiceName, 39 | clients: make(map[string]*Client), 40 | mu: sync.RWMutex{}, 41 | consHash: consistenthash.New(), 42 | } 43 | picker.mu.Lock() 44 | for _, opt := range opts { 45 | opt(&picker) 46 | } 47 | picker.mu.Unlock() 48 | // 增量更新 49 | // TODO: watch closed 50 | picker.set(picker.self) 51 | go func() { 52 | cli, err := clientv3.New(*registry.GlobalClientConfig) 53 | if err != nil { 54 | log.Fatal(err) 55 | return 56 | } 57 | defer cli.Close() 58 | // watcher will watch for changes of the service node 59 | watcher := clientv3.NewWatcher(cli) 60 | watchCh := watcher.Watch(context.Background(), picker.serviceName, clientv3.WithPrefix()) 61 | for { 62 | a := <-watchCh 63 | go func() { 64 | picker.mu.Lock() 65 | defer picker.mu.Unlock() 66 | for _, x := range a.Events { 67 | // x: geek-cache/127.0.0.1:8004 68 | key := string(x.Kv.Key) 69 | idx := strings.Index(key, picker.serviceName) 70 | addr := key[idx+len(picker.serviceName)+1:] 71 | if addr == picker.self { 72 | continue 73 | } 74 | if x.IsCreate() { 75 | if _, ok := picker.clients[addr]; !ok { 76 | picker.set(addr) 77 | } 78 | } else if x.Type == clientv3.EventTypeDelete { 79 | if _, ok := picker.clients[addr]; ok { 80 | picker.remove(addr) 81 | } 82 | } 83 | } 84 | }() 85 | } 86 | }() 87 | // 全量更新 88 | go func() { 89 | picker.mu.Lock() 90 | cli, err := clientv3.New(*registry.GlobalClientConfig) 91 | if err != nil { 92 | log.Fatal(err) 93 | return 94 | } 95 | defer cli.Close() 96 | ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) 97 | defer cancel() 98 | resp, err := cli.Get(ctx, picker.serviceName, clientv3.WithPrefix()) 99 | if err != nil { 100 | log.Panic("[Event] full copy request failed") 101 | } 102 | kvs := resp.OpResponse().Get().Kvs 103 | 104 | defer picker.mu.Unlock() 105 | for _, kv := range kvs { 106 | key := string(kv.Key) 107 | idx := strings.Index(key, picker.serviceName) 108 | addr := key[idx+len(picker.serviceName)+1:] 109 | 110 | if _, ok := picker.clients[addr]; !ok { 111 | picker.set(addr) 112 | } 113 | 114 | } 115 | }() 116 | return &picker 117 | } 118 | 119 | type PickerOptions func(*ClientPicker) 120 | 121 | func PickerServiceName(serviceName string) PickerOptions { 122 | return func(picker *ClientPicker) { 123 | picker.serviceName = serviceName 124 | } 125 | } 126 | 127 | func ConsHashOptions(opts ...consistenthash.ConsOptions) PickerOptions { 128 | return func(picker *ClientPicker) { 129 | picker.consHash = consistenthash.New(opts...) 130 | } 131 | } 132 | 133 | func (p *ClientPicker) set(addr string) { 134 | p.consHash.Add(addr) 135 | p.clients[addr] = NewClient(addr, p.serviceName) 136 | } 137 | 138 | func (p *ClientPicker) remove(addr string) { 139 | p.consHash.Remove(addr) 140 | delete(p.clients, addr) 141 | } 142 | 143 | // PickPeer pick a peer with the consistenthash algorithm 144 | func (s *ClientPicker) PickPeer(key string) (PeerGetter, bool, bool) { 145 | s.mu.RLock() 146 | defer s.mu.RUnlock() 147 | if peer := s.consHash.Get(key); peer != "" { 148 | s.Log("Pick peer %s", peer) 149 | return s.clients[peer], true, peer == s.self 150 | } 151 | return nil, false, false 152 | } 153 | 154 | // Log info 155 | func (s *ClientPicker) Log(format string, path ...interface{}) { 156 | log.Printf("[Server %s] %s", s.self, fmt.Sprintf(format, path...)) 157 | } 158 | -------------------------------------------------------------------------------- /geek/registry/discover.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | clientv3 "go.etcd.io/etcd/client/v3" 5 | "go.etcd.io/etcd/client/v3/naming/resolver" 6 | "google.golang.org/grpc" 7 | ) 8 | 9 | // EtcdDial request a server from grpc 10 | // Connection can be obtained by providing an etcd client and service name 11 | func EtcdDial(c *clientv3.Client, service, target string) (*grpc.ClientConn, error) { 12 | etcdResolver, err := resolver.NewBuilder(c) 13 | if err != nil { 14 | return nil, err 15 | } 16 | return grpc.Dial( 17 | "etcd:///"+service+"/"+target, 18 | grpc.WithResolvers(etcdResolver), 19 | grpc.WithInsecure(), 20 | grpc.WithBlock(), 21 | ) 22 | } 23 | -------------------------------------------------------------------------------- /geek/registry/register.go: -------------------------------------------------------------------------------- 1 | package registry 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "time" 8 | 9 | clientv3 "go.etcd.io/etcd/client/v3" 10 | "go.etcd.io/etcd/client/v3/naming/endpoints" 11 | ) 12 | 13 | var ( 14 | GlobalClientConfig *clientv3.Config = defaultEtcdConfig 15 | defaultEtcdConfig = &clientv3.Config{ 16 | Endpoints: []string{"127.0.0.1:2379"}, 17 | DialTimeout: 5 * time.Second, 18 | } 19 | ) 20 | 21 | // 在租赁模式添加一对kv至etcd 22 | func etcdAdd(c *clientv3.Client, lid clientv3.LeaseID, service, addr string) error { 23 | em, err := endpoints.NewManager(c, service) 24 | if err != nil { 25 | return err 26 | } 27 | return em.AddEndpoint(c.Ctx(), service+"/"+addr, endpoints.Endpoint{Addr: addr}, clientv3.WithLease(lid)) 28 | } 29 | 30 | // Register register a service to etcd 31 | // no return if not error 32 | func Register(service, addr string, stop chan error) error { 33 | cli, err := clientv3.New(*GlobalClientConfig) 34 | if err != nil { 35 | return fmt.Errorf("create etcd client failed: %v", err) 36 | } 37 | defer cli.Close() 38 | // create a lease for 5 seconds 39 | resp, err := cli.Grant(context.Background(), 2) 40 | if err != nil { 41 | return fmt.Errorf("create lease failed: %v", err) 42 | } 43 | leaseId := resp.ID 44 | // register service 45 | err = etcdAdd(cli, leaseId, service, addr) 46 | if err != nil { 47 | return fmt.Errorf("add etcd record failed: %v", err) 48 | } 49 | // set heartbeat 50 | ch, err := cli.KeepAlive(context.Background(), leaseId) 51 | if err != nil { 52 | return fmt.Errorf("set keepalive failed: %v", err) 53 | } 54 | log.Printf("[%s] register service success", addr) 55 | for { 56 | select { 57 | case err := <-stop: 58 | if err != nil { 59 | log.Println(err) 60 | } 61 | case <-cli.Ctx().Done(): 62 | log.Println("service closed") 63 | return nil 64 | case _, ok := <-ch: 65 | // 监听租约 66 | if !ok { 67 | log.Println("keepalive channel closed") 68 | _, err := cli.Revoke(context.Background(), leaseId) 69 | return err 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /geek/server.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/Makonike/geek-cache/geek/utils" 7 | "log" 8 | "net" 9 | "strings" 10 | "sync" 11 | 12 | pb "github.com/Makonike/geek-cache/geek/pb" 13 | registy "github.com/Makonike/geek-cache/geek/registry" 14 | 15 | "google.golang.org/grpc" 16 | "google.golang.org/grpc/reflection" 17 | ) 18 | 19 | const ( 20 | defaultServiceName = "geek-cache" 21 | defaultAddr = "127.0.0.1:7654" 22 | ) 23 | 24 | type Server struct { 25 | pb.UnimplementedGroupCacheServer 26 | self string // self ip 27 | sname string // name of service 28 | status bool // true if the server is running 29 | mu sync.Mutex // guards 30 | stopSignal chan error // signal to stop 31 | } 32 | 33 | type ServerOptions func(*Server) 34 | 35 | func NewServer(self string, opts ...ServerOptions) (*Server, error) { 36 | if self == "" { 37 | self = defaultAddr 38 | } else if !utils.ValidPeerAddr(self) { 39 | return nil, fmt.Errorf("invalid address: %v", self) 40 | } 41 | s := Server{ 42 | self: self, 43 | sname: defaultServiceName, 44 | } 45 | for _, opt := range opts { 46 | opt(&s) 47 | } 48 | return &s, nil 49 | } 50 | 51 | func ServiceName(sname string) ServerOptions { 52 | return func(s *Server) { 53 | s.sname = sname 54 | } 55 | } 56 | 57 | // Log info 58 | func (s *Server) Log(format string, path ...interface{}) { 59 | log.Printf("[Server %s] %s", s.self, fmt.Sprintf(format, path...)) 60 | } 61 | 62 | func (s *Server) Get(ctx context.Context, in *pb.Request) (*pb.ResponseForGet, error) { 63 | group, key := in.GetGroup(), in.GetKey() 64 | out := &pb.ResponseForGet{} 65 | log.Printf("[Geek-Cache %s] Recv RPC Request for get- (%s)/(%s)", s.self, group, key) 66 | 67 | if key == "" { 68 | return out, fmt.Errorf("key required") 69 | } 70 | g := GetGroup(group) 71 | if g == nil { 72 | return out, fmt.Errorf("group not found") 73 | } 74 | view, err := g.Get(key) 75 | if err != nil { 76 | return out, err 77 | } 78 | out.Value = view.ByteSLice() 79 | return out, nil 80 | } 81 | 82 | func (s *Server) Delete(ctx context.Context, in *pb.Request) (*pb.ResponseForDelete, error) { 83 | group, key := in.GetGroup(), in.GetKey() 84 | out := &pb.ResponseForDelete{} 85 | log.Printf("[Geek-Cache %s] Recv RPC Request for delete - (%s)/(%s)", s.self, group, key) 86 | 87 | if key == "" { 88 | return out, fmt.Errorf("key required") 89 | } 90 | g := GetGroup(group) 91 | if g == nil { 92 | return out, fmt.Errorf("group not found") 93 | } 94 | isSuccess, err := g.Delete(key) 95 | if err != nil { 96 | return out, err 97 | } 98 | out.Value = isSuccess 99 | return out, nil 100 | } 101 | 102 | func (s *Server) Start() error { 103 | s.mu.Lock() 104 | if s.status { 105 | s.mu.Unlock() 106 | return fmt.Errorf("server already running") 107 | } 108 | s.status = true 109 | s.stopSignal = make(chan error) 110 | 111 | port := strings.Split(s.self, ":")[1] 112 | l, err := net.Listen("tcp", ":"+port) 113 | if err != nil { 114 | return fmt.Errorf("failed to listen on %s: %v", port, err) 115 | } 116 | grpcServer := grpc.NewServer() 117 | pb.RegisterGroupCacheServer(grpcServer, s) 118 | // 启动 reflection 反射服务 119 | reflection.Register(grpcServer) 120 | go func() { 121 | err := registy.Register(s.sname, s.self, s.stopSignal) 122 | if err != nil { 123 | log.Fatalf(err.Error()) 124 | } 125 | close(s.stopSignal) 126 | err = l.Close() 127 | if err != nil { 128 | log.Fatalf(err.Error()) 129 | } 130 | log.Printf("[%s] Revoke service and close tcp socket ok", s.self) 131 | }() 132 | 133 | s.mu.Unlock() 134 | if err := grpcServer.Serve(l); s.status && err != nil { 135 | return fmt.Errorf("failed to serve on %s: %v", port, err) 136 | } 137 | return nil 138 | } 139 | 140 | func (s *Server) Stop() { 141 | s.mu.Lock() 142 | if !s.status { 143 | s.mu.Unlock() 144 | return 145 | } 146 | s.stopSignal <- nil 147 | s.status = false 148 | s.mu.Unlock() 149 | } 150 | -------------------------------------------------------------------------------- /geek/server_test.go: -------------------------------------------------------------------------------- 1 | package geek 2 | 3 | import ( 4 | "fmt" 5 | "github.com/stretchr/testify/assert" 6 | "log" 7 | "math/rand" 8 | "reflect" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | var server_test_db = map[string]string{ 14 | "Tom": "630", 15 | "Tom2": "631", 16 | "Tom3": "632", 17 | } 18 | 19 | func TestServer(t *testing.T) { 20 | a := assert.New(t) 21 | g := NewGroup("scores", 2<<10, GetterFunc( 22 | func(key string) ([]byte, bool, time.Time) { 23 | log.Println("[SlowDB] search key", key) 24 | if v, ok := server_test_db[key]; ok { 25 | return []byte(v), true, time.Time{} 26 | } 27 | return nil, false, time.Time{} 28 | })) 29 | r := rand.New(rand.NewSource(time.Now().UnixNano())) 30 | port := 50000 + r.Intn(100) 31 | addr := fmt.Sprintf("localhost:%d", port) 32 | 33 | // 添加peerPicker 34 | picker := NewClientPicker(addr) 35 | 36 | g.RegisterPeers(picker) 37 | 38 | view, err := g.Get("Tom") 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | s, err := g.Delete("Tom") 43 | a.True(s) 44 | a.Nil(err) 45 | if !reflect.DeepEqual(view.String(), "630") { 46 | t.Errorf("Tom %s(actual)/%s(ok)", view.String(), "630") 47 | } 48 | view, err = g.Get("Unknown") 49 | if err == nil || view.String() != "" { 50 | t.Errorf("Unknown not exists, but got %s", view.String()) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /geek/singleflight/singleflight.go: -------------------------------------------------------------------------------- 1 | package singleflight 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | // 代表正在进行或已结束的请求 8 | type call struct { 9 | wg sync.WaitGroup 10 | val interface{} 11 | err error 12 | } 13 | 14 | // Group manages all kinds of calls 15 | type Group struct { 16 | mu sync.Mutex // guards for m 17 | m map[string]*call 18 | } 19 | 20 | // Do 针对相同的key，保证多次调用Do()，都只会调用一次fn 21 | func (g *Group) Do(key string, fn func() (interface{}, error)) (interface{}, error) { 22 | // lock protects for m in concurrent calls 23 | g.mu.Lock() 24 | if g.m == nil { 25 | g.m = make(map[string]*call) 26 | } 27 | if c, ok := g.m[key]; ok { 28 | g.mu.Unlock() 29 | c.wg.Wait() // wait for doing request 30 | return c.val, c.err // request completed, return result 31 | } 32 | c := new(call) // a new request, and this is the first request for this key 33 | c.wg.Add(1) 34 | g.m[key] = c 35 | g.mu.Unlock() 36 | 37 | c.val, c.err = fn() // with callback 38 | c.wg.Done() // this request was completed, and other requests for this key will be continue 39 | g.mu.Lock() 40 | delete(g.m, key) 41 | g.mu.Unlock() 42 | 43 | return c.val, c.err 44 | } 45 | -------------------------------------------------------------------------------- /geek/utils/str_encoding.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "unsafe" 4 | 5 | // StrToUUID 将str转换为UUID, 使用了一种简单的hash算法, 可能会有冲突. 6 | // 另外转换后的UUID将无序. 7 | func StrToUUID(str string) uint64 { 8 | var seed uint64 = 13331 9 | var result uint64 10 | for _, b := range str { 11 | result = result*seed + uint64(b) 12 | } 13 | return result 14 | } 15 | 16 | func VarStrToRaw(str string) []byte { 17 | tmp1 := (*[2]uintptr)(unsafe.Pointer(&str)) 18 | tmp2 := [3]uintptr{tmp1[0], tmp1[1], tmp1[1]} 19 | return *(*[]byte)(unsafe.Pointer(&tmp2)) 20 | } 21 | -------------------------------------------------------------------------------- /geek/utils/utils.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | func ValidPeerAddr(addr string) bool { 8 | t1 := strings.Split(addr, ":") 9 | if len(t1) != 2 { 10 | return false 11 | } 12 | // TODO: more selections 13 | t2 := strings.Split(t1[0], ".") 14 | if t1[0] != "localhost" && len(t2) != 4 { 15 | return false 16 | } 17 | return true 18 | } 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/Makonike/geek-cache 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/stretchr/testify v1.7.0 7 | go.etcd.io/etcd/client/v3 v3.5.7 8 | google.golang.org/grpc v1.51.0 9 | google.golang.org/protobuf v1.28.1 10 | ) 11 | 12 | require ( 13 | github.com/coreos/go-semver v0.3.0 // indirect 14 | github.com/coreos/go-systemd/v22 v22.3.2 // indirect 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/gogo/protobuf v1.3.2 // indirect 17 | github.com/golang/protobuf v1.5.2 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | go.etcd.io/etcd/api/v3 v3.5.7 // indirect 20 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect 21 | go.uber.org/atomic v1.7.0 // indirect 22 | go.uber.org/multierr v1.6.0 // indirect 23 | go.uber.org/zap v1.17.0 // indirect 24 | golang.org/x/net v0.7.0 // indirect 25 | golang.org/x/sys v0.5.0 // indirect 26 | golang.org/x/text v0.7.0 // indirect 27 | google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect 28 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= 2 | github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 3 | github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= 4 | github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= 5 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 7 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= 9 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 10 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 11 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= 12 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= 13 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= 14 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= 16 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 17 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 18 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 19 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 20 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 21 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 24 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 25 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 26 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 27 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 28 | go.etcd.io/etcd/api/v3 v3.5.7 h1:sbcmosSVesNrWOJ58ZQFitHMdncusIifYcrBfwrlJSY= 29 | go.etcd.io/etcd/api/v3 v3.5.7/go.mod h1:9qew1gCdDDLu+VwmeG+iFpL+QlpHTo7iubavdVDgCAA= 30 | go.etcd.io/etcd/client/pkg/v3 v3.5.7 h1:y3kf5Gbp4e4q7egZdn5T7W9TSHUvkClN6u+Rq9mEOmg= 31 | go.etcd.io/etcd/client/pkg/v3 v3.5.7/go.mod h1:o0Abi1MK86iad3YrWhgUsbGx1pmTS+hrORWc2CamuhY= 32 | go.etcd.io/etcd/client/v3 v3.5.7 h1:u/OhpiuCgYY8awOHlhIhmGIGpxfBU/GZBUP3m/3/Iz4= 33 | go.etcd.io/etcd/client/v3 v3.5.7/go.mod h1:sOWmj9DZUMyAngS7QQwCyAXXAL6WhgTOPLNS/NabQgw= 34 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 35 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 36 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 37 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 38 | go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= 39 | go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= 40 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 41 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 42 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 43 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 44 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 45 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 46 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 47 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 48 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 49 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 50 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 51 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 52 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 53 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 54 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 55 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 56 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 57 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 58 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 59 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 60 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 61 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 62 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 63 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 64 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 65 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 66 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 67 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 68 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 69 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 70 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 71 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 72 | google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= 73 | google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= 74 | google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= 75 | google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= 76 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 77 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= 78 | google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= 79 | google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= 80 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 81 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 82 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 83 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 84 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 85 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= 86 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 87 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "strconv" 7 | "time" 8 | 9 | "github.com/Makonike/geek-cache/geek" 10 | ) 11 | 12 | func main() { 13 | var port int 14 | flag.IntVar(&port, "port", 8001, "Geecache server port") 15 | flag.Parse() 16 | // mock database or other dataSource 17 | var mysql = map[string]string{ 18 | "Tom": "630", 19 | "Tom1": "631", 20 | "Tom2": "632", 21 | } 22 | // NewGroup create a Group which means a kind of sources 23 | // contain a func that used when misses cache 24 | g := geek.NewGroup("scores", 2<<10, geek.GetterFunc( 25 | func(key string) ([]byte, bool, time.Time) { 26 | log.Println("[SlowDB] search key", key) 27 | if v, ok := mysql[key]; ok { 28 | return []byte(v), true, time.Time{} 29 | } 30 | return nil, false, time.Time{} 31 | })) 32 | 33 | var addr string = "127.0.0.1:" + strconv.Itoa(port) 34 | 35 | server, err := geek.NewServer(addr) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | 40 | picker := geek.NewClientPicker(addr) 41 | g.RegisterPeers(picker) 42 | 43 | for { 44 | err = server.Start() 45 | if err != nil { 46 | log.Println(err.Error()) 47 | return 48 | } 49 | } 50 | } 51 | --------------------------------------------------------------------------------