├── LICENSE ├── README.md ├── _test.ini ├── auto_cache.go ├── auto_cache_entry.go ├── auto_cache_example_test.go ├── auto_cache_fake.go ├── auto_cache_interface.go ├── byte_cache_aerospike.go ├── byte_cache_aerospike_bench_test.go ├── byte_cache_aerospike_config.go ├── byte_cache_aerospike_example_test.go ├── byte_cache_aerospike_indexes.go ├── byte_cache_aerospike_test.go ├── byte_cache_blackhole.go ├── byte_cache_interface.go ├── byte_cache_wrapper.go ├── entry.go ├── errors └── errors.go ├── glide.yaml ├── interface.go ├── key.go ├── metric ├── dummy │ └── metric.go └── metric.go ├── nil_logger.go ├── struct_cache.go ├── struct_cache_benchmark_test.go ├── struct_cache_dummy.go ├── struct_cache_example_test.go ├── struct_cache_helper.go ├── struct_cache_helper_test.go ├── struct_cache_interface.go ├── struct_cache_test.go └── ttls.go /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2017 Lazada Group 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GO-cache # 2 | A nice and fast set of caches by LAZADA! 3 | All cache types are thread safe and supports prometheus metrics. 4 | 5 | ## Have 3 cache interfaces: ## 6 | 1. IAutoCache 7 | 2. IByteCache 8 | 3. IStructCache 9 | 10 | # **IAutoCache:** # 11 | 12 | ## Implementations: ## 13 | * StorageAutoCacheFake 14 | * StorageAutoCache 15 | 16 | ### StorageAutoCacheFake: ### 17 | Can be used in tests 18 | 19 | ### StorageAutoCache: ### 20 | Executes callback when record`s ttl expired. 21 | 22 | #### Example: #### 23 | ```go 24 | package main 25 | 26 | import ( 27 | "fmt" 28 | "math/rand" 29 | "time" 30 | 31 | "go-cache" 32 | ) 33 | 34 | func init() { 35 | rand.Seed(42) 36 | } 37 | 38 | func getValue() (interface{}, error) { 39 | return rand.Intn(100), nil 40 | } 41 | 42 | func main() { 43 | autoCache := cache.NewStorageAutoCacheObject(true, nil) 44 | 45 | const key = "unique-key" 46 | 47 | err := autoCache.Put(getValue, key, 100*time.Second) 48 | if err != nil { 49 | panic(err) 50 | } 51 | 52 | value, err := autoCache.Get(key) 53 | if err != nil { 54 | panic(err) 55 | } 56 | 57 | fmt.Print(value) 58 | // Output: 59 | // 5 60 | } 61 | ``` 62 | 63 | # **IByteCache:** # 64 | 65 | ## Implementations: ## 66 | * BlackholeCache 67 | * AerospikeCache 68 | 69 | ### BlackholeCache: ### 70 | Can be used in tests 71 | 72 | ### AerospikeCache: ### 73 | Use aerospike to store slices of bytes. Can be limited. Can store data into different sets. Also supports tags. 74 | 75 | #### Example: #### 76 | ```go 77 | package main 78 | 79 | import ( 80 | "fmt" 81 | "time" 82 | 83 | "go-cache" 84 | "go-cache/metric/dummy" 85 | ) 86 | 87 | func main() { 88 | config := &cache.AerospikeConfig{ 89 | NameSpace: "test", 90 | Hosts: []string{"localhost:3000"}, 91 | MaxRetries: 5, 92 | } 93 | logger := &cache.AerospikeDummyLogger{} 94 | 95 | client, err := cache.CreateAerospikeClient(config, logger) 96 | if err != nil { 97 | panic(err) 98 | } 99 | 100 | aerospikeCache := cache.NewAerospikeCache(config, client, logger, dummy.NewMetric()) 101 | 102 | const wirth = "But quality of work can be expected only through personal satisfaction, dedication and enjoyment. In our profession, precision and perfection are not a dispensible luxury, but a simple necessity." 103 | data := []byte(wirth) 104 | key := &cache.Key{ 105 | Set: "testset", 106 | Pk: "testExpired", 107 | } 108 | 109 | aerospikeCache.Put(data, key, time.Second) 110 | 111 | cachedData, ok := aerospikeCache.Get(key) 112 | if !ok { 113 | panic("Something went wrong") 114 | } 115 | 116 | fmt.Println(string(cachedData)) 117 | // Output: 118 | // But quality of work can be expected only through personal satisfaction, dedication and enjoyment. In our profession, precision and perfection are not a dispensible luxury, but a simple necessity. 119 | } 120 | ``` 121 | 122 | # **StructCache:** # 123 | Can store type into local cache. Fast and tread safe. 124 | 125 | ## Supports: ## 126 | * Limit 127 | * LRU 128 | * Cache sets 129 | 130 | #### Example: #### 131 | ```go 132 | package main 133 | 134 | import ( 135 | "fmt" 136 | "time" 137 | 138 | "go-cache" 139 | "go-cache/metric/dummy" 140 | ) 141 | 142 | func main() { 143 | structCache := cache.NewStructCacheObject(8000, nil, dummy.NewMetric()) 144 | 145 | k := &cache.Key{ 146 | Set: "set", 147 | Pk: "1", 148 | } 149 | data := "The essential is invisible to the eyes, we can not truly see but with the eyes of the heart." 150 | 151 | ttl := 5*time.Minute 152 | 153 | structCache.Put(data, k, ttl) 154 | 155 | infResult, find := structCache.Get(k) 156 | if !find { 157 | panic("Key is not found") 158 | } 159 | 160 | result, ok := infResult.(string) 161 | if !ok { 162 | panic("Data have wrong type") 163 | } 164 | 165 | fmt.Println(string(result)) 166 | // Output: 167 | // The essential is invisible to the eyes, we can not truly see but with the eyes of the heart. 168 | } 169 | ``` -------------------------------------------------------------------------------- /_test.ini: -------------------------------------------------------------------------------- 1 | [aerospike] 2 | namespace = test 3 | hosts = 127.0.0.1:3000 4 | aerospike_connection_timeout = 1m 5 | aerospike_idle_timeout = 5m -------------------------------------------------------------------------------- /auto_cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "go-cache/errors" 8 | ) 9 | 10 | // StorageAutoCache implements cache that uses auto cache as storage 11 | type StorageAutoCache struct { 12 | logger IAutoCacheLogger 13 | active bool 14 | entries map[string]*EntryAutoCache 15 | lock sync.RWMutex 16 | } 17 | 18 | // NewStorageAutoCacheObject create new instance of StorageAutoCache 19 | func NewStorageAutoCacheObject(active bool, logger IAutoCacheLogger) *StorageAutoCache { 20 | if logger == nil { 21 | logger = NewNilLogger() 22 | } 23 | return &StorageAutoCache{ 24 | logger: logger, 25 | active: active, 26 | entries: map[string]*EntryAutoCache{}, 27 | } 28 | } 29 | 30 | // Get returns data by given key 31 | func (storage *StorageAutoCache) Get(key string) (interface{}, error) { 32 | entry, err := storage.getEntry(key) 33 | if err != nil { 34 | return nil, err 35 | } 36 | 37 | return entry.GetValue() 38 | } 39 | 40 | func (storage *StorageAutoCache) getEntry(key string) (*EntryAutoCache, error) { 41 | var err error 42 | 43 | storage.lock.RLock() 44 | entry, ok := storage.entries[key] 45 | storage.lock.RUnlock() 46 | if !ok { 47 | err = errors.Errorf("Auto cache key %s nof found", key) 48 | } 49 | 50 | return entry, err 51 | } 52 | 53 | // Remove removes value by key 54 | func (storage *StorageAutoCache) Remove(key string) { 55 | entry, err := storage.getEntry(key) 56 | 57 | if err == nil { 58 | entry.Stop() 59 | } 60 | 61 | storage.lock.Lock() 62 | delete(storage.entries, key) 63 | storage.lock.Unlock() 64 | } 65 | 66 | // Put puts data into storage 67 | func (storage *StorageAutoCache) Put(updater func() (interface{}, error), key string, ttl time.Duration) error { 68 | entry := CreateEntryAutoCache(updater, ttl, key, storage.logger) 69 | 70 | if storage.active { 71 | if err := entry.Start(); err != nil { 72 | return errors.Errorf("Auto cache updater \"%s\" error: %s", key, err) 73 | } 74 | } 75 | 76 | storage.lock.Lock() 77 | if oldEntry, ok := storage.entries[key]; ok { 78 | oldEntry.Stop() 79 | } 80 | storage.entries[key] = entry 81 | storage.lock.Unlock() 82 | 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /auto_cache_entry.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "go-cache/errors" 8 | ) 9 | 10 | // EntryAutoCache contains data 11 | type EntryAutoCache struct { 12 | name string 13 | value interface{} 14 | updater func() (interface{}, error) 15 | interval time.Duration 16 | run bool 17 | ticker *time.Ticker 18 | mutex sync.RWMutex 19 | signals chan struct{} 20 | logger IAutoCacheLogger 21 | } 22 | 23 | // CreateEntryAutoCache returns new instance of EntryAutoCache 24 | func CreateEntryAutoCache(updater func() (interface{}, error), interval time.Duration, name string, logger IAutoCacheLogger) *EntryAutoCache { 25 | return &EntryAutoCache{ 26 | name: name, 27 | run: false, 28 | updater: updater, 29 | interval: interval, 30 | logger: logger, 31 | } 32 | } 33 | 34 | // GetValue returns the result of processing the updater 35 | func (entry *EntryAutoCache) GetValue() (interface{}, error) { 36 | entry.mutex.RLock() 37 | 38 | if !entry.run { 39 | entry.mutex.RUnlock() 40 | if err := entry.process(); err != nil { 41 | return nil, err 42 | } 43 | 44 | entry.mutex.RLock() 45 | } 46 | 47 | defer entry.mutex.RUnlock() 48 | 49 | if entry.value == nil { 50 | return nil, errors.Errorf("Value is not set") 51 | } 52 | 53 | return entry.value, nil 54 | } 55 | 56 | // Start starts updater process in goroutine 57 | func (entry *EntryAutoCache) Start() error { 58 | if entry.run { 59 | return nil 60 | } 61 | entry.run = true 62 | entry.signals = make(chan struct{}, 1) 63 | entry.ticker = time.NewTicker(entry.interval) 64 | if err := entry.process(); err != nil { 65 | return err 66 | } 67 | 68 | go func() { 69 | defer func() { 70 | if r := recover(); r != nil { 71 | entry.logger.Criticalf("Panic in entry.loop(), %v", r) 72 | } 73 | }() 74 | entry.loop() 75 | }() 76 | return nil 77 | } 78 | 79 | // Stop stops updater process 80 | func (entry *EntryAutoCache) Stop() { 81 | entry.mutex.Lock() 82 | defer entry.mutex.Unlock() 83 | 84 | if entry.run { 85 | entry.run = false 86 | entry.value = nil 87 | entry.ticker.Stop() 88 | 89 | entry.signals <- struct{}{} 90 | } 91 | } 92 | 93 | func (entry *EntryAutoCache) process() error { 94 | entry.mutex.RLock() 95 | updater := entry.updater 96 | name := entry.name 97 | entry.mutex.RUnlock() 98 | 99 | if value, err := updater(); err == nil { 100 | entry.mutex.Lock() 101 | entry.value = value 102 | entry.mutex.Unlock() 103 | } else { 104 | entry.logger.Errorf("Auto cache updater \"%s\" error: %s", name, err) 105 | return err 106 | } 107 | 108 | return nil 109 | } 110 | 111 | func (entry *EntryAutoCache) loop() { 112 | for { 113 | select { 114 | case <-entry.ticker.C: 115 | entry.process() 116 | case <-entry.signals: 117 | return 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /auto_cache_example_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | ) 8 | 9 | func ExampleStorageAutoCache_Get() { 10 | autoCache := NewStorageAutoCacheObject(true, nil) 11 | 12 | const key = "unique-key" 13 | getValue := func() (interface{}, error) { 14 | rand.Seed(42) 15 | return rand.Intn(100), nil 16 | } 17 | 18 | err := autoCache.Put(getValue, key, 100*time.Second) 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | value, err := autoCache.Get(key) 24 | if err != nil { 25 | panic(err) 26 | } 27 | 28 | fmt.Print(value) 29 | // Output: 30 | // 5 31 | } 32 | 33 | func ExampleStorageAutoCache_Remove() { 34 | autoCache := NewStorageAutoCacheObject(true, nil) 35 | 36 | const key = "unique-key" 37 | getValue := func() (interface{}, error) { 38 | rand.Seed(42) 39 | return rand.Intn(100), nil 40 | } 41 | 42 | err := autoCache.Put(getValue, key, 100*time.Second) 43 | if err != nil { 44 | panic(err) 45 | } 46 | 47 | autoCache.Remove(key) 48 | 49 | _, err = autoCache.Get(key) 50 | 51 | fmt.Print(err) 52 | // Output: 53 | // Auto cache key unique-key nof found 54 | } 55 | -------------------------------------------------------------------------------- /auto_cache_fake.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "go-cache/errors" 8 | ) 9 | 10 | // StorageAutoCacheFake implements cache that uses auto cache as storage 11 | // This type of autocache have no TTL 12 | type StorageAutoCacheFake struct { 13 | updaters map[string]func() (interface{}, error) 14 | mutex sync.RWMutex 15 | } 16 | 17 | // NewStorageAutoCache create new instance of StorageAutoCache 18 | func NewStorageAutoCacheFake() *StorageAutoCacheFake { 19 | return &StorageAutoCacheFake{ 20 | updaters: make(map[string]func() (interface{}, error)), 21 | } 22 | } 23 | 24 | // Get returns data by given key 25 | func (storage *StorageAutoCacheFake) Get(key string) (interface{}, error) { 26 | storage.mutex.RLock() 27 | updater, find := storage.updaters[key] 28 | storage.mutex.RUnlock() 29 | if !find { 30 | return nil, errors.Errorf("Auto cache key %s nof found", key) 31 | } 32 | 33 | return updater() 34 | } 35 | 36 | // Remove removes value by key 37 | func (storage *StorageAutoCacheFake) Remove(key string) { 38 | storage.mutex.Lock() 39 | delete(storage.updaters, key) 40 | storage.mutex.Unlock() 41 | } 42 | 43 | // Put puts data into storage 44 | func (storage *StorageAutoCacheFake) Put(updater func() (interface{}, error), key string, ttl time.Duration) error { 45 | storage.mutex.Lock() 46 | storage.updaters[key] = updater 47 | storage.mutex.Unlock() 48 | 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /auto_cache_interface.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | // IAutoCache defines required interface for caching module (need for auto cache) 6 | type IAutoCache interface { 7 | Get(key string) (data interface{}, err error) 8 | Put(updater func() (interface{}, error), key string, ttl time.Duration) error 9 | Remove(key string) 10 | } 11 | 12 | type IAutoCacheLogger interface { 13 | Errorf(message string, args ...interface{}) 14 | Criticalf(message string, args ...interface{}) 15 | } 16 | -------------------------------------------------------------------------------- /byte_cache_aerospike.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "net" 5 | "strconv" 6 | "sync" 7 | "time" 8 | 9 | "github.com/aerospike/aerospike-client-go" 10 | aerospikeLogger "github.com/aerospike/aerospike-client-go/logger" 11 | "github.com/aerospike/aerospike-client-go/types" 12 | 13 | "go-cache/errors" 14 | "go-cache/metric" 15 | ) 16 | 17 | const ( 18 | dataBin = "data" 19 | tagsBin = "tags" 20 | 21 | defaultReadTimeout = 100 * time.Millisecond 22 | defaultUpdateConnectionCountMetricInterval = time.Second 23 | ) 24 | 25 | // AerospikeCache implements cache that uses Aerospike as storage 26 | type AerospikeCache struct { 27 | ns string 28 | client *aerospike.Client 29 | cachePrefix string 30 | getPolicy *aerospike.BasePolicy 31 | maxRetries int 32 | config *AerospikeConfig 33 | logger IAerospikeCacheLogger 34 | metric metric.Metric 35 | 36 | // connection count metric 37 | updateConnectionCountMetricInterval time.Duration 38 | quitUpdateConnectionCountMetricChan chan struct{} 39 | mu sync.Mutex 40 | } 41 | 42 | var _ IByteCache = &AerospikeCache{} // AerospikeCache implements IByteCache 43 | 44 | // NewAerospikeCache initializes instance of Aerospike-based cache 45 | func NewAerospikeCache(config *AerospikeConfig, client *aerospike.Client, logger IAerospikeCacheLogger, metric metric.Metric) *AerospikeCache { 46 | if client == nil { 47 | return nil 48 | } 49 | if logger == nil { 50 | logger = NewNilLogger() 51 | } 52 | 53 | aeroCache := newAerospike(config, client, logger, metric) 54 | return aeroCache 55 | } 56 | 57 | // CreateAerospikeClient creates AerospikeClient for AerospikeCache 58 | func CreateAerospikeClient(config *AerospikeConfig, logger IAerospikeCacheLogger) (*aerospike.Client, error) { 59 | if config.NameSpace == "" { 60 | return nil, errors.New("need aerospike namespace") 61 | } 62 | 63 | if len(config.Hosts) == 0 || (len(config.Hosts) == 1 && config.Hosts[0] == "") { 64 | return nil, errors.New("aerospike host list is empty") 65 | } 66 | 67 | if logger == nil { 68 | logger = NewNilLogger() 69 | } 70 | aerospikeLogger.Logger.SetLogger(logger) 71 | aerospikeLogger.Logger.SetLevel(aerospikeLogger.LogPriority(config.LogLevel)) 72 | 73 | hosts := make([]*aerospike.Host, len(config.Hosts)) 74 | for i, connStr := range config.Hosts { 75 | hostStr, portStr, err := net.SplitHostPort(connStr) 76 | if err != nil { 77 | return nil, err 78 | } 79 | port, err := strconv.Atoi(portStr) 80 | if err != nil { 81 | return nil, err 82 | } 83 | hosts[i] = aerospike.NewHost(hostStr, port) 84 | } 85 | 86 | clientPolicy := aerospike.NewClientPolicy() 87 | 88 | if config.ConnectionTimeout > 0 { 89 | clientPolicy.Timeout = config.ConnectionTimeout 90 | } 91 | 92 | if config.IdleTimeout > 0 { 93 | clientPolicy.IdleTimeout = config.IdleTimeout 94 | } 95 | 96 | if config.ConnectionQueueSize > 0 { 97 | clientPolicy.ConnectionQueueSize = config.ConnectionQueueSize 98 | } 99 | 100 | if config.LimitConnectionsToQueueSize { 101 | clientPolicy.LimitConnectionsToQueueSize = true 102 | } 103 | 104 | clientPolicy.FailIfNotConnected = config.FailIfNotConnected 105 | 106 | client, err := aerospike.NewClientWithPolicyAndHost(clientPolicy, hosts...) 107 | if err != nil { 108 | return nil, err 109 | } 110 | 111 | return client, nil 112 | } 113 | 114 | // newAerospike internal constructor 115 | func newAerospike(config *AerospikeConfig, client *aerospike.Client, logger IAerospikeCacheLogger, metric metric.Metric) *AerospikeCache { 116 | aerospikeLogger.Logger.SetLogger(logger) 117 | aerospikeLogger.Logger.SetLevel(aerospikeLogger.LogPriority(config.LogLevel)) 118 | 119 | // if update connection count metric interval not set use default (1s) 120 | updateConnectionCountMetricInterval := defaultUpdateConnectionCountMetricInterval 121 | if config.UpdateConnectionCountMetricInterval > 0 { 122 | updateConnectionCountMetricInterval = config.UpdateConnectionCountMetricInterval 123 | } 124 | 125 | getPolicy := aerospike.NewPolicy() 126 | 127 | // if read timeout not set use default (100ms) 128 | getPolicy.Timeout = defaultReadTimeout 129 | if config.ReadTimeout > 0 { 130 | getPolicy.Timeout = config.ReadTimeout 131 | } 132 | 133 | if config.MaxRetries > 0 { 134 | getPolicy.MaxRetries = config.MaxRetries 135 | } 136 | 137 | if config.SleepBetweenRetries > 0 { 138 | getPolicy.SleepBetweenRetries = config.SleepBetweenRetries 139 | } 140 | 141 | ac := &AerospikeCache{ 142 | ns: config.NameSpace, 143 | client: client, 144 | cachePrefix: config.Prefix, 145 | getPolicy: getPolicy, 146 | maxRetries: config.MaxRetries, 147 | config: config, 148 | logger: logger, 149 | metric: metric, 150 | quitUpdateConnectionCountMetricChan: make(chan struct{}), 151 | updateConnectionCountMetricInterval: updateConnectionCountMetricInterval, 152 | } 153 | 154 | go ac.updateConnectionCountMetric() 155 | 156 | return ac 157 | } 158 | 159 | // Client returns aerospike client. 160 | func (a *AerospikeCache) Client() *aerospike.Client { 161 | return a.client 162 | } 163 | 164 | // CreateTagsIndex creates tags index (indexName) by setName 165 | func (a *AerospikeCache) CreateTagsIndex(aerospikeIndex AerospikeIndex) error { 166 | if aerospikeIndex.SetName == "" { 167 | return errors.New("setName cant be empty") 168 | } 169 | if aerospikeIndex.IndexName == "" { 170 | return errors.New("indexName cant be empty") 171 | } 172 | 173 | policy := aerospike.NewWritePolicy(0, 0) 174 | policy.MaxRetries = a.maxRetries 175 | 176 | createTask, err := a.client.CreateComplexIndex( 177 | policy, 178 | a.ns, 179 | aerospikeIndex.SetName, 180 | aerospikeIndex.IndexName, 181 | tagsBin, 182 | aerospikeIndex.IndexType, 183 | aerospike.ICT_LIST, 184 | ) 185 | 186 | if err != nil { 187 | aerospikeError, ok := err.(types.AerospikeError) 188 | if ok && aerospikeError.ResultCode() == types.INDEX_FOUND { 189 | a.logger.Debugf( 190 | "Index %s already exists. Namespace: %s, setName: %s", 191 | aerospikeIndex.IndexName, 192 | a.ns, 193 | aerospikeIndex.SetName, 194 | ) 195 | return nil 196 | } 197 | 198 | return errors.Errorf("Aerospike createIndex error: '%s'", err) 199 | } 200 | 201 | for err := range createTask.OnComplete() { 202 | if err != nil { 203 | return errors.Errorf("Aerospike createIndex error: '%s'", err) 204 | } 205 | } 206 | 207 | a.logger.Debugf( 208 | "Aerospike tags index added. Namespace: %s, setName: %s, indexName: %s", 209 | a.ns, 210 | aerospikeIndex.SetName, 211 | aerospikeIndex.IndexName, 212 | ) 213 | 214 | return nil 215 | } 216 | 217 | // SetCachePrefix defines prefix for user key 218 | func (a *AerospikeCache) SetCachePrefix(prefix string) { 219 | a.mu.Lock() 220 | a.cachePrefix = prefix 221 | a.mu.Unlock() 222 | } 223 | 224 | // Get returns data by given key 225 | func (a *AerospikeCache) Get(key *Key) ([]byte, bool) { 226 | 227 | ts := time.Now() 228 | var ( 229 | ok bool 230 | node *aerospike.Node 231 | buf []byte 232 | err error 233 | ) 234 | 235 | buf, node, ok, err = a.getByPk(key.Set, key.Pk) 236 | 237 | a.updateHitOrMissCount(ok, map[string]string{ 238 | metric.LabelHost: a.getNodeHostName(node), 239 | metric.LabelNamespace: a.ns, 240 | metric.LabelSet: key.Set, 241 | }) 242 | 243 | a.metric.ObserveRT(map[string]string{ 244 | metric.LabelHost: a.getNodeHostName(node), 245 | metric.LabelNamespace: a.ns, 246 | metric.LabelSet: key.Set, 247 | metric.LabelOperation: "get", 248 | metric.LabelIsError: metric.IsError(err), 249 | }, metric.SinceMs(ts)) 250 | 251 | return buf, ok 252 | } 253 | 254 | func (a *AerospikeCache) updateHitOrMissCount(condition bool, labels map[string]string) { 255 | switch condition { 256 | case true: 257 | a.metric.RegisterHit(labels) 258 | case false: 259 | a.metric.RegisterMiss(labels) 260 | } 261 | } 262 | 263 | func (a *AerospikeCache) getByPk(set, pk string) ([]byte, *aerospike.Node, bool, error) { 264 | var ( 265 | data []byte 266 | ok bool 267 | err error 268 | ) 269 | 270 | key, err := a.createKey(set, pk) 271 | if err != nil { 272 | a.logger.Warning(err.Error()) 273 | return data, nil, ok, err 274 | } 275 | 276 | rec, err := a.client.Get(a.getPolicy, key, dataBin) 277 | 278 | if err != nil { 279 | a.logger.Warningf("could not get data for set '%s' by primary key '%s', error: %q", set, pk, err.Error()) 280 | return data, nil, ok, err 281 | } 282 | 283 | if rec == nil { 284 | return data, nil, ok, err 285 | } 286 | 287 | var bin interface{} 288 | bin, ok = rec.Bins[dataBin] 289 | if ok { 290 | data, ok = bin.([]byte) 291 | } 292 | 293 | return data, rec.Node, ok, err 294 | } 295 | 296 | // Put will delayed put cache in Aerospike 297 | func (a *AerospikeCache) Put(data []byte, key *Key, ttl time.Duration) { 298 | a.put(data, key, ttl) 299 | } 300 | 301 | func (a *AerospikeCache) put(data []byte, key *Key, ttl time.Duration) { 302 | ts := time.Now() 303 | var err error 304 | 305 | if len(key.Tags) == 0 { 306 | err = a.putByPk(data, key.Set, key.Pk, ttl) 307 | } else { 308 | err = a.putByPkAndTags(data, key.Set, key.Pk, key.Tags, ttl) 309 | } 310 | 311 | a.metric.ObserveRT(map[string]string{ 312 | metric.LabelNamespace: a.ns, 313 | metric.LabelSet: key.Set, 314 | metric.LabelOperation: "put", 315 | metric.LabelIsError: metric.IsError(err), 316 | }, metric.SinceMs(ts)) 317 | } 318 | 319 | func (a *AerospikeCache) putByPk(data []byte, set, pk string, ttl time.Duration) error { 320 | aeroKey, err := a.createKey(set, pk) 321 | if err != nil { 322 | a.logger.Warning(err.Error()) 323 | return err 324 | } 325 | 326 | bins := []*aerospike.Bin{ 327 | aerospike.NewBin(dataBin, data), 328 | } 329 | 330 | policy := a.getWritePolice(ttl) 331 | 332 | if err = a.client.PutBins(policy, aeroKey, bins...); err != nil { 333 | a.logger.Warningf("could not put into set '%s' by primary key '%s': %+v", set, pk, err) 334 | } 335 | 336 | return err 337 | } 338 | 339 | func (a *AerospikeCache) putByPkAndTags(data []byte, set, pk string, tags []string, ttl time.Duration) error { 340 | var err error 341 | 342 | // build composite primary key for quick data fetching 343 | aeroKey, err := a.createKey(set, pk) 344 | if err != nil { 345 | a.logger.Warning(err.Error()) 346 | return err 347 | } 348 | 349 | prefTags := make([]string, len(tags)) 350 | for i := range prefTags { 351 | prefTags[i] = a.cachePrefix + tags[i] 352 | } 353 | 354 | // add tagsBin to be able to invalidate cache by tag 355 | bins := []*aerospike.Bin{ 356 | aerospike.NewBin(dataBin, data), 357 | aerospike.NewBin(tagsBin, prefTags), 358 | } 359 | 360 | policy := a.getWritePolice(ttl) 361 | if a.config.PutTimeout > 0 { 362 | policy.Timeout = a.config.PutTimeout 363 | } 364 | 365 | if err = a.client.PutBins(policy, aeroKey, bins...); err != nil { 366 | a.logger.Warningf("could not put into set '%s' by primary key '%s': %+v", set, pk, err) 367 | } 368 | 369 | return nil 370 | } 371 | 372 | // ScanKeys return all keys for set 373 | func (a *AerospikeCache) ScanKeys(set string) ([]Key, error) { 374 | 375 | // We do not know how many records will return 376 | var keys []Key 377 | 378 | policy := aerospike.NewScanPolicy() 379 | policy.Priority = aerospike.LOW 380 | policy.IncludeBinData = true 381 | 382 | // Works only with records containing the BINS `id` and `tags` 383 | fields := []string{"id", "tags"} 384 | 385 | var ( 386 | pk string 387 | pkInterface interface{} 388 | 389 | tags []string 390 | binTags interface{} 391 | 392 | ok bool 393 | ) 394 | 395 | r, err := a.client.ScanAll(policy, a.ns, set, fields...) 396 | if err != nil { 397 | return nil, err 398 | } 399 | 400 | for v := range r.Records { 401 | 402 | // We can't use v.Key because v.Key.Value() is nil 403 | if pkInterface, ok = v.Bins["id"]; !ok { 404 | a.logger.Warningf("BINS `id` not found for aerospike set: %q", set) 405 | continue 406 | } 407 | 408 | if pk, ok = pkInterface.(string); !ok { 409 | a.logger.Warningf("BINS `id` contain incorrect value: %v", pkInterface) 410 | continue 411 | } 412 | 413 | if binTags, ok = v.Bins["tags"]; !ok { 414 | a.logger.Warningf("BINS `tags` not found for aerospike set: %q", set) 415 | continue 416 | } 417 | 418 | tags = a.sliceInterfacesToString(binTags) 419 | 420 | keys = append(keys, Key{Set: set, Pk: pk, Tags: tags}) 421 | } 422 | 423 | return keys, nil 424 | } 425 | 426 | func (a *AerospikeCache) sliceInterfacesToString(src interface{}) []string { 427 | var result []string 428 | 429 | if t, ok := src.([]interface{}); ok { 430 | result = make([]string, 0, len(t)) 431 | for i := range t { 432 | if v, ok := t[i].(string); ok { 433 | result = append(result, v) 434 | } 435 | } 436 | } 437 | 438 | return result 439 | } 440 | 441 | // Remove removes data by given cache key 442 | // If tags provided, all records having at least one of them will be removed 443 | // Otherwise only an item with given Primary key will be removed 444 | func (a *AerospikeCache) Remove(key *Key) (err error) { 445 | var ts = time.Now() 446 | 447 | if len(key.Pk) > 0 { 448 | err = a.removeByPk(key.Set, key.Pk) 449 | } 450 | 451 | if err == nil { 452 | for _, tag := range key.Tags { 453 | err = a.removeByTag(key.Set, tag) 454 | if err != nil { 455 | break 456 | } 457 | } 458 | } 459 | 460 | a.metric.ObserveRT(map[string]string{ 461 | metric.LabelNamespace: a.ns, 462 | metric.LabelSet: key.Set, 463 | metric.LabelOperation: "delete", 464 | metric.LabelIsError: metric.IsError(err), 465 | }, metric.SinceMs(ts)) 466 | 467 | return 468 | } 469 | 470 | func (a *AerospikeCache) removeByPk(set, pk string) error { 471 | aeroKey, err := a.createKey(set, pk) 472 | if err != nil { 473 | return err 474 | } 475 | 476 | writePolicy := a.getWritePolice(time.Duration(0)) 477 | 478 | if _, err = a.client.Delete(writePolicy, aeroKey); err != nil { 479 | err = errors.Wrapf(err, "could not remove from set '%s' by primary key '%s'", set, pk) 480 | } 481 | 482 | return err 483 | } 484 | 485 | // removeByTag removes data from Aerospike by PK and tags 486 | func (a *AerospikeCache) removeByTag(set, tag string) error { 487 | var err error 488 | 489 | defer func() { 490 | // once in a while query returns a key with empty digest hash 491 | // then delete command panics with given key (aerospike client bug?) 492 | if r := recover(); r != nil { 493 | err = errors.Errorf("removeByTag panic handled: '%v'", r) 494 | } 495 | }() 496 | 497 | queryPolicy := aerospike.NewQueryPolicy() 498 | queryPolicy.Timeout = a.config.RemoveTimeout 499 | queryPolicy.MaxRetries = a.config.MaxRetries 500 | queryPolicy.WaitUntilMigrationsAreOver = true 501 | 502 | writePolicy := a.getWritePolice(time.Duration(0)) 503 | 504 | stm := aerospike.NewStatement(a.ns, set) 505 | stm.Addfilter(aerospike.NewContainsFilter(tagsBin, aerospike.ICT_LIST, a.cachePrefix+tag)) 506 | 507 | recordSet, err := a.client.Query(queryPolicy, stm) 508 | if err != nil { 509 | return errors.Wrapf(err, "could not select data for deleting from for set '%s' and tag '%s'", set, tag) 510 | } 511 | defer recordSet.Close() 512 | 513 | ch := recordSet.Results() 514 | 515 | for d := range ch { 516 | 517 | if d.Err != nil { 518 | a.logger.Critical(d.Err.Error()) 519 | continue 520 | } 521 | 522 | if d == nil || d.Record == nil { 523 | a.logger.Criticalf("Empty record %s : %s", set, tag) 524 | continue 525 | } 526 | a.logger.Debugf( 527 | "Clear record %s from input params {set: %s, tag: %s} with as prefix %s", 528 | d.Record.Key, 529 | set, 530 | tag, 531 | a.cachePrefix, 532 | ) 533 | 534 | _, err = a.client.Delete(writePolicy, d.Record.Key) 535 | } 536 | 537 | if err != nil { 538 | err = errors.Wrapf(err, "could not delete cache by record") 539 | } 540 | 541 | return err 542 | } 543 | 544 | // Close cache storage Aerospike 545 | func (a *AerospikeCache) Close() { 546 | // send quit message to updateConnectionCountMetric goroutine 547 | a.quitUpdateConnectionCountMetricChan <- struct{}{} 548 | a.client.Close() 549 | } 550 | 551 | //FIXME: This function should be implemented 552 | // Flush cleans all data. 553 | func (a *AerospikeCache) Flush() int { 554 | return 0 555 | } 556 | 557 | //FIXME: implement me 558 | // Count returns count of data in cache 559 | func (a *AerospikeCache) Count() int { 560 | return 0 561 | } 562 | 563 | // ClearSet removes all values in set 564 | func (a *AerospikeCache) ClearSet(set string) error { 565 | result, err := a.client.ScanAll(nil, a.ns, set) 566 | if nil != err { 567 | return err 568 | } 569 | for record := range result.Results() { 570 | if record.Err != nil { 571 | a.logger.Warningf("Record error while clearing set %s: %s", set, err) 572 | continue 573 | } 574 | if _, err := a.client.Delete(nil, record.Record.Key); err != nil { 575 | return errors.Wrapf(err, "could not remove from set '%s'", set) 576 | } 577 | } 578 | a.logger.Debugf("Cache set %s cleared", set) 579 | return nil 580 | } 581 | 582 | func (a *AerospikeCache) createKey(set, key string) (aeroKey *aerospike.Key, err error) { 583 | a.mu.Lock() 584 | aeroKey, err = aerospike.NewKey(a.ns, set, a.cachePrefix+key) 585 | a.mu.Unlock() 586 | 587 | if err != nil { 588 | err = errors.Wrap(err, "could not create cache key") 589 | } 590 | return 591 | } 592 | 593 | func (a *AerospikeCache) updateConnectionCountMetric() { 594 | for { 595 | select { 596 | case <-time.After(a.updateConnectionCountMetricInterval): 597 | for _, node := range a.client.GetNodes() { 598 | host := node.GetHost().Name 599 | if node.IsActive() { 600 | nodeStatistics, err := aerospike.RequestNodeStats(node) 601 | if err != nil { 602 | a.logger.Warningf("Cannot get statistic for node"+node.String()+" Error: ", err.Error()) 603 | continue 604 | } 605 | 606 | connectionsCount, err := strconv.Atoi(nodeStatistics["client_connections"]) 607 | if err != nil { 608 | a.logger.Warningf("Cannot get statistic for node"+node.String()+" Error: ", err.Error()) 609 | continue 610 | } 611 | 612 | a.metric.SetItemCount(host, connectionsCount) 613 | } else { 614 | a.metric.SetItemCount(host, 0) 615 | } 616 | } 617 | case <-a.quitUpdateConnectionCountMetricChan: 618 | return 619 | } 620 | } 621 | } 622 | 623 | func (a *AerospikeCache) getNodeHostName(node *aerospike.Node) string { 624 | if node == nil { 625 | return "" 626 | } 627 | host := node.GetHost() 628 | if host == nil { 629 | return "" 630 | } 631 | return host.Name 632 | } 633 | 634 | func (a *AerospikeCache) getWritePolice(ttl time.Duration) *aerospike.WritePolicy { 635 | policy := aerospike.NewWritePolicy(0, uint32(ttl.Seconds())) 636 | policy.RecordExistsAction = aerospike.REPLACE 637 | policy.MaxRetries = a.maxRetries 638 | 639 | return policy 640 | } 641 | -------------------------------------------------------------------------------- /byte_cache_aerospike_bench_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | ) 7 | 8 | func BenchmarkByteCacheAerospike_Set(b *testing.B) { 9 | cache := initAerospikeByteCache(b, "bench_set") 10 | 11 | setData := []byte("bench") 12 | key := Key{ 13 | Set: "benchset", 14 | Pk: "bench1", 15 | } 16 | 17 | b.ResetTimer() 18 | for i := 0; i < b.N; i++ { 19 | key.Pk = strconv.Itoa(i) 20 | cache.Put(setData, &key, DefaultCacheTTL) 21 | } 22 | } 23 | 24 | func BenchmarkByteCacheAerospike_Get(b *testing.B) { 25 | cache := initAerospikeByteCache(b, "bench_get") 26 | 27 | setData := []byte("bench") 28 | key := Key{ 29 | Set: "benchset", 30 | Pk: "bench1", 31 | } 32 | 33 | for i := 0; i < b.N; i++ { 34 | key.Pk = strconv.Itoa(i) 35 | cache.Put(setData, &key, DefaultCacheTTL) 36 | } 37 | 38 | b.ResetTimer() 39 | for i := 0; i < b.N; i++ { 40 | key.Pk = strconv.Itoa(i) 41 | cache.Get(&key) 42 | } 43 | } 44 | 45 | func BenchmarkByteCacheAerospike_Remove(b *testing.B) { 46 | cache := initAerospikeByteCache(b, "bench_remove") 47 | 48 | setData := []byte("bench") 49 | key := Key{ 50 | Set: "benchset", 51 | Pk: "bench1", 52 | } 53 | 54 | for i := 0; i < b.N; i++ { 55 | key.Pk = strconv.Itoa(i) 56 | cache.Put(setData, &key, DefaultCacheTTL) 57 | } 58 | 59 | b.ResetTimer() 60 | for i := 0; i < b.N; i++ { 61 | key.Pk = strconv.Itoa(i) 62 | cache.Remove(&key) 63 | } 64 | } 65 | 66 | func BenchmarkByteCacheAerospike_SetTagged(b *testing.B) { 67 | cache := initAerospikeByteCache(b, "bench_set_tagged") 68 | 69 | setData := []byte("bench") 70 | key := Key{ 71 | Set: "benchset", 72 | Pk: "bench1", 73 | Tags: []string{"tag"}, 74 | } 75 | 76 | b.ResetTimer() 77 | for i := 0; i < b.N; i++ { 78 | key.Pk = strconv.Itoa(i) 79 | key.Tags[0] = "tag" + key.Pk 80 | cache.Put(setData, &key, DefaultCacheTTL) 81 | } 82 | } 83 | 84 | func BenchmarkByteCacheAerospike_GetTagged(b *testing.B) { 85 | cache := initAerospikeByteCache(b, "bench_get_tagged") 86 | 87 | setData := []byte("bench") 88 | key := Key{ 89 | Set: "benchset", 90 | Pk: "bench1", 91 | Tags: []string{"tag"}, 92 | } 93 | 94 | for i := 0; i < b.N; i++ { 95 | key.Pk = strconv.Itoa(i) 96 | key.Tags[0] = "tag" + key.Pk 97 | cache.Put(setData, &key, DefaultCacheTTL) 98 | } 99 | 100 | b.ResetTimer() 101 | for i := 0; i < b.N; i++ { 102 | key.Pk = strconv.Itoa(i) 103 | cache.Get(&key) 104 | } 105 | } 106 | 107 | func BenchmarkByteCacheAerospike_RemoveTaggedByPk(b *testing.B) { 108 | cache := initAerospikeByteCache(b, "bench_remove_tagged_by_pk") 109 | 110 | setData := []byte("bench") 111 | key := Key{ 112 | Set: "benchset", 113 | Pk: "bench1", 114 | Tags: []string{"tag"}, 115 | } 116 | 117 | for i := 0; i < b.N; i++ { 118 | key.Pk = strconv.Itoa(i) 119 | key.Tags[0] = "tag" + key.Pk 120 | cache.Put(setData, &key, DefaultCacheTTL) 121 | } 122 | 123 | // remove tags to make removing by pk only 124 | key.Tags = []string{} 125 | 126 | b.ResetTimer() 127 | for i := 0; i < b.N; i++ { 128 | key.Pk = strconv.Itoa(i) 129 | cache.Remove(&key) 130 | } 131 | } 132 | 133 | func BenchmarkByteCacheAerospike_RemoveTaggedByTag(b *testing.B) { 134 | cache := initAerospikeByteCache(b, "bench_remove_tagged_by_tag") 135 | 136 | setData := []byte("bench1") 137 | key := Key{ 138 | Set: "benchset", 139 | Pk: "bench1", 140 | Tags: []string{"tag"}, 141 | } 142 | 143 | for i := 0; i < b.N; i++ { 144 | key.Pk = strconv.Itoa(i) 145 | key.Tags[0] = "tag" + key.Pk 146 | cache.Put(setData, &key, DefaultCacheTTL) 147 | } 148 | 149 | // remove pk to make it remove by tags only 150 | key.Pk = "" 151 | 152 | b.ResetTimer() 153 | for i := 0; i < b.N; i++ { 154 | key.Tags[0] = "tag" + key.Pk 155 | cache.Remove(&key) 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /byte_cache_aerospike_config.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | // AerospikeConfig contains configuration for aerospike 6 | type AerospikeConfig struct { 7 | Prefix string `config:"aerospike_prefix" default:"" description:"aerospike prefix"` 8 | NameSpace string `config:"aerospike_namespace" description:"aerospike namespace"` 9 | Hosts []string `config:"aerospike_hosts" description:"aerospike comma-separated host:port list"` 10 | 11 | MaxRetries int `config:"aerospike_max_retries" default:"3" description:"aerospike max retries for get and put operations"` 12 | SleepBetweenRetries time.Duration `config:"aerospike_sleep_between_retries" default:"500ms" description:"aerospike sleep between connection/read/write retries"` 13 | 14 | // tcp connection to aerospike server timeout 15 | ConnectionTimeout time.Duration `config:"aerospike_connection_timeout" default:"1s" description:"aerospike connection timeout"` 16 | 17 | // max unused connection lifetime 18 | IdleTimeout time.Duration `config:"aerospike_idle_timeout" default:"1m" description:"aerospike idle connection lifetime"` 19 | 20 | ReadTimeout time.Duration `config:"aerospike_read_timeout" default:"100ms" description:"aerospike read timeout"` 21 | RemoveTimeout time.Duration `config:"aerospike_remove_timeout" default:"800ms" description:"aerospike remove timeout"` 22 | PutTimeout time.Duration `config:"aerospike_put_timeout" default:"500ms" description:"aerospike put timeout"` 23 | 24 | // max connection pool (queue) size 25 | ConnectionQueueSize int `config:"aerospike_connection_queue_size" default:"256" description:"aerospike connection queue size"` 26 | 27 | // if true - wait for used connection to be released (up to 1ms) 28 | LimitConnectionsToQueueSize bool `config:"aerospike_limit_connections" default:"false" description:"aerospike limit connections count to queue size"` 29 | 30 | LogLevel int `config:"aerospike_log_level" default:"1" description:"aerospike logging level: DEBUG(-1), INFO(0), WARNING(1), ERR(2), OFF(999)"` 31 | 32 | FailIfNotConnected bool `config:"aerospike_fail_if_not_connected" default:"false" description:"aerospike fail if not connected"` 33 | 34 | // connection count metric update time interval 35 | UpdateConnectionCountMetricInterval time.Duration `config:"aerospike_update_connection_count_metric_interval" default:"1s" description:"aerospike update connection count metric interval"` 36 | } 37 | -------------------------------------------------------------------------------- /byte_cache_aerospike_example_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "go-cache/metric/dummy" 8 | ) 9 | 10 | func ExampleAerospikeCache_Get() { 11 | config := &AerospikeConfig{ 12 | NameSpace: "test", 13 | Hosts: []string{"localhost:3000"}, 14 | MaxRetries: 5, 15 | } 16 | logger := &AerospikeDummyLogger{} 17 | 18 | client, err := CreateAerospikeClient(config, logger) 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | aerospikeCache := NewAerospikeCache(config, client, logger, dummy.NewMetric()) 24 | 25 | const wirth = "But quality of work can be expected only through personal satisfaction, dedication and enjoyment. In our profession, precision and perfection are not a dispensible luxury, but a simple necessity." 26 | data := []byte(wirth) 27 | key := &Key{ 28 | Set: "testset", 29 | Pk: "testExpired", 30 | } 31 | 32 | aerospikeCache.Put(data, key, time.Second) 33 | 34 | cachedData, ok := aerospikeCache.Get(key) 35 | if !ok { 36 | panic("Something went wrong") 37 | } 38 | 39 | fmt.Println(string(cachedData)) 40 | // Output: 41 | // But quality of work can be expected only through personal satisfaction, dedication and enjoyment. In our profession, precision and perfection are not a dispensible luxury, but a simple necessity. 42 | } 43 | -------------------------------------------------------------------------------- /byte_cache_aerospike_indexes.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "github.com/aerospike/aerospike-client-go" 5 | ) 6 | 7 | // AerospikeIndex contains info about aerospike index 8 | type AerospikeIndex struct { 9 | SetName string 10 | IndexName string 11 | IndexType aerospike.IndexType 12 | } 13 | -------------------------------------------------------------------------------- /byte_cache_aerospike_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "path" 6 | "runtime" 7 | "strconv" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/aerospike/aerospike-client-go" 13 | "github.com/vaughan0/go-ini" 14 | 15 | "go-cache/metric/dummy" 16 | ) 17 | 18 | type AerospikeDummyLogger struct{} 19 | 20 | func (l *AerospikeDummyLogger) Printf(format string, v ...interface{}) {} 21 | 22 | func (l *AerospikeDummyLogger) Debugf(message string, args ...interface{}) {} 23 | 24 | func (l *AerospikeDummyLogger) Errorf(message string, args ...interface{}) { 25 | fmt.Println("[ERROR]", fmt.Sprintf(message, args...)) 26 | } 27 | 28 | func (l *AerospikeDummyLogger) Warningf(message string, args ...interface{}) { 29 | fmt.Println("[WARNING]", fmt.Sprintf(message, args...)) 30 | } 31 | 32 | func (l *AerospikeDummyLogger) Warning(args ...interface{}) { 33 | fmt.Println("[WARNING]", fmt.Sprint(args...)) 34 | } 35 | 36 | func (l *AerospikeDummyLogger) Criticalf(message string, args ...interface{}) { 37 | fmt.Println("[CRITICAL]", fmt.Sprintf(message, args...)) 38 | } 39 | 40 | func (l *AerospikeDummyLogger) Critical(args ...interface{}) { 41 | fmt.Println("[CRITICAL]", fmt.Sprint(args...)) 42 | } 43 | 44 | func newTestAerospikeByteCache(tb testing.TB, hosts []string, namespace string) *AerospikeCache { 45 | config := &AerospikeConfig{ 46 | NameSpace: namespace, 47 | Hosts: hosts, 48 | MaxRetries: 5, 49 | } 50 | logger := &AerospikeDummyLogger{} 51 | 52 | client, err := CreateAerospikeClient(config, logger) 53 | if err != nil { 54 | tb.Fatalf("Can't create aerospike client: '%s'", err) 55 | } 56 | 57 | return NewAerospikeCache(config, client, logger, dummy.NewMetric()) 58 | } 59 | 60 | func initAerospikeByteCache(tb testing.TB, prefix string) *AerospikeCache { 61 | file, err := ini.LoadFile("./test.ini") 62 | if err != nil { 63 | tb.Fatalf("Unable to parse ini file: '%s'", err) 64 | } 65 | 66 | namespace, ok := file.Get("aerospike", "namespace") 67 | if !ok { 68 | tb.Fatal("'namespace' variable missing from 'aerospike' section") 69 | } 70 | 71 | hosts, ok := file.Get("aerospike", "hosts") 72 | if !ok { 73 | tb.Fatal("'hosts' variable missing from 'aerospike' section") 74 | } 75 | 76 | cache := newTestAerospikeByteCache(tb, strings.Split(hosts, ","), namespace) 77 | cache.SetCachePrefix(prefix) 78 | 79 | cache.CreateTagsIndex(AerospikeIndex{ 80 | "testset_withtags", 81 | "tags_testset_withtags", 82 | aerospike.STRING, 83 | }) 84 | 85 | cache.CreateTagsIndex(AerospikeIndex{ 86 | "benchset", 87 | "tags_benchset", 88 | aerospike.STRING, 89 | }) 90 | 91 | return cache 92 | } 93 | 94 | func TestByteCacheAerospike_TestGet(t *testing.T) { 95 | cache := initAerospikeByteCache(t, "get") 96 | 97 | setData := []byte("test1") 98 | key := Key{ 99 | Set: "testset", 100 | Pk: "test1", 101 | } 102 | cache.Put(setData, &key, DefaultCacheTTL) 103 | 104 | assertByteCacheKeyHasValue(t, cache, &key, "test1") 105 | } 106 | 107 | func TestByteCacheAerospike_GetExpired(t *testing.T) { 108 | cache := initAerospikeByteCache(t, "getexpired") 109 | 110 | setData := []byte("testExpired") 111 | key := Key{ 112 | Set: "testset", 113 | Pk: "testExpired", 114 | } 115 | cache.Put(setData, &key, time.Second) 116 | 117 | assertByteCacheKeyHasValue(t, cache, &key, "testExpired") 118 | 119 | time.Sleep(time.Second * 2) 120 | 121 | assertByteCacheKeyEmpty(t, cache, &key) 122 | } 123 | 124 | func TestByteCacheAerospike_Remove(t *testing.T) { 125 | cache := initAerospikeByteCache(t, "remove") 126 | 127 | setData := []byte("test_remove") 128 | key := Key{ 129 | Set: "testset", 130 | Pk: "test_remove", 131 | } 132 | cache.Put(setData, &key, DefaultCacheTTL) 133 | 134 | assertByteCacheKeyHasValue(t, cache, &key, "test_remove") 135 | 136 | err := cache.Remove(&key) 137 | 138 | if err != nil { 139 | t.Error(err) 140 | } 141 | assertByteCacheKeyEmpty(t, cache, &key) 142 | } 143 | 144 | func TestByteCacheAerospike_RemoveByTag(t *testing.T) { 145 | cache := initAerospikeByteCache(t, "remove_by_tag") 146 | 147 | setData := []byte("test_tag") 148 | key := Key{ 149 | Set: "testset_withtags", 150 | Pk: "test_remove_by_tag", 151 | Tags: []string{"tag1", "tag2"}, 152 | } 153 | cache.Put(setData, &key, DefaultCacheTTL) 154 | 155 | assertByteCacheKeyHasValue(t, cache, &key, "test_tag") 156 | 157 | err := cache.Remove(&Key{ 158 | Set: "testset_withtags", 159 | Tags: []string{"tag2"}, 160 | }) 161 | 162 | if err != nil { 163 | t.Error(err) 164 | } 165 | assertByteCacheKeyEmpty(t, cache, &key) 166 | } 167 | 168 | func TestByteCacheAerospike_RemoveByTag_FewValues(t *testing.T) { 169 | cache := initAerospikeByteCache(t, "remove_by_tag_few_values") 170 | 171 | setData := []byte("test_tag") 172 | keyTag1Tag2 := Key{ 173 | Set: "testset_withtags", 174 | Pk: "test_remove_by_tag_tags12", 175 | Tags: []string{"tag1", "tag2"}, 176 | } 177 | cache.Put(setData, &keyTag1Tag2, DefaultCacheTTL) 178 | 179 | keyTag2Tag3 := Key{ 180 | Set: "testset_withtags", 181 | Pk: "test_remove_by_tag_tags23", 182 | Tags: []string{"tag2", "tag3"}, 183 | } 184 | cache.Put(setData, &keyTag2Tag3, DefaultCacheTTL) 185 | 186 | keyNoTags := Key{ 187 | Set: "testset_withtags", 188 | Pk: "test_remove_by_tag_notags", 189 | } 190 | cache.Put(setData, &keyNoTags, DefaultCacheTTL) 191 | 192 | // check all keys exists and have valid value 193 | assertByteCacheKeyHasValue(t, cache, &keyTag1Tag2, "test_tag") 194 | assertByteCacheKeyHasValue(t, cache, &keyTag2Tag3, "test_tag") 195 | assertByteCacheKeyHasValue(t, cache, &keyNoTags, "test_tag") 196 | 197 | err := cache.Remove(&Key{ 198 | Set: "testset_withtags", 199 | Tags: []string{"tag2"}, 200 | }) 201 | 202 | if err != nil { 203 | t.Error(err) 204 | } 205 | // check tagged keys removed, but untagged one still exists and has valid value 206 | assertByteCacheKeyEmpty(t, cache, &keyTag1Tag2) 207 | assertByteCacheKeyEmpty(t, cache, &keyTag2Tag3) 208 | assertByteCacheKeyHasValue(t, cache, &keyNoTags, "test_tag") 209 | } 210 | 211 | func TestByteCacheAerospike_RemoveByFewTags_FewValues(t *testing.T) { 212 | cache := initAerospikeByteCache(t, "remove_by_few_tags_few_values") 213 | 214 | setData := []byte("test_tag") 215 | keyTag1 := Key{ 216 | Set: "testset_withtags", 217 | Pk: "test_remove_by_tag_tags1", 218 | Tags: []string{"tag1"}, 219 | } 220 | cache.Put(setData, &keyTag1, DefaultCacheTTL) 221 | 222 | keyTag1Tag2 := Key{ 223 | Set: "testset_withtags", 224 | Pk: "test_remove_by_tag_tags12", 225 | Tags: []string{"tag1", "tag2"}, 226 | } 227 | cache.Put(setData, &keyTag1Tag2, DefaultCacheTTL) 228 | 229 | keyTag2Tag3 := Key{ 230 | Set: "testset_withtags", 231 | Pk: "test_remove_by_tag_tags23", 232 | Tags: []string{"tag2", "tag3"}, 233 | } 234 | cache.Put(setData, &keyTag2Tag3, DefaultCacheTTL) 235 | 236 | keyNoTags := Key{ 237 | Set: "testset_withtags", 238 | Pk: "test_remove_by_tag_notags", 239 | } 240 | cache.Put(setData, &keyNoTags, DefaultCacheTTL) 241 | 242 | // check all keys exists and have valid value 243 | assertByteCacheKeyHasValue(t, cache, &keyTag1, "test_tag") 244 | assertByteCacheKeyHasValue(t, cache, &keyTag1Tag2, "test_tag") 245 | assertByteCacheKeyHasValue(t, cache, &keyTag2Tag3, "test_tag") 246 | assertByteCacheKeyHasValue(t, cache, &keyNoTags, "test_tag") 247 | 248 | err := cache.Remove(&Key{ 249 | Set: "testset_withtags", 250 | Tags: []string{"tag1", "tag2"}, 251 | }) 252 | 253 | if err != nil { 254 | t.Error(err) 255 | } 256 | // check tagged keys removed, but untagged one still exists and has valid value 257 | assertByteCacheKeyEmpty(t, cache, &keyTag1) 258 | assertByteCacheKeyEmpty(t, cache, &keyTag1Tag2) 259 | assertByteCacheKeyEmpty(t, cache, &keyTag2Tag3) 260 | assertByteCacheKeyHasValue(t, cache, &keyNoTags, "test_tag") 261 | } 262 | 263 | func TestByteCacheAerospike_RemoveByTagWithPrefix(t *testing.T) { 264 | cache := initAerospikeByteCache(t, "remove_by_tag_with_prefix") 265 | 266 | prefixedCache := newTestAerospikeByteCache(t, cache.config.Hosts, cache.config.NameSpace) 267 | prefixedCache.SetCachePrefix("remove_by_tag_with_other_prefix") 268 | 269 | setData := []byte("test_tag") 270 | key := Key{ 271 | Set: "testset_withtags", 272 | Pk: "test_remove_by_tag_with_prefix", 273 | Tags: []string{"tag1", "tag2"}, 274 | } 275 | cache.Put(setData, &key, DefaultCacheTTL) 276 | prefixedCache.Put(setData, &key, DefaultCacheTTL) 277 | 278 | assertByteCacheKeyHasValue(t, cache, &key, "test_tag") 279 | assertByteCacheKeyHasValue(t, prefixedCache, &key, "test_tag") 280 | 281 | err := prefixedCache.Remove(&Key{ 282 | Set: "testset_withtags", 283 | Tags: []string{"tag2"}, 284 | }) 285 | 286 | if err != nil { 287 | t.Error(err) 288 | } 289 | assertByteCacheKeyHasValue(t, cache, &key, "test_tag") 290 | assertByteCacheKeyEmpty(t, prefixedCache, &key) 291 | } 292 | 293 | func assertByteCacheKeyHasValue(t *testing.T, cache IByteCache, key *Key, expectedValue string) { 294 | getData, success := cache.Get(key) 295 | if !success { 296 | t.Errorf("%s: value doesn't exist (%+v)", callerInfo(), key) 297 | } 298 | if string(getData) != expectedValue { 299 | t.Errorf("%s: %s is not equal to expected value: %s (%+v)", callerInfo(), string(getData), expectedValue, key) 300 | } 301 | } 302 | 303 | func assertByteCacheKeyEmpty(t *testing.T, cache IByteCache, key *Key) { 304 | getData, success := cache.Get(key) 305 | if success { 306 | t.Errorf("%s: value should not exist (key: %+v)", callerInfo(), key) 307 | } 308 | if string(getData) != "" { 309 | t.Errorf("%s: %s should be empty (key: %+v)", callerInfo(), string(getData), key) 310 | } 311 | } 312 | 313 | func callerInfo() string { 314 | _, file, line, _ := runtime.Caller(2) 315 | _, fileNameLine := path.Split(file) 316 | fileNameLine += ":" + strconv.Itoa(line) 317 | 318 | return fileNameLine 319 | } 320 | -------------------------------------------------------------------------------- /byte_cache_blackhole.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | // BlackholeCache implements cache that accept any data but always return null. Used for unit tests 6 | type BlackholeCache struct { 7 | } 8 | 9 | var _ IByteCache = &BlackholeCache{} // BlackholeCache implements IByteCache 10 | 11 | // NewBlackholeCache initializes instance of BlackholeCache 12 | func NewBlackholeCache() *BlackholeCache { 13 | return &BlackholeCache{} 14 | } 15 | 16 | // Get returns nil, do nothing 17 | func (cache *BlackholeCache) Get(key *Key) (data []byte, ok bool) { 18 | return 19 | } 20 | 21 | // Put returns nil, do nothing 22 | func (cache *BlackholeCache) Put(data []byte, key *Key, ttl time.Duration) {} 23 | 24 | // ScanKeys returns nil, do nothing 25 | func (cache *BlackholeCache) ScanKeys(set string) ([]Key, error) { 26 | return nil, nil 27 | } 28 | 29 | // Remove returns nil, do nothing 30 | func (cache *BlackholeCache) Remove(key *Key) (err error) { 31 | return 32 | } 33 | 34 | // Close do nothing 35 | func (cache *BlackholeCache) Close() {} 36 | 37 | // Flush removes all entries from cache and returns number of flushed entries 38 | func (cache *BlackholeCache) Flush() int { 39 | return 0 40 | } 41 | 42 | // Count returns count of data in cache FIXME: implement me 43 | func (cache *BlackholeCache) Count() int { 44 | return 0 45 | } 46 | 47 | // ClearSet returns nil, does nothing 48 | func (cache *BlackholeCache) ClearSet(set string) error { 49 | return nil 50 | } 51 | -------------------------------------------------------------------------------- /byte_cache_interface.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | // IByteCache defines required interface for caching module 6 | type IByteCache interface { 7 | IFlushable 8 | Count() int 9 | Get(key *Key) (data []byte, ok bool) 10 | Put(data []byte, key *Key, ttl time.Duration) 11 | Remove(key *Key) error 12 | Close() 13 | ClearSet(set string) error 14 | ScanKeys(set string) ([]Key, error) 15 | } 16 | 17 | type IMemoryCacheLogger interface { 18 | Errorf(message string, args ...interface{}) 19 | Warning(message ...interface{}) 20 | } 21 | 22 | type IAerospikeCacheLogger interface { 23 | Printf(format string, v ...interface{}) 24 | Debugf(message string, args ...interface{}) 25 | Errorf(message string, args ...interface{}) 26 | Warningf(message string, args ...interface{}) 27 | Warning(...interface{}) 28 | Criticalf(message string, args ...interface{}) 29 | Critical(...interface{}) 30 | } 31 | -------------------------------------------------------------------------------- /byte_cache_wrapper.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | const ( 6 | RetryTimeout = time.Second * 10 7 | ) 8 | 9 | type wrapperCache struct { 10 | realCache IByteCache 11 | stubCache IByteCache 12 | isConnected bool 13 | doneChan chan bool 14 | logger IAerospikeCacheLogger 15 | } 16 | 17 | type fnCreate func() (IByteCache, error) 18 | 19 | // NewEntryCacheWrapper initializes instance of IEntryCache 20 | func NewEntryCacheWrapper(fn fnCreate, logger IAerospikeCacheLogger) IByteCache { 21 | result := &wrapperCache{ 22 | stubCache: NewBlackholeCache(), 23 | logger: logger, 24 | doneChan: make(chan bool, 1), 25 | } 26 | 27 | go result.createRealCache(fn) 28 | 29 | return result 30 | } 31 | 32 | func (this *wrapperCache) getCache() IByteCache { 33 | if this.isConnected { 34 | return this.realCache 35 | } 36 | 37 | return this.stubCache 38 | } 39 | 40 | func (this *wrapperCache) createRealCache(fn fnCreate) { 41 | for { 42 | select { 43 | case <-this.doneChan: 44 | close(this.doneChan) 45 | return 46 | case <-time.After(RetryTimeout): 47 | if cache, err := fn(); err == nil { 48 | this.realCache = cache 49 | this.isConnected = true 50 | this.logger.Debugf("Wrapped cache was created") 51 | return 52 | } 53 | } 54 | } 55 | } 56 | 57 | // Get returns nil, do nothing 58 | func (this *wrapperCache) Get(key *Key) ([]byte, bool) { 59 | return this.getCache().Get(key) 60 | } 61 | 62 | // Put returns nil, do nothing 63 | func (this *wrapperCache) Put(data []byte, key *Key, ttl time.Duration) { 64 | this.getCache().Put(data, key, ttl) 65 | } 66 | 67 | // ScanKeys returns nil, do nothing 68 | func (this *wrapperCache) ScanKeys(set string) ([]Key, error) { 69 | return this.getCache().ScanKeys(set) 70 | } 71 | 72 | // Remove returns nil, do nothing 73 | func (this *wrapperCache) Remove(key *Key) error { 74 | return this.getCache().Remove(key) 75 | } 76 | 77 | // Close do nothing 78 | func (this *wrapperCache) Close() { 79 | this.doneChan <- true 80 | this.getCache().Close() 81 | } 82 | 83 | // Flush removes all entries from cache and returns number of flushed entries 84 | func (this *wrapperCache) Flush() int { 85 | return this.getCache().Flush() 86 | } 87 | 88 | // Count returns count of data in cache 89 | func (this *wrapperCache) Count() int { 90 | return this.getCache().Count() 91 | } 92 | 93 | // ClearSet returns nil, does nothing 94 | func (this *wrapperCache) ClearSet(set string) error { 95 | return this.getCache().ClearSet(set) 96 | } 97 | -------------------------------------------------------------------------------- /entry.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Entry contains data 8 | type Entry struct { 9 | Key *Key 10 | CreateDate time.Time // In UTC 11 | EndDate int64 12 | Data interface{} 13 | } 14 | 15 | // CreateEntry returns new instance of Entry 16 | func CreateEntry(key *Key, endDate int64, data interface{}) *Entry { 17 | return &Entry{ 18 | Key: key, 19 | CreateDate: time.Now().UTC(), 20 | EndDate: endDate, 21 | Data: data, 22 | } 23 | } 24 | 25 | // IsValid returns Entry is valid 26 | func (entry *Entry) IsValid() bool { 27 | return entry.EndDate > time.Now().Unix() 28 | } 29 | -------------------------------------------------------------------------------- /errors/errors.go: -------------------------------------------------------------------------------- 1 | package errors 2 | 3 | import "github.com/pkg/errors" 4 | 5 | var ( 6 | New func(msg string) error 7 | Wrap func(err error, message string) error 8 | Wrapf func(err error, format string, args ...interface{}) error 9 | Errorf func(format string, args ...interface{}) error 10 | ) 11 | 12 | func init() { 13 | SetWithStack(false) 14 | } 15 | 16 | func SetWithStack(errorTraces bool) { 17 | New = errors.New 18 | Wrap = errors.Wrap 19 | Wrapf = errors.Wrapf 20 | Errorf = errors.Errorf 21 | 22 | if errorTraces { 23 | New = func(msg string) error { 24 | return errors.WithStack(New(msg)) 25 | } 26 | 27 | Wrap = func(err error, message string) error { 28 | return errors.WithStack(Wrap(err, message)) 29 | } 30 | 31 | Wrapf = func(err error, format string, args ...interface{}) error { 32 | return errors.WithStack(Wrapf(err, format, args...)) 33 | } 34 | 35 | Errorf = func(format string, args ...interface{}) error { 36 | return errors.WithStack(Errorf(format, args...)) 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: go-cache 2 | import: 3 | - package: github.com/aerospike/aerospike-client-go 4 | subpackages: 5 | - logger 6 | - types 7 | - package: github.com/pkg/errors 8 | version: ^0.8.0 9 | testImports: 10 | - package: github.com/vaughan0/go-ini 11 | vcs: git -------------------------------------------------------------------------------- /interface.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | // IFlushable determines cache that can be flushed 4 | type IFlushable interface { 5 | Flush() int 6 | } 7 | -------------------------------------------------------------------------------- /key.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | const sep byte = '_' 10 | 11 | // Key defines cache key for some record 12 | type Key struct { 13 | Set string 14 | Pk string 15 | Tags []string 16 | } 17 | 18 | // String implements fmt.Stringer interface for readable string representation 19 | func (key Key) String() string { 20 | return fmt.Sprintf("Set: '%s', PK: '%s', Tags: '%v'", key.Set, key.Pk, key.Tags) 21 | } 22 | 23 | // ID returns string that identifies given key 24 | func (key Key) ID() string { 25 | var buf bytes.Buffer 26 | buf.WriteString(key.Set) 27 | buf.WriteByte(sep) 28 | buf.WriteString(key.Pk) 29 | 30 | if len(key.Tags) != 0 { 31 | buf.WriteByte(sep) 32 | buf.WriteString(strings.Join(key.Tags, "_")) 33 | } 34 | 35 | return buf.String() 36 | } 37 | -------------------------------------------------------------------------------- /metric/dummy/metric.go: -------------------------------------------------------------------------------- 1 | package dummy 2 | 3 | type Metric struct{} 4 | 5 | func NewMetric() Metric { 6 | return Metric{} 7 | } 8 | 9 | func (m Metric) ObserveRT(labels map[string]string, timeSince float64) { 10 | return 11 | } 12 | 13 | func (m Metric) RegisterHit(labels map[string]string) { 14 | return 15 | } 16 | 17 | func (m Metric) RegisterMiss(labels map[string]string) { 18 | return 19 | } 20 | 21 | func (m Metric) IncreaseItemCount(set string) { 22 | return 23 | } 24 | 25 | func (m Metric) SetItemCount(set string, n int) { 26 | return 27 | } 28 | -------------------------------------------------------------------------------- /metric/metric.go: -------------------------------------------------------------------------------- 1 | package metric 2 | 3 | import "time" 4 | 5 | const ( 6 | LabelHost = "host" 7 | LabelIsError = "is_error" 8 | LabelNamespace = "namespace" 9 | LabelSet = "set" 10 | LabelOperation = "operation" 11 | ) 12 | 13 | type Metric interface { 14 | ObserveRT(labels map[string]string, timeSince float64) 15 | RegisterHit(labels map[string]string) 16 | RegisterMiss(labels map[string]string) 17 | IncreaseItemCount(set string) 18 | SetItemCount(set string, n int) 19 | } 20 | 21 | // SinceMs just wraps time.Since() with converting result to milliseconds. 22 | // Because Prometheus prefers milliseconds. 23 | func SinceMs(started time.Time) float64 { 24 | return float64(time.Since(started)) / float64(time.Millisecond) 25 | } 26 | 27 | // IsError is a trivial helper for minimize repetitive checks for error 28 | // values. It passing appropriate numbers to metrics. 29 | func IsError(err error) string { 30 | if err != nil { 31 | return "1" 32 | } 33 | return "0" 34 | } 35 | -------------------------------------------------------------------------------- /nil_logger.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | type NilLogger struct { 4 | } 5 | 6 | func NewNilLogger() *NilLogger { 7 | return &NilLogger{} 8 | } 9 | 10 | func (this *NilLogger) IsDebugEnabled() bool { 11 | return false 12 | } 13 | 14 | func (this *NilLogger) Debugf(message string, args ...interface{}) { 15 | } 16 | 17 | func (this *NilLogger) Debug(...interface{}) { 18 | } 19 | 20 | func (this *NilLogger) Warningf(message string, args ...interface{}) { 21 | } 22 | 23 | func (this *NilLogger) Warning(...interface{}) { 24 | } 25 | 26 | func (this *NilLogger) Errorf(message string, args ...interface{}) { 27 | } 28 | 29 | func (this *NilLogger) Criticalf(message string, args ...interface{}) { 30 | } 31 | 32 | func (this *NilLogger) Printf(format string, v ...interface{}) { 33 | } 34 | 35 | func (this *NilLogger) Critical(...interface{}) { 36 | } 37 | -------------------------------------------------------------------------------- /struct_cache.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "strings" 5 | "sync" 6 | "time" 7 | 8 | "container/list" 9 | 10 | "go-cache/errors" 11 | "go-cache/metric" 12 | ) 13 | 14 | var ErrSetAlreadyExists = errors.New("Set already exists") 15 | 16 | type cacheSet struct { 17 | elements map[string]*list.Element 18 | lruList *list.List 19 | keysLock sync.RWMutex 20 | keysLimit int 21 | name string 22 | 23 | logger IStructCacheLogger 24 | 25 | ticker *time.Ticker 26 | metric metric.Metric 27 | 28 | // connection count metric 29 | quitCollectorChan chan struct{} 30 | } 31 | 32 | // StructCache is simple storage with locking 33 | type StructCache struct { 34 | setsCollection map[string]*cacheSet 35 | setsLock sync.RWMutex 36 | defaultLimit int 37 | 38 | logger IStructCacheLogger 39 | 40 | ticker *time.Ticker 41 | metric metric.Metric 42 | } 43 | 44 | var _ IStructCache = &StructCache{} // StructCache implements IStructCache 45 | 46 | // NewStructCacheObject returns new instance of StructCache 47 | func NewStructCacheObject(limit int, logger IStructCacheLogger, metric metric.Metric) *StructCache { 48 | if logger == nil { 49 | logger = NewNilLogger() 50 | } 51 | 52 | cache := &StructCache{ 53 | defaultLimit: limit, 54 | ticker: time.NewTicker(5 * time.Minute), // @todo make it changeable param 55 | setsCollection: make(map[string]*cacheSet), 56 | logger: logger, 57 | metric: metric, 58 | } 59 | 60 | if cache.logger.IsDebugEnabled() { 61 | cache.logger.Debugf("struct_cache: created with %d limit of entries", limit) 62 | } 63 | 64 | return cache 65 | } 66 | 67 | func (cache *StructCache) SetLimit(limit int) { 68 | if cache.defaultLimit == limit { 69 | return 70 | } 71 | 72 | if cache.logger.IsDebugEnabled() { 73 | cache.logger.Debugf("struct_cache: limit has changed, new value: %d", limit) 74 | } 75 | 76 | cache.setsLock.Lock() 77 | cache.defaultLimit = limit 78 | cache.setsLock.Unlock() 79 | } 80 | 81 | func (set *cacheSet) getKeyFromSet(key *Key) (interface{}, time.Time, bool) { 82 | 83 | var ( 84 | created time.Time 85 | data interface{} 86 | ) 87 | 88 | set.keysLock.RLock() 89 | el, ok := set.elements[key.Pk] 90 | if !ok { 91 | set.keysLock.RUnlock() 92 | return data, created, ok 93 | } 94 | 95 | entry, eok := el.Value.(*Entry) 96 | set.keysLock.RUnlock() 97 | 98 | if !eok { 99 | return data, created, false 100 | } 101 | 102 | set.keysLock.RLock() 103 | created = entry.CreateDate 104 | valid := entry.IsValid() 105 | set.keysLock.RUnlock() 106 | 107 | if !valid { 108 | set.remove(key) 109 | return data, created, false 110 | } 111 | 112 | set.keysLock.RLock() 113 | data = entry.Data 114 | set.keysLock.RUnlock() 115 | 116 | set.keysLock.Lock() 117 | set.lruList.MoveToFront(el) 118 | set.keysLock.Unlock() 119 | 120 | return data, created, true 121 | 122 | } 123 | 124 | func (cache *StructCache) getCacheSet(key *Key) (*cacheSet, bool) { 125 | cache.setsLock.RLock() 126 | set, exists := cache.setsCollection[key.Set] 127 | cache.setsLock.RUnlock() 128 | 129 | return set, exists 130 | } 131 | 132 | // GetWithTime returns value and create time(UTC) by key 133 | func (cache *StructCache) GetWithTime(key *Key) (interface{}, time.Time, bool) { 134 | var ( 135 | created time.Time 136 | data interface{} 137 | ok bool 138 | 139 | ts = time.Now() 140 | ) 141 | 142 | set, setFound := cache.getCacheSet(key) 143 | 144 | if setFound { 145 | data, created, ok = set.getKeyFromSet(key) 146 | } 147 | 148 | cache.updateHitOrMissCount(ok, key) 149 | 150 | cache.metric.ObserveRT(map[string]string{ 151 | metric.LabelSet: key.Set, 152 | metric.LabelOperation: "get", 153 | }, metric.SinceMs(ts)) 154 | 155 | return data, created, ok 156 | } 157 | 158 | func (cache *StructCache) updateHitOrMissCount(condition bool, key *Key) { 159 | switch condition { 160 | case true: 161 | cache.metric.RegisterHit(map[string]string{metric.LabelSet: key.Set}) 162 | if cache.logger.IsDebugEnabled() { 163 | cache.logger.Debugf("struct_cache: HIT %v", key) 164 | } 165 | case false: 166 | cache.metric.RegisterMiss(map[string]string{metric.LabelSet: key.Set}) 167 | if cache.logger.IsDebugEnabled() { 168 | cache.logger.Debugf("struct_cache: MISS %v", key) 169 | } 170 | } 171 | } 172 | 173 | // Get returns value by key 174 | func (cache *StructCache) Get(key *Key) (interface{}, bool) { 175 | data, _, ok := cache.GetWithTime(key) 176 | 177 | return data, ok 178 | } 179 | 180 | // Count elements from cache 181 | func (cache *StructCache) Count() int { 182 | var count int 183 | 184 | cache.setsLock.RLock() 185 | for _, set := range cache.setsCollection { 186 | set.keysLock.RLock() 187 | count += set.lruList.Len() 188 | set.keysLock.RUnlock() 189 | } 190 | cache.setsLock.RUnlock() 191 | 192 | if cache.logger.IsDebugEnabled() { 193 | cache.logger.Debugf("struct_cache: count() = %d", count) 194 | } 195 | 196 | return count 197 | } 198 | 199 | // Find search key by mask 200 | func (cache *StructCache) Find(maskedKey string, limit int) []string { 201 | if cache.logger.IsDebugEnabled() { 202 | cache.logger.Debugf("struct_cache: FIND %q", maskedKey) 203 | } 204 | 205 | result := make([]string, 0, limit) 206 | 207 | cache.setsLock.RLock() 208 | for _, set := range cache.setsCollection { 209 | set.keysLock.RLock() 210 | for key := range set.elements { 211 | if strings.Contains(strings.ToLower(key), maskedKey) { 212 | result = append(result, key) 213 | limit-- 214 | } 215 | 216 | if limit <= 0 { 217 | break 218 | } 219 | } 220 | set.keysLock.RUnlock() 221 | 222 | if limit <= 0 { 223 | break 224 | } 225 | } 226 | cache.setsLock.RUnlock() 227 | 228 | return result 229 | } 230 | 231 | func (cache *StructCache) RegisterCacheSet(setName string, limit int, ticker *time.Ticker) error { 232 | cache.setsLock.Lock() 233 | defer cache.setsLock.Unlock() 234 | 235 | if _, exists := cache.setsCollection[setName]; exists { 236 | return ErrSetAlreadyExists 237 | } 238 | 239 | cache.setsCollection[setName] = &cacheSet{ 240 | elements: make(map[string]*list.Element), 241 | lruList: list.New(), 242 | keysLock: sync.RWMutex{}, 243 | keysLimit: limit, 244 | name: setName, 245 | 246 | ticker: ticker, 247 | 248 | logger: cache.logger, 249 | metric: cache.metric, 250 | 251 | quitCollectorChan: make(chan struct{}, 1), 252 | } 253 | 254 | if ticker != nil { 255 | go cache.setsCollection[setName].collector() 256 | } 257 | 258 | return nil 259 | } 260 | 261 | // Put puts elements into storage 262 | func (cache *StructCache) Put(data interface{}, key *Key, ttl time.Duration) error { 263 | if ttl <= 0 || cache.defaultLimit <= 0 { 264 | return errors.New("Cannot put element (ttl or cache limit is not assign)") 265 | } 266 | 267 | if cache.logger.IsDebugEnabled() { 268 | cache.logger.Debugf("struct_cache: PUT %q with TTL: %s", key, ttl) 269 | } 270 | 271 | set, exists := cache.getCacheSet(key) 272 | if !exists { 273 | cache.RegisterCacheSet(key.Set, cache.defaultLimit, cache.ticker) 274 | 275 | set, exists = cache.getCacheSet(key) 276 | if !exists { 277 | return errors.Errorf("Cant create set %q", key.Set) 278 | } 279 | } 280 | 281 | return set.put(data, key, ttl) 282 | } 283 | 284 | func (set *cacheSet) put(data interface{}, key *Key, ttl time.Duration) error { 285 | set.keysLock.Lock() 286 | defer set.keysLock.Unlock() 287 | 288 | ts := time.Now() 289 | 290 | entitiesCount := set.lruList.Len() 291 | 292 | if entitiesCount >= set.keysLimit { 293 | if set.logger.IsDebugEnabled() { 294 | set.logger.Debug("struct_cache: ATTENTION! Entities count exceeds limit") 295 | } 296 | set.trim() 297 | } 298 | 299 | if el, ok := set.elements[key.Pk]; ok { 300 | set.lruList.MoveToFront(el) 301 | if entry, eok := el.Value.(*Entry); eok { 302 | entry.EndDate = time.Now().Unix() + int64(ttl.Seconds()) 303 | 304 | entry.Data = data 305 | return nil 306 | } 307 | } 308 | 309 | entry := CreateEntry(key, time.Now().Unix()+int64(ttl.Seconds()), data) 310 | el := set.lruList.PushFront(entry) 311 | set.elements[key.Pk] = el 312 | 313 | set.metric.ObserveRT(map[string]string{ 314 | metric.LabelSet: key.Set, 315 | metric.LabelOperation: "put", 316 | }, metric.SinceMs(ts)) 317 | set.metric.IncreaseItemCount(set.name) 318 | 319 | return nil 320 | } 321 | 322 | func (cache *StructCache) Close() { 323 | cache.setsLock.RLock() 324 | defer cache.setsLock.RUnlock() 325 | for _, set := range cache.setsCollection { 326 | set.quitCollectorChan <- struct{}{} 327 | } 328 | } 329 | 330 | // trim remove least recently used elements from cache and leave 'limit - 1' elements, to have a change to put one element 331 | func (set *cacheSet) trim() { 332 | if set.logger.IsDebugEnabled() { 333 | set.logger.Debugf("struct_cache: trim (max %d current %d)", set.keysLimit, set.lruList.Len()) 334 | } 335 | 336 | for set.lruList.Len() >= set.keysLimit && set.lruList.Len() > 0 { 337 | el := set.lruList.Back() 338 | if el != nil { 339 | if entry, ok := el.Value.(*Entry); ok { 340 | delete(set.elements, entry.Key.Pk) 341 | set.lruList.Remove(el) 342 | } 343 | } 344 | } 345 | 346 | set.metric.SetItemCount(set.name, set.lruList.Len()) 347 | } 348 | 349 | // Remove removes value by key 350 | func (cache *StructCache) Remove(key *Key) { 351 | set, exists := cache.getCacheSet(key) 352 | if !exists { 353 | return 354 | } 355 | 356 | set.remove(key) 357 | } 358 | 359 | func (set *cacheSet) remove(key *Key) { 360 | k := key.Pk 361 | if set.logger.IsDebugEnabled() { 362 | set.logger.Debugf("struct_cache: REMOVE %q", key) 363 | } 364 | 365 | set.keysLock.Lock() 366 | if el, ok := set.elements[k]; ok { 367 | delete(set.elements, k) 368 | set.lruList.Remove(el) 369 | } 370 | set.keysLock.Unlock() 371 | 372 | set.metric.SetItemCount(set.name, set.lruList.Len()) 373 | } 374 | 375 | func (set *cacheSet) collector() { 376 | for { 377 | select { 378 | case <-set.ticker.C: 379 | set.keysLock.RLock() 380 | i := 0 381 | for _, el := range set.elements { 382 | if i == 1000 { 383 | set.keysLock.RUnlock() 384 | time.Sleep(time.Millisecond * 10) 385 | i = 0 386 | set.keysLock.RLock() 387 | } 388 | i++ 389 | if entry, ok := el.Value.(*Entry); ok { 390 | if entry.IsValid() { 391 | continue 392 | } 393 | i-- 394 | if set.logger.IsDebugEnabled() { 395 | set.logger.Debugf("struct_cache: collector found NOT VALID %q", entry.Key) 396 | } 397 | set.keysLock.RUnlock() 398 | set.remove(entry.Key) 399 | set.keysLock.RLock() 400 | 401 | } 402 | } 403 | set.keysLock.RUnlock() 404 | case <-set.quitCollectorChan: 405 | return 406 | } 407 | } 408 | } 409 | 410 | // Flush removes all entries from cache and returns number of flushed entries 411 | func (cache *StructCache) Flush() int { 412 | if cache.logger.IsDebugEnabled() { 413 | cache.logger.Debug("struct_cache: flush()") 414 | } 415 | 416 | cache.setsLock.RLock() 417 | defer cache.setsLock.RUnlock() 418 | 419 | var count int 420 | 421 | for _, set := range cache.setsCollection { 422 | set.keysLock.Lock() 423 | count += set.lruList.Len() 424 | set.elements = make(map[string]*list.Element) 425 | set.lruList.Init() 426 | set.keysLock.Unlock() 427 | 428 | cache.metric.SetItemCount(set.name, 0) 429 | } 430 | 431 | return count 432 | } 433 | -------------------------------------------------------------------------------- /struct_cache_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "go-cache/metric/dummy" 5 | "math/rand" 6 | "strconv" 7 | "testing" 8 | "time" 9 | ) 10 | 11 | const ( 12 | defaultTTL = 5 * time.Minute 13 | ) 14 | 15 | func prepareDataForBench(structCache *StructCache) { 16 | 17 | for i := 0; i < 1000; i++ { 18 | k := &Key{ 19 | Set: "set0", 20 | Pk: strconv.Itoa(i), 21 | } 22 | structCache.Put(i, k, defaultTTL) 23 | } 24 | 25 | for i := 0; i < 1000; i++ { 26 | k := &Key{ 27 | Set: "set1", 28 | Pk: strconv.Itoa(i), 29 | } 30 | structCache.Put(i, k, defaultTTL) 31 | } 32 | 33 | for i := 0; i < 1000; i++ { 34 | k := &Key{ 35 | Set: "set2", 36 | Pk: strconv.Itoa(i), 37 | } 38 | structCache.Put(i, k, defaultTTL) 39 | } 40 | 41 | for i := 0; i < 1000; i++ { 42 | k := &Key{ 43 | Set: "set3", 44 | Pk: strconv.Itoa(i), 45 | } 46 | structCache.Put(i, k, defaultTTL) 47 | } 48 | 49 | for i := 0; i < 1000; i++ { 50 | k := &Key{ 51 | Set: "set4", 52 | Pk: strconv.Itoa(i), 53 | } 54 | structCache.Put(i, k, defaultTTL) 55 | } 56 | 57 | for i := 0; i < 1000; i++ { 58 | k := &Key{ 59 | Set: "set5", 60 | Pk: strconv.Itoa(i), 61 | } 62 | structCache.Put(i, k, defaultTTL) 63 | } 64 | 65 | for i := 0; i < 1000; i++ { 66 | k := &Key{ 67 | Set: "set6", 68 | Pk: strconv.Itoa(i), 69 | } 70 | structCache.Put(i, k, defaultTTL) 71 | } 72 | 73 | } 74 | 75 | func BenchmarkStructCache_ConcurentGetFrom7DifferentSets(b *testing.B) { 76 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 77 | prepareDataForBench(structCache) 78 | 79 | b.ResetTimer() 80 | b.RunParallel(func(pb *testing.PB) { 81 | for pb.Next() { 82 | caseNum := rand.Intn(7) 83 | set := "set" + strconv.Itoa(caseNum) 84 | key := &Key{Set: set, Pk: "2"} 85 | 86 | v, ok := structCache.Get(key) 87 | if !ok { 88 | b.Fatalf("Get operation is unsuccessfull for %v", key) 89 | } 90 | if v == nil { 91 | b.Fatalf("Got nil value from cache for %v", key) 92 | } 93 | } 94 | }) 95 | } 96 | 97 | func BenchmarkStructCache_ConcurentPutOverLimit(b *testing.B) { 98 | structCache := NewStructCacheObject(5, nil, dummy.NewMetric()) 99 | 100 | b.ResetTimer() 101 | b.RunParallel(func(pb *testing.PB) { 102 | i := 0 103 | for pb.Next() { 104 | i++ 105 | caseNum := rand.Intn(7) 106 | set := "set" + strconv.Itoa(caseNum) 107 | k := &Key{ 108 | Set: set, 109 | Pk: strconv.Itoa(i), 110 | } 111 | 112 | err := structCache.Put(i, k, defaultTTL) 113 | if err != nil { 114 | b.Fatalf("Put operation is unsuccessfull: %v", err) 115 | } 116 | } 117 | }) 118 | } 119 | 120 | func BenchmarkStructCache_ConcurentPutToDifferentSets(b *testing.B) { 121 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 122 | 123 | b.ResetTimer() 124 | b.RunParallel(func(pb *testing.PB) { 125 | i := 0 126 | for pb.Next() { 127 | if i > 8000 { 128 | i = 1 129 | } 130 | 131 | i++ 132 | caseNum := rand.Intn(7) 133 | set := "set" + strconv.Itoa(caseNum) 134 | k := &Key{ 135 | Set: set, 136 | Pk: strconv.Itoa(i), 137 | } 138 | 139 | err := structCache.Put(i, k, defaultTTL) 140 | if err != nil { 141 | b.Fatalf("Put operation is unsuccessfull: %v", err) 142 | } 143 | } 144 | }) 145 | } 146 | 147 | func BenchmarkStructCache_ConcurentPutAndGet(b *testing.B) { 148 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 149 | 150 | b.ResetTimer() 151 | b.RunParallel(func(pb *testing.PB) { 152 | i := 0 153 | for pb.Next() { 154 | if i > 8000 { 155 | i = 1 156 | } 157 | i++ 158 | 159 | k := &Key{ 160 | Set: "set1", 161 | Pk: strconv.Itoa(i), 162 | } 163 | 164 | err := structCache.Put(i, k, defaultTTL) 165 | if err != nil { 166 | b.Fatalf("Put operation is unsuccessfull: %v", err) 167 | } 168 | 169 | kGet := &Key{ 170 | Set: "set1", 171 | Pk: strconv.Itoa(i), 172 | } 173 | v, ok := structCache.Get(kGet) 174 | if !ok { 175 | b.Fatalf("Get operation is unsuccessfull for %v", kGet) 176 | } 177 | if v == nil { 178 | b.Fatalf("Got nil value from cache for %v", kGet) 179 | } 180 | } 181 | }) 182 | } 183 | 184 | func BenchmarkStructCache_ConcurentGetWithoutGC(b *testing.B) { 185 | structCache := NewStructCacheObject(50000000, nil, dummy.NewMetric()) 186 | for i := 0; i < 500000; i++ { 187 | k := &Key{ 188 | Set: "set" + strconv.Itoa(i%50), 189 | Pk: strconv.Itoa(i), 190 | } 191 | structCache.Put(i, k, defaultTTL) 192 | } 193 | 194 | b.ResetTimer() 195 | b.RunParallel(func(pb *testing.PB) { 196 | i := 0 197 | for pb.Next() { 198 | if i > 500000 { 199 | i = 1 200 | } 201 | i++ 202 | 203 | k := &Key{ 204 | Set: "set" + strconv.Itoa(i%50), 205 | Pk: strconv.Itoa(i), 206 | } 207 | 208 | v, ok := structCache.Get(k) 209 | if !ok { 210 | b.Fatalf("Get operation is unsuccessfull for %v", k) 211 | } 212 | if v == nil { 213 | b.Fatalf("Got nil value from cache for %v", k) 214 | } 215 | } 216 | }) 217 | } 218 | 219 | func BenchmarkStructCache_ConcurentPutWithGC(b *testing.B) { 220 | structCache := NewStructCacheObject(50000000, nil, dummy.NewMetric()) 221 | structCache.ticker = time.NewTicker(time.Millisecond * 10) 222 | 223 | for i := 0; i < 500000; i++ { 224 | k := &Key{ 225 | Set: "set" + strconv.Itoa(i%50), 226 | Pk: strconv.Itoa(i), 227 | } 228 | structCache.Put(i, k, defaultTTL) 229 | } 230 | 231 | b.ResetTimer() 232 | b.RunParallel(func(pb *testing.PB) { 233 | i := 500000 234 | for pb.Next() { 235 | i++ 236 | k := &Key{ 237 | Set: "set" + strconv.Itoa(i%50), 238 | Pk: strconv.Itoa(i), 239 | } 240 | 241 | err := structCache.Put(i, k, defaultTTL) 242 | if err != nil { 243 | b.Fatalf("Put operation is unsuccessfull: %v", err) 244 | } 245 | } 246 | }) 247 | } 248 | 249 | func BenchmarkStructCache_ConcurentGet_ManySets_WithGC(b *testing.B) { 250 | structCache := NewStructCacheObject(50000000, nil, dummy.NewMetric()) 251 | structCache.ticker = time.NewTicker(time.Millisecond * 10) 252 | 253 | for i := 0; i < 500000; i++ { 254 | k := &Key{ 255 | Set: "set" + strconv.Itoa(i%50), 256 | Pk: strconv.Itoa(i), 257 | } 258 | structCache.Put(i, k, defaultTTL) 259 | } 260 | 261 | b.ResetTimer() 262 | b.RunParallel(func(pb *testing.PB) { 263 | i := 0 264 | for pb.Next() { 265 | if i > 500000 { 266 | i = 1 267 | } 268 | i++ 269 | 270 | k := &Key{ 271 | Set: "set" + strconv.Itoa(i%50), 272 | Pk: strconv.Itoa(i), 273 | } 274 | 275 | v, ok := structCache.Get(k) 276 | if !ok { 277 | b.Fatalf("Get operation is unsuccessfull for %v", k) 278 | } 279 | if v == nil { 280 | b.Fatalf("Got nil value from cache for %v", k) 281 | } 282 | } 283 | }) 284 | } 285 | 286 | 287 | func BenchmarkStructCache_ConcurentGet_SingleSet_WithGC(b *testing.B) { 288 | structCache := NewStructCacheObject(50000000, nil, dummy.NewMetric()) 289 | structCache.ticker = time.NewTicker(time.Millisecond * 10) 290 | 291 | for i := 0; i < 500000; i++ { 292 | k := &Key{ 293 | Set: "set1", 294 | Pk: strconv.Itoa(i), 295 | } 296 | structCache.Put(i, k, defaultTTL) 297 | } 298 | 299 | b.ResetTimer() 300 | b.RunParallel(func(pb *testing.PB) { 301 | i := 0 302 | for pb.Next() { 303 | if i > 500000 { 304 | i = 1 305 | } 306 | i++ 307 | 308 | k := &Key{ 309 | Set: "set1", 310 | Pk: strconv.Itoa(i), 311 | } 312 | 313 | v, ok := structCache.Get(k) 314 | if !ok { 315 | b.Fatalf("Get operation is unsuccessfull for %v", k) 316 | } 317 | if v == nil { 318 | b.Fatalf("Got nil value from cache for %v", k) 319 | } 320 | } 321 | }) 322 | } 323 | -------------------------------------------------------------------------------- /struct_cache_dummy.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "time" 5 | 6 | "go-cache/errors" 7 | ) 8 | 9 | // StructCacheDummy implements cache that accept any data but always return null 10 | type StructCacheDummy struct { 11 | logger IStructCacheLogger 12 | } 13 | 14 | var _ IStructCache = &StructCacheDummy{} 15 | 16 | // NewStructCacheObjectDummy returns new instance of StructCacheDummy 17 | func NewStructCacheObjectDummy(logger IStructCacheLogger) *StructCacheDummy { 18 | if logger == nil { 19 | logger = NewNilLogger() 20 | } 21 | return &StructCacheDummy{logger} 22 | } 23 | 24 | func (cache *StructCacheDummy) RegisterCacheSet(setName string, limit int, ticker *time.Ticker) error { 25 | cache.logger.Debugf("struct_storage_dummy: registerCacheSet(), setName: %s, limit: %d", setName, limit) 26 | return nil 27 | } 28 | 29 | // Close do nothing 30 | func (cache *StructCacheDummy) Close() { 31 | cache.logger.Debug("struct_storage_dummy: close()") 32 | } 33 | 34 | // GetWithTime returns nil, do nothing 35 | func (cache *StructCacheDummy) GetWithTime(key *Key) (data interface{}, dt time.Time, ok bool) { 36 | cache.logger.Debugf("struct_storage_dummy: getWithTime(), key: %s", key.ID()) 37 | return 38 | } 39 | 40 | // Get returns nil, do nothing 41 | func (cache *StructCacheDummy) Get(key *Key) (data interface{}, ok bool) { 42 | cache.logger.Debugf("struct_storage_dummy: get(), key: %s", key.ID()) 43 | return 44 | } 45 | 46 | // Put returns nil, do nothing 47 | func (cache *StructCacheDummy) Put(data interface{}, key *Key, ttl time.Duration) error { 48 | cache.logger.Debugf("struct_storage_dummy: put(), key: %s", key.ID()) 49 | // For the beginning, let's check if it is pointer. 50 | if isPointer(data) { 51 | return errors.Errorf("struct_storage: It is prohibited to keep pointers in struct cache. %s.", key.String()) 52 | } 53 | return nil 54 | } 55 | 56 | // Remove returns nil, do nothing 57 | func (cache *StructCacheDummy) Remove(key *Key) { 58 | cache.logger.Debugf("struct_storage_dummy: remove(), key: %s", key.ID()) 59 | } 60 | 61 | // Count returns number of cache entries 62 | func (cache *StructCacheDummy) Count() int { 63 | cache.logger.Debug("struct_storage_dummy: count()") 64 | return 0 65 | } 66 | 67 | // Flush removes all entries from cache and returns number of flushed entries 68 | func (cache *StructCacheDummy) Flush() int { 69 | cache.logger.Debug("struct_storage_dummy: flush()") 70 | return 0 71 | } 72 | -------------------------------------------------------------------------------- /struct_cache_example_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | 7 | "go-cache/metric/dummy" 8 | ) 9 | 10 | func ExampleStructCache_Get() { 11 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 12 | 13 | k := &Key{ 14 | Set: "set", 15 | Pk: "1", 16 | } 17 | data := "The essential is invisible to the eyes, we can not truly see but with the eyes of the heart." 18 | 19 | ttl := 5*time.Minute 20 | 21 | structCache.Put(data, k, ttl) 22 | 23 | infResult, find := structCache.Get(k) 24 | if !find { 25 | panic("Key is not found") 26 | } 27 | 28 | result, ok := infResult.(string) 29 | if !ok { 30 | panic("Data have wrong type") 31 | } 32 | 33 | fmt.Println(string(result)) 34 | // Output: 35 | // The essential is invisible to the eyes, we can not truly see but with the eyes of the heart. 36 | } 37 | 38 | func ExampleStructCache_Put() { 39 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 40 | 41 | k := &Key{ 42 | Set: "set1", 43 | Pk: "1", 44 | } 45 | data := "data" 46 | 47 | ttl := 5*time.Minute 48 | 49 | structCache.Put(data, k, ttl) 50 | 51 | data2 := "Your most unhappy customers are your greatest source of learning." 52 | structCache.Put(data2, k, ttl) 53 | 54 | infResult, find := structCache.Get(k) 55 | if !find { 56 | panic("Data was not found") 57 | } 58 | 59 | result, ok := infResult.(string) 60 | if !ok { 61 | panic("Data have wrong type") 62 | } 63 | 64 | fmt.Println(result) 65 | // Output: 66 | // Your most unhappy customers are your greatest source of learning. 67 | } 68 | 69 | func ExampleStructCache_Count() { 70 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 71 | 72 | k := &Key{ 73 | Set: "set1", 74 | Pk: "1", 75 | } 76 | data := "data" 77 | 78 | ttl := 5*time.Minute 79 | 80 | structCache.Put(data, k, ttl) 81 | 82 | k = &Key{ 83 | Set: "set1", 84 | Pk: "2", 85 | } 86 | structCache.Put(data, k, ttl) 87 | 88 | k = &Key{ 89 | Set: "set2", 90 | Pk: "1", 91 | } 92 | structCache.Put(data, k, ttl) 93 | 94 | count := structCache.Count() 95 | 96 | fmt.Println(count) 97 | // Output: 98 | // 3 99 | } 100 | 101 | func ExampleStructCache_Remove() { 102 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 103 | 104 | k := &Key{ 105 | Set: "set1", 106 | Pk: "1", 107 | } 108 | data := "data" 109 | 110 | ttl := 5*time.Minute 111 | 112 | structCache.Put(data, k, ttl) 113 | 114 | structCache.Remove(k) 115 | 116 | count := structCache.Count() 117 | 118 | fmt.Println(count) 119 | // Output: 120 | // 0 121 | } 122 | 123 | func ExampleStructCache_Flush() { 124 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 125 | 126 | k := &Key{ 127 | Set: "set1", 128 | Pk: "1", 129 | } 130 | data := "data" 131 | 132 | ttl := 5*time.Minute 133 | 134 | structCache.Put(data, k, ttl) 135 | 136 | k = &Key{ 137 | Set: "set2", 138 | Pk: "1", 139 | } 140 | structCache.Put(data, k, ttl) 141 | 142 | structCache.Flush() 143 | 144 | count := structCache.Count() 145 | 146 | fmt.Println(count) 147 | // Output: 148 | // 0 149 | } 150 | 151 | func ExampleStructCache_Find() { 152 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 153 | 154 | k := &Key{ 155 | Set: "set1", 156 | Pk: "mask", 157 | } 158 | data := "data" 159 | 160 | ttl := 5*time.Minute 161 | 162 | structCache.Put(data, k, ttl) 163 | 164 | cnt := structCache.Find("as", 1) 165 | 166 | fmt.Println(cnt) 167 | // Output: 168 | // [mask] 169 | } 170 | 171 | func ExampleStructCache_SetLimit() { 172 | structCache := NewStructCacheObject(1000, nil, dummy.NewMetric()) 173 | structCache.SetLimit(1) 174 | 175 | k := &Key{ 176 | Set: "set1", 177 | Pk: "1", 178 | } 179 | data := "data" 180 | 181 | ttl := 5*time.Minute 182 | 183 | structCache.Put(data, k, ttl) 184 | 185 | k = &Key{ 186 | Set: "set1", 187 | Pk: "2", 188 | } 189 | structCache.Put(data, k, ttl) 190 | 191 | cnt := structCache.Count() 192 | 193 | fmt.Println(cnt) 194 | // Output: 195 | // 1 196 | } -------------------------------------------------------------------------------- /struct_cache_helper.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "reflect" 4 | 5 | func isPointer(a interface{}) bool { 6 | kind := reflect.TypeOf(a).Kind() 7 | return kind == reflect.Ptr || kind == reflect.UnsafePointer 8 | } 9 | -------------------------------------------------------------------------------- /struct_cache_helper_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestHelperPointer(t *testing.T) { 8 | type A struct{} 9 | 10 | a := A{} 11 | b := &a 12 | c := "hello" 13 | d := &c 14 | e := make(map[string]string) 15 | f := &e 16 | g := []string{} 17 | h := &g 18 | i := 11 19 | j := &i 20 | k := 11.11 21 | l := &k 22 | 23 | if isPointer(a) { 24 | t.Errorf("expected a is not poiter, got - it is") 25 | } 26 | if !isPointer(b) { 27 | t.Errorf("expected a is poiter, got - it is not") 28 | } 29 | if isPointer(c) { 30 | t.Errorf("expected a is not poiter, got - it is") 31 | } 32 | if !isPointer(d) { 33 | t.Errorf("expected a is poiter, got - it is not") 34 | } 35 | if isPointer(e) { 36 | t.Errorf("expected a is not poiter, got - it is") 37 | } 38 | if !isPointer(f) { 39 | t.Errorf("expected a is poiter, got - it is not") 40 | } 41 | if isPointer(g) { 42 | t.Errorf("expected a is not poiter, got - it is") 43 | } 44 | if !isPointer(h) { 45 | t.Errorf("expected a is poiter, got - it is not") 46 | } 47 | if isPointer(i) { 48 | t.Errorf("expected a is not poiter, got - it is") 49 | } 50 | if !isPointer(j) { 51 | t.Errorf("expected a is poiter, got - it is not") 52 | } 53 | if isPointer(k) { 54 | t.Errorf("expected a is not poiter, got - it is") 55 | } 56 | if !isPointer(l) { 57 | t.Errorf("expected a is poiter, got - it is not") 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /struct_cache_interface.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // IStructCache defines required interface for caching module (to be able to store any kind of data, mostly in memory) 8 | type IStructCache interface { 9 | IFlushable 10 | RegisterCacheSet(setName string, limit int, ticker *time.Ticker) error 11 | Get(key *Key) (data interface{}, ok bool) 12 | GetWithTime(key *Key) (data interface{}, dt time.Time, ok bool) 13 | Put(data interface{}, key *Key, ttl time.Duration) error 14 | Remove(key *Key) 15 | Count() int 16 | Close() 17 | } 18 | 19 | // IStructCacheDebug defines interface for console debug tools 20 | type IStructCacheDebug interface { 21 | IFlushable 22 | Count() int 23 | Find(maskedKey string, limit int) []string 24 | } 25 | 26 | type IStructCacheLogger interface { 27 | IsDebugEnabled() bool 28 | Debugf(message string, args ...interface{}) 29 | Debug(...interface{}) 30 | Warningf(message string, args ...interface{}) 31 | Warning(...interface{}) 32 | } 33 | 34 | type ILimitSetter interface { 35 | SetLimit(int) 36 | } 37 | -------------------------------------------------------------------------------- /struct_cache_test.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import ( 4 | "go-cache/metric/dummy" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestStructCache_Get_Ok(t *testing.T) { 10 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 11 | k := &Key{ 12 | Set: "set1", 13 | Pk: "1", 14 | } 15 | data := "data" 16 | structCache.Put(data, k, time.Minute*5) 17 | 18 | infResult, find := structCache.Get(k) 19 | if !find { 20 | t.Error("Data was not found") 21 | } 22 | 23 | result, ok := infResult.(string) 24 | if !ok { 25 | t.Error("Data have wrong type") 26 | } 27 | 28 | if result != data { 29 | t.Error("Data is not expected") 30 | } 31 | } 32 | 33 | func TestStructCache_RenewExistingKey_Ok(t *testing.T) { 34 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 35 | k := &Key{ 36 | Set: "set1", 37 | Pk: "1", 38 | } 39 | data := "data" 40 | structCache.Put(data, k, time.Minute*5) 41 | 42 | data2 := "data2" 43 | structCache.Put(data2, k, time.Minute*5) 44 | 45 | infResult, find := structCache.Get(k) 46 | if !find { 47 | t.Error("Data was not found") 48 | } 49 | 50 | result, ok := infResult.(string) 51 | if !ok { 52 | t.Error("Data have wrong type") 53 | } 54 | 55 | if result != data2 { 56 | t.Error("Data is not expected") 57 | } 58 | 59 | cnt := structCache.Count() 60 | if cnt != 1 { 61 | t.Error("Count is not expeted") 62 | } 63 | } 64 | 65 | func TestStructCache_Get_KeyNotFoundInExistsingSet(t *testing.T) { 66 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 67 | k := &Key{ 68 | Set: "set1", 69 | Pk: "1", 70 | } 71 | data := "data" 72 | structCache.Put(data, k, time.Minute*5) 73 | 74 | k = &Key{ 75 | Set: "set1", 76 | Pk: "2", 77 | } 78 | 79 | infResult, find := structCache.Get(k) 80 | if find { 81 | t.Error("Key for this set shouldn't exist") 82 | } 83 | 84 | if infResult != nil { 85 | t.Error("Data should be nil") 86 | } 87 | } 88 | 89 | func TestStructCache_Count_Ok(t *testing.T) { 90 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 91 | k := &Key{ 92 | Set: "set1", 93 | Pk: "1", 94 | } 95 | data := "data" 96 | structCache.Put(data, k, time.Minute*5) 97 | 98 | k = &Key{ 99 | Set: "set1", 100 | Pk: "2", 101 | } 102 | structCache.Put(data, k, time.Minute*5) 103 | 104 | k = &Key{ 105 | Set: "set2", 106 | Pk: "1", 107 | } 108 | structCache.Put(data, k, time.Minute*5) 109 | 110 | count := structCache.Count() 111 | 112 | if count != 3 { 113 | t.Error("Count is not expected") 114 | } 115 | } 116 | 117 | func TestStructCache_Remove_Ok(t *testing.T) { 118 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 119 | k := &Key{ 120 | Set: "set1", 121 | Pk: "1", 122 | } 123 | data := "data" 124 | structCache.Put(data, k, time.Minute*5) 125 | 126 | structCache.Remove(k) 127 | 128 | count := structCache.Count() 129 | 130 | if count != 0 { 131 | t.Error("Count is not expected") 132 | } 133 | } 134 | 135 | func TestStructCache_Flush_Ok(t *testing.T) { 136 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 137 | k := &Key{ 138 | Set: "set1", 139 | Pk: "1", 140 | } 141 | data := "data" 142 | structCache.Put(data, k, time.Minute*5) 143 | 144 | k = &Key{ 145 | Set: "set2", 146 | Pk: "1", 147 | } 148 | structCache.Put(data, k, time.Minute*5) 149 | 150 | structCache.Flush() 151 | 152 | count := structCache.Count() 153 | 154 | if count != 0 { 155 | t.Error("Count is not expected") 156 | } 157 | } 158 | 159 | func TestStructCache_Find_Ok(t *testing.T) { 160 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 161 | k := &Key{ 162 | Set: "set1", 163 | Pk: "mask", 164 | } 165 | data := "data" 166 | structCache.Put(data, k, time.Minute*5) 167 | 168 | cnt := structCache.Find("as", 1) 169 | 170 | if len(cnt) != 1 { 171 | t.Error("Count is not expected") 172 | } 173 | 174 | if cnt[0] != "mask" { 175 | t.Error("Find return wrong key") 176 | } 177 | } 178 | 179 | func TestStructCache_RegisterCacheSet_Ok(t *testing.T) { 180 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 181 | 182 | structCache.RegisterCacheSet("set1", 1, nil) 183 | k := &Key{ 184 | Set: "set1", 185 | Pk: "1", 186 | } 187 | data := "data" 188 | structCache.Put(data, k, time.Minute*5) 189 | 190 | k = &Key{ 191 | Set: "set1", 192 | Pk: "2", 193 | } 194 | structCache.Put(data, k, time.Minute*5) 195 | 196 | cnt := structCache.Count() 197 | 198 | if cnt != 1 { 199 | t.Error("Count is not expected") 200 | } 201 | } 202 | 203 | func TestStructCache_RegisterCacheSet_AlreadyExists(t *testing.T) { 204 | structCache := NewStructCacheObject(8000, nil, dummy.NewMetric()) 205 | 206 | err := structCache.RegisterCacheSet("set1", 1, nil) 207 | 208 | if err != nil { 209 | t.Error("CacheSet should be registered") 210 | } 211 | 212 | err = structCache.RegisterCacheSet("set1", 1, nil) 213 | 214 | if err == nil || err != ErrSetAlreadyExists { 215 | t.Error("Should error: cacheSet is already exists") 216 | } 217 | } 218 | 219 | func TestStructCache_Collector_Ok(t *testing.T) { 220 | structCache := NewStructCacheObject(2, nil, dummy.NewMetric()) 221 | structCache.ticker = time.NewTicker(time.Millisecond * 1) 222 | k := &Key{ 223 | Set: "set1", 224 | Pk: "1", 225 | } 226 | data := "data" 227 | structCache.Put(data, k, time.Millisecond*1) 228 | 229 | time.Sleep(time.Millisecond * 20) 230 | 231 | cnt := structCache.Count() 232 | 233 | if cnt != 0 { 234 | t.Error("Count is not expected") 235 | } 236 | } 237 | 238 | func TestStructCache_Close_Ok(t *testing.T) { 239 | structCache := NewStructCacheObject(2, nil, dummy.NewMetric()) 240 | structCache.ticker = time.NewTicker(time.Millisecond * 1) 241 | k := &Key{ 242 | Set: "set1", 243 | Pk: "1", 244 | } 245 | data := "data" 246 | structCache.Put(data, k, time.Millisecond*5) 247 | 248 | // Collector should be closed, so it will not clean expired cache 249 | structCache.Close() 250 | 251 | time.Sleep(time.Millisecond * 20) 252 | 253 | cnt := structCache.Count() 254 | 255 | if cnt != 1 { 256 | t.Error("Count is not expected") 257 | } 258 | } 259 | 260 | func TestStructCache_SetLimit_Ok(t *testing.T) { 261 | structCache := NewStructCacheObject(1000, nil, dummy.NewMetric()) 262 | structCache.SetLimit(1) 263 | k := &Key{ 264 | Set: "set1", 265 | Pk: "1", 266 | } 267 | data := "data" 268 | structCache.Put(data, k, time.Minute*5) 269 | 270 | k = &Key{ 271 | Set: "set1", 272 | Pk: "2", 273 | } 274 | structCache.Put(data, k, time.Minute*5) 275 | 276 | cnt := structCache.Count() 277 | 278 | if cnt != 1 { 279 | t.Error("Count is not expected") 280 | } 281 | } 282 | 283 | func TestStructCache_Get_NotFound(t *testing.T) { 284 | structCache := NewStructCacheObject(1000, nil, dummy.NewMetric()) 285 | k := &Key{ 286 | Set: "set1", 287 | Pk: "1", 288 | } 289 | 290 | data, exists := structCache.Get(k) 291 | 292 | if exists { 293 | t.Error("This key shouldn't exist") 294 | } 295 | 296 | if data != nil { 297 | t.Error("Data for this key should be nill") 298 | } 299 | } 300 | 301 | func TestStructCache_Get_Expired(t *testing.T) { 302 | structCache := NewStructCacheObject(1000, nil, dummy.NewMetric()) 303 | structCache.SetLimit(1) 304 | k := &Key{ 305 | Set: "set1", 306 | Pk: "1", 307 | } 308 | data := "data" 309 | structCache.Put(data, k, time.Millisecond*1) 310 | 311 | time.Sleep(time.Millisecond * 5) 312 | 313 | result, exists := structCache.Get(k) 314 | 315 | if exists { 316 | t.Error("This key is expired") 317 | } 318 | 319 | if result != nil { 320 | t.Error("Data for this key should be nil") 321 | } 322 | 323 | cnt := structCache.Count() 324 | 325 | if cnt != 0 { 326 | t.Error("Expired key should be cleaned on Get operation") 327 | } 328 | } 329 | -------------------------------------------------------------------------------- /ttls.go: -------------------------------------------------------------------------------- 1 | package cache 2 | 3 | import "time" 4 | 5 | const ( 6 | // SmallCacheTTL define a short cache lifetime 7 | SmallCacheTTL = 5 * time.Minute 8 | 9 | // DefaultCacheTTL default cache lifetime 10 | DefaultCacheTTL = 30 * time.Minute 11 | 12 | // TwoHCacheTTL two hours cache lifetime 13 | TwoHCacheTTL = 2 * time.Hour 14 | 15 | // LongCacheTTL large cache lifetime (one day) 16 | LongCacheTTL = 24 * time.Hour 17 | 18 | // VeryLongCacheTTL very large cache lifetime (one mounth) 19 | VeryLongCacheTTL = 30 * 24 * time.Hour 20 | 21 | // EternalLongCacheTTL eternal large cache lifetime (one 10 years) 22 | EternalLongCacheTTL = 10 * 365 * 24 * time.Hour 23 | ) 24 | --------------------------------------------------------------------------------