├── go.mod
├── assets
└── logo.png
├── memcached
├── README.md
└── memcached.go
├── logger.go
├── nocache
└── nocache.go
├── LICENSE
├── ristretto
├── ristretto_test.go
└── ristretto.go
├── memory
└── memory.go
├── redis
├── redis_test.go
└── redis.go
├── options.go
├── key.go
├── remember_test.go
├── remember.go
└── README.md
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/rocketlaunchr/remember-go
2 |
3 | go 1.12
4 |
--------------------------------------------------------------------------------
/assets/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/rocketlaunchr/remember-go/HEAD/assets/logo.png
--------------------------------------------------------------------------------
/memcached/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | # UNTESTED DRIVER
4 |
5 | Work In Progress
6 |
7 | It uses [memcache driver](https://godoc.org/github.com/bradfitz/gomemcache/memcache).
8 |
9 | Feel free to tinker with to make it work. Then submit a Pull Request.
--------------------------------------------------------------------------------
/logger.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package remember
4 |
5 | const logPatternRed = "\x1b[31m%s\x1b[39;49m\n"
6 | const logPatternBlue = "\x1b[36m%s\x1b[39;49m\n"
7 |
8 | // Logger provides an interface to log extra debug information.
9 | // The glog package can be used or alternatively you can defined your own.
10 | //
11 | // Example:
12 | //
13 | // import log
14 | //
15 | // type aLogger struct {}
16 | //
17 | // func (l aLogger) Log(format string, args ...interface{}) {
18 | // log.Printf(format, args...)
19 | // }
20 | type Logger interface {
21 | // Log follows the same pattern as fmt.Printf( ).
22 | Log(format string, args ...interface{})
23 | }
24 |
--------------------------------------------------------------------------------
/nocache/nocache.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package nocache
4 |
5 | import (
6 | "context"
7 | "time"
8 |
9 | "github.com/rocketlaunchr/remember-go"
10 | )
11 |
12 | // NoCache is used for testing purposes.
13 | type NoCache struct{}
14 |
15 | // NewNoCache creates a NoCache struct.
16 | func NewNoCache() *NoCache { return &NoCache{} }
17 |
18 | // Conn will provide a "pretend" connection.
19 | func (nc *NoCache) Conn(ctx context.Context) (remember.Cacher, error) { return nc, nil }
20 |
21 | // StorePointer sets whether a storage driver requires itemToStore to be
22 | // stored as a pointer or as a concrete value.
23 | func (nc *NoCache) StorePointer() bool { return true }
24 |
25 | // Get returns a value from the cache if the key exists.
26 | func (nc *NoCache) Get(key string) (_ interface{}, found bool, _ error) { return nil, false, nil }
27 |
28 | // Set sets a item into the cache for a particular key.
29 | func (nc *NoCache) Set(key string, expiration time.Duration, itemToStore interface{}) error {
30 | return nil
31 | }
32 |
33 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
34 | func (nc *NoCache) Close() { return }
35 |
36 | // Forget clears the value from the cache for the particular key.
37 | func (nc *NoCache) Forget(key string) error { return nil }
38 |
39 | // ForgetAll clears all values from the cache.
40 | func (nc *NoCache) ForgetAll() error { return nil }
41 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | mMIT License
2 |
3 | Copyright (c) 2018-21 PJ Engineering and Business Solutions Pty. Ltd.
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
23 | The usage of this software must not be knowingly used for an application
24 | that will be directly or indirectly used for military purposes.
25 |
26 | All forks of this package must be subject to the above provisions.
27 | This only applies to the fork itself and NOT the product(s) derived from this package
28 | or a fork of this package.
--------------------------------------------------------------------------------
/ristretto/ristretto_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package ristretto_test
4 |
5 | import (
6 | "context"
7 | "testing"
8 | "time"
9 |
10 | rist "github.com/dgraph-io/ristretto"
11 | "github.com/rocketlaunchr/remember-go"
12 | "github.com/rocketlaunchr/remember-go/ristretto"
13 | )
14 |
15 | var cfg = &rist.Config{
16 | NumCounters: 1e7, // number of keys to track frequency of (10M).
17 | MaxCost: 1 << 30, // maximum cost of cache (1GB).
18 | BufferItems: 64, // number of keys per Get buffer.
19 | }
20 |
21 | var ctx = context.Background()
22 |
23 | func TestKeyBasicOperation(t *testing.T) {
24 | var ms = ristretto.NewRistrettoStore(cfg)
25 |
26 | key := "key"
27 | exp := 10 * time.Minute
28 |
29 | slowQuery := func(ctx context.Context) (interface{}, error) {
30 | return "val", nil
31 | }
32 |
33 | actual, _, _ := remember.Cache(ctx, ms, key, exp, slowQuery)
34 |
35 | expected := "val"
36 |
37 | if actual.(string) != expected {
38 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
39 | }
40 | }
41 |
42 | func TestFetchFromCacheAndDisableCache(t *testing.T) {
43 | var ms = ristretto.NewRistrettoStore(cfg)
44 |
45 | key := "key"
46 | exp := 10 * time.Minute
47 |
48 | slowQuery := func(ctx context.Context) (interface{}, error) {
49 | return "val", nil
50 | }
51 |
52 | // warm up cache
53 | remember.Cache(ctx, ms, key, exp, slowQuery)
54 |
55 | // This time fetch from cache
56 | actual, _, _ := remember.Cache(ctx, ms, key, exp, slowQuery)
57 |
58 | expected := "val"
59 |
60 | if actual.(string) != expected {
61 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
62 | }
63 |
64 | // Actual is now "val", Let's change it to "val2" and disable cache usage.
65 |
66 | slowQuery = func(ctx context.Context) (interface{}, error) {
67 | return "val2", nil
68 | }
69 |
70 | actual, _, _ = remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{DisableCacheUsage: true})
71 |
72 | expected = "val2"
73 |
74 | if actual.(string) != expected {
75 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
76 | }
77 | }
78 |
--------------------------------------------------------------------------------
/memory/memory.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package memory
4 |
5 | import (
6 | "context"
7 | "time"
8 |
9 | "github.com/patrickmn/go-cache"
10 | "github.com/rocketlaunchr/remember-go"
11 | )
12 |
13 | // NoExpiration is used to indicate that data should not expire from the cache.
14 | const NoExpiration time.Duration = -1
15 |
16 | // MemoryStore is used to create an in-memory cache.
17 | type MemoryStore struct {
18 | cache *cache.Cache
19 | }
20 |
21 | // NewMemoryStore creates an in-memory cache where the expired items
22 | // are deleted based on the cleanupInterval duration.
23 | func NewMemoryStore(cleanupInterval time.Duration) *MemoryStore {
24 | return &MemoryStore{
25 | cache: cache.New(cache.NoExpiration, cleanupInterval),
26 | }
27 | }
28 |
29 | // NewMemoryStoreFrom creates an in-memory cache directly from a *cache.Cache object.
30 | func NewMemoryStoreFrom(cache *cache.Cache) *MemoryStore {
31 | return &MemoryStore{
32 | cache: cache,
33 | }
34 | }
35 |
36 | // Conn does nothing for this storage driver.
37 | func (c *MemoryStore) Conn(ctx context.Context) (remember.Cacher, error) {
38 | return c, nil
39 | }
40 |
41 | // StorePointer sets whether a storage driver requires itemToStore to be
42 | // stored as a pointer or as a concrete value.
43 | func (c *MemoryStore) StorePointer() bool {
44 | return false
45 | }
46 |
47 | // Get returns a value from the cache if the key exists.
48 | func (c *MemoryStore) Get(key string) (_ interface{}, found bool, _ error) {
49 | item, found := c.cache.Get(key)
50 | return item, found, nil
51 | }
52 |
53 | // Set sets a item into the cache for a particular key.
54 | func (c *MemoryStore) Set(key string, expiration time.Duration, itemToStore interface{}) error {
55 | c.cache.Set(key, itemToStore, expiration)
56 | return nil
57 | }
58 |
59 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
60 | // For this driver, it does nothing.
61 | func (c *MemoryStore) Close() {}
62 |
63 | // Forget clears the value from the cache for the particular key.
64 | func (c *MemoryStore) Forget(key string) error {
65 | c.cache.Delete(key)
66 | return nil
67 | }
68 |
69 | // ForgetAll clears all values from the cache.
70 | func (c *MemoryStore) ForgetAll() error {
71 | c.cache.Flush()
72 | return nil
73 | }
74 |
--------------------------------------------------------------------------------
/redis/redis_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package redis_test
4 |
5 | import (
6 | "context"
7 | "testing"
8 | "time"
9 |
10 | "github.com/alicebob/miniredis/v2"
11 | "github.com/gomodule/redigo/redis"
12 | "github.com/rocketlaunchr/remember-go"
13 | red "github.com/rocketlaunchr/remember-go/redis"
14 | )
15 |
16 | var ctx = context.Background()
17 |
18 | func TestKeyBasicOperation(t *testing.T) {
19 | s, err := miniredis.Run()
20 | if err != nil {
21 | panic(err)
22 | }
23 | defer s.Close()
24 |
25 | var rs = red.NewRedisStore(&redis.Pool{
26 | Dial: func() (redis.Conn, error) {
27 | return redis.Dial("tcp", s.Addr())
28 | },
29 | })
30 |
31 | key := "key"
32 | exp := 10 * time.Minute
33 |
34 | slowQuery := func(ctx context.Context) (interface{}, error) {
35 | return "val", nil
36 | }
37 |
38 | actual, _, _ := remember.Cache(ctx, rs, key, exp, slowQuery)
39 |
40 | expected := "val"
41 |
42 | if actual.(string) != expected {
43 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
44 | }
45 | }
46 |
47 | func TestFetchFromCacheAndDisableCache(t *testing.T) {
48 | s, err := miniredis.Run()
49 | if err != nil {
50 | panic(err)
51 | }
52 | defer s.Close()
53 |
54 | var rs = red.NewRedisStore(&redis.Pool{
55 | Dial: func() (redis.Conn, error) {
56 | return redis.Dial("tcp", s.Addr())
57 | },
58 | })
59 |
60 | key := "key"
61 | exp := 10 * time.Minute
62 |
63 | slowQuery := func(ctx context.Context) (interface{}, error) {
64 | return "val", nil
65 | }
66 |
67 | // warm up cache
68 | remember.Cache(ctx, rs, key, exp, slowQuery)
69 |
70 | // This time fetch from cache
71 | actual, _, _ := remember.Cache(ctx, rs, key, exp, slowQuery)
72 |
73 | expected := "val"
74 |
75 | if actual.(string) != expected {
76 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
77 | }
78 |
79 | // Actual is now "val", Let's change it to "val2" and disable cache usage.
80 |
81 | slowQuery = func(ctx context.Context) (interface{}, error) {
82 | return "val2", nil
83 | }
84 |
85 | actual, _, _ = remember.Cache(ctx, rs, key, exp, slowQuery, remember.Options{DisableCacheUsage: true})
86 |
87 | expected = "val2"
88 |
89 | if actual.(string) != expected {
90 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/options.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package remember
4 |
5 | import (
6 | "context"
7 | "time"
8 | )
9 |
10 | // Options is used to change caching behavior.
11 | type Options struct {
12 |
13 | // DisableCacheUsage disables the cache.
14 | // It can be useful during debugging.
15 | DisableCacheUsage bool
16 |
17 | // UseFreshData will ignore content in the cache and always pull fresh data.
18 | // The pulled data will subsequently be saved in the cache.
19 | UseFreshData bool
20 |
21 | // Logger, when set, will log error and debug messages.
22 | Logger Logger
23 |
24 | // OnlyLogErrors, when set, will only log errors (but not debug messages).
25 | // For production, this should be set to true.
26 | OnlyLogErrors bool
27 |
28 | // GobRegister registers with the gob encoder the data type returned by the
29 | // SlowRetrieve function.
30 | // Some storage drivers may require this to be set.
31 | // Setting this to true will slightly impact concurrency performance.
32 | // It is usually better to set this to false, but register all structs
33 | // inside an init(). Otherwise you will encounter complaints from the gob package
34 | // if a Logger is provided.
35 | // See: https://golang.org/pkg/encoding/gob/#Register
36 | GobRegister bool
37 | }
38 |
39 | // SlowRetrieve obtains a result when the key is not found in the cache.
40 | // It is usually (but not limited to) a query to a database with some additional
41 | // processing of the returned data. The function must return a value that is compatible
42 | // with the gob package for some storage drivers.
43 | type SlowRetrieve func(ctx context.Context) (interface{}, error)
44 |
45 | // Conner allows a storage driver to provide a connection from the pool
46 | // in order to communicate with it.
47 | type Conner interface {
48 | Conn(ctx context.Context) (Cacher, error)
49 | }
50 |
51 | // Cacher is the interface that all storage drivers must implement.
52 | type Cacher interface {
53 | // StorePointer sets whether a storage driver requires itemToStore to be
54 | // stored as a pointer or as a concrete value.
55 | StorePointer() bool
56 |
57 | // Get returns a value from the cache if the key exists.
58 | Get(key string) (item interface{}, found bool, err error)
59 |
60 | // Set sets a item into the cache for a particular key.
61 | Set(key string, expiration time.Duration, itemToStore interface{}) error
62 |
63 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
64 | Close()
65 |
66 | // Forget clears the value from the cache for the particular key.
67 | Forget(key string) error
68 |
69 | // ForgetAll clears all values from the cache.
70 | ForgetAll() error
71 | }
72 |
--------------------------------------------------------------------------------
/memcached/memcached.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package memcached
4 |
5 | import (
6 | "bytes"
7 | "context"
8 | "encoding/gob"
9 | "time"
10 |
11 | "github.com/bradfitz/gomemcache/memcache"
12 | "github.com/rocketlaunchr/remember-go"
13 | )
14 |
15 | // MemcachedStore is used to create a memcached-backed cache.
16 | type MemcachedStore struct {
17 | client *memcache.Client
18 | }
19 |
20 | // NewMemcachedStore creates a memcached-backed cache.
21 | func NewMemcachedStore(server ...string) *MemcachedStore {
22 | return &MemcachedStore{
23 | client: memcache.New(server...),
24 | }
25 | }
26 |
27 | // NewMemachedStoreFromSelector creates a memcached-backed cache.
28 | func NewMemachedStoreFromSelector(ss memcache.ServerSelector) *MemcachedStore {
29 | return &MemcachedStore{
30 | client: memcache.NewFromSelector(ss),
31 | }
32 | }
33 |
34 | // Conn does nothing for this storage driver.
35 | func (c *MemcachedStore) Conn(ctx context.Context) (remember.Cacher, error) {
36 | return c, nil
37 | }
38 |
39 | // StorePointer sets whether a storage driver requires itemToStore to be
40 | // stored as a pointer or as a concrete value.
41 | func (c *MemcachedStore) StorePointer() bool {
42 | return false // Not sure if this should be true or false. Try with both?
43 | }
44 |
45 | // Get retrieves a value from the cache. The key must be at most 250 bytes in length.
46 | func (c *MemcachedStore) Get(key string) (_ interface{}, found bool, _ error) {
47 |
48 | item, err := c.client.Get(key)
49 | if err != nil {
50 | if err == memcache.ErrCacheMiss {
51 | return nil, false, nil
52 | }
53 | return nil, false, err
54 | }
55 |
56 | var output interface{}
57 |
58 | err = gob.NewDecoder(bytes.NewBuffer(item.Value)).Decode(&output)
59 | if err != nil {
60 | return nil, true, err // Could not decode cached data
61 | }
62 |
63 | return output, true, nil
64 | }
65 |
66 | // Set stores a value in the cache. The key must be at most 250 bytes in length.
67 | func (c *MemcachedStore) Set(key string, expiration time.Duration, itemToStore interface{}) error {
68 |
69 | var exp int32
70 |
71 | if expiration != 0 {
72 | exp = int32(time.Now().Add(expiration).Unix())
73 | }
74 |
75 | // Convert item to bytes
76 | b := new(bytes.Buffer)
77 | err := gob.NewEncoder(b).Encode(itemToStore)
78 | if err != nil {
79 | return err
80 | }
81 |
82 | item := &memcache.Item{
83 | Key: key,
84 | Expiration: exp,
85 | Value: b.Bytes(),
86 | }
87 |
88 | return c.client.Set(item)
89 | }
90 |
91 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
92 | func (c *MemcachedStore) Close() {}
93 |
94 | // Forget clears the value from the cache for the particular key.
95 | func (c *MemcachedStore) Forget(key string) error {
96 | return c.client.Delete(key)
97 | }
98 |
99 | // ForgetAll clears all values from the cache.
100 | func (c *MemcachedStore) ForgetAll() error {
101 | return c.client.DeleteAll()
102 | }
103 |
--------------------------------------------------------------------------------
/key.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package remember
4 |
5 | import (
6 | "bytes"
7 | "encoding/json"
8 | "fmt"
9 | "hash/crc32"
10 | "reflect"
11 | "runtime"
12 | "strings"
13 | )
14 |
15 | // CreateKey will generate a key based on the input arguments.
16 | // When prefix is true, the caller's name will be used to prefix the key in an attempt to make it unique.
17 | // The args can also be separated using sep. visual performs no functionality. It is used at code level
18 | // to visually see how the key is structured.
19 | func CreateKey(prefix bool, sep string, visual string, args ...interface{}) string {
20 | var output string
21 |
22 | if prefix {
23 | pc, file, line, ok := runtime.Caller(1)
24 | if !ok {
25 | return fmt.Sprint(args...)
26 | }
27 | details := runtime.FuncForPC(pc)
28 | output = fmt.Sprintf("%s_%s_%d_", details.Name(), file, line)
29 | }
30 |
31 | if sep == "" {
32 | output = output + fmt.Sprint(args...)
33 | } else {
34 | for i, v := range args {
35 | if i != 0 {
36 | output = output + sep
37 | }
38 | output = output + fmt.Sprint(v)
39 | }
40 | }
41 |
42 | return output
43 | }
44 |
45 | // Hash returns a crc32 hashed version of key.
46 | func Hash(key string) string {
47 | return fmt.Sprintf("%08x", crc32.ChecksumIEEE([]byte(key)))
48 | }
49 |
50 | // CreateKeyStruct generates a key by converting a struct into a JSON object.
51 | func CreateKeyStruct(strct interface{}) string {
52 | out := map[string]interface{}{}
53 |
54 | // Encode nil immediately
55 | if strct == nil {
56 | return ""
57 | }
58 |
59 | s := reflect.ValueOf(strct)
60 |
61 | // Check if s is a pointer
62 | if s.Kind() == reflect.Ptr {
63 | s = reflect.Indirect(s)
64 | }
65 | typeOfT := s.Type()
66 |
67 | for i := 0; i < s.NumField(); i++ {
68 | f := typeOfT.Field(i)
69 |
70 | if f.PkgPath != "" {
71 | // Not exported
72 | continue
73 | }
74 |
75 | fieldName := typeOfT.Field(i).Name
76 | fieldTag := f.Tag.Get("json")
77 | fieldValRaw := s.Field(i)
78 | fieldVal := fieldValRaw.Interface()
79 |
80 | // Ignore slices
81 | if fieldValRaw.Kind() == reflect.Slice {
82 | continue
83 | }
84 |
85 | // Check if json parser would ordinarily hide the value anyway
86 | if fieldTag == "-" || (strings.HasSuffix(fieldTag, ",omitempty") && reflect.DeepEqual(fieldVal, reflect.Zero(reflect.TypeOf(fieldVal)).Interface())) {
87 | continue
88 | }
89 |
90 | if fieldTag == "" {
91 | out[fieldName] = fieldVal
92 | } else {
93 | out[strings.TrimSuffix(fieldTag, ",omitempty")] = fieldVal
94 | }
95 | }
96 |
97 | b, _ := json.Marshal(out)
98 | str, _ := compactJson(b)
99 |
100 | return str
101 | }
102 |
103 | // compactJson will remove insignificant spaces to minimize storage space.
104 | func compactJson(src []byte) (string, error) {
105 |
106 | dst := new(bytes.Buffer)
107 |
108 | err := json.Compact(dst, src)
109 | if err != nil {
110 | return "", err
111 | }
112 |
113 | return dst.String(), nil
114 | }
115 |
--------------------------------------------------------------------------------
/redis/redis.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package redis
4 |
5 | import (
6 | "bytes"
7 | "context"
8 | "encoding/gob"
9 | "time"
10 |
11 | "github.com/gomodule/redigo/redis"
12 | "github.com/rocketlaunchr/remember-go"
13 | )
14 |
15 | // NoExpiration is used to indicate that data should not expire from the cache.
16 | const NoExpiration time.Duration = -1
17 |
18 | // RedisStore is used to create a redis-backed cache.
19 | type RedisStore struct {
20 | Pool *redis.Pool
21 | }
22 |
23 | // NewRedisStore creates a redis-backed cache directly from a redis
24 | // pool object.
25 | func NewRedisStore(redisPool *redis.Pool) *RedisStore {
26 | return &RedisStore{
27 | Pool: redisPool,
28 | }
29 | }
30 |
31 | // Conn will provide a single redis connection from the redis connection pool.
32 | func (c *RedisStore) Conn(ctx context.Context) (remember.Cacher, error) {
33 |
34 | conn, err := c.Pool.GetContext(ctx)
35 | if err != nil {
36 | return nil, err
37 | }
38 |
39 | return &RedisConn{
40 | conn: conn,
41 | }, nil
42 | }
43 |
44 | // RedisConn represents a single connection to the redis pool.
45 | type RedisConn struct {
46 | conn redis.Conn
47 | }
48 |
49 | // StorePointer sets whether a storage driver requires itemToStore to be
50 | // stored as a pointer or as a concrete value.
51 | func (c *RedisConn) StorePointer() bool {
52 | return true
53 | }
54 |
55 | // Get returns a value from the cache if the key exists.
56 | func (c *RedisConn) Get(key string) (_ interface{}, found bool, _ error) {
57 |
58 | val, err := redis.Bytes(c.conn.Do("GET", key))
59 | if err != nil {
60 | if err == redis.ErrNil {
61 | // Key not found
62 | return nil, false, nil
63 | }
64 | return nil, false, err
65 | }
66 |
67 | var output interface{}
68 |
69 | err = gob.NewDecoder(bytes.NewBuffer(val)).Decode(&output)
70 | if err != nil {
71 | return nil, true, err // Could not decode cached data
72 | }
73 |
74 | return output, true, nil
75 | }
76 |
77 | // Set sets a item into the cache for a particular key.
78 | func (c *RedisConn) Set(key string, expiration time.Duration, itemToStore interface{}) error {
79 |
80 | // Convert item to bytes
81 | b := new(bytes.Buffer)
82 | err := gob.NewEncoder(b).Encode(itemToStore)
83 | if err != nil {
84 | return err
85 | }
86 |
87 | if expiration == NoExpiration {
88 | _, err = c.conn.Do("SET", key, b.Bytes())
89 | } else {
90 | _, err = c.conn.Do("SET", key, b.Bytes(), "EX", int(expiration.Seconds()))
91 | }
92 |
93 | return err
94 | }
95 |
96 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
97 | func (c *RedisConn) Close() {
98 | c.conn.Close()
99 | }
100 |
101 | // Forget clears the value from the cache for the particular key.
102 | func (c *RedisConn) Forget(key string) error {
103 | _, err := c.conn.Do("DEL", key)
104 | return err
105 | }
106 |
107 | // ForgetAll clears all values from the cache.
108 | func (c *RedisConn) ForgetAll() error {
109 | _, err := c.conn.Do("FLUSHDB")
110 | return err
111 | }
112 |
--------------------------------------------------------------------------------
/remember_test.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package remember_test
4 |
5 | import (
6 | "context"
7 | "log"
8 | "regexp"
9 | "testing"
10 | "time"
11 |
12 | "github.com/rocketlaunchr/remember-go"
13 | "github.com/rocketlaunchr/remember-go/memory"
14 | )
15 |
16 | type aLogger struct{}
17 |
18 | func (l aLogger) Log(format string, args ...interface{}) {
19 | log.Printf(format, args...)
20 | }
21 |
22 | func TestKeyGen(t *testing.T) {
23 |
24 | actuals := []string{
25 | remember.CreateKey(false, "+", "", 1, 2, 3),
26 | remember.CreateKeyStruct(struct {
27 | Search string
28 | notExported string
29 | Ignored string `json:"-"`
30 | Omit string `json:"xxx"`
31 | Page int
32 | }{"search", "z", "y", "ppp", 1}),
33 | remember.Hash("crc32-hash"),
34 | remember.CreateKey(true, "", "", 1, 2, 3),
35 | }
36 |
37 | expected := []string{
38 | "1+2+3",
39 | `{"Page":1,"Search":"search","xxx":"ppp"}`,
40 | "40ffd476",
41 | `^github.com/rocketlaunchr/remember-go_test.TestKeyGen_.+?github.com/rocketlaunchr/remember-go/remember_test.go_\d+_1 2 3$`,
42 | }
43 |
44 | for i := range actuals {
45 | actual := actuals[i]
46 |
47 | if i == 3 {
48 | match, _ := regexp.MatchString(expected[i], actual)
49 | if !match {
50 | t.Errorf("wrong val: expected (regex): %v actual: %v", expected[i], actual)
51 | }
52 | } else {
53 | if actual != expected[i] {
54 | t.Errorf("wrong val: expected: %v actual: %v", expected[i], actual)
55 | }
56 | }
57 | }
58 | }
59 |
60 | func TestKeyBasicOperation(t *testing.T) {
61 | ctx := context.Background()
62 | var ms = memory.NewMemoryStore(10 * time.Minute)
63 |
64 | key := "key"
65 | exp := 10 * time.Minute
66 |
67 | slowQuery := func(ctx context.Context) (interface{}, error) {
68 | return "val", nil
69 | }
70 |
71 | actual, _, _ := remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{Logger: aLogger{}, GobRegister: true})
72 |
73 | expected := "val"
74 |
75 | if actual.(string) != expected {
76 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
77 | }
78 | }
79 |
80 | func TestFetchFromCacheAndDisableCache(t *testing.T) {
81 | ctx := context.Background()
82 | var ms = memory.NewMemoryStore(10 * time.Minute)
83 |
84 | key := "key"
85 | exp := 10 * time.Minute
86 |
87 | slowQuery := func(ctx context.Context) (interface{}, error) {
88 | return "val", nil
89 | }
90 |
91 | // warm up cache
92 | remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{Logger: aLogger{}, GobRegister: true})
93 |
94 | // This time fetch from cache
95 | actual, _, _ := remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{Logger: aLogger{}, GobRegister: true})
96 |
97 | expected := "val"
98 |
99 | if actual.(string) != expected {
100 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
101 | }
102 |
103 | // Actual is now "val", Let's change it to "val2" and disable cache usage.
104 |
105 | slowQuery = func(ctx context.Context) (interface{}, error) {
106 | return "val2", nil
107 | }
108 |
109 | actual, _, _ = remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{Logger: aLogger{}, DisableCacheUsage: true})
110 |
111 | expected = "val2"
112 |
113 | if actual.(string) != expected {
114 | t.Errorf("wrong val: expected: %v actual: %v", expected, actual)
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/ristretto/ristretto.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package ristretto
4 |
5 | import (
6 | "context"
7 | "errors"
8 | "time"
9 |
10 | "github.com/dgraph-io/ristretto"
11 | "github.com/rocketlaunchr/remember-go"
12 | )
13 |
14 | // NoExpiration is used to indicate that data should not expire from the cache.
15 | const NoExpiration time.Duration = 0
16 |
17 | // ErrItemDropped signifies that the item to store was not inserted into the cache.
18 | //
19 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Cache.Set
20 | var ErrItemDropped = errors.New("item dropped")
21 |
22 | // RistrettoStore is used to create an in-memory ristretto cache.
23 | //
24 | // See: https://godoc.org/github.com/dgraph-io/ristretto
25 | type RistrettoStore struct {
26 | Cache *ristretto.Cache
27 | DefaultCost *int64
28 | }
29 |
30 | // NewRistrettoStore creates an in-memory ristretto cache.
31 | //
32 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Config
33 | func NewRistrettoStore(config *ristretto.Config, defaultCost ...int64) *RistrettoStore {
34 | cache, err := ristretto.NewCache(config)
35 | if err != nil {
36 | panic(err)
37 | }
38 |
39 | var dc *int64
40 | if len(defaultCost) > 0 {
41 | dc = &defaultCost[0]
42 | }
43 |
44 | return &RistrettoStore{
45 | Cache: cache,
46 | DefaultCost: dc,
47 | }
48 | }
49 |
50 | // Conn does nothing for this storage driver.
51 | func (r *RistrettoStore) Conn(ctx context.Context) (remember.Cacher, error) {
52 | return r, nil
53 | }
54 |
55 | // StorePointer sets whether a storage driver requires itemToStore to be
56 | // stored as a pointer or as a concrete value.
57 | func (r *RistrettoStore) StorePointer() bool {
58 | return false
59 | }
60 |
61 | // Get returns a value from the cache if the key exists.
62 | // It is possible for nil to be returned while found is also true.
63 | //
64 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Cache.Get
65 | func (r *RistrettoStore) Get(key string) (_ interface{}, found bool, _ error) {
66 | item, found := r.Cache.Get(key)
67 | return item, found, nil
68 | }
69 |
70 | // Set sets a item into the cache for a particular key.
71 | // The cost is always set to 1, unless over-ridden at creation.
72 | //
73 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Cache.SetWithTTL
74 | func (r *RistrettoStore) Set(key string, expiration time.Duration, itemToStore interface{}) error {
75 | var (
76 | stored bool
77 | cost int64 = 1
78 | )
79 | if r.DefaultCost != nil {
80 | cost = *r.DefaultCost
81 | }
82 | if expiration == NoExpiration {
83 | stored = r.Cache.Set(key, itemToStore, cost)
84 | } else {
85 | stored = r.Cache.SetWithTTL(key, itemToStore, cost, expiration)
86 | }
87 |
88 | if stored {
89 | return nil
90 | }
91 | return ErrItemDropped
92 | }
93 |
94 | // Close returns the connection back to the pool for storage drivers that utilize a pool.
95 | // For this driver, it does nothing.
96 | func (r *RistrettoStore) Close() {}
97 |
98 | // Forget clears the value from the cache for the particular key.
99 | //
100 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Cache.Del
101 | func (r *RistrettoStore) Forget(key string) error {
102 | r.Cache.Del(key)
103 | return nil
104 | }
105 |
106 | // ForgetAll clears all values from the cache.
107 | // Note that this is not an atomic operation.
108 | //
109 | // See: https://godoc.org/github.com/dgraph-io/ristretto#Cache.Clear
110 | func (r *RistrettoStore) ForgetAll() error {
111 | r.Cache.Clear()
112 | return nil
113 | }
114 |
--------------------------------------------------------------------------------
/remember.go:
--------------------------------------------------------------------------------
1 | // Copyright 2018-21 PJ Engineering and Business Solutions Pty. Ltd. All rights reserved.
2 |
3 | package remember
4 |
5 | import (
6 | "context"
7 | "encoding/gob"
8 | "fmt"
9 | "log"
10 | "time"
11 | )
12 |
13 | // Cache is used to return a cached value. If it's not available, fn will be called to obtain a value.
14 | // Subsequently, fn's value will be saved into the cache.
15 | func Cache(ctx context.Context, c Conner, key string, expiration time.Duration, fn SlowRetrieve, options ...Options) (_ interface{}, found bool, _ error) {
16 | var (
17 | disableCache bool
18 | fresh bool
19 | logger Logger
20 | gobRegister bool
21 | onlyLogErrors bool
22 | )
23 |
24 | if options != nil {
25 | disableCache = options[0].DisableCacheUsage
26 | fresh = options[0].UseFreshData
27 | logger = options[0].Logger
28 | gobRegister = options[0].GobRegister
29 | onlyLogErrors = options[0].OnlyLogErrors
30 | }
31 |
32 | // Check if cache has been disabled
33 | if disableCache {
34 | if logger != nil && !onlyLogErrors {
35 | logger.Log(logPatternBlue, "[cache disabled] Grabbing from SlowRetrieve key: "+key)
36 | }
37 |
38 | out, err := fn(ctx)
39 | if err != nil {
40 | if logger != nil && !onlyLogErrors {
41 | logger.Log(logPatternBlue, "[cache disabled] Grabbing (cache disabled) from SlowRetrieve key: "+key+" error: "+err.Error())
42 | }
43 | return nil, false, err
44 | }
45 | return out, false, nil
46 | }
47 |
48 | // Obtain cache connection
49 | cache, err := c.Conn(ctx)
50 | if err != nil {
51 | if logger != nil {
52 | logger.Log(logPatternRed, "could not obtain connection for cache")
53 | }
54 | return nil, false, err
55 | }
56 | defer cache.Close()
57 |
58 | var item interface{}
59 |
60 | if fresh {
61 | if logger != nil && !onlyLogErrors {
62 | logger.Log(logPatternBlue, "Grabbing (fresh) from SlowRetrieve key: "+key)
63 | }
64 | goto fresh
65 | }
66 |
67 | // Check if item exists
68 | item, found, err = cache.Get(key)
69 | if err != nil {
70 | // Error when attempting to fetch from cache
71 | if logger != nil {
72 | logger.Log(logPatternRed, "could not fetch from cache key: "+key+" error: "+err.Error())
73 | }
74 | }
75 |
76 | if found && err == nil {
77 | // Item exists in cache
78 | if logger != nil && !onlyLogErrors {
79 | logger.Log(logPatternBlue, "Found in Cache key: "+key)
80 | }
81 | return item, true, nil
82 | }
83 |
84 | if logger != nil && !onlyLogErrors {
85 | logger.Log(logPatternBlue, "Grabbing from SlowRetrieve key: "+key)
86 | }
87 |
88 | fresh:
89 | // Item does not exist in cache so grab it from the fn
90 | itemToStore, err := fn(ctx)
91 | if err != nil {
92 | return nil, false, err
93 | }
94 |
95 | if gobRegister {
96 | func() {
97 | defer func() {
98 | if err := recover(); err != nil {
99 | msg := fmt.Sprintf("gob register: %v", err)
100 | if logger != nil {
101 | logger.Log(logPatternRed, msg)
102 | } else {
103 | log.Printf(logPatternRed, msg)
104 | }
105 | }
106 | }()
107 | gob.Register(itemToStore)
108 | }()
109 | }
110 |
111 | // Store item in Cache
112 | if cache.StorePointer() {
113 | err = cache.Set(key, expiration, &itemToStore)
114 | } else {
115 | err = cache.Set(key, expiration, itemToStore)
116 | }
117 | if err != nil {
118 | // Storage failed
119 | if logger != nil {
120 | logger.Log(logPatternRed, "Could not store item to key: "+key+" "+err.Error()+" "+fmt.Sprintf("%+v", itemToStore))
121 | }
122 | }
123 |
124 | return itemToStore, false, nil
125 | }
126 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | ⭐ the project to show your appreciation. :arrow_upper_right:
3 |
4 |
5 |
6 | 
7 | 
8 | 
9 |
10 |
11 |
12 | 
13 |
14 |
15 | # Cache Slow Database Queries
16 |
17 | This package is used to cache the results of slow database queries in memory or Redis.
18 | It can be used to cache any form of data (eg. function memoization). A Redis and in-memory storage driver is provided.
19 |
20 | See [Article](https://medium.com/@rocketlaunchr.cloud/caching-slow-database-queries-1085d308a0c9) for further details including a tutorial.
21 |
22 | The package is **production ready** and the API is stable. A variant of this package has been used in production for over 4 years.
23 |
24 | ## Installation
25 |
26 | ```
27 | go get -u github.com/rocketlaunchr/remember-go
28 | ```
29 |
30 | ## Create a Key
31 |
32 | Let’s assume the query’s argument is an arbitrary `search` term and a `page` number for pagination.
33 |
34 | ### CreateKeyStruct
35 |
36 | CreateKeyStruct can generate a JSON based key by providing a struct.
37 |
38 | ```go
39 | type Key struct {
40 | Search string
41 | Page int `json:"page"`
42 | }
43 |
44 | var key string = remember.CreateKeyStruct(Key{"golang", 2})
45 | ```
46 |
47 | ### CreateKey
48 |
49 | CreateKey provides more flexibility to generate keys:
50 |
51 | ```go
52 | // Key will be "search-golang-2"
53 | key := remember.CreateKey(false, "-", "search-x-y", "search", "golang", 2)
54 | ```
55 |
56 | ## Initialize the Storage Driver
57 |
58 | ### In-Memory
59 |
60 | ```go
61 | import "github.com/rocketlaunchr/remember-go/memory"
62 |
63 | var ms = memory.NewMemoryStore(10 * time.Minute)
64 | ```
65 |
66 | ### Redis
67 |
68 | The Redis storage driver relies on Gary Burd’s excellent [Redis client library](https://github.com/gomodule/redigo/).
69 |
70 | ```go
71 | import red "github.com/rocketlaunchr/remember-go/redis"
72 | import "github.com/gomodule/redigo/redis"
73 |
74 | var rs = red.NewRedisStore(&redis.Pool{
75 | Dial: func() (redis.Conn, error) {
76 | return redis.Dial("tcp", "localhost:6379")
77 | },
78 | })
79 | ```
80 |
81 | ### Memcached
82 |
83 | An experimental (and untested) memcached driver is provided.
84 | It relies on Brad Fitzpatrick's [memcache driver](https://godoc.org/github.com/bradfitz/gomemcache/memcache).
85 |
86 | ### Ristretto
87 |
88 | DGraph's [Ristretto](https://github.com/dgraph-io/ristretto) is a fast, fixed size, in-memory cache with a dual focus on throughput and hit ratio performance.
89 |
90 | ### Nocache
91 |
92 | This driver is for testing purposes. It does not cache any data.
93 |
94 | ## Create a SlowRetrieve Function
95 |
96 | The package initially checks if data exists in the cache. If it doesn’t, then it elegantly fetches the data directly from the database by calling the `SlowRetrieve` function. It then saves the data into the cache so that next time it doesn’t have to refetch it from the database.
97 |
98 | ```go
99 | type Result struct {
100 | Title string
101 | }
102 |
103 | slowQuery := func(ctx context.Context) (interface{}, error) {
104 | results := []Result{}
105 |
106 | stmt := `
107 | SELECT title
108 | FROM books WHERE title LIKE ?
109 | ORDER BY title LIMIT ?, 20
110 | `
111 |
112 | rows, _ := db.QueryContext(ctx, stmt, search, (page-1)*20)
113 |
114 | for rows.Next() {
115 | var title string
116 | rows.Scan(&title)
117 | results = append(results, Result{title})
118 | }
119 |
120 | return results, nil
121 | }
122 | ```
123 |
124 | ## Usage
125 |
126 | ```go
127 | key := remember.CreateKeyStruct(Key{"golang", 2})
128 | exp := 10*time.Minute
129 |
130 | results, found, err := remember.Cache(ctx, ms, key, exp, slowQuery, remember.Options{GobRegister: false})
131 |
132 | return results.([]Result) // Type assert in order to use
133 | ```
134 |
135 | ## Gob Register Errors
136 |
137 | The Redis storage driver stores the data in a `gob` encoded form. You have to register with the [`gob`](https://golang.org/pkg/encoding/gob/) package the data type returned by the `SlowRetrieve` function. It can be done inside a `func init()`. Alternatively, you can set the `GobRegister` option to true. This will impact concurrency performance and is thus **not recommended**.
138 |
139 | ## Other useful packages
140 |
141 | - [awesome-svelte](https://github.com/rocketlaunchr/awesome-svelte) - Resources for killing react
142 | - [dataframe-go](https://github.com/rocketlaunchr/dataframe-go) - For statistics, machine-learning, and data manipulation/exploration
143 | - [dbq](https://github.com/rocketlaunchr/dbq) - Zero boilerplate database operations for Go
144 | - [electron-alert](https://github.com/rocketlaunchr/electron-alert) - SweetAlert2 for Electron Applications
145 | - [google-search](https://github.com/rocketlaunchr/google-search) - Scrape google search results
146 | - [igo](https://github.com/rocketlaunchr/igo) - A Go transpiler with cool new syntax such as fordefer (defer for for-loops)
147 | - [mysql-go](https://github.com/rocketlaunchr/mysql-go) - Properly cancel slow MySQL queries
148 | - [react](https://github.com/rocketlaunchr/react) - Build front end applications using Go
149 | - [testing-go](https://github.com/rocketlaunchr/testing-go) - Testing framework for unit testing
150 |
151 | #
152 |
153 | ### Legal Information
154 |
155 | The license is a modified MIT license. Refer to `LICENSE` file for more details.
156 |
157 | **© 2019-21 PJ Engineering and Business Solutions Pty. Ltd.**
158 |
159 | ### Final Notes
160 |
161 | Feel free to enhance features by issuing pull-requests.
162 |
--------------------------------------------------------------------------------