├── .gitignore ├── fixtures ├── bench_bbolt.db └── bench_duramap.db ├── .travis.yml ├── Makefile ├── go.mod ├── LICENSE.txt ├── go.sum ├── README.md ├── duramap.go └── duramap_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | fixtures/* 2 | !fixtures/.gitkeep 3 | coverage.txt -------------------------------------------------------------------------------- /fixtures/bench_bbolt.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/notduncansmith/duramap/HEAD/fixtures/bench_bbolt.db -------------------------------------------------------------------------------- /fixtures/bench_duramap.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/notduncansmith/duramap/HEAD/fixtures/bench_duramap.db -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | dist: xenial 2 | 3 | language: go 4 | 5 | go: 6 | - 1.13.x 7 | 8 | install: make build 9 | script: make test 10 | 11 | after_success: 12 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | go build 3 | testunit: 4 | go test -v -coverprofile=coverage.txt 5 | bench: 6 | go test -bench . 7 | testrace: 8 | go test -v -race 9 | test: testunit testrace 10 | all: build test 11 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/notduncansmith/duramap 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/kr/pretty v0.1.0 // indirect 7 | github.com/notduncansmith/mutable v0.0.0-20191105072558-a13a78d07b91 8 | github.com/vmihailenco/msgpack v4.0.4+incompatible 9 | go.etcd.io/bbolt v1.3.3 10 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 11 | golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd // indirect 12 | google.golang.org/appengine v1.6.5 // indirect 13 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2019 Duncan Smith 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 2 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 3 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 4 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 5 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 6 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 7 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 8 | github.com/notduncansmith/mutable v0.0.0-20191105072558-a13a78d07b91 h1:B4WiScuDn9FK9MEG4rQ2q+AnR/uPgnEQezml2rbYE9A= 9 | github.com/notduncansmith/mutable v0.0.0-20191105072558-a13a78d07b91/go.mod h1:FSP687EO4iKB5iYam28rPwVr5LYf+SACbK0N1D6aFC4= 10 | github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI= 11 | github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= 12 | go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= 13 | go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 14 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 15 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 16 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= 17 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 18 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 19 | golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII= 20 | golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 22 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 23 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 24 | google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= 25 | google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 26 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 27 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 28 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Duramap 2 | 3 | [![GoDoc](https://godoc.org/github.com/notduncansmith/duramap?status.svg)](https://godoc.org/github.com/notduncansmith/duramap) [![Build Status](https://travis-ci.com/notduncansmith/duramap.svg?branch=master)](https://travis-ci.com/notduncansmith/duramap) [![codecov](https://codecov.io/gh/notduncansmith/duramap/branch/master/graph/badge.svg)](https://codecov.io/gh/notduncansmith/duramap) 4 | 5 | Duramap wraps the speed of a `map[string]interface{}` with the safety of a [`sync.RWMutex`](https://golang.org/pkg/sync/#RWMutex) and the durability of [`bbolt`](https://github.com/etcd-io/bbolt) (effectively, it is an always-fully-loaded write-through cache). It is intended to be a reliable thread-safe store of mutable data with fast read requirements. 6 | 7 | The internal map cuts most of the cost (~18ms on my machine) of accessing K/V items through BoltDB directly, while serialization of map values adds minimal overhead to writes (values are encoded with [`vmihailenco/msgpack`](https://github.com/vmihailenco/msgpack)). See Benchmarks below for more details. 8 | 9 | ### New and improved! 10 | 11 | The old Duramap used to serialize the entire map contents with MsgPack and store this in a single key, with mutation happening directly to the map passed to `UpdateMap`. Now, the new Duramap now serializes individual key values and writes each to their own corresponding key in bbolt, saving drastically on write overhead for non-tiny maps. Updates now happen through a new `Tx` struct. 12 | 13 | ## Usage 14 | 15 | See [GoDoc](https://godoc.org/github.com/notduncansmith/duramap) for full docs. Example: 16 | 17 | ```go 18 | package mypackage 19 | 20 | import ( 21 | "testing" 22 | 23 | "github.com/notduncansmith/duramap" 24 | ) 25 | 26 | func TestRoundtrip(t *testing.T) { 27 | dm, err := duramap.NewDuramap("./example.db", "example") 28 | 29 | if err != nil { 30 | t.Errorf("Should be able to open database: %v", err) 31 | return 32 | } 33 | 34 | if err = dm.Load(); err != nil { 35 | t.Errorf("Should be able to load map: %v", err) 36 | return 37 | } 38 | 39 | err = dm.UpdateMap(func(tx *Tx) error { 40 | tx.Set("foo", "bar") 41 | if tx.Get("foo") != "bar" { 42 | t.Error("Should be able to read saved value") 43 | } 44 | return nil 45 | }) 46 | 47 | if err != nil { 48 | t.Errorf("Should be able to save value: %v", err) 49 | } 50 | 51 | foo := dm.WithMap(func(m GenericMap) interface{} { 52 | // this is the internal map, do not mutate it! 53 | return m["foo"] 54 | }).(string) 55 | 56 | if foo != "bar" { 57 | t.Error("Should be able to read saved value") 58 | return 59 | } 60 | } 61 | ``` 62 | 63 | ## Benchmarks 64 | 65 | ```sh 66 | goos: darwin 67 | goarch: amd64 68 | pkg: github.com/notduncansmith/duramap 69 | BenchmarkReadsDuramap/int64-12 17755630 57.6 ns/op 70 | BenchmarkReadsDuramap/str64b-12 18237058 56.9 ns/op 71 | BenchmarkReadsDuramap/str128b-12 19252372 56.2 ns/op 72 | BenchmarkReadsDuramap/str256b-12 20408672 54.7 ns/op 73 | BenchmarkWritesDuramap/int64-12 63 19229244 ns/op 74 | BenchmarkWritesDuramap/str64b-12 66 18821763 ns/op 75 | BenchmarkWritesDuramap/str128b-12 63 19051516 ns/op 76 | BenchmarkWritesDuramap/str256b-12 63 18178327 ns/op 77 | BenchmarkReadsBbolt/str256b-12 64 18297425 ns/op 78 | BenchmarkWritesBbolt/str256b-12 63 19110888 ns/op 79 | BenchmarkReadsRWMap/rwmutex-12 75495019 15.7 ns/op 80 | BenchmarkWritesRWMap/rwmutex-12 34192036 34.3 ns/op 81 | BenchmarkReadsMutableMap/mutable-12 29584789 39.0 ns/op 82 | BenchmarkWritesMutableMap/mutable-12 21703112 54.3 ns/op 83 | ``` 84 | 85 | ## License 86 | 87 | Released under [The MIT License](https://opensource.org/licenses/MIT) (see `LICENSE.txt`). 88 | 89 | Copyright 2019 Duncan Smith -------------------------------------------------------------------------------- /duramap.go: -------------------------------------------------------------------------------- 1 | package duramap 2 | 3 | import ( 4 | "crypto/rand" 5 | "errors" 6 | "io" 7 | "sync" 8 | 9 | "golang.org/x/crypto/nacl/secretbox" 10 | 11 | "github.com/notduncansmith/mutable" 12 | mp "github.com/vmihailenco/msgpack" 13 | bolt "go.etcd.io/bbolt" 14 | ) 15 | 16 | // GenericMap is a map of strings to empty interfaces 17 | type GenericMap = map[string]interface{} 18 | 19 | // EncryptionSecret is a 32-byte secret key used to encrypt data with the Secretbox construction 20 | type EncryptionSecret = *[32]byte 21 | 22 | // Tx is a transaction that operates on a Duramap. It has a reference to the internal map. 23 | type Tx struct { 24 | M *GenericMap 25 | writes GenericMap 26 | } 27 | 28 | // Get returns the latest value either written in the Tx or stored in the map 29 | func (tx *Tx) Get(k string) interface{} { 30 | written := tx.writes[k] 31 | if written != nil { 32 | return written 33 | } 34 | return (*tx.M)[k] 35 | } 36 | 37 | // Set writes a new value in the Tx 38 | func (tx *Tx) Set(k string, v interface{}) { 39 | tx.writes[k] = v 40 | } 41 | 42 | // Duramap is a map of strings to ambiguous data structures, which is protected by a Read/Write mutex and backed by a bbolt database 43 | type Duramap struct { 44 | mut *sync.RWMutex 45 | db *bolt.DB 46 | encryptionSecret EncryptionSecret 47 | m GenericMap 48 | Path string 49 | Name string 50 | } 51 | 52 | var dbs = map[string]*bolt.DB{} 53 | var dbsRW = mutable.NewRW("dbs") 54 | 55 | var dms = map[string]*Duramap{} 56 | var dmsRW = mutable.NewRW("dms") 57 | 58 | // NewDuramap returns a new Duramap backed by a bbolt database located at `path` 59 | func NewDuramap(path, name string, secret EncryptionSecret) (*Duramap, error) { 60 | dm := dmsRW.WithRLock(func() interface{} { 61 | return unsafeGetDM(path + ":" + name) 62 | }).(*Duramap) 63 | 64 | if dm != nil { 65 | return dm, nil 66 | } 67 | 68 | var err error 69 | var db *bolt.DB 70 | dbsRW.DoWithRWLock(func() { 71 | if dbs[path] != nil { 72 | db = dbs[path] 73 | return 74 | } 75 | db, err = bolt.Open(path, 0600, nil) 76 | if err == nil { 77 | dbs[path] = db 78 | } 79 | }) 80 | 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | mut := &sync.RWMutex{} 86 | 87 | dm = &Duramap{mut, db, secret, GenericMap{}, path, name} 88 | dmsRW.DoWithRWLock(func() { 89 | unsafeAddDM(dm) 90 | }) 91 | return dm, nil 92 | } 93 | 94 | // Load reads the stored Duramap value and sets it 95 | func (dm *Duramap) Load() error { 96 | if err := dm.db.Update(func(tx *bolt.Tx) error { 97 | _, err := tx.CreateBucketIfNotExists([]byte(dm.Name)) 98 | return err 99 | }); err != nil { 100 | return err 101 | } 102 | 103 | return dm.db.View(func(tx *bolt.Tx) error { 104 | m := GenericMap{} 105 | b := tx.Bucket([]byte(dm.Name)) 106 | c := b.Cursor() 107 | 108 | for k, bz := c.First(); k != nil; k, bz = c.Next() { 109 | v := map[string]interface{}{} 110 | if bz == nil || len(bz) == 0 { 111 | continue 112 | } 113 | if dm.encryptionSecret != nil { 114 | clearbz, err := decrypt(dm.encryptionSecret, bz) 115 | if err != nil { 116 | return err 117 | } 118 | bz = clearbz 119 | } 120 | if err := mp.Unmarshal(bz, &v); err != nil { 121 | return err 122 | } 123 | m[string(k)] = v["."] 124 | } 125 | 126 | dm.mut.Lock() 127 | defer dm.mut.Unlock() 128 | dm.m = m 129 | 130 | return nil 131 | }) 132 | } 133 | 134 | // UpdateMap is like `DoWithMap` but the result of `f` is re-saved afterwards 135 | func (dm *Duramap) UpdateMap(f func(tx *Tx) error) error { 136 | defer dm.mut.Unlock() 137 | dm.mut.Lock() 138 | tx := dm.tx() 139 | if err := f(tx); err != nil { 140 | return err 141 | } 142 | bzWrites := map[string][]byte{} 143 | for k, v := range tx.writes { 144 | bz, err := mp.Marshal(map[string]interface{}{".": v}) 145 | if err != nil { 146 | return err 147 | } 148 | if dm.encryptionSecret == nil { 149 | bzWrites[k] = bz 150 | continue 151 | } 152 | cypherbz, err := encrypt(dm.encryptionSecret, bz) 153 | if err != nil { 154 | return err 155 | } 156 | bzWrites[k] = cypherbz 157 | } 158 | err := dm.db.Update(func(btx *bolt.Tx) error { 159 | b, _ := btx.CreateBucketIfNotExists([]byte(dm.Name)) 160 | for k, vbz := range bzWrites { 161 | if err := b.Put([]byte(k), vbz); err != nil { 162 | return err 163 | } 164 | } 165 | return nil 166 | }) 167 | 168 | if err != nil { 169 | return err 170 | } 171 | 172 | for k, v := range tx.writes { 173 | dm.m[k] = v 174 | } 175 | 176 | return nil 177 | } 178 | 179 | // WithMap returns the result of calling `f` with the internal map 180 | func (dm *Duramap) WithMap(f func(m GenericMap) interface{}) interface{} { 181 | defer dm.mut.RUnlock() 182 | dm.mut.RLock() 183 | return f(dm.m) 184 | } 185 | 186 | // DoWithMap is like `WithMap` but does not return a result 187 | func (dm *Duramap) DoWithMap(f func(m GenericMap)) { 188 | defer dm.mut.RUnlock() 189 | dm.mut.RLock() 190 | f(dm.m) 191 | } 192 | 193 | // Truncate will reset the contents of the Duramap to an empty GenericMap 194 | func (dm *Duramap) Truncate() error { 195 | dm.mut.Lock() 196 | defer dm.mut.Unlock() 197 | err := dm.db.Update(func(tx *bolt.Tx) error { 198 | err := tx.DeleteBucket([]byte(dm.Name)) 199 | if err != nil { 200 | return err 201 | } 202 | _, err = tx.CreateBucket([]byte(dm.Name)) 203 | return err 204 | }) 205 | if err != nil { 206 | return err 207 | } 208 | dm.m = GenericMap{} 209 | return nil 210 | } 211 | 212 | func (dm *Duramap) tx() *Tx { 213 | return &Tx{&dm.m, GenericMap{}} 214 | } 215 | 216 | func unsafeAddDM(dm *Duramap) { 217 | k := dm.Path + ":" + dm.Name 218 | if dms[k] == nil { 219 | dms[k] = dm 220 | } 221 | } 222 | 223 | func unsafeGetDM(k string) *Duramap { 224 | return dms[k] 225 | } 226 | 227 | func encrypt(secret EncryptionSecret, clearbz []byte) ([]byte, error) { 228 | var nonce [24]byte 229 | 230 | if _, err := io.ReadFull(rand.Reader, nonce[:]); err != nil { 231 | return nil, err 232 | } 233 | 234 | encrypted := secretbox.Seal(nil, clearbz, &nonce, secret) 235 | cypherbz := make([]byte, len(encrypted)+24) 236 | copy(cypherbz[:24], nonce[:]) 237 | copy(cypherbz[24:], encrypted) 238 | return cypherbz, nil 239 | } 240 | 241 | func decrypt(secret EncryptionSecret, cypherbz []byte) ([]byte, error) { 242 | var nonce [24]byte 243 | copy(nonce[:], cypherbz[:24]) 244 | clearbz, ok := secretbox.Open(nil, cypherbz[24:], &nonce, secret) 245 | if !ok { 246 | return nil, errors.New("Unable to decrypt") 247 | } 248 | return clearbz, nil 249 | } 250 | -------------------------------------------------------------------------------- /duramap_test.go: -------------------------------------------------------------------------------- 1 | package duramap 2 | 3 | import ( 4 | crand "crypto/rand" 5 | "fmt" 6 | "io" 7 | "math/rand" 8 | "os" 9 | "sync" 10 | "testing" 11 | "time" 12 | 13 | "github.com/notduncansmith/mutable" 14 | bolt "go.etcd.io/bbolt" 15 | ) 16 | 17 | const foo = "foo" 18 | const bar = "bar" 19 | const baz = "baz" 20 | const str64 = "abc4567890123456789012345678901234567890123456789012345678901234" 21 | 22 | var str128 = str64 + str64 23 | var str256 = str128 + str128 24 | 25 | func withDM(name string, t *testing.T, encrypted bool, fn func(dm *Duramap)) { 26 | os.Remove("./fixtures/" + name + ".db") 27 | defer os.Remove("./fixtures/" + name + ".db") 28 | key, err := generateEncryptionSecret() 29 | 30 | if !encrypted { 31 | key = nil 32 | err = nil 33 | } 34 | 35 | if err != nil { 36 | t.Errorf("Unable to generate encryption key: %v", err) 37 | return 38 | } 39 | 40 | dm, err := NewDuramap("./fixtures/"+name+".db", name, key) 41 | 42 | if err != nil { 43 | t.Errorf("Should be able to open database: %v", err) 44 | } 45 | 46 | if err := dm.Load(); err != nil { 47 | t.Errorf("Should be able to load map: %v", err) 48 | return 49 | } 50 | 51 | fn(dm) 52 | } 53 | 54 | func updateMap(t *testing.T, dm *Duramap, fn func(tx *Tx) error) { 55 | err := dm.UpdateMap(fn) 56 | 57 | if err != nil { 58 | t.Errorf("Should be able to save value: %v", err) 59 | } 60 | } 61 | 62 | func expectMap(t *testing.T, dm *Duramap, gm GenericMap) { 63 | m := dm.WithMap(func(m GenericMap) interface{} { 64 | if m == nil { 65 | t.Errorf("Expected map %v, got nil", gm) 66 | } 67 | return m 68 | }).(GenericMap) 69 | 70 | for k, v := range gm { 71 | if m[k] != v { 72 | t.Errorf("Expected m[%v] == %v, got %v", k, gm[k], v) 73 | } 74 | } 75 | 76 | dm.DoWithMap(func(m GenericMap) { 77 | for k := range m { 78 | if gm[k] == nil { 79 | t.Errorf("Unexpected key %v", k) 80 | } 81 | } 82 | }) 83 | } 84 | 85 | func TestRoundtrip(t *testing.T) { 86 | withDM("roundtrip", t, false, func(dm *Duramap) { 87 | updateMap(t, dm, func(tx *Tx) error { 88 | tx.Set(foo, bar) 89 | if tx.Get(foo) != bar { 90 | t.Errorf("Should be able to read saved value %v %v", tx, bar) 91 | } 92 | return nil 93 | }) 94 | updateMap(t, dm, func(tx *Tx) error { 95 | if tx.Get(foo) != bar { 96 | t.Errorf("Should be able to read saved value %v %v", *(tx.M), tx.Get(foo)) 97 | } 98 | return nil 99 | }) 100 | expectMap(t, dm, GenericMap{foo: bar}) 101 | dm.Truncate() 102 | expectMap(t, dm, GenericMap{}) 103 | }) 104 | } 105 | 106 | func TestRoundtripEncrypted(t *testing.T) { 107 | withDM("roundtrip_encrypted", t, true, func(dm *Duramap) { 108 | updateMap(t, dm, func(tx *Tx) error { 109 | tx.Set(foo, bar) 110 | if tx.Get(foo) != bar { 111 | t.Error("Should be able to read saved value") 112 | } 113 | return nil 114 | }) 115 | updateMap(t, dm, func(tx *Tx) error { 116 | if tx.Get(foo) != bar { 117 | t.Error("Should be able to read saved value") 118 | } 119 | return nil 120 | }) 121 | expectMap(t, dm, GenericMap{foo: bar}) 122 | dm.Truncate() 123 | expectMap(t, dm, GenericMap{}) 124 | }) 125 | } 126 | 127 | func TestConcurrentAccess(t *testing.T) { 128 | wg := sync.WaitGroup{} 129 | key, err := generateEncryptionSecret() 130 | if err != nil { 131 | t.Errorf("Unable to generate encryption secret: %v", err) 132 | } 133 | defer os.Remove("./fixtures/concurrent_access.db") 134 | 135 | access := func(i int, w *sync.WaitGroup) { 136 | defer w.Done() 137 | 138 | dm, err := NewDuramap("./fixtures/concurrent_access.db", "concurrent", key) 139 | if err != nil { 140 | t.Errorf("Should be able to open database: %v", err) 141 | return 142 | } 143 | if err = dm.Load(); err != nil { 144 | t.Errorf("Should be able to load map: %v", err) 145 | return 146 | } 147 | for n := 0; n < 9; n++ { 148 | err := dm.UpdateMap(func(tx *Tx) error { 149 | k := fmt.Sprintf("%v-%v", i, n) 150 | tx.Set(k, n) 151 | return nil 152 | }) 153 | if err != nil { 154 | t.Errorf("Should be able to save value: %v", err) 155 | return 156 | } 157 | time.Sleep(time.Millisecond * time.Duration(rand.Intn(10))) 158 | } 159 | for n := 0; n < 9; n++ { 160 | dm.DoWithMap(func(m GenericMap) { 161 | k := fmt.Sprintf("%v-%v", i, n) 162 | if m[k] != n { 163 | t.Errorf("Should be able to read saved value") 164 | } 165 | }) 166 | } 167 | } 168 | 169 | for m := 0; m < 3; m++ { 170 | wg.Add(1) 171 | go access(m, &wg) 172 | } 173 | 174 | wg.Wait() 175 | } 176 | 177 | func TestLoadEmpty(t *testing.T) { 178 | withDM("load_empty", t, true, func(dm *Duramap) { 179 | expectMap(t, dm, GenericMap{}) 180 | }) 181 | } 182 | 183 | func TestLoadRepeated(t *testing.T) { 184 | defer os.Remove("./fixtures/repeated.db") 185 | _, err := NewDuramap("./fixtures/repeated.db", "repeated", nil) 186 | if err != nil { 187 | t.Errorf("Should be able to open database: %v", err) 188 | } 189 | 190 | _, err = NewDuramap("./fixtures/repeated.db", "repeated", nil) 191 | 192 | if err != nil { 193 | t.Errorf("Should be able to open database: %v", err) 194 | } 195 | } 196 | 197 | func BenchmarkReadsDuramap(b *testing.B) { 198 | benchmarkReadsDuramap("int64", int64(123456789), b) 199 | benchmarkReadsDuramap("str64b", str64, b) 200 | benchmarkReadsDuramap("str128b", str128, b) 201 | benchmarkReadsDuramap("str256b", str256, b) 202 | } 203 | 204 | func BenchmarkWritesDuramap(b *testing.B) { 205 | benchmarkWritesDuramap("int64", int64(123456789), b) 206 | benchmarkWritesDuramap("str64b", str64, b) 207 | benchmarkWritesDuramap("str128b", str128, b) 208 | benchmarkWritesDuramap("str256b", str256, b) 209 | } 210 | 211 | func benchmarkReadsDuramap(label string, mapContents interface{}, b *testing.B) { 212 | dm, err := NewDuramap("./fixtures/bench_duramap_reads.db", "test_reads", nil) 213 | defer dm.Truncate() 214 | 215 | if err != nil { 216 | b.Errorf("Should be able to open database: %v", err) 217 | } 218 | 219 | err = dm.UpdateMap(func(tx *Tx) error { 220 | tx.Set(foo, bar) 221 | for i := 0; i < 10000; i++ { 222 | tx.Set("thing-"+fmt.Sprintf("%v", i), mapContents) 223 | } 224 | return nil 225 | }) 226 | 227 | if err != nil { 228 | b.Errorf("Should be able to save value: %v", err) 229 | } 230 | 231 | b.Run(label, func(b *testing.B) { 232 | for n := 0; n < b.N; n++ { 233 | dm.DoWithMap(func(m GenericMap) { 234 | if m[foo] != bar { 235 | b.Error("Should be able to read saved value") 236 | } 237 | }) 238 | } 239 | }) 240 | } 241 | 242 | func benchmarkWritesDuramap(label string, mapContents interface{}, b *testing.B) { 243 | dm, err := NewDuramap("./fixtures/bench_duramap_writes.db", "test_writes", nil) 244 | defer dm.Truncate() 245 | 246 | if err != nil { 247 | b.Errorf("Should be able to open database: %v", err) 248 | } 249 | 250 | err = dm.UpdateMap(func(tx *Tx) error { 251 | tx.Set(foo, bar) 252 | for i := 0; i < 10000; i++ { 253 | tx.Set("thing-"+fmt.Sprintf("%v", i), mapContents) 254 | } 255 | return nil 256 | }) 257 | 258 | if err != nil { 259 | b.Errorf("Should be able to save value: %v", err) 260 | } 261 | 262 | b.Run(label, func(b *testing.B) { 263 | for n := 0; n < b.N; n++ { 264 | err = dm.UpdateMap(func(tx *Tx) error { 265 | tx.Set(foo, baz) 266 | if tx.Get(foo) != baz { 267 | b.Error("Should be able to read saved value") 268 | } 269 | return nil 270 | }) 271 | 272 | if err != nil { 273 | b.Errorf("Should be able to save value: %v", err) 274 | } 275 | } 276 | }) 277 | } 278 | 279 | func BenchmarkReadsBbolt(b *testing.B) { 280 | db, err := bolt.Open("./fixtures/bench_bbolt.db", 0600, nil) 281 | bucketName := []byte("bbolt") 282 | defer db.Close() 283 | if err != nil { 284 | b.Errorf("Should be able to open database: %v", err) 285 | } 286 | 287 | err = db.Update(func(tx *bolt.Tx) error { 288 | bucket, err := tx.CreateBucketIfNotExists(bucketName) 289 | if err != nil { 290 | return err 291 | } 292 | for n := 0; n < 10000; n++ { 293 | err = bucket.Put([]byte("thing-"+fmt.Sprintf("%v", n)), []byte(str256)) 294 | } 295 | return bucket.Put([]byte(foo), []byte(bar)) 296 | }) 297 | 298 | if err != nil { 299 | b.Errorf("Should be able to save value: %v", err) 300 | } 301 | 302 | b.Run("str256b", func(b *testing.B) { 303 | for n := 0; n < b.N; n++ { 304 | err = db.Update(func(tx *bolt.Tx) error { 305 | bucket := tx.Bucket(bucketName) 306 | val := bucket.Get([]byte(foo)) 307 | if string(val) != bar { 308 | b.Errorf("Should be able to read saved value") 309 | } 310 | return nil 311 | }) 312 | if err != nil { 313 | b.Errorf("Should be able to read saved value: %v", err) 314 | } 315 | } 316 | }) 317 | } 318 | 319 | func BenchmarkWritesBbolt(b *testing.B) { 320 | db, err := bolt.Open("./fixtures/bench_bbolt.db", 0600, nil) 321 | bucketName := []byte("bbolt") 322 | defer db.Close() 323 | if err != nil { 324 | b.Errorf("Should be able to open database: %v", err) 325 | } 326 | 327 | err = db.Update(func(tx *bolt.Tx) error { 328 | bucket, err := tx.CreateBucketIfNotExists(bucketName) 329 | if err != nil { 330 | return err 331 | } 332 | for n := 0; n < 10000; n++ { 333 | err = bucket.Put([]byte("thing-"+fmt.Sprintf("%v", n)), []byte(str256)) 334 | } 335 | return bucket.Put([]byte(foo), []byte(bar)) 336 | }) 337 | 338 | if err != nil { 339 | b.Errorf("Should be able to save value: %v", err) 340 | } 341 | 342 | b.Run("str256b", func(b *testing.B) { 343 | for n := 0; n < b.N; n++ { 344 | err = db.Update(func(tx *bolt.Tx) error { 345 | bucket := tx.Bucket(bucketName) 346 | return bucket.Put([]byte(foo), []byte(baz)) 347 | }) 348 | if err != nil { 349 | b.Errorf("Should be able to read saved value: %v", err) 350 | } 351 | } 352 | }) 353 | } 354 | 355 | func BenchmarkReadsRWMap(b *testing.B) { 356 | mut := sync.RWMutex{} 357 | gm := GenericMap{foo: bar} 358 | b.Run("rwmutex", func(b *testing.B) { 359 | for n := 0; n < b.N; n++ { 360 | mut.RLock() 361 | if gm[foo].(string) != bar { 362 | b.Error("Should be able to read saved value") 363 | } 364 | mut.RUnlock() 365 | } 366 | }) 367 | } 368 | 369 | func BenchmarkWritesRWMap(b *testing.B) { 370 | mut := sync.RWMutex{} 371 | gm := GenericMap{foo: bar} 372 | b.Run("rwmutex", func(b *testing.B) { 373 | for n := 0; n < b.N; n++ { 374 | mut.Lock() 375 | gm[foo] = baz 376 | if gm[foo].(string) != baz { 377 | b.Error("Should be able to read saved value") 378 | } 379 | mut.Unlock() 380 | } 381 | }) 382 | } 383 | 384 | func BenchmarkReadsMutableMap(b *testing.B) { 385 | m := mutable.NewRW("benchmark") 386 | gm := GenericMap{foo: bar} 387 | 388 | b.Run("mutable", func(b *testing.B) { 389 | for n := 0; n < b.N; n++ { 390 | m.DoWithRLock(func() { 391 | if gm[foo].(string) != bar { 392 | b.Error("Should be able to read saved value") 393 | } 394 | }) 395 | } 396 | }) 397 | } 398 | 399 | func BenchmarkWritesMutableMap(b *testing.B) { 400 | m := mutable.NewRW("benchmark") 401 | gm := GenericMap{foo: bar} 402 | 403 | b.Run("mutable", func(b *testing.B) { 404 | for n := 0; n < b.N; n++ { 405 | m.DoWithRWLock(func() { 406 | gm[foo] = baz 407 | if gm[foo].(string) != baz { 408 | b.Error("Should be able to read saved value") 409 | } 410 | }) 411 | } 412 | }) 413 | } 414 | 415 | func generateEncryptionSecret() (EncryptionSecret, error) { 416 | var secretKey [32]byte 417 | if _, err := io.ReadFull(crand.Reader, secretKey[:]); err != nil { 418 | return nil, err 419 | } 420 | return &secretKey, nil 421 | } 422 | --------------------------------------------------------------------------------