├── .gitignore ├── LICENSE ├── README.md ├── bench └── main.go ├── chunk.go ├── go.mod ├── go.sum ├── sniper.go └── sniper_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | .DS_Store 15 | /bench/bench 16 | /bench/1 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Vadim Kulibaba 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `sniper` 2 | 3 | [![GoDoc](https://img.shields.io/badge/api-reference-blue.svg?style=flat-square)](https://godoc.org/github.com/recoilme/sniper) 4 | 5 | A simple and efficient thread-safe key/value store for Go. 6 | 7 | 8 | # Getting Started 9 | 10 | ## Features 11 | 12 | * Store hundreds of millions of entries 13 | * Fast. High concurrent. Thread-safe. Scales on multi-core CPUs 14 | * Extremly low memory usage 15 | * Zero GC overhead 16 | * Simple, pure Go implementation 17 | 18 | ## Installing 19 | 20 | To start using `sniper`, install Go and run `go get`: 21 | 22 | ```sh 23 | $ go get -u github.com/recoilme/sniper 24 | ``` 25 | 26 | This will retrieve the library. 27 | 28 | ## Usage 29 | 30 | The `Sniper` includes this methods: 31 | `Set`, `Get`, `Incr`, `Decr`, `Delete`, `Count`, `Open`, `Close`, `FileSize`, `Backup`. 32 | 33 | ```go 34 | s, _ := sniper.Open(sniper.Dir("1")) 35 | s.Set([]byte("hello"), []byte("go")) 36 | res, _ = s.Get([]byte("hello")) 37 | fmt.Println(res) 38 | s.Close() 39 | // Output: 40 | // go 41 | ``` 42 | 43 | ## Performance 44 | 45 | ``` 46 | MacBook Pro 2019 (Quad-Core Intel Core i7 2,8 GHz, 16 ГБ, APPLE SSD AP0512M) 47 | 48 | go version go1.14 darwin/amd64 49 | 50 | number of cpus: 8 51 | number of keys: 10000000 52 | keysize: 10 53 | random seed: 1570109110136449000 54 | 55 | -- sniper -- 56 | set: 10,000,000 ops over 8 threads in 63159ms, 158,331/sec, 6315 ns/op, 644.3 MB, 67 bytes/op 57 | get: 10,000,000 ops over 8 threads in 4455ms, 2,244,629/sec, 445 ns/op, 305.5 MB, 32 bytes/op 58 | del: 10,000,000 ops over 8 threads in 37568ms, 266,182/sec, 3756 ns/op, 122.8 MB, 12 bytes/op 59 | 60 | With fsync 61 | 62 | set: 10,000,000 ops over 8 threads in 85088ms, 117,524/sec, 8508 ns/op, 644.4 MB, 67 bytes/op 63 | get: 10,000,000 ops over 8 threads in 5623ms, 1,778,268/sec, 562 ns/op, 305.5 MB, 32 bytes/op 64 | 65 | ``` 66 | 67 | ## How it is done 68 | 69 | * Sniper database is sharded on 250+ chunks. Each chunk has its own lock (RW), so it supports high concurrent access on multi-core CPUs. 70 | * Each chunk store `hash(key) -> (value addr, value size)`, map. 71 | * Hash is very short, and has collisions. Sniper has collisions resolver. 72 | * Efficient space reuse alghorithm. Every packet has power of 2 size, for inplace rewrite on value update and map of deleted entrys, for reusing space. 73 | 74 | ## Limitations 75 | 76 | * 512 Kb - maximum entry size `len(key) + len(value)` 77 | * ~1 Tb - maximum database size 78 | 79 | ## Mac OS tip 80 | 81 | [How to Change Open Files Limit on OS X and macOS](https://gist.github.com/tombigel/d503800a282fcadbee14b537735d202c) 82 | 83 | ## Contact 84 | 85 | Vadim Kulibaba [@recoilme](https://github.com/recoilme) 86 | 87 | ## License 88 | 89 | `sniper` source code is available under the MIT [License](/LICENSE). -------------------------------------------------------------------------------- /bench/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "math/rand" 7 | "os" 8 | "runtime" 9 | 10 | "github.com/recoilme/sniper" 11 | "github.com/tidwall/lotsa" 12 | ) 13 | 14 | func randKey(rnd *rand.Rand, n int) []byte { 15 | s := make([]byte, n) 16 | rnd.Read(s) 17 | for i := 0; i < n; i++ { 18 | s[i] = 'a' + (s[i] % 26) 19 | } 20 | return s 21 | } 22 | 23 | func seed() ([][]byte, int) { 24 | seed := int64(1570109110136449000) //time.Now().UnixNano() //1570108152262917000 25 | // println(seed) 26 | rng := rand.New(rand.NewSource(seed)) 27 | N := 10_000_000 28 | K := 10 29 | 30 | fmt.Printf("\n") 31 | fmt.Printf("go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) 32 | fmt.Printf("\n") 33 | fmt.Printf(" number of cpus: %d\n", runtime.NumCPU()) 34 | fmt.Printf(" number of keys: %d\n", N) 35 | fmt.Printf(" keysize: %d\n", K) 36 | fmt.Printf(" random seed: %d\n", seed) 37 | 38 | fmt.Printf("\n") 39 | 40 | keysm := make(map[string]bool, N) 41 | for len(keysm) < N { 42 | keysm[string(randKey(rng, K))] = true 43 | } 44 | keys := make([][]byte, 0, N) 45 | for key := range keysm { 46 | keys = append(keys, []byte(key)) 47 | } 48 | return keys, N 49 | } 50 | 51 | func sniperBench(keys [][]byte, N int) { 52 | lotsa.Output = os.Stdout 53 | lotsa.MemUsage = true 54 | 55 | var ms runtime.MemStats 56 | runtime.ReadMemStats(&ms) 57 | fmt.Printf("Alloc = %v MiB Total = %v MiB\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024)) 58 | 59 | fmt.Println("-- sniper --") 60 | sniper.DeleteStore("1") 61 | s, err := sniper.Open(sniper.Dir("1"), sniper.ChunksCollision(2), sniper.ChunksTotal(10)) 62 | if err != nil { 63 | panic(err) 64 | } 65 | fmt.Println("set: ") 66 | coll := 0 67 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 68 | b := make([]byte, 8) 69 | binary.BigEndian.PutUint64(b, uint64(i)) 70 | err := s.Set(keys[i], b) 71 | if err == sniper.ErrCollision { 72 | fmt.Println("ErrCollision, set:", string(keys[i]), err.Error()) 73 | } 74 | if err != nil { 75 | panic(err) 76 | } 77 | }) 78 | 79 | fmt.Printf("Alloc = %v MiB Total = %v MiB Coll=%d\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024), coll) 80 | coll = 0 81 | fmt.Println("get: ") 82 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 83 | b, err := s.Get(keys[i]) 84 | if err != nil { 85 | println("errget", string(keys[i])) 86 | panic(err) 87 | } 88 | v := binary.BigEndian.Uint64(b) 89 | 90 | if uint64(i) != v { 91 | println("get error:", string(keys[i]), i, v) 92 | panic("bad news") 93 | } 94 | }) 95 | 96 | runtime.ReadMemStats(&ms) 97 | fmt.Printf("Alloc = %v MiB Total = %v MiB\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024)) 98 | 99 | fmt.Println("del: ") 100 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 101 | s.Delete(keys[i]) 102 | }) 103 | err = sniper.DeleteStore("1") 104 | fmt.Println(err) 105 | } 106 | 107 | func main() { 108 | keys, N := seed() 109 | 110 | sniperBench(keys, N) 111 | 112 | //uncomment for badger test 113 | 114 | //budgerBench(keys, N) 115 | } 116 | 117 | /* 118 | func budgerBench(keys [][]byte, N int) { 119 | sniper.DeleteStore("badger_test") 120 | bd, err := newBadgerdb("badger_test") 121 | if err != nil { 122 | panic(err) 123 | } 124 | println("-- badger --") 125 | print("set: ") 126 | 127 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 128 | txn := bd.NewTransaction(true) // Read-write txn 129 | b := make([]byte, 8) 130 | binary.BigEndian.PutUint64(b, uint64(i)) 131 | 132 | err = txn.SetEntry(badger.NewEntry(keys[i], b)) 133 | if err != nil { 134 | log.Fatal(err) 135 | } 136 | err = txn.Commit() 137 | if err != nil { 138 | log.Fatal(err) 139 | } 140 | 141 | }) 142 | 143 | print("get: ") 144 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 145 | var val []byte 146 | err := bd.View(func(txn *badger.Txn) error { 147 | item, err := txn.Get(keys[i]) 148 | if err != nil { 149 | return err 150 | } 151 | val, err = item.ValueCopy(val) 152 | return err 153 | }) 154 | if err != nil { 155 | log.Fatal(err) 156 | } 157 | v := binary.BigEndian.Uint64(val) 158 | if uint64(i) != v { 159 | panic("bad news") 160 | } 161 | }) 162 | 163 | print("del: ") 164 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 165 | txn := bd.NewTransaction(true) 166 | err := txn.Delete(keys[i]) 167 | if err != nil { 168 | log.Fatal(err) 169 | } 170 | err = txn.Commit() 171 | if err != nil { 172 | log.Fatal(err) 173 | } 174 | }) 175 | 176 | sniper.DeleteStore("badger_test") 177 | 178 | } 179 | func newBadgerdb(path string) (*badger.DB, error) { 180 | 181 | os.MkdirAll(path, os.FileMode(0777)) 182 | opts := badger.DefaultOptions(path) 183 | opts.SyncWrites = false 184 | opts.Logger = nil 185 | return badger.Open(opts) 186 | } 187 | */ 188 | -------------------------------------------------------------------------------- /chunk.go: -------------------------------------------------------------------------------- 1 | package sniper 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "sync" 11 | "time" 12 | ) 13 | 14 | const ( 15 | currentChunkVersion = 1 16 | versionMarker = 255 17 | deleted = 42 // flag for removed, tribute 2 dbf 18 | ) 19 | 20 | var ( 21 | sizeHeaders = map[int]uint32{0: 8, 1: 12} 22 | sizeHead = sizeHeaders[currentChunkVersion] 23 | forceexit bool 24 | ) 25 | 26 | // chunk - local shard 27 | type chunk struct { 28 | sync.RWMutex 29 | f *os.File // file storage 30 | m map[uint32]uint64 // keys: hash / key meta info 31 | h map[uint32]byte // holes: addr / size 32 | needFsync bool 33 | } 34 | 35 | type Header struct { 36 | sizeb uint8 37 | status uint8 38 | keylen uint16 39 | vallen uint32 40 | expire uint32 41 | } 42 | 43 | func encodeKeyMeta(addr uint32, size byte, expire uint32) uint64 { 44 | return uint64(addr)<<32 | uint64(size)<<24 | uint64(expire)>>9 45 | } 46 | 47 | func decodeKeyMeta(info uint64) (addr uint32, size byte, expire uint32) { 48 | addr = uint32(info >> 32) 49 | size = byte(info >> 24 & 0xff) 50 | expire = uint32(info&0xffffff) << 9 51 | // if expire non zero add 1<<9-1 = 511 sec 52 | if expire != 0 { 53 | expire += 1<<9 - 1 54 | } 55 | return 56 | } 57 | 58 | // https://github.com/thejerf/gomempool/blob/master/pool.go#L519 59 | // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 60 | // suitably modified to work on 32-bit 61 | func nextPowerOf2(v uint32) uint32 { 62 | v-- 63 | v |= v >> 1 64 | v |= v >> 2 65 | v |= v >> 4 66 | v |= v >> 8 67 | v |= v >> 16 68 | v++ 69 | 70 | return v 71 | } 72 | 73 | // NextPowerOf2 return next power of 2 for v and it's value 74 | // return maxuint32 in case of overflow 75 | func NextPowerOf2(v uint32) (power byte, val uint32) { 76 | if v == 0 { 77 | return 0, 0 78 | } 79 | for power = 0; power < 32; power++ { 80 | val = 1 << power 81 | if val >= v { 82 | break 83 | } 84 | } 85 | if power == 32 { 86 | //overflow 87 | val = 4294967295 88 | } 89 | return 90 | } 91 | 92 | func detectChunkVersion(file *os.File) (version int, err error) { 93 | b := make([]byte, 2) 94 | n, errRead := file.Read(b) 95 | if errRead != nil { 96 | return -1, errRead 97 | } 98 | if n != 2 { 99 | return -1, errors.New("File too short") 100 | } 101 | 102 | // 255 version marker 103 | if b[0] == versionMarker { 104 | if b[1] == 0 || b[1] == deleted { 105 | // first version 106 | return 0, nil 107 | } 108 | return int(b[1]), nil 109 | } 110 | if b[1] == 0 || b[1] == deleted { 111 | // first version 112 | return 0, nil 113 | } 114 | return -1, nil 115 | } 116 | 117 | func makeHeader(k, v []byte, expire uint32) (header *Header) { 118 | header = &Header{} 119 | header.status = 0 120 | header.keylen = uint16(len(k)) 121 | header.vallen = uint32(len(v)) 122 | header.expire = expire 123 | sizeb, _ := NextPowerOf2(uint32(header.keylen) + header.vallen + sizeHead) 124 | header.sizeb = sizeb 125 | return 126 | } 127 | 128 | func parseHeaderV0(b []byte) (header *Header) { 129 | header = &Header{} 130 | header.sizeb = b[0] 131 | header.status = b[1] 132 | header.keylen = binary.BigEndian.Uint16(b[2:4]) 133 | header.vallen = binary.BigEndian.Uint32(b[4:8]) 134 | return 135 | } 136 | 137 | func parseHeader(b []byte) (header *Header) { 138 | header = &Header{} 139 | header.sizeb = b[0] 140 | header.status = b[1] 141 | header.keylen = binary.BigEndian.Uint16(b[2:4]) 142 | header.vallen = binary.BigEndian.Uint32(b[4:8]) 143 | header.expire = binary.BigEndian.Uint32(b[8:12]) 144 | return 145 | } 146 | 147 | func readHeader(r io.Reader, version int) (header *Header, err error) { 148 | b := make([]byte, sizeHeaders[version]) 149 | n, err := io.ReadFull(r, b) 150 | if n != int(sizeHeaders[version]) { 151 | if err == io.EOF { 152 | err = nil 153 | } 154 | return 155 | } 156 | switch version { 157 | case 0: 158 | header = parseHeaderV0(b) 159 | case currentChunkVersion: 160 | header = parseHeader(b) 161 | default: 162 | err = fmt.Errorf("Unknov header version %d", version) 163 | } 164 | return 165 | } 166 | 167 | func writeHeader(b []byte, header *Header) { 168 | b[0] = header.sizeb 169 | b[1] = header.status 170 | binary.BigEndian.PutUint16(b[2:4], header.keylen) 171 | binary.BigEndian.PutUint32(b[4:8], header.vallen) 172 | binary.BigEndian.PutUint32(b[8:12], header.expire) 173 | return 174 | } 175 | 176 | func packetMarshal(k, v []byte, expire uint32) (header *Header, b []byte) { 177 | // write head 178 | header = makeHeader(k, v, expire) 179 | size := 1 << header.sizeb 180 | b = make([]byte, size) 181 | writeHeader(b, header) 182 | // write body: val and key 183 | copy(b[sizeHead:], v) 184 | copy(b[sizeHead+header.vallen:], k) 185 | return 186 | } 187 | 188 | func packetUnmarshal(packet []byte) (header *Header, k, v []byte) { 189 | header = parseHeader(packet) 190 | k = packet[sizeHead+header.vallen : sizeHead+header.vallen+uint32(header.keylen)] 191 | v = packet[sizeHead : sizeHead+header.vallen] 192 | return 193 | } 194 | 195 | func (c *chunk) init(name string) (err error) { 196 | c.Lock() 197 | forceexit = false 198 | defer c.Unlock() 199 | 200 | f, err := os.OpenFile(name, os.O_CREATE|os.O_RDWR, os.FileMode(fileMode)) 201 | if err != nil { 202 | return 203 | } 204 | err = f.Sync() 205 | if err != nil { 206 | return err 207 | } 208 | c.f = f 209 | c.m = make(map[uint32]uint64) 210 | c.h = make(map[uint32]byte) 211 | //read if f not empty 212 | if fi, e := c.f.Stat(); e == nil { 213 | // new file 214 | if fi.Size() == 0 { 215 | // write chunk version info 216 | c.f.Write([]byte{versionMarker, currentChunkVersion}) 217 | return 218 | } 219 | 220 | //read file 221 | var seek uint32 222 | // detect chunk version 223 | version, errDetect := detectChunkVersion(c.f) 224 | if errDetect != nil { 225 | err = errDetect 226 | return 227 | } 228 | 229 | if version < 0 || version > currentChunkVersion { 230 | err = errors.New("Unknown chunk version in file " + name) 231 | return 232 | } 233 | 234 | if version == 0 { 235 | // rewind to begin 236 | c.f.Seek(0, 0) 237 | } else { 238 | // real chunk begin 239 | seek = 2 240 | } 241 | 242 | if version > currentChunkVersion { 243 | err = fmt.Errorf("chunk %s unsupported version %d", name, version) 244 | } 245 | 246 | // if load chunk with old version create file in new format 247 | if version < currentChunkVersion { 248 | var newfile *os.File 249 | fmt.Printf("Load from old version chunk %s, do inplace upgrade v%d -> v%d\n", name, version, currentChunkVersion) 250 | newname := name + ".new" 251 | newfile, err = os.OpenFile(newname, os.O_CREATE|os.O_RDWR|os.O_TRUNC, os.FileMode(fileMode)) 252 | if err != nil { 253 | return 254 | } 255 | // write chunk version info 256 | newfile.Write([]byte{versionMarker, currentChunkVersion}) 257 | seek = 2 258 | oldsizehead := sizeHeaders[version] 259 | sizediff := sizeHead - oldsizehead 260 | for { 261 | var header *Header 262 | var errRead error 263 | header, errRead = readHeader(c.f, version) 264 | if errRead != nil { 265 | newfile.Close() 266 | return errRead 267 | } 268 | if header == nil { 269 | break 270 | } 271 | oldsizedata := (1 << header.sizeb) - oldsizehead 272 | sizeb, size := NextPowerOf2(uint32(sizeHead) + uint32(header.keylen) + header.vallen) 273 | header.sizeb = sizeb 274 | b := make([]byte, size+sizediff) 275 | writeHeader(b, header) 276 | n, errRead := c.f.Read(b[sizeHead : sizeHead+oldsizedata]) 277 | if errRead != nil { 278 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 279 | } 280 | if n != int(oldsizedata) { 281 | return fmt.Errorf("n != record length: %w", ErrFormat) 282 | } 283 | 284 | // skip deleted or expired entry 285 | if header.status == deleted || (header.expire != 0 && int64(header.expire) < time.Now().Unix()) { 286 | continue 287 | } 288 | keyidx := int(sizeHead) + int(header.vallen) 289 | h := hash(b[keyidx : keyidx+int(header.keylen)]) 290 | c.m[h] = encodeKeyMeta(seek, header.sizeb, header.expire) 291 | n, errRead = newfile.Write(b[0:size]) 292 | if errRead != nil { 293 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 294 | } 295 | seek += uint32(n) 296 | } 297 | // close old chunk file 298 | errRead := c.f.Close() 299 | if errRead != nil { 300 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 301 | } 302 | // set new file for chunk 303 | c.f = newfile 304 | // remove old chunk file from disk 305 | errRead = os.Remove(name) 306 | if errRead != nil { 307 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 308 | } 309 | // rename new file to old file 310 | errRead = os.Rename(newname, name) 311 | if errRead != nil { 312 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 313 | } 314 | return 315 | } 316 | 317 | var n int 318 | for { 319 | header, errRead := readHeader(c.f, version) 320 | if errRead != nil { 321 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 322 | } 323 | if header == nil { 324 | break 325 | } 326 | // skip val 327 | _, seekerr := c.f.Seek(int64(header.vallen), 1) 328 | if seekerr != nil { 329 | return fmt.Errorf("%s: %w", seekerr.Error(), ErrFormat) 330 | } 331 | // read key 332 | key := make([]byte, header.keylen) 333 | n, errRead = c.f.Read(key) 334 | if errRead != nil { 335 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 336 | } 337 | if n != int(header.keylen) { 338 | return fmt.Errorf("n != key length: %w", ErrFormat) 339 | } 340 | shiftv := 1 << header.sizeb //2^pow 341 | ret, seekerr := c.f.Seek(int64(shiftv-int(header.keylen)-int(header.vallen)-int(sizeHead)), 1) // skip empty tail 342 | if seekerr != nil { 343 | return ErrFormat 344 | } 345 | // map store 346 | if header.status != deleted && (header.expire == 0 || int64(header.expire) >= time.Now().Unix()) { 347 | h := hash(key) 348 | c.m[h] = encodeKeyMeta(seek, header.sizeb, header.expire) 349 | } else { 350 | //deleted blocks store 351 | c.h[seek] = header.sizeb // seek / size 352 | } 353 | seek = uint32(ret) 354 | } 355 | } 356 | 357 | return 358 | } 359 | 360 | // fsync commits the current contents of the file to stable storage 361 | func (c *chunk) fsync() error { 362 | if c.needFsync { 363 | c.Lock() 364 | defer c.Unlock() 365 | c.needFsync = false 366 | return c.f.Sync() 367 | } 368 | return nil 369 | } 370 | 371 | // expirekeys walk all keys and delete expired 372 | // maxruntime - maximum run time 373 | func (c *chunk) expirekeys(maxruntime time.Duration) error { 374 | starttime := time.Now().UnixMilli() 375 | curtime := starttime / 1000 376 | expiredlist := make([]uint32, 0, 1024) 377 | if maxruntime.Seconds() > 1000 { 378 | maxruntime = time.Duration(1000) * time.Second 379 | } 380 | stoptime := starttime + maxruntime.Milliseconds() 381 | 382 | c.RLock() 383 | for h, meta := range c.m { 384 | _, _, expire := decodeKeyMeta(meta) 385 | if expire != 0 && curtime > int64(expire) { 386 | expiredlist = append(expiredlist, h) 387 | } 388 | } 389 | c.RUnlock() 390 | keycount := len(expiredlist) 391 | if keycount == 0 { 392 | return nil 393 | } 394 | sleeptime := maxruntime.Milliseconds() / int64(keycount) / 2 395 | bulk := 1 396 | if sleeptime < 1 { 397 | bulk = keycount/int(maxruntime.Milliseconds()+1) + 1 398 | sleeptime = 1 399 | } else if sleeptime > 10 { 400 | sleeptime = 10 401 | } 402 | 403 | // special case, expire all keys at maximum speed 404 | // maximum run time 300s 405 | if maxruntime == time.Duration(0) { 406 | bulk = 1000 407 | sleeptime = 0 408 | stoptime = starttime + 300000 409 | } 410 | 411 | //fmt.Printf("chunk %s do expire %d keys, sleep %d, bulk %d, starttime %d, stoptime %d\n", c.f.Name(), keycount, sleeptime, bulk, starttime, stoptime) 412 | count := 0 413 | bulkcount := 0 414 | c.Lock() 415 | for _, h := range expiredlist { 416 | if forceexit || time.Now().UnixMilli() >= stoptime { 417 | break 418 | } 419 | count++ 420 | meta, ok := c.m[h] 421 | if ok { 422 | addr, sizeb, expire := decodeKeyMeta(meta) 423 | if expire != 0 && curtime > int64(expire) { 424 | delete(c.m, h) 425 | c.h[addr] = sizeb 426 | } 427 | } 428 | bulkcount++ 429 | if bulkcount >= bulk { 430 | c.Unlock() 431 | time.Sleep(time.Duration(sleeptime) * time.Millisecond) 432 | c.Lock() 433 | bulkcount = 0 434 | } 435 | } 436 | c.Unlock() 437 | //fmt.Printf("chunk %s finish expire %d keys, time %d\n", c.f.Name(), count, time.Now().UnixMilli()-starttime) 438 | return nil 439 | } 440 | 441 | // set - write data to file & in map guarded by mutex 442 | func (c *chunk) set(k, v []byte, h uint32, expire uint32) (err error) { 443 | c.Lock() 444 | defer c.Unlock() 445 | err = c.write_key(k, v, h, expire) 446 | return 447 | } 448 | 449 | // write_key - write data to file & in map 450 | func (c *chunk) write_key(k, v []byte, h uint32, expire uint32) (err error) { 451 | c.needFsync = true 452 | header, b := packetMarshal(k, v, expire) 453 | // write at file 454 | pos := int64(-1) 455 | 456 | if meta, ok := c.m[h]; ok { 457 | addr, size, _ := decodeKeyMeta(meta) 458 | packet := make([]byte, 1<1_000_000_000 of 8 bytes alphabet keys without collision errors) 87 | // different keys may has same hash 88 | // collision chunks needed for resolving this, without collisions errors 89 | // if ChunkColCnt - zero, ErrCollision will return in case of collision 90 | func ChunksCollision(chunks int) OptStore { 91 | return func(s *Store) error { 92 | s.chunkColCnt = chunks 93 | return nil 94 | } 95 | } 96 | 97 | // ChunksTotal - total chunks/shards, default 256 98 | // Must be more then collision chunks 99 | func ChunksTotal(chunks int) OptStore { 100 | return func(s *Store) error { 101 | s.chunksCnt = chunks 102 | return nil 103 | } 104 | } 105 | 106 | // ChunksPrefix - prefix for a chunks filename 107 | func ChunksPrefix(prefix string) OptStore { 108 | return func(s *Store) error { 109 | s.chunksPrefix = prefix 110 | return nil 111 | } 112 | } 113 | 114 | // SyncInterval - how often fsync do, default 0 - OS will do it 115 | func SyncInterval(interv time.Duration) OptStore { 116 | return func(s *Store) error { 117 | s.syncInterval = interv 118 | if interv > 0 { 119 | s.iv = interval.Set(func(t time.Time) { 120 | for i := range s.chunks[:] { 121 | err := s.chunks[i].fsync() 122 | if err != nil { 123 | fmt.Printf("Error fsync:%s\n", err) 124 | //its critical error drive is broken 125 | panic(err) 126 | } 127 | } 128 | }, interv) 129 | } 130 | return nil 131 | } 132 | } 133 | 134 | // ExpireInterval - how often run key expiration process 135 | // expire only one chunk 136 | func ExpireInterval(interv time.Duration) OptStore { 137 | return func(s *Store) error { 138 | s.expireInterval = interv 139 | if interv > 0 { 140 | s.expiv = interval.Set(func(t time.Time) { 141 | err := s.chunks[expirechunk].expirekeys(interv) 142 | if err != nil { 143 | fmt.Printf("Error expire:%s\n", err) 144 | } 145 | expirechunk++ 146 | if expirechunk >= s.chunksCnt { 147 | expirechunk = 0 148 | } 149 | }, interv) 150 | } 151 | return nil 152 | } 153 | } 154 | 155 | func hash(b []byte) uint32 { 156 | // TODO race, test and replace with https://github.com/spaolacci/murmur3/pull/28 157 | return murmur3.Sum32WithSeed(b, 0) 158 | /* 159 | convert to 24 bit hash if you need more memory, but add chunks for collisions 160 | //MASK_24 := uint32((1 << 24) - 1) 161 | //ss := h.Sum32() 162 | //hash := (ss >> 24) ^ (ss & MASK_24) 163 | */ 164 | } 165 | 166 | // Open return new store 167 | // It will create 256 shards 168 | // Each shard store keys and val size and address in map[uint32]uint32 169 | // 170 | // options, see https://gist.github.com/travisjeffery/8265ca411735f638db80e2e34bdbd3ae#gistcomment-3171484 171 | // usage - Open(Dir("1"), SyncInterval(1*time.Second)) 172 | func Open(opts ...OptStore) (s *Store, err error) { 173 | s = &Store{} 174 | //default 175 | s.syncInterval = 0 176 | s.expireInterval = 0 177 | s.chunkColCnt = 4 178 | s.chunksCnt = 256 179 | // call option functions on instance to set options on it 180 | for _, opt := range opts { 181 | err := opt(s) 182 | // if the option func returns an error, add it to the list of errors 183 | if err != nil { 184 | return nil, err 185 | } 186 | } 187 | if s.chunksCnt-s.chunkColCnt < 1 { 188 | return nil, errors.New("chunksCnt must be more then chunkColCnt minimum on 1") 189 | } 190 | s.chunks = make([]chunk, s.chunksCnt) 191 | 192 | chchan := make(chan int, s.chunksCnt) 193 | errchan := make(chan error, 4) 194 | var wg sync.WaitGroup 195 | 196 | exitworkers := false 197 | for i := 0; i < 4; i++ { 198 | wg.Add(1) 199 | go func() { 200 | for i := range chchan { 201 | if exitworkers { 202 | break 203 | } 204 | 205 | var filename string 206 | if s.chunksPrefix != "" { 207 | filename = fmt.Sprintf("%s/%s-%d", s.dir, s.chunksPrefix, i) 208 | } else { 209 | filename = fmt.Sprintf("%s/%d", s.dir, i) 210 | } 211 | err := s.chunks[i].init(filename) 212 | if err != nil { 213 | errchan <- err 214 | exitworkers = true 215 | break 216 | } 217 | } 218 | wg.Done() 219 | }() 220 | } 221 | 222 | // create chuncks 223 | for i := range s.chunks { 224 | chchan <- i 225 | } 226 | close(chchan) 227 | 228 | wg.Wait() 229 | 230 | if len(errchan) > 0 { 231 | err = <-errchan 232 | return 233 | } 234 | s.ss = sortedset.New() 235 | return 236 | } 237 | 238 | func (s *Store) idx(h uint32) uint32 { 239 | return uint32((int(h) % (s.chunksCnt - s.chunkColCnt)) + s.chunkColCnt) 240 | } 241 | 242 | // Set - store key and val in shard 243 | // max packet size is 2^19, 512kb (524288) 244 | // packet size = len(key) + len(val) + 8 245 | func (s *Store) Set(k, v []byte, expire uint32) (err error) { 246 | h := hash(k) 247 | idx := s.idx(h) 248 | err = s.chunks[idx].set(k, v, h, expire) 249 | if err == ErrCollision { 250 | for i := 0; i < int(s.chunkColCnt); i++ { 251 | err = s.chunks[i].set(k, v, h, expire) 252 | if err == ErrCollision { 253 | continue 254 | } 255 | break 256 | } 257 | } 258 | return 259 | } 260 | 261 | // Touch - update key expire 262 | func (s *Store) Touch(k []byte, expire uint32) (err error) { 263 | h := hash(k) 264 | idx := s.idx(h) 265 | err = s.chunks[idx].touch(k, h, expire) 266 | if err == ErrCollision { 267 | for i := 0; i < int(s.chunkColCnt); i++ { 268 | err = s.chunks[i].touch(k, h, expire) 269 | if err == ErrCollision { 270 | continue 271 | } 272 | break 273 | } 274 | } 275 | return 276 | } 277 | 278 | // Get - return val by key 279 | func (s *Store) Get(k []byte) (v []byte, err error) { 280 | h := hash(k) 281 | idx := s.idx(h) 282 | v, _, err = s.chunks[idx].get(k, h) 283 | if err == ErrCollision { 284 | for i := 0; i < int(s.chunkColCnt); i++ { 285 | v, _, err = s.chunks[i].get(k, h) 286 | if err == ErrCollision || err == ErrNotFound { 287 | continue 288 | } 289 | break 290 | } 291 | } 292 | return 293 | } 294 | 295 | // Count return count keys 296 | func (s *Store) Count() (cnt int) { 297 | for i := range s.chunks[:] { 298 | cnt += s.chunks[i].count() 299 | } 300 | return 301 | } 302 | 303 | // Close - close related chunks 304 | func (s *Store) Close() (err error) { 305 | errStr := "" 306 | if s.syncInterval > 0 { 307 | s.iv.Clear() 308 | } 309 | if s.expireInterval > 0 { 310 | s.expiv.Clear() 311 | } 312 | for i := range s.chunks[:] { 313 | err = s.chunks[i].close() 314 | if err != nil { 315 | errStr += err.Error() + "\r\n" 316 | return 317 | } 318 | } 319 | if errStr != "" { 320 | return errors.New(errStr) 321 | } 322 | return 323 | } 324 | 325 | // DeleteStore - remove directory with files 326 | func DeleteStore(dir string) error { 327 | return os.RemoveAll(dir) 328 | } 329 | 330 | // FileSize returns the total size of the disk storage used by the DB. 331 | func (s *Store) FileSize() (fs int64, err error) { 332 | for i := range s.chunks[:] { 333 | is, err := s.chunks[i].fileSize() 334 | if err != nil { 335 | return -1, err 336 | } 337 | fs += is 338 | } 339 | return 340 | } 341 | 342 | // Delete - delete item by key 343 | func (s *Store) Delete(k []byte) (isDeleted bool, err error) { 344 | h := hash(k) 345 | idx := s.idx(h) 346 | isDeleted, err = s.chunks[idx].delete(k, h) 347 | if err == ErrCollision { 348 | for i := 0; i < int(s.chunkColCnt); i++ { 349 | isDeleted, err = s.chunks[i].delete(k, h) 350 | if err == ErrCollision || err == ErrNotFound { 351 | continue 352 | } 353 | break 354 | } 355 | } 356 | return 357 | } 358 | 359 | // Incr - Incr item by uint64 360 | // inited with zero 361 | func (s *Store) Incr(k []byte, v uint64) (uint64, error) { 362 | h := hash(k) 363 | idx := s.idx(h) 364 | return s.chunks[idx].incrdecr(k, h, v, true) 365 | } 366 | 367 | // Decr - Decr item by uint64 368 | // inited with zero 369 | func (s *Store) Decr(k []byte, v uint64) (uint64, error) { 370 | h := hash(k) 371 | idx := s.idx(h) 372 | return s.chunks[idx].incrdecr(k, h, v, false) 373 | } 374 | 375 | // Backup all data to writer 376 | func (s *Store) Backup(w io.Writer) (err error) { 377 | _, err = w.Write([]byte{currentChunkVersion}) 378 | if err != nil { 379 | return 380 | } 381 | for i := range s.chunks[:] { 382 | err = s.chunks[i].backup(w) 383 | if err != nil { 384 | return 385 | } 386 | } 387 | return 388 | } 389 | 390 | // Restore from backup reader 391 | func (s *Store) Restore(r io.Reader) (err error) { 392 | b := make([]byte, 1) 393 | _, err = r.Read(b) 394 | if int(b[0]) != currentChunkVersion { 395 | return fmt.Errorf("Bad backup version %d", b[0]) 396 | } 397 | 398 | for { 399 | var header *Header 400 | var errRead error 401 | header, errRead = readHeader(r, currentChunkVersion) 402 | if errRead != nil { 403 | return errRead 404 | } 405 | if header == nil { 406 | break 407 | } 408 | size := int(sizeHead) + int(header.vallen) + int(header.keylen) // record size 409 | b := make([]byte, size) 410 | writeHeader(b, header) 411 | n, errRead := io.ReadFull(r, b[sizeHead:]) 412 | if errRead != nil { 413 | return fmt.Errorf("%s: %w", errRead.Error(), ErrFormat) 414 | } 415 | if n != size-int(sizeHead) { 416 | return fmt.Errorf("n != record length: %w", ErrFormat) 417 | } 418 | 419 | // skip deleted or expired entry 420 | if header.status == deleted || (header.expire != 0 && int64(header.expire) < time.Now().Unix()) { 421 | continue 422 | } 423 | _, key, val := packetUnmarshal(b) 424 | s.Set(key, val, header.expire) 425 | } 426 | return 427 | } 428 | 429 | // Backup in gzip 430 | func (s *Store) BackupGZ(w io.Writer) (err error) { 431 | gz := gzip.NewWriter(w) 432 | defer gz.Close() 433 | return s.Backup(gz) 434 | } 435 | 436 | // Restore from backup in gzip 437 | func (s *Store) RestoreGZ(r io.Reader) (err error) { 438 | gz, err := gzip.NewReader(r) 439 | if err != nil { 440 | return err 441 | } 442 | defer gz.Close() 443 | return s.Restore(gz) 444 | } 445 | 446 | // Expire - remove expired keys from all chunks 447 | func (s *Store) Expire() (err error) { 448 | for i := range s.chunks[:] { 449 | err = s.chunks[i].expirekeys(time.Duration(0)) 450 | if err != nil { 451 | return 452 | } 453 | } 454 | return 455 | } 456 | 457 | func readUint32(b []byte) uint32 { 458 | _ = b[3] 459 | return uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24 460 | } 461 | 462 | func appendUint32(b []byte, x uint32) []byte { 463 | a := [4]byte{ 464 | byte(x >> 24), 465 | byte(x >> 16), 466 | byte(x >> 8), 467 | byte(x), 468 | } 469 | return append(b, a[:]...) 470 | } 471 | 472 | // Bucket - create new bucket for storing keys with same prefix in memory index 473 | func (s *Store) Bucket(name string) (*sortedset.BucketStore, error) { 474 | // store all buckets in [buckets] key 475 | bKey := []byte("[buckets]") 476 | val, err := s.Get(bKey) 477 | if err == ErrNotFound { 478 | err = nil 479 | } 480 | if err != nil { 481 | return nil, err 482 | } 483 | buckets := string(val) 484 | var isExists bool 485 | for _, bucket := range strings.Split(buckets, ",") { 486 | if bucket == name { 487 | isExists = true 488 | break 489 | } 490 | } 491 | if !isExists { 492 | if buckets != "" { 493 | buckets += "," 494 | } 495 | buckets += name 496 | err = s.Set(bKey, []byte(buckets), 0) 497 | if err != nil { 498 | return nil, err 499 | } 500 | } 501 | return sortedset.Bucket(s.ss, name), nil 502 | } 503 | 504 | // Put - store key and val with Set 505 | // And add key in index (backed by sortedset) 506 | func (s *Store) Put(bucket *sortedset.BucketStore, k, v []byte) (err error) { 507 | key := []byte(bucket.Name) 508 | key = append(key, k...) 509 | err = s.Set(key, v, 0) 510 | if err == nil { 511 | bucket.Put(string(k)) 512 | } 513 | return 514 | } 515 | 516 | // Keys will return keys stored with Put method 517 | // Params: key prefix ("" - return all keys) 518 | // Limit - 0, all 519 | // Offset - 0, zero offset 520 | // Keys will be without prefix and in descending order 521 | func (s *Store) Keys(bucket *sortedset.BucketStore, limit, offset int) []string { 522 | return bucket.Keys(limit, offset) 523 | } 524 | -------------------------------------------------------------------------------- /sniper_test.go: -------------------------------------------------------------------------------- 1 | package sniper 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "math/rand" 8 | "os" 9 | "runtime" 10 | "testing" 11 | "time" 12 | 13 | "bou.ke/monkey" 14 | "github.com/stretchr/testify/assert" 15 | "github.com/tidwall/lotsa" 16 | ) 17 | 18 | func TestPack(t *testing.T) { 19 | addr := 1<<26 - 5 20 | size := byte(5) 21 | curtime := time.Now().Unix() 22 | expire := uint32(curtime>>9) << 9 23 | some64 := encodeKeyMeta(uint32(addr), size, expire) 24 | expire += 1<<9 - 1 25 | s, l, e := decodeKeyMeta(some64) 26 | if s != uint32(addr) || l != 5 || e != expire { 27 | t.Errorf("get addr = %d, size=%d expire=%d", s, l, e) 28 | } 29 | addr = 1<<28 - 1 30 | size = byte(19) 31 | exp, _ := time.Parse("2006-02-01 15:04:05", "2020-10-11 12:34:56") 32 | curtime = exp.Unix() 33 | expire = uint32(curtime>>9) << 9 34 | maxAddrSize := encodeKeyMeta(uint32(addr), size, expire) 35 | expire += 1<<9 - 1 36 | s, l, e = decodeKeyMeta(maxAddrSize) 37 | if s != uint32(addr) || l != 19 || e != expire { 38 | t.Errorf("get addr = %d, size=%d expire=%d", s, l, e) 39 | } 40 | } 41 | 42 | func TestHashCol(t *testing.T) { 43 | //println(1 << 32) 44 | k2 := make([]byte, 8) 45 | binary.BigEndian.PutUint64(k2, uint64(16_123_243)) 46 | k3 := make([]byte, 8) 47 | binary.BigEndian.PutUint64(k3, uint64(106_987_520)) 48 | println(hash(k2), hash(k3)) 49 | //mgdbywinfo uzmqkfjche 720448991 50 | println("str", hash([]byte("mgdbywinfo")), hash([]byte("uzmqkfjche"))) 51 | // 4_294_967_296 52 | sizet := 100_000_000 53 | m := make(map[uint32]int, sizet) 54 | for i := 0; i < sizet; i++ { 55 | k1 := make([]byte, 8) 56 | binary.BigEndian.PutUint64(k1, uint64(i)) 57 | h := hash(k1) 58 | if _, ok := m[h]; ok { 59 | println("collision", h, i, m[h]) 60 | break 61 | } 62 | m[h] = i 63 | } 64 | 65 | } 66 | func TestPower(t *testing.T) { 67 | 68 | p, v := NextPowerOf2(256) 69 | if p != 8 || v != 256 { 70 | t.Errorf("get p = %d,v=%d want 8,256", p, v) 71 | } 72 | 73 | p, v = NextPowerOf2(1023) 74 | if p != 10 || v != 1024 { 75 | t.Errorf("get p = %d,v=%d want 10,1024", p, v) 76 | } 77 | 78 | p, v = NextPowerOf2(4294967294) //2^32-1-1 79 | if p != 32 || v != 4294967295 { 80 | t.Errorf("get p = %d,v=%d want 33,4294967295", p, v) 81 | } 82 | 83 | p, v = NextPowerOf2(3) 84 | if p != 2 || v != 4 { 85 | t.Errorf("get p = %d,v=%d want 2,4", p, v) 86 | } 87 | p, v = NextPowerOf2(0) 88 | if p != 0 || v != 0 { 89 | t.Errorf("get p = %d,v=%d want 0,0", p, v) 90 | } 91 | } 92 | 93 | func TestCmd(t *testing.T) { 94 | err := DeleteStore("1") 95 | assert.NoError(t, err) 96 | 97 | s, err := Open(Dir("1")) 98 | assert.NoError(t, err) 99 | 100 | err = s.Set([]byte("hello"), []byte("go"), 0) 101 | assert.NoError(t, err) 102 | 103 | err = s.Set([]byte("hello"), []byte("world"), 0) 104 | assert.NoError(t, err) 105 | 106 | res, err := s.Get([]byte("hello")) 107 | assert.NoError(t, err) 108 | 109 | assert.Equal(t, true, bytes.Equal(res, []byte("world"))) 110 | 111 | assert.Equal(t, 1, s.Count()) 112 | 113 | err = s.Close() 114 | assert.NoError(t, err) 115 | s, err = Open(Dir("1")) 116 | assert.NoError(t, err) 117 | 118 | res, err = s.Get([]byte("hello")) 119 | assert.NoError(t, err) 120 | 121 | assert.Equal(t, true, bytes.Equal(res, []byte("world"))) 122 | assert.Equal(t, 1, s.Count()) 123 | 124 | deleted, err := s.Delete([]byte("hello")) 125 | assert.NoError(t, err) 126 | assert.True(t, deleted) 127 | assert.Equal(t, 0, s.Count()) 128 | 129 | counter := []byte("counter") 130 | 131 | cnt, err := s.Incr(counter, uint64(1)) 132 | assert.NoError(t, err) 133 | assert.Equal(t, 1, int(cnt)) 134 | cnt, err = s.Incr(counter, uint64(42)) 135 | assert.NoError(t, err) 136 | assert.Equal(t, 43, int(cnt)) 137 | 138 | cnt, err = s.Decr(counter, uint64(2)) 139 | assert.NoError(t, err) 140 | assert.Equal(t, 41, int(cnt)) 141 | 142 | //overflow 143 | cnt, err = s.Decr(counter, uint64(42)) 144 | assert.NoError(t, err) 145 | assert.Equal(t, uint64(18446744073709551615), uint64(cnt)) 146 | 147 | err = s.Close() 148 | assert.NoError(t, err) 149 | 150 | err = DeleteStore("1") 151 | assert.NoError(t, err) 152 | 153 | sniperBench(seed(100_000)) 154 | } 155 | 156 | func randKey(rnd *rand.Rand, n int) []byte { 157 | s := make([]byte, n) 158 | rnd.Read(s) 159 | for i := 0; i < n; i++ { 160 | s[i] = 'a' + (s[i] % 26) 161 | } 162 | return s 163 | } 164 | 165 | func seed(N int) ([][]byte, int) { 166 | seed := int64(1570109110136449000) //time.Now().UnixNano() //1570108152262917000 167 | // println(seed) 168 | rng := rand.New(rand.NewSource(seed)) 169 | 170 | K := 10 171 | 172 | fmt.Printf("\n") 173 | fmt.Printf("go version %s %s/%s\n", runtime.Version(), runtime.GOOS, runtime.GOARCH) 174 | fmt.Printf("\n") 175 | fmt.Printf(" number of cpus: %d\n", runtime.NumCPU()) 176 | fmt.Printf(" number of keys: %d\n", N) 177 | fmt.Printf(" keysize: %d\n", K) 178 | fmt.Printf(" random seed: %d\n", seed) 179 | 180 | fmt.Printf("\n") 181 | 182 | keysm := make(map[string]bool, N) 183 | for len(keysm) < N { 184 | keysm[string(randKey(rng, K))] = true 185 | } 186 | keys := make([][]byte, 0, N) 187 | for key := range keysm { 188 | keys = append(keys, []byte(key)) 189 | } 190 | return keys, N 191 | } 192 | 193 | func sniperBench(keys [][]byte, N int) { 194 | lotsa.Output = os.Stdout 195 | lotsa.MemUsage = true 196 | 197 | fmt.Println("-- sniper --") 198 | DeleteStore("1") 199 | s, err := Open(Dir("1")) //, SyncInterval(1*time.Second)) 200 | if err != nil { 201 | panic(err) 202 | } 203 | var ms runtime.MemStats 204 | runtime.ReadMemStats(&ms) 205 | 206 | fmt.Printf("Alloc = %v MiB Total = %v MiB\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024)) 207 | 208 | fmt.Print("set: ") 209 | coll := 0 210 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 211 | b := make([]byte, 8) 212 | binary.BigEndian.PutUint64(b, uint64(i)) 213 | //println("set", i, keys[i], b) 214 | err := s.Set(keys[i], b, 0) 215 | if err == ErrCollision { 216 | coll++ 217 | err = nil 218 | } 219 | if err != nil { 220 | panic(err) 221 | } 222 | }) 223 | runtime.ReadMemStats(&ms) 224 | 225 | fmt.Printf("Alloc = %v MiB Total = %v MiB Coll=%d\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024), coll) 226 | coll = 0 227 | fmt.Print("get: ") 228 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 229 | b, err := s.Get(keys[i]) 230 | if err != nil { 231 | println("errget", string(keys[i])) 232 | panic(err) 233 | } 234 | v := binary.BigEndian.Uint64(b) 235 | 236 | if uint64(i) != v { 237 | println("get error:", string(keys[i]), i, v) 238 | panic("bad news") 239 | } 240 | }) 241 | 242 | runtime.ReadMemStats(&ms) 243 | 244 | fmt.Printf("Alloc = %v MiB Total = %v MiB\n", (ms.Alloc / 1024 / 1024), (ms.TotalAlloc / 1024 / 1024)) 245 | 246 | fmt.Print("del: ") 247 | lotsa.Ops(N, runtime.NumCPU(), func(i, _ int) { 248 | s.Delete(keys[i]) 249 | }) 250 | err = DeleteStore("1") 251 | if err != nil { 252 | panic("bad news") 253 | } 254 | } 255 | 256 | func TestSync(t *testing.T) { 257 | sniperBench(seed(100_000)) 258 | } 259 | 260 | func TestSingleFile(t *testing.T) { 261 | DeleteStore("2") 262 | s, err := Open(Dir("2"), ChunksCollision(0), ChunksTotal(1)) 263 | assert.NoError(t, err) 264 | err = s.Set([]byte("mgdbywinfo"), []byte("1"), 0) 265 | assert.NoError(t, err) 266 | err = s.Set([]byte("uzmqkfjche"), []byte("2"), 0) 267 | assert.NoError(t, err) 268 | 269 | v, err := s.Get([]byte("uzmqkfjche")) 270 | assert.NoError(t, err) 271 | assert.EqualValues(t, []byte("2"), v) 272 | v, err = s.Get([]byte("mgdbywinfo")) 273 | assert.NoError(t, err) 274 | assert.EqualValues(t, []byte("1"), v) 275 | 276 | err = s.Close() 277 | assert.NoError(t, err) 278 | DeleteStore("2") 279 | } 280 | 281 | func TestBucket(t *testing.T) { 282 | DeleteStore("2") 283 | s, err := Open(Dir("2"), ChunksCollision(0), ChunksTotal(1)) 284 | assert.NoError(t, err) 285 | users, err := s.Bucket("users") 286 | assert.NoError(t, err) 287 | 288 | err = s.Put(users, []byte("01"), []byte("rob")) 289 | assert.NoError(t, err) 290 | 291 | err = s.Put(users, []byte("02"), []byte("bob")) 292 | assert.NoError(t, err) 293 | 294 | v, err := s.Get([]byte("users01")) 295 | assert.NoError(t, err) 296 | assert.Equal(t, []byte("rob"), v) 297 | 298 | v, err = s.Get([]byte("users02")) 299 | assert.NoError(t, err) 300 | assert.Equal(t, []byte("bob"), v) 301 | 302 | assert.Equal(t, []string{"02", "01"}, users.Keys(0, 0)) 303 | 304 | err = s.Close() 305 | assert.NoError(t, err) 306 | DeleteStore("2") 307 | } 308 | 309 | func TestEmptyKey(t *testing.T) { 310 | err := DeleteStore("1") 311 | assert.NoError(t, err) 312 | 313 | s, err := Open(Dir("1")) 314 | assert.NoError(t, err) 315 | 316 | err = s.Set([]byte(""), []byte("go"), 0) 317 | assert.NoError(t, err) 318 | 319 | err = s.Set([]byte(""), []byte("world"), 0) 320 | assert.NoError(t, err) 321 | 322 | res, err := s.Get([]byte("")) 323 | assert.NoError(t, err) 324 | 325 | assert.Equal(t, true, bytes.Equal(res, []byte("world"))) 326 | 327 | assert.Equal(t, 1, s.Count()) 328 | 329 | err = s.Close() 330 | assert.NoError(t, err) 331 | 332 | err = DeleteStore("1") 333 | assert.NoError(t, err) 334 | } 335 | 336 | func TestExpireKey(t *testing.T) { 337 | err := DeleteStore("1") 338 | assert.NoError(t, err) 339 | 340 | s, err := Open(Dir("1")) 341 | assert.NoError(t, err) 342 | 343 | unixtime := uint32(time.Now().Unix()) 344 | 345 | // set key with expire 1 sec 346 | err = s.Set([]byte("key1"), []byte("go"), unixtime+1) 347 | assert.NoError(t, err) 348 | 349 | // set key with expire 3 sec 350 | err = s.Set([]byte("key2"), []byte("world"), unixtime+3) 351 | assert.NoError(t, err) 352 | 353 | // try get key1 354 | res, err := s.Get([]byte("key1")) 355 | assert.NoError(t, err) 356 | 357 | assert.Equal(t, true, bytes.Equal(res, []byte("go"))) 358 | 359 | assert.Equal(t, 2, s.Count()) 360 | 361 | // sleep 2 sec, key1 should expired 362 | time.Sleep(time.Second * 2) 363 | 364 | res, err = s.Get([]byte("key1")) 365 | assert.Equal(t, ErrNotFound, err) 366 | 367 | assert.Equal(t, 1, s.Count()) 368 | 369 | // key2 must exist 370 | res, err = s.Get([]byte("key2")) 371 | assert.NoError(t, err) 372 | 373 | assert.Equal(t, true, bytes.Equal(res, []byte("world"))) 374 | 375 | // sleep 2 sec, key1 should expired 376 | time.Sleep(time.Second * 2) 377 | 378 | res, err = s.Get([]byte("key2")) 379 | assert.Equal(t, ErrNotFound, err) 380 | 381 | // all keys expired 382 | assert.Equal(t, 0, s.Count()) 383 | 384 | /* test Expire method */ 385 | unixtime = uint32(time.Now().Unix()) 386 | 387 | err = s.Set([]byte("key1"), []byte("go"), unixtime+1) 388 | assert.NoError(t, err) 389 | 390 | // shift time forward by 512s force expire delete old keys 391 | patch := monkey.Patch(time.Now, func() time.Time { return time.Unix(int64(unixtime)+512, 0) }) 392 | 393 | err = s.Expire() 394 | assert.NoError(t, err) 395 | patch.Unpatch() 396 | 397 | // all keys expired 398 | assert.Equal(t, 0, s.Count()) 399 | 400 | /* test touch */ 401 | unixtime = uint32(time.Now().Unix()) 402 | 403 | err = s.Set([]byte("key"), []byte("go"), unixtime+4) 404 | assert.NoError(t, err) 405 | 406 | // sleep 2 sec, key1 should stay 407 | time.Sleep(time.Second * 2) 408 | res, err = s.Get([]byte("key")) 409 | assert.NoError(t, err) 410 | 411 | unixtime = uint32(time.Now().Unix()) 412 | err = s.Touch([]byte("key"), unixtime+3) 413 | assert.NoError(t, err) 414 | 415 | // sleep 3 sec, key1 should stay 416 | time.Sleep(time.Second * 3) 417 | res, err = s.Get([]byte("key")) 418 | assert.NoError(t, err) 419 | 420 | // shift time forward by 512s force expire delete old keys 421 | patch = monkey.Patch(time.Now, func() time.Time { return time.Unix(int64(unixtime)+512, 0) }) 422 | err = s.Expire() 423 | assert.NoError(t, err) 424 | patch.Unpatch() 425 | 426 | // all keys expired 427 | assert.Equal(t, 0, s.Count()) 428 | 429 | err = s.Close() 430 | assert.NoError(t, err) 431 | 432 | err = DeleteStore("1") 433 | assert.NoError(t, err) 434 | } 435 | 436 | func getRandKey(rnd *rand.Rand, n int) []byte { 437 | s := make([]byte, n) 438 | rnd.Read(s) 439 | for i := 0; i < n; i++ { 440 | s[i] = 'a' + (s[i] % 26) 441 | } 442 | return s 443 | } 444 | 445 | func TestBackup(t *testing.T) { 446 | var backup = "data1.backup.gz" 447 | 448 | f, _ := os.Create(backup) 449 | defer f.Close() 450 | 451 | err := DeleteStore("1") 452 | assert.NoError(t, err) 453 | 454 | s, err := Open(Dir("1")) 455 | assert.NoError(t, err) 456 | 457 | seed := time.Now().UnixNano() 458 | rng := rand.New(rand.NewSource(seed)) 459 | coll := 0 460 | 461 | for i := 0; i < 1000000; i++ { 462 | b := make([]byte, 8) 463 | binary.BigEndian.PutUint64(b, uint64(i)) 464 | err := s.Set(getRandKey(rng, 10), b, 0) 465 | if err == ErrCollision { 466 | coll++ 467 | err = nil 468 | } 469 | if err != nil { 470 | panic(err) 471 | } 472 | } 473 | // count keys 474 | keys1 := s.Count() 475 | // create backup 476 | err = s.BackupGZ(f) 477 | if err != nil { 478 | panic(err) 479 | } 480 | err = s.Close() 481 | assert.NoError(t, err) 482 | 483 | err = DeleteStore("1") 484 | assert.NoError(t, err) 485 | 486 | s, err = Open(Dir("1")) 487 | assert.NoError(t, err) 488 | 489 | f.Seek(0, 0) 490 | err = s.RestoreGZ(f) 491 | if err != nil { 492 | panic(err) 493 | } 494 | keys2 := s.Count() 495 | assert.Equal(t, keys1, keys2) 496 | 497 | err = s.Close() 498 | assert.NoError(t, err) 499 | 500 | err = DeleteStore("1") 501 | assert.NoError(t, err) 502 | 503 | err = os.Remove(backup) 504 | assert.NoError(t, err) 505 | } 506 | 507 | // test run only when set enviroment variable "TESTCHUNK" 508 | // test use prepared chunk file "testchunk" 509 | func TestExpireChunk(t *testing.T) { 510 | if os.Getenv("TESTCHUNK") == "" { 511 | t.SkipNow() 512 | } 513 | ch := chunk{} 514 | err := ch.init("testchunk") 515 | assert.NoError(t, err) 516 | 517 | keys1 := ch.count() 518 | t.Logf("before expire keys count %d", keys1) 519 | 520 | newtime := time.Now().Add(time.Hour * 168) 521 | // shift time forward by week force expire delete old keys 522 | patch := monkey.Patch(time.Now, func() time.Time { return newtime }) 523 | ch.expirekeys(0) 524 | patch.Unpatch() 525 | keys2 := ch.count() 526 | t.Logf("after expire keys count %d", keys2) 527 | assert.Less(t, keys2, keys1, "Expire not work") 528 | 529 | err = ch.close() 530 | assert.NoError(t, err) 531 | } 532 | --------------------------------------------------------------------------------