├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── api.go ├── doc.go ├── doc_test.go ├── examples └── examples.go ├── go.mod ├── pudge.go └── pudge_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | .DS_Store 14 | .vscode 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.x" 5 | - "1.8" 6 | - "1.10.x" 7 | - master 8 | 9 | script: 10 | - go test -race -coverprofile=coverage.txt -covermode=atomic 11 | 12 | after_success: 13 | - bash <(curl -s https://codecov.io/bash) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Vadim Kulibaba 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Documentation](https://godoc.org/github.com/recoilme/pudge?status.svg)](https://godoc.org/github.com/recoilme/pudge) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/recoilme/pudge)](https://goreportcard.com/report/github.com/recoilme/pudge) 3 | [![Build Status](https://travis-ci.org/recoilme/pudge.svg?branch=master)](https://travis-ci.org/recoilme/pudge) 4 | [![Mentioned in Awesome Go](https://awesome.re/mentioned-badge-flat.svg)](https://github.com/avelino/awesome-go) 5 | 6 | Table of Contents 7 | ================= 8 | 9 | * [Description](#description) 10 | * [Usage](#usage) 11 | * [Cookbook](#cookbook) 12 | * [Disadvantages](#disadvantages) 13 | * [Motivation](#motivation) 14 | * [Benchmarks](#benchmarks) 15 | * [Test 1](#test-1) 16 | * [Test 4](#test-4) 17 | 18 | ## Description 19 | 20 | Package pudge is a fast and simple key/value store written using Go's standard library. 21 | 22 | It presents the following: 23 | * Supporting very efficient lookup, insertions and deletions 24 | * Performance is comparable to hash tables 25 | * Ability to get the data in sorted order, which enables additional operations like range scan 26 | * Select with limit/offset/from key, with ordering or by prefix 27 | * Safe for use in goroutines 28 | * Space efficient 29 | * Very short and simple codebase 30 | * Well tested, used in production 31 | 32 | ![pudge](https://avatars3.githubusercontent.com/u/417177?s=460&v=4) 33 | 34 | ## Usage 35 | 36 | 37 | ```golang 38 | package main 39 | 40 | import ( 41 | "log" 42 | 43 | "github.com/recoilme/pudge" 44 | ) 45 | 46 | func main() { 47 | // Close all database on exit 48 | defer pudge.CloseAll() 49 | 50 | // Set (directories will be created) 51 | pudge.Set("../test/test", "Hello", "World") 52 | 53 | // Get (lazy open db if needed) 54 | output := "" 55 | pudge.Get("../test/test", "Hello", &output) 56 | log.Println("Output:", output) 57 | 58 | ExampleSelect() 59 | } 60 | 61 | 62 | //ExampleSelect 63 | func ExampleSelect() { 64 | cfg := &pudge.Config{ 65 | SyncInterval: 1} // every second fsync 66 | db, err := pudge.Open("../test/db", cfg) 67 | if err != nil { 68 | log.Panic(err) 69 | } 70 | defer db.DeleteFile() 71 | type Point struct { 72 | X int 73 | Y int 74 | } 75 | for i := 100; i >= 0; i-- { 76 | p := &Point{X: i, Y: i} 77 | db.Set(i, p) 78 | } 79 | var point Point 80 | db.Get(8, &point) 81 | log.Println(point) 82 | // Output: {8 8} 83 | // Select 2 keys, from 7 in ascending order 84 | keys, _ := db.Keys(7, 2, 0, true) 85 | for _, key := range keys { 86 | var p Point 87 | db.Get(key, &p) 88 | log.Println(p) 89 | } 90 | // Output: {8 8} 91 | // Output: {9 9} 92 | } 93 | 94 | ``` 95 | 96 | ## Cookbook 97 | 98 | - Store data of any type. Pudge uses Gob encoder/decoder internally. No limits on keys/values size. 99 | 100 | ```golang 101 | pudge.Set("strings", "Hello", "World") 102 | pudge.Set("numbers", 1, 42) 103 | 104 | type User struct { 105 | Id int 106 | Name string 107 | } 108 | u := &User{Id: 1, Name: "name"} 109 | pudge.Set("users", u.Id, u) 110 | 111 | ``` 112 | - Pudge is stateless and safe for use in goroutines. You don't need to create/open files before use. Just write data to pudge, don't worry about state. [web server example](https://github.com/recoilme/pixel) 113 | 114 | - Pudge is parallel. Readers don't block readers, but a writer - does, but by the stateless nature of pudge it's safe to use multiples files for storages. 115 | 116 | ![Illustration from slowpoke (based on pudge)](https://camo.githubusercontent.com/a1b406485fa8cd52a98d820de706e3fd255941e9/68747470733a2f2f686162726173746f726167652e6f72672f776562742f79702f6f6b2f63332f79706f6b63333377702d70316a63657771346132323164693168752e706e67) 117 | 118 | 119 | - Default store system: like memcache + file storage. Pudge uses in-memory hashmap for keys, and writes values to files (no value data stored in memory). But you may use inmemory mode for values, with custom config: 120 | ```golang 121 | cfg = pudge.DefaultConfig() 122 | cfg.StoreMode = 2 123 | db, err := pudge.Open(dbPrefix+"/"+group, cfg) 124 | ... 125 | db.Counter(key, val) 126 | ``` 127 | In that case, all data is stored in memory and will be stored on disk only on Close. 128 | 129 | [Example server for highload, with http api](https://github.com/recoilme/bandit-server) 130 | 131 | - You may use pudge as an engine for creating databases. 132 | 133 | [Example database](https://github.com/recoilme/slowpoke) 134 | 135 | - Don't forget to close all opened databases on shutdown/kill. 136 | ```golang 137 | // Wait for interrupt signal to gracefully shutdown the server 138 | quit := make(chan os.Signal) 139 | signal.Notify(quit, os.Interrupt, os.Kill) 140 | <-quit 141 | log.Println("Shutdown Server ...") 142 | if err := pudge.CloseAll(); err != nil { 143 | log.Println("Pudge Shutdown err:", err) 144 | } 145 | ``` 146 | [example recovery function for gin framework](https://github.com/recoilme/bandit-server/blob/02e6eb9f89913bd68952ec35f6c37fc203d71fc2/bandit-server.go#L89) 147 | 148 | - Pudge has a primitive select/query engine. 149 | ```golang 150 | // Select 2 keys, from 7 in ascending order 151 | keys, _ := db.Keys(7, 2, 0, true) 152 | // select keys from db where key>7 order by keys asc limit 2 offset 0 153 | ``` 154 | 155 | - Pudge will work well on SSD or spined disks. Pudge doesn't eat memory or storage or your sandwich. No hidden compaction/rebalancing/resizing and so on tasks. No LSM Tree. No MMap. It's a very simple database with less than 500 LOC. It's good for [simple social network](https://github.com/recoilme/tgram) or highload system 156 | 157 | 158 | ## Disadvantages 159 | 160 | - No transaction system. All operations are isolated, but you don't may batching them with automatic rollback. 161 | - [Keys](https://godoc.org/github.com/recoilme/pudge#Keys) function (select/query engine) may be slow. Speed of query may vary from 10ms to 1sec per million keys. Pudge don't use BTree/Skiplist or Adaptive radix tree for store keys in ordered way on every insert. Ordering operation is "lazy" and run only if needed. 162 | - If you need storage or database for hundreds of millions keys - take a look at [Sniper](https://github.com/recoilme/sniper) or [b52](https://github.com/recoilme/b52). They are optimized for highload (pudge - not). 163 | - No fsync on every insert. Most of database fsync data by the timer too 164 | - Deleted data don't remove from physically (but upsert will try to reuse space). You may shrink database only with backup right now 165 | ```golang 166 | pudge.BackupAll("backup") 167 | ``` 168 | - Keys automatically convert to binary and ordered with binary comparator. It's simple for use, but ordering will not work correctly for negative numbers for example 169 | - Author of project don't work at Google or Facebook and his name not Howard Chu or Brad Fitzpatrick. But I'm open for issue or contributions. 170 | 171 | 172 | ## Motivation 173 | 174 | Some databases very well for writing. Some of the databases very well for reading. But [pudge is well balanced for both types of operations](https://github.com/recoilme/pogreb-bench). It has small [cute api](https://godoc.org/github.com/recoilme/pudge), and don't have hidden graveyards. It's just hashmap where values written in files. And you may use one database for in-memory/persistent storage in a stateless stressfree way 175 | 176 | 177 | ## Benchmarks 178 | 179 | [All tests here](https://github.com/recoilme/pogreb-bench) 180 | 181 | ***Some tests, MacBook Pro (Retina, 13-inch, Early 2015)*** 182 | 183 | 184 | 185 | ### Test 1 186 | Number of keys: 1000000 187 | Minimum key size: 16, maximum key size: 64 188 | Minimum value size: 128, maximum value size: 512 189 | Concurrency: 2 190 | 191 | 192 | | | pogreb | goleveldb | bolt | badgerdb | pudge | slowpoke | pudge(mem) | 193 | |-----------------------|---------|-----------|--------|----------|--------|----------|------------| 194 | | 1M (Put+Get), seconds | 187 | 38 | 126 | 34 | 23 | 23 | 2 | 195 | | 1M Put, ops/sec | 5336 | 34743 | 8054 | 33539 | 47298 | 46789 | 439581 | 196 | | 1M Get, ops/sec | 1782423 | 98406 | 499871 | 220597 | 499172 | 445783 | 1652069 | 197 | | FileSize,Mb | 568 | 357 | 552 | 487 | 358 | 358 | 358 | 198 | 199 | 200 | ### Test 4 201 | Number of keys: 10000000 202 | Key size: 8 203 | Value size: 16 204 | Concurrency: 100 205 | 206 | 207 | | | goleveldb | badgerdb | pudge | 208 | |-----------------------|-----------|----------|--------| 209 | | 10M (Put+Get), seconds| 165 | 120 | 243 | 210 | | 10M Put, ops/sec | 122933 | 135709 | 43843 | 211 | | 10M Get, ops/sec | 118722 | 214981 | 666067 | 212 | | FileSize,Mb | 312 | 1370 | 381 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package pudge 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "os" 7 | ) 8 | 9 | // DefaultConfig is default config 10 | var DefaultConfig = &Config{ 11 | FileMode: 0644, 12 | DirMode: 0755, 13 | SyncInterval: 0, 14 | StoreMode: 0} 15 | 16 | // Open return db object if it opened. 17 | // Create new db if not exist. 18 | // Read db to obj if exist. 19 | // Or error if any. 20 | // Default Config (if nil): &Config{FileMode: 0644, DirMode: 0755, SyncInterval: 0} 21 | func Open(f string, cfg *Config) (*Db, error) { 22 | if cfg == nil { 23 | cfg = DefaultConfig 24 | } 25 | dbs.RLock() 26 | db, ok := dbs.dbs[f] 27 | if ok { 28 | dbs.RUnlock() 29 | return db, nil 30 | } 31 | dbs.RUnlock() 32 | dbs.Lock() 33 | db, err := newDb(f, cfg) 34 | //log.Println("n", db.name, db.config.StoreMode) 35 | if err == nil { 36 | dbs.dbs[f] = db 37 | } 38 | dbs.Unlock() 39 | return db, err 40 | } 41 | 42 | // Set store any key value to db 43 | func (db *Db) Set(key, value interface{}) error { 44 | db.Lock() 45 | defer db.Unlock() 46 | k, err := KeyToBinary(key) 47 | if err != nil { 48 | return err 49 | } 50 | v, err := ValToBinary(value) 51 | if err != nil { 52 | return err 53 | } 54 | //log.Println("Set:", k, v) 55 | oldCmd, exists := db.vals[string(k)] 56 | //fmt.Println("StoreMode", db.config.StoreMode) 57 | if db.storemode == 2 { 58 | cmd := &Cmd{} 59 | cmd.Size = uint32(len(v)) 60 | cmd.Val = make([]byte, len(v)) 61 | copy(cmd.Val, v) 62 | db.vals[string(k)] = cmd 63 | } else { 64 | cmd, err := writeKeyVal(db.fk, db.fv, k, v, exists, oldCmd) 65 | if err != nil { 66 | return err 67 | } 68 | db.vals[string(k)] = cmd 69 | } 70 | if !exists { 71 | db.appendKey(k) 72 | } 73 | 74 | return err 75 | } 76 | 77 | // Get return value by key 78 | // Return error if any. 79 | func (db *Db) Get(key, value interface{}) error { 80 | db.RLock() 81 | defer db.RUnlock() 82 | k, err := KeyToBinary(key) 83 | if err != nil { 84 | return err 85 | } 86 | if val, ok := db.vals[string(k)]; ok { 87 | switch value.(type) { 88 | case *[]byte: 89 | b := make([]byte, val.Size) 90 | if db.storemode == 2 { 91 | copy(b, val.Val) 92 | } else { 93 | _, err := db.fv.ReadAt(b, int64(val.Seek)) 94 | if err != nil { 95 | return err 96 | } 97 | } 98 | *value.(*[]byte) = b 99 | return nil 100 | default: 101 | 102 | buf := new(bytes.Buffer) 103 | b := make([]byte, val.Size) 104 | if db.storemode == 2 { 105 | //fmt.Println(val) 106 | copy(b, val.Val) 107 | } else { 108 | _, err := db.fv.ReadAt(b, int64(val.Seek)) 109 | if err != nil { 110 | return err 111 | } 112 | } 113 | buf.Write(b) 114 | err = gob.NewDecoder(buf).Decode(value) 115 | return err 116 | } 117 | } 118 | return ErrKeyNotFound 119 | } 120 | 121 | // Close - sync & close files. 122 | // Return error if any. 123 | func (db *Db) Close() error { 124 | if db.cancelSyncer != nil { 125 | db.cancelSyncer() 126 | } 127 | db.Lock() 128 | defer db.Unlock() 129 | 130 | if db.storemode == 2 && db.name != "" { 131 | db.sort() 132 | keys := make([][]byte, len(db.keys)) 133 | 134 | copy(keys, db.keys) 135 | 136 | db.storemode = 0 137 | for _, k := range keys { 138 | if val, ok := db.vals[string(k)]; ok { 139 | writeKeyVal(db.fk, db.fv, k, val.Val, false, nil) 140 | } 141 | } 142 | } 143 | if db.fk != nil { 144 | err := db.fk.Sync() 145 | if err != nil { 146 | return err 147 | } 148 | err = db.fk.Close() 149 | if err != nil { 150 | return err 151 | } 152 | } 153 | if db.fv != nil { 154 | err := db.fv.Sync() 155 | if err != nil { 156 | return err 157 | } 158 | 159 | err = db.fv.Close() 160 | if err != nil { 161 | return err 162 | } 163 | } 164 | 165 | dbs.Lock() 166 | delete(dbs.dbs, db.name) 167 | dbs.Unlock() 168 | return nil 169 | } 170 | 171 | // CloseAll - close all opened Db 172 | func CloseAll() (err error) { 173 | dbs.Lock() 174 | stores := dbs.dbs 175 | dbs.Unlock() 176 | for _, db := range stores { 177 | err = db.Close() 178 | if err != nil { 179 | break 180 | } 181 | } 182 | 183 | return err 184 | } 185 | 186 | // DeleteFile close and delete file 187 | func (db *Db) DeleteFile() error { 188 | return DeleteFile(db.name) 189 | } 190 | 191 | // DeleteFile close db and delete file 192 | func DeleteFile(file string) error { 193 | if file == "" { 194 | return nil 195 | } 196 | dbs.Lock() 197 | db, ok := dbs.dbs[file] 198 | if ok { 199 | dbs.Unlock() 200 | err := db.Close() 201 | if err != nil { 202 | return err 203 | } 204 | } else { 205 | dbs.Unlock() 206 | } 207 | 208 | err := os.Remove(file) 209 | if err != nil { 210 | return err 211 | } 212 | err = os.Remove(file + ".idx") 213 | return err 214 | } 215 | 216 | // Has return true if key exists. 217 | // Return error if any. 218 | func (db *Db) Has(key interface{}) (bool, error) { 219 | db.RLock() 220 | defer db.RUnlock() 221 | k, err := KeyToBinary(key) 222 | if err != nil { 223 | return false, err 224 | } 225 | _, has := db.vals[string(k)] 226 | return has, nil 227 | } 228 | 229 | // FileSize returns the total size of the disk storage used by the DB. 230 | func (db *Db) FileSize() (int64, error) { 231 | db.RLock() 232 | defer db.RUnlock() 233 | var err error 234 | is, err := db.fk.Stat() 235 | if err != nil { 236 | return -1, err 237 | } 238 | ds, err := db.fv.Stat() 239 | if err != nil { 240 | return -1, err 241 | } 242 | return is.Size() + ds.Size(), nil 243 | } 244 | 245 | // Count returns the number of items in the Db. 246 | func (db *Db) Count() (int, error) { 247 | db.RLock() 248 | defer db.RUnlock() 249 | return len(db.keys), nil 250 | } 251 | 252 | // Delete remove key 253 | // Returns error if key not found 254 | func (db *Db) Delete(key interface{}) error { 255 | db.Lock() 256 | defer db.Unlock() 257 | k, err := KeyToBinary(key) 258 | if err != nil { 259 | return err 260 | } 261 | if _, ok := db.vals[string(k)]; ok { 262 | delete(db.vals, string(k)) 263 | db.deleteFromKeys(k) 264 | writeKey(db.fk, 1, 0, 0, k, -1) 265 | return nil 266 | } 267 | return ErrKeyNotFound 268 | } 269 | 270 | // KeysByPrefix return keys with prefix 271 | // in ascending or descending order (false - descending,true - ascending) 272 | // if limit == 0 return all keys 273 | // if offset > 0 - skip offset records 274 | // If from not nil - return keys after from (from not included) 275 | func (db *Db) KeysByPrefix(prefix []byte, limit, offset int, asc bool) ([][]byte, error) { 276 | //log.Println("KeysByPrefix") 277 | db.RLock() 278 | defer db.RUnlock() 279 | // resulting array 280 | arr := make([][]byte, 0, 0) 281 | found := db.foundPref(prefix, asc) 282 | if found < 0 || found >= len(db.keys) || !startFrom(db.keys[found], prefix) { 283 | //not found 284 | return arr, ErrKeyNotFound 285 | } 286 | 287 | start, end := checkInterval(found, limit, offset, 0, len(db.keys), asc) 288 | 289 | if start < 0 || start >= len(db.keys) { 290 | return arr, nil 291 | } 292 | 293 | if asc { 294 | for i := start; i <= end; i++ { 295 | if !startFrom(db.keys[i], prefix) { 296 | break 297 | } 298 | arr = append(arr, db.keys[i]) 299 | } 300 | } else { 301 | for i := start; i >= end; i-- { 302 | if !startFrom(db.keys[i], prefix) { 303 | break 304 | } 305 | arr = append(arr, db.keys[i]) 306 | } 307 | } 308 | return arr, nil 309 | } 310 | 311 | // Keys return keys in ascending or descending order (false - descending,true - ascending) 312 | // if limit == 0 return all keys 313 | // if offset > 0 - skip offset records 314 | // If from not nil - return keys after from (from not included) 315 | func (db *Db) Keys(from interface{}, limit, offset int, asc bool) ([][]byte, error) { 316 | // resulting array 317 | //log.Println("pudge", from, from == nil) 318 | arr := make([][]byte, 0, 0) 319 | excludeFrom := 0 320 | if from != nil { 321 | excludeFrom = 1 322 | 323 | k, err := KeyToBinary(from) 324 | //log.Println(bytes.Equal(k[len(k)-1:], []byte("*"))) 325 | if err != nil { 326 | return arr, err 327 | } 328 | if len(k) > 1 && bytes.Equal(k[len(k)-1:], []byte("*")) { 329 | byteOrStr := false 330 | switch from.(type) { 331 | case []byte: 332 | byteOrStr = true 333 | case string: 334 | byteOrStr = true 335 | } 336 | if byteOrStr { 337 | prefix := make([]byte, len(k)-1) 338 | copy(prefix, k) 339 | return db.KeysByPrefix(prefix, limit, offset, asc) 340 | } 341 | } 342 | } 343 | db.RLock() 344 | defer db.RUnlock() 345 | find, err := db.findKey(from, asc) 346 | if from != nil && err != nil { 347 | return nil, err 348 | } 349 | start, end := checkInterval(find, limit, offset, excludeFrom, len(db.keys), asc) 350 | if start < 0 || start >= len(db.keys) { 351 | return arr, nil 352 | } 353 | 354 | if asc { 355 | for i := start; i <= end; i++ { 356 | arr = append(arr, db.keys[i]) 357 | } 358 | } else { 359 | for i := start; i >= end; i-- { 360 | arr = append(arr, db.keys[i]) 361 | } 362 | } 363 | return arr, nil 364 | } 365 | 366 | // Counter return int64 incremented on incr 367 | func (db *Db) Counter(key interface{}, incr int) (int64, error) { 368 | mutex.Lock() 369 | var counter int64 370 | err := db.Get(key, &counter) 371 | if err != nil && err != ErrKeyNotFound { 372 | return -1, err 373 | } 374 | //mutex.Lock() 375 | counter = counter + int64(incr) 376 | //mutex.Unlock() 377 | err = db.Set(key, counter) 378 | mutex.Unlock() 379 | return counter, err 380 | } 381 | 382 | // Set store any key value to db with opening if needed 383 | func Set(f string, key, value interface{}) error { 384 | db, err := Open(f, nil) 385 | if err != nil { 386 | return err 387 | } 388 | return db.Set(key, value) 389 | } 390 | 391 | // Sets store vals and keys 392 | // Use it for mass insertion 393 | // every pair must contain key and value 394 | func Sets(file string, pairs []interface{}) (err error) { 395 | db, err := Open(file, nil) 396 | if err != nil { 397 | return err 398 | } 399 | for i := range pairs { 400 | if i%2 != 0 { 401 | // on odd - append val and store key 402 | if pairs[i] == nil || pairs[i-1] == nil { 403 | break 404 | } 405 | err = db.Set(pairs[i-1], pairs[i]) 406 | if err != nil { 407 | break 408 | } 409 | } 410 | } 411 | return err 412 | } 413 | 414 | // Get return value by key with opening if needed 415 | // Return error if any. 416 | func Get(f string, key, value interface{}) error { 417 | db, err := Open(f, nil) 418 | if err != nil { 419 | return err 420 | } 421 | return db.Get(key, value) 422 | } 423 | 424 | // Gets return key/value pairs in random order 425 | // result contains key and value 426 | // Gets not return error if key not found 427 | // If no keys found return empty result 428 | func Gets(file string, keys []interface{}) (result [][]byte) { 429 | db, err := Open(file, nil) 430 | if err != nil { 431 | return nil 432 | } 433 | for _, key := range keys { 434 | var v []byte 435 | err := db.Get(key, &v) 436 | if err == nil { 437 | k, err := KeyToBinary(key) 438 | if err == nil { 439 | val, err := ValToBinary(v) 440 | if err == nil { 441 | result = append(result, k) 442 | result = append(result, val) 443 | } 444 | } 445 | } 446 | } 447 | return result 448 | } 449 | 450 | // Counter return int64 incremented on incr with lazy open 451 | func Counter(f string, key interface{}, incr int) (int64, error) { 452 | db, err := Open(f, nil) 453 | if err != nil { 454 | return 0, err 455 | } 456 | return db.Counter(key, incr) 457 | } 458 | 459 | // Delete remove key 460 | // Returns error if key not found 461 | func Delete(f string, key interface{}) error { 462 | db, err := Open(f, nil) 463 | if err != nil { 464 | return err 465 | } 466 | return db.Delete(key) 467 | } 468 | 469 | // Keys return keys in ascending or descending order (false - descending,true - ascending) 470 | // if limit == 0 return all keys 471 | // if offset > 0 - skip offset records 472 | // If from not nil - return keys after from (from not included) 473 | func Keys(f string, from interface{}, limit, offset int, asc bool) ([][]byte, error) { 474 | db, err := Open(f, nil) 475 | if err != nil { 476 | return nil, err 477 | } 478 | return db.Keys(from, limit, offset, asc) 479 | } 480 | 481 | // Has return true if key exists. 482 | // Return error if any. 483 | func Has(f string, key interface{}) (bool, error) { 484 | db, err := Open(f, nil) 485 | if err != nil { 486 | return false, err 487 | } 488 | return db.Has(key) 489 | } 490 | 491 | // Count returns the number of items in the Db. 492 | func Count(f string) (int, error) { 493 | db, err := Open(f, nil) 494 | if err != nil { 495 | return -1, err 496 | } 497 | return db.Count() 498 | } 499 | 500 | // Close - sync & close files. 501 | // Return error if any. 502 | func Close(f string) error { 503 | db, err := Open(f, nil) 504 | if err != nil { 505 | return err 506 | } 507 | return db.Close() 508 | } 509 | 510 | // BackupAll - backup all opened Db 511 | // if dir not set it will be backup 512 | // delete old backup file before run 513 | // ignore all errors 514 | func BackupAll(dir string) (err error) { 515 | if dir == "" { 516 | dir = "backup" 517 | } 518 | dbs.Lock() 519 | stores := dbs.dbs 520 | dbs.Unlock() 521 | //tmp := make(map[string]string) 522 | for _, db := range stores { 523 | backup := dir + "/" + db.name 524 | DeleteFile(backup) 525 | keys, err := db.Keys(nil, 0, 0, true) 526 | if err == nil { 527 | for _, k := range keys { 528 | var b []byte 529 | db.Get(k, &b) 530 | Set(backup, k, b) 531 | } 532 | } 533 | Close(backup) 534 | } 535 | 536 | return err 537 | } 538 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package pudge implements a low-level key/value store in pure Go. 2 | // Keys stored in memory, Value stored on disk 3 | // 4 | // Usage 5 | // 6 | // package main 7 | // 8 | // import ( 9 | // "log" 10 | // 11 | // "github.com/recoilme/pudge" 12 | // ) 13 | // 14 | // func main() { 15 | // cfg := pudge.DefaultConfig() 16 | // cfg.SyncInterval = 0 //disable every second fsync 17 | // db, err := pudge.Open("../test/db", cfg) 18 | // if err != nil { 19 | // log.Panic(err) 20 | // } 21 | // defer db.DeleteFile() 22 | // type Point struct { 23 | // X int 24 | // Y int 25 | // } 26 | // for i := 100; i >= 0; i-- { 27 | // p := &Point{X: i, Y: i} 28 | // db.Set(i, p) 29 | // } 30 | // var point Point 31 | // db.Get(8, &point) 32 | // log.Println(point) 33 | // // Output: {8 8} 34 | // } 35 | package pudge 36 | -------------------------------------------------------------------------------- /doc_test.go: -------------------------------------------------------------------------------- 1 | package pudge 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | func ExampleOpen() { 9 | cfg := &Config{ 10 | SyncInterval: 0} //disable every second fsync 11 | db, err := Open("test/db", cfg) 12 | if err != nil { 13 | log.Panic(err) 14 | } 15 | defer db.DeleteFile() 16 | type Point struct { 17 | X int 18 | Y int 19 | } 20 | for i := 100; i >= 0; i-- { 21 | p := &Point{X: i, Y: i} 22 | db.Set(i, p) 23 | } 24 | var point Point 25 | db.Get(8, &point) 26 | fmt.Println(point) 27 | // Output: {8 8} 28 | } 29 | func ExampleSet() { 30 | Set("test/test", "Hello", "World") 31 | defer CloseAll() 32 | } 33 | 34 | func ExampleGet() { 35 | Set("test/test", "Hello", "World") 36 | output := "" 37 | Get("test/test", "Hello", &output) 38 | defer CloseAll() 39 | fmt.Println(output) 40 | // Output: World 41 | DeleteFile("test/test") 42 | } 43 | -------------------------------------------------------------------------------- /examples/examples.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | 6 | "github.com/recoilme/pudge" 7 | ) 8 | 9 | func main() { 10 | ExampleSet() 11 | ExampleGet() 12 | ExampleDelete() 13 | ExampleDeleteFile() 14 | ExampleOpen() 15 | ExampleInMemoryWithoutPersist() 16 | } 17 | 18 | //ExampleSet lazy 19 | func ExampleSet() { 20 | pudge.Set("../test/test", "Hello", "World") 21 | defer pudge.CloseAll() 22 | } 23 | 24 | //ExampleGet lazy 25 | func ExampleGet() { 26 | output := "" 27 | pudge.Get("../test/test", "Hello", &output) 28 | log.Println("Output:", output) 29 | // Output: World 30 | defer pudge.CloseAll() 31 | } 32 | 33 | //ExampleDelete lazy 34 | func ExampleDelete() { 35 | err := pudge.Delete("../test/test", "Hello") 36 | if err == pudge.ErrKeyNotFound { 37 | log.Println(err) 38 | } 39 | } 40 | 41 | //ExampleDeleteFile lazy 42 | func ExampleDeleteFile() { 43 | err := pudge.DeleteFile("../test/test") 44 | if err != nil { 45 | log.Panic(err) 46 | } 47 | } 48 | 49 | //ExampleOpen complex example 50 | func ExampleOpen() { 51 | cfg := &pudge.Config{ 52 | SyncInterval: 0} //disable every second fsync 53 | db, err := pudge.Open("../test/db", cfg) 54 | if err != nil { 55 | log.Panic(err) 56 | } 57 | defer db.DeleteFile() 58 | type Point struct { 59 | X int 60 | Y int 61 | } 62 | for i := 100; i >= 0; i-- { 63 | p := &Point{X: i, Y: i} 64 | db.Set(i, p) 65 | } 66 | var point Point 67 | db.Get(8, &point) 68 | log.Println(point) 69 | // Output: {8 8} 70 | // Select 2 keys, from 7 in ascending order 71 | keys, _ := db.Keys(7, 2, 0, true) 72 | for _, key := range keys { 73 | var p Point 74 | db.Get(key, &p) 75 | log.Println(p) 76 | } 77 | // Output: {8 8} 78 | // Output: {9 9} 79 | } 80 | 81 | // ExampleInMemoryWithoutPersist -if file is empty in storemode 2 - without persist 82 | func ExampleInMemoryWithoutPersist() { 83 | cfg := &pudge.Config{StoreMode: 2} //in memory 84 | db, err := pudge.Open("", cfg) // if file is empty in storemode 2 - without persist 85 | if err != nil { 86 | log.Panic(err) 87 | } 88 | defer db.Close() //remove from memory 89 | type Point struct { 90 | X int 91 | Y int 92 | } 93 | for i := 100; i >= 0; i-- { 94 | p := &Point{X: i, Y: i} 95 | db.Set(i, p) 96 | } 97 | var point Point 98 | db.Get(89, &point) 99 | log.Println(point) 100 | // Output: {89 89} 101 | keys, _ := db.Keys(77, 2, 0, true) 102 | for _, key := range keys { 103 | var p Point 104 | db.Get(key, &p) 105 | log.Println(p) 106 | } 107 | // Output: {78 78} 108 | // Output: {79 79} 109 | } 110 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/recoilme/pudge 2 | 3 | go 1.13 4 | -------------------------------------------------------------------------------- /pudge.go: -------------------------------------------------------------------------------- 1 | package pudge 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/binary" 7 | "encoding/gob" 8 | "errors" 9 | "io/ioutil" 10 | "os" 11 | "path/filepath" 12 | "sort" 13 | "sync" 14 | "time" 15 | ) 16 | 17 | var ( 18 | dbs struct { 19 | sync.RWMutex 20 | dbs map[string]*Db 21 | } 22 | // ErrKeyNotFound - key not found 23 | ErrKeyNotFound = errors.New("Error: key not found") 24 | mutex = &sync.RWMutex{} 25 | ) 26 | 27 | // Db represent database 28 | type Db struct { 29 | sync.RWMutex 30 | name string 31 | fk *os.File 32 | fv *os.File 33 | keys [][]byte 34 | vals map[string]*Cmd 35 | cancelSyncer context.CancelFunc 36 | storemode int 37 | } 38 | 39 | // Cmd represent keys and vals addresses 40 | type Cmd struct { 41 | Seek uint32 42 | Size uint32 43 | KeySeek uint32 44 | Val []byte 45 | } 46 | 47 | // Config fo db 48 | // Default FileMode = 0644 49 | // Default DirMode = 0755 50 | // Default SyncInterval = 0 sec, 0 - disable sync (os will sync, typically 30 sec or so) 51 | // If StroreMode==2 && file == "" - pure inmemory mode 52 | type Config struct { 53 | FileMode int // 0644 54 | DirMode int // 0755 55 | SyncInterval int // in seconds 56 | StoreMode int // 0 - file first, 2 - memory first(with persist on close), 2 - with empty file - memory without persist 57 | } 58 | 59 | func init() { 60 | dbs.dbs = make(map[string]*Db) 61 | } 62 | 63 | func newDb(f string, cfg *Config) (*Db, error) { 64 | //fmt.Println("newdb2:", f, cfg.StoreMode) 65 | var err error 66 | // create 67 | db := new(Db) 68 | db.Lock() 69 | defer db.Unlock() 70 | // init 71 | db.name = f 72 | db.keys = make([][]byte, 0) 73 | db.vals = make(map[string]*Cmd) 74 | db.storemode = cfg.StoreMode 75 | 76 | // Apply default values 77 | if cfg.FileMode == 0 { 78 | cfg.FileMode = DefaultConfig.FileMode 79 | } 80 | if cfg.DirMode == 0 { 81 | cfg.DirMode = DefaultConfig.DirMode 82 | } 83 | if db.storemode == 2 && db.name == "" { 84 | return db, nil 85 | } 86 | _, err = os.Stat(f) 87 | if err != nil { 88 | // file not exists - create dirs if any 89 | if os.IsNotExist(err) { 90 | if filepath.Dir(f) != "." { 91 | err = os.MkdirAll(filepath.Dir(f), os.FileMode(cfg.DirMode)) 92 | if err != nil { 93 | return nil, err 94 | } 95 | } 96 | } else { 97 | return nil, err 98 | } 99 | } 100 | db.fv, err = os.OpenFile(f, os.O_CREATE|os.O_RDWR, os.FileMode(cfg.FileMode)) 101 | if err != nil { 102 | return nil, err 103 | } 104 | db.fk, err = os.OpenFile(f+".idx", os.O_CREATE|os.O_RDWR, os.FileMode(cfg.FileMode)) 105 | if err != nil { 106 | return nil, err 107 | } 108 | //read keys 109 | buf := new(bytes.Buffer) 110 | b, err := ioutil.ReadAll(db.fk) 111 | if err != nil { 112 | return nil, err 113 | } 114 | buf.Write(b) 115 | var readSeek uint32 116 | for buf.Len() > 0 { 117 | _ = uint8(buf.Next(1)[0]) //format version 118 | t := uint8(buf.Next(1)[0]) 119 | seek := binary.BigEndian.Uint32(buf.Next(4)) 120 | size := binary.BigEndian.Uint32(buf.Next(4)) 121 | _ = buf.Next(4) //time 122 | sizeKey := int(binary.BigEndian.Uint16(buf.Next(2))) 123 | key := buf.Next(sizeKey) 124 | strkey := string(key) 125 | cmd := &Cmd{ 126 | Seek: seek, 127 | Size: size, 128 | KeySeek: readSeek, 129 | } 130 | if db.storemode == 2 { 131 | cmd.Val = make([]byte, size) 132 | db.fv.ReadAt(cmd.Val, int64(seek)) 133 | } 134 | readSeek += uint32(16 + sizeKey) 135 | switch t { 136 | case 0: 137 | if _, exists := db.vals[strkey]; !exists { 138 | //write new key at keys store 139 | db.appendKey(key) 140 | } 141 | db.vals[strkey] = cmd 142 | case 1: 143 | delete(db.vals, strkey) 144 | db.deleteFromKeys(key) 145 | } 146 | } 147 | 148 | if cfg.SyncInterval > 0 { 149 | db.backgroundManager(cfg.SyncInterval) 150 | } 151 | return db, err 152 | } 153 | 154 | // backgroundManager runs continuously in the background and performs various 155 | // operations such as syncing to disk. 156 | func (db *Db) backgroundManager(interval int) { 157 | ctx, cancel := context.WithCancel(context.Background()) 158 | db.cancelSyncer = cancel 159 | go func() { 160 | for { 161 | select { 162 | case <-ctx.Done(): 163 | return 164 | default: 165 | db.Lock() 166 | db.fk.Sync() 167 | db.fv.Sync() 168 | db.Unlock() 169 | time.Sleep(time.Duration(interval) * time.Second) 170 | } 171 | } 172 | }() 173 | } 174 | 175 | //appendKey insert key in slice 176 | func (db *Db) appendKey(b []byte) { 177 | //log.Println("append") 178 | db.keys = append(db.keys, b) 179 | return 180 | } 181 | 182 | // deleteFromKeys delete key from slice keys 183 | func (db *Db) deleteFromKeys(b []byte) { 184 | found := db.found(b, true) 185 | if found < len(db.keys) { 186 | if bytes.Equal(db.keys[found], b) { 187 | db.keys = append(db.keys[:found], db.keys[found+1:]...) 188 | } 189 | } 190 | } 191 | 192 | func (db *Db) sort() { 193 | if !sort.SliceIsSorted(db.keys, db.lessBinary) { 194 | //log.Println("sort") 195 | sort.Slice(db.keys, db.lessBinary) 196 | } 197 | } 198 | 199 | func (db *Db) lessBinary(i, j int) bool { 200 | return bytes.Compare(db.keys[i], db.keys[j]) <= 0 201 | } 202 | 203 | //found return binary search result with sort order 204 | func (db *Db) found(b []byte, asc bool) int { 205 | db.sort() 206 | //if asc { 207 | return sort.Search(len(db.keys), func(i int) bool { 208 | return bytes.Compare(db.keys[i], b) >= 0 209 | }) 210 | //} 211 | //return sort.Search(len(db.keys), func(i int) bool { 212 | // return bytes.Compare(db.keys[i], b) <= 0 213 | //}) 214 | } 215 | 216 | // KeyToBinary return key in bytes 217 | func KeyToBinary(v interface{}) ([]byte, error) { 218 | var err error 219 | 220 | switch v.(type) { 221 | case []byte: 222 | return v.([]byte), nil 223 | case bool, float32, float64, complex64, complex128, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64: 224 | buf := new(bytes.Buffer) 225 | err = binary.Write(buf, binary.BigEndian, v) 226 | return buf.Bytes(), err 227 | case int: 228 | val := uint64(v.(int)) 229 | p := make([]byte, 8) 230 | p[0] = byte(val >> 56) 231 | p[1] = byte(val >> 48) 232 | p[2] = byte(val >> 40) 233 | p[3] = byte(val >> 32) 234 | p[4] = byte(val >> 24) 235 | p[5] = byte(val >> 16) 236 | p[6] = byte(val >> 8) 237 | p[7] = byte(val) 238 | return p, err 239 | case string: 240 | return []byte(v.(string)), nil 241 | default: 242 | buf := new(bytes.Buffer) 243 | err = gob.NewEncoder(buf).Encode(v) 244 | return buf.Bytes(), err 245 | } 246 | } 247 | 248 | // ValToBinary return value in bytes 249 | func ValToBinary(v interface{}) ([]byte, error) { 250 | var err error 251 | switch v.(type) { 252 | case []byte: 253 | return v.([]byte), nil 254 | default: 255 | buf := new(bytes.Buffer) 256 | err = gob.NewEncoder(buf).Encode(v) 257 | if err != nil { 258 | return nil, err 259 | } 260 | return buf.Bytes(), err 261 | } 262 | } 263 | 264 | func writeKeyVal(fk, fv *os.File, readKey, writeVal []byte, exists bool, oldCmd *Cmd) (cmd *Cmd, err error) { 265 | 266 | var seek, newSeek int64 267 | cmd = &Cmd{Size: uint32(len(writeVal))} 268 | if exists { 269 | // key exists 270 | cmd.Seek = oldCmd.Seek 271 | cmd.KeySeek = oldCmd.KeySeek 272 | if oldCmd.Size >= uint32(len(writeVal)) { 273 | //write at old seek new value 274 | _, _, err = writeAtPos(fv, writeVal, int64(oldCmd.Seek)) 275 | } else { 276 | //write at new seek (at the end of file) 277 | seek, _, err = writeAtPos(fv, writeVal, int64(-1)) 278 | cmd.Seek = uint32(seek) 279 | } 280 | if err == nil { 281 | // if no error - store key at KeySeek 282 | newSeek, err = writeKey(fk, 0, cmd.Seek, cmd.Size, []byte(readKey), int64(cmd.KeySeek)) 283 | cmd.KeySeek = uint32(newSeek) 284 | } 285 | } else { 286 | // new key 287 | // write value at the end of file 288 | seek, _, err = writeAtPos(fv, writeVal, int64(-1)) 289 | cmd.Seek = uint32(seek) 290 | if err == nil { 291 | newSeek, err = writeKey(fk, 0, cmd.Seek, cmd.Size, []byte(readKey), -1) 292 | cmd.KeySeek = uint32(newSeek) 293 | } 294 | } 295 | return cmd, err 296 | } 297 | 298 | // if pos<0 store at the end of file 299 | func writeAtPos(f *os.File, b []byte, pos int64) (seek int64, n int, err error) { 300 | seek = pos 301 | if pos < 0 { 302 | seek, err = f.Seek(0, 2) 303 | if err != nil { 304 | return seek, 0, err 305 | } 306 | } 307 | n, err = f.WriteAt(b, seek) 308 | if err != nil { 309 | return seek, n, err 310 | } 311 | return seek, n, err 312 | } 313 | 314 | // writeKey create buffer and store key with val address and size 315 | func writeKey(fk *os.File, t uint8, seek, size uint32, key []byte, keySeek int64) (newSeek int64, err error) { 316 | //get buf from pool 317 | buf := new(bytes.Buffer) 318 | buf.Reset() 319 | buf.Grow(16 + len(key)) 320 | 321 | //encode 322 | binary.Write(buf, binary.BigEndian, uint8(0)) //1byte version 323 | binary.Write(buf, binary.BigEndian, t) //1byte command code(0-set,1-delete) 324 | binary.Write(buf, binary.BigEndian, seek) //4byte seek 325 | binary.Write(buf, binary.BigEndian, size) //4byte size 326 | binary.Write(buf, binary.BigEndian, uint32(time.Now().Unix())) //4byte timestamp 327 | binary.Write(buf, binary.BigEndian, uint16(len(key))) //2byte key size 328 | buf.Write(key) //key 329 | 330 | if keySeek < 0 { 331 | newSeek, _, err = writeAtPos(fk, buf.Bytes(), int64(-1)) 332 | } else { 333 | newSeek, _, err = writeAtPos(fk, buf.Bytes(), int64(keySeek)) 334 | } 335 | 336 | return newSeek, err 337 | } 338 | 339 | // findKey return index of first key in ascending mode 340 | // findKey return index of last key in descending mode 341 | // findKey return 0 or len-1 in case of nil key 342 | func (db *Db) findKey(key interface{}, asc bool) (int, error) { 343 | if key == nil { 344 | db.sort() 345 | if asc { 346 | return 0, ErrKeyNotFound 347 | } 348 | return len(db.keys) - 1, ErrKeyNotFound 349 | } 350 | k, err := KeyToBinary(key) 351 | if err != nil { 352 | return -1, err 353 | } 354 | found := db.found(k, asc) 355 | //log.Println("found", found) 356 | // check found 357 | if found >= len(db.keys) { 358 | return -1, ErrKeyNotFound 359 | } 360 | if !bytes.Equal(db.keys[found], k) { 361 | return -1, ErrKeyNotFound 362 | } 363 | return found, nil 364 | } 365 | 366 | // startFrom return is a start from b in binary 367 | func startFrom(a, b []byte) bool { 368 | if a == nil || b == nil { 369 | return false 370 | } 371 | if len(a) < len(b) { 372 | return false 373 | } 374 | return bytes.Compare(a[:len(b)], b) == 0 375 | } 376 | 377 | func (db *Db) foundPref(b []byte, asc bool) int { 378 | db.sort() 379 | if asc { 380 | return sort.Search(len(db.keys), func(i int) bool { 381 | return bytes.Compare(db.keys[i], b) >= 0 382 | }) 383 | } 384 | var j int 385 | for j = len(db.keys) - 1; j >= 0; j-- { 386 | if startFrom(db.keys[j], b) { 387 | break 388 | } 389 | } 390 | return j 391 | } 392 | 393 | func checkInterval(find, limit, offset, excludeFrom, len int, asc bool) (int, int) { 394 | end := 0 395 | start := find 396 | 397 | if asc { 398 | start += (offset + excludeFrom) 399 | if limit == 0 { 400 | end = len - excludeFrom 401 | } else { 402 | end = (start + limit - 1) 403 | } 404 | } else { 405 | start -= (offset + excludeFrom) 406 | if limit == 0 { 407 | end = 0 408 | } else { 409 | end = start - limit + 1 410 | } 411 | } 412 | 413 | if end < 0 { 414 | end = 0 415 | } 416 | if end >= len { 417 | end = len - 1 418 | } 419 | 420 | return start, end 421 | } 422 | -------------------------------------------------------------------------------- /pudge_test.go: -------------------------------------------------------------------------------- 1 | package pudge 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "math/rand" 7 | "strconv" 8 | "sync" 9 | "testing" 10 | "time" 11 | ) 12 | 13 | const ( 14 | f = "test/1" 15 | ) 16 | 17 | func nrandbin(n int) [][]byte { 18 | i := make([][]byte, n) 19 | for ind := range i { 20 | bin, _ := KeyToBinary(rand.Int()) 21 | i[ind] = bin 22 | } 23 | return i 24 | } 25 | 26 | func TestConfig(t *testing.T) { 27 | _, err := Open("", nil) 28 | if err == nil { 29 | t.Error("Open empty must error") 30 | } 31 | db, err := Open(f, &Config{FileMode: 0777, DirMode: 0777}) 32 | if err != nil { 33 | t.Error(err) 34 | } 35 | err = db.DeleteFile() 36 | if err != nil { 37 | t.Error(err) 38 | } 39 | } 40 | 41 | func TestOpen(t *testing.T) { 42 | db, err := Open(f, nil) 43 | if err != nil { 44 | t.Error(err) 45 | } 46 | err = db.Set(1, 1) 47 | if err != nil { 48 | t.Error(err) 49 | } 50 | db.Close() 51 | db, err = Open(f, nil) 52 | if err != nil { 53 | t.Error(err) 54 | } 55 | v := 1 56 | err = db.Get(1, &v) 57 | if err != nil { 58 | t.Error(err) 59 | } 60 | if v != 1 { 61 | t.Error("not 1") 62 | } 63 | err = db.DeleteFile() 64 | if err != nil { 65 | t.Error(err) 66 | } 67 | } 68 | 69 | func TestSet(t *testing.T) { 70 | db, err := Open(f, nil) 71 | if err != nil { 72 | t.Error(err) 73 | } 74 | err = db.Set(1, 1) 75 | if err != nil { 76 | t.Error(err) 77 | } 78 | err = db.DeleteFile() 79 | if err != nil { 80 | t.Error(err) 81 | } 82 | } 83 | 84 | func TestGet(t *testing.T) { 85 | db, err := Open(f, nil) 86 | if err != nil { 87 | t.Error(err) 88 | } 89 | err = db.Set(1, 1) 90 | if err != nil { 91 | t.Error(err) 92 | } 93 | var val int 94 | err = db.Get(1, &val) 95 | if err != nil { 96 | t.Error(err) 97 | return 98 | } 99 | 100 | if val != 1 { 101 | t.Error("val != 1", val) 102 | return 103 | } 104 | db.Close() 105 | 106 | err = db.DeleteFile() 107 | if err != nil { 108 | t.Error(err) 109 | } 110 | } 111 | 112 | func TestKeys(t *testing.T) { 113 | 114 | f := "test/keys.db" 115 | 116 | db, err := Open(f, nil) 117 | if err != nil { 118 | t.Error(err) 119 | } 120 | defer db.Close() 121 | append := func(i int) { 122 | k := []byte(fmt.Sprintf("%02d", i)) 123 | v := []byte("Val:" + strconv.Itoa(i)) 124 | db.Set(k, v) 125 | } 126 | for i := 22; i >= 1; i-- { 127 | append(i) 128 | } 129 | 130 | //ascending 131 | res, err := db.Keys(nil, 0, 0, true) 132 | if err != nil { 133 | t.Error(err) 134 | } 135 | var s = "" 136 | for _, r := range res { 137 | s += string(r) 138 | } 139 | if s != "01020304050607080910111213141516171819202122" { 140 | t.Error("not asc", s) 141 | } 142 | 143 | //descending 144 | resdesc, _ := db.Keys(nil, 0, 0, false) 145 | s = "" 146 | for _, r := range resdesc { 147 | s += string(r) 148 | } 149 | if s != "22212019181716151413121110090807060504030201" { 150 | t.Error("not desc", s) 151 | } 152 | 153 | //offset limit asc 154 | reslimit, _ := db.Keys(nil, 2, 2, true) 155 | 156 | s = "" 157 | for _, r := range reslimit { 158 | s += string(r) 159 | } 160 | if s != "0304" { 161 | t.Error("not off", s) 162 | } 163 | 164 | //offset limit desc 165 | reslimitdesc, _ := db.Keys(nil, 2, 2, false) 166 | 167 | s = "" 168 | for _, r := range reslimitdesc { 169 | s += string(r) 170 | } 171 | if s != "2019" { 172 | t.Error("not off desc", s) 173 | } 174 | 175 | //from byte asc 176 | resfromasc, _ := db.Keys([]byte("10"), 2, 2, true) 177 | s = "" 178 | for _, r := range resfromasc { 179 | s += string(r) 180 | } 181 | if s != "1314" { 182 | t.Error("not off asc", s) 183 | } 184 | 185 | //from byte desc 186 | resfromdesc, _ := db.Keys([]byte("10"), 2, 2, false) 187 | s = "" 188 | for _, r := range resfromdesc { 189 | s += string(r) 190 | } 191 | if s != "0706" { 192 | t.Error("not off desc", s) 193 | } 194 | 195 | //from byte desc 196 | resnotfound, _ := db.Keys([]byte("100"), 2, 2, false) 197 | s = "" 198 | for _, r := range resnotfound { 199 | s += string(r) 200 | } 201 | if s != "" { 202 | t.Error("resnotfound", s) 203 | } 204 | 205 | //from byte not eq 206 | resnoteq, _ := db.Keys([]byte("33"), 2, 2, false) 207 | s = "" 208 | for _, r := range resnoteq { 209 | s += string(r) 210 | } 211 | if s != "" { 212 | t.Error("resnoteq ", s) 213 | } 214 | 215 | //by prefix 216 | respref, _ := db.Keys([]byte("2*"), 4, 0, false) 217 | s = "" 218 | for _, r := range respref { 219 | s += string(r) 220 | } 221 | if s != "222120" { 222 | t.Error("respref", s) 223 | } 224 | 225 | //by prefix2 226 | respref2, _ := db.Keys([]byte("1*"), 2, 0, false) 227 | s = "" 228 | for _, r := range respref2 { 229 | s += string(r) 230 | } 231 | if s != "1918" { 232 | t.Error("respref2", s) 233 | } 234 | 235 | //by prefixasc 236 | resprefasc, err := db.Keys([]byte("1*"), 2, 0, true) 237 | s = "" 238 | for _, r := range resprefasc { 239 | s += string(r) 240 | } 241 | if s != "1011" { 242 | t.Error("resprefasc", s, err) 243 | } 244 | 245 | //by prefixasc2 246 | resprefasc2, err := db.Keys([]byte("1*"), 0, 0, true) 247 | s = "" 248 | for _, r := range resprefasc2 { 249 | s += string(r) 250 | } 251 | if s != "10111213141516171819" { 252 | t.Error("resprefasc2", s, err) 253 | } 254 | DeleteFile(f) 255 | } 256 | 257 | func TestCounter(t *testing.T) { 258 | f := "test/TestCnt.db" 259 | var counter int64 260 | var err error 261 | db, err := Open(f, nil) 262 | if err != nil { 263 | t.Error(err) 264 | } 265 | key := []byte("postcounter") 266 | for i := 0; i < 10; i++ { 267 | counter, err = db.Counter(key, 1) 268 | if err != nil { 269 | t.Error(err) 270 | } 271 | //log.Println(counter, err) 272 | } 273 | //return 274 | for i := 0; i < 10; i++ { 275 | counter, err = db.Counter(key, 1) 276 | if err != nil { 277 | t.Error(err) 278 | } 279 | } 280 | if counter != 20 { 281 | t.Error("counter!=20") 282 | } 283 | key2 := []byte("counter2") 284 | for i := 0; i < 5; i++ { 285 | counter, _ = db.Counter(key2, 1) 286 | } 287 | 288 | for i := 0; i < 5; i++ { 289 | counter, err = db.Counter(key2, 1) 290 | if err != nil { 291 | t.Error(err) 292 | } 293 | } 294 | if counter != 10 { 295 | t.Error("counter!=10") 296 | } 297 | db.DeleteFile() 298 | } 299 | 300 | func TestLazyOpen(t *testing.T) { 301 | Set(f, 2, 42) 302 | var val int 303 | CloseAll() 304 | Get(f, 2, &val) 305 | if val != 42 { 306 | t.Error("not 42") 307 | } 308 | DeleteFile(f) 309 | } 310 | 311 | func TestAsync(t *testing.T) { 312 | len := 5000 313 | file := "test/async.db" 314 | DeleteFile(file) 315 | defer CloseAll() 316 | 317 | messages := make(chan int) 318 | readmessages := make(chan string) 319 | var wg sync.WaitGroup 320 | 321 | append := func(i int) { 322 | defer wg.Done() 323 | k := ("Key:" + strconv.Itoa(i)) 324 | v := ("Val:" + strconv.Itoa(i)) 325 | err := Set(file, []byte(k), []byte(v)) 326 | if err != nil { 327 | t.Error(err) 328 | } 329 | messages <- i 330 | } 331 | 332 | read := func(i int) { 333 | defer wg.Done() 334 | k := ("Key:" + strconv.Itoa(i)) 335 | v := ("Val:" + strconv.Itoa(i)) 336 | var b []byte 337 | Get(file, []byte(k), &b) 338 | 339 | if string(b) != string(v) { 340 | t.Error("not mutch", string(b), string(v)) 341 | } 342 | readmessages <- fmt.Sprintf("read N:%d content:%s", i, string(b)) 343 | } 344 | 345 | for i := 1; i <= len; i++ { 346 | wg.Add(1) 347 | go append(i) 348 | 349 | } 350 | 351 | go func() { 352 | for i := range messages { 353 | _ = i 354 | //fmt.Println(i) 355 | } 356 | }() 357 | 358 | go func() { 359 | for i := range readmessages { 360 | _ = i 361 | //fmt.Println(i) 362 | } 363 | }() 364 | 365 | wg.Wait() 366 | 367 | for i := 1; i <= len; i++ { 368 | 369 | wg.Add(1) 370 | go read(i) 371 | } 372 | wg.Wait() 373 | DeleteFile(file) 374 | } 375 | 376 | func TestStoreMode(t *testing.T) { 377 | cfg := &Config{StoreMode: 2} 378 | db, err := Open("test/sm", cfg) 379 | if err != nil { 380 | t.Error(err) 381 | } 382 | err = db.Set(1, 2) 383 | if err != nil { 384 | t.Error(err) 385 | } 386 | var v int 387 | err = db.Get(1, &v) 388 | if err != nil { 389 | t.Error(err) 390 | } 391 | if v != 2 { 392 | t.Error("not 2") 393 | } 394 | db.Set(1, 42) 395 | db.Close() 396 | db, err = Open("test/sm", nil) 397 | if err != nil { 398 | t.Error(err) 399 | } 400 | err = db.Get(1, &v) 401 | if err != nil { 402 | t.Error(err) 403 | } 404 | if v != 42 { 405 | t.Error("not 42") 406 | } 407 | DeleteFile("test/sm") 408 | //log.Println(v) 409 | //CloseAll() 410 | } 411 | 412 | // run go test -bench=Store -benchmem 413 | func BenchmarkStore(b *testing.B) { 414 | b.StopTimer() 415 | nums := nrandbin(b.N) 416 | 417 | DeleteFile(f) 418 | 419 | rm, err := Open(f, nil) 420 | if err != nil { 421 | b.Error("Open", err) 422 | } 423 | b.SetBytes(8) 424 | b.StartTimer() 425 | for _, v := range nums { 426 | err = rm.Set(v, v) 427 | if err != nil { 428 | b.Error("Set", err) 429 | } 430 | } 431 | b.StopTimer() 432 | err = DeleteFile(f) 433 | if err != nil { 434 | b.Error("DeleteFile", err) 435 | } 436 | } 437 | 438 | func BenchmarkLoad(b *testing.B) { 439 | b.StopTimer() 440 | nums := nrandbin(b.N) 441 | DeleteFile(f) 442 | rm, err := Open(f, nil) 443 | if err != nil { 444 | b.Error("Open", err) 445 | } 446 | for _, v := range nums { 447 | err = rm.Set(v, v) 448 | if err != nil { 449 | b.Error("Set", err) 450 | } 451 | } 452 | var wg sync.WaitGroup 453 | read := func(db *Db, key []byte) { 454 | defer wg.Done() 455 | var b []byte 456 | db.Get(key, &b) 457 | } 458 | b.StartTimer() 459 | for i := 0; i < b.N; i++ { 460 | wg.Add(1) 461 | go read(rm, nums[i]) 462 | //var v []byte 463 | //err := rm.Get(nums[i], &v) 464 | //if err != nil { 465 | // log.Println("Get", err, nums[i], &v) 466 | // break 467 | //} 468 | } 469 | wg.Wait() 470 | b.StopTimer() 471 | log.Println(rm.Count()) 472 | DeleteFile(f) 473 | } 474 | 475 | func TestBackup(t *testing.T) { 476 | Set("test/1", 1, 2) 477 | Set("test/4", "4", "4") 478 | BackupAll("") 479 | DeleteFile("test/1") 480 | DeleteFile("test/4") 481 | var v1 int 482 | Get("backup/test/1", 1, &v1) 483 | if v1 != 2 { 484 | t.Error("not 2") 485 | } 486 | var v2 = "" 487 | Get("backup/test/4", "4", &v2) 488 | if v2 != "4" { 489 | t.Error("not 4") 490 | } 491 | DeleteFile("backup/test/1") 492 | DeleteFile("backup/test/4") 493 | CloseAll() 494 | } 495 | 496 | func TestMultipleOpen(t *testing.T) { 497 | for i := 1; i < 100000; i++ { 498 | Set("test/m", i, i) 499 | } 500 | Close("test/m") 501 | for i := 1; i < 100; i++ { 502 | go Open("test/m", nil) 503 | } 504 | time.Sleep(1 * time.Millisecond) 505 | DeleteFile("test/m") 506 | } 507 | 508 | func TestInMemory(t *testing.T) { 509 | DefaultConfig.StoreMode = 2 510 | 511 | for i := 0; i < 10; i++ { 512 | fileName := fmt.Sprintf("test/inmemory%d", i) 513 | err := Set(fileName, i, i) 514 | if err != nil { 515 | t.Error(err) 516 | } 517 | } 518 | 519 | err := CloseAll() 520 | if err != nil { 521 | t.Error(err) 522 | } 523 | for i := 0; i < 10; i++ { 524 | fileName := fmt.Sprintf("test/inmemory%d", i) 525 | c, e := Count(fileName) 526 | if c == 0 || e != nil { 527 | t.Error("no persist") 528 | break 529 | } 530 | DeleteFile(fileName) 531 | } 532 | } 533 | 534 | func TestInMemoryWithoutPersist(t *testing.T) { 535 | DefaultConfig.StoreMode = 2 536 | 537 | for i := 0; i < 10000; i++ { 538 | err := Set("", i, i) 539 | if err != nil { 540 | t.Error(err) 541 | } 542 | } 543 | j := 0 544 | Get("", 8, &j) 545 | if j != 8 { 546 | t.Error("j must be 8", j) 547 | } 548 | cnt, e := Count("") 549 | if cnt != 10000 { 550 | t.Error("count must be 10000", cnt, e) 551 | } 552 | for i := 0; i < 10; i++ { 553 | c, e := Count("") 554 | if c != 10000 || e != nil { 555 | t.Error("no persist", c, e) 556 | break 557 | } 558 | } 559 | noerr := DeleteFile("") 560 | if noerr != nil { 561 | t.Error("Delete empty file", noerr) 562 | } 563 | noerr = Close("") 564 | if noerr != nil { 565 | t.Error("Close empty file", noerr) 566 | } 567 | jj := 0 568 | notpresent := Get("", 8, &jj) 569 | if jj == 8 { 570 | t.Error("jj must be 0", j) 571 | } 572 | if notpresent != ErrKeyNotFound { 573 | t.Error("Must be Error: key not found error", notpresent) 574 | } 575 | 576 | } 577 | 578 | func Test42(t *testing.T) { 579 | DefaultConfig.StoreMode = 0 580 | f := "test/int64" 581 | for i := 1; i < 64; i++ { 582 | Set(f, int64(i), int64(i)) 583 | } 584 | keys, err := Keys(f, int64(42), 100, 0, true) 585 | if err != nil { 586 | t.Error(err) 587 | } 588 | if len(keys) != 21 { 589 | t.Error("not 21", len(keys)) 590 | } 591 | DeleteFile(f) 592 | } 593 | 594 | func TestSetsGets(t *testing.T) { 595 | f := "test/setsgets" 596 | DeleteFile(f) 597 | var pairs []interface{} 598 | 599 | for i := 1; i < 64; i++ { 600 | pairs = append(pairs, i) 601 | pairs = append(pairs, i+1) 602 | } 603 | err := Sets(f, pairs) 604 | if err != nil { 605 | t.Error("Sets err", err) 606 | } 607 | var v int 608 | err = Get(f, 63, &v) 609 | if err != nil || v != 64 { 610 | t.Error("Sets err", err, v) 611 | } 612 | //Sets 613 | var pairsBin []interface{} 614 | for i := 0; i < 100; i++ { 615 | k := []byte(fmt.Sprintf("%04d", i)) 616 | pairsBin = append(pairsBin, k) 617 | pairsBin = append(pairsBin, k) 618 | } 619 | err = Sets(f, pairsBin) 620 | if err != nil { 621 | t.Error("Sets err", err) 622 | } 623 | var s []byte 624 | err = Get(f, []byte("0063"), &s) 625 | if err != nil || string(s) != "0063" { 626 | t.Error("Sets err", err, s) 627 | } 628 | var keys []interface{} 629 | for i := 2; i < 4; i++ { 630 | k := []byte(fmt.Sprintf("%04d", i)) 631 | keys = append(keys, k) 632 | } 633 | err = Get(f, []byte("0068"), &s) 634 | if err != nil { 635 | t.Error("Sets err", err) 636 | } 637 | 638 | result := Gets(f, keys) 639 | if len(result) != 4 { 640 | t.Error("Sets err not 4") 641 | } 642 | DeleteFile(f) 643 | } 644 | 645 | func TestEmptyKeysByPrefix(t *testing.T) { 646 | // first pass with asc = false 647 | db, err := Open(f, nil) 648 | if err != nil { 649 | t.Error(err) 650 | } 651 | 652 | prefix, err := KeyToBinary("non-existant-prefix") 653 | if err != nil { 654 | t.Error(err) 655 | } 656 | 657 | keys, err := db.KeysByPrefix(prefix, 0, 0, false) 658 | if err != ErrKeyNotFound { 659 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 660 | } 661 | 662 | if len(keys) != 0 { 663 | t.Errorf("Wrong amount of keys for empty database: %d", len(keys)) 664 | } 665 | 666 | db.Set("some-key", "some-value") 667 | 668 | keys, err = db.KeysByPrefix(prefix, 0, 0, false) 669 | if err != ErrKeyNotFound { 670 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 671 | } 672 | 673 | if len(keys) != 0 { 674 | t.Errorf("Wrong amount of keys: %d", len(keys)) 675 | } 676 | 677 | err = db.DeleteFile() 678 | if err != nil { 679 | t.Fatal(err) 680 | } 681 | 682 | // second pass with asc = true 683 | db, err = Open(f, nil) 684 | if err != nil { 685 | t.Error(err) 686 | } 687 | 688 | prefix, err = KeyToBinary("non-existant-prefix") 689 | if err != nil { 690 | t.Error(err) 691 | } 692 | 693 | keys, err = db.KeysByPrefix(prefix, 0, 0, true) 694 | if err != ErrKeyNotFound { 695 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 696 | } 697 | 698 | if len(keys) != 0 { 699 | t.Errorf("Wrong amount of keys for empty database: %d", len(keys)) 700 | } 701 | 702 | db.Set("some-key", "some-value") 703 | 704 | keys, err = db.KeysByPrefix(prefix, 0, 0, true) 705 | if err != ErrKeyNotFound { 706 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 707 | } 708 | 709 | if len(keys) != 0 { 710 | t.Errorf("Wrong amount of keys: %d", len(keys)) 711 | } 712 | 713 | // third pass with non-zero offset 714 | db, err = Open(f, nil) 715 | if err != nil { 716 | t.Error(err) 717 | } 718 | 719 | prefix, err = KeyToBinary("non-existant-prefix") 720 | if err != nil { 721 | t.Error(err) 722 | } 723 | 724 | keys, err = db.KeysByPrefix(prefix, 0, 1, false) 725 | if err != ErrKeyNotFound { 726 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 727 | } 728 | 729 | if len(keys) != 0 { 730 | t.Errorf("Wrong amount of keys for empty database: %d", len(keys)) 731 | } 732 | 733 | db.Set("some-key", "some-value") 734 | 735 | keys, err = db.KeysByPrefix(prefix, 0, 1, false) 736 | if err != ErrKeyNotFound { 737 | t.Errorf("Error must be ErrKeyNotFound got: %s", err) 738 | } 739 | 740 | if len(keys) != 0 { 741 | t.Errorf("Wrong amount of keys: %d", len(keys)) 742 | } 743 | 744 | err = db.DeleteFile() 745 | if err != nil { 746 | t.Fatal(err) 747 | } 748 | 749 | } 750 | --------------------------------------------------------------------------------