├── img ├── cli.png ├── logo.png └── architecture.png ├── .github └── workflows │ └── go.yml ├── store_test.go ├── .gitignore ├── meta.go ├── go.mod ├── tx_zset_test.go ├── LICENSE ├── config.go ├── evict.go ├── record_test.go ├── db_load_test.go ├── tx_str_test.go ├── tx_str.go ├── tx_set_test.go ├── tx_hash_test.go ├── store.go ├── go.sum ├── flashdb.go ├── db_load.go ├── tx_hash.go ├── record.go ├── tx_set.go ├── txn.go ├── tx_zset.go └── README.md /img/cli.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arriqaaq/flashdb/HEAD/img/cli.png -------------------------------------------------------------------------------- /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arriqaaq/flashdb/HEAD/img/logo.png -------------------------------------------------------------------------------- /img/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arriqaaq/flashdb/HEAD/img/architecture.png -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | # This workflow will build a golang project 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go 3 | 4 | name: Go 5 | 6 | on: 7 | push: 8 | branches: [ "main" ] 9 | pull_request: 10 | branches: [ "main" ] 11 | 12 | jobs: 13 | 14 | build: 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v3 18 | 19 | - name: Set up Go 20 | uses: actions/setup-go@v4 21 | with: 22 | go-version: '1.20' 23 | 24 | - name: Build 25 | run: go build -v ./... 26 | 27 | - name: Test 28 | run: go test -v ./... 29 | -------------------------------------------------------------------------------- /store_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFlashDB_StringStore(t *testing.T) { 11 | s := newStrStore() 12 | 13 | for i := 1; i <= 1000; i++ { 14 | key := fmt.Sprintf("key_%d", i) 15 | value := fmt.Sprintf("value_%d", i) 16 | s.Insert([]byte(key), []byte(value)) 17 | } 18 | 19 | keys := s.Keys() 20 | assert.Equal(t, 1000, len(keys)) 21 | for i := 1; i <= 1000; i++ { 22 | key := fmt.Sprintf("key_%d", i) 23 | value := fmt.Sprintf("value_%d", i) 24 | val, err := s.get(key) 25 | assert.NoError(t, err) 26 | assert.NotEqual(t, value, val) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Binaries for programs and plugins 3 | *.exe 4 | *.exe~ 5 | *.dll 6 | *.so 7 | *.dylib 8 | bin 9 | testbin/* 10 | 11 | # Test binary, build with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Kubernetes Generated files - skip generated files, except for vendored files 18 | 19 | !vendor/**/zz_generated.* 20 | 21 | # editor and IDE paraphernalia 22 | .idea 23 | *.swp 24 | *.swo 25 | *~ 26 | .vscode/ 27 | 28 | controlplane/bin/ 29 | 30 | # Go specific 31 | vendor 32 | coverage.txt 33 | coverage.xml 34 | 35 | 36 | # IDE ignores 37 | **/.idea 38 | **/*.bak 39 | .vscode/* 40 | *.iml 41 | .DS* 42 | **/.DS_Store 43 | .vscode/* 44 | .idea/ 45 | tmp/ 46 | -------------------------------------------------------------------------------- /meta.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | type DataType = string 4 | 5 | const ( 6 | String DataType = "String" 7 | Hash DataType = "Hash" 8 | Set DataType = "Set" 9 | ZSet DataType = "ZSet" 10 | ) 11 | 12 | const ( 13 | StringRecord uint16 = iota 14 | HashRecord 15 | SetRecord 16 | ZSetRecord 17 | ) 18 | 19 | // The operations on Strings. 20 | const ( 21 | StringSet uint16 = iota 22 | StringRem 23 | StringExpire 24 | ) 25 | 26 | // The operations on Hash. 27 | const ( 28 | HashHSet uint16 = iota 29 | HashHDel 30 | HashHClear 31 | HashHExpire 32 | ) 33 | 34 | // The operations on Set. 35 | const ( 36 | SetSAdd uint16 = iota 37 | SetSRem 38 | SetSMove 39 | SetSClear 40 | SetSExpire 41 | ) 42 | 43 | // The operations on Sorted Set. 44 | const ( 45 | ZSetZAdd uint16 = iota 46 | ZSetZRem 47 | ZSetZClear 48 | ZSetZExpire 49 | ) 50 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/arriqaaq/flashdb 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/arriqaaq/aol v0.1.2 7 | github.com/arriqaaq/art v0.1.2 8 | github.com/arriqaaq/hash v0.1.2 9 | github.com/arriqaaq/set v0.1.2 10 | github.com/arriqaaq/zset v0.1.2 11 | github.com/gomodule/redigo v1.8.8 12 | github.com/pelletier/go-toml v1.9.4 13 | github.com/peterh/liner v1.2.2 14 | github.com/stretchr/testify v1.7.1 15 | github.com/tidwall/redcon v1.4.4 16 | ) 17 | 18 | require ( 19 | github.com/davecgh/go-spew v1.1.0 // indirect 20 | github.com/mattn/go-runewidth v0.0.3 // indirect 21 | github.com/pmezard/go-difflib v1.0.0 // indirect 22 | github.com/tidwall/btree v1.1.0 // indirect 23 | github.com/tidwall/match v1.1.1 // indirect 24 | golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 // indirect 25 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /tx_zset_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/arriqaaq/aol" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestRoseDB_ZSet(t *testing.T) { 12 | db := getTestDB() 13 | defer db.Close() 14 | defer os.RemoveAll(tmpDir) 15 | 16 | logPath := "tmp/" 17 | l, err := aol.Open(logPath, nil) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | defer l.Close() 22 | db.log = l 23 | db.persist = true 24 | defer os.RemoveAll("tmp/") 25 | 26 | if err := db.Update(func(tx *Tx) error { 27 | err = tx.ZAdd(testKey, 1, "foo") 28 | assert.NoError(t, err) 29 | err = tx.ZAdd(testKey, 2, "bar") 30 | assert.NoError(t, err) 31 | err = tx.ZAdd(testKey, 3, "baz") 32 | assert.NoError(t, err) 33 | 34 | return nil 35 | }); err != nil { 36 | t.Fatal(err) 37 | } 38 | 39 | if err := db.View(func(tx *Tx) error { 40 | _, s := tx.ZScore(testKey, "foo") 41 | assert.Equal(t, 1.0, s) 42 | 43 | assert.Equal(t, 3, tx.ZCard(testKey)) 44 | return nil 45 | }); err != nil { 46 | t.Fatal(err) 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 arriqaaq 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "strconv" 5 | "time" 6 | ) 7 | 8 | const ( 9 | DefaultAddr = "127.0.0.1:8000" 10 | DefaultMaxKeySize = uint32(1 * 1024) 11 | DefaultMaxValueSize = uint32(8 * 1024) 12 | ) 13 | 14 | type Config struct { 15 | Addr string `json:"addr" toml:"addr"` 16 | Path string `json:"path" toml:"path"` // dir path for append-only logs 17 | EvictionInterval int `json:"eviction_interval" toml:"eviction_interval"` // in seconds 18 | // NoSync disables fsync after writes. This is less durable and puts the 19 | // log at risk of data loss when there's a server crash. 20 | NoSync bool 21 | } 22 | 23 | func (c *Config) validate() { 24 | if c.Addr == "" { 25 | c.Addr = DefaultAddr 26 | } 27 | } 28 | 29 | func (c *Config) evictionInterval() time.Duration { 30 | return time.Duration(c.EvictionInterval) * time.Second 31 | } 32 | 33 | func DefaultConfig() *Config { 34 | return &Config{ 35 | Addr: DefaultAddr, 36 | Path: "/tmp/flashdb", 37 | EvictionInterval: 10, 38 | } 39 | } 40 | 41 | func float64ToStr(val float64) string { 42 | return strconv.FormatFloat(val, 'f', -1, 64) 43 | } 44 | 45 | func strToFloat64(val string) (float64, error) { 46 | return strconv.ParseFloat(val, 64) 47 | } 48 | -------------------------------------------------------------------------------- /evict.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "math/rand" 5 | "runtime" 6 | "time" 7 | 8 | "github.com/arriqaaq/hash" 9 | ) 10 | 11 | const ( 12 | MinimumStartupTime = 500 * time.Millisecond 13 | MaximumStartupTime = 2 * MinimumStartupTime 14 | ) 15 | 16 | // Used to put a random delay before start of each shard, so as to not 17 | // let various shards lock at the same time 18 | func startupDelay() time.Duration { 19 | rand := rand.New(rand.NewSource(time.Now().UnixNano())) 20 | d, delta := MinimumStartupTime, (MaximumStartupTime - MinimumStartupTime) 21 | if delta > 0 { 22 | d += time.Duration(rand.Int63n(int64(delta))) 23 | } 24 | return d 25 | } 26 | 27 | type evictor interface { 28 | run(cache *hash.Hash) 29 | stop() 30 | } 31 | 32 | func newSweeperWithStore(s store, sweepTime time.Duration) evictor { 33 | var swp = &sweeper{ 34 | interval: sweepTime, 35 | stopC: make(chan bool), 36 | store: s, 37 | } 38 | runtime.SetFinalizer(swp, stopSweeper) 39 | return swp 40 | } 41 | 42 | func stopSweeper(c evictor) { 43 | c.stop() 44 | } 45 | 46 | type sweeper struct { 47 | store store 48 | interval time.Duration 49 | stopC chan bool 50 | } 51 | 52 | func (s *sweeper) run(cache *hash.Hash) { 53 | <-time.After(startupDelay()) 54 | ticker := time.NewTicker(s.interval) 55 | for { 56 | select { 57 | case <-ticker.C: 58 | s.store.evict(cache) 59 | case <-s.stopC: 60 | ticker.Stop() 61 | return 62 | } 63 | } 64 | } 65 | 66 | func (s *sweeper) stop() { 67 | s.stopC <- true 68 | } 69 | -------------------------------------------------------------------------------- /record_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/arriqaaq/aol" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func makeRecords(n int) []*record { 13 | rec := make([]*record, 0, n) 14 | for i := 1; i <= n; i++ { 15 | key := fmt.Sprintf("key_%d", i) 16 | value := fmt.Sprintf("value_%d", i) 17 | member := fmt.Sprintf("member_%d", i) 18 | rec = append(rec, newRecordWithValue([]byte(key), []byte(member), []byte(value), ZSetRecord, ZSetZAdd)) 19 | } 20 | return rec 21 | } 22 | 23 | func TestFlashDB_AOL(t *testing.T) { 24 | logPath := "tmp/" 25 | l, err := aol.Open(logPath, nil) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | defer l.Close() 30 | defer os.RemoveAll(logPath) 31 | 32 | recs := makeRecords(100) 33 | for _, r := range recs { 34 | data, err := r.encode() 35 | assert.NoError(t, err) 36 | l.Write(data) 37 | } 38 | 39 | var lastRecord *record 40 | 41 | segs := l.Segments() 42 | for i := 1; i <= segs; i++ { 43 | j := 0 44 | for { 45 | data, err := l.Read(uint64(i), uint64(j)) 46 | if err != nil { 47 | if err == aol.ErrEOF { 48 | break 49 | } 50 | t.Fatalf("expected %v, got %v", nil, err) 51 | } 52 | res, err := decode(data) 53 | assert.NoError(t, err) 54 | lastRecord = res 55 | j++ 56 | } 57 | } 58 | 59 | assert.Equal(t, "key_100", string(lastRecord.meta.key)) 60 | assert.Equal(t, "member_100", string(lastRecord.meta.member)) 61 | assert.Equal(t, "value_100", string(lastRecord.meta.value)) 62 | } 63 | -------------------------------------------------------------------------------- /db_load_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/arriqaaq/aol" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func makeLoadRecords(n int, db *FlashDB) { 13 | for i := 1; i <= n; i++ { 14 | key := fmt.Sprintf("key_%d", i) 15 | member := fmt.Sprintf("member_%d", i) 16 | value := fmt.Sprintf("value_%d", i) 17 | db.Update(func(tx *Tx) error { 18 | tx.HSet(key, member, value) 19 | tx.SAdd(key, member) 20 | tx.Set(key, member) 21 | tx.ZAdd(key, 10.0, member) 22 | return nil 23 | }) 24 | 25 | } 26 | } 27 | 28 | func TestFlashDB_load(t *testing.T) { 29 | db := getTestDB() 30 | logPath := "tmp/" 31 | l, err := aol.Open(logPath, nil) 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | db.log = l 36 | db.persist = true 37 | defer os.RemoveAll("tmp/") 38 | 39 | makeLoadRecords(10, db) 40 | 41 | db.Close() 42 | 43 | p, err := aol.Open(logPath, nil) 44 | if err != nil { 45 | t.Fatal(err) 46 | } 47 | db2 := getTestDB() 48 | db2.log = p 49 | err = db2.load() 50 | assert.NoError(t, err) 51 | 52 | for i := 1; i <= 10; i++ { 53 | key := fmt.Sprintf("key_%d", i) 54 | member := fmt.Sprintf("member_%d", i) 55 | value := fmt.Sprintf("value_%d", i) 56 | db2.View(func(tx *Tx) error { 57 | assert.Equal(t, value, tx.HGet(key, member)) 58 | assert.True(t, tx.SIsMember(key, member)) 59 | val, err := tx.Get(key) 60 | assert.NoError(t, err) 61 | assert.Equal(t, member, val) 62 | ok, score := tx.ZScore(key, member) 63 | assert.True(t, ok) 64 | assert.Equal(t, 10.0, score) 65 | return nil 66 | }) 67 | 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /tx_str_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFlashDB_GetSet(t *testing.T) { 11 | db := getTestDB() 12 | defer db.Close() 13 | defer os.RemoveAll(tmpDir) 14 | 15 | if err := db.Update(func(tx *Tx) error { 16 | err := tx.Set("foo", "bar") 17 | assert.Equal(t, nil, err) 18 | return nil 19 | }); err != nil { 20 | t.Fatal(err) 21 | } 22 | 23 | if err := db.View(func(tx *Tx) error { 24 | val, err := tx.Get("foo") 25 | assert.Equal(t, nil, err) 26 | assert.Equal(t, "bar", val) 27 | return nil 28 | }); err != nil { 29 | t.Fatal(err) 30 | } 31 | } 32 | 33 | func TestFlashDB_SetEx(t *testing.T) { 34 | db := getTestDB() 35 | defer db.Close() 36 | defer os.RemoveAll(tmpDir) 37 | 38 | if err := db.Update(func(tx *Tx) error { 39 | err := tx.SetEx("foo", "1", -4) 40 | assert.NotEmpty(t, err) 41 | 42 | err = tx.SetEx("foo", "1", 993) 43 | assert.Empty(t, err) 44 | return nil 45 | }); err != nil { 46 | t.Fatal(err) 47 | } 48 | } 49 | 50 | func TestFlashDB_Delete(t *testing.T) { 51 | db := getTestDB() 52 | defer db.Close() 53 | defer os.RemoveAll(tmpDir) 54 | 55 | if err := db.Update(func(tx *Tx) error { 56 | err := tx.Set("foo", "bar") 57 | assert.Equal(t, err, nil) 58 | 59 | err = tx.Delete("foo") 60 | assert.Equal(t, err, nil) 61 | 62 | return nil 63 | }); err != nil { 64 | t.Fatal(err) 65 | } 66 | 67 | if err := db.View(func(tx *Tx) error { 68 | val, err := tx.Get("foo") 69 | assert.Empty(t, val) 70 | assert.Equal(t, ErrInvalidKey, err) 71 | return nil 72 | }); err != nil { 73 | t.Fatal(err) 74 | } 75 | 76 | } 77 | 78 | func TestFlashDB_TTL(t *testing.T) { 79 | db := getTestDB() 80 | defer db.Close() 81 | defer os.RemoveAll(tmpDir) 82 | 83 | if err := db.Update(func(tx *Tx) error { 84 | err := tx.SetEx("foo", "bar", 20) 85 | assert.Equal(t, err, nil) 86 | 87 | return nil 88 | }); err != nil { 89 | t.Fatal(err) 90 | } 91 | 92 | if err := db.View(func(tx *Tx) error { 93 | ttl := tx.TTL("foo") 94 | assert.Equal(t, int(ttl), 20) 95 | return nil 96 | }); err != nil { 97 | t.Fatal(err) 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /tx_str.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // Set saves a key-value pair. 8 | func (tx *Tx) Set(key string, value string) error { 9 | e := newRecord([]byte(key), []byte(value), StringRecord, StringSet) 10 | tx.addRecord(e) 11 | 12 | return nil 13 | } 14 | 15 | // SetEx sets key-value pair with given duration time for expiration. 16 | func (tx *Tx) SetEx(key string, value string, duration int64) (err error) { 17 | if duration <= 0 { 18 | return ErrInvalidTTL 19 | } 20 | 21 | ttl := time.Now().Unix() + duration 22 | e := newRecordWithExpire([]byte(key), nil, ttl, StringRecord, StringExpire) 23 | tx.addRecord(e) 24 | 25 | return 26 | } 27 | 28 | // Get returns value of the given key. It may return error if something goes wrong. 29 | func (tx *Tx) Get(key string) (val string, err error) { 30 | val, err = tx.get(key) 31 | if err != nil { 32 | return 33 | } 34 | 35 | return 36 | } 37 | 38 | // Delete deletes the given key. 39 | func (tx *Tx) Delete(key string) error { 40 | e := newRecord([]byte(key), nil, StringRecord, StringRem) 41 | tx.addRecord(e) 42 | 43 | return nil 44 | } 45 | 46 | // Expire adds a expiration time period to the given key. 47 | func (tx *Tx) Expire(key string, duration int64) (err error) { 48 | if duration <= 0 { 49 | return ErrInvalidTTL 50 | } 51 | 52 | if _, err = tx.get(key); err != nil { 53 | return 54 | } 55 | 56 | ttl := time.Now().Unix() + duration 57 | e := newRecordWithExpire([]byte(key), nil, ttl, StringRecord, StringExpire) 58 | tx.addRecord(e) 59 | 60 | return 61 | } 62 | 63 | // TTL returns remaining time of the expiration. 64 | func (tx *Tx) TTL(key string) (ttl int64) { 65 | deadline := tx.db.getTTL(String, key) 66 | if deadline == nil { 67 | return 68 | } 69 | 70 | if tx.db.hasExpired(key, String) { 71 | tx.db.evict(key, String) 72 | return 73 | } 74 | 75 | return deadline.(int64) - time.Now().Unix() 76 | } 77 | 78 | // Exists checks the given key whether exists. Also, if the key is expired, 79 | // the key is evicted and return false. 80 | func (tx *Tx) Exists(key string) bool { 81 | _, err := tx.db.strStore.get(key) 82 | if err != nil { 83 | if err == ErrExpiredKey { 84 | tx.db.evict(key, String) 85 | } 86 | return false 87 | } 88 | 89 | return true 90 | } 91 | 92 | // get is a helper method for retrieving value of the given key from the database. 93 | func (tx *Tx) get(key string) (val string, err error) { 94 | v, err := tx.db.strStore.get(key) 95 | if err != nil { 96 | return "", err 97 | } 98 | 99 | // Check if the key is expired. 100 | if tx.db.hasExpired(key, String) { 101 | tx.db.evict(key, String) 102 | return "", ErrExpiredKey 103 | } 104 | 105 | val = v.(string) 106 | return 107 | } 108 | -------------------------------------------------------------------------------- /tx_set_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFlashDB_SCard(t *testing.T) { 11 | db := getTestDB() 12 | defer db.Close() 13 | defer os.RemoveAll(tmpDir) 14 | 15 | if err := db.Update(func(tx *Tx) error { 16 | tx.SAdd(testKey, "foo", "bar", "baz") 17 | return nil 18 | }); err != nil { 19 | t.Fatal(err) 20 | } 21 | 22 | db.View(func(tx *Tx) error { 23 | cnt := tx.SCard(testKey) 24 | assert.Equal(t, 3, cnt) 25 | return nil 26 | }) 27 | } 28 | 29 | func TestFlashDB_SIsMember(t *testing.T) { 30 | db := getTestDB() 31 | defer db.Close() 32 | defer os.RemoveAll(tmpDir) 33 | 34 | if err := db.Update(func(tx *Tx) error { 35 | tx.SAdd(testKey, "foo", "bar", "baz") 36 | return nil 37 | }); err != nil { 38 | t.Fatal(err) 39 | } 40 | 41 | db.View(func(tx *Tx) error { 42 | assert.True(t, tx.SIsMember(testKey, "foo")) 43 | assert.True(t, tx.SIsMember(testKey, "bar")) 44 | assert.True(t, tx.SIsMember(testKey, "baz")) 45 | return nil 46 | }) 47 | } 48 | 49 | func TestFlashDB_SRem(t *testing.T) { 50 | db := getTestDB() 51 | defer db.Close() 52 | defer os.RemoveAll(tmpDir) 53 | 54 | if err := db.Update(func(tx *Tx) error { 55 | tx.SAdd(testKey, "foo", "bar", "baz") 56 | tx.SRem(testKey, "foo") 57 | return nil 58 | }); err != nil { 59 | t.Fatal(err) 60 | } 61 | 62 | if err := db.View(func(tx *Tx) error { 63 | assert.False(t, tx.SIsMember(testKey, "foo")) 64 | return nil 65 | }); err != nil { 66 | t.Fatal(err) 67 | } 68 | 69 | } 70 | 71 | func TestFlashDB_SClear(t *testing.T) { 72 | db := getTestDB() 73 | defer db.Close() 74 | defer os.RemoveAll(tmpDir) 75 | 76 | if err := db.Update(func(tx *Tx) error { 77 | tx.SAdd(testKey, "foo", "bar", "baz") 78 | return nil 79 | }); err != nil { 80 | t.Fatal(err) 81 | } 82 | 83 | if err := db.Update(func(tx *Tx) error { 84 | resp := tx.SClear(testKey) 85 | assert.NoError(t, resp) 86 | return nil 87 | }); err != nil { 88 | t.Fatal(err) 89 | } 90 | 91 | if err := db.View(func(tx *Tx) error { 92 | assert.False(t, tx.SKeyExists(testKey)) 93 | return nil 94 | }); err != nil { 95 | t.Fatal(err) 96 | } 97 | } 98 | 99 | func TestFlashDB_SDiff(t *testing.T) { 100 | db := getTestDB() 101 | defer db.Close() 102 | defer os.RemoveAll(tmpDir) 103 | 104 | if err := db.Update(func(tx *Tx) error { 105 | err := tx.SAdd("set1", "foo", "bar", "baz") 106 | assert.NoError(t, err) 107 | err = tx.SAdd("set2", "foo", "bar") 108 | assert.NoError(t, err) 109 | return nil 110 | }); err != nil { 111 | t.Fatal(err) 112 | } 113 | 114 | if err := db.View(func(tx *Tx) error { 115 | assert.Equal(t, 3, len(tx.SMembers("set1"))) 116 | assert.Equal(t, 2, len(tx.SMembers("set2"))) 117 | res := tx.SDiff("set1", "set2") 118 | assert.Equal(t, 1, len(res)) 119 | assert.Equal(t, "baz", res[0]) 120 | return nil 121 | }); err != nil { 122 | t.Fatal(err) 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /tx_hash_test.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var ( 11 | testKey = "dummy" 12 | tmpDir = "tmp" 13 | ) 14 | 15 | func testConfig() *Config { 16 | return &Config{ 17 | Addr: DefaultAddr, 18 | Path: tmpDir, 19 | NoSync: true, 20 | } 21 | } 22 | 23 | func getTestDB() *FlashDB { 24 | db, _ := New(testConfig()) 25 | return db 26 | } 27 | 28 | func TestFlashDB_HGetSet(t *testing.T) { 29 | db := getTestDB() 30 | defer db.Close() 31 | defer os.RemoveAll(tmpDir) 32 | 33 | if err := db.Update(func(tx *Tx) error { 34 | _, err := tx.HSet(testKey, "bar", "1") 35 | assert.NoError(t, err) 36 | _, err = tx.HSet(testKey, "baz", "2") 37 | assert.NoError(t, err) 38 | return nil 39 | }); err != nil { 40 | t.Fatal(err) 41 | } 42 | 43 | db.View(func(tx *Tx) error { 44 | val := tx.HGet(testKey, "bar") 45 | assert.Equal(t, "1", val) 46 | val = tx.HGet(testKey, "baz") 47 | assert.Equal(t, "2", val) 48 | return nil 49 | }) 50 | } 51 | 52 | func TestFlashDB_HGetAll(t *testing.T) { 53 | db := getTestDB() 54 | defer db.Close() 55 | defer os.RemoveAll(tmpDir) 56 | 57 | if err := db.Update(func(tx *Tx) error { 58 | tx.HSet(testKey, "bar", "1") 59 | tx.HSet(testKey, "baz", "2") 60 | return nil 61 | }); err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | db.View(func(tx *Tx) error { 66 | values := tx.HGetAll(testKey) 67 | assert.Equal(t, 4, len(values)) 68 | return nil 69 | }) 70 | } 71 | 72 | func TestFlashDB_HDel(t *testing.T) { 73 | db := getTestDB() 74 | defer db.Close() 75 | defer os.RemoveAll(tmpDir) 76 | 77 | if err := db.Update(func(tx *Tx) error { 78 | tx.HSet(testKey, "bar", "1") 79 | tx.HSet(testKey, "baz", "2") 80 | res, err := tx.HDel(testKey, "bar", "baz") 81 | assert.Nil(t, err) 82 | assert.Equal(t, 2, res) 83 | return nil 84 | }); err != nil { 85 | t.Fatal(err) 86 | } 87 | 88 | db.View(func(tx *Tx) error { 89 | assert.Empty(t, tx.HGet(testKey, "bar")) 90 | assert.Empty(t, tx.HGet(testKey, "baz")) 91 | return nil 92 | }) 93 | } 94 | 95 | func TestFlashDB_HExists(t *testing.T) { 96 | db := getTestDB() 97 | defer db.Close() 98 | defer os.RemoveAll(tmpDir) 99 | 100 | if err := db.Update(func(tx *Tx) error { 101 | tx.HSet(testKey, "bar", "1") 102 | tx.HSet(testKey, "baz", "2") 103 | return nil 104 | }); err != nil { 105 | t.Fatal(err) 106 | } 107 | 108 | db.View(func(tx *Tx) error { 109 | assert.True(t, tx.HExists(testKey, "bar")) 110 | assert.True(t, tx.HExists(testKey, "baz")) 111 | assert.False(t, tx.HExists(testKey, "ben")) 112 | return nil 113 | }) 114 | } 115 | 116 | func TestFlashDB_HKeyExists(t *testing.T) { 117 | db := getTestDB() 118 | defer db.Close() 119 | defer os.RemoveAll(tmpDir) 120 | 121 | if err := db.Update(func(tx *Tx) error { 122 | tx.HSet(testKey, "bar", "1") 123 | return nil 124 | }); err != nil { 125 | t.Fatal(err) 126 | } 127 | 128 | db.View(func(tx *Tx) error { 129 | assert.True(t, tx.HKeyExists(testKey)) 130 | assert.False(t, tx.HKeyExists("yolo")) 131 | return nil 132 | }) 133 | } 134 | 135 | func TestFlashDB_HLen(t *testing.T) { 136 | db := getTestDB() 137 | defer db.Close() 138 | defer os.RemoveAll(tmpDir) 139 | 140 | if err := db.Update(func(tx *Tx) error { 141 | tx.HSet(testKey, "bar", "1") 142 | return nil 143 | }); err != nil { 144 | t.Fatal(err) 145 | } 146 | 147 | db.View(func(tx *Tx) error { 148 | assert.Equal(t, tx.HLen(testKey), 1) 149 | return nil 150 | }) 151 | } 152 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/arriqaaq/art" 8 | "github.com/arriqaaq/hash" 9 | "github.com/arriqaaq/set" 10 | "github.com/arriqaaq/zset" 11 | ) 12 | 13 | var ( 14 | _ store = &strStore{} 15 | _ store = &setStore{} 16 | _ store = &zsetStore{} 17 | _ store = &hashStore{} 18 | ) 19 | 20 | type store interface { 21 | evict(cache *hash.Hash) 22 | } 23 | 24 | type strStore struct { 25 | sync.RWMutex 26 | *art.Tree 27 | } 28 | 29 | func newStrStore() *strStore { 30 | n := &strStore{} 31 | n.Tree = art.NewTree() 32 | return n 33 | } 34 | 35 | func (s *strStore) get(key string) (val interface{}, err error) { 36 | val = s.Search([]byte(key)) 37 | if val == nil { 38 | return nil, ErrInvalidKey 39 | } 40 | return 41 | } 42 | 43 | func (s *strStore) Keys() (keys []string) { 44 | s.Each(func(node *art.Node) { 45 | if node.IsLeaf() { 46 | key := string(node.Key()) 47 | keys = append(keys, key) 48 | } 49 | }) 50 | return 51 | } 52 | 53 | func (s *strStore) evict(cache *hash.Hash) { 54 | s.Lock() 55 | defer s.Unlock() 56 | 57 | keys := s.Keys() 58 | expiredKeys := make([]string, 0, 1) 59 | 60 | for _, k := range keys { 61 | ttl := cache.HGet(String, k) 62 | if ttl == nil { 63 | continue 64 | } 65 | if time.Now().Unix() > ttl.(int64) { 66 | expiredKeys = append(expiredKeys, k) 67 | } 68 | } 69 | 70 | for _, k := range expiredKeys { 71 | s.Delete([]byte(k)) 72 | cache.HDel(String, k) 73 | } 74 | } 75 | 76 | type hashStore struct { 77 | sync.RWMutex 78 | *hash.Hash 79 | } 80 | 81 | func newHashStore() *hashStore { 82 | n := &hashStore{} 83 | n.Hash = hash.New() 84 | return n 85 | } 86 | 87 | func (h *hashStore) evict(cache *hash.Hash) { 88 | h.Lock() 89 | defer h.Unlock() 90 | 91 | keys := h.Keys() 92 | expiredKeys := make([]string, 0, 1) 93 | 94 | for _, k := range keys { 95 | ttl := cache.HGet(Hash, k) 96 | if ttl == nil { 97 | continue 98 | } 99 | if time.Now().Unix() > ttl.(int64) { 100 | expiredKeys = append(expiredKeys, k) 101 | } 102 | } 103 | 104 | for _, k := range expiredKeys { 105 | h.HClear(k) 106 | cache.HDel(Hash, k) 107 | } 108 | } 109 | 110 | type setStore struct { 111 | sync.RWMutex 112 | *set.Set 113 | } 114 | 115 | func newSetStore() *setStore { 116 | n := &setStore{} 117 | n.Set = set.New() 118 | return n 119 | } 120 | 121 | func (s *setStore) evict(cache *hash.Hash) { 122 | s.Lock() 123 | defer s.Unlock() 124 | 125 | keys := s.Keys() 126 | expiredKeys := make([]string, 0, 1) 127 | 128 | for _, k := range keys { 129 | ttl := cache.HGet(Set, k) 130 | if ttl == nil { 131 | continue 132 | } 133 | if time.Now().Unix() > ttl.(int64) { 134 | expiredKeys = append(expiredKeys, k) 135 | } 136 | } 137 | 138 | for _, k := range expiredKeys { 139 | s.SClear(k) 140 | cache.HDel(Set, k) 141 | } 142 | } 143 | 144 | type zsetStore struct { 145 | sync.RWMutex 146 | *zset.ZSet 147 | } 148 | 149 | func newZSetStore() *zsetStore { 150 | n := &zsetStore{} 151 | n.ZSet = zset.New() 152 | return n 153 | } 154 | 155 | func (z *zsetStore) evict(cache *hash.Hash) { 156 | z.Lock() 157 | defer z.Unlock() 158 | 159 | keys := z.Keys() 160 | expiredKeys := make([]string, 0, 1) 161 | 162 | for _, k := range keys { 163 | ttl := cache.HGet(ZSet, k) 164 | if ttl == nil { 165 | continue 166 | } 167 | if time.Now().Unix() > ttl.(int64) { 168 | expiredKeys = append(expiredKeys, k) 169 | } 170 | } 171 | 172 | for _, k := range expiredKeys { 173 | z.ZClear(k) 174 | cache.HDel(ZSet, k) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= 2 | github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= 3 | github.com/arriqaaq/aol v0.1.2 h1:uGEMA1mYu968vmRUSE9d2++YUaDGObbBN1QzD6RlgC4= 4 | github.com/arriqaaq/aol v0.1.2/go.mod h1:GQ7Jdsp3RYIbUsovCh0XdIbRj0mvBNBFQeIfL1oKKKE= 5 | github.com/arriqaaq/art v0.1.2 h1:GoAiArsfRTV8wPb2X7/58ZFNBUwoPfGfsTvweeiYgzg= 6 | github.com/arriqaaq/art v0.1.2/go.mod h1:11r05D8a+owc6lTpI/yVypn16g7LrsVif+8lU9riSDM= 7 | github.com/arriqaaq/hash v0.1.2 h1:q/Az5T00nkqEc/XhlnrGcPeZBmgBM5ktKi26nr70s9c= 8 | github.com/arriqaaq/hash v0.1.2/go.mod h1:WB3wOwaJdkNPb8DX+J2YHivF9y+sJkXhM1B4gH7xphE= 9 | github.com/arriqaaq/set v0.1.2 h1:uiIarvVXvsAWr/RQ4L9C7HgOXh4jteaBeFMwU/faWlk= 10 | github.com/arriqaaq/set v0.1.2/go.mod h1:Sj43MSlXuSY6tqVu2XZ1jf30fQylGp3FCM2O6qFBSd8= 11 | github.com/arriqaaq/skiplist v0.1.6 h1:OtJ/6pcMFYnV22RwFUbz8CophthySLRq0vVAfWRAvzc= 12 | github.com/arriqaaq/skiplist v0.1.6/go.mod h1:iFkbyk/oh3K2w9sPqtijfDrMeTcoGu6uI+9DSGL/Zxw= 13 | github.com/arriqaaq/zset v0.1.2 h1:vF/B95Unz/zA0Ttmc1XHCQVyxIXd7B6PNc3H4aCOdyo= 14 | github.com/arriqaaq/zset v0.1.2/go.mod h1:5DoabYGc0lHZnhhzZoYawG015X7i+bphIsa6qEekFuI= 15 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 16 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 17 | github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= 18 | github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= 19 | github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= 20 | github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= 21 | github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= 22 | github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= 23 | github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= 24 | github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 28 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 29 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 30 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 31 | github.com/tidwall/btree v1.1.0 h1:5P+9WU8ui5uhmcg3SoPyTwoI0mVyZ1nps7YQzTZFkYM= 32 | github.com/tidwall/btree v1.1.0/go.mod h1:TzIRzen6yHbibdSfK6t8QimqbUnoxUSrZfeW7Uob0q4= 33 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 34 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 35 | github.com/tidwall/redcon v1.4.4 h1:N3ZwZx6n5dqNxB3cfmj9D/8zNboFia5FAv1wt+azwyU= 36 | github.com/tidwall/redcon v1.4.4/go.mod h1:p5Wbsgeyi2VSTBWOcA5vRXrOb9arFTcU2+ZzFjqV75Y= 37 | golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= 38 | golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 40 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 41 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 42 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 43 | -------------------------------------------------------------------------------- /flashdb.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "time" 7 | 8 | "github.com/arriqaaq/aol" 9 | "github.com/arriqaaq/hash" 10 | ) 11 | 12 | var ( 13 | ErrInvalidKey = errors.New("invalid key") 14 | ErrInvalidTTL = errors.New("invalid ttl") 15 | ErrExpiredKey = errors.New("key has expired") 16 | ErrTxClosed = errors.New("tx closed") 17 | ErrDatabaseClosed = errors.New("database closed") 18 | ErrTxNotWritable = errors.New("tx not writable") 19 | ) 20 | 21 | type ( 22 | FlashDB struct { 23 | mu sync.RWMutex 24 | config *Config 25 | exps *hash.Hash // hashmap of ttl keys 26 | log *aol.Log 27 | 28 | closed bool // set when the database has been closed 29 | persist bool // do we write to disk 30 | 31 | strStore *strStore 32 | hashStore *hashStore 33 | setStore *setStore 34 | zsetStore *zsetStore 35 | 36 | evictors []evictor // background manager to delete keys periodically 37 | } 38 | ) 39 | 40 | func New(config *Config) (*FlashDB, error) { 41 | 42 | config.validate() 43 | 44 | db := &FlashDB{ 45 | config: config, 46 | strStore: newStrStore(), 47 | setStore: newSetStore(), 48 | hashStore: newHashStore(), 49 | zsetStore: newZSetStore(), 50 | exps: hash.New(), 51 | } 52 | 53 | evictionInterval := config.evictionInterval() 54 | if evictionInterval > 0 { 55 | db.evictors = []evictor{ 56 | newSweeperWithStore(db.strStore, evictionInterval), 57 | newSweeperWithStore(db.setStore, evictionInterval), 58 | newSweeperWithStore(db.hashStore, evictionInterval), 59 | newSweeperWithStore(db.zsetStore, evictionInterval), 60 | } 61 | for _, evictor := range db.evictors { 62 | go evictor.run(db.exps) 63 | } 64 | } 65 | 66 | db.persist = config.Path != "" 67 | if db.persist { 68 | opts := aol.DefaultOptions 69 | opts.NoSync = config.NoSync 70 | 71 | l, err := aol.Open(config.Path, opts) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | db.log = l 77 | 78 | // load data from append-only log 79 | err = db.load() 80 | if err != nil { 81 | return nil, err 82 | } 83 | } 84 | 85 | return db, nil 86 | } 87 | 88 | func (db *FlashDB) setTTL(dType DataType, key string, ttl int64) { 89 | db.exps.HSet(dType, key, ttl) 90 | } 91 | 92 | func (db *FlashDB) getTTL(dType DataType, key string) interface{} { 93 | return db.exps.HGet(dType, key) 94 | } 95 | 96 | func (db *FlashDB) hasExpired(key string, dType DataType) (expired bool) { 97 | ttl := db.exps.HGet(dType, key) 98 | if ttl == nil { 99 | return 100 | } 101 | if time.Now().Unix() > ttl.(int64) { 102 | expired = true 103 | } 104 | return 105 | } 106 | 107 | func (db *FlashDB) evict(key string, dType DataType) { 108 | ttl := db.exps.HGet(dType, key) 109 | if ttl == nil { 110 | return 111 | } 112 | 113 | var r *record 114 | if time.Now().Unix() > ttl.(int64) { 115 | switch dType { 116 | case String: 117 | r = newRecord([]byte(key), nil, StringRecord, StringRem) 118 | db.strStore.Delete([]byte(key)) 119 | case Hash: 120 | r = newRecord([]byte(key), nil, HashRecord, HashHClear) 121 | db.hashStore.HClear(key) 122 | case Set: 123 | r = newRecord([]byte(key), nil, SetRecord, SetSClear) 124 | db.setStore.SClear(key) 125 | case ZSet: 126 | r = newRecord([]byte(key), nil, ZSetRecord, ZSetZClear) 127 | db.zsetStore.ZClear(key) 128 | } 129 | 130 | if err := db.write(r); err != nil { 131 | panic(err) 132 | } 133 | 134 | db.exps.HDel(dType, key) 135 | } 136 | } 137 | 138 | func (db *FlashDB) Close() error { 139 | db.closed = true 140 | for _, evictor := range db.evictors { 141 | evictor.stop() 142 | } 143 | if db.log != nil { 144 | err := db.log.Close() 145 | if err != nil { 146 | return err 147 | } 148 | } 149 | return nil 150 | } 151 | 152 | func (db *FlashDB) write(r *record) error { 153 | if db.log == nil { 154 | return nil 155 | } 156 | encVal, err := r.encode() 157 | if err != nil { 158 | return err 159 | } 160 | 161 | return db.log.Write(encVal) 162 | } 163 | -------------------------------------------------------------------------------- /db_load.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/arriqaaq/aol" 7 | ) 8 | 9 | // load String, Hash, Set and ZSet stores from append-only log 10 | func (db *FlashDB) load() error { 11 | if db.log == nil { 12 | return nil 13 | } 14 | 15 | noOfSegments := db.log.Segments() 16 | for i := 1; i <= noOfSegments; i++ { 17 | j := 0 18 | 19 | for { 20 | data, err := db.log.Read(uint64(i), uint64(j)) 21 | if err != nil { 22 | if err == aol.ErrEOF { 23 | break 24 | } 25 | return err 26 | } 27 | 28 | record, err := decode(data) 29 | if err != nil { 30 | return err 31 | } 32 | 33 | if len(record.meta.key) > 0 { 34 | if err := db.loadRecord(record); err != nil { 35 | return err 36 | } 37 | } 38 | 39 | j++ 40 | } 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func (db *FlashDB) loadRecord(r *record) (err error) { 47 | 48 | switch r.getType() { 49 | case StringRecord: 50 | err = db.buildStringRecord(r) 51 | case HashRecord: 52 | err = db.buildHashRecord(r) 53 | case SetRecord: 54 | err = db.buildSetRecord(r) 55 | case ZSetRecord: 56 | err = db.buildZsetRecord(r) 57 | } 58 | return 59 | } 60 | 61 | /* 62 | Utility functions to build stores from aol Record 63 | */ 64 | 65 | func (db *FlashDB) buildStringRecord(r *record) error { 66 | 67 | key := string(r.meta.key) 68 | member := string(r.meta.member) 69 | 70 | switch r.getMark() { 71 | case StringSet: 72 | db.strStore.Insert([]byte(key), member) 73 | case StringRem: 74 | db.strStore.Delete([]byte(key)) 75 | db.exps.HDel(String, key) 76 | case StringExpire: 77 | if r.timestamp < uint64(time.Now().Unix()) { 78 | db.strStore.Delete([]byte(key)) 79 | db.exps.HDel(String, key) 80 | } else { 81 | db.setTTL(String, key, int64(r.timestamp)) 82 | } 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (db *FlashDB) buildHashRecord(r *record) error { 89 | 90 | key := string(r.meta.key) 91 | member := string(r.meta.member) 92 | value := string(r.meta.value) 93 | 94 | switch r.getMark() { 95 | case HashHSet: 96 | db.hashStore.HSet(key, member, value) 97 | case HashHDel: 98 | db.hashStore.HDel(key, member) 99 | case HashHClear: 100 | db.hashStore.HClear(key) 101 | db.exps.HDel(Hash, key) 102 | case HashHExpire: 103 | if r.timestamp < uint64(time.Now().Unix()) { 104 | db.hashStore.HClear(key) 105 | db.exps.HDel(Hash, key) 106 | } else { 107 | db.setTTL(Hash, key, int64(r.timestamp)) 108 | } 109 | } 110 | 111 | return nil 112 | } 113 | 114 | func (db *FlashDB) buildSetRecord(r *record) error { 115 | 116 | key := string(r.meta.key) 117 | member := string(r.meta.member) 118 | value := string(r.meta.value) 119 | 120 | switch r.getMark() { 121 | case SetSAdd: 122 | db.setStore.SAdd(key, member) 123 | case SetSRem: 124 | db.setStore.SRem(key, member) 125 | case SetSMove: 126 | db.setStore.SMove(key, value, member) 127 | case SetSClear: 128 | db.setStore.SClear(key) 129 | db.exps.HDel(Set, key) 130 | case SetSExpire: 131 | if r.timestamp < uint64(time.Now().Unix()) { 132 | db.setStore.SClear(key) 133 | db.exps.HDel(Set, key) 134 | } else { 135 | db.setTTL(Set, key, int64(r.timestamp)) 136 | } 137 | } 138 | 139 | return nil 140 | } 141 | 142 | func (db *FlashDB) buildZsetRecord(r *record) error { 143 | 144 | key := string(r.meta.key) 145 | member := string(r.meta.member) 146 | value := string(r.meta.value) 147 | 148 | switch r.getMark() { 149 | case ZSetZAdd: 150 | score, err := strToFloat64(value) 151 | if err != nil { 152 | return err 153 | } 154 | db.zsetStore.ZAdd(key, score, member, nil) 155 | case ZSetZRem: 156 | db.zsetStore.ZRem(key, member) 157 | case ZSetZClear: 158 | db.zsetStore.ZClear(key) 159 | db.exps.HDel(ZSet, key) 160 | case ZSetZExpire: 161 | if r.timestamp < uint64(time.Now().Unix()) { 162 | db.zsetStore.ZClear(key) 163 | db.exps.HDel(ZSet, key) 164 | } else { 165 | db.setTTL(ZSet, key, int64(r.timestamp)) 166 | } 167 | } 168 | 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /tx_hash.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // HSet sets field in the hash stored at key to value. 8 | func (tx *Tx) HSet(key string, field string, value string) (res int, err error) { 9 | existVal := tx.HGet(key, field) 10 | if existVal == value { 11 | return 12 | } 13 | 14 | e := newRecordWithValue([]byte(key), []byte(field), []byte(value), HashRecord, HashHSet) 15 | tx.addRecord(e) 16 | return 17 | } 18 | 19 | // HGet returns the value associated with field in the hash stored at key. If 20 | // the key has expired, the key is evicted and empty string is returned. 21 | func (tx *Tx) HGet(key string, field string) string { 22 | if tx.db.hasExpired(key, Hash) { 23 | tx.db.evict(key, Hash) 24 | return "" 25 | } 26 | 27 | return toString(tx.db.hashStore.HGet(key, field)) 28 | } 29 | 30 | // HGetAll returns all fields and values stored at key. If the key has expired, 31 | // the key is evicted. 32 | func (tx *Tx) HGetAll(key string) []string { 33 | if tx.db.hasExpired(key, Hash) { 34 | tx.db.evict(key, Hash) 35 | return nil 36 | } 37 | 38 | vals := tx.db.hashStore.HGetAll(key) 39 | values := make([]string, 0, 1) 40 | 41 | for _, v := range vals { 42 | values = append(values, toString(v)) 43 | } 44 | 45 | return values 46 | } 47 | 48 | // HDel deletes the fields stored at key. 49 | func (tx *Tx) HDel(key string, fields ...string) (res int, err error) { 50 | for _, f := range fields { 51 | e := newRecord([]byte(key), []byte(f), HashRecord, HashHDel) 52 | tx.addRecord(e) 53 | res++ 54 | } 55 | return 56 | } 57 | 58 | // HKeyExists determines whether the key is exists. If the key has expired, the 59 | // key is evicted. 60 | func (tx *Tx) HKeyExists(key string) (ok bool) { 61 | if tx.db.hasExpired(key, Hash) { 62 | tx.db.evict(key, Hash) 63 | return 64 | } 65 | return tx.db.hashStore.HKeyExists(key) 66 | } 67 | 68 | // HExists determines whether the key and field are exists. If the key has 69 | // expired, the key is evicted. 70 | func (tx *Tx) HExists(key, field string) (ok bool) { 71 | if tx.db.hasExpired(key, Hash) { 72 | tx.db.evict(key, Hash) 73 | return 74 | } 75 | 76 | return tx.db.hashStore.HExists(key, field) 77 | } 78 | 79 | // HLen returns number of the fields stored at key. If the key has expired, the 80 | // key is evicted. 81 | func (tx *Tx) HLen(key string) int { 82 | if tx.db.hasExpired(key, Hash) { 83 | tx.db.evict(key, Hash) 84 | return 0 85 | } 86 | 87 | return tx.db.hashStore.HLen(key) 88 | } 89 | 90 | // HKeys returns all fields stored at key. If the key has expired, the key is evicted. 91 | func (tx *Tx) HKeys(key string) (val []string) { 92 | if tx.db.hasExpired(key, Hash) { 93 | tx.db.evict(key, Hash) 94 | return nil 95 | } 96 | 97 | return tx.db.hashStore.HKeys(key) 98 | } 99 | 100 | // HVals returns all values stored at key. If the key has expired, the key 101 | // is evicted. 102 | func (tx *Tx) HVals(key string) (values []string) { 103 | if tx.db.hasExpired(key, Hash) { 104 | tx.db.evict(key, Hash) 105 | return nil 106 | } 107 | 108 | vals := tx.db.hashStore.HVals(key) 109 | for _, v := range vals { 110 | values = append(values, toString(v)) 111 | } 112 | 113 | return 114 | } 115 | 116 | // HExpire adds an expiry time for key. If the duration is not positive, expiry 117 | // time is not set. 118 | func (tx *Tx) HExpire(key string, duration int64) (err error) { 119 | if duration <= 0 { 120 | return ErrInvalidTTL 121 | } 122 | 123 | if !tx.HKeyExists(key) { 124 | return ErrInvalidKey 125 | } 126 | 127 | ttl := time.Now().Unix() + duration 128 | e := newRecordWithExpire([]byte(key), nil, ttl, HashRecord, HashHExpire) 129 | tx.addRecord(e) 130 | 131 | return 132 | } 133 | 134 | // HTTL returns remaining time for deadline. If the key has expired, the key is evicted. 135 | func (tx *Tx) HTTL(key string) (ttl int64) { 136 | if tx.db.hasExpired(key, Hash) { 137 | tx.db.evict(key, Hash) 138 | return 139 | } 140 | 141 | deadline := tx.db.getTTL(Hash, key) 142 | if deadline == nil { 143 | return 144 | } 145 | return deadline.(int64) - time.Now().Unix() 146 | } 147 | 148 | // HClear clears the key. If the key has expired, the key is evicted. 149 | func (tx *Tx) HClear(key string) (err error) { 150 | if tx.db.hasExpired(key, Hash) { 151 | tx.db.evict(key, Hash) 152 | return 153 | } 154 | 155 | e := newRecord([]byte(key), nil, HashRecord, HashHClear) 156 | tx.addRecord(e) 157 | return 158 | } 159 | 160 | func toString(val interface{}) string { 161 | if val == nil { 162 | return "" 163 | } 164 | return val.(string) 165 | } 166 | -------------------------------------------------------------------------------- /record.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "time" 7 | ) 8 | 9 | /* 10 | Record is based on bitcask entry model design in nutsdb 11 | */ 12 | 13 | var ( 14 | ErrInvalidEntry = errors.New("invalid entry") 15 | ) 16 | 17 | const ( 18 | // keySize, memberSize, valueSize is uint32 type，4 bytes each. 19 | // timestamp 8 bytes, state 2 bytes. 20 | // 4 + 4 + 4 + 8 + 2 = 22 21 | entryHeaderSize = 22 22 | ) 23 | 24 | type ( 25 | record struct { 26 | meta *meta 27 | state uint16 // state represents two fields, high 8 bits is the data type, low 8 bits is operation mark. 28 | timestamp uint64 // Timestamp is the time when entry was written. 29 | } 30 | 31 | // Meta meta info. 32 | meta struct { 33 | key []byte 34 | member []byte 35 | value []byte 36 | keySize uint32 37 | memberSize uint32 38 | valueSize uint32 39 | } 40 | ) 41 | 42 | func newInternal(key, member, value []byte, state uint16, timestamp uint64) *record { 43 | return &record{ 44 | state: state, timestamp: timestamp, 45 | meta: &meta{ 46 | key: key, 47 | member: member, 48 | value: value, 49 | keySize: uint32(len(key)), 50 | memberSize: uint32(len(member)), 51 | valueSize: uint32(len(value)), 52 | }, 53 | } 54 | } 55 | 56 | func newRecord(key, member []byte, t, mark uint16) *record { 57 | var state uint16 = 0 58 | // set type and mark. 59 | state = state | (t << 8) 60 | state = state | mark 61 | return newInternal(key, member, nil, state, uint64(time.Now().UnixNano())) 62 | } 63 | 64 | func newRecordWithValue(key, member, value []byte, t, mark uint16) *record { 65 | var state uint16 = 0 66 | // set type and mark. 67 | state = state | (t << 8) 68 | state = state | mark 69 | return newInternal(key, member, value, state, uint64(time.Now().UnixNano())) 70 | } 71 | 72 | func newRecordWithExpire(key, member []byte, deadline int64, t, mark uint16) *record { 73 | var state uint16 = 0 74 | // set type and mark. 75 | state = state | (t << 8) 76 | state = state | mark 77 | 78 | return newInternal(key, member, nil, state, uint64(deadline)) 79 | } 80 | 81 | func (e *record) size() uint32 { 82 | return entryHeaderSize + e.meta.keySize + e.meta.memberSize + e.meta.valueSize 83 | } 84 | 85 | // Encode returns the slice after the entry be encoded. 86 | // 87 | // the entry stored format: 88 | // |----------------------------------------------------------------------------------------------------------------| 89 | // | ks | ms | vs | state | timestamp | key | member | value | 90 | // |----------------------------------------------------------------------------------------------------------------| 91 | // | uint32| uint32 | uint32 | uint16 | uint64 | []byte | []byte | []byte | 92 | // |----------------------------------------------------------------------------------------------------------------| 93 | // 94 | 95 | func (e *record) encode() ([]byte, error) { 96 | if e == nil || e.meta.keySize == 0 { 97 | return nil, ErrInvalidEntry 98 | } 99 | 100 | ks, ms := e.meta.keySize, e.meta.memberSize 101 | vs := e.meta.valueSize 102 | buf := make([]byte, e.size()) 103 | 104 | binary.BigEndian.PutUint32(buf[0:4], ks) 105 | binary.BigEndian.PutUint32(buf[4:8], ms) 106 | binary.BigEndian.PutUint32(buf[8:12], vs) 107 | binary.BigEndian.PutUint16(buf[12:14], e.state) 108 | binary.BigEndian.PutUint64(buf[14:22], e.timestamp) 109 | copy(buf[entryHeaderSize:entryHeaderSize+ks], e.meta.key) 110 | copy(buf[entryHeaderSize+ks:(entryHeaderSize+ks+ms)], e.meta.member) 111 | if vs > 0 { 112 | copy(buf[(entryHeaderSize+ks+ms):(entryHeaderSize+ks+ms+vs)], e.meta.value) 113 | } 114 | 115 | return buf, nil 116 | } 117 | 118 | func decode(buf []byte) (*record, error) { 119 | ks := binary.BigEndian.Uint32(buf[0:4]) 120 | ms := binary.BigEndian.Uint32(buf[4:8]) 121 | vs := binary.BigEndian.Uint32(buf[8:12]) 122 | state := binary.BigEndian.Uint16(buf[12:14]) 123 | timestamp := binary.BigEndian.Uint64(buf[14:22]) 124 | 125 | return &record{ 126 | meta: &meta{ 127 | keySize: ks, 128 | memberSize: ms, 129 | valueSize: vs, 130 | key: buf[entryHeaderSize : entryHeaderSize+ks], 131 | member: buf[entryHeaderSize+ks : (entryHeaderSize + ks + ms)], 132 | value: buf[(entryHeaderSize + ks + ms):(entryHeaderSize + ks + ms + vs)], 133 | }, 134 | state: state, 135 | timestamp: timestamp, 136 | }, nil 137 | } 138 | 139 | func (e *record) getType() uint16 { 140 | return e.state >> 8 141 | } 142 | 143 | func (e *record) getMark() uint16 { 144 | return e.state & (2<<7 - 1) 145 | } 146 | -------------------------------------------------------------------------------- /tx_set.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // SAdd adds one or more members to the set stored at key. If a member exists at 8 | // key, it is skipped. 9 | func (tx *Tx) SAdd(key string, members ...string) (err error) { 10 | for _, m := range members { 11 | exist := tx.db.setStore.SIsMember(key, m) 12 | if !exist { 13 | e := newRecord([]byte(key), []byte(m), SetRecord, SetSAdd) 14 | tx.addRecord(e) 15 | } 16 | } 17 | return 18 | } 19 | 20 | // SIsMember checks the member is a member of set stored at key. If the key has 21 | // expired, the key is evicted. 22 | func (tx *Tx) SIsMember(key string, member string) bool { 23 | if tx.db.hasExpired(key, Set) { 24 | tx.db.evict(key, Set) 25 | return false 26 | } 27 | return tx.db.setStore.SIsMember(key, member) 28 | } 29 | 30 | // SRandMember returns random elements stored at key. If the key has expired, 31 | // the key is evicted. 32 | func (tx *Tx) SRandMember(key string, count int) (values []string) { 33 | if tx.db.hasExpired(key, Set) { 34 | tx.db.evict(key, Set) 35 | return nil 36 | } 37 | 38 | vals := tx.db.setStore.SRandMember(key, count) 39 | for _, v := range vals { 40 | values = append(values, toString(v)) 41 | } 42 | return 43 | } 44 | 45 | // SRem removes one or more members from the set stored at key. It returns 46 | // number of removed members from the set. If the key has expired, the key 47 | // is evicted. 48 | func (tx *Tx) SRem(key string, members ...string) (res int, err error) { 49 | if tx.db.hasExpired(key, Set) { 50 | tx.db.evict(key, Set) 51 | return 52 | } 53 | 54 | for _, m := range members { 55 | e := newRecord([]byte(key), []byte(m), SetRecord, SetSRem) 56 | tx.addRecord(e) 57 | res++ 58 | } 59 | return 60 | } 61 | 62 | // SMove moves a member from src to dst. If both keys have expired, the key is 63 | // evicted. 64 | func (tx *Tx) SMove(src, dst string, member string) error { 65 | if tx.db.hasExpired(src, Set) { 66 | tx.db.evict(src, Hash) 67 | return ErrExpiredKey 68 | } 69 | if tx.db.hasExpired(dst, Set) { 70 | tx.db.evict(dst, Hash) 71 | return ErrExpiredKey 72 | } 73 | 74 | ok := tx.db.setStore.SMove(src, dst, member) 75 | if ok { 76 | e := newRecordWithValue([]byte(src), []byte(member), []byte(dst), SetRecord, SetSMove) 77 | tx.addRecord(e) 78 | } 79 | return nil 80 | } 81 | 82 | // SCard returns the cardinality of the set stored at key. If the key has expired, 83 | // the key is evicted. 84 | func (tx *Tx) SCard(key string) int { 85 | if tx.db.hasExpired(key, Set) { 86 | tx.db.evict(key, Set) 87 | return 0 88 | } 89 | return tx.db.setStore.SCard(key) 90 | } 91 | 92 | // SMembers returns the members stored at key. If the key has expired, the key 93 | // is evicted. 94 | func (tx *Tx) SMembers(key string) (values []string) { 95 | if tx.db.hasExpired(key, Set) { 96 | tx.db.evict(key, Set) 97 | return 98 | } 99 | 100 | vals := tx.db.setStore.SMembers(key) 101 | for _, v := range vals { 102 | values = append(values, toString(v)) 103 | } 104 | return 105 | } 106 | 107 | // SUnion returns the members of the set resulting from union of all the given 108 | // keys. The members' type is string. If any key has expired, the key is evicted. 109 | func (tx *Tx) SUnion(keys ...string) (values []string) { 110 | var activeKeys []string 111 | for _, k := range keys { 112 | if tx.db.hasExpired(k, Set) { 113 | tx.db.evict(k, Hash) 114 | continue 115 | } 116 | activeKeys = append(activeKeys, k) 117 | } 118 | 119 | vals := tx.db.setStore.SUnion(activeKeys...) 120 | for _, v := range vals { 121 | values = append(values, toString(v)) 122 | } 123 | return 124 | } 125 | 126 | // SDiff returns the members if the set resulting from difference between the 127 | // first and all the remaining keys. If any key has expired, the key is evicted. 128 | func (tx *Tx) SDiff(keys ...string) (values []string) { 129 | var activeKeys []string 130 | for _, k := range keys { 131 | if tx.db.hasExpired(k, Set) { 132 | tx.db.evict(k, Hash) 133 | continue 134 | } 135 | activeKeys = append(activeKeys, k) 136 | } 137 | 138 | vals := tx.db.setStore.SDiff(activeKeys...) 139 | for _, v := range vals { 140 | values = append(values, toString(v)) 141 | } 142 | return 143 | } 144 | 145 | // SKeyExists returns if the key exists. 146 | func (tx *Tx) SKeyExists(key string) (ok bool) { 147 | if tx.db.hasExpired(key, Set) { 148 | tx.db.evict(key, Set) 149 | 150 | return 151 | } 152 | 153 | ok = tx.db.setStore.SKeyExists(key) 154 | return 155 | } 156 | 157 | // SClear clear the specified key in set. 158 | func (tx *Tx) SClear(key string) (err error) { 159 | if !tx.SKeyExists(key) { 160 | return ErrInvalidKey 161 | } 162 | 163 | e := newRecord([]byte(key), nil, SetRecord, SetSClear) 164 | tx.addRecord(e) 165 | return 166 | } 167 | 168 | // SExpire set expired time for the key in set. 169 | func (tx *Tx) SExpire(key string, duration int64) (err error) { 170 | if duration <= 0 { 171 | return ErrInvalidTTL 172 | } 173 | if !tx.SKeyExists(key) { 174 | return ErrInvalidKey 175 | } 176 | 177 | ttl := time.Now().Unix() + duration 178 | e := newRecordWithExpire([]byte(key), nil, ttl, SetRecord, SetSExpire) 179 | tx.addRecord(e) 180 | return 181 | } 182 | 183 | // STTL return time to live for the key in set. 184 | func (tx *Tx) STTL(key string) (ttl int64) { 185 | if tx.db.hasExpired(key, Set) { 186 | tx.db.evict(key, Set) 187 | return 188 | } 189 | 190 | deadline := tx.db.getTTL(Set, key) 191 | if deadline == nil { 192 | return 193 | } 194 | 195 | return deadline.(int64) - time.Now().Unix() 196 | } 197 | -------------------------------------------------------------------------------- /txn.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "github.com/arriqaaq/aol" 5 | ) 6 | 7 | // Tx represents a transaction on the database. This transaction can either be 8 | // read-only or read/write. Read-only transactions can be used for retrieving 9 | // values for keys and iterating through keys and values. Read/write 10 | // transactions can set and delete keys. 11 | // 12 | // All transactions must be committed or rolled-back when done. 13 | type Tx struct { 14 | db *FlashDB // the underlying database. 15 | writable bool // when false mutable operations fail. 16 | wc *txWriteContext // context for writable transactions. 17 | } 18 | 19 | func (tx *Tx) addRecord(r *record) { 20 | tx.wc.commitItems = append(tx.wc.commitItems, r) 21 | } 22 | 23 | type txWriteContext struct { 24 | commitItems []*record // details for committing tx. 25 | } 26 | 27 | // lock locks the database based on the transaction type. 28 | func (tx *Tx) lock() { 29 | if tx.writable { 30 | tx.db.mu.Lock() 31 | } else { 32 | tx.db.mu.RLock() 33 | } 34 | } 35 | 36 | // unlock unlocks the database based on the transaction type. 37 | func (tx *Tx) unlock() { 38 | if tx.writable { 39 | tx.db.mu.Unlock() 40 | } else { 41 | tx.db.mu.RUnlock() 42 | } 43 | } 44 | 45 | // managed calls a block of code that is fully contained in a transaction. 46 | // This method is intended to be wrapped by Update and View 47 | func (db *FlashDB) managed(writable bool, fn func(tx *Tx) error) (err error) { 48 | var tx *Tx 49 | tx, err = db.Begin(writable) 50 | if err != nil { 51 | return 52 | } 53 | defer func() { 54 | if err != nil { 55 | // The caller returned an error. We must rollback. 56 | _ = tx.Rollback() 57 | return 58 | } 59 | if writable { 60 | // Everything went well. Lets Commit() 61 | err = tx.Commit() 62 | } else { 63 | // read-only transaction can only roll back. 64 | err = tx.Rollback() 65 | } 66 | }() 67 | err = fn(tx) 68 | return 69 | } 70 | 71 | // Begin opens a new transaction. 72 | // Multiple read-only transactions can be opened at the same time but there can 73 | // only be one read/write transaction at a time. Attempting to open a read/write 74 | // transactions while another one is in progress will result in blocking until 75 | // the current read/write transaction is completed. 76 | // 77 | // All transactions must be closed by calling Commit() or Rollback() when done. 78 | func (db *FlashDB) Begin(writable bool) (*Tx, error) { 79 | tx := &Tx{ 80 | db: db, 81 | writable: writable, 82 | } 83 | tx.lock() 84 | if db.closed { 85 | tx.unlock() 86 | return nil, ErrDatabaseClosed 87 | } 88 | if writable { 89 | tx.wc = &txWriteContext{} 90 | if db.persist { 91 | tx.wc.commitItems = make([]*record, 0, 1) 92 | } 93 | } 94 | return tx, nil 95 | } 96 | 97 | // Commit writes all changes to disk. 98 | // An error is returned when a write error occurs, or when a Commit() is called 99 | // from a read-only transaction. 100 | func (tx *Tx) Commit() error { 101 | if tx.db == nil { 102 | return ErrTxClosed 103 | } else if !tx.writable { 104 | return ErrTxNotWritable 105 | } 106 | var err error 107 | if tx.db.persist && (len(tx.wc.commitItems) > 0) && tx.writable { 108 | batch := new(aol.Batch) 109 | // Each committed record is written to disk 110 | for _, r := range tx.wc.commitItems { 111 | rec, err := r.encode() 112 | if err != nil { 113 | return err 114 | } 115 | batch.Write(rec) 116 | } 117 | // If this operation fails then the write did failed and we must 118 | // rollback. 119 | err = tx.db.log.WriteBatch(batch) 120 | if err != nil { 121 | tx.rollback() 122 | } 123 | } 124 | 125 | // apply all commands 126 | err = tx.buildRecords(tx.wc.commitItems) 127 | // Unlock the database and allow for another writable transaction. 128 | tx.unlock() 129 | // Clear the db field to disable this transaction from future use. 130 | tx.db = nil 131 | return err 132 | } 133 | 134 | // View executes a function within a managed read-only transaction. 135 | // When a non-nil error is returned from the function that error will be return 136 | // to the caller of View(). 137 | func (db *FlashDB) View(fn func(tx *Tx) error) error { 138 | return db.managed(false, fn) 139 | } 140 | 141 | // Update executes a function within a managed read/write transaction. 142 | // The transaction has been committed when no error is returned. 143 | // In the event that an error is returned, the transaction will be rolled back. 144 | // When a non-nil error is returned from the function, the transaction will be 145 | // rolled back and the that error will be return to the caller of Update(). 146 | func (db *FlashDB) Update(fn func(tx *Tx) error) error { 147 | return db.managed(true, fn) 148 | } 149 | 150 | // Rollback closes the transaction and reverts all mutable operations that 151 | // were performed on the transaction such as Set() and Delete(). 152 | // 153 | // Read-only transactions can only be rolled back, not committed. 154 | func (tx *Tx) Rollback() error { 155 | if tx.db == nil { 156 | return ErrTxClosed 157 | } 158 | // The rollback func does the heavy lifting. 159 | if tx.writable { 160 | tx.rollback() 161 | } 162 | // unlock the database for more transactions. 163 | tx.unlock() 164 | // Clear the db field to disable this transaction from future use. 165 | tx.db = nil 166 | return nil 167 | } 168 | 169 | // rollback handles the underlying rollback logic. 170 | // Intended to be called from Commit() and Rollback(). 171 | func (tx *Tx) rollback() { 172 | tx.wc.commitItems = nil 173 | } 174 | 175 | func (tx *Tx) buildRecords(recs []*record) (err error) { 176 | for _, r := range recs { 177 | switch r.getType() { 178 | case StringRecord: 179 | err = tx.db.buildStringRecord(r) 180 | case HashRecord: 181 | err = tx.db.buildHashRecord(r) 182 | case SetRecord: 183 | err = tx.db.buildSetRecord(r) 184 | case ZSetRecord: 185 | err = tx.db.buildZsetRecord(r) 186 | } 187 | } 188 | return 189 | } 190 | -------------------------------------------------------------------------------- /tx_zset.go: -------------------------------------------------------------------------------- 1 | package flashdb 2 | 3 | import ( 4 | "time" 5 | ) 6 | 7 | // ZAdd adds key-member pair with the score. If the key-member pair already 8 | // exists and the old score is the same as the new score, it doesn't do anything. 9 | func (tx *Tx) ZAdd(key string, score float64, member string) error { 10 | if ok, oldScore := tx.ZScore(key, member); ok && oldScore == score { 11 | return nil 12 | } 13 | 14 | value := float64ToStr(score) 15 | e := newRecordWithValue([]byte(key), []byte(member), []byte(value), ZSetRecord, ZSetZAdd) 16 | tx.addRecord(e) 17 | 18 | return nil 19 | } 20 | 21 | // ZScore returns score of the given key-member pair.If the key has expired, 22 | // the key is evicted. 23 | func (tx *Tx) ZScore(key string, member string) (ok bool, score float64) { 24 | if tx.db.hasExpired(key, ZSet) { 25 | tx.db.evict(key, ZSet) 26 | return 27 | } 28 | 29 | return tx.db.zsetStore.ZScore(key, member) 30 | } 31 | 32 | // ZCard returns sorted set cardinality(number of elements) of the sorted set 33 | // stored at key. If the key has expired, the key is evicted. 34 | func (tx *Tx) ZCard(key string) int { 35 | if tx.db.hasExpired(key, ZSet) { 36 | tx.db.evict(key, ZSet) 37 | return 0 38 | } 39 | 40 | return tx.db.zsetStore.ZCard(key) 41 | } 42 | 43 | // ZRank returns the rank of the member at key, with the scores ordered from 44 | // low to high. If the key has expired, the key is evicted. 45 | func (tx *Tx) ZRank(key string, member string) int64 { 46 | if tx.db.hasExpired(key, ZSet) { 47 | tx.db.evict(key, ZSet) 48 | return -1 49 | } 50 | 51 | return tx.db.zsetStore.ZRank(key, member) 52 | } 53 | 54 | // ZRevRank returns the rank of the member at key, with the scores ordered from 55 | // high to low. If the key has expired, the key is evicted. 56 | func (tx *Tx) ZRevRank(key string, member string) int64 { 57 | if tx.db.hasExpired(key, ZSet) { 58 | tx.db.evict(key, ZSet) 59 | return -1 60 | } 61 | 62 | return tx.db.zsetStore.ZRevRank(key, member) 63 | } 64 | 65 | // ZRange returns the specified range of elements in the sorted set stored at 66 | // key. If the key has expired, the key is evicted. 67 | func (tx *Tx) ZRange(key string, start, stop int) []interface{} { 68 | if tx.db.hasExpired(key, ZSet) { 69 | tx.db.evict(key, ZSet) 70 | return nil 71 | } 72 | 73 | return tx.db.zsetStore.ZRange(key, start, stop) 74 | } 75 | 76 | // ZRangeWithScores returns the specified range of elements with scores in the 77 | // sorted set stored at key. If the key has expired, the key is evicted. 78 | func (tx *Tx) ZRangeWithScores(key string, start, stop int) []interface{} { 79 | if tx.db.hasExpired(key, ZSet) { 80 | tx.db.evict(key, ZSet) 81 | return nil 82 | } 83 | 84 | return tx.db.zsetStore.ZRangeWithScores(key, start, stop) 85 | } 86 | 87 | // ZRevRange returns the specified range of elements in the sorted set stored at 88 | // key. The elements are ordered from the highest score to the lowest score. If 89 | // key has expired, the key is evicted. 90 | func (tx *Tx) ZRevRange(key string, start, stop int) []interface{} { 91 | if tx.db.hasExpired(key, ZSet) { 92 | tx.db.evict(key, ZSet) 93 | return nil 94 | } 95 | 96 | return tx.db.zsetStore.ZRevRange(key, start, stop) 97 | } 98 | 99 | // ZRevRangeWithScores returns the specified range of elements in the sorted set 100 | // at key. The elements are ordered from the highest to the lowest score. If key 101 | // has expired, the key is evicted. 102 | func (tx *Tx) ZRevRangeWithScores(key string, start, stop int) []interface{} { 103 | if tx.db.hasExpired(key, ZSet) { 104 | tx.db.evict(key, ZSet) 105 | return nil 106 | } 107 | 108 | return tx.db.zsetStore.ZRevRangeWithScores(key, start, stop) 109 | } 110 | 111 | // ZRem removes the member from the sorted set at key. 112 | func (tx *Tx) ZRem(key string, member string) (ok bool, err error) { 113 | if tx.db.hasExpired(key, ZSet) { 114 | tx.db.evict(key, ZSet) 115 | return 116 | } 117 | 118 | ok = tx.db.zsetStore.ZRem(key, member) 119 | if ok { 120 | e := newRecord([]byte(key), []byte(member), ZSetRecord, ZSetZRem) 121 | tx.addRecord(e) 122 | } 123 | 124 | return 125 | } 126 | 127 | // ZGetByRank returns the members by given rank at key. If the key has expired, 128 | // the key is evicted. 129 | func (tx *Tx) ZGetByRank(key string, rank int) []interface{} { 130 | if tx.db.hasExpired(key, ZSet) { 131 | tx.db.evict(key, ZSet) 132 | return nil 133 | } 134 | 135 | return tx.db.zsetStore.ZGetByRank(key, rank) 136 | } 137 | 138 | // ZRevGetByRank returns the members by given rank at key. The members are 139 | // returned reverse ordered. If the key has expired, the key is evicted. 140 | func (tx *Tx) ZRevGetByRank(key string, rank int) []interface{} { 141 | if tx.db.hasExpired(key, ZSet) { 142 | tx.db.evict(key, ZSet) 143 | return nil 144 | } 145 | 146 | return tx.db.zsetStore.ZRevGetByRank(key, rank) 147 | } 148 | 149 | // ZScoreRange returns the members in given range at key. If the key has expired, 150 | // the key is evicted. 151 | func (tx *Tx) ZScoreRange(key string, min, max float64) []interface{} { 152 | if tx.db.hasExpired(key, ZSet) { 153 | tx.db.evict(key, ZSet) 154 | return nil 155 | } 156 | 157 | return tx.db.zsetStore.ZScoreRange(key, min, max) 158 | } 159 | 160 | // ZRevScoreRange returns the members in given range at key. The members are 161 | // returned in reverse order. If the key has expired, the key is evicted. 162 | func (tx *Tx) ZRevScoreRange(key string, max, min float64) []interface{} { 163 | if tx.db.hasExpired(key, ZSet) { 164 | tx.db.evict(key, ZSet) 165 | return nil 166 | } 167 | 168 | return tx.db.zsetStore.ZRevScoreRange(key, max, min) 169 | } 170 | 171 | // ZKeyExists checks the sorted set whether the key exists. If the key has expired, 172 | // the key is evicted. 173 | func (tx *Tx) ZKeyExists(key string) (ok bool) { 174 | if tx.db.hasExpired(key, ZSet) { 175 | tx.db.evict(key, ZSet) 176 | return 177 | } 178 | 179 | ok = tx.db.zsetStore.ZKeyExists(key) 180 | return 181 | } 182 | 183 | // ZClear clears the members at key. 184 | func (tx *Tx) ZClear(key string) (err error) { 185 | e := newRecord([]byte(key), nil, ZSetRecord, ZSetZClear) 186 | tx.addRecord(e) 187 | 188 | return 189 | } 190 | 191 | // ZExpire sets expire time at key. duration should be more than zero. 192 | func (tx *Tx) ZExpire(key string, duration int64) (err error) { 193 | if duration <= 0 { 194 | return ErrInvalidTTL 195 | } 196 | if !tx.ZKeyExists(key) { 197 | return ErrInvalidKey 198 | } 199 | 200 | ttl := time.Now().Unix() + duration 201 | e := newRecordWithExpire([]byte(key), nil, ttl, ZSetRecord, ZSetZExpire) 202 | tx.addRecord(e) 203 | return 204 | } 205 | 206 | // ZTTL returns the remaining TTL of the given key. 207 | func (tx *Tx) ZTTL(key string) (ttl int64) { 208 | if !tx.ZKeyExists(key) { 209 | return 210 | } 211 | 212 | deadline := tx.db.getTTL(ZSet, key) 213 | if deadline == nil { 214 | return 215 | } 216 | return deadline.(int64) - time.Now().Unix() 217 | } 218 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | FlashDB 4 |

5 | 6 | # flashdb 7 | 8 | FlashDB is a simple, in-memory, key/value store in pure Go. 9 | It persists to disk, is ACID compliant, and uses locking for multiple 10 | readers and a single writer. It supports redis like operations for 11 | data structures like SET, SORTED SET, HASH and STRING. 12 | 13 | Features 14 | ======== 15 | 16 | - In-memory database for [fast reads and writes](#performance) 17 | - Embeddable with a simple API 18 | - Supports Redis like operations for SET, SORTED SET, HASH and STRING 19 | - [Durable append-only file](#append-only-file) format for persistence 20 | - Option to evict old items with an [expiration](#data-expiration) TTL 21 | - ACID semantics with locking [transactions](#transactions) that support rollbacks 22 | 23 | 24 | Architecture 25 | ============= 26 | 27 | FlashDB is made of composable libraries that can be used independently and are easy to understand. The idea is to bridge the 28 | learning for anyone new on how to build a simple ACID database. 29 | 30 | 31 | - [Set](https://github.com/arriqaaq/set) 32 | - [ZSet](https://github.com/arriqaaq/zset) 33 | - [String](https://github.com/arriqaaq/art) 34 | - [Hash](https://github.com/arriqaaq/hash) 35 | - [Append Only Log](https://github.com/arriqaaq/aol) 36 | 37 | 38 | Getting Started 39 | =============== 40 | 41 | ## Installing 42 | 43 | To start using FlashDB, install Go and run `go get`: 44 | 45 | ```sh 46 | $ go get -u github.com/arriqaaq/flashdb 47 | ``` 48 | 49 | This will retrieve the library. 50 | 51 | 52 | ## Opening a database 53 | 54 | The primary object in FlashDB is a `DB`. To open or create your 55 | database, use the `flashdb.New()` function: 56 | 57 | ```go 58 | package main 59 | 60 | import ( 61 | "log" 62 | 63 | "github.com/arriqaaq/flashdb" 64 | ) 65 | 66 | func main() { 67 | config := &flashdb.Config{Path:"/tmp", EvictionInterval: 10} 68 | db, err := flashdb.New(config) 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | defer db.Close() 73 | 74 | ... 75 | } 76 | ``` 77 | 78 | It's also possible to open a database that does not persist to disk by keeping the path in the config empty. 79 | 80 | ```go 81 | config := &flashdb.Config{Path:"", EvictionInterval: 10} 82 | flashdb.New(config) 83 | ``` 84 | 85 | ## Transactions 86 | All reads and writes must be performed from inside a transaction. FlashDB can have one write transaction opened at a time, but can have many concurrent read transactions. Each transaction maintains a stable view of the database. In other words, once a transaction has begun, the data for that transaction cannot be changed by other transactions. 87 | 88 | When a transaction fails, it will roll back, and revert all changes that occurred to the database during that transaction. When a read/write transaction succeeds all changes are persisted to disk. 89 | 90 | ### Read-only Transactions 91 | A read-only transaction should be used when you don't need to make changes to the data. The advantage of a read-only transaction is that there can be many running concurrently. 92 | 93 | ```go 94 | err := db.View(func(tx *flashdb.Tx) error { 95 | ... 96 | return nil 97 | }) 98 | ``` 99 | 100 | ### Read/write Transactions 101 | A read/write transaction is used when you need to make changes to your data. There can only be one read/write transaction running at a time. So make sure you close it as soon as you are done with it. 102 | 103 | ```go 104 | err := db.Update(func(tx *flashdb.Tx) error { 105 | ... 106 | return nil 107 | }) 108 | ``` 109 | 110 | ### Setting and getting key/values 111 | 112 | To set a value you must open a read/write transaction: 113 | 114 | ```go 115 | err := db.Update(func(tx *flashdb.Tx) error { 116 | err := tx.Set("mykey", "myvalue") 117 | return err 118 | }) 119 | ``` 120 | 121 | 122 | To get the value: 123 | 124 | ```go 125 | err := db.View(func(tx *flashdb.Tx) error { 126 | val, err := tx.Get("mykey") 127 | if err != nil{ 128 | return err 129 | } 130 | fmt.Printf("value is %s\n", val) 131 | return nil 132 | }) 133 | ``` 134 | 135 | Commands 136 | ======== 137 | | String | Hash | Set | ZSet | 138 | |--------|---------|-------------|----------------| 139 | | SET | HSET | SADD | ZADD | 140 | | GET | HGET | SISMEMBER | ZSCORE | 141 | | DELETE | HGETALL | SRANDMEMBER | ZCARD | 142 | | EXPIRE | HDEL | SREM | ZRANK | 143 | | TTL | HEXISTS | SMOVE | ZREVRANK | 144 | | | HLEN | SCARD | ZRANGE | 145 | | | HKEYS | SMEMBERS | ZREVRANGE | 146 | | | HVALS | SUNION | ZREM | 147 | | | HCLEAR | SDIFF | ZGETBYRANK | 148 | | | | SCLEAR | ZREVGETBYRANK | 149 | | | | | ZSCORERANGE | 150 | | | | | ZREVSCORERANGE | 151 | | | | | ZCLEAR | 152 | 153 | Benchmarks 154 | ========== 155 | 156 | * Go Version : go1.11.4 darwin/amd64 157 | * OS: Mac OS X 10.13.6 158 | * Architecture: x86_64 159 | * 16 GB 2133 MHz LPDDR3 160 | * CPU: 3.1 GHz Intel Core i7 161 | 162 | ``` 163 | badger 2022/03/09 14:04:44 INFO: All 0 tables opened in 0s 164 | goos: darwin 165 | goarch: amd64 166 | pkg: github.com/arriqaaq/flashbench 167 | cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz 168 | 169 | BenchmarkBadgerDBPutValue64B-16 9940 141844 ns/op 2208 B/op 68 allocs/op 170 | BenchmarkBadgerDBPutValue128B-16 7701 192942 ns/op 2337 B/op 68 allocs/op 171 | BenchmarkBadgerDBPutValue256B-16 7368 142600 ns/op 2637 B/op 69 allocs/op 172 | BenchmarkBadgerDBPutValue512B-16 6980 148056 ns/op 3149 B/op 69 allocs/op 173 | BenchmarkBadgerDBGet-16 1000000 1388 ns/op 408 B/op 9 allocs/op 174 | 175 | BenchmarkFlashDBPutValue64B-16 204318 5129 ns/op 1385 B/op 19 allocs/op 176 | BenchmarkFlashDBPutValue128B-16 231177 5318 ns/op 1976 B/op 16 allocs/op 177 | BenchmarkFlashDBPutValue256B-16 189516 6202 ns/op 3263 B/op 15 allocs/op 178 | BenchmarkFlashDBPutValue512B-16 165580 8110 ns/op 5866 B/op 16 allocs/op 179 | BenchmarkFlashDBGet-16 4053836 295 ns/op 32 B/op 2 allocs/op 180 | 181 | PASS 182 | ok github.com/arriqaaq/flashbench 28.947s 183 | ``` 184 | 185 | #### With fsync enabled for every update transaction 186 | 187 | ``` 188 | badger 2022/03/09 14:04:44 INFO: All 0 tables opened in 0s 189 | goos: darwin 190 | goarch: amd64 191 | pkg: github.com/arriqaaq/flashbench 192 | cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz 193 | 194 | BenchmarkNutsDBPutValue64B-16 52 20301019 ns/op 1315 B/op 17 allocs/op 195 | BenchmarkNutsDBPutValue128B-16 63 23496536 ns/op 1059 B/op 15 allocs/op 196 | BenchmarkNutsDBPutValue256B-16 62 20037952 ns/op 1343 B/op 15 allocs/op 197 | BenchmarkNutsDBPutValue512B-16 62 20090731 ns/op 1754 B/op 15 allocs/op 198 | 199 | BenchmarkFlashDBPutValue64B-16 62 18364330 ns/op 692 B/op 15 allocs/op 200 | BenchmarkFlashDBPutValue128B-16 64 18315903 ns/op 1015 B/op 15 allocs/op 201 | BenchmarkFlashDBPutValue256B-16 64 19250441 ns/op 1694 B/op 15 allocs/op 202 | BenchmarkFlashDBPutValue512B-16 61 18811900 ns/op 2976 B/op 15 allocs/op 203 | BenchmarkFlashDBGet-16 3599500 340.7 ns/op 32 B/op 2 allocs/op 204 | 205 | PASS 206 | ok github.com/arriqaaq/flashbench 28.947s 207 | ``` 208 | 209 | The benchmark code can be found here [flashdb-bench](https://github.com/arriqaaq/flashdb-bench). 210 | 211 | 212 | 213 | TODO 214 | ==== 215 | 216 | FlashDB is in early stages of development. A couple of to-do tasks listed: 217 | 218 | - Add more comprehensive unit test cases 219 | - Add explicit documentation on various usecases 220 | 221 | 222 | References 223 | ========== 224 | 225 | FlashDB is inspired by NutsDB and BuntDB. 226 | 227 | 228 | ## Contact 229 | Farhan Khan [@arriqaaq](http://twitter.com/arriqaaq) 230 | 231 | ## License 232 | FlashDB source code is available under the MIT [License](/LICENSE) 233 | --------------------------------------------------------------------------------