├── LISENSE ├── README.md ├── compress.go ├── compress_test.go ├── config.go ├── consts.go ├── errors.go ├── header.go ├── index.go ├── kvstore.go ├── kvstore_test.go └── types.go /LISENSE: -------------------------------------------------------------------------------- 1 | kvstore is an Open Source project licensed under the terms of 2 | the LGPLv3 license. Please see 3 | for license text. 4 | 5 | Copyright (c) Chris Lee(feilee1987@gmail.com). All rights reserved. 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | kvstore 2 | ============== 3 | Kvstore for time series data, with indexing and data compression. 4 | 5 | Features 6 | --------------- 7 | * Store data with a uint64 key, for example use date(20160101) as key 8 | * Support Get and Append 9 | * Support data compression, both gzip and snappy 10 | 11 | Architectures 12 | --------------- 13 | CompressCodec | FileCapacity | FileDataNums 14 | -----------------|------|--------- 15 | Index(Key, Offset, Size) | Index | ... ... 16 | Data Block | Data Block | ... ... 17 | 18 | Each datafile contains three parts: 19 | * File header 20 | * File header contains CompressCodec, FileCapacity and FileDataNums. 21 | * Indexes 22 | * Indexes are sorted by key. Each index contains Key, Offset and Size for the corresponding data block. 23 | * Data blocks 24 | * Data blocks can be compressed, support gzip and snappy. 25 | 26 | Usages 27 | --------------- 28 | ```go 29 | var ( 30 | key1 uint64 = 20150110 31 | key2 uint64 = 20150120 32 | key3 uint64 = 20150130 33 | 34 | buf1 []byte = []byte("hello world 1.") 35 | buf2 []byte = []byte("hello world 2.") 36 | buf3 []byte = []byte("hello world 3.") 37 | ) 38 | 39 | func main() { 40 | c := &Config{ 41 | FileName: "./data/gzip.file", 42 | FileCapacity: 3, 43 | CompressCodec: 1} 44 | 45 | kv, _ := New(c) 46 | 47 | kv.Append(key1, buf1) 48 | kv.Append(key2, buf2) 49 | kv.Append(key3, buf3) 50 | 51 | buf, _ := kv.Get(key1) 52 | 53 | kv.Close() 54 | } 55 | 56 | ``` 57 | 58 | Discussion 59 | --------------- 60 | [Discussion](https://forum.golangbridge.org/t/kvstore-kvstore-for-time-series-data-with-indexing-and-data-compression/5218) at Go Forum. 61 | -------------------------------------------------------------------------------- /compress.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "io" 7 | 8 | "github.com/golang/snappy" 9 | ) 10 | 11 | // None 12 | func NoneCompress(bs []byte) []byte { 13 | return bs 14 | } 15 | 16 | func NoneUnCompress(bs []byte) []byte { 17 | return bs 18 | } 19 | 20 | // Gzip 21 | func GzipCompress(bs []byte) []byte { 22 | var buf bytes.Buffer 23 | zw := gzip.NewWriter(&buf) 24 | zw.Write(bs) 25 | zw.Flush() 26 | zw.Close() 27 | 28 | return buf.Bytes() 29 | } 30 | 31 | func GzipUnCompress(bs []byte) []byte { 32 | var buf bytes.Buffer 33 | bsBuf := bytes.NewBuffer(bs) 34 | zr, _ := gzip.NewReader(bsBuf) 35 | io.Copy(&buf, zr) 36 | zr.Close() 37 | 38 | return buf.Bytes() 39 | } 40 | 41 | // Snappy 42 | func SnappyCompress(bs []byte) []byte { 43 | return snappy.Encode(nil, bs) 44 | } 45 | 46 | func SnappyUnCompress(bs []byte) []byte { 47 | buf, _ := snappy.Decode(nil, bs) 48 | return buf 49 | } 50 | -------------------------------------------------------------------------------- /compress_test.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | var poem string = ` 9 | O how much more doth beauty beauteous seem, 10 | By that sweet ornament which truth doth give! 11 | The rose looks fair, but fairer we it deem 12 | For that sweet odour which doth in it live. 13 | The canker-blooms have full as deep a dye 14 | As the perfumed tincture of the roses, 15 | Hang on such thorns and play as wantonly 16 | When summer's breath their masked buds discloses: 17 | But, for their virtue only is their show, 18 | They live unwoo'd and unrespected fade, 19 | Die to themselves. Sweet roses do not so; 20 | Of their sweet deaths are sweetest odours made: 21 | And so of you, beauteous and lovely youth, 22 | When that shall fade, my verse distills your truth. 23 | ` 24 | 25 | func TestGzip(t *testing.T) { 26 | bs := []byte(poem) 27 | b1 := GzipCompress(bs) 28 | b2 := GzipUnCompress(b1) 29 | 30 | if !reflect.DeepEqual(bs, b2) { 31 | t.Errorf("After GzipCompress and GzipUncompress, slice of bytes should be the same.") 32 | } 33 | } 34 | 35 | func TestSnappy(t *testing.T) { 36 | bs := []byte(poem) 37 | b1 := SnappyCompress(bs) 38 | b2 := SnappyUnCompress(b1) 39 | 40 | if !reflect.DeepEqual(bs, b2) { 41 | t.Errorf("After SnappyCompress and SnappyUncompress, slice of bytes should be the same.") 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | type Config struct { 4 | FileName string 5 | FileCapacity uint64 // only use for create file(default 10000); for reopen file, ignore 6 | CompressCodec uint64 // only use for create file(default none) ; for reopen file, ignore 7 | } 8 | 9 | func (c *Config) GetCompressCodec() (string, error) { 10 | switch c.CompressCodec { 11 | case NoneCompressCodec: 12 | return "None", nil 13 | case GzipCompressCodec: 14 | return "Gzip", nil 15 | case SnappyCompressCodec: 16 | return "Snappy", nil 17 | default: 18 | return "Error compress codec", ErrCompressCodec 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /consts.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | const ( 4 | DefaultFileCapacity uint64 = 10000 5 | SizeOfFileHeader uint64 = 24 6 | SizeOfOneIndex uint64 = 24 7 | StartOffsetForIndexes uint64 = 24 8 | ) 9 | 10 | const ( 11 | NoneCompressCodec uint64 = 0 12 | GzipCompressCodec uint64 = 1 13 | SnappyCompressCodec uint64 = 2 14 | ) 15 | 16 | func (kv *KvStore) setCompressFunc() { 17 | switch kv.CompressCodec { 18 | case NoneCompressCodec: 19 | kv.Compress = NoneCompress 20 | kv.UnCompress = NoneUnCompress 21 | case GzipCompressCodec: 22 | kv.Compress = GzipCompress 23 | kv.UnCompress = GzipUnCompress 24 | case SnappyCompressCodec: 25 | kv.Compress = SnappyCompress 26 | kv.UnCompress = SnappyUnCompress 27 | default: 28 | kv.Compress = NoneCompress 29 | kv.UnCompress = NoneUnCompress 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | import ( 4 | "errors" 5 | ) 6 | 7 | var ( 8 | ErrOutOfCapacity = errors.New("Out of capacity, can't store more data!") 9 | ErrDataNotFound = errors.New("No matching date!") 10 | ErrCompressCodec = errors.New("Error compress codec!") 11 | ErrWrongConfig = errors.New("Wrong config!") 12 | ErrFailedCreateFile = errors.New("Failed to create file!") 13 | ErrFailedOpenFile = errors.New("Failed to open file!") 14 | ErrAppendFail = errors.New("Append with key less than last key in file!") 15 | ) 16 | -------------------------------------------------------------------------------- /header.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | type FileHeader struct { 4 | CompressCodec uint64 5 | FileCapacity uint64 6 | FileDataNums uint64 7 | } 8 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | type Index struct { 4 | Key uint64 // for example, use date: 20160101 5 | Offset uint64 6 | Size uint64 7 | } 8 | -------------------------------------------------------------------------------- /kvstore.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | import ( 4 | "encoding/binary" 5 | "os" 6 | "sort" 7 | "sync" 8 | ) 9 | 10 | type KvStore struct { 11 | sync.RWMutex 12 | 13 | FileHeader 14 | Indexs []Index 15 | File *os.File 16 | Compress func([]byte) []byte 17 | UnCompress func([]byte) []byte 18 | } 19 | 20 | func New(c *Config) (*KvStore, error) { 21 | if c == nil || c.FileName == "" { 22 | return nil, ErrWrongConfig 23 | } 24 | 25 | kv := new(KvStore) 26 | // File doesn't exist 27 | if _, err := os.Stat(c.FileName); os.IsNotExist(err) { 28 | if c.FileCapacity <= 0 { 29 | c.FileCapacity = DefaultFileCapacity 30 | 31 | } 32 | if _, err := c.GetCompressCodec(); err != nil { 33 | c.CompressCodec = 0 34 | } 35 | 36 | f, err := os.OpenFile(c.FileName, os.O_RDWR|os.O_CREATE, 0755) 37 | if err != nil { 38 | return nil, ErrFailedCreateFile 39 | } 40 | 41 | kv.CompressCodec = c.CompressCodec 42 | kv.FileCapacity = c.FileCapacity 43 | kv.FileDataNums = 0 44 | kv.File = f 45 | kv.updateFileHeader() 46 | kv.setCompressFunc() 47 | } else { 48 | f, err := os.OpenFile(c.FileName, os.O_RDWR|os.O_CREATE, 0755) 49 | if err != nil { 50 | return nil, ErrFailedOpenFile 51 | } 52 | kv.File = f 53 | kv.loadFileHeader() 54 | kv.setCompressFunc() 55 | kv.Indexs = make([]Index, kv.FileDataNums) 56 | kv.loadIndexes() 57 | } 58 | 59 | return kv, nil 60 | } 61 | 62 | func (kv *KvStore) Close() error { 63 | return kv.File.Close() 64 | } 65 | 66 | func (kv *KvStore) updateFileHeader() error { 67 | kv.Lock() 68 | defer kv.Unlock() 69 | 70 | return kv.updateFileHeaderNoLock() 71 | } 72 | 73 | func (kv *KvStore) updateFileHeaderNoLock() error { 74 | buf := make([]byte, SizeOfFileHeader) 75 | binary.LittleEndian.PutUint64(buf[:8], kv.CompressCodec) 76 | binary.LittleEndian.PutUint64(buf[8:16], kv.FileCapacity) 77 | binary.LittleEndian.PutUint64(buf[16:], kv.FileDataNums) 78 | 79 | _, err := kv.WriteAt(buf, 0) 80 | return err 81 | } 82 | 83 | func (kv *KvStore) appendLastIndexNoLock() error { 84 | idx := kv.Indexs[len(kv.Indexs)-1] 85 | buf := make([]byte, SizeOfOneIndex) 86 | binary.LittleEndian.PutUint64(buf[:8], idx.Key) 87 | binary.LittleEndian.PutUint64(buf[8:16], idx.Offset) 88 | binary.LittleEndian.PutUint64(buf[16:], idx.Size) 89 | 90 | _, err := kv.WriteAt(buf, StartOffsetForIndexes+SizeOfOneIndex*(uint64)(len(kv.Indexs)-1)) 91 | return err 92 | } 93 | 94 | func (kv *KvStore) loadFileHeader() { 95 | kv.RLock() 96 | defer kv.RUnlock() 97 | 98 | buf := make([]byte, SizeOfFileHeader) 99 | kv.File.ReadAt(buf, 0) 100 | kv.CompressCodec = binary.LittleEndian.Uint64(buf[:8]) 101 | kv.FileCapacity = binary.LittleEndian.Uint64(buf[8:16]) 102 | kv.FileDataNums = binary.LittleEndian.Uint64(buf[16:]) 103 | } 104 | 105 | func (kv *KvStore) loadIndexes() { 106 | kv.RLock() 107 | defer kv.RUnlock() 108 | 109 | buf := make([]byte, SizeOfOneIndex*kv.FileDataNums) 110 | kv.ReadAt(buf, StartOffsetForIndexes) 111 | for i := uint64(0); i < kv.FileDataNums; i++ { 112 | idx := Index{} 113 | offset := i * SizeOfOneIndex 114 | idx.Key = binary.LittleEndian.Uint64(buf[offset : offset+8]) 115 | idx.Offset = binary.LittleEndian.Uint64(buf[offset+8 : offset+16]) 116 | idx.Size = binary.LittleEndian.Uint64(buf[offset+16 : offset+24]) 117 | kv.Indexs = append(kv.Indexs, idx) 118 | } 119 | } 120 | 121 | func (kv *KvStore) Get(key uint64) ([]byte, error) { 122 | kv.RLock() 123 | defer kv.RUnlock() 124 | 125 | idx := sort.Search(len(kv.Indexs), func(i int) bool { return kv.Indexs[i].Key >= key }) 126 | if idx == len(kv.Indexs) || kv.Indexs[idx].Key != key { 127 | return nil, ErrDataNotFound 128 | } 129 | offset := kv.Indexs[idx].Offset 130 | size := kv.Indexs[idx].Size 131 | buf := make([]byte, size) 132 | kv.ReadAt(buf, offset) 133 | return kv.UnCompress(buf), nil 134 | } 135 | 136 | func (kv *KvStore) Append(key uint64, buf []byte) error { 137 | kv.Lock() 138 | defer kv.Unlock() 139 | 140 | if kv.FileDataNums >= kv.FileCapacity { 141 | return ErrOutOfCapacity 142 | } 143 | 144 | n := len(kv.Indexs) 145 | if n > 0 && kv.Indexs[n-1].Key >= key { 146 | return ErrAppendFail 147 | } 148 | 149 | wb := kv.Compress(buf) 150 | 151 | // FileHeader 152 | kv.FileDataNums += 1 153 | kv.updateFileHeaderNoLock() 154 | 155 | // Indexes 156 | var idx Index 157 | idx.Key = key 158 | idx.Size = uint64(len(wb)) 159 | if n == 0 { 160 | idx.Offset = StartOffsetForIndexes + SizeOfOneIndex*DefaultFileCapacity 161 | } else { 162 | idx.Offset = kv.Indexs[n-1].Offset + kv.Indexs[n-1].Size 163 | } 164 | kv.Indexs = append(kv.Indexs, idx) 165 | kv.appendLastIndexNoLock() 166 | 167 | // FileBody 168 | kv.WriteAt(wb, idx.Offset) 169 | return nil 170 | } 171 | -------------------------------------------------------------------------------- /kvstore_test.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | import ( 4 | "os" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | var ( 10 | key1 uint64 = 20150110 11 | key2 uint64 = 20150120 12 | key3 uint64 = 20150130 13 | 14 | keylesser uint64 = 20150101 15 | keymiss uint64 = 20150115 16 | keymore uint64 = 20150131 17 | 18 | buf0 []byte = []byte("hello world.") 19 | buf1 []byte = []byte("hello world 1.") 20 | buf2 []byte = []byte("hello world 2.") 21 | buf3 []byte = []byte("hello world 3.") 22 | ) 23 | 24 | func TestWrongConfig(t *testing.T) { 25 | c := &Config{} 26 | _, err := New(c) 27 | 28 | if err != ErrWrongConfig { 29 | t.Errorf("Config without filename, can't create kvstore!") 30 | } 31 | } 32 | 33 | func TestCreateEmptyFileAndGet(t *testing.T) { 34 | os.Remove("./data/empty.gzip.file") 35 | 36 | c := &Config{ 37 | FileName: "./data/empty.gzip.file", 38 | FileCapacity: 10, 39 | CompressCodec: 1} 40 | 41 | kv, err := New(c) 42 | if err != nil { 43 | t.Errorf("Failed to create kv, reason: %v\n", err) 44 | } 45 | 46 | _, err = kv.Get(key1) 47 | if err != ErrDataNotFound { 48 | t.Errorf("Get empty file, return value should be nil!") 49 | } 50 | 51 | kv.Close() 52 | } 53 | 54 | func TestAppendAndGet(t *testing.T) { 55 | os.Remove("./data/gzip.file") 56 | 57 | c := &Config{ 58 | FileName: "./data/gzip.file", 59 | FileCapacity: 3, 60 | CompressCodec: 1} 61 | 62 | kv, err := New(c) 63 | if err != nil { 64 | t.Errorf("Failed to create kv, reason: %v\n", err) 65 | } 66 | 67 | err = kv.Append(key1, buf1) 68 | if err != nil { 69 | t.Errorf("Failed to create key1") 70 | } 71 | 72 | err = kv.Append(key1, buf1) 73 | if err != ErrAppendFail { 74 | t.Errorf("key1 equal key1, shouldn't been appended.") 75 | } 76 | 77 | err = kv.Append(keylesser, buf0) 78 | if err != ErrAppendFail { 79 | t.Errorf("keylesser less than key1, shouldn't been appended.") 80 | } 81 | 82 | err = kv.Append(key2, buf2) 83 | if err != nil { 84 | t.Errorf("Failed to create key2, reason: %v\n", err) 85 | } 86 | 87 | err = kv.Append(key3, buf3) 88 | if err != nil { 89 | t.Errorf("Failed to create key3, reason: %v\n", err) 90 | } 91 | 92 | err = kv.Append(keymore, buf0) 93 | if err != ErrOutOfCapacity { 94 | t.Errorf("Out out the capacity, should not be appended. Reason: %v\n", err) 95 | } 96 | 97 | buf, err := kv.Get(key1) 98 | if !reflect.DeepEqual(buf, buf1) { 99 | t.Errorf("buf should equal buf1") 100 | } 101 | 102 | buf, err = kv.Get(key2) 103 | if !reflect.DeepEqual(buf, buf2) { 104 | t.Errorf("buf should equal buf2") 105 | } 106 | 107 | buf, err = kv.Get(key3) 108 | if !reflect.DeepEqual(buf, buf3) { 109 | t.Errorf("buf should equal buf3") 110 | } 111 | 112 | buf, err = kv.Get(keylesser) 113 | if err != ErrDataNotFound { 114 | t.Errorf("key is lesser, should not be found!") 115 | } 116 | 117 | buf, err = kv.Get(keymiss) 118 | if err != ErrDataNotFound { 119 | t.Errorf("key is miss, should not be found!") 120 | } 121 | 122 | buf, err = kv.Get(keymore) 123 | if err != ErrDataNotFound { 124 | t.Errorf("key is bigger, should not be found!") 125 | } 126 | 127 | kv.Close() 128 | } 129 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package kvstore 2 | 3 | func (kv *KvStore) ReadAt(b []byte, offset uint64) (n int, err error) { 4 | return kv.File.ReadAt(b, int64(offset)) 5 | } 6 | 7 | func (kv *KvStore) WriteAt(b []byte, offset uint64) (n int, err error) { 8 | return kv.File.WriteAt(b, int64(offset)) 9 | } 10 | --------------------------------------------------------------------------------