├── .gitignore ├── options.go ├── example ├── bitcask_main.go └── decode_main.go ├── doc └── doc.md ├── entry.go ├── README.md ├── keydir.go ├── http └── bitcaskhttp.go ├── bitcask_test.go ├── encoding_test.go ├── encode.go ├── file.go ├── util.go └── bitcask.go /.gitignore: -------------------------------------------------------------------------------- 1 | ### Go template 2 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 3 | *.o 4 | *.a 5 | *.so 6 | 7 | # Folders 8 | _obj 9 | _test 10 | 11 | # Architecture specific extensions/prefixes 12 | *.[568vq] 13 | [568vq].out 14 | 15 | *.cgo1.go 16 | *.cgo2.c 17 | _cgo_defun.c 18 | _cgo_gotypes.go 19 | _cgo_export.* 20 | 21 | _testmain.go 22 | 23 | *.exe 24 | *.test 25 | *.prof 26 | 27 | # Created by .ignore support plugin (hsz.mobi) 28 | .idea/ 29 | split1Bitcask/ 30 | split2Bitcask/ 31 | testBitcask/ 32 | exampleBitcaskDir/ 33 | example/exampleBitcaskDir/ -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | const ( 4 | defaultExpirySecs = 0 5 | defaultMaxFileSize = 1 << 31 // 2G 6 | defaultTimeoutSecs = 10 7 | defaultValueMaxSize = 1 << 20 // 1m 8 | defaultCheckSumCrc32 = false 9 | ) 10 | 11 | // Options . 12 | // now, just MaxFileSize is used 13 | type Options struct { 14 | ExpirySecs int 15 | MaxFileSize uint64 16 | OpenTimeoutSecs int 17 | ReadWrite bool 18 | MergeSecs int 19 | CheckSumCrc32 bool 20 | ValueMaxSize uint64 21 | } 22 | 23 | // NewOptions ... 24 | func NewOptions(expirySecs int, maxFileSize uint64, openTimeoutSecs, mergeSecs int, readWrite bool) Options { 25 | if expirySecs < 0 { 26 | expirySecs = defaultExpirySecs 27 | } 28 | 29 | if maxFileSize <= 0 { 30 | maxFileSize = defaultMaxFileSize 31 | } 32 | 33 | if openTimeoutSecs < 0 { 34 | openTimeoutSecs = defaultTimeoutSecs 35 | } 36 | 37 | return Options{ 38 | ExpirySecs: expirySecs, 39 | OpenTimeoutSecs: openTimeoutSecs, 40 | MaxFileSize: maxFileSize, 41 | ReadWrite: readWrite, 42 | CheckSumCrc32: defaultCheckSumCrc32, 43 | ValueMaxSize: defaultValueMaxSize, 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /example/bitcask_main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/laohanlinux/bitcask" 7 | "github.com/laohanlinux/go-logger/logger" 8 | ) 9 | 10 | func main() { 11 | os.RemoveAll("exampleBitcaskDir") 12 | bc, err := bitcask.Open("exampleBitcaskDir", nil) 13 | if err != nil { 14 | logger.Fatal(err) 15 | } 16 | defer bc.Close() 17 | k1 := []byte("xiaoMing") 18 | v1 := []byte("毕业于新东方推土机学院") 19 | 20 | k2 := []byte("zhanSan") 21 | v2 := []byte("毕业于新东方厨师学院") 22 | 23 | bc.Put(k1, v1) 24 | bc.Put(k2, v2) 25 | 26 | v1, _ = bc.Get(k1) 27 | v2, _ = bc.Get(k2) 28 | logger.Info(string(k1), string(v1)) 29 | logger.Info(string(k2), string(v2)) 30 | // override 31 | v2 = []byte("毕业于新东方美容美发学院") 32 | bc.Put(k2, v2) 33 | v2, _ = bc.Get(k2) 34 | logger.Info(string(k2), string(v2)) 35 | 36 | bc.Del(k1) 37 | bc.Del(k2) 38 | logger.Info("毕业后的数据库：") 39 | v1, e := bc.Get(k1) 40 | if e != bitcask.ErrNotFound { 41 | logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound) 42 | } else { 43 | logger.Info(string(k1), "已经毕业.") 44 | } 45 | v2, e = bc.Get(k2) 46 | if e != bitcask.ErrNotFound { 47 | logger.Info(string(k1), "shoud be:", bitcask.ErrNotFound) 48 | } else { 49 | logger.Info(string(k2), "已经毕业.") 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /doc/doc.md: -------------------------------------------------------------------------------- 1 | # BitCask Design 2 | 3 | 本项目基于`basho`的`bitcask`论文所设计的`键/值`存储系统。 4 | 5 | ## 磁盘源数据域 6 | 7 | ![源数据域](http://pic.yupoo.com/iammutex/BwqvSyJo/qQIps.jpg) 8 | 9 | 10 | ## HashMap结构 11 | 12 | 存储于内存中。 13 | 14 | ![](http://pic.yupoo.com/iammutex/BwqvSLXE/F43A2.jpg) 15 | 16 | ## Hint File 17 | 18 | 用于重建`HashMap`数据结构以及`HashMap`的持久化。 19 | 20 | ![](http://pic.yupoo.com/iammutex/BwqvTat7/o6LeV.jpg) 21 | 22 | ## 其他参数说明 23 | 24 | 文件的删除标志由ksz 和 valuesz 决定 25 | 26 | 如果ksz 和 valuesz 都为0，则表示该记录的操作是删除的操作。 27 | 28 | - sync.open_timeout 29 | 30 | ``` 31 | none — lets the operating system manage syncing writes (default) 32 | o_sync — uses the O_SYNC flag, which forces syncs on every write 33 | Time interval — Riak will force Bitcask to sync at specified intervals 34 | ``` 35 | 36 | - merge Policy 37 | 38 | ``` 39 | always — No restrictions on when merge operations can occur (default) 40 | never — Merge will never be attempted 41 | window — Merge operations occur during specified hours 42 | ``` 43 | 44 | ``` 45 | windows.start = 3 46 | windows.end = 7 47 | ``` 48 | 49 | - Merge Interval 50 | 51 | ``` 52 | merge_check_interval = 3m 53 | ``` 54 | 55 | - Fold Keys Threshold 56 | 57 | ``` 58 | max_age = 0.5s 59 | max_puts = 1000 60 | ``` 61 | 62 | - Automatic Expiration 63 | 64 | `not impleted` 65 | -------------------------------------------------------------------------------- /entry.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import "fmt" 4 | 5 | // {fileID:value_sz:value_pos:tstamp} 6 | // 4 * 4 + 64 = 80Bit 7 | type entry struct { 8 | fileID uint32 // file id 9 | valueSz uint32 // value size in data block 10 | valueOffset uint64 // value offset in data block 11 | timeStamp uint32 // file access time spot 12 | } 13 | 14 | func (e *entry) toString() string { 15 | return fmt.Sprintf("timeStamp:%d, fileID:%d, valuesz:%d, offset:%d", e.timeStamp, 16 | e.fileID, e.valueSz, e.valueOffset) 17 | } 18 | 19 | // if all attr equal to old entry, return false 20 | func (e *entry) isNewerThan(old *entry) bool { 21 | if old.timeStamp < e.timeStamp { 22 | return true 23 | } else if old.timeStamp > e.timeStamp { 24 | return false 25 | } 26 | 27 | if old.fileID < e.fileID { 28 | return true 29 | } else if old.fileID > e.fileID { 30 | return false 31 | } 32 | 33 | if old.valueOffset < e.valueOffset { 34 | return true 35 | } else if old.valueOffset > e.valueOffset { 36 | return false 37 | } 38 | 39 | return false 40 | } 41 | 42 | // if all attr equal to old entry, return true 43 | func (e *entry) isNewerThan1(old *entry) bool { 44 | if old.timeStamp < e.timeStamp { 45 | return true 46 | } 47 | if old.timeStamp > e.timeStamp { 48 | return false 49 | } 50 | return true 51 | } 52 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bitcask 2 | this is storage backend for riot 3 | 4 | [Design Doc](https://github.com/laohanlinux/bitcask/blob/master/doc/doc.md) 5 | 6 | [riot](https://github.com/laohanlinux/riot) 7 | 8 | 9 | 10 | # Example 11 | 12 | ``` 13 | package main 14 | 15 | import ( 16 | "github.com/laohanlinux/bitcask" 17 | "github.com/laohanlinux/go-logger/logger" 18 | ) 19 | 20 | func main() { 21 | bc, err := bitcask.Open("exampleBitcaskDir", nil) 22 | if err != nil { 23 | logger.Fatal(err) 24 | } 25 | defer bc.Close() 26 | k1 := []byte("xiaoMing") 27 | v1 := []byte("毕业于新东方推土机学院") 28 | 29 | k2 := []byte("zhanSan") 30 | v2 := []byte("毕业于新东方厨师学院") 31 | 32 | bc.Put(k1, v1) 33 | bc.Put(k2, v2) 34 | 35 | v1, _ = bc.Get(k1) 36 | v2, _ = bc.Get(k2) 37 | logger.Info(string(k1), string(v1)) 38 | logger.Info(string(k2), string(v2)) 39 | // time.Sleep(time.Second * 10) 40 | // override 41 | v2 = []byte("毕业于新东方美容美发学院") 42 | bc.Put(k2, v2) 43 | v2, _ = bc.Get(k2) 44 | logger.Info(string(k2), string(v2)) 45 | 46 | } 47 | 48 | ``` 49 | 50 | `go run example/bitcask_main.go` 51 | 52 | ``` 53 | 2016/01/16 16:56:11 bitcask_main.go:25 [info [xiaoMing 毕业于新东方推土机学院]] 54 | 2016/01/16 16:56:11 bitcask_main.go:26 [info [zhanSan 毕业于新东方厨师学院]] 55 | 2016/01/16 16:56:11 bitcask_main.go:32 [info [zhanSan 毕业于新东方美容美发学院]] 56 | 2016/01/16 16:56:11 bitcask_main.go:36 [info [毕业后的数据库：]] 57 | 2016/01/16 16:56:11 bitcask_main.go:41 [info [xiaoMing 已经毕业.]] 58 | 2016/01/16 16:56:11 bitcask_main.go:47 [info [zhanSan 已经毕业.]] 59 | ``` 60 | 61 | other Example: find it in `xxxx_test.go` 62 | 63 | # TODO 64 | 65 | - 优化数据结构，减少内存占用 66 | - 增加merge功能 67 | -------------------------------------------------------------------------------- /keydir.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import "sync" 4 | 5 | // KeyDirs for HashMap 6 | var keyDirsLock *sync.RWMutex 7 | 8 | var keyDirs *KeyDirs 9 | var keyDirsOnce sync.Once 10 | 11 | func init() { 12 | keyDirsLock = &sync.RWMutex{} 13 | } 14 | 15 | // KeyDirs ... 16 | type KeyDirs struct { 17 | entrys map[string]*entry 18 | } 19 | 20 | // NewKeyDir return a KeyDir Obj 21 | func NewKeyDir(dirName string) *KeyDirs { 22 | //filepath.Abs(fp.Name()) 23 | keyDirsLock.Lock() 24 | defer keyDirsLock.Unlock() 25 | 26 | keyDirsOnce.Do(func() { 27 | if keyDirs == nil { 28 | keyDirs = &KeyDirs{ 29 | entrys: make(map[string]*entry), 30 | } 31 | } 32 | }) 33 | return keyDirs 34 | } 35 | func (keyDirs *KeyDirs) get(key string) *entry { 36 | keyDirsLock.RLock() 37 | defer keyDirsLock.RUnlock() 38 | e, _ := keyDirs.entrys[key] 39 | return e 40 | } 41 | 42 | func (keyDirs *KeyDirs) del(key string) { 43 | keyDirsLock.Lock() 44 | defer keyDirsLock.Unlock() 45 | delete(keyDirs.entrys, key) 46 | } 47 | 48 | // put a key with value into bitcask 49 | func (keyDirs *KeyDirs) put(key string, e *entry) { 50 | keyDirsLock.Lock() 51 | defer keyDirsLock.Unlock() 52 | keyDirs.entrys[key] = e 53 | } 54 | 55 | func (keyDirs *KeyDirs) setCompare(key string, e *entry) bool { 56 | keyDirsLock.Lock() 57 | defer keyDirsLock.Unlock() 58 | old, ok := keyDirs.entrys[key] 59 | //println("old:", old.toString()) 60 | //println("new:", e.toString()) 61 | if !ok || e.isNewerThan1(old) { 62 | // logger.Info("update data:", key, e.fileID, e.timeStamp) 63 | keyDirs.entrys[key] = e 64 | return true 65 | } 66 | // logger.Error("update fail:", key, e.fileID, e.timeStamp) 67 | return false 68 | } 69 | 70 | func (keyDirs *KeyDirs) updateFileID(oldID, newID uint32) { 71 | keyDirsLock.Lock() 72 | defer keyDirsLock.Unlock() 73 | for _, e := range keyDirs.entrys { 74 | if e.fileID == oldID { 75 | e.fileID = newID 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /http/bitcaskhttp.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "io/ioutil" 6 | "net/http" 7 | "runtime/debug" 8 | 9 | "github.com/laohanlinux/bitcask" 10 | "github.com/laohanlinux/go-logger/logger" 11 | "github.com/laohanlinux/mux" 12 | ) 13 | 14 | var addr string 15 | var storagePath string 16 | var maxSize uint64 17 | var logLevel int 18 | 19 | var bc *bitcask.BitCask 20 | 21 | func main() { 22 | flag.StringVar(&addr, "addr", "127.0.0.1:80", "bitcask http listen addr") 23 | flag.StringVar(&storagePath, "s", "bitcaskStorage", "data storage path") 24 | flag.Uint64Var(&maxSize, "ms", 1<<32, "single data file maxsize") 25 | flag.IntVar(&logLevel, "l", 0, "logger level") 26 | flag.Parse() 27 | 28 | logger.SetLevel(1) 29 | opts := &bitcask.Options{ 30 | MaxFileSize: maxSize, 31 | } 32 | var err error 33 | bc, err = bitcask.Open(storagePath, opts) 34 | if err != nil { 35 | logger.Fatal(err) 36 | } 37 | defer bc.Close() 38 | 39 | defer func() { 40 | if err := recover(); err != nil { 41 | logger.Error(err) 42 | debug.PrintStack() 43 | } 44 | }() 45 | 46 | r := mux.NewRouter() 47 | r.HandleFunc("/{key}", bitcaskGetHandle).Methods("GET") 48 | r.HandleFunc("/{key}", bitcaskDelHandle).Methods("DELETE") 49 | r.HandleFunc("/{key}", bitcaskPutHandle).Methods("POST") 50 | logger.Info("bitcask server listen:", addr) 51 | if err := http.ListenAndServe(addr, r); err != nil { 52 | logger.Error(err) 53 | } 54 | } 55 | 56 | func bitcaskGetHandle(w http.ResponseWriter, r *http.Request) { 57 | vars := mux.Vars(r) 58 | key := vars["key"] 59 | if len(key) <= 0 { 60 | w.Write([]byte("key invalid")) 61 | return 62 | } 63 | 64 | value, err := bc.Get([]byte(key)) 65 | if err != nil && err != bitcask.ErrNotFound { 66 | logger.Error(err) 67 | w.WriteHeader(500) 68 | return 69 | } 70 | 71 | if err == bitcask.ErrNotFound { 72 | w.WriteHeader(404) 73 | w.Write([]byte(bitcask.ErrNotFound.Error())) 74 | return 75 | } 76 | 77 | w.Write(value) 78 | } 79 | 80 | func bitcaskPutHandle(w http.ResponseWriter, r *http.Request) { 81 | vars := mux.Vars(r) 82 | key := vars["key"] 83 | if len(key) <= 0 { 84 | w.Write([]byte("key invalid")) 85 | return 86 | } 87 | value, err := ioutil.ReadAll(r.Body) 88 | if err != nil { 89 | logger.Error(err) 90 | w.WriteHeader(500) 91 | return 92 | } 93 | 94 | bc.Put([]byte(key), value) 95 | w.Write([]byte("Success")) 96 | } 97 | 98 | func bitcaskDelHandle(w http.ResponseWriter, r *http.Request) { 99 | vars := mux.Vars(r) 100 | key := vars["key"] 101 | if len(key) <= 0 { 102 | w.Write([]byte("key invalid")) 103 | return 104 | } 105 | 106 | err := bc.Del([]byte(key)) 107 | if err != nil && err != bitcask.ErrNotFound { 108 | logger.Error(err) 109 | w.WriteHeader(500) 110 | return 111 | } 112 | if err == bitcask.ErrNotFound { 113 | w.WriteHeader(404) 114 | w.Write([]byte(bitcask.ErrNotFound.Error())) 115 | return 116 | } 117 | 118 | w.Write([]byte("Success")) 119 | } 120 | -------------------------------------------------------------------------------- /example/decode_main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "time" 8 | 9 | "github.com/laohanlinux/bitcask" 10 | "github.com/laohanlinux/go-logger/logger" 11 | ) 12 | 13 | func main() { 14 | d1() 15 | } 16 | 17 | func h1() { 18 | buf := make([]byte, bitcask.HintHeaderSize) 19 | fmt.Println("hfp:", os.Args[1], "dfp:", os.Args[2]) 20 | fp, err := os.Open(os.Args[1]) 21 | dfp, _ := os.Open(os.Args[2]) 22 | if err != nil { 23 | logger.Fatal(err) 24 | } 25 | 26 | for { 27 | n, err := fp.Read(buf) 28 | if err != nil && err != io.EOF { 29 | logger.Fatal(err) 30 | } 31 | 32 | if err == io.EOF { 33 | break 34 | } 35 | 36 | if n != len(buf) || n != bitcask.HintHeaderSize { 37 | logger.Fatal(n) 38 | } 39 | 40 | htStamp, hksz, hvaluesz, hvaluePos := bitcask.DecodeHint(buf) 41 | logger.Debug("hintSize:", hksz) 42 | time.Sleep(time.Second * 3) 43 | key := make([]byte, hksz) 44 | fp.Read(key) 45 | fmt.Println("Hint:", "key:", string(key), htStamp, "ksz:", hksz, "valuesize:", hvaluesz, "pos:", hvaluePos) 46 | if err != nil { 47 | logger.Fatal(err) 48 | } 49 | // read 50 | dbuf := make([]byte, bitcask.HeaderSize+hksz+hvaluesz) 51 | dfp.ReadAt(dbuf, int64(hvaluePos)) 52 | dvalue, err := bitcask.DecodeEntry(dbuf) 53 | if err != nil { 54 | logger.Fatal(err) 55 | } 56 | fmt.Println("dvalue:", string(dvalue)) 57 | os.Exit(0) 58 | } 59 | } 60 | 61 | func d1() { 62 | buf := make([]byte, bitcask.HeaderSize) 63 | fp, err := os.Open(os.Args[1]) 64 | if err != nil { 65 | logger.Fatal(err) 66 | } 67 | 68 | offset := int64(0) 69 | for { 70 | n, err := fp.ReadAt(buf, offset) 71 | if err != nil && err != io.EOF { 72 | logger.Fatal(err) 73 | } 74 | if err == io.EOF { 75 | break 76 | } 77 | if n != len(buf) || n != bitcask.HeaderSize { 78 | logger.Fatal(n) 79 | } 80 | offset += int64(n) 81 | // parse data header 82 | c32, tStamp, ksz, valuesz := bitcask.DecodeEntryHeader(buf) 83 | logger.Info(c32, tStamp, "ksz:", ksz, "valuesz:", valuesz) 84 | if err != nil { 85 | logger.Fatal(err) 86 | } 87 | 88 | if ksz+valuesz == 0 { 89 | continue 90 | } 91 | 92 | keyValue := make([]byte, ksz+valuesz) 93 | n, err = fp.ReadAt(keyValue, offset) 94 | if err != nil && err != io.EOF { 95 | logger.Fatal(err) 96 | } 97 | if err == io.EOF { 98 | break 99 | } 100 | offset += int64(n) 101 | fmt.Println(string(keyValue[:ksz]), string(keyValue[ksz:])) 102 | } 103 | } 104 | 105 | func d2() { 106 | buf := make([]byte, bitcask.HeaderSize) 107 | fp, err := os.Open(os.Args[1]) 108 | if err != nil { 109 | logger.Fatal(err) 110 | } 111 | 112 | for { 113 | n, err := fp.Read(buf[0:]) 114 | if err != nil && err != io.EOF { 115 | logger.Fatal(err) 116 | } 117 | if n != len(buf) { 118 | logger.Fatal(n) 119 | } 120 | value, err := bitcask.DecodeEntry(buf) 121 | logger.Info(value) 122 | if err != nil { 123 | logger.Fatal(err) 124 | } 125 | //logger.Info(c32, tStamp, ksz, valuesz, key, value) 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /bitcask_test.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "testing" 7 | "time" 8 | 9 | "github.com/laohanlinux/assert" 10 | "github.com/laohanlinux/go-logger/logger" 11 | ) 12 | 13 | // rebuild file test 14 | func TestSplit1(t *testing.T) { 15 | storagePath := "split1Bitcask" 16 | os.RemoveAll(storagePath) 17 | bc, err := Open(storagePath, nil) 18 | assert.Nil(t, err) 19 | testKey := []byte("Foo") 20 | 21 | value := []byte("Bar") 22 | bc.Put(testKey, value) 23 | v, err := bc.Get(testKey) 24 | assert.Nil(t, err) 25 | assert.Equal(t, v, value) 26 | 27 | bc.Close() 28 | 29 | storagePath = "split1Bitcask" 30 | bc, err = Open(storagePath, nil) 31 | assert.Nil(t, err) 32 | testKey = []byte("Foo") 33 | 34 | value = []byte("Bar") 35 | bc.Put(testKey, value) 36 | v, err = bc.Get(testKey) 37 | assert.Nil(t, err) 38 | assert.Equal(t, v, value) 39 | bc.Close() 40 | } 41 | 42 | func TestSplit2(t *testing.T) { 43 | storagePath := "split2Bitcask" 44 | os.RemoveAll(storagePath) 45 | opts := &Options{ 46 | MaxFileSize: 2, 47 | } 48 | bc, err := Open(storagePath, opts) 49 | assert.Nil(t, err) 50 | testKey := []byte("Foo") 51 | 52 | value := []byte("Bar") 53 | bc.Put(testKey, value) 54 | v, err := bc.Get(testKey) 55 | assert.Nil(t, err) 56 | assert.Equal(t, v, value) 57 | logger.Info("==============================") 58 | time.Sleep(time.Second * 2) 59 | 60 | // cause split file 61 | value = []byte("Apple") 62 | bc.Put(testKey, value) 63 | v, err = bc.Get(testKey) 64 | assert.Nil(t, err) 65 | assert.Equal(t, v, value) 66 | bc.Close() 67 | } 68 | 69 | func TestBitCask(t *testing.T) { 70 | // clear dirty 71 | os.RemoveAll("testBitcask") 72 | b, err := Open("testBitcask", nil) 73 | logger.Info(err) 74 | assert.Nil(t, err) 75 | assert.NotNil(t, b) 76 | 77 | testKey := []byte("Foo") 78 | value := []byte("Bar") 79 | b.Put(testKey, value) 80 | v, err := b.Get(testKey) 81 | assert.Nil(t, err) 82 | logger.Info("value:", string(v)) 83 | assert.Equal(t, v, value) 84 | 85 | testKey = []byte("xiaoMing") 86 | value = []byte("abc") 87 | b.Put(testKey, value) 88 | v, err = b.Get(testKey) 89 | logger.Info("value:", string(v)) 90 | assert.Equal(t, v, value) 91 | 92 | // hintFile: 93 | value = []byte("ddddd") 94 | b.Put(testKey, value) 95 | v, err = b.Get(testKey) 96 | logger.Info("value:", string(v)) 97 | assert.Equal(t, v, value) 98 | 99 | b.Close() 100 | } 101 | 102 | func BenchmarkBitcaskCurrency(b *testing.B) { 103 | storagePath := "benchMarkBitcask" 104 | os.RemoveAll(storagePath) 105 | opts := &Options{ 106 | MaxFileSize: 1 << 12, 107 | } 108 | bc, err := Open(storagePath, opts) 109 | if err != nil { 110 | logger.Fatal(err) 111 | } 112 | 113 | keyValues := make(map[int]string) 114 | 115 | for i := 0; i < b.N/2; i++ { 116 | key := strconv.Itoa(i) 117 | value := strconv.Itoa(int(time.Now().Unix())) 118 | bc.Put([]byte(key), []byte(value)) 119 | keyValues[i] = value 120 | } 121 | logger.Warn(b.N) 122 | logger.Info("Put all Data") 123 | for i := 0; i < b.N/2; i++ { 124 | k := strconv.Itoa(i) 125 | v, _ := bc.Get([]byte(k)) 126 | if string(v) != keyValues[i] { 127 | logger.Error(string(v), keyValues[i]) 128 | os.Exit(-1) 129 | } 130 | } 131 | logger.Info("Get all data") 132 | // delete all data 133 | for i := 0; i < b.N/2; i++ { 134 | k := strconv.Itoa(i) 135 | //v, _ := bc.Get([]byte(k)) 136 | err := bc.Del([]byte(k)) 137 | if err != nil { 138 | logger.Error(err) 139 | } 140 | } 141 | logger.Info("Delete all data") 142 | // Get all data 143 | for i := 0; i < b.N/2; i++ { 144 | k := strconv.Itoa(i) 145 | v, err := bc.Get([]byte(k)) 146 | if err != ErrNotFound { 147 | logger.Error(string(v), keyValues[i]) 148 | } 149 | } 150 | logger.Info("all data is not found, pass test") 151 | //mergeWorker.Staop() 152 | bc.Close() 153 | } 154 | -------------------------------------------------------------------------------- /encoding_test.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "hash/crc32" 7 | "testing" 8 | "time" 9 | 10 | "github.com/laohanlinux/assert" 11 | ) 12 | 13 | func TestEncodeDecodeEntry(t *testing.T) { 14 | /** 15 | crc32 : tStamp : ksz : valueSz : key : value 16 | 4 : 4 : 4 : 4 : xxxx : xxxx 17 | **/ 18 | //func encodeEntry(tStamp, keySize, valueSize uint32, key, value []byte) []byte { 19 | // EncodeEntry 20 | tStamp := uint32(time.Now().Unix()) 21 | key := []byte("Foo") 22 | value := []byte("Bar") 23 | ksz := uint32(len(key)) 24 | valuesz := uint32(len(value)) 25 | buf := make([]byte, HeaderSize+ksz+valuesz) 26 | binary.LittleEndian.PutUint32(buf[4:8], tStamp) 27 | binary.LittleEndian.PutUint32(buf[8:12], ksz) 28 | binary.LittleEndian.PutUint32(buf[12:16], valuesz) 29 | copy(buf[16:(16+ksz)], key) 30 | copy(buf[(16+ksz):(16+ksz+valuesz)], value) 31 | c32 := crc32.ChecksumIEEE(buf[4:]) 32 | binary.LittleEndian.PutUint32(buf[0:4], uint32(c32)) 33 | // Test decode 34 | 35 | ksz = binary.LittleEndian.Uint32(buf[8:12]) 36 | valuesz = binary.LittleEndian.Uint32(buf[12:16]) 37 | tStamp = binary.LittleEndian.Uint32(buf[4:8]) 38 | c32 = binary.LittleEndian.Uint32(buf[:4]) 39 | assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32) 40 | assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp) 41 | assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz) 42 | assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz) 43 | assert.Equal(t, buf[HeaderSize:(HeaderSize+ksz)], key) 44 | assert.Equal(t, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)], value) 45 | 46 | // EncodeEntry , ksz = 0, valueSz = 0 47 | ksz = uint32(0) 48 | valuesz = uint32(0) 49 | buf = make([]byte, HeaderSize+ksz+valuesz, HeaderSize+ksz+valuesz) 50 | binary.LittleEndian.PutUint32(buf[4:8], tStamp) 51 | binary.LittleEndian.PutUint32(buf[8:12], ksz) 52 | binary.LittleEndian.PutUint32(buf[12:16], valuesz) 53 | c32 = crc32.ChecksumIEEE(buf[4:]) 54 | binary.LittleEndian.PutUint32(buf[0:4], c32) 55 | // decodeEntry, ksz =0, valueSz = 0 56 | assert.Equal(t, binary.LittleEndian.Uint32(buf[0:4]), c32) 57 | assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), tStamp) 58 | assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), ksz) 59 | assert.Equal(t, binary.LittleEndian.Uint32(buf[12:16]), valuesz) 60 | } 61 | 62 | func TestEncodeDecodeHint(t *testing.T) { 63 | /** 64 | tStamp : ksz : valueSz : valuePos : key 65 | 4 : 4 : 4 : 8 : xxxxx 66 | **/ 67 | // encodeHint 68 | tStamp := uint32(time.Now().Unix()) 69 | key := []byte("Foo") 70 | value := []byte("Bar") 71 | ksz := uint32(len(key)) 72 | valuesz := uint32(len(value)) 73 | valuePos := uint64(8) 74 | buf := make([]byte, HintHeaderSize+ksz, HintHeaderSize+ksz) 75 | binary.LittleEndian.PutUint32(buf[0:4], tStamp) 76 | binary.LittleEndian.PutUint32(buf[4:8], ksz) 77 | binary.LittleEndian.PutUint32(buf[8:12], valuesz) 78 | binary.LittleEndian.PutUint64(buf[12:20], valuePos) 79 | copy(buf[HintHeaderSize:], key) 80 | // decodeHint 81 | assert.Equal(t, binary.LittleEndian.Uint32(buf[:4]), tStamp) 82 | assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), ksz) 83 | assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), valuesz) 84 | assert.Equal(t, binary.LittleEndian.Uint64(buf[12:20]), valuePos) 85 | fmt.Println(string(buf[HintHeaderSize : HintHeaderSize+ksz])) 86 | assert.Equal(t, buf[HintHeaderSize:], key) 87 | 88 | ksz = 0 89 | valuesz = 0 90 | valuePos = 0 91 | buf = make([]byte, HintHeaderSize+ksz, HintHeaderSize+ksz) 92 | binary.LittleEndian.PutUint32(buf[0:4], tStamp) 93 | binary.LittleEndian.PutUint32(buf[4:8], ksz) 94 | binary.LittleEndian.PutUint32(buf[8:12], valuesz) 95 | binary.LittleEndian.PutUint64(buf[12:20], valuePos) 96 | assert.Equal(t, binary.LittleEndian.Uint32(buf[:4]), tStamp) 97 | assert.Equal(t, binary.LittleEndian.Uint32(buf[4:8]), ksz) 98 | assert.Equal(t, binary.LittleEndian.Uint32(buf[8:12]), valuesz) 99 | assert.Equal(t, binary.LittleEndian.Uint64(buf[12:20]), valuePos) 100 | } 101 | -------------------------------------------------------------------------------- /encode.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "hash/crc32" 7 | 8 | "github.com/laohanlinux/go-logger/logger" 9 | ) 10 | 11 | // ErrCrc32 ... 12 | var ErrCrc32 = fmt.Errorf("checksumIEEE error") 13 | 14 | func encodeEntry(tStamp, keySize, valueSize uint32, key, value []byte) []byte { 15 | /** 16 | crc32 : tStamp : ksz : valueSz : key : value 17 | 4 : 4 : 4 : 4 : xxxx : xxxx 18 | **/ 19 | bufSize := HeaderSize + keySize + valueSize 20 | buf := make([]byte, bufSize) 21 | binary.LittleEndian.PutUint32(buf[4:8], tStamp) 22 | binary.LittleEndian.PutUint32(buf[8:12], keySize) 23 | binary.LittleEndian.PutUint32(buf[12:16], valueSize) 24 | copy(buf[HeaderSize:(HeaderSize+keySize)], key) 25 | copy(buf[(HeaderSize+keySize):(HeaderSize+keySize+valueSize)], value) 26 | 27 | c32 := crc32.ChecksumIEEE(buf[4:]) 28 | binary.LittleEndian.PutUint32(buf[0:4], uint32(c32)) 29 | return buf 30 | } 31 | 32 | // DecodeEntry ... 33 | func DecodeEntry(buf []byte) ([]byte, error) { 34 | /** 35 | crc32 : tStamp : ksz : valueSz : key : value 36 | 4 : 4 : 4 : 4 : xxxx : xxxx 37 | **/ 38 | ksz := binary.LittleEndian.Uint32(buf[8:12]) 39 | 40 | valuesz := binary.LittleEndian.Uint32(buf[12:HeaderSize]) 41 | c32 := binary.LittleEndian.Uint32(buf[:4]) 42 | value := make([]byte, valuesz) 43 | copy(value, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)]) 44 | logger.Info(c32) 45 | if crc32.ChecksumIEEE(buf[4:]) != c32 { 46 | return nil, ErrCrc32 47 | } 48 | return value, nil 49 | } 50 | 51 | // DecodeEntryHeader ... 52 | func DecodeEntryHeader(buf []byte) (uint32, uint32, uint32, uint32) { 53 | /** 54 | crc32 : tStamp : ksz : valueSz : key : value 55 | 4 : 4 : 4 : 4 : xxxx : xxxx 56 | **/ 57 | c32 := binary.LittleEndian.Uint32(buf[:4]) 58 | tStamp := binary.LittleEndian.Uint32(buf[4:8]) 59 | ksz := binary.LittleEndian.Uint32(buf[8:12]) 60 | valuesz := binary.LittleEndian.Uint32(buf[12:HeaderSize]) 61 | return c32, tStamp, ksz, valuesz 62 | } 63 | 64 | 65 | // DecodeEntryDetail ... 66 | func DecodeEntryDetail(buf []byte) (uint32, uint32, uint32, uint32, []byte, []byte, error) { 67 | /** 68 | crc32 : tStamp : ksz : valueSz : key : value 69 | 4 : 4 : 4 : 4 : xxxx : xxxx 70 | **/ 71 | tStamp := binary.LittleEndian.Uint32(buf[4:8]) 72 | ksz := binary.LittleEndian.Uint32(buf[8:12]) 73 | valuesz := binary.LittleEndian.Uint32(buf[12:HeaderSize]) 74 | c32 := binary.LittleEndian.Uint32(buf[:4]) 75 | if crc32.ChecksumIEEE(buf[4:]) != c32 { 76 | //return 0, 0, 0, 0, nil, nil, ErrCrc32 77 | return c32, tStamp, ksz, valuesz, nil, nil, ErrCrc32 78 | } 79 | 80 | if ksz+valuesz == 0 { 81 | return c32, tStamp, ksz, valuesz, nil, nil, nil 82 | } 83 | 84 | key := make([]byte, ksz) 85 | value := make([]byte, valuesz) 86 | copy(key, buf[HeaderSize:HeaderSize+ksz]) 87 | copy(value, buf[(HeaderSize+ksz):(HeaderSize+ksz+valuesz)]) 88 | return c32, tStamp, ksz, valuesz, key, value, nil 89 | } 90 | 91 | func encodeHint(tStamp, ksz, valueSz uint32, valuePos uint64, key []byte) []byte { 92 | /** 93 | tStamp : ksz : valueSz : valuePos : key 94 | 4 : 4 : 4 : 8 : xxxxx 95 | **/ 96 | buf := make([]byte, HintHeaderSize+len(key), HintHeaderSize+len(key)) 97 | binary.LittleEndian.PutUint32(buf[0:4], tStamp) 98 | binary.LittleEndian.PutUint32(buf[4:8], ksz) 99 | binary.LittleEndian.PutUint32(buf[8:12], valueSz) 100 | binary.LittleEndian.PutUint64(buf[12:HintHeaderSize], valuePos) 101 | copy(buf[HintHeaderSize:], []byte(key)) 102 | return buf 103 | } 104 | 105 | // DecodeHint ... 106 | func DecodeHint(buf []byte) (uint32, uint32, uint32, uint64) { 107 | /** 108 | tStamp : ksz : valueSz : valuePos : key 109 | 4 : 4 : 4 : 8 : xxxxx 110 | **/ 111 | tStamp := binary.LittleEndian.Uint32(buf[:4]) 112 | ksz := binary.LittleEndian.Uint32(buf[4:8]) 113 | valueSz := binary.LittleEndian.Uint32(buf[8:12]) 114 | valueOffset := binary.LittleEndian.Uint64(buf[12:HintHeaderSize]) 115 | return tStamp, ksz, valueSz, valueOffset 116 | } 117 | -------------------------------------------------------------------------------- /file.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "os" 5 | "strconv" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | const ( 11 | // HeaderSize : 4 + 4 + 4 + 4 12 | /** 13 | crc32 : tStamp : ksz : valueSz : key : value 14 | 4 : 4 : 4 : 4 : xxxx : xxxx 15 | **/ 16 | HeaderSize = 16 17 | // HintHeaderSize : 4 + 4 + 4 + 8 = 20 byte 18 | /** 19 | tstamp : ksz : valuesz ： valuePos : key 20 | 4 : 4 : 4 : 8 : xxxx 21 | */ 22 | HintHeaderSize = 20 23 | ) 24 | 25 | // BFiles ... 26 | type BFiles struct { 27 | bfs map[uint32]*BFile 28 | rwLock *sync.RWMutex 29 | } 30 | 31 | func newBFiles() *BFiles { 32 | return &BFiles{ 33 | bfs: make(map[uint32]*BFile), 34 | rwLock: &sync.RWMutex{}, 35 | } 36 | } 37 | 38 | func (bfs *BFiles) get(fileID uint32) *BFile { 39 | bfs.rwLock.RLock() 40 | defer bfs.rwLock.RUnlock() 41 | bf, _ := bfs.bfs[fileID] 42 | return bf 43 | } 44 | 45 | func (bfs *BFiles) put(bf *BFile, fileID uint32) { 46 | bfs.rwLock.Lock() 47 | defer bfs.rwLock.Unlock() 48 | bfs.bfs[fileID] = bf 49 | } 50 | 51 | func (bfs *BFiles) close() { 52 | bfs.rwLock.Lock() 53 | defer bfs.rwLock.Unlock() 54 | for _, bf := range bfs.bfs { 55 | bf.fp.Close() 56 | bf.hintFp.Close() 57 | } 58 | } 59 | 60 | // BFile 可写文件信息 1: datafile and hint file 61 | type BFile struct { 62 | // fp is the writeable file 63 | fp *os.File 64 | fileID uint32 65 | writeOffset uint64 66 | // hintFp is the hint file 67 | hintFp *os.File 68 | } 69 | 70 | // 71 | func newBFile() *BFile { 72 | return &BFile{} 73 | } 74 | 75 | func openBFile(dirName string, tStamp int) (*BFile, error) { 76 | fp, err := os.OpenFile(dirName+"/"+strconv.Itoa(tStamp)+".data", os.O_RDONLY, os.ModePerm) 77 | if err != nil { 78 | return nil, err 79 | } 80 | 81 | return &BFile{ 82 | fileID: uint32(tStamp), 83 | fp: fp, 84 | hintFp: nil, 85 | writeOffset: 0, 86 | }, nil 87 | } 88 | 89 | func (bf *BFile) read(offset uint64, length uint32) ([]byte, error) { 90 | /** 91 | crc32 : tStamp : ksz : valueSz : key : value 92 | 4 : 4 : 4: 4 : xxxx : xxxx 93 | **/ 94 | value := make([]byte, length) 95 | //TODO 96 | // assert read function and crc32 97 | bf.fp.Seek(int64(offset), 0) 98 | _, err := bf.fp.Read(value) 99 | if err != nil { 100 | return nil, err 101 | } 102 | return value, err 103 | } 104 | 105 | // including writing data file and hint file 106 | func (bf *BFile) writeDatat(key []byte, value []byte) (entry, error) { 107 | // 1. write into datafile 108 | timeStamp := uint32(time.Now().Unix()) 109 | keySize := uint32(len(key)) 110 | valueSize := uint32(len(value)) 111 | vec := encodeEntry(timeStamp, keySize, valueSize, key, value) 112 | entrySize := HeaderSize + keySize + valueSize 113 | 114 | valueOffset := bf.writeOffset + uint64(HeaderSize+keySize) 115 | // write data file into disk 116 | // TODO 117 | // assert WriteAt function 118 | _, err := appendWriteFile(bf.fp, vec) 119 | if err != nil { 120 | panic(err) 121 | } 122 | //logger.Debug("has write into data file:", n) 123 | 124 | // 2. write hint file disk 125 | hintData := encodeHint(timeStamp, keySize, valueSize, valueOffset, key) 126 | // TODO 127 | // assert write function 128 | _, err = appendWriteFile(bf.hintFp, hintData) 129 | if err != nil { 130 | panic(err) 131 | } 132 | //logger.Debug("has write into hint file:", n) 133 | bf.writeOffset += uint64(entrySize) 134 | 135 | return entry{ 136 | fileID: bf.fileID, 137 | valueSz: valueSize, 138 | valueOffset: valueOffset, 139 | timeStamp: timeStamp, 140 | }, nil 141 | } 142 | 143 | func (bf *BFile) del(key []byte) error { 144 | // 1. write into datafile 145 | timeStamp := uint32(time.Now().Unix()) 146 | keySize := uint32(0) 147 | valueSize := uint32(0) 148 | vec := encodeEntry(timeStamp, keySize, valueSize, key, nil) 149 | //logger.Info(len(vec), keySize, valueSize) 150 | entrySize := HeaderSize + keySize + valueSize 151 | // TODO 152 | // race data 153 | valueOffset := bf.writeOffset + uint64(HeaderSize+keySize) 154 | // write data file into disk 155 | // TODO 156 | // assert WriteAt function 157 | _, err := appendWriteFile(bf.fp, vec) 158 | if err != nil { 159 | panic(err) 160 | } 161 | 162 | //logger.Debug("has write into data file:", n) 163 | // 2. write hint file disk 164 | hintData := encodeHint(timeStamp, keySize, valueSize, valueOffset, key) 165 | 166 | // TODO 167 | // assert write function 168 | _, err = appendWriteFile(bf.hintFp, hintData) 169 | if err != nil { 170 | panic(err) 171 | } 172 | //logger.Debug("has write into hint file:", n) 173 | bf.writeOffset += uint64(entrySize) 174 | 175 | return nil 176 | } 177 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "os" 5 | "sort" 6 | "strconv" 7 | "strings" 8 | "time" 9 | 10 | "github.com/laohanlinux/go-logger/logger" 11 | ) 12 | 13 | const ( 14 | lockFileName = "bitcask.lock" 15 | ) 16 | 17 | // if writeableFile size large than Opts.MaxFileSize and the fileID not equal to local time stamp; 18 | // if will create a new writeable file 19 | func checkWriteableFile(bc *BitCask) { 20 | if bc.writeFile.writeOffset > bc.Opts.MaxFileSize && bc.writeFile.fileID != uint32(time.Now().Unix()) { 21 | logger.Info("open a new data/hint file:", bc.writeFile.writeOffset, bc.Opts.MaxFileSize) 22 | //close data/hint fp 23 | bc.writeFile.hintFp.Close() 24 | bc.writeFile.fp.Close() 25 | 26 | writeFp, fileID := setWriteableFile(0, bc.dirFile) 27 | hintFp := setHintFile(fileID, bc.dirFile) 28 | bf := &BFile{ 29 | fp: writeFp, 30 | fileID: fileID, 31 | writeOffset: 0, 32 | hintFp: hintFp, 33 | } 34 | bc.writeFile = bf 35 | // update pid 36 | writePID(bc.lockFile, fileID) 37 | } 38 | } 39 | 40 | // return the hint file lists 41 | func listHintFiles(bc *BitCask) ([]string, error) { 42 | filterFiles := []string{lockFileName} 43 | dirFp, err := os.OpenFile(bc.dirFile, os.O_RDONLY, os.ModeDir) 44 | if err != nil { 45 | return nil, err 46 | } 47 | defer dirFp.Close() 48 | // 49 | lists, err := dirFp.Readdirnames(-1) 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | var hintLists []string 55 | for _, v := range lists { 56 | if strings.Contains(v, "hint") && !existsSuffixs(filterFiles, v) { 57 | hintLists = append(hintLists, v) 58 | } 59 | } 60 | return hintLists, nil 61 | } 62 | 63 | // return the data file lists 64 | func listDataFiles(bc *BitCask) ([]string, error) { 65 | filterFiles := []string{lockFileName} 66 | dirFp, err := os.OpenFile(bc.dirFile, os.O_RDONLY, os.ModeDir) 67 | if err != nil { 68 | return nil, err 69 | } 70 | defer dirFp.Close() 71 | // 72 | lists, err := dirFp.Readdirnames(-1) 73 | if err != nil { 74 | return nil, err 75 | } 76 | 77 | var dataFileLists []string 78 | for _, v := range lists { 79 | if strings.Contains(v, ".data") && !existsSuffixs(filterFiles, v) { 80 | dataFileLists = append(dataFileLists, v) 81 | } 82 | } 83 | sort.Strings(dataFileLists) 84 | return dataFileLists, nil 85 | } 86 | 87 | // lock a file by fp locker; the file must exits 88 | func lockFile(fileName string) (*os.File, error) { 89 | return os.OpenFile(fileName, os.O_EXCL|os.O_CREATE|os.O_RDWR, os.ModePerm) 90 | } 91 | 92 | func existsSuffixs(suffixs []string, src string) (b bool) { 93 | for _, suffix := range suffixs { 94 | if b = strings.HasSuffix(src, suffix); b { 95 | return 96 | } 97 | } 98 | return 99 | } 100 | 101 | func writePID(pidFp *os.File, fileID uint32) { 102 | pidFp.WriteAt([]byte(strconv.Itoa(os.Getpid())+"\t"+strconv.Itoa(int(fileID))+".data"), 0) 103 | } 104 | 105 | // get file last hint file info 106 | func lastFileInfo(files []*os.File) (uint32, *os.File) { 107 | if files == nil { 108 | return uint32(0), nil 109 | } 110 | lastFp := files[0] 111 | 112 | fileName := lastFp.Name() 113 | s := strings.LastIndex(fileName, "/") + 1 114 | e := strings.LastIndex(fileName, ".hint") 115 | idx, _ := strconv.Atoi(fileName[s:e]) 116 | lastID := idx 117 | for i := 0; i < len(files); i++ { 118 | idxFp := files[i] 119 | fileName = idxFp.Name() 120 | s = strings.LastIndex(fileName, "/") + 1 121 | e = strings.LastIndex(fileName, ".hint") 122 | idx, _ = strconv.Atoi(fileName[s:e]) 123 | if lastID < idx { 124 | lastID = idx 125 | lastFp = idxFp 126 | } 127 | } 128 | return uint32(lastID), lastFp 129 | } 130 | 131 | func closeReadHintFp(files []*os.File, fileID uint32) { 132 | for _, fp := range files { 133 | if !strings.Contains(fp.Name(), strconv.Itoa(int(fileID))) { 134 | fp.Close() 135 | } 136 | } 137 | } 138 | 139 | func setWriteableFile(fileID uint32, dirName string) (*os.File, uint32) { 140 | var fp *os.File 141 | var err error 142 | if fileID == 0 { 143 | fileID = uint32(time.Now().Unix()) 144 | } 145 | fileName := dirName + "/" + strconv.Itoa(int(fileID)) + ".data" 146 | fp, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0755) 147 | if err != nil { 148 | panic(err) 149 | } 150 | return fp, fileID 151 | } 152 | 153 | func setHintFile(fileID uint32, dirName string) *os.File { 154 | var fp *os.File 155 | var err error 156 | if fileID == 0 { 157 | fileID = uint32(time.Now().Unix()) 158 | } 159 | fileName := dirName + "/" + strconv.Itoa(int(fileID)) + ".hint" 160 | fp, err = os.OpenFile(fileName, os.O_CREATE|os.O_RDWR, 0755) 161 | if err != nil { 162 | panic(err) 163 | } 164 | return fp 165 | } 166 | 167 | func appendWriteFile(fp *os.File, buf []byte) (int, error) { 168 | stat, err := fp.Stat() 169 | if err != nil { 170 | return -1, err 171 | } 172 | 173 | return fp.WriteAt(buf, stat.Size()) 174 | } 175 | 176 | // return a unique not exists file name by timeStamp 177 | func uniqueFileName(root, suffix string) string { 178 | for { 179 | tStamp := strconv.Itoa(int(time.Now().Unix())) 180 | _, err := os.Stat(root + "/" + tStamp + "." + suffix) 181 | if err != nil && os.IsNotExist(err) { 182 | return tStamp + "." + suffix 183 | } 184 | time.Sleep(time.Second) 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /bitcask.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "strconv" 8 | "strings" 9 | "sync" 10 | 11 | "github.com/laohanlinux/go-logger/logger" 12 | ) 13 | 14 | // ErrNotFound ... 15 | var ( 16 | ErrNotFound = fmt.Errorf("Not Found.") 17 | ErrIsNotDir = fmt.Errorf("the file is not dir") 18 | ) 19 | 20 | // Open ... 21 | func Open(dirName string, opts *Options) (*BitCask, error) { 22 | if opts == nil { 23 | opts1 := NewOptions(0, 0, -1, 60, true) 24 | opts = &opts1 25 | } 26 | 27 | //make sure the fileName is exits 28 | _, err := os.Stat(dirName) 29 | if err != nil && !os.IsNotExist(err) { 30 | return nil, err 31 | } 32 | 33 | if os.IsNotExist(err) { 34 | err = os.Mkdir(dirName, 0755) 35 | if err != nil { 36 | return nil, err 37 | } 38 | } 39 | 40 | b := &BitCask{ 41 | Opts: opts, 42 | dirFile: dirName, 43 | oldFile: newBFiles(), 44 | rwLock: &sync.RWMutex{}, 45 | } 46 | // lock file 47 | b.lockFile, err = lockFile(dirName + "/" + lockFileName) 48 | if err != nil { 49 | return nil, err 50 | } 51 | 52 | b.keyDirs = NewKeyDir(dirName) 53 | // scan readAble file 54 | files, _ := b.readableFiles() 55 | b.parseHint(files) 56 | // get the last fileid 57 | fileID, hintFp := lastFileInfo(files) 58 | 59 | var writeFp *os.File 60 | writeFp, fileID = setWriteableFile(fileID, dirName) 61 | 62 | hintFp = setHintFile(fileID, dirName) 63 | // close other hint 64 | closeReadHintFp(files, fileID) 65 | // setting writeable file, only one 66 | dataStat, _ := writeFp.Stat() 67 | bf := &BFile{ 68 | fp: writeFp, 69 | fileID: fileID, 70 | writeOffset: uint64(dataStat.Size()), 71 | hintFp: hintFp, 72 | } 73 | b.writeFile = bf 74 | // save pid into bitcask.lock file 75 | writePID(b.lockFile, fileID) 76 | return b, nil 77 | } 78 | 79 | // BitCask ... 80 | type BitCask struct { 81 | Opts *Options // opts for bitcask 82 | oldFile *BFiles // hint file, data file 83 | lockFile *os.File // lock file with process 84 | keyDirs *KeyDirs // key/value hashMap, building with hint file 85 | dirFile string // bitcask storage root dir 86 | writeFile *BFile // writeable file 87 | rwLock *sync.RWMutex // rwlocker for bitcask Get and put Operation 88 | } 89 | 90 | // Close opening fp 91 | func (bc *BitCask) Close() { 92 | // close ActiveFiles 93 | bc.oldFile.close() 94 | // close writeable file 95 | bc.writeFile.fp.Close() 96 | bc.writeFile.hintFp.Close() 97 | // close lockFile 98 | bc.lockFile.Close() 99 | // delete lockFile 100 | os.Remove(bc.dirFile + "/" + lockFileName) 101 | } 102 | 103 | // Put key/value 104 | func (bc *BitCask) Put(key []byte, value []byte) error { 105 | bc.rwLock.Lock() 106 | defer bc.rwLock.Unlock() 107 | checkWriteableFile(bc) 108 | // write data into writeable file 109 | e, err := bc.writeFile.writeDatat(key, value) 110 | if err != nil { 111 | bc.rwLock.Unlock() 112 | return err 113 | } 114 | // add key/value into keydirs 115 | keyDirs.put(string(key), &e) 116 | return nil 117 | } 118 | 119 | // Get ... 120 | func (bc *BitCask) Get(key []byte) ([]byte, error) { 121 | e := keyDirs.get(string(key)) 122 | if e == nil { 123 | return nil, ErrNotFound 124 | } 125 | 126 | fileID := e.fileID 127 | bf, err := bc.getFileState(fileID) 128 | if err != nil && os.IsNotExist(err) { 129 | logger.Warn("key:", string(key), "=>the file is not exits:", fileID) 130 | //time.Sleep(time.Second) 131 | return nil, err 132 | } 133 | 134 | //TODO 135 | // assrt file crc32 136 | //logger.Info("fileID", fileID, "entry offset:", e.valueOffset, "\t entryLen:", e.valueSz) 137 | return bf.read(e.valueOffset, e.valueSz) 138 | } 139 | 140 | // Del value by key 141 | func (bc *BitCask) Del(key []byte) error { 142 | bc.rwLock.Lock() 143 | defer bc.rwLock.Unlock() 144 | if bc.writeFile == nil { 145 | return fmt.Errorf("Can Not Read The Bitcask Root Director") 146 | } 147 | e := keyDirs.get(string(key)) 148 | if e == nil { 149 | return ErrNotFound 150 | } 151 | 152 | checkWriteableFile(bc) 153 | // write data into writeable file 154 | err := bc.writeFile.del(key) 155 | if err != nil { 156 | return err 157 | } 158 | // delete key/value from keydirs 159 | keyDirs.del(string(key)) 160 | return nil 161 | } 162 | 163 | // return readable hint file: xxxx.hint 164 | func (bc *BitCask) readableFiles() ([]*os.File, error) { 165 | filterFiles := []string{lockFileName} 166 | ldfs, err := listHintFiles(bc) 167 | if err != nil { 168 | return nil, err 169 | } 170 | 171 | fps := make([]*os.File, 0, len(ldfs)) 172 | for _, filePath := range ldfs { 173 | if existsSuffixs(filterFiles, filePath) { 174 | continue 175 | } 176 | fp, err := os.OpenFile(bc.dirFile+"/"+filePath, os.O_RDONLY, 0755) 177 | if err != nil { 178 | return nil, err 179 | } 180 | fps = append(fps, fp) 181 | } 182 | if len(fps) == 0 { 183 | return nil, nil 184 | } 185 | return fps, nil 186 | } 187 | 188 | func (bc *BitCask) getFileState(fileID uint32) (*BFile, error) { 189 | // lock up it from write able file 190 | if fileID == bc.writeFile.fileID { 191 | return bc.writeFile, nil 192 | } 193 | // if not exits in write able file, look up it from OldFile 194 | bf := bc.oldFile.get(fileID) 195 | if bf != nil { 196 | return bf, nil 197 | } 198 | 199 | bf, err := openBFile(bc.dirFile, int(fileID)) 200 | if err != nil { 201 | return nil, err 202 | } 203 | bc.oldFile.put(bf, fileID) 204 | return bf, nil 205 | } 206 | 207 | func (bc *BitCask) parseHint(hintFps []*os.File) { 208 | 209 | b := make([]byte, HintHeaderSize, HintHeaderSize) 210 | for _, fp := range hintFps { 211 | offset := int64(0) 212 | hintName := fp.Name() 213 | s := strings.LastIndex(hintName, "/") + 1 214 | e := strings.LastIndex(hintName, ".hint") 215 | fileID, _ := strconv.ParseInt(hintName[s:e], 10, 32) 216 | 217 | for { 218 | // parse hint header 219 | n, err := fp.ReadAt(b, offset) 220 | offset += int64(n) 221 | if err != nil && err != io.EOF { 222 | panic(err) 223 | } 224 | if err == io.EOF { 225 | break 226 | } 227 | 228 | if n != HintHeaderSize { 229 | panic(n) 230 | } 231 | 232 | tStamp, ksz, valueSz, valuePos := DecodeHint(b) 233 | //logger.Info("ksz:", ksz, "offset:", offset) 234 | if ksz+valueSz == 0 { // the record is deleted 235 | continue 236 | } 237 | 238 | // parse hint key 239 | keyByte := make([]byte, ksz) 240 | n, err = fp.ReadAt(keyByte, offset) 241 | if err != nil && err != io.EOF { 242 | panic(err) 243 | } 244 | if err == io.EOF { 245 | break 246 | } 247 | if n != int(ksz) { 248 | panic(n) 249 | } 250 | key := string(keyByte) 251 | 252 | e := &entry{ 253 | fileID: uint32(fileID), 254 | valueSz: valueSz, 255 | valueOffset: valuePos, 256 | timeStamp: tStamp, 257 | } 258 | offset += int64(ksz) 259 | // put entry into keyDirs 260 | keyDirs.put(key, e) 261 | } 262 | } 263 | } 264 | --------------------------------------------------------------------------------