├── .gitignore ├── Check.go ├── Database.go ├── Databsase.go ├── LSM_test.go ├── MemTable.go ├── RWData.go ├── Readme.md ├── config └── Config.go ├── docs └── 步骤.md ├── example └── main.go ├── go.mod ├── iMemTable.go ├── kv ├── Value.go └── Value_test.go ├── orderTable ├── OrderTableInterface.go ├── skipList │ ├── SkipList.go │ └── SkipList.go:Zone.Identifier └── sortTree │ ├── SortTree.go │ ├── SortTree.go:Zone.Identifier │ ├── Stack.go │ └── Stack.go:Zone.Identifier ├── sortTree ├── SortTree.go ├── SortTree_test.go └── Stack.go ├── ssTable ├── Init.go ├── MetaInfo.go ├── Position.go ├── Search.go ├── SsTable.go ├── TableTree.go ├── TableTreeInit.go ├── compaction.go ├── creatTable.go ├── createTable.go ├── dbFile.go ├── initload.go └── tree.go ├── start.go └── wal ├── Wal.go └── Wal.go:Zone.Identifier /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | 10 | # Test binary, built with `go test -c` 11 | *.test 12 | 13 | # Output of the go coverage tool, specifically when used with LiteIDE 14 | *.out 15 | 16 | # Dependency directories (remove the comment below to include it) 17 | # vendor/ 18 | -------------------------------------------------------------------------------- /Check.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/config" 5 | "log" 6 | "time" 7 | ) 8 | 9 | func Check() { 10 | //con := config.GetConfig() 11 | ticker := time.Tick(50 * time.Millisecond) 12 | for range ticker { 13 | log.Println("Performing background checks...") 14 | // 检查内存 15 | checkMemory() 16 | // 检查压缩数据库文件 17 | database.TableTree.Check() 18 | } 19 | } 20 | 21 | func checkMemory() { 22 | con := config.GetConfig() 23 | count := database.MemTable.OrderTable.GetCount() 24 | if count < con.Threshold { 25 | return 26 | } 27 | // 交互内存 28 | log.Println("Compressing memory") 29 | database.Swap() 30 | } 31 | 32 | // CompressMemory 会监听iMemTable，当iMemTable有数据的时候就进行压缩 33 | func CompressMemory() { 34 | con := config.GetConfig() 35 | ticker := time.Tick(time.Duration(con.CompressInterval) * time.Millisecond) 36 | for range ticker { 37 | //fmt.Println(database.iMemTable.Getlen()) 38 | for database.iMemTable.Getlen() != 0 { 39 | log.Println("Compressing iMemTable") 40 | preTable := database.iMemTable.PopTable() 41 | database.TableTree.CreatNewTable(preTable.OrderTable.GetValues()) 42 | preTable.Wal.DeleteFile() 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Database.go: -------------------------------------------------------------------------------- 1 | package lsm 2 | 3 | import ( 4 | "github.com/huiming23344/lsm/ssTable" 5 | "github.com/huiming23344/lsm/wal" 6 | "log" 7 | "os" 8 | "path" 9 | ) 10 | 11 | type Database struct { 12 | // 内存表 13 | MemTable *MemTable 14 | // 只读内存表 15 | iMemTable *ReadOnlyMemTables 16 | // SSTable 列表 17 | TableTree *ssTable.TableTree 18 | } 19 | 20 | // 数据库，全局唯一实例 21 | var database *Database 22 | 23 | func (d *Database) loadAllWalFiles(dir string) { 24 | infos, err := os.ReadDir(dir) 25 | if err != nil { 26 | log.Println("Failed to read the database file") 27 | panic(err) 28 | } 29 | tree := d.MemTable.MemoryTree 30 | for _, info := range infos { 31 | // 如果是 wal.log 文件 32 | name := info.Name() 33 | if path.Ext(name) == ".log" { 34 | preWal := &wal.Wal{} 35 | preTree := preWal.LoadFromFile(path.Join(dir, info.Name()), tree) 36 | table := &MemTable{ 37 | MemoryTree: preTree, 38 | Wal: preWal, 39 | } 40 | log.Printf("add table to iMemTable, table: %v\n", table) 41 | d.iMemTable.AddTable(table) 42 | } 43 | } 44 | return 45 | } 46 | 47 | func (d *Database) Swap() { 48 | table := d.MemTable.Swap() 49 | // 将内存表存储到 iMemTable 中 50 | log.Printf("add table to iMemTable, table: %v\n", table) 51 | d.iMemTable.AddTable(table) 52 | } 53 | -------------------------------------------------------------------------------- /Databsase.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/ssTable" 5 | "LSM/wal" 6 | "log" 7 | "os" 8 | "path" 9 | ) 10 | 11 | type Database struct { 12 | MemTable *MemTable 13 | iMemTable *ReadOnlyMemTables 14 | TableTree *ssTable.TableTree 15 | } 16 | 17 | var database *Database 18 | 19 | func (d *Database) loadAllWalFiles(dir string) { 20 | infos, err := os.ReadDir(dir) 21 | if err != nil { 22 | log.Println("Failed to read the database file") 23 | panic(err) 24 | } 25 | orderTable := d.MemTable.OrderTable 26 | for _, info := range infos { 27 | name := info.Name() 28 | if path.Ext(name) == ".log" { 29 | preWal := &wal.Wal{} 30 | preTable := preWal.LoadFromFile(path.Join(dir, info.Name()), orderTable) 31 | table := &MemTable{ 32 | OrderTable: preTable, 33 | Wal: preWal, 34 | } 35 | log.Printf("add table to iMemTable, table: %v\n", table) 36 | d.iMemTable.AddTable(table) 37 | } 38 | } 39 | return 40 | } 41 | func (d *Database) Swap() { 42 | table := d.MemTable.Swap() 43 | log.Printf("add table to iMemTable, table: %v\n", table) 44 | d.iMemTable.AddTable(table) 45 | } 46 | -------------------------------------------------------------------------------- /LSM_test.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/config" 5 | "fmt" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | type TestValue struct { 11 | A int64 12 | B int64 13 | C int64 14 | D string 15 | } 16 | 17 | func insert() { 18 | 19 | // 64 个字节 20 | testV := TestValue{ 21 | A: 1, 22 | B: 1, 23 | C: 3, 24 | D: "00000000000000000000000000000000000000", 25 | } 26 | 27 | count := 0 28 | start := time.Now() 29 | key := []byte{'a', 'a', 'a', 'a', 'a', 'a'} 30 | Set(string(key), testV) 31 | for a := 0; a < 1; a++ { 32 | for b := 0; b < 1; b++ { 33 | for c := 0; c < 26; c++ { 34 | for d := 0; d < 26; d++ { 35 | for e := 0; e < 26; e++ { 36 | for f := 0; f < 26; f++ { 37 | key[0] = 'a' + byte(a) 38 | key[1] = 'a' + byte(b) 39 | key[2] = 'a' + byte(c) 40 | key[3] = 'a' + byte(d) 41 | key[4] = 'a' + byte(e) 42 | key[5] = 'a' + byte(f) 43 | Set(string(key), testV) 44 | count++ 45 | } 46 | } 47 | } 48 | } 49 | } 50 | } 51 | elapse := time.Since(start) 52 | fmt.Println("插入完成，数据量：", count, ",消耗时间：", elapse) 53 | } 54 | func Test_Sampletest(t *testing.T) { 55 | Start(config.Config{ 56 | DataDir: `SSD`, 57 | Level0Size: 1, 58 | PartSize: 4, 59 | Threshold: 10, 60 | CheckInterval: 50, // 压缩时间间隔 61 | }) 62 | // 64 个字节 63 | testV := TestValue{ 64 | A: 1, 65 | B: 1, 66 | C: 3, 67 | D: "00000000000000000000000000000000000000", 68 | } 69 | Set("aaa", testV) 70 | value, success := Get[TestValue]("aaa") 71 | if success && value == testV { 72 | fmt.Println(value) 73 | } else { 74 | fmt.Errorf("NotOK") 75 | } 76 | Delete[TestValue]("aaa") 77 | } 78 | func Test_Insert(t *testing.T) { 79 | Start(config.Config{ 80 | DataDir: `SSD`, 81 | Level0Size: 1, // 0 层 SSTable 文件大小 82 | PartSize: 10, // 每层文件数量 83 | Threshold: 500, // 内存表阈值 84 | CheckInterval: 3, // 压缩时间间隔 85 | CompressInterval: 3, 86 | }) 87 | insert() 88 | time.Sleep(8 * time.Second) 89 | } 90 | func Test_SampleCompress(t *testing.T) { 91 | Start(config.Config{ 92 | DataDir: `SSD`, 93 | Level0Size: 10, // 0 层 SSTable 文件大小 94 | PartSize: 4, // 每层文件数量 95 | Threshold: 30, // 内存表阈值 96 | CheckInterval: 50, // 压缩时间间隔 97 | CompressInterval: 3, 98 | }) 99 | testV := TestValue{ 100 | A: 1, 101 | B: 1, 102 | C: 3, 103 | D: "00000000000000000000000000000000000000", 104 | } 105 | count := 0 106 | start := time.Now() 107 | key := []byte{'a', 'a', 'a', 'a'} 108 | for a := 0; a < 1; a++ { 109 | for b := 0; b < 1; b++ { 110 | for c := 0; c < 26; c++ { 111 | for d := 0; d < 26; d++ { 112 | key[0] = 'a' + byte(a) 113 | key[1] = 'a' + byte(b) 114 | key[2] = 'a' + byte(c) 115 | key[3] = 'a' + byte(d) 116 | Set(string(key), testV) 117 | count++ 118 | } 119 | } 120 | } 121 | } 122 | elapse := time.Since(start) 123 | fmt.Println("插入完成，数据量：", count, ",消耗时间：", elapse) 124 | time.Sleep(4 * time.Second) 125 | } 126 | func TestInsertSearch(t *testing.T) { 127 | Start(config.Config{ 128 | DataDir: `SSD`, 129 | Level0Size: 10, // 0 层 SSTable 文件大小 130 | PartSize: 4, // 每层文件数量 131 | Threshold: 30, // 内存表阈值 132 | CheckInterval: 3, // 压缩时间间隔 133 | CompressInterval: 3, 134 | }) 135 | insert() 136 | testV := TestValue{ 137 | A: 1, 138 | B: 1, 139 | C: 3, 140 | D: "00000000000000000000000000000000000000", 141 | } 142 | 143 | count := 0 144 | key := []byte{'a', 'a', 'a', 'a', 'a', 'a'} 145 | //Set(string(key), testV) 146 | for a := 0; a < 1; a++ { 147 | for b := 0; b < 1; b++ { 148 | for c := 0; c < 26; c++ { 149 | for d := 0; d < 26; d++ { 150 | for e := 0; e < 26; e++ { 151 | for f := 0; f < 26; f++ { 152 | key[0] = 'a' + byte(a) 153 | key[1] = 'a' + byte(b) 154 | key[2] = 'a' + byte(c) 155 | key[3] = 'a' + byte(d) 156 | key[4] = 'a' + byte(e) 157 | key[5] = 'a' + byte(f) 158 | v, _ := Get[TestValue](string(key)) 159 | if v != testV { 160 | fmt.Errorf("NotEqual") 161 | } else { 162 | fmt.Println("SearchCorrect ", key) 163 | } 164 | count++ 165 | } 166 | } 167 | } 168 | } 169 | } 170 | } 171 | 172 | } 173 | -------------------------------------------------------------------------------- /MemTable.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/config" 5 | "LSM/kv" 6 | "LSM/orderTable" 7 | "LSM/orderTable/skipList" 8 | "LSM/wal" 9 | "log" 10 | "sync" 11 | ) 12 | 13 | type MemTable struct { 14 | OrderTable orderTable.OrderInterface 15 | Wal *wal.Wal 16 | swapLock *sync.RWMutex 17 | } 18 | 19 | func (m *MemTable) InitMemTree() { 20 | log.Println("Initializing MemTable MemTree...") 21 | m.OrderTable = &skipList.SkipList{} 22 | m.OrderTable.Init() 23 | m.swapLock = &sync.RWMutex{} 24 | } 25 | func (m *MemTable) InitWal(dir string) { 26 | log.Println("Initializing Wal...") 27 | m.Wal = &wal.Wal{} 28 | m.Wal.Init(dir) 29 | } 30 | func (m *MemTable) Swap() *MemTable { 31 | con := config.GetConfig() 32 | m.swapLock.Lock() 33 | defer m.swapLock.Unlock() 34 | tmpTable := m.OrderTable.Swap() 35 | table := &MemTable{ 36 | OrderTable: tmpTable, 37 | Wal: m.Wal, 38 | swapLock: &sync.RWMutex{}, 39 | } 40 | newWal := &wal.Wal{} 41 | newWal.Init(con.DataDir) 42 | m.Wal = newWal 43 | return table 44 | } 45 | func (m *MemTable) Search(key string) (kv.Value, kv.SearchResult) { 46 | m.swapLock.RLock() 47 | defer m.swapLock.RUnlock() 48 | return m.OrderTable.Search(key) 49 | } 50 | func (m *MemTable) Set(key string, value []byte) (kv.Value, bool) { 51 | m.swapLock.Lock() 52 | defer m.swapLock.Unlock() 53 | oldValue, hasOld := m.OrderTable.Set(key, value) 54 | m.Wal.Write(kv.Value{ 55 | Key: key, 56 | Value: value, 57 | Delete: false, 58 | }) 59 | return oldValue, hasOld 60 | } 61 | func (m *MemTable) Delete(key string) (kv.Value, bool) { 62 | m.swapLock.Lock() 63 | defer m.swapLock.Unlock() 64 | oldValue, success := m.OrderTable.Delete(key) 65 | if success { 66 | m.Wal.Write(kv.Value{ 67 | Key: key, 68 | Value: nil, 69 | Delete: true, 70 | }) 71 | } 72 | return oldValue, success 73 | } 74 | -------------------------------------------------------------------------------- /RWData.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/kv" 5 | "encoding/json" 6 | "log" 7 | ) 8 | 9 | func Get[T any](key string) (T, bool) { 10 | log.Print("Get ", key) 11 | value, result := database.MemTable.Search(key) 12 | if result == kv.Success { 13 | return getInstance[T](value.Value) 14 | } 15 | value, result = database.iMemTable.Search(key) 16 | if result == kv.Success { 17 | return getInstance[T](value.Value) 18 | } 19 | if database.TableTree != nil { 20 | value, result = database.TableTree.Search(key) 21 | if result == kv.Success { 22 | return getInstance[T](value.Value) 23 | } 24 | } 25 | var nilV T 26 | return nilV, false 27 | } 28 | func Set[T any](key string, value T) bool { 29 | //log.Print("Set ", key, value) 30 | data, err := kv.Convert(value) 31 | if err != nil { 32 | log.Println(err) 33 | return false 34 | } 35 | _, _ = database.MemTable.Set(key, data) 36 | return true 37 | } 38 | 39 | // DeleteAndGet 删除元素并尝试获取旧的值， 40 | // 返回的 bool 表示是否有旧值，不表示是否删除成功 41 | func DeleteAndGet[T any](key string) (T, bool) { 42 | log.Print("Delete ", key) 43 | value, success := database.MemTable.Delete(key) 44 | if success { 45 | return getInstance[T](value.Value) 46 | } 47 | var nilV T 48 | return nilV, false 49 | } 50 | func Delete[T any](key string) { 51 | log.Print("Delete ", key) 52 | database.MemTable.Delete(key) 53 | } 54 | func getInstance[T any](data []byte) (T, bool) { 55 | var value T 56 | err := json.Unmarshal(data, &value) 57 | if err != nil { 58 | log.Println(err) 59 | } 60 | return value, true 61 | } 62 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # lsm 2 | 主要代码来自于[whuanle/lsm](https://github.com/whuanle/lsm)， 3 | 为了方便应用到[huiming23344/kv-raft](https://github.com/huiming23344/kv-raft)中，在原作者基础上进行了一些修改和完善。 4 | 5 | ## 主要改动 6 | 加入`iMemTable`结构，解决了Compress时WalF会被误删的问题 7 | 8 | ## 使用方法 9 | 10 | 下载依赖包： 11 | ```go 12 | go get -u github.com/huiming/lsm@v1.0.0 13 | ``` 14 | 15 | 16 | 17 | ## 参考 18 | 19 | - [lsm on github by whuanle](https://github.com/whuanle/lsm) 20 | - [痴者工良的cnblogs](https://www.cnblogs.com/whuanle/p/16297025.html) -------------------------------------------------------------------------------- /config/Config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import "sync" 4 | 5 | // Config 数据库启动配置 6 | type Config struct { 7 | // 数据目录 8 | DataDir string 9 | // 0 层的 所有 SsTable 文件大小总和的最大值，单位 MB，超过此值，该层 SsTable 将会被压缩到下一层 10 | Level0Size int 11 | // 每层中 SsTable 表数量的阈值，该层 SsTable 将会被压缩到下一层 12 | PartSize int 13 | // 内存表的 kv 最大数量，超出这个阈值，内存表将会被保存到 SsTable 中 14 | Threshold int 15 | // 检查内存树大小的时间间隔，多久进行一次检查，如果超出就放入iMemTable中 16 | CheckInterval int 17 | // 压缩内存的时间间隔，多久进行一次检查iMemTable不为空的压缩工作 18 | CompressInterval int 19 | } 20 | 21 | var once *sync.Once = &sync.Once{} 22 | 23 | // 常驻内存 24 | var config Config 25 | 26 | // Init 初始化数据库配置 27 | func Init(con Config) { 28 | once.Do(func() { 29 | config = con 30 | }) 31 | } 32 | 33 | // GetConfig 获取数据库配置 34 | func GetConfig() Config { 35 | return config 36 | } 37 | -------------------------------------------------------------------------------- /docs/步骤.md: -------------------------------------------------------------------------------- 1 | 1，先写一个有序树 -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "github.com/huiming23344/lsm" 7 | "github.com/huiming23344/lsm/config" 8 | "os" 9 | "time" 10 | ) 11 | 12 | type TestValue struct { 13 | A int64 14 | B int64 15 | C int64 16 | D string 17 | } 18 | 19 | func main() { 20 | testResetWalBug() 21 | } 22 | 23 | func testResetWalBug() { 24 | defer func() { 25 | r := recover() 26 | if r != nil { 27 | fmt.Println(r) 28 | inputReader := bufio.NewReader(os.Stdin) 29 | _, _ = inputReader.ReadString('\n') 30 | } 31 | }() 32 | lsm.Start(config.Config{ 33 | DataDir: "./data", 34 | Level0Size: 100, 35 | PartSize: 4, 36 | Threshold: 9, 37 | CheckInterval: 2, 38 | CompressInterval: 2, 39 | }) 40 | //注释掉以下五行代码，再次运行以检验wal中新数据是否被删除 41 | //insert 10 to start compress 42 | insertValuesByCount(10, 10) 43 | // wait for compress and insert kvs when compress begins 44 | time.Sleep(2 * time.Second) 45 | insertValuesByCount(6, 0) 46 | keys := []string{"0", "1", "2", "3", "4", "5"} 47 | queryByKeys(keys) 48 | time.Sleep(10 * time.Second) 49 | } 50 | 51 | func queryByKeys(keys []string) { 52 | for _, key := range keys { 53 | start := time.Now() 54 | v, _ := lsm.Get[TestValue](key) 55 | elapse := time.Since(start) 56 | fmt.Println("查找", key, " 完成，消耗时间：", elapse) 57 | fmt.Println(v) 58 | } 59 | } 60 | 61 | func query() { 62 | start := time.Now() 63 | v, _ := lsm.Get[TestValue]("4") 64 | elapse := time.Since(start) 65 | fmt.Println("查找 aaaaaa 完成，消耗时间：", elapse) 66 | fmt.Println(v) 67 | 68 | start = time.Now() 69 | v, _ = lsm.Get[TestValue]("2") 70 | elapse = time.Since(start) 71 | fmt.Println("查找 aazzzz 完成，消耗时间：", elapse) 72 | fmt.Println(v) 73 | } 74 | 75 | func insertValuesByCount(count, startFrom int) { 76 | start := time.Now() 77 | // 64 个字节 78 | testV := TestValue{ 79 | A: 1, 80 | B: 1, 81 | C: 3, 82 | D: "00000000000000000000000000000000000000", 83 | } 84 | for i := 0; i < count; i++ { 85 | lsm.Set(fmt.Sprint(i+startFrom), testV) 86 | } 87 | elapse := time.Since(start) 88 | fmt.Println("插入完成，数据量：", count, ",消耗时间：", elapse) 89 | } 90 | 91 | func insert() { 92 | // 64 个字节 93 | testV := TestValue{ 94 | A: 1, 95 | B: 1, 96 | C: 3, 97 | D: "00000000000000000000000000000000000000", 98 | } 99 | 100 | //testVData, _ := json.Marshal(testV) 101 | //// 131 个字节 102 | //kvData, _ := kv.Encode(kv.Value{ 103 | // Key: "abcdef", 104 | // Value: testVData, 105 | // Deleted: false, 106 | //}) 107 | //fmt.Println(len(kvData)) 108 | //position := ssTable.Position{} 109 | //// 35 个字节 110 | //positionData, _ := json.Marshal(position) 111 | //fmt.Println(len(positionData)) 112 | // 113 | count := 0 114 | start := time.Now() 115 | key := []byte{'a', 'a', 'a', 'a', 'a', 'a'} 116 | lsm.Set(string(key), testV) 117 | for a := 0; a < 26; a++ { 118 | for b := 0; b < 26; b++ { 119 | for c := 0; c < 26; c++ { 120 | for d := 0; d < 26; d++ { 121 | for e := 0; e < 26; e++ { 122 | for f := 0; f < 26; f++ { 123 | key[0] = 'a' + byte(a) 124 | key[1] = 'a' + byte(b) 125 | key[2] = 'a' + byte(c) 126 | key[3] = 'a' + byte(d) 127 | key[4] = 'a' + byte(e) 128 | key[5] = 'a' + byte(f) 129 | lsm.Set(string(key), testV) 130 | count++ 131 | } 132 | } 133 | } 134 | } 135 | } 136 | } 137 | elapse := time.Since(start) 138 | fmt.Println("插入完成，数据量：", count, ",消耗时间：", elapse) 139 | } 140 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module LSM 2 | 3 | go 1.24 4 | -------------------------------------------------------------------------------- /iMemTable.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/kv" 5 | "sync" 6 | ) 7 | 8 | type ReadOnlyMemTables struct { 9 | readonlyTable []*MemTable 10 | lock *sync.RWMutex 11 | } 12 | 13 | func (r *ReadOnlyMemTables) Init() { 14 | r.readonlyTable = make([]*MemTable, 0) 15 | r.lock = &sync.RWMutex{} 16 | } 17 | func (r *ReadOnlyMemTables) Getlen() int { 18 | r.lock.Lock() 19 | defer r.lock.Unlock() 20 | return len(r.readonlyTable) 21 | } 22 | func (r *ReadOnlyMemTables) AddTable(table *MemTable) { 23 | r.lock.Lock() 24 | defer r.lock.Unlock() 25 | r.readonlyTable = append(r.readonlyTable, table) 26 | } 27 | func (r *ReadOnlyMemTables) PopTable() *MemTable { 28 | r.lock.Lock() 29 | defer r.lock.Unlock() 30 | table := r.readonlyTable[0] 31 | r.readonlyTable = r.readonlyTable[1:] 32 | return table 33 | } 34 | func (r *ReadOnlyMemTables) Search(key string) (kv.Value, kv.SearchResult) { 35 | r.lock.RLock() 36 | defer r.lock.RUnlock() 37 | for _, table := range r.readonlyTable { 38 | value, result := table.Search(key) 39 | if result == kv.Success { 40 | return value, result 41 | } 42 | } 43 | return kv.Value{}, kv.NotFind 44 | } 45 | -------------------------------------------------------------------------------- /kv/Value.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "encoding/json" 5 | ) 6 | 7 | type SearchResult int 8 | 9 | const ( 10 | NotFind = iota 11 | Delete 12 | Success 13 | ) 14 | 15 | type Value struct { 16 | Key string 17 | Value []byte 18 | Delete bool 19 | } 20 | 21 | func (v *Value) Copy() *Value { 22 | return &Value{ 23 | Key: v.Key, 24 | Value: v.Value, 25 | Delete: v.Delete, 26 | } 27 | } 28 | func Get[T any](v *Value) (T, error) { 29 | var value T 30 | err := json.Unmarshal(v.Value, &value) 31 | return value, err 32 | } 33 | func Convert[T any](value T) ([]byte, error) { 34 | return json.Marshal(&value) 35 | } 36 | func Decode(data []byte) (Value, error) { 37 | var value Value 38 | err := json.Unmarshal(data, &value) 39 | return value, err 40 | } 41 | func Encode(value Value) ([]byte, error) { 42 | return json.Marshal(&value) 43 | } 44 | -------------------------------------------------------------------------------- /kv/Value_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | type vTest struct { 9 | A int 10 | B int 11 | } 12 | 13 | var testData []byte = []byte{123, 34, 65, 34, 58, 49, 50, 51, 44, 34, 66, 34, 58, 49, 50, 51, 125} 14 | 15 | func Test_Value_Convert(t *testing.T) { 16 | data, err := Convert(vTest{ 17 | A: 123, 18 | B: 123, 19 | }) 20 | if err != nil { 21 | t.Error(err) 22 | } 23 | if reflect.DeepEqual(data, testData) == false { 24 | t.Fatal() 25 | } 26 | } 27 | 28 | func Test_Value_Get(t *testing.T) { 29 | 30 | } 31 | 32 | func Test_Value_Encode(t *testing.T) { 33 | 34 | } 35 | 36 | func Test_Value_Decode(t *testing.T) { 37 | 38 | } 39 | -------------------------------------------------------------------------------- /orderTable/OrderTableInterface.go: -------------------------------------------------------------------------------- 1 | package orderTable 2 | 3 | import "LSM/kv" 4 | 5 | type OrderInterface interface { 6 | Set(key string, value []byte) (oldValue kv.Value, hasOld bool) 7 | GetCount() int 8 | Delete(key string) (oldValue kv.Value, hasOld bool) 9 | GetValues() []kv.Value 10 | Init() 11 | //Swap() *OrderInterface 12 | Swap() OrderInterface 13 | Search(key string) (kv.Value, kv.SearchResult) 14 | } 15 | -------------------------------------------------------------------------------- /orderTable/skipList/SkipList.go: -------------------------------------------------------------------------------- 1 | package skipList 2 | 3 | import ( 4 | "LSM/kv" 5 | "LSM/orderTable" 6 | "math/rand" 7 | "sync" 8 | ) 9 | 10 | const MAXLEVEL = 16 11 | 12 | type ListNode struct { 13 | KV kv.Value 14 | next []*ListNode 15 | } 16 | type SkipList struct { 17 | head *ListNode 18 | level int 19 | mutex *sync.RWMutex 20 | count int 21 | } 22 | 23 | func NewNode(value kv.Value, level int) *ListNode { 24 | return &ListNode{ 25 | KV: value, 26 | next: make([]*ListNode, level), 27 | } 28 | } 29 | func NewSkipList() *SkipList { 30 | return &SkipList{ 31 | head: NewNode(kv.Value{}, MAXLEVEL), 32 | level: 1, 33 | } 34 | } 35 | func (sl *SkipList) randomLevel() int { 36 | level := 1 37 | for rand.Float32() < 0.5 && level < MAXLEVEL { 38 | level++ 39 | } 40 | return level 41 | } 42 | func (sl *SkipList) Init() { 43 | sl.head = NewNode(kv.Value{}, MAXLEVEL) 44 | sl.count = 0 45 | sl.mutex = &sync.RWMutex{} 46 | sl.level = 1 47 | } 48 | func (sl *SkipList) Set(key string, value []byte) (oldValue kv.Value, hasOld bool) { 49 | sl.mutex.Lock() 50 | defer sl.mutex.Unlock() 51 | update := make([]*ListNode, MAXLEVEL) 52 | current := sl.head 53 | for i := sl.level - 1; i >= 0; i-- { 54 | for current.next[i] != nil && current.next[i].KV.Key < key { 55 | current = current.next[i] 56 | } 57 | update[i] = current 58 | } 59 | if current.next[0] != nil && current.next[0].KV.Key == key { 60 | oldValue = current.next[0].KV 61 | hasOld = true 62 | current.next[0].KV.Value = value 63 | current.next[0].KV.Delete = false 64 | return 65 | } 66 | level := sl.randomLevel() 67 | if level > sl.level { 68 | for i := sl.level; i < level; i++ { 69 | update[i] = sl.head 70 | } 71 | sl.level = level 72 | } 73 | newNode := NewNode(kv.Value{Key: key, Value: value, Delete: false}, level) 74 | for i := 0; i < level; i++ { 75 | newNode.next[i] = update[i].next[i] 76 | update[i].next[i] = newNode 77 | } 78 | sl.count++ 79 | return kv.Value{}, false 80 | 81 | } 82 | func (sl *SkipList) GetCount() int { 83 | sl.mutex.RLock() 84 | defer sl.mutex.RUnlock() 85 | return sl.count 86 | 87 | } 88 | func (sl *SkipList) Delete(key string) (oldValue kv.Value, hasOld bool) { 89 | sl.mutex.Lock() 90 | defer sl.mutex.Unlock() 91 | update := make([]*ListNode, MAXLEVEL) 92 | current := sl.head 93 | for i := sl.level - 1; i >= 0; i-- { 94 | for current.next[i] != nil && current.next[i].KV.Key < key { 95 | current = current.next[i] 96 | } 97 | update[i] = current 98 | } 99 | if current.next[0] != nil && current.next[0].KV.Key == key { 100 | current.next[0].KV.Delete = true 101 | return current.next[0].KV, true 102 | } 103 | level := sl.randomLevel() 104 | if level > sl.level { 105 | for i := sl.level; i < level; i++ { 106 | update[i] = sl.head 107 | } 108 | sl.level = level 109 | } 110 | newNode := NewNode(kv.Value{key, nil, true}, level) 111 | for i := sl.level; i < level; i++ { 112 | newNode.next[i] = update[i].next[i] 113 | update[i].next[i] = newNode 114 | } 115 | sl.count++ 116 | return kv.Value{}, false 117 | 118 | } 119 | func (sl *SkipList) GetValues() []kv.Value { 120 | sl.mutex.RLock() 121 | defer sl.mutex.RUnlock() 122 | var values []kv.Value 123 | current := sl.head.next[0] 124 | for current != nil { 125 | values = append(values, current.KV) 126 | current = current.next[0] 127 | } 128 | return values 129 | 130 | } 131 | func (sl *SkipList) Search(key string) (kv.Value, kv.SearchResult) { 132 | sl.mutex.RLock() // 加读锁保证并发安全 133 | defer sl.mutex.RUnlock() // 函数结束时释放锁 134 | 135 | // 从最高层开始搜索 136 | current := sl.head 137 | for i := sl.level - 1; i >= 0; i-- { 138 | // 在当前层向右搜索，直到找到大于或等于key的节点 139 | for current.next[i] != nil && current.next[i].KV.Key < key { 140 | current = current.next[i] 141 | } 142 | } 143 | 144 | // 移动到最底层的下一个节点（可能是目标节点） 145 | current = current.next[0] 146 | 147 | // 检查是否找到匹配的键 148 | if current != nil && current.KV.Key == key { 149 | // 找到匹配的键，返回值和成功状态 150 | return current.KV, kv.Success 151 | } 152 | 153 | // 未找到键，返回空值和未找到状态 154 | return kv.Value{}, kv.NotFind 155 | } 156 | func (sl *SkipList) Swap() orderTable.OrderInterface { 157 | sl.mutex.Lock() 158 | defer sl.mutex.Unlock() 159 | newSl := NewSkipList() 160 | newSl.head, sl.head = sl.head, newSl.head 161 | newSl.level, sl.level = sl.level, newSl.level 162 | newSl.mutex = sl.mutex 163 | sl.mutex = &sync.RWMutex{} 164 | sl.count = 0 165 | return newSl 166 | } 167 | -------------------------------------------------------------------------------- /orderTable/skipList/SkipList.go:Zone.Identifier: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whuanle/lsm/9309f47f33f6586c23edebe596aa2ac25349a5e1/orderTable/skipList/SkipList.go:Zone.Identifier -------------------------------------------------------------------------------- /orderTable/sortTree/SortTree.go: -------------------------------------------------------------------------------- 1 | package sortTree 2 | 3 | import ( 4 | "LSM/kv" 5 | "LSM/orderTable" 6 | "log" 7 | "sync" 8 | ) 9 | 10 | const ( 11 | NotFound = "KeyNotFound" 12 | ) 13 | 14 | type treeNode struct { 15 | KV kv.Value 16 | Left *treeNode 17 | Right *treeNode 18 | } 19 | type Tree struct { 20 | root *treeNode 21 | count int 22 | RWLock *sync.RWMutex 23 | } 24 | 25 | func (tree *Tree) Init() { 26 | tree.RWLock = &sync.RWMutex{} 27 | } 28 | func (tree *Tree) Count() int { 29 | return tree.count 30 | } 31 | func (tree *Tree) Search(key string) (kv.Value, kv.SearchResult) { 32 | tree.RWLock.RLock() 33 | defer tree.RWLock.RUnlock() 34 | 35 | if tree.root == nil { 36 | return kv.Value{}, kv.NotFind 37 | log.Fatal("The tree is nil") 38 | } 39 | currentNode := tree.root 40 | for currentNode != nil { 41 | if key == currentNode.KV.Key { 42 | if currentNode.KV.Delete == false { 43 | return currentNode.KV, kv.Success 44 | } 45 | return kv.Value{}, kv.Delete 46 | } 47 | if key < currentNode.KV.Key { 48 | currentNode = currentNode.Left 49 | } else { 50 | currentNode = currentNode.Right 51 | } 52 | } 53 | return kv.Value{}, kv.NotFind 54 | } 55 | func (tree *Tree) GetCount() int { 56 | return tree.count 57 | } 58 | 59 | // Set 设置 Key 的值并返回旧值 60 | func (tree *Tree) Set(key string, value []byte) (oldValue kv.Value, hasOld bool) { 61 | tree.RWLock.Lock() 62 | defer tree.RWLock.Unlock() 63 | currentNode := tree.root 64 | newNode := &treeNode{ 65 | KV: kv.Value{ 66 | Key: key, 67 | Value: value, 68 | }, 69 | } 70 | if currentNode == nil { 71 | tree.root = newNode 72 | tree.count++ 73 | return kv.Value{}, false 74 | } 75 | for currentNode != nil { 76 | if key == currentNode.KV.Key { 77 | oldKV := currentNode.KV.Copy() 78 | currentNode.KV.Value = value 79 | currentNode.KV.Delete = false 80 | if oldKV.Delete { 81 | return kv.Value{}, false 82 | } 83 | return *oldKV, true 84 | } 85 | if key < currentNode.KV.Key { 86 | if currentNode.Left == nil { 87 | currentNode.Left = newNode 88 | tree.count++ 89 | return kv.Value{}, false 90 | } 91 | currentNode = currentNode.Left 92 | } else { 93 | if currentNode.Right == nil { 94 | currentNode.Right = newNode 95 | tree.count++ 96 | return kv.Value{}, false 97 | } 98 | currentNode = currentNode.Right 99 | } 100 | } 101 | return kv.Value{}, false 102 | } 103 | func (tree *Tree) Delete(key string) (oldValue kv.Value, hasOld bool) { 104 | tree.RWLock.Lock() 105 | defer tree.RWLock.Unlock() 106 | currentNode := tree.root 107 | if currentNode == nil { 108 | return kv.Value{}, false 109 | } 110 | for currentNode != nil { 111 | if key == currentNode.KV.Key { 112 | if currentNode.KV.Delete { 113 | return kv.Value{}, false 114 | } 115 | oldKV := currentNode.KV.Copy() 116 | currentNode.KV.Value = nil 117 | currentNode.KV.Delete = true 118 | tree.count-- 119 | return *oldKV, true 120 | } 121 | if key < currentNode.KV.Key { 122 | currentNode = currentNode.Left 123 | } else { 124 | currentNode = currentNode.Right 125 | } 126 | } 127 | return kv.Value{}, false 128 | } 129 | func (tree *Tree) GetValues() []kv.Value { 130 | tree.RWLock.RLock() 131 | defer tree.RWLock.RUnlock() 132 | stack := InitStack(tree.count / 2) 133 | values := make([]kv.Value, 0) 134 | currentNode := tree.root 135 | for { 136 | if currentNode != nil { 137 | stack.Push(currentNode) 138 | currentNode = currentNode.Left 139 | } else { 140 | popNode, success := stack.Pop() 141 | if !success { 142 | break 143 | } 144 | values = append(values, popNode.KV) 145 | currentNode = popNode.Right 146 | } 147 | } 148 | return values 149 | } 150 | func (tree *Tree) Swap() orderTable.OrderInterface { 151 | tree.RWLock.Lock() 152 | defer tree.RWLock.Unlock() 153 | newTree := &Tree{} 154 | newTree.Init() 155 | newTree.root = tree.root 156 | tree.root = nil 157 | tree.count = 0 158 | return newTree 159 | } 160 | -------------------------------------------------------------------------------- /orderTable/sortTree/SortTree.go:Zone.Identifier: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whuanle/lsm/9309f47f33f6586c23edebe596aa2ac25349a5e1/orderTable/sortTree/SortTree.go:Zone.Identifier -------------------------------------------------------------------------------- /orderTable/sortTree/Stack.go: -------------------------------------------------------------------------------- 1 | package sortTree 2 | 3 | // Stack 顺序栈 4 | type Stack struct { 5 | stack []*treeNode 6 | base int // 栈底索引 7 | top int // 栈顶索引 8 | } 9 | 10 | // 简化的栈，不存在栈满的情况 11 | 12 | // InitStack 初始化栈 13 | func InitStack(n int) Stack { 14 | stack := Stack{ 15 | stack: make([]*treeNode, n), 16 | } 17 | return stack 18 | } 19 | 20 | // Push 入栈 21 | func (stack *Stack) Push(value *treeNode) { 22 | // 栈满 23 | if stack.top == len(stack.stack) { 24 | stack.stack = append(stack.stack, value) 25 | } else { 26 | stack.stack[stack.top] = value 27 | } 28 | stack.top++ 29 | } 30 | 31 | // Pop 出栈 32 | func (stack *Stack) Pop() (*treeNode, bool) { 33 | // 空栈 34 | if stack.top == stack.base { 35 | return nil, false 36 | } 37 | // 下退一个位置 38 | stack.top-- 39 | return stack.stack[stack.top], true 40 | } 41 | -------------------------------------------------------------------------------- /orderTable/sortTree/Stack.go:Zone.Identifier: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whuanle/lsm/9309f47f33f6586c23edebe596aa2ac25349a5e1/orderTable/sortTree/Stack.go:Zone.Identifier -------------------------------------------------------------------------------- /sortTree/SortTree.go: -------------------------------------------------------------------------------- 1 | package sortTree 2 | 3 | import ( 4 | "github.com/huiming23344/lsm/kv" 5 | "log" 6 | "sync" 7 | ) 8 | 9 | // treeNode 有序树节点 10 | type treeNode struct { 11 | KV kv.Value 12 | Left *treeNode 13 | Right *treeNode 14 | } 15 | 16 | // Tree 有序树 17 | type Tree struct { 18 | root *treeNode 19 | count int 20 | rWLock *sync.RWMutex 21 | } 22 | 23 | // Init 初始化树 24 | func (tree *Tree) Init() { 25 | tree.rWLock = &sync.RWMutex{} 26 | } 27 | 28 | // GetCount 获取树中的元素数量 29 | func (tree *Tree) GetCount() int { 30 | return tree.count 31 | } 32 | 33 | // Search 查找 Key 的值 34 | func (tree *Tree) Search(key string) (kv.Value, kv.SearchResult) { 35 | tree.rWLock.RLock() 36 | defer tree.rWLock.RUnlock() 37 | 38 | if tree == nil { 39 | log.Fatal("The tree is nil") 40 | } 41 | 42 | currentNode := tree.root 43 | // 有序查找 44 | for currentNode != nil { 45 | if key == currentNode.KV.Key { 46 | if currentNode.KV.Deleted == false { 47 | return currentNode.KV, kv.Success 48 | } else { 49 | return kv.Value{}, kv.Deleted 50 | } 51 | } 52 | if key < currentNode.KV.Key { 53 | // 继续对比下一层 54 | currentNode = currentNode.Left 55 | } else { 56 | // 继续对比下一层 57 | currentNode = currentNode.Right 58 | } 59 | } 60 | return kv.Value{}, kv.None 61 | } 62 | 63 | // Set 设置 Key 的值并返回旧值 64 | func (tree *Tree) Set(key string, value []byte) (oldValue kv.Value, hasOld bool) { 65 | tree.rWLock.Lock() 66 | defer tree.rWLock.Unlock() 67 | 68 | if tree == nil { 69 | log.Fatal("The tree is nil") 70 | } 71 | 72 | current := tree.root 73 | newNode := &treeNode{ 74 | KV: kv.Value{ 75 | Key: key, 76 | Value: value, 77 | }, 78 | } 79 | 80 | if current == nil { 81 | tree.root = newNode 82 | tree.count++ 83 | return kv.Value{}, false 84 | } 85 | 86 | for current != nil { 87 | // 如果已经存在键，则替换值 88 | if key == current.KV.Key { 89 | oldKV := current.KV.Copy() 90 | current.KV.Value = value 91 | current.KV.Deleted = false 92 | // 返回旧值 93 | if oldKV.Deleted { 94 | return kv.Value{}, false 95 | } else { 96 | return *oldKV, true 97 | } 98 | } 99 | // 要插入左边 100 | if key < current.KV.Key { 101 | // 左孩为空，直接插入左边 102 | if current.Left == nil { 103 | current.Left = newNode 104 | tree.count++ 105 | return kv.Value{}, false 106 | } 107 | // 继续对比下一层 108 | current = current.Left 109 | } else { 110 | // 右孩为空，直接插入右边 111 | if current.Right == nil { 112 | current.Right = newNode 113 | tree.count++ 114 | return kv.Value{}, false 115 | } 116 | // 继续对比下一层 117 | current = current.Right 118 | } 119 | } 120 | log.Fatalf("The tree fail to Set value, key: %s, value: %v", key, value) 121 | return kv.Value{}, false 122 | } 123 | 124 | // Delete 删除 key 并返回旧值 125 | func (tree *Tree) Delete(key string) (oldValue kv.Value, hasOld bool) { 126 | tree.rWLock.Lock() 127 | defer tree.rWLock.Unlock() 128 | 129 | if tree == nil { 130 | log.Fatal("The tree is nil") 131 | } 132 | 133 | newNode := &treeNode{ 134 | KV: kv.Value{ 135 | Key: key, 136 | Value: nil, 137 | Deleted: true, 138 | }, 139 | } 140 | 141 | currentNode := tree.root 142 | if currentNode == nil { 143 | tree.root = newNode 144 | return kv.Value{}, false 145 | } 146 | 147 | for currentNode != nil { 148 | if key == currentNode.KV.Key { 149 | // 存在且未被删除 150 | if currentNode.KV.Deleted == false { 151 | oldKV := currentNode.KV.Copy() 152 | currentNode.KV.Value = nil 153 | currentNode.KV.Deleted = true 154 | // count 应该是统计当前树中存在的有效节点，但是如果删除一个不存在的key，这个count会计算错误 155 | // 应该要在添加删除Node的时候count增加一下来保证count数量正确 156 | tree.count-- 157 | return *oldKV, true 158 | } else { // 已被删除过 159 | return kv.Value{}, false 160 | } 161 | } 162 | // 往下一层查找 163 | if key < currentNode.KV.Key { 164 | // 如果不存在此 key，则插入一个删除标记 165 | if currentNode.Left == nil { 166 | currentNode.Left = newNode 167 | tree.count++ 168 | } 169 | // 继续对比下一层 170 | currentNode = currentNode.Left 171 | } else { 172 | // 如果不存在此 key，则插入一个删除标记 173 | if currentNode.Right == nil { 174 | currentNode.Right = newNode 175 | tree.count++ 176 | } 177 | // 继续对比下一层 178 | currentNode = currentNode.Right 179 | } 180 | } 181 | log.Fatalf("The tree fail to delete key, key: %s", key) 182 | return kv.Value{}, false 183 | } 184 | 185 | // GetValues 获取树中的所有元素，这是一个有序元素列表 186 | func (tree *Tree) GetValues() []kv.Value { 187 | tree.rWLock.RLock() 188 | defer tree.rWLock.RUnlock() 189 | 190 | // 使用栈，而非递归，栈使用了切片，可以自动扩展大小，不必担心栈满 191 | stack := InitStack(tree.count / 2) 192 | values := make([]kv.Value, 0) 193 | 194 | // 从小到大获取树的元素 195 | currentNode := tree.root 196 | for { 197 | if currentNode != nil { 198 | stack.Push(currentNode) 199 | currentNode = currentNode.Left 200 | } else { 201 | popNode, success := stack.Pop() 202 | if success == false { 203 | break 204 | } 205 | values = append(values, popNode.KV) 206 | currentNode = popNode.Right 207 | } 208 | } 209 | return values 210 | } 211 | 212 | func (tree *Tree) Swap() *Tree { 213 | tree.rWLock.Lock() 214 | defer tree.rWLock.Unlock() 215 | 216 | newTree := &Tree{} 217 | newTree.Init() 218 | newTree.root = tree.root 219 | tree.root = nil 220 | tree.count = 0 221 | return newTree 222 | } 223 | -------------------------------------------------------------------------------- /sortTree/SortTree_test.go: -------------------------------------------------------------------------------- 1 | package sortTree 2 | 3 | import ( 4 | "github.com/huiming23344/lsm/kv" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func Test_SortTree_Insert(t *testing.T) { 10 | tree := &Tree{} 11 | tree.Init() 12 | _, hasOld := tree.Set("a", []byte{1, 2, 3}) 13 | if hasOld == true { 14 | t.Error(hasOld) 15 | } 16 | 17 | oldKV, hasOld := tree.Set("a", []byte{2, 3, 4}) 18 | if !hasOld { 19 | t.Error("fail to test the set function, the 'hasOld' should be true") 20 | } 21 | if !reflect.DeepEqual(oldKV.Value, []byte{1, 2, 3}) { 22 | t.Error("fail to test the set function, the 'oldKV' is invalid") 23 | } 24 | 25 | count := tree.GetCount() 26 | if count != 1 { 27 | t.Error(count) 28 | } 29 | 30 | tree.Set("b", []byte{1, 2, 3}) 31 | tree.Set("c", []byte{1, 2, 3}) 32 | 33 | count = tree.GetCount() 34 | if count != 3 { 35 | t.Error(count) 36 | } 37 | 38 | tree.Delete("a") 39 | tree.Delete("a") 40 | 41 | count = tree.GetCount() 42 | if count != 2 { 43 | t.Error(count) 44 | } 45 | 46 | data, success := tree.Search("a") 47 | if success != kv.Deleted { 48 | t.Error(success) 49 | } 50 | 51 | data, success = tree.Search("b") 52 | if success != kv.Success { 53 | t.Error(success) 54 | } 55 | 56 | if data.Value[0] != 1 { 57 | t.Error(data) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /sortTree/Stack.go: -------------------------------------------------------------------------------- 1 | package sortTree 2 | 3 | // Stack 顺序栈 4 | type Stack struct { 5 | stack []*treeNode 6 | base int // 栈底索引 7 | top int // 栈顶索引 8 | } 9 | 10 | // 简化的栈，不存在栈满的情况 11 | 12 | // InitStack 初始化栈 13 | func InitStack(n int) Stack { 14 | stack := Stack{ 15 | stack: make([]*treeNode, n), 16 | } 17 | return stack 18 | } 19 | 20 | // Push 入栈 21 | func (stack *Stack) Push(value *treeNode) { 22 | // 栈满 23 | if stack.top == len(stack.stack) { 24 | stack.stack = append(stack.stack, value) 25 | } else { 26 | stack.stack[stack.top] = value 27 | } 28 | stack.top++ 29 | } 30 | 31 | // Pop 出栈 32 | func (stack *Stack) Pop() (*treeNode, bool) { 33 | // 空栈 34 | if stack.top == stack.base { 35 | return nil, false 36 | } 37 | // 下退一个位置 38 | stack.top-- 39 | return stack.stack[stack.top], true 40 | } 41 | -------------------------------------------------------------------------------- /ssTable/Init.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "github.com/huiming23344/lsm/config" 5 | "io/ioutil" 6 | "log" 7 | "path" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var levelMaxSize []int 13 | 14 | // Init 初始化 TableTree 15 | func (tree *TableTree) Init(dir string) { 16 | log.Println("The SSTable list are being loaded") 17 | start := time.Now() 18 | defer func() { 19 | elapse := time.Since(start) 20 | log.Println("The SSTable list are being loaded,consumption of time : ", elapse) 21 | }() 22 | 23 | // 初始化每一层 SSTable 的文件总最大值 24 | con := config.GetConfig() 25 | levelMaxSize = make([]int, 10) 26 | levelMaxSize[0] = con.Level0Size 27 | levelMaxSize[1] = levelMaxSize[0] * 10 28 | levelMaxSize[2] = levelMaxSize[1] * 10 29 | levelMaxSize[3] = levelMaxSize[2] * 10 30 | levelMaxSize[4] = levelMaxSize[3] * 10 31 | levelMaxSize[5] = levelMaxSize[4] * 10 32 | levelMaxSize[6] = levelMaxSize[5] * 10 33 | levelMaxSize[7] = levelMaxSize[6] * 10 34 | levelMaxSize[8] = levelMaxSize[7] * 10 35 | levelMaxSize[9] = levelMaxSize[8] * 10 36 | 37 | tree.levels = make([]*tableNode, 10) 38 | tree.lock = &sync.RWMutex{} 39 | infos, err := ioutil.ReadDir(dir) 40 | if err != nil { 41 | log.Println("Failed to read the database file") 42 | panic(err) 43 | } 44 | for _, info := range infos { 45 | // 如果是 SSTable 文件 46 | if path.Ext(info.Name()) == ".db" { 47 | tree.loadDbFile(path.Join(dir, info.Name())) 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /ssTable/MetaInfo.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | /* 4 | 5 | 索引是从数据区开始！ 6 | 0 ─────────────────────────────────────────────────────────► 7 | ◄─────────────────────────── 8 | dataLen ◄────────────────── 9 | indexLen ◄──────────────┐ 10 | ┌──────────────────────────┬─────────────────┬──────────────┤ 11 | │ │ │ │ 12 | │ 数据区 │ 稀疏索引区 │ 元数据 │ 13 | │ │ │ │ 14 | └──────────────────────────┴─────────────────┴──────────────┘ 15 | */ 16 | 17 | // MetaInfo 是 SSTable 的元数据， 18 | // 元数据出现在磁盘文件的末尾 19 | type MetaInfo struct { 20 | // 版本号 21 | version int64 22 | // 数据区起始索引 23 | dataStart int64 24 | // 数据区长度 25 | dataLen int64 26 | // 稀疏索引区起始索引 27 | indexStart int64 28 | // 稀疏索引区长度 29 | indexLen int64 30 | } 31 | -------------------------------------------------------------------------------- /ssTable/Position.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | // Position 元素定位，存储在稀疏索引区中，表示一个元素的起始位置和长度 4 | type Position struct { 5 | // 起始索引 6 | Start int64 7 | // 长度 8 | Len int64 9 | // Key 已经被删除 10 | Deleted bool 11 | } 12 | -------------------------------------------------------------------------------- /ssTable/Search.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "github.com/huiming23344/lsm/kv" 5 | "log" 6 | ) 7 | 8 | // Search 查找元素， 9 | // 先使用二分查找法从内存中的 keys 列表查找 Key，如果存在，找到 Position ，再通过从数据区加载 10 | func (table *SSTable) Search(key string) (value kv.Value, result kv.SearchResult) { 11 | table.lock.Lock() 12 | defer table.lock.Unlock() 13 | 14 | // 元素定位 15 | var position = Position{ 16 | Start: -1, 17 | } 18 | l := 0 19 | r := len(table.sortIndex) - 1 20 | 21 | // 二分查找法，查找 key 是否存在 22 | for l <= r { 23 | mid := (l + r) / 2 24 | if table.sortIndex[mid] == key { 25 | // 获取元素定位 26 | position = table.sparseIndex[key] 27 | // 如果元素已被删除，则返回 28 | if position.Deleted { 29 | return kv.Value{}, kv.Deleted 30 | } 31 | break 32 | } else if table.sortIndex[mid] < key { 33 | l = mid + 1 34 | } else if table.sortIndex[mid] > key { 35 | r = mid - 1 36 | } 37 | } 38 | 39 | if position.Start == -1 { 40 | return kv.Value{}, kv.None 41 | } 42 | 43 | // Todo：如果读取失败，需要增加错误处理过程 44 | // 从磁盘文件中查找 45 | bytes := make([]byte, position.Len) 46 | if _, err := table.f.Seek(position.Start, 0); err != nil { 47 | log.Println(err) 48 | return kv.Value{}, kv.None 49 | } 50 | if _, err := table.f.Read(bytes); err != nil { 51 | log.Println(err) 52 | return kv.Value{}, kv.None 53 | } 54 | 55 | value, err := kv.Decode(bytes) 56 | if err != nil { 57 | log.Println(err) 58 | return kv.Value{}, kv.None 59 | } 60 | return value, kv.Success 61 | } 62 | -------------------------------------------------------------------------------- /ssTable/SsTable.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "LSM/kv" 5 | "log" 6 | "os" 7 | "sync" 8 | ) 9 | 10 | type SSTable struct { 11 | f *os.File 12 | filePath string 13 | //元数据 14 | tableMetaInfo MetaInfo 15 | //文件稀疏索引 16 | sparseIndex map[string]Position 17 | //排序后的key列表 18 | sortIndex []string 19 | lock sync.Locker 20 | } 21 | 22 | func (table *SSTable) Init(filePath string) { 23 | table.filePath = filePath 24 | table.lock = &sync.Mutex{} 25 | table.loadFileHandle() 26 | } 27 | func (table *SSTable) Search(key string) (value kv.Value, result kv.SearchResult) { 28 | table.lock.Lock() 29 | defer table.lock.Unlock() 30 | var position = Position{ 31 | Start: -1, 32 | } 33 | l := 0 34 | r := len(table.sortIndex) - 1 35 | for l <= r { 36 | mid := (l + r) / 2 37 | if table.sortIndex[mid] == key { 38 | position = table.sparseIndex[key] 39 | if position.Deleted { 40 | return kv.Value{}, kv.Delete 41 | } 42 | break 43 | } else if table.sortIndex[mid] < key { 44 | l = mid + 1 45 | } else { 46 | r = mid - 1 47 | } 48 | } 49 | if position.Start == -1 { 50 | return kv.Value{}, kv.NotFind 51 | } 52 | bytes := make([]byte, position.Len) 53 | if _, err := table.f.Seek(position.Start, 0); err != nil { 54 | log.Println(err) 55 | return kv.Value{}, kv.NotFind 56 | } 57 | if _, err := table.f.Read(bytes); err != nil { 58 | log.Println(err) 59 | return kv.Value{}, kv.NotFind 60 | } 61 | value, err := kv.Decode(bytes) 62 | if err != nil { 63 | log.Println(err) 64 | return kv.Value{}, kv.NotFind 65 | } 66 | return value, kv.Success 67 | } 68 | -------------------------------------------------------------------------------- /ssTable/TableTree.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "LSM/kv" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | type TableTree struct { 10 | levels []*tableNode 11 | lock *sync.RWMutex 12 | levellocks []*sync.RWMutex 13 | } 14 | 15 | type tableNode struct { 16 | index int 17 | table *SSTable 18 | next *tableNode 19 | tableNodeL string 20 | tableNodeR string 21 | needToDelete bool 22 | } 23 | 24 | func (tree *TableTree) Search(key string) (kv.Value, kv.SearchResult) { 25 | tree.lock.RLock() 26 | defer tree.lock.RUnlock() 27 | for level, node := range tree.levels { 28 | tables := make([]*tableNode, 0) 29 | for node != nil { 30 | tables = append(tables, node) 31 | node = node.next 32 | } 33 | if level == 0 { 34 | for i := len(tables) - 1; i >= 0; i-- { 35 | value, searchResult := tables[i].table.Search(key) 36 | if searchResult == kv.NotFind { 37 | continue 38 | } 39 | return value, searchResult 40 | } 41 | } else { 42 | for i := 0; i < len(tables); i++ { 43 | if tables[i].tableNodeL > key || tables[i].tableNodeR < key { 44 | continue 45 | } 46 | value, searchResult := tables[i].table.Search(key) 47 | if searchResult == kv.NotFind { 48 | continue 49 | } 50 | return value, searchResult 51 | } 52 | } 53 | } 54 | return kv.Value{}, kv.NotFind 55 | } 56 | func (tree *TableTree) Insert(table *SSTable, level int) (index int) { 57 | node := tree.levels[level] 58 | newNode := &tableNode{ 59 | table: table, 60 | next: nil, 61 | index: 0, 62 | tableNodeL: table.sortIndex[0], 63 | tableNodeR: table.sortIndex[len(table.sortIndex)-1], 64 | needToDelete: false, 65 | } 66 | if node == nil { 67 | tree.levels[level] = newNode 68 | } else { 69 | for node != nil { 70 | if node.next == nil { 71 | node.next = newNode 72 | newNode.index = node.index + 1 73 | break 74 | } 75 | node = node.next 76 | } 77 | } 78 | return newNode.index 79 | } 80 | func (tree *TableTree) getMaxIndex(level int) int { 81 | node := tree.levels[level] 82 | index := 0 83 | for node != nil { 84 | index = node.index 85 | node = node.next 86 | } 87 | return index 88 | } 89 | func (tree *TableTree) getLevelCount(level int) int { 90 | node := tree.levels[level] 91 | count := 0 92 | for node != nil { 93 | count++ 94 | node = node.next 95 | } 96 | return count 97 | } 98 | func getLevel(name string) (level int, index int, err error) { 99 | n, err := fmt.Sscanf(name, "%d.%d.db", &level, &index) 100 | if n != 2 && err != nil { 101 | return 0, 0, fmt.Errorf("incorrect data file name: %q", name) 102 | } 103 | return level, index, nil 104 | } 105 | -------------------------------------------------------------------------------- /ssTable/TableTreeInit.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "LSM/config" 5 | "io/ioutil" 6 | "log" 7 | "path" 8 | "sync" 9 | "time" 10 | ) 11 | 12 | var levelMaxSize []int 13 | 14 | func (tree *TableTree) Init(dir string) { 15 | log.Println("The SSTable list are being loaded") 16 | start := time.Now() 17 | defer func() { 18 | elapse := time.Since(start) 19 | log.Println("The SSTable list are being loaded,consumption of time : ", elapse) 20 | }() 21 | con := config.GetConfig() 22 | levelMaxSize = make([]int, 10) 23 | levelMaxSize[0] = con.Level0Size 24 | levelMaxSize[1] = con.Level0Size * 10 25 | levelMaxSize[2] = levelMaxSize[1] * 10 26 | levelMaxSize[3] = levelMaxSize[2] * 10 27 | levelMaxSize[4] = levelMaxSize[3] * 10 28 | levelMaxSize[5] = levelMaxSize[4] * 10 29 | levelMaxSize[6] = levelMaxSize[5] * 10 30 | levelMaxSize[7] = levelMaxSize[6] * 10 31 | levelMaxSize[8] = levelMaxSize[7] * 10 32 | levelMaxSize[9] = levelMaxSize[8] * 10 33 | tree.levels = make([]*tableNode, 10) 34 | tree.lock = &sync.RWMutex{} 35 | tree.levellocks = make([]*sync.RWMutex, 10) 36 | for i := 0; i < 10; i++ { 37 | tree.levellocks[i] = &sync.RWMutex{} 38 | } 39 | infos, err := ioutil.ReadDir(dir) 40 | if err != nil { 41 | log.Println("Failed to read the database file") 42 | panic(err) 43 | } 44 | for _, info := range infos { 45 | if path.Ext(info.Name()) == ".db" { 46 | tree.loadDbFile(path.Join(dir, info.Name())) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /ssTable/compaction.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "LSM/config" 5 | "LSM/kv" 6 | "LSM/orderTable/skipList" 7 | "log" 8 | "math" 9 | "os" 10 | "sort" 11 | "strconv" 12 | "time" 13 | ) 14 | 15 | func (tree *TableTree) Check() { 16 | tree.majorCompaction() 17 | } 18 | func (tree *TableTree) majorCompaction() { 19 | con := config.GetConfig() 20 | for levelIndex, _ := range tree.levels { 21 | //fmt.Println("-------cmp------", tree.getLevelCount(levelIndex), con.PartSize*int(math.Pow(10, float64(levelIndex)))) 22 | if tree.getLevelCount(levelIndex) > con.PartSize*int(math.Pow(10, float64(levelIndex))) { 23 | tree.majorCompactionLevel(levelIndex) 24 | } 25 | } 26 | } 27 | func GetTableValues(table *SSTable) []kv.Value { 28 | tableCache := make([]byte, 100000) 29 | memoryList := skipList.SkipList{} 30 | memoryList.Init() 31 | if int64(len(tableCache)) < table.tableMetaInfo.dataLen { 32 | tableCache = make([]byte, table.tableMetaInfo.dataLen) 33 | } 34 | newSlice := tableCache[0:table.tableMetaInfo.dataLen] 35 | if _, err := table.f.Seek(0, 0); err != nil { 36 | log.Println("error open file", table.filePath) 37 | panic(err) 38 | } 39 | if _, err := table.f.Read(newSlice); err != nil { 40 | log.Println("error read file", table.filePath) 41 | panic(err) 42 | } 43 | for k, position := range table.sparseIndex { 44 | if position.Deleted == false { 45 | value, err := kv.Decode(newSlice[position.Start:(position.Start + position.Len)]) 46 | if err != nil { 47 | log.Fatal(err) 48 | } 49 | memoryList.Set(k, value.Value) 50 | } else { 51 | memoryList.Delete(k) 52 | } 53 | } 54 | return memoryList.GetValues() 55 | } 56 | func (tree *TableTree) majorCompactionLevel(level int) { 57 | log.Println("Compressing layer ", level, " files") 58 | start := time.Now() 59 | defer func() { 60 | elapse := time.Since(start) 61 | log.Println("Completed compression,consumption of time : ", elapse) 62 | }() 63 | con := config.GetConfig() 64 | if level == 0 { 65 | tree.levellocks[0].Lock() 66 | tree.levellocks[1].Lock() 67 | tableCache := make([]byte, levelMaxSize[level]) 68 | currentNode := tree.levels[level] 69 | memoryList := skipList.SkipList{} 70 | memoryList.Init() 71 | for currentNode != nil { 72 | table := currentNode.table 73 | if int64(len(tableCache)) < table.tableMetaInfo.dataLen { 74 | tableCache = make([]byte, table.tableMetaInfo.dataLen) 75 | } 76 | newSlice := tableCache[0:table.tableMetaInfo.dataLen] 77 | if _, err := table.f.Seek(0, 0); err != nil { 78 | log.Println("error open file", table.filePath) 79 | panic(err) 80 | } 81 | if _, err := table.f.Read(newSlice); err != nil { 82 | log.Println("error read file", table.filePath) 83 | panic(err) 84 | } 85 | for k, position := range table.sparseIndex { 86 | if position.Deleted == false { 87 | value, err := kv.Decode(newSlice[position.Start:(position.Start + position.Len)]) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | memoryList.Set(k, value.Value) 92 | } else { 93 | memoryList.Delete(k) 94 | } 95 | } 96 | currentNode.needToDelete = true 97 | currentNode = currentNode.next 98 | } //已经将第一层的数据放进跳表里 99 | memoryValue := memoryList.GetValues() 100 | //log.Println("-----------the last", memoryValue[len(memoryValue)-1].Key) 101 | nowMemoryR := memoryValue[len(memoryValue)-1].Key 102 | tableCache = make([]byte, levelMaxSize[level+1]) 103 | currentNode = tree.levels[level+1] 104 | nowPtr := 0 105 | mergeToList := skipList.SkipList{} 106 | mergeToList.Init() 107 | mergeToSize := 0 108 | var mergeValue []kv.Value 109 | for currentNode != nil || nowPtr < len(memoryValue) { 110 | if currentNode != nil { 111 | if currentNode.tableNodeL > nowMemoryR || currentNode.tableNodeR < memoryValue[nowPtr].Key { 112 | currentNode = currentNode.next 113 | continue 114 | } 115 | } 116 | if currentNode != nil { 117 | mergeValue = GetTableValues(currentNode.table) 118 | } else { 119 | break 120 | mergeValue = make([]kv.Value, 0) 121 | } 122 | for i := 0; i < len(mergeValue); { 123 | value := mergeValue[i] 124 | if nowPtr == len(memoryValue) { 125 | if !value.Delete { 126 | mergeToList.Set(value.Key, value.Value) 127 | } else { 128 | mergeToList.Delete(value.Key) 129 | } 130 | i++ 131 | } else if memoryValue[nowPtr].Key < value.Key { 132 | if !memoryValue[nowPtr].Delete { 133 | mergeToList.Set(memoryValue[nowPtr].Key, memoryValue[nowPtr].Value) 134 | } else { 135 | mergeToList.Delete(memoryValue[nowPtr].Key) 136 | } 137 | nowPtr++ 138 | } else if memoryValue[nowPtr].Key > value.Key { 139 | if !value.Delete { 140 | mergeToList.Set(value.Key, value.Value) 141 | } else { 142 | mergeToList.Delete(value.Key) 143 | } 144 | i++ 145 | } else { 146 | if !memoryValue[nowPtr].Delete { 147 | mergeToList.Set(value.Key, value.Value) 148 | } else { 149 | mergeToList.Delete(value.Key) 150 | } 151 | i++ 152 | nowPtr++ 153 | } 154 | mergeToSize++ 155 | if mergeToSize >= con.Threshold { 156 | tree.creatTable(mergeToList.Swap().GetValues(), level+1) 157 | mergeToSize = 0 158 | } 159 | } 160 | currentNode.needToDelete = true 161 | currentNode = currentNode.next 162 | } 163 | for nowPtr < len(memoryValue) { 164 | if !memoryValue[nowPtr].Delete { 165 | mergeToList.Set(memoryValue[nowPtr].Key, memoryValue[nowPtr].Value) 166 | } else { 167 | mergeToList.Delete(memoryValue[nowPtr].Key) 168 | } 169 | nowPtr++ 170 | mergeToSize++ 171 | if mergeToSize >= con.Threshold { 172 | //fmt.Println(mergeToSize, "\n", "\n") 173 | tree.creatTable(mergeToList.Swap().GetValues(), level+1) 174 | mergeToSize = 0 175 | } 176 | } 177 | if mergeToSize != 0 { 178 | tree.creatTable(mergeToList.Swap().GetValues(), level+1) 179 | mergeToSize = 0 180 | } 181 | tree.clearLevel(0) 182 | tree.levellocks[0].Unlock() 183 | tree.levellocks[1].Unlock() 184 | } else { 185 | tree.levellocks[level].Lock() 186 | tree.levellocks[level+1].Lock() 187 | upCurrentNode := tree.levels[level] 188 | downCurrentNode := tree.levels[level+1] 189 | mergeToList := skipList.SkipList{} 190 | mergeToList.Init() 191 | mergeToSize := 0 192 | tmpNode := tree.levels[level] 193 | upL := upCurrentNode.tableNodeL 194 | for tmpNode.next != nil { 195 | tmpNode = tmpNode.next 196 | } 197 | upR := tmpNode.tableNodeR 198 | var upValues []kv.Value 199 | var downValues []kv.Value 200 | for downCurrentNode != nil { 201 | if downCurrentNode.tableNodeR > upL || downCurrentNode.next == nil { 202 | break 203 | } 204 | downCurrentNode = downCurrentNode.next 205 | } 206 | upPtr := 0 207 | downPtr := 0 208 | upValues = GetTableValues(upCurrentNode.table) 209 | if downCurrentNode != nil { 210 | downValues = GetTableValues(downCurrentNode.table) 211 | } 212 | for upCurrentNode != nil || downCurrentNode != nil { 213 | if upPtr == len(upValues) { 214 | upCurrentNode.needToDelete = true 215 | upCurrentNode = upCurrentNode.next 216 | if upCurrentNode != nil { 217 | upPtr = 0 218 | upValues = GetTableValues(upCurrentNode.table) 219 | } else { 220 | upPtr = -1 221 | } 222 | } 223 | if downPtr == len(downValues) { 224 | if downCurrentNode != nil { 225 | downCurrentNode.needToDelete = true 226 | downCurrentNode = downCurrentNode.next 227 | if downCurrentNode != nil || downCurrentNode.tableNodeL <= upR { 228 | downPtr = 0 229 | downValues = GetTableValues(downCurrentNode.table) 230 | } else { 231 | downPtr = -1 232 | } 233 | } else { 234 | downPtr = -1 235 | } 236 | 237 | } 238 | if upPtr == -1 && downPtr == -1 { 239 | break 240 | } 241 | if downPtr == -1 || upValues[upPtr].Key == downValues[downPtr].Key || upValues[upPtr].Key < downValues[downPtr].Key { 242 | if !upValues[upPtr].Delete { 243 | mergeToList.Set(upValues[upPtr].Key, upValues[upPtr].Value) 244 | } else { 245 | mergeToList.Delete(upValues[upPtr].Key) 246 | } 247 | mergeToSize++ 248 | upPtr++ 249 | } else if upPtr == -1 || upValues[upPtr].Key > downValues[downPtr].Key { 250 | if !upValues[upPtr].Delete { 251 | mergeToList.Set(downValues[downPtr].Key, downValues[downPtr].Value) 252 | } else { 253 | mergeToList.Delete(downValues[downPtr].Key) 254 | } 255 | mergeToSize++ 256 | downPtr++ 257 | } 258 | if mergeToSize >= con.Threshold { 259 | tree.creatTable(mergeToList.Swap().GetValues(), level+1) 260 | mergeToSize = 0 261 | } 262 | } 263 | if mergeToSize != 0 { 264 | tree.creatTable(mergeToList.Swap().GetValues(), level+1) 265 | mergeToSize = 0 266 | } 267 | tree.clearLevel(level) 268 | tree.clearLevel(level + 1) 269 | tree.levellocks[level].Unlock() 270 | tree.levellocks[level+1].Unlock() 271 | } 272 | //tree.levels[level] = nil 273 | } 274 | 275 | func (tree *TableTree) clearLevel(level int) { 276 | log.Println("Clear level", level) 277 | currentNode := tree.levels[level] 278 | tmpNodes := make([]*tableNode, 0) 279 | for currentNode != nil { 280 | if currentNode.needToDelete == false { 281 | tmpNodes = append(tmpNodes, currentNode) 282 | } else { 283 | err := currentNode.table.f.Close() 284 | if err != nil { 285 | log.Println("error close file,", err) 286 | panic(err) 287 | } 288 | err = os.Remove(currentNode.table.filePath) 289 | if err != nil { 290 | log.Println("error remove file,", currentNode.table.filePath, err) 291 | panic(err) 292 | } 293 | currentNode.table.f = nil 294 | currentNode.table = nil 295 | } 296 | currentNode = currentNode.next 297 | } 298 | sort.Slice(tmpNodes, func(i, j int) bool { 299 | return tmpNodes[i].tableNodeL < tmpNodes[j].tableNodeL 300 | }) 301 | if len(tmpNodes) == 0 { 302 | tree.levels[level] = nil 303 | return 304 | } 305 | for i := 1; i < len(tmpNodes); i++ { 306 | tmpNodes[i-1].next = tmpNodes[i] 307 | } 308 | tmpNodes[len(tmpNodes)-1].next = nil 309 | tree.levels[level] = tmpNodes[0] 310 | for i := 0; i < len(tmpNodes); i++ { 311 | os.Rename(tmpNodes[i].table.filePath, config.Config{}.DataDir+"/"+strconv.Itoa(level)+"."+strconv.Itoa(i)+".db") 312 | } 313 | log.Println("Cleared level leave", tree.getLevelCount(level)) 314 | return 315 | } 316 | -------------------------------------------------------------------------------- /ssTable/creatTable.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "LSM/config" 5 | "LSM/kv" 6 | "encoding/json" 7 | "log" 8 | "os" 9 | "sort" 10 | "strconv" 11 | "sync" 12 | ) 13 | 14 | func (tree *TableTree) CreatNewTable(values []kv.Value) { 15 | tree.levellocks[0].Lock() 16 | tree.creatTable(values, 0) 17 | tree.levellocks[0].Unlock() 18 | } 19 | func (tree *TableTree) creatTable(values []kv.Value, level int) *SSTable { 20 | //println("creatTable---------------\n", level) 21 | keys := make([]string, 0, len(values)) 22 | positions := make(map[string]Position) 23 | dataArea := make([]byte, 0) 24 | for _, value := range values { 25 | data, err := kv.Encode(value) 26 | if err != nil { 27 | log.Println("Failed to insert Key: ", value.Key, err) 28 | continue 29 | } 30 | keys = append(keys, value.Key) 31 | positions[value.Key] = Position{ 32 | Start: int64(len(dataArea)), 33 | Len: int64(len(data)), 34 | Deleted: value.Delete, 35 | } 36 | dataArea = append(dataArea, data...) 37 | } 38 | sort.Strings(keys) 39 | 40 | indexArea, err := json.Marshal(positions) 41 | if err != nil { 42 | log.Fatal("An SSTable file cannot be created,", err) 43 | } 44 | meta := MetaInfo{ 45 | version: 0, 46 | dataStart: 0, 47 | dataLen: int64(len(dataArea)), 48 | indexStart: int64(len(dataArea)), 49 | indexLen: int64(len(indexArea)), 50 | } 51 | table := &SSTable{ 52 | tableMetaInfo: meta, 53 | sparseIndex: positions, 54 | sortIndex: keys, 55 | lock: &sync.RWMutex{}, 56 | } 57 | index := tree.Insert(table, level) 58 | //log.Printf("Create a new SSTable,level: %d ,index: %d\r\n", level, index) 59 | con := config.GetConfig() 60 | filePath := con.DataDir + "/" + strconv.Itoa(level) + "." + strconv.Itoa(index) + ".db" 61 | table.filePath = filePath 62 | writeDataToFile(filePath, dataArea, indexArea, meta) 63 | f, err := os.OpenFile(table.filePath, os.O_RDWR|os.O_CREATE, 0666) 64 | if err != nil { 65 | log.Fatal("Failed to open new SSTable,", err) 66 | panic(err) 67 | } 68 | table.f = f 69 | return table 70 | } 71 | -------------------------------------------------------------------------------- /ssTable/createTable.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "encoding/json" 5 | "github.com/huiming23344/lsm/config" 6 | "github.com/huiming23344/lsm/kv" 7 | "log" 8 | "os" 9 | "sort" 10 | "strconv" 11 | "sync" 12 | ) 13 | 14 | // CreateNewTable 创建新的 SSTable 15 | func (tree *TableTree) CreateNewTable(values []kv.Value) { 16 | tree.createTable(values, 0) 17 | } 18 | 19 | // 创建新的 SSTable，插入到合适的层 20 | func (tree *TableTree) createTable(values []kv.Value, level int) *SSTable { 21 | // 生成数据区 22 | keys := make([]string, 0, len(values)) 23 | positions := make(map[string]Position) 24 | dataArea := make([]byte, 0) 25 | for _, value := range values { 26 | data, err := kv.Encode(value) 27 | if err != nil { 28 | log.Println("Failed to insert Key: ", value.Key, err) 29 | continue 30 | } 31 | keys = append(keys, value.Key) 32 | // 文件定位记录 33 | positions[value.Key] = Position{ 34 | Start: int64(len(dataArea)), 35 | Len: int64(len(data)), 36 | Deleted: value.Deleted, 37 | } 38 | dataArea = append(dataArea, data...) 39 | } 40 | sort.Strings(keys) 41 | 42 | // 生成稀疏索引区 43 | // map[string]Position to json 44 | indexArea, err := json.Marshal(positions) 45 | if err != nil { 46 | log.Fatal("An SSTable file cannot be created,", err) 47 | } 48 | 49 | // 生成 MetaInfo 50 | meta := MetaInfo{ 51 | version: 0, 52 | dataStart: 0, 53 | dataLen: int64(len(dataArea)), 54 | indexStart: int64(len(dataArea)), 55 | indexLen: int64(len(indexArea)), 56 | } 57 | 58 | table := &SSTable{ 59 | tableMetaInfo: meta, 60 | sparseIndex: positions, 61 | sortIndex: keys, 62 | lock: &sync.RWMutex{}, 63 | } 64 | index := tree.insert(table, level) 65 | log.Printf("Create a new SSTable,level: %d ,index: %d\r\n", level, index) 66 | con := config.GetConfig() 67 | filePath := con.DataDir + "/" + strconv.Itoa(level) + "." + strconv.Itoa(index) + ".db" 68 | table.filePath = filePath 69 | 70 | writeDataToFile(filePath, dataArea, indexArea, meta) 71 | // 以只读的形式打开文件 72 | f, err := os.OpenFile(table.filePath, os.O_RDONLY, 0666) 73 | if err != nil { 74 | log.Println(" error open file ", table.filePath) 75 | panic(err) 76 | } 77 | table.f = f 78 | 79 | return table 80 | } 81 | -------------------------------------------------------------------------------- /ssTable/dbFile.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "encoding/binary" 5 | "log" 6 | "os" 7 | ) 8 | 9 | func (table *SSTable) GetDbSize() int64 { 10 | info, err := os.Stat(table.filePath) 11 | if err != nil { 12 | log.Println(err) 13 | panic(err) 14 | } 15 | return info.Size() 16 | } 17 | 18 | func (tree *TableTree) GetLevelSize(level int) int64 { 19 | var add int64 20 | add = 0 21 | currentNode := tree.levels[level] 22 | for currentNode != nil { 23 | add += currentNode.table.GetDbSize() 24 | currentNode = currentNode.next 25 | } 26 | return add 27 | } 28 | func writeDataToFile(filename string, dataArea []byte, indexArea []byte, meta MetaInfo) { 29 | f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0666) 30 | if err != nil { 31 | log.Fatal(" error create file,", err) 32 | } 33 | _, err = f.Write(dataArea) 34 | if err != nil { 35 | log.Fatal(" error write file,", err) 36 | } 37 | _, err = f.Write(indexArea) 38 | if err != nil { 39 | log.Fatal(" error write file,", err) 40 | } 41 | // 写入元数据到文件末尾 42 | // 注意，右侧必须能够识别字节长度的类型，不能使用 int 这种类型，只能使用 int32、int64 等 43 | _ = binary.Write(f, binary.LittleEndian, &meta.version) 44 | _ = binary.Write(f, binary.LittleEndian, &meta.dataStart) 45 | _ = binary.Write(f, binary.LittleEndian, &meta.dataLen) 46 | _ = binary.Write(f, binary.LittleEndian, &meta.indexStart) 47 | _ = binary.Write(f, binary.LittleEndian, &meta.indexLen) 48 | err = f.Sync() 49 | if err != nil { 50 | log.Fatal(" error write file,", err) 51 | } 52 | err = f.Close() 53 | if err != nil { 54 | log.Fatal(" error close file,", err) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /ssTable/initload.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "sort" 10 | "time" 11 | ) 12 | 13 | func (tree *TableTree) loadDbFile(path string) { 14 | log.Println("Loading the ", path) 15 | start := time.Now() 16 | defer func() { 17 | elapse := time.Since(start) 18 | log.Println("Loading the ", path, ",Consumption of time : ", elapse) 19 | }() 20 | level, index, err := getLevel(filepath.Base(path)) 21 | if err != nil { 22 | return 23 | } 24 | table := &SSTable{} 25 | table.Init(path) 26 | newNode := &tableNode{ 27 | index: index, 28 | table: table, 29 | } 30 | currentNode := tree.levels[level] 31 | if currentNode == nil { 32 | tree.levels[level] = newNode 33 | newNode.next = nil 34 | return 35 | } 36 | if currentNode.index > newNode.index { 37 | newNode.next = currentNode 38 | tree.levels[level] = newNode 39 | return 40 | } 41 | for currentNode.next != nil { 42 | if currentNode.next == nil || newNode.index < currentNode.next.index { 43 | newNode.next = currentNode.next 44 | currentNode.next = newNode 45 | break 46 | } else { 47 | currentNode = currentNode.next 48 | } 49 | } 50 | } 51 | func (table *SSTable) loadFileHandle() { 52 | if table.f == nil { 53 | f, err := os.OpenFile(table.filePath, os.O_RDONLY, 0666) 54 | if err != nil { 55 | log.Println("Error opening file ", table.filePath) 56 | } 57 | table.f = f 58 | } 59 | table.loadMetaInfo() 60 | table.loadSparseIndex() 61 | } 62 | func (table *SSTable) loadMetaInfo() { 63 | f := table.f 64 | _, err := f.Seek(0, 0) 65 | if err != nil { 66 | log.Println("Error seeking file ", table.filePath) 67 | panic(err) 68 | } 69 | info, _ := f.Stat() 70 | _, err = f.Seek(info.Size()-8*5, 0) 71 | if err != nil { 72 | log.Println("Error reading metadata ", table.filePath) 73 | panic(err) 74 | } 75 | _ = binary.Read(f, binary.LittleEndian, &table.tableMetaInfo.version) 76 | _, err = f.Seek(info.Size()-8*4, 0) 77 | if err != nil { 78 | log.Println("Error reading metadata ", table.filePath) 79 | panic(err) 80 | } 81 | _ = binary.Read(f, binary.LittleEndian, &table.tableMetaInfo.dataStart) 82 | 83 | _, err = f.Seek(info.Size()-8*3, 0) 84 | if err != nil { 85 | log.Println("Error reading metadata ", table.filePath) 86 | panic(err) 87 | } 88 | _ = binary.Read(f, binary.LittleEndian, &table.tableMetaInfo.dataLen) 89 | 90 | _, err = f.Seek(info.Size()-8*2, 0) 91 | if err != nil { 92 | log.Println("Error reading metadata ", table.filePath) 93 | panic(err) 94 | } 95 | _ = binary.Read(f, binary.LittleEndian, &table.tableMetaInfo.indexStart) 96 | 97 | _, err = f.Seek(info.Size()-8*1, 0) 98 | if err != nil { 99 | log.Println("Error reading metadata ", table.filePath) 100 | panic(err) 101 | } 102 | _ = binary.Read(f, binary.LittleEndian, &table.tableMetaInfo.indexLen) 103 | } 104 | func (table *SSTable) loadSparseIndex() { 105 | bytes := make([]byte, table.tableMetaInfo.indexLen) 106 | if _, err := table.f.Seek(table.tableMetaInfo.indexStart, 0); err != nil { 107 | log.Println(" error open file ", table.filePath) 108 | panic(err) 109 | } 110 | if _, err := table.f.Read(bytes); err != nil { 111 | log.Println("error open file ", table.filePath) 112 | panic(err) 113 | } 114 | table.sparseIndex = make(map[string]Position) 115 | err := json.Unmarshal(bytes, &table.sparseIndex) 116 | if err != nil { 117 | log.Println(" error open file ", table.filePath) 118 | panic(err) 119 | } 120 | _, _ = table.f.Seek(0, 0) 121 | keys := make([]string, 0, len(table.sparseIndex)) 122 | for k := range table.sparseIndex { 123 | keys = append(keys, k) 124 | } 125 | sort.Strings(keys) 126 | table.sortIndex = keys 127 | } 128 | -------------------------------------------------------------------------------- /ssTable/tree.go: -------------------------------------------------------------------------------- 1 | package ssTable 2 | 3 | // 插入一个 SSTable 到指定层 4 | func (tree *TableTree) insert(table *SSTable, level int) (index int) { 5 | tree.lock.Lock() 6 | defer tree.lock.Unlock() 7 | 8 | // 每次插入的，都出现在最后面 9 | node := tree.levels[level] 10 | newNode := &tableNode{ 11 | table: table, 12 | next: nil, 13 | index: 0, 14 | } 15 | 16 | if node == nil { 17 | tree.levels[level] = newNode 18 | } else { 19 | for node != nil { 20 | if node.next == nil { 21 | newNode.index = node.index + 1 22 | node.next = newNode 23 | break 24 | } else { 25 | node = node.next 26 | } 27 | } 28 | } 29 | return newNode.index 30 | } 31 | -------------------------------------------------------------------------------- /start.go: -------------------------------------------------------------------------------- 1 | package LSM 2 | 3 | import ( 4 | "LSM/config" 5 | "LSM/ssTable" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func Start(con config.Config) { 11 | if database != nil { 12 | return 13 | } 14 | log.Println("Loading a Configuration File") 15 | config.Init(con) 16 | log.Println("Starting Server") 17 | initDatabase(con.DataDir) 18 | log.Println("Performing background checks...") 19 | checkMemory() 20 | // 检查压缩数据库文件 21 | database.TableTree.Check() 22 | go Check() 23 | go CompressMemory() 24 | } 25 | func initDatabase(dir string) { 26 | database = &Database{ 27 | MemTable: &MemTable{}, 28 | iMemTable: &ReadOnlyMemTables{}, 29 | TableTree: &ssTable.TableTree{}, 30 | } 31 | if _, err := os.Stat(dir); err != nil { 32 | log.Printf("The %s directory does not exist. The directory is being created\r\n", dir) 33 | err := os.MkdirAll(dir, 0700) 34 | if err != nil { 35 | log.Println("Failed to create the database directory") 36 | panic(err) 37 | } 38 | } 39 | database.iMemTable.Init() 40 | database.MemTable.InitMemTree() 41 | log.Println("Loading all wal.log...") 42 | database.loadAllWalFiles(dir) 43 | database.MemTable.InitWal(dir) 44 | log.Println("Loading database...") 45 | database.TableTree.Init(dir) 46 | } 47 | -------------------------------------------------------------------------------- /wal/Wal.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "LSM/kv" 5 | "LSM/orderTable" 6 | "LSM/orderTable/skipList" 7 | "LSM/orderTable/sortTree" 8 | "bytes" 9 | "encoding/binary" 10 | "encoding/json" 11 | "fmt" 12 | "log" 13 | "os" 14 | "path" 15 | "sync" 16 | "time" 17 | ) 18 | 19 | type Wal struct { 20 | f *os.File 21 | path string 22 | lock sync.Locker 23 | } 24 | 25 | func (w *Wal) Init(dir string) { 26 | log.Println("Loading wal.log...") 27 | start := time.Now() 28 | defer func() { 29 | elapsed := time.Since(start) 30 | log.Println("Loaded wal.log,Consumption of time :", elapsed) 31 | }() 32 | uuidStr := time.Now().Format("2006-01-02-15-04-05.000") 33 | walPath := path.Join(dir, fmt.Sprintf("%s_wal.log", uuidStr)) 34 | log.Printf("init wal.log: walPath: %s\n", walPath) 35 | f, err := os.OpenFile(walPath, os.O_RDWR|os.O_CREATE, 0666) 36 | if err != nil { 37 | log.Println("The wal.log file cannot be created") 38 | panic(err) 39 | } 40 | w.f = f 41 | w.path = walPath 42 | w.lock = &sync.Mutex{} 43 | } 44 | func (w *Wal) LoadFromFile(path string, table orderTable.OrderInterface) orderTable.OrderInterface { 45 | f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0666) 46 | if err != nil { 47 | log.Println("The wal.log file cannot be opened") 48 | panic(err) 49 | } 50 | w.f = f 51 | w.path = path 52 | w.lock = &sync.Mutex{} 53 | return w.LoadToMemory(table) 54 | } 55 | func (w *Wal) LoadToMemory(table orderTable.OrderInterface) orderTable.OrderInterface { 56 | w.lock.Lock() 57 | defer w.lock.Unlock() 58 | var preTable orderTable.OrderInterface 59 | if _, ok := table.(*sortTree.Tree); ok { 60 | preTable = &sortTree.Tree{} 61 | } else if _, ok := table.(*skipList.SkipList); ok { 62 | preTable = &skipList.SkipList{} 63 | } 64 | //preTree := &sortTree.Tree{} 65 | preTable.Init() 66 | info, _ := os.Stat(w.path) 67 | size := info.Size() 68 | if size == 0 { 69 | return preTable 70 | } 71 | _, err := w.f.Seek(0, 0) 72 | if err != nil { 73 | log.Println("The wal.log file cannot be opened") 74 | panic(err) 75 | } 76 | defer func(f *os.File, offset int64, whence int) { 77 | _, err = f.Seek(offset, whence) 78 | if err != nil { 79 | log.Println("The wal.log file cannot be opened") 80 | panic(err) 81 | } 82 | }(w.f, size-1, 0) 83 | data := make([]byte, size) 84 | _, err = w.f.Read(data) 85 | if err != nil { 86 | log.Println("The wal.log file cannot be opened") 87 | panic(err) 88 | } 89 | dataLen := int64(0) 90 | index := int64(0) 91 | for index < size { 92 | indexData := data[index:(index + 8)] 93 | buf := bytes.NewBuffer(indexData) 94 | err = binary.Read(buf, binary.LittleEndian, &dataLen) 95 | if err != nil { 96 | log.Println("The wal.log file cannot be opened") 97 | panic(err) 98 | } 99 | index += 8 100 | dataArea := data[index:(index + dataLen)] 101 | var value kv.Value 102 | err = json.Unmarshal(dataArea, &value) 103 | if err != nil { 104 | log.Println("The wal.log file cannot be opened") 105 | panic(err) 106 | } 107 | if value.Delete { 108 | table.Delete(value.Key) 109 | preTable.Delete(value.Key) 110 | } else { 111 | table.Set(value.Key, value.Value) 112 | preTable.Set(value.Key, value.Value) 113 | } 114 | index += dataLen 115 | } 116 | return preTable 117 | } 118 | func (w *Wal) Write(value kv.Value) { 119 | w.lock.Lock() 120 | defer w.lock.Unlock() 121 | if value.Delete { 122 | log.Println("wal.log: delete ", value.Key) 123 | } else { 124 | //log.Println("wal.log: insert ", value.Key) 125 | } 126 | data, _ := json.Marshal(value) 127 | err := binary.Write(w.f, binary.LittleEndian, int64(len(data))) 128 | if err != nil { 129 | log.Println("The wal.log file cannot be written") 130 | panic(err) 131 | } 132 | err = binary.Write(w.f, binary.LittleEndian, data) 133 | if err != nil { 134 | log.Println("Failed to write the wal.log") 135 | panic(err) 136 | } 137 | } 138 | func (w *Wal) Reset() { 139 | w.lock.Lock() 140 | defer w.lock.Unlock() 141 | 142 | log.Println("wal.log: reset") 143 | err := w.f.Close() 144 | if err != nil { 145 | panic(err) 146 | } 147 | w.f = nil 148 | err = os.Remove(w.path) 149 | if err != nil { 150 | panic(err) 151 | } 152 | f, err := os.OpenFile(w.path, os.O_RDWR|os.O_CREATE, 0666) 153 | if err != nil { 154 | panic(err) 155 | } 156 | w.f = f 157 | } 158 | func (w *Wal) DeleteFile() { 159 | w.lock.Lock() 160 | defer w.lock.Unlock() 161 | log.Printf("Deleting the wal.log file: %s\n", w.path) 162 | err := w.f.Close() 163 | if err != nil { 164 | panic(err) 165 | } 166 | err = os.Remove(w.path) 167 | if err != nil { 168 | log.Println("Failed to delete the wal.log") 169 | panic(err) 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /wal/Wal.go:Zone.Identifier: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/whuanle/lsm/9309f47f33f6586c23edebe596aa2ac25349a5e1/wal/Wal.go:Zone.Identifier --------------------------------------------------------------------------------