├── .gitignore ├── README.md ├── batch.go ├── db.go ├── errors.go ├── examples ├── basic │ └── main.go └── batch │ └── main.go ├── go.mod ├── go.sum ├── img ├── image-20231205180841632.png ├── image-20231205183622480.png ├── image-20231205191018169.png ├── image-20231205193509342.png ├── image-20231205193959491.png └── image-20231205194331633.png ├── logrecord.go ├── logrecord_test.go ├── memtable.go ├── options.go ├── tinywal ├── chunk.go ├── const.go ├── error.go ├── example │ └── main.go ├── options.go ├── pool.go ├── reader.go ├── segmentfile.go ├── segmentfile_test.go └── tinywal.go └── utils ├── path.go └── rand_kv.go /.gitignore: -------------------------------------------------------------------------------- 1 | .VSCodeCounter -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Golang实现自己的KV存储引擎 2 | 3 | 通过本项目可以学到什么？ 4 | 5 | - `WAL`预写日志的实现 6 | - LSM Tree（Log-Structed-Merge Tree) 7 | - 如何构架一个企业级的项目 8 | - KV数据的序列化和反序列化存储 9 | - Golang的基本语法 10 | 11 | 项目已托管GitHub : https://github.com/gofish2020/easydb 欢迎学习Star，如果有看不懂的欢迎私信 12 | 13 | 整个代码量1000多行，基本的大框架也好理解，容易出现错误的点在于代码细节的处理（特别是wal文件的读写部分） 14 | 15 | ![](./img/image-20231205194331633.png) 16 | 17 | ## 简介 18 | 19 | 内存数据采用`SkipList` 存储 20 | 21 | 通过`WAL (Write Ahead Log)`保证内存数据`durability`和`crash-safe`能力 22 | 23 | ![](./img/image-20231205180841632.png) 24 | 25 | # 代码逻辑结构 26 | 27 | ![](./img/image-20231205183622480.png) 28 | 29 | 1. 先通过`easydb.Open`打开数据库`*DB`对象；`*DB`内部基于`WAL`恢复内存数据`openAllMemtables`【就是读取`segment`文件，解析出一个个`LogRecord`保存到`skiplist`中】，同时将内存数据分成活跃内存和不可变内存【参考 LSM Tree结构】。每个内存对象`*memtable`内部除了定义`skiplist`记录内存数据，同时定义`*wal`对象记录磁盘，`*wal`对象中的磁盘文件按照指定的大小分段保存（这里类似kafka中日志数据文件分段原理） 30 | 2. 然后调用 `db.Put`方法，内部通过`batch`开启写事物（对db上锁），将数据批量保存`batch`的`pendingWrites`中,然后在`batch.Commit`一次性全部保存到内存和预写日志中同时关闭写事物（对db解锁） 31 | 3. 调用`db.Get`方法，内部通过`batch`开启读事物（对db上锁），读取所有的内存对象中的数据（倒序）方式，也就是从最近的内存对象`*memtable`开始读，读取结束，提交事物(关闭读事物，对db解锁) 32 | 33 | 建议从下面 `Open Get Put`这几个函数开始看起 34 | 35 | ```go 36 | package main 37 | 38 | import ( 39 | "fmt" 40 | 41 | "github.com/gofish2020/easydb" 42 | "github.com/gofish2020/easydb/utils" 43 | ) 44 | 45 | // this file shows how to use the basic operations of EasyDB 46 | func main() { 47 | // specify the options 48 | options := easydb.DefaultOptions 49 | options.DirPath = utils.ExecDir() + "/data" 50 | 51 | // open a database 52 | db, err := easydb.Open(options) 53 | if err != nil { 54 | panic(err) 55 | } 56 | defer func() { 57 | _ = db.Close() 58 | }() 59 | 60 | // put a key 61 | err = db.Put([]byte("name"), []byte("easydb"), nil) 62 | if err != nil { 63 | panic(err) 64 | } 65 | 66 | // get a key 67 | val, err := db.Get([]byte("name")) 68 | if err != nil { 69 | panic(err) 70 | } 71 | println(string(val)) 72 | 73 | // delete a key 74 | err = db.Delete([]byte("name"), nil) 75 | if err != nil { 76 | panic(err) 77 | } 78 | 79 | // get a key 80 | val, err = db.Get([]byte("name")) 81 | if err != nil { 82 | if err == easydb.ErrKeyNotFound { 83 | fmt.Println("key not exist") 84 | return 85 | } 86 | panic(err) 87 | } 88 | println(string(val)) 89 | } 90 | 91 | ``` 92 | 93 | # WAL日志格式 94 | 95 | ![](./img/image-20231205191018169.png) 96 | 97 | - `WAL`日志文件按照`SegmentSize`分成一个个的段文件； 98 | - 每个段文件，按照` 32KB `为一块存储区域，存储 多个 `chunk`实际数据 99 | - 每个`chunk`由 7 字节`header` + 数据`payload` 组成；`header`头包括 4字节校验码，2字节数据长度 1字节数据类型；校验码校验的范围为：【length + type + payload】确保数据没有损坏 100 | - 一个数据可能由多个`chunk`组成 101 | 102 | ![](./img/image-20231205193509342.png) 103 | 104 | - 当在block中保存了多个chunk后，block剩余的空间不够保存数据，多余的空间浪费掉，填充一些无效字节即可 105 | 106 | ![](./img/image-20231205193959491.png) 107 | 108 | 109 | 110 | Ps：本项目主要参考 LotusDB 实现 -------------------------------------------------------------------------------- /batch.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import ( 4 | "fmt" 5 | "sync" 6 | 7 | "github.com/bwmarrin/snowflake" 8 | ) 9 | 10 | func makeBatch() interface{} { 11 | node, err := snowflake.NewNode(1) 12 | if err != nil { 13 | panic(fmt.Sprintf("snowflake.NewNode(1) failed: %v", err)) 14 | } 15 | return &Batch{ 16 | options: DefaultBatchOptions, 17 | batchId: node, 18 | } 19 | } 20 | 21 | type Batch struct { 22 | db *DB 23 | pendingWrites map[string]*LogRecord 24 | options BatchOptions 25 | mu sync.RWMutex 26 | committed bool 27 | batchId *snowflake.Node 28 | } 29 | 30 | // NewBatch开启读/写事物 31 | func (db *DB) NewBatch(options BatchOptions) *Batch { 32 | batch := &Batch{ 33 | db: db, 34 | options: options, 35 | committed: false, 36 | } 37 | if !options.ReadOnly { 38 | batch.pendingWrites = make(map[string]*LogRecord) 39 | node, err := snowflake.NewNode(1) 40 | if err != nil { 41 | panic(fmt.Sprintf("snowflake.NewNode(1) failed: %v", err)) 42 | } 43 | batch.batchId = node 44 | } 45 | // 开启实务（对db加锁） 46 | batch.lock() 47 | return batch 48 | } 49 | 50 | // 锁定db 51 | func (b *Batch) lock() { 52 | if b.options.ReadOnly { 53 | b.db.mu.RLock() 54 | } else { 55 | b.db.mu.Lock() 56 | } 57 | } 58 | 59 | // 解锁db 60 | func (b *Batch) unlock() { 61 | if b.options.ReadOnly { 62 | b.db.mu.RUnlock() 63 | } else { 64 | b.db.mu.Unlock() 65 | } 66 | } 67 | 68 | // 保存数据到 pendingWrites中 69 | func (b *Batch) Put(key []byte, value []byte) error { 70 | if len(key) == 0 { 71 | return ErrKeyIsEmpty 72 | } 73 | if b.db.closed { 74 | return ErrDBClosed 75 | } 76 | if b.options.ReadOnly { 77 | return ErrReadOnlyBatch 78 | } 79 | 80 | b.mu.Lock() 81 | // 将写操作缓存在 pendingWrites 中 82 | b.pendingWrites[string(key)] = &LogRecord{ 83 | Key: key, 84 | Value: value, 85 | Type: LogRecordNormal, 86 | } 87 | b.mu.Unlock() 88 | 89 | return nil 90 | } 91 | 92 | // 获取数据 93 | func (b *Batch) Get(key []byte) ([]byte, error) { 94 | if len(key) == 0 { 95 | return nil, ErrKeyIsEmpty 96 | } 97 | if b.db.closed { 98 | return nil, ErrDBClosed 99 | } 100 | 101 | // 从pendWrites读取数据 102 | if b.pendingWrites != nil { 103 | b.mu.RLock() 104 | if record := b.pendingWrites[string(key)]; record != nil { 105 | if record.Type == LogRecordDeleted { 106 | b.mu.RUnlock() 107 | return nil, ErrKeyNotFound 108 | } 109 | b.mu.RUnlock() 110 | return record.Value, nil 111 | } 112 | b.mu.RUnlock() 113 | } 114 | 115 | // 遍历所有内存(倒序读取内存中保存的数据) 116 | tables := b.db.getMemTables() 117 | for _, table := range tables { 118 | // 从内存中skl中读取 119 | deleted, value := table.get(key) 120 | if deleted { 121 | return nil, ErrKeyNotFound 122 | } 123 | if len(value) != 0 { 124 | return value, nil 125 | } 126 | } 127 | 128 | return nil, ErrKeyNotFound 129 | } 130 | 131 | // 删除key 132 | func (b *Batch) Delete(key []byte) error { 133 | if len(key) == 0 { 134 | return ErrKeyIsEmpty 135 | } 136 | if b.db.closed { 137 | return ErrDBClosed 138 | } 139 | if b.options.ReadOnly { 140 | return ErrReadOnlyBatch 141 | } 142 | 143 | // 构建删除记录 144 | b.mu.Lock() 145 | b.pendingWrites[string(key)] = &LogRecord{ 146 | Key: key, 147 | Type: LogRecordDeleted, 148 | } 149 | b.mu.Unlock() 150 | 151 | return nil 152 | } 153 | 154 | // 批量提交 155 | func (b *Batch) Commit(options *WriteOptions) error { 156 | // use the default options if options is nil 157 | if options == nil { 158 | options = &WriteOptions{Sync: false, DisableWal: false} 159 | } 160 | // 关闭实务（对db解锁） 161 | defer b.unlock() 162 | if b.db.closed { 163 | return ErrDBClosed 164 | } 165 | 166 | if b.options.ReadOnly || len(b.pendingWrites) == 0 { 167 | return nil 168 | } 169 | 170 | b.mu.Lock() 171 | defer b.mu.Unlock() 172 | 173 | // check if committed 174 | if b.committed { 175 | return ErrBatchCommitted 176 | } 177 | 178 | // wait for memtable space 179 | if err := b.db.waitMemtableSpace(); err != nil { 180 | return err 181 | } 182 | batchId := b.batchId.Generate() 183 | // call memtable put batch 184 | err := b.db.activeMem.putBatch(b.pendingWrites, batchId, options) 185 | if err != nil { 186 | return err 187 | } 188 | 189 | b.committed = true 190 | return nil 191 | } 192 | 193 | func (b *Batch) init(rdonly, sync bool, db *DB) *Batch { 194 | b.options.ReadOnly = rdonly 195 | b.options.Sync = sync 196 | b.db = db 197 | b.lock() 198 | return b 199 | } 200 | 201 | func (b *Batch) reset() { 202 | b.db = nil 203 | b.pendingWrites = nil 204 | b.committed = false 205 | } 206 | 207 | func (b *Batch) withPendingWrites() *Batch { 208 | b.pendingWrites = make(map[string]*LogRecord) 209 | return b 210 | } 211 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "sync" 7 | 8 | "github.com/gofrs/flock" 9 | ) 10 | 11 | const ( 12 | fileLockName = "FLOCK" 13 | ) 14 | 15 | // 用来保存kv数据，写入到activeMem，读取从所有的 mem中读取；immutableMem可以用来归档 16 | type DB struct { 17 | mu sync.RWMutex 18 | activeMem *memtable // 活跃内存 19 | immutableMem []*memtable // 不可变内存 20 | closed bool 21 | 22 | batchPool sync.Pool 23 | } 24 | 25 | func Open(options Options) (*DB, error) { 26 | 27 | // 判断目录是否存在 28 | if _, err := os.Stat(options.DirPath); err != nil { 29 | if err := os.MkdirAll(options.DirPath, os.ModePerm); err != nil { 30 | return nil, err 31 | } 32 | } 33 | // 目录锁 34 | fileLock := flock.New(filepath.Join(options.DirPath, fileLockName)) 35 | hold, err := fileLock.TryLock() 36 | if err != nil { 37 | return nil, err 38 | } 39 | if !hold { 40 | return nil, ErrDatabaseIsUsing 41 | } 42 | 43 | // 打开所有内存文件，构造内存数据 44 | memtables, err := openAllMemtables(options) 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | db := &DB{ 50 | activeMem: memtables[len(memtables)-1], 51 | immutableMem: memtables[:len(memtables)-1], 52 | batchPool: sync.Pool{New: makeBatch}, 53 | } 54 | return db, nil 55 | } 56 | 57 | func (db *DB) getMemTables() []*memtable { 58 | 59 | // 倒序的方式（也就是从最新的mem读） 60 | var tables []*memtable 61 | tables = append(tables, db.activeMem) 62 | 63 | last := len(db.immutableMem) - 1 64 | for i := range db.immutableMem { 65 | tables = append(tables, db.immutableMem[last-i]) // 倒序 66 | } 67 | return tables 68 | } 69 | 70 | func (db *DB) waitMemtableSpace() error { 71 | if !db.activeMem.isFull() { 72 | return nil 73 | } 74 | 75 | // 当前活跃，变成不可变 76 | db.immutableMem = append(db.immutableMem, db.activeMem) 77 | // 重新生成一个新的活跃内存 78 | options := db.activeMem.option 79 | options.id++ 80 | table, err := openMemtable(options) 81 | if err != nil { 82 | return err 83 | } 84 | db.activeMem = table 85 | return nil 86 | } 87 | 88 | func (db *DB) Close() error { 89 | db.mu.Lock() 90 | defer db.mu.Unlock() 91 | 92 | // close all memtables 93 | for _, table := range db.immutableMem { 94 | if err := table.close(); err != nil { 95 | return err 96 | } 97 | } 98 | if err := db.activeMem.close(); err != nil { 99 | return err 100 | } 101 | db.closed = true 102 | return nil 103 | } 104 | 105 | func (db *DB) Put(key []byte, value []byte, options *WriteOptions) error { 106 | batch := db.batchPool.Get().(*Batch) 107 | defer func() { 108 | batch.reset() 109 | db.batchPool.Put(batch) 110 | }() 111 | // This is a single put operation, we can set Sync to false. 112 | // Because the data will be written to the WAL, 113 | // and the WAL file will be synced to disk according to the DB options. 114 | batch.init(false, false, db).withPendingWrites() 115 | if err := batch.Put(key, value); err != nil { 116 | batch.unlock() 117 | return err 118 | } 119 | return batch.Commit(options) 120 | } 121 | 122 | func (db *DB) Get(key []byte) ([]byte, error) { 123 | batch := db.batchPool.Get().(*Batch) 124 | batch.init(true, false, db) 125 | defer func() { 126 | _ = batch.Commit(nil) 127 | batch.reset() 128 | db.batchPool.Put(batch) 129 | }() 130 | return batch.Get(key) 131 | } 132 | 133 | func (db *DB) Delete(key []byte, options *WriteOptions) error { 134 | batch := db.batchPool.Get().(*Batch) 135 | defer func() { 136 | batch.reset() 137 | db.batchPool.Put(batch) 138 | }() 139 | // This is a single delete operation, we can set Sync to false. 140 | // Because the data will be written to the WAL, 141 | // and the WAL file will be synced to disk according to the DB options. 142 | batch.init(false, false, db).withPendingWrites() 143 | if err := batch.Delete(key); err != nil { 144 | batch.unlock() 145 | return err 146 | } 147 | return batch.Commit(options) 148 | } 149 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import "errors" 4 | 5 | var ( 6 | ErrKeyIsEmpty = errors.New("the key is empty") 7 | ErrKeyNotFound = errors.New("key not found in database") 8 | ErrDatabaseIsUsing = errors.New("the database directory is used by another process") 9 | ErrReadOnlyBatch = errors.New("the batch is read only") 10 | ErrBatchCommitted = errors.New("the batch is committed") 11 | ErrDBClosed = errors.New("the database is closed") 12 | ) 13 | -------------------------------------------------------------------------------- /examples/basic/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/gofish2020/easydb" 7 | "github.com/gofish2020/easydb/utils" 8 | ) 9 | 10 | // this file shows how to use the basic operations of EasyDB 11 | func main() { 12 | // specify the options 13 | options := easydb.DefaultOptions 14 | options.DirPath = utils.ExecDir() + "/data" 15 | 16 | // open a database 17 | db, err := easydb.Open(options) 18 | if err != nil { 19 | panic(err) 20 | } 21 | defer func() { 22 | _ = db.Close() 23 | }() 24 | 25 | // put a key 26 | err = db.Put([]byte("name"), []byte("easydb"), nil) 27 | if err != nil { 28 | panic(err) 29 | } 30 | 31 | // get a key 32 | val, err := db.Get([]byte("name")) 33 | if err != nil { 34 | panic(err) 35 | } 36 | println(string(val)) 37 | 38 | // delete a key 39 | err = db.Delete([]byte("name"), nil) 40 | if err != nil { 41 | panic(err) 42 | } 43 | 44 | // get a key 45 | val, err = db.Get([]byte("name")) 46 | if err != nil { 47 | if err == easydb.ErrKeyNotFound { 48 | fmt.Println("key not exist") 49 | return 50 | } 51 | panic(err) 52 | } 53 | println(string(val)) 54 | } 55 | -------------------------------------------------------------------------------- /examples/batch/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gofish2020/easydb" 5 | "github.com/gofish2020/easydb/utils" 6 | ) 7 | 8 | // this file shows how to use the batch operations of EasyDB 9 | func main() { 10 | // specify the options 11 | options := easydb.DefaultOptions 12 | options.DirPath = utils.ExecDir() + "/data" 13 | 14 | // open a database 15 | db, err := easydb.Open(options) 16 | if err != nil { 17 | panic(err) 18 | } 19 | defer func() { 20 | _ = db.Close() 21 | }() 22 | 23 | // create a batch 24 | batch := db.NewBatch(easydb.DefaultBatchOptions) 25 | 26 | // set a key 27 | _ = batch.Put([]byte("name"), []byte("EasyDB")) 28 | 29 | // get a key 30 | val, _ := batch.Get([]byte("name")) 31 | println(string(val)) 32 | 33 | // delete a key 34 | _ = batch.Delete([]byte("name")) 35 | 36 | // commit the batch 37 | _ = batch.Commit(nil) 38 | 39 | // _ = batch.Put([]byte("name1"), []byte("EasyDB1")) // don't do this!!! 40 | } 41 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gofish2020/easydb 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/bwmarrin/snowflake v0.3.0 7 | github.com/dgraph-io/badger/v4 v4.2.0 8 | github.com/gofrs/flock v0.8.1 9 | github.com/hashicorp/golang-lru/v2 v2.0.7 10 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 11 | github.com/stretchr/testify v1.8.4 12 | ) 13 | 14 | require ( 15 | github.com/cespare/xxhash/v2 v2.2.0 // indirect 16 | github.com/davecgh/go-spew v1.1.1 // indirect 17 | github.com/dgraph-io/ristretto v0.1.1 // indirect 18 | github.com/dustin/go-humanize v1.0.0 // indirect 19 | github.com/gogo/protobuf v1.3.2 // indirect 20 | github.com/golang/glog v1.0.0 // indirect 21 | github.com/golang/snappy v0.0.3 // indirect 22 | github.com/klauspost/compress v1.12.3 // indirect 23 | github.com/kr/text v0.2.0 // indirect 24 | github.com/pkg/errors v0.9.1 // indirect 25 | github.com/pmezard/go-difflib v1.0.0 // indirect 26 | golang.org/x/net v0.7.0 // indirect 27 | golang.org/x/sys v0.5.0 // indirect 28 | gopkg.in/yaml.v3 v3.0.1 // indirect 29 | ) 30 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= 2 | github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= 3 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 4 | github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= 5 | github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 6 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/dgraph-io/badger/v4 v4.2.0 h1:kJrlajbXXL9DFTNuhhu9yCx7JJa4qpYWxtE8BzuWsEs= 11 | github.com/dgraph-io/badger/v4 v4.2.0/go.mod h1:qfCqhPoWDFJRx1gp5QwwyGo8xk1lbHUxvK9nK0OGAak= 12 | github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= 13 | github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= 14 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= 15 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= 16 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= 17 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= 18 | github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= 19 | github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= 20 | github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= 21 | github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= 22 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 23 | github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= 24 | github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= 25 | github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= 26 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 27 | github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= 28 | github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= 29 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= 30 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= 31 | github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= 32 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 33 | github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= 34 | github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= 35 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 36 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 37 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 38 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 39 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 40 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 41 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 42 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 43 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 44 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 45 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 46 | github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 47 | github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= 48 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 49 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 50 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 51 | golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 52 | golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= 53 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 54 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 55 | golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 56 | golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= 57 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 58 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 59 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 60 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 61 | golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 62 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 63 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 64 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 65 | golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 66 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 67 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 68 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 69 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 70 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 71 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 72 | golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= 73 | golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= 74 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 75 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 76 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 77 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 78 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 79 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= 80 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 81 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 82 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 83 | -------------------------------------------------------------------------------- /img/image-20231205180841632.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205180841632.png -------------------------------------------------------------------------------- /img/image-20231205183622480.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205183622480.png -------------------------------------------------------------------------------- /img/image-20231205191018169.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205191018169.png -------------------------------------------------------------------------------- /img/image-20231205193509342.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205193509342.png -------------------------------------------------------------------------------- /img/image-20231205193959491.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205193959491.png -------------------------------------------------------------------------------- /img/image-20231205194331633.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gofish2020/easydb/d406c96223569095f5d95ee08dbec5a581635113/img/image-20231205194331633.png -------------------------------------------------------------------------------- /logrecord.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import "encoding/binary" 4 | 5 | type LogRecordType = byte 6 | 7 | const ( 8 | LogRecordNormal = iota 9 | LogRecordDeleted 10 | LogRecordBatchEnd 11 | ) 12 | 13 | const maxLogRecordHeaderSize = 1 + binary.MaxVarintLen64 + binary.MaxVarintLen32 + binary.MaxVarintLen32 14 | 15 | type LogRecord struct { 16 | Key []byte 17 | Value []byte 18 | Type LogRecordType 19 | BatchId uint64 20 | } 21 | 22 | func NewLogRecord() *LogRecord { 23 | return &LogRecord{} 24 | } 25 | 26 | /* 27 | 序列化 28 | type + batchid + keysize + valuesize + key + value 29 | 1 varint(max 10) varint(max 5) varint(max 5) 30 | */ 31 | 32 | func (l *LogRecord) Encode() []byte { 33 | header := make([]byte, maxLogRecordHeaderSize) 34 | header[0] = l.Type 35 | index := 1 36 | index += binary.PutUvarint(header[index:], l.BatchId) 37 | index += binary.PutVarint(header[index:], int64(len(l.Key))) 38 | index += binary.PutVarint(header[index:], int64(len(l.Value))) 39 | 40 | result := make([]byte, index+len(l.Key)+len(l.Value)) 41 | 42 | //copy header 43 | copy(result, header[:index]) 44 | // copy key 45 | copy(result[index:], l.Key) 46 | // copy value 47 | copy(result[index+len(l.Key):], l.Value) 48 | 49 | return result 50 | } 51 | 52 | func (l *LogRecord) Decode(data []byte) { 53 | 54 | l.Type = data[0] // 类型 55 | 56 | index := 1 57 | n := 0 58 | l.BatchId, n = binary.Uvarint(data[index:]) 59 | index += n 60 | keyLen, n := binary.Varint(data[index:]) 61 | index += n 62 | valueLen, n := binary.Varint(data[index:]) 63 | index += n 64 | 65 | key := make([]byte, keyLen) 66 | copy(key, data[index:index+int(keyLen)]) 67 | index += int(keyLen) 68 | 69 | value := make([]byte, valueLen) 70 | copy(value, data[index:index+int(valueLen)]) 71 | 72 | l.Key = key 73 | l.Value = value 74 | } 75 | -------------------------------------------------------------------------------- /logrecord_test.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gofish2020/easydb/utils" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLogRecord(t *testing.T) { 11 | 12 | logRecord := &LogRecord{ 13 | Key: utils.GetTestKey(10), 14 | Value: utils.RandomValue(10), 15 | Type: LogRecordNormal, 16 | BatchId: 1911888181881, 17 | } 18 | 19 | data := logRecord.Encode() 20 | 21 | newRecord := &LogRecord{} 22 | 23 | newRecord.Decode(data) 24 | 25 | assert.Equal(t, logRecord.Key, newRecord.Key) 26 | assert.Equal(t, logRecord.Value, newRecord.Value) 27 | assert.Equal(t, logRecord.Type, newRecord.Type) 28 | assert.Equal(t, logRecord.BatchId, newRecord.BatchId) 29 | 30 | t.Log(string(newRecord.Key)) 31 | t.Log(string(newRecord.Value)) 32 | t.Log(newRecord.Type) 33 | t.Log(newRecord.BatchId) 34 | } 35 | -------------------------------------------------------------------------------- /memtable.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "math" 7 | "os" 8 | "sort" 9 | "sync" 10 | 11 | "github.com/bwmarrin/snowflake" 12 | badgerskl "github.com/dgraph-io/badger/v4/skl" 13 | "github.com/dgraph-io/badger/v4/y" 14 | "github.com/gofish2020/easydb/tinywal" 15 | ) 16 | 17 | const ( 18 | walFileExt = ".MEM.%d" 19 | initialTableID = 1 20 | ) 21 | 22 | // memtable: 内存对象 23 | type memtable struct { 24 | wal *tinywal.TinyWal // 保证内存数据的持久化 25 | option memtableOptions 26 | 27 | mu sync.RWMutex 28 | skl *badgerskl.Skiplist // 内存数据（跳表） 29 | } 30 | 31 | type memtableOptions struct { 32 | sklMemSize uint32 // skl内存表大小 33 | id int // 内存表编号 34 | walDir string // 文件目录 35 | walCacheSize uint64 // wal的缓存大小 36 | walIsSync bool // 是否立即刷盘 37 | walBytesPerSync uint64 // 多少字节刷盘 38 | } 39 | 40 | func openAllMemtables(options Options) ([]*memtable, error) { 41 | 42 | entries, err := os.ReadDir(options.DirPath) 43 | if err != nil { 44 | return nil, err 45 | } 46 | 47 | // 获取内存编号 id 48 | var tableIDs []int 49 | for _, entry := range entries { 50 | if entry.IsDir() { 51 | continue 52 | } 53 | var id int 54 | var prefix int 55 | _, err := fmt.Sscanf(entry.Name(), "%d"+walFileExt, &prefix, &id) 56 | if err != nil { 57 | continue 58 | } 59 | tableIDs = append(tableIDs, id) 60 | } 61 | // 默认初始化一个内存id 62 | if len(tableIDs) == 0 { 63 | tableIDs = append(tableIDs, initialTableID) 64 | } 65 | // 排序 66 | sort.Ints(tableIDs) 67 | 68 | // 多个内存 69 | tables := make([]*memtable, len(tableIDs)) 70 | 71 | for i, table := range tableIDs { 72 | table, err := openMemtable(memtableOptions{ 73 | walDir: options.DirPath, 74 | id: table, 75 | sklMemSize: options.MemtableSize, 76 | walIsSync: options.Sync, 77 | walBytesPerSync: uint64(options.BytesPerSync), 78 | }) 79 | if err != nil { 80 | return nil, err 81 | } 82 | tables[i] = table 83 | } 84 | 85 | return tables, nil 86 | } 87 | 88 | // 通过wal构建skl 89 | func openMemtable(option memtableOptions) (*memtable, error) { 90 | 91 | // 初始化跳表 92 | skl := badgerskl.NewSkiplist(int64(float64(option.sklMemSize) * 1.5)) 93 | 94 | // 初始化内存对象 95 | memTable := &memtable{option: option, skl: skl} 96 | // 构建wal对象 97 | wal, err := tinywal.Open(tinywal.Options{ 98 | Dir: option.walDir, 99 | SegmentFileExt: fmt.Sprintf(walFileExt, option.id), // 一个内存表对应一个文件 100 | SegmentSize: math.MaxInt64, // 无限大 101 | LocalCacheSize: option.walCacheSize, 102 | IsSync: option.walIsSync, 103 | BytesPerSync: option.walBytesPerSync, 104 | }) 105 | if err != nil { 106 | return nil, err 107 | } 108 | memTable.wal = wal 109 | 110 | indexRecords := make(map[uint64][]*LogRecord) 111 | // 遍历wal中的文件，构建skl内存数据 112 | reader := wal.NewReader() 113 | for { 114 | data, _, err := reader.Next() 115 | if err != nil { 116 | if err == io.EOF { // 说明所有文件都读取完了 117 | break 118 | } 119 | // 说明出错了 120 | return nil, err 121 | } 122 | 123 | log := NewLogRecord() 124 | log.Decode(data) 125 | 126 | // batch data save to skiplist 127 | if log.Type == LogRecordBatchEnd { 128 | batchId, err := snowflake.ParseBytes(log.Key) 129 | if err != nil { 130 | return nil, err 131 | } 132 | 133 | for _, idxRecord := range indexRecords[uint64(batchId)] { 134 | memTable.skl.Put(y.KeyWithTs(idxRecord.Key, 0), 135 | y.ValueStruct{Value: idxRecord.Value, Meta: idxRecord.Type}) 136 | } 137 | delete(indexRecords, uint64(batchId)) 138 | } else { 139 | indexRecords[log.BatchId] = append(indexRecords[log.BatchId], log) 140 | } 141 | } 142 | 143 | return memTable, nil 144 | } 145 | 146 | // 从skl中读取数据 147 | func (mt *memtable) get(key []byte) (bool, []byte) { 148 | mt.mu.RLock() 149 | defer mt.mu.RUnlock() 150 | 151 | valueStruct := mt.skl.Get(y.KeyWithTs(key, 0)) 152 | deleted := valueStruct.Meta == LogRecordDeleted 153 | return deleted, valueStruct.Value 154 | } 155 | 156 | func (mt *memtable) isFull() bool { 157 | return mt.skl.MemSize() >= int64(mt.option.sklMemSize) 158 | } 159 | 160 | func (mt *memtable) putBatch(pendingWrites map[string]*LogRecord, 161 | batchId snowflake.ID, options *WriteOptions) error { 162 | 163 | // if wal is not disabled, write to wal first to ensure durability and atomicity 164 | if options == nil || !options.DisableWal { 165 | // add record to wal.pendingWrites 166 | for _, record := range pendingWrites { 167 | record.BatchId = uint64(batchId) 168 | if err := mt.wal.PendingWrites(record.Encode()); err != nil { 169 | return err 170 | } 171 | } 172 | 173 | // add a record to indicate the end of the batch 174 | log := NewLogRecord() 175 | log.Key = batchId.Bytes() 176 | log.Type = LogRecordBatchEnd 177 | 178 | if err := mt.wal.PendingWrites(log.Encode()); err != nil { 179 | return err 180 | } 181 | 182 | // write wal.pendingWrites 183 | if _, err := mt.wal.WriteAll(); err != nil { 184 | return err 185 | } 186 | // flush wal if necessary 187 | if options.Sync && !mt.option.walIsSync { 188 | if err := mt.wal.Sync(); err != nil { 189 | return err 190 | } 191 | } 192 | } 193 | 194 | mt.mu.Lock() 195 | // write to in-memory skip list 196 | for key, record := range pendingWrites { 197 | mt.skl.Put(y.KeyWithTs([]byte(key), 0), y.ValueStruct{Value: record.Value, Meta: record.Type}) 198 | } 199 | mt.mu.Unlock() 200 | 201 | return nil 202 | } 203 | 204 | func (mt *memtable) close() error { 205 | if mt.wal != nil { 206 | return mt.wal.Close() 207 | } 208 | return nil 209 | } 210 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package easydb 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/gofish2020/easydb/tinywal" 7 | ) 8 | 9 | type Options struct { 10 | // DirPath specifies the directory path where all the database files will be stored. 11 | DirPath string 12 | 13 | // MemtableSize represents the maximum size in bytes for a memtable. 14 | // It means that each memtable will occupy so much memory. 15 | // Default value is 64MB. 16 | MemtableSize uint32 17 | 18 | // BlockCache specifies the size of the block cache in number of bytes. 19 | // A block cache is used to store recently accessed data blocks, improving read performance. 20 | // If BlockCache is set to 0, no block cache will be used. 21 | BlockCache uint32 22 | 23 | // Sync is whether to synchronize writes through os buffer cache and down onto the actual disk. 24 | // Setting sync is required for durability of a single write operation, but also results in slower writes. 25 | // 26 | // If false, and the machine crashes, then some recent writes may be lost. 27 | // Note that if it is just the process that crashes (machine does not) then no writes will be lost. 28 | // 29 | // In other words, Sync being false has the same semantics as a write 30 | // system call. Sync being true means write followed by fsync. 31 | Sync bool 32 | 33 | // BytesPerSync specifies the number of bytes to write before calling fsync. 34 | BytesPerSync uint32 35 | } 36 | 37 | func tempDBDir() string { 38 | dir, _ := os.MkdirTemp("", "easydb-temp") 39 | return dir 40 | } 41 | 42 | var DefaultOptions = Options{ 43 | DirPath: tempDBDir(), 44 | MemtableSize: 64 * tinywal.MB, 45 | BlockCache: 0, 46 | Sync: false, 47 | BytesPerSync: 0, 48 | } 49 | 50 | // BatchOptions specifies the options for creating a batch. 51 | type BatchOptions struct { 52 | // Sync has the same semantics as Options.Sync. 53 | Sync bool 54 | 55 | // ReadOnly specifies whether the batch is read only. 56 | ReadOnly bool 57 | } 58 | 59 | var DefaultBatchOptions = BatchOptions{ 60 | Sync: true, 61 | ReadOnly: false, 62 | } 63 | 64 | type WriteOptions struct { 65 | // Sync is whether to synchronize writes through os buffer cache and down onto the actual disk. 66 | // Setting sync is required for durability of a single write operation, but also results in slower writes. 67 | // 68 | // If false, and the machine crashes, then some recent writes may be lost. 69 | // Note that if it is just the process that crashes (machine does not) then no writes will be lost. 70 | // 71 | // In other words, Sync being false has the same semantics as a write 72 | // system call. Sync being true means write followed by fsync. 73 | 74 | // Default value is false. 75 | Sync bool 76 | 77 | // DisableWal if true, writes will not first go to the write ahead log, and the write may get lost after a crash. 78 | // Setting true only if don`t care about the data loss. 79 | // Default value is false. 80 | DisableWal bool 81 | } 82 | -------------------------------------------------------------------------------- /tinywal/chunk.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | type ChunkType = byte 4 | 5 | const ( 6 | ChunkTypeFull ChunkType = iota 7 | ChunkTypeStart 8 | ChunkTypeMiddle 9 | ChunkTypeEnd 10 | ) 11 | 12 | type ChunkPosition struct { 13 | // 文件id 14 | SegmentFileId SegmentFileId 15 | // block索引 16 | BlockIndex uint32 17 | // 在block内到偏移位置 18 | ChunkOffset uint32 19 | // chunk写入，实际占用的总大小 20 | ChunkSize uint32 21 | } 22 | -------------------------------------------------------------------------------- /tinywal/const.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | const ( 4 | B = 1 5 | KB = 1024 * B 6 | MB = 1024 * KB 7 | GB = 1024 * MB 8 | ) 9 | 10 | // segment 11 | const ( 12 | //segment 第一个文件编号 13 | firstSegmentFileId = 1 14 | // segment file的权限 15 | segmentFileModePerm = 0644 16 | ) 17 | 18 | // block 19 | const ( 20 | // 单个Block 32KB 21 | blockSize = 32 * KB 22 | //blockSize = 10 * B 23 | ) 24 | 25 | // chunk 26 | const ( 27 | // 块头默认大小 4(checksum) + 2(数据大小) + 1(数据类型) 28 | chunkHeaderSize = 7 29 | ) 30 | 31 | func BlockSize() int { 32 | return blockSize 33 | } 34 | -------------------------------------------------------------------------------- /tinywal/error.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | import "errors" 4 | 5 | var ErrDataTooLarge = errors.New("数据太大") 6 | 7 | var ErrClosed = errors.New("文件已关闭") 8 | 9 | var ErrInvalidCRC = errors.New("invalid crc, the data may be corrupted") 10 | 11 | var ErrPendingSizeTooLarge = errors.New("the upper bound of pendingWrites can't larger than segment size") 12 | -------------------------------------------------------------------------------- /tinywal/example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/gofish2020/easydb/tinywal" 8 | "github.com/gofish2020/easydb/utils" 9 | ) 10 | 11 | func main() { 12 | 13 | option := tinywal.DefaultWalOption 14 | option.Dir = utils.ExecDir() + "/data" 15 | option.SegmentSize = 20 * tinywal.B // 设置文件大小20B 16 | option.LocalCacheSize = uint64(2 * tinywal.BlockSize()) 17 | option.IsSync = false 18 | option.BytesPerSync = 0 19 | 20 | wal, err := tinywal.Open(option) 21 | if err != nil { 22 | return 23 | } 24 | 25 | pos, err := wal.Write([]byte("333444")) // 写入 13 字节 26 | if err != nil { 27 | return 28 | } 29 | 30 | result, err := wal.Read(pos) 31 | if err != nil && err != io.EOF { 32 | return 33 | } 34 | fmt.Println(result) 35 | 36 | result1, err := wal.Read(pos) 37 | if err != nil && err != io.EOF { 38 | return 39 | } 40 | fmt.Println(result1) 41 | 42 | } 43 | -------------------------------------------------------------------------------- /tinywal/options.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | type Options struct { 4 | // 存放segment文件目录 5 | Dir string 6 | // Segment 文件后缀名，例如：.log / .seg 7 | SegmentFileExt string 8 | // 单个Segment文件最大大小 9 | SegmentSize int64 10 | // 缓存大小(unit : Byte)，用来缓存最近读取过的Block（所有文件共用的缓存，不是每个文件一个缓存） 11 | LocalCacheSize uint64 12 | // 是否将os pagecache中的数据，刷到磁盘中 13 | IsSync bool 14 | //每写入BytesPerSync字节，主动将os pagecache刷入磁盘中 15 | BytesPerSync uint64 16 | } 17 | 18 | var DefaultWalOption = Options{ 19 | Dir: "/Users/mac/easydb/data", 20 | SegmentFileExt: ".log", 21 | SegmentSize: 1 * GB, 22 | LocalCacheSize: 500 * MB, 23 | IsSync: true, 24 | BytesPerSync: 0, 25 | } 26 | -------------------------------------------------------------------------------- /tinywal/pool.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | ) 7 | 8 | // 写 缓存 9 | type bufferPool struct { 10 | buffer sync.Pool 11 | } 12 | 13 | var DefaultBuffer = newBufferPool() 14 | 15 | func newBufferPool() *bufferPool { 16 | return &bufferPool{ 17 | buffer: sync.Pool{ 18 | New: func() any { 19 | return new(bytes.Buffer) 20 | }, 21 | }, 22 | } 23 | } 24 | 25 | func (b *bufferPool) Get() *bytes.Buffer { 26 | return b.buffer.Get().(*bytes.Buffer) 27 | } 28 | 29 | func (b *bufferPool) Put(buf *bytes.Buffer) { 30 | buf.Reset() 31 | b.buffer.Put(buf) 32 | } 33 | 34 | // 读 缓存 35 | 36 | var DefaultReadBuffer = newReadbufferPool() 37 | 38 | func newReadbufferPool() *readBufferPool { 39 | return &readBufferPool{ 40 | buffer: sync.Pool{ 41 | New: newBlockAndHeader, 42 | }, 43 | } 44 | } 45 | 46 | type readBufferPool struct { 47 | buffer sync.Pool 48 | } 49 | 50 | func (b *readBufferPool) Get() *blockAndHeader { 51 | return b.buffer.Get().(*blockAndHeader) 52 | } 53 | 54 | func (b *readBufferPool) Put(buf *blockAndHeader) { 55 | b.buffer.Put(buf) 56 | } 57 | 58 | type blockAndHeader struct { 59 | block []byte 60 | header []byte 61 | } 62 | 63 | func newBlockAndHeader() interface{} { 64 | return &blockAndHeader{ 65 | block: make([]byte, blockSize), // block 32KB 66 | header: make([]byte, chunkHeaderSize), // chunk 头 7字节 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /tinywal/reader.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | import "io" 4 | 5 | type Reader struct { 6 | allSegmentReader []*segmentReader // 所有文件 7 | progress int // 文件进度 8 | } 9 | 10 | func (r *Reader) Next() ([]byte, *ChunkPosition, error) { 11 | 12 | if r.progress >= len(r.allSegmentReader) { // 所有文件都完成读 13 | return nil, nil, io.EOF 14 | } 15 | 16 | data, chunkPos, err := r.allSegmentReader[r.progress].Next() 17 | if err == io.EOF { // 说明当前文件读取到尾部 18 | r.progress++ // 继续下一个文件 19 | return r.Next() // 从中读取数据 20 | } 21 | 22 | return data, chunkPos, nil 23 | } 24 | -------------------------------------------------------------------------------- /tinywal/segmentfile.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | import ( 4 | "bytes" 5 | "encoding/binary" 6 | "fmt" 7 | "hash/crc32" 8 | "io" 9 | "os" 10 | "path/filepath" 11 | 12 | lru "github.com/hashicorp/golang-lru/v2" 13 | ) 14 | 15 | type SegmentFileId = uint32 16 | 17 | type segmentFile struct { 18 | // 文件Id 19 | segmentFileId SegmentFileId 20 | // 文件句柄 21 | fd *os.File 22 | // 最后一个block的索引 23 | lastBlockIndex uint32 24 | // 最后一个block的大小 25 | lastBlockSize uint32 26 | // 7字节chunkheader(复用，避免内存重复分配) 27 | header []byte 28 | closed bool 29 | //localCache 30 | localCache *lru.Cache[uint64, []byte] 31 | } 32 | 33 | // 拼接文件名 34 | func segmentFileName(dir, ext string, id uint32) string { 35 | return filepath.Join(dir, fmt.Sprintf("%010d"+ext, id)) 36 | } 37 | 38 | // openSegmentFile 打开一个文件 39 | func openSegmentFile(dir string, ext string, segmentFileId uint32, localCache *lru.Cache[uint64, []byte]) (*segmentFile, error) { 40 | 41 | fd, err := os.OpenFile(segmentFileName(dir, ext, segmentFileId), os.O_CREATE|os.O_RDWR|os.O_APPEND, segmentFileModePerm) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | fileInfo, err := fd.Stat() 47 | if err != nil { 48 | return nil, err 49 | } 50 | size := fileInfo.Size() // 当前文件大小 51 | 52 | return &segmentFile{ 53 | segmentFileId: segmentFileId, 54 | fd: fd, 55 | lastBlockIndex: uint32(size / blockSize), 56 | lastBlockSize: uint32(size % blockSize), 57 | localCache: localCache, 58 | header: make([]byte, chunkHeaderSize), 59 | }, nil 60 | } 61 | 62 | // close file 63 | func (seg *segmentFile) Close() error { 64 | if seg.closed { 65 | return nil 66 | } 67 | seg.closed = true 68 | return seg.fd.Close() 69 | } 70 | 71 | // file size 72 | func (seg *segmentFile) Size() int64 { 73 | return int64(seg.lastBlockIndex*blockSize + seg.lastBlockSize) 74 | } 75 | 76 | // 文件刷盘 77 | func (seg *segmentFile) Sync() error { 78 | if seg.closed { 79 | return nil 80 | } 81 | return seg.fd.Sync() 82 | } 83 | 84 | // 追加chunk到buf中 85 | func (seg *segmentFile) appendChunk2Buffer(buf *bytes.Buffer, data []byte, chunkType ChunkType) error { 86 | // 设置header中的长度 87 | binary.LittleEndian.PutUint16(seg.header[4:6], uint16(len(data))) 88 | // 设置header中的类型 89 | seg.header[6] = chunkType 90 | 91 | // 对 len + type + data 求 checksum 92 | sum := crc32.ChecksumIEEE(seg.header[4:]) 93 | sum = crc32.Update(sum, crc32.IEEETable, data) 94 | // 设置header中的校验和 95 | binary.LittleEndian.PutUint32(seg.header[:4], sum) 96 | //将一个完整的chunk写入buf中（header + payload 就是一个chunk） 97 | _, err := buf.Write(seg.header[:]) 98 | if err != nil { 99 | return err 100 | } 101 | _, err = buf.Write(data) 102 | if err != nil { 103 | return err 104 | } 105 | return nil 106 | } 107 | 108 | // 将数据data写入到buf中 109 | func (seg *segmentFile) writeBuffer(data []byte, buf *bytes.Buffer) (*ChunkPosition, error) { 110 | 111 | startBufferLen := buf.Len() 112 | if seg.closed { 113 | return nil, ErrClosed 114 | } 115 | 116 | padding := uint32(0) 117 | //1.判断block剩余空间过小（chunkheader都没法写入） 118 | if seg.lastBlockSize+chunkHeaderSize >= blockSize { 119 | // block的剩余空间过小，只写入【填充字节】 120 | size := blockSize - seg.lastBlockSize 121 | _, err := buf.Write(make([]byte, size)) 122 | if err != nil { 123 | return nil, err 124 | } 125 | padding += size 126 | // 新block 127 | seg.lastBlockIndex++ 128 | seg.lastBlockSize = 0 129 | } 130 | 131 | // 数据写入的起始位置 132 | pos := &ChunkPosition{ 133 | SegmentFileId: seg.segmentFileId, 134 | BlockIndex: seg.lastBlockIndex, 135 | ChunkOffset: seg.lastBlockSize, 136 | } 137 | 138 | dataLen := uint32(len(data)) 139 | //2.block剩余空间可以写入完整chunk数据 140 | if seg.lastBlockSize+chunkHeaderSize+dataLen <= blockSize { 141 | err := seg.appendChunk2Buffer(buf, data, ChunkTypeFull) 142 | if err != nil { 143 | return nil, err 144 | } 145 | pos.ChunkSize = dataLen + chunkHeaderSize 146 | } else { // 将data保存为多个chunk 147 | 148 | var ( 149 | leftSize = dataLen // 剩余数据大小 150 | curBlockSize = seg.lastBlockSize // block 占用的大小 151 | chunkNum uint32 = 0 // 记录分成了几块 152 | ) 153 | 154 | for leftSize > 0 { 155 | chunkType := ChunkTypeMiddle // 默认都是中间的数据 156 | if leftSize == dataLen { // 第一次 157 | chunkType = ChunkTypeStart 158 | } 159 | // block剩余可以保存的大小 160 | freeSize := blockSize - curBlockSize - chunkHeaderSize 161 | if freeSize >= leftSize { // 最后一次 162 | freeSize = leftSize // 剩下的数据可以全部保存 163 | chunkType = ChunkTypeEnd 164 | } 165 | 166 | // 截取data中的数据 167 | err := seg.appendChunk2Buffer(buf, data[dataLen-leftSize:dataLen-leftSize+freeSize], chunkType) 168 | if err != nil { 169 | return nil, err 170 | } 171 | chunkNum += 1 172 | // 剩余的数据更新 173 | leftSize -= freeSize 174 | // block 占用的大小更新 175 | curBlockSize = (curBlockSize + chunkHeaderSize + freeSize) % blockSize 176 | } 177 | // chunk 占用的大小 178 | pos.ChunkSize = chunkNum*chunkHeaderSize + dataLen 179 | } 180 | 181 | endBufferLen := buf.Len() 182 | // 校验用 183 | if pos.ChunkSize+padding != uint32(endBufferLen-startBufferLen) { 184 | panic(fmt.Sprintf("wrong!!! the chunk size %d is not equal to the buffer len %d", 185 | pos.ChunkSize+padding, endBufferLen-startBufferLen)) 186 | } 187 | 188 | // 更新seg中block相关的两个值 189 | seg.lastBlockSize += pos.ChunkSize // 追加的数据大小 190 | if seg.lastBlockSize >= blockSize { 191 | seg.lastBlockIndex += seg.lastBlockSize / blockSize 192 | seg.lastBlockSize = seg.lastBlockSize % blockSize 193 | } 194 | return pos, nil 195 | } 196 | 197 | // 将buf写入fd中 198 | func (seg *segmentFile) writeBuffer2File(buf *bytes.Buffer) error { 199 | if seg.lastBlockSize > blockSize { 200 | panic("can not exceed the block size") 201 | } 202 | 203 | if _, err := seg.fd.Write(buf.Bytes()); err != nil { 204 | return err 205 | } 206 | return nil 207 | } 208 | 209 | // 数据写入当前文件 210 | func (seg *segmentFile) Write(data []byte) (*ChunkPosition, error) { 211 | if seg.closed { 212 | return nil, ErrClosed 213 | } 214 | lastBlockIndex := seg.lastBlockIndex 215 | lastBlockSize := seg.lastBlockSize 216 | 217 | var err error 218 | buf := DefaultBuffer.Get() 219 | defer func() { 220 | // 万一写入失败，恢复文件的游标值 221 | if err != nil { 222 | seg.lastBlockIndex = lastBlockIndex 223 | seg.lastBlockSize = lastBlockSize 224 | } 225 | DefaultBuffer.Put(buf) 226 | }() 227 | // 将数据临时写到buffer 228 | pos, err := seg.writeBuffer(data, buf) 229 | if err != nil { 230 | return nil, err 231 | } 232 | // 将buffer写入文件 233 | err = seg.writeBuffer2File(buf) 234 | if err != nil { 235 | return nil, err 236 | } 237 | return pos, nil 238 | } 239 | 240 | // 从block中读取完整的数据 241 | func (seg *segmentFile) Read(blockIndex uint32, chunkOffset uint32) ([]byte, error) { 242 | data, _, err := seg.readInternal(blockIndex, chunkOffset) 243 | return data, err 244 | } 245 | 246 | // 文件id | blockIndex 247 | func (seg *segmentFile) getCacheKey(blockIndex uint32) uint64 { 248 | return uint64(seg.segmentFileId)<<32 | uint64(blockIndex) 249 | } 250 | func (seg *segmentFile) readInternal(blockIndex uint32, chunkOffset uint32) ([]byte, *ChunkPosition, error) { 251 | 252 | if seg.closed { 253 | return nil, nil, ErrClosed 254 | } 255 | 256 | var ( 257 | result []byte 258 | segmengSize = seg.Size() 259 | buf = DefaultReadBuffer.Get() 260 | nextChunk = &ChunkPosition{SegmentFileId: seg.segmentFileId} 261 | ) 262 | 263 | defer func() { 264 | DefaultReadBuffer.Put(buf) 265 | }() 266 | 267 | for { 268 | // 当前block默认大小 269 | curBlockSize := int64(blockSize) 270 | // 当前block之前的大小 271 | beforeBlockSize := int64(blockIndex) * blockSize 272 | if beforeBlockSize+curBlockSize > segmengSize { 273 | // 当前block【实际大小】 274 | curBlockSize = segmengSize - beforeBlockSize 275 | } 276 | // chunk起始位置越界 277 | if int64(chunkOffset) >= curBlockSize { // curBlockSize 可能是负数，表示读取到文件尾部 278 | return nil, nil, io.EOF 279 | } 280 | 281 | var ( 282 | ok bool 283 | cachedBlock []byte 284 | ) 285 | if seg.localCache != nil { // 判断缓存是否存在 286 | cachedBlock, ok = seg.localCache.Get(seg.getCacheKey(blockIndex)) 287 | } 288 | 289 | if ok { 290 | copy(buf.block, cachedBlock) 291 | } else { 292 | // 读取一个block 293 | _, err := seg.fd.ReadAt(buf.block[0:curBlockSize], beforeBlockSize) 294 | if err != nil { 295 | return nil, nil, err 296 | } 297 | 298 | // 设置缓存 299 | if seg.localCache != nil && curBlockSize == blockSize { 300 | cache := make([]byte, blockSize) 301 | copy(cache, buf.block) 302 | seg.localCache.Add(seg.getCacheKey(blockIndex), cache) 303 | } 304 | } 305 | 306 | // 从block读chunk header 307 | copy(buf.header, buf.block[chunkOffset:chunkOffset+chunkHeaderSize]) 308 | 309 | // payload length 310 | length := binary.LittleEndian.Uint16(buf.header[4:6]) 311 | // 读取payload 312 | start := chunkOffset + chunkHeaderSize 313 | result = append(result, buf.block[start:start+uint32(length)]...) 314 | 315 | // 校验数据完整性 316 | oldSum := binary.LittleEndian.Uint32(buf.header[:4]) 317 | end := chunkOffset + chunkHeaderSize + uint32(length) 318 | sum := crc32.ChecksumIEEE(buf.block[chunkOffset+4 : end]) 319 | if sum != oldSum { 320 | return nil, nil, ErrInvalidCRC 321 | } 322 | 323 | chunkType := buf.header[6] 324 | 325 | if chunkType == ChunkTypeFull || chunkType == ChunkTypeEnd { 326 | // 下一个数据的起始位置 327 | nextChunk.BlockIndex = blockIndex 328 | nextChunk.ChunkOffset = end 329 | 330 | if end+chunkHeaderSize >= blockSize { // 说明有填充区 331 | nextChunk.BlockIndex++ 332 | nextChunk.ChunkOffset = 0 333 | } 334 | break 335 | } 336 | 337 | blockIndex++ 338 | chunkOffset = 0 339 | } 340 | 341 | return result, nextChunk, nil 342 | } 343 | 344 | func (seg *segmentFile) NewSegmentReader() *segmentReader { 345 | return &segmentReader{ 346 | seg: seg, 347 | blockidx: 0, 348 | chunkoffset: 0, 349 | } 350 | } 351 | 352 | // writeAll write batch data to the segment file. 353 | func (seg *segmentFile) writeAll(data [][]byte) (positions []*ChunkPosition, err error) { 354 | if seg.closed { 355 | return nil, ErrClosed 356 | } 357 | 358 | // if any error occurs, restore the segment status 359 | originBlockNumber := seg.lastBlockIndex 360 | originBlockSize := seg.lastBlockSize 361 | 362 | // init chunk buffer 363 | buf := DefaultBuffer.Get() 364 | defer func() { 365 | if err != nil { 366 | seg.lastBlockIndex = originBlockNumber 367 | seg.lastBlockSize = originBlockSize 368 | } 369 | DefaultBuffer.Put(buf) 370 | }() 371 | 372 | // write all data to the chunk buffer 373 | var pos *ChunkPosition 374 | positions = make([]*ChunkPosition, len(data)) 375 | for i := 0; i < len(positions); i++ { 376 | pos, err = seg.writeBuffer(data[i], buf) 377 | if err != nil { 378 | return 379 | } 380 | positions[i] = pos 381 | } 382 | // write the chunk buffer to the segment file 383 | if err = seg.writeBuffer2File(buf); err != nil { 384 | return 385 | } 386 | return 387 | } 388 | 389 | // *****************单个文件读取的进度****************** 390 | type segmentReader struct { 391 | seg *segmentFile 392 | blockidx uint32 393 | chunkoffset uint32 394 | } 395 | 396 | // 返回数据和当前数据的位置 397 | func (segReader *segmentReader) Next() ([]byte, *ChunkPosition, error) { 398 | if segReader.seg.closed { 399 | return nil, nil, ErrClosed 400 | } 401 | 402 | curChunk := &ChunkPosition{ 403 | SegmentFileId: segReader.seg.segmentFileId, 404 | BlockIndex: segReader.blockidx, 405 | ChunkOffset: segReader.chunkoffset, 406 | } 407 | 408 | data, nextChunk, err := segReader.seg.readInternal(curChunk.BlockIndex, curChunk.ChunkOffset) 409 | if err != nil { 410 | return nil, nil, err 411 | } 412 | // 估值 413 | curChunk.ChunkSize = nextChunk.BlockIndex*blockSize + nextChunk.ChunkOffset - 414 | (curChunk.BlockIndex*blockSize + curChunk.ChunkOffset) 415 | 416 | // 下一个读取chunk起始位置（记录下来） 417 | segReader.blockidx = nextChunk.BlockIndex 418 | segReader.chunkoffset = nextChunk.ChunkOffset 419 | 420 | return data, curChunk, nil 421 | } 422 | -------------------------------------------------------------------------------- /tinywal/segmentfile_test.go: -------------------------------------------------------------------------------- 1 | package tinywal 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSegmentFileName(t *testing.T) { 10 | t.Log(segmentFileName("/nash", ".log", 1)) 11 | } 12 | 13 | func TestOpen(t *testing.T) { 14 | 15 | //t.Log(DefaultOption) 16 | _, err := Open(DefaultWalOption) 17 | assert.Nil(t, err) 18 | 19 | //t.Log(err) 20 | } 21 | -------------------------------------------------------------------------------- /tinywal/tinywal.go: -------------------------------------------------------------------------------- 1 | // 一个目录下有多个文件 -> 每个文件对应一个segment(activeSegment immutableSegment) -> 一个文件由多个block组成，block中保存chunk记录（实际数据由多个chunk组成） 2 | 3 | package tinywal 4 | 5 | import ( 6 | "errors" 7 | "fmt" 8 | "io/fs" 9 | "os" 10 | "sort" 11 | "strings" 12 | "sync" 13 | 14 | lru "github.com/hashicorp/golang-lru/v2" 15 | ) 16 | 17 | type TinyWal struct { 18 | //wal 配置信息 19 | option Options 20 | // 写锁 21 | mutex sync.RWMutex 22 | // 活跃segment 23 | activeSegmentFile *segmentFile 24 | // 不可变segment 25 | immutableSegmentFile map[SegmentFileId]*segmentFile 26 | // lru (key/value 内存缓存) 所有segment文件复用【key = 文件ID+blockid & value = 32kb的block】 27 | localCache *lru.Cache[uint64, []byte] 28 | // 记录已经写入的字节数 29 | bytesWrite uint64 30 | 31 | pendingWritesLock sync.Mutex 32 | pendingWrites [][]byte 33 | pendingSize int64 34 | } 35 | 36 | // Open 基于目录中的segment file构建tinywal对象 37 | func Open(option Options) (*TinyWal, error) { 38 | 39 | // 判断文件后缀格式是否正确 40 | if !strings.HasPrefix(option.SegmentFileExt, ".") { 41 | return nil, errors.New("文件后缀必须以.开头") 42 | } 43 | 44 | // 如果目录不存在创建目录；如果目录存在啥也不做 45 | err := os.MkdirAll(option.Dir, fs.ModePerm) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | tinywal := TinyWal{ 51 | option: option, 52 | immutableSegmentFile: make(map[SegmentFileId]*segmentFile), 53 | activeSegmentFile: nil, 54 | } 55 | 56 | // LocalCacheSize > 0 需要缓存 57 | if option.LocalCacheSize > 0 { 58 | // 计算缓存block个数 59 | blockNum := option.LocalCacheSize / blockSize 60 | if option.LocalCacheSize%blockSize != 0 { 61 | blockNum++ 62 | } 63 | tinywal.localCache, err = lru.New[uint64, []byte](int(blockNum)) 64 | if err != nil { 65 | return nil, err 66 | } 67 | } 68 | 69 | // 读取目录中所有文件 70 | dirEntries, err := os.ReadDir(option.Dir) 71 | if err != nil { 72 | return nil, err 73 | } 74 | 75 | var segmentFileIds []int 76 | for _, entry := range dirEntries { 77 | if entry.IsDir() { 78 | continue 79 | } 80 | // 提取文件名中以SegmentFileExt后缀的【前面的名字】 81 | segmentFileId := 0 82 | _, err = fmt.Sscanf(entry.Name(), "%d"+option.SegmentFileExt, &segmentFileId) 83 | if err != nil { // 说明这个文件名格式不符合 84 | continue 85 | } 86 | segmentFileIds = append(segmentFileIds, segmentFileId) 87 | } 88 | 89 | if len(segmentFileIds) == 0 { // 说明第一次 90 | segment, err := openSegmentFile(option.Dir, option.SegmentFileExt, firstSegmentFileId, tinywal.localCache) 91 | if err != nil { 92 | return nil, err 93 | } 94 | tinywal.activeSegmentFile = segment 95 | } else { 96 | 97 | sort.Ints(segmentFileIds) 98 | for i, segmentFileId := range segmentFileIds { // 打开所有的文件 99 | segment, err := openSegmentFile(option.Dir, option.SegmentFileExt, uint32(segmentFileId), tinywal.localCache) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | if i == len(segmentFileIds)-1 { // 最后一个文件为活跃segment 105 | tinywal.activeSegmentFile = segment 106 | } else { 107 | tinywal.immutableSegmentFile[uint32(segmentFileId)] = segment 108 | } 109 | } 110 | } 111 | return &tinywal, nil 112 | } 113 | 114 | func (tinywal *TinyWal) isFull(delta int64) bool { 115 | return tinywal.activeSegmentFile.Size()+tinywal.maxWriteSize(delta) > int64(tinywal.option.SegmentSize) 116 | } 117 | 118 | // 可能占用的最大空间 119 | func (tinywal *TinyWal) maxWriteSize(delta int64) int64 { 120 | return chunkHeaderSize + delta + (delta/blockSize+1)*chunkHeaderSize 121 | } 122 | 123 | // 创建新的 activeSegmentFile 替换代替当前 activeSegmentFile 124 | func (tinywal *TinyWal) replaceActiveSegmentFile() error { 125 | 126 | if err := tinywal.activeSegmentFile.Sync(); err != nil { 127 | return err 128 | } 129 | tinywal.bytesWrite = 0 130 | segmentFile, err := openSegmentFile(tinywal.option.Dir, tinywal.option.SegmentFileExt, tinywal.activeSegmentFile.segmentFileId+1, tinywal.localCache) 131 | if err != nil { 132 | return err 133 | } 134 | 135 | tinywal.immutableSegmentFile[tinywal.activeSegmentFile.segmentFileId] = tinywal.activeSegmentFile 136 | tinywal.activeSegmentFile = segmentFile 137 | return nil 138 | } 139 | 140 | // Write 往tinywal对象中写入字节流 141 | func (tinywal *TinyWal) Write(data []byte) (*ChunkPosition, error) { 142 | tinywal.mutex.Lock() 143 | defer tinywal.mutex.Unlock() 144 | 145 | // 1.判断数据是否比文件还大 146 | if (tinywal.maxWriteSize(int64(len(data)))) > tinywal.option.SegmentSize { 147 | return nil, ErrDataTooLarge 148 | } 149 | //2. 当前文件的剩余空间，能否写入数据 150 | if tinywal.isFull(int64(len(data))) { 151 | // 新建一个文件 152 | if err := tinywal.replaceActiveSegmentFile(); err != nil { 153 | return nil, err 154 | } 155 | } 156 | //3. 将数据写入到 activeSegmentFile中 157 | pos, err := tinywal.activeSegmentFile.Write(data) 158 | if err != nil { 159 | return nil, err 160 | } 161 | 162 | // 4.记录已经写入多少字节 163 | tinywal.bytesWrite += uint64(pos.ChunkSize) 164 | 165 | isSync := tinywal.option.IsSync 166 | //5.是否将数据刷盘（基于配置） 167 | if !isSync && tinywal.bytesWrite > tinywal.option.BytesPerSync { 168 | isSync = true 169 | } 170 | // 6.执行刷盘 171 | if isSync { 172 | if err := tinywal.activeSegmentFile.Sync(); err != nil { 173 | return nil, err 174 | } 175 | tinywal.bytesWrite = 0 176 | } 177 | return pos, nil 178 | } 179 | 180 | // Read 从tinywal中读取数据 181 | func (tinywal *TinyWal) Read(pos *ChunkPosition) ([]byte, error) { 182 | tinywal.mutex.RLock() 183 | defer tinywal.mutex.RUnlock() 184 | 185 | var segment *segmentFile 186 | if pos.SegmentFileId == tinywal.activeSegmentFile.segmentFileId { 187 | segment = tinywal.activeSegmentFile 188 | } else { 189 | if seg, ok := tinywal.immutableSegmentFile[pos.SegmentFileId]; ok { 190 | segment = seg 191 | } 192 | } 193 | 194 | if segment == nil { 195 | return nil, fmt.Errorf("segment file %d%s not found", pos.SegmentFileId, tinywal.option.SegmentFileExt) 196 | } 197 | 198 | // 从文件中读取数据 199 | return segment.Read(pos.BlockIndex, pos.ChunkOffset) 200 | } 201 | 202 | func (tinywal *TinyWal) NewReader() *Reader { 203 | tinywal.mutex.RLock() 204 | defer tinywal.mutex.RUnlock() 205 | 206 | var readers []*segmentReader 207 | for _, segmentFile := range tinywal.immutableSegmentFile { 208 | readers = append(readers, segmentFile.NewSegmentReader()) 209 | } 210 | 211 | readers = append(readers, tinywal.activeSegmentFile.NewSegmentReader()) 212 | 213 | // 基于文件编号排序 214 | sort.Slice(readers, func(i, j int) bool { 215 | return readers[i].seg.segmentFileId < readers[j].seg.segmentFileId 216 | }) 217 | 218 | return &Reader{ 219 | allSegmentReader: readers, 220 | progress: 0, 221 | } 222 | } 223 | 224 | func (tinywal *TinyWal) PendingWrites(data []byte) error { 225 | tinywal.pendingWritesLock.Lock() 226 | defer tinywal.pendingWritesLock.Unlock() 227 | 228 | size := tinywal.maxWriteSize(int64(len(data))) 229 | tinywal.pendingSize += size 230 | tinywal.pendingWrites = append(tinywal.pendingWrites, data) 231 | return nil 232 | } 233 | 234 | // 清空 235 | func (tinywal *TinyWal) ClearPendingWrites() { 236 | tinywal.pendingWritesLock.Lock() 237 | defer tinywal.pendingWritesLock.Unlock() 238 | 239 | tinywal.pendingSize = 0 240 | tinywal.pendingWrites = tinywal.pendingWrites[:0] 241 | } 242 | 243 | func (tinywal *TinyWal) WriteAll() ([]*ChunkPosition, error) { 244 | if len(tinywal.pendingWrites) == 0 { 245 | return make([]*ChunkPosition, 0), nil 246 | } 247 | 248 | tinywal.mutex.Lock() 249 | defer func() { 250 | tinywal.ClearPendingWrites() 251 | tinywal.mutex.Unlock() 252 | }() 253 | 254 | // 数据不能超过文件大小 255 | if tinywal.pendingSize > tinywal.option.SegmentSize { 256 | return nil, ErrPendingSizeTooLarge 257 | } 258 | 259 | // 文件剩余空间不够 260 | if tinywal.activeSegmentFile.Size()+tinywal.pendingSize > tinywal.option.SegmentSize { 261 | if err := tinywal.replaceActiveSegmentFile(); err != nil { 262 | return nil, err 263 | } 264 | } 265 | 266 | // write all data to the active segment file. 267 | positions, err := tinywal.activeSegmentFile.writeAll(tinywal.pendingWrites) 268 | if err != nil { 269 | return nil, err 270 | } 271 | 272 | return positions, nil 273 | } 274 | 275 | func (tinywal *TinyWal) Sync() error { 276 | tinywal.mutex.Lock() 277 | defer tinywal.mutex.Unlock() 278 | 279 | return tinywal.activeSegmentFile.Sync() 280 | } 281 | 282 | func (wal *TinyWal) Close() error { 283 | wal.mutex.Lock() 284 | defer wal.mutex.Unlock() 285 | 286 | // purge the block cache. 287 | if wal.localCache != nil { 288 | wal.localCache.Purge() 289 | } 290 | 291 | // close all segment files. 292 | for _, segment := range wal.immutableSegmentFile { 293 | if err := segment.Close(); err != nil { 294 | return err 295 | } 296 | } 297 | wal.immutableSegmentFile = nil 298 | 299 | // close the active segment file. 300 | return wal.activeSegmentFile.Close() 301 | } 302 | -------------------------------------------------------------------------------- /utils/path.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import "github.com/kardianos/osext" 4 | 5 | // ExecDir 当前可执行程序目录 6 | func ExecDir() string { 7 | 8 | path, err := osext.ExecutableFolder() 9 | if err != nil { 10 | return "" 11 | } 12 | return path 13 | } 14 | -------------------------------------------------------------------------------- /utils/rand_kv.go: -------------------------------------------------------------------------------- 1 | package utils 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "time" 7 | ) 8 | 9 | var ( 10 | //lock = sync.Mutex{} 11 | randStr = rand.New(rand.NewSource(time.Now().Unix())) 12 | letters = []byte("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") 13 | ) 14 | 15 | func GetTestKey(i int) []byte { 16 | return []byte(fmt.Sprintf("easydb-test-key-%09d", i)) 17 | } 18 | 19 | func RandomValue(n int) []byte { 20 | b := make([]byte, n) 21 | for i := range b { 22 | b[i] = letters[randStr.Intn(len(letters))] 23 | } 24 | return []byte("easydb-test-value-" + string(b)) 25 | } 26 | --------------------------------------------------------------------------------