├── README.md ├── wal.go └── wal_test.go /README.md: -------------------------------------------------------------------------------- 1 | # SimpleWAL 2 | A demo implementation of write ahead log. 3 | 4 | A more elaborate explanable in https://eileen-code4fun.medium.com/building-an-append-only-log-from-scratch-e8712b49c924. 5 | 6 | Benchmark test for sync vs async: 7 | 8 | | Test | Iterations | Cost | 9 | | :------------------ | :--------: | -------------: | 10 | | BenchmarkSyncWAL-8 | 1000000000 | 0.363 ns/op | 11 | | BenchmarkAsyncWAL-8 | 1000000000 | 0.000796 ns/op | 12 | -------------------------------------------------------------------------------- /wal.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "hash/crc32" 7 | "os" 8 | ) 9 | 10 | const ( 11 | dataLenSize = 4 12 | crcSize = 4 13 | ) 14 | 15 | type WAL struct { 16 | f *os.File 17 | buffer []byte 18 | bi int 19 | fsync bool 20 | bufferSize, maxRecordSize int 21 | } 22 | 23 | func NewWAL(filename string, fsync bool, bufferSize, maxRecordSize int) (*WAL, error) { 24 | if maxRecordSize + dataLenSize + crcSize > bufferSize { 25 | return nil, fmt.Errorf("error in size configuration") 26 | } 27 | file, err := os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0660) 28 | if err != nil { 29 | return nil, err 30 | } 31 | return &WAL{ 32 | f: file, 33 | buffer: make([]byte, bufferSize), 34 | bi: 0, 35 | fsync: fsync, 36 | bufferSize: bufferSize, 37 | maxRecordSize: maxRecordSize + dataLenSize + crcSize, 38 | }, nil 39 | } 40 | 41 | func (w *WAL) AddRecord(data []byte) error { 42 | if dataLenSize + len(data) > w.maxRecordSize { 43 | return fmt.Errorf("record size %d exceeds limit %d", len(data), w.maxRecordSize - dataLenSize - crcSize) 44 | } 45 | if w.bi + w.maxRecordSize > w.bufferSize { 46 | if err := w.Flush(); err != nil { 47 | return err 48 | } 49 | } 50 | // Write data len. 51 | binary.LittleEndian.PutUint32(w.buffer[w.bi:], uint32(len(data))) 52 | w.bi += dataLenSize 53 | copy(w.buffer[w.bi:], data) 54 | w.bi += len(data) 55 | crc := crc32.ChecksumIEEE(w.buffer[w.bi-len(data)-dataLenSize:w.bi]) 56 | binary.LittleEndian.PutUint32(w.buffer[w.bi:], crc) 57 | w.bi += crcSize 58 | return nil 59 | } 60 | 61 | func (w *WAL) Flush() error { 62 | for ; w.bi < w.bufferSize; w.bi ++ { 63 | // Pad the remaining space with 0. 64 | w.buffer[w.bi] = 0 65 | } 66 | for i := 0; i < w.bi; { 67 | n, err := w.f.Write(w.buffer[i:]) 68 | if err != nil { 69 | return err 70 | } 71 | i += n 72 | } 73 | w.bi = 0 74 | if w.fsync { 75 | return w.f.Sync() 76 | } 77 | return nil 78 | } 79 | 80 | func (w *WAL) Close() error { 81 | if err := w.Flush(); err != nil { 82 | return err 83 | } 84 | return w.f.Close() 85 | } 86 | 87 | type LogIterator struct { 88 | f *os.File 89 | buffer []byte 90 | bi int 91 | bufferSize, maxRecordSize int 92 | } 93 | 94 | func NewLogIterator(filename string, bufferSize, maxRecordSize int) (*LogIterator, error) { 95 | if maxRecordSize + dataLenSize + crcSize > bufferSize { 96 | return nil, fmt.Errorf("error in size configuration") 97 | } 98 | file, err := os.Open(filename) 99 | if err != nil { 100 | return nil, err 101 | } 102 | itr := &LogIterator{ 103 | f: file, 104 | buffer: make([]byte, bufferSize), 105 | bi: 0, 106 | bufferSize: bufferSize, 107 | maxRecordSize: maxRecordSize + dataLenSize + crcSize, 108 | } 109 | return itr, itr.read() 110 | } 111 | 112 | func (itr *LogIterator) read() error { 113 | for i := 0; i < itr.bufferSize; { 114 | n, err := itr.f.Read(itr.buffer[i:]) 115 | if err != nil { 116 | return err 117 | } 118 | i += n 119 | } 120 | return nil 121 | } 122 | 123 | func (itr *LogIterator) Next() ([]byte, error) { 124 | if itr.bi + itr.maxRecordSize > itr.bufferSize { 125 | if err := itr.read(); err != nil { 126 | return nil, err 127 | } 128 | itr.bi = 0 129 | } 130 | len := int(binary.LittleEndian.Uint32(itr.buffer[itr.bi:])) 131 | itr.bi += dataLenSize 132 | data := make([]byte, len) 133 | copy(data, itr.buffer[itr.bi:]) 134 | itr.bi += len 135 | crc := binary.LittleEndian.Uint32(itr.buffer[itr.bi:]) 136 | expectedCRC := crc32.ChecksumIEEE(itr.buffer[itr.bi-len-dataLenSize:itr.bi]) 137 | itr.bi += crcSize 138 | if crc != expectedCRC { 139 | return nil, fmt.Errorf("crc mismatch; want %d; stored %d", expectedCRC, crc) 140 | } 141 | return data, nil 142 | } 143 | 144 | func (itr *LogIterator) Close() error { 145 | return itr.f.Close() 146 | } 147 | -------------------------------------------------------------------------------- /wal_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | // go test -v -run TestWAL 4 | 5 | import ( 6 | "fmt" 7 | "io" 8 | "os" 9 | "reflect" 10 | "testing" 11 | ) 12 | 13 | func write(w *WAL, data string, t *testing.T) { 14 | if err := w.AddRecord([]byte(data)); err != nil { 15 | t.Fatalf("failed to write data %s; %v", data, err) 16 | } 17 | } 18 | 19 | func read(itr *LogIterator, t *testing.T) (string, bool) { 20 | data, err := itr.Next() 21 | if err != nil && err != io.EOF { 22 | t.Fatalf("failed to read log; %v", err) 23 | } 24 | return string(data), err == io.EOF 25 | } 26 | 27 | func TestWAL(t *testing.T) { 28 | defer os.Remove("test.log") 29 | bufferSize := 35 30 | maxRecordSize := 15 31 | w, err := NewWAL("test.log", true, bufferSize, maxRecordSize) 32 | if err != nil { 33 | t.Fatalf("failed to create WAL; %v", err) 34 | } 35 | data := []string{"hello world", "hello again", "hi world", "hi again"} 36 | for i, d := range data { 37 | t.Logf("write log entry: %d; content: %s", i, d) 38 | write(w, d, t) 39 | } 40 | if err := w.Close(); err != nil { 41 | t.Fatalf("failed to close; %v", err) 42 | } 43 | itr, err := NewLogIterator("test.log", bufferSize, maxRecordSize) 44 | if err != nil { 45 | t.Fatalf("failed to create iterator; %v", err) 46 | } 47 | var log []string 48 | for i := 0;; i ++{ 49 | t.Logf("reading log entry: %d", i) 50 | l, done := read(itr, t) 51 | if done { 52 | break 53 | } 54 | log = append(log, l) 55 | } 56 | if err := itr.Close(); err != nil { 57 | t.Fatalf("failed to close; %v", err) 58 | } 59 | if !reflect.DeepEqual(data, log) { 60 | t.Errorf("want log %v; got %v", data, log) 61 | } 62 | } 63 | 64 | // go test -bench=. 65 | 66 | func benchmark(fsync bool, b *testing.B) { 67 | filename := fmt.Sprintf("btest_%t.log", fsync) 68 | defer os.Remove(filename) 69 | bufferSize := 35 70 | maxRecordSize := 15 71 | w, err := NewWAL(filename, fsync, bufferSize, maxRecordSize) 72 | if err != nil { 73 | b.Fatalf("failed to create WAL; %v", err) 74 | } 75 | data := make([]byte, 12) 76 | for i := 0; i < 100; i ++ { 77 | if err := w.AddRecord(data); err != nil { 78 | b.Fatalf("failed to write data; %v", err) 79 | } 80 | } 81 | if err := w.Close(); err != nil { 82 | b.Fatalf("failed to close; %v", err) 83 | } 84 | } 85 | 86 | func BenchmarkSyncWAL(b *testing.B) { 87 | benchmark(true, b) 88 | } 89 | 90 | func BenchmarkAsyncWAL(b *testing.B) { 91 | benchmark(false, b) 92 | } --------------------------------------------------------------------------------