├── .github └── workflows │ └── go.yml ├── .gitignore ├── LICENSE ├── README.md ├── go.mod ├── go.sum ├── wal.go └── wal_test.go /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | 21 | - name: Check out code into the Go module directory 22 | uses: actions/checkout@v2 23 | 24 | - name: Get dependencies 25 | run: | 26 | go get -v -t -d ./... 27 | if [ -f Gopkg.toml ]; then 28 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 29 | dep ensure 30 | fi 31 | 32 | - name: Build 33 | run: go build -v . 34 | 35 | - name: Test 36 | run: go test -v . 37 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | testlog 2 | .idea 3 | vendor -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Joshua J Baker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `wal` 2 | [![GoDoc](https://godoc.org/github.com/tidwall/wal?status.svg)](https://godoc.org/github.com/tidwall/wal) 3 | 4 | A simple and fast write ahead log for Go. 5 | 6 | ## Features 7 | 8 | - High durability 9 | - Fast operations 10 | - Monotonic indexes 11 | - Batch writes 12 | - Log truncation from front or back. 13 | 14 | ## Getting Started 15 | 16 | ### Installing 17 | 18 | To start using `wal`, install Go and run `go get`: 19 | 20 | ```sh 21 | $ go get -u github.com/tidwall/wal 22 | ``` 23 | 24 | This will retrieve the library. 25 | 26 | ### Example 27 | 28 | ```go 29 | // open a new log file 30 | log, err := wal.Open("mylog", nil) 31 | 32 | // write some entries 33 | err = log.Write(1, []byte("first entry")) 34 | err = log.Write(2, []byte("second entry")) 35 | err = log.Write(3, []byte("third entry")) 36 | 37 | // read an entry 38 | data, err := log.Read(1) 39 | println(string(data)) // output: first entry 40 | 41 | // close the log 42 | err = log.Close() 43 | ``` 44 | 45 | Batch writes: 46 | 47 | ```go 48 | 49 | // write three entries as a batch 50 | batch := new(wal.Batch) 51 | batch.Write(1, []byte("first entry")) 52 | batch.Write(2, []byte("second entry")) 53 | batch.Write(3, []byte("third entry")) 54 | 55 | err = log.WriteBatch(batch) 56 | ``` 57 | 58 | Truncating: 59 | 60 | ```go 61 | // write some entries 62 | err = log.Write(1, []byte("first entry")) 63 | ... 64 | err = log.Write(1000, []byte("thousandth entry")) 65 | 66 | // truncate the log from index starting 350 and ending with 950. 67 | err = log.TruncateFront(350) 68 | err = log.TruncateBack(950) 69 | ``` 70 | 71 | 72 | 73 | ## Contact 74 | 75 | Josh Baker [@tidwall](http://twitter.com/tidwall) 76 | 77 | ## License 78 | 79 | `wal` source code is available under the MIT [License](/LICENSE). 80 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tidwall/wal 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/tidwall/gjson v1.10.2 7 | github.com/tidwall/tinylru v1.1.0 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= 2 | github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 3 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 4 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 5 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 6 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 7 | github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I= 8 | github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8= 9 | -------------------------------------------------------------------------------- /wal.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "encoding/binary" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "path/filepath" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "unicode/utf8" 17 | "unsafe" 18 | 19 | "github.com/tidwall/gjson" 20 | "github.com/tidwall/tinylru" 21 | ) 22 | 23 | var ( 24 | // ErrCorrupt is returns when the log is corrupt. 25 | ErrCorrupt = errors.New("log corrupt") 26 | 27 | // ErrClosed is returned when an operation cannot be completed because 28 | // the log is closed. 29 | ErrClosed = errors.New("log closed") 30 | 31 | // ErrNotFound is returned when an entry is not found. 32 | ErrNotFound = errors.New("not found") 33 | 34 | // ErrOutOfOrder is returned from Write() when the index is not equal to 35 | // LastIndex()+1. It's required that log monotonically grows by one and has 36 | // no gaps. Thus, the series 10,11,12,13,14 is valid, but 10,11,13,14 is 37 | // not because there's a gap between 11 and 13. Also, 10,12,11,13 is not 38 | // valid because 12 and 11 are out of order. 39 | ErrOutOfOrder = errors.New("out of order") 40 | 41 | // ErrOutOfRange is returned from TruncateFront() and TruncateBack() when 42 | // the index not in the range of the log's first and last index. Or, this 43 | // may be returned when the caller is attempting to remove *all* entries; 44 | // The log requires that at least one entry exists following a truncate. 45 | ErrOutOfRange = errors.New("out of range") 46 | ) 47 | 48 | // LogFormat is the format of the log files. 49 | type LogFormat byte 50 | 51 | const ( 52 | // Binary format writes entries in binary. This is the default and, unless 53 | // a good reason otherwise, should be used in production. 54 | Binary LogFormat = 0 55 | // JSON format writes entries as JSON lines. This causes larger, human 56 | // readable files. 57 | JSON LogFormat = 1 58 | ) 59 | 60 | // Options for Log 61 | type Options struct { 62 | // NoSync disables fsync after writes. This is less durable and puts the 63 | // log at risk of data loss when there's a server crash. 64 | NoSync bool 65 | // SegmentSize of each segment. This is just a target value, actual size 66 | // may differ. Default is 20 MB. 67 | SegmentSize int 68 | // LogFormat is the format of the log files. Default is Binary. 69 | LogFormat LogFormat 70 | // SegmentCacheSize is the maximum number of segments that will be held in 71 | // memory for caching. Increasing this value may enhance performance for 72 | // concurrent read operations. Default is 1 73 | SegmentCacheSize int 74 | // NoCopy allows for the Read() operation to return the raw underlying data 75 | // slice. This is an optimization to help minimize allocations. When this 76 | // option is set, do not modify the returned data because it may affect 77 | // other Read calls. Default false 78 | NoCopy bool 79 | // Perms represents the datafiles modes and permission bits 80 | DirPerms os.FileMode 81 | FilePerms os.FileMode 82 | } 83 | 84 | // DefaultOptions for Open(). 85 | var DefaultOptions = &Options{ 86 | NoSync: false, // Fsync after every write 87 | SegmentSize: 20971520, // 20 MB log segment files. 88 | LogFormat: Binary, // Binary format is small and fast. 89 | SegmentCacheSize: 2, // Number of cached in-memory segments 90 | NoCopy: false, // Make a new copy of data for every Read call. 91 | DirPerms: 0750, // Permissions for the created directories 92 | FilePerms: 0640, // Permissions for the created data files 93 | } 94 | 95 | // Log represents a write ahead log 96 | type Log struct { 97 | mu sync.RWMutex 98 | path string // absolute path to log directory 99 | opts Options // log options 100 | closed bool // log is closed 101 | corrupt bool // log may be corrupt 102 | segments []*segment // all known log segments 103 | firstIndex uint64 // index of the first entry in log 104 | lastIndex uint64 // index of the last entry in log 105 | sfile *os.File // tail segment file handle 106 | wbatch Batch // reusable write batch 107 | scache tinylru.LRU // segment entries cache 108 | } 109 | 110 | // segment represents a single segment file. 111 | type segment struct { 112 | path string // path of segment file 113 | index uint64 // first index of segment 114 | ebuf []byte // cached entries buffer 115 | epos []bpos // cached entries positions in buffer 116 | } 117 | 118 | type bpos struct { 119 | pos int // byte position 120 | end int // one byte past pos 121 | } 122 | 123 | // Open a new write ahead log 124 | func Open(path string, opts *Options) (*Log, error) { 125 | if opts == nil { 126 | opts = DefaultOptions 127 | } 128 | if opts.SegmentCacheSize <= 0 { 129 | opts.SegmentCacheSize = DefaultOptions.SegmentCacheSize 130 | } 131 | if opts.SegmentSize <= 0 { 132 | opts.SegmentSize = DefaultOptions.SegmentSize 133 | } 134 | if opts.DirPerms == 0 { 135 | opts.DirPerms = DefaultOptions.DirPerms 136 | } 137 | if opts.FilePerms == 0 { 138 | opts.FilePerms = DefaultOptions.FilePerms 139 | } 140 | 141 | var err error 142 | path, err = abs(path) 143 | if err != nil { 144 | return nil, err 145 | } 146 | l := &Log{path: path, opts: *opts} 147 | l.scache.Resize(l.opts.SegmentCacheSize) 148 | if err := os.MkdirAll(path, l.opts.DirPerms); err != nil { 149 | return nil, err 150 | } 151 | if err := l.load(); err != nil { 152 | return nil, err 153 | } 154 | return l, nil 155 | } 156 | 157 | func abs(path string) (string, error) { 158 | if path == ":memory:" { 159 | return "", errors.New("in-memory log not supported") 160 | } 161 | return filepath.Abs(path) 162 | } 163 | 164 | func (l *Log) pushCache(segIdx int) { 165 | _, _, _, v, evicted := 166 | l.scache.SetEvicted(segIdx, l.segments[segIdx]) 167 | if evicted { 168 | s := v.(*segment) 169 | s.ebuf = nil 170 | s.epos = nil 171 | } 172 | } 173 | 174 | // load all the segments. This operation also cleans up any START/END segments. 175 | func (l *Log) load() error { 176 | fis, err := ioutil.ReadDir(l.path) 177 | if err != nil { 178 | return err 179 | } 180 | startIdx := -1 181 | endIdx := -1 182 | for _, fi := range fis { 183 | name := fi.Name() 184 | if fi.IsDir() || len(name) < 20 { 185 | continue 186 | } 187 | index, err := strconv.ParseUint(name[:20], 10, 64) 188 | if err != nil || index == 0 { 189 | continue 190 | } 191 | isStart := len(name) == 26 && strings.HasSuffix(name, ".START") 192 | isEnd := len(name) == 24 && strings.HasSuffix(name, ".END") 193 | if len(name) == 20 || isStart || isEnd { 194 | if isStart { 195 | startIdx = len(l.segments) 196 | } else if isEnd && endIdx == -1 { 197 | endIdx = len(l.segments) 198 | } 199 | l.segments = append(l.segments, &segment{ 200 | index: index, 201 | path: filepath.Join(l.path, name), 202 | }) 203 | } 204 | } 205 | if len(l.segments) == 0 { 206 | // Create a new log 207 | l.segments = append(l.segments, &segment{ 208 | index: 1, 209 | path: filepath.Join(l.path, segmentName(1)), 210 | }) 211 | l.firstIndex = 1 212 | l.lastIndex = 0 213 | l.sfile, err = os.OpenFile(l.segments[0].path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, l.opts.FilePerms) 214 | return err 215 | } 216 | // Open existing log. Clean up log if START of END segments exists. 217 | if startIdx != -1 { 218 | if endIdx != -1 { 219 | // There should not be a START and END at the same time 220 | return ErrCorrupt 221 | } 222 | // Delete all files leading up to START 223 | for i := 0; i < startIdx; i++ { 224 | if err := os.Remove(l.segments[i].path); err != nil { 225 | return err 226 | } 227 | } 228 | l.segments = append([]*segment{}, l.segments[startIdx:]...) 229 | // Rename the START segment 230 | orgPath := l.segments[0].path 231 | finalPath := orgPath[:len(orgPath)-len(".START")] 232 | err := os.Rename(orgPath, finalPath) 233 | if err != nil { 234 | return err 235 | } 236 | l.segments[0].path = finalPath 237 | } 238 | if endIdx != -1 { 239 | // Delete all files following END 240 | for i := len(l.segments) - 1; i > endIdx; i-- { 241 | if err := os.Remove(l.segments[i].path); err != nil { 242 | return err 243 | } 244 | } 245 | l.segments = append([]*segment{}, l.segments[:endIdx+1]...) 246 | if len(l.segments) > 1 && l.segments[len(l.segments)-2].index == 247 | l.segments[len(l.segments)-1].index { 248 | // remove the segment prior to the END segment because it shares 249 | // the same starting index. 250 | l.segments[len(l.segments)-2] = l.segments[len(l.segments)-1] 251 | l.segments = l.segments[:len(l.segments)-1] 252 | } 253 | // Rename the END segment 254 | orgPath := l.segments[len(l.segments)-1].path 255 | finalPath := orgPath[:len(orgPath)-len(".END")] 256 | err := os.Rename(orgPath, finalPath) 257 | if err != nil { 258 | return err 259 | } 260 | l.segments[len(l.segments)-1].path = finalPath 261 | } 262 | l.firstIndex = l.segments[0].index 263 | // Open the last segment for appending 264 | lseg := l.segments[len(l.segments)-1] 265 | l.sfile, err = os.OpenFile(lseg.path, os.O_WRONLY, l.opts.FilePerms) 266 | if err != nil { 267 | return err 268 | } 269 | if _, err := l.sfile.Seek(0, 2); err != nil { 270 | return err 271 | } 272 | // Load the last segment entries 273 | if err := l.loadSegmentEntries(lseg); err != nil { 274 | return err 275 | } 276 | l.lastIndex = lseg.index + uint64(len(lseg.epos)) - 1 277 | return nil 278 | } 279 | 280 | // segmentName returns a 20-byte textual representation of an index 281 | // for lexical ordering. This is used for the file names of log segments. 282 | func segmentName(index uint64) string { 283 | return fmt.Sprintf("%020d", index) 284 | } 285 | 286 | // Close the log. 287 | func (l *Log) Close() error { 288 | l.mu.Lock() 289 | defer l.mu.Unlock() 290 | if l.closed { 291 | if l.corrupt { 292 | return ErrCorrupt 293 | } 294 | return ErrClosed 295 | } 296 | if err := l.sfile.Sync(); err != nil { 297 | return err 298 | } 299 | if err := l.sfile.Close(); err != nil { 300 | return err 301 | } 302 | l.closed = true 303 | if l.corrupt { 304 | return ErrCorrupt 305 | } 306 | return nil 307 | } 308 | 309 | // Write an entry to the log. 310 | func (l *Log) Write(index uint64, data []byte) error { 311 | l.mu.Lock() 312 | defer l.mu.Unlock() 313 | if l.corrupt { 314 | return ErrCorrupt 315 | } else if l.closed { 316 | return ErrClosed 317 | } 318 | l.wbatch.Clear() 319 | l.wbatch.Write(index, data) 320 | return l.writeBatch(&l.wbatch) 321 | } 322 | 323 | func (l *Log) appendEntry(dst []byte, index uint64, data []byte) (out []byte, 324 | epos bpos) { 325 | if l.opts.LogFormat == JSON { 326 | return appendJSONEntry(dst, index, data) 327 | } 328 | return appendBinaryEntry(dst, data) 329 | } 330 | 331 | // Cycle the old segment for a new segment. 332 | func (l *Log) cycle() error { 333 | if err := l.sfile.Sync(); err != nil { 334 | return err 335 | } 336 | if err := l.sfile.Close(); err != nil { 337 | return err 338 | } 339 | // cache the previous segment 340 | l.pushCache(len(l.segments) - 1) 341 | s := &segment{ 342 | index: l.lastIndex + 1, 343 | path: filepath.Join(l.path, segmentName(l.lastIndex+1)), 344 | } 345 | var err error 346 | l.sfile, err = os.OpenFile(s.path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, l.opts.FilePerms) 347 | if err != nil { 348 | return err 349 | } 350 | l.segments = append(l.segments, s) 351 | return nil 352 | } 353 | 354 | func appendJSONEntry(dst []byte, index uint64, data []byte) (out []byte, 355 | epos bpos) { 356 | // {"index":number,"data":string} 357 | pos := len(dst) 358 | dst = append(dst, `{"index":"`...) 359 | dst = strconv.AppendUint(dst, index, 10) 360 | dst = append(dst, `","data":`...) 361 | dst = appendJSONData(dst, data) 362 | dst = append(dst, '}', '\n') 363 | return dst, bpos{pos, len(dst)} 364 | } 365 | 366 | func appendJSONData(dst []byte, s []byte) []byte { 367 | if utf8.Valid(s) { 368 | b, _ := json.Marshal(*(*string)(unsafe.Pointer(&s))) 369 | dst = append(dst, '"', '+') 370 | return append(dst, b[1:]...) 371 | } 372 | dst = append(dst, '"', '$') 373 | dst = append(dst, base64.URLEncoding.EncodeToString(s)...) 374 | return append(dst, '"') 375 | } 376 | 377 | func appendBinaryEntry(dst []byte, data []byte) (out []byte, epos bpos) { 378 | // data_size + data 379 | pos := len(dst) 380 | dst = appendUvarint(dst, uint64(len(data))) 381 | dst = append(dst, data...) 382 | return dst, bpos{pos, len(dst)} 383 | } 384 | 385 | func appendUvarint(dst []byte, x uint64) []byte { 386 | var buf [10]byte 387 | n := binary.PutUvarint(buf[:], x) 388 | dst = append(dst, buf[:n]...) 389 | return dst 390 | } 391 | 392 | // Batch of entries. Used to write multiple entries at once using WriteBatch(). 393 | type Batch struct { 394 | entries []batchEntry 395 | datas []byte 396 | } 397 | 398 | type batchEntry struct { 399 | index uint64 400 | size int 401 | } 402 | 403 | // Write an entry to the batch 404 | func (b *Batch) Write(index uint64, data []byte) { 405 | b.entries = append(b.entries, batchEntry{index, len(data)}) 406 | b.datas = append(b.datas, data...) 407 | } 408 | 409 | // Clear the batch for reuse. 410 | func (b *Batch) Clear() { 411 | b.entries = b.entries[:0] 412 | b.datas = b.datas[:0] 413 | } 414 | 415 | // WriteBatch writes the entries in the batch to the log in the order that they 416 | // were added to the batch. The batch is cleared upon a successful return. 417 | func (l *Log) WriteBatch(b *Batch) error { 418 | l.mu.Lock() 419 | defer l.mu.Unlock() 420 | if l.corrupt { 421 | return ErrCorrupt 422 | } else if l.closed { 423 | return ErrClosed 424 | } 425 | if len(b.entries) == 0 { 426 | return nil 427 | } 428 | return l.writeBatch(b) 429 | } 430 | 431 | func (l *Log) writeBatch(b *Batch) error { 432 | // check that all indexes in batch are sane 433 | for i := 0; i < len(b.entries); i++ { 434 | if b.entries[i].index != l.lastIndex+uint64(i+1) { 435 | return ErrOutOfOrder 436 | } 437 | } 438 | // load the tail segment 439 | s := l.segments[len(l.segments)-1] 440 | if len(s.ebuf) > l.opts.SegmentSize { 441 | // tail segment has reached capacity. Close it and create a new one. 442 | if err := l.cycle(); err != nil { 443 | return err 444 | } 445 | s = l.segments[len(l.segments)-1] 446 | } 447 | 448 | mark := len(s.ebuf) 449 | datas := b.datas 450 | for i := 0; i < len(b.entries); i++ { 451 | data := datas[:b.entries[i].size] 452 | var epos bpos 453 | s.ebuf, epos = l.appendEntry(s.ebuf, b.entries[i].index, data) 454 | s.epos = append(s.epos, epos) 455 | if len(s.ebuf) >= l.opts.SegmentSize { 456 | // segment has reached capacity, cycle now 457 | if _, err := l.sfile.Write(s.ebuf[mark:]); err != nil { 458 | return err 459 | } 460 | l.lastIndex = b.entries[i].index 461 | if err := l.cycle(); err != nil { 462 | return err 463 | } 464 | s = l.segments[len(l.segments)-1] 465 | mark = 0 466 | } 467 | datas = datas[b.entries[i].size:] 468 | } 469 | if len(s.ebuf)-mark > 0 { 470 | if _, err := l.sfile.Write(s.ebuf[mark:]); err != nil { 471 | return err 472 | } 473 | l.lastIndex = b.entries[len(b.entries)-1].index 474 | } 475 | if !l.opts.NoSync { 476 | if err := l.sfile.Sync(); err != nil { 477 | return err 478 | } 479 | } 480 | b.Clear() 481 | return nil 482 | } 483 | 484 | // FirstIndex returns the index of the first entry in the log. Returns zero 485 | // when log has no entries. 486 | func (l *Log) FirstIndex() (index uint64, err error) { 487 | l.mu.RLock() 488 | defer l.mu.RUnlock() 489 | if l.corrupt { 490 | return 0, ErrCorrupt 491 | } else if l.closed { 492 | return 0, ErrClosed 493 | } 494 | // We check the lastIndex for zero because the firstIndex is always one or 495 | // more, even when there's no entries 496 | if l.lastIndex == 0 { 497 | return 0, nil 498 | } 499 | return l.firstIndex, nil 500 | } 501 | 502 | // LastIndex returns the index of the last entry in the log. Returns zero when 503 | // log has no entries. 504 | func (l *Log) LastIndex() (index uint64, err error) { 505 | l.mu.RLock() 506 | defer l.mu.RUnlock() 507 | if l.corrupt { 508 | return 0, ErrCorrupt 509 | } else if l.closed { 510 | return 0, ErrClosed 511 | } 512 | if l.lastIndex == 0 { 513 | return 0, nil 514 | } 515 | return l.lastIndex, nil 516 | } 517 | 518 | // findSegment performs a bsearch on the segments 519 | func (l *Log) findSegment(index uint64) int { 520 | i, j := 0, len(l.segments) 521 | for i < j { 522 | h := i + (j-i)/2 523 | if index >= l.segments[h].index { 524 | i = h + 1 525 | } else { 526 | j = h 527 | } 528 | } 529 | return i - 1 530 | } 531 | 532 | func (l *Log) loadSegmentEntries(s *segment) error { 533 | data, err := ioutil.ReadFile(s.path) 534 | if err != nil { 535 | return err 536 | } 537 | ebuf := data 538 | var epos []bpos 539 | var pos int 540 | for exidx := s.index; len(data) > 0; exidx++ { 541 | var n int 542 | if l.opts.LogFormat == JSON { 543 | n, err = loadNextJSONEntry(data) 544 | } else { 545 | n, err = loadNextBinaryEntry(data) 546 | } 547 | if err != nil { 548 | return err 549 | } 550 | data = data[n:] 551 | epos = append(epos, bpos{pos, pos + n}) 552 | pos += n 553 | } 554 | s.ebuf = ebuf 555 | s.epos = epos 556 | return nil 557 | } 558 | 559 | func loadNextJSONEntry(data []byte) (n int, err error) { 560 | // {"index":number,"data":string} 561 | idx := bytes.IndexByte(data, '\n') 562 | if idx == -1 { 563 | return 0, ErrCorrupt 564 | } 565 | line := data[:idx] 566 | dres := gjson.Get(*(*string)(unsafe.Pointer(&line)), "data") 567 | if dres.Type != gjson.String { 568 | return 0, ErrCorrupt 569 | } 570 | return idx + 1, nil 571 | } 572 | 573 | func loadNextBinaryEntry(data []byte) (n int, err error) { 574 | // data_size + data 575 | size, n := binary.Uvarint(data) 576 | if n <= 0 { 577 | return 0, ErrCorrupt 578 | } 579 | if uint64(len(data)-n) < size { 580 | return 0, ErrCorrupt 581 | } 582 | return n + int(size), nil 583 | } 584 | 585 | // loadSegment loads the segment entries into memory, pushes it to the front 586 | // of the lru cache, and returns it. 587 | func (l *Log) loadSegment(index uint64) (*segment, error) { 588 | // check the last segment first. 589 | lseg := l.segments[len(l.segments)-1] 590 | if index >= lseg.index { 591 | return lseg, nil 592 | } 593 | // check the most recent cached segment 594 | var rseg *segment 595 | l.scache.Range(func(_, v interface{}) bool { 596 | s := v.(*segment) 597 | if index >= s.index && index < s.index+uint64(len(s.epos)) { 598 | rseg = s 599 | } 600 | return false 601 | }) 602 | if rseg != nil { 603 | return rseg, nil 604 | } 605 | // find in the segment array 606 | idx := l.findSegment(index) 607 | s := l.segments[idx] 608 | if len(s.epos) == 0 { 609 | // load the entries from cache 610 | if err := l.loadSegmentEntries(s); err != nil { 611 | return nil, err 612 | } 613 | } 614 | // push the segment to the front of the cache 615 | l.pushCache(idx) 616 | return s, nil 617 | } 618 | 619 | // Read an entry from the log. Returns a byte slice containing the data entry. 620 | func (l *Log) Read(index uint64) (data []byte, err error) { 621 | l.mu.RLock() 622 | defer l.mu.RUnlock() 623 | if l.corrupt { 624 | return nil, ErrCorrupt 625 | } else if l.closed { 626 | return nil, ErrClosed 627 | } 628 | if index == 0 || index < l.firstIndex || index > l.lastIndex { 629 | return nil, ErrNotFound 630 | } 631 | s, err := l.loadSegment(index) 632 | if err != nil { 633 | return nil, err 634 | } 635 | epos := s.epos[index-s.index] 636 | edata := s.ebuf[epos.pos:epos.end] 637 | if l.opts.LogFormat == JSON { 638 | return readJSON(edata) 639 | } 640 | // binary read 641 | size, n := binary.Uvarint(edata) 642 | if n <= 0 { 643 | return nil, ErrCorrupt 644 | } 645 | if uint64(len(edata)-n) < size { 646 | return nil, ErrCorrupt 647 | } 648 | if l.opts.NoCopy { 649 | data = edata[n : uint64(n)+size] 650 | } else { 651 | data = make([]byte, size) 652 | copy(data, edata[n:]) 653 | } 654 | return data, nil 655 | } 656 | 657 | //go:noinline 658 | func readJSON(edata []byte) ([]byte, error) { 659 | var data []byte 660 | s := gjson.Get(*(*string)(unsafe.Pointer(&edata)), "data").String() 661 | if len(s) > 0 && s[0] == '$' { 662 | var err error 663 | data, err = base64.URLEncoding.DecodeString(s[1:]) 664 | if err != nil { 665 | return nil, ErrCorrupt 666 | } 667 | } else if len(s) > 0 && s[0] == '+' { 668 | data = make([]byte, len(s[1:])) 669 | copy(data, s[1:]) 670 | } else { 671 | return nil, ErrCorrupt 672 | } 673 | return data, nil 674 | } 675 | 676 | // ClearCache clears the segment cache 677 | func (l *Log) ClearCache() error { 678 | l.mu.Lock() 679 | defer l.mu.Unlock() 680 | if l.corrupt { 681 | return ErrCorrupt 682 | } else if l.closed { 683 | return ErrClosed 684 | } 685 | l.clearCache() 686 | return nil 687 | } 688 | func (l *Log) clearCache() { 689 | l.scache.Range(func(_, v interface{}) bool { 690 | s := v.(*segment) 691 | s.ebuf = nil 692 | s.epos = nil 693 | return true 694 | }) 695 | l.scache = tinylru.LRU{} 696 | l.scache.Resize(l.opts.SegmentCacheSize) 697 | } 698 | 699 | // TruncateFront truncates the front of the log by removing all entries that 700 | // are before the provided `index`. In other words the entry at 701 | // `index` becomes the first entry in the log. 702 | func (l *Log) TruncateFront(index uint64) error { 703 | l.mu.Lock() 704 | defer l.mu.Unlock() 705 | if l.corrupt { 706 | return ErrCorrupt 707 | } else if l.closed { 708 | return ErrClosed 709 | } 710 | return l.truncateFront(index) 711 | } 712 | func (l *Log) truncateFront(index uint64) (err error) { 713 | if index == 0 || l.lastIndex == 0 || 714 | index < l.firstIndex || index > l.lastIndex { 715 | return ErrOutOfRange 716 | } 717 | if index == l.firstIndex { 718 | // nothing to truncate 719 | return nil 720 | } 721 | segIdx := l.findSegment(index) 722 | var s *segment 723 | s, err = l.loadSegment(index) 724 | if err != nil { 725 | return err 726 | } 727 | epos := s.epos[index-s.index:] 728 | ebuf := s.ebuf[epos[0].pos:] 729 | // Create a temp file contains the truncated segment. 730 | tempName := filepath.Join(l.path, "TEMP") 731 | if err = func() error { 732 | f, err := os.OpenFile(tempName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, l.opts.FilePerms) 733 | if err != nil { 734 | return err 735 | } 736 | defer f.Close() 737 | if _, err := f.Write(ebuf); err != nil { 738 | return err 739 | } 740 | if err := f.Sync(); err != nil { 741 | return err 742 | } 743 | return f.Close() 744 | }(); err != nil { 745 | return fmt.Errorf("failed to create temp file for new start segment: %w", err) 746 | } 747 | // Rename the TEMP file to it's START file name. 748 | startName := filepath.Join(l.path, segmentName(index)+".START") 749 | if err = os.Rename(tempName, startName); err != nil { 750 | return err 751 | } 752 | // The log was truncated but still needs some file cleanup. Any errors 753 | // following this message will not cause an on-disk data ocorruption, but 754 | // may cause an inconsistency with the current program, so we'll return 755 | // ErrCorrupt so the the user can attempt a recover by calling Close() 756 | // followed by Open(). 757 | defer func() { 758 | if v := recover(); v != nil { 759 | err = ErrCorrupt 760 | l.corrupt = true 761 | } 762 | }() 763 | if segIdx == len(l.segments)-1 { 764 | // Close the tail segment file 765 | if err = l.sfile.Close(); err != nil { 766 | return err 767 | } 768 | } 769 | // Delete truncated segment files 770 | for i := 0; i <= segIdx; i++ { 771 | if err = os.Remove(l.segments[i].path); err != nil { 772 | return err 773 | } 774 | } 775 | // Rename the START file to the final truncated segment name. 776 | newName := filepath.Join(l.path, segmentName(index)) 777 | if err = os.Rename(startName, newName); err != nil { 778 | return err 779 | } 780 | s.path = newName 781 | s.index = index 782 | if segIdx == len(l.segments)-1 { 783 | // Reopen the tail segment file 784 | if l.sfile, err = os.OpenFile(newName, os.O_WRONLY, l.opts.FilePerms); err != nil { 785 | return err 786 | } 787 | var n int64 788 | if n, err = l.sfile.Seek(0, 2); err != nil { 789 | return err 790 | } 791 | if n != int64(len(ebuf)) { 792 | err = errors.New("invalid seek") 793 | return err 794 | } 795 | // Load the last segment entries 796 | if err = l.loadSegmentEntries(s); err != nil { 797 | return err 798 | } 799 | } 800 | l.segments = append([]*segment{}, l.segments[segIdx:]...) 801 | l.firstIndex = index 802 | l.clearCache() 803 | return nil 804 | } 805 | 806 | // TruncateBack truncates the back of the log by removing all entries that 807 | // are after the provided `index`. In other words the entry at `index` 808 | // becomes the last entry in the log. 809 | func (l *Log) TruncateBack(index uint64) error { 810 | l.mu.Lock() 811 | defer l.mu.Unlock() 812 | if l.corrupt { 813 | return ErrCorrupt 814 | } else if l.closed { 815 | return ErrClosed 816 | } 817 | return l.truncateBack(index) 818 | } 819 | 820 | func (l *Log) truncateBack(index uint64) (err error) { 821 | if index == 0 || l.lastIndex == 0 || 822 | index < l.firstIndex || index > l.lastIndex { 823 | return ErrOutOfRange 824 | } 825 | if index == l.lastIndex { 826 | // nothing to truncate 827 | return nil 828 | } 829 | segIdx := l.findSegment(index) 830 | var s *segment 831 | s, err = l.loadSegment(index) 832 | if err != nil { 833 | return err 834 | } 835 | epos := s.epos[:index-s.index+1] 836 | ebuf := s.ebuf[:epos[len(epos)-1].end] 837 | // Create a temp file contains the truncated segment. 838 | tempName := filepath.Join(l.path, "TEMP") 839 | if err = func() error { 840 | f, err := os.OpenFile(tempName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, l.opts.FilePerms) 841 | if err != nil { 842 | return err 843 | } 844 | defer f.Close() 845 | if _, err := f.Write(ebuf); err != nil { 846 | return err 847 | } 848 | if err := f.Sync(); err != nil { 849 | return err 850 | } 851 | return f.Close() 852 | }(); err != nil { 853 | return fmt.Errorf("failed to create temp file for new end segment: %w", err) 854 | } 855 | // Rename the TEMP file to it's END file name. 856 | endName := filepath.Join(l.path, segmentName(s.index)+".END") 857 | if err = os.Rename(tempName, endName); err != nil { 858 | return err 859 | } 860 | // The log was truncated but still needs some file cleanup. Any errors 861 | // following this message will not cause an on-disk data ocorruption, but 862 | // may cause an inconsistency with the current program, so we'll return 863 | // ErrCorrupt so the the user can attempt a recover by calling Close() 864 | // followed by Open(). 865 | defer func() { 866 | if v := recover(); v != nil { 867 | err = ErrCorrupt 868 | l.corrupt = true 869 | } 870 | }() 871 | 872 | // Close the tail segment file 873 | if err = l.sfile.Close(); err != nil { 874 | return err 875 | } 876 | // Delete truncated segment files 877 | for i := segIdx; i < len(l.segments); i++ { 878 | if err = os.Remove(l.segments[i].path); err != nil { 879 | return err 880 | } 881 | } 882 | // Rename the END file to the final truncated segment name. 883 | newName := filepath.Join(l.path, segmentName(s.index)) 884 | if err = os.Rename(endName, newName); err != nil { 885 | return err 886 | } 887 | // Reopen the tail segment file 888 | if l.sfile, err = os.OpenFile(newName, os.O_WRONLY, l.opts.FilePerms); err != nil { 889 | return err 890 | } 891 | var n int64 892 | n, err = l.sfile.Seek(0, 2) 893 | if err != nil { 894 | return err 895 | } 896 | if n != int64(len(ebuf)) { 897 | err = errors.New("invalid seek") 898 | return err 899 | } 900 | s.path = newName 901 | l.segments = append([]*segment{}, l.segments[:segIdx+1]...) 902 | l.lastIndex = index 903 | l.clearCache() 904 | if err = l.loadSegmentEntries(s); err != nil { 905 | return err 906 | } 907 | return nil 908 | } 909 | 910 | // Sync performs an fsync on the log. This is not necessary when the 911 | // NoSync option is set to false. 912 | func (l *Log) Sync() error { 913 | l.mu.Lock() 914 | defer l.mu.Unlock() 915 | if l.corrupt { 916 | return ErrCorrupt 917 | } else if l.closed { 918 | return ErrClosed 919 | } 920 | return l.sfile.Sync() 921 | } 922 | -------------------------------------------------------------------------------- /wal_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "math/rand" 7 | "os" 8 | "strings" 9 | "sync/atomic" 10 | "testing" 11 | ) 12 | 13 | func dataStr(index uint64) string { 14 | if index%2 == 0 { 15 | return fmt.Sprintf("data-\"%d\"", index) 16 | } 17 | return fmt.Sprintf("data-'%d'", index) 18 | } 19 | 20 | func testLog(t *testing.T, opts *Options, N int) { 21 | logPath := "testlog/" + strings.Join(strings.Split(t.Name(), "/")[1:], "/") 22 | l, err := Open(logPath, opts) 23 | if err != nil { 24 | t.Fatal(err) 25 | } 26 | defer l.Close() 27 | 28 | // FirstIndex - should be zero 29 | n, err := l.FirstIndex() 30 | if err != nil { 31 | t.Fatal(err) 32 | } 33 | if n != 0 { 34 | t.Fatalf("expected %d, got %d", 0, n) 35 | } 36 | 37 | // LastIndex - should be zero 38 | n, err = l.LastIndex() 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | if n != 0 { 43 | t.Fatalf("expected %d, got %d", 0, n) 44 | } 45 | 46 | for i := 1; i <= N; i++ { 47 | // Write - try to append previous index, should fail 48 | err = l.Write(uint64(i-1), nil) 49 | if err != ErrOutOfOrder { 50 | t.Fatalf("expected %v, got %v", ErrOutOfOrder, err) 51 | } 52 | // Write - append next item 53 | err = l.Write(uint64(i), []byte(dataStr(uint64(i)))) 54 | if err != nil { 55 | t.Fatalf("expected %v, got %v", nil, err) 56 | } 57 | // Write - get next item 58 | data, err := l.Read(uint64(i)) 59 | if err != nil { 60 | t.Fatalf("expected %v, got %v", nil, err) 61 | } 62 | if string(data) != dataStr(uint64(i)) { 63 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 64 | } 65 | } 66 | 67 | // Read -- should fail, not found 68 | _, err = l.Read(0) 69 | if err != ErrNotFound { 70 | t.Fatalf("expected %v, got %v", ErrNotFound, err) 71 | } 72 | // Read -- read back all entries 73 | for i := 1; i <= N; i++ { 74 | data, err := l.Read(uint64(i)) 75 | if err != nil { 76 | t.Fatalf("error while getting %d", i) 77 | } 78 | if string(data) != dataStr(uint64(i)) { 79 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 80 | } 81 | } 82 | // Read -- read back first half entries 83 | for i := 1; i <= N/2; i++ { 84 | data, err := l.Read(uint64(i)) 85 | if err != nil { 86 | t.Fatalf("error while getting %d", i) 87 | } 88 | if string(data) != dataStr(uint64(i)) { 89 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 90 | } 91 | } 92 | // Read -- read second third entries 93 | for i := N / 3; i <= N/3+N/3; i++ { 94 | data, err := l.Read(uint64(i)) 95 | if err != nil { 96 | t.Fatalf("error while getting %d", i) 97 | } 98 | if string(data) != dataStr(uint64(i)) { 99 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 100 | } 101 | } 102 | 103 | // Read -- random access 104 | for _, v := range rand.Perm(N) { 105 | index := uint64(v + 1) 106 | data, err := l.Read(index) 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | if dataStr(index) != string(data) { 111 | t.Fatalf("expected %v, got %v", dataStr(index), string(data)) 112 | } 113 | } 114 | 115 | // FirstIndex/LastIndex -- check valid first and last indexes 116 | n, err = l.FirstIndex() 117 | if err != nil { 118 | t.Fatal(err) 119 | } 120 | if n != 1 { 121 | t.Fatalf("expected %d, got %d", 1, n) 122 | } 123 | n, err = l.LastIndex() 124 | if err != nil { 125 | t.Fatal(err) 126 | } 127 | if n != uint64(N) { 128 | t.Fatalf("expected %d, got %d", N, n) 129 | } 130 | 131 | // Close -- close the log 132 | if err := l.Close(); err != nil { 133 | t.Fatal(err) 134 | } 135 | 136 | // Write - try while closed 137 | err = l.Write(1, nil) 138 | if err != ErrClosed { 139 | t.Fatalf("expected %v, got %v", ErrClosed, err) 140 | } 141 | // WriteBatch - try while closed 142 | err = l.WriteBatch(nil) 143 | if err != ErrClosed { 144 | t.Fatalf("expected %v, got %v", ErrClosed, err) 145 | } 146 | // FirstIndex - try while closed 147 | _, err = l.FirstIndex() 148 | if err != ErrClosed { 149 | t.Fatalf("expected %v, got %v", ErrClosed, err) 150 | } 151 | // LastIndex - try while closed 152 | _, err = l.LastIndex() 153 | if err != ErrClosed { 154 | t.Fatalf("expected %v, got %v", ErrClosed, err) 155 | } 156 | // Get - try while closed 157 | _, err = l.Read(0) 158 | if err != ErrClosed { 159 | t.Fatalf("expected %v, got %v", ErrClosed, err) 160 | } 161 | // TruncateFront - try while closed 162 | err = l.TruncateFront(0) 163 | if err != ErrClosed { 164 | t.Fatalf("expected %v, got %v", ErrClosed, err) 165 | } 166 | // TruncateBack - try while closed 167 | err = l.TruncateBack(0) 168 | if err != ErrClosed { 169 | t.Fatalf("expected %v, got %v", ErrClosed, err) 170 | } 171 | 172 | // Open -- reopen log 173 | l, err = Open(logPath, opts) 174 | if err != nil { 175 | t.Fatal(err) 176 | } 177 | defer l.Close() 178 | 179 | // Read -- read back all entries 180 | for i := 1; i <= N; i++ { 181 | data, err := l.Read(uint64(i)) 182 | if err != nil { 183 | t.Fatalf("error while getting %d, err=%s", i, err) 184 | } 185 | if string(data) != dataStr(uint64(i)) { 186 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 187 | } 188 | } 189 | // FirstIndex/LastIndex -- check valid first and last indexes 190 | n, err = l.FirstIndex() 191 | if err != nil { 192 | t.Fatal(err) 193 | } 194 | if n != 1 { 195 | t.Fatalf("expected %d, got %d", 1, n) 196 | } 197 | n, err = l.LastIndex() 198 | if err != nil { 199 | t.Fatal(err) 200 | } 201 | if n != uint64(N) { 202 | t.Fatalf("expected %d, got %d", N, n) 203 | } 204 | // Write -- add 50 more items 205 | for i := N + 1; i <= N+50; i++ { 206 | index := uint64(i) 207 | if err := l.Write(index, []byte(dataStr(index))); err != nil { 208 | t.Fatal(err) 209 | } 210 | data, err := l.Read(index) 211 | if err != nil { 212 | t.Fatal(err) 213 | } 214 | if string(data) != dataStr(index) { 215 | t.Fatalf("expected %v, got %v", dataStr(index), string(data)) 216 | } 217 | } 218 | N += 50 219 | // FirstIndex/LastIndex -- check valid first and last indexes 220 | n, err = l.FirstIndex() 221 | if err != nil { 222 | t.Fatal(err) 223 | } 224 | if n != 1 { 225 | t.Fatalf("expected %d, got %d", 1, n) 226 | } 227 | n, err = l.LastIndex() 228 | if err != nil { 229 | t.Fatal(err) 230 | } 231 | if n != uint64(N) { 232 | t.Fatalf("expected %d, got %d", N, n) 233 | } 234 | // Batch -- test batch writes 235 | b := new(Batch) 236 | b.Write(1, nil) 237 | b.Write(2, nil) 238 | b.Write(3, nil) 239 | // WriteBatch -- should fail out of order 240 | err = l.WriteBatch(b) 241 | if err != ErrOutOfOrder { 242 | t.Fatalf("expected %v, got %v", ErrOutOfOrder, nil) 243 | } 244 | // Clear -- clear the batch 245 | b.Clear() 246 | // WriteBatch -- should succeed 247 | err = l.WriteBatch(b) 248 | if err != nil { 249 | t.Fatal(err) 250 | } 251 | // Write 100 entries in batches of 10 252 | for i := 0; i < 10; i++ { 253 | for i := N + 1; i <= N+10; i++ { 254 | index := uint64(i) 255 | b.Write(index, []byte(dataStr(index))) 256 | } 257 | err = l.WriteBatch(b) 258 | if err != nil { 259 | t.Fatal(err) 260 | } 261 | N += 10 262 | } 263 | // Read -- read back all entries 264 | for i := 1; i <= N; i++ { 265 | data, err := l.Read(uint64(i)) 266 | if err != nil { 267 | t.Fatalf("error while getting %d", i) 268 | } 269 | if string(data) != dataStr(uint64(i)) { 270 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 271 | } 272 | } 273 | 274 | // Write -- one entry, so the buffer might be activated 275 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 276 | if err != nil { 277 | t.Fatal(err) 278 | } 279 | N++ 280 | // Read -- one random read, so there is an opened reader 281 | data, err := l.Read(uint64(N / 2)) 282 | if err != nil { 283 | t.Fatal(err) 284 | } 285 | if string(data) != dataStr(uint64(N/2)) { 286 | t.Fatalf("expected %v, got %v", dataStr(uint64(N/2)), string(data)) 287 | } 288 | 289 | // TruncateFront -- should fail, out of range 290 | for _, i := range []int{0, N + 1} { 291 | index := uint64(i) 292 | if err = l.TruncateFront(index); err != ErrOutOfRange { 293 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 294 | } 295 | testFirstLast(t, l, uint64(1), uint64(N), nil) 296 | } 297 | 298 | // TruncateBack -- should fail, out of range 299 | err = l.TruncateFront(0) 300 | if err != ErrOutOfRange { 301 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 302 | } 303 | testFirstLast(t, l, uint64(1), uint64(N), nil) 304 | 305 | // TruncateFront -- Remove no entries 306 | if err = l.TruncateFront(1); err != nil { 307 | t.Fatal(err) 308 | } 309 | testFirstLast(t, l, uint64(1), uint64(N), nil) 310 | 311 | // TruncateFront -- Remove first 80 entries 312 | if err = l.TruncateFront(81); err != nil { 313 | t.Fatal(err) 314 | } 315 | testFirstLast(t, l, uint64(81), uint64(N), nil) 316 | 317 | // Write -- one entry, so the buffer might be activated 318 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 319 | if err != nil { 320 | t.Fatal(err) 321 | } 322 | N++ 323 | testFirstLast(t, l, uint64(81), uint64(N), nil) 324 | 325 | // Read -- one random read, so there is an opened reader 326 | data, err = l.Read(uint64(N / 2)) 327 | if err != nil { 328 | t.Fatal(err) 329 | } 330 | if string(data) != dataStr(uint64(N/2)) { 331 | t.Fatalf("expected %v, got %v", dataStr(uint64(N/2)), string(data)) 332 | } 333 | 334 | // TruncateBack -- should fail, out of range 335 | for _, i := range []int{0, 80} { 336 | index := uint64(i) 337 | if err = l.TruncateBack(index); err != ErrOutOfRange { 338 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 339 | } 340 | testFirstLast(t, l, uint64(81), uint64(N), nil) 341 | } 342 | 343 | // TruncateBack -- Remove no entries 344 | if err = l.TruncateBack(uint64(N)); err != nil { 345 | t.Fatal(err) 346 | } 347 | testFirstLast(t, l, uint64(81), uint64(N), nil) 348 | // TruncateBack -- Remove last 80 entries 349 | if err = l.TruncateBack(uint64(N - 80)); err != nil { 350 | t.Fatal(err) 351 | } 352 | N -= 80 353 | testFirstLast(t, l, uint64(81), uint64(N), nil) 354 | 355 | // Read -- read back all entries 356 | for i := 81; i <= N; i++ { 357 | data, err := l.Read(uint64(i)) 358 | if err != nil { 359 | t.Fatalf("error while getting %d", i) 360 | } 361 | if string(data) != dataStr(uint64(i)) { 362 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 363 | } 364 | } 365 | 366 | // Close -- close log after truncating 367 | if err = l.Close(); err != nil { 368 | t.Fatal(err) 369 | } 370 | 371 | // Open -- open log after truncating 372 | l, err = Open(logPath, opts) 373 | if err != nil { 374 | t.Fatal(err) 375 | } 376 | defer l.Close() 377 | 378 | testFirstLast(t, l, uint64(81), uint64(N), nil) 379 | 380 | // Read -- read back all entries 381 | for i := 81; i <= N; i++ { 382 | data, err := l.Read(uint64(i)) 383 | if err != nil { 384 | t.Fatalf("error while getting %d", i) 385 | } 386 | if string(data) != dataStr(uint64(i)) { 387 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 388 | } 389 | } 390 | 391 | // TruncateFront -- truncate all entries but one 392 | if err = l.TruncateFront(uint64(N)); err != nil { 393 | t.Fatal(err) 394 | } 395 | testFirstLast(t, l, uint64(N), uint64(N), nil) 396 | 397 | // Write -- write on entry 398 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 399 | if err != nil { 400 | t.Fatal(err) 401 | } 402 | N++ 403 | testFirstLast(t, l, uint64(N-1), uint64(N), nil) 404 | 405 | // TruncateBack -- truncate all entries but one 406 | if err = l.TruncateBack(uint64(N - 1)); err != nil { 407 | t.Fatal(err) 408 | } 409 | N-- 410 | testFirstLast(t, l, uint64(N), uint64(N), nil) 411 | 412 | if err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))); err != nil { 413 | t.Fatal(err) 414 | } 415 | N++ 416 | 417 | l.Sync() 418 | testFirstLast(t, l, uint64(N-1), uint64(N), nil) 419 | } 420 | 421 | func testFirstLast(t *testing.T, l *Log, expectFirst, expectLast uint64, data func(index uint64) []byte) { 422 | t.Helper() 423 | fi, err := l.FirstIndex() 424 | if err != nil { 425 | t.Fatal(err) 426 | } 427 | li, err := l.LastIndex() 428 | if err != nil { 429 | t.Fatal(err) 430 | } 431 | if fi != expectFirst || li != expectLast { 432 | t.Fatalf("expected %v/%v, got %v/%v", expectFirst, expectLast, fi, li) 433 | } 434 | for i := fi; i <= li; i++ { 435 | dt1, err := l.Read(i) 436 | if err != nil { 437 | t.Fatal(err) 438 | } 439 | if data != nil { 440 | dt2 := data(i) 441 | if string(dt1) != string(dt2) { 442 | t.Fatalf("mismatch '%s' != '%s'", dt2, dt1) 443 | } 444 | } 445 | } 446 | 447 | } 448 | 449 | func TestLog(t *testing.T) { 450 | os.RemoveAll("testlog") 451 | defer os.RemoveAll("testlog") 452 | 453 | t.Run("nil-opts", func(t *testing.T) { 454 | testLog(t, nil, 100) 455 | }) 456 | t.Run("no-sync", func(t *testing.T) { 457 | t.Run("json", func(t *testing.T) { 458 | testLog(t, makeOpts(512, true, JSON), 100) 459 | }) 460 | t.Run("binary", func(t *testing.T) { 461 | testLog(t, makeOpts(512, true, Binary), 100) 462 | }) 463 | }) 464 | t.Run("sync", func(t *testing.T) { 465 | t.Run("json", func(t *testing.T) { 466 | testLog(t, makeOpts(512, false, JSON), 100) 467 | }) 468 | t.Run("binary", func(t *testing.T) { 469 | testLog(t, makeOpts(512, false, Binary), 100) 470 | }) 471 | }) 472 | } 473 | 474 | func TestOutliers(t *testing.T) { 475 | // Create some scenarios where the log has been corrupted, operations 476 | // fail, or various weirdnesses. 477 | t.Run("fail-in-memory", func(t *testing.T) { 478 | if l, err := Open(":memory:", nil); err == nil { 479 | l.Close() 480 | t.Fatal("expected error") 481 | } 482 | }) 483 | t.Run("fail-not-a-directory", func(t *testing.T) { 484 | defer os.RemoveAll("testlog/file") 485 | if err := os.MkdirAll("testlog", 0777); err != nil { 486 | t.Fatal(err) 487 | } else if f, err := os.Create("testlog/file"); err != nil { 488 | t.Fatal(err) 489 | } else if err := f.Close(); err != nil { 490 | t.Fatal(err) 491 | } else if l, err := Open("testlog/file", nil); err == nil { 492 | l.Close() 493 | t.Fatal("expected error") 494 | } 495 | }) 496 | t.Run("load-with-junk-files", func(t *testing.T) { 497 | // junk should be ignored 498 | defer os.RemoveAll("testlog/junk") 499 | if err := os.MkdirAll("testlog/junk/other1", 0777); err != nil { 500 | t.Fatal(err) 501 | } 502 | f, err := os.Create("testlog/junk/other2") 503 | if err != nil { 504 | t.Fatal(err) 505 | } 506 | f.Close() 507 | f, err = os.Create("testlog/junk/" + strings.Repeat("A", 20)) 508 | if err != nil { 509 | t.Fatal(err) 510 | } 511 | f.Close() 512 | l, err := Open("testlog/junk", nil) 513 | if err != nil { 514 | t.Fatal(err) 515 | } 516 | l.Close() 517 | }) 518 | 519 | t.Run("fail-corrupted-tail-json", func(t *testing.T) { 520 | defer os.RemoveAll("testlog/corrupt-tail") 521 | opts := makeOpts(512, true, JSON) 522 | os.MkdirAll("testlog/corrupt-tail", 0777) 523 | ioutil.WriteFile( 524 | "testlog/corrupt-tail/00000000000000000001", 525 | []byte("\n"), 0666) 526 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 527 | l.Close() 528 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 529 | } 530 | ioutil.WriteFile( 531 | "testlog/corrupt-tail/00000000000000000001", 532 | []byte(`{}`+"\n"), 0666) 533 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 534 | l.Close() 535 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 536 | } 537 | ioutil.WriteFile( 538 | "testlog/corrupt-tail/00000000000000000001", 539 | []byte(`{"index":"1"}`+"\n"), 0666) 540 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 541 | l.Close() 542 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 543 | } 544 | ioutil.WriteFile( 545 | "testlog/corrupt-tail/00000000000000000001", 546 | []byte(`{"index":"1","data":"?"}`), 0666) 547 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 548 | l.Close() 549 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 550 | } 551 | }) 552 | 553 | t.Run("start-marker-file", func(t *testing.T) { 554 | lpath := "testlog/start-marker" 555 | opts := makeOpts(512, true, JSON) 556 | l := must(Open(lpath, opts)).(*Log) 557 | defer l.Close() 558 | for i := uint64(1); i <= 100; i++ { 559 | must(nil, l.Write(i, []byte(dataStr(i)))) 560 | } 561 | path := l.segments[l.findSegment(35)].path 562 | firstIndex := l.segments[l.findSegment(35)].index 563 | must(nil, l.Close()) 564 | data := must(ioutil.ReadFile(path)).([]byte) 565 | must(nil, ioutil.WriteFile(path+".START", data, 0666)) 566 | l = must(Open(lpath, opts)).(*Log) 567 | defer l.Close() 568 | testFirstLast(t, l, firstIndex, 100, nil) 569 | 570 | }) 571 | // t.Run("end-marker-file", func(t *testing.T) { 572 | // lpath := "testlog/end-marker" 573 | // opts := makeOpts(512, true, JSON) 574 | // l := must(Open(lpath, opts)).(*Log) 575 | // defer l.Close() 576 | // for i := uint64(1); i <= 100; i++ { 577 | // must(nil, l.Write(i, []byte(dataStr(i)))) 578 | // } 579 | // path := l.segments[l.findSegment(35)].path 580 | // data := must(ioutil.ReadFile(path)).([]byte) 581 | // var lastIndex uint64 582 | // for { 583 | 584 | // n, _, err := readEntry(rd, JSON, true) 585 | // if err == io.EOF { 586 | // break 587 | // } 588 | // if err != nil { 589 | // t.Fatal(err) 590 | // } 591 | // lastIndex = n 592 | // } 593 | 594 | // must(nil, l.Close()) 595 | // data = must(ioutil.ReadFile(path)).([]byte) 596 | // must(nil, ioutil.WriteFile(path+".END", data, 0666)) 597 | // l = must(Open(lpath, opts)).(*Log) 598 | // defer l.Close() 599 | // testFirstLast(t, l, 1, lastIndex) 600 | // }) 601 | 602 | } 603 | 604 | func makeOpts(segSize int, noSync bool, lf LogFormat) *Options { 605 | opts := *DefaultOptions 606 | opts.SegmentSize = segSize 607 | opts.NoSync = noSync 608 | opts.LogFormat = lf 609 | return &opts 610 | } 611 | 612 | // https://github.com/tidwall/wal/issues/1 613 | func TestIssue1(t *testing.T) { 614 | in := []byte{0, 0, 0, 0, 0, 0, 0, 1, 37, 108, 131, 178, 151, 17, 77, 32, 615 | 27, 48, 23, 159, 63, 14, 240, 202, 206, 151, 131, 98, 45, 165, 151, 67, 616 | 38, 180, 54, 23, 138, 238, 246, 16, 0, 0, 0, 0} 617 | opts := *DefaultOptions 618 | opts.LogFormat = JSON 619 | os.RemoveAll("testlog") 620 | l, err := Open("testlog", &opts) 621 | if err != nil { 622 | t.Fatal(err) 623 | } 624 | defer l.Close() 625 | if err := l.Write(1, in); err != nil { 626 | t.Fatal(err) 627 | } 628 | out, err := l.Read(1) 629 | if err != nil { 630 | t.Fatal(err) 631 | } 632 | if string(in) != string(out) { 633 | t.Fatal("data mismatch") 634 | } 635 | } 636 | 637 | func TestSimpleTruncateFront(t *testing.T) { 638 | os.RemoveAll("testlog") 639 | 640 | opts := &Options{ 641 | NoSync: true, 642 | LogFormat: JSON, 643 | SegmentSize: 100, 644 | } 645 | 646 | l, err := Open("testlog", opts) 647 | if err != nil { 648 | t.Fatal(err) 649 | } 650 | defer func() { 651 | l.Close() 652 | }() 653 | 654 | makeData := func(index uint64) []byte { 655 | return []byte(fmt.Sprintf("data-%d", index)) 656 | } 657 | 658 | valid := func(t *testing.T, first, last uint64) { 659 | t.Helper() 660 | index, err := l.FirstIndex() 661 | if err != nil { 662 | t.Fatal(err) 663 | } 664 | if index != first { 665 | t.Fatalf("expected %v, got %v", first, index) 666 | } 667 | index, err = l.LastIndex() 668 | if err != nil { 669 | t.Fatal(err) 670 | } 671 | if index != last { 672 | t.Fatalf("expected %v, got %v", last, index) 673 | } 674 | for i := first; i <= last; i++ { 675 | data, err := l.Read(i) 676 | if err != nil { 677 | t.Fatal(err) 678 | } 679 | if string(data) != string(makeData(i)) { 680 | t.Fatalf("expcted '%s', got '%s'", makeData(i), data) 681 | } 682 | } 683 | } 684 | validReopen := func(t *testing.T, first, last uint64) { 685 | t.Helper() 686 | valid(t, first, last) 687 | if err := l.Close(); err != nil { 688 | t.Fatal(err) 689 | } 690 | l, err = Open("testlog", opts) 691 | if err != nil { 692 | t.Fatal(err) 693 | } 694 | valid(t, first, last) 695 | } 696 | for i := 1; i <= 100; i++ { 697 | err := l.Write(uint64(i), makeData(uint64(i))) 698 | if err != nil { 699 | t.Fatal(err) 700 | } 701 | } 702 | validReopen(t, 1, 100) 703 | 704 | if err := l.TruncateFront(1); err != nil { 705 | t.Fatal(err) 706 | } 707 | validReopen(t, 1, 100) 708 | 709 | if err := l.TruncateFront(2); err != nil { 710 | t.Fatal(err) 711 | } 712 | validReopen(t, 2, 100) 713 | 714 | if err := l.TruncateFront(4); err != nil { 715 | t.Fatal(err) 716 | } 717 | validReopen(t, 4, 100) 718 | 719 | if err := l.TruncateFront(5); err != nil { 720 | t.Fatal(err) 721 | } 722 | validReopen(t, 5, 100) 723 | 724 | if err := l.TruncateFront(99); err != nil { 725 | t.Fatal(err) 726 | } 727 | validReopen(t, 99, 100) 728 | 729 | if err := l.TruncateFront(100); err != nil { 730 | t.Fatal(err) 731 | } 732 | validReopen(t, 100, 100) 733 | 734 | } 735 | 736 | func TestSimpleTruncateBack(t *testing.T) { 737 | os.RemoveAll("testlog") 738 | 739 | opts := &Options{ 740 | NoSync: true, 741 | LogFormat: JSON, 742 | SegmentSize: 100, 743 | } 744 | 745 | l, err := Open("testlog", opts) 746 | if err != nil { 747 | t.Fatal(err) 748 | } 749 | defer func() { 750 | l.Close() 751 | }() 752 | 753 | makeData := func(index uint64) []byte { 754 | return []byte(fmt.Sprintf("data-%d", index)) 755 | } 756 | 757 | valid := func(t *testing.T, first, last uint64) { 758 | t.Helper() 759 | index, err := l.FirstIndex() 760 | if err != nil { 761 | t.Fatal(err) 762 | } 763 | if index != first { 764 | t.Fatalf("expected %v, got %v", first, index) 765 | } 766 | index, err = l.LastIndex() 767 | if err != nil { 768 | t.Fatal(err) 769 | } 770 | if index != last { 771 | t.Fatalf("expected %v, got %v", last, index) 772 | } 773 | for i := first; i <= last; i++ { 774 | data, err := l.Read(i) 775 | if err != nil { 776 | t.Fatal(err) 777 | } 778 | if string(data) != string(makeData(i)) { 779 | t.Fatalf("expcted '%s', got '%s'", makeData(i), data) 780 | } 781 | } 782 | } 783 | validReopen := func(t *testing.T, first, last uint64) { 784 | t.Helper() 785 | valid(t, first, last) 786 | if err := l.Close(); err != nil { 787 | t.Fatal(err) 788 | } 789 | l, err = Open("testlog", opts) 790 | if err != nil { 791 | t.Fatal(err) 792 | } 793 | valid(t, first, last) 794 | } 795 | for i := 1; i <= 100; i++ { 796 | err := l.Write(uint64(i), makeData(uint64(i))) 797 | if err != nil { 798 | t.Fatal(err) 799 | } 800 | } 801 | validReopen(t, 1, 100) 802 | 803 | ///////////////////////////////////////////////////////////// 804 | if err := l.TruncateBack(100); err != nil { 805 | t.Fatal(err) 806 | } 807 | validReopen(t, 1, 100) 808 | if err := l.Write(101, makeData(101)); err != nil { 809 | t.Fatal(err) 810 | } 811 | validReopen(t, 1, 101) 812 | 813 | ///////////////////////////////////////////////////////////// 814 | if err := l.TruncateBack(99); err != nil { 815 | t.Fatal(err) 816 | } 817 | validReopen(t, 1, 99) 818 | if err := l.Write(100, makeData(100)); err != nil { 819 | t.Fatal(err) 820 | } 821 | validReopen(t, 1, 100) 822 | 823 | if err := l.TruncateBack(94); err != nil { 824 | t.Fatal(err) 825 | } 826 | validReopen(t, 1, 94) 827 | 828 | if err := l.TruncateBack(93); err != nil { 829 | t.Fatal(err) 830 | } 831 | validReopen(t, 1, 93) 832 | 833 | if err := l.TruncateBack(92); err != nil { 834 | t.Fatal(err) 835 | } 836 | validReopen(t, 1, 92) 837 | 838 | } 839 | 840 | func TestConcurrency(t *testing.T) { 841 | os.RemoveAll("testlog") 842 | 843 | l, err := Open("testlog", &Options{ 844 | NoSync: true, 845 | NoCopy: true, 846 | }) 847 | if err != nil { 848 | t.Fatal(err) 849 | } 850 | defer l.Close() 851 | 852 | // Write 1000 entries 853 | for i := 1; i <= 1000; i++ { 854 | err := l.Write(uint64(i), []byte(dataStr(uint64(i)))) 855 | if err != nil { 856 | t.Fatal(err) 857 | } 858 | } 859 | 860 | // Perform 100,000 reads (over 100 threads) 861 | finished := int32(0) 862 | maxIndex := int32(1000) 863 | numReads := int32(0) 864 | for i := 0; i < 100; i++ { 865 | go func() { 866 | defer atomic.AddInt32(&finished, 1) 867 | 868 | for i := 0; i < 1_000; i++ { 869 | index := rand.Int31n(atomic.LoadInt32(&maxIndex)) + 1 870 | if _, err := l.Read(uint64(index)); err != nil { 871 | panic(err) 872 | } 873 | atomic.AddInt32(&numReads, 1) 874 | } 875 | }() 876 | } 877 | 878 | // continue writing 879 | for index := maxIndex + 1; atomic.LoadInt32(&finished) < 100; index++ { 880 | err := l.Write(uint64(index), []byte(dataStr(uint64(index)))) 881 | if err != nil { 882 | t.Fatal(err) 883 | } 884 | atomic.StoreInt32(&maxIndex, index) 885 | } 886 | 887 | // confirm total reads 888 | if exp := int32(100_000); numReads != exp { 889 | t.Fatalf("expected %d reads, but god %d", exp, numReads) 890 | } 891 | } 892 | 893 | func must(v interface{}, err error) interface{} { 894 | if err != nil { 895 | panic(err) 896 | } 897 | return v 898 | } 899 | --------------------------------------------------------------------------------