├── .gitignore ├── go.mod ├── go.sum ├── .github └── workflows │ └── go.yml ├── LICENSE ├── README.md ├── wal.go └── wal_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | testlog 2 | .idea 3 | vendor -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tidwall/wal 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/tidwall/gjson v1.10.2 7 | github.com/tidwall/tinylru v1.1.0 8 | ) 9 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= 2 | github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 3 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 4 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 5 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 6 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 7 | github.com/tidwall/tinylru v1.1.0 h1:XY6IUfzVTU9rpwdhKUF6nQdChgCdGjkMfLzbWyiau6I= 8 | github.com/tidwall/tinylru v1.1.0/go.mod h1:3+bX+TJ2baOLMWTnlyNWHh4QMnFyARg2TLTQ6OFbzw8= 9 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.13 20 | 21 | - name: Check out code into the Go module directory 22 | uses: actions/checkout@v2 23 | 24 | - name: Get dependencies 25 | run: | 26 | go get -v -t -d ./... 27 | if [ -f Gopkg.toml ]; then 28 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 29 | dep ensure 30 | fi 31 | 32 | - name: Build 33 | run: go build -v . 34 | 35 | - name: Test 36 | run: go test -v . 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Joshua J Baker 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # `wal` 2 | [![GoDoc](https://godoc.org/github.com/tidwall/wal?status.svg)](https://godoc.org/github.com/tidwall/wal) 3 | 4 | A simple and fast write ahead log for Go. 5 | 6 | ## Features 7 | 8 | - High durability 9 | - Fast operations 10 | - Monotonic indexes 11 | - Batch writes 12 | - Log truncation from front or back. 13 | 14 | ## Getting Started 15 | 16 | ### Installing 17 | 18 | To start using `wal`, install Go and run `go get`: 19 | 20 | ```sh 21 | $ go get -u github.com/tidwall/wal 22 | ``` 23 | 24 | This will retrieve the library. 25 | 26 | ### Example 27 | 28 | ```go 29 | // open a new log file 30 | log, err := wal.Open("mylog", nil) 31 | 32 | // write some entries 33 | err = log.Write(1, []byte("first entry")) 34 | err = log.Write(2, []byte("second entry")) 35 | err = log.Write(3, []byte("third entry")) 36 | 37 | // read an entry 38 | data, err := log.Read(1) 39 | println(string(data)) // output: first entry 40 | 41 | // close the log 42 | err = log.Close() 43 | ``` 44 | 45 | Batch writes: 46 | 47 | ```go 48 | 49 | // write three entries as a batch 50 | batch := new(wal.Batch) 51 | batch.Write(1, []byte("first entry")) 52 | batch.Write(2, []byte("second entry")) 53 | batch.Write(3, []byte("third entry")) 54 | 55 | err = log.WriteBatch(batch) 56 | ``` 57 | 58 | Truncating: 59 | 60 | ```go 61 | // write some entries 62 | err = log.Write(1, []byte("first entry")) 63 | ... 64 | err = log.Write(1000, []byte("thousandth entry")) 65 | 66 | // truncate the log from index starting 350 and ending with 950. 67 | err = log.TruncateFront(350) 68 | err = log.TruncateBack(950) 69 | ``` 70 | 71 | 72 | 73 | ## Contact 74 | 75 | Josh Baker [@tidwall](http://twitter.com/tidwall) 76 | 77 | ## License 78 | 79 | `wal` source code is available under the MIT [License](/LICENSE). 80 | -------------------------------------------------------------------------------- /wal.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "bytes" 5 | "encoding/base64" 6 | "encoding/binary" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "io/ioutil" 11 | "os" 12 | "path/filepath" 13 | "strconv" 14 | "strings" 15 | "sync" 16 | "unicode/utf8" 17 | "unsafe" 18 | 19 | "github.com/tidwall/gjson" 20 | "github.com/tidwall/tinylru" 21 | ) 22 | 23 | var ( 24 | // ErrCorrupt is returns when the log is corrupt. 25 | ErrCorrupt = errors.New("log corrupt") 26 | 27 | // ErrClosed is returned when an operation cannot be completed because 28 | // the log is closed. 29 | ErrClosed = errors.New("log closed") 30 | 31 | // ErrNotFound is returned when an entry is not found. 32 | ErrNotFound = errors.New("not found") 33 | 34 | // ErrOutOfOrder is returned from Write() when the index is not equal to 35 | // LastIndex()+1. It's required that log monotonically grows by one and has 36 | // no gaps. Thus, the series 10,11,12,13,14 is valid, but 10,11,13,14 is 37 | // not because there's a gap between 11 and 13. Also, 10,12,11,13 is not 38 | // valid because 12 and 11 are out of order. 39 | ErrOutOfOrder = errors.New("out of order") 40 | 41 | // ErrOutOfRange is returned from TruncateFront() and TruncateBack() when 42 | // the index not in the range of the log's first and last index. Or, this 43 | // may be returned when the caller is attempting to remove *all* entries; 44 | // The log requires that at least one entry exists following a truncate. 45 | ErrOutOfRange = errors.New("out of range") 46 | 47 | // ErrEmptyLog is returned by Open() when the `AllowEmpty` option was not 48 | // provided and log has been emptied due to the use of TruncateFront() or 49 | // TruncateBack(). 50 | ErrEmptyLog = errors.New("empty log") 51 | ) 52 | 53 | // LogFormat is the format of the log files. 54 | type LogFormat byte 55 | 56 | const ( 57 | // Binary format writes entries in binary. This is the default and, unless 58 | // a good reason otherwise, should be used in production. 59 | Binary LogFormat = 0 60 | // JSON format writes entries as JSON lines. This causes larger, human 61 | // readable files. 62 | JSON LogFormat = 1 63 | ) 64 | 65 | // Options for Log 66 | type Options struct { 67 | // NoSync disables fsync after writes. This is less durable and puts the 68 | // log at risk of data loss when there's a server crash. 69 | NoSync bool 70 | // SegmentSize of each segment. This is just a target value, actual size 71 | // may differ. Default 20 MB 72 | SegmentSize int 73 | // LogFormat is the format of the log files. Default Binary 74 | LogFormat LogFormat 75 | // SegmentCacheSize is the maximum number of segments that will be held in 76 | // memory for caching. Increasing this value may enhance performance for 77 | // concurrent read operations. Default 1 78 | SegmentCacheSize int 79 | // NoCopy allows for the Read() operation to return the raw underlying data 80 | // slice. This is an optimization to help minimize allocations. When this 81 | // option is set, do not modify the returned data because it may affect 82 | // other Read calls. Default false 83 | NoCopy bool 84 | // AllowEmpty allows for a log to have all entries removed through the use 85 | // of TruncateFront() or TruncateBack(). Otherwise without this option, 86 | // at least one entry must always remain following a truncate operation. 87 | // Default false 88 | // 89 | // Warning: using this option changes the behavior of the log in the 90 | // following ways: 91 | // - An empty log will always have the FirstIndex() be equal to 92 | // LastIndex()+1. 93 | // - For a newly created log that has no entries, FirstIndex() and 94 | // LastIndex() return 1 and 0, respectively. 95 | // Without AllowEmpty, both return 0. 96 | AllowEmpty bool 97 | // Perms represents the datafiles modes and permission bits 98 | DirPerms os.FileMode 99 | FilePerms os.FileMode 100 | } 101 | 102 | // DefaultOptions for Open(). 103 | var DefaultOptions = &Options{ 104 | NoSync: false, // Fsync after every write 105 | SegmentSize: 20971520, // 20 MB log segment files 106 | LogFormat: Binary, // Binary format is small and fast 107 | SegmentCacheSize: 2, // Number of cached in-memory segments 108 | NoCopy: false, // Make a new copy of data for every Read call 109 | AllowEmpty: false, // Do not allow empty log. 1+ entries required 110 | DirPerms: 0750, // Permissions for the created directories 111 | FilePerms: 0640, // Permissions for the created data files 112 | } 113 | 114 | // Log represents a write ahead log 115 | type Log struct { 116 | mu sync.RWMutex 117 | path string // absolute path to log directory 118 | opts Options // log options 119 | closed bool // log is closed 120 | corrupt bool // log may be corrupt 121 | segments []*segment // all known log segments 122 | firstIndex uint64 // index of the first entry in log 123 | lastIndex uint64 // index of the last entry in log 124 | sfile *os.File // tail segment file handle 125 | wbatch Batch // reusable write batch 126 | scache tinylru.LRU // segment entries cache 127 | } 128 | 129 | // segment represents a single segment file. 130 | type segment struct { 131 | path string // path of segment file 132 | index uint64 // first index of segment 133 | ebuf []byte // cached entries buffer 134 | epos []bpos // cached entries positions in buffer 135 | } 136 | 137 | type bpos struct { 138 | pos int // byte position 139 | end int // one byte past pos 140 | } 141 | 142 | // Open a new write ahead log 143 | func Open(path string, opts *Options) (*Log, error) { 144 | if opts == nil { 145 | opts = DefaultOptions 146 | } 147 | if opts.SegmentCacheSize <= 0 { 148 | opts.SegmentCacheSize = DefaultOptions.SegmentCacheSize 149 | } 150 | if opts.SegmentSize <= 0 { 151 | opts.SegmentSize = DefaultOptions.SegmentSize 152 | } 153 | if opts.DirPerms == 0 { 154 | opts.DirPerms = DefaultOptions.DirPerms 155 | } 156 | if opts.FilePerms == 0 { 157 | opts.FilePerms = DefaultOptions.FilePerms 158 | } 159 | 160 | var err error 161 | path, err = abs(path) 162 | if err != nil { 163 | return nil, err 164 | } 165 | l := &Log{path: path, opts: *opts} 166 | l.scache.Resize(l.opts.SegmentCacheSize) 167 | if err := os.MkdirAll(path, l.opts.DirPerms); err != nil { 168 | return nil, err 169 | } 170 | if err := l.load(); err != nil { 171 | return nil, err 172 | } 173 | return l, nil 174 | } 175 | 176 | func abs(path string) (string, error) { 177 | if path == ":memory:" { 178 | return "", errors.New("in-memory log not supported") 179 | } 180 | return filepath.Abs(path) 181 | } 182 | 183 | func (l *Log) pushCache(segIdx int) { 184 | _, _, _, v, evicted := 185 | l.scache.SetEvicted(segIdx, l.segments[segIdx]) 186 | if evicted { 187 | s := v.(*segment) 188 | s.ebuf = nil 189 | s.epos = nil 190 | } 191 | } 192 | 193 | // load all the segments. This operation also cleans up any START/END segments. 194 | func (l *Log) load() error { 195 | fis, err := ioutil.ReadDir(l.path) 196 | if err != nil { 197 | return err 198 | } 199 | startIdx := -1 200 | endIdx := -1 201 | for _, fi := range fis { 202 | name := fi.Name() 203 | if fi.IsDir() || len(name) < 20 { 204 | continue 205 | } 206 | index, err := strconv.ParseUint(name[:20], 10, 64) 207 | if err != nil || index == 0 { 208 | continue 209 | } 210 | isStart := len(name) == 26 && strings.HasSuffix(name, ".START") 211 | isEnd := len(name) == 24 && strings.HasSuffix(name, ".END") 212 | if len(name) == 20 || isStart || isEnd { 213 | if isStart { 214 | startIdx = len(l.segments) 215 | } else if isEnd && endIdx == -1 { 216 | endIdx = len(l.segments) 217 | } 218 | l.segments = append(l.segments, &segment{ 219 | index: index, 220 | path: filepath.Join(l.path, name), 221 | }) 222 | } 223 | } 224 | if len(l.segments) == 0 { 225 | // Create a new log 226 | l.segments = append(l.segments, &segment{ 227 | index: 1, 228 | path: filepath.Join(l.path, segmentName(1)), 229 | }) 230 | l.firstIndex = 1 231 | l.lastIndex = 0 232 | l.sfile, err = os.OpenFile(l.segments[0].path, 233 | os.O_CREATE|os.O_RDWR|os.O_TRUNC, l.opts.FilePerms) 234 | return err 235 | } 236 | // Open existing log. Clean up log if START of END segments exists. 237 | if startIdx != -1 { 238 | if endIdx != -1 { 239 | // There should not be a START and END at the same time 240 | return ErrCorrupt 241 | } 242 | // Delete all files leading up to START 243 | for i := 0; i < startIdx; i++ { 244 | if err := os.Remove(l.segments[i].path); err != nil { 245 | return err 246 | } 247 | } 248 | l.segments = append([]*segment{}, l.segments[startIdx:]...) 249 | // Rename the START segment 250 | orgPath := l.segments[0].path 251 | finalPath := orgPath[:len(orgPath)-len(".START")] 252 | err := os.Rename(orgPath, finalPath) 253 | if err != nil { 254 | return err 255 | } 256 | l.segments[0].path = finalPath 257 | } 258 | if endIdx != -1 { 259 | // Delete all files following END 260 | for i := len(l.segments) - 1; i > endIdx; i-- { 261 | if err := os.Remove(l.segments[i].path); err != nil { 262 | return err 263 | } 264 | } 265 | l.segments = append([]*segment{}, l.segments[:endIdx+1]...) 266 | if len(l.segments) > 1 && l.segments[len(l.segments)-2].index == 267 | l.segments[len(l.segments)-1].index { 268 | // remove the segment prior to the END segment because it shares 269 | // the same starting index. 270 | l.segments[len(l.segments)-2] = l.segments[len(l.segments)-1] 271 | l.segments = l.segments[:len(l.segments)-1] 272 | } 273 | // Rename the END segment 274 | orgPath := l.segments[len(l.segments)-1].path 275 | finalPath := orgPath[:len(orgPath)-len(".END")] 276 | err := os.Rename(orgPath, finalPath) 277 | if err != nil { 278 | return err 279 | } 280 | l.segments[len(l.segments)-1].path = finalPath 281 | } 282 | l.firstIndex = l.segments[0].index 283 | // Open the last segment for appending 284 | lseg := l.segments[len(l.segments)-1] 285 | l.sfile, err = os.OpenFile(lseg.path, os.O_WRONLY, l.opts.FilePerms) 286 | if err != nil { 287 | return err 288 | } 289 | if _, err := l.sfile.Seek(0, 2); err != nil { 290 | return err 291 | } 292 | // Load the last segment entries 293 | if err := l.loadSegmentEntries(lseg); err != nil { 294 | return err 295 | } 296 | l.lastIndex = lseg.index + uint64(len(lseg.epos)) - 1 297 | if l.lastIndex > 0 && l.firstIndex > l.lastIndex && !l.opts.AllowEmpty { 298 | return ErrEmptyLog 299 | } 300 | return nil 301 | } 302 | 303 | // segmentName returns a 20-byte textual representation of an index 304 | // for lexical ordering. This is used for the file names of log segments. 305 | func segmentName(index uint64) string { 306 | return fmt.Sprintf("%020d", index) 307 | } 308 | 309 | // Close the log. 310 | func (l *Log) Close() error { 311 | l.mu.Lock() 312 | defer l.mu.Unlock() 313 | if l.closed { 314 | if l.corrupt { 315 | return ErrCorrupt 316 | } 317 | return ErrClosed 318 | } 319 | if err := l.sfile.Sync(); err != nil { 320 | return err 321 | } 322 | if err := l.sfile.Close(); err != nil { 323 | return err 324 | } 325 | l.closed = true 326 | if l.corrupt { 327 | return ErrCorrupt 328 | } 329 | return nil 330 | } 331 | 332 | // Write an entry to the log. 333 | func (l *Log) Write(index uint64, data []byte) error { 334 | l.mu.Lock() 335 | defer l.mu.Unlock() 336 | if l.corrupt { 337 | return ErrCorrupt 338 | } else if l.closed { 339 | return ErrClosed 340 | } 341 | l.wbatch.Clear() 342 | l.wbatch.Write(index, data) 343 | return l.writeBatch(&l.wbatch) 344 | } 345 | 346 | func (l *Log) appendEntry(dst []byte, index uint64, data []byte) (out []byte, 347 | epos bpos) { 348 | if l.opts.LogFormat == JSON { 349 | return appendJSONEntry(dst, index, data) 350 | } 351 | return appendBinaryEntry(dst, data) 352 | } 353 | 354 | // Cycle the old segment for a new segment. 355 | func (l *Log) cycle() error { 356 | if err := l.sfile.Sync(); err != nil { 357 | return err 358 | } 359 | if err := l.sfile.Close(); err != nil { 360 | return err 361 | } 362 | // cache the previous segment 363 | l.pushCache(len(l.segments) - 1) 364 | s := &segment{ 365 | index: l.lastIndex + 1, 366 | path: filepath.Join(l.path, segmentName(l.lastIndex+1)), 367 | } 368 | var err error 369 | l.sfile, err = os.OpenFile(s.path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 370 | l.opts.FilePerms) 371 | if err != nil { 372 | return err 373 | } 374 | l.segments = append(l.segments, s) 375 | return nil 376 | } 377 | 378 | func appendJSONEntry(dst []byte, index uint64, data []byte) (out []byte, 379 | epos bpos) { 380 | // {"index":number,"data":string} 381 | pos := len(dst) 382 | dst = append(dst, `{"index":"`...) 383 | dst = strconv.AppendUint(dst, index, 10) 384 | dst = append(dst, `","data":`...) 385 | dst = appendJSONData(dst, data) 386 | dst = append(dst, '}', '\n') 387 | return dst, bpos{pos, len(dst)} 388 | } 389 | 390 | func appendJSONData(dst []byte, s []byte) []byte { 391 | if utf8.Valid(s) { 392 | b, _ := json.Marshal(*(*string)(unsafe.Pointer(&s))) 393 | dst = append(dst, '"', '+') 394 | return append(dst, b[1:]...) 395 | } 396 | dst = append(dst, '"', '$') 397 | dst = append(dst, base64.URLEncoding.EncodeToString(s)...) 398 | return append(dst, '"') 399 | } 400 | 401 | func appendBinaryEntry(dst []byte, data []byte) (out []byte, epos bpos) { 402 | // data_size + data 403 | pos := len(dst) 404 | dst = appendUvarint(dst, uint64(len(data))) 405 | dst = append(dst, data...) 406 | return dst, bpos{pos, len(dst)} 407 | } 408 | 409 | func appendUvarint(dst []byte, x uint64) []byte { 410 | var buf [10]byte 411 | n := binary.PutUvarint(buf[:], x) 412 | dst = append(dst, buf[:n]...) 413 | return dst 414 | } 415 | 416 | // Batch of entries. Used to write multiple entries at once using WriteBatch(). 417 | type Batch struct { 418 | entries []batchEntry 419 | datas []byte 420 | } 421 | 422 | type batchEntry struct { 423 | index uint64 424 | size int 425 | } 426 | 427 | // Write an entry to the batch 428 | func (b *Batch) Write(index uint64, data []byte) { 429 | b.entries = append(b.entries, batchEntry{index, len(data)}) 430 | b.datas = append(b.datas, data...) 431 | } 432 | 433 | // Clear the batch for reuse. 434 | func (b *Batch) Clear() { 435 | b.entries = b.entries[:0] 436 | b.datas = b.datas[:0] 437 | } 438 | 439 | // WriteBatch writes the entries in the batch to the log in the order that they 440 | // were added to the batch. The batch is cleared upon a successful return. 441 | func (l *Log) WriteBatch(b *Batch) error { 442 | l.mu.Lock() 443 | defer l.mu.Unlock() 444 | if l.corrupt { 445 | return ErrCorrupt 446 | } else if l.closed { 447 | return ErrClosed 448 | } 449 | if len(b.entries) == 0 { 450 | return nil 451 | } 452 | return l.writeBatch(b) 453 | } 454 | 455 | func (l *Log) writeBatch(b *Batch) error { 456 | // check that all indexes in batch are sane 457 | for i := 0; i < len(b.entries); i++ { 458 | if b.entries[i].index != l.lastIndex+uint64(i+1) { 459 | return ErrOutOfOrder 460 | } 461 | } 462 | // load the tail segment 463 | s := l.segments[len(l.segments)-1] 464 | if len(s.ebuf) > l.opts.SegmentSize { 465 | // tail segment has reached capacity. Close it and create a new one. 466 | if err := l.cycle(); err != nil { 467 | return err 468 | } 469 | s = l.segments[len(l.segments)-1] 470 | } 471 | 472 | mark := len(s.ebuf) 473 | datas := b.datas 474 | for i := 0; i < len(b.entries); i++ { 475 | data := datas[:b.entries[i].size] 476 | var epos bpos 477 | s.ebuf, epos = l.appendEntry(s.ebuf, b.entries[i].index, data) 478 | s.epos = append(s.epos, epos) 479 | if len(s.ebuf) >= l.opts.SegmentSize { 480 | // segment has reached capacity, cycle now 481 | if _, err := l.sfile.Write(s.ebuf[mark:]); err != nil { 482 | return err 483 | } 484 | l.lastIndex = b.entries[i].index 485 | if err := l.cycle(); err != nil { 486 | return err 487 | } 488 | s = l.segments[len(l.segments)-1] 489 | mark = 0 490 | } 491 | datas = datas[b.entries[i].size:] 492 | } 493 | if len(s.ebuf)-mark > 0 { 494 | if _, err := l.sfile.Write(s.ebuf[mark:]); err != nil { 495 | return err 496 | } 497 | l.lastIndex = b.entries[len(b.entries)-1].index 498 | } 499 | if !l.opts.NoSync { 500 | if err := l.sfile.Sync(); err != nil { 501 | return err 502 | } 503 | } 504 | b.Clear() 505 | return nil 506 | } 507 | 508 | // FirstIndex returns the index of the first entry in the log. 509 | // Returns zero when log has no entries. 510 | // When using the `AllowEmpty` option and when the log is empty, this will 511 | // return LastIndex+1, which is the next future index. 512 | func (l *Log) FirstIndex() (index uint64, err error) { 513 | l.mu.RLock() 514 | defer l.mu.RUnlock() 515 | if l.corrupt { 516 | return 0, ErrCorrupt 517 | } else if l.closed { 518 | return 0, ErrClosed 519 | } 520 | if !l.opts.AllowEmpty && l.lastIndex == 0 { 521 | return 0, nil 522 | } 523 | return l.firstIndex, nil 524 | } 525 | 526 | // LastIndex returns the index of the last entry in the log. 527 | // Returns zero when log has no entries. 528 | // When using the `AllowEmpty` option and when the log is empty, this will 529 | // return FirstIndex()-1, which is the last known deleted index. 530 | func (l *Log) LastIndex() (index uint64, err error) { 531 | l.mu.RLock() 532 | defer l.mu.RUnlock() 533 | if l.corrupt { 534 | return 0, ErrCorrupt 535 | } else if l.closed { 536 | return 0, ErrClosed 537 | } 538 | if !l.opts.AllowEmpty && l.firstIndex == 0 { 539 | return 0, nil 540 | } 541 | return l.lastIndex, nil 542 | } 543 | 544 | // findSegment performs a bsearch on the segments 545 | func (l *Log) findSegment(index uint64) int { 546 | i, j := 0, len(l.segments) 547 | for i < j { 548 | h := i + (j-i)/2 549 | if index >= l.segments[h].index { 550 | i = h + 1 551 | } else { 552 | j = h 553 | } 554 | } 555 | return i - 1 556 | } 557 | 558 | func (l *Log) loadSegmentEntries(s *segment) error { 559 | data, err := ioutil.ReadFile(s.path) 560 | if err != nil { 561 | return err 562 | } 563 | ebuf := data 564 | var epos []bpos 565 | var pos int 566 | for exidx := s.index; len(data) > 0; exidx++ { 567 | var n int 568 | if l.opts.LogFormat == JSON { 569 | n, err = loadNextJSONEntry(data) 570 | } else { 571 | n, err = loadNextBinaryEntry(data) 572 | } 573 | if err != nil { 574 | return err 575 | } 576 | data = data[n:] 577 | epos = append(epos, bpos{pos, pos + n}) 578 | pos += n 579 | } 580 | s.ebuf = ebuf 581 | s.epos = epos 582 | return nil 583 | } 584 | 585 | func loadNextJSONEntry(data []byte) (n int, err error) { 586 | // {"index":number,"data":string} 587 | idx := bytes.IndexByte(data, '\n') 588 | if idx == -1 { 589 | return 0, ErrCorrupt 590 | } 591 | line := data[:idx] 592 | dres := gjson.Get(*(*string)(unsafe.Pointer(&line)), "data") 593 | if dres.Type != gjson.String { 594 | return 0, ErrCorrupt 595 | } 596 | return idx + 1, nil 597 | } 598 | 599 | func loadNextBinaryEntry(data []byte) (n int, err error) { 600 | // data_size + data 601 | size, n := binary.Uvarint(data) 602 | if n <= 0 { 603 | return 0, ErrCorrupt 604 | } 605 | if uint64(len(data)-n) < size { 606 | return 0, ErrCorrupt 607 | } 608 | return n + int(size), nil 609 | } 610 | 611 | // loadSegment loads the segment entries into memory, pushes it to the front 612 | // of the lru cache, and returns it. 613 | func (l *Log) loadSegment(index uint64) (*segment, error) { 614 | // check the last segment first. 615 | lseg := l.segments[len(l.segments)-1] 616 | if index >= lseg.index { 617 | return lseg, nil 618 | } 619 | // check the most recent cached segment 620 | var rseg *segment 621 | l.scache.Range(func(_, v interface{}) bool { 622 | s := v.(*segment) 623 | if index >= s.index && index < s.index+uint64(len(s.epos)) { 624 | rseg = s 625 | } 626 | return false 627 | }) 628 | if rseg != nil { 629 | return rseg, nil 630 | } 631 | // find in the segment array 632 | idx := l.findSegment(index) 633 | s := l.segments[idx] 634 | if len(s.epos) == 0 { 635 | // load the entries from cache 636 | if err := l.loadSegmentEntries(s); err != nil { 637 | return nil, err 638 | } 639 | } 640 | // push the segment to the front of the cache 641 | l.pushCache(idx) 642 | return s, nil 643 | } 644 | 645 | // Read an entry from the log. Returns a byte slice containing the data entry. 646 | func (l *Log) Read(index uint64) (data []byte, err error) { 647 | l.mu.RLock() 648 | defer l.mu.RUnlock() 649 | if l.corrupt { 650 | return nil, ErrCorrupt 651 | } else if l.closed { 652 | return nil, ErrClosed 653 | } 654 | if index < l.firstIndex || index > l.lastIndex { 655 | return nil, ErrNotFound 656 | } 657 | s, err := l.loadSegment(index) 658 | if err != nil { 659 | return nil, err 660 | } 661 | epos := s.epos[index-s.index] 662 | edata := s.ebuf[epos.pos:epos.end] 663 | if l.opts.LogFormat == JSON { 664 | return readJSON(edata) 665 | } 666 | // binary read 667 | size, n := binary.Uvarint(edata) 668 | if n <= 0 { 669 | return nil, ErrCorrupt 670 | } 671 | if uint64(len(edata)-n) < size { 672 | return nil, ErrCorrupt 673 | } 674 | if l.opts.NoCopy { 675 | data = edata[n : uint64(n)+size] 676 | } else { 677 | data = make([]byte, size) 678 | copy(data, edata[n:]) 679 | } 680 | return data, nil 681 | } 682 | 683 | //go:noinline 684 | func readJSON(edata []byte) ([]byte, error) { 685 | var data []byte 686 | s := gjson.Get(*(*string)(unsafe.Pointer(&edata)), "data").String() 687 | if len(s) > 0 && s[0] == '$' { 688 | var err error 689 | data, err = base64.URLEncoding.DecodeString(s[1:]) 690 | if err != nil { 691 | return nil, ErrCorrupt 692 | } 693 | } else if len(s) > 0 && s[0] == '+' { 694 | data = make([]byte, len(s[1:])) 695 | copy(data, s[1:]) 696 | } else { 697 | return nil, ErrCorrupt 698 | } 699 | return data, nil 700 | } 701 | 702 | // ClearCache clears the segment cache. 703 | // This only frees internal buffers and the LRU cache and does not modify the 704 | // contents of the log. 705 | func (l *Log) ClearCache() error { 706 | l.mu.Lock() 707 | defer l.mu.Unlock() 708 | if l.corrupt { 709 | return ErrCorrupt 710 | } else if l.closed { 711 | return ErrClosed 712 | } 713 | l.clearCache() 714 | return nil 715 | } 716 | 717 | func (l *Log) clearCache() { 718 | l.scache.Range(func(_, v interface{}) bool { 719 | s := v.(*segment) 720 | s.ebuf = nil 721 | s.epos = nil 722 | return true 723 | }) 724 | l.scache = tinylru.LRU{} 725 | l.scache.Resize(l.opts.SegmentCacheSize) 726 | } 727 | 728 | // atomicWrite performs an temp write + rename to ensure the file writing is 729 | // and atomic operation. One os.WriteFile alone is not good enough. 730 | func (l *Log) atomicWrite(name string, data []byte) error { 731 | // Create a TEMP file 732 | tempName := name + ".TEMP" 733 | defer os.RemoveAll(tempName) 734 | if err := func() error { 735 | f, err := os.OpenFile(tempName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 736 | l.opts.FilePerms) 737 | if err != nil { 738 | return err 739 | } 740 | defer f.Close() 741 | if _, err := f.Write(data); err != nil { 742 | return err 743 | } 744 | if err := f.Sync(); err != nil { 745 | return err 746 | } 747 | return f.Close() 748 | }(); err != nil { 749 | return err 750 | } 751 | // Rename the TEMP file to final name 752 | return os.Rename(tempName, name) 753 | } 754 | 755 | // TruncateFront truncates the front of the log by removing all entries that 756 | // are before the provided `index`. In other words the entry at `index` becomes 757 | // the first entry in the log. 758 | // 759 | // The `AllowEmpty` option may be used to allow for removing all entries in the 760 | // log by providing `LastIndex+1` as the index. Otherwise without `AllowEmpty`, 761 | // at least one entry must always remain following a truncate. 762 | func (l *Log) TruncateFront(index uint64) error { 763 | l.mu.Lock() 764 | defer l.mu.Unlock() 765 | if l.corrupt { 766 | return ErrCorrupt 767 | } else if l.closed { 768 | return ErrClosed 769 | } 770 | return l.truncateFront(index) 771 | } 772 | 773 | func (l *Log) truncateFront(index uint64) (err error) { 774 | if index < l.firstIndex || index > l.lastIndex+1 { 775 | return ErrOutOfRange 776 | } 777 | if !l.opts.AllowEmpty && index == l.lastIndex+1 { 778 | return ErrOutOfRange 779 | } 780 | if index == l.firstIndex { 781 | // nothing to truncate 782 | return nil 783 | } 784 | var segIdx int 785 | var s *segment 786 | var ebuf []byte 787 | if index == l.lastIndex+1 { 788 | // Truncate all entries, only care about the last segment 789 | segIdx = len(l.segments) - 1 790 | s = l.segments[segIdx] 791 | ebuf = nil 792 | } else { 793 | segIdx = l.findSegment(index) 794 | s, err = l.loadSegment(index) 795 | if err != nil { 796 | return err 797 | } 798 | epos := s.epos[index-s.index:] 799 | ebuf = s.ebuf[epos[0].pos:] 800 | } 801 | // Create a START file contains the truncated segment. 802 | startName := filepath.Join(l.path, segmentName(index)+".START") 803 | if err = l.atomicWrite(startName, ebuf); err != nil { 804 | return fmt.Errorf("failed to create start segment: %w", err) 805 | } 806 | // The log was truncated but still needs some file cleanup. Any errors 807 | // following this message will not cause an on-disk data ocorruption, but 808 | // may cause an inconsistency with the current program, so we'll return 809 | // ErrCorrupt so the the user can attempt a recover by calling Close() 810 | // followed by Open(). 811 | defer func() { 812 | if v := recover(); v != nil { 813 | err = ErrCorrupt 814 | } 815 | if err != nil { 816 | l.corrupt = true 817 | } 818 | }() 819 | if segIdx == len(l.segments)-1 { 820 | // Close the tail segment file 821 | if err = l.sfile.Close(); err != nil { 822 | return err 823 | } 824 | } 825 | // Delete truncated segment files 826 | for i := 0; i <= segIdx; i++ { 827 | if err = os.Remove(l.segments[i].path); err != nil { 828 | return err 829 | } 830 | } 831 | // Rename the START file to the final truncated segment name. 832 | newName := filepath.Join(l.path, segmentName(index)) 833 | if err = os.Rename(startName, newName); err != nil { 834 | return err 835 | } 836 | s.path = newName 837 | s.index = index 838 | if segIdx == len(l.segments)-1 { 839 | // Reopen the tail segment file 840 | l.sfile, err = os.OpenFile(newName, os.O_WRONLY, l.opts.FilePerms) 841 | if err != nil { 842 | return err 843 | } 844 | var n int64 845 | if n, err = l.sfile.Seek(0, 2); err != nil { 846 | return err 847 | } 848 | if n != int64(len(ebuf)) { 849 | err = errors.New("invalid seek") 850 | return err 851 | } 852 | // Load the last segment entries 853 | if err = l.loadSegmentEntries(s); err != nil { 854 | return err 855 | } 856 | } 857 | l.segments = append([]*segment{}, l.segments[segIdx:]...) 858 | l.firstIndex = index 859 | l.clearCache() 860 | return nil 861 | } 862 | 863 | // TruncateBack truncates the back of the log by removing all entries that 864 | // are after the provided `index`. In other words the entry at `index` becomes 865 | // the last entry in the log. 866 | // 867 | // The `AllowEmpty` option may be used to allow for removing all entries in the 868 | // log by providing `FirstIndex()-1` as the index. Otherwise without 869 | // `AllowEmpty`, at least one entry must always remain following a truncate. 870 | func (l *Log) TruncateBack(index uint64) error { 871 | l.mu.Lock() 872 | defer l.mu.Unlock() 873 | if l.corrupt { 874 | return ErrCorrupt 875 | } else if l.closed { 876 | return ErrClosed 877 | } 878 | return l.truncateBack(index) 879 | } 880 | 881 | func (l *Log) truncateBack(index uint64) (err error) { 882 | if index < l.firstIndex-1 || index > l.lastIndex { 883 | return ErrOutOfRange 884 | } 885 | if !l.opts.AllowEmpty && index == l.firstIndex-1 { 886 | return ErrOutOfRange 887 | } 888 | if index == l.lastIndex { 889 | // nothing to truncate 890 | return nil 891 | } 892 | var segIdx int 893 | var s *segment 894 | var ebuf []byte 895 | if index == l.firstIndex-1 { 896 | // Truncate all entries, only care about the first segment 897 | segIdx = 0 898 | s = l.segments[segIdx] 899 | ebuf = nil 900 | } else { 901 | segIdx = l.findSegment(index) 902 | s, err = l.loadSegment(index) 903 | if err != nil { 904 | return err 905 | } 906 | epos := s.epos[:index-s.index+1] 907 | ebuf = s.ebuf[:epos[len(epos)-1].end] 908 | } 909 | // Create an END file contains the truncated segment. 910 | endName := filepath.Join(l.path, segmentName(s.index)+".END") 911 | if err = l.atomicWrite(endName, ebuf); err != nil { 912 | return fmt.Errorf("failed to create end segment: %w", err) 913 | } 914 | // The log was truncated but still needs some file cleanup. Any errors 915 | // following this message will not cause an on-disk data ocorruption, but 916 | // may cause an inconsistency with the current program, so we'll return 917 | // ErrCorrupt so the the user can attempt a recover by calling Close() 918 | // followed by Open(). 919 | defer func() { 920 | if v := recover(); v != nil { 921 | err = ErrCorrupt 922 | } 923 | if err != nil { 924 | l.corrupt = true 925 | } 926 | }() 927 | 928 | // Close the tail segment file 929 | if err = l.sfile.Close(); err != nil { 930 | return err 931 | } 932 | // Delete truncated segment files 933 | for i := segIdx; i < len(l.segments); i++ { 934 | if err = os.Remove(l.segments[i].path); err != nil { 935 | return err 936 | } 937 | } 938 | // Rename the END file to the final truncated segment name. 939 | newName := filepath.Join(l.path, segmentName(s.index)) 940 | if err = os.Rename(endName, newName); err != nil { 941 | return err 942 | } 943 | // Reopen the tail segment file 944 | l.sfile, err = os.OpenFile(newName, os.O_WRONLY, l.opts.FilePerms) 945 | if err != nil { 946 | return err 947 | } 948 | var n int64 949 | n, err = l.sfile.Seek(0, 2) 950 | if err != nil { 951 | return err 952 | } 953 | if n != int64(len(ebuf)) { 954 | err = errors.New("invalid seek") 955 | return err 956 | } 957 | s.path = newName 958 | l.segments = append([]*segment{}, l.segments[:segIdx+1]...) 959 | l.lastIndex = index 960 | l.clearCache() 961 | if err = l.loadSegmentEntries(s); err != nil { 962 | return err 963 | } 964 | return nil 965 | } 966 | 967 | // Sync performs an fsync on the log. This is not necessary when the 968 | // NoSync option is set to false. 969 | func (l *Log) Sync() error { 970 | l.mu.Lock() 971 | defer l.mu.Unlock() 972 | if l.corrupt { 973 | return ErrCorrupt 974 | } else if l.closed { 975 | return ErrClosed 976 | } 977 | return l.sfile.Sync() 978 | } 979 | 980 | // IsEmpty returns true if there are no entries in the log. 981 | func (l *Log) IsEmpty() (bool, error) { 982 | l.mu.Lock() 983 | defer l.mu.Unlock() 984 | if l.corrupt { 985 | return false, ErrCorrupt 986 | } else if l.closed { 987 | return false, ErrClosed 988 | } 989 | return (l.firstIndex == 0 && l.lastIndex == 0) || 990 | l.firstIndex > l.lastIndex, nil 991 | } 992 | -------------------------------------------------------------------------------- /wal_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io/ioutil" 7 | "math/rand" 8 | "os" 9 | "strings" 10 | "sync" 11 | "sync/atomic" 12 | "testing" 13 | ) 14 | 15 | func dataStr(index uint64) string { 16 | if index%2 == 0 { 17 | return fmt.Sprintf("data-\"%d\"", index) 18 | } 19 | return fmt.Sprintf("data-'%d'", index) 20 | } 21 | 22 | func testLog(t *testing.T, opts *Options, N int) { 23 | logPath := "testlog/" + strings.Join(strings.Split(t.Name(), "/")[1:], "/") 24 | l, err := Open(logPath, opts) 25 | if err != nil { 26 | t.Fatal(err) 27 | } 28 | defer l.Close() 29 | 30 | // FirstIndex - should be zero or one, depending on allow empty. 31 | n, err := l.FirstIndex() 32 | if err != nil { 33 | t.Fatal(err) 34 | } 35 | if !l.opts.AllowEmpty { 36 | if n != 0 { 37 | t.Fatalf("expected %d, got %d", 0, n) 38 | } 39 | } else { 40 | if n != 1 { 41 | t.Fatalf("expected %d, got %d", 1, n) 42 | } 43 | } 44 | 45 | // LastIndex - should be zero 46 | n, err = l.LastIndex() 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | if n != 0 { 51 | t.Fatalf("expected %d, got %d", 0, n) 52 | } 53 | 54 | for i := 1; i <= N; i++ { 55 | // Write - try to append previous index, should fail 56 | err = l.Write(uint64(i-1), nil) 57 | if err != ErrOutOfOrder { 58 | t.Fatalf("expected %v, got %v", ErrOutOfOrder, err) 59 | } 60 | // Write - append next item 61 | err = l.Write(uint64(i), []byte(dataStr(uint64(i)))) 62 | if err != nil { 63 | t.Fatalf("expected %v, got %v", nil, err) 64 | } 65 | // Write - get next item 66 | data, err := l.Read(uint64(i)) 67 | if err != nil { 68 | t.Fatalf("expected %v, got %v", nil, err) 69 | } 70 | if string(data) != dataStr(uint64(i)) { 71 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 72 | } 73 | } 74 | 75 | // Read -- should fail, not found 76 | _, err = l.Read(0) 77 | if err != ErrNotFound { 78 | t.Fatalf("expected %v, got %v", ErrNotFound, err) 79 | } 80 | // Read -- read back all entries 81 | for i := 1; i <= N; i++ { 82 | data, err := l.Read(uint64(i)) 83 | if err != nil { 84 | t.Fatalf("error while getting %d", i) 85 | } 86 | if string(data) != dataStr(uint64(i)) { 87 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 88 | } 89 | } 90 | // Read -- read back first half entries 91 | for i := 1; i <= N/2; i++ { 92 | data, err := l.Read(uint64(i)) 93 | if err != nil { 94 | t.Fatalf("error while getting %d", i) 95 | } 96 | if string(data) != dataStr(uint64(i)) { 97 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 98 | } 99 | } 100 | // Read -- read second third entries 101 | for i := N / 3; i <= N/3+N/3; i++ { 102 | data, err := l.Read(uint64(i)) 103 | if err != nil { 104 | t.Fatalf("error while getting %d", i) 105 | } 106 | if string(data) != dataStr(uint64(i)) { 107 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 108 | } 109 | } 110 | 111 | // Read -- random access 112 | for _, v := range rand.Perm(N) { 113 | index := uint64(v + 1) 114 | data, err := l.Read(index) 115 | if err != nil { 116 | t.Fatal(err) 117 | } 118 | if dataStr(index) != string(data) { 119 | t.Fatalf("expected %v, got %v", dataStr(index), string(data)) 120 | } 121 | } 122 | 123 | // FirstIndex/LastIndex -- check valid first and last indexes 124 | n, err = l.FirstIndex() 125 | if err != nil { 126 | t.Fatal(err) 127 | } 128 | if n != 1 { 129 | t.Fatalf("expected %d, got %d", 1, n) 130 | } 131 | n, err = l.LastIndex() 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | if n != uint64(N) { 136 | t.Fatalf("expected %d, got %d", N, n) 137 | } 138 | 139 | // Close -- close the log 140 | if err := l.Close(); err != nil { 141 | t.Fatal(err) 142 | } 143 | 144 | // Write - try while closed 145 | err = l.Write(1, nil) 146 | if err != ErrClosed { 147 | t.Fatalf("expected %v, got %v", ErrClosed, err) 148 | } 149 | // WriteBatch - try while closed 150 | err = l.WriteBatch(nil) 151 | if err != ErrClosed { 152 | t.Fatalf("expected %v, got %v", ErrClosed, err) 153 | } 154 | // FirstIndex - try while closed 155 | _, err = l.FirstIndex() 156 | if err != ErrClosed { 157 | t.Fatalf("expected %v, got %v", ErrClosed, err) 158 | } 159 | // LastIndex - try while closed 160 | _, err = l.LastIndex() 161 | if err != ErrClosed { 162 | t.Fatalf("expected %v, got %v", ErrClosed, err) 163 | } 164 | // Get - try while closed 165 | _, err = l.Read(0) 166 | if err != ErrClosed { 167 | t.Fatalf("expected %v, got %v", ErrClosed, err) 168 | } 169 | // TruncateFront - try while closed 170 | err = l.TruncateFront(0) 171 | if err != ErrClosed { 172 | t.Fatalf("expected %v, got %v", ErrClosed, err) 173 | } 174 | // TruncateBack - try while closed 175 | err = l.TruncateBack(0) 176 | if err != ErrClosed { 177 | t.Fatalf("expected %v, got %v", ErrClosed, err) 178 | } 179 | 180 | // Open -- reopen log 181 | l, err = Open(logPath, opts) 182 | if err != nil { 183 | t.Fatal(err) 184 | } 185 | defer l.Close() 186 | 187 | // Read -- read back all entries 188 | for i := 1; i <= N; i++ { 189 | data, err := l.Read(uint64(i)) 190 | if err != nil { 191 | t.Fatalf("error while getting %d, err=%s", i, err) 192 | } 193 | if string(data) != dataStr(uint64(i)) { 194 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 195 | } 196 | } 197 | // FirstIndex/LastIndex -- check valid first and last indexes 198 | n, err = l.FirstIndex() 199 | if err != nil { 200 | t.Fatal(err) 201 | } 202 | if n != 1 { 203 | t.Fatalf("expected %d, got %d", 1, n) 204 | } 205 | n, err = l.LastIndex() 206 | if err != nil { 207 | t.Fatal(err) 208 | } 209 | if n != uint64(N) { 210 | t.Fatalf("expected %d, got %d", N, n) 211 | } 212 | // Write -- add 50 more items 213 | for i := N + 1; i <= N+50; i++ { 214 | index := uint64(i) 215 | if err := l.Write(index, []byte(dataStr(index))); err != nil { 216 | t.Fatal(err) 217 | } 218 | data, err := l.Read(index) 219 | if err != nil { 220 | t.Fatal(err) 221 | } 222 | if string(data) != dataStr(index) { 223 | t.Fatalf("expected %v, got %v", dataStr(index), string(data)) 224 | } 225 | } 226 | N += 50 227 | // FirstIndex/LastIndex -- check valid first and last indexes 228 | n, err = l.FirstIndex() 229 | if err != nil { 230 | t.Fatal(err) 231 | } 232 | if n != 1 { 233 | t.Fatalf("expected %d, got %d", 1, n) 234 | } 235 | n, err = l.LastIndex() 236 | if err != nil { 237 | t.Fatal(err) 238 | } 239 | if n != uint64(N) { 240 | t.Fatalf("expected %d, got %d", N, n) 241 | } 242 | // Batch -- test batch writes 243 | b := new(Batch) 244 | b.Write(1, nil) 245 | b.Write(2, nil) 246 | b.Write(3, nil) 247 | // WriteBatch -- should fail out of order 248 | err = l.WriteBatch(b) 249 | if err != ErrOutOfOrder { 250 | t.Fatalf("expected %v, got %v", ErrOutOfOrder, nil) 251 | } 252 | // Clear -- clear the batch 253 | b.Clear() 254 | // WriteBatch -- should succeed 255 | err = l.WriteBatch(b) 256 | if err != nil { 257 | t.Fatal(err) 258 | } 259 | // Write 100 entries in batches of 10 260 | for i := 0; i < 10; i++ { 261 | for i := N + 1; i <= N+10; i++ { 262 | index := uint64(i) 263 | b.Write(index, []byte(dataStr(index))) 264 | } 265 | err = l.WriteBatch(b) 266 | if err != nil { 267 | t.Fatal(err) 268 | } 269 | N += 10 270 | } 271 | // Read -- read back all entries 272 | for i := 1; i <= N; i++ { 273 | data, err := l.Read(uint64(i)) 274 | if err != nil { 275 | t.Fatalf("error while getting %d", i) 276 | } 277 | if string(data) != dataStr(uint64(i)) { 278 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 279 | } 280 | } 281 | 282 | // Write -- one entry, so the buffer might be activated 283 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 284 | if err != nil { 285 | t.Fatal(err) 286 | } 287 | N++ 288 | // Read -- one random read, so there is an opened reader 289 | data, err := l.Read(uint64(N / 2)) 290 | if err != nil { 291 | t.Fatal(err) 292 | } 293 | if string(data) != dataStr(uint64(N/2)) { 294 | t.Fatalf("expected %v, got %v", dataStr(uint64(N/2)), string(data)) 295 | } 296 | 297 | // TruncateFront -- should fail, out of range 298 | for _, i := range []int{0, N + 2} { 299 | index := uint64(i) 300 | if err = l.TruncateFront(index); err != ErrOutOfRange { 301 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 302 | } 303 | testFirstLast(t, l, uint64(1), uint64(N), nil) 304 | } 305 | 306 | // TruncateBack -- should fail, out of range 307 | err = l.TruncateFront(0) 308 | if err != ErrOutOfRange { 309 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 310 | } 311 | testFirstLast(t, l, uint64(1), uint64(N), nil) 312 | 313 | // TruncateFront -- Remove no entries 314 | if err = l.TruncateFront(1); err != nil { 315 | t.Fatal(err) 316 | } 317 | testFirstLast(t, l, uint64(1), uint64(N), nil) 318 | 319 | // TruncateFront -- Remove first 80 entries 320 | if err = l.TruncateFront(81); err != nil { 321 | t.Fatal(err) 322 | } 323 | testFirstLast(t, l, uint64(81), uint64(N), nil) 324 | 325 | // Write -- one entry, so the buffer might be activated 326 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 327 | if err != nil { 328 | t.Fatal(err) 329 | } 330 | N++ 331 | testFirstLast(t, l, uint64(81), uint64(N), nil) 332 | 333 | // Read -- one random read, so there is an opened reader 334 | data, err = l.Read(uint64(N / 2)) 335 | if err != nil { 336 | t.Fatal(err) 337 | } 338 | if string(data) != dataStr(uint64(N/2)) { 339 | t.Fatalf("expected %v, got %v", dataStr(uint64(N/2)), string(data)) 340 | } 341 | 342 | // TruncateBack -- should fail, out of range 343 | for _, i := range []int{0, 79} { 344 | index := uint64(i) 345 | if err = l.TruncateBack(index); err != ErrOutOfRange { 346 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 347 | } 348 | testFirstLast(t, l, uint64(81), uint64(N), nil) 349 | } 350 | 351 | // TruncateBack -- Remove no entries 352 | if err = l.TruncateBack(uint64(N)); err != nil { 353 | t.Fatal(err) 354 | } 355 | testFirstLast(t, l, uint64(81), uint64(N), nil) 356 | // TruncateBack -- Remove last 80 entries 357 | if err = l.TruncateBack(uint64(N - 80)); err != nil { 358 | t.Fatal(err) 359 | } 360 | N -= 80 361 | testFirstLast(t, l, uint64(81), uint64(N), nil) 362 | 363 | // Read -- read back all entries 364 | for i := 81; i <= N; i++ { 365 | data, err := l.Read(uint64(i)) 366 | if err != nil { 367 | t.Fatalf("error while getting %d", i) 368 | } 369 | if string(data) != dataStr(uint64(i)) { 370 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 371 | } 372 | } 373 | 374 | // Close -- close log after truncating 375 | if err = l.Close(); err != nil { 376 | t.Fatal(err) 377 | } 378 | 379 | // Open -- open log after truncating 380 | l, err = Open(logPath, opts) 381 | if err != nil { 382 | t.Fatal(err) 383 | } 384 | defer l.Close() 385 | 386 | testFirstLast(t, l, uint64(81), uint64(N), nil) 387 | 388 | // Read -- read back all entries 389 | for i := 81; i <= N; i++ { 390 | data, err := l.Read(uint64(i)) 391 | if err != nil { 392 | t.Fatalf("error while getting %d", i) 393 | } 394 | if string(data) != dataStr(uint64(i)) { 395 | t.Fatalf("expected %s, got %s", dataStr(uint64(i)), data) 396 | } 397 | } 398 | 399 | // TruncateFront -- truncate all entries but one 400 | if err = l.TruncateFront(uint64(N)); err != nil { 401 | t.Fatal(err) 402 | } 403 | testFirstLast(t, l, uint64(N), uint64(N), nil) 404 | 405 | // Write -- write on entry 406 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 407 | if err != nil { 408 | t.Fatal(err) 409 | } 410 | N++ 411 | testFirstLast(t, l, uint64(N-1), uint64(N), nil) 412 | 413 | // TruncateBack -- truncate all entries but one 414 | if err = l.TruncateBack(uint64(N - 1)); err != nil { 415 | t.Fatal(err) 416 | } 417 | N-- 418 | testFirstLast(t, l, uint64(N), uint64(N), nil) 419 | 420 | if err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))); err != nil { 421 | t.Fatal(err) 422 | } 423 | N++ 424 | 425 | l.Sync() 426 | testFirstLast(t, l, uint64(N-1), uint64(N), nil) 427 | 428 | allowEmpty := opts != nil && opts.AllowEmpty 429 | // TruncateFront -- truncate all entries 430 | 431 | err = l.TruncateFront(uint64(N + 1)) 432 | if allowEmpty { 433 | if err != nil { 434 | t.Fatal(err) 435 | } 436 | testFirstLast(t, l, uint64(N+1), uint64(N), nil) 437 | } else { 438 | if err != ErrOutOfRange { 439 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 440 | } 441 | } 442 | 443 | err = l.Write(uint64(N+1), []byte(dataStr(uint64(N+1)))) 444 | if err != nil { 445 | t.Fatal(err) 446 | } 447 | N++ 448 | if allowEmpty { 449 | testFirstLast(t, l, uint64(N), uint64(N), nil) 450 | } else { 451 | testFirstLast(t, l, uint64(N-2), uint64(N), nil) 452 | } 453 | 454 | // TruncateBack -- truncate all entries 455 | fidx, err := l.FirstIndex() 456 | if err != nil { 457 | t.Fatal(err) 458 | } 459 | if allowEmpty { 460 | if fidx != uint64(N) { 461 | t.Fatalf("expected %v, got %v", N, fidx) 462 | } 463 | } else { 464 | if fidx != uint64(N-2) { 465 | t.Fatalf("expected %v, got %v", N-2, fidx) 466 | } 467 | } 468 | 469 | err = l.TruncateBack(uint64(fidx - 1)) 470 | if allowEmpty { 471 | if err != nil { 472 | t.Fatal(err) 473 | } 474 | testFirstLast(t, l, uint64(N), uint64(N-1), nil) 475 | } else { 476 | if err != ErrOutOfRange { 477 | t.Fatalf("expected %v, got %v", ErrOutOfRange, err) 478 | } 479 | } 480 | } 481 | 482 | func testFirstLast(t *testing.T, l *Log, expectFirst, expectLast uint64, 483 | data func(index uint64) []byte, 484 | ) { 485 | t.Helper() 486 | fi, err := l.FirstIndex() 487 | if err != nil { 488 | t.Fatal(err) 489 | } 490 | li, err := l.LastIndex() 491 | if err != nil { 492 | t.Fatal(err) 493 | } 494 | if fi != expectFirst || li != expectLast { 495 | t.Fatalf("expected %v/%v, got %v/%v", expectFirst, expectLast, fi, li) 496 | } 497 | is, err := l.IsEmpty() 498 | if err != nil { 499 | t.Fatal(err) 500 | } 501 | if expectFirst > expectLast { 502 | if !is { 503 | t.Fatalf("expected true") 504 | } 505 | } else { 506 | if is { 507 | t.Fatalf("expected false") 508 | } 509 | } 510 | for i := fi; i <= li; i++ { 511 | dt1, err := l.Read(i) 512 | if err != nil { 513 | t.Fatal(err) 514 | } 515 | if data != nil { 516 | dt2 := data(i) 517 | if string(dt1) != string(dt2) { 518 | t.Fatalf("mismatch '%s' != '%s'", dt2, dt1) 519 | } 520 | } 521 | } 522 | 523 | } 524 | 525 | func TestLog(t *testing.T) { 526 | os.RemoveAll("testlog") 527 | defer os.RemoveAll("testlog") 528 | 529 | t.Run("nil-opts", func(t *testing.T) { 530 | testLog(t, nil, 500) 531 | }) 532 | t.Run("allow-empty", func(t *testing.T) { 533 | testLog(t, makeOpts(512, true, JSON, true), 500) 534 | }) 535 | t.Run("no-sync", func(t *testing.T) { 536 | t.Run("json", func(t *testing.T) { 537 | t.Run("no-empty", func(t *testing.T) { 538 | testLog(t, makeOpts(512, true, JSON, false), 500) 539 | }) 540 | t.Run("allow-empty", func(t *testing.T) { 541 | testLog(t, makeOpts(512, true, JSON, true), 500) 542 | }) 543 | }) 544 | t.Run("binary", func(t *testing.T) { 545 | t.Run("no-empty", func(t *testing.T) { 546 | testLog(t, makeOpts(512, true, Binary, false), 500) 547 | }) 548 | t.Run("allow-empty", func(t *testing.T) { 549 | testLog(t, makeOpts(512, true, Binary, true), 500) 550 | }) 551 | }) 552 | }) 553 | t.Run("sync", func(t *testing.T) { 554 | t.Run("json", func(t *testing.T) { 555 | t.Run("no-empty", func(t *testing.T) { 556 | testLog(t, makeOpts(512, false, JSON, false), 100) 557 | }) 558 | t.Run("allow-empty", func(t *testing.T) { 559 | testLog(t, makeOpts(512, false, JSON, true), 100) 560 | }) 561 | }) 562 | t.Run("binary", func(t *testing.T) { 563 | t.Run("no-empty", func(t *testing.T) { 564 | testLog(t, makeOpts(512, false, Binary, false), 100) 565 | }) 566 | t.Run("allow-empty", func(t *testing.T) { 567 | testLog(t, makeOpts(512, false, Binary, true), 100) 568 | }) 569 | }) 570 | }) 571 | } 572 | 573 | func TestOutliers(t *testing.T) { 574 | // Create some scenarios where the log has been corrupted, operations 575 | // fail, or various weirdnesses. 576 | t.Run("fail-in-memory", func(t *testing.T) { 577 | if l, err := Open(":memory:", nil); err == nil { 578 | l.Close() 579 | t.Fatal("expected error") 580 | } 581 | }) 582 | t.Run("fail-not-a-directory", func(t *testing.T) { 583 | defer os.RemoveAll("testlog/file") 584 | if err := os.MkdirAll("testlog", 0777); err != nil { 585 | t.Fatal(err) 586 | } else if f, err := os.Create("testlog/file"); err != nil { 587 | t.Fatal(err) 588 | } else if err := f.Close(); err != nil { 589 | t.Fatal(err) 590 | } else if l, err := Open("testlog/file", nil); err == nil { 591 | l.Close() 592 | t.Fatal("expected error") 593 | } 594 | }) 595 | t.Run("load-with-junk-files", func(t *testing.T) { 596 | // junk should be ignored 597 | defer os.RemoveAll("testlog/junk") 598 | if err := os.MkdirAll("testlog/junk/other1", 0777); err != nil { 599 | t.Fatal(err) 600 | } 601 | f, err := os.Create("testlog/junk/other2") 602 | if err != nil { 603 | t.Fatal(err) 604 | } 605 | f.Close() 606 | f, err = os.Create("testlog/junk/" + strings.Repeat("A", 20)) 607 | if err != nil { 608 | t.Fatal(err) 609 | } 610 | f.Close() 611 | l, err := Open("testlog/junk", nil) 612 | if err != nil { 613 | t.Fatal(err) 614 | } 615 | l.Close() 616 | }) 617 | 618 | t.Run("fail-corrupted-tail-json", func(t *testing.T) { 619 | defer os.RemoveAll("testlog/corrupt-tail") 620 | opts := makeOpts(512, true, JSON, false) 621 | os.MkdirAll("testlog/corrupt-tail", 0777) 622 | ioutil.WriteFile( 623 | "testlog/corrupt-tail/00000000000000000001", 624 | []byte("\n"), 0666) 625 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 626 | l.Close() 627 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 628 | } 629 | ioutil.WriteFile( 630 | "testlog/corrupt-tail/00000000000000000001", 631 | []byte(`{}`+"\n"), 0666) 632 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 633 | l.Close() 634 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 635 | } 636 | ioutil.WriteFile( 637 | "testlog/corrupt-tail/00000000000000000001", 638 | []byte(`{"index":"1"}`+"\n"), 0666) 639 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 640 | l.Close() 641 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 642 | } 643 | ioutil.WriteFile( 644 | "testlog/corrupt-tail/00000000000000000001", 645 | []byte(`{"index":"1","data":"?"}`), 0666) 646 | if l, err := Open("testlog/corrupt-tail", opts); err != ErrCorrupt { 647 | l.Close() 648 | t.Fatalf("expected %v, got %v", ErrCorrupt, err) 649 | } 650 | }) 651 | 652 | t.Run("start-marker-file", func(t *testing.T) { 653 | lpath := "testlog/start-marker" 654 | opts := makeOpts(512, true, JSON, false) 655 | l := must(Open(lpath, opts)).(*Log) 656 | defer l.Close() 657 | for i := uint64(1); i <= 100; i++ { 658 | must(nil, l.Write(i, []byte(dataStr(i)))) 659 | } 660 | path := l.segments[l.findSegment(35)].path 661 | firstIndex := l.segments[l.findSegment(35)].index 662 | must(nil, l.Close()) 663 | data := must(ioutil.ReadFile(path)).([]byte) 664 | must(nil, ioutil.WriteFile(path+".START", data, 0666)) 665 | l = must(Open(lpath, opts)).(*Log) 666 | defer l.Close() 667 | testFirstLast(t, l, firstIndex, 100, nil) 668 | }) 669 | } 670 | 671 | func makeOpts(segSize int, noSync bool, lf LogFormat, allowEmpty bool, 672 | ) *Options { 673 | opts := *DefaultOptions 674 | opts.SegmentSize = segSize 675 | opts.NoSync = noSync 676 | opts.LogFormat = lf 677 | opts.AllowEmpty = allowEmpty 678 | return &opts 679 | } 680 | 681 | // https://github.com/tidwall/wal/issues/1 682 | func TestIssue1(t *testing.T) { 683 | in := []byte{0, 0, 0, 0, 0, 0, 0, 1, 37, 108, 131, 178, 151, 17, 77, 32, 684 | 27, 48, 23, 159, 63, 14, 240, 202, 206, 151, 131, 98, 45, 165, 151, 67, 685 | 38, 180, 54, 23, 138, 238, 246, 16, 0, 0, 0, 0} 686 | opts := *DefaultOptions 687 | opts.LogFormat = JSON 688 | os.RemoveAll("testlog") 689 | l, err := Open("testlog", &opts) 690 | if err != nil { 691 | t.Fatal(err) 692 | } 693 | defer l.Close() 694 | if err := l.Write(1, in); err != nil { 695 | t.Fatal(err) 696 | } 697 | out, err := l.Read(1) 698 | if err != nil { 699 | t.Fatal(err) 700 | } 701 | if string(in) != string(out) { 702 | t.Fatal("data mismatch") 703 | } 704 | } 705 | 706 | func TestSimpleTruncateFront(t *testing.T) { 707 | os.RemoveAll("testlog") 708 | 709 | opts := &Options{ 710 | NoSync: true, 711 | LogFormat: JSON, 712 | SegmentSize: 100, 713 | } 714 | 715 | l, err := Open("testlog", opts) 716 | if err != nil { 717 | t.Fatal(err) 718 | } 719 | defer func() { 720 | l.Close() 721 | }() 722 | 723 | makeData := func(index uint64) []byte { 724 | return []byte(fmt.Sprintf("data-%d", index)) 725 | } 726 | 727 | valid := func(t *testing.T, first, last uint64) { 728 | t.Helper() 729 | index, err := l.FirstIndex() 730 | if err != nil { 731 | t.Fatal(err) 732 | } 733 | if index != first { 734 | t.Fatalf("expected %v, got %v", first, index) 735 | } 736 | index, err = l.LastIndex() 737 | if err != nil { 738 | t.Fatal(err) 739 | } 740 | if index != last { 741 | t.Fatalf("expected %v, got %v", last, index) 742 | } 743 | for i := first; i <= last; i++ { 744 | data, err := l.Read(i) 745 | if err != nil { 746 | t.Fatal(err) 747 | } 748 | if string(data) != string(makeData(i)) { 749 | t.Fatalf("expcted '%s', got '%s'", makeData(i), data) 750 | } 751 | } 752 | } 753 | validReopen := func(t *testing.T, first, last uint64) { 754 | t.Helper() 755 | valid(t, first, last) 756 | if err := l.Close(); err != nil { 757 | t.Fatal(err) 758 | } 759 | l, err = Open("testlog", opts) 760 | if err != nil { 761 | t.Fatal(err) 762 | } 763 | valid(t, first, last) 764 | } 765 | for i := 1; i <= 100; i++ { 766 | err := l.Write(uint64(i), makeData(uint64(i))) 767 | if err != nil { 768 | t.Fatal(err) 769 | } 770 | } 771 | validReopen(t, 1, 100) 772 | 773 | if err := l.TruncateFront(1); err != nil { 774 | t.Fatal(err) 775 | } 776 | validReopen(t, 1, 100) 777 | 778 | if err := l.TruncateFront(2); err != nil { 779 | t.Fatal(err) 780 | } 781 | validReopen(t, 2, 100) 782 | 783 | if err := l.TruncateFront(4); err != nil { 784 | t.Fatal(err) 785 | } 786 | validReopen(t, 4, 100) 787 | 788 | if err := l.TruncateFront(5); err != nil { 789 | t.Fatal(err) 790 | } 791 | validReopen(t, 5, 100) 792 | 793 | if err := l.TruncateFront(99); err != nil { 794 | t.Fatal(err) 795 | } 796 | validReopen(t, 99, 100) 797 | 798 | if err := l.TruncateFront(100); err != nil { 799 | t.Fatal(err) 800 | } 801 | validReopen(t, 100, 100) 802 | 803 | } 804 | 805 | func TestSimpleTruncateBack(t *testing.T) { 806 | os.RemoveAll("testlog") 807 | 808 | opts := &Options{ 809 | NoSync: true, 810 | LogFormat: JSON, 811 | SegmentSize: 100, 812 | } 813 | 814 | l, err := Open("testlog", opts) 815 | if err != nil { 816 | t.Fatal(err) 817 | } 818 | defer func() { 819 | l.Close() 820 | }() 821 | 822 | makeData := func(index uint64) []byte { 823 | return []byte(fmt.Sprintf("data-%d", index)) 824 | } 825 | 826 | valid := func(t *testing.T, first, last uint64) { 827 | t.Helper() 828 | index, err := l.FirstIndex() 829 | if err != nil { 830 | t.Fatal(err) 831 | } 832 | if index != first { 833 | t.Fatalf("expected %v, got %v", first, index) 834 | } 835 | index, err = l.LastIndex() 836 | if err != nil { 837 | t.Fatal(err) 838 | } 839 | if index != last { 840 | t.Fatalf("expected %v, got %v", last, index) 841 | } 842 | for i := first; i <= last; i++ { 843 | data, err := l.Read(i) 844 | if err != nil { 845 | t.Fatal(err) 846 | } 847 | if string(data) != string(makeData(i)) { 848 | t.Fatalf("expcted '%s', got '%s'", makeData(i), data) 849 | } 850 | } 851 | } 852 | validReopen := func(t *testing.T, first, last uint64) { 853 | t.Helper() 854 | valid(t, first, last) 855 | if err := l.Close(); err != nil { 856 | t.Fatal(err) 857 | } 858 | l, err = Open("testlog", opts) 859 | if err != nil { 860 | t.Fatal(err) 861 | } 862 | valid(t, first, last) 863 | } 864 | for i := 1; i <= 100; i++ { 865 | err := l.Write(uint64(i), makeData(uint64(i))) 866 | if err != nil { 867 | t.Fatal(err) 868 | } 869 | } 870 | validReopen(t, 1, 100) 871 | 872 | ///////////////////////////////////////////////////////////// 873 | if err := l.TruncateBack(100); err != nil { 874 | t.Fatal(err) 875 | } 876 | validReopen(t, 1, 100) 877 | if err := l.Write(101, makeData(101)); err != nil { 878 | t.Fatal(err) 879 | } 880 | validReopen(t, 1, 101) 881 | 882 | ///////////////////////////////////////////////////////////// 883 | if err := l.TruncateBack(99); err != nil { 884 | t.Fatal(err) 885 | } 886 | validReopen(t, 1, 99) 887 | if err := l.Write(100, makeData(100)); err != nil { 888 | t.Fatal(err) 889 | } 890 | validReopen(t, 1, 100) 891 | 892 | if err := l.TruncateBack(94); err != nil { 893 | t.Fatal(err) 894 | } 895 | validReopen(t, 1, 94) 896 | 897 | if err := l.TruncateBack(93); err != nil { 898 | t.Fatal(err) 899 | } 900 | validReopen(t, 1, 93) 901 | 902 | if err := l.TruncateBack(92); err != nil { 903 | t.Fatal(err) 904 | } 905 | validReopen(t, 1, 92) 906 | 907 | } 908 | 909 | func TestConcurrency(t *testing.T) { 910 | os.RemoveAll("testlog") 911 | 912 | l, err := Open("testlog", &Options{ 913 | NoSync: true, 914 | NoCopy: true, 915 | }) 916 | if err != nil { 917 | t.Fatal(err) 918 | } 919 | defer l.Close() 920 | 921 | // Write 1000 entries 922 | for i := 1; i <= 1000; i++ { 923 | err := l.Write(uint64(i), []byte(dataStr(uint64(i)))) 924 | if err != nil { 925 | t.Fatal(err) 926 | } 927 | } 928 | 929 | // Perform 100,000 reads (over 100 threads) 930 | finished := int32(0) 931 | maxIndex := int32(1000) 932 | numReads := int32(0) 933 | for i := 0; i < 100; i++ { 934 | go func() { 935 | defer atomic.AddInt32(&finished, 1) 936 | 937 | for i := 0; i < 1_000; i++ { 938 | index := rand.Int31n(atomic.LoadInt32(&maxIndex)) + 1 939 | if _, err := l.Read(uint64(index)); err != nil { 940 | panic(err) 941 | } 942 | atomic.AddInt32(&numReads, 1) 943 | } 944 | }() 945 | } 946 | 947 | // continue writing 948 | for index := maxIndex + 1; atomic.LoadInt32(&finished) < 100; index++ { 949 | err := l.Write(uint64(index), []byte(dataStr(uint64(index)))) 950 | if err != nil { 951 | t.Fatal(err) 952 | } 953 | atomic.StoreInt32(&maxIndex, index) 954 | } 955 | 956 | // confirm total reads 957 | if exp := int32(100_000); numReads != exp { 958 | t.Fatalf("expected %d reads, but god %d", exp, numReads) 959 | } 960 | } 961 | 962 | func TestRWConcurrency(t *testing.T) { 963 | os.RemoveAll("testlog") 964 | 965 | l, err := Open("testlog", &Options{ 966 | NoSync: true, 967 | NoCopy: true, 968 | AllowEmpty: true, 969 | }) 970 | if err != nil { 971 | t.Fatal(err) 972 | } 973 | defer l.Close() 974 | 975 | wg := sync.WaitGroup{} 976 | wg.Add(2) 977 | notify := make(chan struct{}, 1) 978 | count := 100 979 | 980 | go func() { 981 | defer wg.Done() 982 | defer close(notify) 983 | idx, _ := l.LastIndex() 984 | for i := 0; i < count; i++ { 985 | idx++ 986 | if err := l.Write(idx, []byte(dataStr(uint64(i)))); err != nil { 987 | panic(err) 988 | } 989 | select { 990 | case notify <- struct{}{}: 991 | default: 992 | } 993 | } 994 | if idx != uint64(count) { 995 | panic(fmt.Sprintf("expected last index %d, got %d", count, idx)) 996 | } 997 | }() 998 | 999 | go func() { 1000 | defer wg.Done() 1001 | idx, _ := l.FirstIndex() 1002 | for range notify { 1003 | for { 1004 | _, err := l.Read(idx) 1005 | if errors.Is(err, ErrNotFound) { 1006 | break 1007 | } 1008 | if err := l.TruncateFront(idx + 1); err != nil { 1009 | panic(err) 1010 | } 1011 | idx++ 1012 | } 1013 | } 1014 | if idx != uint64(count+1) { 1015 | panic(fmt.Sprintf("expected first index %d, got %d", count+1, idx)) 1016 | } 1017 | }() 1018 | 1019 | wg.Wait() 1020 | } 1021 | 1022 | func must(v interface{}, err error) interface{} { 1023 | if err != nil { 1024 | panic(err) 1025 | } 1026 | return v 1027 | } 1028 | 1029 | func TestNoAllowEmpty(t *testing.T) { 1030 | os.RemoveAll("testlog") 1031 | l, err := Open("testlog", &Options{ 1032 | NoSync: true, 1033 | NoCopy: true, 1034 | AllowEmpty: false, 1035 | SegmentSize: 4096, 1036 | LogFormat: JSON, 1037 | }) 1038 | if err != nil { 1039 | t.Fatal(err) 1040 | } 1041 | defer func() { 1042 | if l != nil { 1043 | l.Close() 1044 | } 1045 | }() 1046 | firstIndex, err := l.FirstIndex() 1047 | if err != nil { 1048 | t.Fatal(err) 1049 | } 1050 | lastIndex, err := l.LastIndex() 1051 | if err != nil { 1052 | t.Fatal(err) 1053 | } 1054 | if firstIndex != 0 || lastIndex != 0 { 1055 | t.Fatalf("expected %d %d, got %d %d\n", 0, 0, firstIndex, lastIndex) 1056 | } 1057 | N := 1000 1058 | for i := 0; i < N; i++ { 1059 | err := l.Write(uint64(i)+1, []byte(fmt.Sprintf("%d", i))) 1060 | if err != nil { 1061 | t.Fatal(err) 1062 | } 1063 | } 1064 | err = l.TruncateFront(uint64(N + 1)) 1065 | if err != ErrOutOfRange { 1066 | t.Fatalf("expected %v, got %v\n", ErrOutOfRange, err) 1067 | } 1068 | err = l.TruncateBack(0) 1069 | if err != ErrOutOfRange { 1070 | t.Fatalf("expected %v, got %v\n", ErrOutOfRange, err) 1071 | } 1072 | if err := l.Close(); err != nil { 1073 | t.Fatal(err) 1074 | } 1075 | // Reopen, allowing empty 1076 | l, err = Open("testlog", &Options{ 1077 | NoSync: true, 1078 | NoCopy: true, 1079 | AllowEmpty: true, 1080 | SegmentSize: 4096, 1081 | LogFormat: JSON, 1082 | }) 1083 | if err != nil { 1084 | t.Fatal(err) 1085 | } 1086 | err = l.TruncateFront(uint64(N + 1)) 1087 | if err != nil { 1088 | t.Fatal(err) 1089 | } 1090 | if err := l.Close(); err != nil { 1091 | t.Fatal(err) 1092 | } 1093 | // Reopen, not allowing empty 1094 | l, err = Open("testlog", &Options{ 1095 | NoSync: true, 1096 | NoCopy: true, 1097 | AllowEmpty: false, 1098 | SegmentSize: 4096, 1099 | LogFormat: JSON, 1100 | }) 1101 | if err != ErrEmptyLog { 1102 | t.Fatalf("expected %v, got %v", ErrEmptyLog, err) 1103 | } 1104 | } 1105 | 1106 | func TestEmptyOpenFromNothing(t *testing.T) { 1107 | os.RemoveAll("testlog") 1108 | l, err := Open("testlog", &Options{ 1109 | NoSync: true, 1110 | NoCopy: true, 1111 | AllowEmpty: true, 1112 | }) 1113 | if err != nil { 1114 | t.Fatal(err) 1115 | } 1116 | defer l.Close() 1117 | isEmpty, err := l.IsEmpty() 1118 | if err != nil { 1119 | t.Fatal(err) 1120 | } 1121 | if isEmpty == false { 1122 | t.Fatal("expected true") 1123 | } 1124 | firstIndex, err := l.FirstIndex() 1125 | if err != nil { 1126 | t.Fatal(err) 1127 | } 1128 | lastIndex, err := l.LastIndex() 1129 | if err != nil { 1130 | t.Fatal(err) 1131 | } 1132 | if firstIndex != 1 || lastIndex != 0 { 1133 | t.Fatalf("expected %d %d, got %d %d\n", 1, 0, firstIndex, lastIndex) 1134 | } 1135 | if err := l.Close(); err != nil { 1136 | t.Fatal(err) 1137 | } 1138 | } 1139 | 1140 | func TestEmptyOpenFromExisting1(t *testing.T) { 1141 | os.RemoveAll("testlog") 1142 | os.Mkdir("testlog", 0777) 1143 | os.WriteFile("testlog/00000000000000000001", nil, 0666) 1144 | l, err := Open("testlog", &Options{ 1145 | NoSync: true, 1146 | NoCopy: true, 1147 | AllowEmpty: true, 1148 | }) 1149 | if err != nil { 1150 | t.Fatal(err) 1151 | } 1152 | defer l.Close() 1153 | isEmpty, err := l.IsEmpty() 1154 | if err != nil { 1155 | t.Fatal(err) 1156 | } 1157 | if isEmpty == false { 1158 | t.Fatal("expected true") 1159 | } 1160 | firstIndex, err := l.FirstIndex() 1161 | if err != nil { 1162 | t.Fatal(err) 1163 | } 1164 | lastIndex, err := l.LastIndex() 1165 | if err != nil { 1166 | t.Fatal(err) 1167 | } 1168 | if firstIndex != 1 || lastIndex != 0 { 1169 | t.Fatalf("expected %d %d, got %d %d\n", 1, 0, firstIndex, lastIndex) 1170 | } 1171 | } 1172 | 1173 | func TestEmptyOpenFromExisting1001(t *testing.T) { 1174 | os.RemoveAll("testlog") 1175 | os.Mkdir("testlog", 0777) 1176 | os.WriteFile("testlog/00000000000000001001", nil, 0666) 1177 | l, err := Open("testlog", &Options{ 1178 | NoSync: true, 1179 | NoCopy: true, 1180 | AllowEmpty: true, 1181 | }) 1182 | if err != nil { 1183 | t.Fatal(err) 1184 | } 1185 | defer l.Close() 1186 | isEmpty, err := l.IsEmpty() 1187 | if err != nil { 1188 | t.Fatal(err) 1189 | } 1190 | if isEmpty == false { 1191 | t.Fatal("expected true") 1192 | } 1193 | firstIndex, err := l.FirstIndex() 1194 | if err != nil { 1195 | t.Fatal(err) 1196 | } 1197 | lastIndex, err := l.LastIndex() 1198 | if err != nil { 1199 | t.Fatal(err) 1200 | } 1201 | if firstIndex != 1001 || lastIndex != 1000 { 1202 | t.Fatalf("expected %d %d, got %d %d\n", 0, 0, firstIndex, lastIndex) 1203 | } 1204 | } 1205 | 1206 | func TestEmptyTruncateFrontTwice(t *testing.T) { 1207 | os.RemoveAll("testlog") 1208 | l, err := Open("testlog", &Options{ 1209 | NoSync: true, 1210 | NoCopy: true, 1211 | AllowEmpty: true, 1212 | SegmentSize: 4096, 1213 | LogFormat: JSON, 1214 | }) 1215 | if err != nil { 1216 | t.Fatal(err) 1217 | } 1218 | defer l.Close() 1219 | N := 1000 1220 | for i := 0; i < N; i++ { 1221 | err := l.Write(uint64(i)+1, []byte(fmt.Sprintf("%d", i))) 1222 | if err != nil { 1223 | t.Fatal(err) 1224 | } 1225 | } 1226 | err = l.TruncateFront(uint64(N) + 1) 1227 | if err != nil { 1228 | t.Fatal(err) 1229 | } 1230 | empty, err := l.IsEmpty() 1231 | if err != nil { 1232 | t.Fatal(err) 1233 | } 1234 | if !empty { 1235 | t.Fatalf("expected %v, got %v", true, empty) 1236 | } 1237 | index, err := l.FirstIndex() 1238 | if err != nil { 1239 | t.Fatal(err) 1240 | } 1241 | if index != uint64(N+1) { 1242 | t.Fatalf("expected %v, got %v", uint64(N+1), index) 1243 | } 1244 | err = l.TruncateFront(uint64(N) + 1) 1245 | if err != nil { 1246 | t.Fatal(err) 1247 | } 1248 | empty, err = l.IsEmpty() 1249 | if err != nil { 1250 | t.Fatal(err) 1251 | } 1252 | if !empty { 1253 | t.Fatalf("expected %v, got %v", true, empty) 1254 | } 1255 | index, err = l.FirstIndex() 1256 | if err != nil { 1257 | t.Fatal(err) 1258 | } 1259 | if index != uint64(N+1) { 1260 | t.Fatalf("expected %v, got %v", uint64(N+1), index) 1261 | } 1262 | } 1263 | 1264 | func TestEmptyTruncateBackTwice(t *testing.T) { 1265 | os.RemoveAll("testlog") 1266 | l, err := Open("testlog", &Options{ 1267 | NoSync: true, 1268 | NoCopy: true, 1269 | AllowEmpty: true, 1270 | SegmentSize: 4096, 1271 | LogFormat: JSON, 1272 | }) 1273 | if err != nil { 1274 | t.Fatal(err) 1275 | } 1276 | defer l.Close() 1277 | N := 1000 1278 | for i := 0; i < N; i++ { 1279 | err := l.Write(uint64(i)+1, []byte(fmt.Sprintf("%d", i))) 1280 | if err != nil { 1281 | t.Fatal(err) 1282 | } 1283 | } 1284 | err = l.TruncateBack(0) 1285 | if err != nil { 1286 | t.Fatal(err) 1287 | } 1288 | fidx, err := l.FirstIndex() 1289 | if err != nil { 1290 | t.Fatal(err) 1291 | } 1292 | lidx, err := l.LastIndex() 1293 | if err != nil { 1294 | t.Fatal(err) 1295 | } 1296 | if fidx != 1 || lidx != 0 { 1297 | t.Fatalf("expected %v/%v, got %v/%v", 1, 0, fidx, lidx) 1298 | } 1299 | empty, err := l.IsEmpty() 1300 | if err != nil { 1301 | t.Fatal(err) 1302 | } 1303 | if !empty { 1304 | t.Fatalf("expected %v, got %v", true, empty) 1305 | } 1306 | err = l.TruncateBack(0) 1307 | if err != nil { 1308 | t.Fatal(err) 1309 | } 1310 | fidx, err = l.FirstIndex() 1311 | if err != nil { 1312 | t.Fatal(err) 1313 | } 1314 | lidx, err = l.LastIndex() 1315 | if err != nil { 1316 | t.Fatal(err) 1317 | } 1318 | if fidx != 1 || lidx != 0 { 1319 | t.Fatalf("expected %v/%v, got %v/%v", 1, 0, fidx, lidx) 1320 | } 1321 | empty, err = l.IsEmpty() 1322 | if err != nil { 1323 | t.Fatal(err) 1324 | } 1325 | if !empty { 1326 | t.Fatalf("expected %v, got %v", true, empty) 1327 | } 1328 | } 1329 | 1330 | func TestIssue33(t *testing.T) { 1331 | // Create a fresh log without allowempty, close, and reopen. 1332 | // This should not fail with an ErrEmptyLog 1333 | os.RemoveAll("testlog") 1334 | l, err := Open("testlog", nil) 1335 | if err != nil { 1336 | t.Fatal(err) 1337 | } 1338 | l.Close() 1339 | l, err = Open("testlog", nil) 1340 | if err != nil { 1341 | t.Fatal(err) 1342 | } 1343 | l.Close() 1344 | } 1345 | --------------------------------------------------------------------------------