├── .envrc ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── api.go ├── bench_test.go ├── blocking.go ├── compact ├── deletes.go ├── deletes_test.go ├── updates.go └── updates_test.go ├── delete.go ├── delete_test.go ├── flake.lock ├── flake.nix ├── go.mod ├── go.sum ├── index ├── format.go ├── format_test.go ├── index.go ├── keys.go ├── keys_test.go ├── offset.go ├── offset_test.go ├── times.go └── times_test.go ├── log.go ├── log_test.go ├── message ├── format.go ├── format_test.go └── message.go ├── notify.go ├── notify_test.go ├── reader.go ├── segment ├── index.go ├── index_test.go ├── segment.go ├── segment_test.go ├── segments.go ├── segments_test.go └── utils.go ├── trim ├── age.go ├── age_test.go ├── count.go ├── count_test.go ├── offset.go ├── offset_test.go ├── size.go └── size_test.go ├── typed.go ├── typed_blocking.go ├── typed_codec.go ├── typed_test.go └── writer.go /.envrc: -------------------------------------------------------------------------------- 1 | use flake 2 | dotenv_if_exists 3 | layout go 4 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [ "main" ] 7 | 8 | jobs: 9 | build: 10 | name: Build and test 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: cachix/install-nix-action@v27 15 | with: 16 | nix_path: nixpkgs=channel:nixos-unstable 17 | github_access_token: ${{ secrets.GITHUB_TOKEN }} 18 | - uses: DeterminateSystems/magic-nix-cache-action@main 19 | - name: Build 20 | run: nix develop --command make build 21 | - name: Test 22 | run: nix develop --command make test-verbose 23 | - name: Lint 24 | run: nix develop --command make lint 25 | - name: Tidy 26 | run: nix develop --command go mod tidy 27 | - name: Check if tidy changed anything 28 | run: git diff --exit-code 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | .direnv 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 klev-dev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | default: test 2 | 3 | .PHONY: test test-verbose lint 4 | test: 5 | go test -cover ./... 6 | 7 | test-verbose: 8 | go test -cover -v ./... 9 | 10 | lint: 11 | golangci-lint run 12 | 13 | .PHONY: 14 | build: 15 | go build -v ./... 16 | 17 | .PHONY: bench bench-publish bench-consume bench-get bench-multi 18 | bench: 19 | go test -bench=. -benchmem -run XXX 20 | 21 | bench-publish: 22 | go test -bench=BenchmarkSingle/Publish -benchmem -run XXX 23 | 24 | bench-consume: 25 | go test -bench=BenchmarkSingle/Consume -benchmem -run XXX 26 | 27 | bench-get: 28 | go test -bench=BenchmarkSingle/Get -benchmem -run XXX 29 | 30 | bench-multi: 31 | go test -bench=BenchmarkMulti -benchmem -run XXX 32 | 33 | .PHONY: update-libs 34 | update-libs: 35 | go get -u ./... 36 | go mod tidy 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # klevdb 2 | 3 | [![CI](https://github.com/klev-dev/klevdb/actions/workflows/ci.yml/badge.svg)](https://github.com/klev-dev/klevdb/actions) 4 | [![Go Reference](https://pkg.go.dev/badge/github.com/klev-dev/klevdb.svg)](https://pkg.go.dev/github.com/klev-dev/klevdb) 5 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 6 | 7 | klevdb is a fast message store, written in Go. Think single partition on kafka, but stored locally. 8 | 9 | In addition to basic consuming by offset, you can also configure klevdb to index times and keys. Times index allow you to quickly find a message by its time (or the first message after a certain time). Keys index allow you to quickly find the last message with a given key. 10 | 11 | ## Usage 12 | 13 | To add klevdb to your package use: 14 | 15 | ``` 16 | go get github.com/klev-dev/klevdb 17 | ``` 18 | 19 | To use klevdb: 20 | 21 | ``` 22 | package main 23 | 24 | import ( 25 | "github.com/klev-dev/klevdb" 26 | ) 27 | 28 | func main() { 29 | db, _ := klevdb.Open("/tmp/kdb", klevdb.Options{ 30 | CreateDirs: true, 31 | KeyIndex: true, 32 | }) 33 | defer db.Close() 34 | 35 | publishNext, _ := db.Publish([]klevdb.Message{ 36 | { 37 | Key: []byte("key1"), 38 | Value: []byte("val1"), 39 | }, 40 | { 41 | Key: []byte("key1"), 42 | Value: []byte("val2"), 43 | }, 44 | }) 45 | fmt.Println("published, next offset:", publishNext) 46 | 47 | consumeNext, msgs, _ := db.Consume(klevdb.OffsetOldest, 1) 48 | fmt.Println("consumed:", msgs, "value:", string(msgs[0].Value)) 49 | fmt.Println("next consume offset:", consumeNext) 50 | 51 | msg, _ := db.GetByKey([]byte("key1")) 52 | fmt.Println("got:", msg, "value:", string(msg.Value)) 53 | } 54 | ``` 55 | 56 | Running the above program, outputs the following: 57 | ``` 58 | published, next offset: 2 59 | consumed: [{0 2009-11-10 23:00:00 +0000 UTC [107 101 121 49] [118 97 108 49]}] value: val1 60 | next consume offset: 1 61 | got: {1 2009-11-10 23:00:00 +0000 UTC [107 101 121 49] [118 97 108 50]} value: val2 62 | ``` 63 | 64 | Further documentation is available at [GoDoc](https://pkg.go.dev/github.com/klev-dev/klevdb) 65 | 66 | ## Performance 67 | 68 | Benchmarks on framework gen1 i5: 69 | ``` 70 | goos: linux 71 | goarch: amd64 72 | pkg: github.com/klev-dev/klevdb 73 | cpu: 11th Gen Intel(R) Core(TM) i5-1135G7 @ 2.40GHz 74 | ``` 75 | 76 | ### Publish 77 | ``` 78 | ≻ make bench-publish 79 | go test -bench=BenchmarkSingle/Publish -benchmem 80 | 81 | BenchmarkSingle/Publish/1/No-8 347466 3337 ns/op 54.54 MB/s 156 B/op 1 allocs/op 82 | BenchmarkSingle/Publish/1/Times-8 391308 3585 ns/op 53.00 MB/s 162 B/op 1 allocs/op 83 | BenchmarkSingle/Publish/1/Keys-8 314779 3960 ns/op 47.98 MB/s 305 B/op 7 allocs/op 84 | BenchmarkSingle/Publish/1/All-8 319302 3907 ns/op 50.68 MB/s 310 B/op 7 allocs/op 85 | BenchmarkSingle/Publish/8/No-8 397518 3266 ns/op 445.81 MB/s 156 B/op 0 allocs/op 86 | BenchmarkSingle/Publish/8/Times-8 451623 3402 ns/op 446.73 MB/s 161 B/op 0 allocs/op 87 | BenchmarkSingle/Publish/8/Keys-8 309150 3821 ns/op 397.78 MB/s 298 B/op 5 allocs/op 88 | BenchmarkSingle/Publish/8/All-8 382129 3797 ns/op 417.17 MB/s 303 B/op 5 allocs/op 89 | 90 | PASS 91 | ok github.com/klev-dev/klevdb 12.433s 92 | ``` 93 | With default rollover of 1MB, for messages with keys 10B and values 128B: 94 | * ~300,000 writes/sec, no indexes 95 | * ~250,000 writes/sec, with all indexes enabled 96 | * scales lineary with the batch size 97 | 98 | ### Consume 99 | ``` 100 | ≻ make bench-consume 101 | 102 | BenchmarkSingle/Consume/W/1-8 4372142 279.5 ns/op 651.05 MB/s 224 B/op 2 allocs/op 103 | BenchmarkSingle/Consume/RW/1-8 4377028 287.1 ns/op 633.94 MB/s 274 B/op 2 allocs/op 104 | BenchmarkSingle/Consume/R/1-8 4356441 299.0 ns/op 608.71 MB/s 274 B/op 2 allocs/op 105 | BenchmarkSingle/Consume/W/8-8 6508213 178.4 ns/op 8163.31 MB/s 294 B/op 1 allocs/op 106 | BenchmarkSingle/Consume/RW/8-8 6069168 194.8 ns/op 7475.85 MB/s 344 B/op 1 allocs/op 107 | BenchmarkSingle/Consume/R/8-8 6271984 196.4 ns/op 7413.22 MB/s 344 B/op 1 allocs/op 108 | 109 | PASS 110 | ok github.com/klev-dev/klevdb 147.152s 111 | ``` 112 | With default rollover of 1MB, for messages with keys 10B and values 128B: 113 | * ~3,500,000 reads/sec, single message consume 114 | * ~5,500,000 reads/sec, 8 message batches 115 | 116 | ### Get 117 | ``` 118 | ≻ make bench-get 119 | go test -bench=BenchmarkSingle/Get -benchmem 120 | 121 | BenchmarkSingle/Get/ByOffset-8 5355378 225.2 ns/op 808.24 MB/s 144 B/op 1 allocs/op 122 | BenchmarkSingle/Get/ByKey-8 1000000 3583 ns/op 53.04 MB/s 152 B/op 2 allocs/op 123 | BenchmarkSingle/Get/ByKey/R-8 1000000 3794 ns/op 50.08 MB/s 345 B/op 7 allocs/op 124 | BenchmarkSingle/Get/ByTime-8 1000000 2197 ns/op 86.48 MB/s 144 B/op 1 allocs/op 125 | BenchmarkSingle/Get/ByTime/R-8 1000000 2178 ns/op 87.25 MB/s 202 B/op 1 allocs/op 126 | 127 | PASS 128 | ok github.com/klev-dev/klevdb 52.528s 129 | ``` 130 | With default rollover of 1MB, for messages with keys 10B and values 128B: 131 | * ~4,400,000 gets/sec, across all offsets 132 | * ~270,000 key reads/sec, across all keys 133 | * ~450,000 time reads/sec, across all times 134 | 135 | ### Multi 136 | ``` 137 | ≻ make bench-multi 138 | go test -bench=BenchmarkMulti -benchmem 139 | 140 | BenchmarkMulti/Base-8 282462 4433 ns/op 673 B/op 7 allocs/op 141 | BenchmarkMulti/Publish-8 30628 40717 ns/op 19.45 MB/s 2974 B/op 56 allocs/op 142 | BenchmarkMulti/Consume-8 1289114 909.9 ns/op 835.24 MB/s 2842 B/op 17 allocs/op 143 | BenchmarkMulti/GetKey-8 459753 5729 ns/op 34.56 MB/s 1520 B/op 20 allocs/op 144 | 145 | PASS 146 | ok github.com/klev-dev/klevdb 22.973s 147 | ``` 148 | -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "errors" 5 | "time" 6 | 7 | "github.com/klev-dev/klevdb/index" 8 | "github.com/klev-dev/klevdb/message" 9 | "github.com/klev-dev/klevdb/segment" 10 | ) 11 | 12 | const ( 13 | // OffsetOldest represents the smallest offset still available 14 | // Use it to consume all messages, starting at the first available 15 | OffsetOldest = message.OffsetOldest 16 | // OffsetNewest represents the offset that will be used for the next produce 17 | // Use it to consume only new messages 18 | OffsetNewest = message.OffsetNewest 19 | // OffsetInvalid is the offset returned when error is detected 20 | OffsetInvalid = message.OffsetInvalid 21 | ) 22 | 23 | type Message = message.Message 24 | 25 | // InvalidMessage returned when an error have occurred 26 | var InvalidMessage = message.Invalid 27 | 28 | // ErrInvalidOffset error is returned when the offset attribute is invalid or out of bounds 29 | var ErrInvalidOffset = message.ErrInvalidOffset 30 | 31 | // ErrNotFound error is returned when the offset, key or timestamp is not found 32 | var ErrNotFound = message.ErrNotFound 33 | 34 | // ErrNoIndex error is returned when we try to use key or timestamp, but the log doesn't include index on them 35 | var ErrNoIndex = errors.New("no index") 36 | 37 | // ErrReadonly error is returned when attempting to modify (e.g. publish or delete) from a log that is open as a readonly 38 | var ErrReadonly = errors.New("log opened in readonly mode") 39 | 40 | type Stats = segment.Stats 41 | 42 | type Options struct { 43 | // When set will try to create all directories 44 | CreateDirs bool 45 | // Open the store in readonly mode 46 | Readonly bool 47 | // Index message keys, enabling GetByKey and OffsetByKey 48 | KeyIndex bool 49 | // Index message times, enabling GetByTime and OffsetByTime 50 | TimeIndex bool 51 | // Force filesystem sync after each Publish 52 | AutoSync bool 53 | // At what segment size it will rollover to a new segment. Defaults to 1mb. 54 | Rollover int64 55 | // Check the head segment for integrity, before opening it for reading/writing. 56 | Check bool 57 | } 58 | 59 | type Log interface { 60 | // Publish appends messages to the log. 61 | // It returns the offset of the next message to be appended. 62 | // The offset of the message is ignored, set to the actual offset. 63 | // If the time of the message is 0, it set to the current UTC time. 64 | Publish(messages []Message) (nextOffset int64, err error) 65 | 66 | // NextOffset returns the offset of the next message to be published. 67 | NextOffset() (nextOffset int64, err error) 68 | 69 | // Consume retrieves messages from the log, starting at the offset. 70 | // It returns offset, which can be used to retrieve for the next consume. 71 | // If offset == OffsetOldest, the first message will be the oldest 72 | // message still available on the log. If the log is empty, 73 | // it will return no error, nextOffset will be 0 74 | // If offset == OffsetNewest, no actual messages will be returned, 75 | // but nextOffset will be set to the offset that will be used 76 | // for the next Publish call 77 | // If offset is before the first available on the log, or is after 78 | // NextOffset, it returns ErrInvalidOffset 79 | // If the exact offset is already deleted, it will start consuming 80 | // from the next available offset. 81 | // Consume is allowed to return no messages, but with increasing nextOffset 82 | // in case messages between offset and nextOffset have been deleted. 83 | // NextOffset is always bigger then offset, unless we are caught up 84 | // to the head of the log in which case they are equal. 85 | Consume(offset int64, maxCount int64) (nextOffset int64, messages []Message, err error) 86 | 87 | // ConsumeByKey is similar to Consume, but only returns messages matching the key 88 | ConsumeByKey(key []byte, offset int64, maxCount int64) (nextOffset int64, messages []Message, err error) 89 | 90 | // Get retrieves a single message, by its offset 91 | // If offset == OffsetOldest, it returns the first message on the log 92 | // If offset == OffsetNewest, it returns the last message on the log 93 | // If offset is before the first available on the log, or is after 94 | // NextOffset, it returns ErrInvalidOffset 95 | // If log is empty, it returns ErrInvalidOffset 96 | // If the exact offset have been deleted, it returns ErrNotFound 97 | Get(offset int64) (message Message, err error) 98 | 99 | // GetByKey retrieves the last message in the log for this key 100 | // If no such message is found, it returns ErrNotFound 101 | GetByKey(key []byte) (message Message, err error) 102 | // OffsetByKey retrieves the last message offset in the log for this key 103 | // If no such message is found, it returns ErrNotFound 104 | OffsetByKey(key []byte) (offset int64, err error) 105 | 106 | // GetByTime retrieves the first message after start time 107 | // If start time is after all messages in the log, it returns ErrNotFound 108 | GetByTime(start time.Time) (message Message, err error) 109 | // OffsetByTime retrieves the first message offset and its time after start time 110 | // If start time is after all messages in the log, it returns ErrNotFound 111 | OffsetByTime(start time.Time) (offset int64, messageTime time.Time, err error) 112 | 113 | // Delete tries to delete a set of messages by their offset 114 | // from the log and returns the amount of storage deleted 115 | // It does not guarantee that it will delete all messages, 116 | // it returns the set of actually deleted offsets. 117 | Delete(offsets map[int64]struct{}) (deletedOffsets map[int64]struct{}, deletedSize int64, err error) 118 | 119 | // Size returns the amount of storage associated with a message 120 | Size(m Message) int64 121 | 122 | // Stat returns log stats like disk space, number of messages 123 | Stat() (Stats, error) 124 | 125 | // Backup takes a backup snapshot of this log to another location 126 | Backup(dir string) error 127 | 128 | // Sync forces persisting data to the disk. It returns the nextOffset 129 | // at the time of the Sync, so clients can determine what portion 130 | // of the log is now durable. 131 | Sync() (nextOffset int64, err error) 132 | 133 | // GC releases any unused resources associated with this log 134 | GC(unusedFor time.Duration) error 135 | 136 | // Close closes the log 137 | Close() error 138 | } 139 | 140 | // Stat stats a store directory, without opening the store 141 | func Stat(dir string, opts Options) (Stats, error) { 142 | return segment.StatDir(dir, index.Params{ 143 | Times: opts.TimeIndex, 144 | Keys: opts.KeyIndex, 145 | }) 146 | } 147 | 148 | // Backup backups a store directory to another location, without opening the store 149 | func Backup(src, dst string) error { 150 | return segment.BackupDir(src, dst) 151 | } 152 | 153 | // Check runs an integrity check, without opening the store 154 | func Check(dir string, opts Options) error { 155 | return segment.CheckDir(dir, index.Params{ 156 | Times: opts.TimeIndex, 157 | Keys: opts.KeyIndex, 158 | }) 159 | } 160 | 161 | // Recover rewrites the storage to include all messages prior the first that fails an integrity check 162 | func Recover(dir string, opts Options) error { 163 | return segment.RecoverDir(dir, index.Params{ 164 | Times: opts.TimeIndex, 165 | Keys: opts.KeyIndex, 166 | }) 167 | } 168 | -------------------------------------------------------------------------------- /bench_test.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | "golang.org/x/sync/errgroup" 11 | 12 | "github.com/klev-dev/klevdb/message" 13 | ) 14 | 15 | func BenchmarkSingle(b *testing.B) { 16 | b.Run("Publish", benchmarkPublish) 17 | b.Run("Consume", benchmarkConsume) 18 | b.Run("Get", benchmarkGet) 19 | } 20 | 21 | func BenchmarkMulti(b *testing.B) { 22 | b.Run("Base", benchmarkBaseMulti) 23 | b.Run("Publish", benchmarkPublishMulti) 24 | b.Run("Consume", benchmarkConsumeMulti) 25 | b.Run("GetKey", benchmarkGetKeyMulti) 26 | } 27 | 28 | func MkdirBench(b *testing.B) string { 29 | name := strings.Replace(b.Name(), "/", "_", -1) 30 | 31 | currentDir, err := os.Getwd() 32 | require.NoError(b, err) 33 | 34 | dir, err := os.MkdirTemp(currentDir, name) 35 | require.NoError(b, err) 36 | return dir 37 | } 38 | 39 | func benchmarkPublish(b *testing.B) { 40 | var cases = []struct { 41 | name string 42 | opts Options 43 | }{ 44 | {"No", Options{}}, 45 | {"Times", Options{TimeIndex: true}}, 46 | {"Keys", Options{KeyIndex: true}}, 47 | {"All", Options{TimeIndex: true, KeyIndex: true}}, 48 | } 49 | for _, bn := range []int{1, 8} { 50 | for _, c := range cases { 51 | b.Run(fmt.Sprintf("%d/%s", bn, c.name), func(b *testing.B) { 52 | dir := MkdirBench(b) 53 | defer os.RemoveAll(dir) 54 | 55 | s, err := Open(dir, c.opts) 56 | require.NoError(b, err) 57 | defer s.Close() 58 | 59 | msgs := message.Gen(b.N) 60 | 61 | b.SetBytes(s.Size(msgs[0]) * int64(bn)) 62 | b.ResetTimer() 63 | 64 | for i := 0; i < b.N; i += bn { 65 | top := i + bn 66 | if top > b.N { 67 | top = b.N 68 | } 69 | 70 | if _, err := s.Publish(msgs[i:top]); err != nil { 71 | b.Fatal(err) 72 | } 73 | } 74 | 75 | b.StopTimer() 76 | }) 77 | } 78 | } 79 | } 80 | 81 | func benchmarkPublishMulti(b *testing.B) { 82 | dir := MkdirBench(b) 83 | defer os.RemoveAll(dir) 84 | 85 | s, err := Open(dir, Options{TimeIndex: true, KeyIndex: true}) 86 | require.NoError(b, err) 87 | defer s.Close() 88 | 89 | msgs := message.Gen(b.N) 90 | 91 | b.SetBytes(s.Size(msgs[0]) * 4) 92 | b.ResetTimer() 93 | 94 | g := new(errgroup.Group) 95 | for k := 0; k < 10; k++ { 96 | g.Go(func() error { 97 | for i := 0; i < b.N; i += 4 { 98 | top := i + 4 99 | if top > b.N { 100 | top = b.N 101 | } 102 | if _, err := s.Publish(msgs[i:top]); err != nil { 103 | return err 104 | } 105 | } 106 | return nil 107 | }) 108 | } 109 | require.NoError(b, g.Wait()) 110 | 111 | b.StopTimer() 112 | } 113 | 114 | func fillLog(b *testing.B, l Log) []Message { 115 | msgs := message.Gen(b.N) 116 | for i := 0; i < b.N; i += 4 { 117 | top := i + 4 118 | if top > b.N { 119 | top = b.N 120 | } 121 | if _, err := l.Publish(msgs[i:top]); err != nil { 122 | b.Fatal(err) 123 | } 124 | } 125 | return msgs 126 | } 127 | 128 | func benchmarkConsume(b *testing.B) { 129 | var cases = []struct { 130 | name string 131 | opts Options 132 | }{ 133 | {"No", Options{}}, 134 | {"Times", Options{TimeIndex: true}}, 135 | {"Keys", Options{KeyIndex: true}}, 136 | {"All", Options{TimeIndex: true, KeyIndex: true}}, 137 | } 138 | for _, bn := range []int{1, 8} { 139 | for _, c := range cases { 140 | b.Run(fmt.Sprintf("W/%s/%d", c.name, bn), func(b *testing.B) { 141 | dir := MkdirBench(b) 142 | defer os.RemoveAll(dir) 143 | 144 | l, err := Open(dir, c.opts) 145 | require.NoError(b, err) 146 | defer l.Close() 147 | 148 | msgs := fillLog(b, l) 149 | 150 | b.SetBytes(l.Size(msgs[0]) * int64(bn)) 151 | b.ResetTimer() 152 | 153 | for i := 0; i < b.N; i += bn { 154 | if _, _, err := l.Consume(int64(i), int64(bn)); err != nil { 155 | b.Fatal(err) 156 | } 157 | } 158 | 159 | b.StopTimer() 160 | }) 161 | 162 | b.Run(fmt.Sprintf("RW/%s/%d", c.name, bn), func(b *testing.B) { 163 | dir := MkdirBench(b) 164 | defer os.RemoveAll(dir) 165 | 166 | l, err := Open(dir, c.opts) 167 | require.NoError(b, err) 168 | defer l.Close() 169 | 170 | msgs := fillLog(b, l) 171 | require.NoError(b, l.Close()) 172 | 173 | b.SetBytes(l.Size(msgs[0]) * int64(bn)) 174 | b.ResetTimer() 175 | 176 | l, err = Open(dir, c.opts) 177 | require.NoError(b, err) 178 | defer l.Close() 179 | 180 | for i := 0; i < b.N; i += bn { 181 | if _, _, err := l.Consume(int64(i), int64(bn)); err != nil { 182 | b.Fatal(err) 183 | } 184 | } 185 | 186 | b.StopTimer() 187 | }) 188 | 189 | b.Run(fmt.Sprintf("R/%s/%d", c.name, bn), func(b *testing.B) { 190 | dir := MkdirBench(b) 191 | defer os.RemoveAll(dir) 192 | 193 | l, err := Open(dir, c.opts) 194 | require.NoError(b, err) 195 | defer l.Close() 196 | 197 | msgs := fillLog(b, l) 198 | require.NoError(b, l.Close()) 199 | 200 | b.SetBytes(l.Size(msgs[0]) * int64(bn)) 201 | b.ResetTimer() 202 | 203 | opts := c.opts 204 | opts.Readonly = true 205 | l, err = Open(dir, opts) 206 | require.NoError(b, err) 207 | defer l.Close() 208 | 209 | for i := 0; i < b.N; i += bn { 210 | if _, _, err := l.Consume(int64(i), int64(bn)); err != nil { 211 | b.Fatal(err) 212 | } 213 | } 214 | 215 | b.StopTimer() 216 | }) 217 | } 218 | } 219 | } 220 | 221 | func benchmarkConsumeMulti(b *testing.B) { 222 | dir := MkdirBench(b) 223 | defer os.RemoveAll(dir) 224 | 225 | s, err := Open(dir, Options{KeyIndex: true}) 226 | require.NoError(b, err) 227 | defer s.Close() 228 | 229 | msgs := message.Gen(b.N) 230 | for i := range msgs { 231 | if _, err := s.Publish(msgs[i : i+1]); err != nil { 232 | b.Fatal(err) 233 | } 234 | } 235 | 236 | b.SetBytes(s.Size(msgs[0]) * 4) 237 | b.ResetTimer() 238 | 239 | g := new(errgroup.Group) 240 | for k := 0; k < 10; k++ { 241 | g.Go(func() error { 242 | for i := 0; i < b.N; i += 4 { 243 | if _, _, err := s.Consume(int64(i), 4); err != nil { 244 | return err 245 | } 246 | } 247 | return nil 248 | }) 249 | } 250 | require.NoError(b, g.Wait()) 251 | 252 | b.StopTimer() 253 | } 254 | 255 | func benchmarkGet(b *testing.B) { 256 | b.Run("ByOffset", func(b *testing.B) { 257 | dir := MkdirBench(b) 258 | defer os.RemoveAll(dir) 259 | 260 | l, err := Open(dir, Options{}) 261 | require.NoError(b, err) 262 | defer l.Close() 263 | 264 | msgs := fillLog(b, l) 265 | 266 | b.SetBytes(l.Size(msgs[0])) 267 | b.ResetTimer() 268 | 269 | for i := 0; i < b.N; i++ { 270 | if _, err := l.Get(int64(i)); err != nil { 271 | b.Fatal(err) 272 | } 273 | } 274 | 275 | b.StopTimer() 276 | }) 277 | 278 | b.Run("ByKey", func(b *testing.B) { 279 | dir := MkdirBench(b) 280 | defer os.RemoveAll(dir) 281 | 282 | l, err := Open(dir, Options{KeyIndex: true}) 283 | require.NoError(b, err) 284 | defer l.Close() 285 | 286 | msgs := fillLog(b, l) 287 | 288 | b.SetBytes(l.Size(msgs[0])) 289 | b.ResetTimer() 290 | 291 | for i := 0; i < b.N; i++ { 292 | if _, err := l.GetByKey(msgs[i].Key); err != nil { 293 | b.Fatal(err) 294 | } 295 | } 296 | 297 | b.StopTimer() 298 | }) 299 | 300 | b.Run("ByKey/R", func(b *testing.B) { 301 | dir := MkdirBench(b) 302 | defer os.RemoveAll(dir) 303 | 304 | l, err := Open(dir, Options{KeyIndex: true}) 305 | require.NoError(b, err) 306 | defer l.Close() 307 | 308 | msgs := fillLog(b, l) 309 | require.NoError(b, l.Close()) 310 | 311 | b.SetBytes(l.Size(msgs[0])) 312 | b.ResetTimer() 313 | 314 | l, err = Open(dir, Options{KeyIndex: true, Readonly: true}) 315 | require.NoError(b, err) 316 | defer l.Close() 317 | 318 | for i := 0; i < b.N; i++ { 319 | if _, err := l.GetByKey(msgs[i].Key); err != nil { 320 | b.Fatal(err) 321 | } 322 | } 323 | 324 | b.StopTimer() 325 | }) 326 | 327 | b.Run("ByTime", func(b *testing.B) { 328 | dir := MkdirBench(b) 329 | defer os.RemoveAll(dir) 330 | 331 | l, err := Open(dir, Options{TimeIndex: true}) 332 | require.NoError(b, err) 333 | defer l.Close() 334 | 335 | msgs := fillLog(b, l) 336 | 337 | b.SetBytes(l.Size(msgs[0])) 338 | b.ResetTimer() 339 | 340 | for i := 0; i < b.N; i++ { 341 | if _, err := l.GetByTime(msgs[i].Time); err != nil { 342 | b.Fatal(err) 343 | } 344 | } 345 | 346 | b.StopTimer() 347 | }) 348 | 349 | b.Run("ByTime/R", func(b *testing.B) { 350 | dir := MkdirBench(b) 351 | defer os.RemoveAll(dir) 352 | 353 | l, err := Open(dir, Options{TimeIndex: true}) 354 | require.NoError(b, err) 355 | defer l.Close() 356 | 357 | msgs := fillLog(b, l) 358 | require.NoError(b, l.Close()) 359 | 360 | b.SetBytes(l.Size(msgs[0])) 361 | b.ResetTimer() 362 | 363 | l, err = Open(dir, Options{TimeIndex: true, Readonly: true}) 364 | require.NoError(b, err) 365 | defer l.Close() 366 | 367 | for i := 0; i < b.N; i++ { 368 | if _, err := l.GetByTime(msgs[i].Time); err != nil { 369 | b.Fatal(err) 370 | } 371 | } 372 | 373 | b.StopTimer() 374 | }) 375 | } 376 | 377 | func benchmarkGetKeyMulti(b *testing.B) { 378 | dir := MkdirBench(b) 379 | defer os.RemoveAll(dir) 380 | 381 | s, err := Open(dir, Options{KeyIndex: true, TimeIndex: true}) 382 | require.NoError(b, err) 383 | defer s.Close() 384 | 385 | msgs := message.Gen(b.N) 386 | for i := 0; i < b.N; i += 10 { 387 | top := i + 10 388 | if top > b.N { 389 | top = b.N 390 | } 391 | if _, err := s.Publish(msgs[i:top]); err != nil { 392 | b.Fatal(err) 393 | } 394 | } 395 | 396 | b.SetBytes(s.Size(msgs[0])) 397 | b.ResetTimer() 398 | 399 | g := new(errgroup.Group) 400 | for k := 0; k < 10; k++ { 401 | g.Go(func() error { 402 | for i := 0; i < b.N; i++ { 403 | if _, err := s.GetByKey(msgs[i].Key); err != nil { 404 | return err 405 | } 406 | } 407 | return nil 408 | }) 409 | } 410 | require.NoError(b, g.Wait()) 411 | 412 | b.StopTimer() 413 | } 414 | 415 | func benchmarkBaseMulti(b *testing.B) { 416 | dir := MkdirBench(b) 417 | defer os.RemoveAll(dir) 418 | 419 | s, err := Open(dir, Options{KeyIndex: true, TimeIndex: true}) 420 | require.NoError(b, err) 421 | defer s.Close() 422 | 423 | msgs := message.Gen(b.N) 424 | 425 | b.ResetTimer() 426 | 427 | g := new(errgroup.Group) 428 | 429 | g.Go(func() error { 430 | for i := 0; i < b.N; i += 10 { 431 | top := i + 10 432 | if top > b.N { 433 | top = b.N 434 | } 435 | if _, err := s.Publish(msgs[i:top]); err != nil { 436 | return err 437 | } 438 | } 439 | return nil 440 | }) 441 | 442 | g.Go(func() error { 443 | offset := OffsetOldest 444 | for offset < int64(len(msgs)) { 445 | next, _, err := s.Consume(offset, 10) 446 | if err != nil { 447 | return err 448 | } 449 | offset = next 450 | } 451 | return nil 452 | }) 453 | require.NoError(b, g.Wait()) 454 | 455 | b.StopTimer() 456 | } 457 | -------------------------------------------------------------------------------- /blocking.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import "context" 4 | 5 | // BlockingLog enhances log adding blocking consume 6 | type BlockingLog interface { 7 | Log 8 | 9 | // ConsumeBlocking is similar to Consume, but if offset is equal to the next offset it will block until next message is produced 10 | ConsumeBlocking(ctx context.Context, offset int64, maxCount int64) (nextOffset int64, messages []Message, err error) 11 | 12 | // ConsumeByKeyBlocking is similar to ConsumeBlocking, but only returns messages matching the key 13 | ConsumeByKeyBlocking(ctx context.Context, key []byte, offset int64, maxCount int64) (nextOffset int64, messages []Message, err error) 14 | } 15 | 16 | // OpenBlocking opens log and wraps it with support for blocking consume 17 | func OpenBlocking(dir string, opts Options) (BlockingLog, error) { 18 | l, err := Open(dir, opts) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return WrapBlocking(l) 23 | } 24 | 25 | // WrapBlocking wraps log with support for blocking consume 26 | func WrapBlocking(l Log) (BlockingLog, error) { 27 | next, err := l.NextOffset() 28 | if err != nil { 29 | return nil, err 30 | } 31 | return &blockingLog{l, NewOffsetNotify(next)}, nil 32 | } 33 | 34 | type blockingLog struct { 35 | Log 36 | notify *OffsetNotify 37 | } 38 | 39 | func (l *blockingLog) Publish(messages []Message) (int64, error) { 40 | nextOffset, err := l.Log.Publish(messages) 41 | if err != nil { 42 | return OffsetInvalid, err 43 | } 44 | 45 | l.notify.Set(nextOffset) 46 | return nextOffset, nil 47 | } 48 | 49 | func (l *blockingLog) ConsumeBlocking(ctx context.Context, offset int64, maxCount int64) (int64, []Message, error) { 50 | if err := l.notify.Wait(ctx, offset); err != nil { 51 | return 0, nil, err 52 | } 53 | return l.Log.Consume(offset, maxCount) 54 | } 55 | 56 | func (l *blockingLog) ConsumeByKeyBlocking(ctx context.Context, key []byte, offset int64, maxCount int64) (int64, []Message, error) { 57 | if err := l.notify.Wait(ctx, offset); err != nil { 58 | return 0, nil, err 59 | } 60 | return l.Log.ConsumeByKey(key, offset, maxCount) 61 | } 62 | 63 | func (l *blockingLog) Close() error { 64 | if err := l.notify.Close(); err != nil { 65 | return err 66 | } 67 | return l.Log.Close() 68 | } 69 | -------------------------------------------------------------------------------- /compact/deletes.go: -------------------------------------------------------------------------------- 1 | package compact 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | art "github.com/plar/go-adaptive-radix-tree/v2" 8 | 9 | "github.com/klev-dev/klevdb" 10 | ) 11 | 12 | // FindDeletes returns a set of offsets for messages with 13 | // nil value for a given key, before a given time. 14 | // 15 | // Messages that have a nil value are considered deletes 16 | // for this key, and therefore eligible for deletion. 17 | func FindDeletes(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, error) { 18 | maxOffset, err := l.NextOffset() 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | var keyOffset = art.New() 24 | var offsets = map[int64]struct{}{} 25 | 26 | SEARCH: 27 | for offset := klevdb.OffsetOldest; offset < maxOffset; { 28 | nextOffset, msgs, err := l.Consume(offset, 32) 29 | if err != nil { 30 | return nil, err 31 | } 32 | offset = nextOffset 33 | 34 | for _, msg := range msgs { 35 | if msg.Time.After(before) { 36 | break SEARCH 37 | } 38 | 39 | // we've seen this previously, we can delete only the first instance 40 | if _, ok := keyOffset.Search(msg.Key); ok { 41 | continue 42 | } 43 | 44 | // not seen it (first instance) whithout value (e.g. delete) 45 | if msg.Value == nil { 46 | offsets[msg.Offset] = struct{}{} 47 | } 48 | 49 | // add it to the set of seen keys, so later instances are not deleted 50 | keyOffset.Insert(msg.Key, msg.Offset) 51 | } 52 | 53 | if err := ctx.Err(); err != nil { 54 | return nil, err 55 | } 56 | } 57 | 58 | if err := ctx.Err(); err != nil { 59 | return nil, err 60 | } 61 | 62 | return offsets, nil 63 | } 64 | 65 | // Deletes tries to remove messages with nil value before given time. 66 | // It will not remove messages for keys it sees before that offset. 67 | // 68 | // This is similar to removing keys, which were deleted (e.g. value set to nil) 69 | // and are therfore no longer relevant/active. 70 | // 71 | // returns the offsets it deleted and the amount of storage freed 72 | func Deletes(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, int64, error) { 73 | offsets, err := FindDeletes(ctx, l, before) 74 | if err != nil { 75 | return nil, 0, err 76 | } 77 | return l.Delete(offsets) 78 | } 79 | 80 | // DeletesMulti is similar to Deletes, but will try to remove messages from multiple segments 81 | func DeletesMulti(ctx context.Context, l klevdb.Log, before time.Time, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 82 | offsets, err := FindDeletes(ctx, l, before) 83 | if err != nil { 84 | return nil, 0, err 85 | } 86 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 87 | } 88 | -------------------------------------------------------------------------------- /compact/deletes_test.go: -------------------------------------------------------------------------------- 1 | package compact 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/klev-dev/klevdb" 9 | "github.com/klev-dev/klevdb/message" 10 | "github.com/stretchr/testify/require" 11 | "golang.org/x/exp/slices" 12 | ) 13 | 14 | func TestDeletes(t *testing.T) { 15 | msgs := message.Gen(5) 16 | 17 | t.Run("Empty", func(t *testing.T) { 18 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 19 | require.NoError(t, err) 20 | defer l.Close() 21 | 22 | off, cmp, err := Deletes(context.TODO(), l, time.Now()) 23 | require.NoError(t, err) 24 | require.Empty(t, off) 25 | require.Equal(t, int64(0), cmp) 26 | }) 27 | 28 | t.Run("None", func(t *testing.T) { 29 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 30 | require.NoError(t, err) 31 | defer l.Close() 32 | 33 | _, err = l.Publish(msgs) 34 | require.NoError(t, err) 35 | 36 | off, cmp, err := Deletes(context.TODO(), l, time.Now()) 37 | require.NoError(t, err) 38 | require.Empty(t, off) 39 | require.Equal(t, int64(0), cmp) 40 | }) 41 | 42 | t.Run("Dups", func(t *testing.T) { 43 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 44 | require.NoError(t, err) 45 | defer l.Close() 46 | 47 | nmsgs := slices.Clone(msgs) 48 | for i := range nmsgs { 49 | nmsgs[i].Value = nil 50 | } 51 | 52 | _, err = l.Publish(nmsgs) 53 | require.NoError(t, err) 54 | 55 | _, err = l.Publish(msgs) 56 | require.NoError(t, err) 57 | 58 | off, cmp, err := Deletes(context.TODO(), l, time.Now()) 59 | require.NoError(t, err) 60 | require.Len(t, off, 5) 61 | for i := range nmsgs { 62 | require.Contains(t, off, int64(i)) 63 | } 64 | require.Equal(t, l.Size(nmsgs[0])*int64(len(nmsgs)), cmp) 65 | }) 66 | } 67 | -------------------------------------------------------------------------------- /compact/updates.go: -------------------------------------------------------------------------------- 1 | package compact 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | art "github.com/plar/go-adaptive-radix-tree/v2" 8 | 9 | "github.com/klev-dev/klevdb" 10 | ) 11 | 12 | // FindUpdates returns a set of offsets for messages that have 13 | // the same key further in the log, before a given time. 14 | // 15 | // Messages before the last one for a given key are considered updates 16 | // that are no longer relevant, and therefore are eligible for deletion. 17 | func FindUpdates(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, error) { 18 | maxOffset, err := l.NextOffset() 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | var keyOffset = art.New() 24 | var offsets = map[int64]struct{}{} 25 | 26 | SEARCH: 27 | for offset := klevdb.OffsetOldest; offset < maxOffset; { 28 | nextOffset, msgs, err := l.Consume(offset, 32) 29 | if err != nil { 30 | return nil, err 31 | } 32 | offset = nextOffset 33 | 34 | for _, msg := range msgs { 35 | if msg.Time.After(before) { 36 | break SEARCH 37 | } 38 | 39 | if prevMsgOffset, ok := keyOffset.Insert(msg.Key, msg.Offset); ok { 40 | offsets[prevMsgOffset.(int64)] = struct{}{} 41 | } 42 | } 43 | 44 | if err := ctx.Err(); err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | if err := ctx.Err(); err != nil { 50 | return nil, err 51 | } 52 | 53 | return offsets, nil 54 | } 55 | 56 | // Updates tries to remove messages before given time that are repeated 57 | // further in the log leaving only the last message for a given key. 58 | // 59 | // This is similar to removing the old value updates, 60 | // leaving only the current value (last update) for a key. 61 | // 62 | // returns the offsets it deleted and the amount of storage freed 63 | func Updates(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, int64, error) { 64 | offsets, err := FindUpdates(ctx, l, before) 65 | if err != nil { 66 | return nil, 0, err 67 | } 68 | return l.Delete(offsets) 69 | } 70 | 71 | // UpdatesMulti is similar to Updates, but will try to remove messages from multiple segments 72 | func UpdatesMulti(ctx context.Context, l klevdb.Log, before time.Time, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 73 | offsets, err := FindUpdates(ctx, l, before) 74 | if err != nil { 75 | return nil, 0, err 76 | } 77 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 78 | } 79 | -------------------------------------------------------------------------------- /compact/updates_test.go: -------------------------------------------------------------------------------- 1 | package compact 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/klev-dev/klevdb" 9 | "github.com/klev-dev/klevdb/message" 10 | "github.com/stretchr/testify/require" 11 | "golang.org/x/exp/slices" 12 | ) 13 | 14 | func TestUpdates(t *testing.T) { 15 | msgs := message.Gen(5) 16 | 17 | t.Run("Empty", func(t *testing.T) { 18 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 19 | require.NoError(t, err) 20 | defer l.Close() 21 | 22 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 23 | require.NoError(t, err) 24 | require.Empty(t, off) 25 | require.Equal(t, int64(0), cmp) 26 | }) 27 | 28 | t.Run("None", func(t *testing.T) { 29 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 30 | require.NoError(t, err) 31 | defer l.Close() 32 | 33 | _, err = l.Publish(msgs) 34 | require.NoError(t, err) 35 | 36 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 37 | require.NoError(t, err) 38 | require.Empty(t, off) 39 | require.Equal(t, int64(0), cmp) 40 | 41 | gmsg, err := l.GetByKey(msgs[0].Key) 42 | require.NoError(t, err) 43 | require.Equal(t, msgs[0], gmsg) 44 | }) 45 | 46 | t.Run("First", func(t *testing.T) { 47 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 48 | require.NoError(t, err) 49 | defer l.Close() 50 | 51 | _, err = l.Publish(msgs) 52 | require.NoError(t, err) 53 | 54 | dmsgs := []message.Message{msgs[0]} 55 | dmsgs[0].Value = []byte("abc") 56 | _, err = l.Publish(dmsgs) 57 | require.NoError(t, err) 58 | 59 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 60 | require.NoError(t, err) 61 | require.Len(t, off, 1) 62 | require.Contains(t, off, int64(0)) 63 | require.Equal(t, l.Size(msgs[0]), cmp) 64 | 65 | gmsg, err := l.GetByKey(msgs[0].Key) 66 | require.NoError(t, err) 67 | require.Equal(t, dmsgs[0], gmsg) 68 | }) 69 | 70 | t.Run("Last", func(t *testing.T) { 71 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 72 | require.NoError(t, err) 73 | defer l.Close() 74 | 75 | _, err = l.Publish(msgs) 76 | require.NoError(t, err) 77 | 78 | dmsgs := []message.Message{msgs[4]} 79 | dmsgs[0].Value = []byte("abc") 80 | _, err = l.Publish(dmsgs) 81 | require.NoError(t, err) 82 | 83 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 84 | require.NoError(t, err) 85 | require.Len(t, off, 1) 86 | require.Contains(t, off, int64(4)) 87 | require.Equal(t, l.Size(msgs[0]), cmp) 88 | 89 | gmsg, err := l.GetByKey(msgs[4].Key) 90 | require.NoError(t, err) 91 | require.Equal(t, dmsgs[0], gmsg) 92 | }) 93 | 94 | t.Run("Multi", func(t *testing.T) { 95 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 96 | require.NoError(t, err) 97 | defer l.Close() 98 | 99 | _, err = l.Publish(msgs) 100 | require.NoError(t, err) 101 | 102 | _, err = l.Publish(msgs) 103 | require.NoError(t, err) 104 | 105 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 106 | require.NoError(t, err) 107 | require.Len(t, off, len(msgs)) 108 | for i := range msgs { 109 | require.Contains(t, off, int64(i)) 110 | } 111 | require.Equal(t, l.Size(msgs[0])*int64(len(msgs)), cmp) 112 | 113 | gmsg, err := l.GetByKey(msgs[1].Key) 114 | require.NoError(t, err) 115 | require.Equal(t, msgs[1], gmsg) 116 | }) 117 | 118 | t.Run("Time", func(t *testing.T) { 119 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 120 | require.NoError(t, err) 121 | defer l.Close() 122 | 123 | _, err = l.Publish(msgs) 124 | require.NoError(t, err) 125 | 126 | nmsgs := slices.Clone(msgs) 127 | for i := range nmsgs { 128 | nmsgs[i].Time = nmsgs[i].Time.Add(time.Hour) 129 | } 130 | _, err = l.Publish(nmsgs) 131 | require.NoError(t, err) 132 | 133 | off, cmp, err := Updates(context.TODO(), l, nmsgs[2].Time) 134 | require.NoError(t, err) 135 | require.Len(t, off, 3) 136 | for i := 0; i < 3; i++ { 137 | require.Contains(t, off, int64(i)) 138 | } 139 | require.Equal(t, l.Size(msgs[0])*3, cmp) 140 | 141 | gmsg, err := l.GetByKey(msgs[1].Key) 142 | require.NoError(t, err) 143 | require.Equal(t, nmsgs[1], gmsg) 144 | }) 145 | 146 | t.Run("NilKey", func(t *testing.T) { 147 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{KeyIndex: true}) 148 | require.NoError(t, err) 149 | defer l.Close() 150 | 151 | _, err = l.Publish([]klevdb.Message{ 152 | {Key: []byte("x")}, 153 | {}, 154 | {}, 155 | }) 156 | require.NoError(t, err) 157 | 158 | off, cmp, err := Updates(context.TODO(), l, time.Now()) 159 | require.NoError(t, err) 160 | require.Len(t, off, 1) 161 | require.Contains(t, off, int64(1)) 162 | require.Equal(t, l.Size(klevdb.Message{}), cmp) 163 | }) 164 | } 165 | -------------------------------------------------------------------------------- /delete.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "context" 5 | "time" 6 | 7 | "golang.org/x/exp/maps" 8 | ) 9 | 10 | // DeleteMultiBackoff is call on each iteration of 11 | // DeleteMulti to give applications opportunity to not overload 12 | // the target log with deletes 13 | type DeleteMultiBackoff func(context.Context) error 14 | 15 | // DeleteMultiWithWait returns a backoff func that sleeps/waits 16 | // for a certain duration. If context is canceled while executing 17 | // it returns the associated error 18 | func DeleteMultiWithWait(d time.Duration) DeleteMultiBackoff { 19 | return func(ctx context.Context) error { 20 | select { 21 | case <-time.After(d): 22 | return nil 23 | case <-ctx.Done(): 24 | return ctx.Err() 25 | } 26 | } 27 | } 28 | 29 | // DeleteMulti tries to delete all messages with offsets 30 | // 31 | // from the log and returns the amount of storage deleted 32 | // 33 | // If error is encountered, it will return the deleted offsets 34 | // 35 | // and size, together with the error 36 | // 37 | // DeleteMultiBackoff is called on each iteration to give 38 | // 39 | // others a chanse to work with the log, while being deleted 40 | func DeleteMulti(ctx context.Context, l Log, offsets map[int64]struct{}, backoff DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 41 | var deletedOffsets = map[int64]struct{}{} 42 | var deletedSize int64 43 | 44 | for len(offsets) > 0 { 45 | deleted, size, err := l.Delete(offsets) 46 | switch { 47 | case err != nil: 48 | return deletedOffsets, deletedSize, err 49 | case len(deleted) == 0: 50 | return deletedOffsets, deletedSize, nil 51 | } 52 | 53 | maps.Copy(deletedOffsets, deleted) 54 | deletedSize += size 55 | maps.DeleteFunc(offsets, func(k int64, v struct{}) bool { 56 | _, ok := deleted[k] 57 | return ok 58 | }) 59 | 60 | if err := backoff(ctx); err != nil { 61 | return deletedOffsets, deletedSize, err 62 | } 63 | } 64 | 65 | return deletedOffsets, deletedSize, nil 66 | } 67 | -------------------------------------------------------------------------------- /delete_test.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/klev-dev/klevdb/message" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestDeleteMulti(t *testing.T) { 13 | msgs := message.Gen(10) 14 | 15 | l, err := Open(t.TempDir(), Options{ 16 | TimeIndex: true, 17 | KeyIndex: true, 18 | Rollover: 2 * (message.Size(msgs[0]) - 1), 19 | }) 20 | require.NoError(t, err) 21 | defer l.Close() 22 | 23 | publishBatched(t, l, msgs, 1) 24 | 25 | stats, err := l.Stat() 26 | require.NoError(t, err) 27 | require.Equal(t, 10, stats.Messages) 28 | require.Equal(t, 5, stats.Segments) 29 | 30 | offsets, sz, err := DeleteMulti(context.TODO(), l, map[int64]struct{}{ 31 | 0: {}, 32 | 2: {}, 33 | 3: {}, 34 | 4: {}, 35 | }, DeleteMultiWithWait(time.Millisecond)) 36 | require.NoError(t, err) 37 | require.Len(t, offsets, 4) 38 | require.Contains(t, offsets, int64(0)) 39 | require.Contains(t, offsets, int64(2)) 40 | require.Contains(t, offsets, int64(3)) 41 | require.Contains(t, offsets, int64(4)) 42 | require.Equal(t, l.Size(msgs[0])*4, sz) 43 | 44 | stats, err = l.Stat() 45 | require.NoError(t, err) 46 | require.Equal(t, 6, stats.Messages) 47 | require.Equal(t, 4, stats.Segments) 48 | } 49 | -------------------------------------------------------------------------------- /flake.lock: -------------------------------------------------------------------------------- 1 | { 2 | "nodes": { 3 | "flake-utils": { 4 | "inputs": { 5 | "systems": "systems" 6 | }, 7 | "locked": { 8 | "lastModified": 1731533236, 9 | "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", 10 | "owner": "numtide", 11 | "repo": "flake-utils", 12 | "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", 13 | "type": "github" 14 | }, 15 | "original": { 16 | "owner": "numtide", 17 | "repo": "flake-utils", 18 | "type": "github" 19 | } 20 | }, 21 | "nixpkgs": { 22 | "locked": { 23 | "lastModified": 1737717945, 24 | "narHash": "sha256-ET91TMkab3PmOZnqiJQYOtSGvSTvGeHoegAv4zcTefM=", 25 | "owner": "NixOS", 26 | "repo": "nixpkgs", 27 | "rev": "ecd26a469ac56357fd333946a99086e992452b6a", 28 | "type": "github" 29 | }, 30 | "original": { 31 | "owner": "NixOS", 32 | "ref": "nixpkgs-unstable", 33 | "repo": "nixpkgs", 34 | "type": "github" 35 | } 36 | }, 37 | "root": { 38 | "inputs": { 39 | "flake-utils": "flake-utils", 40 | "nixpkgs": "nixpkgs" 41 | } 42 | }, 43 | "systems": { 44 | "locked": { 45 | "lastModified": 1681028828, 46 | "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", 47 | "owner": "nix-systems", 48 | "repo": "default", 49 | "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", 50 | "type": "github" 51 | }, 52 | "original": { 53 | "owner": "nix-systems", 54 | "repo": "default", 55 | "type": "github" 56 | } 57 | } 58 | }, 59 | "root": "root", 60 | "version": 7 61 | } 62 | -------------------------------------------------------------------------------- /flake.nix: -------------------------------------------------------------------------------- 1 | { 2 | description = "klevdb is a log storage db"; 3 | 4 | inputs = { 5 | nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; 6 | flake-utils.url = "github:numtide/flake-utils"; 7 | }; 8 | 9 | outputs = { self, nixpkgs, flake-utils }: 10 | flake-utils.lib.eachDefaultSystem (system: 11 | let 12 | pkgs = import nixpkgs { 13 | inherit system; 14 | }; 15 | in 16 | { 17 | formatter = pkgs.nixpkgs-fmt; 18 | devShell = pkgs.mkShell { 19 | buildInputs = with pkgs; [ 20 | go 21 | gopls 22 | golangci-lint 23 | ]; 24 | }; 25 | }); 26 | } 27 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/klev-dev/klevdb 2 | 3 | go 1.23.0 4 | 5 | toolchain go1.23.4 6 | 7 | require ( 8 | github.com/gofrs/flock v0.12.1 9 | github.com/mr-tron/base58 v1.2.0 10 | github.com/plar/go-adaptive-radix-tree/v2 v2.0.3 11 | github.com/stretchr/testify v1.9.0 12 | golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b 13 | golang.org/x/sync v0.15.0 14 | ) 15 | 16 | require ( 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/kr/text v0.2.0 // indirect 19 | github.com/pmezard/go-difflib v1.0.0 // indirect 20 | github.com/rogpeppe/go-internal v1.13.1 // indirect 21 | golang.org/x/sys v0.33.0 // indirect 22 | gopkg.in/yaml.v3 v3.0.1 // indirect 23 | ) 24 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= 5 | github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= 6 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 7 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 8 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 9 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 10 | github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= 11 | github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= 12 | github.com/plar/go-adaptive-radix-tree/v2 v2.0.3 h1:cJx/EUTduV4q10O5HSzHgPrViApJkJQk9OSeaT7UYUU= 13 | github.com/plar/go-adaptive-radix-tree/v2 v2.0.3/go.mod h1:8yf9K81YK94H4gKh/K3hCBeC2s4JA/PYgqMkkOadwvk= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 17 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 18 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 19 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 20 | golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= 21 | golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= 22 | golang.org/x/sync v0.15.0 h1:KWH3jNZsfyT6xfAfKiz6MRNmd46ByHDYaZ7KSkCtdW8= 23 | golang.org/x/sync v0.15.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= 24 | golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= 25 | golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 26 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 27 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 28 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 29 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 30 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 31 | -------------------------------------------------------------------------------- /index/format.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | ) 10 | 11 | var ErrCorrupted = errors.New("index corrupted") 12 | var errIndexSize = fmt.Errorf("%w: unaligned index size", ErrCorrupted) 13 | 14 | type Writer struct { 15 | opts Params 16 | f *os.File 17 | pos int64 18 | buff []byte 19 | keyOffset int 20 | } 21 | 22 | func OpenWriter(path string, opts Params) (*Writer, error) { 23 | f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) 24 | if err != nil { 25 | return nil, fmt.Errorf("write index open: %w", err) 26 | } 27 | 28 | stat, err := f.Stat() 29 | if err != nil { 30 | return nil, fmt.Errorf("write index stat: %w", err) 31 | } 32 | 33 | return &Writer{ 34 | opts: opts, 35 | f: f, 36 | pos: stat.Size(), 37 | keyOffset: opts.keyOffset(), 38 | }, nil 39 | } 40 | 41 | func (w *Writer) Write(it Item) error { 42 | if w.buff == nil { 43 | w.buff = make([]byte, w.opts.Size()) 44 | } 45 | 46 | binary.BigEndian.PutUint64(w.buff[0:], uint64(it.Offset)) 47 | binary.BigEndian.PutUint64(w.buff[8:], uint64(it.Position)) 48 | 49 | if w.opts.Times { 50 | binary.BigEndian.PutUint64(w.buff[16:], uint64(it.Timestamp)) 51 | } 52 | 53 | if w.opts.Keys { 54 | binary.BigEndian.PutUint64(w.buff[w.keyOffset:], it.KeyHash) 55 | } 56 | 57 | if n, err := w.f.Write(w.buff); err != nil { 58 | return fmt.Errorf("write index: %w", err) 59 | } else { 60 | w.pos += int64(n) 61 | } 62 | 63 | return nil 64 | } 65 | 66 | func (w *Writer) Size() int64 { 67 | return w.pos 68 | } 69 | 70 | func (w *Writer) Sync() error { 71 | if err := w.f.Sync(); err != nil { 72 | return fmt.Errorf("write index sync: %w", err) 73 | } 74 | return nil 75 | } 76 | 77 | func (w *Writer) Close() error { 78 | if err := w.f.Close(); err != nil { 79 | return fmt.Errorf("write index close: %w", err) 80 | } 81 | return nil 82 | } 83 | 84 | func (w *Writer) SyncAndClose() error { 85 | if err := w.Sync(); err != nil { 86 | return err 87 | } 88 | return w.Close() 89 | } 90 | 91 | func Write(path string, opts Params, index []Item) error { 92 | w, err := OpenWriter(path, opts) 93 | if err != nil { 94 | return err 95 | } 96 | defer w.Close() 97 | 98 | for _, item := range index { 99 | if err := w.Write(item); err != nil { 100 | return err 101 | } 102 | } 103 | 104 | return w.SyncAndClose() 105 | } 106 | 107 | func Read(path string, opts Params) ([]Item, error) { 108 | f, err := os.Open(path) 109 | if err != nil { 110 | return nil, fmt.Errorf("read index open: %w", err) 111 | } 112 | defer f.Close() 113 | 114 | stat, err := os.Stat(path) 115 | if err != nil { 116 | return nil, fmt.Errorf("read index stat: %w", err) 117 | } 118 | dataSize := stat.Size() 119 | 120 | itemSize := opts.Size() 121 | if dataSize%itemSize > 0 { 122 | return nil, errIndexSize 123 | } 124 | 125 | data := make([]byte, dataSize) 126 | if _, err = io.ReadFull(f, data); err != nil { 127 | return nil, fmt.Errorf("read index: %w", err) 128 | } 129 | 130 | var keyOffset = opts.keyOffset() 131 | 132 | var items = make([]Item, dataSize/int64(itemSize)) 133 | for i := range items { 134 | pos := i * int(itemSize) 135 | 136 | items[i].Offset = int64(binary.BigEndian.Uint64(data[pos:])) 137 | items[i].Position = int64(binary.BigEndian.Uint64(data[pos+8:])) 138 | 139 | if opts.Times { 140 | items[i].Timestamp = int64(binary.BigEndian.Uint64(data[pos+16:])) 141 | } 142 | 143 | if opts.Keys { 144 | items[i].KeyHash = binary.BigEndian.Uint64(data[pos+keyOffset:]) 145 | } 146 | } 147 | return items, nil 148 | } 149 | -------------------------------------------------------------------------------- /index/format_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | var indexSz = 10000 11 | 12 | var iopts = Params{true, true} 13 | 14 | func createIndex(dir string, itemCount int) (string, error) { 15 | var items = make([]Item, itemCount) 16 | for i := range items { 17 | items[i].Offset = int64(i) 18 | items[i].Position = int64(i) 19 | items[i].Timestamp = int64(i) 20 | items[i].KeyHash = uint64(i) 21 | } 22 | filename := filepath.Join(dir, "index") 23 | return filename, Write(filename, iopts, items) 24 | } 25 | 26 | func TestWriteRead(t *testing.T) { 27 | dir := t.TempDir() 28 | 29 | filename, err := createIndex(dir, indexSz) 30 | require.NoError(t, err) 31 | 32 | items, err := Read(filename, iopts) 33 | require.NoError(t, err) 34 | require.Len(t, items, indexSz) 35 | 36 | for i, item := range items { 37 | require.Equal(t, int64(i), item.Offset) 38 | require.Equal(t, int64(i), item.Position) 39 | require.Equal(t, int64(i), item.Timestamp) 40 | require.Equal(t, uint64(i), item.KeyHash) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /index/index.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "github.com/klev-dev/klevdb/message" 5 | ) 6 | 7 | type Item struct { 8 | Offset int64 9 | Position int64 10 | Timestamp int64 11 | KeyHash uint64 12 | } 13 | 14 | type Params struct { 15 | Times bool 16 | Keys bool 17 | } 18 | 19 | func (o Params) keyOffset() int { 20 | off := 8 + 8 // offset + position 21 | if o.Times { 22 | off += 8 23 | } 24 | return off 25 | } 26 | 27 | func (o Params) Size() int64 { 28 | sz := int64(8 + 8) // offset + position 29 | if o.Times { 30 | sz += 8 31 | } 32 | if o.Keys { 33 | sz += 8 34 | } 35 | return sz 36 | } 37 | 38 | func (o Params) NewItem(m message.Message, position int64, prevts int64) Item { 39 | it := Item{Offset: m.Offset, Position: position} 40 | 41 | if o.Times { 42 | it.Timestamp = m.Time.UnixMicro() 43 | // guarantee timestamp monotonic increase 44 | if it.Timestamp < prevts { 45 | it.Timestamp = prevts 46 | } 47 | } 48 | 49 | if o.Keys { 50 | it.KeyHash = KeyHash(m.Key) 51 | } 52 | 53 | return it 54 | } 55 | -------------------------------------------------------------------------------- /index/keys.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "encoding/binary" 5 | "fmt" 6 | "hash/fnv" 7 | 8 | "github.com/klev-dev/klevdb/message" 9 | art "github.com/plar/go-adaptive-radix-tree/v2" 10 | ) 11 | 12 | var ErrKeyNotFound = fmt.Errorf("key: %w", message.ErrNotFound) 13 | 14 | func KeyHash(key []byte) uint64 { 15 | hasher := fnv.New64a() 16 | hasher.Write(key) 17 | return hasher.Sum64() 18 | } 19 | 20 | func KeyHashEncoded(h uint64) []byte { 21 | hash := make([]byte, 8) 22 | binary.BigEndian.PutUint64(hash, h) 23 | return hash 24 | } 25 | 26 | func AppendKeys(keys art.Tree, items []Item) { 27 | hash := make([]byte, 8) 28 | for _, item := range items { 29 | binary.BigEndian.PutUint64(hash, item.KeyHash) 30 | 31 | var positions []int64 32 | if v, found := keys.Search(hash); found { 33 | positions = v.([]int64) 34 | } 35 | positions = append(positions, item.Position) 36 | 37 | keys.Insert(hash, positions) 38 | } 39 | } 40 | 41 | func Keys(keys art.Tree, keyHash []byte) ([]int64, error) { 42 | if v, found := keys.Search(keyHash); found { 43 | return v.([]int64), nil 44 | } 45 | 46 | return nil, ErrKeyNotFound 47 | } 48 | -------------------------------------------------------------------------------- /index/keys_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/klev-dev/klevdb/message" 7 | art "github.com/plar/go-adaptive-radix-tree/v2" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestKeys(t *testing.T) { 12 | t.Run("Empty", func(t *testing.T) { 13 | keys := art.New() 14 | pos, err := Keys(keys, KeyHashEncoded(1)) 15 | require.ErrorIs(t, err, message.ErrNotFound) 16 | require.Empty(t, pos) 17 | }) 18 | 19 | t.Run("Single", func(t *testing.T) { 20 | item := Item{Position: 1, KeyHash: 123} 21 | 22 | keys := art.New() 23 | AppendKeys(keys, []Item{item}) 24 | 25 | pos, err := Keys(keys, KeyHashEncoded(item.KeyHash)) 26 | require.NoError(t, err) 27 | require.ElementsMatch(t, []int64{item.Position}, pos) 28 | 29 | pos, err = Keys(keys, KeyHashEncoded(321)) 30 | require.ErrorIs(t, err, message.ErrNotFound) 31 | require.Empty(t, pos) 32 | }) 33 | 34 | t.Run("Duplicate", func(t *testing.T) { 35 | item1 := Item{Position: 1, KeyHash: 123} 36 | item2 := Item{Position: 2, KeyHash: 123} 37 | item3 := Item{Position: 3, KeyHash: 213} 38 | 39 | keys := art.New() 40 | AppendKeys(keys, []Item{item1, item2, item3}) 41 | 42 | pos, err := Keys(keys, KeyHashEncoded(item1.KeyHash)) 43 | require.NoError(t, err) 44 | require.ElementsMatch(t, []int64{item1.Position, item2.Position}, pos) 45 | 46 | pos, err = Keys(keys, KeyHashEncoded(item3.KeyHash)) 47 | require.NoError(t, err) 48 | require.ElementsMatch(t, []int64{item3.Position}, pos) 49 | 50 | pos, err = Keys(keys, KeyHashEncoded(321)) 51 | require.ErrorIs(t, err, message.ErrNotFound) 52 | require.Empty(t, pos) 53 | }) 54 | } 55 | -------------------------------------------------------------------------------- /index/offset.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/klev-dev/klevdb/message" 7 | ) 8 | 9 | var ErrOffsetIndexEmpty = fmt.Errorf("%w: no offset items", message.ErrInvalidOffset) 10 | var ErrOffsetBeforeStart = fmt.Errorf("%w: offset before start", message.ErrNotFound) 11 | var ErrOffsetAfterEnd = fmt.Errorf("%w: offset after end", message.ErrInvalidOffset) 12 | var ErrOffsetNotFound = fmt.Errorf("%w: offset not found", message.ErrNotFound) 13 | 14 | func Consume(items []Item, offset int64) (int64, int64, error) { 15 | if len(items) == 0 { 16 | return 0, 0, ErrOffsetIndexEmpty 17 | } 18 | 19 | switch offset { 20 | case message.OffsetOldest: 21 | return items[0].Position, items[len(items)-1].Position, nil 22 | case message.OffsetNewest: 23 | last := items[len(items)-1] 24 | return last.Position, last.Position, nil 25 | } 26 | 27 | beginIndex := 0 28 | beginItem := items[beginIndex] 29 | switch { 30 | case offset <= beginItem.Offset: 31 | return beginItem.Position, items[len(items)-1].Position, nil 32 | } 33 | 34 | endIndex := len(items) - 1 35 | endItem := items[endIndex] 36 | switch { 37 | case offset > endItem.Offset: 38 | return 0, 0, ErrOffsetAfterEnd 39 | case offset == endItem.Offset: 40 | return endItem.Position, endItem.Position, nil 41 | } 42 | 43 | for beginIndex <= endIndex { 44 | midIndex := (beginIndex + endIndex) / 2 45 | midItem := items[midIndex] 46 | switch { 47 | case midItem.Offset < offset: 48 | beginIndex = midIndex + 1 49 | case midItem.Offset > offset: 50 | endIndex = midIndex - 1 51 | default: 52 | return midItem.Position, endItem.Position, nil 53 | } 54 | } 55 | 56 | return items[beginIndex].Position, endItem.Position, nil 57 | } 58 | 59 | func Get(items []Item, offset int64) (int64, error) { 60 | if len(items) == 0 { 61 | return 0, ErrOffsetIndexEmpty 62 | } 63 | 64 | switch offset { 65 | case message.OffsetOldest: 66 | return items[0].Position, nil 67 | case message.OffsetNewest: 68 | return items[len(items)-1].Position, nil 69 | } 70 | 71 | beginIndex := 0 72 | beginItem := items[beginIndex] 73 | switch { 74 | case offset < beginItem.Offset: 75 | return 0, ErrOffsetBeforeStart 76 | case offset == beginItem.Offset: 77 | return beginItem.Position, nil 78 | } 79 | 80 | endIndex := len(items) - 1 81 | endItem := items[endIndex] 82 | switch { 83 | case offset > endItem.Offset: 84 | return 0, ErrOffsetAfterEnd 85 | case offset == endItem.Offset: 86 | return endItem.Position, nil 87 | } 88 | 89 | for beginIndex <= endIndex { 90 | midIndex := (beginIndex + endIndex) / 2 91 | midItem := items[midIndex] 92 | switch { 93 | case midItem.Offset < offset: 94 | beginIndex = midIndex + 1 95 | case midItem.Offset > offset: 96 | endIndex = midIndex - 1 97 | default: 98 | return midItem.Position, nil 99 | } 100 | } 101 | 102 | return 0, ErrOffsetNotFound 103 | } 104 | -------------------------------------------------------------------------------- /index/offset_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/klev-dev/klevdb/message" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func genItems(offsets ...int64) []Item { 12 | items := make([]Item, len(offsets)) 13 | for i, v := range offsets { 14 | items[i] = Item{Offset: v, Position: v} 15 | } 16 | return items 17 | } 18 | 19 | func TestConsume(t *testing.T) { 20 | var tests = []struct { 21 | items []int64 22 | offset int64 23 | position int64 24 | max int64 25 | err error 26 | }{ 27 | // empty tests 28 | {items: nil, offset: 0, err: ErrOffsetIndexEmpty}, 29 | // single item tests 30 | {items: []int64{1}, offset: message.OffsetOldest, position: 1, max: 1}, 31 | {items: []int64{1}, offset: message.OffsetNewest, position: 1, max: 1}, 32 | {items: []int64{1}, offset: 0, position: 1, max: 1}, 33 | {items: []int64{1}, offset: 1, position: 1, max: 1}, 34 | {items: []int64{1}, offset: 2, err: ErrOffsetAfterEnd}, 35 | // continuous tests 36 | {items: []int64{1, 2, 3}, offset: message.OffsetOldest, position: 1, max: 3}, 37 | {items: []int64{1, 2, 3}, offset: message.OffsetNewest, position: 3, max: 3}, 38 | {items: []int64{1, 2, 3}, offset: 0, position: 1, max: 3}, 39 | {items: []int64{1, 2, 3}, offset: 1, position: 1, max: 3}, 40 | {items: []int64{1, 2, 3}, offset: 3, position: 3, max: 3}, 41 | {items: []int64{1, 2, 3}, offset: 4, err: ErrOffsetAfterEnd}, 42 | // gaps tests 43 | {items: []int64{1, 3}, offset: message.OffsetOldest, position: 1, max: 3}, 44 | {items: []int64{1, 3}, offset: message.OffsetNewest, position: 3, max: 3}, 45 | {items: []int64{1, 3}, offset: 0, position: 1, max: 3}, 46 | {items: []int64{1, 3}, offset: 1, position: 1, max: 3}, 47 | {items: []int64{1, 3}, offset: 2, position: 3, max: 3}, 48 | {items: []int64{1, 3}, offset: 3, position: 3, max: 3}, 49 | {items: []int64{1, 3}, offset: 4, err: ErrOffsetAfterEnd}, 50 | {items: []int64{1, 3, 5}, offset: message.OffsetOldest, position: 1, max: 5}, 51 | {items: []int64{1, 3, 5}, offset: message.OffsetNewest, position: 5, max: 5}, 52 | {items: []int64{1, 3, 5}, offset: 0, position: 1, max: 5}, 53 | {items: []int64{1, 3, 5}, offset: 1, position: 1, max: 5}, 54 | {items: []int64{1, 3, 5}, offset: 2, position: 3, max: 5}, 55 | {items: []int64{1, 3, 5}, offset: 3, position: 3, max: 5}, 56 | {items: []int64{1, 3, 5}, offset: 4, position: 5, max: 5}, 57 | {items: []int64{1, 3, 5}, offset: 5, position: 5, max: 5}, 58 | {items: []int64{1, 3, 5}, offset: 6, err: ErrOffsetAfterEnd}, 59 | } 60 | 61 | for _, tc := range tests { 62 | t.Run(fmt.Sprintf("%v:%d", tc.items, tc.offset), func(t *testing.T) { 63 | position, maxPosition, err := Consume(genItems(tc.items...), tc.offset) 64 | require.Equal(t, tc.position, position) 65 | require.Equal(t, tc.max, maxPosition) 66 | require.Equal(t, tc.err, err) 67 | }) 68 | } 69 | } 70 | 71 | func TestGet(t *testing.T) { 72 | var tests = []struct { 73 | items []int64 74 | offset int64 75 | position int64 76 | err error 77 | }{ 78 | // empty tests 79 | {items: nil, offset: 0, err: ErrOffsetIndexEmpty}, 80 | // single item tests 81 | {items: []int64{1}, offset: message.OffsetOldest, position: 1}, 82 | {items: []int64{1}, offset: message.OffsetNewest, position: 1}, 83 | {items: []int64{1}, offset: 0, err: ErrOffsetBeforeStart}, 84 | {items: []int64{1}, offset: 1, position: 1}, 85 | {items: []int64{1}, offset: 2, err: ErrOffsetAfterEnd}, 86 | // continuous tests 87 | {items: []int64{1, 2, 3}, offset: message.OffsetOldest, position: 1}, 88 | {items: []int64{1, 2, 3}, offset: message.OffsetNewest, position: 3}, 89 | {items: []int64{1, 2, 3}, offset: 0, err: ErrOffsetBeforeStart}, 90 | {items: []int64{1, 2, 3}, offset: 1, position: 1}, 91 | {items: []int64{1, 2, 3}, offset: 3, position: 3}, 92 | {items: []int64{1, 2, 3}, offset: 4, err: ErrOffsetAfterEnd}, 93 | // gaps tests 94 | {items: []int64{1, 3}, offset: message.OffsetOldest, position: 1}, 95 | {items: []int64{1, 3}, offset: message.OffsetNewest, position: 3}, 96 | {items: []int64{1, 3}, offset: 0, err: ErrOffsetBeforeStart}, 97 | {items: []int64{1, 3}, offset: 1, position: 1}, 98 | {items: []int64{1, 3}, offset: 2, err: ErrOffsetNotFound}, 99 | {items: []int64{1, 3}, offset: 3, position: 3}, 100 | {items: []int64{1, 3}, offset: 4, err: ErrOffsetAfterEnd}, 101 | {items: []int64{1, 3, 5}, offset: message.OffsetOldest, position: 1}, 102 | {items: []int64{1, 3, 5}, offset: message.OffsetNewest, position: 5}, 103 | {items: []int64{1, 3, 5}, offset: 0, err: ErrOffsetBeforeStart}, 104 | {items: []int64{1, 3, 5}, offset: 1, position: 1}, 105 | {items: []int64{1, 3, 5}, offset: 2, err: ErrOffsetNotFound}, 106 | {items: []int64{1, 3, 5}, offset: 3, position: 3}, 107 | {items: []int64{1, 3, 5}, offset: 4, err: ErrOffsetNotFound}, 108 | {items: []int64{1, 3, 5}, offset: 5, position: 5}, 109 | {items: []int64{1, 3, 5}, offset: 6, err: ErrOffsetAfterEnd}, 110 | } 111 | 112 | for _, tc := range tests { 113 | t.Run(fmt.Sprintf("%v:%d", tc.items, tc.offset), func(t *testing.T) { 114 | position, err := Get(genItems(tc.items...), tc.offset) 115 | require.Equal(t, tc.position, position) 116 | require.Equal(t, tc.err, err) 117 | }) 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /index/times.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "sort" 7 | 8 | "github.com/klev-dev/klevdb/message" 9 | ) 10 | 11 | var ErrTimeIndexEmpty = fmt.Errorf("%w: no time items", message.ErrInvalidOffset) 12 | var ErrTimeBeforeStart = errors.New("time before start") 13 | var ErrTimeAfterEnd = errors.New("time after end") 14 | 15 | func Time(items []Item, ts int64) (int64, error) { 16 | if len(items) == 0 { 17 | return 0, ErrTimeIndexEmpty 18 | } 19 | 20 | beginIndex := 0 21 | beginItem := items[beginIndex] 22 | switch { 23 | case ts < beginItem.Timestamp: 24 | return 0, ErrTimeBeforeStart 25 | case ts == beginItem.Timestamp: 26 | return beginItem.Position, nil 27 | } 28 | 29 | endIndex := len(items) - 1 30 | endItem := items[endIndex] 31 | switch { 32 | case endItem.Timestamp < ts: 33 | return 0, ErrTimeAfterEnd 34 | } 35 | 36 | foundIndex := sort.Search(len(items), func(midIndex int) bool { 37 | return items[midIndex].Timestamp >= ts 38 | }) 39 | return items[foundIndex].Position, nil 40 | } 41 | -------------------------------------------------------------------------------- /index/times_test.go: -------------------------------------------------------------------------------- 1 | package index 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestTime(t *testing.T) { 11 | gen := func(ts ...int64) []Item { 12 | items := make([]Item, len(ts)) 13 | for i := range items { 14 | items[i].Timestamp = ts[i] 15 | items[i].Position = int64(i) 16 | } 17 | return items 18 | } 19 | 20 | t.Run("Empty", func(t *testing.T) { 21 | items := gen() 22 | _, err := Time(items, 1) 23 | require.ErrorIs(t, ErrTimeIndexEmpty, err) 24 | }) 25 | 26 | t.Run("Before", func(t *testing.T) { 27 | items := gen(1) 28 | _, err := Time(items, 0) 29 | require.ErrorIs(t, ErrTimeBeforeStart, err) 30 | }) 31 | 32 | t.Run("After", func(t *testing.T) { 33 | items := gen(1) 34 | _, err := Time(items, 2) 35 | require.ErrorIs(t, ErrTimeAfterEnd, err) 36 | }) 37 | 38 | t.Run("Exact", func(t *testing.T) { 39 | for i := 1; i < 6; i++ { 40 | itemsProto := make([]int64, i) 41 | for k := range itemsProto { 42 | itemsProto[k] = int64(k + 1) 43 | } 44 | items := gen(itemsProto...) 45 | for m, it := range itemsProto { 46 | t.Run(fmt.Sprintf("%d/%d", i, it), func(t *testing.T) { 47 | pos, err := Time(items, it) 48 | require.NoError(t, err) 49 | require.Equal(t, int64(m), pos) 50 | }) 51 | } 52 | } 53 | }) 54 | 55 | t.Run("RepeatSingle", func(t *testing.T) { 56 | for i := 1; i < 6; i++ { 57 | itemsProto := make([]int64, i) 58 | for k := range itemsProto { 59 | itemsProto[k] = 1 60 | } 61 | items := gen(itemsProto...) 62 | t.Run(fmt.Sprintf("Exact/%d", i), func(t *testing.T) { 63 | pos, err := Time(items, 1) 64 | require.NoError(t, err) 65 | require.Equal(t, int64(0), pos) 66 | }) 67 | } 68 | }) 69 | 70 | t.Run("RepeatMulti", func(t *testing.T) { 71 | for i := 1; i < 6; i++ { 72 | itemsProto := make([]int64, i*3) 73 | for k := 0; k < i; k++ { 74 | itemsProto[k] = 1 75 | itemsProto[k+i] = 3 76 | itemsProto[k+i*2] = 5 77 | } 78 | items := gen(itemsProto...) 79 | 80 | t.Run(fmt.Sprintf("Start/%d", i), func(t *testing.T) { 81 | pos, err := Time(items, 1) 82 | require.NoError(t, err) 83 | require.Equal(t, int64(0), pos) 84 | }) 85 | 86 | t.Run(fmt.Sprintf("Mid/%d", i), func(t *testing.T) { 87 | pos, err := Time(items, 3) 88 | require.NoError(t, err) 89 | require.Equal(t, int64(i), pos) 90 | }) 91 | 92 | t.Run(fmt.Sprintf("End/%d", i), func(t *testing.T) { 93 | pos, err := Time(items, 5) 94 | require.NoError(t, err) 95 | require.Equal(t, int64(i*2), pos) 96 | }) 97 | 98 | t.Run(fmt.Sprintf("RelLow/%d", i), func(t *testing.T) { 99 | pos, err := Time(items, 2) 100 | require.NoError(t, err) 101 | require.Equal(t, int64(i), pos) 102 | }) 103 | 104 | t.Run(fmt.Sprintf("RelHigh/%d", i), func(t *testing.T) { 105 | pos, err := Time(items, 4) 106 | require.NoError(t, err) 107 | require.Equal(t, int64(i*2), pos) 108 | }) 109 | } 110 | }) 111 | } 112 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "sync" 9 | "time" 10 | 11 | "github.com/gofrs/flock" 12 | "golang.org/x/exp/maps" 13 | "golang.org/x/exp/slices" 14 | 15 | "github.com/klev-dev/klevdb/index" 16 | "github.com/klev-dev/klevdb/message" 17 | "github.com/klev-dev/klevdb/segment" 18 | ) 19 | 20 | var errNoKeyIndex = fmt.Errorf("%w by key", ErrNoIndex) 21 | var errKeyNotFound = fmt.Errorf("key %w", message.ErrNotFound) 22 | var errNoTimeIndex = fmt.Errorf("%w by time", ErrNoIndex) 23 | var errTimeNotFound = fmt.Errorf("time %w", message.ErrNotFound) 24 | var errDeleteRelative = fmt.Errorf("%w: delete relative offsets", message.ErrInvalidOffset) 25 | 26 | // Open create a log based on a dir and set of options 27 | func Open(dir string, opts Options) (result Log, err error) { 28 | if opts.Rollover == 0 { 29 | opts.Rollover = 1024 * 1024 30 | } 31 | 32 | if opts.CreateDirs { 33 | if err := os.MkdirAll(dir, 0700); err != nil { 34 | return nil, fmt.Errorf("open create dirs: %w", err) 35 | } 36 | } 37 | 38 | lock := flock.New(filepath.Join(dir, ".lock")) 39 | if opts.Readonly { 40 | switch ok, err := lock.TryRLock(); { 41 | case err != nil: 42 | return nil, fmt.Errorf("open read lock: %w", err) 43 | case !ok: 44 | return nil, fmt.Errorf("open already writing locked") 45 | } 46 | } else { 47 | switch ok, err := lock.TryLock(); { 48 | case err != nil: 49 | return nil, fmt.Errorf("open lock: %w", err) 50 | case !ok: 51 | return nil, fmt.Errorf("open already locked") 52 | } 53 | } 54 | defer func() { 55 | if err != nil { 56 | if lerr := lock.Unlock(); lerr != nil { 57 | err = fmt.Errorf("%w: open release lock: %w", err, lerr) 58 | } 59 | } 60 | }() 61 | 62 | params := index.Params{Times: opts.TimeIndex, Keys: opts.KeyIndex} 63 | 64 | l := &log{ 65 | dir: dir, 66 | opts: opts, 67 | params: params, 68 | lock: lock, 69 | } 70 | 71 | segments, err := segment.Find(dir) 72 | if err != nil { 73 | return nil, err 74 | } 75 | 76 | if len(segments) == 0 { 77 | if opts.Readonly { 78 | ix := newReaderIndex(nil, params.Keys, 0, true) 79 | rdr := reopenReader(segment.New(dir, 0), params, ix) 80 | l.readers = []*reader{rdr} 81 | } else { 82 | w, err := openWriter(segment.New(dir, 0), params, 0) 83 | if err != nil { 84 | return nil, err 85 | } 86 | l.writer = w 87 | l.readers = []*reader{w.reader} 88 | } 89 | } else { 90 | head := segments[len(segments)-1] 91 | if opts.Check { 92 | if err := head.Check(params); err != nil { 93 | return nil, err 94 | } 95 | } 96 | 97 | for _, seg := range segments[:len(segments)-1] { 98 | rdr := openReader(seg, params, false) 99 | l.readers = append(l.readers, rdr) 100 | } 101 | 102 | if opts.Readonly { 103 | rdr := openReader(head, params, true) 104 | l.readers = append(l.readers, rdr) 105 | } else { 106 | wrt, err := openWriter(head, params, 0) 107 | if err != nil { 108 | return nil, err 109 | } 110 | l.writer = wrt 111 | l.readers = append(l.readers, wrt.reader) 112 | } 113 | } 114 | 115 | return l, nil 116 | } 117 | 118 | type log struct { 119 | dir string 120 | opts Options 121 | params index.Params 122 | lock *flock.Flock 123 | 124 | writer *writer 125 | writerMu sync.Mutex 126 | 127 | readers []*reader 128 | readersMu sync.RWMutex 129 | 130 | deleteMu sync.Mutex 131 | } 132 | 133 | func (l *log) Publish(msgs []message.Message) (int64, error) { 134 | if l.opts.Readonly { 135 | return OffsetInvalid, ErrReadonly 136 | } 137 | 138 | l.writerMu.Lock() 139 | defer l.writerMu.Unlock() 140 | 141 | if l.writer.NeedsRollover(l.opts.Rollover) { 142 | oldWriter := l.writer 143 | if err := oldWriter.Sync(); err != nil { 144 | return OffsetInvalid, err 145 | } 146 | 147 | oldReader, nextOffset, nextTime := l.writer.ReopenReader() 148 | newWriter, err := openWriter(segment.New(l.dir, nextOffset), l.params, nextTime) 149 | if err != nil { 150 | return OffsetInvalid, err 151 | } 152 | 153 | l.readersMu.Lock() 154 | 155 | l.readers[len(l.readers)-1] = oldReader 156 | l.writer = newWriter 157 | l.readers = append(l.readers, newWriter.reader) 158 | 159 | l.readersMu.Unlock() 160 | 161 | if err := oldWriter.Close(); err != nil { 162 | return OffsetInvalid, err 163 | } 164 | } 165 | 166 | nextOffset, err := l.writer.Publish(msgs) 167 | if err != nil { 168 | return OffsetInvalid, err 169 | } 170 | 171 | if l.opts.AutoSync { 172 | if err := l.writer.Sync(); err != nil { 173 | return OffsetInvalid, err 174 | } 175 | } 176 | 177 | return nextOffset, nil 178 | } 179 | 180 | func (l *log) NextOffset() (int64, error) { 181 | if l.opts.Readonly { 182 | l.readersMu.RLock() 183 | defer l.readersMu.RUnlock() 184 | 185 | rdr := l.readers[len(l.readers)-1] 186 | return rdr.GetNextOffset() 187 | } 188 | 189 | l.writerMu.Lock() 190 | defer l.writerMu.Unlock() 191 | 192 | return l.writer.GetNextOffset() 193 | } 194 | 195 | func (l *log) Consume(offset int64, maxCount int64) (int64, []message.Message, error) { 196 | l.readersMu.RLock() 197 | defer l.readersMu.RUnlock() 198 | 199 | rdr, segmentIndex := segment.Consume(l.readers, offset) 200 | 201 | nextOffset, msgs, err := rdr.Consume(offset, maxCount) 202 | if err == index.ErrOffsetAfterEnd && segmentIndex < len(l.readers)-1 { 203 | // this is after the end, consume starting the next one 204 | next := l.readers[segmentIndex+1] 205 | return next.Consume(message.OffsetOldest, maxCount) 206 | } 207 | 208 | return nextOffset, msgs, err 209 | } 210 | 211 | func (l *log) ConsumeByKey(key []byte, offset int64, maxCount int64) (int64, []message.Message, error) { 212 | if !l.opts.KeyIndex { 213 | return OffsetInvalid, nil, errNoKeyIndex 214 | } 215 | 216 | hash := index.KeyHashEncoded(index.KeyHash(key)) 217 | 218 | l.readersMu.RLock() 219 | defer l.readersMu.RUnlock() 220 | 221 | rdr, index := segment.Consume(l.readers, offset) 222 | for { 223 | nextOffset, msgs, err := rdr.ConsumeByKey(key, hash, offset, maxCount) 224 | if err != nil { 225 | return nextOffset, msgs, err 226 | } 227 | if len(msgs) > 0 { 228 | return nextOffset, msgs, err 229 | } 230 | if index >= len(l.readers)-1 { 231 | return nextOffset, msgs, err 232 | } 233 | 234 | index += 1 235 | rdr = l.readers[index] 236 | offset = message.OffsetOldest 237 | } 238 | } 239 | 240 | func (l *log) Get(offset int64) (message.Message, error) { 241 | l.readersMu.RLock() 242 | defer l.readersMu.RUnlock() 243 | 244 | rdr, segmentIndex, err := segment.Get(l.readers, offset) 245 | if err != nil { 246 | return message.Invalid, err 247 | } 248 | 249 | msg, err := rdr.Get(offset) 250 | if err == index.ErrOffsetAfterEnd && segmentIndex < len(l.readers)-1 { 251 | return msg, index.ErrOffsetNotFound 252 | } 253 | return msg, err 254 | } 255 | 256 | func (l *log) GetByKey(key []byte) (message.Message, error) { 257 | if !l.opts.KeyIndex { 258 | return message.Invalid, errNoKeyIndex 259 | } 260 | 261 | hash := index.KeyHashEncoded(index.KeyHash(key)) 262 | tctx := time.Now().UnixMicro() 263 | 264 | l.readersMu.RLock() 265 | defer l.readersMu.RUnlock() 266 | 267 | for i := len(l.readers) - 1; i >= 0; i-- { 268 | rdr := l.readers[i] 269 | 270 | switch msg, err := rdr.GetByKey(key, hash, tctx); { 271 | case err == nil: 272 | return msg, nil 273 | case err == index.ErrKeyNotFound: 274 | // not in this segment, try the rest 275 | default: 276 | return message.Invalid, err 277 | } 278 | } 279 | 280 | // not in any segment, so just return the error 281 | return message.Invalid, errKeyNotFound 282 | } 283 | 284 | func (l *log) OffsetByKey(key []byte) (int64, error) { 285 | msg, err := l.GetByKey(key) 286 | if err != nil { 287 | return OffsetInvalid, err 288 | } 289 | return msg.Offset, nil 290 | } 291 | 292 | func (l *log) GetByTime(start time.Time) (message.Message, error) { 293 | if !l.opts.TimeIndex { 294 | return message.Invalid, errNoTimeIndex 295 | } 296 | 297 | ts := start.UnixMicro() 298 | tctx := time.Now().UnixMicro() 299 | 300 | l.readersMu.RLock() 301 | defer l.readersMu.RUnlock() 302 | 303 | for i := len(l.readers) - 1; i >= 0; i-- { 304 | rdr := l.readers[i] 305 | 306 | switch msg, err := rdr.GetByTime(ts, tctx); { 307 | case err == nil: 308 | return msg, nil 309 | case err == index.ErrTimeBeforeStart: 310 | // not in this segment, try the rest 311 | if i == 0 { 312 | return rdr.Get(message.OffsetOldest) 313 | } 314 | case err == index.ErrTimeAfterEnd: 315 | // time is between end of this and begin next 316 | if i < len(l.readers)-1 { 317 | nextRdr := l.readers[i+1] 318 | return nextRdr.Get(message.OffsetOldest) 319 | } 320 | return message.Invalid, errTimeNotFound 321 | default: 322 | return message.Invalid, err 323 | } 324 | } 325 | 326 | return message.Invalid, errTimeNotFound 327 | } 328 | 329 | func (l *log) OffsetByTime(start time.Time) (int64, time.Time, error) { 330 | msg, err := l.GetByTime(start) 331 | if err != nil { 332 | return OffsetInvalid, time.Time{}, err 333 | } 334 | return msg.Offset, msg.Time, nil 335 | } 336 | 337 | func (l *log) Delete(offsets map[int64]struct{}) (map[int64]struct{}, int64, error) { 338 | if l.opts.Readonly { 339 | return nil, 0, ErrReadonly 340 | } 341 | 342 | if len(offsets) == 0 { 343 | return nil, 0, nil 344 | } 345 | 346 | l.deleteMu.Lock() 347 | defer l.deleteMu.Unlock() 348 | 349 | rdr, err := l.findDeleteReader(offsets) 350 | if err != nil { 351 | return nil, 0, err 352 | } 353 | 354 | l.writerMu.Lock() 355 | if l.writer.reader == rdr { 356 | if err := l.writer.Sync(); err != nil { 357 | l.writerMu.Unlock() 358 | return nil, 0, err 359 | } 360 | } 361 | l.writerMu.Unlock() 362 | 363 | rs, err := rdr.segment.Rewrite(offsets, l.params) 364 | if err != nil { 365 | return nil, 0, err 366 | } 367 | 368 | if len(rs.DeletedOffsets) == 0 { 369 | // deleted nothing, just remove rewrite files 370 | return nil, 0, rs.Segment.Remove() 371 | } 372 | 373 | // check if we are deleting in the writing segment 374 | l.writerMu.Lock() 375 | if l.writer.reader == rdr { 376 | defer l.writerMu.Unlock() 377 | 378 | l.readersMu.Lock() 379 | defer l.readersMu.Unlock() 380 | 381 | newWriter, newReader, err := l.writer.Delete(rs) 382 | switch { 383 | case err == errSegmentChanged: 384 | return nil, 0, nil 385 | case err != nil: 386 | return nil, 0, err 387 | } 388 | 389 | l.writer = newWriter 390 | if newReader == nil { 391 | l.readers[len(l.readers)-1] = newWriter.reader 392 | } else { 393 | l.readers[len(l.readers)-1] = newReader 394 | l.readers = append(l.readers, newWriter.reader) 395 | } 396 | 397 | return rs.DeletedOffsets, rs.DeletedSize, nil 398 | } 399 | l.writerMu.Unlock() 400 | 401 | // we are deleting in a reader segment 402 | l.readersMu.Lock() 403 | defer l.readersMu.Unlock() 404 | 405 | newReader, err := rdr.Delete(rs) 406 | if err != nil { 407 | return nil, 0, err 408 | } 409 | 410 | var newReaders []*reader 411 | for _, r := range l.readers { 412 | if r.segment == rdr.segment { 413 | if newReader != nil { 414 | newReaders = append(newReaders, newReader) 415 | } 416 | } else { 417 | newReaders = append(newReaders, r) 418 | } 419 | } 420 | l.readers = newReaders 421 | 422 | return rs.DeletedOffsets, rs.DeletedSize, nil 423 | } 424 | 425 | func (l *log) findDeleteReader(offsets map[int64]struct{}) (*reader, error) { 426 | orderedOffsets := maps.Keys(offsets) 427 | slices.Sort(orderedOffsets) 428 | lowestOffset := orderedOffsets[0] 429 | 430 | if lowestOffset < 0 { 431 | return nil, errDeleteRelative 432 | } 433 | 434 | l.readersMu.RLock() 435 | defer l.readersMu.RUnlock() 436 | 437 | rdr, _, err := segment.Get(l.readers, lowestOffset) 438 | return rdr, err 439 | } 440 | 441 | func (l *log) Size(m message.Message) int64 { 442 | return message.Size(m) + l.params.Size() 443 | } 444 | 445 | func (l *log) Stat() (segment.Stats, error) { 446 | l.readersMu.RLock() 447 | defer l.readersMu.RUnlock() 448 | 449 | if l.opts.Readonly && len(l.readers) == 1 { 450 | segStats, err := l.readers[0].Stat() 451 | if err != nil && errors.Is(err, os.ErrNotExist) { 452 | return segment.Stats{}, nil 453 | } 454 | return segStats, err 455 | } 456 | 457 | stats := segment.Stats{} 458 | for _, reader := range l.readers { 459 | segStats, err := reader.Stat() 460 | if err != nil { 461 | return segment.Stats{}, err 462 | } 463 | 464 | stats.Segments += segStats.Segments 465 | stats.Messages += segStats.Messages 466 | stats.Size += segStats.Size 467 | } 468 | return stats, nil 469 | } 470 | 471 | func (l *log) Backup(dir string) error { 472 | l.readersMu.RLock() 473 | defer l.readersMu.RUnlock() 474 | 475 | if l.opts.Readonly && len(l.readers) == 1 { 476 | err := l.readers[0].Backup(dir) 477 | if err != nil && errors.Is(err, os.ErrNotExist) { 478 | return nil 479 | } 480 | return err 481 | } 482 | 483 | for _, reader := range l.readers { 484 | if err := reader.Backup(dir); err != nil { 485 | return err 486 | } 487 | } 488 | 489 | return nil 490 | } 491 | 492 | func (l *log) Sync() (int64, error) { 493 | if l.opts.Readonly { 494 | l.readersMu.RLock() 495 | defer l.readersMu.RUnlock() 496 | 497 | rdr := l.readers[len(l.readers)-1] 498 | return rdr.GetNextOffset() 499 | } 500 | 501 | l.writerMu.Lock() 502 | defer l.writerMu.Unlock() 503 | 504 | if err := l.writer.Sync(); err != nil { 505 | return OffsetInvalid, nil 506 | } 507 | return l.writer.GetNextOffset() 508 | } 509 | 510 | func (l *log) GC(unusedFor time.Duration) error { 511 | l.readersMu.RLock() 512 | defer l.readersMu.RUnlock() 513 | 514 | for _, reader := range l.readers { 515 | if err := reader.GC(unusedFor); err != nil { 516 | return err 517 | } 518 | } 519 | 520 | return nil 521 | } 522 | 523 | func (l *log) Close() error { 524 | if l.opts.Readonly { 525 | l.readersMu.Lock() 526 | defer l.readersMu.Unlock() 527 | 528 | for _, reader := range l.readers { 529 | if err := reader.Close(); err != nil { 530 | return err 531 | } 532 | } 533 | } else { 534 | l.writerMu.Lock() 535 | defer l.writerMu.Unlock() 536 | 537 | l.readersMu.Lock() 538 | defer l.readersMu.Unlock() 539 | 540 | if err := l.writer.Sync(); err != nil { 541 | return err 542 | } 543 | 544 | if err := l.writer.Close(); err != nil { 545 | return err 546 | } 547 | 548 | for _, reader := range l.readers[:len(l.readers)-1] { 549 | if err := reader.Close(); err != nil { 550 | return err 551 | } 552 | } 553 | } 554 | 555 | if err := l.lock.Unlock(); err != nil { 556 | return fmt.Errorf("close unlock: %w", err) 557 | } 558 | 559 | return nil 560 | } 561 | -------------------------------------------------------------------------------- /log_test.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "sync" 11 | "testing" 12 | "time" 13 | 14 | "github.com/stretchr/testify/require" 15 | "golang.org/x/sync/errgroup" 16 | 17 | "github.com/klev-dev/klevdb/message" 18 | "github.com/klev-dev/klevdb/segment" 19 | ) 20 | 21 | func publishBatched(t *testing.T, l Log, msgs []Message, batchLen int) { 22 | for begin := 0; begin < len(msgs); begin += batchLen { 23 | end := begin + batchLen 24 | if end > len(msgs) { 25 | end = len(msgs) 26 | } 27 | startOffset, err := l.NextOffset() 28 | require.NoError(t, err) 29 | 30 | nextOffset, err := l.Publish(msgs[begin:end]) 31 | require.NoError(t, err) 32 | require.Equal(t, startOffset+int64(end-begin), nextOffset) 33 | } 34 | } 35 | 36 | func TestBasic(t *testing.T) { 37 | msgs := message.Gen(6) 38 | dir := t.TempDir() 39 | 40 | var l Log 41 | var err error 42 | t.Run("Open", func(t *testing.T) { 43 | l, err = Open(dir, Options{ 44 | Rollover: 3 * message.Size(msgs[0]), 45 | }) 46 | require.NoError(t, err) 47 | 48 | next, err := l.NextOffset() 49 | require.NoError(t, err) 50 | require.Equal(t, int64(0), next) 51 | 52 | stat, err := l.Stat() 53 | require.NoError(t, err) 54 | require.Equal(t, 1, stat.Segments) 55 | require.Equal(t, 0, stat.Messages) 56 | require.Equal(t, int64(0), stat.Size) 57 | }) 58 | 59 | var coff int64 60 | var cmsgs []Message 61 | 62 | t.Run("EmptyRelative", func(t *testing.T) { 63 | coff, cmsgs, err = l.Consume(OffsetOldest, 10) 64 | require.NoError(t, err) 65 | require.Equal(t, int64(0), coff) 66 | require.Nil(t, cmsgs) 67 | 68 | coff, cmsgs, err = l.Consume(OffsetNewest, 10) 69 | require.NoError(t, err) 70 | require.Equal(t, int64(0), coff) 71 | require.Nil(t, cmsgs) 72 | }) 73 | 74 | t.Run("EmptyAbsolute", func(t *testing.T) { 75 | coff, cmsgs, err = l.Consume(0, 10) 76 | require.NoError(t, err) 77 | require.Equal(t, int64(0), coff) 78 | require.Nil(t, cmsgs) 79 | 80 | coff, cmsgs, err = l.Consume(1, 10) 81 | require.ErrorIs(t, err, ErrInvalidOffset) 82 | require.Equal(t, OffsetInvalid, coff) 83 | require.Nil(t, cmsgs) 84 | }) 85 | 86 | t.Run("EmptyPublish", func(t *testing.T) { 87 | nextOffset, err := l.Publish(nil) 88 | require.NoError(t, err) 89 | require.Equal(t, int64(0), nextOffset) 90 | 91 | nextOffset, err = l.NextOffset() 92 | require.NoError(t, err) 93 | require.Equal(t, int64(0), nextOffset) 94 | }) 95 | 96 | t.Run("Publish", func(t *testing.T) { 97 | for i := range msgs[0:3] { 98 | nextOffset, err := l.Publish(msgs[i : i+1]) 99 | require.NoError(t, err) 100 | require.Equal(t, int64(i+1), nextOffset) 101 | 102 | nextOffset, err = l.NextOffset() 103 | require.NoError(t, err) 104 | require.Equal(t, int64(i+1), nextOffset) 105 | } 106 | 107 | nextOffset, err := l.NextOffset() 108 | require.NoError(t, err) 109 | require.Equal(t, int64(3), nextOffset) 110 | 111 | stat, err := l.Stat() 112 | require.NoError(t, err) 113 | require.Equal(t, 1, stat.Segments) 114 | require.Equal(t, 3, stat.Messages) 115 | require.Equal(t, 3*l.Size(msgs[0]), stat.Size) 116 | }) 117 | 118 | t.Run("Relative", func(t *testing.T) { 119 | coff, cmsgs, err = l.Consume(OffsetOldest, 10) 120 | require.NoError(t, err) 121 | require.Equal(t, int64(3), coff) 122 | require.Equal(t, msgs[0:3], cmsgs) 123 | 124 | coff, cmsgs, err = l.Consume(OffsetNewest, 10) 125 | require.NoError(t, err) 126 | require.Equal(t, int64(3), coff) 127 | require.Nil(t, cmsgs) 128 | }) 129 | 130 | t.Run("Absolute", func(t *testing.T) { 131 | coff, cmsgs, err = l.Consume(0, 10) 132 | require.NoError(t, err) 133 | require.Equal(t, int64(3), coff) 134 | require.Equal(t, msgs[0:3], cmsgs) 135 | 136 | coff, cmsgs, err = l.Consume(2, 10) 137 | require.NoError(t, err) 138 | require.Equal(t, int64(3), coff) 139 | require.Equal(t, msgs[2:3], cmsgs) 140 | 141 | coff, cmsgs, err = l.Consume(3, 10) 142 | require.NoError(t, err) 143 | require.Equal(t, int64(3), coff) 144 | require.Nil(t, cmsgs) 145 | 146 | coff, cmsgs, err = l.Consume(4, 10) 147 | require.ErrorIs(t, err, ErrInvalidOffset) 148 | require.Equal(t, OffsetInvalid, coff) 149 | require.Nil(t, cmsgs) 150 | }) 151 | 152 | t.Run("Rollover", func(t *testing.T) { 153 | nextOffset, err := l.Publish(msgs[3:]) 154 | require.NoError(t, err) 155 | require.Equal(t, int64(len(msgs)), nextOffset) 156 | 157 | nextOffset, err = l.NextOffset() 158 | require.NoError(t, err) 159 | require.Equal(t, int64(len(msgs)), nextOffset) 160 | 161 | stat, err := l.Stat() 162 | require.NoError(t, err) 163 | require.Equal(t, 2, stat.Segments) 164 | require.Equal(t, 6, stat.Messages) 165 | require.Equal(t, 6*l.Size(msgs[0]), stat.Size) 166 | }) 167 | 168 | t.Run("RolloverRelative", func(t *testing.T) { 169 | coff, cmsgs, err = l.Consume(OffsetOldest, 10) 170 | require.NoError(t, err) 171 | require.Equal(t, int64(3), coff) 172 | require.Equal(t, msgs[0:3], cmsgs) 173 | 174 | coff, cmsgs, err = l.Consume(OffsetNewest, 10) 175 | require.NoError(t, err) 176 | require.Equal(t, int64(6), coff) 177 | require.Nil(t, cmsgs) 178 | }) 179 | 180 | t.Run("RolloverAbsolute", func(t *testing.T) { 181 | coff, cmsgs, err = l.Consume(0, 3) 182 | require.NoError(t, err) 183 | require.Equal(t, int64(3), coff) 184 | require.Equal(t, cmsgs, msgs[0:3]) 185 | 186 | coff, cmsgs, err = l.Consume(coff, 3) 187 | require.NoError(t, err) 188 | require.Equal(t, int64(6), coff) 189 | require.Equal(t, cmsgs, msgs[3:]) 190 | 191 | coff, cmsgs, err = l.Consume(coff, 10) 192 | require.NoError(t, err) 193 | require.Equal(t, int64(6), coff) 194 | require.Nil(t, cmsgs) 195 | 196 | coff, cmsgs, err = l.Consume(coff+1, 10) 197 | require.ErrorIs(t, err, ErrInvalidOffset) 198 | require.Equal(t, OffsetInvalid, coff) 199 | require.Nil(t, cmsgs) 200 | }) 201 | 202 | t.Run("GC", func(t *testing.T) { 203 | err := l.GC(0) 204 | require.NoError(t, err) 205 | 206 | coff, cmsgs, err = l.Consume(0, 3) 207 | require.NoError(t, err) 208 | require.Equal(t, int64(3), coff) 209 | require.Equal(t, cmsgs, msgs[0:3]) 210 | }) 211 | 212 | t.Run("Close", func(t *testing.T) { 213 | err := l.Close() 214 | require.NoError(t, err) 215 | }) 216 | } 217 | 218 | func TestGet(t *testing.T) { 219 | msgs := message.Gen(4) 220 | 221 | l, err := Open(t.TempDir(), Options{ 222 | Rollover: 2 * message.Size(msgs[0]), 223 | }) 224 | require.NoError(t, err) 225 | defer l.Close() 226 | 227 | var gmsg Message 228 | 229 | t.Run("Empty", func(t *testing.T) { 230 | gmsg, err = l.Get(OffsetOldest) 231 | require.ErrorIs(t, err, ErrInvalidOffset) 232 | require.Equal(t, InvalidMessage, gmsg) 233 | 234 | gmsg, err = l.Get(OffsetNewest) 235 | require.ErrorIs(t, err, ErrInvalidOffset) 236 | require.Equal(t, InvalidMessage, gmsg) 237 | 238 | gmsg, err = l.Get(0) 239 | require.ErrorIs(t, err, ErrInvalidOffset) 240 | require.Equal(t, InvalidMessage, gmsg) 241 | 242 | gmsg, err = l.Get(3) 243 | require.ErrorIs(t, err, ErrInvalidOffset) 244 | require.Equal(t, InvalidMessage, gmsg) 245 | }) 246 | 247 | publishBatched(t, l, msgs, 1) 248 | 249 | t.Run("Absolute", func(t *testing.T) { 250 | for _, msg := range msgs { 251 | gmsg, err = l.Get(msg.Offset) 252 | require.NoError(t, err) 253 | require.Equal(t, msg, gmsg) 254 | } 255 | }) 256 | 257 | t.Run("Relative", func(t *testing.T) { 258 | gmsg, err = l.Get(OffsetInvalid) 259 | require.ErrorIs(t, err, ErrInvalidOffset) 260 | require.Equal(t, InvalidMessage, gmsg) 261 | 262 | gmsg, err = l.Get(OffsetOldest) 263 | require.NoError(t, err) 264 | require.Equal(t, msgs[0], gmsg) 265 | 266 | gmsg, err = l.Get(OffsetNewest) 267 | require.NoError(t, err) 268 | require.Equal(t, msgs[len(msgs)-1], gmsg) 269 | }) 270 | 271 | t.Run("Invalid", func(t *testing.T) { 272 | gmsg, err = l.Get(msgs[len(msgs)-1].Offset + 1) 273 | require.ErrorIs(t, err, ErrInvalidOffset) 274 | require.Equal(t, InvalidMessage, gmsg) 275 | }) 276 | } 277 | 278 | func TestByKey(t *testing.T) { 279 | t.Run("NoIndex", func(t *testing.T) { 280 | l, err := Open(t.TempDir(), Options{}) 281 | require.NoError(t, err) 282 | defer l.Close() 283 | 284 | gmsg, err := l.GetByKey([]byte("key")) 285 | require.ErrorIs(t, err, ErrNoIndex) 286 | require.Equal(t, InvalidMessage, gmsg) 287 | 288 | ooff, err := l.OffsetByKey([]byte("key")) 289 | require.ErrorIs(t, err, ErrNoIndex) 290 | require.Equal(t, OffsetInvalid, ooff) 291 | 292 | coff, cmsgs, err := l.ConsumeByKey([]byte("key"), OffsetOldest, 32) 293 | require.ErrorIs(t, err, ErrNoIndex) 294 | require.Equal(t, OffsetInvalid, coff) 295 | require.Nil(t, cmsgs) 296 | }) 297 | 298 | msgs := message.Gen(4) 299 | l, err := Open(t.TempDir(), Options{ 300 | KeyIndex: true, 301 | Rollover: 2 * message.Size(msgs[0]), 302 | }) 303 | require.NoError(t, err) 304 | defer l.Close() 305 | 306 | t.Run("Empty", func(t *testing.T) { 307 | gmsg, err := l.GetByKey(msgs[0].Key) 308 | require.ErrorIs(t, err, ErrNotFound) 309 | require.Equal(t, InvalidMessage, gmsg) 310 | 311 | ooff, err := l.OffsetByKey(msgs[0].Key) 312 | require.ErrorIs(t, err, ErrNotFound) 313 | require.Equal(t, OffsetInvalid, ooff) 314 | 315 | coff, cmsgs, err := l.ConsumeByKey(msgs[0].Key, OffsetOldest, 32) 316 | require.NoError(t, err) 317 | require.Equal(t, int64(0), coff) 318 | require.Nil(t, cmsgs) 319 | }) 320 | 321 | t.Run("Put", func(t *testing.T) { 322 | publishBatched(t, l, msgs, 1) 323 | 324 | for i, msg := range msgs { 325 | gmsg, err := l.GetByKey(msg.Key) 326 | require.NoError(t, err) 327 | require.Equal(t, msg, gmsg) 328 | 329 | ooff, err := l.OffsetByKey(msg.Key) 330 | require.NoError(t, err) 331 | require.Equal(t, msg.Offset, ooff) 332 | 333 | coff, cmsgs, err := l.ConsumeByKey(msg.Key, OffsetOldest, 32) 334 | require.NoError(t, err) 335 | require.Equal(t, int64(i+1), coff) 336 | require.Len(t, cmsgs, 1) 337 | require.Equal(t, msg, cmsgs[0]) 338 | 339 | // another search would return empty 340 | coff, cmsgs, err = l.ConsumeByKey(msg.Key, coff, 32) 341 | require.NoError(t, err) 342 | require.Equal(t, int64(4), coff) 343 | require.Nil(t, cmsgs) 344 | } 345 | }) 346 | 347 | t.Run("After", func(t *testing.T) { 348 | coff, cmsgs, err := l.ConsumeByKey(msgs[0].Key, msgs[1].Offset, 32) 349 | require.NoError(t, err) 350 | require.Equal(t, int64(4), coff) 351 | require.Nil(t, cmsgs) 352 | }) 353 | 354 | t.Run("Missing", func(t *testing.T) { 355 | gmsg, err := l.GetByKey([]byte("key")) 356 | require.ErrorIs(t, err, ErrNotFound) 357 | require.Equal(t, InvalidMessage, gmsg) 358 | 359 | ooff, err := l.OffsetByKey([]byte("key")) 360 | require.ErrorIs(t, err, ErrNotFound) 361 | require.Equal(t, OffsetInvalid, ooff) 362 | 363 | coff, cmsgs, err := l.ConsumeByKey([]byte("key"), OffsetOldest, 32) 364 | require.NoError(t, err) 365 | require.Equal(t, int64(4), coff) 366 | require.Nil(t, cmsgs) 367 | }) 368 | 369 | t.Run("Update", func(t *testing.T) { 370 | nmsgs := make([]Message, len(msgs)) 371 | for i := range msgs { 372 | nmsgs[i].Time = msgs[i].Time.Add(time.Hour) 373 | nmsgs[i].Key = msgs[i].Key 374 | nmsgs[i].Value = append(msgs[i].Value, "world"...) 375 | } 376 | 377 | publishBatched(t, l, nmsgs, 1) 378 | 379 | for i := range msgs { 380 | gmsg, err := l.GetByKey(msgs[i].Key) 381 | require.NoError(t, err) 382 | require.Equal(t, nmsgs[i], gmsg) 383 | 384 | ooff, err := l.OffsetByKey(msgs[i].Key) 385 | require.NoError(t, err) 386 | require.Equal(t, nmsgs[i].Offset, ooff) 387 | 388 | coff, cmsgs, err := l.ConsumeByKey(msgs[i].Key, OffsetOldest, 32) 389 | require.NoError(t, err) 390 | require.Equal(t, int64(i+1), coff) 391 | require.Len(t, cmsgs, 1) 392 | require.Equal(t, msgs[i], cmsgs[0]) 393 | 394 | coff, cmsgs, err = l.ConsumeByKey(msgs[i].Key, coff, 32) 395 | require.NoError(t, err) 396 | require.Equal(t, int64(len(msgs)+i+1), coff) 397 | require.Len(t, cmsgs, 1) 398 | require.Equal(t, nmsgs[i], cmsgs[0]) 399 | } 400 | }) 401 | } 402 | 403 | func TestByTime(t *testing.T) { 404 | t.Run("NoIndex", func(t *testing.T) { 405 | l, err := Open(t.TempDir(), Options{}) 406 | require.NoError(t, err) 407 | defer l.Close() 408 | 409 | gmsg, err := l.GetByTime(time.Now()) 410 | require.ErrorIs(t, err, ErrNoIndex) 411 | require.Equal(t, InvalidMessage, gmsg) 412 | 413 | ooff, ots, err := l.OffsetByTime(time.Now()) 414 | require.ErrorIs(t, err, ErrNoIndex) 415 | require.Equal(t, OffsetInvalid, ooff) 416 | require.Zero(t, ots) 417 | }) 418 | 419 | msgs := message.Gen(4) 420 | l, err := Open(t.TempDir(), Options{ 421 | TimeIndex: true, 422 | Rollover: 2 * message.Size(msgs[0]), 423 | }) 424 | require.NoError(t, err) 425 | defer l.Close() 426 | 427 | t.Run("Empty", func(t *testing.T) { 428 | gmsg, err := l.GetByTime(msgs[0].Time) 429 | require.ErrorIs(t, err, ErrInvalidOffset) 430 | require.Equal(t, InvalidMessage, gmsg) 431 | 432 | ooff, ots, err := l.OffsetByTime(msgs[0].Time) 433 | require.ErrorIs(t, err, ErrInvalidOffset) 434 | require.Equal(t, OffsetInvalid, ooff) 435 | require.Zero(t, ots) 436 | }) 437 | 438 | t.Run("Absolute", func(t *testing.T) { 439 | publishBatched(t, l, msgs, 1) 440 | 441 | for _, msg := range msgs { 442 | gmsg, err := l.GetByTime(msg.Time) 443 | require.NoError(t, err) 444 | require.Equal(t, msg, gmsg) 445 | 446 | ooff, ots, err := l.OffsetByTime(msg.Time) 447 | require.NoError(t, err) 448 | require.Equal(t, msg.Offset, ooff) 449 | require.Equal(t, msg.Time, ots) 450 | } 451 | }) 452 | 453 | t.Run("Before", func(t *testing.T) { 454 | before := msgs[0].Time.Add(-time.Hour) 455 | 456 | gmsg, err := l.GetByTime(before) 457 | require.NoError(t, err) 458 | require.Equal(t, msgs[0], gmsg) 459 | 460 | ooff, ots, err := l.OffsetByTime(before) 461 | require.NoError(t, err) 462 | require.Equal(t, msgs[0].Offset, ooff) 463 | require.Equal(t, msgs[0].Time, ots) 464 | }) 465 | 466 | t.Run("After", func(t *testing.T) { 467 | before := msgs[len(msgs)-1].Time.Add(time.Hour) 468 | 469 | gmsg, err := l.GetByTime(before) 470 | require.ErrorIs(t, err, ErrNotFound) 471 | require.Equal(t, InvalidMessage, gmsg) 472 | 473 | ooff, ots, err := l.OffsetByTime(before) 474 | require.ErrorIs(t, err, ErrNotFound) 475 | require.Equal(t, OffsetInvalid, ooff) 476 | require.Zero(t, ots) 477 | }) 478 | 479 | t.Run("Mid", func(t *testing.T) { 480 | before := msgs[2].Time.Add(-time.Microsecond) 481 | 482 | gmsg, err := l.GetByTime(before) 483 | require.NoError(t, err) 484 | require.Equal(t, msgs[2], gmsg) 485 | 486 | ooff, ots, err := l.OffsetByTime(before) 487 | require.NoError(t, err) 488 | require.Equal(t, msgs[2].Offset, ooff) 489 | require.Equal(t, msgs[2].Time, ots) 490 | }) 491 | } 492 | 493 | func TestByTimeMono(t *testing.T) { 494 | l, err := Open(t.TempDir(), Options{ 495 | TimeIndex: true, 496 | }) 497 | require.NoError(t, err) 498 | defer l.Close() 499 | 500 | msgs := message.Gen(5) 501 | msgs[1], msgs[3] = msgs[3], msgs[1] 502 | publishBatched(t, l, msgs, 1) 503 | 504 | gmsg, err := l.GetByTime(msgs[1].Time) 505 | require.NoError(t, err) 506 | require.Equal(t, msgs[1], gmsg) 507 | 508 | gmsg, err = l.GetByTime(msgs[2].Time) 509 | require.NoError(t, err) 510 | require.Equal(t, msgs[1], gmsg) 511 | 512 | gmsg, err = l.GetByTime(msgs[3].Time) 513 | require.NoError(t, err) 514 | require.Equal(t, msgs[1], gmsg) 515 | } 516 | 517 | func TestReopen(t *testing.T) { 518 | t.Run("Segment", testReopenSegment) 519 | t.Run("Segments", testReopenSegments) 520 | } 521 | 522 | func testReopenSegment(t *testing.T) { 523 | msgs := message.Gen(4) 524 | dir := t.TempDir() 525 | 526 | l, err := Open(dir, Options{ 527 | TimeIndex: true, 528 | KeyIndex: true, 529 | }) 530 | require.NoError(t, err) 531 | 532 | publishBatched(t, l, msgs, 1) 533 | 534 | require.NoError(t, l.Close()) 535 | 536 | l, err = Open(dir, Options{ 537 | TimeIndex: true, 538 | KeyIndex: true, 539 | }) 540 | require.NoError(t, err) 541 | defer l.Close() 542 | 543 | coff, cmsgs, err := l.Consume(OffsetOldest, 4) 544 | require.NoError(t, err) 545 | require.Equal(t, int64(len(msgs)), coff) 546 | require.Equal(t, msgs, cmsgs) 547 | 548 | for i, msg := range msgs { 549 | gmsg, err := l.GetByKey(msgs[i].Key) 550 | require.NoError(t, err) 551 | require.Equal(t, msg, gmsg) 552 | } 553 | } 554 | 555 | func testReopenSegments(t *testing.T) { 556 | msgs := message.Gen(4) 557 | dir := t.TempDir() 558 | 559 | l, err := Open(dir, Options{ 560 | TimeIndex: true, 561 | KeyIndex: true, 562 | Rollover: 2 * message.Size(msgs[0]), 563 | }) 564 | require.NoError(t, err) 565 | 566 | publishBatched(t, l, msgs, 1) 567 | 568 | require.NoError(t, l.Close()) 569 | 570 | l, err = Open(dir, Options{ 571 | TimeIndex: true, 572 | KeyIndex: true, 573 | }) 574 | require.NoError(t, err) 575 | defer l.Close() 576 | 577 | coff, cmsgs, err := l.Consume(OffsetOldest, 4) 578 | require.NoError(t, err) 579 | require.Equal(t, int64(2), coff) 580 | require.Equal(t, msgs[0:2], cmsgs) 581 | 582 | coff, cmsgs, err = l.Consume(coff, 4) 583 | require.NoError(t, err) 584 | require.Equal(t, int64(4), coff) 585 | require.Equal(t, msgs[2:], cmsgs) 586 | 587 | for i, msg := range msgs { 588 | gmsg, err := l.GetByKey(msgs[i].Key) 589 | require.NoError(t, err) 590 | require.Equal(t, msg, gmsg) 591 | } 592 | } 593 | 594 | func TestReadonly(t *testing.T) { 595 | t.Run("Empty", testReadonlyEmpty) 596 | t.Run("Segment", testReadonlySegment) 597 | t.Run("Segments", testReadonlySegments) 598 | } 599 | 600 | func testReadonlyEmpty(t *testing.T) { 601 | l, err := Open(t.TempDir(), Options{ 602 | TimeIndex: true, 603 | KeyIndex: true, 604 | Readonly: true, 605 | }) 606 | require.NoError(t, err) 607 | defer l.Close() 608 | 609 | _, err = l.Publish(nil) 610 | require.ErrorIs(t, err, ErrReadonly) 611 | 612 | noff, err := l.NextOffset() 613 | require.NoError(t, err) 614 | require.Equal(t, int64(0), noff) 615 | 616 | _, _, err = l.Delete(nil) 617 | require.ErrorIs(t, err, ErrReadonly) 618 | 619 | // Consume checks 620 | coff, cmsgs, err := l.Consume(OffsetOldest, 1) 621 | require.NoError(t, err) 622 | require.Equal(t, int64(0), coff) 623 | require.Empty(t, cmsgs) 624 | 625 | coff, cmsgs, err = l.Consume(OffsetNewest, 1) 626 | require.NoError(t, err) 627 | require.Equal(t, int64(0), coff) 628 | require.Empty(t, cmsgs) 629 | 630 | coff, cmsgs, err = l.Consume(0, 1) 631 | require.NoError(t, err) 632 | require.Equal(t, int64(0), coff) 633 | require.Empty(t, cmsgs) 634 | 635 | coff, cmsgs, err = l.Consume(1, 1) 636 | require.ErrorIs(t, err, ErrInvalidOffset) 637 | require.Equal(t, OffsetInvalid, coff) 638 | require.Empty(t, cmsgs) 639 | 640 | // Get checks 641 | _, err = l.Get(OffsetOldest) 642 | require.ErrorIs(t, err, ErrInvalidOffset) 643 | 644 | _, err = l.Get(OffsetNewest) 645 | require.ErrorIs(t, err, ErrInvalidOffset) 646 | 647 | _, err = l.Get(0) 648 | require.ErrorIs(t, err, ErrInvalidOffset) 649 | 650 | _, err = l.Get(1) 651 | require.ErrorIs(t, err, ErrInvalidOffset) 652 | 653 | // Other getters checks 654 | _, err = l.GetByKey([]byte("abc")) 655 | require.ErrorIs(t, err, ErrNotFound) 656 | 657 | _, err = l.GetByTime(time.Now().UTC()) 658 | require.ErrorIs(t, err, ErrInvalidOffset) 659 | 660 | // Others 661 | stat, err := l.Stat() 662 | require.NoError(t, err) 663 | require.Equal(t, 0, stat.Segments) 664 | require.Equal(t, 0, stat.Messages) 665 | require.Equal(t, int64(0), stat.Size) 666 | 667 | err = l.Backup(t.TempDir()) 668 | require.NoError(t, err) 669 | 670 | noff, err = l.Sync() 671 | require.NoError(t, err) 672 | require.Equal(t, int64(0), noff) 673 | 674 | err = l.Close() 675 | require.NoError(t, err) 676 | } 677 | 678 | func testReadonlySegment(t *testing.T) { 679 | msgs := message.Gen(4) 680 | dir := t.TempDir() 681 | 682 | l, err := Open(dir, Options{ 683 | TimeIndex: true, 684 | KeyIndex: true, 685 | }) 686 | require.NoError(t, err) 687 | 688 | publishBatched(t, l, msgs, 1) 689 | 690 | err = l.Close() 691 | require.NoError(t, err) 692 | 693 | l, err = Open(dir, Options{ 694 | TimeIndex: true, 695 | KeyIndex: true, 696 | Readonly: true, 697 | }) 698 | require.NoError(t, err) 699 | defer l.Close() 700 | 701 | _, err = l.Publish(nil) 702 | require.ErrorIs(t, err, ErrReadonly) 703 | 704 | noff, err := l.NextOffset() 705 | require.NoError(t, err) 706 | require.Equal(t, int64(4), noff) 707 | 708 | _, _, err = l.Delete(nil) 709 | require.ErrorIs(t, err, ErrReadonly) 710 | 711 | // Consume checks 712 | coff, cmsgs, err := l.Consume(OffsetOldest, 4) 713 | require.NoError(t, err) 714 | require.Equal(t, int64(4), coff) 715 | require.Equal(t, msgs, cmsgs) 716 | 717 | coff, cmsgs, err = l.Consume(OffsetNewest, 4) 718 | require.NoError(t, err) 719 | require.Equal(t, int64(4), coff) 720 | require.Empty(t, cmsgs) 721 | 722 | coff, cmsgs, err = l.Consume(0, 1) 723 | require.NoError(t, err) 724 | require.Equal(t, int64(1), coff) 725 | require.Equal(t, msgs[0:1], cmsgs) 726 | 727 | coff, cmsgs, err = l.Consume(4, 4) 728 | require.NoError(t, err) 729 | require.Equal(t, int64(4), coff) 730 | require.Empty(t, cmsgs) 731 | 732 | coff, cmsgs, err = l.Consume(5, 1) 733 | require.ErrorIs(t, err, ErrInvalidOffset) 734 | require.Equal(t, OffsetInvalid, coff) 735 | require.Empty(t, cmsgs) 736 | 737 | // Get checks 738 | gmsg, err := l.Get(OffsetOldest) 739 | require.NoError(t, err) 740 | require.Equal(t, msgs[0], gmsg) 741 | 742 | gmsg, err = l.Get(OffsetNewest) 743 | require.NoError(t, err) 744 | require.Equal(t, msgs[3], gmsg) 745 | 746 | gmsg, err = l.Get(2) 747 | require.NoError(t, err) 748 | require.Equal(t, msgs[2], gmsg) 749 | 750 | _, err = l.Get(5) 751 | require.ErrorIs(t, err, ErrInvalidOffset) 752 | 753 | // Other getters checks 754 | gmsg, err = l.GetByKey(msgs[1].Key) 755 | require.NoError(t, err) 756 | require.Equal(t, msgs[1], gmsg) 757 | 758 | gmsg, err = l.GetByTime(msgs[2].Time) 759 | require.NoError(t, err) 760 | require.Equal(t, msgs[2], gmsg) 761 | 762 | // Others 763 | stat, err := l.Stat() 764 | require.NoError(t, err) 765 | require.Equal(t, 1, stat.Segments) 766 | require.Equal(t, 4, stat.Messages) 767 | require.Equal(t, 4*l.Size(msgs[0]), stat.Size) 768 | 769 | bdir := t.TempDir() 770 | err = l.Backup(bdir) 771 | require.NoError(t, err) 772 | 773 | bstat, err := Stat(bdir, Options{TimeIndex: true, KeyIndex: true}) 774 | require.NoError(t, err) 775 | require.Equal(t, stat, bstat) 776 | 777 | noff, err = l.Sync() 778 | require.NoError(t, err) 779 | require.Equal(t, int64(4), noff) 780 | 781 | err = l.Close() 782 | require.NoError(t, err) 783 | } 784 | 785 | func testReadonlySegments(t *testing.T) { 786 | msgs := message.Gen(4) 787 | dir := t.TempDir() 788 | 789 | l, err := Open(dir, Options{ 790 | TimeIndex: true, 791 | KeyIndex: true, 792 | Rollover: 2 * message.Size(msgs[0]), 793 | }) 794 | require.NoError(t, err) 795 | 796 | publishBatched(t, l, msgs, 1) 797 | 798 | err = l.Close() 799 | require.NoError(t, err) 800 | 801 | l, err = Open(dir, Options{ 802 | TimeIndex: true, 803 | KeyIndex: true, 804 | Readonly: true, 805 | }) 806 | require.NoError(t, err) 807 | defer l.Close() 808 | 809 | _, err = l.Publish(nil) 810 | require.ErrorIs(t, err, ErrReadonly) 811 | 812 | noff, err := l.NextOffset() 813 | require.NoError(t, err) 814 | require.Equal(t, int64(4), noff) 815 | 816 | _, _, err = l.Delete(nil) 817 | require.ErrorIs(t, err, ErrReadonly) 818 | 819 | // Consume checks 820 | coff, cmsgs, err := l.Consume(OffsetOldest, 4) 821 | require.NoError(t, err) 822 | require.Equal(t, int64(2), coff) 823 | require.Equal(t, msgs[0:2], cmsgs) 824 | 825 | coff, cmsgs, err = l.Consume(OffsetNewest, 4) 826 | require.NoError(t, err) 827 | require.Equal(t, int64(4), coff) 828 | require.Empty(t, cmsgs) 829 | 830 | coff, cmsgs, err = l.Consume(2, 4) 831 | require.NoError(t, err) 832 | require.Equal(t, int64(4), coff) 833 | require.Equal(t, msgs[2:], cmsgs) 834 | 835 | coff, cmsgs, err = l.Consume(4, 4) 836 | require.NoError(t, err) 837 | require.Equal(t, int64(4), coff) 838 | require.Empty(t, cmsgs) 839 | 840 | coff, cmsgs, err = l.Consume(5, 1) 841 | require.ErrorIs(t, err, ErrInvalidOffset) 842 | require.Equal(t, OffsetInvalid, coff) 843 | require.Empty(t, cmsgs) 844 | 845 | // Get checks 846 | gmsg, err := l.Get(OffsetOldest) 847 | require.NoError(t, err) 848 | require.Equal(t, msgs[0], gmsg) 849 | 850 | gmsg, err = l.Get(OffsetNewest) 851 | require.NoError(t, err) 852 | require.Equal(t, msgs[3], gmsg) 853 | 854 | gmsg, err = l.Get(2) 855 | require.NoError(t, err) 856 | require.Equal(t, msgs[2], gmsg) 857 | 858 | _, err = l.Get(5) 859 | require.ErrorIs(t, err, ErrInvalidOffset) 860 | 861 | // Other getters checks 862 | gmsg, err = l.GetByKey(msgs[1].Key) 863 | require.NoError(t, err) 864 | require.Equal(t, msgs[1], gmsg) 865 | 866 | gmsg, err = l.GetByTime(msgs[2].Time) 867 | require.NoError(t, err) 868 | require.Equal(t, msgs[2], gmsg) 869 | 870 | // Others 871 | stat, err := l.Stat() 872 | require.NoError(t, err) 873 | require.Equal(t, 2, stat.Segments) 874 | require.Equal(t, 4, stat.Messages) 875 | require.Equal(t, 4*l.Size(msgs[0]), stat.Size) 876 | 877 | bdir := t.TempDir() 878 | err = l.Backup(bdir) 879 | require.NoError(t, err) 880 | 881 | bstat, err := Stat(bdir, Options{TimeIndex: true, KeyIndex: true}) 882 | require.NoError(t, err) 883 | require.Equal(t, stat, bstat) 884 | 885 | noff, err = l.Sync() 886 | require.NoError(t, err) 887 | require.Equal(t, int64(4), noff) 888 | 889 | err = l.Close() 890 | require.NoError(t, err) 891 | } 892 | 893 | func TestStat(t *testing.T) { 894 | t.Run("Segment", testStatSegment) 895 | t.Run("Segments", testStatSegments) 896 | } 897 | 898 | func testStatSegment(t *testing.T) { 899 | msgs := message.Gen(3) 900 | 901 | l, err := Open(t.TempDir(), Options{ 902 | TimeIndex: true, 903 | KeyIndex: true, 904 | }) 905 | require.NoError(t, err) 906 | defer l.Close() 907 | 908 | publishBatched(t, l, msgs, 1) 909 | 910 | sz := int64(0) 911 | for _, msg := range msgs { 912 | sz += l.Size(msg) 913 | } 914 | 915 | stats, err := l.Stat() 916 | require.NoError(t, err) 917 | require.Equal(t, 1, stats.Segments) 918 | require.Equal(t, len(msgs), stats.Messages) 919 | require.Equal(t, sz, stats.Size) 920 | } 921 | 922 | func testStatSegments(t *testing.T) { 923 | msgs := message.Gen(3) 924 | 925 | l, err := Open(t.TempDir(), Options{ 926 | TimeIndex: true, 927 | KeyIndex: true, 928 | Rollover: 2 * message.Size(msgs[0]), 929 | }) 930 | require.NoError(t, err) 931 | defer l.Close() 932 | 933 | publishBatched(t, l, msgs, 1) 934 | 935 | sz := int64(0) 936 | for _, msg := range msgs { 937 | sz += l.Size(msg) 938 | } 939 | 940 | stats, err := l.Stat() 941 | require.NoError(t, err) 942 | require.Equal(t, 2, stats.Segments) 943 | require.Equal(t, len(msgs), stats.Messages) 944 | require.Equal(t, sz, stats.Size) 945 | } 946 | 947 | func TestBackup(t *testing.T) { 948 | t.Run("Segment", testBackupSegment) 949 | t.Run("Segments", testBackupSegments) 950 | } 951 | 952 | func testBackupSegment(t *testing.T) { 953 | msgs := message.Gen(3) 954 | 955 | l, err := Open(t.TempDir(), Options{TimeIndex: true, KeyIndex: true}) 956 | require.NoError(t, err) 957 | defer l.Close() 958 | 959 | publishBatched(t, l, msgs, 1) 960 | 961 | stat, err := l.Stat() 962 | require.NoError(t, err) 963 | require.Equal(t, 1, stat.Segments) 964 | 965 | bdir := t.TempDir() 966 | err = l.Backup(bdir) 967 | require.NoError(t, err) 968 | 969 | bl, err := Open(bdir, Options{TimeIndex: true, KeyIndex: true}) 970 | require.NoError(t, err) 971 | defer bl.Close() 972 | 973 | startOffset := OffsetOldest 974 | for { 975 | coff, cmsgs, err := l.Consume(startOffset, 1) 976 | require.NoError(t, err) 977 | 978 | boff, bmsgs, err := bl.Consume(startOffset, 1) 979 | require.NoError(t, err) 980 | 981 | require.Equal(t, coff, boff) 982 | require.Equal(t, cmsgs, bmsgs) 983 | 984 | if startOffset == coff { 985 | break 986 | } 987 | startOffset = coff 988 | } 989 | } 990 | 991 | func testBackupSegments(t *testing.T) { 992 | msgs := message.Gen(3) 993 | 994 | l, err := Open(t.TempDir(), Options{ 995 | TimeIndex: true, 996 | KeyIndex: true, 997 | Rollover: 2 * message.Size(msgs[0]), 998 | }) 999 | require.NoError(t, err) 1000 | defer l.Close() 1001 | 1002 | publishBatched(t, l, msgs, 1) 1003 | 1004 | stat, err := l.Stat() 1005 | require.NoError(t, err) 1006 | require.Equal(t, 2, stat.Segments) 1007 | 1008 | bdir := t.TempDir() 1009 | err = l.Backup(bdir) 1010 | require.NoError(t, err) 1011 | 1012 | bl, err := Open(bdir, Options{TimeIndex: true, KeyIndex: true}) 1013 | require.NoError(t, err) 1014 | defer bl.Close() 1015 | 1016 | startOffset := OffsetOldest 1017 | for { 1018 | coff, cmsgs, err := l.Consume(startOffset, 1) 1019 | require.NoError(t, err) 1020 | 1021 | boff, bmsgs, err := bl.Consume(startOffset, 1) 1022 | require.NoError(t, err) 1023 | 1024 | require.Equal(t, coff, boff) 1025 | require.Equal(t, cmsgs, bmsgs) 1026 | 1027 | if startOffset == coff { 1028 | break 1029 | } 1030 | startOffset = coff 1031 | } 1032 | } 1033 | 1034 | func TestReindex(t *testing.T) { 1035 | msgs := message.Gen(4) 1036 | 1037 | dir := t.TempDir() 1038 | 1039 | l, err := Open(dir, Options{ 1040 | TimeIndex: true, 1041 | KeyIndex: true, 1042 | Rollover: 2 * message.Size(msgs[0]), 1043 | }) 1044 | require.NoError(t, err) 1045 | 1046 | publishBatched(t, l, msgs, 1) 1047 | require.NoError(t, l.Close()) 1048 | 1049 | // delete all index files 1050 | files, err := os.ReadDir(dir) 1051 | require.NoError(t, err) 1052 | for _, f := range files { 1053 | if strings.HasSuffix(f.Name(), ".index") { 1054 | name := filepath.Join(dir, f.Name()) 1055 | err := os.Remove(name) 1056 | require.NoError(t, err) 1057 | } 1058 | } 1059 | 1060 | l, err = Open(dir, Options{ 1061 | TimeIndex: true, 1062 | KeyIndex: true, 1063 | Rollover: 2 * message.Size(msgs[0]), 1064 | }) 1065 | require.NoError(t, err) 1066 | 1067 | coff, cmsgs, err := l.Consume(message.OffsetOldest, 32) 1068 | require.NoError(t, err) 1069 | require.Equal(t, int64(2), coff) 1070 | require.Equal(t, msgs[0:2], cmsgs) 1071 | 1072 | coff, cmsgs, err = l.Consume(coff, 32) 1073 | require.NoError(t, err) 1074 | require.Equal(t, int64(4), coff) 1075 | require.Equal(t, msgs[2:], cmsgs) 1076 | 1077 | gmsg, err := l.GetByKey(msgs[1].Key) 1078 | require.NoError(t, err) 1079 | require.Equal(t, msgs[1], gmsg) 1080 | } 1081 | 1082 | func TestCorruptReopen(t *testing.T) { 1083 | msgs := message.Gen(4) 1084 | logOpts := Options{ 1085 | TimeIndex: true, 1086 | KeyIndex: true, 1087 | Rollover: 2 * message.Size(msgs[0]), 1088 | } 1089 | 1090 | dir := t.TempDir() 1091 | 1092 | l, err := Open(dir, logOpts) 1093 | require.NoError(t, err) 1094 | 1095 | publishBatched(t, l, msgs, 1) 1096 | require.NoError(t, l.Close()) 1097 | 1098 | segments, err := segment.Find(dir) 1099 | require.NoError(t, err) 1100 | require.Len(t, segments, 2) 1101 | lastSegment := segments[len(segments)-1] 1102 | require.NoError(t, os.WriteFile(lastSegment.Index, []byte("random data characters"), 0700)) 1103 | 1104 | l, err = Open(dir, logOpts) 1105 | require.Error(t, err) 1106 | require.Nil(t, l) 1107 | 1108 | for _, seg := range segments { 1109 | require.NoError(t, os.Remove(seg.Index)) 1110 | } 1111 | 1112 | l, err = Open(dir, logOpts) 1113 | require.NoError(t, err) 1114 | 1115 | coff, cmsgs, err := l.Consume(message.OffsetOldest, 32) 1116 | require.NoError(t, err) 1117 | require.Equal(t, int64(2), coff) 1118 | require.Equal(t, msgs[0:2], cmsgs) 1119 | 1120 | coff, cmsgs, err = l.Consume(coff, 32) 1121 | require.NoError(t, err) 1122 | require.Equal(t, int64(4), coff) 1123 | require.Equal(t, msgs[2:], cmsgs) 1124 | 1125 | gmsg, err := l.GetByKey(msgs[1].Key) 1126 | require.NoError(t, err) 1127 | require.Equal(t, msgs[1], gmsg) 1128 | } 1129 | 1130 | func TestDelete(t *testing.T) { 1131 | t.Run("ReaderPartial", testDeleteReaderPartial) 1132 | t.Run("ReaderPartialReload", testDeleteReaderPartialReload) 1133 | t.Run("ReaderFull", testDeleteReaderFull) 1134 | t.Run("WriterSingle", testDeleteWriterSingle) 1135 | t.Run("WriterLast", testDeleteWriterLast) 1136 | t.Run("WriterPartial", testDeleteWriterPartial) 1137 | t.Run("WriterFull", testDeleteWriterFull) 1138 | t.Run("All", testDeleteAll) 1139 | } 1140 | 1141 | func testDeleteReaderPartial(t *testing.T) { 1142 | msgs := message.Gen(4) 1143 | 1144 | l, err := Open(t.TempDir(), Options{ 1145 | TimeIndex: true, 1146 | KeyIndex: true, 1147 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1148 | }) 1149 | require.NoError(t, err) 1150 | defer l.Close() 1151 | 1152 | publishBatched(t, l, msgs, 1) 1153 | 1154 | stats, err := l.Stat() 1155 | require.NoError(t, err) 1156 | require.Equal(t, 4, stats.Messages) 1157 | require.Equal(t, 2, stats.Segments) 1158 | 1159 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1160 | 0: {}, 1161 | }) 1162 | require.NoError(t, err) 1163 | require.Len(t, offsets, 1) 1164 | require.Contains(t, offsets, int64(0)) 1165 | require.Equal(t, l.Size(msgs[0]), sz) 1166 | 1167 | stats, err = l.Stat() 1168 | require.NoError(t, err) 1169 | require.Equal(t, 3, stats.Messages) 1170 | require.Equal(t, 2, stats.Segments) 1171 | 1172 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1173 | require.NoError(t, err) 1174 | require.Equal(t, int64(2), doff) 1175 | require.Equal(t, msgs[1], dmsgs[0]) 1176 | 1177 | _, err = l.Get(0) 1178 | require.ErrorIs(t, err, ErrNotFound) 1179 | 1180 | _, err = l.GetByKey(msgs[0].Key) 1181 | require.ErrorIs(t, err, ErrNotFound) 1182 | } 1183 | 1184 | func testDeleteReaderPartialReload(t *testing.T) { 1185 | dir := t.TempDir() 1186 | msgs := message.Gen(4) 1187 | 1188 | l, err := Open(dir, Options{ 1189 | TimeIndex: true, 1190 | KeyIndex: true, 1191 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1192 | }) 1193 | require.NoError(t, err) 1194 | publishBatched(t, l, msgs, 1) 1195 | 1196 | err = l.Close() 1197 | require.NoError(t, err) 1198 | 1199 | l, err = Open(dir, Options{ 1200 | TimeIndex: true, 1201 | KeyIndex: true, 1202 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1203 | }) 1204 | require.NoError(t, err) 1205 | defer l.Close() 1206 | 1207 | stats, err := l.Stat() 1208 | require.NoError(t, err) 1209 | require.Equal(t, 4, stats.Messages) 1210 | require.Equal(t, 2, stats.Segments) 1211 | 1212 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1213 | 0: {}, 1214 | }) 1215 | require.NoError(t, err) 1216 | require.Len(t, offsets, 1) 1217 | require.Contains(t, offsets, int64(0)) 1218 | require.Equal(t, l.Size(msgs[0]), sz) 1219 | 1220 | stats, err = l.Stat() 1221 | require.NoError(t, err) 1222 | require.Equal(t, 3, stats.Messages) 1223 | require.Equal(t, 2, stats.Segments) 1224 | 1225 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1226 | require.NoError(t, err) 1227 | require.Equal(t, int64(2), doff) 1228 | require.Equal(t, msgs[1], dmsgs[0]) 1229 | 1230 | _, err = l.Get(0) 1231 | require.ErrorIs(t, err, ErrNotFound) 1232 | 1233 | _, err = l.GetByKey(msgs[0].Key) 1234 | require.ErrorIs(t, err, ErrNotFound) 1235 | } 1236 | 1237 | func testDeleteReaderFull(t *testing.T) { 1238 | msgs := message.Gen(4) 1239 | 1240 | l, err := Open(t.TempDir(), Options{ 1241 | TimeIndex: true, 1242 | KeyIndex: true, 1243 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1244 | }) 1245 | require.NoError(t, err) 1246 | defer l.Close() 1247 | 1248 | publishBatched(t, l, msgs, 1) 1249 | 1250 | stats, err := l.Stat() 1251 | require.NoError(t, err) 1252 | require.Equal(t, 4, stats.Messages) 1253 | require.Equal(t, 2, stats.Segments) 1254 | 1255 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1256 | 0: {}, 1257 | 1: {}, 1258 | }) 1259 | require.NoError(t, err) 1260 | require.Len(t, offsets, 2) 1261 | require.Contains(t, offsets, int64(0)) 1262 | require.Contains(t, offsets, int64(1)) 1263 | require.Equal(t, l.Size(msgs[0])*2, sz) 1264 | 1265 | stats, err = l.Stat() 1266 | require.NoError(t, err) 1267 | require.Equal(t, 2, stats.Messages) 1268 | require.Equal(t, 1, stats.Segments) 1269 | 1270 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1271 | require.NoError(t, err) 1272 | require.Equal(t, int64(4), doff) 1273 | require.Equal(t, msgs[2:], dmsgs) 1274 | } 1275 | 1276 | func testDeleteWriterSingle(t *testing.T) { 1277 | msgs := message.Gen(4) 1278 | 1279 | l, err := Open(t.TempDir(), Options{ 1280 | TimeIndex: true, 1281 | KeyIndex: true, 1282 | Rollover: 4 * (message.Size(msgs[0]) - 1), 1283 | }) 1284 | require.NoError(t, err) 1285 | defer l.Close() 1286 | 1287 | publishBatched(t, l, msgs, 1) 1288 | 1289 | stats, err := l.Stat() 1290 | require.NoError(t, err) 1291 | require.Equal(t, 4, stats.Messages) 1292 | require.Equal(t, 1, stats.Segments) 1293 | 1294 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1295 | 0: {}, 1296 | 1: {}, 1297 | }) 1298 | require.NoError(t, err) 1299 | require.Len(t, offsets, 2) 1300 | require.Contains(t, offsets, int64(0)) 1301 | require.Contains(t, offsets, int64(1)) 1302 | require.Equal(t, l.Size(msgs[0])*2, sz) 1303 | 1304 | stats, err = l.Stat() 1305 | require.NoError(t, err) 1306 | require.Equal(t, 2, stats.Messages) 1307 | require.Equal(t, 1, stats.Segments) 1308 | 1309 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1310 | require.NoError(t, err) 1311 | require.Equal(t, int64(4), doff) 1312 | require.Equal(t, msgs[2:4], dmsgs) 1313 | 1314 | doff, dmsgs, err = l.Consume(1, 32) 1315 | require.NoError(t, err) 1316 | require.Equal(t, int64(4), doff) 1317 | require.Equal(t, msgs[2:4], dmsgs) 1318 | } 1319 | 1320 | func testDeleteWriterLast(t *testing.T) { 1321 | msgs := message.Gen(4) 1322 | dir := t.TempDir() 1323 | 1324 | l, err := Open(dir, Options{ 1325 | TimeIndex: true, 1326 | KeyIndex: true, 1327 | }) 1328 | require.NoError(t, err) 1329 | defer l.Close() 1330 | 1331 | publishBatched(t, l, msgs, 1) 1332 | nextOffset, err := l.NextOffset() 1333 | require.NoError(t, err) 1334 | require.Equal(t, int64(4), nextOffset) 1335 | 1336 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1337 | 3: {}, 1338 | }) 1339 | require.NoError(t, err) 1340 | require.Len(t, offsets, 1) 1341 | require.Contains(t, offsets, int64(3)) 1342 | require.Equal(t, l.Size(msgs[0]), sz) 1343 | 1344 | stats, err := l.Stat() 1345 | require.NoError(t, err) 1346 | require.Equal(t, 3, stats.Messages) 1347 | require.Equal(t, 2, stats.Segments) 1348 | 1349 | err = l.Close() 1350 | require.NoError(t, err) 1351 | 1352 | l, err = Open(dir, Options{ 1353 | TimeIndex: true, 1354 | KeyIndex: true, 1355 | }) 1356 | require.NoError(t, err) 1357 | defer l.Close() 1358 | 1359 | nextOffset, err = l.NextOffset() 1360 | require.NoError(t, err) 1361 | require.Equal(t, int64(4), nextOffset) 1362 | } 1363 | 1364 | func testDeleteWriterPartial(t *testing.T) { 1365 | msgs := message.Gen(4) 1366 | 1367 | l, err := Open(t.TempDir(), Options{ 1368 | TimeIndex: true, 1369 | KeyIndex: true, 1370 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1371 | }) 1372 | require.NoError(t, err) 1373 | defer l.Close() 1374 | 1375 | publishBatched(t, l, msgs, 1) 1376 | 1377 | stats, err := l.Stat() 1378 | require.NoError(t, err) 1379 | require.Equal(t, 4, stats.Messages) 1380 | require.Equal(t, 2, stats.Segments) 1381 | 1382 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1383 | 2: {}, 1384 | }) 1385 | require.NoError(t, err) 1386 | require.Len(t, offsets, 1) 1387 | require.Contains(t, offsets, int64(2)) 1388 | require.Equal(t, l.Size(msgs[0]), sz) 1389 | 1390 | stats, err = l.Stat() 1391 | require.NoError(t, err) 1392 | require.Equal(t, 3, stats.Messages) 1393 | require.Equal(t, 2, stats.Segments) 1394 | 1395 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1396 | require.NoError(t, err) 1397 | require.Equal(t, int64(2), doff) 1398 | require.Equal(t, msgs[0:2], dmsgs) 1399 | 1400 | doff, dmsgs, err = l.Consume(2, 32) 1401 | require.NoError(t, err) 1402 | require.Equal(t, int64(4), doff) 1403 | require.Equal(t, msgs[3:], dmsgs) 1404 | 1405 | _, err = l.Get(2) 1406 | require.ErrorIs(t, err, ErrNotFound) 1407 | } 1408 | 1409 | func testDeleteWriterFull(t *testing.T) { 1410 | msgs := message.Gen(4) 1411 | 1412 | l, err := Open(t.TempDir(), Options{ 1413 | TimeIndex: true, 1414 | KeyIndex: true, 1415 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1416 | }) 1417 | require.NoError(t, err) 1418 | defer l.Close() 1419 | 1420 | publishBatched(t, l, msgs, 1) 1421 | 1422 | stats, err := l.Stat() 1423 | require.NoError(t, err) 1424 | require.Equal(t, 4, stats.Messages) 1425 | require.Equal(t, 2, stats.Segments) 1426 | 1427 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1428 | 2: {}, 1429 | 3: {}, 1430 | }) 1431 | require.NoError(t, err) 1432 | require.Len(t, offsets, 2) 1433 | require.Contains(t, offsets, int64(2)) 1434 | require.Contains(t, offsets, int64(3)) 1435 | require.Equal(t, l.Size(msgs[0])*2, sz) 1436 | 1437 | stats, err = l.Stat() 1438 | require.NoError(t, err) 1439 | require.Equal(t, 2, stats.Messages) 1440 | require.Equal(t, 2, stats.Segments) 1441 | 1442 | doff, dmsgs, err := l.Consume(message.OffsetOldest, 32) 1443 | require.NoError(t, err) 1444 | require.Equal(t, int64(2), doff) 1445 | require.Equal(t, msgs[0:2], dmsgs) 1446 | 1447 | doff, dmsgs, err = l.Consume(2, 32) 1448 | require.NoError(t, err) 1449 | require.Equal(t, int64(4), doff) 1450 | require.Empty(t, dmsgs) 1451 | 1452 | poff, err := l.Publish(msgs[2:]) 1453 | require.NoError(t, err) 1454 | require.Equal(t, int64(6), poff) 1455 | 1456 | doff, dmsgs, err = l.Consume(2, 32) 1457 | require.NoError(t, err) 1458 | require.Equal(t, int64(6), doff) 1459 | require.Equal(t, msgs[2:], dmsgs) 1460 | } 1461 | 1462 | func testDeleteAll(t *testing.T) { 1463 | msgs := message.Gen(4) 1464 | 1465 | l, err := Open(t.TempDir(), Options{ 1466 | TimeIndex: true, 1467 | KeyIndex: true, 1468 | Rollover: 2 * (message.Size(msgs[0]) - 1), 1469 | }) 1470 | require.NoError(t, err) 1471 | defer l.Close() 1472 | 1473 | publishBatched(t, l, msgs, 1) 1474 | 1475 | // delete the writer segment 1476 | offsets, sz, err := l.Delete(map[int64]struct{}{ 1477 | 2: {}, 1478 | 3: {}, 1479 | }) 1480 | require.NoError(t, err) 1481 | require.Len(t, offsets, 2) 1482 | require.Contains(t, offsets, int64(2)) 1483 | require.Contains(t, offsets, int64(3)) 1484 | require.Equal(t, l.Size(msgs[0])*2, sz) 1485 | 1486 | // delete the reader segment 1487 | offsets, sz, err = l.Delete(map[int64]struct{}{ 1488 | 0: {}, 1489 | 1: {}, 1490 | }) 1491 | require.NoError(t, err) 1492 | require.Len(t, offsets, 2) 1493 | require.Contains(t, offsets, int64(0)) 1494 | require.Contains(t, offsets, int64(1)) 1495 | require.Equal(t, l.Size(msgs[0])*2, sz) 1496 | 1497 | stats, err := l.Stat() 1498 | require.NoError(t, err) 1499 | require.Equal(t, 0, stats.Messages) 1500 | require.Equal(t, 1, stats.Segments) 1501 | 1502 | coff, cmsgs, err := l.Consume(message.OffsetOldest, 32) 1503 | require.NoError(t, err) 1504 | require.Equal(t, int64(4), coff) 1505 | require.Empty(t, cmsgs) 1506 | 1507 | coff, cmsgs, err = l.Consume(1, 32) 1508 | require.NoError(t, err) 1509 | require.Equal(t, int64(4), coff) 1510 | require.Empty(t, cmsgs) 1511 | 1512 | poff, err := l.Publish(msgs[0:1]) 1513 | require.NoError(t, err) 1514 | require.Equal(t, int64(len(msgs)+1), poff) 1515 | } 1516 | 1517 | func TestConcurrent(t *testing.T) { 1518 | t.Run("PubsubRecent", testConcurrentPubsubRecent) 1519 | t.Run("Consume", testConcurrentConsume) 1520 | t.Run("Delete", testConcurrentDelete) 1521 | t.Run("GC", testConcurrentGC) 1522 | } 1523 | 1524 | func testConcurrentPubsubRecent(t *testing.T) { 1525 | defer os.RemoveAll("test_pubsub") 1526 | s, err := Open("test_pubsub", Options{ 1527 | CreateDirs: true, 1528 | AutoSync: true, 1529 | Check: true, 1530 | Rollover: 1024 * 64, 1531 | }) 1532 | require.NoError(t, err) 1533 | defer s.Close() 1534 | 1535 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 1536 | defer cancel() 1537 | 1538 | g, ctx := errgroup.WithContext(ctx) 1539 | 1540 | g.Go(func() error { 1541 | for i := 0; ctx.Err() == nil; i++ { 1542 | msgs := []Message{{ 1543 | Key: []byte(fmt.Sprintf("%010d", i)), 1544 | }} 1545 | _, err := s.Publish(msgs) 1546 | if err != nil { 1547 | return err 1548 | } 1549 | } 1550 | return nil 1551 | }) 1552 | 1553 | g.Go(func() error { 1554 | var offset = OffsetOldest 1555 | for ctx.Err() == nil { 1556 | next, msgs, err := s.Consume(offset, 32) 1557 | if err != nil { 1558 | return fmt.Errorf("could not consume offset %d: %w", offset, err) 1559 | } 1560 | 1561 | if offset == next { 1562 | offset = offset - 16 1563 | continue 1564 | } 1565 | 1566 | offset = next 1567 | for _, msg := range msgs { 1568 | require.Equal(t, []byte(fmt.Sprintf("%010d", msg.Offset)), msg.Key) 1569 | } 1570 | } 1571 | return nil 1572 | }) 1573 | 1574 | g.Go(func() error { 1575 | for ctx.Err() == nil { 1576 | _, msgs, err := s.Consume(OffsetOldest, 32) 1577 | if err != nil { 1578 | return err 1579 | } 1580 | 1581 | var del = make(map[int64]struct{}, len(msgs)) 1582 | for _, msg := range msgs { 1583 | del[msg.Offset] = struct{}{} 1584 | } 1585 | _, _, err = s.Delete(del) 1586 | if err != nil { 1587 | return err 1588 | } 1589 | } 1590 | return nil 1591 | }) 1592 | 1593 | require.NoError(t, g.Wait()) 1594 | } 1595 | 1596 | func testConcurrentConsume(t *testing.T) { 1597 | dir := t.TempDir() 1598 | 1599 | s, err := Open(dir, Options{KeyIndex: true, TimeIndex: true}) 1600 | require.NoError(t, err) 1601 | defer s.Close() 1602 | 1603 | var wg sync.WaitGroup 1604 | for i := 0; i < 3; i++ { 1605 | wg.Add(1) 1606 | go func() { 1607 | defer wg.Done() 1608 | 1609 | for i := 0; i < 10000; i++ { 1610 | msgs := []Message{{ 1611 | Key: []byte(fmt.Sprintf("%02d", i)), 1612 | }} 1613 | _, err := s.Publish(msgs) 1614 | require.NoError(t, err) 1615 | } 1616 | }() 1617 | 1618 | wg.Add(1) 1619 | go func() { 1620 | defer wg.Done() 1621 | 1622 | offset := OffsetOldest 1623 | for offset < 30000 { 1624 | next, _, err := s.Consume(offset, 1) 1625 | require.NoError(t, err) 1626 | offset = next 1627 | } 1628 | }() 1629 | 1630 | wg.Add(1) 1631 | go func() { 1632 | defer wg.Done() 1633 | 1634 | time.Sleep(time.Millisecond) 1635 | for i := 0; i < 10000; i++ { 1636 | k := []byte(fmt.Sprintf("%02d", i)) 1637 | _, err := s.GetByKey(k) 1638 | if errors.Is(err, ErrNotFound) { 1639 | i-- 1640 | continue 1641 | } 1642 | require.NoError(t, err, "key %s", k) 1643 | } 1644 | }() 1645 | } 1646 | 1647 | wg.Wait() 1648 | } 1649 | 1650 | func testConcurrentDelete(t *testing.T) { 1651 | dir := t.TempDir() 1652 | 1653 | s, err := Open(dir, Options{KeyIndex: true, TimeIndex: true}) 1654 | require.NoError(t, err) 1655 | defer s.Close() 1656 | 1657 | msgs := message.Gen(10000) 1658 | msgSize := s.Size(msgs[0]) 1659 | 1660 | var wg sync.WaitGroup 1661 | wg.Add(1) 1662 | wg.Add(1) 1663 | 1664 | go func() { 1665 | defer wg.Done() 1666 | 1667 | for i := 0; i < len(msgs); i += 10 { 1668 | _, err := s.Publish(msgs[i : i+10]) 1669 | require.NoError(t, err) 1670 | } 1671 | }() 1672 | 1673 | go func() { 1674 | defer wg.Done() 1675 | 1676 | for !t.Failed() { 1677 | next, msgs, err := s.Consume(OffsetOldest, 10) 1678 | require.NoError(t, err) 1679 | offsets := make(map[int64]struct{}) 1680 | for _, msg := range msgs { 1681 | offsets[msg.Offset] = struct{}{} 1682 | } 1683 | deleted, sz, err := s.Delete(offsets) 1684 | require.NoError(t, err) 1685 | require.Len(t, deleted, len(offsets)) 1686 | require.Equal(t, int64(len(msgs))*msgSize, sz) 1687 | if next >= int64(len(msgs)) { 1688 | break 1689 | } 1690 | if len(msgs) == 0 { 1691 | time.Sleep(time.Millisecond) 1692 | } 1693 | } 1694 | }() 1695 | 1696 | wg.Wait() 1697 | } 1698 | 1699 | func testConcurrentGC(t *testing.T) { 1700 | dir := t.TempDir() 1701 | 1702 | msgs := message.Gen(100) 1703 | msgSize := message.Size(msgs[0]) 1704 | 1705 | l, err := Open(dir, Options{ 1706 | Rollover: msgSize * 10, 1707 | }) 1708 | require.NoError(t, err) 1709 | defer l.Close() 1710 | 1711 | publishBatched(t, l, msgs, 10) 1712 | 1713 | ctx, cancel := context.WithTimeout(context.Background(), time.Second) 1714 | defer cancel() 1715 | 1716 | g, ctx := errgroup.WithContext(ctx) 1717 | 1718 | g.Go(func() error { 1719 | for ctx.Err() == nil { 1720 | next, consumed, err := l.Consume(OffsetOldest, 32) 1721 | if err != nil { 1722 | return err 1723 | } 1724 | require.Equal(t, int64(10), next) 1725 | require.Equal(t, msgs[0:10], consumed) 1726 | } 1727 | return nil 1728 | }) 1729 | 1730 | g.Go(func() error { 1731 | for ctx.Err() == nil { 1732 | if err := l.GC(0); err != nil { 1733 | return err 1734 | } 1735 | time.Sleep(time.Millisecond) 1736 | } 1737 | return nil 1738 | }) 1739 | 1740 | require.NoError(t, g.Wait()) 1741 | } 1742 | -------------------------------------------------------------------------------- /message/format.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "fmt" 7 | "hash/crc32" 8 | "io" 9 | "os" 10 | "time" 11 | 12 | "golang.org/x/exp/mmap" 13 | ) 14 | 15 | var ErrCorrupted = errors.New("log corrupted") 16 | var errShortHeader = fmt.Errorf("%w: short header", ErrCorrupted) 17 | var errShortMessage = fmt.Errorf("%w: short message", ErrCorrupted) 18 | var errNoMessage = fmt.Errorf("%w: no message", ErrCorrupted) 19 | var errCrcFailed = fmt.Errorf("%w: crc failed", ErrCorrupted) 20 | 21 | var crc32cTable = crc32.MakeTable(crc32.Castagnoli) 22 | 23 | func Size(m Message) int64 { 24 | return int64(28 + len(m.Key) + len(m.Value)) 25 | } 26 | 27 | type Writer struct { 28 | Path string 29 | f *os.File 30 | pos int64 31 | buff []byte 32 | } 33 | 34 | func OpenWriter(path string) (*Writer, error) { 35 | f, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600) 36 | if err != nil { 37 | return nil, fmt.Errorf("write log open: %w", err) 38 | } 39 | 40 | stat, err := f.Stat() 41 | if err != nil { 42 | return nil, fmt.Errorf("write log stat: %w", err) 43 | } 44 | 45 | return &Writer{Path: path, f: f, pos: stat.Size()}, nil 46 | } 47 | 48 | func (w *Writer) Write(m Message) (int64, error) { 49 | var fullSize = 8 + // offset 50 | 8 + // unix micro 51 | 4 + // key size 52 | 4 + // value size 53 | 4 + // crc 54 | len(m.Key) + len(m.Value) 55 | 56 | if w.buff == nil || cap(w.buff) < fullSize { 57 | w.buff = make([]byte, fullSize) 58 | } else { 59 | w.buff = w.buff[:fullSize] 60 | } 61 | 62 | binary.BigEndian.PutUint64(w.buff[0:], uint64(m.Offset)) 63 | binary.BigEndian.PutUint64(w.buff[8:], uint64(m.Time.UnixMicro())) 64 | binary.BigEndian.PutUint32(w.buff[16:], uint32(len(m.Key))) 65 | binary.BigEndian.PutUint32(w.buff[20:], uint32(len(m.Value))) 66 | 67 | copy(w.buff[28:], m.Key) 68 | copy(w.buff[28+len(m.Key):], m.Value) 69 | 70 | crc := crc32.Checksum(w.buff[28:], crc32cTable) 71 | binary.BigEndian.PutUint32(w.buff[24:], crc) 72 | 73 | pos := w.pos 74 | if n, err := w.f.Write(w.buff); err != nil { 75 | return 0, fmt.Errorf("write log: %w", err) 76 | } else { 77 | w.pos += int64(n) 78 | } 79 | return pos, nil 80 | } 81 | 82 | func (w *Writer) Size() int64 { 83 | return w.pos 84 | } 85 | 86 | func (w *Writer) Sync() error { 87 | if err := w.f.Sync(); err != nil { 88 | return fmt.Errorf("write log sync: %w", err) 89 | } 90 | return nil 91 | } 92 | 93 | func (w *Writer) Close() error { 94 | if err := w.f.Close(); err != nil { 95 | return fmt.Errorf("write log close: %w", err) 96 | } 97 | return nil 98 | } 99 | 100 | func (w *Writer) SyncAndClose() error { 101 | if err := w.Sync(); err != nil { 102 | return err 103 | } 104 | return w.Close() 105 | } 106 | 107 | type Reader struct { 108 | Path string 109 | r *os.File 110 | ra *mmap.ReaderAt 111 | } 112 | 113 | func OpenReader(path string) (*Reader, error) { 114 | f, err := os.Open(path) 115 | if err != nil { 116 | return nil, fmt.Errorf("read log open: %w", err) 117 | } 118 | 119 | return &Reader{ 120 | Path: path, 121 | r: f, 122 | }, nil 123 | } 124 | 125 | func OpenReaderMem(path string) (*Reader, error) { 126 | f, err := mmap.Open(path) 127 | if err != nil { 128 | return nil, fmt.Errorf("read mem log open: %w", err) 129 | } 130 | 131 | return &Reader{ 132 | Path: path, 133 | ra: f, 134 | }, nil 135 | } 136 | 137 | func (r *Reader) Consume(position, maxPosition int64, maxCount int64) ([]Message, error) { 138 | var msgs = make([]Message, int(maxCount)) 139 | var i int64 140 | for ; i < maxCount && position <= maxPosition; i++ { 141 | next, err := r.read(position, &msgs[i]) 142 | switch { 143 | case err == nil: 144 | position = next 145 | case errors.Is(err, io.EOF): 146 | return msgs[:i], nil 147 | default: 148 | return nil, err 149 | } 150 | } 151 | return msgs[:i], nil 152 | } 153 | 154 | func (r *Reader) Get(position int64) (msg Message, err error) { 155 | _, err = r.read(position, &msg) 156 | return 157 | } 158 | 159 | func (r *Reader) Read(position int64) (msg Message, nextPosition int64, err error) { 160 | nextPosition, err = r.read(position, &msg) 161 | return 162 | } 163 | 164 | func (r *Reader) read(position int64, msg *Message) (nextPosition int64, err error) { 165 | var headerBytes [8 + 8 + 4 + 4 + 4]byte 166 | if r.ra != nil { 167 | _, err = r.ra.ReadAt(headerBytes[:], position) 168 | } else { 169 | _, err = r.r.ReadAt(headerBytes[:], position) 170 | } 171 | switch { 172 | case err == nil: 173 | // all good, continue 174 | case errors.Is(err, io.ErrUnexpectedEOF): 175 | return -1, errShortHeader 176 | default: 177 | return -1, fmt.Errorf("read header: %w", err) 178 | } 179 | 180 | msg.Offset = int64(binary.BigEndian.Uint64(headerBytes[0:])) 181 | msg.Time = time.UnixMicro(int64(binary.BigEndian.Uint64(headerBytes[8:]))).UTC() 182 | keySize := int32(binary.BigEndian.Uint32(headerBytes[16:])) 183 | valueSize := int32(binary.BigEndian.Uint32(headerBytes[20:])) 184 | expectedCRC := binary.BigEndian.Uint32(headerBytes[24:]) 185 | 186 | position += int64(len(headerBytes)) 187 | messageBytes := make([]byte, keySize+valueSize) 188 | if r.ra != nil { 189 | _, err = r.ra.ReadAt(messageBytes, position) 190 | } else { 191 | _, err = r.r.ReadAt(messageBytes, position) 192 | } 193 | switch { 194 | case err == nil: 195 | // all good, continue 196 | case errors.Is(err, io.ErrUnexpectedEOF): 197 | return -1, errShortMessage 198 | case errors.Is(err, io.EOF): 199 | return -1, errNoMessage 200 | default: 201 | return -1, fmt.Errorf("read message: %w", err) 202 | } 203 | 204 | actualCRC := crc32.Checksum(messageBytes, crc32cTable) 205 | if expectedCRC != actualCRC { 206 | return -1, errCrcFailed 207 | } 208 | 209 | if keySize > 0 { 210 | msg.Key = messageBytes[:keySize] 211 | } 212 | if valueSize > 0 { 213 | msg.Value = messageBytes[keySize:] 214 | } 215 | 216 | position += int64(len(messageBytes)) 217 | return position, nil 218 | } 219 | 220 | func (r *Reader) Close() error { 221 | if r.ra != nil { 222 | if err := r.ra.Close(); err != nil { 223 | return fmt.Errorf("write mem log close: %w", err) 224 | } 225 | } else { 226 | if err := r.r.Close(); err != nil { 227 | return fmt.Errorf("write log close: %w", err) 228 | } 229 | } 230 | return nil 231 | } 232 | -------------------------------------------------------------------------------- /message/format_test.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestWriteRead(t *testing.T) { 11 | msgs := Gen(2) 12 | for i := range msgs { 13 | msgs[i].Offset = int64(i + 5) 14 | } 15 | 16 | path := filepath.Join(t.TempDir(), "test.log") 17 | w, err := OpenWriter(path) 18 | require.NoError(t, err) 19 | 20 | pos, err := w.Write(msgs[0]) 21 | require.NoError(t, err) 22 | require.Equal(t, int64(0), pos) 23 | 24 | pos, err = w.Write(msgs[1]) 25 | require.NoError(t, err) 26 | require.Equal(t, Size(msgs[0]), pos) 27 | 28 | err = w.SyncAndClose() 29 | require.NoError(t, err) 30 | 31 | t.Run("Direct", func(t *testing.T) { 32 | r, err := OpenReader(path) 33 | require.NoError(t, err) 34 | 35 | msg, err := r.Get(0) 36 | require.NoError(t, err) 37 | require.Equal(t, msgs[0], msg) 38 | 39 | msg, err = r.Get(pos) 40 | require.NoError(t, err) 41 | require.Equal(t, msgs[1], msg) 42 | 43 | err = r.Close() 44 | require.NoError(t, err) 45 | }) 46 | 47 | t.Run("Mem", func(t *testing.T) { 48 | r, err := OpenReaderMem(path) 49 | require.NoError(t, err) 50 | 51 | msg, err := r.Get(0) 52 | require.NoError(t, err) 53 | require.Equal(t, msgs[0], msg) 54 | 55 | msg, err = r.Get(pos) 56 | require.NoError(t, err) 57 | require.Equal(t, msgs[1], msg) 58 | 59 | err = r.Close() 60 | require.NoError(t, err) 61 | }) 62 | } 63 | 64 | func TestLogSize(t *testing.T) { 65 | path := filepath.Join(t.TempDir(), "test.log") 66 | w, err := OpenWriter(path) 67 | require.NoError(t, err) 68 | 69 | msg := Message{ 70 | Key: []byte("abc"), 71 | Value: []byte("abcde"), 72 | } 73 | pos, err := w.Write(msg) 74 | require.NoError(t, err) 75 | require.Equal(t, int64(0), pos) 76 | 77 | require.Equal(t, w.pos, Size(msg)) 78 | } 79 | -------------------------------------------------------------------------------- /message/message.go: -------------------------------------------------------------------------------- 1 | package message 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | "time" 8 | ) 9 | 10 | const ( 11 | // OffsetOldest represents the smallest offset still available 12 | // Use it to consume all messages, starting at the first available 13 | OffsetOldest int64 = -2 14 | // OffsetNewest represents the offset that will be used for the next produce 15 | // Use it to consume only new messages 16 | OffsetNewest int64 = -1 17 | // OffsetInvalid is the offset returned when error is detected 18 | OffsetInvalid int64 = -3 19 | ) 20 | 21 | var ErrInvalidOffset = errors.New("invalid offset") 22 | var ErrNotFound = errors.New("not found") 23 | 24 | type Message struct { 25 | Offset int64 26 | Time time.Time 27 | Key []byte 28 | Value []byte 29 | } 30 | 31 | var Invalid = Message{Offset: OffsetInvalid} 32 | 33 | func Gen(count int) []Message { 34 | var msgs = make([]Message, count) 35 | for i := range msgs { 36 | msgs[i] = Message{ 37 | Time: time.Date(2023, 1, 1, 0, 0, i, 0, time.UTC), 38 | Key: []byte(fmt.Sprintf("%10d", i)), 39 | Value: []byte(strings.Repeat(" ", 128)), 40 | } 41 | } 42 | return msgs 43 | } 44 | -------------------------------------------------------------------------------- /notify.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "sync/atomic" 7 | ) 8 | 9 | var ErrOffsetNotifyClosed = errors.New("offset notify already closed") 10 | 11 | type OffsetNotify struct { 12 | nextOffset atomic.Int64 13 | barrier chan chan struct{} 14 | } 15 | 16 | func NewOffsetNotify(nextOffset int64) *OffsetNotify { 17 | w := &OffsetNotify{ 18 | barrier: make(chan chan struct{}, 1), 19 | } 20 | 21 | w.nextOffset.Store(nextOffset) 22 | w.barrier <- make(chan struct{}) 23 | 24 | return w 25 | } 26 | 27 | func (w *OffsetNotify) Wait(ctx context.Context, offset int64) error { 28 | // quick path, just load and check 29 | if w.nextOffset.Load() > offset { 30 | return nil 31 | } 32 | 33 | // acquire current barrier 34 | b, ok := <-w.barrier 35 | if !ok { 36 | // already closed, return error 37 | return ErrOffsetNotifyClosed 38 | } 39 | 40 | // probe the current offset 41 | updated := w.nextOffset.Load() > offset 42 | 43 | // release current barrier 44 | w.barrier <- b 45 | 46 | // already has a new value, return 47 | if updated { 48 | return nil 49 | } 50 | 51 | // now wait for something to happen 52 | select { 53 | case <-b: 54 | return nil 55 | case <-ctx.Done(): 56 | return ctx.Err() 57 | } 58 | } 59 | 60 | func (w *OffsetNotify) Set(nextOffset int64) { 61 | // acquire current barrier 62 | b, ok := <-w.barrier 63 | if !ok { 64 | // already closed 65 | return 66 | } 67 | 68 | // set the new offset 69 | if w.nextOffset.Load() < nextOffset { 70 | w.nextOffset.Store(nextOffset) 71 | } 72 | 73 | // close the current barrier, e.g. broadcasting update 74 | close(b) 75 | 76 | // create new barrier 77 | w.barrier <- make(chan struct{}) 78 | } 79 | 80 | func (w *OffsetNotify) Close() error { 81 | // acquire current barrier 82 | b, ok := <-w.barrier 83 | if !ok { 84 | // already closed, return an error 85 | return ErrOffsetNotifyClosed 86 | } 87 | 88 | // close the current barrier, e.g. broadcasting update 89 | close(b) 90 | 91 | // close the barrier channel, completing process 92 | close(w.barrier) 93 | 94 | return nil 95 | } 96 | -------------------------------------------------------------------------------- /notify_test.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestNotify(t *testing.T) { 13 | t.Run("unblock", func(t *testing.T) { 14 | n := NewOffsetNotify(10) 15 | 16 | err := n.Wait(context.TODO(), 5) 17 | require.NoError(t, err) 18 | }) 19 | 20 | t.Run("blocked", func(t *testing.T) { 21 | n := NewOffsetNotify(10) 22 | ch := make(chan struct{}) 23 | var wg sync.WaitGroup 24 | 25 | wg.Add(1) 26 | go func() { 27 | defer wg.Done() 28 | close(ch) 29 | 30 | err := n.Wait(context.TODO(), 15) 31 | require.NoError(t, err) 32 | }() 33 | 34 | <-ch 35 | time.Sleep(10 * time.Millisecond) 36 | n.Set(20) 37 | wg.Wait() 38 | }) 39 | 40 | t.Run("cancel", func(t *testing.T) { 41 | n := NewOffsetNotify(10) 42 | ch := make(chan struct{}) 43 | var wg sync.WaitGroup 44 | ctx, cancel := context.WithCancel(context.TODO()) 45 | defer cancel() 46 | 47 | wg.Add(1) 48 | go func() { 49 | defer wg.Done() 50 | close(ch) 51 | 52 | err := n.Wait(ctx, 15) 53 | require.ErrorIs(t, err, context.Canceled) 54 | }() 55 | 56 | <-ch 57 | time.Sleep(10 * time.Millisecond) 58 | cancel() 59 | wg.Wait() 60 | }) 61 | 62 | t.Run("close", func(t *testing.T) { 63 | n := NewOffsetNotify(10) 64 | ch := make(chan struct{}) 65 | var wg sync.WaitGroup 66 | 67 | wg.Add(1) 68 | go func() { 69 | defer wg.Done() 70 | close(ch) 71 | 72 | err := n.Wait(context.TODO(), 15) 73 | require.NoError(t, err) 74 | }() 75 | 76 | <-ch 77 | time.Sleep(10 * time.Millisecond) 78 | n.Close() 79 | wg.Wait() 80 | }) 81 | 82 | t.Run("close_err", func(t *testing.T) { 83 | n := NewOffsetNotify(10) 84 | err := n.Close() 85 | require.NoError(t, err) 86 | 87 | err = n.Wait(context.TODO(), 15) 88 | require.ErrorIs(t, err, ErrOffsetNotifyClosed) 89 | 90 | err = n.Close() 91 | require.ErrorIs(t, err, ErrOffsetNotifyClosed) 92 | }) 93 | } 94 | -------------------------------------------------------------------------------- /reader.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | 9 | art "github.com/plar/go-adaptive-radix-tree/v2" 10 | 11 | "github.com/klev-dev/klevdb/index" 12 | "github.com/klev-dev/klevdb/message" 13 | "github.com/klev-dev/klevdb/segment" 14 | ) 15 | 16 | type reader struct { 17 | segment segment.Segment 18 | params index.Params 19 | head bool 20 | 21 | messages *message.Reader 22 | messagesMu sync.RWMutex 23 | messagesInuse atomic.Int64 24 | 25 | index indexer 26 | indexMu sync.RWMutex 27 | indexLastAccess atomic.Int64 28 | } 29 | 30 | type indexer interface { 31 | GetNextOffset() (int64, error) 32 | Consume(offset int64) (int64, int64, int64, error) 33 | Get(offset int64) (int64, error) 34 | Keys(hash []byte) ([]int64, error) 35 | Time(ts int64) (int64, error) 36 | Len() int 37 | } 38 | 39 | func openReader(seg segment.Segment, params index.Params, head bool) *reader { 40 | return &reader{ 41 | segment: seg, 42 | params: params, 43 | head: head, 44 | } 45 | } 46 | 47 | func reopenReader(seg segment.Segment, params index.Params, ix indexer) *reader { 48 | return &reader{ 49 | segment: seg, 50 | params: params, 51 | head: false, 52 | 53 | index: ix, 54 | } 55 | } 56 | 57 | func openReaderAppend(seg segment.Segment, params index.Params, ix indexer) (*reader, error) { 58 | messages, err := message.OpenReader(seg.Log) 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return &reader{ 64 | segment: seg, 65 | params: params, 66 | head: true, 67 | 68 | messages: messages, 69 | index: ix, 70 | }, nil 71 | } 72 | 73 | func (r *reader) GetOffset() int64 { 74 | return r.segment.GetOffset() 75 | } 76 | 77 | func (r *reader) GetNextOffset() (int64, error) { 78 | index, err := r.getIndexNow() 79 | if err != nil { 80 | return 0, err 81 | } 82 | return index.GetNextOffset() 83 | } 84 | 85 | func (r *reader) Consume(offset, maxCount int64) (int64, []message.Message, error) { 86 | index, err := r.getIndexNow() 87 | if err != nil { 88 | return OffsetInvalid, nil, err 89 | } 90 | 91 | if offset == OffsetNewest { 92 | nextOffset, err := index.GetNextOffset() 93 | if err != nil { 94 | return OffsetInvalid, nil, err 95 | } 96 | return nextOffset, nil, nil 97 | } 98 | 99 | position, maxPosition, nextOffset, err := index.Consume(offset) 100 | switch { 101 | case err != nil: 102 | return OffsetInvalid, nil, err 103 | case position == -1: 104 | return nextOffset, nil, nil 105 | } 106 | 107 | messages, err := r.getMessages() 108 | if err != nil { 109 | return OffsetInvalid, nil, err 110 | } 111 | defer r.messagesInuse.Add(-1) 112 | 113 | msgs, err := messages.Consume(position, maxPosition, maxCount) 114 | if err != nil { 115 | return OffsetInvalid, nil, err 116 | } 117 | return msgs[len(msgs)-1].Offset + 1, msgs, nil 118 | } 119 | 120 | func (r *reader) ConsumeByKey(key, keyHash []byte, offset, maxCount int64) (int64, []message.Message, error) { 121 | ix, err := r.getIndexNow() 122 | if err != nil { 123 | return OffsetInvalid, nil, err 124 | } 125 | 126 | if offset == OffsetNewest { 127 | nextOffset, err := ix.GetNextOffset() 128 | if err != nil { 129 | return OffsetInvalid, nil, err 130 | } 131 | return nextOffset, nil, nil 132 | } 133 | 134 | positions, err := ix.Keys(keyHash) 135 | switch { 136 | case err == nil: 137 | break 138 | case err == index.ErrKeyNotFound: 139 | nextOffset, err := ix.GetNextOffset() 140 | if err != nil { 141 | return OffsetInvalid, nil, err 142 | } 143 | return nextOffset, nil, nil 144 | default: 145 | return OffsetInvalid, nil, err 146 | } 147 | 148 | messages, err := r.getMessages() 149 | if err != nil { 150 | return OffsetInvalid, nil, err 151 | } 152 | defer r.messagesInuse.Add(-1) 153 | 154 | var msgs []message.Message 155 | for i := 0; i < len(positions); i++ { 156 | msg, err := messages.Get(positions[i]) 157 | if err != nil { 158 | return OffsetInvalid, nil, err 159 | } 160 | if msg.Offset < offset { 161 | continue 162 | } 163 | if bytes.Equal(key, msg.Key) { 164 | msgs = append(msgs, msg) 165 | if len(msgs) >= int(maxCount) { 166 | break 167 | } 168 | } 169 | } 170 | 171 | if len(msgs) == 0 { 172 | nextOffset, err := ix.GetNextOffset() 173 | if err != nil { 174 | return OffsetInvalid, nil, err 175 | } 176 | return nextOffset, nil, nil 177 | } 178 | 179 | return msgs[len(msgs)-1].Offset + 1, msgs, nil 180 | } 181 | 182 | func (r *reader) Get(offset int64) (message.Message, error) { 183 | index, err := r.getIndexNow() 184 | if err != nil { 185 | return message.Invalid, err 186 | } 187 | 188 | position, err := index.Get(offset) 189 | if err != nil { 190 | return message.Invalid, err 191 | } 192 | 193 | messages, err := r.getMessages() 194 | if err != nil { 195 | return message.Invalid, err 196 | } 197 | defer r.messagesInuse.Add(-1) 198 | 199 | return messages.Get(position) 200 | } 201 | 202 | func (r *reader) GetByKey(key, keyHash []byte, tctx int64) (message.Message, error) { 203 | ix, err := r.getIndexAt(tctx) 204 | if err != nil { 205 | return message.Invalid, err 206 | } 207 | 208 | positions, err := ix.Keys(keyHash) 209 | if err != nil { 210 | return message.Invalid, err 211 | } 212 | 213 | messages, err := r.getMessages() 214 | if err != nil { 215 | return message.Invalid, err 216 | } 217 | defer r.messagesInuse.Add(-1) 218 | 219 | for i := len(positions) - 1; i >= 0; i-- { 220 | msg, err := messages.Get(positions[i]) 221 | if err != nil { 222 | return message.Invalid, err 223 | } 224 | if bytes.Equal(key, msg.Key) { 225 | return msg, nil 226 | } 227 | } 228 | 229 | return message.Invalid, index.ErrKeyNotFound 230 | } 231 | 232 | func (r *reader) GetByTime(ts int64, tctx int64) (message.Message, error) { 233 | index, err := r.getIndexAt(tctx) 234 | if err != nil { 235 | return message.Invalid, err 236 | } 237 | 238 | position, err := index.Time(ts) 239 | if err != nil { 240 | return message.Invalid, err 241 | } 242 | 243 | messages, err := r.getMessages() 244 | if err != nil { 245 | return message.Invalid, err 246 | } 247 | defer r.messagesInuse.Add(-1) 248 | 249 | return messages.Get(position) 250 | } 251 | 252 | func (r *reader) Stat() (segment.Stats, error) { 253 | return r.segment.Stat(r.params) 254 | } 255 | 256 | func (r *reader) Backup(dir string) error { 257 | return r.segment.Backup(dir) 258 | } 259 | 260 | func (r *reader) Delete(rs *segment.RewriteSegment) (*reader, error) { 261 | // log already has reader lock exclusively, no need to sync here 262 | if err := r.Close(); err != nil { 263 | return nil, err 264 | } 265 | 266 | if len(rs.SurviveOffsets) == 0 { 267 | // nothing left in reader, drop empty files 268 | if err := rs.Segment.Remove(); err != nil { 269 | return nil, err 270 | } 271 | 272 | return nil, r.segment.Remove() 273 | } 274 | 275 | nseg := rs.GetNewSegment() 276 | if nseg != r.segment { 277 | // the starting offset of the new segment is different 278 | 279 | // first move the replacement 280 | if err := rs.Segment.Rename(nseg); err != nil { 281 | return nil, err 282 | } 283 | 284 | // then delete this segment 285 | if err := r.segment.Remove(); err != nil { 286 | return nil, err 287 | } 288 | 289 | return &reader{segment: nseg, params: r.params}, nil 290 | } 291 | 292 | // the rewritten segment has the same starting offset 293 | if err := rs.Segment.Override(r.segment); err != nil { 294 | return nil, err 295 | } 296 | 297 | return r, nil 298 | } 299 | 300 | func (r *reader) getIndexNow() (indexer, error) { 301 | r.indexLastAccess.Store(time.Now().UnixMicro()) 302 | return r.getIndexMarked() 303 | } 304 | 305 | func (r *reader) getIndexAt(tctx int64) (indexer, error) { 306 | r.indexLastAccess.Store(tctx) 307 | return r.getIndexMarked() 308 | } 309 | 310 | func (r *reader) getIndexMarked() (indexer, error) { 311 | r.indexMu.RLock() 312 | if ix := r.index; ix != nil { 313 | defer r.indexMu.RUnlock() 314 | return ix, nil 315 | } 316 | r.indexMu.RUnlock() 317 | 318 | r.indexMu.Lock() 319 | defer r.indexMu.Unlock() 320 | 321 | if ix := r.index; ix != nil { 322 | return ix, nil 323 | } 324 | 325 | items, err := r.segment.ReindexAndReadIndex(r.params) 326 | if err != nil { 327 | return nil, err 328 | } 329 | 330 | r.index = newReaderIndex(items, r.params.Keys, r.segment.Offset, r.head) 331 | return r.index, nil 332 | } 333 | 334 | func (r *reader) getMessages() (*message.Reader, error) { 335 | r.messagesMu.RLock() 336 | if msgs := r.messages; msgs != nil { 337 | r.messagesInuse.Add(1) 338 | r.messagesMu.RUnlock() 339 | return msgs, nil 340 | } 341 | r.messagesMu.RUnlock() 342 | 343 | r.messagesMu.Lock() 344 | defer r.messagesMu.Unlock() 345 | 346 | if msgs := r.messages; msgs != nil { 347 | r.messagesInuse.Add(1) 348 | return msgs, nil 349 | } 350 | 351 | msgs, err := message.OpenReaderMem(r.segment.Log) 352 | if err != nil { 353 | return nil, err 354 | } 355 | 356 | r.messages = msgs 357 | r.messagesInuse.Add(1) 358 | return msgs, nil 359 | } 360 | 361 | func (r *reader) closeIndex() { 362 | r.indexMu.Lock() 363 | defer r.indexMu.Unlock() 364 | 365 | r.index = nil 366 | } 367 | 368 | func (r *reader) GC(unusedFor time.Duration) error { 369 | if r.head { 370 | // we never GC an actively writing segment 371 | return nil 372 | } 373 | 374 | indexLastAccess := time.UnixMicro(r.indexLastAccess.Load()) 375 | if time.Since(indexLastAccess) < unusedFor { 376 | // only unload segments unused for defined time 377 | return nil 378 | } 379 | 380 | r.closeIndex() 381 | 382 | r.messagesMu.Lock() 383 | defer r.messagesMu.Unlock() 384 | 385 | if r.messages == nil || r.messagesInuse.Load() > 0 { 386 | return nil 387 | } 388 | 389 | if err := r.messages.Close(); err != nil { 390 | return err 391 | } 392 | r.messages = nil 393 | return nil 394 | } 395 | 396 | func (r *reader) Close() error { 397 | r.closeIndex() 398 | 399 | r.messagesMu.Lock() 400 | defer r.messagesMu.Unlock() 401 | 402 | if r.messages == nil { 403 | return nil 404 | } 405 | 406 | if err := r.messages.Close(); err != nil { 407 | return err 408 | } 409 | r.messages = nil 410 | return nil 411 | } 412 | 413 | type readerIndex struct { 414 | items []index.Item 415 | keys art.Tree 416 | nextOffset int64 417 | head bool 418 | } 419 | 420 | func newReaderIndex(items []index.Item, hasKeys bool, offset int64, head bool) *readerIndex { 421 | var keys art.Tree 422 | if hasKeys { 423 | keys = art.New() 424 | index.AppendKeys(keys, items) 425 | } 426 | 427 | nextOffset := offset 428 | if len(items) > 0 { 429 | nextOffset = items[len(items)-1].Offset + 1 430 | } 431 | 432 | return &readerIndex{ 433 | items: items, 434 | keys: keys, 435 | nextOffset: nextOffset, 436 | head: head, 437 | } 438 | } 439 | 440 | func (ix *readerIndex) GetNextOffset() (int64, error) { 441 | return ix.nextOffset, nil 442 | } 443 | 444 | func (ix *readerIndex) Consume(offset int64) (int64, int64, int64, error) { 445 | position, maxPosition, err := index.Consume(ix.items, offset) 446 | if (err == index.ErrOffsetIndexEmpty || err == index.ErrOffsetAfterEnd) && ix.head && offset <= ix.nextOffset { 447 | return -1, -1, ix.nextOffset, nil 448 | } 449 | return position, maxPosition, offset, err 450 | } 451 | 452 | func (ix *readerIndex) Get(offset int64) (int64, error) { 453 | position, err := index.Get(ix.items, offset) 454 | if err == index.ErrOffsetAfterEnd && ix.head && offset >= ix.nextOffset { 455 | return -1, message.ErrInvalidOffset 456 | } 457 | return position, err 458 | } 459 | 460 | func (ix *readerIndex) Keys(keyHash []byte) ([]int64, error) { 461 | return index.Keys(ix.keys, keyHash) 462 | } 463 | 464 | func (ix *readerIndex) Time(ts int64) (int64, error) { 465 | return index.Time(ix.items, ts) 466 | } 467 | 468 | func (ix *readerIndex) Len() int { 469 | return len(ix.items) 470 | } 471 | -------------------------------------------------------------------------------- /segment/index.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/klev-dev/klevdb/message" 7 | ) 8 | 9 | type Offsetter interface { 10 | GetOffset() int64 11 | } 12 | 13 | func Consume[S ~[]O, O Offsetter](segments S, offset int64) (O, int) { 14 | switch offset { 15 | case message.OffsetOldest: 16 | return segments[0], 0 17 | case message.OffsetNewest: 18 | return segments[len(segments)-1], len(segments) - 1 19 | } 20 | 21 | beginIndex := 0 22 | beginSegment := segments[beginIndex] 23 | if offset <= beginSegment.GetOffset() { 24 | return beginSegment, beginIndex 25 | } 26 | 27 | endIndex := len(segments) - 1 28 | endSegment := segments[endIndex] 29 | if endSegment.GetOffset() <= offset { 30 | return endSegment, endIndex 31 | } 32 | 33 | for beginIndex < endIndex { 34 | midIndex := (beginIndex + endIndex) / 2 35 | midSegment := segments[midIndex] 36 | switch { 37 | case midSegment.GetOffset() < offset: 38 | beginIndex = midIndex + 1 39 | case midSegment.GetOffset() > offset: 40 | endIndex = midIndex - 1 41 | default: 42 | return midSegment, midIndex 43 | } 44 | } 45 | 46 | if segments[beginIndex].GetOffset() > offset { 47 | return segments[beginIndex-1], beginIndex - 1 48 | } 49 | return segments[beginIndex], beginIndex 50 | } 51 | 52 | var ErrOffsetRelative = fmt.Errorf("%w: get relative offset", message.ErrInvalidOffset) 53 | var ErrOffsetBeforeStart = fmt.Errorf("%w: offset before start", message.ErrNotFound) 54 | 55 | func Get[S ~[]O, O Offsetter](segments S, offset int64) (O, int, error) { 56 | switch offset { 57 | case message.OffsetOldest: 58 | return segments[0], 0, nil 59 | case message.OffsetNewest: 60 | return segments[len(segments)-1], len(segments) - 1, nil 61 | } 62 | 63 | beginIndex := 0 64 | beginSegment := segments[beginIndex] 65 | switch { 66 | case offset < beginSegment.GetOffset(): 67 | var v O 68 | if beginSegment.GetOffset() == 0 { 69 | return v, -1, ErrOffsetRelative 70 | } 71 | return v, -1, ErrOffsetBeforeStart 72 | case offset == beginSegment.GetOffset(): 73 | return beginSegment, 0, nil 74 | } 75 | 76 | endIndex := len(segments) - 1 77 | endSegment := segments[endIndex] 78 | if endSegment.GetOffset() <= offset { 79 | return endSegment, endIndex, nil 80 | } 81 | 82 | for beginIndex < endIndex { 83 | midIndex := (beginIndex + endIndex) / 2 84 | midSegment := segments[midIndex] 85 | switch { 86 | case midSegment.GetOffset() < offset: 87 | beginIndex = midIndex + 1 88 | case midSegment.GetOffset() > offset: 89 | endIndex = midIndex - 1 90 | default: 91 | return midSegment, midIndex, nil 92 | } 93 | } 94 | 95 | if segments[beginIndex].GetOffset() > offset { 96 | return segments[beginIndex-1], beginIndex - 1, nil 97 | } 98 | return segments[beginIndex], beginIndex, nil 99 | } 100 | -------------------------------------------------------------------------------- /segment/index_test.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func genSegments(offsets ...int64) []Segment { 11 | var segments []Segment 12 | for _, offset := range offsets { 13 | segments = append(segments, New("", offset)) 14 | } 15 | return segments 16 | } 17 | 18 | func TestConsumeSegment(t *testing.T) { 19 | var segments = genSegments(500, 1500, 2500, 3500, 4500, 5500) 20 | 21 | var tests = []struct { 22 | in int64 23 | out int64 24 | }{ 25 | // low bound 26 | {0, 500}, 27 | {250, 500}, 28 | {500, 500}, 29 | // high bound 30 | {5500, 5500}, 31 | {6000, 5500}, 32 | // middles 33 | {1000, 500}, 34 | {1500, 1500}, 35 | {2000, 1500}, 36 | {3000, 2500}, 37 | {4000, 3500}, 38 | {5000, 4500}, 39 | } 40 | 41 | for _, tc := range tests { 42 | t.Run(fmt.Sprintf("%4d|%4d", tc.in, tc.out), func(t *testing.T) { 43 | s, _ := Consume(segments, tc.in) 44 | require.Equal(t, tc.out, s.Offset) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /segment/segment.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | 10 | "golang.org/x/exp/maps" 11 | "golang.org/x/exp/slices" 12 | 13 | "github.com/klev-dev/klevdb/index" 14 | "github.com/klev-dev/klevdb/message" 15 | ) 16 | 17 | type Segment struct { 18 | Dir string 19 | Offset int64 20 | 21 | Log string 22 | Index string 23 | } 24 | 25 | func (s Segment) GetOffset() int64 { 26 | return s.Offset 27 | } 28 | 29 | func New(dir string, offset int64) Segment { 30 | return Segment{ 31 | Dir: dir, 32 | Offset: offset, 33 | 34 | Log: filepath.Join(dir, fmt.Sprintf("%020d.log", offset)), 35 | Index: filepath.Join(dir, fmt.Sprintf("%020d.index", offset)), 36 | } 37 | } 38 | 39 | type Stats struct { 40 | Segments int 41 | Messages int 42 | Size int64 43 | } 44 | 45 | func (s Segment) Stat(params index.Params) (Stats, error) { 46 | logStat, err := os.Stat(s.Log) 47 | if err != nil { 48 | return Stats{}, fmt.Errorf("stat log: %w", err) 49 | } 50 | 51 | indexStat, err := os.Stat(s.Index) 52 | if err != nil { 53 | return Stats{}, fmt.Errorf("stat index: %w", err) 54 | } 55 | 56 | return Stats{ 57 | Segments: 1, 58 | Messages: int(indexStat.Size() / params.Size()), 59 | Size: logStat.Size() + indexStat.Size(), 60 | }, nil 61 | } 62 | 63 | var errIndexSize = fmt.Errorf("%w: incorrect size", index.ErrCorrupted) 64 | var errIndexItem = fmt.Errorf("%w: incorrect item", index.ErrCorrupted) 65 | 66 | func (s Segment) Check(params index.Params) error { 67 | log, err := message.OpenReader(s.Log) 68 | if err != nil { 69 | return err 70 | } 71 | defer log.Close() 72 | 73 | var position, indexTime int64 74 | var logIndex []index.Item 75 | for { 76 | msg, nextPosition, err := log.Read(position) 77 | if errors.Is(err, io.EOF) { 78 | break 79 | } else if err != nil { 80 | return err 81 | } 82 | 83 | item := params.NewItem(msg, position, indexTime) 84 | logIndex = append(logIndex, item) 85 | 86 | position = nextPosition 87 | indexTime = item.Timestamp 88 | } 89 | 90 | switch items, err := index.Read(s.Index, params); { 91 | case errors.Is(err, os.ErrNotExist): 92 | return nil 93 | case err != nil: 94 | return err 95 | case len(logIndex) != len(items): 96 | return errIndexSize 97 | default: 98 | for i, item := range logIndex { 99 | if item != items[i] { 100 | return errIndexItem 101 | } 102 | } 103 | } 104 | 105 | return nil 106 | } 107 | 108 | func (s Segment) Recover(params index.Params) error { 109 | log, err := message.OpenReader(s.Log) 110 | if err != nil { 111 | return err 112 | } 113 | defer log.Close() 114 | 115 | restore, err := message.OpenWriter(s.Log + ".recover") 116 | if err != nil { 117 | return err 118 | } 119 | defer restore.Close() 120 | 121 | var position, indexTime int64 122 | var corrupted = false 123 | var logIndex []index.Item 124 | for { 125 | msg, nextPosition, err := log.Read(position) 126 | if errors.Is(err, io.EOF) { 127 | break 128 | } else if errors.Is(err, message.ErrCorrupted) { 129 | corrupted = true 130 | break 131 | } else if err != nil { 132 | return err 133 | } 134 | 135 | if _, err := restore.Write(msg); err != nil { 136 | return err 137 | } 138 | 139 | item := params.NewItem(msg, position, indexTime) 140 | logIndex = append(logIndex, item) 141 | 142 | position = nextPosition 143 | indexTime = item.Timestamp 144 | } 145 | 146 | if err := log.Close(); err != nil { 147 | return err 148 | } 149 | if err := restore.SyncAndClose(); err != nil { 150 | return err 151 | } 152 | 153 | if corrupted { 154 | if err := os.Rename(restore.Path, log.Path); err != nil { 155 | return fmt.Errorf("restore log rename: %w", err) 156 | } 157 | } else { 158 | if err := os.Remove(restore.Path); err != nil { 159 | return fmt.Errorf("restore log delete: %w", err) 160 | } 161 | } 162 | 163 | var corruptedIndex = false 164 | switch items, err := index.Read(s.Index, params); { 165 | case errors.Is(err, os.ErrNotExist): 166 | return nil 167 | case errors.Is(err, index.ErrCorrupted): 168 | corruptedIndex = true 169 | case err != nil: 170 | return err 171 | case len(logIndex) != len(items): 172 | corruptedIndex = true 173 | default: 174 | for i, item := range logIndex { 175 | if item != items[i] { 176 | corruptedIndex = true 177 | break 178 | } 179 | } 180 | } 181 | 182 | if corruptedIndex { 183 | if err := os.Remove(s.Index); err != nil { 184 | return fmt.Errorf("restore index delete: %w", err) 185 | } 186 | } 187 | 188 | return nil 189 | } 190 | 191 | func (s Segment) NeedsReindex() (bool, error) { 192 | switch info, err := os.Stat(s.Index); { 193 | case os.IsNotExist(err): 194 | return true, nil 195 | case err != nil: 196 | return false, fmt.Errorf("needs reindex stat: %w", err) 197 | case info.Size() == 0: 198 | return true, nil 199 | default: 200 | return false, nil 201 | } 202 | } 203 | 204 | func (s Segment) ReindexAndReadIndex(params index.Params) ([]index.Item, error) { 205 | switch reindex, err := s.NeedsReindex(); { 206 | case err != nil: 207 | return nil, err 208 | case reindex: 209 | return s.Reindex(params) 210 | default: 211 | return index.Read(s.Index, params) 212 | } 213 | } 214 | 215 | func (s Segment) Reindex(params index.Params) ([]index.Item, error) { 216 | log, err := message.OpenReader(s.Log) 217 | if err != nil { 218 | return nil, err 219 | } 220 | defer log.Close() 221 | 222 | return s.ReindexReader(params, log) 223 | } 224 | 225 | func (s Segment) ReindexReader(params index.Params, log *message.Reader) ([]index.Item, error) { 226 | var position, indexTime int64 227 | var items []index.Item 228 | for { 229 | msg, nextPosition, err := log.Read(position) 230 | if errors.Is(err, io.EOF) { 231 | break 232 | } else if err != nil { 233 | return nil, err 234 | } 235 | 236 | item := params.NewItem(msg, position, indexTime) 237 | items = append(items, item) 238 | 239 | position = nextPosition 240 | indexTime = item.Timestamp 241 | } 242 | 243 | if err := index.Write(s.Index, params, items); err != nil { 244 | return nil, err 245 | } 246 | return items, nil 247 | } 248 | 249 | func (s Segment) Backup(targetDir string) error { 250 | logName, err := filepath.Rel(s.Dir, s.Log) 251 | if err != nil { 252 | return fmt.Errorf("backup log rel: %w", err) 253 | } 254 | targetLog := filepath.Join(targetDir, logName) 255 | if err := copyFile(s.Log, targetLog); err != nil { 256 | return fmt.Errorf("backup log copy: %w", err) 257 | } 258 | 259 | indexName, err := filepath.Rel(s.Dir, s.Index) 260 | if err != nil { 261 | return fmt.Errorf("backup index rel: %w", err) 262 | } 263 | targetIndex := filepath.Join(targetDir, indexName) 264 | if err := copyFile(s.Index, targetIndex); err != nil { 265 | return fmt.Errorf("backup index copy: %w", err) 266 | } 267 | 268 | return nil 269 | } 270 | 271 | func (s Segment) ForRewrite() (Segment, error) { 272 | randStr, err := randStr(8) 273 | if err != nil { 274 | return Segment{}, err 275 | } 276 | 277 | s.Log = fmt.Sprintf("%s.rewrite.%s", s.Log, randStr) 278 | s.Index = fmt.Sprintf("%s.rewrite.%s", s.Index, randStr) 279 | return s, nil 280 | } 281 | 282 | func (olds Segment) Rename(news Segment) error { 283 | if err := os.Rename(olds.Log, news.Log); err != nil { 284 | return fmt.Errorf("rename log rename: %w", err) 285 | } 286 | 287 | if err := os.Rename(olds.Index, news.Index); err != nil { 288 | return fmt.Errorf("rename index rename: %w", err) 289 | } 290 | 291 | return nil 292 | } 293 | 294 | func (olds Segment) Override(news Segment) error { 295 | // remove index segment so we don't have invalid index 296 | if err := os.Remove(news.Index); err != nil { 297 | return fmt.Errorf("override index delete: %w", err) 298 | } 299 | 300 | if err := os.Rename(olds.Log, news.Log); err != nil { 301 | return fmt.Errorf("override log rename: %w", err) 302 | } 303 | 304 | if err := os.Rename(olds.Index, news.Index); err != nil { 305 | return fmt.Errorf("override index rename: %w", err) 306 | } 307 | 308 | return nil 309 | } 310 | 311 | func (s Segment) Remove() error { 312 | if err := os.Remove(s.Index); err != nil { 313 | return fmt.Errorf("remove index delete: %w", err) 314 | } 315 | if err := os.Remove(s.Log); err != nil { 316 | return fmt.Errorf("remove log delete: %w", err) 317 | } 318 | return nil 319 | } 320 | 321 | type RewriteSegment struct { 322 | Segment Segment 323 | Stats Stats 324 | 325 | SurviveOffsets map[int64]struct{} 326 | DeletedOffsets map[int64]struct{} 327 | DeletedSize int64 328 | } 329 | 330 | func (r *RewriteSegment) GetNewSegment() Segment { 331 | orderedOffsets := maps.Keys(r.SurviveOffsets) 332 | slices.Sort(orderedOffsets) 333 | lowestOffset := orderedOffsets[0] 334 | return New(r.Segment.Dir, lowestOffset) 335 | } 336 | 337 | func (src Segment) Rewrite(dropOffsets map[int64]struct{}, params index.Params) (*RewriteSegment, error) { 338 | dst, err := src.ForRewrite() 339 | if err != nil { 340 | return nil, err 341 | } 342 | 343 | result := &RewriteSegment{Segment: dst} 344 | 345 | srcLog, err := message.OpenReader(src.Log) 346 | if err != nil { 347 | return nil, err 348 | } 349 | defer srcLog.Close() 350 | 351 | dstLog, err := message.OpenWriter(dst.Log) 352 | if err != nil { 353 | return nil, err 354 | } 355 | defer dstLog.Close() 356 | 357 | result.SurviveOffsets = map[int64]struct{}{} 358 | result.DeletedOffsets = map[int64]struct{}{} 359 | 360 | var srcPosition, indexTime int64 361 | var dstItems []index.Item 362 | for { 363 | msg, nextSrcPosition, err := srcLog.Read(srcPosition) 364 | if err != nil { 365 | if errors.Is(err, io.EOF) { 366 | break 367 | } 368 | return nil, err 369 | } 370 | 371 | if _, ok := dropOffsets[msg.Offset]; ok { 372 | result.DeletedOffsets[msg.Offset] = struct{}{} 373 | result.DeletedSize += message.Size(msg) + params.Size() 374 | } else { 375 | dstPosition, err := dstLog.Write(msg) 376 | if err != nil { 377 | return nil, err 378 | } 379 | result.SurviveOffsets[msg.Offset] = struct{}{} 380 | 381 | item := params.NewItem(msg, dstPosition, indexTime) 382 | dstItems = append(dstItems, item) 383 | indexTime = item.Timestamp 384 | } 385 | 386 | srcPosition = nextSrcPosition 387 | } 388 | 389 | if err := srcLog.Close(); err != nil { 390 | return nil, err 391 | } 392 | if err := dstLog.SyncAndClose(); err != nil { 393 | return nil, err 394 | } 395 | if err := index.Write(dst.Index, params, dstItems); err != nil { 396 | return nil, err 397 | } 398 | 399 | result.Stats, err = dst.Stat(params) 400 | if err != nil { 401 | return nil, err 402 | } 403 | return result, nil 404 | } 405 | -------------------------------------------------------------------------------- /segment/segment_test.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "hash/fnv" 5 | "os" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stretchr/testify/require" 10 | 11 | "github.com/klev-dev/klevdb/index" 12 | "github.com/klev-dev/klevdb/message" 13 | ) 14 | 15 | func clearLastByte(fn string) error { 16 | f, err := os.OpenFile(fn, os.O_RDWR, 0600) 17 | if err != nil { 18 | return err 19 | } 20 | 21 | _, err = f.Seek(-1, 2) 22 | if err != nil { 23 | return err 24 | } 25 | 26 | _, err = f.Write([]byte{0}) 27 | if err != nil { 28 | return err 29 | } 30 | 31 | return f.Close() 32 | } 33 | 34 | func TestRecover(t *testing.T) { 35 | params := index.Params{Times: true, Keys: true} 36 | msgs := []message.Message{ 37 | { 38 | Offset: 0, 39 | Time: time.Date(2022, 04, 03, 14, 58, 0, 0, time.UTC), 40 | Key: []byte("key"), 41 | Value: []byte("value"), 42 | }, 43 | { 44 | Offset: 1, 45 | Time: time.Date(2022, 04, 03, 15, 58, 0, 0, time.UTC), 46 | Key: []byte("key1"), 47 | Value: []byte("value"), 48 | }, 49 | } 50 | 51 | var tests = []struct { 52 | name string 53 | in []message.Message 54 | corrupt func(s Segment) error 55 | out []message.Message 56 | }{ 57 | { 58 | "Ok", 59 | msgs, 60 | func(s Segment) error { return nil }, 61 | msgs, 62 | }, 63 | { 64 | "MessageMissing", 65 | msgs, 66 | func(s Segment) error { 67 | return os.Truncate(s.Log, message.Size(msgs[0])) 68 | }, 69 | msgs[0:1], 70 | }, 71 | { 72 | "MessageShortHeader", 73 | msgs, 74 | func(s Segment) error { 75 | return os.Truncate(s.Log, message.Size(msgs[0])+4) 76 | }, 77 | msgs[0:1], 78 | }, 79 | { 80 | "MessageShortData", 81 | msgs, 82 | func(s Segment) error { 83 | return os.Truncate(s.Log, message.Size(msgs[0])+params.Size()+4) 84 | }, 85 | msgs[0:1], 86 | }, 87 | { 88 | "MessageCRC", 89 | msgs, 90 | func(s Segment) error { 91 | return clearLastByte(s.Log) 92 | }, 93 | msgs[0:1], 94 | }, 95 | { 96 | "IndexMissing", 97 | msgs, 98 | func(s Segment) error { 99 | return os.Remove(s.Index) 100 | }, 101 | msgs, 102 | }, 103 | { 104 | "IndexItemMissing", 105 | msgs, 106 | func(s Segment) error { 107 | return os.Truncate(s.Index, params.Size()) 108 | }, 109 | msgs, 110 | }, 111 | { 112 | "IndexItemPartial", 113 | msgs, 114 | func(s Segment) error { 115 | return os.Truncate(s.Index, params.Size()+4) 116 | }, 117 | msgs, 118 | }, 119 | { 120 | "IndexItemWrong", 121 | msgs, 122 | func(s Segment) error { 123 | return clearLastByte(s.Index) 124 | }, 125 | msgs, 126 | }, 127 | } 128 | 129 | for _, test := range tests { 130 | t.Run(test.name, func(t *testing.T) { 131 | seg := New(t.TempDir(), 0) 132 | writeMessages(t, seg, params, test.in) 133 | 134 | require.NoError(t, test.corrupt(seg)) 135 | 136 | require.NoError(t, seg.Recover(params)) 137 | 138 | assertMessages(t, seg, params, test.out) 139 | }) 140 | } 141 | } 142 | 143 | func TestBackup(t *testing.T) { 144 | params := index.Params{Times: true, Keys: true} 145 | msgs := []message.Message{ 146 | { 147 | Offset: 0, 148 | Time: time.Date(2022, 04, 03, 14, 58, 0, 0, time.UTC), 149 | Key: []byte("key"), 150 | Value: []byte("value"), 151 | }, 152 | { 153 | Offset: 1, 154 | Time: time.Date(2022, 04, 03, 15, 58, 0, 0, time.UTC), 155 | Key: []byte("key1"), 156 | Value: []byte("value"), 157 | }, 158 | } 159 | 160 | var tests = []struct { 161 | name string 162 | in []message.Message 163 | backup func(t *testing.T, s Segment, dir string) error 164 | out []message.Message 165 | }{ 166 | { 167 | name: "Simple", 168 | in: msgs, 169 | backup: func(t *testing.T, s Segment, dir string) error { 170 | return s.Backup(dir) 171 | }, 172 | out: msgs, 173 | }, 174 | { 175 | name: "Repeated", 176 | in: msgs, 177 | backup: func(t *testing.T, s Segment, dir string) error { 178 | if err := s.Backup(dir); err != nil { 179 | return err 180 | } 181 | return s.Backup(dir) 182 | }, 183 | out: msgs, 184 | }, 185 | { 186 | name: "Incremental", 187 | in: msgs[0:1], 188 | backup: func(t *testing.T, s Segment, dir string) error { 189 | if err := s.Backup(dir); err != nil { 190 | return err 191 | } 192 | 193 | writeMessages(t, s, params, msgs[1:]) 194 | return s.Backup(dir) 195 | }, 196 | out: msgs, 197 | }, 198 | } 199 | 200 | for _, test := range tests { 201 | t.Run(test.name, func(t *testing.T) { 202 | seg := New(t.TempDir(), 0) 203 | writeMessages(t, seg, params, test.in) 204 | 205 | ndir := t.TempDir() 206 | require.NoError(t, test.backup(t, seg, ndir)) 207 | 208 | nseg := New(ndir, 0) 209 | assertMessages(t, nseg, params, test.out) 210 | }) 211 | } 212 | } 213 | 214 | func writeMessages(t *testing.T, seg Segment, params index.Params, msgs []message.Message) { 215 | lw, err := message.OpenWriter(seg.Log) 216 | require.NoError(t, err) 217 | iw, err := index.OpenWriter(seg.Index, params) 218 | require.NoError(t, err) 219 | 220 | var indexTime int64 221 | for _, msg := range msgs { 222 | pos, err := lw.Write(msg) 223 | require.NoError(t, err) 224 | item := params.NewItem(msg, pos, indexTime) 225 | err = iw.Write(item) 226 | require.NoError(t, err) 227 | indexTime = item.Timestamp 228 | } 229 | 230 | require.NoError(t, iw.Close()) 231 | require.NoError(t, lw.Close()) 232 | } 233 | 234 | func assertMessages(t *testing.T, seg Segment, params index.Params, expMsgs []message.Message) { 235 | index, err := seg.ReindexAndReadIndex(params) 236 | require.NoError(t, err) 237 | require.Len(t, index, len(expMsgs)) 238 | 239 | lr, err := message.OpenReader(seg.Log) 240 | require.NoError(t, err) 241 | 242 | for i, expMsg := range expMsgs { 243 | actIndex := index[i] 244 | 245 | require.Equal(t, expMsg.Offset, actIndex.Offset) 246 | require.Equal(t, expMsg.Time.UnixMicro(), actIndex.Timestamp) 247 | 248 | hasher := fnv.New64a() 249 | hasher.Write(expMsg.Key) 250 | require.Equal(t, hasher.Sum64(), actIndex.KeyHash) 251 | 252 | actMsg, err := lr.Get(actIndex.Position) 253 | require.NoError(t, err) 254 | 255 | require.Equal(t, expMsg.Offset, actMsg.Offset) 256 | require.Equal(t, expMsg.Time, actMsg.Time) 257 | require.Equal(t, expMsg.Key, actMsg.Key) 258 | require.Equal(t, expMsg.Value, actMsg.Value) 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /segment/segments.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/klev-dev/klevdb/index" 11 | ) 12 | 13 | func Find(dir string) ([]Segment, error) { 14 | files, err := os.ReadDir(dir) 15 | if err != nil { 16 | return nil, fmt.Errorf("find read dir: %w", err) 17 | } 18 | 19 | var segments []Segment 20 | for _, f := range files { 21 | if strings.HasSuffix(f.Name(), ".log") { 22 | offsetStr := strings.TrimSuffix(f.Name(), ".log") 23 | 24 | offset, err := strconv.ParseInt(offsetStr, 10, 64) 25 | if err != nil { 26 | return nil, fmt.Errorf("find parse offset: %w", err) 27 | } 28 | 29 | segments = append(segments, New(dir, offset)) 30 | } 31 | } 32 | 33 | return segments, nil 34 | } 35 | 36 | func StatDir(dir string, params index.Params) (Stats, error) { 37 | segments, err := Find(dir) 38 | switch { 39 | case errors.Is(err, os.ErrNotExist): 40 | return Stats{}, nil 41 | case err != nil: 42 | return Stats{}, err 43 | } 44 | return Stat(segments, params) 45 | } 46 | 47 | func Stat(segments []Segment, params index.Params) (Stats, error) { 48 | var total = Stats{} 49 | for _, seg := range segments { 50 | segStat, err := seg.Stat(params) 51 | if err != nil { 52 | return Stats{}, fmt.Errorf("stat %d: %w", seg.Offset, err) 53 | } 54 | 55 | total.Segments += segStat.Segments 56 | total.Messages += segStat.Messages 57 | total.Size += segStat.Size 58 | } 59 | return total, nil 60 | } 61 | 62 | func CheckDir(dir string, params index.Params) error { 63 | switch segments, err := Find(dir); { 64 | case errors.Is(err, os.ErrNotExist): 65 | return nil 66 | case err != nil: 67 | return err 68 | case len(segments) == 0: 69 | return nil 70 | default: 71 | seg := segments[len(segments)-1] 72 | if err := seg.Check(params); err != nil { 73 | return fmt.Errorf("check %d: %w", seg.Offset, err) 74 | } 75 | return nil 76 | } 77 | } 78 | 79 | func RecoverDir(dir string, params index.Params) error { 80 | switch segments, err := Find(dir); { 81 | case errors.Is(err, os.ErrNotExist): 82 | return nil 83 | case err != nil: 84 | return err 85 | case len(segments) == 0: 86 | return nil 87 | default: 88 | seg := segments[len(segments)-1] 89 | if err := seg.Recover(params); err != nil { 90 | return fmt.Errorf("recover %d: %w", seg.Offset, err) 91 | } 92 | return nil 93 | } 94 | } 95 | 96 | func BackupDir(dir, target string) error { 97 | switch segments, err := Find(dir); { 98 | case errors.Is(err, os.ErrNotExist): 99 | return nil 100 | case err != nil: 101 | return err 102 | default: 103 | if err := os.MkdirAll(target, 0700); err != nil { 104 | return fmt.Errorf("backup dir create: %w", err) 105 | } 106 | 107 | return Backup(segments, target) 108 | } 109 | } 110 | 111 | func Backup(segments []Segment, dir string) error { 112 | for _, seg := range segments { 113 | if err := seg.Backup(dir); err != nil { 114 | return fmt.Errorf("backup %d: %w", seg.Offset, err) 115 | } 116 | } 117 | 118 | return nil 119 | } 120 | -------------------------------------------------------------------------------- /segment/segments_test.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "path/filepath" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | 9 | "github.com/klev-dev/klevdb/index" 10 | ) 11 | 12 | func TestRecoverDir(t *testing.T) { 13 | t.Run("Missing", func(t *testing.T) { 14 | dir := t.TempDir() 15 | missing := filepath.Join(dir, "abc") 16 | require.NoError(t, RecoverDir(missing, index.Params{})) 17 | }) 18 | 19 | t.Run("Empty", func(t *testing.T) { 20 | dir := t.TempDir() 21 | require.NoError(t, RecoverDir(dir, index.Params{})) 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /segment/utils.go: -------------------------------------------------------------------------------- 1 | package segment 2 | 3 | import ( 4 | "crypto/rand" 5 | "fmt" 6 | "io" 7 | "os" 8 | 9 | "github.com/mr-tron/base58" 10 | ) 11 | 12 | func randStr(length int) (string, error) { 13 | k := make([]byte, length) 14 | if _, err := io.ReadFull(rand.Reader, k); err != nil { 15 | return "", fmt.Errorf("rand read: %w", err) 16 | } 17 | return base58.Encode(k), nil 18 | } 19 | 20 | func copyFile(src, dst string) error { 21 | fsrc, err := os.Open(src) 22 | if err != nil { 23 | return fmt.Errorf("copy src open: %w", err) 24 | } 25 | defer fsrc.Close() 26 | 27 | stat, err := fsrc.Stat() 28 | if err != nil { 29 | return fmt.Errorf("copy src stat: %w", err) 30 | } 31 | 32 | fdst, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600) 33 | if os.IsExist(err) { 34 | switch dstStat, err := os.Stat(dst); { 35 | case err != nil: 36 | return fmt.Errorf("copy dst stat: %w", err) 37 | case stat.Size() == dstStat.Size() && stat.ModTime().Equal(dstStat.ModTime()): 38 | // TODO do we need a safer version of this? 39 | return nil 40 | } 41 | fdst, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) 42 | } 43 | if err != nil { 44 | return fmt.Errorf("copy dst open: %w", err) 45 | } 46 | defer fdst.Close() 47 | 48 | switch n, err := io.Copy(fdst, fsrc); { 49 | case err != nil: 50 | return fmt.Errorf("copy: %w", err) 51 | case n < stat.Size(): 52 | return fmt.Errorf("partial copy (%d/%d)", n, stat.Size()) 53 | } 54 | 55 | if err := fdst.Sync(); err != nil { 56 | return fmt.Errorf("copy dst sync: %w", err) 57 | } 58 | if err := fdst.Close(); err != nil { 59 | return fmt.Errorf("copy dst close: %w", err) 60 | } 61 | if err := os.Chtimes(dst, stat.ModTime(), stat.ModTime()); err != nil { 62 | return fmt.Errorf("copy dst chtimes: %w", err) 63 | } 64 | 65 | return nil 66 | } 67 | -------------------------------------------------------------------------------- /trim/age.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "time" 7 | 8 | "github.com/klev-dev/klevdb" 9 | ) 10 | 11 | // FindByAge returns a set of offsets for messages that are 12 | // at the start of the log and before given time. 13 | func FindByAge(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, error) { 14 | maxOffset, _, err := l.OffsetByTime(before) 15 | switch { 16 | case err == nil: 17 | // we've found the max offset, start collecting offsets to delete 18 | break 19 | case errors.Is(err, klevdb.ErrNoIndex): 20 | // this log is not indexed by time, use the max as a bound 21 | maxOffset, err = l.NextOffset() 22 | if err != nil { 23 | return nil, err 24 | } 25 | case errors.Is(err, klevdb.ErrNotFound): 26 | // all messages are before, again use the max as a bound 27 | maxOffset, err = l.NextOffset() 28 | if err != nil { 29 | return nil, err 30 | } 31 | default: 32 | // something else went wrong 33 | if err != nil { 34 | return nil, err 35 | } 36 | } 37 | 38 | var offsets = map[int64]struct{}{} 39 | 40 | SEARCH: 41 | for offset := klevdb.OffsetOldest; offset < maxOffset; { 42 | nextOffset, msgs, err := l.Consume(offset, 32) 43 | if err != nil { 44 | return nil, err 45 | } 46 | offset = nextOffset 47 | 48 | for _, msg := range msgs { 49 | if msg.Time.After(before) { 50 | break SEARCH 51 | } 52 | 53 | offsets[msg.Offset] = struct{}{} 54 | } 55 | 56 | if err := ctx.Err(); err != nil { 57 | return nil, err 58 | } 59 | } 60 | 61 | if err := ctx.Err(); err != nil { 62 | return nil, err 63 | } 64 | 65 | return offsets, nil 66 | } 67 | 68 | // ByAge tries to remove the messages at the start of the log before given time. 69 | // 70 | // returns the offsets it deleted and the amount of storage freed 71 | func ByAge(ctx context.Context, l klevdb.Log, before time.Time) (map[int64]struct{}, int64, error) { 72 | offsets, err := FindByAge(ctx, l, before) 73 | if err != nil { 74 | return nil, 0, err 75 | } 76 | return l.Delete(offsets) 77 | } 78 | 79 | // ByAgeMulti is similar to ByAge, but will try to remove messages from multiple segments 80 | func ByAgeMulti(ctx context.Context, l klevdb.Log, before time.Time, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 81 | offsets, err := FindByAge(ctx, l, before) 82 | if err != nil { 83 | return nil, 0, err 84 | } 85 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 86 | } 87 | -------------------------------------------------------------------------------- /trim/age_test.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | "time" 7 | 8 | "github.com/klev-dev/klevdb" 9 | "github.com/klev-dev/klevdb/message" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestByAge(t *testing.T) { 14 | t.Run("Partial", testByAgePartial) 15 | t.Run("NoIndex", testByAgeNoIndex) 16 | t.Run("All", testByAgeAll) 17 | } 18 | 19 | func testByAgePartial(t *testing.T) { 20 | msgs := message.Gen(20) 21 | 22 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{TimeIndex: true}) 23 | require.NoError(t, err) 24 | defer l.Close() 25 | 26 | _, err = l.Publish(msgs) 27 | require.NoError(t, err) 28 | 29 | msg, err := l.Get(klevdb.OffsetOldest) 30 | require.NoError(t, err) 31 | require.Equal(t, int64(0), msg.Offset) 32 | 33 | trimTime := msgs[10].Time.Add(-time.Millisecond) 34 | _, trim, err := ByAge(context.TODO(), l, trimTime) 35 | require.NoError(t, err) 36 | require.Equal(t, l.Size(msgs[0])*10, trim) 37 | 38 | msg, err = l.Get(klevdb.OffsetOldest) 39 | require.NoError(t, err) 40 | require.Equal(t, int64(10), msg.Offset) 41 | } 42 | 43 | func testByAgeNoIndex(t *testing.T) { 44 | msgs := message.Gen(20) 45 | 46 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{}) 47 | require.NoError(t, err) 48 | defer l.Close() 49 | 50 | _, err = l.Publish(msgs) 51 | require.NoError(t, err) 52 | 53 | msg, err := l.Get(klevdb.OffsetOldest) 54 | require.NoError(t, err) 55 | require.Equal(t, int64(0), msg.Offset) 56 | 57 | trimTime := msgs[10].Time.Add(-time.Millisecond) 58 | _, trim, err := ByAge(context.TODO(), l, trimTime) 59 | require.NoError(t, err) 60 | require.Equal(t, l.Size(msgs[0])*10, trim) 61 | 62 | msg, err = l.Get(klevdb.OffsetOldest) 63 | require.NoError(t, err) 64 | require.Equal(t, int64(10), msg.Offset) 65 | } 66 | 67 | func testByAgeAll(t *testing.T) { 68 | msgs := message.Gen(20) 69 | 70 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{TimeIndex: true}) 71 | require.NoError(t, err) 72 | defer l.Close() 73 | 74 | _, err = l.Publish(msgs) 75 | require.NoError(t, err) 76 | 77 | trimTime := msgs[len(msgs)-1].Time.Add(time.Millisecond) 78 | off, sz, err := ByAge(context.TODO(), l, trimTime) 79 | require.NoError(t, err) 80 | require.Len(t, off, 20) 81 | require.Equal(t, l.Size(msgs[0])*20, sz) 82 | 83 | coff, cmsgs, err := l.Consume(klevdb.OffsetOldest, 32) 84 | require.NoError(t, err) 85 | require.Equal(t, int64(20), coff) 86 | require.Empty(t, cmsgs) 87 | } 88 | -------------------------------------------------------------------------------- /trim/count.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/klev-dev/klevdb" 7 | ) 8 | 9 | // FindByCount returns a set of offsets for messages that when 10 | // removed will keep number of the messages in the log less then max 11 | func FindByCount(ctx context.Context, l klevdb.Log, max int) (map[int64]struct{}, error) { 12 | stats, err := l.Stat() 13 | switch { 14 | case err != nil: 15 | return nil, err 16 | case stats.Messages <= max: 17 | return nil, nil 18 | } 19 | 20 | maxOffset, err := l.NextOffset() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | var offsets = map[int64]struct{}{} 26 | 27 | toRemove := stats.Messages - max 28 | for offset := klevdb.OffsetOldest; offset < maxOffset && toRemove > 0; { 29 | nextOffset, msgs, err := l.Consume(offset, 32) 30 | if err != nil { 31 | return nil, err 32 | } 33 | offset = nextOffset 34 | 35 | for _, msg := range msgs { 36 | offsets[msg.Offset] = struct{}{} 37 | toRemove-- 38 | 39 | if toRemove <= 0 { 40 | break 41 | } 42 | } 43 | 44 | if err := ctx.Err(); err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | if err := ctx.Err(); err != nil { 50 | return nil, err 51 | } 52 | 53 | return offsets, nil 54 | } 55 | 56 | // ByCount tries to remove messages to keep the number of messages 57 | // in the log under max count. 58 | // 59 | // returns the offsets it deleted and the amount of storage freed 60 | func ByCount(ctx context.Context, l klevdb.Log, max int) (map[int64]struct{}, int64, error) { 61 | offsets, err := FindByCount(ctx, l, max) 62 | if err != nil { 63 | return nil, 0, err 64 | } 65 | return l.Delete(offsets) 66 | } 67 | 68 | // ByCountMulti is similar to ByCount, but will try to remove messages from multiple segments 69 | func ByCountMulti(ctx context.Context, l klevdb.Log, max int, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 70 | offsets, err := FindByCount(ctx, l, max) 71 | if err != nil { 72 | return nil, 0, err 73 | } 74 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 75 | } 76 | -------------------------------------------------------------------------------- /trim/count_test.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/klev-dev/klevdb" 8 | "github.com/klev-dev/klevdb/message" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestByCount(t *testing.T) { 13 | msgs := message.Gen(20) 14 | 15 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{}) 16 | require.NoError(t, err) 17 | defer l.Close() 18 | 19 | _, err = l.Publish(msgs) 20 | require.NoError(t, err) 21 | 22 | stat, err := l.Stat() 23 | require.NoError(t, err) 24 | require.Equal(t, len(msgs), stat.Messages) 25 | 26 | msg, err := l.Get(klevdb.OffsetOldest) 27 | require.NoError(t, err) 28 | require.Equal(t, int64(0), msg.Offset) 29 | 30 | t.Run("None", func(t *testing.T) { 31 | off, sz, err := ByCount(context.TODO(), l, 21) 32 | require.Len(t, off, 0) 33 | require.NoError(t, err) 34 | require.Equal(t, int64(0), sz) 35 | 36 | stat, err = l.Stat() 37 | require.NoError(t, err) 38 | require.Equal(t, len(msgs), stat.Messages) 39 | 40 | msg, err = l.Get(klevdb.OffsetOldest) 41 | require.NoError(t, err) 42 | require.Equal(t, int64(0), msg.Offset) 43 | }) 44 | 45 | t.Run("Half", func(t *testing.T) { 46 | off, sz, err := ByCount(context.TODO(), l, 10) 47 | require.Len(t, off, 10) 48 | require.NoError(t, err) 49 | require.Equal(t, l.Size(msgs[0])*10, sz) 50 | 51 | stat, err = l.Stat() 52 | require.NoError(t, err) 53 | require.Equal(t, 10, stat.Messages) 54 | 55 | msg, err = l.Get(klevdb.OffsetOldest) 56 | require.NoError(t, err) 57 | require.Equal(t, int64(10), msg.Offset) 58 | }) 59 | } 60 | -------------------------------------------------------------------------------- /trim/offset.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/klev-dev/klevdb" 7 | "github.com/klev-dev/klevdb/message" 8 | ) 9 | 10 | // FindByOffset returns a set of offsets for messages that 11 | // offset is before a given offset 12 | func FindByOffset(ctx context.Context, l klevdb.Log, before int64) (map[int64]struct{}, error) { 13 | if before == message.OffsetOldest { 14 | return map[int64]struct{}{}, nil 15 | } 16 | 17 | maxOffset, err := l.NextOffset() 18 | if err != nil { 19 | return nil, err 20 | } 21 | if before == message.OffsetNewest { 22 | before = maxOffset 23 | } else if maxOffset > before { 24 | maxOffset = before 25 | } 26 | 27 | var offsets = map[int64]struct{}{} 28 | for offset := klevdb.OffsetOldest; offset < maxOffset; { 29 | nextOffset, msgs, err := l.Consume(offset, 32) 30 | if err != nil { 31 | return nil, err 32 | } 33 | offset = nextOffset 34 | 35 | for _, msg := range msgs { 36 | if msg.Offset >= before { 37 | break 38 | } 39 | offsets[msg.Offset] = struct{}{} 40 | } 41 | 42 | if err := ctx.Err(); err != nil { 43 | return nil, err 44 | } 45 | } 46 | 47 | if err := ctx.Err(); err != nil { 48 | return nil, err 49 | } 50 | 51 | return offsets, nil 52 | } 53 | 54 | // ByOffset tries to remove the messages at the start of the log before offset 55 | // 56 | // returns the offsets it deleted and the amount of storage freed 57 | func ByOffset(ctx context.Context, l klevdb.Log, before int64) (map[int64]struct{}, int64, error) { 58 | offsets, err := FindByOffset(ctx, l, before) 59 | if err != nil { 60 | return nil, 0, err 61 | } 62 | return l.Delete(offsets) 63 | } 64 | 65 | // ByOffsetMulti is similar to ByOffset, but will try to remove messages from multiple segments 66 | func ByOffsetMulti(ctx context.Context, l klevdb.Log, before int64, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 67 | offsets, err := FindByOffset(ctx, l, before) 68 | if err != nil { 69 | return nil, 0, err 70 | } 71 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 72 | } 73 | -------------------------------------------------------------------------------- /trim/offset_test.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/klev-dev/klevdb" 8 | "github.com/klev-dev/klevdb/message" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestByOffset(t *testing.T) { 13 | msgs := message.Gen(20) 14 | 15 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{}) 16 | require.NoError(t, err) 17 | defer l.Close() 18 | 19 | _, err = l.Publish(msgs) 20 | require.NoError(t, err) 21 | 22 | stat, err := l.Stat() 23 | require.NoError(t, err) 24 | require.Equal(t, len(msgs), stat.Messages) 25 | 26 | msg, err := l.Get(klevdb.OffsetOldest) 27 | require.NoError(t, err) 28 | require.Equal(t, int64(0), msg.Offset) 29 | 30 | t.Run("None", func(t *testing.T) { 31 | off, sz, err := ByOffset(context.TODO(), l, 0) 32 | require.Len(t, off, 0) 33 | require.NoError(t, err) 34 | require.Equal(t, int64(0), sz) 35 | 36 | stat, err = l.Stat() 37 | require.NoError(t, err) 38 | require.Equal(t, len(msgs), stat.Messages) 39 | 40 | msg, err = l.Get(klevdb.OffsetOldest) 41 | require.NoError(t, err) 42 | require.Equal(t, int64(0), msg.Offset) 43 | }) 44 | 45 | t.Run("Half", func(t *testing.T) { 46 | off, sz, err := ByOffset(context.TODO(), l, 10) 47 | require.Len(t, off, 10) 48 | require.NoError(t, err) 49 | require.Equal(t, l.Size(msgs[0])*10, sz) 50 | 51 | stat, err = l.Stat() 52 | require.NoError(t, err) 53 | require.Equal(t, 10, stat.Messages) 54 | 55 | msg, err = l.Get(klevdb.OffsetOldest) 56 | require.NoError(t, err) 57 | require.Equal(t, int64(10), msg.Offset) 58 | }) 59 | 60 | t.Run("All", func(t *testing.T) { 61 | off, sz, err := ByOffset(context.TODO(), l, 100) 62 | require.Len(t, off, 10) 63 | require.NoError(t, err) 64 | require.Equal(t, l.Size(msgs[0])*10, sz) 65 | 66 | stat, err = l.Stat() 67 | require.NoError(t, err) 68 | require.Equal(t, 0, stat.Messages) 69 | 70 | msg, err = l.Get(klevdb.OffsetOldest) 71 | require.ErrorIs(t, err, message.ErrInvalidOffset) 72 | }) 73 | } 74 | 75 | func TestByOffsetRelative(t *testing.T) { 76 | msgs := message.Gen(20) 77 | 78 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{}) 79 | require.NoError(t, err) 80 | defer l.Close() 81 | 82 | _, err = l.Publish(msgs) 83 | require.NoError(t, err) 84 | 85 | stat, err := l.Stat() 86 | require.NoError(t, err) 87 | require.Equal(t, len(msgs), stat.Messages) 88 | 89 | msg, err := l.Get(klevdb.OffsetOldest) 90 | require.NoError(t, err) 91 | require.Equal(t, int64(0), msg.Offset) 92 | 93 | t.Run("Oldest", func(t *testing.T) { 94 | off, sz, err := ByOffset(context.TODO(), l, message.OffsetOldest) 95 | require.Len(t, off, 0) 96 | require.NoError(t, err) 97 | require.Equal(t, int64(0), sz) 98 | 99 | stat, err = l.Stat() 100 | require.NoError(t, err) 101 | require.Equal(t, len(msgs), stat.Messages) 102 | 103 | msg, err = l.Get(klevdb.OffsetOldest) 104 | require.NoError(t, err) 105 | require.Equal(t, int64(0), msg.Offset) 106 | }) 107 | 108 | t.Run("Newest", func(t *testing.T) { 109 | off, sz, err := ByOffset(context.TODO(), l, message.OffsetNewest) 110 | require.Len(t, off, 20) 111 | require.NoError(t, err) 112 | require.Equal(t, l.Size(msgs[0])*20, sz) 113 | 114 | stat, err = l.Stat() 115 | require.NoError(t, err) 116 | require.Equal(t, 0, stat.Messages) 117 | 118 | msg, err = l.Get(klevdb.OffsetOldest) 119 | require.ErrorIs(t, err, message.ErrInvalidOffset) 120 | }) 121 | } 122 | -------------------------------------------------------------------------------- /trim/size.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/klev-dev/klevdb" 7 | ) 8 | 9 | // FindBySize returns a set of offsets for messages that 10 | // if deleted will decrease the log size to sz 11 | func FindBySize(ctx context.Context, l klevdb.Log, sz int64) (map[int64]struct{}, error) { 12 | stats, err := l.Stat() 13 | switch { 14 | case err != nil: 15 | return nil, err 16 | case stats.Size < sz: 17 | return nil, nil 18 | } 19 | 20 | maxOffset, err := l.NextOffset() 21 | if err != nil { 22 | return nil, err 23 | } 24 | 25 | var offsets = map[int64]struct{}{} 26 | 27 | total := stats.Size 28 | for offset := klevdb.OffsetOldest; offset < maxOffset && total >= sz; { 29 | nextOffset, msgs, err := l.Consume(offset, 32) 30 | if err != nil { 31 | return nil, err 32 | } 33 | offset = nextOffset 34 | 35 | for _, msg := range msgs { 36 | offsets[msg.Offset] = struct{}{} 37 | total -= l.Size(msg) 38 | 39 | if total < sz { 40 | break 41 | } 42 | } 43 | 44 | if err := ctx.Err(); err != nil { 45 | return nil, err 46 | } 47 | } 48 | 49 | if err := ctx.Err(); err != nil { 50 | return nil, err 51 | } 52 | 53 | return offsets, nil 54 | } 55 | 56 | // BySize tries to remove messages until log size is less then sz 57 | // 58 | // returns the offsets it deleted and the amount of storage freed 59 | func BySize(ctx context.Context, l klevdb.Log, sz int64) (map[int64]struct{}, int64, error) { 60 | offsets, err := FindBySize(ctx, l, sz) 61 | if err != nil { 62 | return nil, 0, err 63 | } 64 | return l.Delete(offsets) 65 | } 66 | 67 | // BySizeMulti is similar to BySize, but will try to remove messages from multiple segments 68 | func BySizeMulti(ctx context.Context, l klevdb.Log, sz int64, backoff klevdb.DeleteMultiBackoff) (map[int64]struct{}, int64, error) { 69 | offsets, err := FindBySize(ctx, l, sz) 70 | if err != nil { 71 | return nil, 0, err 72 | } 73 | return klevdb.DeleteMulti(ctx, l, offsets, backoff) 74 | } 75 | -------------------------------------------------------------------------------- /trim/size_test.go: -------------------------------------------------------------------------------- 1 | package trim 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/klev-dev/klevdb" 8 | "github.com/klev-dev/klevdb/message" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestBySize(t *testing.T) { 13 | msgs := message.Gen(20) 14 | 15 | l, err := klevdb.Open(t.TempDir(), klevdb.Options{}) 16 | require.NoError(t, err) 17 | defer l.Close() 18 | 19 | _, err = l.Publish(msgs) 20 | require.NoError(t, err) 21 | 22 | stat, err := l.Stat() 23 | require.NoError(t, err) 24 | require.Equal(t, l.Size(msgs[0])*20, stat.Size) 25 | 26 | msg, err := l.Get(klevdb.OffsetOldest) 27 | require.NoError(t, err) 28 | require.Equal(t, int64(0), msg.Offset) 29 | 30 | t.Run("None", func(t *testing.T) { 31 | off, sz, err := BySize(context.TODO(), l, l.Size(msgs[0])*21) 32 | require.Len(t, off, 0) 33 | require.NoError(t, err) 34 | require.Equal(t, int64(0), sz) 35 | 36 | stat, err = l.Stat() 37 | require.NoError(t, err) 38 | require.Equal(t, l.Size(msgs[0])*20, stat.Size) 39 | 40 | msg, err = l.Get(klevdb.OffsetOldest) 41 | require.NoError(t, err) 42 | require.Equal(t, int64(0), msg.Offset) 43 | }) 44 | 45 | t.Run("Half", func(t *testing.T) { 46 | toTrimSize := l.Size(msgs[0]) * 11 47 | off, sz, err := BySize(context.TODO(), l, toTrimSize) 48 | require.Len(t, off, 10) 49 | require.NoError(t, err) 50 | require.Equal(t, l.Size(msgs[0])*10, sz) 51 | 52 | stat, err = l.Stat() 53 | require.NoError(t, err) 54 | require.Equal(t, l.Size(msgs[0])*10, stat.Size) 55 | 56 | msg, err = l.Get(klevdb.OffsetOldest) 57 | require.NoError(t, err) 58 | require.Equal(t, int64(10), msg.Offset) 59 | }) 60 | } 61 | -------------------------------------------------------------------------------- /typed.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import "time" 4 | 5 | type TMessage[K any, V any] struct { 6 | Offset int64 7 | Time time.Time 8 | Key K 9 | KeyEmpty bool 10 | Value V 11 | ValueEmpty bool 12 | } 13 | 14 | // TLog is a typed log 15 | type TLog[K any, V any] interface { 16 | // Publish see Log.Publish 17 | Publish(messages []TMessage[K, V]) (nextOffset int64, err error) 18 | 19 | // NextOffset see Log.NextOffset 20 | NextOffset() (nextOffset int64, err error) 21 | 22 | // Consume see Log.Consume 23 | Consume(offset int64, maxCount int64) (nextOffset int64, messages []TMessage[K, V], err error) 24 | 25 | // ConsumeByKey see Log.ConsumeByKey 26 | ConsumeByKey(key K, empty bool, offset int64, maxCount int64) (nextOffset int64, messages []TMessage[K, V], err error) 27 | 28 | // Get see Log.Get 29 | Get(offset int64) (message TMessage[K, V], err error) 30 | 31 | // GetByKey see Log.GetByKey 32 | GetByKey(key K, empty bool) (message TMessage[K, V], err error) 33 | 34 | // GetByTime see Log.GetByTime 35 | GetByTime(start time.Time) (message TMessage[K, V], err error) 36 | 37 | // Delete see Log.Delete 38 | Delete(offsets map[int64]struct{}) (deletedOffsets map[int64]struct{}, deletedSize int64, err error) 39 | 40 | // Size see Log.Size 41 | Size(m Message) int64 42 | 43 | // Stat see Log.Stat 44 | Stat() (Stats, error) 45 | 46 | // Backup see Log.Backup 47 | Backup(dir string) error 48 | 49 | // Sync see Log.Sync 50 | Sync() (nextOffset int64, err error) 51 | 52 | // GC see Log.GC 53 | GC(unusedFor time.Duration) error 54 | 55 | // Close see Log.Close 56 | Close() error 57 | 58 | // Raw returns the wrapped in log 59 | Raw() Log 60 | } 61 | 62 | // OpenT opens a typed log with specified key/value codecs 63 | func OpenT[K any, V any](dir string, opts Options, keyCodec Codec[K], valueCodec Codec[V]) (TLog[K, V], error) { 64 | l, err := Open(dir, opts) 65 | if err != nil { 66 | return nil, err 67 | } 68 | return &tlog[K, V]{l, keyCodec, valueCodec}, nil 69 | } 70 | 71 | // WrapT wraps a log with specified key/value codecs 72 | func WrapT[K any, V any](l Log, keyCodec Codec[K], valueCodec Codec[V]) (TLog[K, V], error) { 73 | return &tlog[K, V]{l, keyCodec, valueCodec}, nil 74 | } 75 | 76 | type tlog[K any, V any] struct { 77 | Log 78 | 79 | keyCodec Codec[K] 80 | valueCodec Codec[V] 81 | } 82 | 83 | func (l *tlog[K, V]) Publish(tmessages []TMessage[K, V]) (int64, error) { 84 | var err error 85 | messages := make([]Message, len(tmessages)) 86 | for i, tmsg := range tmessages { 87 | messages[i], err = l.encode(tmsg) 88 | if err != nil { 89 | return OffsetInvalid, err 90 | } 91 | } 92 | 93 | return l.Log.Publish(messages) 94 | } 95 | 96 | func (l *tlog[K, V]) Consume(offset int64, maxCount int64) (int64, []TMessage[K, V], error) { 97 | nextOffset, messages, err := l.Log.Consume(offset, maxCount) 98 | if err != nil { 99 | return OffsetInvalid, nil, err 100 | } 101 | if len(messages) == 0 { 102 | return nextOffset, nil, nil 103 | } 104 | 105 | tmessages := make([]TMessage[K, V], len(messages)) 106 | for i, msg := range messages { 107 | tmessages[i], err = l.decode(msg) 108 | if err != nil { 109 | return OffsetInvalid, nil, err 110 | } 111 | } 112 | return nextOffset, tmessages, nil 113 | } 114 | 115 | func (l *tlog[K, V]) ConsumeByKey(key K, empty bool, offset int64, maxCount int64) (int64, []TMessage[K, V], error) { 116 | kbytes, err := l.keyCodec.Encode(key, empty) 117 | if err != nil { 118 | return OffsetInvalid, nil, err 119 | } 120 | 121 | nextOffset, messages, err := l.Log.ConsumeByKey(kbytes, offset, maxCount) 122 | if err != nil { 123 | return OffsetInvalid, nil, err 124 | } 125 | if len(messages) == 0 { 126 | return nextOffset, nil, nil 127 | } 128 | 129 | tmessages := make([]TMessage[K, V], len(messages)) 130 | for i, msg := range messages { 131 | tmessages[i], err = l.decode(msg) 132 | if err != nil { 133 | return OffsetInvalid, nil, err 134 | } 135 | } 136 | return nextOffset, tmessages, nil 137 | } 138 | 139 | func (l *tlog[K, V]) Get(offset int64) (TMessage[K, V], error) { 140 | msg, err := l.Log.Get(offset) 141 | if err != nil { 142 | return TMessage[K, V]{Offset: OffsetInvalid}, err 143 | } 144 | return l.decode(msg) 145 | } 146 | 147 | func (l *tlog[K, V]) GetByKey(key K, empty bool) (TMessage[K, V], error) { 148 | kbytes, err := l.keyCodec.Encode(key, empty) 149 | if err != nil { 150 | return TMessage[K, V]{Offset: OffsetInvalid}, err 151 | } 152 | msg, err := l.Log.GetByKey(kbytes) 153 | if err != nil { 154 | return TMessage[K, V]{Offset: OffsetInvalid}, err 155 | } 156 | return l.decode(msg) 157 | } 158 | 159 | func (l *tlog[K, V]) GetByTime(start time.Time) (TMessage[K, V], error) { 160 | msg, err := l.Log.GetByTime(start) 161 | if err != nil { 162 | return TMessage[K, V]{Offset: OffsetInvalid}, err 163 | } 164 | return l.decode(msg) 165 | } 166 | 167 | func (l *tlog[K, V]) Raw() Log { 168 | return l.Log 169 | } 170 | 171 | func (l *tlog[K, V]) encode(tmsg TMessage[K, V]) (msg Message, err error) { 172 | msg.Offset = tmsg.Offset 173 | msg.Time = tmsg.Time 174 | 175 | msg.Key, err = l.keyCodec.Encode(tmsg.Key, tmsg.KeyEmpty) 176 | if err != nil { 177 | return InvalidMessage, err 178 | } 179 | 180 | msg.Value, err = l.valueCodec.Encode(tmsg.Value, tmsg.ValueEmpty) 181 | if err != nil { 182 | return InvalidMessage, err 183 | } 184 | 185 | return msg, nil 186 | } 187 | 188 | func (l *tlog[K, V]) decode(msg Message) (tmsg TMessage[K, V], err error) { 189 | tmsg.Offset = msg.Offset 190 | tmsg.Time = msg.Time 191 | 192 | tmsg.Key, tmsg.KeyEmpty, err = l.keyCodec.Decode(msg.Key) 193 | if err != nil { 194 | return TMessage[K, V]{Offset: OffsetInvalid}, err 195 | } 196 | 197 | tmsg.Value, tmsg.ValueEmpty, err = l.valueCodec.Decode(msg.Value) 198 | if err != nil { 199 | return TMessage[K, V]{Offset: OffsetInvalid}, err 200 | } 201 | 202 | return tmsg, nil 203 | } 204 | -------------------------------------------------------------------------------- /typed_blocking.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import "context" 4 | 5 | // TBlockingLog enhances tlog adding blocking consume 6 | type TBlockingLog[K any, V any] interface { 7 | TLog[K, V] 8 | 9 | // ConsumeBlocking see BlockingLog.ConsumeBlocking 10 | ConsumeBlocking(ctx context.Context, offset int64, maxCount int64) (nextOffset int64, messages []TMessage[K, V], err error) 11 | 12 | // ConsumeByKeyBlocking see BlockingLog.ConsumeByKeyBlocking 13 | ConsumeByKeyBlocking(ctx context.Context, key K, empty bool, offset int64, maxCount int64) (nextOffset int64, messages []TMessage[K, V], err error) 14 | } 15 | 16 | // OpenBlocking opens tlog and wraps it with support for blocking consume 17 | func OpenTBlocking[K any, V any](dir string, opts Options, keyCodec Codec[K], valueCodec Codec[V]) (TBlockingLog[K, V], error) { 18 | l, err := OpenT(dir, opts, keyCodec, valueCodec) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return WrapTBlocking(l) 23 | } 24 | 25 | // WrapBlocking wraps tlog with support for blocking consume 26 | func WrapTBlocking[K any, V any](l TLog[K, V]) (TBlockingLog[K, V], error) { 27 | next, err := l.NextOffset() 28 | if err != nil { 29 | return nil, err 30 | } 31 | return &tlogBlocking[K, V]{l, NewOffsetNotify(next)}, nil 32 | } 33 | 34 | type tlogBlocking[K any, V any] struct { 35 | TLog[K, V] 36 | notify *OffsetNotify 37 | } 38 | 39 | func (l *tlogBlocking[K, V]) Publish(tmessages []TMessage[K, V]) (int64, error) { 40 | nextOffset, err := l.TLog.Publish(tmessages) 41 | if err != nil { 42 | return OffsetInvalid, err 43 | } 44 | 45 | l.notify.Set(nextOffset) 46 | return nextOffset, err 47 | } 48 | 49 | func (l *tlogBlocking[K, V]) ConsumeBlocking(ctx context.Context, offset int64, maxCount int64) (int64, []TMessage[K, V], error) { 50 | if err := l.notify.Wait(ctx, offset); err != nil { 51 | return 0, nil, err 52 | } 53 | return l.TLog.Consume(offset, maxCount) 54 | } 55 | 56 | func (l *tlogBlocking[K, V]) ConsumeByKeyBlocking(ctx context.Context, key K, empty bool, offset int64, maxCount int64) (int64, []TMessage[K, V], error) { 57 | if err := l.notify.Wait(ctx, offset); err != nil { 58 | return 0, nil, err 59 | } 60 | return l.TLog.ConsumeByKey(key, empty, offset, maxCount) 61 | } 62 | 63 | func (l *tlogBlocking[K, V]) Close() error { 64 | if err := l.notify.Close(); err != nil { 65 | return err 66 | } 67 | return l.TLog.Close() 68 | } 69 | -------------------------------------------------------------------------------- /typed_codec.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "encoding/binary" 5 | "encoding/json" 6 | "errors" 7 | ) 8 | 9 | // Codec is interface satisfied by all codecs 10 | type Codec[T any] interface { 11 | Encode(t T, empty bool) (b []byte, err error) 12 | Decode(b []byte) (t T, empty bool, err error) 13 | } 14 | 15 | // JsonCodec supports coding values as a JSON 16 | type JsonCodec[T any] struct{} 17 | 18 | func (c JsonCodec[T]) Encode(t T, empty bool) ([]byte, error) { 19 | if empty { 20 | return nil, nil 21 | } 22 | return json.Marshal(t) 23 | } 24 | 25 | func (c JsonCodec[T]) Decode(b []byte) (T, bool, error) { 26 | var t T 27 | if b == nil { 28 | return t, true, nil 29 | } 30 | err := json.Unmarshal(b, &t) 31 | return t, false, err 32 | } 33 | 34 | type stringOptCodec struct{} 35 | 36 | func (c stringOptCodec) Encode(t string, empty bool) ([]byte, error) { 37 | if empty { 38 | return nil, nil 39 | } 40 | return json.Marshal(t) 41 | } 42 | 43 | func (c stringOptCodec) Decode(b []byte) (string, bool, error) { 44 | if b == nil { 45 | return "", true, nil 46 | } 47 | var s string 48 | err := json.Unmarshal(b, &s) 49 | return s, false, err 50 | } 51 | 52 | // StringOptCodec supports coding an optional string, e.g. differantiates between "" and nil strings 53 | var StringOptCodec = stringOptCodec{} 54 | 55 | type stringCodec struct{} 56 | 57 | func (c stringCodec) Encode(t string, empty bool) ([]byte, error) { 58 | return []byte(t), nil 59 | } 60 | 61 | func (c stringCodec) Decode(b []byte) (string, bool, error) { 62 | return string(b), false, nil 63 | } 64 | 65 | // StringCodec supports coding a string 66 | var StringCodec = stringCodec{} 67 | 68 | type varintCodec struct{} 69 | 70 | func (c varintCodec) Encode(t int64, empty bool) ([]byte, error) { 71 | if empty { 72 | return nil, nil 73 | } 74 | return binary.AppendVarint(nil, t), nil 75 | } 76 | 77 | var errShortBuffer = errors.New("varint: short buffer") 78 | var errOverflow = errors.New("varint: overflow") 79 | 80 | func (c varintCodec) Decode(b []byte) (int64, bool, error) { 81 | if b == nil { 82 | return 0, true, nil 83 | } 84 | t, n := binary.Varint(b) 85 | switch { 86 | case n == 0: 87 | return 0, true, errShortBuffer 88 | case n < 0: 89 | return 0, true, errOverflow 90 | default: 91 | return t, false, nil 92 | } 93 | } 94 | 95 | // VarintCodec supports coding integers as varint 96 | var VarintCodec = varintCodec{} 97 | -------------------------------------------------------------------------------- /typed_test.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | type tobj struct { 10 | V string `json:"v"` 11 | } 12 | 13 | func TestKV(t *testing.T) { 14 | dir := t.TempDir() 15 | l, err := OpenT[tobj, tobj](dir, Options{}, JsonCodec[tobj]{}, JsonCodec[tobj]{}) 16 | require.NoError(t, err) 17 | 18 | _, err = l.Publish([]TMessage[tobj, tobj]{ 19 | { 20 | Key: tobj{"hello"}, 21 | Value: tobj{"world"}, 22 | }, 23 | { 24 | Key: tobj{"hello"}, 25 | KeyEmpty: true, 26 | Value: tobj{"world"}, 27 | }, 28 | { 29 | Key: tobj{"hello"}, 30 | Value: tobj{"world"}, 31 | ValueEmpty: true, 32 | }, 33 | { 34 | Key: tobj{"hello"}, 35 | KeyEmpty: true, 36 | Value: tobj{"world"}, 37 | ValueEmpty: true, 38 | }, 39 | }) 40 | require.NoError(t, err) 41 | 42 | _, msgs, err := l.Consume(OffsetOldest, 4) 43 | require.NoError(t, err) 44 | 45 | require.Equal(t, tobj{"hello"}, msgs[0].Key) 46 | require.False(t, msgs[0].KeyEmpty) 47 | require.Equal(t, tobj{"world"}, msgs[0].Value) 48 | require.False(t, msgs[0].ValueEmpty) 49 | 50 | require.Equal(t, tobj{""}, msgs[1].Key) 51 | require.True(t, msgs[1].KeyEmpty) 52 | require.Equal(t, tobj{"world"}, msgs[1].Value) 53 | require.False(t, msgs[1].ValueEmpty) 54 | 55 | require.Equal(t, tobj{"hello"}, msgs[2].Key) 56 | require.False(t, msgs[2].KeyEmpty) 57 | require.Equal(t, tobj{""}, msgs[2].Value) 58 | require.True(t, msgs[2].ValueEmpty) 59 | 60 | require.Equal(t, tobj{""}, msgs[3].Key) 61 | require.True(t, msgs[3].KeyEmpty) 62 | require.Equal(t, tobj{""}, msgs[3].Value) 63 | require.True(t, msgs[3].ValueEmpty) 64 | } 65 | -------------------------------------------------------------------------------- /writer.go: -------------------------------------------------------------------------------- 1 | package klevdb 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "sync/atomic" 7 | "time" 8 | 9 | art "github.com/plar/go-adaptive-radix-tree/v2" 10 | 11 | "github.com/klev-dev/klevdb/index" 12 | "github.com/klev-dev/klevdb/message" 13 | "github.com/klev-dev/klevdb/segment" 14 | ) 15 | 16 | type writer struct { 17 | segment segment.Segment 18 | params index.Params 19 | 20 | messages *message.Writer 21 | items *index.Writer 22 | index *writerIndex 23 | reader *reader 24 | } 25 | 26 | func openWriter(seg segment.Segment, params index.Params, nextTime int64) (*writer, error) { 27 | messages, err := message.OpenWriter(seg.Log) 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | var ix *writerIndex 33 | if messages.Size() > 0 { 34 | indexItems, err := seg.ReindexAndReadIndex(params) 35 | if err != nil { 36 | return nil, err 37 | } 38 | ix = newWriterIndex(indexItems, params.Keys, seg.Offset, nextTime) 39 | } else { 40 | ix = newWriterIndex(nil, params.Keys, seg.Offset, nextTime) 41 | } 42 | 43 | items, err := index.OpenWriter(seg.Index, params) 44 | if err != nil { 45 | return nil, err 46 | } 47 | 48 | reader, err := openReaderAppend(seg, params, ix) 49 | if err != nil { 50 | return nil, err 51 | } 52 | 53 | return &writer{ 54 | segment: seg, 55 | params: params, 56 | 57 | messages: messages, 58 | items: items, 59 | index: ix, 60 | reader: reader, 61 | }, nil 62 | } 63 | 64 | func (w *writer) GetNextOffset() (int64, error) { 65 | return w.index.GetNextOffset() 66 | } 67 | 68 | func (w *writer) NeedsRollover(rollover int64) bool { 69 | return (w.messages.Size() + w.items.Size()) > rollover 70 | } 71 | 72 | func (w *writer) Publish(msgs []message.Message) (int64, error) { 73 | nextOffset, indexTime := w.index.getNext() 74 | 75 | items := make([]index.Item, len(msgs)) 76 | for i := range msgs { 77 | msgs[i].Offset = nextOffset + int64(i) 78 | if msgs[i].Time.IsZero() { 79 | msgs[i].Time = time.Now().UTC() 80 | } 81 | 82 | position, err := w.messages.Write(msgs[i]) 83 | if err != nil { 84 | return OffsetInvalid, err 85 | } 86 | 87 | items[i] = w.params.NewItem(msgs[i], position, indexTime) 88 | if err := w.items.Write(items[i]); err != nil { 89 | return OffsetInvalid, err 90 | } 91 | indexTime = items[i].Timestamp 92 | } 93 | 94 | return w.index.append(items), nil 95 | } 96 | 97 | func (w *writer) ReopenReader() (*reader, int64, int64) { 98 | rdr := reopenReader(w.segment, w.params, w.index.reader()) 99 | nextOffset, nextTime := w.index.getNext() 100 | return rdr, nextOffset, nextTime 101 | } 102 | 103 | var errSegmentChanged = errors.New("writing segment changed") 104 | 105 | func (w *writer) Delete(rs *segment.RewriteSegment) (*writer, *reader, error) { 106 | if err := w.Sync(); err != nil { 107 | return nil, nil, err 108 | } 109 | 110 | if len(rs.SurviveOffsets)+len(rs.DeletedOffsets) != w.index.Len() { 111 | // the number of messages changed, nothing to drop 112 | if err := rs.Segment.Remove(); err != nil { 113 | return nil, nil, err 114 | } 115 | return nil, nil, errSegmentChanged 116 | } 117 | 118 | if err := w.Close(); err != nil { 119 | return nil, nil, err 120 | } 121 | 122 | if len(rs.SurviveOffsets) == 0 { 123 | if err := rs.Segment.Remove(); err != nil { 124 | return nil, nil, err 125 | } 126 | 127 | nextOffset, nextTime := w.index.getNext() 128 | nseg := segment.New(w.segment.Dir, nextOffset) 129 | nwrt, err := openWriter(nseg, w.params, nextTime) 130 | if err != nil { 131 | return nil, nil, err 132 | } 133 | 134 | if err := w.segment.Remove(); err != nil { 135 | return nil, nil, err 136 | } 137 | 138 | return nwrt, nil, nil 139 | } 140 | 141 | nseg := rs.GetNewSegment() 142 | if nseg != w.segment { 143 | // the starting offset of the new segment is different 144 | if err := rs.Segment.Rename(nseg); err != nil { 145 | return nil, nil, err 146 | } 147 | 148 | if err := w.segment.Remove(); err != nil { 149 | return nil, nil, err 150 | } 151 | 152 | // first move the replacement 153 | nextOffset, nextTime := w.index.getNext() 154 | if _, ok := rs.DeletedOffsets[w.index.getLastOffset()]; ok { 155 | rdr := openReader(nseg, w.params, false) 156 | wrt, err := openWriter(segment.New(w.segment.Dir, nextOffset), w.params, nextTime) 157 | return wrt, rdr, err 158 | } else { 159 | wrt, err := openWriter(nseg, w.params, nextTime) 160 | return wrt, nil, err 161 | } 162 | } 163 | 164 | if err := rs.Segment.Override(w.segment); err != nil { 165 | return nil, nil, err 166 | } 167 | 168 | nextOffset, nextTime := w.index.getNext() 169 | if _, ok := rs.DeletedOffsets[w.index.getLastOffset()]; ok { 170 | rdr := openReader(w.segment, w.params, false) 171 | wrt, err := openWriter(segment.New(w.segment.Dir, nextOffset), w.params, nextTime) 172 | return wrt, rdr, err 173 | } else { 174 | wrt, err := openWriter(w.segment, w.params, nextTime) 175 | return wrt, nil, err 176 | } 177 | } 178 | 179 | func (w *writer) Sync() error { 180 | if err := w.messages.Sync(); err != nil { 181 | return err 182 | } 183 | if err := w.items.Sync(); err != nil { 184 | return err 185 | } 186 | return nil 187 | } 188 | 189 | func (w *writer) Close() error { 190 | if err := w.messages.Close(); err != nil { 191 | return err 192 | } 193 | if err := w.items.Close(); err != nil { 194 | return err 195 | } 196 | 197 | return w.reader.Close() 198 | } 199 | 200 | type writerIndex struct { 201 | items []index.Item 202 | keys art.Tree 203 | nextOffset atomic.Int64 204 | nextTime atomic.Int64 205 | 206 | mu sync.RWMutex 207 | } 208 | 209 | func newWriterIndex(items []index.Item, hasKeys bool, offset int64, timestamp int64) *writerIndex { 210 | var keys art.Tree 211 | if hasKeys { 212 | keys = art.New() 213 | index.AppendKeys(keys, items) 214 | } 215 | 216 | ix := &writerIndex{ 217 | items: items, 218 | keys: keys, 219 | } 220 | 221 | nextOffset := offset 222 | nextTime := timestamp 223 | if len(items) > 0 { 224 | nextOffset = items[len(items)-1].Offset + 1 225 | nextTime = items[len(items)-1].Timestamp 226 | } 227 | ix.nextOffset.Store(nextOffset) 228 | ix.nextTime.Store(nextTime) 229 | 230 | return ix 231 | } 232 | 233 | func (ix *writerIndex) GetNextOffset() (int64, error) { 234 | return ix.nextOffset.Load(), nil 235 | } 236 | 237 | func (ix *writerIndex) getNext() (int64, int64) { 238 | return ix.nextOffset.Load(), ix.nextTime.Load() 239 | } 240 | 241 | func (ix *writerIndex) getLastOffset() int64 { 242 | ix.mu.RLock() 243 | defer ix.mu.RUnlock() 244 | 245 | return ix.items[len(ix.items)-1].Offset 246 | } 247 | 248 | func (ix *writerIndex) append(items []index.Item) int64 { 249 | ix.mu.Lock() 250 | defer ix.mu.Unlock() 251 | 252 | ix.items = append(ix.items, items...) 253 | if ix.keys != nil { 254 | index.AppendKeys(ix.keys, items) 255 | } 256 | if ln := len(items); ln > 0 { 257 | ix.nextTime.Store(items[ln-1].Timestamp) 258 | ix.nextOffset.Store(items[ln-1].Offset + 1) 259 | } 260 | return ix.nextOffset.Load() 261 | } 262 | 263 | func (ix *writerIndex) reader() *readerIndex { 264 | ix.mu.RLock() 265 | defer ix.mu.RUnlock() 266 | 267 | return &readerIndex{ix.items, ix.keys, ix.nextOffset.Load(), false} 268 | } 269 | 270 | func (ix *writerIndex) Consume(offset int64) (int64, int64, int64, error) { 271 | ix.mu.RLock() 272 | defer ix.mu.RUnlock() 273 | 274 | position, maxPosition, err := index.Consume(ix.items, offset) 275 | if err == index.ErrOffsetIndexEmpty || err == index.ErrOffsetAfterEnd { 276 | if nextOffset := ix.nextOffset.Load(); offset <= nextOffset { 277 | return -1, -1, nextOffset, nil 278 | } 279 | } 280 | return position, maxPosition, offset, err 281 | } 282 | 283 | func (ix *writerIndex) Get(offset int64) (int64, error) { 284 | ix.mu.RLock() 285 | defer ix.mu.RUnlock() 286 | 287 | position, err := index.Get(ix.items, offset) 288 | if err == index.ErrOffsetAfterEnd { 289 | if nextOffset := ix.nextOffset.Load(); offset >= nextOffset { 290 | return 0, message.ErrInvalidOffset 291 | } 292 | } 293 | return position, err 294 | } 295 | 296 | func (ix *writerIndex) Keys(keyHash []byte) ([]int64, error) { 297 | ix.mu.RLock() 298 | defer ix.mu.RUnlock() 299 | 300 | return index.Keys(ix.keys, keyHash) 301 | } 302 | 303 | func (ix *writerIndex) Time(ts int64) (int64, error) { 304 | ix.mu.RLock() 305 | defer ix.mu.RUnlock() 306 | 307 | return index.Time(ix.items, ts) 308 | } 309 | 310 | func (ix *writerIndex) Len() int { 311 | ix.mu.RLock() 312 | defer ix.mu.RUnlock() 313 | 314 | return len(ix.items) 315 | } 316 | --------------------------------------------------------------------------------