├── .github └── workflows │ ├── codeql-analysis.yml │ ├── simpledb-test.yml │ └── unit-test.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── _examples ├── memstore.go ├── proto │ ├── hello_world.pb.go │ ├── hello_world.proto │ ├── mutation.pb.go │ └── mutation.proto ├── recordio.go ├── simpledb.go ├── skiplist.go ├── sstables.go └── wal.go ├── benchmark ├── README.md ├── proto │ ├── bench.pb.go │ └── bench.proto ├── recordio_read_test.go ├── recordio_write_test.go ├── simpledb_test.go ├── sstable_read_test.go └── sstable_write_test.go ├── go.mod ├── go.sum ├── kaitai ├── README.md ├── gokaitai │ ├── recordio_v2.go │ ├── recordio_v2_test.go │ ├── recordio_v3.go │ ├── recordio_v3_test.go │ ├── recordio_v4.go │ ├── recordio_v4_test.go │ └── vlq_base128_le.go ├── recordio_v2.ksy ├── recordio_v3.ksy ├── recordio_v4.ksy └── vlq_base128_le.ksy ├── memstore ├── README.md ├── memstore.go ├── memstore_sstable_iterator.go └── memstore_test.go ├── pq ├── priority_queue.go └── priority_queue_test.go ├── recordio ├── README.md ├── buffered_io.go ├── buffered_io_test.go ├── bufio_vendor.go ├── bufio_vendor_test.go ├── checksum_byte_reader.go ├── checksum_byte_reader_test.go ├── common_reader.go ├── compressor │ ├── compressor.go │ ├── gzip_compression.go │ ├── gzip_compression_test.go │ ├── lzw_compessor_test.go │ ├── lzw_compressor.go │ ├── snappy_compression.go │ └── snappy_compression_test.go ├── counting_buffered_reader.go ├── counting_buffered_reader_test.go ├── direct_io.go ├── direct_io_test.go ├── file_reader.go ├── file_reader_generator_test.go ├── file_reader_test.go ├── file_reader_v1compat_test.go ├── file_reader_v2compat_test.go ├── file_reader_v3compat_test.go ├── file_writer.go ├── file_writer_test.go ├── io_factory.go ├── mmap_reader.go ├── mmap_reader_test.go ├── mmap_reader_v1compat_test.go ├── mmap_reader_v2compat_test.go ├── mmap_reader_v3compat_test.go ├── proto │ ├── mmap_proto_reader.go │ ├── proto_reader.go │ ├── proto_writer.go │ ├── recordio_proto.go │ └── recordio_proto_test.go ├── recordio.go ├── recordio_test.go └── test_files │ ├── berlin52.tsp │ ├── text_line.pb.go │ ├── text_line.proto │ ├── v1_compat │ ├── recordio_SnappyWriterMultiRecord_asc │ ├── recordio_UncompressedSingleRecord │ ├── recordio_UncompressedSingleRecord_comp1 │ ├── recordio_UncompressedSingleRecord_comp2 │ ├── recordio_UncompressedSingleRecord_mnm │ ├── recordio_UncompressedSingleRecord_v0 │ ├── recordio_UncompressedSingleRecord_v256 │ └── recordio_UncompressedWriterMultiRecord_asc │ ├── v2_compat │ ├── recordio_SnappyWriterMultiRecord_asc │ ├── recordio_UncompressedSingleRecord │ ├── recordio_UncompressedSingleRecord_comp1 │ ├── recordio_UncompressedSingleRecord_comp2 │ ├── recordio_UncompressedSingleRecord_comp300 │ ├── recordio_UncompressedSingleRecord_directio │ ├── recordio_UncompressedSingleRecord_directio_trailer │ ├── recordio_UncompressedSingleRecord_mnm │ ├── recordio_UncompressedSingleRecord_v0 │ ├── recordio_UncompressedSingleRecord_v256 │ └── recordio_UncompressedWriterMultiRecord_asc │ ├── v3_compat │ ├── recordio_SnappyWriterMultiRecord_asc │ ├── recordio_UncompressedMagicNumberContent │ ├── recordio_UncompressedNilAndEmptyRecord │ ├── recordio_UncompressedSingleRecord │ ├── recordio_UncompressedSingleRecord_comp1 │ ├── recordio_UncompressedSingleRecord_comp2 │ ├── recordio_UncompressedSingleRecord_comp300 │ ├── recordio_UncompressedSingleRecord_directio │ ├── recordio_UncompressedSingleRecord_directio_trailer │ ├── recordio_UncompressedSingleRecord_mnm │ ├── recordio_UncompressedSingleRecord_v0 │ ├── recordio_UncompressedSingleRecord_v256 │ └── recordio_UncompressedWriterMultiRecord_asc │ └── v4_compat │ ├── recordio_SnappyWriterMultiRecord_asc │ ├── recordio_UncompressedCrcFailure │ ├── recordio_UncompressedMagicNumberContent │ ├── recordio_UncompressedNilAndEmptyRecord │ ├── recordio_UncompressedSingleRecord │ ├── recordio_UncompressedSingleRecord_comp1 │ ├── recordio_UncompressedSingleRecord_comp2 │ ├── recordio_UncompressedSingleRecord_comp300 │ ├── recordio_UncompressedSingleRecord_directio │ ├── recordio_UncompressedSingleRecord_directio_trailer │ ├── recordio_UncompressedSingleRecord_mnm │ ├── recordio_UncompressedSingleRecord_v0 │ ├── recordio_UncompressedSingleRecord_v256 │ └── recordio_UncompressedWriterMultiRecord_asc ├── simpledb ├── README.md ├── _crash_tests │ ├── .gitignore │ ├── crash_test.go │ └── simpledb_web_server.go ├── compaction.go ├── compaction_test.go ├── db.go ├── db_e2e_test.go ├── db_test.go ├── flush.go ├── flush_test.go ├── porcupine │ ├── .gitignore │ ├── db_recorder.go │ ├── linearizability_test.go │ └── model.go ├── proto │ ├── compaction_metadata.pb.go │ ├── compaction_metadata.proto │ ├── wal_mutation.pb.go │ └── wal_mutation.proto ├── recovery.go ├── recovery_test.go ├── rw_memstore.go ├── rw_memstore_test.go ├── sstable_manager.go └── sstable_manager_test.go ├── skiplist ├── README.md ├── map_generic.go └── map_generic_test.go ├── sstables ├── README.md ├── disk_key_index.go ├── empty_sstable_reader.go ├── empty_sstable_reader_test.go ├── map_key_index.go ├── proto │ ├── sstable.pb.go │ └── sstable.proto ├── skiplist_index.go ├── slice_key_index.go ├── sstable.go ├── sstable_index.go ├── sstable_index_test.go ├── sstable_iterator.go ├── sstable_merger.go ├── sstable_merger_test.go ├── sstable_reader.go ├── sstable_reader_generator_test.go ├── sstable_reader_test.go ├── sstable_reader_v0compat_test.go ├── sstable_test.go ├── sstable_writer.go ├── sstable_writer_test.go ├── super_sstable_reader.go ├── super_sstable_reader_test.go └── test_files │ ├── SimpleWriteHappyPathSSTable │ ├── data.rio │ └── index.rio │ ├── SimpleWriteHappyPathSSTableRecordIOV2 │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithBloom │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithCRCHashes │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithCRCHashesMismatch │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithMetaData │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ └── v0_compat │ ├── SimpleWriteHappyPathSSTable │ ├── data.rio │ └── index.rio │ ├── SimpleWriteHappyPathSSTableRecordIOV2 │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin │ ├── SimpleWriteHappyPathSSTableWithBloom │ ├── bloom.bf.gz │ ├── data.rio │ └── index.rio │ └── SimpleWriteHappyPathSSTableWithMetaData │ ├── bloom.bf.gz │ ├── data.rio │ ├── index.rio │ └── meta.pb.bin ├── test_examples.sh └── wal ├── README.md ├── appender.go ├── appender_test.go ├── cleaner.go ├── cleaner_test.go ├── proto ├── proto_bindings.go └── proto_bindings_test.go ├── replayer.go ├── replayer_test.go ├── test_files ├── seq_number.pb.go └── seq_number.proto ├── write_ahead_log.go └── write_ahead_log_test.go /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [ main ] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [ main ] 14 | schedule: 15 | - cron: '0 11 * * 1' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | language: [ 'go' ] 26 | steps: 27 | - name: Checkout repository 28 | uses: actions/checkout@v2 29 | with: 30 | # We must fetch at least the immediate parents so that if this is 31 | # a pull request then we can checkout the head. 32 | fetch-depth: 2 33 | 34 | # If this run was triggered by a pull request event, then checkout 35 | # the head of the pull request instead of the merge commit. 36 | - run: git checkout HEAD^2 37 | if: ${{ github.event_name == 'pull_request' }} 38 | 39 | # Initializes the CodeQL tools for scanning. 40 | - name: Initialize CodeQL 41 | uses: github/codeql-action/init@v2 42 | with: 43 | languages: ${{ matrix.language }} 44 | - name: Autobuild 45 | uses: github/codeql-action/autobuild@v2 46 | - name: Perform CodeQL Analysis 47 | uses: github/codeql-action/analyze@v2 48 | with: 49 | languages: ${{ matrix.language }} -------------------------------------------------------------------------------- /.github/workflows/simpledb-test.yml: -------------------------------------------------------------------------------- 1 | name: simpledb-test 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Go 13 | uses: actions/setup-go@v2 14 | with: 15 | go-version: 1.22 16 | - name: Linearization Test 17 | run: make linear-simpledb 18 | - uses: actions/upload-artifact@v4 19 | if: always() 20 | with: 21 | path: simpledb/porcupine/*.html 22 | - name: Crash Test 23 | run: make crash-simpledb 24 | -------------------------------------------------------------------------------- /.github/workflows/unit-test.yml: -------------------------------------------------------------------------------- 1 | name: unit-test 2 | on: 3 | push: 4 | branches: [ main ] 5 | pull_request: 6 | branches: [ main ] 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v2 12 | - name: Set up Go 13 | uses: actions/setup-go@v2 14 | with: 15 | go-version: 1.22 16 | - name: Go vet 17 | run: make vet 18 | - name: Unit Test 19 | run: make unit-test 20 | - name: Example Test 21 | run: make example-test 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | .idea/ -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # go option 2 | GO ?= go 3 | TAGS := 4 | BUILDFLAGS := -race 5 | TESTS := ./... 6 | TESTFLAGS := -race 7 | LDFLAGS := 8 | GOFLAGS := 9 | BINARIES := sstables 10 | VERSION := v1.7.1 11 | 12 | # Required for globs to work correctly 13 | SHELL=/bin/bash 14 | 15 | .DEFAULT_GOAL := unit-test 16 | 17 | .PHONY: compile-kaitai 18 | compile-kaitai: 19 | @echo 20 | @echo "==> Compiling Kaitai files <==" 21 | kaitai-struct-compiler -t go --go-package gokaitai kaitai/recordio_v2.ksy 22 | kaitai-struct-compiler -t go --go-package gokaitai kaitai/recordio_v3.ksy 23 | kaitai-struct-compiler -t go --go-package gokaitai kaitai/recordio_v4.ksy 24 | mv gokaitai/*.go kaitai/gokaitai/ 25 | rm -rf gokaitai 26 | 27 | .PHONY: compile-proto 28 | compile-proto: 29 | @echo 30 | @echo "==> Compiling Protobuf files <==" 31 | protoc --go_out=. --go_opt=paths=source_relative recordio/test_files/text_line.proto 32 | protoc --go_out=. --go_opt=paths=source_relative simpledb/proto/wal_mutation.proto 33 | protoc --go_out=. --go_opt=paths=source_relative simpledb/proto/compaction_metadata.proto 34 | protoc --go_out=. --go_opt=paths=source_relative wal/test_files/seq_number.proto 35 | protoc --go_out=. --go_opt=paths=source_relative _examples/proto/hello_world.proto 36 | protoc --go_out=. --go_opt=paths=source_relative _examples/proto/mutation.proto 37 | protoc --go_out=. --go_opt=paths=source_relative benchmark/proto/bench.proto 38 | protoc --go_out=. --go_opt=paths=source_relative sstables/proto/sstable.proto 39 | 40 | .PHONY: release 41 | release: 42 | @echo 43 | @echo "==> Preparing the release $(VERSION) <==" 44 | go mod tidy 45 | git tag ${VERSION} 46 | 47 | .PHONY: bench 48 | bench: 49 | $(GO) test -v -benchmem -bench=RecordIO ./benchmark 50 | $(GO) test -v -benchmem -bench=SSTable ./benchmark 51 | 52 | .PHONY: bench-simpledb 53 | bench-simpledb: 54 | $(GO) test -v -benchmem -bench=SimpleDB ./benchmark 55 | 56 | .PHONY: unit-test 57 | unit-test: 58 | @echo 59 | @echo "==> Building <==" 60 | $(GO) build $(BUILDFLAGS) -race $(TESTS) 61 | @echo "==> Running unit tests <==" 62 | $(GO) clean -testcache 63 | $(GO) test $(GOFLAGS) $(TESTFLAGS) $(TESTS) 64 | # separately test simpledb, because the race detector 65 | # increases the runtime of the end2end tests too much (10-20m) 66 | # the race-simpledb target can be used to test that 67 | $(GO) test -v --tags simpleDBe2e $(GOFLAGS) ./simpledb 68 | 69 | .PHONY: race-simpledb 70 | race-simpledb: 71 | @echo 72 | @echo "==> Running simpledb race tests <==" 73 | $(GO) clean -testcache 74 | $(GO) test -v -timeout 30m --tags simpleDBe2e $(GOFLAGS) ./simpledb $(TESTFLAGS) 75 | 76 | .PHONY: crash-simpledb 77 | crash-simpledb: 78 | @echo 79 | @echo "==> Running simpledb crash tests <==" 80 | $(GO) clean -testcache 81 | $(GO) test -v -timeout 30m --tags simpleDBcrash $(GOFLAGS) ./simpledb/_crash_tests $(TESTFLAGS) 82 | 83 | .PHONY: linear-simpledb 84 | linear-simpledb: 85 | @echo 86 | @echo "==> Running simpledb linearizability tests <==" 87 | $(GO) clean -testcache 88 | $(GO) test -v -timeout 30m --tags simpleDBlinear $(GOFLAGS) ./simpledb/porcupine $(TESTFLAGS) 89 | 90 | .PHONY: example-test 91 | example-test: 92 | /bin/bash test_examples.sh 93 | 94 | .PHONY: generate-test-files 95 | generate-test-files: 96 | @echo 97 | @echo "==> Generate Test Files <==" 98 | $(GO) clean -testcache 99 | export generate_compatfiles=true && $(GO) test $(GOFLAGS) $(TESTS) -run .*TestGenerateTestFiles.* 100 | 101 | .PHONY: vet 102 | vet: 103 | @echo 104 | @echo "==> Go vet <==" 105 | $(GO) vet $(TESTS) 106 | 107 | .PHONY: test 108 | test: vet unit-test linear-simpledb race-simpledb crash-simpledb 109 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![unit-test](https://github.com/thomasjungblut/go-sstables/actions/workflows/unit-test.yml/badge.svg)](https://github.com/thomasjungblut/go-sstables/actions/workflows/unit-test.yml) 2 | [![CodeQL](https://github.com/thomasjungblut/go-sstables/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/thomasjungblut/go-sstables/actions/workflows/codeql-analysis.yml) 3 | [![Go Reference](https://pkg.go.dev/badge/github.com/thomasjungblut/go-sstables.svg)](https://pkg.go.dev/github.com/thomasjungblut/go-sstables) 4 | 5 | ## go-sstables 6 | 7 | `go-sstables` is a Go library that contains NoSQL database building blocks like a sequential record format (recordio), a 8 | sorted string table (sstable), a write-ahead-log (WAL), and a memory store (memstore) that stores key/value pairs in 9 | memory using a skip list. 10 | 11 | You can frequently find those in embedded key-value stores or databases as well, notable examples 12 | are [RocksDB](https://github.com/facebook/rocksdb) or [LevelDB](https://github.com/google/leveldb). 13 | 14 | There is an example embedded key-value database in this library, you can find it in the simpledb folder - please don't 15 | use it for any production workload. 16 | 17 | While plain `[]byte` are at the core of this library, there are wrappers and bindings for protobuf to enable more 18 | convenient serialization. 19 | 20 | ## Installation 21 | 22 | This is a library as it does not contain any installable binary, which means you can just directly add it to your 23 | dependency via `go get`: 24 | 25 | > go get -d github.com/thomasjungblut/go-sstables 26 | 27 | ## Documentation 28 | 29 | [RocksDB has a great overview](https://github.com/facebook/rocksdb/wiki/RocksDB-Overview#3-high-level-architecture) of 30 | how the components usually play together to get an idea: 31 | 32 | ![rocksdb architecture overview](https://user-images.githubusercontent.com/62277872/119747261-310fb300-be47-11eb-92c3-c11719fa8a0c.png) 33 | 34 | You will find basically all of those mentioned pieces above and all of them stitched together as SimpleDB. The 35 | documentation is now separated by package for easier browsing, each of those READMEs contain examples - there is 36 | also [/examples](_examples) when you prefer browsing the code directly. 37 | 38 | * [RecordIO](recordio/README.md) 39 | * [Benchmark](benchmark/README.md#recordio) 40 | * [sstables](sstables/README.md) 41 | * [Benchmark](benchmark/README.md#sstable) 42 | * [Memstore](memstore/README.md) 43 | * [SkipList](skiplist/README.md) 44 | * [WriteAheadLog](wal/README.md) 45 | * [SimpleDB](simpledb/README.md) 46 | * [Benchmark](benchmark/README.md#simpledb) 47 | 48 | You can also find all interface and 49 | method [documentation on `pkg.go.dev`](https://pkg.go.dev/github.com/thomasjungblut/go-sstables/sstables#section-documentation) 50 | . 51 | 52 | ## Kaitai Support 53 | 54 | As you might want to read the data and files in other languages, I've added support for [Kaitai](https://kaitai.io/). 55 | Kaitai is a declarative schema file to define a binary format. From that `ksy` file you can generate code for a lot of 56 | other languages and read the data. 57 | 58 | Currently, there is support for: 59 | 60 | * [RecordIO (v2)](kaitai/recordio_v2.ksy) 61 | * [RecordIO (v3)](kaitai/recordio_v3.ksy) 62 | * [RecordIO (v4)](kaitai/recordio_v4.ksy) 63 | 64 | You can find more information on how to generate Kaitai readers in [kaitai/README.md](kaitai/README.md). 65 | 66 | 67 | ## Development on go-sstables 68 | 69 | ### Thank you 70 | 71 | I want to deeply thank Sébastien Heitzmann (@sebheitzmann) for his continued support, reviews and usage of this library. 72 | 73 | 74 | ### Generating protobufs 75 | 76 | This needs some pre-requisites installed, namely 77 | the [protobuf compiler](https://github.com/protocolbuffers/protobuf/releases) and the go generator plugin. The latter 78 | can be installed as a go package: 79 | 80 | ``` 81 | go install google.golang.org/protobuf/cmd/protoc-gen-go 82 | ``` 83 | 84 | Full installation details can be found in 85 | the [protobuf dev documentation](https://developers.google.com/protocol-buffers/docs/gotutorial#compiling-your-protocol-buffers) 86 | . 87 | 88 | Once installed, one can generate the protobuf structs using: 89 | 90 | ``` 91 | make compile-proto 92 | ``` 93 | 94 | ### Releasing the Go Module 95 | 96 | [General Guidance](https://github.com/golang/go/wiki/Modules#releasing-modules-all-versions) 97 | 98 | In short, run these commands: 99 | 100 | ``` 101 | make unit-test 102 | make release 103 | git push --tags 104 | ``` 105 | -------------------------------------------------------------------------------- /_examples/memstore.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/thomasjungblut/go-sstables/memstore" 5 | "github.com/thomasjungblut/go-sstables/sstables" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func main() { 11 | path := "/tmp/sstable-ms-ex/" 12 | defer os.RemoveAll(path) 13 | 14 | ms := memstore.NewMemStore() 15 | ms.Add([]byte{1}, []byte{1}) 16 | ms.Add([]byte{2}, []byte{2}) 17 | ms.Upsert([]byte{1}, []byte{2}) 18 | ms.Delete([]byte{2}) 19 | ms.DeleteIfExists([]byte{3}) 20 | value, _ := ms.Get([]byte{1}) 21 | log.Printf("value for key 1: %d", value) // yields 2 22 | 23 | size := ms.EstimatedSizeInBytes() 24 | log.Printf("memstore size is %d bytes", size) // yields 3 25 | 26 | ms.Flush(sstables.WriteBasePath(path)) 27 | 28 | } 29 | -------------------------------------------------------------------------------- /_examples/proto/hello_world.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/go-sstables/examples/proto"; 4 | 5 | message HelloWorld { 6 | string message = 1; 7 | } 8 | -------------------------------------------------------------------------------- /_examples/proto/mutation.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/go-sstables/examples/proto"; 4 | 5 | message UpdateMutation { 6 | string columnName = 1; 7 | string columnValue = 2; 8 | } 9 | 10 | message DeleteMutation { 11 | string columnName = 1; 12 | } 13 | 14 | message Mutation { 15 | uint64 seqNumber = 1; 16 | oneof mutation { 17 | UpdateMutation update = 2; 18 | DeleteMutation delete = 3; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /_examples/recordio.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log" 7 | "os" 8 | 9 | "github.com/thomasjungblut/go-sstables/_examples/proto" 10 | "github.com/thomasjungblut/go-sstables/recordio" 11 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 12 | ) 13 | 14 | func main() { 15 | path := "/tmp/some_file.snap" 16 | defer os.Remove(path) 17 | 18 | simpleWrite(path) 19 | simpleRead(path) 20 | 21 | simpleReadAtOffset(path) 22 | } 23 | 24 | func simpleRead(path string) { 25 | reader, err := rProto.NewReader(rProto.ReaderPath(path)) 26 | if err != nil { 27 | log.Fatalf("error: %v", err) 28 | } 29 | 30 | err = reader.Open() 31 | if err != nil { 32 | log.Fatalf("error: %v", err) 33 | } 34 | 35 | for { 36 | record := &proto.HelloWorld{} 37 | _, err := reader.ReadNext(record) 38 | // io.EOF signals that no records are left to be read 39 | if errors.Is(err, io.EOF) { 40 | break 41 | } 42 | 43 | if err != nil { 44 | log.Fatalf("error: %v", err) 45 | } 46 | 47 | log.Printf("%s", record.GetMessage()) 48 | } 49 | 50 | err = reader.Close() 51 | if err != nil { 52 | log.Fatalf("error: %v", err) 53 | } 54 | } 55 | 56 | func simpleWrite(path string) { 57 | writer, err := rProto.NewWriter(rProto.Path(path), rProto.CompressionType(recordio.CompressionTypeSnappy)) 58 | if err != nil { 59 | log.Fatalf("error: %v", err) 60 | } 61 | err = writer.Open() 62 | if err != nil { 63 | log.Fatalf("error: %v", err) 64 | } 65 | record := &proto.HelloWorld{Message: "Hello World"} 66 | recordOffset, err := writer.Write(record) 67 | if err != nil { 68 | log.Fatalf("error: %v", err) 69 | } 70 | log.Printf("wrote a record at offset of %d bytes", recordOffset) 71 | 72 | err = writer.Close() 73 | if err != nil { 74 | log.Fatalf("error: %v", err) 75 | } 76 | } 77 | 78 | func simpleReadAtOffset(path string) { 79 | reader, err := rProto.NewMMapProtoReaderWithPath(path) 80 | if err != nil { 81 | log.Fatalf("error: %v", err) 82 | } 83 | 84 | err = reader.Open() 85 | if err != nil { 86 | log.Fatalf("error: %v", err) 87 | } 88 | 89 | record := &proto.HelloWorld{} 90 | _, err = reader.ReadNextAt(record, 8) 91 | if err != nil { 92 | log.Fatalf("error: %v", err) 93 | } 94 | 95 | log.Printf("Reading message at offset 8: %s", record.GetMessage()) 96 | 97 | err = reader.Close() 98 | if err != nil { 99 | log.Fatalf("error: %v", err) 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /_examples/simpledb.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "os" 7 | 8 | "github.com/thomasjungblut/go-sstables/simpledb" 9 | ) 10 | 11 | func main() { 12 | path := "/tmp/simpledb_example/" 13 | os.MkdirAll(path, 0777) 14 | defer os.RemoveAll(path) 15 | 16 | db, err := simpledb.NewSimpleDB(path) 17 | if err != nil { 18 | log.Fatalf("error: %v", err) 19 | } 20 | 21 | err = db.Open() 22 | if err != nil { 23 | log.Fatalf("error: %v", err) 24 | } 25 | 26 | err = db.Put("hello", "world") 27 | if err != nil { 28 | log.Fatalf("error: %v", err) 29 | } 30 | 31 | get, err := db.Get("hello") 32 | if err != nil { 33 | log.Fatalf("error: %v", err) 34 | } 35 | 36 | log.Printf("get 'hello' = %s", get) 37 | 38 | _, err = db.Get("not found") 39 | if errors.Is(err, simpledb.ErrNotFound) { 40 | log.Printf("not found!") 41 | } 42 | 43 | err = db.Delete("hello") 44 | if err != nil { 45 | log.Fatalf("error: %v", err) 46 | } 47 | 48 | err = db.Close() 49 | if err != nil { 50 | log.Fatalf("error: %v", err) 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /_examples/skiplist.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/skiplist" 6 | "log" 7 | ) 8 | 9 | func main() { 10 | 11 | skipListMap := skiplist.NewSkipListMap[int, int](skiplist.OrderedComparator[int]{}) 12 | skipListMap.Insert(13, 91) 13 | skipListMap.Insert(3, 1) 14 | skipListMap.Insert(5, 2) 15 | log.Printf("size: %d", skipListMap.Size()) 16 | 17 | it, _ := skipListMap.Iterator() 18 | for { 19 | k, v, err := it.Next() 20 | if errors.Is(err, skiplist.Done) { 21 | break 22 | } 23 | log.Printf("key: %d, value: %d", k, v) 24 | } 25 | 26 | log.Printf("starting at key: %d", 5) 27 | it, _ = skipListMap.IteratorStartingAt(5) 28 | for { 29 | k, v, err := it.Next() 30 | if errors.Is(err, skiplist.Done) { 31 | break 32 | } 33 | log.Printf("key: %d, value: %d", k, v) 34 | } 35 | 36 | log.Printf("between: %d and %d", 8, 50) 37 | it, _ = skipListMap.IteratorBetween(8, 50) 38 | for { 39 | k, v, err := it.Next() 40 | if errors.Is(err, skiplist.Done) { 41 | break 42 | } 43 | log.Printf("key: %d, value: %d", k, v) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /_examples/sstables.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/skiplist" 6 | "github.com/thomasjungblut/go-sstables/sstables" 7 | "log" 8 | "os" 9 | ) 10 | 11 | func main() { 12 | path := "/tmp/sstable_example/" 13 | os.MkdirAll(path, 0777) 14 | defer os.RemoveAll(path) 15 | 16 | mainWriteSimple(path) 17 | 18 | mainSimpleRead(path) 19 | } 20 | 21 | func mainSimpleRead(path string) { 22 | reader, err := sstables.NewSSTableReader(sstables.ReadBasePath("/tmp/sstable_example/")) 23 | if err != nil { 24 | log.Fatalf("error: %v", err) 25 | } 26 | defer reader.Close() 27 | 28 | metadata := reader.MetaData() 29 | log.Printf("reading table with %d records, minKey %d and maxKey %d", metadata.NumRecords, metadata.MinKey, metadata.MaxKey) 30 | 31 | contains, err := reader.Contains([]byte{1}) 32 | if err != nil { 33 | log.Fatalf("error: %v", err) 34 | } 35 | val, err := reader.Get([]byte{1}) 36 | if err != nil { 37 | log.Fatalf("error: %v", err) 38 | } 39 | log.Printf("table contains value for key? %t = %d", contains, val) 40 | 41 | it, err := reader.ScanRange([]byte{1}, []byte{2}) 42 | for { 43 | k, v, err := it.Next() 44 | // io.EOF signals that no records are left to be read 45 | if errors.Is(err, sstables.Done) { 46 | break 47 | } 48 | 49 | if err != nil { 50 | log.Fatalf("error: %v", err) 51 | } 52 | 53 | log.Printf("%d = %d", k, v) 54 | } 55 | } 56 | 57 | func mainWriteSimple(path string) { 58 | writer, err := sstables.NewSSTableSimpleWriter( 59 | sstables.WriteBasePath(path), 60 | sstables.WithKeyComparator(skiplist.BytesComparator{})) 61 | if err != nil { 62 | log.Fatalf("error: %v", err) 63 | } 64 | 65 | skipListMap := skiplist.NewSkipListMap[[]byte, []byte](skiplist.BytesComparator{}) 66 | skipListMap.Insert([]byte{1}, []byte{1}) 67 | skipListMap.Insert([]byte{2}, []byte{2}) 68 | skipListMap.Insert([]byte{3}, []byte{3}) 69 | 70 | err = writer.WriteSkipListMap(skipListMap) 71 | if err != nil { 72 | log.Fatalf("error: %v", err) 73 | } 74 | } 75 | 76 | func mainWriteStreaming() { 77 | path := "/tmp/sstable_example/" 78 | os.MkdirAll(path, 0777) 79 | defer os.RemoveAll(path) 80 | 81 | writer, err := sstables.NewSSTableStreamWriter( 82 | sstables.WriteBasePath(path), 83 | sstables.WithKeyComparator(skiplist.BytesComparator{})) 84 | if err != nil { 85 | log.Fatalf("error: %v", err) 86 | } 87 | 88 | err = writer.Open() 89 | if err != nil { 90 | log.Fatalf("error: %v", err) 91 | } 92 | 93 | err = writer.WriteNext([]byte{1}, []byte{1}) 94 | if err != nil { 95 | log.Fatalf("error: %v", err) 96 | } 97 | err = writer.WriteNext([]byte{2}, []byte{2}) 98 | if err != nil { 99 | log.Fatalf("error: %v", err) 100 | } 101 | err = writer.WriteNext([]byte{3}, []byte{3}) 102 | if err != nil { 103 | log.Fatalf("error: %v", err) 104 | } 105 | 106 | err = writer.Close() 107 | if err != nil { 108 | log.Fatalf("error: %v", err) 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /_examples/wal.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | 6 | exProto "github.com/thomasjungblut/go-sstables/_examples/proto" 7 | . "github.com/thomasjungblut/go-sstables/wal" 8 | wProto "github.com/thomasjungblut/go-sstables/wal/proto" 9 | pb "google.golang.org/protobuf/proto" 10 | "log" 11 | "os" 12 | ) 13 | 14 | func main() { 15 | path := "/tmp/wal_example/" 16 | _ = os.MkdirAll(path, 0777) 17 | defer os.RemoveAll(path) 18 | 19 | opts, err := NewWriteAheadLogOptions(BasePath(path)) 20 | if err != nil { 21 | log.Fatalf("error: %v", err) 22 | } 23 | wal, err := wProto.NewProtoWriteAheadLog(opts) 24 | if err != nil { 25 | log.Fatalf("error: %v", err) 26 | } 27 | 28 | updateMutation := exProto.UpdateMutation{ 29 | ColumnName: "some_col", 30 | ColumnValue: "some_val", 31 | } 32 | mutation := exProto.Mutation{ 33 | SeqNumber: 1, 34 | Mutation: &exProto.Mutation_Update{Update: &updateMutation}, 35 | } 36 | 37 | err = wal.AppendSync(&mutation) 38 | if err != nil { 39 | log.Fatalf("error: %v", err) 40 | } 41 | 42 | deleteMutation := exProto.DeleteMutation{ 43 | ColumnName: "some_col", 44 | } 45 | mutation = exProto.Mutation{ 46 | SeqNumber: 2, 47 | Mutation: &exProto.Mutation_Delete{Delete: &deleteMutation}, 48 | } 49 | 50 | err = wal.AppendSync(&mutation) 51 | if err != nil { 52 | log.Fatalf("error: %v", err) 53 | } 54 | 55 | err = wal.Close() 56 | if err != nil { 57 | log.Fatalf("error: %v", err) 58 | } 59 | 60 | err = wal.Replay(func() pb.Message { 61 | return &exProto.Mutation{} 62 | }, func(record pb.Message) error { 63 | mutation := record.(*exProto.Mutation) 64 | fmt.Printf("seq no: %d\n", mutation.SeqNumber) 65 | switch x := mutation.Mutation.(type) { 66 | case *exProto.Mutation_Update: 67 | fmt.Printf("update with colname %s and val %s\n", x.Update.ColumnName, x.Update.ColumnValue) 68 | case *exProto.Mutation_Delete: 69 | fmt.Printf("delete with colname %s\n", x.Delete.ColumnName) 70 | default: 71 | return fmt.Errorf("proto.Mutation has unexpected oneof type %T", x) 72 | } 73 | return nil 74 | }) 75 | 76 | if err != nil { 77 | log.Fatalf("error: %v", err) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /benchmark/proto/bench.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/go-sstables/benchmark/proto"; 4 | 5 | message BytesMsg { 6 | bytes key = 1; 7 | } 8 | -------------------------------------------------------------------------------- /benchmark/recordio_read_test.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | bProto "github.com/thomasjungblut/go-sstables/benchmark/proto" 7 | "github.com/thomasjungblut/go-sstables/recordio" 8 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 9 | "io" 10 | "os" 11 | "testing" 12 | ) 13 | 14 | func BenchmarkRecordIORead(b *testing.B) { 15 | benchmarks := []struct { 16 | name string 17 | fileSize int 18 | }{ 19 | {"32mb", 1024 * 1024 * 32}, 20 | {"64mb", 1024 * 1024 * 64}, 21 | {"128mb", 1024 * 1024 * 128}, 22 | {"256mb", 1024 * 1024 * 256}, 23 | {"512mb", 1024 * 1024 * 512}, 24 | {"1024mb", 1024 * 1024 * 1024}, 25 | {"2048mb", 1024 * 1024 * 1024 * 2}, 26 | {"4096mb", 1024 * 1024 * 1024 * 4}, 27 | {"8192mb", 1024 * 1024 * 1024 * 8}, 28 | } 29 | 30 | for _, bm := range benchmarks { 31 | b.Run(bm.name, func(b *testing.B) { 32 | bytes := randomRecordOfSize(1024) 33 | tmpFile, err := os.CreateTemp("", "recordio_Bench") 34 | assert.NoError(b, err) 35 | defer os.Remove(tmpFile.Name()) 36 | 37 | writer, err := recordio.NewFileWriter(recordio.File(tmpFile)) 38 | assert.NoError(b, err) 39 | assert.NoError(b, writer.Open()) 40 | 41 | for writer.Size() < uint64(bm.fileSize) { 42 | _, _ = writer.Write(bytes) 43 | } 44 | b.SetBytes(int64(writer.Size())) 45 | assert.NoError(b, writer.Close()) 46 | 47 | b.ResetTimer() 48 | for n := 0; n < b.N; n++ { 49 | reader, err := recordio.NewFileReader(recordio.ReaderPath(tmpFile.Name())) 50 | assert.NoError(b, err) 51 | assert.NoError(b, reader.Open()) 52 | 53 | for { 54 | _, err := reader.ReadNext() 55 | if errors.Is(err, io.EOF) { 56 | break 57 | } 58 | } 59 | 60 | assert.NoError(b, reader.Close()) 61 | } 62 | }) 63 | } 64 | } 65 | 66 | func BenchmarkRecordIOProtoRead(b *testing.B) { 67 | benchmarks := []struct { 68 | name string 69 | fileSize int 70 | }{ 71 | {"32mb", 1024 * 1024 * 32}, 72 | {"64mb", 1024 * 1024 * 64}, 73 | {"128mb", 1024 * 1024 * 128}, 74 | {"256mb", 1024 * 1024 * 256}, 75 | {"512mb", 1024 * 1024 * 512}, 76 | {"1024mb", 1024 * 1024 * 1024}, 77 | {"2048mb", 1024 * 1024 * 1024 * 2}, 78 | {"4096mb", 1024 * 1024 * 1024 * 4}, 79 | {"8192mb", 1024 * 1024 * 1024 * 8}, 80 | } 81 | 82 | for _, bm := range benchmarks { 83 | b.Run(bm.name, func(b *testing.B) { 84 | bytes := randomRecordOfSize(1024) 85 | tmpFile, err := os.CreateTemp("", "recordio_Bench") 86 | assert.NoError(b, err) 87 | defer os.Remove(tmpFile.Name()) 88 | 89 | writer, err := rProto.NewWriter(rProto.File(tmpFile)) 90 | assert.NoError(b, err) 91 | assert.NoError(b, writer.Open()) 92 | 93 | msg := &bProto.BytesMsg{Key: bytes} 94 | for writer.Size() < uint64(bm.fileSize) { 95 | _, _ = writer.Write(msg) 96 | } 97 | b.SetBytes(int64(writer.Size())) 98 | assert.NoError(b, writer.Close()) 99 | 100 | b.ResetTimer() 101 | for n := 0; n < b.N; n++ { 102 | reader, err := rProto.NewReader(rProto.ReaderPath(tmpFile.Name())) 103 | assert.NoError(b, err) 104 | assert.NoError(b, reader.Open()) 105 | 106 | msg := &bProto.BytesMsg{} 107 | for { 108 | _, err := reader.ReadNext(msg) 109 | if errors.Is(err, io.EOF) { 110 | break 111 | } 112 | } 113 | 114 | assert.NoError(b, reader.Close()) 115 | } 116 | }) 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /benchmark/recordio_write_test.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/thomasjungblut/go-sstables/recordio" 6 | "math/rand" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func BenchmarkRecordIOWrite(b *testing.B) { 12 | benchmarks := []struct { 13 | name string 14 | recSize int 15 | sync bool 16 | compType int 17 | }{ 18 | {"RecordSize1k", 1024, false, recordio.CompressionTypeNone}, 19 | {"RecordSize10k", 1024 * 10, false, recordio.CompressionTypeNone}, 20 | {"RecordSize100k", 1024 * 100, false, recordio.CompressionTypeNone}, 21 | {"RecordSize1M", 1024 * 1000, false, recordio.CompressionTypeNone}, 22 | 23 | {"GzipRecordSize1k", 1024, false, recordio.CompressionTypeGZIP}, 24 | {"GzipRecordSize10k", 1024 * 10, false, recordio.CompressionTypeGZIP}, 25 | {"GzipRecordSize100k", 1024 * 100, false, recordio.CompressionTypeGZIP}, 26 | {"GzipRecordSize1M", 1024 * 1000, false, recordio.CompressionTypeGZIP}, 27 | 28 | {"SnappyRecordSize1k", 1024, false, recordio.CompressionTypeSnappy}, 29 | {"SnappyRecordSize10k", 1024 * 10, false, recordio.CompressionTypeSnappy}, 30 | {"SnappyRecordSize100k", 1024 * 100, false, recordio.CompressionTypeSnappy}, 31 | {"SnappyRecordSize1M", 1024 * 1000, false, recordio.CompressionTypeSnappy}, 32 | 33 | {"SyncRecordSize1k", 1024, true, recordio.CompressionTypeNone}, 34 | {"SyncRecordSize10k", 1024 * 10, true, recordio.CompressionTypeNone}, 35 | {"SyncRecordSize100k", 1024 * 100, true, recordio.CompressionTypeNone}, 36 | {"SyncRecordSize1M", 1024 * 1000, true, recordio.CompressionTypeNone}, 37 | } 38 | 39 | for _, bm := range benchmarks { 40 | b.Run(bm.name, func(b *testing.B) { 41 | bytes := randomRecordOfSize(bm.recSize) 42 | tmpFile, err := os.CreateTemp("", "recordio_Bench") 43 | assert.Nil(b, err) 44 | defer os.Remove(tmpFile.Name()) 45 | 46 | writer, err := recordio.NewFileWriter(recordio.File(tmpFile), recordio.CompressionType(bm.compType)) 47 | assert.Nil(b, err) 48 | assert.Nil(b, writer.Open()) 49 | 50 | b.ResetTimer() 51 | for n := 0; n < b.N; n++ { 52 | if bm.sync { 53 | _, _ = writer.WriteSync(bytes) 54 | } else { 55 | _, _ = writer.Write(bytes) 56 | } 57 | b.SetBytes(int64(len(bytes))) 58 | } 59 | 60 | assert.Nil(b, writer.Close()) 61 | stat, err := os.Stat(tmpFile.Name()) 62 | assert.Nil(b, err) 63 | assert.Truef(b, stat.Size() > int64(len(bytes)*b.N), "unexpected small file size %d", stat.Size()) 64 | }) 65 | } 66 | 67 | } 68 | 69 | func randomRecordOfSize(l int) []byte { 70 | bytes := make([]byte, l) 71 | for i := 0; i < l; i++ { 72 | bytes[i] = byte(rand.Intn(255)) 73 | } 74 | 75 | return bytes 76 | } 77 | -------------------------------------------------------------------------------- /benchmark/simpledb_test.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "log" 7 | "math/rand" 8 | "os" 9 | "runtime" 10 | "strconv" 11 | "strings" 12 | "sync" 13 | "sync/atomic" 14 | "testing" 15 | 16 | "github.com/stretchr/testify/require" 17 | 18 | "github.com/thomasjungblut/go-sstables/simpledb" 19 | ) 20 | 21 | func BenchmarkSimpleDBReadLatency(b *testing.B) { 22 | log.SetOutput(io.Discard) 23 | dbSizes := []int{100, 1000, 10000, 100000} 24 | 25 | for _, n := range dbSizes { 26 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 27 | tmpDir, err := os.MkdirTemp("", "simpledb_Bench") 28 | require.Nil(b, err) 29 | defer func() { require.Nil(b, os.RemoveAll(tmpDir)) }() 30 | db, err := simpledb.NewSimpleDB(tmpDir, 31 | simpledb.MemstoreSizeBytes(1024*1024*1024)) 32 | require.Nil(b, err) 33 | defer func() { require.Nil(b, db.Close()) }() 34 | require.Nil(b, db.Open()) 35 | 36 | parallelWriteDB(db, runtime.NumCPU(), n) 37 | 38 | b.ResetTimer() 39 | i := 0 40 | for n := 0; n < b.N; n++ { 41 | k := strconv.Itoa(i) 42 | val, err := db.Get(k) 43 | if err != simpledb.ErrNotFound { 44 | b.SetBytes(int64(len(k) + len(val))) 45 | } 46 | i++ 47 | if i >= n { 48 | i = 0 49 | } 50 | } 51 | }) 52 | } 53 | } 54 | 55 | func BenchmarkSimpleDBWriteLatency(b *testing.B) { 56 | log.SetOutput(io.Discard) 57 | dbSizes := []int{100, 1000, 10000, 100000, 1000000} 58 | 59 | for _, n := range dbSizes { 60 | b.Run(fmt.Sprintf("%d", n), func(b *testing.B) { 61 | tmpDir, err := os.MkdirTemp("", "simpledb_Bench") 62 | require.Nil(b, err) 63 | defer func() { require.Nil(b, os.RemoveAll(tmpDir)) }() 64 | 65 | memstoreSize := uint64(1024 * 1024 * 1024) 66 | db, err := simpledb.NewSimpleDB(tmpDir, 67 | simpledb.MemstoreSizeBytes(memstoreSize)) 68 | require.Nil(b, err) 69 | defer func() { require.Nil(b, db.Close()) }() 70 | require.Nil(b, db.Open()) 71 | 72 | b.ResetTimer() 73 | for n := 0; n < b.N; n++ { 74 | bytes := parallelWriteDB(db, runtime.NumCPU(), n) 75 | b.SetBytes(bytes) 76 | } 77 | }) 78 | } 79 | } 80 | 81 | func parallelWriteDB(db *simpledb.DB, numGoRoutines int, numRecords int) int64 { 82 | numRecordsWritten := int64(0) 83 | bytesWritten := int64(0) 84 | wg := sync.WaitGroup{} 85 | recordsPerRoutine := numRecords / numGoRoutines 86 | val := randomString() 87 | for n := 0; n < numGoRoutines; n++ { 88 | wg.Add(1) 89 | go func(db *simpledb.DB, start, end int) { 90 | for i := start; i < end; i++ { 91 | k := strconv.Itoa(i) 92 | _ = db.Put(k, val) 93 | atomic.AddInt64(&bytesWritten, int64(len(k)+len(val))) 94 | atomic.AddInt64(&numRecordsWritten, 1) 95 | } 96 | wg.Done() 97 | }(db, n*recordsPerRoutine, n*recordsPerRoutine+recordsPerRoutine) 98 | } 99 | 100 | wg.Wait() 101 | return bytesWritten 102 | } 103 | 104 | func randomString() string { 105 | return randomStringSize(10000) 106 | } 107 | 108 | func randomStringSize(n int) string { 109 | builder := strings.Builder{} 110 | for i := 0; i < n; i++ { 111 | builder.WriteRune(rand.Int31n(255)) 112 | } 113 | return builder.String() 114 | } 115 | -------------------------------------------------------------------------------- /benchmark/sstable_write_test.go: -------------------------------------------------------------------------------- 1 | package benchmark 2 | 3 | import ( 4 | "encoding/binary" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/thomasjungblut/go-sstables/memstore" 7 | "github.com/thomasjungblut/go-sstables/skiplist" 8 | "github.com/thomasjungblut/go-sstables/sstables" 9 | "os" 10 | "testing" 11 | ) 12 | 13 | func BenchmarkSSTableMemstoreFlush(b *testing.B) { 14 | benchmarks := []struct { 15 | name string 16 | memstoreSize int 17 | }{ 18 | {"32mb", 1024 * 1024 * 32}, 19 | {"64mb", 1024 * 1024 * 64}, 20 | {"128mb", 1024 * 1024 * 128}, 21 | {"256mb", 1024 * 1024 * 256}, 22 | {"512mb", 1024 * 1024 * 512}, 23 | {"1024mb", 1024 * 1024 * 1024}, 24 | {"2048mb", 1024 * 1024 * 1024 * 2}, 25 | } 26 | 27 | cmp := skiplist.BytesComparator{} 28 | for _, bm := range benchmarks { 29 | b.Run(bm.name, func(b *testing.B) { 30 | mStore := memstore.NewMemStore() 31 | bytes := randomRecordOfSize(1024) 32 | 33 | i := 0 34 | for mStore.EstimatedSizeInBytes() < uint64(bm.memstoreSize) { 35 | k := make([]byte, 4) 36 | binary.BigEndian.PutUint32(k, uint32(i)) 37 | assert.Nil(b, mStore.Add(k, bytes)) 38 | i++ 39 | } 40 | 41 | var tmpDirs []string 42 | for n := 0; n < b.N; n++ { 43 | tmpDir, err := os.MkdirTemp("", "sstable_BenchWrite") 44 | assert.Nil(b, err) 45 | tmpDirs = append(tmpDirs, tmpDir) 46 | } 47 | 48 | defer func() { 49 | for i := 0; i < b.N; i++ { 50 | assert.Nil(b, os.RemoveAll(tmpDirs[i])) 51 | } 52 | }() 53 | 54 | b.ResetTimer() 55 | for n := 0; n < b.N; n++ { 56 | err := mStore.Flush(sstables.WriteBasePath(tmpDirs[n]), 57 | sstables.WithKeyComparator(cmp), sstables.WriteBufferSizeBytes(1024*1024*4)) 58 | assert.Nil(b, err) 59 | b.SetBytes(int64(mStore.EstimatedSizeInBytes())) 60 | } 61 | }) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/thomasjungblut/go-sstables 2 | 3 | require ( 4 | capnproto.org/go/capnp/v3 v3.1.0-alpha.1 5 | github.com/anishathalye/porcupine v0.1.2 6 | github.com/golang/snappy v0.0.4 7 | github.com/kaitai-io/kaitai_struct_go_runtime v0.10.0 8 | github.com/ncw/directio v1.0.5 9 | github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 10 | github.com/stretchr/testify v1.9.0 11 | golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 12 | google.golang.org/protobuf v1.34.2 13 | ) 14 | 15 | require ( 16 | github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 // indirect 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/pmezard/go-difflib v1.0.0 // indirect 19 | github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 // indirect 20 | golang.org/x/text v0.3.8 // indirect 21 | gopkg.in/yaml.v3 v3.0.1 // indirect 22 | ) 23 | 24 | replace github.com/anishathalye/porcupine v0.1.2 => github.com/tjungblu/porcupine v0.0.0-20221116095144-377185aa0569 25 | 26 | go 1.23 27 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | capnproto.org/go/capnp/v3 v3.1.0-alpha.1 h1:8/sMnWuatR99G0L0vmnrXj0zVP0MrlyClRqSmqGYydo= 2 | capnproto.org/go/capnp/v3 v3.1.0-alpha.1/go.mod h1:2vT5D2dtG8sJGEoEKU17e+j7shdaYp1Myl8X03B3hmc= 3 | github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381 h1:d5EKgQfRQvO97jnISfR89AiCCCJMwMFoSxUiU0OGCRU= 4 | github.com/colega/zeropool v0.0.0-20230505084239-6fb4a4f75381/go.mod h1:OU76gHeRo8xrzGJU3F3I1CqX1ekM8dfJw0+wPeMwnp0= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 8 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 9 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 10 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 11 | github.com/kaitai-io/kaitai_struct_go_runtime v0.10.0 h1:bxazq0XLMSVMm/DIVFLl9BqIWehrqcLsyVWSacEjIKE= 12 | github.com/kaitai-io/kaitai_struct_go_runtime v0.10.0/go.mod h1:fBebEoDoc0xNbZsIcRQWqDp4jViaTKv6uxAUjmCFGgM= 13 | github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= 14 | github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570 h1:gIlAHnH1vJb5vwEjIp5kBj/eu99p/bl0Ay2goiPe5xE= 18 | github.com/steakknife/bloomfilter v0.0.0-20180922174646-6819c0d2a570/go.mod h1:8OR4w3TdeIHIh1g6EMY5p0gVNOovcWC+1vpc7naMuAw= 19 | github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3 h1:njlZPzLwU639dk2kqnCPPv+wNjq7Xb6EfUxe/oX0/NM= 20 | github.com/steakknife/hamming v0.0.0-20180906055917-c99c65617cd3/go.mod h1:hpGUWaI9xL8pRQCTXQgocU38Qw1g0Us7n5PxxTwTCYU= 21 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 22 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 23 | github.com/tjungblu/porcupine v0.0.0-20221116095144-377185aa0569 h1:acDBvgSBtnyBidmpmTEgxStjXeuYyfG3fF72khP24/Y= 24 | github.com/tjungblu/porcupine v0.0.0-20221116095144-377185aa0569/go.mod h1:+z336r1WR0gcwl1ALfoNBpDTCW06vO5DzBwunEcSvcs= 25 | golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 h1:yixxcjnhBmY0nkL253HFVIm0JsFHwrHdT3Yh6szTnfY= 26 | golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8/go.mod h1:jj3sYF3dwk5D+ghuXyeI3r5MFf+NT2An6/9dOA95KSI= 27 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 28 | golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= 29 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 30 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 31 | google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= 32 | google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= 33 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 34 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 35 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 36 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 37 | -------------------------------------------------------------------------------- /kaitai/README.md: -------------------------------------------------------------------------------- 1 | ## Using Kaitai Struct 2 | 3 | The best about Kaitai is its excellent documentation. You can find how to install and use it from their [website](https://kaitai.io/#quick-start). Below we make a quick example on how to compile and use it in Java, other languages work the same way. 4 | 5 | All we need here is the `ksc` command (or `kaitai-struct-compiler`), the respective `ksy` file and the target of compilation. 6 | 7 | Here, for example, we compile for Java: 8 | 9 | > ksc recordio_v4.ksy --target java 10 | 11 | In the case of recordio v4 we need receive two files: one for the vint compression schema and one for the actual recordio files. 12 | 13 | The Java code is fairly easy to use, you would just need to run: 14 | 15 | ```java 16 | RecordioV4 records = RecordioV4.fromFile("/some/record.io"); 17 | for(Record r : records.record()) { 18 | ... 19 | } 20 | ``` 21 | -------------------------------------------------------------------------------- /kaitai/gokaitai/recordio_v2.go: -------------------------------------------------------------------------------- 1 | // Code generated by kaitai-struct-compiler from a .ksy source file. DO NOT EDIT. 2 | 3 | package gokaitai 4 | 5 | import ( 6 | "github.com/kaitai-io/kaitai_struct_go_runtime/kaitai" 7 | "bytes" 8 | ) 9 | 10 | 11 | type RecordioV2_Compression int 12 | const ( 13 | RecordioV2_Compression__None RecordioV2_Compression = 0 14 | RecordioV2_Compression__Snappy RecordioV2_Compression = 1 15 | RecordioV2_Compression__Gzip RecordioV2_Compression = 2 16 | ) 17 | type RecordioV2 struct { 18 | FileHeader *RecordioV2_FileHeader 19 | Record []*RecordioV2_Record 20 | _io *kaitai.Stream 21 | _root *RecordioV2 22 | _parent interface{} 23 | } 24 | func NewRecordioV2() *RecordioV2 { 25 | return &RecordioV2{ 26 | } 27 | } 28 | 29 | func (this *RecordioV2) Read(io *kaitai.Stream, parent interface{}, root *RecordioV2) (err error) { 30 | this._io = io 31 | this._parent = parent 32 | this._root = root 33 | 34 | tmp1 := NewRecordioV2_FileHeader() 35 | err = tmp1.Read(this._io, this, this._root) 36 | if err != nil { 37 | return err 38 | } 39 | this.FileHeader = tmp1 40 | for i := 1;; i++ { 41 | tmp2, err := this._io.EOF() 42 | if err != nil { 43 | return err 44 | } 45 | if tmp2 { 46 | break 47 | } 48 | tmp3 := NewRecordioV2_Record() 49 | err = tmp3.Read(this._io, this, this._root) 50 | if err != nil { 51 | return err 52 | } 53 | this.Record = append(this.Record, tmp3) 54 | } 55 | return err 56 | } 57 | 58 | /** 59 | * recordio header format to figure out the version it was written and whether the records are compressed. 60 | */ 61 | type RecordioV2_FileHeader struct { 62 | Version uint32 63 | CompressionType RecordioV2_Compression 64 | _io *kaitai.Stream 65 | _root *RecordioV2 66 | _parent *RecordioV2 67 | } 68 | func NewRecordioV2_FileHeader() *RecordioV2_FileHeader { 69 | return &RecordioV2_FileHeader{ 70 | } 71 | } 72 | 73 | func (this *RecordioV2_FileHeader) Read(io *kaitai.Stream, parent *RecordioV2, root *RecordioV2) (err error) { 74 | this._io = io 75 | this._parent = parent 76 | this._root = root 77 | 78 | tmp4, err := this._io.ReadU4le() 79 | if err != nil { 80 | return err 81 | } 82 | this.Version = uint32(tmp4) 83 | tmp5, err := this._io.ReadU4le() 84 | if err != nil { 85 | return err 86 | } 87 | this.CompressionType = RecordioV2_Compression(tmp5) 88 | return err 89 | } 90 | 91 | /** 92 | * The version of the recordio format used in this file. 93 | */ 94 | 95 | /** 96 | * The compression algorithm used. 0 means no compression, 1 means Snappy, 2 means Gzip. 97 | */ 98 | 99 | /** 100 | * recordio record is an "infinite" stream of magic number separated and length encoded byte arrays. 101 | */ 102 | type RecordioV2_Record struct { 103 | Magic []byte 104 | UncompressedPayloadLen *VlqBase128Le 105 | CompressedPayloadLen *VlqBase128Le 106 | Payload []byte 107 | _io *kaitai.Stream 108 | _root *RecordioV2 109 | _parent *RecordioV2 110 | _f_lenPayload bool 111 | lenPayload int 112 | } 113 | func NewRecordioV2_Record() *RecordioV2_Record { 114 | return &RecordioV2_Record{ 115 | } 116 | } 117 | 118 | func (this *RecordioV2_Record) Read(io *kaitai.Stream, parent *RecordioV2, root *RecordioV2) (err error) { 119 | this._io = io 120 | this._parent = parent 121 | this._root = root 122 | 123 | tmp6, err := this._io.ReadBytes(int(3)) 124 | if err != nil { 125 | return err 126 | } 127 | this.Magic = tmp6 128 | if !(bytes.Equal(this.Magic, []uint8{145, 141, 76})) { 129 | return kaitai.NewValidationNotEqualError([]uint8{145, 141, 76}, this.Magic, this._io, "/types/record/seq/0") 130 | } 131 | tmp7 := NewVlqBase128Le() 132 | err = tmp7.Read(this._io, this, nil) 133 | if err != nil { 134 | return err 135 | } 136 | this.UncompressedPayloadLen = tmp7 137 | tmp8 := NewVlqBase128Le() 138 | err = tmp8.Read(this._io, this, nil) 139 | if err != nil { 140 | return err 141 | } 142 | this.CompressedPayloadLen = tmp8 143 | tmp9, err := this.LenPayload() 144 | if err != nil { 145 | return err 146 | } 147 | tmp10, err := this._io.ReadBytes(int(tmp9)) 148 | if err != nil { 149 | return err 150 | } 151 | this.Payload = tmp10 152 | return err 153 | } 154 | 155 | /** 156 | * The size is either the compressed or uncompressed length. 157 | */ 158 | func (this *RecordioV2_Record) LenPayload() (v int, err error) { 159 | if (this._f_lenPayload) { 160 | return this.lenPayload, nil 161 | } 162 | tmp11, err := this.UncompressedPayloadLen.Value() 163 | if err != nil { 164 | return 0, err 165 | } 166 | tmp12, err := this.CompressedPayloadLen.Value() 167 | if err != nil { 168 | return 0, err 169 | } 170 | this.lenPayload = int((tmp11 ^ tmp12)) 171 | this._f_lenPayload = true 172 | return this.lenPayload, nil 173 | } 174 | -------------------------------------------------------------------------------- /kaitai/gokaitai/recordio_v2_test.go: -------------------------------------------------------------------------------- 1 | package gokaitai 2 | 3 | import ( 4 | "github.com/kaitai-io/kaitai_struct_go_runtime/kaitai" 5 | "github.com/stretchr/testify/require" 6 | "github.com/thomasjungblut/go-sstables/recordio" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func TestHappyPathReadV2(t *testing.T) { 12 | path := "../../recordio/test_files/v2_compat/recordio_UncompressedWriterMultiRecord_asc" 13 | f, err := os.Open(path) 14 | require.NoError(t, err) 15 | defer f.Close() 16 | 17 | rio := NewRecordioV2() 18 | err = rio.Read(kaitai.NewStream(f), nil, rio) 19 | require.NoError(t, err) 20 | 21 | require.Equal(t, uint32(2), rio.FileHeader.Version) 22 | require.Equal(t, RecordioV2_Compression__None, rio.FileHeader.CompressionType) 23 | for i := 0; i < len(rio.Record); i++ { 24 | record := rio.Record[i] 25 | require.Equal(t, i, len(record.Payload)) 26 | require.Equal(t, recordio.MagicNumberSeparatorLongBytes, record.Magic) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /kaitai/gokaitai/recordio_v3.go: -------------------------------------------------------------------------------- 1 | // Code generated by kaitai-struct-compiler from a .ksy source file. DO NOT EDIT. 2 | 3 | package gokaitai 4 | 5 | import ( 6 | "github.com/kaitai-io/kaitai_struct_go_runtime/kaitai" 7 | "bytes" 8 | ) 9 | 10 | 11 | type RecordioV3_Compression int 12 | const ( 13 | RecordioV3_Compression__None RecordioV3_Compression = 0 14 | RecordioV3_Compression__Snappy RecordioV3_Compression = 1 15 | RecordioV3_Compression__Gzip RecordioV3_Compression = 2 16 | ) 17 | type RecordioV3 struct { 18 | FileHeader *RecordioV3_FileHeader 19 | Record []*RecordioV3_Record 20 | _io *kaitai.Stream 21 | _root *RecordioV3 22 | _parent interface{} 23 | } 24 | func NewRecordioV3() *RecordioV3 { 25 | return &RecordioV3{ 26 | } 27 | } 28 | 29 | func (this *RecordioV3) Read(io *kaitai.Stream, parent interface{}, root *RecordioV3) (err error) { 30 | this._io = io 31 | this._parent = parent 32 | this._root = root 33 | 34 | tmp1 := NewRecordioV3_FileHeader() 35 | err = tmp1.Read(this._io, this, this._root) 36 | if err != nil { 37 | return err 38 | } 39 | this.FileHeader = tmp1 40 | for i := 1;; i++ { 41 | tmp2, err := this._io.EOF() 42 | if err != nil { 43 | return err 44 | } 45 | if tmp2 { 46 | break 47 | } 48 | tmp3 := NewRecordioV3_Record() 49 | err = tmp3.Read(this._io, this, this._root) 50 | if err != nil { 51 | return err 52 | } 53 | this.Record = append(this.Record, tmp3) 54 | } 55 | return err 56 | } 57 | 58 | /** 59 | * recordio header format to figure out the version it was written and whether the records are compressed. 60 | */ 61 | type RecordioV3_FileHeader struct { 62 | Version uint32 63 | CompressionType RecordioV3_Compression 64 | _io *kaitai.Stream 65 | _root *RecordioV3 66 | _parent *RecordioV3 67 | } 68 | func NewRecordioV3_FileHeader() *RecordioV3_FileHeader { 69 | return &RecordioV3_FileHeader{ 70 | } 71 | } 72 | 73 | func (this *RecordioV3_FileHeader) Read(io *kaitai.Stream, parent *RecordioV3, root *RecordioV3) (err error) { 74 | this._io = io 75 | this._parent = parent 76 | this._root = root 77 | 78 | tmp4, err := this._io.ReadU4le() 79 | if err != nil { 80 | return err 81 | } 82 | this.Version = uint32(tmp4) 83 | tmp5, err := this._io.ReadU4le() 84 | if err != nil { 85 | return err 86 | } 87 | this.CompressionType = RecordioV3_Compression(tmp5) 88 | return err 89 | } 90 | 91 | /** 92 | * The version of the recordio format used in this file. 93 | */ 94 | 95 | /** 96 | * The compression algorithm used. 0 means no compression, 1 means Snappy, 2 means Gzip. 97 | */ 98 | 99 | /** 100 | * recordio record is an "infinite" stream of magic number separated and length encoded byte arrays. 101 | */ 102 | type RecordioV3_Record struct { 103 | Magic []byte 104 | RecordNil uint8 105 | UncompressedPayloadLen *VlqBase128Le 106 | CompressedPayloadLen *VlqBase128Le 107 | Payload []byte 108 | _io *kaitai.Stream 109 | _root *RecordioV3 110 | _parent *RecordioV3 111 | _f_lenPayload bool 112 | lenPayload int 113 | } 114 | func NewRecordioV3_Record() *RecordioV3_Record { 115 | return &RecordioV3_Record{ 116 | } 117 | } 118 | 119 | func (this *RecordioV3_Record) Read(io *kaitai.Stream, parent *RecordioV3, root *RecordioV3) (err error) { 120 | this._io = io 121 | this._parent = parent 122 | this._root = root 123 | 124 | tmp6, err := this._io.ReadBytes(int(3)) 125 | if err != nil { 126 | return err 127 | } 128 | this.Magic = tmp6 129 | if !(bytes.Equal(this.Magic, []uint8{145, 141, 76})) { 130 | return kaitai.NewValidationNotEqualError([]uint8{145, 141, 76}, this.Magic, this._io, "/types/record/seq/0") 131 | } 132 | tmp7, err := this._io.ReadU1() 133 | if err != nil { 134 | return err 135 | } 136 | this.RecordNil = tmp7 137 | tmp8 := NewVlqBase128Le() 138 | err = tmp8.Read(this._io, this, nil) 139 | if err != nil { 140 | return err 141 | } 142 | this.UncompressedPayloadLen = tmp8 143 | tmp9 := NewVlqBase128Le() 144 | err = tmp9.Read(this._io, this, nil) 145 | if err != nil { 146 | return err 147 | } 148 | this.CompressedPayloadLen = tmp9 149 | tmp10, err := this.LenPayload() 150 | if err != nil { 151 | return err 152 | } 153 | tmp11, err := this._io.ReadBytes(int(tmp10)) 154 | if err != nil { 155 | return err 156 | } 157 | this.Payload = tmp11 158 | return err 159 | } 160 | 161 | /** 162 | * The size is either the compressed or uncompressed length. 163 | */ 164 | func (this *RecordioV3_Record) LenPayload() (v int, err error) { 165 | if (this._f_lenPayload) { 166 | return this.lenPayload, nil 167 | } 168 | tmp12, err := this.UncompressedPayloadLen.Value() 169 | if err != nil { 170 | return 0, err 171 | } 172 | tmp13, err := this.CompressedPayloadLen.Value() 173 | if err != nil { 174 | return 0, err 175 | } 176 | this.lenPayload = int((tmp12 ^ tmp13)) 177 | this._f_lenPayload = true 178 | return this.lenPayload, nil 179 | } 180 | 181 | /** 182 | * 1 means the record is nil, 0 otherwise 183 | */ 184 | -------------------------------------------------------------------------------- /kaitai/gokaitai/recordio_v3_test.go: -------------------------------------------------------------------------------- 1 | package gokaitai 2 | 3 | import ( 4 | "github.com/kaitai-io/kaitai_struct_go_runtime/kaitai" 5 | "github.com/stretchr/testify/require" 6 | "github.com/thomasjungblut/go-sstables/recordio" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func TestHappyPathReadV3(t *testing.T) { 12 | path := "../../recordio/test_files/v3_compat/recordio_UncompressedWriterMultiRecord_asc" 13 | f, err := os.Open(path) 14 | require.NoError(t, err) 15 | defer f.Close() 16 | 17 | rio := NewRecordioV3() 18 | err = rio.Read(kaitai.NewStream(f), nil, rio) 19 | require.NoError(t, err) 20 | 21 | require.Equal(t, uint32(3), rio.FileHeader.Version) 22 | require.Equal(t, RecordioV3_Compression__None, rio.FileHeader.CompressionType) 23 | for i := 0; i < len(rio.Record); i++ { 24 | record := rio.Record[i] 25 | require.Equal(t, i, len(record.Payload)) 26 | require.Equal(t, recordio.MagicNumberSeparatorLongBytes, record.Magic) 27 | require.Equal(t, uint8(0), record.RecordNil) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /kaitai/gokaitai/recordio_v4_test.go: -------------------------------------------------------------------------------- 1 | package gokaitai 2 | 3 | import ( 4 | "github.com/kaitai-io/kaitai_struct_go_runtime/kaitai" 5 | "github.com/stretchr/testify/require" 6 | "github.com/thomasjungblut/go-sstables/recordio" 7 | "os" 8 | "testing" 9 | ) 10 | 11 | func TestHappyPathReadV4(t *testing.T) { 12 | path := "../../recordio/test_files/v4_compat/recordio_UncompressedWriterMultiRecord_asc" 13 | f, err := os.Open(path) 14 | require.NoError(t, err) 15 | defer f.Close() 16 | 17 | rio := NewRecordioV4() 18 | err = rio.Read(kaitai.NewStream(f), nil, rio) 19 | require.NoError(t, err) 20 | 21 | require.Equal(t, uint32(4), rio.FileHeader.Version) 22 | require.Equal(t, RecordioV4_Compression__None, rio.FileHeader.CompressionType) 23 | for i := 0; i < len(rio.Record); i++ { 24 | record := rio.Record[i] 25 | require.Equal(t, i, len(record.Payload)) 26 | require.Equal(t, recordio.MagicNumberSeparatorLongBytes, record.Magic) 27 | require.Equal(t, uint8(0), record.RecordNil) 28 | 29 | value, err := record.Crc32Checksum.Value() 30 | require.NoError(t, err) 31 | require.NotZero(t, value) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /kaitai/recordio_v2.ksy: -------------------------------------------------------------------------------- 1 | meta: 2 | id: recordio_v2 3 | endian: le 4 | imports: 5 | - vlq_base128_le 6 | seq: 7 | - id: file_header 8 | type: file_header 9 | - id: record 10 | type: record 11 | repeat: eos 12 | types: 13 | file_header: 14 | doc: | 15 | recordio header format to figure out the version it was written and whether the records are compressed. 16 | seq: 17 | - id: version 18 | type: u4 19 | doc: The version of the recordio format used in this file. 20 | - id: compression_type 21 | type: u4 22 | enum: compression 23 | doc: The compression algorithm used. 0 means no compression, 1 means Snappy, 2 means Gzip. 24 | record: 25 | doc: | 26 | recordio record is an "infinite" stream of magic number separated and length encoded byte arrays. 27 | seq: 28 | - id: magic 29 | contents: [0x91, 0x8D, 0x4C] 30 | - id: uncompressed_payload_len 31 | type: vlq_base128_le 32 | - id: compressed_payload_len 33 | type: vlq_base128_le 34 | - id: payload 35 | size: len_payload 36 | instances: 37 | len_payload: 38 | value: uncompressed_payload_len.value ^ compressed_payload_len.value 39 | doc: The size is either the compressed or uncompressed length. 40 | enums: 41 | compression: 42 | 0: none 43 | 1: snappy 44 | 2: gzip -------------------------------------------------------------------------------- /kaitai/recordio_v3.ksy: -------------------------------------------------------------------------------- 1 | meta: 2 | id: recordio_v3 3 | endian: le 4 | imports: 5 | - vlq_base128_le 6 | seq: 7 | - id: file_header 8 | type: file_header 9 | - id: record 10 | type: record 11 | repeat: eos 12 | types: 13 | file_header: 14 | doc: | 15 | recordio header format to figure out the version it was written and whether the records are compressed. 16 | seq: 17 | - id: version 18 | type: u4 19 | doc: The version of the recordio format used in this file. 20 | - id: compression_type 21 | type: u4 22 | enum: compression 23 | doc: The compression algorithm used. 0 means no compression, 1 means Snappy, 2 means Gzip. 24 | record: 25 | doc: | 26 | recordio record is an "infinite" stream of magic number separated and length encoded byte arrays. 27 | seq: 28 | - id: magic 29 | contents: [0x91, 0x8D, 0x4C] 30 | - id: record_nil 31 | type: u1 32 | doc: 1 means the record is nil, 0 otherwise 33 | - id: uncompressed_payload_len 34 | type: vlq_base128_le 35 | - id: compressed_payload_len 36 | type: vlq_base128_le 37 | - id: payload 38 | size: len_payload 39 | instances: 40 | len_payload: 41 | value: uncompressed_payload_len.value ^ compressed_payload_len.value 42 | doc: The size is either the compressed or uncompressed length. 43 | enums: 44 | compression: 45 | 0: none 46 | 1: snappy 47 | 2: gzip -------------------------------------------------------------------------------- /kaitai/recordio_v4.ksy: -------------------------------------------------------------------------------- 1 | meta: 2 | id: recordio_v4 3 | endian: le 4 | imports: 5 | - vlq_base128_le 6 | seq: 7 | - id: file_header 8 | type: file_header 9 | - id: record 10 | type: record 11 | repeat: eos 12 | types: 13 | file_header: 14 | doc: | 15 | recordio header format to figure out the version it was written and whether the records are compressed. 16 | seq: 17 | - id: version 18 | type: u4 19 | doc: The version of the recordio format used in this file. 20 | - id: compression_type 21 | type: u4 22 | enum: compression 23 | doc: The compression algorithm used. 0 means no compression, 1 means Snappy, 2 means Gzip. 24 | record: 25 | doc: | 26 | recordio record is an "infinite" stream of magic number separated and length encoded byte arrays. 27 | seq: 28 | - id: magic 29 | contents: [0x91, 0x8D, 0x4C] 30 | - id: record_nil 31 | type: u1 32 | doc: 1 means the record is nil, 0 otherwise 33 | - id: uncompressed_payload_len 34 | type: vlq_base128_le 35 | - id: compressed_payload_len 36 | type: vlq_base128_le 37 | - id: crc32_checksum 38 | type: vlq_base128_le 39 | doc: The checksum is a CRC32 (Castagnoli table mapping) built from the previous fields of the record header. 40 | - id: payload 41 | size: len_payload 42 | instances: 43 | len_payload: 44 | value: uncompressed_payload_len.value ^ compressed_payload_len.value 45 | doc: The size is either the compressed or uncompressed length. 46 | enums: 47 | compression: 48 | 0: none 49 | 1: snappy 50 | 2: gzip -------------------------------------------------------------------------------- /kaitai/vlq_base128_le.ksy: -------------------------------------------------------------------------------- 1 | # source: https://github.com/kaitai-io/kaitai_struct_formats/blob/d1e58e36f26ecff1375c6022dc040342914ee4d1/common/vlq_base128_le.ksy 2 | meta: 3 | id: vlq_base128_le 4 | title: Variable length quantity, unsigned integer, base128, little-endian 5 | license: CC0-1.0 6 | ks-version: 0.7 7 | doc: | 8 | A variable-length unsigned integer using base128 encoding. 1-byte groups 9 | consist of 1-bit flag of continuation and 7-bit value chunk, and are ordered 10 | "least significant group first", i.e. in "little-endian" manner. 11 | 12 | This particular encoding is specified and used in: 13 | 14 | * DWARF debug file format, where it's dubbed "unsigned LEB128" or "ULEB128". 15 | http://dwarfstd.org/doc/dwarf-2.0.0.pdf - page 139 16 | * Google Protocol Buffers, where it's called "Base 128 Varints". 17 | https://developers.google.com/protocol-buffers/docs/encoding?csw=1#varints 18 | * Apache Lucene, where it's called "VInt" 19 | http://lucene.apache.org/core/3_5_0/fileformats.html#VInt 20 | * Apache Avro uses this as a basis for integer encoding, adding ZigZag on 21 | top of it for signed ints 22 | http://avro.apache.org/docs/current/spec.html#binary_encode_primitive 23 | 24 | More information on this encoding is available at https://en.wikipedia.org/wiki/LEB128 25 | 26 | This particular implementation supports serialized values to up 8 bytes long. 27 | seq: 28 | - id: groups 29 | type: group 30 | repeat: until 31 | repeat-until: not _.has_next 32 | types: 33 | group: 34 | doc: | 35 | One byte group, clearly divided into 7-bit "value" chunk and 1-bit "continuation" flag. 36 | seq: 37 | - id: b 38 | type: u1 39 | instances: 40 | has_next: 41 | value: (b & 0b1000_0000) != 0 42 | doc: If true, then we have more bytes to read 43 | value: 44 | value: b & 0b0111_1111 45 | doc: The 7-bit (base128) numeric value chunk of this group 46 | instances: 47 | len: 48 | value: groups.size 49 | value: 50 | value: >- 51 | groups[0].value 52 | + (len >= 2 ? (groups[1].value << 7) : 0) 53 | + (len >= 3 ? (groups[2].value << 14) : 0) 54 | + (len >= 4 ? (groups[3].value << 21) : 0) 55 | + (len >= 5 ? (groups[4].value << 28) : 0) 56 | + (len >= 6 ? (groups[5].value << 35) : 0) 57 | + (len >= 7 ? (groups[6].value << 42) : 0) 58 | + (len >= 8 ? (groups[7].value << 49) : 0) 59 | doc: Resulting value as normal integer -------------------------------------------------------------------------------- /memstore/README.md: -------------------------------------------------------------------------------- 1 | ## Using MemStore 2 | 3 | Memstore acts like a sorted dictionary that can be flushed into an SSTable representation on disk. 4 | It allows you to add, update, retrieve and delete elements by their key, of which both are represented by byte slices. 5 | 6 | A simple example below illustrates all functionality of the memstore: 7 | 8 | ```go 9 | path := "/tmp/sstable-ms-ex/" 10 | defer os.RemoveAll(path) 11 | 12 | ms := memstore.NewMemStore() 13 | ms.Add([]byte{1}, []byte{1}) 14 | ms.Add([]byte{2}, []byte{2}) 15 | ms.Upsert([]byte{1}, []byte{2}) 16 | ms.Delete([]byte{2}) 17 | ms.DeleteIfExists([]byte{3}) 18 | value, _ := ms.Get([]byte{1}) 19 | log.Printf("value for key 1: %d", value) // yields 2 20 | 21 | size := ms.EstimatedSizeInBytes() 22 | log.Printf("memstore size in bytes: %d", size) // yields 3 23 | 24 | ms.Flush(sstables.WriteBasePath(path)) 25 | ``` 26 | 27 | You can get the full example from [examples/memstore.go](/_examples/memstore.go). 28 | -------------------------------------------------------------------------------- /memstore/memstore_sstable_iterator.go: -------------------------------------------------------------------------------- 1 | package memstore 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/skiplist" 6 | "github.com/thomasjungblut/go-sstables/sstables" 7 | ) 8 | 9 | type SkipListSStableIterator struct { 10 | iterator skiplist.IteratorI[[]byte, ValueStruct] 11 | } 12 | 13 | func (s SkipListSStableIterator) Next() ([]byte, []byte, error) { 14 | key, val, err := s.iterator.Next() 15 | if err != nil { 16 | if errors.Is(err, skiplist.Done) { 17 | return nil, nil, sstables.Done 18 | } else { 19 | return nil, nil, err 20 | } 21 | } 22 | return key, *val.value, nil 23 | } 24 | -------------------------------------------------------------------------------- /pq/priority_queue.go: -------------------------------------------------------------------------------- 1 | package pq 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/thomasjungblut/go-sstables/skiplist" 7 | ) 8 | 9 | // Done indicates an iterator has returned all items. 10 | // https://github.com/GoogleCloudPlatform/google-cloud-go/wiki/Iterator-Guidelines 11 | var Done = errors.New("no more items in iterator") 12 | 13 | type IteratorWithContext[K any, V any, CTX any] interface { 14 | // Next returns the next key, value in sequence. 15 | // Returns Done as the error when the iterator is exhausted. 16 | Next() (K, V, error) 17 | // Context returns the context to identify the given iterator. 18 | Context() CTX 19 | } 20 | 21 | type PriorityQueueI[K any, V any, CTX any] interface { 22 | // Next returns the next key, value and context in sequence. 23 | // Returns Done as the error when the iterator is exhausted. 24 | Next() (K, V, CTX, error) 25 | } 26 | 27 | type Element[K any, V any, CTX any] struct { 28 | heapIndex int 29 | key K 30 | value V 31 | iterator IteratorWithContext[K, V, CTX] 32 | } 33 | 34 | type PriorityQueue[K any, V any, CTX any] struct { 35 | size int 36 | heap []*Element[K, V, CTX] 37 | comp skiplist.Comparator[K] 38 | } 39 | 40 | func (pq *PriorityQueue[K, V, CTX]) lessThan(i, j *Element[K, V, CTX]) bool { 41 | return pq.comp.Compare(i.key, j.key) < 0 42 | } 43 | 44 | func (pq *PriorityQueue[K, V, CTX]) swap(i, j int) { 45 | pq.heap[i], pq.heap[j] = pq.heap[j], pq.heap[i] 46 | pq.heap[i].heapIndex = i 47 | pq.heap[j].heapIndex = j 48 | } 49 | 50 | func (pq *PriorityQueue[K, V, CTX]) init(iterators []IteratorWithContext[K, V, CTX]) error { 51 | // reserve the 0th element for nil, makes it easier to implement the rest of the logic 52 | pq.heap = []*Element[K, V, CTX]{nil} 53 | for i, it := range iterators { 54 | e := &Element[K, V, CTX]{heapIndex: i, iterator: it} 55 | err := pq.fillNext(e) 56 | if err == nil { 57 | pq.heap = append(pq.heap, e) 58 | pq.size++ 59 | pq.upHeap(pq.size) 60 | } else if !errors.Is(err, Done) { 61 | return fmt.Errorf("INIT couldn't fill next heap entry: %w", err) 62 | } 63 | } 64 | 65 | return nil 66 | } 67 | 68 | func (pq *PriorityQueue[K, V, CTX]) Next() (_ K, _ V, _ CTX, err error) { 69 | err = Done 70 | 71 | if pq.size == 0 { 72 | return 73 | } 74 | // since we reserved index 0 for nil, the minimum element is always at index 1 75 | top := pq.heap[1] 76 | k := top.key 77 | v := top.value 78 | c := top.iterator.Context() 79 | err = pq.fillNext(top) 80 | // if we encounter a real error, we're returning immediately 81 | if err != nil && !errors.Is(err, Done) { 82 | err = fmt.Errorf("NEXT couldn't fill next heap entry: %w", err) 83 | return 84 | } 85 | 86 | // remove the element from the heap completely if its iterator is exhausted 87 | if errors.Is(err, Done) { 88 | // move the root away to the bottom leaf 89 | pq.swap(1, pq.size) 90 | // and chop it off the slice 91 | pq.heap = pq.heap[0:pq.size] 92 | pq.size-- 93 | } 94 | 95 | // always down the heap at the end 96 | pq.downHeap() 97 | 98 | return k, v, c, nil 99 | } 100 | 101 | func (pq *PriorityQueue[K, V, CTX]) upHeap(i int) { 102 | element := pq.heap[i] 103 | j := i >> 1 104 | for j > 0 && pq.lessThan(element, pq.heap[j]) { 105 | pq.heap[i] = pq.heap[j] 106 | i = j 107 | j = j >> 1 108 | } 109 | pq.heap[i] = element 110 | } 111 | 112 | func (pq *PriorityQueue[K, V, CTX]) downHeap() { 113 | if pq.size == 0 { 114 | return 115 | } 116 | 117 | i := 1 118 | element := pq.heap[i] 119 | j := i << 1 120 | k := j + 1 121 | if k <= pq.size && pq.lessThan(pq.heap[k], pq.heap[j]) { 122 | j = k 123 | } 124 | for j <= pq.size && pq.lessThan(pq.heap[j], element) { 125 | pq.heap[i] = pq.heap[j] 126 | i = j 127 | j = i << 1 128 | k = j + 1 129 | if k <= pq.size && pq.lessThan(pq.heap[k], pq.heap[j]) { 130 | j = k 131 | } 132 | } 133 | pq.heap[i] = element 134 | } 135 | 136 | func (pq *PriorityQueue[K, V, CTX]) fillNext(item *Element[K, V, CTX]) error { 137 | k, v, e := item.iterator.Next() 138 | item.key = k 139 | item.value = v 140 | return e 141 | } 142 | 143 | func NewPriorityQueue[K any, V any, CTX any](comp skiplist.Comparator[K], iterators []IteratorWithContext[K, V, CTX]) (PriorityQueueI[K, V, CTX], error) { 144 | q := &PriorityQueue[K, V, CTX]{comp: comp} 145 | err := q.init(iterators) 146 | if err != nil { 147 | return nil, err 148 | } 149 | return q, nil 150 | } 151 | -------------------------------------------------------------------------------- /pq/priority_queue_test.go: -------------------------------------------------------------------------------- 1 | package pq 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/stretchr/testify/require" 7 | "github.com/thomasjungblut/go-sstables/skiplist" 8 | "sort" 9 | "testing" 10 | ) 11 | 12 | type SliceIterator struct { 13 | list []int // every int is treated as key/value as a single element byte array 14 | currentIndex int 15 | context int 16 | } 17 | 18 | func (l *SliceIterator) Next() (int, int, error) { 19 | if l.currentIndex >= len(l.list) { 20 | return 0, 0, Done 21 | } 22 | 23 | k := l.list[l.currentIndex] 24 | l.currentIndex++ 25 | 26 | return k, k + 1, nil 27 | } 28 | 29 | func (l *SliceIterator) Context() int { 30 | return l.context 31 | } 32 | 33 | func TestTwoListsHappyPath(t *testing.T) { 34 | assertMergeAndListMatches(t, []int{1, 3}, []int{2, 4}) 35 | } 36 | 37 | func TestSingleList(t *testing.T) { 38 | assertMergeAndListMatches(t, []int{1, 3}) 39 | } 40 | 41 | func TestSingleListEmpty(t *testing.T) { 42 | assertMergeAndListMatches(t, []int{}) 43 | } 44 | 45 | func TestTwoListsSameItems(t *testing.T) { 46 | assertMergeAndListMatches(t, []int{1, 3}, []int{1, 3}) 47 | } 48 | 49 | func TestMultiList(t *testing.T) { 50 | assertMergeAndListMatches(t, []int{1, 2}, []int{4, 6}, []int{0, 9}, []int{5, 8}) 51 | } 52 | 53 | func TestTwoListsOneLonger(t *testing.T) { 54 | assertMergeAndListMatches(t, []int{1, 5, 7, 8, 9}, []int{3, 4, 6, 10, 11, 12, 13, 14, 15}) 55 | } 56 | 57 | func TestTwoListsLeftLonger(t *testing.T) { 58 | assertMergeAndListMatches(t, []int{1, 5, 7, 8, 9, 25, 27, 100, 250}, []int{3, 4, 6, 10, 14, 15}) 59 | } 60 | 61 | func TestMultiListConsecutive(t *testing.T) { 62 | assertMergeAndListMatches(t, []int{1, 2}, []int{3, 4}, []int{5, 6}) 63 | } 64 | 65 | func TestMultiListConsecutiveReversed(t *testing.T) { 66 | assertMergeAndListMatches(t, []int{5, 6}, []int{3, 4}, []int{1, 2}) 67 | } 68 | 69 | func TestMultiListMixed(t *testing.T) { 70 | assertMergeAndListMatches(t, []int{1, 5, 8, 19}, []int{2, 3, 4, 12}, []int{6, 9, 25}) 71 | } 72 | 73 | func TestMultiListShortExhaust(t *testing.T) { 74 | assertMergeAndListMatches(t, []int{4, 5, 8, 19}, []int{4, 6, 9, 12}, []int{1, 2, 3}) 75 | } 76 | 77 | func TestMultiListEmptyMiddle(t *testing.T) { 78 | assertMergeAndListMatches(t, []int{1, 9}, []int{}, []int{5, 6}) 79 | } 80 | 81 | func TestMultiListAllEmpty(t *testing.T) { 82 | assertMergeAndListMatches(t, []int{}, []int{}, []int{}) 83 | } 84 | 85 | func assertMergeAndListMatches(t *testing.T, lists ...[]int) { 86 | var iterators []IteratorWithContext[int, int, int] 87 | var expected []int 88 | 89 | for _, v := range lists { 90 | iterators = append(iterators, &SliceIterator{list: v, context: len(v)}) 91 | expected = append(expected, v...) 92 | } 93 | 94 | pq, err := NewPriorityQueue[int, int, int](skiplist.OrderedComparator[int]{}, iterators) 95 | require.Nil(t, err) 96 | 97 | var actualKeys []int 98 | for { 99 | k, v, _, err := pq.Next() 100 | if errors.Is(err, Done) { 101 | break 102 | } 103 | 104 | assert.Equal(t, k+1, v) 105 | actualKeys = append(actualKeys, k) 106 | } 107 | 108 | sort.Ints(expected) 109 | assert.Exactly(t, expected, actualKeys) 110 | } 111 | -------------------------------------------------------------------------------- /recordio/buffered_io.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | type BufferedIOFactory struct { 8 | } 9 | 10 | func (d BufferedIOFactory) CreateNewReader(filePath string, bufSize int) (*os.File, ByteReaderResetCount, error) { 11 | readFile, err := os.OpenFile(filePath, os.O_RDONLY|os.O_CREATE, 0666) 12 | if err != nil { 13 | return nil, nil, err 14 | } 15 | 16 | block := make([]byte, bufSize) 17 | return readFile, NewCountingByteReader(NewReaderBuf(readFile, block)), nil 18 | } 19 | 20 | func (d BufferedIOFactory) CreateNewWriter(filePath string, bufSize int) (*os.File, WriteSeekerCloserFlusher, error) { 21 | writeFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666) 22 | if err != nil { 23 | return nil, nil, err 24 | } 25 | 26 | block := make([]byte, bufSize) 27 | return writeFile, NewWriterBuf(writeFile, block), nil 28 | } 29 | -------------------------------------------------------------------------------- /recordio/buffered_io_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestBufferedIOFactory_CreateNewReader(t *testing.T) { 11 | temp, err := os.CreateTemp("", "TestBufferedIOFactory_CreateNewReader") 12 | require.NoError(t, err) 13 | require.NoError(t, temp.Close()) 14 | 15 | f := BufferedIOFactory{} 16 | file, buf, err := f.CreateNewReader(temp.Name(), 4096) 17 | require.NoError(t, err) 18 | defer closeCleanFile(t, file) 19 | 20 | assert.Equal(t, 4096, buf.Size()) 21 | } 22 | 23 | func TestBufferedIOFactory_CreateNewWriter(t *testing.T) { 24 | temp, err := os.CreateTemp("", "TestBufferedIOFactory_CreateNewWriter") 25 | require.NoError(t, err) 26 | require.NoError(t, temp.Close()) 27 | 28 | f := BufferedIOFactory{} 29 | file, buf, err := f.CreateNewWriter(temp.Name(), 4096) 30 | require.NoError(t, err) 31 | defer closeCleanFile(t, file) 32 | 33 | assert.Equal(t, 4096, buf.Size()) 34 | } 35 | -------------------------------------------------------------------------------- /recordio/bufio_vendor_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "io" 7 | "testing" 8 | ) 9 | 10 | type closingWriter struct { 11 | buf []byte 12 | } 13 | 14 | func (w *closingWriter) Write(p []byte) (n int, err error) { 15 | copy(w.buf, p) // simply overwrite for testing purposes 16 | return len(p), nil 17 | } 18 | 19 | func (w *closingWriter) Seek(offset int64, whence int) (int64, error) { 20 | return offset, nil 21 | } 22 | 23 | func (*closingWriter) Close() error { 24 | return nil 25 | } 26 | 27 | func TestCreateNewBufferWithSlice(t *testing.T) { 28 | sink := &closingWriter{make([]byte, 6)} 29 | wBuf := NewWriterBuf(sink, make([]byte, 4)) 30 | assert.Equal(t, 4, wBuf.Size()) 31 | 32 | _, err := wBuf.Write([]byte{13, 6, 91, 22}) 33 | require.NoError(t, err) 34 | // buffer should not been flushed so far 35 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0}, sink.buf) 36 | require.NoError(t, wBuf.Flush()) 37 | assert.Equal(t, []byte{13, 6, 91, 22, 0, 0}, sink.buf) 38 | } 39 | 40 | func TestCreateNewBufferCloseFlushes(t *testing.T) { 41 | sink := &closingWriter{make([]byte, 6)} 42 | wBuf := NewWriterBuf(sink, make([]byte, 4)) 43 | assert.Equal(t, 4, wBuf.Size()) 44 | 45 | _, err := wBuf.Write([]byte{13, 6, 91, 22}) 46 | require.NoError(t, err) 47 | // buffer should not been flushed so far 48 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0}, sink.buf) 49 | require.NoError(t, wBuf.Close()) 50 | assert.Equal(t, []byte{13, 6, 91, 22, 0, 0}, sink.buf) 51 | } 52 | 53 | func TestSeekFlushes(t *testing.T) { 54 | sink := &closingWriter{make([]byte, 6)} 55 | wBuf := NewWriterBuf(sink, make([]byte, 4)) 56 | assert.Equal(t, 4, wBuf.Size()) 57 | 58 | _, err := wBuf.Write([]byte{13, 6, 91, 22}) 59 | require.NoError(t, err) 60 | // buffer should not been flushed so far 61 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0}, sink.buf) 62 | 63 | _, err = wBuf.Seek(0, io.SeekStart) 64 | require.NoError(t, err) 65 | assert.Equal(t, []byte{13, 6, 91, 22, 0, 0}, sink.buf) 66 | 67 | require.NoError(t, wBuf.Close()) 68 | assert.Equal(t, []byte{13, 6, 91, 22, 0, 0}, sink.buf) 69 | } 70 | 71 | func TestCreateNewBufferWithAlignedSlice(t *testing.T) { 72 | sink := &closingWriter{make([]byte, 8)} 73 | wBuf := NewAlignedWriterBuf(sink, make([]byte, 4)) 74 | assert.Equal(t, 4, wBuf.Size()) 75 | 76 | _, err := wBuf.Write([]byte{13, 6, 91}) 77 | require.NoError(t, err) 78 | // buffer should not been flushed so far 79 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0}, sink.buf) 80 | require.NoError(t, wBuf.Flush()) 81 | assert.Equal(t, []byte{13, 6, 91, 0, 0, 0, 0, 0}, sink.buf) 82 | } 83 | 84 | func TestCreateNewBufferWithAlignedSliceZerosBuffer(t *testing.T) { 85 | sink := &closingWriter{make([]byte, 8)} 86 | dirtyBuf := []byte{1, 1, 1, 1} 87 | wBuf := NewAlignedWriterBuf(sink, dirtyBuf) 88 | assert.Equal(t, 4, wBuf.Size()) 89 | 90 | _, err := wBuf.Write([]byte{13, 6}) 91 | require.NoError(t, err) 92 | // buffer should not been flushed so far 93 | assert.Equal(t, []byte{0, 0, 0, 0, 0, 0, 0, 0}, sink.buf) 94 | require.NoError(t, wBuf.Flush()) 95 | assert.Equal(t, []byte{13, 6, 0, 0, 0, 0, 0, 0}, sink.buf) 96 | } 97 | -------------------------------------------------------------------------------- /recordio/checksum_byte_reader.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "fmt" 5 | "hash" 6 | "hash/crc32" 7 | "io" 8 | ) 9 | 10 | // checksumByteReader generates a checksum on the bytes read so far 11 | type checksumByteReader struct { 12 | io.ByteReader 13 | 14 | bytes []byte 15 | crc hash.Hash32 16 | idx int 17 | } 18 | 19 | func (h *checksumByteReader) ReadByte() (byte, error) { 20 | b, err := h.ByteReader.ReadByte() 21 | if err != nil { 22 | return b, err 23 | } 24 | 25 | if h.idx >= len(h.bytes) { 26 | return b, fmt.Errorf("checksum byte reader out of range: %d, only have %d", h.idx, len(h.bytes)) 27 | } 28 | 29 | h.bytes[h.idx] = b 30 | h.idx++ 31 | 32 | return b, err 33 | } 34 | 35 | func (h *checksumByteReader) Reset() { 36 | h.crc.Reset() 37 | h.idx = 0 38 | } 39 | 40 | func (h *checksumByteReader) Count() int { 41 | return h.idx 42 | } 43 | 44 | func (h *checksumByteReader) Checksum() (uint64, error) { 45 | _, err := h.crc.Write(h.bytes[:h.idx]) 46 | if err != nil { 47 | return 0, err 48 | } 49 | return uint64(h.crc.Sum32()), nil 50 | } 51 | 52 | func newChecksumByteReader(r io.ByteReader, cachedBytes []byte) *checksumByteReader { 53 | crc := crc32.New(crc32.MakeTable(crc32.Castagnoli)) 54 | return &checksumByteReader{ 55 | ByteReader: r, 56 | bytes: cachedBytes, 57 | crc: crc, 58 | idx: 0, 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /recordio/checksum_byte_reader_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "github.com/stretchr/testify/require" 8 | "io" 9 | "testing" 10 | ) 11 | 12 | func TestChecksumHappyPath(t *testing.T) { 13 | reader := newChecksumByteReader(bytes.NewReader(MagicNumberSeparatorLongBytes), 14 | make([]byte, len(MagicNumberSeparatorLongBytes))) 15 | var actual []byte 16 | for { 17 | b, err := reader.ReadByte() 18 | if err != nil { 19 | if errors.Is(err, io.EOF) { 20 | break 21 | } 22 | require.NoError(t, err) 23 | } 24 | actual = append(actual, b) 25 | } 26 | 27 | require.Equal(t, MagicNumberSeparatorLongBytes, actual) 28 | checksum, err := reader.Checksum() 29 | require.NoError(t, err) 30 | require.Equal(t, uint64(0x967294b), checksum) 31 | 32 | reader.Reset() 33 | checksum, err = reader.Checksum() 34 | require.NoError(t, err) 35 | require.Equal(t, uint64(0), checksum) 36 | } 37 | 38 | func TestChecksumOutOfRange(t *testing.T) { 39 | reader := newChecksumByteReader(bytes.NewReader(MagicNumberSeparatorLongBytes), 40 | make([]byte, 0)) 41 | 42 | _, err := reader.ReadByte() 43 | require.Equal(t, fmt.Errorf("checksum byte reader out of range: 0, only have 0"), err) 44 | } 45 | -------------------------------------------------------------------------------- /recordio/compressor/compressor.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | type CompressionI interface { 4 | // Compress compresses the given record of bytes 5 | Compress(record []byte) ([]byte, error) 6 | // Decompress decompresses the given byte buffer 7 | Decompress(buf []byte) ([]byte, error) 8 | 9 | // CompressWithBuf compresses the given record of bytes and a buffer where to compress into. 10 | // If the buffer doesn't fit, it will resize it (truncate/enlarging copy). 11 | // Thus, it's important to use the returned buffer value. 12 | CompressWithBuf(record []byte, destinationBuffer []byte) ([]byte, error) 13 | // DecompressWithBuf decompresses the given byte buffer and a buffer where to decompress into. 14 | // If the buffer doesn't fit, it will resize it (truncate/enlarging copy). 15 | // Thus, it's important to use the returned buffer value. 16 | DecompressWithBuf(buf []byte, destinationBuffer []byte) ([]byte, error) 17 | } 18 | -------------------------------------------------------------------------------- /recordio/compressor/gzip_compression.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | ) 7 | 8 | type GzipCompressor struct { 9 | } 10 | 11 | func (c *GzipCompressor) Compress(record []byte) ([]byte, error) { 12 | var buf bytes.Buffer 13 | return compressWithBytesBuffer(record, &buf) 14 | } 15 | 16 | func (c *GzipCompressor) CompressWithBuf(record []byte, destinationBuffer []byte) ([]byte, error) { 17 | // we have to set the length of the buffer (keeping capacity) to make sure gzip doesn't append 18 | destinationBuffer = destinationBuffer[:0] 19 | buf := bytes.NewBuffer(destinationBuffer) 20 | return compressWithBytesBuffer(record, buf) 21 | } 22 | 23 | func compressWithBytesBuffer(record []byte, buf *bytes.Buffer) ([]byte, error) { 24 | writer, err := gzip.NewWriterLevel(buf, gzip.DefaultCompression) 25 | if err != nil { 26 | return nil, err 27 | } 28 | _, err = writer.Write(record) 29 | if err != nil { 30 | return nil, err 31 | } 32 | err = writer.Close() 33 | if err != nil { 34 | return nil, err 35 | } 36 | return buf.Bytes(), nil 37 | } 38 | 39 | func (c *GzipCompressor) Decompress(buf []byte) ([]byte, error) { 40 | reader, err := gzip.NewReader(bytes.NewBuffer(buf)) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | var resultBuffer bytes.Buffer 46 | _, err = resultBuffer.ReadFrom(reader) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | return resultBuffer.Bytes(), nil 52 | } 53 | 54 | func (c *GzipCompressor) DecompressWithBuf(buf []byte, destinationBuffer []byte) ([]byte, error) { 55 | // we have to set the length of the buffer (keeping capacity) to make sure gzip doesn't append 56 | destinationBuffer = destinationBuffer[:0] 57 | reader, err := gzip.NewReader(bytes.NewBuffer(buf)) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | resultBuffer := bytes.NewBuffer(destinationBuffer) 63 | _, err = resultBuffer.ReadFrom(reader) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return resultBuffer.Bytes(), nil 69 | } 70 | -------------------------------------------------------------------------------- /recordio/compressor/gzip_compression_test.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSimpleGzipCompression(t *testing.T) { 9 | comp := GzipCompressor{} 10 | data := "some data" 11 | 12 | compressedBytes, err := comp.Compress([]byte(data)) 13 | assert.Nil(t, err) 14 | assert.Equal(t, 33, len(compressedBytes)) 15 | 16 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 17 | } 18 | 19 | func TestSimpleGzipCompressionWithBuffers(t *testing.T) { 20 | comp := GzipCompressor{} 21 | data := "some data" 22 | 23 | destBuf := make([]byte, 33) 24 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 25 | assert.Nil(t, err) 26 | assert.Equal(t, 33, len(compressedBytes)) 27 | 28 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 29 | } 30 | 31 | func TestSimpleGzipCompressionWithSmallerBuffer(t *testing.T) { 32 | comp := GzipCompressor{} 33 | data := "some data" 34 | 35 | destBuf := make([]byte, 30) 36 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 37 | assert.Nil(t, err) 38 | assert.Equal(t, 33, len(compressedBytes)) 39 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 40 | } 41 | 42 | func TestSimpleGzipCompressionWithLargerBuffer(t *testing.T) { 43 | comp := GzipCompressor{} 44 | data := "some data" 45 | 46 | destBuf := make([]byte, 40) 47 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 48 | assert.Nil(t, err) 49 | assert.Equal(t, 33, len(compressedBytes)) 50 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 51 | } 52 | -------------------------------------------------------------------------------- /recordio/compressor/lzw_compessor_test.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSimpleLzwCompression(t *testing.T) { 9 | comp := LzwCompressor{} 10 | data := "some data" 11 | 12 | compressedBytes, err := comp.Compress([]byte(data)) 13 | assert.Nil(t, err) 14 | assert.Equal(t, 13, len(compressedBytes)) 15 | 16 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 17 | } 18 | 19 | func TestSimpleLzwCompressionWithBuffers(t *testing.T) { 20 | comp := LzwCompressor{} 21 | data := "some data" 22 | 23 | destBuf := make([]byte, 33) 24 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 25 | assert.Nil(t, err) 26 | assert.Equal(t, 13, len(compressedBytes)) 27 | 28 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 29 | } 30 | 31 | func TestSimpleLzwCompressionWithSmallerBuffer(t *testing.T) { 32 | comp := LzwCompressor{} 33 | data := "some data" 34 | 35 | destBuf := make([]byte, 30) 36 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 37 | assert.Nil(t, err) 38 | assert.Equal(t, 13, len(compressedBytes)) 39 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 40 | } 41 | 42 | func TestSimpleLzwCompressionWithLargerBuffer(t *testing.T) { 43 | comp := LzwCompressor{} 44 | data := "some data" 45 | 46 | destBuf := make([]byte, 40) 47 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 48 | assert.Nil(t, err) 49 | assert.Equal(t, 13, len(compressedBytes)) 50 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 51 | } 52 | -------------------------------------------------------------------------------- /recordio/compressor/lzw_compressor.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "bytes" 5 | "compress/lzw" 6 | ) 7 | 8 | type LzwCompressor struct { 9 | } 10 | 11 | func (l LzwCompressor) Compress(record []byte) ([]byte, error) { 12 | var buf bytes.Buffer 13 | writer := lzw.NewWriter(&buf, lzw.LSB, 8) 14 | _, err := writer.Write(record) 15 | if err != nil { 16 | return nil, err 17 | } 18 | err = writer.Close() 19 | if err != nil { 20 | return nil, err 21 | } 22 | return buf.Bytes(), nil 23 | } 24 | 25 | func (l LzwCompressor) Decompress(buf []byte) ([]byte, error) { 26 | reader := lzw.NewReader(bytes.NewBuffer(buf), lzw.LSB, 8) 27 | var resultBuffer bytes.Buffer 28 | _, err := resultBuffer.ReadFrom(reader) 29 | if err != nil { 30 | return nil, err 31 | } 32 | 33 | return resultBuffer.Bytes(), nil 34 | } 35 | 36 | func (l LzwCompressor) CompressWithBuf(record []byte, destinationBuffer []byte) ([]byte, error) { 37 | // we have to set the length of the buffer (keeping capacity) to make sure gzip doesn't append 38 | destinationBuffer = destinationBuffer[:0] 39 | buf := bytes.NewBuffer(destinationBuffer) 40 | writer := lzw.NewWriter(buf, lzw.LSB, 8) 41 | _, err := writer.Write(record) 42 | if err != nil { 43 | return nil, err 44 | } 45 | err = writer.Close() 46 | if err != nil { 47 | return nil, err 48 | } 49 | return buf.Bytes(), nil 50 | } 51 | 52 | func (l LzwCompressor) DecompressWithBuf(buf []byte, destinationBuffer []byte) ([]byte, error) { 53 | // we have to set the length of the buffer (keeping capacity) to make sure gzip doesn't append 54 | destinationBuffer = destinationBuffer[:0] 55 | reader := lzw.NewReader(bytes.NewBuffer(buf), lzw.LSB, 8) 56 | resultBuffer := bytes.NewBuffer(destinationBuffer) 57 | _, err := resultBuffer.ReadFrom(reader) 58 | if err != nil { 59 | return nil, err 60 | } 61 | 62 | return resultBuffer.Bytes(), nil 63 | } 64 | -------------------------------------------------------------------------------- /recordio/compressor/snappy_compression.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "github.com/golang/snappy" 5 | ) 6 | 7 | type SnappyCompressor struct { 8 | } 9 | 10 | func (c *SnappyCompressor) Compress(record []byte) ([]byte, error) { 11 | return snappy.Encode(nil, record), nil 12 | } 13 | 14 | func (c *SnappyCompressor) Decompress(buf []byte) ([]byte, error) { 15 | return snappy.Decode(nil, buf) 16 | } 17 | 18 | func (c *SnappyCompressor) CompressWithBuf(record []byte, destinationBuffer []byte) ([]byte, error) { 19 | return snappy.Encode(destinationBuffer, record), nil 20 | } 21 | 22 | func (c *SnappyCompressor) DecompressWithBuf(buf []byte, destinationBuffer []byte) ([]byte, error) { 23 | return snappy.Decode(destinationBuffer, buf) 24 | } 25 | -------------------------------------------------------------------------------- /recordio/compressor/snappy_compression_test.go: -------------------------------------------------------------------------------- 1 | package compressor 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "testing" 6 | ) 7 | 8 | func TestSimpleSnappyCompression(t *testing.T) { 9 | comp := SnappyCompressor{} 10 | data := "some data" 11 | 12 | compressedBytes, err := comp.Compress([]byte(data)) 13 | assert.Nil(t, err) 14 | assert.Equal(t, 11, len(compressedBytes)) 15 | 16 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 17 | } 18 | 19 | func TestSimpleSnappyCompressionWithBuffers(t *testing.T) { 20 | comp := SnappyCompressor{} 21 | data := "some data" 22 | 23 | destBuf := make([]byte, 11) 24 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 25 | assert.Nil(t, err) 26 | assert.Equal(t, 11, len(compressedBytes)) 27 | 28 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 29 | } 30 | 31 | func TestSimpleSnappyCompressionWithSmallerBuffer(t *testing.T) { 32 | comp := SnappyCompressor{} 33 | data := "some data" 34 | 35 | destBuf := make([]byte, 10) 36 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 37 | assert.Nil(t, err) 38 | assert.Equal(t, 11, len(compressedBytes)) 39 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 40 | } 41 | 42 | func TestSimpleSnappyCompressionWithLargerBuffer(t *testing.T) { 43 | comp := SnappyCompressor{} 44 | data := "some data" 45 | 46 | destBuf := make([]byte, 15) 47 | compressedBytes, err := comp.CompressWithBuf([]byte(data), destBuf) 48 | assert.Nil(t, err) 49 | assert.Equal(t, 11, len(compressedBytes)) 50 | decompressAndCheck(t, &comp, compressedBytes, data, 9) 51 | } 52 | 53 | func decompressAndCheck(t *testing.T, comp CompressionI, compressedBytes []byte, expectedData string, expectedByteSize int) { 54 | decompressedBytes, err := comp.Decompress(compressedBytes) 55 | assert.Nil(t, err) 56 | assert.Equal(t, expectedByteSize, len(decompressedBytes)) 57 | assert.Equal(t, expectedData, string(decompressedBytes)) 58 | } 59 | -------------------------------------------------------------------------------- /recordio/counting_buffered_reader.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "io" 5 | ) 6 | 7 | type Reset interface { 8 | Reset(r io.Reader) 9 | } 10 | 11 | type ByteReaderReset interface { 12 | io.ByteReader 13 | io.Reader 14 | Reset 15 | Size() int 16 | } 17 | 18 | type ByteReaderResetCount interface { 19 | ByteReaderReset 20 | Count() uint64 21 | } 22 | 23 | type CountingBufferedReader struct { 24 | r ByteReaderReset 25 | count uint64 26 | } 27 | 28 | // ReadByte reads and returns a single byte. If no byte is available, returns an error. 29 | func (c *CountingBufferedReader) ReadByte() (byte, error) { 30 | b, err := c.r.ReadByte() 31 | if err == nil { 32 | c.count = c.count + 1 33 | } 34 | return b, err 35 | } 36 | 37 | // Read reads data into p. 38 | // It returns the number of bytes read into p. 39 | // The bytes are taken from at most one Read on the underlying Reader, 40 | // hence n may be less than len(p). 41 | // To read exactly len(p) bytes, use io.ReadFull(b, p). 42 | // At EOF, the count will be zero and err will be io.EOF. 43 | func (c *CountingBufferedReader) Read(p []byte) (n int, err error) { 44 | read, err := c.r.Read(p) 45 | if err == nil { 46 | c.count = c.count + uint64(read) 47 | } 48 | return read, err 49 | } 50 | 51 | // Reset discards any buffered data, resets all state, and switches 52 | // the buffered reader to read from r. 53 | func (c *CountingBufferedReader) Reset(r io.Reader) { 54 | c.r.Reset(r) 55 | } 56 | 57 | func (c *CountingBufferedReader) Count() uint64 { 58 | return c.count 59 | } 60 | 61 | func (c *CountingBufferedReader) Size() int { 62 | return c.r.Size() 63 | } 64 | 65 | func NewCountingByteReader(reader ByteReaderReset) ByteReaderResetCount { 66 | return &CountingBufferedReader{r: reader, count: 0} 67 | } 68 | -------------------------------------------------------------------------------- /recordio/counting_buffered_reader_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "github.com/stretchr/testify/assert" 7 | "io" 8 | "testing" 9 | ) 10 | 11 | var testBuf = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13} 12 | 13 | func TestReaderHappyCountingSingleBytes(t *testing.T) { 14 | reader := testReader() 15 | 16 | idx := 0 17 | for { 18 | b, err := reader.ReadByte() 19 | if err == io.EOF { 20 | break 21 | } 22 | 23 | assert.Nil(t, err) 24 | assert.Equal(t, testBuf[idx], b) 25 | idx++ 26 | } 27 | 28 | assert.Equal(t, len(testBuf), idx) 29 | assert.Equal(t, len(testBuf), int(reader.Count())) 30 | } 31 | 32 | func TestReaderHappyCountingBufferedBytes(t *testing.T) { 33 | reader := testReader() 34 | 35 | buf := make([]byte, 5) 36 | idx := 0 37 | for { 38 | read, err := reader.Read(buf) 39 | if err == io.EOF { 40 | break 41 | } 42 | 43 | assert.Nil(t, err) 44 | for i := 0; i < read; i++ { 45 | assert.Equal(t, testBuf[idx], buf[i]) 46 | idx++ 47 | } 48 | } 49 | 50 | assert.Equal(t, len(testBuf), idx) 51 | assert.Equal(t, len(testBuf), int(reader.Count())) 52 | } 53 | 54 | func TestReaderHappyResetAndRead(t *testing.T) { 55 | reader := testReader() 56 | 57 | idx := 0 58 | for { 59 | b, err := reader.ReadByte() 60 | if err == io.EOF { 61 | break 62 | } 63 | 64 | assert.Nil(t, err) 65 | assert.Equal(t, testBuf[idx], b) 66 | idx++ 67 | 68 | // this basically simulates how we reset from offset 5 to 8 (eg in seeking) 69 | if b == 5 { 70 | reader.Reset(bufio.NewReader(bytes.NewReader(testBuf[8:]))) 71 | idx = 8 72 | } 73 | } 74 | 75 | assert.Equal(t, len(testBuf), idx) 76 | assert.Equal(t, 12, int(reader.Count())) // 12 because we have been skipping that many bytes in reading 77 | } 78 | 79 | func testReader() ByteReaderResetCount { 80 | return NewCountingByteReader(bufio.NewReader(bytes.NewReader(testBuf))) 81 | } 82 | -------------------------------------------------------------------------------- /recordio/direct_io.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "syscall" 7 | 8 | "github.com/ncw/directio" 9 | ) 10 | 11 | type DirectIOFactory struct { 12 | } 13 | 14 | func (d DirectIOFactory) CreateNewReader(filePath string, bufSize int) (*os.File, ByteReaderResetCount, error) { 15 | readFile, err := directio.OpenFile(filePath, os.O_RDONLY|os.O_CREATE, 0666) 16 | if err != nil { 17 | return nil, nil, err 18 | } 19 | 20 | block := directio.AlignedBlock(bufSize) 21 | return readFile, NewCountingByteReader(NewReaderBuf(readFile, block)), nil 22 | } 23 | 24 | func (d DirectIOFactory) CreateNewWriter(filePath string, bufSize int) (*os.File, WriteSeekerCloserFlusher, error) { 25 | writeFile, err := directio.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0666) 26 | if err != nil { 27 | return nil, nil, err 28 | } 29 | 30 | block := directio.AlignedBlock(bufSize) 31 | return writeFile, NewAlignedWriterBuf(writeFile, block), nil 32 | } 33 | 34 | // IsDirectIOAvailable tests whether DirectIO is available (on the OS / filesystem). 35 | // It will return (true, nil) if that's the case, if it's not available it will be (false, nil). 36 | // Any other error will be indicated by the error (either true/false). 37 | func IsDirectIOAvailable() (available bool, err error) { 38 | // the only way to check is to create a tmp file and check whether the error is EINVAL, which indicates it's not available. 39 | tmpFile, err := os.CreateTemp("", "directio-test") 40 | if err != nil { 41 | return 42 | } 43 | 44 | err = tmpFile.Close() 45 | if err != nil { 46 | return 47 | } 48 | 49 | defer func(name string) { 50 | _ = os.Remove(name) 51 | }(tmpFile.Name()) 52 | 53 | tmpFile, err = directio.OpenFile(tmpFile.Name(), os.O_WRONLY|os.O_CREATE, 0666) 54 | if err != nil { 55 | // this syscall specifically signals that DirectIO is not supported 56 | if errors.Is(err, syscall.EINVAL) { 57 | available = false 58 | err = nil 59 | return 60 | } 61 | } 62 | 63 | // at this point we can be sure a file can be opened with DirectIO flags correctly 64 | available = true 65 | 66 | err = tmpFile.Close() 67 | if err != nil { 68 | return 69 | } 70 | 71 | return 72 | } 73 | -------------------------------------------------------------------------------- /recordio/direct_io_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestDirectIOFactory_CreateNewReader(t *testing.T) { 11 | ok, err := IsDirectIOAvailable() 12 | require.NoError(t, err) 13 | if !ok { 14 | t.Skip("directio not available here") 15 | return 16 | } 17 | 18 | temp, err := os.CreateTemp("", "TestDirectIOFactory_CreateNewReader") 19 | require.NoError(t, err) 20 | require.NoError(t, temp.Close()) 21 | 22 | f := DirectIOFactory{} 23 | file, buf, err := f.CreateNewReader(temp.Name(), 4096) 24 | require.NoError(t, err) 25 | defer closeCleanFile(t, file) 26 | 27 | assert.Equal(t, 4096, buf.Size()) 28 | } 29 | 30 | func TestDirectIOFactory_CreateNewWriter(t *testing.T) { 31 | ok, err := IsDirectIOAvailable() 32 | require.NoError(t, err) 33 | if !ok { 34 | t.Skip("directio not available here") 35 | return 36 | } 37 | 38 | temp, err := os.CreateTemp("", "TestDirectIOFactory_CreateNewWriter") 39 | require.NoError(t, err) 40 | require.NoError(t, temp.Close()) 41 | 42 | f := DirectIOFactory{} 43 | file, buf, err := f.CreateNewWriter(temp.Name(), 4096) 44 | require.NoError(t, err) 45 | defer closeCleanFile(t, file) 46 | 47 | assert.Equal(t, 4096, buf.Size()) 48 | } 49 | -------------------------------------------------------------------------------- /recordio/io_factory.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | type IOFactory interface { 8 | CreateNewReader(filePath string, bufSize int) (*os.File, ByteReaderResetCount, error) 9 | CreateNewWriter(filePath string, bufSize int) (*os.File, WriteSeekerCloserFlusher, error) 10 | } 11 | -------------------------------------------------------------------------------- /recordio/mmap_reader_v1compat_test.go: -------------------------------------------------------------------------------- 1 | // this file exists for backward compatibility with the V1 files 2 | // is basically a 1:1 copy of mmap_reader_test, which has additional tests and goes to the different folder 3 | package recordio 4 | 5 | import ( 6 | "errors" 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | "testing" 10 | ) 11 | 12 | func TestMMapReaderHappyPathSingleRecordV1(t *testing.T) { 13 | reader := newOpenedTestMMapReader(t, "test_files/v1_compat/recordio_UncompressedSingleRecord") 14 | defer closeMMapReader(t, reader) 15 | 16 | // should contain an ascending 13 byte buffer 17 | buf, err := reader.ReadNextAt(FileHeaderSizeBytes) 18 | require.Nil(t, err) 19 | assertAscendingBytes(t, buf, 13) 20 | } 21 | 22 | func TestMMapReaderSingleRecordMisalignedOffsetV1(t *testing.T) { 23 | reader := newOpenedTestMMapReader(t, "test_files/v1_compat/recordio_UncompressedSingleRecord") 24 | defer closeMMapReader(t, reader) 25 | 26 | _, err := reader.ReadNextAt(FileHeaderSizeBytes + 1) 27 | assert.Equal(t, errors.New("magic number mismatch"), errors.Unwrap(err)) 28 | } 29 | 30 | func TestMMapReaderSingleRecordOffsetBiggerThanFileV1(t *testing.T) { 31 | reader := newOpenedTestMMapReader(t, "test_files/v1_compat/recordio_UncompressedSingleRecord") 32 | defer closeMMapReader(t, reader) 33 | 34 | _, err := reader.ReadNextAt(42000) 35 | assert.Equal(t, errors.New("mmap: invalid ReadAt offset 42000"), errors.Unwrap(err)) 36 | } 37 | 38 | func TestMMapReaderV1VersionMismatchV0(t *testing.T) { 39 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord_v0", t) 40 | expectErrorStringOnOpen(t, reader, "version mismatch, expected a value from 1 to 4 but was 0") 41 | } 42 | 43 | func TestMMapReaderV1VersionMismatchV256(t *testing.T) { 44 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord_v256", t) 45 | expectErrorStringOnOpen(t, reader, "version mismatch, expected a value from 1 to 4 but was 256") 46 | } 47 | 48 | func TestMMapReaderV1CompressionGzipHeader(t *testing.T) { 49 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord_comp1", t) 50 | err := reader.Open() 51 | require.Nil(t, err) 52 | defer closeMMapReader(t, reader) 53 | assert.Equal(t, 1, reader.header.compressionType) 54 | } 55 | 56 | func TestMMapReaderV1CompressionSnappyHeader(t *testing.T) { 57 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord_comp2", t) 58 | err := reader.Open() 59 | require.Nil(t, err) 60 | defer closeMMapReader(t, reader) 61 | assert.Equal(t, 2, reader.header.compressionType) 62 | } 63 | 64 | func TestMMapReaderForbidsClosedReaderV1(t *testing.T) { 65 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord", t) 66 | err := reader.Close() 67 | require.Nil(t, err) 68 | _, err = reader.ReadNextAt(100) 69 | assert.Contains(t, err.Error(), "opened yet or is closed already") 70 | err = reader.Open() 71 | assert.Contains(t, err.Error(), "already closed") 72 | } 73 | 74 | func TestMMapReaderForbidsDoubleOpensV1(t *testing.T) { 75 | reader := newTestMMapReader("test_files/v1_compat/recordio_UncompressedSingleRecord", t) 76 | err := reader.Open() 77 | require.Nil(t, err) 78 | expectErrorStringOnOpen(t, reader, "already opened") 79 | } 80 | -------------------------------------------------------------------------------- /recordio/mmap_reader_v2compat_test.go: -------------------------------------------------------------------------------- 1 | package recordio 2 | 3 | import ( 4 | "errors" 5 | "github.com/stretchr/testify/assert" 6 | "github.com/stretchr/testify/require" 7 | "io" 8 | "testing" 9 | ) 10 | 11 | func TestMMapReaderHappyPathSingleRecordV2(t *testing.T) { 12 | reader := newOpenedTestMMapReader(t, "test_files/v2_compat/recordio_UncompressedSingleRecord") 13 | defer closeMMapReader(t, reader) 14 | 15 | // should contain an ascending 13 byte buffer 16 | buf, err := reader.ReadNextAt(FileHeaderSizeBytes) 17 | require.Nil(t, err) 18 | assertAscendingBytes(t, buf, 13) 19 | } 20 | 21 | func TestMMapReaderSingleRecordMisalignedOffsetV2(t *testing.T) { 22 | reader := newOpenedTestMMapReader(t, "test_files/v2_compat/recordio_UncompressedSingleRecord") 23 | defer closeMMapReader(t, reader) 24 | 25 | _, err := reader.ReadNextAt(FileHeaderSizeBytes + 1) 26 | assert.Equal(t, errors.New("magic number mismatch"), errors.Unwrap(err)) 27 | } 28 | 29 | func TestMMapReaderSingleRecordOffsetBiggerThanFileV2(t *testing.T) { 30 | reader := newOpenedTestMMapReader(t, "test_files/v2_compat/recordio_UncompressedSingleRecord") 31 | defer closeMMapReader(t, reader) 32 | 33 | _, err := reader.ReadNextAt(42000) 34 | assert.Equal(t, errors.New("mmap: invalid ReadAt offset 42000"), errors.Unwrap(err)) 35 | } 36 | 37 | func TestMMapReaderV2VersionMismatchV0(t *testing.T) { 38 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord_v0", t) 39 | expectErrorStringOnOpen(t, reader, "version mismatch, expected a value from 1 to 4 but was 0") 40 | } 41 | 42 | func TestMMapReaderV2VersionMismatchV256(t *testing.T) { 43 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord_v256", t) 44 | expectErrorStringOnOpen(t, reader, "version mismatch, expected a value from 1 to 4 but was 256") 45 | } 46 | 47 | func TestMMapReaderCompressionGzipHeaderV2(t *testing.T) { 48 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord_comp1", t) 49 | err := reader.Open() 50 | require.Nil(t, err) 51 | defer closeMMapReader(t, reader) 52 | assert.Equal(t, 1, reader.header.compressionType) 53 | } 54 | 55 | func TestMMapReaderCompressionSnappyHeaderV2(t *testing.T) { 56 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord_comp2", t) 57 | err := reader.Open() 58 | require.Nil(t, err) 59 | defer closeMMapReader(t, reader) 60 | assert.Equal(t, 2, reader.header.compressionType) 61 | } 62 | 63 | func TestMMapReaderCompressionUnknownV2(t *testing.T) { 64 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord_comp300", t) 65 | expectErrorStringOnOpen(t, reader, "unknown compression type [300]") 66 | } 67 | 68 | func TestMMapReaderForbidsClosedReaderV2(t *testing.T) { 69 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord", t) 70 | err := reader.Close() 71 | require.Nil(t, err) 72 | _, err = reader.ReadNextAt(100) 73 | assert.Contains(t, err.Error(), "was either not opened yet or is closed already") 74 | err = reader.Open() 75 | assert.Contains(t, err.Error(), "already closed") 76 | } 77 | 78 | func TestMMapReaderForbidsDoubleOpensV2(t *testing.T) { 79 | reader := newTestMMapReader("test_files/v2_compat/recordio_UncompressedSingleRecord", t) 80 | err := reader.Open() 81 | require.Nil(t, err) 82 | expectErrorStringOnOpen(t, reader, "already opened") 83 | } 84 | 85 | // this is explicitly testing the difference in mmap semantics, where we would get an EOF error due to the following: 86 | // * record header is very small (5 bytes) 87 | // * record itself is smaller than the remainder of the buffer (RecordHeaderSizeBytesV1V2 - 5 bytes of the header = 15 bytes) 88 | // * only the EOF follows 89 | // this basically triggers the mmap.ReaderAt to fill a buffer of RecordHeaderSizeBytesV1V2 size (up until the EOF) AND return the io.EOF as an error. 90 | // that caused some failed tests in the sstable reader, so it makes sense to have an explicit test for it 91 | func TestMMapReaderReadsSmallVarIntHeaderEOFCorrectlyV2(t *testing.T) { 92 | reader := newOpenedTestMMapReader(t, "test_files/v2_compat/recordio_UncompressedSingleRecord") 93 | bytes, err := reader.ReadNextAt(FileHeaderSizeBytes) 94 | require.Nil(t, err) 95 | assertAscendingBytes(t, bytes, 13) 96 | bytes, err = reader.ReadNextAt(uint64(FileHeaderSizeBytes + 5 + len(bytes))) 97 | require.Nil(t, bytes) 98 | assert.Equal(t, io.EOF, err) 99 | 100 | // testing the boundaries around, which should give us a magic number mismatch 101 | bytes, err = reader.ReadNextAt(uint64(FileHeaderSizeBytes + 4 + len(bytes))) 102 | require.Nil(t, bytes) 103 | assert.Equal(t, errors.New("magic number mismatch"), errors.Unwrap(err)) 104 | } 105 | -------------------------------------------------------------------------------- /recordio/proto/mmap_proto_reader.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "github.com/thomasjungblut/go-sstables/recordio" 5 | "google.golang.org/protobuf/proto" 6 | ) 7 | 8 | type MMapProtoReader struct { 9 | recordio.ReadAtI 10 | } 11 | 12 | func (r *MMapProtoReader) ReadNextAt(record proto.Message, offset uint64) (proto.Message, error) { 13 | bytes, err := r.ReadAtI.ReadNextAt(offset) 14 | if err != nil { 15 | return nil, err 16 | } 17 | 18 | err = proto.Unmarshal(bytes, record) 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | return record, nil 24 | } 25 | 26 | func (r *MMapProtoReader) SeekNext(record proto.Message, offset uint64) (uint64, proto.Message, error) { 27 | off, bytes, err := r.ReadAtI.SeekNext(offset) 28 | if err != nil { 29 | return off, nil, err 30 | } 31 | 32 | err = proto.Unmarshal(bytes, record) 33 | if err != nil { 34 | return off, nil, err 35 | } 36 | 37 | return off, record, nil 38 | } 39 | 40 | func NewMMapProtoReaderWithPath(path string) (ReadAtI, error) { 41 | r, err := recordio.NewMemoryMappedReaderWithPath(path) 42 | if err != nil { 43 | return nil, err 44 | } 45 | 46 | return &MMapProtoReader{r}, nil 47 | } 48 | -------------------------------------------------------------------------------- /recordio/proto/proto_reader.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "errors" 5 | "google.golang.org/protobuf/encoding/protowire" 6 | "os" 7 | 8 | "github.com/thomasjungblut/go-sstables/recordio" 9 | gproto "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | type Reader struct { 13 | recordio.ReaderI 14 | opts *gproto.UnmarshalOptions 15 | } 16 | 17 | func (r *Reader) ReadNext(record gproto.Message) (gproto.Message, error) { 18 | bytes, err := r.ReaderI.ReadNext() 19 | if err != nil { 20 | return nil, err 21 | } 22 | 23 | err = r.opts.Unmarshal(bytes, record) 24 | if err != nil { 25 | return nil, err 26 | } 27 | 28 | return record, nil 29 | } 30 | 31 | // options 32 | 33 | type ReaderOptions struct { 34 | path string 35 | file *os.File 36 | bufSizeBytes int 37 | } 38 | 39 | type ReaderOption func(*ReaderOptions) 40 | 41 | func ReaderPath(p string) ReaderOption { 42 | return func(args *ReaderOptions) { 43 | args.path = p 44 | } 45 | } 46 | 47 | func ReaderFile(p *os.File) ReaderOption { 48 | return func(args *ReaderOptions) { 49 | args.file = p 50 | } 51 | } 52 | 53 | func ReadBufferSizeBytes(p int) ReaderOption { 54 | return func(args *ReaderOptions) { 55 | args.bufSizeBytes = p 56 | } 57 | } 58 | 59 | // create a new reader with the given options. Either Path or File must be supplied 60 | func NewReader(readerOptions ...ReaderOption) (ReaderI, error) { 61 | opts := &ReaderOptions{ 62 | path: "", 63 | file: nil, 64 | bufSizeBytes: 1024 * 1024 * 4, 65 | } 66 | 67 | for _, readerOption := range readerOptions { 68 | readerOption(opts) 69 | } 70 | 71 | if (opts.file != nil) && (opts.path != "") { 72 | return nil, errors.New("either os.File or string path must be supplied, never both") 73 | } 74 | 75 | if opts.file == nil { 76 | if opts.path == "" { 77 | return nil, errors.New("path was not supplied") 78 | } 79 | } 80 | reader, err := recordio.NewFileReader( 81 | recordio.ReaderPath(opts.path), 82 | recordio.ReaderFile(opts.file), 83 | recordio.ReaderBufferSizeBytes(opts.bufSizeBytes)) 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | return &Reader{ 89 | ReaderI: reader, 90 | opts: &gproto.UnmarshalOptions{ 91 | RecursionLimit: protowire.DefaultRecursionLimit, 92 | }, 93 | }, nil 94 | 95 | } 96 | 97 | // Deprecated: use the NewProtoReader with options. 98 | func NewProtoReaderWithPath(path string) (ReaderI, error) { 99 | return NewReader(ReaderPath(path)) 100 | } 101 | 102 | // Deprecated: use the NewProtoReader with options. 103 | func NewProtoReaderWithFile(file *os.File) (ReaderI, error) { 104 | return NewReader(ReaderFile(file)) 105 | } 106 | -------------------------------------------------------------------------------- /recordio/proto/proto_writer.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | "github.com/ncw/directio" 8 | "github.com/thomasjungblut/go-sstables/recordio" 9 | "google.golang.org/protobuf/proto" 10 | ) 11 | 12 | type Writer struct { 13 | writer recordio.WriterI 14 | } 15 | 16 | func (w *Writer) Open() error { 17 | return w.writer.Open() 18 | } 19 | 20 | func (w *Writer) Write(record proto.Message) (uint64, error) { 21 | bytes, err := proto.Marshal(record) 22 | if err != nil { 23 | return 0, err 24 | } 25 | return w.writer.Write(bytes) 26 | } 27 | 28 | func (w *Writer) WriteSync(record proto.Message) (uint64, error) { 29 | bytes, err := proto.Marshal(record) 30 | if err != nil { 31 | return 0, err 32 | } 33 | return w.writer.WriteSync(bytes) 34 | } 35 | 36 | func (w *Writer) Close() error { 37 | return w.writer.Close() 38 | } 39 | 40 | func (w *Writer) Size() uint64 { 41 | return w.writer.Size() 42 | } 43 | 44 | // options 45 | 46 | type WriterOptions struct { 47 | path string 48 | file *os.File 49 | compressionType int 50 | bufSizeBytes int 51 | useDirectIO bool 52 | } 53 | 54 | type WriterOption func(*WriterOptions) 55 | 56 | func Path(p string) WriterOption { 57 | return func(args *WriterOptions) { 58 | args.path = p 59 | } 60 | } 61 | 62 | func File(p *os.File) WriterOption { 63 | return func(args *WriterOptions) { 64 | args.file = p 65 | } 66 | } 67 | 68 | func CompressionType(p int) WriterOption { 69 | return func(args *WriterOptions) { 70 | args.compressionType = p 71 | } 72 | } 73 | 74 | func WriteBufferSizeBytes(p int) WriterOption { 75 | return func(args *WriterOptions) { 76 | args.bufSizeBytes = p 77 | } 78 | } 79 | 80 | func DirectIO() WriterOption { 81 | return func(args *WriterOptions) { 82 | args.useDirectIO = true 83 | } 84 | } 85 | 86 | // create a new writer with the given options. Either Path or File must be supplied, compression is optional and 87 | // turned off by default. 88 | func NewWriter(writerOptions ...WriterOption) (WriterI, error) { 89 | opts := &WriterOptions{ 90 | path: "", 91 | file: nil, 92 | compressionType: recordio.CompressionTypeNone, 93 | bufSizeBytes: 1024 * 1024 * 4, 94 | useDirectIO: false, 95 | } 96 | 97 | for _, writeOption := range writerOptions { 98 | writeOption(opts) 99 | } 100 | 101 | if (opts.file != nil) && (opts.path != "") { 102 | return nil, errors.New("either os.File or string path must be supplied, never both") 103 | } 104 | 105 | if opts.file == nil { 106 | if opts.path == "" { 107 | return nil, errors.New("path was not supplied") 108 | } 109 | if opts.useDirectIO { 110 | f, err := directio.OpenFile(opts.path, os.O_WRONLY|os.O_CREATE, 0666) 111 | if err != nil { 112 | return nil, err 113 | } 114 | opts.file = f 115 | } else { 116 | f, err := os.OpenFile(opts.path, os.O_WRONLY|os.O_CREATE, 0666) 117 | if err != nil { 118 | return nil, err 119 | } 120 | opts.file = f 121 | } 122 | } 123 | 124 | writer, err := recordio.NewFileWriter( 125 | recordio.File(opts.file), 126 | recordio.CompressionType(opts.compressionType), 127 | recordio.BufferSizeBytes(opts.bufSizeBytes)) 128 | if err != nil { 129 | return nil, err 130 | } 131 | 132 | return &Writer{writer: writer}, nil 133 | } 134 | -------------------------------------------------------------------------------- /recordio/proto/recordio_proto.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import "google.golang.org/protobuf/proto" 4 | import "github.com/thomasjungblut/go-sstables/recordio" 5 | 6 | type ReaderI interface { 7 | recordio.OpenClosableI 8 | // ReadNext reads the next record into the passed message record, EOF error when it reaches the end signalled by (nil, io.EOF) 9 | ReadNext(record proto.Message) (proto.Message, error) 10 | // SkipNext skips the next record, EOF error when it reaches the end signalled by io.EOF as the error 11 | SkipNext() error 12 | } 13 | 14 | // ReadAtI is used to randomly read protobuf through byte offsets. This type is thread-safe 15 | type ReadAtI interface { 16 | recordio.OpenClosableI 17 | recordio.SizeI 18 | 19 | // ReadNextAt reads the next record at the given offset into the passed message record, EOF error when it reaches the end signalled by (nil, io.EOF), implementation must be thread-safe 20 | ReadNextAt(record proto.Message, offset uint64) (proto.Message, error) 21 | 22 | // SeekNext reads the next full record that comes after the provided offset. The main difference to ReadNextAt is 23 | // that this function seeks to the next record marker, whereas ReadNextAt always needs to be pointed to the start of 24 | // the record. This function returns any io related error, for example io.EOF, or a wrapped equivalent, when the end is reached. 25 | SeekNext(record proto.Message, offset uint64) (uint64, proto.Message, error) 26 | } 27 | 28 | type WriterI interface { 29 | recordio.OpenClosableI 30 | recordio.SizeI 31 | // Write appends a record, returns the current offset this item was written to 32 | Write(record proto.Message) (uint64, error) 33 | // WriteSync appends a record and forces a disk sync, returns the current offset this item was written to 34 | WriteSync(record proto.Message) (uint64, error) 35 | } 36 | -------------------------------------------------------------------------------- /recordio/test_files/berlin52.tsp: -------------------------------------------------------------------------------- 1 | NAME: berlin52 2 | TYPE: TSP 3 | COMMENT: 52 locations in Berlin (Groetschel) 4 | DIMENSION: 52 5 | EDGE_WEIGHT_TYPE: EUC_2D 6 | NODE_COORD_SECTION 7 | 1 565.0 575.0 8 | 2 25.0 185.0 9 | 3 345.0 750.0 10 | 4 945.0 685.0 11 | 5 845.0 655.0 12 | 6 880.0 660.0 13 | 7 25.0 230.0 14 | 8 525.0 1000.0 15 | 9 580.0 1175.0 16 | 10 650.0 1130.0 17 | 11 1605.0 620.0 18 | 12 1220.0 580.0 19 | 13 1465.0 200.0 20 | 14 1530.0 5.0 21 | 15 845.0 680.0 22 | 16 725.0 370.0 23 | 17 145.0 665.0 24 | 18 415.0 635.0 25 | 19 510.0 875.0 26 | 20 560.0 365.0 27 | 21 300.0 465.0 28 | 22 520.0 585.0 29 | 23 480.0 415.0 30 | 24 835.0 625.0 31 | 25 975.0 580.0 32 | 26 1215.0 245.0 33 | 27 1320.0 315.0 34 | 28 1250.0 400.0 35 | 29 660.0 180.0 36 | 30 410.0 250.0 37 | 31 420.0 555.0 38 | 32 575.0 665.0 39 | 33 1150.0 1160.0 40 | 34 700.0 580.0 41 | 35 685.0 595.0 42 | 36 685.0 610.0 43 | 37 770.0 610.0 44 | 38 795.0 645.0 45 | 39 720.0 635.0 46 | 40 760.0 650.0 47 | 41 475.0 960.0 48 | 42 95.0 260.0 49 | 43 875.0 920.0 50 | 44 700.0 500.0 51 | 45 555.0 815.0 52 | 46 830.0 485.0 53 | 47 1170.0 65.0 54 | 48 830.0 610.0 55 | 49 605.0 625.0 56 | 50 595.0 360.0 57 | 51 1340.0 725.0 58 | 52 1740.0 245.0 59 | EOF -------------------------------------------------------------------------------- /recordio/test_files/text_line.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package test_files; 3 | option go_package = "github.com/thomasjungblut/go-sstables/recordio/test_files"; 4 | 5 | message TextLine { 6 | int32 lineNumber = 1; 7 | string line = 2; 8 | } 9 | 10 | -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_SnappyWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_SnappyWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_comp1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_comp1 -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_comp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_comp2 -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_mnm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_mnm -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_v0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_v0 -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_v256: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedSingleRecord_v256 -------------------------------------------------------------------------------- /recordio/test_files/v1_compat/recordio_UncompressedWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v1_compat/recordio_UncompressedWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_SnappyWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_SnappyWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp1 -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp2 -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp300: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_comp300 -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_directio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_directio -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_directio_trailer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_directio_trailer -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_mnm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_mnm -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_v0: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedSingleRecord_v256: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /recordio/test_files/v2_compat/recordio_UncompressedWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v2_compat/recordio_UncompressedWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_SnappyWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_SnappyWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedMagicNumberContent: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedMagicNumberContent -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedNilAndEmptyRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedNilAndEmptyRecord -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp1 -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp2 -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp300: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_comp300 -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_directio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_directio -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_directio_trailer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_directio_trailer -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_mnm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_mnm -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_v0: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedSingleRecord_v256: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /recordio/test_files/v3_compat/recordio_UncompressedWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v3_compat/recordio_UncompressedWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_SnappyWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_SnappyWriterMultiRecord_asc -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedCrcFailure: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedCrcFailure -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedMagicNumberContent: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedMagicNumberContent -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedNilAndEmptyRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedNilAndEmptyRecord -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp1 -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp2 -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp300: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_comp300 -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_directio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_directio -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_directio_trailer: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_directio_trailer -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_mnm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_mnm -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_v0: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedSingleRecord_v256: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /recordio/test_files/v4_compat/recordio_UncompressedWriterMultiRecord_asc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/recordio/test_files/v4_compat/recordio_UncompressedWriterMultiRecord_asc -------------------------------------------------------------------------------- /simpledb/_crash_tests/.gitignore: -------------------------------------------------------------------------------- 1 | *.html -------------------------------------------------------------------------------- /simpledb/_crash_tests/simpledb_web_server.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log" 7 | "net/http" 8 | "os" 9 | "time" 10 | 11 | "github.com/thomasjungblut/go-sstables/simpledb" 12 | ) 13 | 14 | const address = "0.0.0.0:29071" 15 | const dataEndpoint = "/data" 16 | const fullDataEndpoint = address + dataEndpoint 17 | 18 | type Server struct { 19 | db simpledb.DatabaseI 20 | } 21 | 22 | func (s *Server) handleGet(w http.ResponseWriter, key string) { 23 | get, err := s.db.Get(key) 24 | if err != nil { 25 | if errors.Is(err, simpledb.ErrNotFound) { 26 | w.WriteHeader(http.StatusNotFound) 27 | return 28 | } else { 29 | log.Printf("error while getting key '%s': %v\n", key, err) 30 | w.WriteHeader(http.StatusInternalServerError) 31 | return 32 | } 33 | } 34 | 35 | w.WriteHeader(http.StatusOK) 36 | _, err = w.Write([]byte(get)) 37 | if err != nil { 38 | log.Printf("error while writing value for key '%s': %v\n", key, err) 39 | return 40 | } 41 | } 42 | 43 | func (s *Server) Data(w http.ResponseWriter, r *http.Request) { 44 | 45 | key := r.URL.Query().Get("key") 46 | if key == "" { 47 | w.WriteHeader(http.StatusBadRequest) 48 | return 49 | } 50 | 51 | switch r.Method { 52 | case http.MethodGet: 53 | s.handleGet(w, key) 54 | case http.MethodDelete: 55 | err := s.db.Delete(key) 56 | if err != nil { 57 | log.Printf("error while deleting key '%s': %v\n", key, err) 58 | w.WriteHeader(http.StatusInternalServerError) 59 | return 60 | } 61 | w.WriteHeader(http.StatusOK) 62 | case http.MethodPut: 63 | allBytes, err := io.ReadAll(r.Body) 64 | if err != nil { 65 | log.Printf("error while getting body for key '%s': %v\n", key, err) 66 | w.WriteHeader(http.StatusInternalServerError) 67 | return 68 | } 69 | 70 | err = s.db.Put(key, string(allBytes)) 71 | if err != nil { 72 | log.Printf("error while getting putting value for key '%s': %v\n", key, err) 73 | w.WriteHeader(http.StatusInternalServerError) 74 | return 75 | } 76 | w.WriteHeader(http.StatusOK) 77 | default: 78 | log.Printf("not supported method '%s'\n", r.Method) 79 | w.WriteHeader(http.StatusInternalServerError) 80 | } 81 | 82 | } 83 | 84 | func main() { 85 | if len(os.Args) != 2 { 86 | log.Fatal("missing db folder argument\n") 87 | } 88 | baseDir := os.Args[1] 89 | 90 | _, err := os.Stat(baseDir) 91 | if err != nil { 92 | if os.IsNotExist(err) { 93 | log.Fatal("db folder does not exist\n") 94 | } else { 95 | log.Fatalf("stat: %v\n", err) 96 | } 97 | } 98 | 99 | // make sure we sync to WAL all the time and have a small memstore size for flushing often 100 | db, err := simpledb.NewSimpleDB(baseDir, 101 | simpledb.CompactionFileThreshold(5), 102 | simpledb.CompactionRunInterval(1*time.Second), 103 | simpledb.MemstoreSizeBytes(1024*1024*4)) 104 | if err != nil { 105 | log.Fatalf("newDB: %v\n", err) 106 | } 107 | defer func() { 108 | err := db.Close() 109 | log.Fatalf("error while closing db: %v\n", err) 110 | }() 111 | 112 | err = db.Open() 113 | if err != nil { 114 | log.Fatalf("openDB: %v\n", err) 115 | } 116 | 117 | serv := &Server{db: db} 118 | http.HandleFunc(dataEndpoint, serv.Data) 119 | log.Printf("running on %s\n", address) 120 | log.Fatal(http.ListenAndServe(address, nil)) 121 | } 122 | -------------------------------------------------------------------------------- /simpledb/compaction.go: -------------------------------------------------------------------------------- 1 | package simpledb 2 | 3 | import ( 4 | "errors" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "sort" 9 | "strings" 10 | "time" 11 | 12 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 13 | "github.com/thomasjungblut/go-sstables/simpledb/proto" 14 | "github.com/thomasjungblut/go-sstables/skiplist" 15 | "github.com/thomasjungblut/go-sstables/sstables" 16 | ) 17 | 18 | func backgroundCompaction(db *DB) { 19 | defer func() { 20 | db.doneCompactionChannel <- true 21 | }() 22 | 23 | if !db.enableCompactions { 24 | return 25 | } 26 | 27 | err := func(db *DB) error { 28 | for { 29 | select { 30 | case <-db.compactionTickerStopChannel: 31 | return nil 32 | case <-db.compactionTicker.C: 33 | metadata, err := executeCompaction(db) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | // nothing that was compacted, wait for the next tick 39 | if metadata == nil { 40 | continue 41 | } 42 | 43 | err = db.sstableManager.reflectCompactionResult(metadata) 44 | if err != nil { 45 | return err 46 | } 47 | } 48 | 49 | } 50 | }(db) 51 | 52 | if err != nil { 53 | log.Panicf("error while compacting, error was %v", err) 54 | } 55 | } 56 | 57 | func executeCompaction(db *DB) (compactionMetadata *proto.CompactionMetadata, err error) { 58 | compactionAction := db.sstableManager.candidateTablesForCompaction(db.compactedMaxSizeBytes, db.compactionRatio) 59 | paths := compactionAction.pathsToCompact 60 | numRecords := compactionAction.totalRecords 61 | if len(paths) <= db.compactionFileThreshold { 62 | return nil, nil 63 | } 64 | 65 | // make sure we're always compacting with the right order in mind 66 | sort.Strings(paths) 67 | 68 | start := time.Now() 69 | writeFolder, err := os.MkdirTemp(db.basePath, SSTableCompactionPathPrefix) 70 | if err != nil { 71 | return nil, err 72 | } 73 | 74 | log.Printf("starting compaction of %d files in %v with %v\n", len(paths), writeFolder, strings.Join(paths, ",")) 75 | 76 | writer, err := sstables.NewSSTableStreamWriter( 77 | sstables.WriteBasePath(writeFolder), 78 | sstables.WithKeyComparator(skiplist.BytesComparator{}), 79 | sstables.BloomExpectedNumberOfElements(numRecords)) 80 | if err != nil { 81 | return nil, err 82 | } 83 | 84 | err = writer.Open() 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | defer func() { 90 | err = errors.Join(err, writer.Close()) 91 | }() 92 | 93 | var readers []sstables.SSTableReaderI 94 | var iterators []sstables.SSTableMergeIteratorContext 95 | for i := 0; i < len(paths); i++ { 96 | reader, err := sstables.NewSSTableReader( 97 | sstables.ReadBasePath(paths[i]), 98 | sstables.ReadWithKeyComparator(db.cmp), 99 | ) 100 | if err != nil { 101 | return nil, err 102 | } 103 | 104 | scanner, err := reader.Scan() 105 | if err != nil { 106 | return nil, err 107 | } 108 | 109 | readers = append(readers, reader) 110 | iterators = append(iterators, sstables.NewMergeIteratorContext(i, scanner)) 111 | } 112 | 113 | defer func() { 114 | for _, reader := range readers { 115 | err = errors.Join(err, reader.Close()) 116 | } 117 | }() 118 | 119 | reduceFunc := sstables.ScanReduceLatestWinsSkipTombstones 120 | err = sstables.NewSSTableMerger(db.cmp).MergeCompact(iterators, writer, reduceFunc) 121 | if err != nil { 122 | return nil, err 123 | } 124 | 125 | // in order to be portable, we are taking only relative paths from the db base path 126 | // later in reconstruction they are "rebased" over the database base path 127 | for i := 0; i < len(paths); i++ { 128 | paths[i] = filepath.Base(paths[i]) 129 | } 130 | 131 | compactionMetadata = &proto.CompactionMetadata{ 132 | WritePath: filepath.Base(writeFolder), 133 | ReplacementPath: paths[0], 134 | SstablePaths: paths, 135 | } 136 | 137 | // at this point the compaction is finished, we save the metadata that this was successful for potential recoveries 138 | err = saveCompactionMetadata(writeFolder, compactionMetadata) 139 | if err != nil { 140 | return nil, err 141 | } 142 | 143 | log.Printf("done compacting %d sstables in %v. Path: [%s]\n", len(paths), time.Since(start), writeFolder) 144 | 145 | return compactionMetadata, nil 146 | } 147 | 148 | func saveCompactionMetadata(writeFolder string, compactionMetadata *proto.CompactionMetadata) (err error) { 149 | metaWriter, err := rProto.NewWriter( 150 | rProto.Path(filepath.Join(writeFolder, CompactionFinishedSuccessfulFileName)), 151 | rProto.WriteBufferSizeBytes(4*1024), 152 | ) 153 | 154 | if err != nil { 155 | return err 156 | } 157 | err = metaWriter.Open() 158 | if err != nil { 159 | return err 160 | } 161 | 162 | defer func() { 163 | err = errors.Join(err, metaWriter.Close()) 164 | }() 165 | 166 | _, err = metaWriter.Write(compactionMetadata) 167 | if err != nil { 168 | return err 169 | } 170 | 171 | return nil 172 | } 173 | -------------------------------------------------------------------------------- /simpledb/flush.go: -------------------------------------------------------------------------------- 1 | package simpledb 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "sync/atomic" 9 | "time" 10 | 11 | "github.com/thomasjungblut/go-sstables/memstore" 12 | "github.com/thomasjungblut/go-sstables/sstables" 13 | ) 14 | 15 | func flushMemstoreContinuously(db *DB) { 16 | defer func() { db.doneFlushChannel <- true }() 17 | err := func(db *DB) error { 18 | for flushAction := range db.storeFlushChannel { 19 | err := executeFlush(db, flushAction) 20 | if err != nil { 21 | return err 22 | } 23 | } 24 | return nil 25 | }(db) 26 | 27 | if err != nil { 28 | log.Panicf("error while merging sstable at %s, error was %v", db.currentSSTablePath, err) 29 | } 30 | } 31 | 32 | func executeFlush(db *DB, flushAction memStoreFlushAction) error { 33 | walPath := flushAction.walPath 34 | memStoreToFlush := *flushAction.memStore 35 | numElements := uint64(memStoreToFlush.Size()) 36 | // we can skip if there is nothing to write, usually that indicates a proper "close" was done. 37 | if memStoreToFlush.Size() == 0 { 38 | log.Printf("no memstore flush necessary due to empty store, skipping\n") 39 | return nil 40 | } 41 | 42 | start := time.Now() 43 | 44 | gen := atomic.AddUint64(&db.currentGeneration, uint64(1)) 45 | writePath := filepath.Join(db.basePath, fmt.Sprintf(SSTablePattern, gen)) 46 | err := os.MkdirAll(writePath, 0700) 47 | if err != nil { 48 | return err 49 | } 50 | 51 | err = memStoreToFlush.FlushWithTombstones( 52 | sstables.WriteBasePath(writePath), 53 | sstables.WithKeyComparator(db.cmp), 54 | sstables.WriteBufferSizeBytes(int(db.writeBufferSizeBytes)), 55 | sstables.BloomExpectedNumberOfElements(numElements)) 56 | if err != nil { 57 | return err 58 | } 59 | 60 | if walPath != "" { 61 | err = os.Remove(walPath) 62 | if err != nil { 63 | return err 64 | } 65 | } 66 | 67 | reader, err := sstables.NewSSTableReader( 68 | sstables.ReadBasePath(writePath), 69 | sstables.ReadWithKeyComparator(db.cmp), 70 | sstables.ReadBufferSizeBytes(int(db.readBufferSizeBytes)), 71 | ) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | elapsedDuration := time.Since(start) 77 | totalBytes := reader.MetaData().TotalBytes 78 | throughput := float64(totalBytes) / 1024 / 1024 / elapsedDuration.Seconds() 79 | log.Printf("done flushing memstore to sstable of size %d bytes (%2.f mb/s) in %v. Path: [%s]\n", 80 | totalBytes, throughput, elapsedDuration, writePath) 81 | 82 | // add the newly created reader into the rotation 83 | // note that this CAN block here waiting on a current compaction to finish 84 | db.sstableManager.addReader(reader) 85 | 86 | return nil 87 | } 88 | 89 | func (db *DB) rotateWalAndFlushMemstore() error { 90 | walPath, err := db.wal.Rotate() 91 | if err != nil { 92 | return err 93 | } 94 | db.storeFlushChannel <- memStoreFlushAction{ 95 | memStore: swapMemstore(db), 96 | walPath: walPath, 97 | } 98 | return nil 99 | } 100 | 101 | func swapMemstore(db *DB) *memstore.MemStoreI { 102 | storeToFlush := db.memStore.writeStore 103 | db.memStore = &RWMemstore{ 104 | readStore: storeToFlush, 105 | writeStore: memstore.NewMemStore(), 106 | } 107 | return &storeToFlush 108 | } 109 | -------------------------------------------------------------------------------- /simpledb/flush_test.go: -------------------------------------------------------------------------------- 1 | package simpledb 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "sort" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/stretchr/testify/assert" 14 | "github.com/thomasjungblut/go-sstables/memstore" 15 | "github.com/thomasjungblut/go-sstables/skiplist" 16 | ) 17 | 18 | func TestFlushHappyPath(t *testing.T) { 19 | tmpDummyWalFile, err := os.CreateTemp("", "simpledb_flushHappyPath") 20 | assert.Nil(t, err) 21 | assert.Nil(t, tmpDummyWalFile.Close()) 22 | 23 | defer os.Remove(tmpDummyWalFile.Name()) 24 | tmpDir, err := os.MkdirTemp("", "simpledb_flushHappyPath") 25 | assert.Nil(t, err) 26 | defer func(path string) { 27 | err := os.RemoveAll(path) 28 | assert.Nil(t, err) 29 | }(tmpDir) 30 | 31 | action := memStoreFlushAction{ 32 | memStore: &setupPrefilledRWMemstore(t).writeStore, 33 | walPath: tmpDummyWalFile.Name(), 34 | } 35 | 36 | db := &DB{ 37 | cmp: skiplist.BytesComparator{}, 38 | basePath: tmpDir, 39 | currentGeneration: 42, 40 | rwLock: &sync.RWMutex{}, 41 | sstableManager: NewSSTableManager(skiplist.BytesComparator{}, &sync.RWMutex{}, tmpDir), 42 | readBufferSizeBytes: DefaultReadBufferSizeBytes, 43 | writeBufferSizeBytes: DefaultWriteBufferSizeBytes, 44 | } 45 | err = executeFlush(db, action) 46 | assert.Nil(t, err) 47 | 48 | // ensure that the wal file was cleaned and a sstable with the right name was created 49 | _, err = os.Stat(tmpDummyWalFile.Name()) 50 | assert.True(t, os.IsNotExist(err)) 51 | _, err = os.Stat(filepath.Join(tmpDir, "sstable_000000000000043")) 52 | assert.Nil(t, err) 53 | 54 | assert.Nil(t, db.sstableManager.currentReader.Close()) 55 | } 56 | 57 | func TestFlushEmptyMemstore(t *testing.T) { 58 | m := memstore.NewMemStore() 59 | action := memStoreFlushAction{ 60 | memStore: &m, 61 | walPath: "some", 62 | } 63 | 64 | db := &DB{currentGeneration: 0} 65 | err := executeFlush(db, action) 66 | assert.Nil(t, err) 67 | assert.Equal(t, uint64(0), db.currentGeneration) 68 | } 69 | 70 | func TestFlushPathsSortCorrectly(t *testing.T) { 71 | var tables []string 72 | for i := 0; i < 10001; i++ { 73 | tables = append(tables, fmt.Sprintf(SSTablePattern, i)) 74 | } 75 | 76 | sort.Strings(tables) 77 | for i := 0; i < 10001; i++ { 78 | is := strings.Split(tables[i], "_") 79 | atoi, err := strconv.Atoi(is[1]) 80 | assert.Nil(t, err) 81 | assert.Equal(t, i, atoi) 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /simpledb/porcupine/.gitignore: -------------------------------------------------------------------------------- 1 | *.html -------------------------------------------------------------------------------- /simpledb/porcupine/db_recorder.go: -------------------------------------------------------------------------------- 1 | package porcupine 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/anishathalye/porcupine" 7 | "github.com/thomasjungblut/go-sstables/simpledb" 8 | ) 9 | 10 | type DatabaseClientRecorder struct { 11 | clientId int 12 | db simpledb.DatabaseI 13 | 14 | operations []porcupine.Operation[Input, Output] 15 | } 16 | 17 | func NewDatabaseRecorder(db simpledb.DatabaseI, clientId int) *DatabaseClientRecorder { 18 | return &DatabaseClientRecorder{ 19 | clientId: clientId, 20 | db: db, 21 | operations: []porcupine.Operation[Input, Output]{}, 22 | } 23 | } 24 | 25 | func (d *DatabaseClientRecorder) Operations() []porcupine.Operation[Input, Output] { 26 | return d.operations 27 | } 28 | 29 | func (d *DatabaseClientRecorder) GetBytes(key []byte) ([]byte, error) { 30 | return nil, nil 31 | } 32 | 33 | func (d *DatabaseClientRecorder) Get(key string) (string, error) { 34 | start := time.Now() 35 | val, err := d.db.Get(key) 36 | end := time.Now() 37 | d.operations = append(d.operations, porcupine.Operation[Input, Output]{ 38 | ClientId: d.clientId, 39 | Input: Input{ 40 | Operation: GetOp, 41 | Key: key, 42 | }, 43 | Call: start.UnixNano(), 44 | Output: Output{ 45 | Key: key, 46 | Val: val, 47 | Err: err, 48 | }, 49 | Return: end.UnixNano(), 50 | }) 51 | 52 | return val, err 53 | } 54 | 55 | func (d *DatabaseClientRecorder) Put(key, value string) error { 56 | start := time.Now() 57 | err := d.db.Put(key, value) 58 | end := time.Now() 59 | d.operations = append(d.operations, porcupine.Operation[Input, Output]{ 60 | ClientId: d.clientId, 61 | Input: Input{ 62 | Operation: PutOp, 63 | Key: key, 64 | Val: value, 65 | }, 66 | Call: start.UnixNano(), 67 | Output: Output{ 68 | Key: key, 69 | Val: value, 70 | Err: err, 71 | }, 72 | Return: end.UnixNano(), 73 | }) 74 | 75 | return err 76 | } 77 | 78 | func (d *DatabaseClientRecorder) PutBytes(key, value []byte) error { 79 | return nil 80 | } 81 | 82 | func (d *DatabaseClientRecorder) Delete(key string) error { 83 | start := time.Now() 84 | err := d.db.Delete(key) 85 | end := time.Now() 86 | d.operations = append(d.operations, porcupine.Operation[Input, Output]{ 87 | ClientId: d.clientId, 88 | Input: Input{ 89 | Operation: DelOp, 90 | Key: key, 91 | Val: "", 92 | }, 93 | Call: start.UnixNano(), 94 | Output: Output{ 95 | Key: key, 96 | Val: "", 97 | Err: err, 98 | }, 99 | Return: end.UnixNano(), 100 | }) 101 | 102 | return err 103 | } 104 | 105 | func (d *DatabaseClientRecorder) DeleteBytes(key []byte) error { 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /simpledb/porcupine/linearizability_test.go: -------------------------------------------------------------------------------- 1 | //go:build simpleDBlinear 2 | // +build simpleDBlinear 3 | 4 | package porcupine 5 | 6 | import ( 7 | "math/rand" 8 | "os" 9 | "strings" 10 | "sync" 11 | "testing" 12 | 13 | "github.com/anishathalye/porcupine" 14 | "github.com/stretchr/testify/require" 15 | "github.com/thomasjungblut/go-sstables/simpledb" 16 | ) 17 | 18 | type TestDatabase struct { 19 | db *simpledb.DB 20 | basePath string 21 | } 22 | 23 | func TestHappyPath(t *testing.T) { 24 | db := newOpenedSimpleDB(t, "linearizability_HappyPath") 25 | defer cleanDatabaseFolder(t, db) 26 | defer closeDatabase(t, db) 27 | 28 | key := "key" 29 | client := NewDatabaseRecorder(db.db, 0) 30 | for i := 0; i < 100; i++ { 31 | _, _ = client.Get(key) 32 | _ = client.Put(key, randomString(5)) 33 | if rand.Float32() < 0.25 { 34 | _ = client.Delete(key) 35 | } 36 | } 37 | 38 | VerifyOperations(t, client.operations) 39 | } 40 | 41 | func TestHappyPathMultiKey(t *testing.T) { 42 | db := newOpenedSimpleDB(t, "linearizability_HappyPathMultiKey") 43 | defer cleanDatabaseFolder(t, db) 44 | defer closeDatabase(t, db) 45 | 46 | client := NewDatabaseRecorder(db.db, 0) 47 | for i := 0; i < 100; i++ { 48 | key := randomString(5) 49 | _, _ = client.Get(key) 50 | _ = client.Put(key, randomString(5)) 51 | _, _ = client.Get(key) 52 | if rand.Float32() < 0.5 { 53 | _ = client.Delete(key) 54 | } 55 | _, _ = client.Get(key) 56 | } 57 | 58 | VerifyOperations(t, client.operations) 59 | } 60 | 61 | func TestHappyPathMultiThread(t *testing.T) { 62 | db := newOpenedSimpleDB(t, "linearizability_HappyPathMultiThread") 63 | defer cleanDatabaseFolder(t, db) 64 | defer closeDatabase(t, db) 65 | 66 | operations := parallelWriteGetDelete(db.db, 4, 100, 2500) 67 | VerifyOperations(t, operations) 68 | } 69 | 70 | func TestMultiTriggerFlush(t *testing.T) { 71 | db := newOpenedSimpleDB(t, "linearizability_HappyPathMultiThread") 72 | defer cleanDatabaseFolder(t, db) 73 | defer closeDatabase(t, db) 74 | 75 | operations := parallelWriteGetDelete(db.db, 8, 8*1000, 2500) 76 | VerifyOperations(t, operations) 77 | } 78 | 79 | func newSimpleDBWithTemp(t *testing.T, name string) *TestDatabase { 80 | tmpDir, err := os.MkdirTemp("", name) 81 | require.Nil(t, err) 82 | 83 | //for testing purposes we will flush with a tiny amount of 1mb 84 | db, err := simpledb.NewSimpleDB(tmpDir, simpledb.MemstoreSizeBytes(1024*1024*1)) 85 | require.Nil(t, err) 86 | return &TestDatabase{ 87 | db: db, 88 | basePath: tmpDir, 89 | } 90 | } 91 | 92 | func newOpenedSimpleDB(t *testing.T, name string) *TestDatabase { 93 | db := newSimpleDBWithTemp(t, name) 94 | require.Nil(t, db.db.Open()) 95 | return db 96 | } 97 | 98 | func closeDatabase(t *testing.T, db *TestDatabase) { 99 | func(t *testing.T, db *simpledb.DB) { require.Nil(t, db.Close()) }(t, db.db) 100 | } 101 | 102 | func cleanDatabaseFolder(t *testing.T, db *TestDatabase) { 103 | func(t *testing.T, basePath string) { require.Nil(t, os.RemoveAll(basePath)) }(t, db.basePath) 104 | } 105 | 106 | func randomString(size int) string { 107 | builder := strings.Builder{} 108 | for i := 0; i < size; i++ { 109 | builder.WriteRune(rand.Int31n(26) + 97) 110 | } 111 | 112 | return builder.String() 113 | } 114 | 115 | func parallelWriteGetDelete(db *simpledb.DB, numGoRoutines int, numRecords int, valSizeBytes int) []porcupine.Operation[Input, Output] { 116 | var operations []porcupine.Operation[Input, Output] 117 | var opsLock sync.Mutex 118 | wg := sync.WaitGroup{} 119 | recordsPerRoutine := numRecords / numGoRoutines 120 | var keys []string 121 | var values []string 122 | for i := 0; i < recordsPerRoutine; i++ { 123 | keys = append(keys, randomString(5)) 124 | values = append(values, randomString(valSizeBytes)) 125 | } 126 | for n := 0; n < numGoRoutines; n++ { 127 | wg.Add(1) 128 | go func(db *simpledb.DB, id, start, end int) { 129 | defer wg.Done() 130 | 131 | client := NewDatabaseRecorder(db, id) 132 | rnd := rand.New(rand.NewSource(int64(id))) 133 | for i := start; i < end; i++ { 134 | // that ensures some overlap in the requests 135 | key := keys[rnd.Intn(len(keys))] 136 | val := values[rnd.Intn(len(keys))] 137 | _, _ = client.Get(key) 138 | _ = client.Put(key, val) 139 | _, _ = client.Get(key) 140 | if rnd.Float32() < 0.5 { 141 | _ = client.Delete(key) 142 | } 143 | _, _ = client.Get(key) 144 | } 145 | 146 | opsLock.Lock() 147 | defer opsLock.Unlock() 148 | 149 | operations = append(operations, client.operations...) 150 | 151 | }(db, n, n*recordsPerRoutine, n*recordsPerRoutine+recordsPerRoutine) 152 | } 153 | 154 | wg.Wait() 155 | 156 | return operations 157 | } 158 | -------------------------------------------------------------------------------- /simpledb/porcupine/model.go: -------------------------------------------------------------------------------- 1 | package porcupine 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "reflect" 8 | "testing" 9 | 10 | pp "github.com/anishathalye/porcupine" 11 | "github.com/stretchr/testify/require" 12 | "github.com/thomasjungblut/go-sstables/simpledb" 13 | ) 14 | 15 | const ( 16 | GetOp = iota 17 | PutOp = iota 18 | DelOp = iota 19 | ) 20 | 21 | type MapState struct { 22 | m map[string]string 23 | } 24 | 25 | type Input struct { 26 | Operation uint8 27 | Key string 28 | Val string 29 | } 30 | 31 | type Output struct { 32 | Key string 33 | Val string 34 | Err error 35 | } 36 | 37 | func (s MapState) Clone() MapState { 38 | sx := make(map[string]string, len(s.m)) 39 | for k, v := range s.m { 40 | sx[k] = v 41 | } 42 | return MapState{m: sx} 43 | } 44 | 45 | func (s MapState) Equals(otherState MapState) bool { 46 | return reflect.DeepEqual(s.m, otherState.m) 47 | } 48 | 49 | func shorten(s string, size int) string { 50 | if len(s) > size { 51 | return s[:size] 52 | } 53 | return s 54 | } 55 | 56 | func (s MapState) String() string { 57 | shortValueState := map[string]string{} 58 | for k, v := range s.m { 59 | shortValueState[k] = shorten(v, 5) 60 | } 61 | 62 | return fmt.Sprintf("%v", shortValueState) 63 | } 64 | 65 | func NewMapState() MapState { 66 | return MapState{m: map[string]string{}} 67 | } 68 | 69 | var Model = pp.Model[MapState, Input, Output]{ 70 | Init: NewMapState, 71 | Partition: func(history []pp.Operation[Input, Output]) [][]pp.Operation[Input, Output] { 72 | indexMap := map[string]int{} 73 | var partitions [][]pp.Operation[Input, Output] 74 | for _, op := range history { 75 | i := op.Input 76 | ix, found := indexMap[i.Key] 77 | if !found { 78 | partitions = append(partitions, []pp.Operation[Input, Output]{op}) 79 | indexMap[i.Key] = len(partitions) - 1 80 | } else { 81 | partitions[ix] = append(partitions[ix], op) 82 | } 83 | } 84 | return partitions 85 | }, 86 | Step: func(s MapState, i Input, o Output) (bool, MapState) { 87 | stateVal, found := s.m[i.Key] 88 | 89 | switch i.Operation { 90 | case GetOp: 91 | if errors.Is(o.Err, simpledb.ErrNotFound) { 92 | return !found, s 93 | } else if stateVal == o.Val { 94 | return true, s 95 | } 96 | break 97 | case PutOp: 98 | if o.Err == nil { 99 | s.m[i.Key] = i.Val 100 | return true, s 101 | } 102 | break 103 | case DelOp: 104 | if o.Err == nil { 105 | delete(s.m, i.Key) 106 | return true, s 107 | } 108 | break 109 | } 110 | 111 | if o.Err != nil { 112 | log.Printf("unexpected error state found for key: [%s] %v\n", i.Key, o.Err) 113 | panic(o.Err) 114 | } 115 | 116 | return false, s 117 | }, 118 | DescribeOperation: func(i Input, o Output) string { 119 | opName := "" 120 | switch i.Operation { 121 | case GetOp: 122 | opName = "Get" 123 | break 124 | case PutOp: 125 | opName = "Put" 126 | break 127 | case DelOp: 128 | opName = "Del" 129 | break 130 | } 131 | 132 | return fmt.Sprintf("%s(%s) -> %s", opName, i.Key, shorten(o.Val, 5)) 133 | }, 134 | } 135 | 136 | func VerifyOperations(t *testing.T, operations []pp.Operation[Input, Output]) { 137 | result, info := pp.CheckOperationsVerbose(Model, operations, 0) 138 | require.NoError(t, pp.VisualizePath(Model, info, t.Name()+"_porcupine.html")) 139 | require.Equal(t, pp.Ok, result, "output was not linearizable") 140 | } 141 | -------------------------------------------------------------------------------- /simpledb/proto/compaction_metadata.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/simpledb/proto"; 4 | 5 | message CompactionMetadata { 6 | // database root relative write path of the compaction result 7 | string writePath = 1; 8 | // database root relative desired replacement path for the compaction result 9 | string replacementPath = 2; 10 | // database root relative set of paths that contributed to that compaction result 11 | repeated string sstablePaths = 3; 12 | } 13 | -------------------------------------------------------------------------------- /simpledb/proto/wal_mutation.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/simpledb/proto"; 4 | 5 | message UpsertMutation { 6 | string key = 1; 7 | string value = 2; 8 | bytes keyBytes = 3; 9 | bytes valueBytes = 4; 10 | } 11 | 12 | message DeleteTombstoneMutation { 13 | string key = 1; 14 | bytes keyBytes = 2; 15 | } 16 | 17 | message WalMutation { 18 | oneof mutation { 19 | UpsertMutation addition = 1; 20 | DeleteTombstoneMutation deleteTombStone = 2; 21 | } 22 | // don't forget leave couple of indices for the oneof 23 | } 24 | -------------------------------------------------------------------------------- /simpledb/rw_memstore.go: -------------------------------------------------------------------------------- 1 | package simpledb 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/memstore" 6 | "github.com/thomasjungblut/go-sstables/sstables" 7 | ) 8 | 9 | // RWMemstore the RW memstore contains two memstores, one for reading, one for writing. 10 | // the one that writes takes precedence over the read store (for the same key). 11 | type RWMemstore struct { 12 | readStore memstore.MemStoreI 13 | writeStore memstore.MemStoreI 14 | } 15 | 16 | // read paths 17 | 18 | func (c *RWMemstore) Contains(key []byte) bool { 19 | return c.writeStore.Contains(key) || c.readStore.Contains(key) 20 | } 21 | 22 | func (c *RWMemstore) Get(key []byte) ([]byte, error) { 23 | // the write memstore always wins here, if the key is not found 24 | // in the writeStore the readStore is the source of truth 25 | writeVal, writeErr := c.writeStore.Get(key) 26 | if writeErr != nil { 27 | if errors.Is(writeErr, memstore.KeyNotFound) { 28 | return c.readStore.Get(key) 29 | } 30 | 31 | // that also includes the tombstones 32 | return nil, writeErr 33 | } 34 | 35 | return writeVal, nil 36 | } 37 | 38 | // write paths, just proxy to the writeStore 39 | 40 | func (c *RWMemstore) Add(key []byte, value []byte) error { 41 | return c.writeStore.Add(key, value) 42 | } 43 | 44 | func (c *RWMemstore) Upsert(key []byte, value []byte) error { 45 | return c.writeStore.Upsert(key, value) 46 | } 47 | 48 | func (c *RWMemstore) Delete(key []byte) error { 49 | err := c.writeStore.Delete(key) 50 | if errors.Is(err, memstore.KeyNotFound) { 51 | return c.Tombstone(key) 52 | } 53 | 54 | return err 55 | } 56 | 57 | func (c *RWMemstore) DeleteIfExists(key []byte) error { 58 | return c.Delete(key) 59 | } 60 | 61 | func (c *RWMemstore) Tombstone(key []byte) error { 62 | return c.writeStore.Tombstone(key) 63 | } 64 | 65 | func (c *RWMemstore) EstimatedSizeInBytes() uint64 { 66 | return c.writeStore.EstimatedSizeInBytes() 67 | } 68 | 69 | func (c *RWMemstore) SStableIterator() sstables.SSTableIteratorI { 70 | return c.writeStore.SStableIterator() 71 | } 72 | 73 | func (c *RWMemstore) Flush(opts ...sstables.WriterOption) error { 74 | return c.writeStore.Flush(opts...) 75 | } 76 | 77 | func (c *RWMemstore) Size() int { 78 | return c.writeStore.Size() 79 | } 80 | -------------------------------------------------------------------------------- /simpledb/rw_memstore_test.go: -------------------------------------------------------------------------------- 1 | package simpledb 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/thomasjungblut/go-sstables/memstore" 9 | ) 10 | 11 | func TestRWMemstoreAddShouldGoToWriteStore(t *testing.T) { 12 | rw := setupPrefilledRWMemstore(t) 13 | k := asStringBytes(42) 14 | assert.Nil(t, rw.Add(k, k)) 15 | _, err := rw.readStore.Get(k) 16 | assert.Equal(t, memstore.KeyNotFound, err) 17 | v, err := rw.writeStore.Get(k) 18 | assert.Equal(t, k, v) 19 | } 20 | 21 | func TestRWMemstoreContains(t *testing.T) { 22 | rw := setupPrefilledRWMemstore(t) 23 | for i := 0; i < 15; i++ { 24 | assert.True(t, rw.Contains(asStringBytes(i))) 25 | } 26 | 27 | assert.False(t, rw.Contains(asStringBytes(-1))) 28 | assert.False(t, rw.Contains(asStringBytes(16))) 29 | } 30 | 31 | func TestRWMemstoreGet(t *testing.T) { 32 | rw := setupPrefilledRWMemstore(t) 33 | for i := 0; i < 15; i++ { 34 | k := asStringBytes(i) 35 | v, err := rw.Get(k) 36 | assert.Nil(t, err) 37 | assert.Equal(t, k, v) 38 | } 39 | 40 | _, err := rw.Get(asStringBytes(-1)) 41 | assert.Equal(t, memstore.KeyNotFound, err) 42 | 43 | _, err = rw.Get(asStringBytes(16)) 44 | assert.Equal(t, memstore.KeyNotFound, err) 45 | 46 | _, err = rw.Get(asStringBytes(42)) 47 | assert.Equal(t, memstore.KeyNotFound, err) 48 | } 49 | 50 | func TestRWMemstoreTombstoning(t *testing.T) { 51 | rw := setupPrefilledRWMemstore(t) 52 | 53 | for i := 0; i < 15; i++ { 54 | k := asStringBytes(i) 55 | assert.Nil(t, rw.Tombstone(k)) 56 | 57 | _, err := rw.Get(k) 58 | assert.Equal(t, memstore.KeyTombstoned, err) 59 | } 60 | } 61 | 62 | func TestRWMemstoreDelete(t *testing.T) { 63 | rw := setupPrefilledRWMemstore(t) 64 | 65 | for i := 0; i < 10; i++ { 66 | k := asStringBytes(i) 67 | assert.Nil(t, rw.Delete(k)) 68 | 69 | _, err := rw.Get(k) 70 | assert.Equal(t, memstore.KeyTombstoned, err) 71 | } 72 | 73 | for i := 10; i < 20; i++ { 74 | k := asStringBytes(i) 75 | assert.Nil(t, rw.DeleteIfExists(k)) 76 | 77 | _, err := rw.Get(k) 78 | assert.Equal(t, memstore.KeyTombstoned, err) 79 | } 80 | } 81 | 82 | func setupPrefilledRWMemstore(t *testing.T) *RWMemstore { 83 | rw := &RWMemstore{ 84 | readStore: memstore.NewMemStore(), 85 | writeStore: memstore.NewMemStore(), 86 | } 87 | 88 | for i := 0; i < 10; i++ { 89 | is := asStringBytes(i) 90 | assert.Nil(t, rw.readStore.Add(is, is)) 91 | } 92 | 93 | for i := 5; i < 15; i++ { 94 | is := asStringBytes(i) 95 | assert.Nil(t, rw.writeStore.Add(is, is)) 96 | } 97 | 98 | return rw 99 | } 100 | 101 | func asStringBytes(i int) []byte { 102 | return []byte(strconv.Itoa(i)) 103 | } 104 | -------------------------------------------------------------------------------- /skiplist/README.md: -------------------------------------------------------------------------------- 1 | # SkipLists 2 | 3 | Whenever you find yourself in need of a sorted list/map for range scans or ordered iteration, you can resort to a `SkipList`. The `SkipList` in this project is based on [LevelDBs skiplist](https://github.com/google/leveldb/blob/master/db/skiplist.h) 4 | 5 | This package is the only one compatible with versions pre-generics. 6 | 7 | ## Using skiplist.Map (generics Go >=1.18) 8 | 9 | You can get the full example from [examples/skiplist.go](/_examples/skiplist.go). 10 | 11 | ```go 12 | import ( 13 | "github.com/thomasjungblut/go-sstables/skiplist" 14 | "log" 15 | ) 16 | 17 | func main() { 18 | 19 | skipListMap := skiplist.NewSkipListMap[int, int](skiplist.OrderedComparator[int]{}) 20 | skipListMap.Insert(13, 91) 21 | skipListMap.Insert(3, 1) 22 | skipListMap.Insert(5, 2) 23 | log.Printf("size: %d", skipListMap.Size()) 24 | 25 | it, _ := skipListMap.Iterator() 26 | for { 27 | k, v, err := it.Next() 28 | if errors.is(err, skiplist.Done) { 29 | break 30 | } 31 | log.Printf("key: %d, value: %d", k, v) 32 | } 33 | 34 | log.Printf("starting at key: %d", 5) 35 | it, _ = skipListMap.IteratorStartingAt(5) 36 | for { 37 | k, v, err := it.Next() 38 | if errors.is(err, skiplist.Done) { 39 | break 40 | } 41 | log.Printf("key: %d, value: %d", k, v) 42 | } 43 | 44 | log.Printf("between: %d and %d", 8, 50) 45 | it, _ = skipListMap.IteratorBetween(8, 50) 46 | for { 47 | k, v, err := it.Next() 48 | if errors.is(err, skiplist.Done) { 49 | break 50 | } 51 | log.Printf("key: %d, value: %d", k, v) 52 | } 53 | } 54 | ``` 55 | -------------------------------------------------------------------------------- /sstables/empty_sstable_reader.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "github.com/thomasjungblut/go-sstables/sstables/proto" 5 | ) 6 | 7 | // these types are mostly used for testing or default behaviour of an empty sstable 8 | 9 | type EmptySStableReader struct{} 10 | 11 | func (EmptySStableReader) Contains(_ []byte) (bool, error) { 12 | return false, nil 13 | } 14 | 15 | func (EmptySStableReader) Get(_ []byte) ([]byte, error) { 16 | return nil, NotFound 17 | } 18 | 19 | func (EmptySStableReader) Scan() (SSTableIteratorI, error) { 20 | return EmptySSTableIterator{}, nil 21 | } 22 | 23 | func (EmptySStableReader) ScanStartingAt(_ []byte) (SSTableIteratorI, error) { 24 | return EmptySSTableIterator{}, nil 25 | } 26 | 27 | func (EmptySStableReader) ScanRange(_ []byte, _ []byte) (SSTableIteratorI, error) { 28 | return EmptySSTableIterator{}, nil 29 | } 30 | 31 | func (EmptySStableReader) Close() error { 32 | return nil 33 | } 34 | 35 | func (EmptySStableReader) MetaData() *proto.MetaData { 36 | return &proto.MetaData{ 37 | NumRecords: 0, 38 | MinKey: nil, 39 | MaxKey: nil, 40 | } 41 | } 42 | 43 | func (EmptySStableReader) BasePath() string { 44 | return "" 45 | } 46 | 47 | type EmptySSTableIterator struct{} 48 | 49 | func (EmptySSTableIterator) Next() ([]byte, []byte, error) { 50 | return nil, nil, Done 51 | } 52 | -------------------------------------------------------------------------------- /sstables/empty_sstable_reader_test.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "testing" 7 | ) 8 | 9 | func TestContains(t *testing.T) { 10 | reader := EmptySStableReader{} 11 | contains, err := reader.Contains([]byte{}) 12 | require.NoError(t, err) 13 | assert.False(t, contains, "contains returned true") 14 | } 15 | 16 | func TestGet(t *testing.T) { 17 | reader := EmptySStableReader{} 18 | _, err := reader.Get([]byte{}) 19 | assert.Equal(t, NotFound, err) 20 | } 21 | 22 | func TestMetaData(t *testing.T) { 23 | reader := EmptySStableReader{} 24 | assert.Equal(t, 0, int(reader.MetaData().NumRecords)) 25 | require.Nil(t, reader.MetaData().MinKey) 26 | require.Nil(t, reader.MetaData().MaxKey) 27 | } 28 | 29 | func TestScan(t *testing.T) { 30 | reader := EmptySStableReader{} 31 | it, err := reader.Scan() 32 | require.Nil(t, err) 33 | testConsumeEmptyIterator(t, it) 34 | } 35 | 36 | func TestEmptyScanStartingAt(t *testing.T) { 37 | reader := EmptySStableReader{} 38 | it, err := reader.ScanStartingAt([]byte{1, 2, 3}) 39 | require.Nil(t, err) 40 | testConsumeEmptyIterator(t, it) 41 | } 42 | 43 | func TestEmptyScanRange(t *testing.T) { 44 | reader := EmptySStableReader{} 45 | it, err := reader.ScanRange([]byte{1, 2, 3}, []byte{1, 2, 5}) 46 | require.Nil(t, err) 47 | testConsumeEmptyIterator(t, it) 48 | } 49 | 50 | func testConsumeEmptyIterator(t *testing.T, it SSTableIteratorI) { 51 | _, _, err := it.Next() 52 | assert.Equal(t, Done, err) 53 | } 54 | -------------------------------------------------------------------------------- /sstables/map_key_index.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 7 | "github.com/thomasjungblut/go-sstables/skiplist" 8 | "github.com/thomasjungblut/go-sstables/sstables/proto" 9 | "io" 10 | ) 11 | 12 | type ByteKeyMapper[T comparable] interface { 13 | MapBytes(data []byte) T 14 | } 15 | 16 | type Byte4KeyMapper struct { 17 | } 18 | 19 | func (s *Byte4KeyMapper) MapBytes(data []byte) [4]byte { 20 | if len(data) > 4 { 21 | panic(fmt.Sprintf("data length is too large, found %d but expected 4", len(data))) 22 | } 23 | var result [4]byte 24 | copy(result[:], data[:]) 25 | return result 26 | } 27 | 28 | type Byte20KeyMapper struct { 29 | } 30 | 31 | func (s *Byte20KeyMapper) MapBytes(data []byte) [20]byte { 32 | if len(data) > 20 { 33 | panic(fmt.Sprintf("data length is too large, found %d but expected 20", len(data))) 34 | } 35 | var result [20]byte 36 | copy(result[:], data[:]) 37 | return result 38 | } 39 | 40 | // MapKeyIndex is keeping the entire index as a slice and a map in memory and uses binary search to 41 | // find the given keys for range lookups. This is useful for fast Contains/Get lookups. 42 | type MapKeyIndex[T comparable] struct { 43 | SliceKeyIndex 44 | index map[T]IndexVal 45 | mapper ByteKeyMapper[T] 46 | } 47 | 48 | func (s *MapKeyIndex[T]) Contains(key []byte) (bool, error) { 49 | _, found := s.index[s.mapper.MapBytes(key)] 50 | return found, nil 51 | } 52 | func (s *MapKeyIndex[T]) Get(key []byte) (IndexVal, error) { 53 | val, found := s.index[s.mapper.MapBytes(key)] 54 | if found { 55 | return val, nil 56 | } 57 | 58 | return IndexVal{}, skiplist.NotFound 59 | } 60 | 61 | type MapKeyIndexLoader[T comparable] struct { 62 | ReadBufferSize int 63 | Mapper ByteKeyMapper[T] 64 | } 65 | 66 | func (s *MapKeyIndexLoader[T]) Load(indexPath string, metadata *proto.MetaData) (SortedKeyIndex, error) { 67 | if s.Mapper == nil { 68 | return nil, fmt.Errorf("error loader need a Mapper for sstable '%s'", indexPath) 69 | } 70 | 71 | reader, err := rProto.NewReader( 72 | rProto.ReaderPath(indexPath), 73 | rProto.ReadBufferSizeBytes(s.ReadBufferSize), 74 | ) 75 | if err != nil { 76 | return nil, fmt.Errorf("error while creating index reader of sstable in '%s': %w", indexPath, err) 77 | } 78 | 79 | err = reader.Open() 80 | if err != nil { 81 | return nil, fmt.Errorf("error while opening index reader of sstable in '%s': %w", indexPath, err) 82 | } 83 | 84 | defer func() { 85 | err = errors.Join(err, reader.Close()) 86 | }() 87 | 88 | capacity := uint64(0) 89 | if metadata != nil { 90 | capacity = metadata.NumRecords 91 | } 92 | 93 | smap := make(map[T]IndexVal, capacity) 94 | sx := make([]sliceKey, 0, capacity) 95 | 96 | record := &proto.IndexEntry{} 97 | var i = 0 98 | for { 99 | _, err := reader.ReadNext(record) 100 | // io.EOF signals that no records are left to be read 101 | if errors.Is(err, io.EOF) { 102 | break 103 | } 104 | 105 | if err != nil { 106 | return nil, fmt.Errorf("error while reading index records of sstable in '%s': %w", indexPath, err) 107 | } 108 | 109 | kBytes := s.Mapper.MapBytes(record.Key) 110 | smap[kBytes] = IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum} 111 | sx = append(sx, sliceKey{IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum}, record.Key}) 112 | 113 | i++ 114 | } 115 | 116 | return &MapKeyIndex[T]{ 117 | SliceKeyIndex: SliceKeyIndex{index: sx}, 118 | index: smap, 119 | mapper: s.Mapper, 120 | }, nil 121 | } 122 | -------------------------------------------------------------------------------- /sstables/proto/sstable.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package proto; 3 | option go_package = "github.com/thomasjungblut/go-sstables/sstables/proto"; 4 | 5 | message IndexEntry { 6 | bytes key = 1; 7 | uint64 valueOffset = 2; 8 | uint64 checksum = 3; // a golang crc-64 checksum of the respective dataEntry 9 | } 10 | 11 | // deprecated, it's unnecessary overhead to marshal the bytes once more 12 | message DataEntry { 13 | bytes value = 1; 14 | } 15 | 16 | message MetaData { 17 | uint64 numRecords = 1; 18 | bytes minKey = 2; 19 | bytes maxKey = 3; 20 | uint64 dataBytes = 4; 21 | uint64 indexBytes = 5; 22 | uint64 totalBytes = 6; 23 | uint32 version = 7; // currently version 1, the default is version 0 with protos as values 24 | uint64 skippedRecords = 8; 25 | uint64 nullValues = 9; // in simpleDB that corresponds to the number of tombstones 26 | } 27 | -------------------------------------------------------------------------------- /sstables/skiplist_index.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 7 | "github.com/thomasjungblut/go-sstables/skiplist" 8 | "github.com/thomasjungblut/go-sstables/sstables/proto" 9 | "io" 10 | ) 11 | 12 | type SkipListIndex struct { 13 | skiplist.MapI[[]byte, IndexVal] 14 | NoOpOpenClose 15 | } 16 | 17 | func (s *SkipListIndex) Contains(key []byte) (bool, error) { 18 | return s.MapI.Contains(key), nil 19 | } 20 | 21 | type SkipListIndexLoader struct { 22 | KeyComparator skiplist.Comparator[[]byte] 23 | ReadBufferSize int 24 | } 25 | 26 | func (l *SkipListIndexLoader) Load(indexPath string, _ *proto.MetaData) (_ SortedKeyIndex, err error) { 27 | reader, err := rProto.NewReader( 28 | rProto.ReaderPath(indexPath), 29 | rProto.ReadBufferSizeBytes(l.ReadBufferSize), 30 | ) 31 | if err != nil { 32 | return nil, fmt.Errorf("error while creating index reader of sstable in '%s': %w", indexPath, err) 33 | } 34 | 35 | err = reader.Open() 36 | if err != nil { 37 | return nil, fmt.Errorf("error while opening index reader of sstable in '%s': %w", indexPath, err) 38 | } 39 | 40 | defer func() { 41 | err = errors.Join(err, reader.Close()) 42 | }() 43 | 44 | indexMap := skiplist.NewSkipListMap[[]byte, IndexVal](l.KeyComparator) 45 | record := &proto.IndexEntry{} 46 | 47 | for { 48 | _, err := reader.ReadNext(record) 49 | // io.EOF signals that no records are left to be read 50 | if errors.Is(err, io.EOF) { 51 | break 52 | } 53 | 54 | if err != nil { 55 | return nil, fmt.Errorf("error while reading index records of sstable in '%s': %w", indexPath, err) 56 | } 57 | 58 | indexMap.Insert(record.Key, IndexVal{ 59 | Offset: record.ValueOffset, 60 | Checksum: record.Checksum, 61 | }) 62 | } 63 | 64 | return &SkipListIndex{indexMap, NoOpOpenClose{}}, nil 65 | } 66 | -------------------------------------------------------------------------------- /sstables/slice_key_index.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 8 | "github.com/thomasjungblut/go-sstables/skiplist" 9 | "github.com/thomasjungblut/go-sstables/sstables/proto" 10 | "golang.org/x/exp/slices" 11 | "io" 12 | ) 13 | 14 | type sliceKey struct { 15 | IndexVal 16 | key []byte 17 | } 18 | 19 | // SliceKeyIndex is keeping the entire index as a slice in memory and uses binary search to find the given keys. 20 | type SliceKeyIndex struct { 21 | NoOpOpenClose 22 | index []sliceKey 23 | } 24 | 25 | func (s *SliceKeyIndex) search(key []byte) (int, bool) { 26 | return slices.BinarySearchFunc(s.index, key, func(entry sliceKey, k []byte) int { 27 | return bytes.Compare(entry.key, k) 28 | }) 29 | } 30 | 31 | func (s *SliceKeyIndex) Get(key []byte) (IndexVal, error) { 32 | idx, found := s.search(key) 33 | if found { 34 | return s.index[idx].IndexVal, nil 35 | } 36 | 37 | return IndexVal{}, skiplist.NotFound 38 | } 39 | 40 | func (s *SliceKeyIndex) Contains(key []byte) (bool, error) { 41 | _, found := s.search(key) 42 | return found, nil 43 | } 44 | 45 | func (s *SliceKeyIndex) Iterator() (skiplist.IteratorI[[]byte, IndexVal], error) { 46 | return &SliceKeyIndexIterator{index: s.index, endIndexExcl: len(s.index)}, nil 47 | } 48 | 49 | func (s *SliceKeyIndex) IteratorStartingAt(key []byte) (skiplist.IteratorI[[]byte, IndexVal], error) { 50 | idx, _ := s.search(key) 51 | return &SliceKeyIndexIterator{index: s.index, currentIndex: idx, endIndexExcl: len(s.index)}, nil 52 | } 53 | 54 | func (s *SliceKeyIndex) IteratorBetween(keyLower []byte, keyHigher []byte) (skiplist.IteratorI[[]byte, IndexVal], error) { 55 | if bytes.Compare(keyLower, keyHigher) > 0 { 56 | return nil, errors.New("keyHigher is lower than keyLower") 57 | } 58 | 59 | startIdx, _ := s.search(keyLower) 60 | endIdx, _ := s.search(keyHigher) 61 | 62 | // we need to adjust the ending a bit, because our iterator always includes the keyHigher in the result 63 | if endIdx >= 0 && endIdx < len(s.index) { 64 | if bytes.Compare(s.index[endIdx].key, keyHigher) <= 0 { 65 | endIdx = endIdx + 1 66 | } 67 | } 68 | 69 | return &SliceKeyIndexIterator{index: s.index, currentIndex: startIdx, endIndexExcl: endIdx}, nil 70 | } 71 | 72 | type SliceKeyIndexIterator struct { 73 | index []sliceKey 74 | endIndexExcl int 75 | currentIndex int 76 | } 77 | 78 | func (s *SliceKeyIndexIterator) Next() ([]byte, IndexVal, error) { 79 | if s.currentIndex >= s.endIndexExcl { 80 | return nil, IndexVal{}, skiplist.Done 81 | } 82 | cx := s.index[s.currentIndex] 83 | s.currentIndex += 1 84 | return cx.key, cx.IndexVal, nil 85 | } 86 | 87 | type SliceKeyIndexLoader struct { 88 | ReadBufferSize int 89 | } 90 | 91 | func (s *SliceKeyIndexLoader) Load(indexPath string, metadata *proto.MetaData) (SortedKeyIndex, error) { 92 | reader, err := rProto.NewReader( 93 | rProto.ReaderPath(indexPath), 94 | rProto.ReadBufferSizeBytes(s.ReadBufferSize), 95 | ) 96 | if err != nil { 97 | return nil, fmt.Errorf("error while creating index reader of sstable in '%s': %w", indexPath, err) 98 | } 99 | 100 | err = reader.Open() 101 | if err != nil { 102 | return nil, fmt.Errorf("error while opening index reader of sstable in '%s': %w", indexPath, err) 103 | } 104 | 105 | defer func() { 106 | err = errors.Join(err, reader.Close()) 107 | }() 108 | 109 | capacity := uint64(0) 110 | if metadata != nil { 111 | capacity = metadata.NumRecords 112 | } 113 | 114 | sx := make([]sliceKey, 0, capacity) 115 | 116 | record := &proto.IndexEntry{} 117 | for { 118 | _, err := reader.ReadNext(record) 119 | // io.EOF signals that no records are left to be read 120 | if errors.Is(err, io.EOF) { 121 | break 122 | } 123 | 124 | if err != nil { 125 | return nil, fmt.Errorf("error while reading index records of sstable in '%s': %w", indexPath, err) 126 | } 127 | 128 | sx = append(sx, sliceKey{IndexVal{Offset: record.ValueOffset, Checksum: record.Checksum}, record.Key}) 129 | } 130 | 131 | return &SliceKeyIndex{NoOpOpenClose{}, sx}, nil 132 | } 133 | -------------------------------------------------------------------------------- /sstables/sstable.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/skiplist" 6 | "github.com/thomasjungblut/go-sstables/sstables/proto" 7 | ) 8 | 9 | var IndexFileName = "index.rio" 10 | var DataFileName = "data.rio" 11 | var BloomFileName = "bloom.bf.gz" 12 | var MetaFileName = "meta.pb.bin" 13 | 14 | var Version = uint32(1) 15 | 16 | // Done indicates an iterator has returned all items. 17 | // https://github.com/GoogleCloudPlatform/google-cloud-go/wiki/Iterator-Guidelines 18 | var Done = errors.New("no more items in iterator") 19 | var NotFound = errors.New("key was not found") 20 | 21 | type SSTableIteratorI interface { 22 | // Next returns the next key, value in sequence. 23 | // Returns Done as the error when the iterator is exhausted 24 | Next() ([]byte, []byte, error) 25 | } 26 | 27 | type SSTableReaderI interface { 28 | // Contains returns true when the given key exists, false otherwise 29 | Contains(key []byte) (bool, error) 30 | // Get returns the value associated with the given key, NotFound as the error otherwise 31 | Get(key []byte) ([]byte, error) 32 | // Scan returns an iterator over the whole sorted sequence. Scan uses a more optimized version that iterates the 33 | // data file sequentially, whereas the other Scan* functions use the index and random access using mmap. 34 | Scan() (SSTableIteratorI, error) 35 | // ScanStartingAt returns an iterator over the sorted sequence starting at the given key (inclusive if key is in the list). 36 | // Using a key that is out of the sequence range will result in either an empty iterator or the full sequence. 37 | ScanStartingAt(key []byte) (SSTableIteratorI, error) 38 | // ScanRange returns an iterator over the sorted sequence starting at the given keyLower (inclusive if key is in the list) 39 | // and until the given keyHigher was reached (inclusive if key is in the list). 40 | // Using keys that are out of the sequence range will result in either an empty iterator or the full sequence. 41 | // If keyHigher is lower than keyLower an error will be returned. 42 | ScanRange(keyLower []byte, keyHigher []byte) (SSTableIteratorI, error) 43 | // Close closes this sstable reader 44 | Close() error 45 | // MetaData returns the metadata of this sstable 46 | MetaData() *proto.MetaData 47 | // BasePath returns the base path / root path of this sstable that contains all the files. 48 | BasePath() string 49 | } 50 | 51 | type SSTableSimpleWriterI interface { 52 | // WriteSkipList writes all records of that SkipList to a sstable disk structure, expects []byte as key and value 53 | WriteSkipList(skipListMap *skiplist.MapI[[]byte, []byte]) error 54 | } 55 | 56 | type SSTableStreamWriterI interface { 57 | // Open opens the sstable files. 58 | Open() error 59 | // WriteNext writes the next record to a sstable disk structure, expects keys to be ordered. 60 | WriteNext(key []byte, value []byte) error 61 | // Close closes the sstable files. 62 | Close() error 63 | } 64 | 65 | type SSTableMergerI interface { 66 | // Merge merges/writes the given Iterators into a single sorted SSTable 67 | Merge(iterators []SSTableIteratorI, writer SSTableStreamWriterI) error 68 | // MergeCompact is like merge, but accumulates values for the same key and presents it as a 69 | // "reduction" function to compact values for the same key. 70 | // reduce receives a key and a slice of values - it then needs to return a single key and value. 71 | MergeCompact(iterators []SSTableIteratorI, writer SSTableStreamWriterI, 72 | reduce func([]byte, [][]byte) ([]byte, []byte)) error 73 | } 74 | -------------------------------------------------------------------------------- /sstables/sstable_index.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "github.com/thomasjungblut/go-sstables/recordio" 5 | "github.com/thomasjungblut/go-sstables/skiplist" 6 | "github.com/thomasjungblut/go-sstables/sstables/proto" 7 | ) 8 | 9 | type IndexVal struct { 10 | Offset uint64 11 | Checksum uint64 12 | } 13 | 14 | type NoOpOpenClose struct { 15 | } 16 | 17 | func (s *NoOpOpenClose) Open() error { 18 | return nil 19 | } 20 | 21 | func (s *NoOpOpenClose) Close() error { 22 | return nil 23 | } 24 | 25 | type SortedKeyIndex interface { 26 | recordio.OpenClosableI 27 | 28 | // Contains returns true if the given key can be found in the index 29 | Contains(key []byte) (bool, error) 30 | // Get returns the IndexVal that compares equal to the key supplied or returns skiplist.NotFound if it does not exist. 31 | Get(key []byte) (IndexVal, error) 32 | // Iterator returns an iterator over the entire sorted sequence 33 | Iterator() (skiplist.IteratorI[[]byte, IndexVal], error) 34 | // IteratorStartingAt returns an iterator over the sorted sequence starting at the given key (inclusive if key is in the index). 35 | // Using a key that is out of the sequence range will result in either an empty iterator or the full sequence. 36 | IteratorStartingAt(key []byte) (skiplist.IteratorI[[]byte, IndexVal], error) 37 | // IteratorBetween Returns an iterator over the sorted sequence starting at the given keyLower (inclusive if key is in the index) 38 | // and until the given keyHigher was reached (inclusive if key is in the index). 39 | // Using keys that are out of the sequence range will result in either an empty iterator or the full sequence. 40 | // If keyHigher is lower than keyLower an error will be returned 41 | IteratorBetween(keyLower []byte, keyHigher []byte) (skiplist.IteratorI[[]byte, IndexVal], error) 42 | } 43 | 44 | type IndexLoader interface { 45 | // Load is creating a SortedKeyIndex from the given path. 46 | Load(path string, metadata *proto.MetaData) (SortedKeyIndex, error) 47 | } 48 | -------------------------------------------------------------------------------- /sstables/sstable_iterator.go: -------------------------------------------------------------------------------- 1 | package sstables 2 | 3 | import ( 4 | "errors" 5 | "github.com/thomasjungblut/go-sstables/recordio" 6 | rProto "github.com/thomasjungblut/go-sstables/recordio/proto" 7 | "github.com/thomasjungblut/go-sstables/skiplist" 8 | "github.com/thomasjungblut/go-sstables/sstables/proto" 9 | ) 10 | 11 | type SSTableIterator struct { 12 | reader *SSTableReader 13 | keyIterator skiplist.IteratorI[[]byte, IndexVal] 14 | } 15 | 16 | func (it *SSTableIterator) Next() ([]byte, []byte, error) { 17 | key, iv, err := it.keyIterator.Next() 18 | if err != nil { 19 | if errors.Is(err, skiplist.Done) { 20 | return nil, nil, Done 21 | } else { 22 | return nil, nil, err 23 | } 24 | } 25 | 26 | valBytes, err := it.reader.getValueAtOffset(iv, it.reader.opts.skipHashCheckOnRead) 27 | if err != nil { 28 | return nil, nil, err 29 | } 30 | 31 | return key, valBytes, nil 32 | } 33 | 34 | // V0SSTableFullScanIterator deprecated, since this is for the v0 protobuf based sstables. 35 | // this is an optimized iterator that does a sequential read over the index+data files instead of a 36 | // sequential read on the index with a random access lookup on the data file via mmap 37 | type V0SSTableFullScanIterator struct { 38 | keyIterator skiplist.IteratorI[[]byte, IndexVal] 39 | dataReader rProto.ReaderI 40 | } 41 | 42 | func (it *V0SSTableFullScanIterator) Next() ([]byte, []byte, error) { 43 | key, _, err := it.keyIterator.Next() 44 | if err != nil { 45 | if errors.Is(err, skiplist.Done) { 46 | return nil, nil, Done 47 | } else { 48 | return nil, nil, err 49 | } 50 | } 51 | 52 | value := &proto.DataEntry{} 53 | _, err = it.dataReader.ReadNext(value) 54 | if err != nil { 55 | return nil, nil, err 56 | } 57 | 58 | return key, value.Value, nil 59 | } 60 | 61 | func newV0SStableFullScanIterator(keyIterator skiplist.IteratorI[[]byte, IndexVal], dataReader rProto.ReaderI) (SSTableIteratorI, error) { 62 | return &V0SSTableFullScanIterator{ 63 | keyIterator: keyIterator, 64 | dataReader: dataReader, 65 | }, nil 66 | } 67 | 68 | // SSTableFullScanIterator this is an optimized iterator that does a sequential read over the index+data files instead of a 69 | // sequential read on the index with a random access lookup on the data file via mmap 70 | type SSTableFullScanIterator struct { 71 | keyIterator skiplist.IteratorI[[]byte, IndexVal] 72 | dataReader recordio.ReaderI 73 | 74 | skipHashCheck bool 75 | } 76 | 77 | func (it *SSTableFullScanIterator) Next() ([]byte, []byte, error) { 78 | key, iVal, err := it.keyIterator.Next() 79 | if err != nil { 80 | if errors.Is(err, skiplist.Done) { 81 | return nil, nil, Done 82 | } else { 83 | return nil, nil, err 84 | } 85 | } 86 | 87 | next, err := it.dataReader.ReadNext() 88 | if err != nil { 89 | return nil, nil, err 90 | } 91 | 92 | if it.skipHashCheck { 93 | return key, next, nil 94 | } 95 | 96 | checksum, err := checksumValue(next) 97 | if err != nil { 98 | return nil, nil, err 99 | } 100 | 101 | if checksum != iVal.Checksum { 102 | // this mismatch could come from default values, reading older formats 103 | if iVal.Checksum == 0 { 104 | return key, next, nil 105 | } 106 | 107 | return key, next, ChecksumError{checksum, iVal.Checksum} 108 | } 109 | 110 | return key, next, err 111 | } 112 | 113 | func newSStableFullScanIterator( 114 | keyIterator skiplist.IteratorI[[]byte, IndexVal], 115 | dataReader recordio.ReaderI, 116 | skipHashCheck bool) (SSTableIteratorI, error) { 117 | return &SSTableFullScanIterator{ 118 | keyIterator: keyIterator, 119 | dataReader: dataReader, 120 | skipHashCheck: skipHashCheck, 121 | }, nil 122 | } 123 | -------------------------------------------------------------------------------- /sstables/sstable_reader_generator_test.go: -------------------------------------------------------------------------------- 1 | // this does not really test anything, it generates the test_files that can be used to test the file_reader 2 | // you can switch it on by setting the "generate_compatfiles" env variable to something non-empty 3 | package sstables 4 | 5 | import ( 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "github.com/thomasjungblut/go-sstables/skiplist" 9 | "os" 10 | "path" 11 | "testing" 12 | ) 13 | 14 | func TestGenerateTestFiles(t *testing.T) { 15 | if os.Getenv("generate_compatfiles") == "" { 16 | t.Skip("not requested to generate compatibility files") 17 | return 18 | } 19 | 20 | prefix := "test_files/" 21 | writeHappyPathSSTable(t, prefix+"SimpleWriteHappyPathSSTableRecordIOV2") 22 | writeHappyPathSSTable(t, prefix+"SimpleWriteHappyPathSSTableWithBloom") 23 | writeHappyPathSSTable(t, prefix+"SimpleWriteHappyPathSSTableWithMetaData") 24 | writeHappyPathSSTable(t, prefix+"SimpleWriteHappyPathSSTableWithCRCHashes") 25 | writeHappyPathSSTableWithEmptyValues(t, prefix+"SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues") 26 | 27 | writeHappyPathSSTable(t, prefix+"SimpleWriteHappyPathSSTableWithCRCHashesMismatch") 28 | imputeError(t, prefix+"SimpleWriteHappyPathSSTableWithCRCHashesMismatch") 29 | } 30 | 31 | // this will change a byte at a specific offset for crc hash test cases 32 | func imputeError(t *testing.T, p string) { 33 | f, err := os.OpenFile(path.Join(p, DataFileName), os.O_RDWR, 0655) 34 | require.NoError(t, err) 35 | defer func() { 36 | require.NoError(t, f.Close()) 37 | }() 38 | 39 | _, err = f.WriteAt([]byte{0x15}, 51) 40 | require.NoError(t, err) 41 | } 42 | 43 | func writeHappyPathSSTable(t *testing.T, path string) { 44 | writer := newSimpleBytesWriterAt(t, path) 45 | list := TEST_ONLY_NewSkipListMapWithElements([]int{1, 2, 3, 4, 5, 6, 7}) 46 | err := writer.WriteSkipListMap(list) 47 | assert.Nil(t, err) 48 | } 49 | 50 | func writeHappyPathSSTableWithEmptyValues(t *testing.T, path string) { 51 | writer := newSimpleBytesWriterAt(t, path) 52 | list := skiplist.NewSkipListMap[[]byte, []byte](skiplist.BytesComparator{}) 53 | list.Insert(intToByteSlice(42), intToByteSlice(0)) 54 | list.Insert(intToByteSlice(45), []byte{}) 55 | err := writer.WriteSkipListMap(list) 56 | assert.Nil(t, err) 57 | } 58 | 59 | func newSimpleBytesWriterAt(t *testing.T, path string) *SSTableSimpleWriter { 60 | _ = os.RemoveAll(path) 61 | _ = os.MkdirAll(path, 0666) 62 | writer, e := NewSSTableSimpleWriter(WriteBasePath(path), WithKeyComparator(skiplist.BytesComparator{})) 63 | assert.Nil(t, e) 64 | return writer 65 | } 66 | -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTable/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTable/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTable/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTable/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/meta.pb.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableRecordIOV2/meta.pb.bin -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/meta.pb.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithBloom/meta.pb.bin -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/meta.pb.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashes/meta.pb.bin -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesEmptyValues/meta.pb.bin: -------------------------------------------------------------------------------- 1 | *- (,0E8 -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/meta.pb.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithCRCHashesMismatch/meta.pb.bin -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/data.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/index.rio -------------------------------------------------------------------------------- /sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/meta.pb.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/SimpleWriteHappyPathSSTableWithMetaData/meta.pb.bin -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTable/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTable/data.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTable/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTable/index.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/data.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/index.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableRecordIOV2/meta.pb.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/data.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithBloom/index.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/bloom.bf.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/bloom.bf.gz -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/data.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/data.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/index.rio: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/thomasjungblut/go-sstables/723e28a555ac23e8bd7f03d06047eb9383caf7f6/sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/index.rio -------------------------------------------------------------------------------- /sstables/test_files/v0_compat/SimpleWriteHappyPathSSTableWithMetaData/meta.pb.bin: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /test_examples.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ex_files=$(ls _examples/*.go) 4 | for f_name in $ex_files; do 5 | echo "running $f_name" 6 | go run $f_name 7 | if [ $? -ne 0 ]; then 8 | echo "failure while running $f_name" 9 | exit 1 10 | fi 11 | done 12 | -------------------------------------------------------------------------------- /wal/appender.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "fmt" 5 | "github.com/thomasjungblut/go-sstables/recordio" 6 | "path/filepath" 7 | ) 8 | 9 | // this is an implicitly hardcoded limit of one mio. WAL files, I hope that nobody needs more than that. 10 | const defaultWalSuffix = ".wal" 11 | const defaultWalFilePattern = "%06d" + defaultWalSuffix 12 | 13 | type Appender struct { 14 | nextWriterNumber uint 15 | walFileNamePattern string 16 | currentWriter recordio.WriterI 17 | currentWriterPath string 18 | walOptions *Options 19 | } 20 | 21 | func (a *Appender) Append(record []byte) error { 22 | err := checkSizeAndRotate(a, len(record)) 23 | if err != nil { 24 | return fmt.Errorf("error while rotating wal writer '%s': %w", a.currentWriterPath, err) 25 | } 26 | _, err = a.currentWriter.Write(record) 27 | if err != nil { 28 | return fmt.Errorf("error while appending to wal writer '%s': %w", a.currentWriterPath, err) 29 | } 30 | 31 | return nil 32 | } 33 | 34 | func (a *Appender) AppendSync(record []byte) error { 35 | err := checkSizeAndRotate(a, len(record)) 36 | if err != nil { 37 | return fmt.Errorf("error while rotating sync wal writer '%s': %w", a.currentWriterPath, err) 38 | } 39 | _, err = a.currentWriter.WriteSync(record) 40 | if err != nil { 41 | return fmt.Errorf("error while appending to sync wal writer '%s': %w", a.currentWriterPath, err) 42 | } 43 | 44 | return nil 45 | } 46 | 47 | func (a *Appender) Rotate() (string, error) { 48 | currentPath := a.currentWriterPath 49 | err := a.currentWriter.Close() 50 | if err != nil { 51 | return "", fmt.Errorf("error while closing current rotation writer '%s': %w", a.currentWriterPath, err) 52 | } 53 | 54 | err = setupNextWriter(a) 55 | if err != nil { 56 | return "", fmt.Errorf("error while setting up new rotation writer '%s': %w", a.currentWriterPath, err) 57 | } 58 | 59 | return currentPath, nil 60 | } 61 | 62 | func (a *Appender) Close() error { 63 | err := a.currentWriter.Close() 64 | if err != nil { 65 | return fmt.Errorf("error while closing appender and current rotation writer '%s': %w", a.currentWriterPath, err) 66 | } 67 | 68 | return nil 69 | } 70 | 71 | func checkSizeAndRotate(a *Appender, nextRecordSize int) error { 72 | if (a.currentWriter.Size() + uint64(nextRecordSize)) > a.walOptions.maxWalFileSize { 73 | _, err := a.Rotate() 74 | if err != nil { 75 | return fmt.Errorf("error rotating appender at '%s': %w", a.currentWriterPath, err) 76 | } 77 | } 78 | 79 | return nil 80 | } 81 | 82 | func setupNextWriter(a *Appender) error { 83 | if a.nextWriterNumber >= 1000000 { 84 | return fmt.Errorf("not supporting more than one million wal files at the minute. "+ 85 | "Current limit exceeded: %d", a.nextWriterNumber) 86 | } 87 | 88 | writerPath := filepath.Join(a.walOptions.basePath, fmt.Sprintf(defaultWalFilePattern, a.nextWriterNumber)) 89 | currentWriter, err := a.walOptions.writerFactory(writerPath) 90 | if err != nil { 91 | return fmt.Errorf("error while creating new wal appender writer under '%s': %w", writerPath, err) 92 | } 93 | 94 | err = currentWriter.Open() 95 | if err != nil { 96 | return fmt.Errorf("error while opening new wal appender writer under '%s': %w", writerPath, err) 97 | } 98 | 99 | a.nextWriterNumber++ 100 | a.currentWriter = currentWriter 101 | a.currentWriterPath = writerPath 102 | 103 | return nil 104 | } 105 | 106 | func NewAppender(walOpts *Options) (WriteAheadLogAppendI, error) { 107 | appender := &Appender{ 108 | walOptions: walOpts, 109 | nextWriterNumber: 0, 110 | walFileNamePattern: defaultWalFilePattern, 111 | currentWriter: nil, 112 | } 113 | 114 | err := setupNextWriter(appender) 115 | if err != nil { 116 | return nil, err 117 | } 118 | return appender, nil 119 | } 120 | -------------------------------------------------------------------------------- /wal/cleaner.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | type Cleaner struct { 9 | walOptions *Options 10 | } 11 | 12 | func (c *Cleaner) Clean() error { 13 | err := os.RemoveAll(c.walOptions.basePath) 14 | if err != nil { 15 | return fmt.Errorf("error while cleaning wal folders under '%s': %w", c.walOptions.basePath, err) 16 | } 17 | return nil 18 | } 19 | 20 | func NewCleaner(opts *Options) WriteAheadLogCleanI { 21 | return &Cleaner{walOptions: opts} 22 | } 23 | -------------------------------------------------------------------------------- /wal/cleaner_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | "github.com/stretchr/testify/require" 6 | "os" 7 | "testing" 8 | ) 9 | 10 | func TestSimpleDeleteHappyPath(t *testing.T) { 11 | log, _ := singleRecordWal(t, "wal_simpleDeleteHappyPath") 12 | info, err := os.Stat(log.walOptions.basePath) 13 | require.Nil(t, err) 14 | assert.True(t, info.IsDir()) 15 | err = NewCleaner(log.walOptions).Clean() 16 | require.Nil(t, err) 17 | _, err = os.Stat(log.walOptions.basePath) 18 | assert.NotNil(t, err) 19 | } 20 | -------------------------------------------------------------------------------- /wal/proto/proto_bindings.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "github.com/thomasjungblut/go-sstables/recordio" 5 | w "github.com/thomasjungblut/go-sstables/wal" 6 | "google.golang.org/protobuf/proto" 7 | ) 8 | 9 | type WriteAheadLogReplayI interface { 10 | // Replays the whole WAL from start, calling the given process function 11 | // for each record in guaranteed order. 12 | // This needs a factory to create the respective message type to use for deserialization. 13 | Replay(messageFactory func() proto.Message, process func(record proto.Message) error) error 14 | } 15 | 16 | type WriteAheadLogAppendI interface { 17 | recordio.CloseableI 18 | // Appends a given record and execute fsync to guarantee the persistence of the record. 19 | // Has considerably less throughput than Append. 20 | AppendSync(record proto.Message) error 21 | } 22 | 23 | type WriteAheadLogI interface { 24 | WriteAheadLogAppendI 25 | WriteAheadLogReplayI 26 | w.WriteAheadLogCleanI 27 | } 28 | 29 | type WriteAheadLog struct { 30 | wal w.WriteAheadLogI 31 | } 32 | 33 | func (p *WriteAheadLog) Clean() error { 34 | return p.wal.Clean() 35 | } 36 | 37 | func (p *WriteAheadLog) Replay(messageFactory func() proto.Message, process func(record proto.Message) error) error { 38 | err := p.wal.Replay(func(bytes []byte) error { 39 | msg := messageFactory() 40 | err := proto.Unmarshal(bytes, msg) 41 | if err != nil { 42 | return err 43 | } 44 | return process(msg) 45 | }) 46 | 47 | return err 48 | } 49 | 50 | func (p *WriteAheadLog) AppendSync(record proto.Message) error { 51 | bytes, err := proto.Marshal(record) 52 | if err != nil { 53 | return err 54 | } 55 | return p.wal.AppendSync(bytes) 56 | } 57 | 58 | func (p *WriteAheadLog) Close() error { 59 | return p.wal.Close() 60 | } 61 | 62 | func NewProtoWriteAheadLog(opts *w.Options) (WriteAheadLogI, error) { 63 | wal, err := w.NewWriteAheadLog(opts) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return &WriteAheadLog{ 69 | wal: wal, 70 | }, nil 71 | } 72 | -------------------------------------------------------------------------------- /wal/proto/proto_bindings_test.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | import ( 4 | "github.com/stretchr/testify/assert" 5 | w "github.com/thomasjungblut/go-sstables/wal" 6 | "github.com/thomasjungblut/go-sstables/wal/test_files" 7 | "google.golang.org/protobuf/proto" 8 | "os" 9 | "testing" 10 | ) 11 | 12 | const TestMaxWalFileSize uint64 = 8 * 1024 // 8k 13 | 14 | func TestProtoWALEndToEndHappyPath(t *testing.T) { 15 | wal := newTestProtoWal(t, "wal_proto_e2e_happy_path") 16 | 17 | maxNum := uint64(250) 18 | for i := uint64(0); i < maxNum; i++ { 19 | msg := test_files.SequenceNumber{SequenceNumber: i} 20 | err := wal.AppendSync(&msg) 21 | assert.Nil(t, err) 22 | } 23 | 24 | assert.Nil(t, wal.Close()) 25 | 26 | sequenceNum := &test_files.SequenceNumber{} 27 | msgFactory := func() proto.Message { 28 | return sequenceNum 29 | } 30 | 31 | expected := uint64(0) 32 | err := wal.Replay(msgFactory, func(msg proto.Message) error { 33 | assert.Equal(t, expected, msg.(*test_files.SequenceNumber).SequenceNumber) 34 | expected++ 35 | return nil 36 | }) 37 | 38 | assert.Nil(t, err) 39 | assert.Equal(t, maxNum, expected) 40 | } 41 | 42 | func newTestProtoWal(t *testing.T, tmpDirName string) *WriteAheadLog { 43 | tmpDir, err := os.MkdirTemp("", tmpDirName) 44 | assert.Nil(t, err) 45 | 46 | opts, err := w.NewWriteAheadLogOptions( 47 | w.BasePath(tmpDir), 48 | w.MaximumWalFileSizeBytes(TestMaxWalFileSize)) 49 | assert.Nil(t, err) 50 | 51 | wal, err := NewProtoWriteAheadLog(opts) 52 | assert.Nil(t, err) 53 | t.Cleanup(func() { 54 | _ = wal.Clean() 55 | }) 56 | 57 | return wal.(*WriteAheadLog) 58 | } 59 | -------------------------------------------------------------------------------- /wal/replayer.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/thomasjungblut/go-sstables/recordio" 7 | "io" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | "strings" 12 | ) 13 | 14 | type Replayer struct { 15 | walOptions *Options 16 | } 17 | 18 | func (r *Replayer) Replay(process func(record []byte) error) (err error) { 19 | var walFiles []string 20 | err = filepath.Walk(r.walOptions.basePath, func(path string, info os.FileInfo, err error) error { 21 | if err != nil { 22 | return err 23 | } 24 | 25 | if !info.IsDir() && strings.HasSuffix(info.Name(), defaultWalSuffix) { 26 | walFiles = append(walFiles, path) 27 | } 28 | 29 | return nil 30 | }) 31 | 32 | if err != nil { 33 | return fmt.Errorf("error while walking WAL structure under '%s': %w", r.walOptions.basePath, err) 34 | } 35 | 36 | // do not rely on the order of the FS, we do an additional sort to make sure we start reading from 0000 to 9999 37 | sort.Strings(walFiles) 38 | 39 | var toClose []recordio.ReaderI 40 | defer func() { 41 | for _, reader := range toClose { 42 | err = errors.Join(err, reader.Close()) 43 | } 44 | }() 45 | 46 | for _, path := range walFiles { 47 | reader, err := r.walOptions.readerFactory(path) 48 | if err != nil { 49 | return fmt.Errorf("error while creating WAL reader under '%s': %w", path, err) 50 | } 51 | toClose = append(toClose, reader) 52 | 53 | err = reader.Open() 54 | if err != nil { 55 | return fmt.Errorf("error while opening WAL reader under '%s': %w", path, err) 56 | } 57 | 58 | for { 59 | bytes, err := reader.ReadNext() 60 | // io.EOF signals that no records are left to be read 61 | if errors.Is(err, io.EOF) { 62 | break 63 | } 64 | 65 | if err != nil { 66 | return fmt.Errorf("error while reading WAL records under '%s': %w", path, err) 67 | } 68 | 69 | err = process(bytes) 70 | if err != nil { 71 | return fmt.Errorf("error while processing WAL record under '%s': %w", path, err) 72 | } 73 | } 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func NewReplayer(walOpts *Options) (WriteAheadLogReplayI, error) { 80 | stat, err := os.Stat(walOpts.basePath) 81 | if err != nil { 82 | return nil, fmt.Errorf("error creating replayer by stat the path at '%s': %w", walOpts.basePath, err) 83 | } 84 | 85 | if !stat.IsDir() { 86 | return nil, fmt.Errorf("given base path %s is not a directory", walOpts.basePath) 87 | } 88 | 89 | return &Replayer{ 90 | walOptions: walOpts, 91 | }, nil 92 | } 93 | -------------------------------------------------------------------------------- /wal/replayer_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "os" 9 | "path/filepath" 10 | "testing" 11 | ) 12 | 13 | // we largely exercise replays in wal_appender_test already, here we focus only on edge cases that cause errors 14 | 15 | func TestReplayFileFails(t *testing.T) { 16 | file, err := os.CreateTemp("", "wal_replayfilefails") 17 | require.Nil(t, err) 18 | opts, err := NewWriteAheadLogOptions(BasePath(file.Name())) 19 | require.Nil(t, err) 20 | _, err = NewReplayer(opts) 21 | assert.Equal(t, fmt.Errorf("given base path %s is not a directory", file.Name()), err) 22 | } 23 | 24 | func TestReplayFolderDoesNotExist(t *testing.T) { 25 | opts, err := NewWriteAheadLogOptions(BasePath("somepaththathopefullydoesnotexistanywhere")) 26 | require.Nil(t, err) 27 | _, err = NewReplayer(opts) 28 | assert.NotNil(t, err) 29 | } 30 | 31 | func TestReplayerIgnoresNonWalFiles(t *testing.T) { 32 | log, recorder := singleRecordWal(t, "wal_replayignorewal") 33 | 34 | err := os.WriteFile(filepath.Join(log.walOptions.basePath, "some-not-so-wal-file"), []byte{1, 2, 3}, os.ModePerm) 35 | require.Nil(t, err) 36 | 37 | assertRecorderMatchesReplay(t, log.walOptions, recorder) 38 | } 39 | 40 | func TestReplayHonorsCallbackErrors(t *testing.T) { 41 | log, _ := singleRecordWal(t, "wal_replayhonorscallback") 42 | repl, err := NewReplayer(log.walOptions) 43 | require.Nil(t, err) 44 | testErr := errors.New("test") 45 | err = repl.Replay(func(record []byte) error { 46 | return testErr 47 | }) 48 | assert.True(t, errors.Is(err, testErr)) 49 | } 50 | -------------------------------------------------------------------------------- /wal/test_files/seq_number.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package test_files; 3 | option go_package = "github.com/thomasjungblut/go-sstables/wal/test_files"; 4 | 5 | message SequenceNumber { 6 | uint64 sequenceNumber = 1; 7 | } 8 | -------------------------------------------------------------------------------- /wal/write_ahead_log.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/thomasjungblut/go-sstables/recordio" 7 | ) 8 | 9 | const DefaultMaxWalSize uint64 = 128 * 1024 * 1024 // 128mb 10 | 11 | type WriteAheadLogReplayI interface { 12 | // Replay the whole WAL from start, calling the given process function 13 | // for each record in guaranteed order. 14 | Replay(process func(record []byte) error) error 15 | } 16 | 17 | type WriteAheadLogAppendI interface { 18 | recordio.CloseableI 19 | // Append a given record and does NOT execute fsync to guarantee the persistence of the record. 20 | Append(record []byte) error 21 | // AppendSync a given record and execute fsync to guarantee the persistence of the record. 22 | // Has considerably less throughput than Append. 23 | AppendSync(record []byte) error 24 | 25 | // Rotate - The WAL usually auto-rotates after a certain size - this method allows to force this rotation. 26 | // This can be useful in scenarios where you want to flush a memstore and rotate the WAL at the same time. 27 | // Therefore, this returns the path of the previous wal file that was closed through this operation. 28 | Rotate() (string, error) 29 | } 30 | 31 | type WriteAheadLogCleanI interface { 32 | // Clean Removes all WAL files and the directory it is contained in 33 | Clean() error 34 | } 35 | 36 | type WriteAheadLogCompactI interface { 37 | // Compact should compact the WAL, but isn't properly implemented just yet 38 | Compact() error 39 | } 40 | 41 | type WriteAheadLogI interface { 42 | WriteAheadLogAppendI 43 | WriteAheadLogReplayI 44 | WriteAheadLogCleanI 45 | } 46 | 47 | type WriteAheadLog struct { 48 | WriteAheadLogAppendI 49 | WriteAheadLogReplayI 50 | WriteAheadLogCleanI 51 | } 52 | 53 | // NewWriteAheadLog creates a new WAL by supplying options, for example using a base path: wal.NewWriteAheadLogOptions(wal.BasePath("some_directory")) 54 | func NewWriteAheadLog(opts *Options) (WriteAheadLogI, error) { 55 | appender, err := NewAppender(opts) 56 | if err != nil { 57 | return nil, err 58 | } 59 | replayer, err := NewReplayer(opts) 60 | if err != nil { 61 | return nil, err 62 | } 63 | return &WriteAheadLog{ 64 | appender, 65 | replayer, 66 | NewCleaner(opts), 67 | }, nil 68 | } 69 | 70 | // NewWriteAheadLogOptions handles WAL configurations, the minimal required option is the base path: wal.NewWriteAheadLogOptions(wal.BasePath("some_directory")) 71 | func NewWriteAheadLogOptions(walOptions ...Option) (*Options, error) { 72 | opts := &Options{ 73 | basePath: "", 74 | maxWalFileSize: DefaultMaxWalSize, 75 | writerFactory: func(path string) (recordio.WriterI, error) { 76 | return recordio.NewFileWriter(recordio.Path(path)) 77 | }, 78 | readerFactory: func(path string) (recordio.ReaderI, error) { 79 | return recordio.NewFileReaderWithPath(path) 80 | }, 81 | } 82 | 83 | for _, walOption := range walOptions { 84 | walOption(opts) 85 | } 86 | 87 | if opts.basePath == "" { 88 | return nil, errors.New("basePath was not supplied") 89 | } 90 | 91 | return opts, nil 92 | } 93 | 94 | // options 95 | 96 | type Options struct { 97 | maxWalFileSize uint64 98 | basePath string 99 | // TODO(thomas): this should be ideally in a writer-only option 100 | writerFactory func(path string) (recordio.WriterI, error) 101 | // TODO(thomas): this should be ideally in a reader-only option 102 | readerFactory func(path string) (recordio.ReaderI, error) 103 | } 104 | 105 | type Option func(*Options) 106 | 107 | func BasePath(p string) Option { 108 | return func(args *Options) { 109 | args.basePath = p 110 | } 111 | } 112 | 113 | func MaximumWalFileSizeBytes(p uint64) Option { 114 | return func(args *Options) { 115 | args.maxWalFileSize = p 116 | } 117 | } 118 | 119 | func WriterFactory(writerFactory func(path string) (recordio.WriterI, error)) Option { 120 | return func(args *Options) { 121 | args.writerFactory = writerFactory 122 | } 123 | } 124 | 125 | func ReaderFactory(readerFactory func(path string) (recordio.ReaderI, error)) Option { 126 | return func(args *Options) { 127 | args.readerFactory = readerFactory 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /wal/write_ahead_log_test.go: -------------------------------------------------------------------------------- 1 | package wal 2 | 3 | import ( 4 | "encoding/binary" 5 | "errors" 6 | "github.com/stretchr/testify/assert" 7 | "github.com/stretchr/testify/require" 8 | "github.com/thomasjungblut/go-sstables/recordio" 9 | "os" 10 | "testing" 11 | ) 12 | 13 | func TestWALEndToEndHappyPath(t *testing.T) { 14 | wal := newTestWal(t, "wal_e2e_happy_path") 15 | 16 | maxNum := uint64(2500) 17 | for i := uint64(0); i < maxNum; i++ { 18 | record := make([]byte, 8) 19 | binary.BigEndian.PutUint64(record, i) 20 | err := wal.AppendSync(record) 21 | require.Nil(t, err) 22 | } 23 | 24 | require.Nil(t, wal.Close()) 25 | 26 | expected := uint64(0) 27 | err := wal.Replay(func(record []byte) error { 28 | n := binary.BigEndian.Uint64(record) 29 | assert.Equal(t, expected, n) 30 | expected++ 31 | return nil 32 | }) 33 | require.Nil(t, err) 34 | assert.Equal(t, maxNum, expected) 35 | } 36 | 37 | func TestWALCrashRecovery(t *testing.T) { 38 | // this is a very naive test to figure out whether we can still read a partially written WAL 39 | // ideally we would start a new process to continuously write data into a WAL and make it crash. 40 | // this certainly also does not simulate when we actually have a full power-down scenario, 41 | // but that's the best we can do in a unit test. 42 | wal := newTestWal(t, "wal_e2e_crash_recovery") 43 | 44 | maxNum := uint64(100) 45 | for i := uint64(0); i < maxNum; i++ { 46 | record := make([]byte, 8) 47 | binary.BigEndian.PutUint64(record, i) 48 | err := wal.AppendSync(record) 49 | require.Nil(t, err) 50 | 51 | // mind: we do not close the wal between, to test if we are actually fsync'ing properly and 52 | // we can always read from whatever was appended already 53 | expected := uint64(0) 54 | err = wal.Replay(func(record []byte) error { 55 | n := binary.BigEndian.Uint64(record) 56 | assert.Equal(t, expected, n) 57 | expected++ 58 | return nil 59 | }) 60 | assert.Equal(t, i+1, expected) 61 | require.Nil(t, err) 62 | } 63 | } 64 | 65 | func TestOptionMissingBasePath(t *testing.T) { 66 | _, err := NewWriteAheadLogOptions(MaximumWalFileSizeBytes(TestMaxWalFileSize)) 67 | assert.Equal(t, errors.New("basePath was not supplied"), err) 68 | } 69 | 70 | func newTestWal(t *testing.T, tmpDirName string) *WriteAheadLog { 71 | tmpDir, err := os.MkdirTemp("", tmpDirName) 72 | require.Nil(t, err) 73 | 74 | opts, err := NewWriteAheadLogOptions(BasePath(tmpDir), 75 | MaximumWalFileSizeBytes(TestMaxWalFileSize), 76 | WriterFactory(func(path string) (recordio.WriterI, error) { 77 | return recordio.NewFileWriter(recordio.Path(path), recordio.CompressionType(recordio.CompressionTypeSnappy)) 78 | }), 79 | ReaderFactory(func(path string) (recordio.ReaderI, error) { 80 | return recordio.NewFileReaderWithPath(path) 81 | }), 82 | ) 83 | require.Nil(t, err) 84 | 85 | wal, err := NewWriteAheadLog(opts) 86 | require.Nil(t, err) 87 | t.Cleanup(func() { 88 | _ = wal.Close() 89 | _ = wal.Clean() 90 | }) 91 | 92 | return wal.(*WriteAheadLog) 93 | } 94 | --------------------------------------------------------------------------------