├── go.mod ├── store.go ├── Makefile ├── .gitignore ├── .github └── workflows │ └── build.yml ├── memory_store.go ├── memory_store_test.go ├── workshop.md ├── LICENSE ├── format_test.go ├── disk_store_test.go ├── hints.md ├── format.go ├── disk_store.go └── README.md /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/avinassh/go-caskdb 2 | 3 | go 1.19 4 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | type Store interface { 4 | Get(key string) string 5 | Set(key string, value string) 6 | Close() bool 7 | } 8 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | build: 2 | go build 3 | 4 | test: 5 | go test -v ./... 6 | 7 | lint: 8 | go fmt ./... 9 | 10 | coverage: 11 | go test -coverprofile=coverage.txt ./... 12 | 13 | html: coverage 14 | go tool cover -html=coverage.txt -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | vendor/ 16 | 17 | # coverage report file 18 | coverage.txt -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'master' 7 | pull_request: {} 8 | 9 | jobs: 10 | tests: 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | os: [ ubuntu-latest, macos-latest, windows-latest ] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - uses: actions/setup-go@v3 18 | - name: tests 19 | run: | 20 | make test 21 | -------------------------------------------------------------------------------- /memory_store.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | type MemoryStore struct { 4 | data map[string]string 5 | } 6 | 7 | func NewMemoryStore() *MemoryStore { 8 | return &MemoryStore{make(map[string]string)} 9 | } 10 | 11 | func (m *MemoryStore) Get(key string) string { 12 | return m.data[key] 13 | } 14 | 15 | func (m *MemoryStore) Set(key string, value string) { 16 | m.data[key] = value 17 | } 18 | 19 | func (m *MemoryStore) Close() bool { 20 | return true 21 | } 22 | -------------------------------------------------------------------------------- /memory_store_test.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | import "testing" 4 | 5 | func TestMemoryStore_Get(t *testing.T) { 6 | store := NewMemoryStore() 7 | store.Set("name", "jojo") 8 | if val := store.Get("name"); val != "jojo" { 9 | t.Errorf("Get() = %v, want %v", val, "jojo") 10 | } 11 | } 12 | 13 | func TestMemoryStore_InvalidGet(t *testing.T) { 14 | store := NewMemoryStore() 15 | if val := store.Get("some rando key"); val != "" { 16 | t.Errorf("Get() = %v, want %v", val, "") 17 | } 18 | } 19 | 20 | func TestMemoryStore_Close(t *testing.T) { 21 | store := NewMemoryStore() 22 | if !store.Close() { 23 | t.Errorf("Close() failed") 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /workshop.md: -------------------------------------------------------------------------------- 1 | # Workshop 2 | 3 | I don't have any workshops scheduled shortly. [Follow me on Twitter](https://twitter.com/iavins/) for updates. [Drop me an email](http://scr.im/avii) if you wish to arrange a workshop for your team/company. 4 | 5 | A typical workshop session is 4 hours and can span over two days. It is a hands-on session where I will live code with you. Though I will code in Python, you / your team may pick any programming language you are comfortable with. We will also pair/mob program together. 6 | 7 | | Session | Duration | 8 | |---------------------------------------|----------| 9 | | The building blocks of the database | 1 Hour | 10 | | Coffee break | 5 Min | 11 | | Live coding | 2 Hour | 12 | | Break | 10 Min | 13 | | Some Level 1 challenges | 30 Min | 14 | | [Optional] Covering the Bitcask Paper | 30 Min | 15 | | [Optional] Discussion of Level 2, 3 | 30 Min | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Avinash Sajjanshetty 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /format_test.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func Test_encodeHeader(t *testing.T) { 8 | tests := []struct { 9 | timestamp uint32 10 | keySize uint32 11 | valueSize uint32 12 | }{ 13 | {10, 10, 10}, 14 | {0, 0, 0}, 15 | {10000, 10000, 10000}, 16 | } 17 | for _, tt := range tests { 18 | data := encodeHeader(tt.timestamp, tt.keySize, tt.valueSize) 19 | timestamp, keySize, valueSize := decodeHeader(data) 20 | if timestamp != tt.timestamp { 21 | t.Errorf("encodeHeader() timestamp = %v, want %v", timestamp, tt.timestamp) 22 | } 23 | if keySize != tt.keySize { 24 | t.Errorf("encodeHeader() keySize = %v, want %v", keySize, tt.keySize) 25 | } 26 | if valueSize != tt.valueSize { 27 | t.Errorf("encodeHeader() valueSize = %v, want %v", valueSize, tt.valueSize) 28 | } 29 | } 30 | } 31 | 32 | func Test_encodeKV(t *testing.T) { 33 | tests := []struct { 34 | timestamp uint32 35 | key string 36 | value string 37 | size int 38 | }{ 39 | {10, "hello", "world", headerSize + 10}, 40 | {0, "", "", headerSize}, 41 | {100, "🔑", "", headerSize + 4}, 42 | } 43 | for _, tt := range tests { 44 | size, data := encodeKV(tt.timestamp, tt.key, tt.value) 45 | timestamp, key, value := decodeKV(data) 46 | if timestamp != tt.timestamp { 47 | t.Errorf("encodeKV() timestamp = %v, want %v", timestamp, tt.timestamp) 48 | } 49 | if key != tt.key { 50 | t.Errorf("encodeKV() key = %v, want %v", key, tt.key) 51 | } 52 | if value != tt.value { 53 | t.Errorf("encodeKV() value = %v, want %v", value, tt.value) 54 | } 55 | if size != tt.size { 56 | t.Errorf("encodeKV() size = %v, want %v", size, tt.size) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /disk_store_test.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | func TestDiskStore_Get(t *testing.T) { 9 | store, err := NewDiskStore("test.db") 10 | if err != nil { 11 | t.Fatalf("failed to create disk store: %v", err) 12 | } 13 | defer os.Remove("test.db") 14 | store.Set("name", "jojo") 15 | if val := store.Get("name"); val != "jojo" { 16 | t.Errorf("Get() = %v, want %v", val, "jojo") 17 | } 18 | } 19 | 20 | func TestDiskStore_GetInvalid(t *testing.T) { 21 | store, err := NewDiskStore("test.db") 22 | if err != nil { 23 | t.Fatalf("failed to create disk store: %v", err) 24 | } 25 | defer os.Remove("test.db") 26 | if val := store.Get("some key"); val != "" { 27 | t.Errorf("Get() = %v, want %v", val, "") 28 | } 29 | } 30 | 31 | func TestDiskStore_SetWithPersistence(t *testing.T) { 32 | store, err := NewDiskStore("test.db") 33 | if err != nil { 34 | t.Fatalf("failed to create disk store: %v", err) 35 | } 36 | defer os.Remove("test.db") 37 | 38 | tests := map[string]string{ 39 | "crime and punishment": "dostoevsky", 40 | "anna karenina": "tolstoy", 41 | "war and peace": "tolstoy", 42 | "hamlet": "shakespeare", 43 | "othello": "shakespeare", 44 | "brave new world": "huxley", 45 | "dune": "frank herbert", 46 | } 47 | for key, val := range tests { 48 | store.Set(key, val) 49 | if store.Get(key) != val { 50 | t.Errorf("Get() = %v, want %v", store.Get(key), val) 51 | } 52 | } 53 | store.Close() 54 | store, err = NewDiskStore("test.db") 55 | if err != nil { 56 | t.Fatalf("failed to create disk store: %v", err) 57 | } 58 | for key, val := range tests { 59 | if store.Get(key) != val { 60 | t.Errorf("Get() = %v, want %v", store.Get(key), val) 61 | } 62 | } 63 | store.Close() 64 | } 65 | 66 | func TestDiskStore_Delete(t *testing.T) { 67 | store, err := NewDiskStore("test.db") 68 | if err != nil { 69 | t.Fatalf("failed to create disk store: %v", err) 70 | } 71 | defer os.Remove("test.db") 72 | 73 | tests := map[string]string{ 74 | "crime and punishment": "dostoevsky", 75 | "anna karenina": "tolstoy", 76 | "war and peace": "tolstoy", 77 | "hamlet": "shakespeare", 78 | "othello": "shakespeare", 79 | "brave new world": "huxley", 80 | "dune": "frank herbert", 81 | } 82 | for key, val := range tests { 83 | store.Set(key, val) 84 | } 85 | for key := range tests { 86 | store.Set(key, "") 87 | } 88 | store.Set("end", "yes") 89 | store.Close() 90 | 91 | store, err = NewDiskStore("test.db") 92 | if err != nil { 93 | t.Fatalf("failed to create disk store: %v", err) 94 | } 95 | for key := range tests { 96 | if store.Get(key) != "" { 97 | t.Errorf("Get() = %v, want '' (empty)", store.Get(key)) 98 | } 99 | } 100 | if store.Get("end") != "yes" { 101 | t.Errorf("Get() = %v, want %v", store.Get("end"), "yes") 102 | } 103 | store.Close() 104 | } 105 | -------------------------------------------------------------------------------- /hints.md: -------------------------------------------------------------------------------- 1 | # Hints 2 | 3 | _contains spoilers, proceed with caution!_ 4 | 5 | ## Tests 6 | Below is the order in which you should pass the tests: 7 | 8 | 1. `Test_encodeHeader` 9 | 2. `Test_encodeKV` 10 | 3. `TestDiskStore_Set` 11 | 4. `TestDiskStore_Get` and `TestDiskStore_GetInvalid` 12 | 5. `TestDiskStore_SetWithPersistence` 13 | 14 | ## Tasks 15 | 16 | I have relisted the tasks here again, but with more details. If you have difficulty understanding the steps, the following should help you. 17 | 18 | ### Read the paper 19 | The word 'paper' might scare you, but [Bitcask's paper](https://riak.com/assets/bitcask-intro.pdf) is very approachable. It is only six pages long, half of them being diagrams. 20 | 21 | ### Header 22 | 23 | | Test | Test_encodeHeader | 24 | |------|-------------------| 25 | 26 | The next step is to implement a fixed-sized header similar to Bitcask. Every record in our DB contains a header holding metadata and helps our DB read values from the disk. The DB will read a bunch of bytes from the disk, so we need information on how many bytes to read and from which byte offset. 27 | 28 | **Some more details:** 29 | 30 | The header contains three fields timestamp, key size, and value size. 31 | 32 | | Field | Type | Size | 33 | |------------|------|------| 34 | | timestamp | int | 4B | 35 | | key_size | int | 4B | 36 | | value_size | int | 4B | 37 | 38 | We need to implement a function which takes all these three fields and serialises them to bytes. The function signature looks like this: 39 | 40 | ```go 41 | func encodeHeader(timestamp uint32, keySize uint32, valueSize uint32) 42 | ``` 43 | 44 | Then we also need to write the opposite of the above: 45 | 46 | ```go 47 | func decodeHeader(header []byte) (uint32, uint32, uint32) 48 | ``` 49 | 50 | **More Hints:** 51 | - Read this [comment](https://github.com/avinassh/go-caskdb/blob/0ae4fab/format.go#L3,L37) to understand why do we need serialiser methods 52 | - Not sure how to come up with a file format? Read the comment in the [format module](https://github.com/avinassh/go-caskdb/blob/0ae4fab/format.go#L41,L63) 53 | 54 | ### Key Value Serialisers 55 | 56 | | Test | Test_encodeKV | 57 | |------|---------------| 58 | 59 | Now we will write encode and decode methods for key and value. 60 | 61 | The method signatures: 62 | ```go 63 | func encodeKV(timestamp uint32, key string, value string) (int, []byte) 64 | func decodeKV(data []byte) (uint32, string, string) 65 | ``` 66 | 67 | Note that `encodeKV` method returns the bytes and the bytes' size. 68 | 69 | ### Storing to Disk 70 | 71 | | Test | TestDiskStore_Set | 72 | |------|-------------------| 73 | 74 | This step involves figuring out the persistence layer, saving the data to the disk, and keeping the pointer to the inserted record in the memory. 75 | 76 | So, implement the `DiskStore.Set` class in `disk_store.go` 77 | 78 | **Hints:** 79 | - Some meta info on the DiskStore and inner workings of the DiskStore are [here](https://github.com/avinassh/go-caskdb/blob/0ae4fab/disk_store.go#L24,L63). 80 | 81 | ### Start up tasks 82 | 83 | | Test | TestDiskStore_SetWithPersistence | 84 | |------|----------------------------------| 85 | 86 | DiskStore is a persistent key-value store, so we need to load the existing keys into the `keyDir` at the start of the database. 87 | -------------------------------------------------------------------------------- /format.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | import ( 4 | "encoding/binary" 5 | ) 6 | 7 | // format file provides encode/decode functions for serialisation and deserialisation 8 | // operations 9 | // 10 | // format methods are generic and does not have any disk or memory specific code. 11 | // 12 | // The disk storage deals with bytes; you cannot just store a string or object without 13 | // converting it to bytes. The programming languages provide abstractions where you 14 | // don't have to think about all this when storing things in memory (i.e. RAM). 15 | // Consider the following example where you are storing stuff in a hash table: 16 | // 17 | // books = {} 18 | // books["hamlet"] = "shakespeare" 19 | // books["anna karenina"] = "tolstoy" 20 | // 21 | // In the above, the language deals with all the complexities: 22 | // 23 | // - allocating space on the RAM so that it can store data of `books` 24 | // - whenever you add data to `books`, convert that to bytes and keep it in the memory 25 | // - whenever the size of `books` increases, move that to somewhere in the RAM so that 26 | // we can add new items 27 | // 28 | // Unfortunately, when it comes to disks, we have to do all this by ourselves, write 29 | // code which can allocate space, convert objects to/from bytes and many other operations. 30 | // 31 | // This file has two functions which help us with serialisation of data. 32 | // 33 | // encodeKV - takes the key value pair and encodes them into bytes 34 | // decodeKV - takes a bunch of bytes and decodes them into key value pairs 35 | // 36 | //**workshop note** 37 | // 38 | //For the workshop, the functions will have the following signature: 39 | // 40 | // func encodeKV(timestamp uint32, key string, value string) (int, []byte) 41 | // func decodeKV(data []byte) (uint32, string, string) 42 | 43 | // headerSize specifies the total header size. Our key value pair, when stored on disk 44 | // looks like this: 45 | // 46 | // ┌───────────┬──────────┬────────────┬─────┬───────┐ 47 | // │ timestamp │ key_size │ value_size │ key │ value │ 48 | // └───────────┴──────────┴────────────┴─────┴───────┘ 49 | // 50 | // This is analogous to a typical database's row (or a record). The total length of 51 | // the row is variable, depending on the contents of the key and value. 52 | // 53 | // The first three fields form the header: 54 | // 55 | // ┌───────────────┬──────────────┬────────────────┐ 56 | // │ timestamp(4B) │ key_size(4B) │ value_size(4B) │ 57 | // └───────────────┴──────────────┴────────────────┘ 58 | // 59 | // These three fields store unsigned integers of size 4 bytes, giving our header a 60 | // fixed length of 12 bytes. Timestamp field stores the time the record we 61 | // inserted in unix epoch seconds. Key size and value size fields store the length of 62 | // bytes occupied by the key and value. The maximum integer 63 | // stored by 4 bytes is 4,294,967,295 (2 ** 32 - 1), roughly ~4.2GB. So, the size of 64 | // each key or value cannot exceed this. Theoretically, a single row can be as large 65 | // as ~8.4GB. 66 | const headerSize = 12 67 | 68 | // KeyEntry keeps the metadata about the KV, specially the position of 69 | // the byte offset in the file. Whenever we insert/update a key, we create a new 70 | // KeyEntry object and insert that into keyDir. 71 | type KeyEntry struct { 72 | Offset uint32 73 | Size uint32 74 | Timestamp uint32 75 | } 76 | 77 | func NewKeyEntry(timestamp uint32, position uint32, totalSize uint32) KeyEntry { 78 | return KeyEntry{ 79 | Timestamp: timestamp, 80 | Offset: position, 81 | Size: totalSize, 82 | } 83 | } 84 | 85 | func encodeHeader(timestamp uint32, keySize uint32, valueSize uint32) []byte { 86 | header := make([]byte, 0, headerSize) 87 | header = binary.BigEndian.AppendUint32(header, timestamp) 88 | header = binary.BigEndian.AppendUint32(header, keySize) 89 | header = binary.BigEndian.AppendUint32(header, valueSize) 90 | return header 91 | } 92 | 93 | func decodeHeader(header []byte) (uint32, uint32, uint32) { 94 | if len(header) != 12 { 95 | panic("Invalid header") 96 | } 97 | timestamp := binary.BigEndian.Uint32(header[0:4]) 98 | keySize := binary.BigEndian.Uint32(header[4:8]) 99 | valueSize := binary.BigEndian.Uint32(header[8:]) 100 | return timestamp, keySize, valueSize 101 | } 102 | 103 | func encodeKV(timestamp uint32, key string, value string) (int, []byte) { 104 | keySize := len(key) 105 | valueSize := len(value) 106 | header := encodeHeader(timestamp, uint32(keySize), uint32(valueSize)) 107 | kv := make([]byte, 0, headerSize+keySize+valueSize) 108 | kv = append(kv, header...) 109 | kv = append(kv, []byte(key)...) 110 | kv = append(kv, []byte(value)...) 111 | return headerSize + keySize + valueSize, kv 112 | 113 | } 114 | 115 | func decodeKV(data []byte) (uint32, string, string) { 116 | timestamp, keySize, valueSize := decodeHeader(data[0:12]) 117 | key := data[12 : 12+keySize] 118 | value := data[12+keySize : 12+keySize+valueSize] 119 | 120 | return timestamp, string(key), string(value) 121 | 122 | } 123 | -------------------------------------------------------------------------------- /disk_store.go: -------------------------------------------------------------------------------- 1 | package caskdb 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "io/fs" 8 | "os" 9 | "time" 10 | ) 11 | 12 | // DiskStore is a Log-Structured Hash Table as described in the BitCask paper. We 13 | // keep appending the data to a file, like a log. DiskStorage maintains an in-memory 14 | // hash table called KeyDir, which keeps the row's location on the disk. 15 | // 16 | // The idea is simple yet brilliant: 17 | // - Write the record to the disk 18 | // - Update the internal hash table to point to that byte offset 19 | // - Whenever we get a read request, check the internal hash table for the address, 20 | // fetch that and return 21 | // 22 | // KeyDir does not store values, only their locations. 23 | // 24 | // The above approach solves a lot of problems: 25 | // - Writes are insanely fast since you are just appending to the file 26 | // - Reads are insanely fast since you do only one disk seek. In B-Tree backed 27 | // storage, there could be 2-3 disk seeks 28 | // 29 | // However, there are drawbacks too: 30 | // - We need to maintain an in-memory hash table KeyDir. A database with a large 31 | // number of keys would require more RAM 32 | // - Since we need to build the KeyDir at initialisation, it will affect the startup 33 | // time too 34 | // - Deleted keys need to be purged from the file to reduce the file size 35 | // 36 | // Read the paper for more details: https://riak.com/assets/bitcask-intro.pdf 37 | // 38 | // DiskStore provides two simple operations to get and set key value pairs. Both key 39 | // and value need to be of string type, and all the data is persisted to disk. 40 | // During startup, DiskStorage loads all the existing KV pair metadata, and it will 41 | // throw an error if the file is invalid or corrupt. 42 | // 43 | // Note that if the database file is large, the initialisation will take time 44 | // accordingly. The initialisation is also a blocking operation; till it is completed, 45 | // we cannot use the database. 46 | // 47 | // Typical usage example: 48 | // 49 | // store, _ := NewDiskStore("books.db") 50 | // store.Set("othello", "shakespeare") 51 | // author := store.Get("othello") 52 | type DiskStore struct { 53 | keyDir map[string]KeyEntry 54 | readFileHandle *os.File 55 | writeFileHandle *os.File 56 | currentOffset uint32 57 | } 58 | 59 | func isFileExists(fileName string) bool { 60 | // https://stackoverflow.com/a/12518877 61 | if _, err := os.Stat(fileName); err == nil || errors.Is(err, fs.ErrExist) { 62 | return true 63 | } 64 | return false 65 | } 66 | 67 | func getKeyDir(fileName string) (map[string]KeyEntry, error) { 68 | var f *os.File 69 | defer f.Close() 70 | keyDir := make(map[string]KeyEntry) 71 | var err error 72 | if isFileExists(fileName) { 73 | f, err = os.Open(fileName) 74 | if err != nil { 75 | return nil, err 76 | } 77 | } else { 78 | f, err = os.Create(fileName) 79 | if err != nil { 80 | return nil, err 81 | } 82 | } 83 | offset := 0 84 | for { 85 | headerBuffer := make([]byte, headerSize) 86 | n, err := f.Read(headerBuffer) 87 | if err == io.EOF || n == 0 { 88 | break 89 | } 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | timestamp, keySize, valueSize := decodeHeader(headerBuffer) 95 | kvBuffer := make([]byte, keySize+valueSize) 96 | n, err = f.Read(kvBuffer) 97 | if err != nil { 98 | return nil, err 99 | } 100 | if n == 0 { 101 | return nil, errors.New("EOF reading key") 102 | } 103 | data := append(headerBuffer, kvBuffer...) 104 | _, key, _ := decodeKV(data) 105 | totalSize := headerSize + keySize + valueSize 106 | keyDir[key] = NewKeyEntry(timestamp, uint32(offset), totalSize) 107 | offset += int(totalSize) 108 | } 109 | return keyDir, err 110 | } 111 | 112 | func NewDiskStore(fileName string) (*DiskStore, error) { 113 | var err error 114 | var writeFileHandle *os.File 115 | var readFileHandle *os.File 116 | keyDir, err := getKeyDir(fileName) 117 | if err != nil { 118 | return nil, err 119 | } 120 | 121 | writeFileHandle, err = os.OpenFile(fileName, os.O_APPEND|os.O_WRONLY, 0644) 122 | if err != nil { 123 | return nil, err 124 | } 125 | readFileHandle, err = os.Open(fileName) 126 | 127 | return &DiskStore{ 128 | keyDir: keyDir, 129 | writeFileHandle: writeFileHandle, 130 | readFileHandle: readFileHandle, 131 | currentOffset: 0, 132 | }, err 133 | } 134 | 135 | func (d *DiskStore) Get(key string) string { 136 | var value string 137 | if keyEntry, ok := d.keyDir[key]; ok { 138 | d.readFileHandle.Seek(int64(keyEntry.Offset), 0) 139 | kvBuffer := make([]byte, keyEntry.Size) 140 | d.readFileHandle.Read(kvBuffer) 141 | _, _, value = decodeKV(kvBuffer) 142 | } 143 | 144 | return value 145 | } 146 | 147 | func (d *DiskStore) Set(key string, value string) { 148 | timestamp := uint32(time.Now().Unix()) 149 | totalSize, encodedKV := encodeKV(timestamp, key, value) 150 | d.keyDir[key] = NewKeyEntry(timestamp, d.currentOffset, uint32(totalSize)) 151 | d.writeFileHandle.Write(encodedKV) 152 | d.currentOffset += uint32(totalSize) 153 | err := d.writeFileHandle.Sync() 154 | if err != nil { 155 | panic(fmt.Sprintf("Failed to sync to disk %s", err.Error())) 156 | } 157 | } 158 | 159 | func (d *DiskStore) Close() bool { 160 | d.readFileHandle.Close() 161 | d.writeFileHandle.Close() 162 | return true 163 | } 164 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![logo](https://github.com/avinassh/py-caskdb/raw/master/assets/logo.svg) 2 | # CaskDB - Disk based Log Structured Hash Table Store 3 | 4 | ![made-with-go](https://img.shields.io/badge/Made%20with-Go-1f425f.svg) 5 | [![build](https://github.com/avinassh/py-caskdb/actions/workflows/build.yml/badge.svg)](https://github.com/avinassh/go-caskdb/actions/workflows/build.yml) 6 | [![codecov](https://codecov.io/gh/avinassh/py-caskdb/branch/master/graph/badge.svg?token=9SA8Q4L7AZ)](https://codecov.io/gh/avinassh/py-caskdb) 7 | [![MIT license](https://camo.githubusercontent.com/f7358a0a5a91ec17974d36c9d426073a0ac67a958b22319be1ba5aa32542c28d/68747470733a2f2f62616467656e2e6e65742f6769746875622f6c6963656e73652f4e61657265656e2f5374726170646f776e2e6a73)](https://github.com/avinassh/go-caskdb/blob/master/LICENSE) 8 | [![twitter@iavins](https://img.shields.io/twitter/follow/iavins?style=social)](https://twitter.com/iavins) 9 | 10 | ![architecture](https://user-images.githubusercontent.com/640792/167299554-0fc44510-d500-4347-b680-258e224646fa.png) 11 | 12 | CaskDB is a disk-based, embedded, persistent, key-value store based on the [Riak's bitcask paper](https://riak.com/assets/bitcask-intro.pdf), written in Go. It is more focused on the educational capabilities than using it in production. The file format is platform, machine, and programming language independent. Say, the database file created from Go on macOS should be compatible with Rust on Windows. 13 | 14 | This project aims to help anyone, even a beginner in databases, build a persistent database in a few hours. There are no external dependencies; only the Go standard library is enough. 15 | 16 | If you are interested in writing the database yourself, head to the workshop section. 17 | 18 | ## Features 19 | - Low latency for reads and writes 20 | - High throughput 21 | - Easy to back up / restore 22 | - Simple and easy to understand 23 | - Store data much larger than the RAM 24 | 25 | ## Limitations 26 | Most of the following limitations are of CaskDB. However, there are some due to design constraints by the Bitcask paper. 27 | 28 | - Single file stores all data, and deleted keys still take up the space 29 | - CaskDB does not offer range scans 30 | - CaskDB requires keeping all the keys in the internal memory. With a lot of keys, RAM usage will be high 31 | - Slow startup time since it needs to load all the keys in memory 32 | 33 | ## Dependencies 34 | CaskDB does not require any external libraries to run. Go standard library is enough. 35 | 36 | ## Installation 37 | ```shell 38 | go get github.com/avinassh/go-caskdb 39 | ``` 40 | 41 | ## Usage 42 | 43 | ```go 44 | store, _ := NewDiskStore("books.db") 45 | store.Set("othello", "shakespeare") 46 | author := store.Get("othello") 47 | ``` 48 | 49 | ## Cask DB (Python) 50 | This project is a Go version of the [same project in Python](https://github.com/avinassh/py-caskdb). 51 | 52 | ## Prerequisites 53 | The workshop is for intermediate-advanced programmers. Knowing basics of Go helps, and you can build the database in any language you wish. 54 | 55 | Not sure where you stand? You are ready if you have done the following in any language: 56 | - If you have used a dictionary or hash table data structure 57 | - Converting an object (class, struct, or dict) to JSON and converting JSON back to the things 58 | - Open a file to write or read anything. A common task is dumping a dictionary contents to disk and reading back 59 | 60 | ## Workshop 61 | **NOTE:** I don't have any [workshops](workshop.md) scheduled shortly. [Follow me on Twitter](https://twitter.com/iavins/) for updates. [Drop me an email](http://scr.im/avii) if you wish to arrange a workshop for your team/company. 62 | 63 | CaskDB comes with a full test suite and a wide range of tools to help you write a database quickly. [A Github action](https://github.com/avinassh/go-caskdb/blob/master/.github/workflows/build.yml) is present with an automated tests runner. Fork the repo, push the code, and pass the tests! 64 | 65 | Throughout the workshop, you will implement the following: 66 | - Serialiser methods take a bunch of objects and serialise them into bytes. Also, the procedures take a bunch of bytes and deserialise them back to the things. 67 | - Come up with a data format with a header and data to store the bytes on the disk. The header would contain metadata like timestamp, key size, and value. 68 | - Store and retrieve data from the disk 69 | - Read an existing CaskDB file to load all keys 70 | 71 | ### Tasks 72 | 1. Read [the paper](https://riak.com/assets/bitcask-intro.pdf). Fork this repo and checkout the `start-here` branch 73 | 2. Implement the fixed-sized header, which can encode timestamp (uint, 4 bytes), key size (uint, 4 bytes), value size (uint, 4 bytes) together 74 | 3. Implement the key, value serialisers, and pass the tests from `format_test.go` 75 | 4. Figure out how to store the data on disk and the row pointer in the memory. Implement the get/set operations. Tests for the same are in `disk_store_test.go` 76 | 5. Code from the task #2 and #3 should be enough to read an existing CaskDB file and load the keys into memory 77 | 78 | Run `make test` to run the tests locally. Push the code to Github, and tests will run on different OS: ubuntu, mac, and windows. 79 | 80 | Not sure how to proceed? Then check the [hints](hints.md) file which contains more details on the tasks and hints. 81 | 82 | ### Hints 83 | - Not sure how to come up with a file format? Read the comment in the [format file](format.go) 84 | 85 | ## What next? 86 | I often get questions about what is next after the basic implementation. Here are some challenges (with different levels of difficulties) 87 | 88 | ### Level 1: 89 | - Crash safety: the bitcask paper stores CRC in the row, and while fetching the row back, it verifies the data 90 | - Key deletion: CaskDB does not have a delete API. Read the paper and implement it 91 | - Instead of using a hash table, use a data structure like the red-black tree to support range scans 92 | - CaskDB accepts only strings as keys and values. Make it generic and take other data structures like int or bytes. 93 | 94 | ### Level 2: 95 | - Hint file to improve the startup time. The paper has more details on it 96 | - Implement an internal cache which stores some of the key-value pairs. You may explore and experiment with different cache eviction strategies like LRU, LFU, FIFO etc. 97 | - Split the data into multiple files when the files hit a specific capacity 98 | 99 | ### Level 3: 100 | - Support for multiple processes 101 | - Garbage collector: keys which got updated and deleted remain in the file and take up space. Write a garbage collector to remove such stale data 102 | - Add SQL query engine layer 103 | - Store JSON in values and explore making CaskDB as a document database like Mongo 104 | - Make CaskDB distributed by exploring algorithms like raft, paxos, or consistent hashing 105 | 106 | ## Line Count 107 | 108 | ```shell 109 | $ tokei -f format.go disk_store.go 110 | =============================================================================== 111 | Language Files Lines Code Comments Blanks 112 | =============================================================================== 113 | Go 2 320 133 168 19 114 | ------------------------------------------------------------------------------- 115 | format.go 111 35 67 9 116 | disk_store.go 209 98 101 10 117 | =============================================================================== 118 | Total 2 320 133 168 19 119 | =============================================================================== 120 | ``` 121 | 122 | ## License 123 | The MIT license. Please check `LICENSE` for more details. 124 | --------------------------------------------------------------------------------