├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── db.go ├── db_test.go ├── go.mod ├── go.sum ├── header ├── header.go └── header_test.go ├── options.go ├── setup.go └── table ├── mmap.go ├── table.go └── table_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | 16 | *.db 17 | *.index 18 | *.idx 19 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.10" 5 | - master 6 | 7 | before_script: 8 | - make deps 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Tom Bevan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | test: 2 | go test -v ./... --cover 3 | 4 | deps: 5 | go get github.com/purehyperbole/rad 6 | go get github.com/google/uuid 7 | go get github.com/stretchr/testify 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LunarDB [![GoDoc](https://godoc.org/github.com/purehyperbole/lunar?status.svg)](https://godoc.org/github.com/purehyperbole/lunar) [![Go Report Card](https://goreportcard.com/badge/github.com/purehyperbole/lunar)](https://goreportcard.com/report/github.com/purehyperbole/lunar) [![Build Status](https://travis-ci.org/purehyperbole/lunar.svg?branch=master)](https://travis-ci.org/purehyperbole/lunar) 2 | 3 | A simple embedded, persistent key value store for go. 4 | 5 | The index makes use of a lock free radix tree, which is kept only in memory. 6 | 7 | Data persistence is handled via a memory mapped file (MMAP). 8 | 9 | # Motivation 10 | 11 | This project was built for fun and learning. It probably has lots of bugs and shouldn't be used for any real workloads (yet!) 12 | 13 | # Installation 14 | 15 | To start using lunar, you can run: 16 | 17 | `$ go get github.com/purehyperbole/lunar` 18 | 19 | # Usage 20 | 21 | `Open` will open a database file. This will create a data and accompanying index file if the specified file(s) don't exist. 22 | 23 | ```go 24 | package main 25 | 26 | import ( 27 | "github.com/purehyperbole/lunar" 28 | ) 29 | 30 | func main() { 31 | // open a new or existing database file. 32 | db, err := lunar.Open("test.db") 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | defer db.Close() 38 | } 39 | ``` 40 | 41 | `Get` allows data to be retrieved. 42 | 43 | ```go 44 | data, err := db.Get([]byte("myKey1234")) 45 | ``` 46 | 47 | `Set` allows data to be stored. 48 | 49 | ```go 50 | err := db.Set([]byte("myKey1234"), []byte(`{"status": "ok"}`)) 51 | ``` 52 | 53 | # Features/Wishlist 54 | 55 | - [x] Persistence 56 | - [x] Lock free index (Radix) 57 | - [ ] Data file compaction 58 | - [ ] Configurable sync on write options 59 | - [ ] Transactions (MVCC) 60 | 61 | ## Versioning 62 | 63 | For transparency into our release cycle and in striving to maintain backward 64 | compatibility, this project is maintained under [the Semantic Versioning guidelines](http://semver.org/). 65 | 66 | ## Copyright and License 67 | 68 | Code and documentation copyright since 2018 purehyperbole. 69 | 70 | Code released under 71 | [the MIT License](LICENSE). 72 | -------------------------------------------------------------------------------- /db.go: -------------------------------------------------------------------------------- 1 | package lunar 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/purehyperbole/lunar/header" 7 | "github.com/purehyperbole/lunar/table" 8 | "github.com/purehyperbole/rad" 9 | ) 10 | 11 | // DB Database 12 | type DB struct { 13 | index *rad.Radix 14 | data *table.Table 15 | compaction bool // compaction on file open 16 | } 17 | 18 | type entry struct { 19 | offset int64 20 | size int64 21 | } 22 | 23 | var ( 24 | ErrNotFound = errors.New("key not found") 25 | ) 26 | 27 | // Open open a database table and index, will create both if they dont exist 28 | func Open(path string, opts ...func(*DB) error) (*DB, error) { 29 | var db DB 30 | 31 | for _, opt := range opts { 32 | err := opt(&db) 33 | if err != nil { 34 | return nil, err 35 | } 36 | } 37 | 38 | return &db, db.setup(path) 39 | } 40 | 41 | // Close unmaps and closes data and index files 42 | func (db *DB) Close() error { 43 | return db.data.Close() 44 | } 45 | 46 | // Get get a value by key 47 | func (db *DB) Get(key []byte) ([]byte, error) { 48 | v := db.index.Lookup(key) 49 | 50 | entry, ok := v.(*entry) 51 | 52 | if v == nil || !ok { 53 | return nil, ErrNotFound 54 | } 55 | 56 | data, err := db.data.Read(entry.size, entry.offset) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | h := header.Deserialize(data[:header.HeaderSize]) 62 | 63 | return data[h.DataOffset():], nil 64 | } 65 | 66 | // Set set value by key 67 | func (db *DB) Set(key, value []byte) error { 68 | var h header.Header 69 | h.SetKeySize(int64(len(key))) 70 | h.SetDataSize(int64(len(value))) 71 | 72 | data := make([]byte, h.TotalSize()) 73 | copy(data[0:], header.Serialize(&h)) 74 | copy(data[header.HeaderSize:], key) 75 | copy(data[h.DataOffset():], value) 76 | 77 | off, err := db.data.Write(data) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | db.index.MustInsert(key, &entry{ 83 | size: h.TotalSize(), 84 | offset: off, 85 | }) 86 | 87 | return nil 88 | } 89 | 90 | // Gets get a value by string key 91 | func (db *DB) Gets(key string) ([]byte, error) { 92 | return db.Get([]byte(key)) 93 | } 94 | 95 | // Sets set a value by string key 96 | func (db *DB) Sets(key string, value []byte) error { 97 | return db.Set([]byte(key), value) 98 | } 99 | -------------------------------------------------------------------------------- /db_test.go: -------------------------------------------------------------------------------- 1 | package lunar 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "testing" 8 | "time" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | func cleanup(db *DB) { 15 | db.Close() 16 | os.Remove("test.db") 17 | os.Remove("test.db.idx") 18 | } 19 | 20 | func TestDBOpen(t *testing.T) { 21 | // open new 22 | db, err := Open("test.db") 23 | defer cleanup(db) 24 | 25 | require.Nil(t, err) 26 | assert.NotNil(t, db) 27 | 28 | dstat, derr := os.Stat("test.db") 29 | 30 | assert.Nil(t, derr) 31 | assert.Equal(t, int64(1<<16), dstat.Size()) 32 | assert.Nil(t, db.Close()) 33 | 34 | // open existing 35 | db, err = Open("test.db") 36 | require.Nil(t, err) 37 | assert.NotNil(t, db) 38 | } 39 | 40 | func TestDBSet(t *testing.T) { 41 | db, err := Open("test.db") 42 | defer cleanup(db) 43 | 44 | require.Nil(t, err) 45 | 46 | // insert new key & retrieve 47 | err = db.Sets("test-key", []byte("test")) 48 | 49 | require.Nil(t, err) 50 | 51 | data, err := db.Gets("test-key") 52 | 53 | require.Nil(t, err) 54 | assert.Equal(t, []byte("test"), data) 55 | 56 | // update existing key 57 | err = db.Sets("test-key", []byte("test-1234")) 58 | 59 | require.Nil(t, err) 60 | 61 | data, err = db.Gets("test-key") 62 | 63 | require.Nil(t, err) 64 | assert.Equal(t, []byte("test-1234"), data) 65 | } 66 | 67 | func TestDBGet(t *testing.T) { 68 | db, err := Open("test.db") 69 | defer cleanup(db) 70 | 71 | require.Nil(t, err) 72 | 73 | // get a nonexistant key 74 | data, err := db.Gets("test-key") 75 | require.NotNil(t, err) 76 | assert.Nil(t, data) 77 | 78 | // get an existing key 79 | err = db.Sets("test-key", []byte("test-1234")) 80 | require.Nil(t, err) 81 | 82 | data, err = db.Gets("test-key") 83 | require.Nil(t, err) 84 | assert.Equal(t, []byte("test-1234"), data) 85 | 86 | // test persistence 87 | err = db.Close() 88 | require.Nil(t, err) 89 | 90 | db, err = Open("test.db") 91 | defer cleanup(db) 92 | require.Nil(t, err) 93 | 94 | err = db.Sets("test-4567", []byte("test-4567")) 95 | require.Nil(t, err) 96 | 97 | data, err = db.Gets("test-4567") 98 | require.Nil(t, err) 99 | assert.Equal(t, []byte("test-4567"), data) 100 | 101 | data, err = db.Gets("test-key") 102 | require.Nil(t, err) 103 | assert.Equal(t, []byte("test-1234"), data) 104 | } 105 | 106 | func TestPersistence(t *testing.T) { 107 | db, err := Open("test.db") 108 | defer cleanup(db) 109 | defer os.Remove("test.db.backup") 110 | 111 | require.Nil(t, err) 112 | 113 | err = db.Sets("test-key", []byte("test")) 114 | require.Nil(t, err) 115 | 116 | err = db.Sets("test-key-2", []byte("test-1")) 117 | require.Nil(t, err) 118 | 119 | err = db.Sets("test-key-2", []byte("test-2")) 120 | require.Nil(t, err) 121 | 122 | pos := db.data.Position() 123 | 124 | db.Close() 125 | 126 | // test reopen 127 | 128 | db, err = Open("test.db") 129 | require.Nil(t, err) 130 | 131 | assert.Equal(t, pos, db.data.Position()) 132 | 133 | data, err := db.Gets("test-key") 134 | require.Nil(t, err) 135 | assert.Equal(t, []byte("test"), data) 136 | 137 | data, err = db.Gets("test-key-2") 138 | require.Nil(t, err) 139 | assert.Equal(t, []byte("test-2"), data) 140 | 141 | db.Close() 142 | 143 | // with compaction 144 | db, err = Open("test.db", Compact(true)) 145 | require.Nil(t, err) 146 | 147 | fmt.Println(db.data.Position()) 148 | 149 | data, err = db.Gets("test-key") 150 | require.Nil(t, err) 151 | assert.Equal(t, []byte("test"), data) 152 | 153 | data, err = db.Gets("test-key-2") 154 | require.Nil(t, err) 155 | assert.Equal(t, []byte("test-2"), data) 156 | } 157 | 158 | func BenchmarkDBSet(b *testing.B) { 159 | db, err := Open("test.db") 160 | defer cleanup(db) 161 | defer os.Remove("test.db.backup") 162 | require.Nil(b, err) 163 | 164 | key := make([]byte, 20) 165 | value := make([]byte, 100) 166 | 167 | start := time.Now() 168 | 169 | b.ResetTimer() 170 | 171 | for i := 0; i < b.N; i++ { 172 | rand.Read(key) 173 | 174 | err := db.Set(key, value) 175 | if err != nil { 176 | require.Nil(b, err) 177 | } 178 | } 179 | 180 | fmt.Println(time.Since(start)) 181 | } 182 | 183 | func BenchmarkDBGet(b *testing.B) { 184 | db, err := Open("test.db") 185 | defer cleanup(db) 186 | defer os.Remove("test.db.backup") 187 | require.Nil(b, err) 188 | 189 | key := make([]byte, 20) 190 | value := make([]byte, 100) 191 | 192 | wrnd := rand.New(rand.NewSource(1921)) 193 | rrnd := rand.New(rand.NewSource(1921)) 194 | 195 | for i := 0; i < b.N; i++ { 196 | wrnd.Read(key) 197 | 198 | err := db.Set(key, value) 199 | if err != nil { 200 | require.Nil(b, err) 201 | } 202 | } 203 | 204 | start := time.Now() 205 | 206 | b.ResetTimer() 207 | 208 | for i := 0; i < b.N; i++ { 209 | rrnd.Read(key) 210 | 211 | _, err := db.Get(key) 212 | if err != nil { 213 | require.Nil(b, err) 214 | } 215 | } 216 | 217 | fmt.Println(time.Since(start)) 218 | 219 | time.Sleep(time.Second * 20) 220 | } 221 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/purehyperbole/lunar 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/google/uuid v1.1.1 7 | github.com/purehyperbole/rad v1.0.0 8 | github.com/stretchr/testify v1.4.0 9 | ) 10 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= 4 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 5 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 6 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 7 | github.com/purehyperbole/rad v1.0.0 h1:/u1E2ix4n+A8j3mjtNGYmjQc5NoNkkBRoA5+QoUDyD8= 8 | github.com/purehyperbole/rad v1.0.0/go.mod h1:qFtZpbD0Dld+lPM6MbrB/JcPvl1pa0uZDRlMFVzDY6Y= 9 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 10 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 11 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 12 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 13 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 14 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 15 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 16 | -------------------------------------------------------------------------------- /header/header.go: -------------------------------------------------------------------------------- 1 | package header 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "unsafe" 7 | ) 8 | 9 | const ( 10 | // HeaderSize the allocated size of the header 11 | HeaderSize = 48 12 | ) 13 | 14 | // Header data header stores 15 | // info about a given data value 16 | type Header struct { 17 | xmin uint64 // transaction id that created the node's data 18 | xmax uint64 // transaction id that updated/deleted the node's data 19 | psize int64 // size of the previous version of this data, including header 20 | poffset int64 // offset of the previous version of this data 21 | size int64 // size of current data 22 | ksize int64 // size of the current key 23 | } 24 | 25 | // Xmin returns the transaction if of the node that created the data 26 | func (h *Header) Xmin() uint64 { 27 | return h.xmin 28 | } 29 | 30 | // Xmax returns the transaction if of the node that updated or deleted the data 31 | func (h *Header) Xmax() uint64 { 32 | return h.xmax 33 | } 34 | 35 | // DataSize returns the size of the values data 36 | func (h *Header) DataSize() int64 { 37 | return h.size 38 | } 39 | 40 | // KeySize returns the size of the tuples key 41 | func (h *Header) KeySize() int64 { 42 | return h.ksize 43 | } 44 | 45 | // TotalSize returns the total size of header + data 46 | func (h *Header) TotalSize() int64 { 47 | return HeaderSize + h.ksize + h.size 48 | } 49 | 50 | // DataOffset returs the offset that the data starts at 51 | func (h *Header) DataOffset() int64 { 52 | return HeaderSize + h.ksize 53 | } 54 | 55 | // Previous returns the size and offset of the previous version's data 56 | func (h *Header) Previous() (int64, int64) { 57 | return h.psize, h.poffset 58 | } 59 | 60 | // HasPrevious returns true if there is a previous version of the data 61 | func (h *Header) HasPrevious() bool { 62 | return h.psize != 0 63 | } 64 | 65 | // SetXmin sets the transaction if of the node that created the data 66 | func (h *Header) SetXmin(txid uint64) { 67 | h.xmin = txid 68 | } 69 | 70 | // SetXmax sets the transaction if of the node that updated or deleted the data 71 | func (h *Header) SetXmax(txid uint64) { 72 | h.xmax = txid 73 | } 74 | 75 | // SetPrevious sets the offset of the previous version's data 76 | func (h *Header) SetPrevious(size, offset int64) { 77 | h.psize = size 78 | h.poffset = offset 79 | } 80 | 81 | // SetDataSize sets the size of the keys data 82 | func (h *Header) SetDataSize(size int64) { 83 | h.size = size 84 | } 85 | 86 | // SetKeySize sets the size of the tuples key 87 | func (h *Header) SetKeySize(size int64) { 88 | h.ksize = size 89 | } 90 | 91 | // Serialize serialize a node to a byteslice 92 | func Serialize(h *Header) []byte { 93 | data := make([]byte, 48) 94 | 95 | xmin := *(*[8]byte)(unsafe.Pointer(&h.xmin)) 96 | copy(data[0:], xmin[:]) 97 | 98 | xmax := *(*[8]byte)(unsafe.Pointer(&h.xmax)) 99 | copy(data[8:], xmax[:]) 100 | 101 | psize := *(*[8]byte)(unsafe.Pointer(&h.psize)) 102 | copy(data[16:], psize[:]) 103 | 104 | poffset := *(*[8]byte)(unsafe.Pointer(&h.poffset)) 105 | copy(data[24:], poffset[:]) 106 | 107 | size := *(*[8]byte)(unsafe.Pointer(&h.size)) 108 | copy(data[32:], size[:]) 109 | 110 | ksize := *(*[8]byte)(unsafe.Pointer(&h.ksize)) 111 | copy(data[40:], ksize[:]) 112 | 113 | return data 114 | } 115 | 116 | // Deserialize deserialize from a byteslice to a Node 117 | func Deserialize(data []byte) *Header { 118 | return &Header{ 119 | xmin: *(*uint64)(unsafe.Pointer(&data[0])), 120 | xmax: *(*uint64)(unsafe.Pointer(&data[8])), 121 | psize: *(*int64)(unsafe.Pointer(&data[16])), 122 | poffset: *(*int64)(unsafe.Pointer(&data[24])), 123 | size: *(*int64)(unsafe.Pointer(&data[32])), 124 | ksize: *(*int64)(unsafe.Pointer(&data[40])), 125 | } 126 | } 127 | 128 | // Prepend prepends header information to data 129 | func Prepend(h *Header, data []byte) []byte { 130 | hdr := Serialize(h) 131 | // may be more performant to write the header seperately 132 | // as this append creates a copy of the data 133 | hdr = append(hdr, data...) 134 | return hdr 135 | } 136 | 137 | // Print prints header information to stdout 138 | func Print(h *Header) { 139 | output := []string{"{"} 140 | 141 | output = append(output, fmt.Sprintf(" Xmin: %d", h.xmin)) 142 | output = append(output, fmt.Sprintf(" Xmax: %d", h.xmax)) 143 | output = append(output, fmt.Sprintf(" Previous Version Size: %d", h.psize)) 144 | output = append(output, fmt.Sprintf(" Previous Version Offset: %d", h.poffset)) 145 | 146 | output = append(output, "}") 147 | 148 | fmt.Println(strings.Join(output, "\n")) 149 | } 150 | -------------------------------------------------------------------------------- /header/header_test.go: -------------------------------------------------------------------------------- 1 | package header 2 | 3 | import ( 4 | "testing" 5 | "unsafe" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func testBuildBytes() []byte { 11 | data := make([]byte, 48) 12 | 13 | var scratch []byte 14 | xmin := uint64(2) 15 | xmax := uint64(15) 16 | psize := int64(8192) 17 | poffset := int64(4096) 18 | size := int64(2048) 19 | ksize := int64(8) 20 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&xmin))[:]...) 21 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&xmax))[:]...) 22 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&psize))[:]...) 23 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&poffset))[:]...) 24 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&size))[:]...) 25 | scratch = append(scratch, (*[8]byte)(unsafe.Pointer(&ksize))[:]...) 26 | 27 | copy(data[0:], scratch[:]) 28 | 29 | return data 30 | } 31 | 32 | func TestSerialize(t *testing.T) { 33 | hdr := Header{ 34 | xmin: 0, 35 | xmax: 5, 36 | psize: 4096, 37 | poffset: 4096, 38 | size: 2048, 39 | ksize: 8, 40 | } 41 | 42 | data := Serialize(&hdr) 43 | 44 | assert.Len(t, data, 48) 45 | assert.Equal(t, uint64(0), *(*uint64)(unsafe.Pointer(&data[0]))) 46 | assert.Equal(t, uint64(5), *(*uint64)(unsafe.Pointer(&data[8]))) 47 | assert.Equal(t, int64(4096), *(*int64)(unsafe.Pointer(&data[16]))) 48 | assert.Equal(t, int64(4096), *(*int64)(unsafe.Pointer(&data[24]))) 49 | assert.Equal(t, int64(2048), *(*int64)(unsafe.Pointer(&data[32]))) 50 | assert.Equal(t, int64(8), *(*int64)(unsafe.Pointer(&data[40]))) 51 | } 52 | 53 | func TestDeserialize(t *testing.T) { 54 | data := testBuildBytes() 55 | 56 | hdr := Deserialize(data) 57 | 58 | sz, off := hdr.Previous() 59 | 60 | assert.Equal(t, uint64(2), hdr.Xmin()) 61 | assert.Equal(t, uint64(15), hdr.Xmax()) 62 | assert.Equal(t, int64(8192), sz) 63 | assert.Equal(t, int64(4096), off) 64 | } 65 | -------------------------------------------------------------------------------- /options.go: -------------------------------------------------------------------------------- 1 | package lunar 2 | 3 | // Compact option used when opening the database 4 | // Compaction only happens once when the data table is loaded 5 | // The existing database will be copied to a backup file and a new database table created 6 | func Compact(c bool) func(db *DB) error { 7 | return func(db *DB) error { 8 | db.compaction = c 9 | return nil 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /setup.go: -------------------------------------------------------------------------------- 1 | package lunar 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | 7 | "github.com/purehyperbole/lunar/header" 8 | "github.com/purehyperbole/lunar/table" 9 | "github.com/purehyperbole/rad" 10 | ) 11 | 12 | func (db *DB) setup(datapath string) error { 13 | var err error 14 | var rt *table.Table 15 | 16 | db.index = rad.New() 17 | 18 | if db.compaction && exists(datapath) { 19 | backup := datapath + ".backup" 20 | 21 | if exists(backup) { 22 | return errors.New("could not backup data file") 23 | } 24 | 25 | err = os.Rename(datapath, backup) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | rt, err = table.New(backup) 31 | } else { 32 | rt, err = table.New(datapath) 33 | } 34 | 35 | if err != nil { 36 | return err 37 | } 38 | 39 | db.data, err = table.New(datapath) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | if exists(datapath) { 45 | return db.reload(rt, db.data) 46 | } 47 | 48 | return nil 49 | } 50 | 51 | func (db *DB) reload(rt, wt *table.Table) error { 52 | var pos int64 53 | 54 | dsz := rt.Size() 55 | 56 | for { 57 | if pos+header.HeaderSize > dsz { 58 | return nil 59 | } 60 | 61 | // read header 62 | data, err := rt.Read(header.HeaderSize, pos) 63 | if err != nil { 64 | return err 65 | } 66 | 67 | h := header.Deserialize(data) 68 | 69 | if h.KeySize() < 1 { 70 | if !db.compaction { 71 | wt.SetPosition(pos) 72 | } 73 | break 74 | } 75 | 76 | // skip old records 77 | if db.compaction && h.Xmax() > 0 { 78 | continue 79 | } 80 | 81 | // get key from data 82 | key := make([]byte, h.KeySize()) 83 | 84 | kd, err := rt.Read(h.KeySize(), pos+header.HeaderSize) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | copy(key, kd) 90 | 91 | np := pos 92 | 93 | if db.compaction { 94 | data, err := rt.Read(h.TotalSize(), pos) 95 | if err != nil { 96 | return err 97 | } 98 | 99 | np, err = wt.Write(data) 100 | if err != nil { 101 | return err 102 | } 103 | } 104 | 105 | db.index.Insert(key, &entry{ 106 | size: h.TotalSize(), 107 | offset: np, 108 | }) 109 | 110 | pos = pos + h.TotalSize() 111 | } 112 | 113 | db.data = wt 114 | 115 | return nil 116 | } 117 | 118 | func exists(path string) bool { 119 | _, err := os.Stat(path) 120 | if err != nil { 121 | return false 122 | } 123 | 124 | return true 125 | } 126 | -------------------------------------------------------------------------------- /table/mmap.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "reflect" 8 | "sync/atomic" 9 | "syscall" 10 | "time" 11 | "unsafe" 12 | ) 13 | 14 | var ( 15 | ErrMappingClosed = errors.New("mapping closed") 16 | ) 17 | 18 | type mmap struct { 19 | fd *os.File // file descriptor 20 | size int64 // file Size 21 | active int32 // active read or write operations 22 | closed int32 // mapping is closed. probably should just use a rwmutex with write lock protecting the resize, but that would be no fun 23 | mapping []byte // mmap mapping 24 | } 25 | 26 | func newmmap(fd *os.File) (*mmap, error) { 27 | stat, err := fd.Stat() 28 | if err != nil { 29 | return nil, err 30 | } 31 | 32 | m := mmap{ 33 | fd: fd, 34 | size: stat.Size(), 35 | mapping: make([]byte, 0), 36 | } 37 | 38 | return &m, m.mmap() 39 | } 40 | 41 | func (m *mmap) mmap() error { 42 | mapping, err := syscall.Mmap( 43 | int(m.fd.Fd()), 44 | 0, 45 | int(m.size), 46 | syscall.PROT_WRITE|syscall.PROT_READ, syscall.MAP_SHARED, 47 | ) 48 | 49 | if err != nil { 50 | return err 51 | } 52 | 53 | m.mapping = mapping 54 | 55 | return nil 56 | } 57 | 58 | func (m *mmap) munmap() error { 59 | fmt.Println("munmap", m.size) 60 | return syscall.Munmap(m.mapping) 61 | } 62 | 63 | func (m *mmap) mremap(newSize int64) error { 64 | sh := (*reflect.SliceHeader)(unsafe.Pointer(&m.mapping)) 65 | 66 | r1, _, err := syscall.Syscall6(syscall.SYS_MREMAP, sh.Data, uintptr(sh.Len), uintptr(newSize), uintptr(1), 0, 0) 67 | if err != 0 { 68 | return syscall.Errno(err) 69 | } 70 | 71 | nsh := &reflect.SliceHeader{ 72 | Data: r1, 73 | Len: int(newSize), 74 | Cap: int(newSize), 75 | } 76 | 77 | m.mapping = *(*[]byte)(unsafe.Pointer(nsh)) 78 | 79 | return nil 80 | } 81 | 82 | func (m *mmap) read(size, offset int64) ([]byte, error) { 83 | if atomic.AddInt32(&m.active, 1) < 1 { 84 | return nil, ErrMappingClosed 85 | } 86 | 87 | defer atomic.AddInt32(&m.active, -1) 88 | 89 | if m.size < (offset + size) { 90 | return nil, ErrBoundsViolation 91 | } 92 | 93 | return m.mapping[offset:(offset + size)], nil 94 | } 95 | 96 | func (m *mmap) write(data []byte, offset int64) error { 97 | atomic.AddInt32(&m.active, 1) 98 | defer atomic.AddInt32(&m.active, -1) 99 | 100 | if atomic.LoadInt32(&m.closed) == 1 { 101 | fmt.Println("mapping closed") 102 | return ErrMappingClosed 103 | } 104 | 105 | if len(data) > MaxStep { 106 | return ErrDataSizeTooLarge 107 | } 108 | 109 | if m.size < (offset + int64(len(data))) { 110 | return ErrBoundsViolation 111 | } 112 | 113 | copy(m.mapping[offset:], data) 114 | 115 | return nil 116 | } 117 | 118 | func (m *mmap) close() error { 119 | atomic.StoreInt32(&m.closed, 1) 120 | for atomic.LoadInt32(&m.active) > 0 { 121 | time.Sleep(time.Millisecond) 122 | } 123 | return m.munmap() 124 | } 125 | -------------------------------------------------------------------------------- /table/table.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "sync" 7 | "sync/atomic" 8 | "unsafe" 9 | ) 10 | 11 | const ( 12 | // PageSize page size 13 | PageSize = 1 << 12 // 4kb 14 | // MinStep the smallest increment that the table can grow 15 | MinStep = 1 << 16 // 64 kb 16 | // MaxStep the largest increment that the table can grow 17 | MaxStep = 1 << 30 // 1 GB 18 | // MaxTableSize maximum size of table 19 | MaxTableSize = 0x7FFFFFFFFFFFFFFF 20 | ) 21 | 22 | var ( 23 | // ErrBoundsViolation the specified segment of memory does not exist 24 | ErrBoundsViolation = errors.New("specified offset and size exceeds size of mapping") 25 | // ErrDataSizeTooLarge the provided value data exceeds the maximum size limit 26 | ErrDataSizeTooLarge = errors.New("data exceeds maximum limit") 27 | ) 28 | 29 | // Table mmaped file 30 | type Table struct { 31 | fd *os.File 32 | position int64 33 | mapping unsafe.Pointer 34 | mu sync.Mutex 35 | } 36 | 37 | // New loads a new table 38 | func New(path string) (*Table, error) { 39 | fd, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0766) 40 | if err != nil { 41 | return nil, err 42 | } 43 | 44 | stat, err := fd.Stat() 45 | if err != nil { 46 | return nil, err 47 | } 48 | 49 | if stat.Size() < 1 { 50 | err = fd.Truncate(MinStep) 51 | if err != nil { 52 | return nil, err 53 | } 54 | } 55 | 56 | mapping, err := newmmap(fd) 57 | if err != nil { 58 | return nil, err 59 | } 60 | 61 | t := Table{ 62 | fd: fd, 63 | mapping: unsafe.Pointer(mapping), 64 | } 65 | 66 | return &t, nil 67 | } 68 | 69 | // Read reads from table at a given offset 70 | func (t *Table) Read(size, offset int64) ([]byte, error) { 71 | mapping := (*mmap)(atomic.LoadPointer(&t.mapping)) 72 | return mapping.read(size, offset) 73 | } 74 | 75 | // Write writes to table at a given offset 76 | func (t *Table) Write(data []byte) (int64, error) { 77 | ds := int64(len(data)) 78 | 79 | offset := atomic.AddInt64(&t.position, ds) - ds 80 | 81 | if t.Size() < offset+ds { 82 | err := t.resize(ds, offset) 83 | if err != nil { 84 | return 0, err 85 | } 86 | } 87 | 88 | err := (*mmap)(atomic.LoadPointer(&t.mapping)).write(data, offset) 89 | for err == ErrMappingClosed { 90 | err = (*mmap)(atomic.LoadPointer(&t.mapping)).write(data, offset) 91 | } 92 | 93 | return offset, err 94 | } 95 | 96 | // WriteAt write to a given offset 97 | func (t *Table) WriteAt(data []byte, offset int64) error { 98 | ds := int64(len(data)) 99 | 100 | if t.Size() < offset+ds { 101 | err := t.resize(ds, offset) 102 | if err != nil { 103 | return err 104 | } 105 | } 106 | 107 | err := (*mmap)(atomic.LoadPointer(&t.mapping)).write(data, offset) 108 | for err == ErrMappingClosed { 109 | err = (*mmap)(atomic.LoadPointer(&t.mapping)).write(data, offset) 110 | } 111 | 112 | return err 113 | } 114 | 115 | // Position returns the tables current position 116 | func (t *Table) Position() int64 { 117 | return atomic.LoadInt64(&t.position) 118 | } 119 | 120 | // SetPosition updates the position of a table 121 | func (t *Table) SetPosition(pos int64) { 122 | atomic.StoreInt64(&t.position, pos) 123 | } 124 | 125 | // Size the size of the table 126 | func (t *Table) Size() int64 { 127 | return (*mmap)(atomic.LoadPointer(&t.mapping)).size 128 | } 129 | 130 | // Close close table file descriptor and unmap 131 | func (t *Table) Close() error { 132 | mapping := (*mmap)(atomic.LoadPointer(&t.mapping)) 133 | 134 | err := mapping.close() 135 | if err != nil { 136 | return err 137 | } 138 | 139 | err = t.sync() 140 | if err != nil { 141 | return err 142 | } 143 | 144 | return t.fd.Close() 145 | } 146 | 147 | func (t *Table) sync() error { 148 | return t.fd.Sync() 149 | } 150 | 151 | func (t *Table) resize(size, offset int64) error { 152 | t.mu.Lock() 153 | defer t.mu.Unlock() 154 | 155 | if t.Size() > size+offset { 156 | return nil 157 | } 158 | 159 | newSize := t.growadvise(size) 160 | 161 | err := t.fd.Truncate(newSize) 162 | if err != nil { 163 | return err 164 | } 165 | 166 | oldMapping := (*mmap)(atomic.LoadPointer(&t.mapping)) 167 | 168 | newMapping, err := newmmap(t.fd) 169 | if err != nil { 170 | return err 171 | } 172 | 173 | atomic.StorePointer(&t.mapping, unsafe.Pointer(newMapping)) 174 | 175 | if oldMapping != nil { 176 | go oldMapping.close() 177 | } 178 | 179 | return nil 180 | } 181 | 182 | func (t *Table) growadvise(size int64) int64 { 183 | tsz := t.Size() 184 | 185 | if size < tsz { 186 | size = tsz * 2 187 | } 188 | 189 | if size < MinStep { 190 | return tsz + MinStep 191 | } 192 | 193 | if size > MaxStep { 194 | return tsz + MaxStep 195 | } 196 | 197 | return size + size%PageSize 198 | } 199 | -------------------------------------------------------------------------------- /table/table_test.go: -------------------------------------------------------------------------------- 1 | package table 2 | 3 | import ( 4 | "os" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/google/uuid" 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestNew(t *testing.T) { 14 | testpath := "test.db" 15 | 16 | db, err := New(testpath) 17 | require.Nil(t, err) 18 | require.NotNil(t, db) 19 | 20 | err = db.Close() 21 | require.Nil(t, err) 22 | } 23 | 24 | func TestWrite(t *testing.T) { 25 | data := []byte("test1234") 26 | comparison := make([]byte, len(data)) 27 | 28 | db, err := New("test.db") 29 | require.Nil(t, err) 30 | 31 | offset, err := db.Write(data) 32 | require.Nil(t, err) 33 | assert.Equal(t, int64(0), offset) 34 | 35 | mapping := (*mmap)(db.mapping) 36 | 37 | // check mapping 38 | assert.Equal(t, data, mapping.mapping[:len(data)]) 39 | 40 | // check file 41 | db.fd.ReadAt(comparison, 0) 42 | assert.Equal(t, data, comparison) 43 | 44 | // clean file 45 | os.Remove(db.fd.Name()) 46 | } 47 | 48 | func TestRead(t *testing.T) { 49 | data := []byte("test4567") 50 | 51 | db, err := New("test.db") 52 | require.Nil(t, err) 53 | 54 | _, err = db.Write(data) 55 | require.Nil(t, err) 56 | 57 | comparison, err := db.Read(int64(len(data)), 0) 58 | require.Nil(t, err) 59 | assert.Equal(t, data, comparison) 60 | 61 | // clean file 62 | os.Remove(db.fd.Name()) 63 | } 64 | 65 | func TestConcurrentWrite(t *testing.T) { 66 | var wg sync.WaitGroup 67 | 68 | db, err := New("test.db") 69 | require.Nil(t, err) 70 | 71 | defer os.Remove(db.fd.Name()) 72 | 73 | wg.Add(16) 74 | 75 | for i := 0; i < 16; i++ { 76 | go func() { 77 | v := []byte(uuid.New().String()) 78 | for x := 0; x < 1000000; x++ { 79 | _, err := db.Write(v) 80 | 81 | if err != nil { 82 | panic(err) 83 | } 84 | } 85 | wg.Done() 86 | }() 87 | } 88 | 89 | wg.Wait() 90 | } 91 | --------------------------------------------------------------------------------