├── .drone.yml ├── .gitignore ├── .goreleaser.yml ├── CODE_OF_CONDUCT.md ├── LICENSE ├── Makefile ├── README.md ├── _config.yml ├── bitcask.go ├── bitcask_test.go ├── cmd └── bitcask │ ├── del.go │ ├── get.go │ ├── keys.go │ ├── main.go │ ├── merge.go │ ├── root.go │ └── set.go ├── datafile.go ├── entry.go ├── go.mod ├── go.sum ├── keydir.go ├── proto ├── doc.go ├── entry.pb.go └── entry.proto ├── streampb └── stream.go ├── tools └── release.sh ├── version.go └── version_test.go /.drone.yml: -------------------------------------------------------------------------------- 1 | kind: pipeline 2 | name: default 3 | 4 | steps: 5 | - name: build 6 | image: golang:latest 7 | commands: 8 | - go test -v -short -cover -coverprofile=coverage.txt ./... 9 | 10 | - name: coverage 11 | image: plugins/codecov 12 | settings: 13 | token: 14 | from_secret: codecov-token 15 | 16 | - name: notify 17 | image: plugins/webhook 18 | urls: https://msgbus.mills.io/ci.mills.io 19 | when: 20 | status: 21 | - success 22 | - failure 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~* 2 | *.bak 3 | 4 | /coverage.txt 5 | /bitcask 6 | /tmp 7 | /dist 8 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - 3 | main: ./cmd/bitcask 4 | flags: -tags "static_build" 5 | ldflags: -w -X .Version={{.Version}} -X .Commit={{.Commit}} 6 | env: 7 | - CGO_ENABLED=0 8 | sign: 9 | artifacts: checksum 10 | archive: 11 | replacements: 12 | darwin: Darwin 13 | linux: Linux 14 | windows: Windows 15 | 386: i386 16 | amd64: x86_64 17 | checksum: 18 | name_template: 'checksums.txt' 19 | snapshot: 20 | name_template: "{{ .Tag }}-next" 21 | changelog: 22 | sort: asc 23 | filters: 24 | exclude: 25 | - '^docs:' 26 | - '^test:' 27 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | In the interest of fostering an open and welcoming environment, we as 6 | contributors and maintainers pledge to making participation in our project and 7 | our community a harassment-free experience for everyone, regardless of age, body 8 | size, disability, ethnicity, sex characteristics, gender identity and expression, 9 | level of experience, education, socio-economic status, nationality, personal 10 | appearance, race, religion, or sexual identity and orientation. 11 | 12 | ## Our Standards 13 | 14 | Examples of behavior that contributes to creating a positive environment 15 | include: 16 | 17 | * Using welcoming and inclusive language 18 | * Being respectful of differing viewpoints and experiences 19 | * Gracefully accepting constructive criticism 20 | * Focusing on what is best for the community 21 | * Showing empathy towards other community members 22 | 23 | Examples of unacceptable behavior by participants include: 24 | 25 | * The use of sexualized language or imagery and unwelcome sexual attention or 26 | advances 27 | * Trolling, insulting/derogatory comments, and personal or political attacks 28 | * Public or private harassment 29 | * Publishing others' private information, such as a physical or electronic 30 | address, without explicit permission 31 | * Other conduct which could reasonably be considered inappropriate in a 32 | professional setting 33 | 34 | ## Our Responsibilities 35 | 36 | Project maintainers are responsible for clarifying the standards of acceptable 37 | behavior and are expected to take appropriate and fair corrective action in 38 | response to any instances of unacceptable behavior. 39 | 40 | Project maintainers have the right and responsibility to remove, edit, or 41 | reject comments, commits, code, wiki edits, issues, and other contributions 42 | that are not aligned to this Code of Conduct, or to ban temporarily or 43 | permanently any contributor for other behaviors that they deem inappropriate, 44 | threatening, offensive, or harmful. 45 | 46 | ## Scope 47 | 48 | This Code of Conduct applies both within project spaces and in public spaces 49 | when an individual is representing the project or its community. Examples of 50 | representing a project or community include using an official project e-mail 51 | address, posting via an official social media account, or acting as an appointed 52 | representative at an online or offline event. Representation of a project may be 53 | further defined and clarified by project maintainers. 54 | 55 | ## Enforcement 56 | 57 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 58 | reported by contacting the project team at . All 59 | complaints will be reviewed and investigated and will result in a response that 60 | is deemed necessary and appropriate to the circumstances. The project team is 61 | obligated to maintain confidentiality with regard to the reporter of an incident. 62 | Further details of specific enforcement policies may be posted separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution 69 | 70 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, 71 | available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html 72 | 73 | [homepage]: https://www.contributor-covenant.org 74 | 75 | For answers to common questions about this code of conduct, see 76 | https://www.contributor-covenant.org/faq 77 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 James Mills 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: dev build generate install image release profile bench test clean 2 | 3 | CGO_ENABLED=0 4 | COMMIT=$(shell git rev-parse --short HEAD) 5 | 6 | all: dev 7 | 8 | dev: build 9 | @./bitcask --version 10 | 11 | build: clean generate 12 | @go build \ 13 | -tags "netgo static_build" -installsuffix netgo \ 14 | -ldflags "-w -X $(shell go list)/.Commit=$(COMMIT)" \ 15 | ./cmd/bitcask/... 16 | 17 | generate: 18 | @go generate $(shell go list)/... 19 | 20 | install: build 21 | @go install ./cmd/bitcask/... 22 | 23 | image: 24 | @docker build -t prologic/bitcask . 25 | 26 | release: 27 | @./tools/release.sh 28 | 29 | profile: build 30 | @go test -cpuprofile cpu.prof -memprofile mem.prof -v -bench ./... 31 | 32 | bench: build 33 | @go test -v -benchmem -bench=. ./... 34 | 35 | test: build 36 | @go test -v -cover -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... -race ./... 37 | 38 | clean: 39 | @git clean -f -d -X 40 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # bitcask 2 | 3 | [![Build Status](https://cloud.drone.io/api/badges/prologic/bitcask/status.svg)](https://cloud.drone.io/prologic/bitcask) 4 | [![CodeCov](https://codecov.io/gh/prologic/bitcask/branch/master/graph/badge.svg)](https://codecov.io/gh/prologic/bitcask) 5 | [![Go Report Card](https://goreportcard.com/badge/prologic/bitcask)](https://goreportcard.com/report/prologic/bitcask) 6 | [![GoDoc](https://godoc.org/github.com/prologic/bitcask?status.svg)](https://godoc.org/github.com/prologic/bitcask) 7 | [![Sourcegraph](https://sourcegraph.com/github.com/prologic/bitcask/-/badge.svg)](https://sourcegraph.com/github.com/prologic/bitcask?badge) 8 | 9 | A Bitcask (LSM+WAL) Key/Value Store written in Go. 10 | 11 | ## Features 12 | 13 | * Embeddable 14 | * Builtin CLI 15 | 16 | ## Install 17 | 18 | ```#!bash 19 | $ go get github.com/prologic/bitcask 20 | ``` 21 | 22 | ## Usage (library) 23 | 24 | Install the package into your project: 25 | 26 | ```#!bash 27 | $ go get github.com/prologic/bitcask 28 | ``` 29 | 30 | ```#!go 31 | package main 32 | 33 | import ( 34 | "log" 35 | 36 | "github.com/prologic/bitcask" 37 | ) 38 | 39 | func main() { 40 | db, _ := bitcask.Open("/tmp/db") 41 | db.Set("Hello", []byte("World")) 42 | db.Close() 43 | } 44 | ``` 45 | 46 | See the [godoc](https://godoc.org/github.com/prologic/bitcask) for further 47 | documentation and other examples. 48 | 49 | ## Usage (tool) 50 | 51 | ```#!bash 52 | $ bitcask -p /tmp/db set Hello World 53 | $ bitcask -p /tmp/db get Hello 54 | World 55 | ``` 56 | 57 | ## Performance 58 | 59 | Benchmarks run on a 11" Macbook with a 1.4Ghz Intel Core i7: 60 | 61 | ``` 62 | $ make bench 63 | ... 64 | BenchmarkGet-4 300000 5065 ns/op 144 B/op 4 allocs/op 65 | BenchmarkPut-4 100000 14640 ns/op 699 B/op 7 allocs/op 66 | ``` 67 | 68 | * ~30,000 reads/sec for non-active data 69 | * ~180,000 reads/sec for active data 70 | * ~60,000 writes/sec 71 | 72 | ## License 73 | 74 | bitcask is licensed under the [MIT License](https://github.com/prologic/bitcask/blob/master/LICENSE) 75 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-hacker -------------------------------------------------------------------------------- /bitcask.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "io" 7 | "io/ioutil" 8 | "os" 9 | "path/filepath" 10 | "sort" 11 | "strconv" 12 | "strings" 13 | "time" 14 | ) 15 | 16 | const ( 17 | DefaultMaxDatafileSize = 1 << 20 // 1MB 18 | ) 19 | 20 | var ( 21 | ErrKeyNotFound = errors.New("error: key not found") 22 | ) 23 | 24 | type Bitcask struct { 25 | path string 26 | curr *Datafile 27 | keydir *Keydir 28 | 29 | maxDatafileSize int64 30 | } 31 | 32 | func (b *Bitcask) Close() error { 33 | return b.curr.Close() 34 | } 35 | 36 | func (b *Bitcask) Sync() error { 37 | return b.curr.Sync() 38 | } 39 | 40 | func (b *Bitcask) Get(key string) ([]byte, error) { 41 | item, ok := b.keydir.Get(key) 42 | if !ok { 43 | return nil, ErrKeyNotFound 44 | } 45 | 46 | var ( 47 | df *Datafile 48 | err error 49 | ) 50 | 51 | // Optimization 52 | if item.FileID == b.curr.id { 53 | df = b.curr 54 | } else { 55 | // TODO: Pre-open non-active Datafiles and cache the file pointers? 56 | df, err = NewDatafile(b.path, item.FileID, true) 57 | if err != nil { 58 | return nil, err 59 | } 60 | defer df.Close() 61 | } 62 | 63 | e, err := df.ReadAt(item.Index) 64 | if err != nil { 65 | return nil, err 66 | } 67 | 68 | return e.Value, nil 69 | } 70 | 71 | func (b *Bitcask) Put(key string, value []byte) error { 72 | index, err := b.put(key, value) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | b.keydir.Add(key, b.curr.id, index, time.Now().Unix()) 78 | 79 | return nil 80 | } 81 | 82 | func (b *Bitcask) Delete(key string) error { 83 | _, err := b.put(key, []byte{}) 84 | if err != nil { 85 | return err 86 | } 87 | 88 | b.keydir.Delete(key) 89 | 90 | return nil 91 | } 92 | 93 | func (b *Bitcask) Fold(f func(key string) error) error { 94 | for key := range b.keydir.Keys() { 95 | if err := f(key); err != nil { 96 | return err 97 | } 98 | } 99 | return nil 100 | } 101 | 102 | func (b *Bitcask) put(key string, value []byte) (int64, error) { 103 | size, err := b.curr.Size() 104 | if err != nil { 105 | return -1, err 106 | } 107 | 108 | if size >= b.maxDatafileSize { 109 | err := b.curr.Close() 110 | if err != nil { 111 | return -1, err 112 | } 113 | 114 | id := b.curr.id + 1 115 | curr, err := NewDatafile(b.path, id, false) 116 | if err != nil { 117 | return -1, err 118 | } 119 | b.curr = curr 120 | } 121 | 122 | e := NewEntry(key, value) 123 | return b.curr.Write(e) 124 | } 125 | 126 | func (b *Bitcask) setMaxDatafileSize(size int64) error { 127 | b.maxDatafileSize = size 128 | return nil 129 | } 130 | 131 | func MaxDatafileSize(size int64) func(*Bitcask) error { 132 | return func(b *Bitcask) error { 133 | return b.setMaxDatafileSize(size) 134 | } 135 | } 136 | 137 | func getDatafiles(path string) ([]string, error) { 138 | fns, err := filepath.Glob(fmt.Sprintf("%s/*.data", path)) 139 | if err != nil { 140 | return nil, err 141 | } 142 | sort.Strings(fns) 143 | return fns, nil 144 | } 145 | 146 | func parseIds(fns []string) ([]int, error) { 147 | var ids []int 148 | for _, fn := range fns { 149 | fn = filepath.Base(fn) 150 | ext := filepath.Ext(fn) 151 | if ext != ".data" { 152 | continue 153 | } 154 | id, err := strconv.ParseInt(strings.TrimSuffix(fn, ext), 10, 32) 155 | if err != nil { 156 | return nil, err 157 | } 158 | ids = append(ids, int(id)) 159 | } 160 | sort.Ints(ids) 161 | return ids, nil 162 | } 163 | 164 | func Merge(path string, force bool) error { 165 | fns, err := getDatafiles(path) 166 | if err != nil { 167 | return err 168 | } 169 | 170 | ids, err := parseIds(fns) 171 | if err != nil { 172 | return err 173 | } 174 | 175 | // Do not merge if we only have 1 Datafile 176 | if len(ids) <= 1 { 177 | return nil 178 | } 179 | 180 | // Don't merge the Active Datafile (the last one) 181 | fns = fns[:len(fns)-1] 182 | ids = ids[:len(ids)-1] 183 | 184 | temp, err := ioutil.TempDir("", "bitcask") 185 | if err != nil { 186 | return err 187 | } 188 | 189 | for i, fn := range fns { 190 | // Don't merge Datafiles whose .hint files we've already generated 191 | // (they are already merged); unless we set the force flag to true 192 | // (forcing a re-merge). 193 | if filepath.Ext(fn) == ".hint" && !force { 194 | // Already merged 195 | continue 196 | } 197 | 198 | id := ids[i] 199 | 200 | keydir := NewKeydir() 201 | 202 | df, err := NewDatafile(path, id, true) 203 | if err != nil { 204 | return err 205 | } 206 | defer df.Close() 207 | 208 | for { 209 | e, err := df.Read() 210 | if err != nil { 211 | if err == io.EOF { 212 | break 213 | } 214 | return err 215 | } 216 | 217 | // Tombstone value (deleted key) 218 | if len(e.Value) == 0 { 219 | keydir.Delete(e.Key) 220 | continue 221 | } 222 | 223 | keydir.Add(e.Key, ids[i], e.Index, e.Timestamp) 224 | } 225 | 226 | tempdf, err := NewDatafile(temp, id, false) 227 | if err != nil { 228 | return err 229 | } 230 | defer tempdf.Close() 231 | 232 | for key := range keydir.Keys() { 233 | item, _ := keydir.Get(key) 234 | e, err := df.ReadAt(item.Index) 235 | if err != nil { 236 | return err 237 | } 238 | 239 | _, err = tempdf.Write(e) 240 | if err != nil { 241 | return err 242 | } 243 | } 244 | 245 | err = tempdf.Close() 246 | if err != nil { 247 | return err 248 | } 249 | 250 | err = df.Close() 251 | if err != nil { 252 | return err 253 | } 254 | 255 | err = os.Rename(tempdf.Name(), df.Name()) 256 | if err != nil { 257 | return err 258 | } 259 | 260 | hint := strings.TrimSuffix(df.Name(), ".data") + ".hint" 261 | err = keydir.Save(hint) 262 | if err != nil { 263 | return err 264 | } 265 | } 266 | 267 | return nil 268 | } 269 | 270 | func Open(path string, options ...func(*Bitcask) error) (*Bitcask, error) { 271 | if err := os.MkdirAll(path, 0755); err != nil { 272 | return nil, err 273 | } 274 | 275 | err := Merge(path, false) 276 | if err != nil { 277 | return nil, err 278 | } 279 | 280 | keydir := NewKeydir() 281 | 282 | fns, err := getDatafiles(path) 283 | if err != nil { 284 | return nil, err 285 | } 286 | 287 | ids, err := parseIds(fns) 288 | if err != nil { 289 | return nil, err 290 | } 291 | 292 | for i, fn := range fns { 293 | if filepath.Ext(fn) == ".hint" { 294 | f, err := os.Open(filepath.Join(path, fn)) 295 | if err != nil { 296 | return nil, err 297 | } 298 | defer f.Close() 299 | 300 | hint, err := NewKeydirFromBytes(f) 301 | if err != nil { 302 | return nil, err 303 | } 304 | 305 | for key := range hint.Keys() { 306 | item, _ := hint.Get(key) 307 | keydir.Add(key, item.FileID, item.Index, item.Timestamp) 308 | } 309 | } else { 310 | df, err := NewDatafile(path, ids[i], true) 311 | if err != nil { 312 | return nil, err 313 | } 314 | 315 | for { 316 | e, err := df.Read() 317 | if err != nil { 318 | if err == io.EOF { 319 | break 320 | } 321 | return nil, err 322 | } 323 | 324 | // Tombstone value (deleted key) 325 | if len(e.Value) == 0 { 326 | keydir.Delete(e.Key) 327 | continue 328 | } 329 | 330 | keydir.Add(e.Key, ids[i], e.Index, e.Timestamp) 331 | } 332 | } 333 | } 334 | 335 | var id int 336 | if len(ids) > 0 { 337 | id = ids[(len(ids) - 1)] 338 | } 339 | curr, err := NewDatafile(path, id, false) 340 | if err != nil { 341 | return nil, err 342 | } 343 | 344 | bitcask := &Bitcask{ 345 | path: path, 346 | curr: curr, 347 | keydir: keydir, 348 | 349 | maxDatafileSize: DefaultMaxDatafileSize, 350 | } 351 | 352 | for _, option := range options { 353 | err = option(bitcask) 354 | if err != nil { 355 | return nil, err 356 | } 357 | } 358 | 359 | return bitcask, nil 360 | } 361 | -------------------------------------------------------------------------------- /bitcask_test.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestAll(t *testing.T) { 13 | var ( 14 | db *Bitcask 15 | testdir string 16 | err error 17 | ) 18 | 19 | assert := assert.New(t) 20 | 21 | testdir, err = ioutil.TempDir("", "bitcask") 22 | assert.NoError(err) 23 | 24 | t.Run("Open", func(t *testing.T) { 25 | db, err = Open(testdir) 26 | assert.NoError(err) 27 | }) 28 | 29 | t.Run("Put", func(t *testing.T) { 30 | err = db.Put("foo", []byte("bar")) 31 | assert.NoError(err) 32 | }) 33 | 34 | t.Run("Get", func(t *testing.T) { 35 | val, err := db.Get("foo") 36 | assert.NoError(err) 37 | assert.Equal([]byte("bar"), val) 38 | }) 39 | 40 | t.Run("Delete", func(t *testing.T) { 41 | err := db.Delete("foo") 42 | assert.NoError(err) 43 | _, err = db.Get("foo") 44 | assert.Error(err) 45 | assert.Equal(err.Error(), "error: key not found") 46 | }) 47 | 48 | t.Run("Sync", func(t *testing.T) { 49 | err = db.Sync() 50 | assert.NoError(err) 51 | }) 52 | 53 | t.Run("Close", func(t *testing.T) { 54 | err = db.Close() 55 | assert.NoError(err) 56 | }) 57 | } 58 | 59 | func TestDeletedKeys(t *testing.T) { 60 | assert := assert.New(t) 61 | 62 | testdir, err := ioutil.TempDir("", "bitcask") 63 | assert.NoError(err) 64 | 65 | t.Run("Setup", func(t *testing.T) { 66 | var ( 67 | db *Bitcask 68 | err error 69 | ) 70 | 71 | t.Run("Open", func(t *testing.T) { 72 | db, err = Open(testdir) 73 | assert.NoError(err) 74 | }) 75 | 76 | t.Run("Put", func(t *testing.T) { 77 | err = db.Put("foo", []byte("bar")) 78 | assert.NoError(err) 79 | }) 80 | 81 | t.Run("Get", func(t *testing.T) { 82 | val, err := db.Get("foo") 83 | assert.NoError(err) 84 | assert.Equal([]byte("bar"), val) 85 | }) 86 | 87 | t.Run("Delete", func(t *testing.T) { 88 | err := db.Delete("foo") 89 | assert.NoError(err) 90 | _, err = db.Get("foo") 91 | assert.Error(err) 92 | assert.Equal("error: key not found", err.Error()) 93 | }) 94 | 95 | t.Run("Sync", func(t *testing.T) { 96 | err = db.Sync() 97 | assert.NoError(err) 98 | }) 99 | 100 | t.Run("Close", func(t *testing.T) { 101 | err = db.Close() 102 | assert.NoError(err) 103 | }) 104 | }) 105 | 106 | t.Run("Reopen", func(t *testing.T) { 107 | var ( 108 | db *Bitcask 109 | err error 110 | ) 111 | 112 | t.Run("Open", func(t *testing.T) { 113 | db, err = Open(testdir) 114 | assert.NoError(err) 115 | }) 116 | 117 | t.Run("Get", func(t *testing.T) { 118 | _, err = db.Get("foo") 119 | assert.Error(err) 120 | assert.Equal("error: key not found", err.Error()) 121 | }) 122 | 123 | t.Run("Close", func(t *testing.T) { 124 | err = db.Close() 125 | assert.NoError(err) 126 | }) 127 | }) 128 | } 129 | 130 | func TestMerge(t *testing.T) { 131 | assert := assert.New(t) 132 | 133 | testdir, err := ioutil.TempDir("", "bitcask") 134 | assert.NoError(err) 135 | 136 | t.Run("Setup", func(t *testing.T) { 137 | var ( 138 | db *Bitcask 139 | err error 140 | ) 141 | 142 | t.Run("Open", func(t *testing.T) { 143 | db, err = Open(testdir, MaxDatafileSize(1024)) 144 | assert.NoError(err) 145 | }) 146 | 147 | t.Run("Put", func(t *testing.T) { 148 | for i := 0; i < 1024; i++ { 149 | err = db.Put(string(i), []byte(strings.Repeat(" ", 1024))) 150 | assert.NoError(err) 151 | } 152 | }) 153 | 154 | t.Run("Get", func(t *testing.T) { 155 | for i := 0; i < 32; i++ { 156 | err = db.Put(string(i), []byte(strings.Repeat(" ", 1024))) 157 | assert.NoError(err) 158 | val, err := db.Get(string(i)) 159 | assert.NoError(err) 160 | assert.Equal([]byte(strings.Repeat(" ", 1024)), val) 161 | } 162 | }) 163 | 164 | t.Run("Sync", func(t *testing.T) { 165 | err = db.Sync() 166 | assert.NoError(err) 167 | }) 168 | 169 | t.Run("Close", func(t *testing.T) { 170 | err = db.Close() 171 | assert.NoError(err) 172 | }) 173 | }) 174 | 175 | t.Run("Merge", func(t *testing.T) { 176 | var ( 177 | db *Bitcask 178 | err error 179 | ) 180 | 181 | t.Run("Open", func(t *testing.T) { 182 | db, err = Open(testdir) 183 | assert.NoError(err) 184 | }) 185 | 186 | t.Run("Get", func(t *testing.T) { 187 | for i := 0; i < 32; i++ { 188 | val, err := db.Get(string(i)) 189 | assert.NoError(err) 190 | assert.Equal([]byte(strings.Repeat(" ", 1024)), val) 191 | } 192 | }) 193 | 194 | t.Run("Close", func(t *testing.T) { 195 | err = db.Close() 196 | assert.NoError(err) 197 | }) 198 | }) 199 | } 200 | 201 | func BenchmarkGet(b *testing.B) { 202 | testdir, err := ioutil.TempDir("", "bitcask") 203 | if err != nil { 204 | b.Fatal(err) 205 | } 206 | 207 | db, err := Open(testdir) 208 | if err != nil { 209 | b.Fatal(err) 210 | } 211 | defer db.Close() 212 | 213 | err = db.Put("foo", []byte("bar")) 214 | if err != nil { 215 | b.Fatal(err) 216 | } 217 | 218 | b.ResetTimer() 219 | for i := 0; i < b.N; i++ { 220 | val, err := db.Get("foo") 221 | if err != nil { 222 | b.Fatal(err) 223 | } 224 | if string(val) != "bar" { 225 | b.Errorf("expected val=bar got=%s", val) 226 | } 227 | } 228 | } 229 | 230 | func BenchmarkPut(b *testing.B) { 231 | testdir, err := ioutil.TempDir("", "bitcask") 232 | if err != nil { 233 | b.Fatal(err) 234 | } 235 | 236 | db, err := Open(testdir) 237 | if err != nil { 238 | b.Fatal(err) 239 | } 240 | defer db.Close() 241 | 242 | b.ResetTimer() 243 | for i := 0; i < b.N; i++ { 244 | err := db.Put(fmt.Sprintf("key%d", i), []byte("bar")) 245 | if err != nil { 246 | b.Fatal(err) 247 | } 248 | } 249 | } 250 | -------------------------------------------------------------------------------- /cmd/bitcask/del.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | 10 | "github.com/prologic/bitcask" 11 | ) 12 | 13 | var delCmd = &cobra.Command{ 14 | Use: "del ", 15 | Aliases: []string{"delete", "remove", "rm"}, 16 | Short: "Delete a key and its value", 17 | Long: `This deletes a key and its value`, 18 | Args: cobra.ExactArgs(1), 19 | Run: func(cmd *cobra.Command, args []string) { 20 | path := viper.GetString("path") 21 | 22 | key := args[0] 23 | 24 | os.Exit(del(path, key)) 25 | }, 26 | } 27 | 28 | func init() { 29 | RootCmd.AddCommand(delCmd) 30 | } 31 | 32 | func del(path, key string) int { 33 | db, err := bitcask.Open(path) 34 | if err != nil { 35 | log.WithError(err).Error("error opening database") 36 | return 1 37 | } 38 | 39 | err = db.Delete(key) 40 | if err != nil { 41 | log.WithError(err).Error("error deleting key") 42 | return 1 43 | } 44 | 45 | return 0 46 | } 47 | -------------------------------------------------------------------------------- /cmd/bitcask/get.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | 11 | "github.com/prologic/bitcask" 12 | ) 13 | 14 | var getCmd = &cobra.Command{ 15 | Use: "get ", 16 | Aliases: []string{"view"}, 17 | Short: "Get a new Key and display its Value", 18 | Long: `This retrieves a key and display its value`, 19 | Args: cobra.ExactArgs(1), 20 | Run: func(cmd *cobra.Command, args []string) { 21 | path := viper.GetString("path") 22 | 23 | key := args[0] 24 | 25 | os.Exit(get(path, key)) 26 | }, 27 | } 28 | 29 | func init() { 30 | RootCmd.AddCommand(getCmd) 31 | } 32 | 33 | func get(path, key string) int { 34 | db, err := bitcask.Open(path) 35 | if err != nil { 36 | log.WithError(err).Error("error opening database") 37 | return 1 38 | } 39 | 40 | value, err := db.Get(key) 41 | if err != nil { 42 | log.WithError(err).Error("error reading key") 43 | return 1 44 | } 45 | 46 | fmt.Printf("%s\n", string(value)) 47 | log.WithField("key", key).WithField("value", value).Debug("key/value") 48 | 49 | return 0 50 | } 51 | -------------------------------------------------------------------------------- /cmd/bitcask/keys.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | 11 | "github.com/prologic/bitcask" 12 | ) 13 | 14 | var keysCmd = &cobra.Command{ 15 | Use: "keys", 16 | Aliases: []string{"list", "ls"}, 17 | Short: "Display all keys in Database", 18 | Long: `This displays all known keys in the Database"`, 19 | Args: cobra.ExactArgs(0), 20 | Run: func(cmd *cobra.Command, args []string) { 21 | path := viper.GetString("path") 22 | 23 | os.Exit(keys(path)) 24 | }, 25 | } 26 | 27 | func init() { 28 | RootCmd.AddCommand(keysCmd) 29 | } 30 | 31 | func keys(path string) int { 32 | db, err := bitcask.Open(path) 33 | if err != nil { 34 | log.WithError(err).Error("error opening database") 35 | return 1 36 | } 37 | 38 | err = db.Fold(func(key string) error { 39 | fmt.Printf("%s\n", key) 40 | return nil 41 | }) 42 | if err != nil { 43 | log.WithError(err).Error("error listing keys") 44 | return 1 45 | } 46 | 47 | return 0 48 | } 49 | -------------------------------------------------------------------------------- /cmd/bitcask/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | func main() { 4 | Execute() 5 | } 6 | -------------------------------------------------------------------------------- /cmd/bitcask/merge.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | log "github.com/sirupsen/logrus" 7 | "github.com/spf13/cobra" 8 | "github.com/spf13/viper" 9 | 10 | "github.com/prologic/bitcask" 11 | ) 12 | 13 | var mergeCmd = &cobra.Command{ 14 | Use: "merge", 15 | Aliases: []string{"clean", "compact", "defrag"}, 16 | Short: "Merges the Datafiles in the Database", 17 | Long: `This merges all non-active Datafiles in the Database and 18 | compacts the data stored on disk. Old values are removed as well as deleted 19 | keys.`, 20 | Args: cobra.ExactArgs(0), 21 | Run: func(cmd *cobra.Command, args []string) { 22 | path := viper.GetString("path") 23 | force, err := cmd.Flags().GetBool("force") 24 | if err != nil { 25 | log.WithError(err).Error("error parsing force flag") 26 | os.Exit(1) 27 | } 28 | 29 | os.Exit(merge(path, force)) 30 | }, 31 | } 32 | 33 | func init() { 34 | RootCmd.AddCommand(mergeCmd) 35 | 36 | mergeCmd.Flags().BoolP( 37 | "force", "f", false, 38 | "Force a re-merge even if .hint files exist", 39 | ) 40 | } 41 | 42 | func merge(path string, force bool) int { 43 | err := bitcask.Merge(path, force) 44 | if err != nil { 45 | log.WithError(err).Error("error merging database") 46 | return 1 47 | } 48 | 49 | return 0 50 | } 51 | -------------------------------------------------------------------------------- /cmd/bitcask/root.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | log "github.com/sirupsen/logrus" 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | 11 | "github.com/prologic/bitcask" 12 | ) 13 | 14 | // RootCmd represents the base command when called without any subcommands 15 | var RootCmd = &cobra.Command{ 16 | Use: "bitcask", 17 | Version: bitcask.FullVersion(), 18 | Short: "Command-line tools for bitcask", 19 | Long: `This is the command-line tool to interact with a bitcask database. 20 | 21 | This lets you get, set and delete key/value pairs as well as perform merge 22 | (or compaction) operations. This tool serves as an example implementation 23 | however is also intended to be useful in shell scripts.`, 24 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 25 | // set logging level 26 | if viper.GetBool("debug") { 27 | log.SetLevel(log.DebugLevel) 28 | } else { 29 | log.SetLevel(log.InfoLevel) 30 | } 31 | }, 32 | } 33 | 34 | // Execute adds all child commands to the root command 35 | // and sets flags appropriately. 36 | // This is called by main.main(). It only needs to happen once to the rootCmd. 37 | func Execute() { 38 | if err := RootCmd.Execute(); err != nil { 39 | fmt.Println(err) 40 | os.Exit(1) 41 | } 42 | } 43 | 44 | func init() { 45 | RootCmd.PersistentFlags().BoolP( 46 | "debug", "d", false, 47 | "Enable debug logging", 48 | ) 49 | 50 | RootCmd.PersistentFlags().StringP( 51 | "path", "p", "/tmp/bitcask", 52 | "Path to Bitcask database", 53 | ) 54 | 55 | viper.BindPFlag("path", RootCmd.PersistentFlags().Lookup("path")) 56 | viper.SetDefault("path", "/tmp/bitcask") 57 | 58 | viper.BindPFlag("debug", RootCmd.PersistentFlags().Lookup("debug")) 59 | viper.SetDefault("debug", false) 60 | } 61 | -------------------------------------------------------------------------------- /cmd/bitcask/set.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "io" 6 | "io/ioutil" 7 | "os" 8 | 9 | log "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "github.com/spf13/viper" 12 | 13 | "github.com/prologic/bitcask" 14 | ) 15 | 16 | var setCmd = &cobra.Command{ 17 | Use: "set []", 18 | Aliases: []string{"add"}, 19 | Short: "Add/Set a new Key/Value pair", 20 | Long: `This adds or sets a new key/value pair. 21 | 22 | If the value is not specified as an argument it is read from standard input.`, 23 | Args: cobra.MinimumNArgs(1), 24 | Run: func(cmd *cobra.Command, args []string) { 25 | path := viper.GetString("path") 26 | 27 | key := args[0] 28 | 29 | var value io.Reader 30 | if len(args) > 1 { 31 | value = bytes.NewBufferString(args[1]) 32 | } else { 33 | value = os.Stdin 34 | } 35 | 36 | os.Exit(set(path, key, value)) 37 | }, 38 | } 39 | 40 | func init() { 41 | RootCmd.AddCommand(setCmd) 42 | } 43 | 44 | func set(path, key string, value io.Reader) int { 45 | db, err := bitcask.Open(path) 46 | if err != nil { 47 | log.WithError(err).Error("error opening database") 48 | return 1 49 | } 50 | 51 | data, err := ioutil.ReadAll(value) 52 | if err != nil { 53 | log.WithError(err).Error("error writing key") 54 | return 1 55 | } 56 | 57 | err = db.Put(key, data) 58 | if err != nil { 59 | log.WithError(err).Error("error writing key") 60 | return 1 61 | } 62 | 63 | return 0 64 | } 65 | -------------------------------------------------------------------------------- /datafile.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "os" 7 | "path/filepath" 8 | "time" 9 | 10 | pb "github.com/prologic/bitcask/proto" 11 | "github.com/prologic/bitcask/streampb" 12 | ) 13 | 14 | const ( 15 | DefaultDatafileFilename = "%09d.data" 16 | ) 17 | 18 | var ( 19 | ErrReadonly = errors.New("error: read only datafile") 20 | ) 21 | 22 | type Datafile struct { 23 | id int 24 | r *os.File 25 | w *os.File 26 | dec *streampb.Decoder 27 | enc *streampb.Encoder 28 | } 29 | 30 | func NewDatafile(path string, id int, readonly bool) (*Datafile, error) { 31 | var ( 32 | r *os.File 33 | w *os.File 34 | err error 35 | ) 36 | 37 | fn := filepath.Join(path, fmt.Sprintf(DefaultDatafileFilename, id)) 38 | 39 | if !readonly { 40 | w, err = os.OpenFile(fn, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0640) 41 | if err != nil { 42 | return nil, err 43 | } 44 | } 45 | 46 | r, err = os.Open(fn) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | dec := streampb.NewDecoder(r) 52 | enc := streampb.NewEncoder(w) 53 | 54 | return &Datafile{ 55 | id: id, 56 | r: r, 57 | w: w, 58 | dec: dec, 59 | enc: enc, 60 | }, nil 61 | } 62 | 63 | func (df *Datafile) Name() string { 64 | return df.r.Name() 65 | } 66 | 67 | func (df *Datafile) Close() error { 68 | if df.w == nil { 69 | return df.r.Close() 70 | } 71 | 72 | err := df.Sync() 73 | if err != nil { 74 | return err 75 | } 76 | return df.w.Close() 77 | } 78 | 79 | func (df *Datafile) Sync() error { 80 | if df.w == nil { 81 | return nil 82 | } 83 | return df.w.Sync() 84 | } 85 | 86 | func (df *Datafile) Size() (int64, error) { 87 | var ( 88 | stat os.FileInfo 89 | err error 90 | ) 91 | 92 | if df.w == nil { 93 | stat, err = df.r.Stat() 94 | } else { 95 | stat, err = df.w.Stat() 96 | } 97 | 98 | if err != nil { 99 | return -1, err 100 | } 101 | 102 | return stat.Size(), nil 103 | } 104 | 105 | func (df *Datafile) Read() (pb.Entry, error) { 106 | var e pb.Entry 107 | return e, df.dec.Decode(&e) 108 | } 109 | 110 | func (df *Datafile) ReadAt(index int64) (e pb.Entry, err error) { 111 | _, err = df.r.Seek(index, os.SEEK_SET) 112 | if err != nil { 113 | return 114 | } 115 | return df.Read() 116 | } 117 | 118 | func (df *Datafile) Write(e pb.Entry) (int64, error) { 119 | if df.w == nil { 120 | return -1, ErrReadonly 121 | } 122 | 123 | stat, err := df.w.Stat() 124 | if err != nil { 125 | return -1, err 126 | } 127 | 128 | index := stat.Size() 129 | 130 | e.Index = index 131 | e.Timestamp = time.Now().Unix() 132 | 133 | err = df.enc.Encode(&e) 134 | if err != nil { 135 | return -1, err 136 | } 137 | 138 | return index, nil 139 | } 140 | -------------------------------------------------------------------------------- /entry.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "hash/crc32" 5 | 6 | pb "github.com/prologic/bitcask/proto" 7 | ) 8 | 9 | func NewEntry(key string, value []byte) pb.Entry { 10 | crc := crc32.ChecksumIEEE(value) 11 | 12 | return pb.Entry{ 13 | CRC: crc, 14 | Key: key, 15 | Value: value, 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/prologic/bitcask 2 | 3 | require ( 4 | github.com/gogo/protobuf v1.2.1 5 | github.com/golang/protobuf v1.2.0 6 | github.com/gorilla/websocket v1.4.0 // indirect 7 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 // indirect 8 | github.com/mitchellh/go-homedir v1.1.0 9 | github.com/pkg/errors v0.8.1 10 | github.com/prologic/msgbus v0.1.1 11 | github.com/prometheus/client_golang v0.9.2 // indirect 12 | github.com/sirupsen/logrus v1.3.0 13 | github.com/spf13/cobra v0.0.3 14 | github.com/spf13/viper v1.3.1 15 | github.com/stretchr/testify v1.3.0 16 | gopkg.in/vmihailenco/msgpack.v2 v2.9.1 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 2 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= 3 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 4 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 5 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 6 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 7 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 10 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 11 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 12 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 13 | github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= 14 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 15 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 16 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 17 | github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= 18 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 19 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 20 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 21 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= 22 | github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= 23 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 24 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 25 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 26 | github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= 27 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 28 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 29 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 30 | github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= 31 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 32 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 33 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 34 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 35 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 36 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 37 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 38 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 39 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 40 | github.com/prologic/msgbus v0.1.1/go.mod h1:B3Qu4/U2FP08x93jUzp9E8bl155+cIgDH2DUGRK6OZk= 41 | github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= 42 | github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= 43 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= 44 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 45 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= 46 | github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 47 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= 48 | github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 49 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= 50 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 51 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 52 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 53 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 54 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 55 | github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= 56 | github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= 57 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 58 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 59 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 60 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 61 | github.com/spf13/viper v1.3.1 h1:5+8j8FTpnFV4nEImW/ofkzEt8VoOiLXxdYIDsB73T38= 62 | github.com/spf13/viper v1.3.1/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 63 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 64 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 65 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 66 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 67 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 68 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 69 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 70 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 71 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= 72 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 73 | golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 74 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 75 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 76 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A= 77 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 78 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 79 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 80 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 81 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 82 | gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM= 83 | gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= 84 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 85 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 86 | -------------------------------------------------------------------------------- /keydir.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "bytes" 5 | "encoding/gob" 6 | "io" 7 | "io/ioutil" 8 | "sync" 9 | ) 10 | 11 | type Item struct { 12 | FileID int 13 | Index int64 14 | Timestamp int64 15 | } 16 | 17 | type Keydir struct { 18 | sync.RWMutex 19 | kv map[string]Item 20 | } 21 | 22 | func NewKeydir() *Keydir { 23 | return &Keydir{ 24 | kv: make(map[string]Item), 25 | } 26 | } 27 | 28 | func (k *Keydir) Add(key string, fileid int, index, timestamp int64) { 29 | k.Lock() 30 | defer k.Unlock() 31 | 32 | k.kv[key] = Item{ 33 | FileID: fileid, 34 | Index: index, 35 | Timestamp: timestamp, 36 | } 37 | } 38 | 39 | func (k *Keydir) Get(key string) (Item, bool) { 40 | k.RLock() 41 | defer k.RUnlock() 42 | 43 | item, ok := k.kv[key] 44 | return item, ok 45 | } 46 | 47 | func (k *Keydir) Delete(key string) { 48 | k.Lock() 49 | defer k.Unlock() 50 | 51 | delete(k.kv, key) 52 | } 53 | 54 | func (k *Keydir) Keys() chan string { 55 | ch := make(chan string) 56 | go func() { 57 | for k := range k.kv { 58 | ch <- k 59 | } 60 | close(ch) 61 | }() 62 | return ch 63 | } 64 | 65 | func (k *Keydir) Bytes() ([]byte, error) { 66 | var buf bytes.Buffer 67 | enc := gob.NewEncoder(&buf) 68 | err := enc.Encode(k.kv) 69 | if err != nil { 70 | return nil, err 71 | } 72 | return buf.Bytes(), nil 73 | } 74 | 75 | func (k *Keydir) Save(fn string) error { 76 | data, err := k.Bytes() 77 | if err != nil { 78 | return err 79 | } 80 | 81 | return ioutil.WriteFile(fn, data, 0644) 82 | } 83 | 84 | func NewKeydirFromBytes(r io.Reader) (*Keydir, error) { 85 | k := NewKeydir() 86 | dec := gob.NewDecoder(r) 87 | err := dec.Decode(&k.kv) 88 | if err != nil { 89 | return nil, err 90 | } 91 | return k, nil 92 | } 93 | -------------------------------------------------------------------------------- /proto/doc.go: -------------------------------------------------------------------------------- 1 | package proto 2 | 3 | //go:generate protoc --go_out=. entry.proto 4 | -------------------------------------------------------------------------------- /proto/entry.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: entry.proto 3 | 4 | package proto 5 | 6 | import proto "github.com/golang/protobuf/proto" 7 | import fmt "fmt" 8 | import math "math" 9 | 10 | // Reference imports to suppress errors if they are not otherwise used. 11 | var _ = proto.Marshal 12 | var _ = fmt.Errorf 13 | var _ = math.Inf 14 | 15 | // This is a compile-time assertion to ensure that this generated file 16 | // is compatible with the proto package it is being compiled against. 17 | // A compilation error at this line likely means your copy of the 18 | // proto package needs to be updated. 19 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 20 | 21 | type Entry struct { 22 | CRC uint32 `protobuf:"varint,1,opt,name=CRC,proto3" json:"CRC,omitempty"` 23 | Key string `protobuf:"bytes,2,opt,name=Key,proto3" json:"Key,omitempty"` 24 | Index int64 `protobuf:"varint,3,opt,name=Index,proto3" json:"Index,omitempty"` 25 | Value []byte `protobuf:"bytes,4,opt,name=Value,proto3" json:"Value,omitempty"` 26 | Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` 27 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 28 | XXX_unrecognized []byte `json:"-"` 29 | XXX_sizecache int32 `json:"-"` 30 | } 31 | 32 | func (m *Entry) Reset() { *m = Entry{} } 33 | func (m *Entry) String() string { return proto.CompactTextString(m) } 34 | func (*Entry) ProtoMessage() {} 35 | func (*Entry) Descriptor() ([]byte, []int) { 36 | return fileDescriptor_entry_4f5906245d08394f, []int{0} 37 | } 38 | func (m *Entry) XXX_Unmarshal(b []byte) error { 39 | return xxx_messageInfo_Entry.Unmarshal(m, b) 40 | } 41 | func (m *Entry) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 42 | return xxx_messageInfo_Entry.Marshal(b, m, deterministic) 43 | } 44 | func (dst *Entry) XXX_Merge(src proto.Message) { 45 | xxx_messageInfo_Entry.Merge(dst, src) 46 | } 47 | func (m *Entry) XXX_Size() int { 48 | return xxx_messageInfo_Entry.Size(m) 49 | } 50 | func (m *Entry) XXX_DiscardUnknown() { 51 | xxx_messageInfo_Entry.DiscardUnknown(m) 52 | } 53 | 54 | var xxx_messageInfo_Entry proto.InternalMessageInfo 55 | 56 | func (m *Entry) GetCRC() uint32 { 57 | if m != nil { 58 | return m.CRC 59 | } 60 | return 0 61 | } 62 | 63 | func (m *Entry) GetKey() string { 64 | if m != nil { 65 | return m.Key 66 | } 67 | return "" 68 | } 69 | 70 | func (m *Entry) GetIndex() int64 { 71 | if m != nil { 72 | return m.Index 73 | } 74 | return 0 75 | } 76 | 77 | func (m *Entry) GetValue() []byte { 78 | if m != nil { 79 | return m.Value 80 | } 81 | return nil 82 | } 83 | 84 | func (m *Entry) GetTimestamp() int64 { 85 | if m != nil { 86 | return m.Timestamp 87 | } 88 | return 0 89 | } 90 | 91 | func init() { 92 | proto.RegisterType((*Entry)(nil), "proto.Entry") 93 | } 94 | 95 | func init() { proto.RegisterFile("entry.proto", fileDescriptor_entry_4f5906245d08394f) } 96 | 97 | var fileDescriptor_entry_4f5906245d08394f = []byte{ 98 | // 134 bytes of a gzipped FileDescriptorProto 99 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4e, 0xcd, 0x2b, 0x29, 100 | 0xaa, 0xd4, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0xa5, 0x5c, 0xac, 0xae, 101 | 0x20, 0x51, 0x21, 0x01, 0x2e, 0x66, 0xe7, 0x20, 0x67, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xde, 0x20, 102 | 0x10, 0x13, 0x24, 0xe2, 0x9d, 0x5a, 0x29, 0xc1, 0xa4, 0xc0, 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a, 103 | 0x89, 0x70, 0xb1, 0x7a, 0xe6, 0xa5, 0xa4, 0x56, 0x48, 0x30, 0x2b, 0x30, 0x6a, 0x30, 0x07, 0x41, 104 | 0x38, 0x20, 0xd1, 0xb0, 0xc4, 0x9c, 0xd2, 0x54, 0x09, 0x16, 0x05, 0x46, 0x0d, 0x9e, 0x20, 0x08, 105 | 0x47, 0x48, 0x86, 0x8b, 0x33, 0x24, 0x33, 0x37, 0xb5, 0xb8, 0x24, 0x31, 0xb7, 0x40, 0x82, 0x15, 106 | 0xac, 0x1e, 0x21, 0x90, 0xc4, 0x06, 0xb6, 0xdd, 0x18, 0x10, 0x00, 0x00, 0xff, 0xff, 0x07, 0x99, 107 | 0x47, 0xb9, 0x93, 0x00, 0x00, 0x00, 108 | } 109 | -------------------------------------------------------------------------------- /proto/entry.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | package proto; 4 | 5 | message Entry { 6 | uint32 CRC = 1; 7 | string Key = 2; 8 | int64 Index = 3; 9 | bytes Value = 4; 10 | int64 Timestamp = 5; 11 | } 12 | -------------------------------------------------------------------------------- /streampb/stream.go: -------------------------------------------------------------------------------- 1 | package streampb 2 | 3 | import ( 4 | "encoding/binary" 5 | "io" 6 | 7 | "github.com/gogo/protobuf/proto" 8 | "github.com/pkg/errors" 9 | ) 10 | 11 | const ( 12 | // prefixSize is the number of bytes we preallocate for storing 13 | // our big endian lenth prefix buffer. 14 | prefixSize = 8 15 | ) 16 | 17 | // NewEncoder creates a streaming protobuf encoder. 18 | func NewEncoder(w io.Writer) *Encoder { 19 | return &Encoder{w: w, prefixBuf: make([]byte, prefixSize)} 20 | } 21 | 22 | // Encoder wraps an underlying io.Writer and allows you to stream 23 | // proto encodings on it. 24 | type Encoder struct { 25 | w io.Writer 26 | prefixBuf []byte 27 | } 28 | 29 | // Encode takes any proto.Message and streams it to the underlying writer. 30 | // Messages are framed with a length prefix. 31 | func (e *Encoder) Encode(msg proto.Message) error { 32 | buf, err := proto.Marshal(msg) 33 | if err != nil { 34 | return err 35 | } 36 | binary.BigEndian.PutUint64(e.prefixBuf, uint64(len(buf))) 37 | 38 | if _, err := e.w.Write(e.prefixBuf); err != nil { 39 | return errors.Wrap(err, "failed writing length prefix") 40 | } 41 | 42 | _, err = e.w.Write(buf) 43 | return errors.Wrap(err, "failed writing marshaled data") 44 | } 45 | 46 | // NewDecoder creates a streaming protobuf decoder. 47 | func NewDecoder(r io.Reader) *Decoder { 48 | return &Decoder{ 49 | r: r, 50 | prefixBuf: make([]byte, prefixSize), 51 | } 52 | } 53 | 54 | // Decoder wraps an underlying io.Reader and allows you to stream 55 | // proto decodings on it. 56 | type Decoder struct { 57 | r io.Reader 58 | prefixBuf []byte 59 | } 60 | 61 | // Decode takes a proto.Message and unmarshals the next payload in the 62 | // underlying io.Reader. It returns an EOF when it's done. 63 | func (d *Decoder) Decode(v proto.Message) error { 64 | _, err := io.ReadFull(d.r, d.prefixBuf) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | n := binary.BigEndian.Uint64(d.prefixBuf) 70 | 71 | buf := make([]byte, n) 72 | 73 | idx := uint64(0) 74 | for idx < n { 75 | m, err := d.r.Read(buf[idx:n]) 76 | if err != nil { 77 | return errors.Wrap(translateError(err), "failed reading marshaled data") 78 | } 79 | idx += uint64(m) 80 | } 81 | return proto.Unmarshal(buf[:n], v) 82 | } 83 | 84 | func translateError(err error) error { 85 | if err == io.EOF { 86 | return io.ErrUnexpectedEOF 87 | } 88 | return err 89 | } 90 | -------------------------------------------------------------------------------- /tools/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Get the highest tag number 4 | VERSION="$(git describe --abbrev=0 --tags)" 5 | VERSION=${VERSION:-'0.0.0'} 6 | 7 | # Get number parts 8 | MAJOR="${VERSION%%.*}"; VERSION="${VERSION#*.}" 9 | MINOR="${VERSION%%.*}"; VERSION="${VERSION#*.}" 10 | PATCH="${VERSION%%.*}"; VERSION="${VERSION#*.}" 11 | 12 | # Increase version 13 | PATCH=$((PATCH+1)) 14 | 15 | TAG="${1}" 16 | 17 | if [ "${TAG}" = "" ]; then 18 | TAG="${MAJOR}.${MINOR}.${PATCH}" 19 | fi 20 | 21 | echo "Releasing ${TAG} ..." 22 | 23 | git tag -a -s -m "Relase ${TAG}" "${TAG}" 24 | git push --tags 25 | goreleaser release --rm-dist 26 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | var ( 8 | // Version release version 9 | Version = "0.0.1" 10 | 11 | // Commit will be overwritten automatically by the build system 12 | Commit = "HEAD" 13 | ) 14 | 15 | // FullVersion returns the full version and commit hash 16 | func FullVersion() string { 17 | return fmt.Sprintf("%s@%s", Version, Commit) 18 | } 19 | -------------------------------------------------------------------------------- /version_test.go: -------------------------------------------------------------------------------- 1 | package bitcask 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestFullVersion(t *testing.T) { 11 | assert := assert.New(t) 12 | 13 | expected := fmt.Sprintf("%s@%s", Version, Commit) 14 | assert.Equal(expected, FullVersion()) 15 | } 16 | --------------------------------------------------------------------------------