├── .gitignore ├── stores ├── writer.go ├── reader.go ├── context-keys.go ├── store.go ├── record.go ├── in-memory-log.go ├── serializable │ ├── two-phase-lock.go │ └── key-locker.go └── persist.go ├── command.go ├── Dockerfile ├── tcp.Dockerfile ├── protobuf ├── record.proto ├── writer.go ├── reader.go ├── to-proto.go └── record.pb.go ├── go.mod ├── commands ├── noop.go ├── quit.go ├── get.go ├── rollback.go ├── commit.go ├── begin.go ├── delete.go └── set.go ├── LICENSE ├── README.md ├── cmd ├── kvapi │ ├── handlers │ │ └── kv.go │ └── main.go └── kv-tcp │ └── main.go ├── go.sum └── transactors └── transactor.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | -------------------------------------------------------------------------------- /stores/writer.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type Writer interface { 8 | Write(ctx context.Context, record Record) error 9 | } 10 | -------------------------------------------------------------------------------- /stores/reader.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | type Reader interface { 8 | Read(ctx context.Context, records chan<- Record) error 9 | } 10 | -------------------------------------------------------------------------------- /stores/context-keys.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | type contextKey struct { 4 | name string 5 | } 6 | 7 | // ContextKeyTransactionID is a context key for the transaction ID. 8 | var ContextKeyTransactionID = contextKey{"TXID"} 9 | -------------------------------------------------------------------------------- /command.go: -------------------------------------------------------------------------------- 1 | package kvdb 2 | 3 | import "context" 4 | 5 | // Command is a thing to do. 6 | type Command interface { 7 | Execute(ctx context.Context) error 8 | Undo(ctx context.Context) error 9 | ShouldAutoTransact() bool 10 | } 11 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang as build 2 | 3 | WORKDIR /kvdb 4 | 5 | COPY . . 6 | 7 | RUN CGO_ENABLED=0 go build -o /bin/kvapi cmd/kvapi/*.go 8 | 9 | FROM scratch 10 | 11 | COPY --from=build /bin/kvapi /kvapi 12 | 13 | EXPOSE 3001 14 | ENTRYPOINT ["/kvapi"] 15 | -------------------------------------------------------------------------------- /tcp.Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang as build 2 | 3 | WORKDIR /kvdb 4 | 5 | COPY . . 6 | 7 | RUN CGO_ENABLED=0 go build -o /bin/kv-tcp cmd/kv-tcp/*.go 8 | 9 | FROM scratch 10 | 11 | COPY --from=build /bin/kv-tcp /kv-tcp 12 | 13 | EXPOSE 8888 14 | ENTRYPOINT ["/kv-tcp"] 15 | -------------------------------------------------------------------------------- /protobuf/record.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | package protobuf; 3 | 4 | message Record { 5 | enum RecordKind { 6 | SET = 0; 7 | DEL = 1; 8 | CMT = 2; 9 | } 10 | 11 | RecordKind kind = 1; 12 | string key = 2; 13 | string value = 3; 14 | int64 transactionId = 4; 15 | } 16 | -------------------------------------------------------------------------------- /stores/store.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import "context" 4 | 5 | type Store interface { 6 | Set(ctx context.Context, key, value string) error 7 | Get(ctx context.Context, key string) (string, error) 8 | Delete(ctx context.Context, key string) error 9 | Keys(ctx context.Context) ([]string, error) 10 | Release(ctx context.Context) 11 | } 12 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/christianalexander/kvdb 2 | 3 | require ( 4 | github.com/gogo/protobuf v1.1.1 5 | github.com/golang/protobuf v1.2.0 6 | github.com/gorilla/context v1.1.1 // indirect 7 | github.com/gorilla/mux v1.6.2 8 | github.com/sirupsen/logrus v1.1.1 9 | golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 // indirect 10 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f // indirect 11 | ) 12 | -------------------------------------------------------------------------------- /stores/record.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import "fmt" 4 | 5 | type RecordKind string 6 | 7 | const ( 8 | RecordKindSet RecordKind = "SET" 9 | RecordKindDelete = "DEL" 10 | RecordKindCommit = "COMMIT" 11 | ) 12 | 13 | type Record struct { 14 | Kind RecordKind 15 | TransactionID int64 16 | Key string 17 | Value string 18 | } 19 | 20 | func (r Record) String() string { 21 | return fmt.Sprintf("%s:%d:%s:%s", r.Kind, r.TransactionID, r.Key, r.Value) 22 | } 23 | -------------------------------------------------------------------------------- /commands/noop.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/christianalexander/kvdb" 7 | ) 8 | 9 | // noop is a command that does nothing. 10 | type noop struct{} 11 | 12 | // Execute satisfies the command interface. 13 | func (q noop) Execute(ctx context.Context) error { 14 | return nil 15 | } 16 | 17 | func (q noop) Undo(ctx context.Context) error { 18 | return nil 19 | } 20 | 21 | func (q noop) ShouldAutoTransact() bool { 22 | return false 23 | } 24 | 25 | // NewNoop creates a new noop command. 26 | func NewNoop() kvdb.Command { 27 | return noop{} 28 | } 29 | -------------------------------------------------------------------------------- /commands/quit.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/christianalexander/kvdb" 7 | ) 8 | 9 | // quit is a command that closes the connection. 10 | type quit struct { 11 | cancelConnection func() error 12 | } 13 | 14 | // Execute satisfies the command interface. 15 | func (q quit) Execute(context.Context) error { 16 | return q.cancelConnection() 17 | } 18 | 19 | func (q quit) Undo(ctx context.Context) error { 20 | return nil 21 | } 22 | 23 | func (q quit) ShouldAutoTransact() bool { 24 | return false 25 | } 26 | 27 | // NewQuit creates a new quit command. 28 | func NewQuit(cancelConnection func() error) kvdb.Command { 29 | return quit{cancelConnection} 30 | } 31 | -------------------------------------------------------------------------------- /commands/get.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/christianalexander/kvdb" 9 | "github.com/christianalexander/kvdb/stores" 10 | ) 11 | 12 | // get is a command that gets a value from the store. 13 | type get struct { 14 | writer io.Writer 15 | store stores.Store 16 | key string 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q get) Execute(ctx context.Context) error { 21 | val, err := q.store.Get(ctx, q.key) 22 | if err != nil { 23 | q.writer.Write([]byte("\r\n")) 24 | return nil 25 | } 26 | 27 | fmt.Fprintln(q.writer, val) 28 | return nil 29 | } 30 | 31 | func (q get) Undo(ctx context.Context) error { 32 | return nil 33 | } 34 | 35 | func (q get) ShouldAutoTransact() bool { 36 | return true 37 | } 38 | 39 | // NewGet creates a new get command. 40 | func NewGet(writer io.Writer, store stores.Store, key string) kvdb.Command { 41 | return get{writer, store, key} 42 | } 43 | -------------------------------------------------------------------------------- /protobuf/writer.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/christianalexander/kvdb/stores" 11 | proto "github.com/golang/protobuf/proto" 12 | ) 13 | 14 | type protoWriter struct { 15 | writer io.Writer 16 | } 17 | 18 | func NewWriter(writer io.Writer) stores.Writer { 19 | return protoWriter{writer} 20 | } 21 | 22 | func (w protoWriter) Write(ctx context.Context, record stores.Record) error { 23 | protoRecord := RecordToProto(record) 24 | 25 | out, err := proto.Marshal(protoRecord) 26 | if err != nil { 27 | return fmt.Errorf("failed to marshal record %s: %v", record, err) 28 | } 29 | 30 | lBuf := make([]byte, binary.MaxVarintLen64) 31 | n := binary.PutUvarint(lBuf, uint64(len(out))) 32 | 33 | mr := io.MultiReader(bytes.NewBuffer(lBuf[:n]), bytes.NewBuffer(out)) 34 | 35 | _, err = io.Copy(w.writer, mr) 36 | if err != nil { 37 | return fmt.Errorf("failed to write record to log: %v", err) 38 | } 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /commands/rollback.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/christianalexander/kvdb" 9 | "github.com/christianalexander/kvdb/transactors" 10 | ) 11 | 12 | // rollback is a command that completes a transaction. 13 | type rollback struct { 14 | writer io.Writer 15 | transactor transactors.Transactor 16 | setTxID func(int64) 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q rollback) Execute(ctx context.Context) error { 21 | q.transactor.Rollback(ctx) 22 | q.writer.Write([]byte("OK\r\n")) 23 | q.setTxID(0) 24 | 25 | return nil 26 | } 27 | 28 | func (q rollback) Undo(ctx context.Context) error { 29 | return fmt.Errorf("cannot undo a rollback command") 30 | } 31 | 32 | func (q rollback) ShouldAutoTransact() bool { 33 | return false 34 | } 35 | 36 | // NewRollback creates a new rollback command. 37 | func NewRollback(writer io.Writer, transactor transactors.Transactor, setTxID func(int64)) kvdb.Command { 38 | return rollback{writer, transactor, setTxID} 39 | } 40 | -------------------------------------------------------------------------------- /protobuf/reader.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "encoding/binary" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/christianalexander/kvdb/stores" 11 | "github.com/gogo/protobuf/proto" 12 | ) 13 | 14 | type protoReader struct { 15 | reader io.Reader 16 | } 17 | 18 | func NewReader(reader io.Reader) stores.Reader { 19 | return protoReader{reader} 20 | } 21 | 22 | func (r protoReader) Read(ctx context.Context, records chan<- stores.Record) error { 23 | br := bufio.NewReader(r.reader) 24 | 25 | for { 26 | select { 27 | case <-ctx.Done(): 28 | return ctx.Err() 29 | default: 30 | l, err := binary.ReadUvarint(br) 31 | if err != nil { 32 | return fmt.Errorf("failed to read from record file: %v", err) 33 | } 34 | 35 | buf := make([]byte, l) 36 | br.Read(buf) 37 | 38 | var record Record 39 | err = proto.Unmarshal(buf, &record) 40 | if err != nil { 41 | return fmt.Errorf("failed to unmarshal record: %v", err) 42 | } 43 | 44 | records <- *record.ToRecord() 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2018 Christian Alexander 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /commands/commit.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | 8 | "github.com/christianalexander/kvdb" 9 | "github.com/christianalexander/kvdb/transactors" 10 | ) 11 | 12 | // commit is a command that completes a transaction. 13 | type commit struct { 14 | writer io.Writer 15 | transactor transactors.Transactor 16 | setTxID func(int64) 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q commit) Execute(ctx context.Context) error { 21 | err := q.transactor.Commit(ctx) 22 | if err == nil { 23 | q.writer.Write([]byte("OK\r\n")) 24 | q.setTxID(0) 25 | } 26 | 27 | return err 28 | } 29 | 30 | func (q commit) Undo(ctx context.Context) error { 31 | return fmt.Errorf("cannot undo a commit command") 32 | } 33 | 34 | func (q commit) ShouldAutoTransact() bool { 35 | return false 36 | } 37 | 38 | // NewCommit creates a new commit command. 39 | func NewCommit(writer io.Writer, transactor transactors.Transactor, setTxID func(int64)) kvdb.Command { 40 | return commit{writer, transactor, setTxID} 41 | } 42 | -------------------------------------------------------------------------------- /commands/begin.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/christianalexander/kvdb" 8 | "github.com/christianalexander/kvdb/transactors" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // begin is a command that begins a transaction. 13 | type begin struct { 14 | writer io.Writer 15 | transactor transactors.Transactor 16 | setTxID func(int64) 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q begin) Execute(ctx context.Context) error { 21 | txID, err := q.transactor.Begin(ctx) 22 | if err == nil { 23 | q.writer.Write([]byte("OK\r\n")) 24 | logrus.Printf("Begin setting txid to %d", txID) 25 | q.setTxID(txID) 26 | } 27 | 28 | return err 29 | } 30 | 31 | func (q begin) Undo(ctx context.Context) error { 32 | return nil 33 | } 34 | 35 | func (q begin) ShouldAutoTransact() bool { 36 | return false 37 | } 38 | 39 | // NewBegin creates a new begin command. 40 | func NewBegin(writer io.Writer, transactor transactors.Transactor, setTxID func(int64)) kvdb.Command { 41 | return begin{writer, transactor, setTxID} 42 | } 43 | -------------------------------------------------------------------------------- /protobuf/to-proto.go: -------------------------------------------------------------------------------- 1 | package protobuf 2 | 3 | import "github.com/christianalexander/kvdb/stores" 4 | 5 | func RecordToProto(r stores.Record) *Record { 6 | return &Record{ 7 | Kind: recordKindToProto(r.Kind), 8 | TransactionId: r.TransactionID, 9 | Key: r.Key, 10 | Value: r.Value, 11 | } 12 | } 13 | 14 | func recordKindToProto(k stores.RecordKind) Record_RecordKind { 15 | switch k { 16 | case stores.RecordKindSet: 17 | return Record_SET 18 | case stores.RecordKindDelete: 19 | return Record_DEL 20 | case stores.RecordKindCommit: 21 | return Record_CMT 22 | } 23 | 24 | return Record_SET 25 | } 26 | 27 | func recordKindFromProto(k Record_RecordKind) stores.RecordKind { 28 | switch k { 29 | case Record_SET: 30 | return stores.RecordKindSet 31 | case Record_DEL: 32 | return stores.RecordKindDelete 33 | case Record_CMT: 34 | return stores.RecordKindCommit 35 | } 36 | 37 | return stores.RecordKindSet 38 | } 39 | 40 | func (r Record) ToRecord() *stores.Record { 41 | return &stores.Record{ 42 | Kind: recordKindFromProto(r.Kind), 43 | TransactionID: r.TransactionId, 44 | Key: r.Key, 45 | Value: r.Value, 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /commands/delete.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/christianalexander/kvdb" 8 | "github.com/christianalexander/kvdb/stores" 9 | ) 10 | 11 | // delete is a command that Deletes a value from the store. 12 | type delete struct { 13 | writer io.Writer 14 | store stores.Store 15 | key string 16 | previousValue string 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q *delete) Execute(ctx context.Context) error { 21 | val, _ := q.store.Get(ctx, q.key) 22 | q.previousValue = val 23 | 24 | err := q.store.Delete(ctx, q.key) 25 | if err == nil { 26 | q.writer.Write([]byte("OK\r\n")) 27 | } 28 | 29 | return err 30 | } 31 | 32 | func (q *delete) Undo(ctx context.Context) error { 33 | if q.previousValue != "" { 34 | return q.store.Set(ctx, q.key, q.previousValue) 35 | } 36 | return nil 37 | } 38 | 39 | func (q delete) ShouldAutoTransact() bool { 40 | return true 41 | } 42 | 43 | // NewDelete creates a new delete command. 44 | func NewDelete(writer io.Writer, store stores.Store, key string) kvdb.Command { 45 | return &delete{ 46 | writer: writer, 47 | store: store, 48 | key: key, 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /commands/set.go: -------------------------------------------------------------------------------- 1 | package commands 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/christianalexander/kvdb" 8 | "github.com/christianalexander/kvdb/stores" 9 | "github.com/sirupsen/logrus" 10 | ) 11 | 12 | // set is a command that sets a value in the store. 13 | type set struct { 14 | writer io.Writer 15 | store stores.Store 16 | key, value, previousValue string 17 | } 18 | 19 | // Execute satisfies the command interface. 20 | func (q *set) Execute(ctx context.Context) error { 21 | val, _ := q.store.Get(ctx, q.key) 22 | q.previousValue = val 23 | 24 | err := q.store.Set(ctx, q.key, q.value) 25 | if err == nil { 26 | q.writer.Write([]byte("OK\r\n")) 27 | } 28 | 29 | return err 30 | } 31 | 32 | func (q *set) Undo(ctx context.Context) error { 33 | logrus.Print(q.previousValue) 34 | if q.previousValue == "" { 35 | return q.store.Delete(ctx, q.key) 36 | } 37 | 38 | return q.store.Set(ctx, q.key, q.previousValue) 39 | } 40 | 41 | func (q set) ShouldAutoTransact() bool { 42 | return true 43 | } 44 | 45 | // NewSet creates a new set command. 46 | func NewSet(writer io.Writer, store stores.Store, key, value string) kvdb.Command { 47 | return &set{ 48 | writer: writer, 49 | store: store, 50 | key: key, 51 | value: value, 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /stores/in-memory-log.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | ) 8 | 9 | type inMemoryStore struct { 10 | mu sync.RWMutex 11 | values map[string]string 12 | } 13 | 14 | func NewInMemoryStore() Store { 15 | return &inMemoryStore{ 16 | values: make(map[string]string), 17 | } 18 | } 19 | 20 | func (s *inMemoryStore) Set(ctx context.Context, key string, value string) error { 21 | s.mu.Lock() 22 | defer s.mu.Unlock() 23 | 24 | s.values[key] = value 25 | 26 | return nil 27 | } 28 | 29 | func (s *inMemoryStore) Get(ctx context.Context, key string) (string, error) { 30 | s.mu.RLock() 31 | defer s.mu.RUnlock() 32 | 33 | v, ok := s.values[key] 34 | if !ok { 35 | return "", fmt.Errorf("value for key '%s' not found", key) 36 | } 37 | 38 | return v, nil 39 | } 40 | 41 | func (s *inMemoryStore) Delete(ctx context.Context, key string) error { 42 | s.mu.Lock() 43 | defer s.mu.Unlock() 44 | 45 | delete(s.values, key) 46 | 47 | return nil 48 | } 49 | 50 | func (s *inMemoryStore) Keys(ctx context.Context) ([]string, error) { 51 | s.mu.RLock() 52 | defer s.mu.RUnlock() 53 | 54 | var result []string 55 | for k := range s.values { 56 | result = append(result, k) 57 | } 58 | 59 | return result, nil 60 | } 61 | 62 | func (s *inMemoryStore) Release(context.Context) {} 63 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # KVDB 2 | 3 | A key/value database for the [Phoenix Golang Meetup](https://www.meetup.com/Golang-Phoenix)'s [Build a Database Server](https://www.meetup.com/Golang-Phoenix/events/255183136/) challenge. 4 | 5 | Heavily inspired by chapter 7 of [Martin Kleppmann](https://martin.kleppmann.com/)'s [Designing Data-Intensive Applications](https://dataintensive.net/) book. 6 | 7 | ## Structure 8 | 9 | - [`cmd`](cmd) - TCP and HTTP frontends for the DB 10 | - [`commands`](commands) - Implementations of execuatable and undoable actions 11 | - [`protobuf`](protobuf) - Protobuf implementations of store persistence 12 | - [`stores`](stores) - Stuff to do with storage 13 | - [`transactors`](transactors) - Implementation of a transaction orchestrator 14 | 15 | ## Isolation 16 | 17 | This DB implmements serializable isolation with a 2-phase lock. 18 | 19 | ## Binary Log 20 | 21 | This DB has a protobuf binary log for disk persistence. 22 | 23 | ## See Also 24 | 25 | ["Transactions: myths, surprises and opportunities"](https://www.youtube.com/watch?v=5ZjhNTM8XU8) - Martin Kleppmann at Strange Loop 26 | 27 | [Command Pattern - Wikipedia](https://en.wikipedia.org/wiki/Command_pattern) 28 | 29 | [Protocol Buffers](https://developers.google.com/protocol-buffers/) 30 | 31 | [Varint - Go Standard Library](https://golang.org/src/encoding/binary/varint.go) 32 | -------------------------------------------------------------------------------- /cmd/kvapi/handlers/kv.go: -------------------------------------------------------------------------------- 1 | package handlers 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | 8 | "github.com/christianalexander/kvdb/stores" 9 | "github.com/gorilla/mux" 10 | ) 11 | 12 | func GetGetHandler(store stores.Store) http.Handler { 13 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 14 | vars := mux.Vars(r) 15 | key := vars["Key"] 16 | 17 | v, err := store.Get(r.Context(), key) 18 | if err != nil { 19 | http.Error(w, fmt.Sprintf("Failed to get '%s': %v", key, err), http.StatusNotFound) 20 | return 21 | } 22 | 23 | w.WriteHeader(http.StatusOK) 24 | w.Write([]byte(v)) 25 | }) 26 | } 27 | 28 | func GetSetHandler(store stores.Store) http.Handler { 29 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 30 | vars := mux.Vars(r) 31 | key := vars["Key"] 32 | 33 | body, err := ioutil.ReadAll(r.Body) 34 | if err != nil { 35 | http.Error(w, fmt.Sprintf("Failed to read body: %v", err), http.StatusBadRequest) 36 | return 37 | } 38 | 39 | err = store.Set(r.Context(), key, string(body)) 40 | if err != nil { 41 | http.Error(w, fmt.Sprintf("Failed to set value: %v", err), http.StatusInternalServerError) 42 | return 43 | } 44 | 45 | w.Header().Add("Location", fmt.Sprintf("/%s", key)) 46 | w.WriteHeader(http.StatusCreated) 47 | w.Write([]byte("OK")) 48 | }) 49 | } 50 | 51 | func GetDeleteHandler(store stores.Store) http.Handler { 52 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 53 | vars := mux.Vars(r) 54 | key := vars["Key"] 55 | 56 | err := store.Delete(r.Context(), key) 57 | if err != nil { 58 | http.Error(w, fmt.Sprintf("Failed to delete value: %v", err), http.StatusInternalServerError) 59 | return 60 | } 61 | 62 | w.WriteHeader(http.StatusNoContent) 63 | }) 64 | } 65 | -------------------------------------------------------------------------------- /cmd/kvapi/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "net/http" 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | "time" 11 | 12 | "github.com/christianalexander/kvdb/protobuf" 13 | 14 | "github.com/christianalexander/kvdb/cmd/kvapi/handlers" 15 | "github.com/christianalexander/kvdb/stores" 16 | "github.com/sirupsen/logrus" 17 | 18 | "github.com/gorilla/mux" 19 | ) 20 | 21 | var inPath string 22 | var outPath string 23 | 24 | func init() { 25 | flag.StringVar(&inPath, "in", "", "The path to the log input file") 26 | flag.StringVar(&outPath, "out", "", "The path to the log out file") 27 | 28 | flag.Parse() 29 | } 30 | 31 | func main() { 32 | logrus.Infoln("Starting KV API") 33 | 34 | cctx, cancel := context.WithCancel(context.Background()) 35 | 36 | store := stores.NewInMemoryStore() 37 | 38 | if inPath != "" { 39 | inFile, err := os.Open(inPath) 40 | if err != nil { 41 | logrus.Fatalf("Failed to open inPath file ('%s'): %v", inPath, err) 42 | } 43 | 44 | reader := protobuf.NewReader(inFile) 45 | s, err := stores.FromPersistence(cctx, reader, store) 46 | if err != nil { 47 | logrus.Fatalf("Failed to read from persistence: %v", err) 48 | } 49 | 50 | store = s 51 | } 52 | 53 | if outPath != "" { 54 | outFile, err := os.OpenFile(outPath, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0664) 55 | if err != nil { 56 | logrus.Fatalf("Failed to open outPath file ('%s'): %v", outPath, err) 57 | } 58 | 59 | writer := protobuf.NewWriter(outFile) 60 | store = stores.WithPersistence(writer, store) 61 | } 62 | 63 | r := mux.NewRouter() 64 | 65 | r.Handle("/{Key}", handlers.GetGetHandler(store)).Methods(http.MethodGet) 66 | r.Handle("/{Key}", handlers.GetSetHandler(store)).Methods(http.MethodPut, http.MethodPost) 67 | r.Handle("/{Key}", handlers.GetDeleteHandler(store)).Methods(http.MethodDelete) 68 | 69 | srv := http.Server{Handler: r, Addr: ":3001"} 70 | 71 | sig := make(chan os.Signal, 1) 72 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM, syscall.SIGABRT) 73 | 74 | go func() { 75 | s := <-sig 76 | logrus.Infof("Signal received: %s", s) 77 | cancel() 78 | 79 | logrus.Infoln("Shutting down server within one second") 80 | tctx, cancel := context.WithTimeout(context.Background(), time.Second) 81 | srv.RegisterOnShutdown(cancel) 82 | srv.Shutdown(tctx) 83 | }() 84 | 85 | logrus.Infoln("Listening on port 3001") 86 | logrus.Fatal(srv.ListenAndServe()) 87 | } 88 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= 4 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 5 | github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= 6 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 7 | github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= 8 | github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= 9 | github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= 10 | github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 11 | github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe h1:CHRGQ8V7OlCYtwaKPJi3iA7J+YdNKdo8j7nG5IgDhjs= 12 | github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 13 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 14 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 15 | github.com/sirupsen/logrus v1.1.1 h1:VzGj7lhU7KEB9e9gMpAV/v5XT2NVSvLJhJLCWbnkgXg= 16 | github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= 17 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 18 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 19 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793 h1:u+LnwYTOOW7Ukr/fppxEb1Nwz0AtPflrblfvUudpo+I= 20 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 21 | golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1 h1:Y/KGZSOdz/2r0WJ9Mkmz6NJBusp0kiNx1Cn82lzJQ6w= 22 | golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 23 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= 24 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 25 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= 26 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 27 | -------------------------------------------------------------------------------- /stores/serializable/two-phase-lock.go: -------------------------------------------------------------------------------- 1 | package serializable 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/christianalexander/kvdb/stores" 8 | ) 9 | 10 | // twoPhaseLockStore implements two-phase locking for serializable isolation. 11 | type twoPhaseLockStore struct { 12 | stores.Store 13 | lm lockerMap 14 | } 15 | 16 | // NewTwoPhaseLockStore returns a store with two-phase locking for serializable isolation. 17 | func NewTwoPhaseLockStore(store stores.Store) stores.Store { 18 | return &twoPhaseLockStore{ 19 | Store: store, 20 | lm: lockerMap{ 21 | lockers: make(map[string]*keyLocker), 22 | keys: make(map[int64][]string), 23 | }, 24 | } 25 | } 26 | 27 | func (ts *twoPhaseLockStore) Set(ctx context.Context, key, value string) error { 28 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 29 | if !ok || txID == 0 { 30 | return fmt.Errorf("two phase lock store could not set without a transaction ID") 31 | } 32 | 33 | err := ts.lm.Acquire(ctx, txID, key) 34 | if err != nil { 35 | return err 36 | } 37 | 38 | err = ts.Store.Set(ctx, key, value) 39 | if err != nil { 40 | return err 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func (ts *twoPhaseLockStore) Get(ctx context.Context, key string) (string, error) { 47 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 48 | if !ok || txID == 0 { 49 | return "", fmt.Errorf("two phase lock store could not get without a transaction ID") 50 | } 51 | 52 | err := ts.lm.RAcquire(ctx, txID, key) 53 | if err != nil { 54 | return "", err 55 | } 56 | v, err := ts.Store.Get(ctx, key) 57 | if err != nil { 58 | return "", err 59 | } 60 | 61 | return v, nil 62 | } 63 | 64 | func (ts *twoPhaseLockStore) Delete(ctx context.Context, key string) error { 65 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 66 | if !ok || txID == 0 { 67 | return fmt.Errorf("two phase lock store could not delete without a transaction ID") 68 | } 69 | 70 | err := ts.lm.Acquire(ctx, txID, key) 71 | if err != nil { 72 | return err 73 | } 74 | 75 | err = ts.Store.Delete(ctx, key) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | return nil 81 | } 82 | 83 | func (ts *twoPhaseLockStore) Keys(ctx context.Context) ([]string, error) { 84 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 85 | if !ok || txID == 0 { 86 | return nil, fmt.Errorf("two phase lock store could not get keys without a transaction ID") 87 | } 88 | 89 | keys, err := ts.Keys(ctx) 90 | if err != nil { 91 | return nil, err 92 | } 93 | 94 | for _, k := range keys { 95 | err := ts.lm.RAcquire(ctx, txID, k) 96 | if err != nil { 97 | return nil, err 98 | } 99 | } 100 | 101 | return keys, nil 102 | } 103 | 104 | func (ts *twoPhaseLockStore) Release(ctx context.Context) { 105 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 106 | if !ok || txID == 0 { 107 | return 108 | } 109 | ts.lm.Release(txID) 110 | } 111 | -------------------------------------------------------------------------------- /stores/persist.go: -------------------------------------------------------------------------------- 1 | package stores 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type withPersistence struct { 11 | Store 12 | writer Writer 13 | } 14 | 15 | func WithPersistence(writer Writer, store Store) Store { 16 | return &withPersistence{ 17 | Store: store, 18 | writer: writer, 19 | } 20 | } 21 | 22 | func (s *withPersistence) Set(ctx context.Context, key string, value string) error { 23 | txID := ctx.Value(ContextKeyTransactionID).(int64) 24 | err := s.writer.Write(ctx, Record{ 25 | Kind: RecordKindSet, 26 | TransactionID: txID, 27 | Key: key, 28 | Value: value, 29 | }) 30 | if err != nil { 31 | return fmt.Errorf("failed to write set operation: %v", err) 32 | } 33 | 34 | return s.Store.Set(ctx, key, value) 35 | } 36 | 37 | func (s *withPersistence) Delete(ctx context.Context, key string) error { 38 | txID := ctx.Value(ContextKeyTransactionID).(int64) 39 | err := s.writer.Write(ctx, Record{ 40 | Kind: RecordKindDelete, 41 | TransactionID: txID, 42 | Key: key, 43 | }) 44 | if err != nil { 45 | return fmt.Errorf("failed to write delete operation: %v", err) 46 | } 47 | 48 | return s.Store.Delete(ctx, key) 49 | } 50 | 51 | func applyRecord(ctx context.Context, pendingTransactionRecords map[int64][]Record, store Store, record Record) { 52 | logrus.Debugln(record.String()) 53 | switch record.Kind { 54 | case RecordKindSet: 55 | if record.TransactionID != 0 { 56 | pendingTransactionRecords[record.TransactionID] = append(pendingTransactionRecords[record.TransactionID], record) 57 | } else { 58 | err := store.Set(ctx, record.Key, record.Value) 59 | if err != nil { 60 | logrus.Warnf("Failed to replay set record: %v", err) 61 | } 62 | } 63 | case RecordKindDelete: 64 | if record.TransactionID != 0 { 65 | pendingTransactionRecords[record.TransactionID] = append(pendingTransactionRecords[record.TransactionID], record) 66 | } else { 67 | err := store.Delete(ctx, record.Key) 68 | if err != nil { 69 | logrus.Warnf("Failed to replay delete record: %v", err) 70 | } 71 | } 72 | case RecordKindCommit: 73 | if records, ok := pendingTransactionRecords[record.TransactionID]; ok { 74 | for _, r := range records { 75 | r.TransactionID = 0 76 | applyRecord(ctx, pendingTransactionRecords, store, r) 77 | } 78 | } 79 | default: 80 | logrus.Warnf("Received record of unknown type '%s'", record.Kind) 81 | } 82 | } 83 | 84 | func FromPersistence(ctx context.Context, reader Reader, store Store) (Store, error) { 85 | records := make(chan Record) 86 | pendingTransactionRecords := make(map[int64][]Record) 87 | 88 | go func() { 89 | reader.Read(ctx, records) 90 | close(records) 91 | }() 92 | 93 | for { 94 | select { 95 | case <-ctx.Done(): 96 | return nil, ctx.Err() 97 | case r, ok := <-records: 98 | if !ok { 99 | return store, nil 100 | } 101 | 102 | applyRecord(ctx, pendingTransactionRecords, store, r) 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /transactors/transactor.go: -------------------------------------------------------------------------------- 1 | package transactors 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "sync/atomic" 8 | 9 | "github.com/christianalexander/kvdb" 10 | "github.com/christianalexander/kvdb/stores" 11 | "github.com/sirupsen/logrus" 12 | ) 13 | 14 | // A Transactor is able to orchestrate transactions. 15 | type Transactor interface { 16 | Execute(ctx context.Context, command kvdb.Command) error 17 | Begin(ctx context.Context) (transactionID int64, err error) 18 | Commit(ctx context.Context) error 19 | Rollback(ctx context.Context) error 20 | } 21 | 22 | type transactor struct { 23 | store stores.Store 24 | mu sync.Mutex 25 | transactionCommands map[int64][]kvdb.Command 26 | latestTransactionID int64 27 | writer stores.Writer 28 | } 29 | 30 | // New creates a new Transactor. 31 | func New(store stores.Store, writer stores.Writer) Transactor { 32 | return &transactor{ 33 | store: store, 34 | transactionCommands: make(map[int64][]kvdb.Command), 35 | writer: writer, 36 | } 37 | } 38 | 39 | func (t *transactor) Execute(ctx context.Context, command kvdb.Command) error { 40 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 41 | if command.ShouldAutoTransact() && (!ok || txID == 0) { 42 | logrus.Printf("Assigned txID %d", txID) 43 | txID = atomic.AddInt64(&t.latestTransactionID, 1) 44 | ctx = context.WithValue(ctx, stores.ContextKeyTransactionID, txID) 45 | defer t.Commit(ctx) 46 | } 47 | 48 | err := command.Execute(ctx) 49 | 50 | t.mu.Lock() 51 | t.transactionCommands[txID] = append(t.transactionCommands[txID], command) 52 | t.mu.Unlock() 53 | 54 | return err 55 | } 56 | 57 | func (t *transactor) Begin(ctx context.Context) (transactionID int64, err error) { 58 | existingID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 59 | if ok && existingID != 0 { 60 | return 0, fmt.Errorf("can not start a transaction within the existing transaction '%d'", existingID) 61 | } 62 | 63 | return atomic.AddInt64(&t.latestTransactionID, 1), nil 64 | } 65 | 66 | func (t *transactor) Commit(ctx context.Context) error { 67 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 68 | if !ok || txID == 0 { 69 | return fmt.Errorf("can not commit without a transaction") 70 | } 71 | 72 | t.store.Release(ctx) 73 | 74 | t.mu.Lock() 75 | delete(t.transactionCommands, txID) 76 | t.mu.Unlock() 77 | 78 | if t.writer != nil { 79 | t.writer.Write(ctx, stores.Record{ 80 | Kind: stores.RecordKindCommit, 81 | TransactionID: txID, 82 | }) 83 | } 84 | 85 | return nil 86 | } 87 | 88 | func (t *transactor) Rollback(ctx context.Context) error { 89 | txID, ok := ctx.Value(stores.ContextKeyTransactionID).(int64) 90 | if !ok || txID == 0 { 91 | return fmt.Errorf("can not rollback without a transaction") 92 | } 93 | 94 | commands, ok := t.transactionCommands[txID] 95 | if !ok { 96 | return fmt.Errorf("can not roll back transaction without command history") 97 | } 98 | 99 | for i := len(commands) - 1; i >= 0; i-- { 100 | commands[i].Undo(ctx) 101 | } 102 | 103 | t.store.Release(ctx) 104 | 105 | t.mu.Lock() 106 | delete(t.transactionCommands, txID) 107 | t.mu.Unlock() 108 | 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /protobuf/record.pb.go: -------------------------------------------------------------------------------- 1 | // Code generated by protoc-gen-go. DO NOT EDIT. 2 | // source: record.proto 3 | 4 | package protobuf 5 | 6 | import ( 7 | fmt "fmt" 8 | proto "github.com/golang/protobuf/proto" 9 | math "math" 10 | ) 11 | 12 | // Reference imports to suppress errors if they are not otherwise used. 13 | var _ = proto.Marshal 14 | var _ = fmt.Errorf 15 | var _ = math.Inf 16 | 17 | // This is a compile-time assertion to ensure that this generated file 18 | // is compatible with the proto package it is being compiled against. 19 | // A compilation error at this line likely means your copy of the 20 | // proto package needs to be updated. 21 | const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package 22 | 23 | type Record_RecordKind int32 24 | 25 | const ( 26 | Record_SET Record_RecordKind = 0 27 | Record_DEL Record_RecordKind = 1 28 | Record_CMT Record_RecordKind = 2 29 | ) 30 | 31 | var Record_RecordKind_name = map[int32]string{ 32 | 0: "SET", 33 | 1: "DEL", 34 | 2: "CMT", 35 | } 36 | 37 | var Record_RecordKind_value = map[string]int32{ 38 | "SET": 0, 39 | "DEL": 1, 40 | "CMT": 2, 41 | } 42 | 43 | func (x Record_RecordKind) String() string { 44 | return proto.EnumName(Record_RecordKind_name, int32(x)) 45 | } 46 | 47 | func (Record_RecordKind) EnumDescriptor() ([]byte, []int) { 48 | return fileDescriptor_bf94fd919e302a1d, []int{0, 0} 49 | } 50 | 51 | type Record struct { 52 | Kind Record_RecordKind `protobuf:"varint,1,opt,name=kind,proto3,enum=protobuf.Record_RecordKind" json:"kind,omitempty"` 53 | Key string `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` 54 | Value string `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` 55 | TransactionId int64 `protobuf:"varint,4,opt,name=transactionId,proto3" json:"transactionId,omitempty"` 56 | XXX_NoUnkeyedLiteral struct{} `json:"-"` 57 | XXX_unrecognized []byte `json:"-"` 58 | XXX_sizecache int32 `json:"-"` 59 | } 60 | 61 | func (m *Record) Reset() { *m = Record{} } 62 | func (m *Record) String() string { return proto.CompactTextString(m) } 63 | func (*Record) ProtoMessage() {} 64 | func (*Record) Descriptor() ([]byte, []int) { 65 | return fileDescriptor_bf94fd919e302a1d, []int{0} 66 | } 67 | 68 | func (m *Record) XXX_Unmarshal(b []byte) error { 69 | return xxx_messageInfo_Record.Unmarshal(m, b) 70 | } 71 | func (m *Record) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { 72 | return xxx_messageInfo_Record.Marshal(b, m, deterministic) 73 | } 74 | func (m *Record) XXX_Merge(src proto.Message) { 75 | xxx_messageInfo_Record.Merge(m, src) 76 | } 77 | func (m *Record) XXX_Size() int { 78 | return xxx_messageInfo_Record.Size(m) 79 | } 80 | func (m *Record) XXX_DiscardUnknown() { 81 | xxx_messageInfo_Record.DiscardUnknown(m) 82 | } 83 | 84 | var xxx_messageInfo_Record proto.InternalMessageInfo 85 | 86 | func (m *Record) GetKind() Record_RecordKind { 87 | if m != nil { 88 | return m.Kind 89 | } 90 | return Record_SET 91 | } 92 | 93 | func (m *Record) GetKey() string { 94 | if m != nil { 95 | return m.Key 96 | } 97 | return "" 98 | } 99 | 100 | func (m *Record) GetValue() string { 101 | if m != nil { 102 | return m.Value 103 | } 104 | return "" 105 | } 106 | 107 | func (m *Record) GetTransactionId() int64 { 108 | if m != nil { 109 | return m.TransactionId 110 | } 111 | return 0 112 | } 113 | 114 | func init() { 115 | proto.RegisterEnum("protobuf.Record_RecordKind", Record_RecordKind_name, Record_RecordKind_value) 116 | proto.RegisterType((*Record)(nil), "protobuf.Record") 117 | } 118 | 119 | func init() { proto.RegisterFile("record.proto", fileDescriptor_bf94fd919e302a1d) } 120 | 121 | var fileDescriptor_bf94fd919e302a1d = []byte{ 122 | // 173 bytes of a gzipped FileDescriptorProto 123 | 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x29, 0x4a, 0x4d, 0xce, 124 | 0x2f, 0x4a, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x00, 0x53, 0x49, 0xa5, 0x69, 0x4a, 125 | 0x1b, 0x18, 0xb9, 0xd8, 0x82, 0xc0, 0x52, 0x42, 0xfa, 0x5c, 0x2c, 0xd9, 0x99, 0x79, 0x29, 0x12, 126 | 0x8c, 0x0a, 0x8c, 0x1a, 0x7c, 0x46, 0xd2, 0x7a, 0x30, 0x35, 0x7a, 0x10, 0x79, 0x28, 0xe5, 0x9d, 127 | 0x99, 0x97, 0x12, 0x04, 0x56, 0x28, 0x24, 0xc0, 0xc5, 0x9c, 0x9d, 0x5a, 0x29, 0xc1, 0xa4, 0xc0, 128 | 0xa8, 0xc1, 0x19, 0x04, 0x62, 0x0a, 0x89, 0x70, 0xb1, 0x96, 0x25, 0xe6, 0x94, 0xa6, 0x4a, 0x30, 129 | 0x83, 0xc5, 0x20, 0x1c, 0x21, 0x15, 0x2e, 0xde, 0x92, 0xa2, 0xc4, 0xbc, 0xe2, 0xc4, 0xe4, 0x92, 130 | 0xcc, 0xfc, 0x3c, 0xcf, 0x14, 0x09, 0x16, 0x05, 0x46, 0x0d, 0xe6, 0x20, 0x54, 0x41, 0x25, 0x75, 131 | 0x2e, 0x2e, 0x84, 0x0d, 0x42, 0xec, 0x5c, 0xcc, 0xc1, 0xae, 0x21, 0x02, 0x0c, 0x20, 0x86, 0x8b, 132 | 0xab, 0x8f, 0x00, 0x23, 0x88, 0xe1, 0xec, 0x1b, 0x22, 0xc0, 0x94, 0xc4, 0x06, 0x76, 0x98, 0x31, 133 | 0x20, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x7a, 0x8e, 0x71, 0xd3, 0x00, 0x00, 0x00, 134 | } 135 | -------------------------------------------------------------------------------- /stores/serializable/key-locker.go: -------------------------------------------------------------------------------- 1 | package serializable 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | // A lockerMap holds a locker for each key, and the keys for each transaction. 11 | type lockerMap struct { 12 | mu sync.RWMutex 13 | lockers map[string]*keyLocker 14 | keys map[int64][]string 15 | } 16 | 17 | // A keyLocker is the implementation of a lock for a given key. 18 | type keyLocker struct { 19 | mu sync.Mutex 20 | 21 | waitingReaders []waiter 22 | waitingWriters []waiter 23 | 24 | // activeTransactions stores the active transaction IDs 25 | activeTransactions map[int64]struct{} 26 | 27 | // writeLockTxID is set if a transaction has an exclusive lock 28 | writeLockTxID int64 29 | } 30 | 31 | type waiter struct { 32 | txID int64 33 | ready chan struct{} 34 | } 35 | 36 | func (lm *lockerMap) getKeyLocker(key string) *keyLocker { 37 | lm.mu.Lock() 38 | locker, ok := lm.lockers[key] 39 | if !ok { 40 | l, ok := lm.lockers[key] 41 | if !ok { 42 | lm.lockers[key] = &keyLocker{ 43 | activeTransactions: make(map[int64]struct{}), 44 | } 45 | locker = lm.lockers[key] 46 | } else { 47 | locker = l 48 | } 49 | } 50 | lm.mu.Unlock() 51 | 52 | return locker 53 | } 54 | 55 | func (lm *lockerMap) setTxKey(txID int64, key string) { 56 | lm.mu.Lock() 57 | lm.keys[txID] = append(lm.keys[txID], key) 58 | lm.mu.Unlock() 59 | } 60 | 61 | // Acquire gets an exclusive lock on a key for a transaction. 62 | func (lm *lockerMap) Acquire(ctx context.Context, txID int64, key string) error { 63 | locker := lm.getKeyLocker(key) 64 | logrus.WithField("txID", txID).Debugf("Acquiring write lock for %s", key) 65 | 66 | for { 67 | locker.mu.Lock() 68 | if locker.writeLockTxID == txID { 69 | locker.mu.Unlock() 70 | logrus.WithField("txID", txID).Debugf("Write lock acquired for %s, already write holder", key) 71 | return nil 72 | } 73 | 74 | activeTxCount := len(locker.activeTransactions) 75 | if activeTxCount == 0 { 76 | locker.activeTransactions[txID] = struct{}{} 77 | lm.setTxKey(txID, key) 78 | locker.writeLockTxID = txID 79 | locker.mu.Unlock() 80 | logrus.WithField("txID", txID).Debugf("Write lock acquired for %s", key) 81 | return nil 82 | } else if activeTxCount == 1 { 83 | if _, ok := locker.activeTransactions[txID]; ok { 84 | locker.writeLockTxID = txID 85 | locker.mu.Unlock() 86 | return nil 87 | } 88 | } 89 | 90 | ready := make(chan struct{}) 91 | me := waiter{txID, ready} 92 | locker.waitingWriters = append(locker.waitingWriters, me) 93 | locker.mu.Unlock() 94 | 95 | logrus.WithField("txID", txID).Debugf("Entering write wait for '%s'", key) 96 | select { 97 | case <-ctx.Done(): 98 | locker.mu.Lock() 99 | ww := locker.waitingWriters 100 | for i, w := range ww { 101 | if w.txID == txID { 102 | locker.waitingWriters = append(locker.waitingWriters[:i], locker.waitingWriters[i+1:]...) 103 | } 104 | } 105 | locker.mu.Unlock() 106 | return ctx.Err() 107 | case <-ready: 108 | break 109 | } 110 | } 111 | } 112 | 113 | // RAcquire gets a shared lock on a key for a transaction. 114 | func (lm *lockerMap) RAcquire(ctx context.Context, txID int64, key string) error { 115 | locker := lm.getKeyLocker(key) 116 | logrus.WithField("txID", txID).Debugf("Acquiring read lock for %s", key) 117 | 118 | for { 119 | locker.mu.Lock() 120 | if locker.writeLockTxID == txID { 121 | locker.mu.Unlock() 122 | logrus.WithField("txID", txID).Debugf("Read lock acquired for %s, already write holder", key) 123 | return nil 124 | } 125 | 126 | _, txExists := locker.activeTransactions[txID] 127 | if locker.writeLockTxID == 0 && (txExists || len(locker.waitingWriters) == 0) { 128 | locker.activeTransactions[txID] = struct{}{} 129 | lm.setTxKey(txID, key) 130 | locker.mu.Unlock() 131 | logrus.WithField("txID", txID).Debugf("Read lock acquired for %s", key) 132 | return nil 133 | } 134 | 135 | ready := make(chan struct{}) 136 | me := waiter{txID, ready} 137 | locker.waitingReaders = append(locker.waitingReaders, me) 138 | locker.mu.Unlock() 139 | 140 | logrus.WithField("txID", txID).Debugf("Entering read wait for '%s'", key) 141 | select { 142 | case <-ctx.Done(): 143 | wr := locker.waitingReaders 144 | for i, w := range wr { 145 | if w.txID == txID { 146 | locker.waitingReaders = append(locker.waitingReaders[:i], locker.waitingReaders[i+1:]...) 147 | } 148 | } 149 | locker.mu.Unlock() 150 | return ctx.Err() 151 | case <-ready: 152 | break 153 | } 154 | } 155 | } 156 | 157 | // Release gives up all locks held by a transaction. 158 | func (lm *lockerMap) Release(txID int64) { 159 | logrus.WithField("txID", txID).Debug("Releasing transaction") 160 | keys, ok := lm.keys[txID] 161 | if !ok { 162 | return 163 | } 164 | 165 | for _, key := range keys { 166 | logrus.WithField("txID", txID).Debugf("Releasing from key %s", key) 167 | locker := lm.getKeyLocker(key) 168 | 169 | locker.mu.Lock() 170 | delete(locker.activeTransactions, txID) 171 | 172 | // if this was the writer 173 | if locker.writeLockTxID == txID { 174 | // release the writer 175 | locker.writeLockTxID = 0 176 | 177 | // Release all readers 178 | for _, r := range locker.waitingReaders { 179 | logrus.WithField("txID", txID).Debugf("Releasing reader for tx %d - key %s", r.txID, key) 180 | close(r.ready) 181 | } 182 | // Remove all reader waiters 183 | locker.waitingReaders = nil 184 | } 185 | 186 | if len(locker.activeTransactions) == 0 && len(locker.waitingWriters) != 0 { 187 | w := locker.waitingWriters[0] 188 | locker.waitingWriters = locker.waitingWriters[1:] 189 | close(w.ready) 190 | } else if len(locker.activeTransactions) == 1 { 191 | var activeTxID int64 192 | for k := range locker.activeTransactions { 193 | activeTxID = k 194 | } 195 | 196 | ww := locker.waitingWriters 197 | for i, w := range ww { 198 | if w.txID == activeTxID { 199 | close(w.ready) 200 | locker.waitingWriters = append(locker.waitingWriters[:i], locker.waitingWriters[i+1:]...) 201 | } 202 | } 203 | } 204 | 205 | locker.mu.Unlock() 206 | } 207 | 208 | lm.mu.Lock() 209 | delete(lm.keys, txID) 210 | lm.mu.Unlock() 211 | } 212 | -------------------------------------------------------------------------------- /cmd/kv-tcp/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "net" 10 | "os" 11 | "strings" 12 | "time" 13 | 14 | "github.com/christianalexander/kvdb" 15 | "github.com/christianalexander/kvdb/commands" 16 | "github.com/christianalexander/kvdb/protobuf" 17 | "github.com/christianalexander/kvdb/stores" 18 | "github.com/christianalexander/kvdb/stores/serializable" 19 | "github.com/christianalexander/kvdb/transactors" 20 | 21 | "github.com/sirupsen/logrus" 22 | ) 23 | 24 | type contextKey struct { 25 | name string 26 | } 27 | 28 | var ctxKeyServer = contextKey{"SERVER"} 29 | 30 | var inPath string 31 | var outPath string 32 | 33 | func init() { 34 | flag.StringVar(&inPath, "in", "", "The path to the log input file") 35 | flag.StringVar(&outPath, "out", "", "The path to the log out file") 36 | 37 | flag.Parse() 38 | } 39 | 40 | func main() { 41 | logrus.Infoln("Starting KV TCP API") 42 | 43 | ln, err := net.Listen("tcp", ":8888") 44 | if err != nil { 45 | logrus.Fatalf("Failed to start listener: %v", err) 46 | } 47 | 48 | logrus.SetLevel(logrus.DebugLevel) 49 | 50 | logrus.Infoln("Listening on port 8888") 51 | 52 | store := stores.NewInMemoryStore() 53 | 54 | if inPath != "" { 55 | inFile, err := os.Open(inPath) 56 | if err != nil { 57 | logrus.Fatalf("Failed to open inPath file ('%s'): %v", inPath, err) 58 | } 59 | 60 | reader := protobuf.NewReader(inFile) 61 | s, err := stores.FromPersistence(context.Background(), reader, store) 62 | if err != nil { 63 | logrus.Fatalf("Failed to read from persistence: %v", err) 64 | } 65 | 66 | store = s 67 | } 68 | 69 | var writer stores.Writer 70 | if outPath != "" { 71 | outFile, err := os.OpenFile(outPath, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0664) 72 | if err != nil { 73 | logrus.Fatalf("Failed to open outPath file ('%s'): %v", outPath, err) 74 | } 75 | 76 | w := protobuf.NewWriter(outFile) 77 | writer = w 78 | store = stores.WithPersistence(writer, store) 79 | } 80 | 81 | store = serializable.NewTwoPhaseLockStore(store) 82 | transactor := transactors.New(store, writer) 83 | 84 | server{store, transactor}.serve(ln.(*net.TCPListener)) 85 | } 86 | 87 | type server struct { 88 | store stores.Store 89 | transactor transactors.Transactor 90 | } 91 | 92 | func (s server) serve(l net.Listener) error { 93 | defer l.Close() 94 | 95 | var tempDelay time.Duration 96 | ctx := context.Background() 97 | for { 98 | rw, e := l.Accept() 99 | if e != nil { 100 | if ne, ok := e.(net.Error); ok && ne.Temporary() { 101 | if tempDelay == 0 { 102 | tempDelay = 5 * time.Millisecond 103 | } else { 104 | tempDelay *= 2 105 | } 106 | if max := 1 * time.Second; tempDelay > max { 107 | tempDelay = max 108 | } 109 | logrus.Warnf("http: Accept error: %v; retrying in %v", e, tempDelay) 110 | time.Sleep(tempDelay) 111 | continue 112 | } 113 | return e 114 | } 115 | tempDelay = 0 116 | c := newConn(rw) 117 | ctx := context.WithValue(ctx, ctxKeyServer, s) 118 | go c.serve(ctx) 119 | } 120 | } 121 | 122 | type conn struct { 123 | nc net.Conn 124 | close chan struct{} 125 | txID int64 126 | } 127 | 128 | func newConn(c net.Conn) *conn { 129 | return &conn{nc: c, close: make(chan struct{})} 130 | } 131 | 132 | func (c *conn) serve(ctx context.Context) { 133 | defer c.nc.Close() 134 | 135 | reader := bufio.NewReaderSize(c.nc, 4<<10) 136 | 137 | cctx, cancel := context.WithCancel(ctx) 138 | defer cancel() 139 | 140 | go func() { 141 | <-cctx.Done() 142 | if c.txID != 0 { 143 | srv := ctx.Value(ctxKeyServer).(server) 144 | srv.transactor.Rollback(context.WithValue(cctx, stores.ContextKeyTransactionID, c.txID)) 145 | } 146 | }() 147 | 148 | for { 149 | select { 150 | case <-c.close: 151 | return 152 | case <-cctx.Done(): 153 | return 154 | default: 155 | l, _, err := reader.ReadLine() 156 | if err != nil { 157 | if err != io.EOF { 158 | logrus.Warnf("Failed to read request: %v", err) 159 | } 160 | return 161 | } 162 | 163 | n, p1, p2, ok := parseCommandLine(string(l)) 164 | if !ok { 165 | logrus.Warnf("Failed to parse line '%s'", l) 166 | continue 167 | } 168 | 169 | cmd, err := c.GetCommand(cctx, n, p1, p2) 170 | if err != nil { 171 | logrus.Warnln(err) 172 | fmt.Fprintf(c.nc, "%v\r\n", err) 173 | return 174 | } 175 | 176 | srv := ctx.Value(ctxKeyServer).(server) 177 | srv.transactor.Execute(context.WithValue(cctx, stores.ContextKeyTransactionID, c.txID), cmd) 178 | if err != nil { 179 | logrus.Warnf("Failed to execute command: %v", err) 180 | fmt.Fprintf(c.nc, "%v\r\n", err) 181 | continue 182 | } 183 | } 184 | } 185 | } 186 | 187 | func (c *conn) GetCommand(ctx context.Context, commandName, p1, p2 string) (kvdb.Command, error) { 188 | switch strings.ToUpper(commandName) { 189 | case "QUIT": 190 | return commands.NewQuit(func() error { 191 | close(c.close) 192 | return nil 193 | }), nil 194 | case "SET": 195 | if p1 == "" || p2 == "" { 196 | return nil, fmt.Errorf("expected 'SET ', got 'SET %s %s'", p1, p2) 197 | } 198 | srv := ctx.Value(ctxKeyServer).(server) 199 | return commands.NewSet(c.nc, srv.store, p1, p2), nil 200 | case "GET": 201 | if p1 == "" { 202 | return nil, fmt.Errorf("expected 'GET ', but no key specified") 203 | } 204 | srv := ctx.Value(ctxKeyServer).(server) 205 | return commands.NewGet(c.nc, srv.store, p1), nil 206 | case "DEL": 207 | if p1 == "" { 208 | return nil, fmt.Errorf("expected 'DEL ', but no key specified") 209 | } 210 | srv := ctx.Value(ctxKeyServer).(server) 211 | return commands.NewDelete(c.nc, srv.store, p1), nil 212 | case "BEGIN": 213 | if c.txID != 0 { 214 | return nil, fmt.Errorf("cannot begin transaction within an active transaction") 215 | } 216 | srv := ctx.Value(ctxKeyServer).(server) 217 | return commands.NewBegin(c.nc, srv.transactor, func(txID int64) { 218 | c.txID = txID 219 | }), nil 220 | case "COMMIT": 221 | if c.txID == 0 { 222 | return nil, fmt.Errorf("cannot commit without a transaction") 223 | } 224 | srv := ctx.Value(ctxKeyServer).(server) 225 | return commands.NewCommit(c.nc, srv.transactor, func(txID int64) { 226 | c.txID = txID 227 | }), nil 228 | case "ROLLBACK": 229 | if c.txID == 0 { 230 | return nil, fmt.Errorf("cannot rollback without a transaction") 231 | } 232 | srv := ctx.Value(ctxKeyServer).(server) 233 | return commands.NewRollback(c.nc, srv.transactor, func(txID int64) { 234 | c.txID = txID 235 | }), nil 236 | } 237 | 238 | return nil, fmt.Errorf("invalid command '%s'", commandName) 239 | } 240 | 241 | func parseCommandLine(line string) (commandName, p1, p2 string, ok bool) { 242 | s1 := strings.Index(line, " ") 243 | s2 := strings.Index(line[s1+1:], " ") 244 | if s1 < 0 { 245 | return line, "", "", true 246 | } 247 | if s2 < 0 { 248 | return line[:s1], line[s1+1:], "", true 249 | } 250 | s2 += s1 + 1 251 | return line[:s1], line[s1+1 : s2], line[s2+1:], true 252 | } 253 | --------------------------------------------------------------------------------