├── .travis.yml ├── .gitignore ├── LICENSE ├── items.go ├── README.md ├── store_benchmark_test.go ├── store_test.go └── store.go /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.2 4 | - 1.3 5 | - 1.4 6 | - release 7 | - tip 8 | 9 | script: 10 | - go test -v ./... 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Serge Zaitsev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /items.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "encoding/gob" 5 | "encoding/json" 6 | "io" 7 | "io/ioutil" 8 | ) 9 | 10 | // Simple raw item encoding - copies value bytes as is 11 | type ByteItem struct { 12 | Value []byte 13 | } 14 | 15 | func (b *ByteItem) WriteTo(w io.Writer) (int64, error) { 16 | n, err := io.WriteString(w, string(b.Value)) 17 | return int64(n), err 18 | } 19 | 20 | func (b *ByteItem) ReadFrom(r io.Reader) (int64, error) { 21 | data, err := ioutil.ReadAll(r) 22 | if err == nil { 23 | b.Value = data 24 | } 25 | return int64(len(data)), err 26 | } 27 | 28 | // JSON encoding - serializes arbitrary value to/from JSON 29 | type JSONItem struct { 30 | Value interface{} 31 | } 32 | 33 | func (e *JSONItem) WriteTo(w io.Writer) (int64, error) { 34 | if err := json.NewEncoder(w).Encode(e.Value); err != nil { 35 | return 0, err 36 | } 37 | return 0, nil 38 | } 39 | 40 | func (e *JSONItem) ReadFrom(r io.Reader) (int64, error) { 41 | if err := json.NewDecoder(r).Decode(e.Value); err != nil { 42 | return 0, err 43 | } 44 | return 0, nil 45 | } 46 | 47 | // Gob encoding - serializes arbitrary value to/from gob format 48 | type GobItem struct { 49 | Value interface{} 50 | } 51 | 52 | func (e *GobItem) WriteTo(w io.Writer) (int64, error) { 53 | if err := gob.NewEncoder(w).Encode(e.Value); err != nil { 54 | return 0, err 55 | } 56 | return 0, nil 57 | } 58 | 59 | func (e *GobItem) ReadFrom(r io.Reader) (int64, error) { 60 | if err := gob.NewDecoder(r).Decode(e.Value); err != nil { 61 | return 0, err 62 | } 63 | return 0, nil 64 | } 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kv 2 | 3 | [![Build Status](https://travis-ci.org/zserge/kv.svg)](https://travis-ci.org/zserge/kv) [![](https://godoc.org/github.com/zserge/kv?status.svg)](http://godoc.org/github.com/zserge/kv) 4 | 5 | An ultimately minimal persistent key-value store + LRU cache 6 | 7 | ## Benchmark 8 | 9 | kv: 10 | 11 | ``` 12 | BenchmarkStoreGet-2 20000 59301 ns/op 84.32 MB/s 13 | BenchmarkStoreSet-2 20000 68880 ns/op 72.59 MB/s 14 | BenchmarkStoreCacheGet-2 3000000 404 ns/op 12347.96 MB/s 15 | BenchmarkStoreCacheSet-2 2000000 715 ns/op 6983.40 MB/s 16 | // Benchmarks below are copied from diskv tests 17 | BenchmarkWrite__32B-2 50000 30470 ns/op 1.05 MB/s 18 | BenchmarkWrite__1KB-2 50000 53329 ns/op 19.20 MB/s 19 | BenchmarkWrite__4KB-2 30000 66490 ns/op 61.60 MB/s 20 | BenchmarkWrite_10KB-2 20000 119928 ns/op 85.38 MB/s 21 | BenchmarkRead__32B_NoCache-2 2000 523594 ns/op 0.06 MB/s 22 | BenchmarkRead__1KB_NoCache-2 3000 641618 ns/op 1.60 MB/s 23 | BenchmarkRead__4KB_NoCache-2 2000 733945 ns/op 5.58 MB/s 24 | BenchmarkRead_10KB_NoCache-2 2000 803028 ns/op 12.75 MB/s 25 | BenchmarkRead__32B_WithCache-2 1000000 1509 ns/op 21.20 MB/s 26 | BenchmarkRead__1KB_WithCache-2 1000000 1221 ns/op 838.37 MB/s 27 | BenchmarkRead__4KB_WithCache-2 1000000 1061 ns/op 3859.21 MB/s 28 | BenchmarkRead_10KB_WithCache-2 1000000 1059 ns/op 9668.19 MB/s 29 | ``` 30 | 31 | Comparing to Diskv tested on the same machine: 32 | 33 | ``` 34 | BenchmarkWrite__32B_NoIndex-2 10000 113520 ns/op 0.28 MB/s 35 | BenchmarkWrite__1KB_NoIndex-2 10000 111664 ns/op 9.17 MB/s 36 | BenchmarkWrite__4KB_NoIndex-2 10000 118777 ns/op 34.48 MB/s 37 | BenchmarkWrite_10KB_NoIndex-2 10000 136848 ns/op 74.83 MB/s 38 | BenchmarkRead__32B_NoCache-2 50000 31648 ns/op 1.01 MB/s 39 | BenchmarkRead__1KB_NoCache-2 50000 34720 ns/op 29.49 MB/s 40 | BenchmarkRead__4KB_NoCache-2 20000 70228 ns/op 58.32 MB/s 41 | BenchmarkRead_10KB_NoCache-2 10000 103389 ns/op 99.04 MB/s 42 | BenchmarkRead__32B_WithCache-2 200000 7671 ns/op 4.17 MB/s 43 | BenchmarkRead__1KB_WithCache-2 200000 9299 ns/op 110.11 MB/s 44 | BenchmarkRead__4KB_WithCache-2 30000 33615 ns/op 121.85 MB/s 45 | BenchmarkRead_10KB_WithCache-2 20000 93018 ns/op 110.09 MB/s 46 | ``` 47 | 48 | I don't say that `kv` is faster or better, but it's fast enough for a simple 49 | key-value file-based storage. 50 | -------------------------------------------------------------------------------- /store_benchmark_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "math/rand" 7 | "testing" 8 | ) 9 | 10 | const StoreBenchPath = "store-bench" 11 | 12 | func BenchmarkStoreGet(b *testing.B) { 13 | store := NewStore(StoreBenchPath) 14 | item := &ByteItem{bytes.Repeat([]byte("hello"), 1000)} 15 | store.Set("foo", item) 16 | 17 | b.SetBytes(int64(len(item.Value))) 18 | for i := 0; i < b.N; i++ { 19 | store.Get("foo", item) 20 | } 21 | } 22 | 23 | func BenchmarkStoreSet(b *testing.B) { 24 | store := NewStore(StoreBenchPath) 25 | item := &ByteItem{bytes.Repeat([]byte("hello"), 1000)} 26 | 27 | b.SetBytes(int64(len(item.Value))) 28 | for i := 0; i < b.N; i++ { 29 | store.Set("foo", item) 30 | } 31 | } 32 | 33 | func BenchmarkStoreCacheGet(b *testing.B) { 34 | store := NewLRU(1, NewStore(StoreBenchPath)) 35 | item := &ByteItem{bytes.Repeat([]byte("hello"), 1000)} 36 | store.Set("foo", item) 37 | 38 | b.SetBytes(int64(len(item.Value))) 39 | for i := 0; i < b.N; i++ { 40 | store.Get("foo", item) 41 | } 42 | } 43 | 44 | func BenchmarkStoreCacheSet(b *testing.B) { 45 | store := NewLRU(1, NewStore(StoreBenchPath)) 46 | item := &ByteItem{bytes.Repeat([]byte("hello"), 1000)} 47 | 48 | b.SetBytes(int64(len(item.Value))) 49 | for i := 0; i < b.N; i++ { 50 | store.Set("foo", item) 51 | } 52 | } 53 | 54 | func BenchmarkWrite__32B(b *testing.B) { 55 | benchWrite(b, 32, true) 56 | } 57 | 58 | func BenchmarkWrite__1KB(b *testing.B) { 59 | benchWrite(b, 1024, true) 60 | } 61 | 62 | func BenchmarkWrite__4KB(b *testing.B) { 63 | benchWrite(b, 4096, true) 64 | } 65 | 66 | func BenchmarkWrite_10KB(b *testing.B) { 67 | benchWrite(b, 10240, true) 68 | } 69 | 70 | func BenchmarkRead__32B_NoCache(b *testing.B) { 71 | benchRead(b, 32, 1) 72 | } 73 | 74 | func BenchmarkRead__1KB_NoCache(b *testing.B) { 75 | benchRead(b, 1024, 1) 76 | } 77 | 78 | func BenchmarkRead__4KB_NoCache(b *testing.B) { 79 | benchRead(b, 4096, 1) 80 | } 81 | 82 | func BenchmarkRead_10KB_NoCache(b *testing.B) { 83 | benchRead(b, 10240, 1) 84 | } 85 | 86 | func BenchmarkRead__32B_WithCache(b *testing.B) { 87 | benchRead(b, 32, keyCount) 88 | } 89 | 90 | func BenchmarkRead__1KB_WithCache(b *testing.B) { 91 | benchRead(b, 1024, keyCount) 92 | } 93 | 94 | func BenchmarkRead__4KB_WithCache(b *testing.B) { 95 | benchRead(b, 4096, keyCount) 96 | } 97 | 98 | func BenchmarkRead_10KB_WithCache(b *testing.B) { 99 | benchRead(b, 10240, keyCount) 100 | } 101 | 102 | func shuffle(keys []string) { 103 | ints := rand.Perm(len(keys)) 104 | for i := range keys { 105 | keys[i], keys[ints[i]] = keys[ints[i]], keys[i] 106 | } 107 | } 108 | 109 | func genValue(size int) []byte { 110 | v := make([]byte, size) 111 | for i := 0; i < size; i++ { 112 | v[i] = uint8((rand.Int() % 26) + 97) // a-z 113 | } 114 | return v 115 | } 116 | 117 | const ( 118 | keyCount = 1000 119 | ) 120 | 121 | func genKeys() []string { 122 | keys := make([]string, keyCount) 123 | for i := 0; i < keyCount; i++ { 124 | keys[i] = fmt.Sprintf("%d", i) 125 | } 126 | return keys 127 | } 128 | 129 | func load(store Store, keys []string, val []byte) { 130 | for _, key := range keys { 131 | store.Set(key, &ByteItem{val}) 132 | } 133 | } 134 | 135 | func benchRead(b *testing.B, size, cachesz int) { 136 | b.StopTimer() 137 | store := NewLRU(cachesz, NewStore(StoreBenchPath)) 138 | keys := genKeys() 139 | value := genValue(size) 140 | load(store, keys, value) 141 | shuffle(keys) 142 | b.SetBytes(int64(size)) 143 | 144 | b.StartTimer() 145 | for i := 0; i < b.N; i++ { 146 | store.Get(keys[i%len(keys)], &ByteItem{}) 147 | } 148 | b.StopTimer() 149 | } 150 | 151 | func benchWrite(b *testing.B, size int, withIndex bool) { 152 | b.StopTimer() 153 | 154 | store := NewStore(StoreBenchPath) 155 | keys := genKeys() 156 | value := genValue(size) 157 | shuffle(keys) 158 | b.SetBytes(int64(size)) 159 | 160 | b.StartTimer() 161 | for i := 0; i < b.N; i++ { 162 | store.Set(keys[i%len(keys)], &ByteItem{value}) 163 | } 164 | b.StopTimer() 165 | } 166 | -------------------------------------------------------------------------------- /store_test.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | ) 7 | 8 | const StoreTestPath = "store-test" 9 | 10 | func TestStoreEmpty(t *testing.T) { 11 | store := NewStore(StoreTestPath) 12 | defer os.RemoveAll(StoreTestPath) 13 | if keys := store.List(""); len(keys) != 0 { 14 | t.Error(keys) 15 | } 16 | if item := store.Get("foo", &ByteItem{[]byte{}}); item != nil { 17 | t.Error(nil) 18 | } 19 | } 20 | 21 | func TestStoreSet(t *testing.T) { 22 | store := NewStore(StoreTestPath) 23 | defer os.RemoveAll(StoreTestPath) 24 | <-store.Set("foo", &ByteItem{[]byte("Hello")}) 25 | item := store.Get("foo", &ByteItem{}).(*ByteItem) 26 | if string(item.Value) != "Hello" { 27 | t.Error(item) 28 | } 29 | <-store.Set("foo", &ByteItem{[]byte("World")}) 30 | item = store.Get("foo", &ByteItem{}).(*ByteItem) 31 | if string(item.Value) != "World" { 32 | t.Error(item) 33 | } 34 | if keys := store.List(""); len(keys) != 1 || keys[0] != "foo" { 35 | t.Error(keys) 36 | } 37 | } 38 | 39 | func TestStoreDel(t *testing.T) { 40 | store := NewStore(StoreTestPath) 41 | defer os.RemoveAll(StoreTestPath) 42 | <-store.Set("foo", &ByteItem{[]byte("Hello")}) 43 | item := store.Get("foo", &ByteItem{}).(*ByteItem) 44 | if string(item.Value) != "Hello" { 45 | t.Error(item) 46 | } 47 | <-store.Set("foo", nil) 48 | if item := store.Get("foo", &ByteItem{}); item != nil { 49 | t.Error(item) 50 | } 51 | if err := <-store.Set("missing key", nil); err == nil { 52 | t.Error("missing key should return error on removal") 53 | } 54 | if item := store.Get("foo", &ByteItem{}); item != nil { 55 | t.Error(item) 56 | } 57 | if keys := store.List(""); len(keys) != 0 { 58 | t.Error(keys) 59 | } 60 | } 61 | 62 | func TestStoreList(t *testing.T) { 63 | store := NewStore(StoreTestPath) 64 | defer os.RemoveAll(StoreTestPath) 65 | store.Set("foo", &ByteItem{[]byte("Hello")}) 66 | store.Set("bar", &ByteItem{[]byte("World")}) 67 | store.Set("baz", &ByteItem{[]byte("!")}) 68 | <-store.Flush() 69 | if keys := store.List(""); len(keys) != 3 { 70 | t.Error(keys) 71 | } 72 | if keys := store.List("foo"); len(keys) != 1 { 73 | t.Error(keys) 74 | } 75 | if keys := store.List("ba"); len(keys) != 2 { 76 | t.Error(keys) 77 | } 78 | } 79 | 80 | func TestStoreError(t *testing.T) { 81 | store := NewStore(StoreTestPath) 82 | defer os.RemoveAll(StoreTestPath) 83 | if err := <-store.Set("foo", &ByteItem{[]byte("Hello")}); err != nil { 84 | t.Error(err) 85 | } 86 | if err := <-store.Set("", &ByteItem{[]byte("Hello")}); err == nil { 87 | t.Error("Error expected (is a directory), but nil received") 88 | } 89 | } 90 | 91 | func TestStoreFlush(t *testing.T) { 92 | store := NewStore(StoreTestPath) 93 | defer os.RemoveAll(StoreTestPath) 94 | store.Set("foo", &ByteItem{[]byte("Hello")}) 95 | store.Set("bar", &ByteItem{[]byte("World")}) 96 | store.Set("baz", &ByteItem{[]byte("!")}) 97 | <-store.Flush() 98 | <-store.Flush() 99 | <-store.Flush() 100 | } 101 | 102 | func TestLRUWithoutBackend(t *testing.T) { 103 | store := NewLRU(2, nil) 104 | store.Set("foo", &ByteItem{[]byte("Hello")}) 105 | store.Set("bar", &ByteItem{[]byte("World")}) 106 | store.Set("baz", &ByteItem{[]byte("!")}) 107 | item := store.Get("baz", &ByteItem{}).(*ByteItem) 108 | if string(item.Value) != "!" { 109 | t.Error(item.Value) 110 | } 111 | item = store.Get("bar", &ByteItem{}).(*ByteItem) 112 | if string(item.Value) != "World" { 113 | t.Error(item.Value) 114 | } 115 | // "foo" will be dropped out of Store 116 | if nilItem := store.Get("foo", &ByteItem{}); nilItem != nil { 117 | t.Error(nilItem) 118 | } 119 | store.Set("foo", &ByteItem{[]byte("Hello")}) 120 | item = store.Get("foo", &ByteItem{}).(*ByteItem) 121 | if string(item.Value) != "Hello" { 122 | t.Error(item.Value) 123 | } 124 | // now "baz" item will be dropped out and it's been least recently updated 125 | if nilItem := store.Get("baz", &ByteItem{}); nilItem != nil { 126 | t.Error(nilItem) 127 | } 128 | if keys := store.List(""); len(keys) != 2 { 129 | t.Error(keys) 130 | } 131 | if keys := store.List("b"); len(keys) != 1 || keys[0] != "bar" { 132 | t.Error(keys) 133 | } 134 | } 135 | 136 | func TestLRUWithBackend(t *testing.T) { 137 | dir := NewStore(StoreTestPath) 138 | defer os.RemoveAll(StoreTestPath) 139 | store := NewLRU(2, dir) 140 | store.Set("foo", &ByteItem{[]byte("Begin")}) 141 | store.Set("foo", &ByteItem{[]byte("Hello")}) 142 | store.Set("bar", &ByteItem{[]byte("World")}) 143 | 144 | // This kicks "foo" out of cache to the backend store, updated "foo" value 145 | // should be written to disk 146 | <-store.Set("baz", &ByteItem{[]byte("!")}) 147 | if keys := dir.List(""); len(keys) != 1 || keys[0] != "foo" { 148 | t.Error(keys) 149 | } 150 | 151 | // check locally cached values 152 | item := store.Get("baz", &ByteItem{}).(*ByteItem) 153 | if string(item.Value) != "!" { 154 | t.Error(item.Value) 155 | } 156 | item = store.Get("bar", &ByteItem{}).(*ByteItem) 157 | if string(item.Value) != "World" { 158 | t.Error(item.Value) 159 | } 160 | 161 | // "foo" is not in the cache, but will be read from the backend store anyway 162 | item = store.Get("foo", &ByteItem{}).(*ByteItem) 163 | if string(item.Value) != "Hello" { 164 | t.Error(item.Value) 165 | } 166 | 167 | // when "foo" was read "baz" has been dropped out, so backend store should 168 | // have 2 items: "foo" and "baz" 169 | item = dir.Get("foo", &ByteItem{}).(*ByteItem) 170 | if string(item.Value) != "Hello" { 171 | t.Error(item.Value) 172 | } 173 | item = dir.Get("baz", &ByteItem{}).(*ByteItem) 174 | if string(item.Value) != "!" { 175 | t.Error(item.Value) 176 | } 177 | 178 | if keys := dir.List(""); len(keys) != 2 { 179 | t.Error(keys) 180 | } 181 | 182 | // Sync all cached items to disk 183 | <-store.Flush() 184 | if keys := dir.List(""); len(keys) != 3 { 185 | t.Error(keys) 186 | } 187 | } 188 | 189 | func TestItemJSON(t *testing.T) { 190 | type jsonItem struct { 191 | Foo string `json:"foo"` 192 | Bar int `json:"bar"` 193 | } 194 | a := &jsonItem{"Hello", 1} 195 | 196 | defer os.RemoveAll(StoreTestPath) 197 | store := NewStore(StoreTestPath) 198 | <-store.Set("foo", &JSONItem{a}) 199 | 200 | byteItem := store.Get("foo", &ByteItem{}).(*ByteItem) 201 | if string(byteItem.Value) != `{"foo":"Hello","bar":1}`+"\n" { 202 | t.Error(string(byteItem.Value)) 203 | } 204 | b := store.Get("foo", &JSONItem{&jsonItem{}}).(*JSONItem).Value.(*jsonItem) 205 | if b.Foo != "Hello" || b.Bar != 1 { 206 | t.Error(b) 207 | } 208 | } 209 | 210 | func TestItemGob(t *testing.T) { 211 | a := []string{"a", "b"} 212 | 213 | defer os.RemoveAll(StoreTestPath) 214 | store := NewStore(StoreTestPath) 215 | <-store.Set("foo", &GobItem{a}) 216 | 217 | b := []string{} 218 | store.Get("foo", &GobItem{&b}) 219 | if len(b) != 2 || b[0] != "a" || b[1] != "b" { 220 | t.Error(b) 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package kv 2 | 3 | import ( 4 | "container/list" 5 | "io" 6 | "net/url" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "sync" 11 | "syscall" 12 | ) 13 | 14 | // Item is something that can be put into a Store. Items should be able to 15 | // read their values from io.Reader and write them into io.Writer. 16 | // Several helper implemenations are provided - for raw bytes, for JSON and for 17 | // gob format. 18 | type Item interface { 19 | io.ReaderFrom 20 | io.WriterTo 21 | } 22 | 23 | // 24 | // Store is the interface that wraps basic get/set functions for a simple 25 | // key-value storage. 26 | // 27 | // Store can be implemented as on-disk persistent storage, or as in-memory 28 | // cache. 29 | // 30 | // Get fulfils the item with the value associated with the key and returns the 31 | // item. Caller must provide the item instance beforehand, because the way how 32 | // item serializes/deserializes itself depends on its type. Get returns no 33 | // errors explicitly, but nil is returned if key is missing or any other I/O 34 | // error happended. Get is synchronous, your goroutine is blocked until item is 35 | // fully read from the store. 36 | // 37 | // Set writes item contents to the store at the given key. It's an asynchronous 38 | // operation, but caller can read from the returned channel to wait for write 39 | // completion and to get notified if I/O error occurred. If item is nil the key 40 | // is removed from the store 41 | // 42 | // List returns list of keys that exists and in the store and start with the 43 | // given prefix. If prefix is an empty string - all keys are returned. List 44 | // function is syncrhonous. 45 | // 46 | // Flush waits for all writing goroutines to finish and syncs all store data to 47 | // the disk. Flush can be called asynchronously, or the caller can wait for the 48 | // actual flush to happen by reading from the returned channel. 49 | type Store interface { 50 | Get(key string, item Item) Item 51 | Set(key string, item Item) <-chan error 52 | List(prefix string) []string 53 | Flush() <-chan error 54 | } 55 | 56 | // Store implementation that keeps each item in its own file in the given 57 | // directory 58 | type dirStore struct { 59 | mutex sync.RWMutex 60 | wg sync.WaitGroup 61 | path string 62 | } 63 | 64 | // Creates a new store from the given path. Keys are file names relative to the 65 | // path, values are file contents. 66 | func NewStore(path string) Store { 67 | return &dirStore{ 68 | path: path, 69 | } 70 | } 71 | 72 | func mkpath(root, s string) string { 73 | parts := strings.Split(s, "/") 74 | for i, s := range parts { 75 | parts[i] = url.QueryEscape(s) 76 | } 77 | return filepath.Join(root, filepath.Join(parts...)) 78 | } 79 | 80 | func (store *dirStore) Get(key string, item Item) Item { 81 | store.mutex.RLock() 82 | defer store.mutex.RUnlock() 83 | 84 | if f, err := os.Open(mkpath(store.path, key)); err == nil { 85 | defer f.Close() 86 | if _, err := item.ReadFrom(f); err == nil { 87 | return item 88 | } 89 | } 90 | return nil 91 | } 92 | 93 | func (store *dirStore) Set(key string, item Item) <-chan error { 94 | c := make(chan error, 1) 95 | 96 | store.mutex.Lock() 97 | defer store.mutex.Unlock() 98 | 99 | store.wg.Add(1) 100 | go func() { 101 | defer store.wg.Done() 102 | defer close(c) 103 | s := mkpath(store.path, key) 104 | if item == nil { 105 | if err := os.Remove(s); err != nil { 106 | c <- err 107 | } 108 | } else { 109 | os.MkdirAll(filepath.Dir(s), 0700) 110 | 111 | if f, err := os.OpenFile(s, os.O_WRONLY|os.O_CREATE, 0600); err != nil { 112 | c <- err 113 | } else { 114 | // FIXME make this atomic (using file move/rename) 115 | defer f.Close() 116 | item.WriteTo(f) 117 | } 118 | } 119 | }() 120 | return c 121 | } 122 | 123 | func (store *dirStore) Flush() <-chan error { 124 | // In disk store flush does not really return any errors because it can not 125 | // know which write goroutines are now running so it can't collect their 126 | // errors, also Sync() doesn't return errors either. So error channel is just 127 | // to meet the interface requirements and to wait for Flush() to complete. 128 | c := make(chan error) 129 | go func() { 130 | store.wg.Wait() 131 | syscall.Sync() 132 | close(c) 133 | }() 134 | return c 135 | } 136 | 137 | func (store *dirStore) List(prefix string) []string { 138 | store.mutex.RLock() 139 | defer store.mutex.RUnlock() 140 | files := []string{} 141 | if prefix == "" { 142 | prefix = "/" 143 | } 144 | // FIXME: should get rid of this hack and use filepath utils instead 145 | glob := mkpath(store.path, prefix+"x") 146 | glob = glob[:len(glob)-1] 147 | if matches, err := filepath.Glob(glob + "*"); err == nil { 148 | for _, file := range matches { 149 | if stat, err := os.Stat(file); err == nil && stat.Mode().IsRegular() { 150 | if s, err := url.QueryUnescape(file); err == nil { 151 | files = append(files, strings.TrimPrefix(s, store.path+"/")) 152 | } 153 | } 154 | } 155 | } 156 | return files 157 | } 158 | 159 | type lruItem struct { 160 | K string 161 | V Item 162 | } 163 | 164 | type lru struct { 165 | l *list.List 166 | m map[string]*list.Element 167 | mutex sync.Mutex 168 | size int 169 | backend Store 170 | } 171 | 172 | // Returns LRU cache which is backed up to some other store. 173 | func NewLRU(size int, backend Store) Store { 174 | return &lru{ 175 | l: list.New(), 176 | m: make(map[string]*list.Element), 177 | size: size, 178 | backend: backend, 179 | } 180 | } 181 | 182 | var noErrChan = func() <-chan error { 183 | c := make(chan error) 184 | close(c) 185 | return c 186 | }() 187 | 188 | func (store *lru) Get(key string, item Item) Item { 189 | store.mutex.Lock() 190 | defer store.mutex.Unlock() 191 | 192 | if el, ok := store.m[key]; ok { 193 | store.l.MoveToFront(el) 194 | return el.Value.(*lruItem).V 195 | } else if store.backend != nil { 196 | if item := store.backend.Get(key, item); item != nil { 197 | <-store.put(key, item, noErrChan) 198 | return item 199 | } 200 | } 201 | return nil 202 | } 203 | 204 | func (store *lru) put(key string, item Item, c <-chan error) <-chan error { 205 | if len(store.m) < store.size { 206 | store.m[key] = store.l.PushFront(&lruItem{key, item}) 207 | } else { 208 | el := store.l.Back() 209 | value := el.Value.(*lruItem) 210 | if store.backend != nil { 211 | c = store.backend.Set(value.K, value.V) 212 | } 213 | delete(store.m, value.K) 214 | el.Value = &lruItem{key, item} 215 | store.l.MoveToFront(el) 216 | store.m[key] = el 217 | } 218 | return c 219 | } 220 | 221 | func (store *lru) Set(key string, item Item) <-chan error { 222 | store.mutex.Lock() 223 | defer store.mutex.Unlock() 224 | 225 | c := noErrChan 226 | 227 | if el, ok := store.m[key]; ok { 228 | el.Value = &lruItem{key, item} 229 | store.l.MoveToFront(el) 230 | } else if item != nil { 231 | c = store.put(key, item, noErrChan) 232 | } 233 | return c 234 | } 235 | 236 | func (store *lru) List(prefix string) []string { 237 | store.mutex.Lock() 238 | defer store.mutex.Unlock() 239 | keys := []string{} 240 | for k, _ := range store.m { 241 | if strings.HasPrefix(k, prefix) { 242 | keys = append(keys, k) 243 | } 244 | } 245 | return keys 246 | } 247 | 248 | func (store *lru) Flush() <-chan error { 249 | c := make(chan error) 250 | if store.backend != nil { 251 | go func() { 252 | store.mutex.Lock() 253 | defer store.mutex.Unlock() 254 | for _, v := range store.m { 255 | pair := v.Value.(*lruItem) 256 | store.backend.Set(pair.K, pair.V) 257 | } 258 | if err, ok := <-store.backend.Flush(); ok { 259 | c <- err 260 | } 261 | close(c) 262 | }() 263 | } 264 | return c 265 | } 266 | --------------------------------------------------------------------------------