├── .drone.yml ├── .github └── FUNDING.yml ├── .gitignore ├── Benchmarks_test.go ├── Cluster_test.go ├── Collection.go ├── Configuration.go ├── KeyValue.go ├── LICENSE ├── Namespace.go ├── Namespace_test.go ├── Network.go ├── Node.go ├── Node_test.go ├── PacketType.go ├── README.md ├── README.src.md ├── Utils_test.go ├── go.mod └── go.sum /.drone.yml: -------------------------------------------------------------------------------- 1 | kind: pipeline 2 | name: default 3 | 4 | steps: 5 | - name: test 6 | image: golang 7 | environment: 8 | GO111MODULE: on 9 | commands: 10 | - go version 11 | - go get ./... 12 | - go build ./... 13 | - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.18.0 14 | - golangci-lint run --enable dupl --enable goconst --enable gocritic --enable misspell --enable prealloc --enable unconvert --enable unparam --disable typecheck 15 | - go test -v -race -coverprofile=coverage.txt ./... 16 | 17 | - name: coverage 18 | image: plugins/codecov 19 | settings: 20 | token: 21 | from_secret: codecov-token 22 | files: 23 | - coverage.txt 24 | 25 | - name: discord 26 | image: appleboy/drone-discord 27 | when: 28 | status: 29 | - failure 30 | settings: 31 | webhook_id: 32 | from_secret: discord-id 33 | webhook_token: 34 | from_secret: discord-token -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: akyoto 2 | patreon: eduardurbach -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.exe 2 | *.dll 3 | *.so 4 | *.dylib 5 | *.test 6 | *.out 7 | *.dat -------------------------------------------------------------------------------- /Benchmarks_test.go: -------------------------------------------------------------------------------- 1 | package nano_test 2 | 3 | import ( 4 | "strconv" 5 | "sync/atomic" 6 | "testing" 7 | "time" 8 | 9 | "github.com/aerogo/nano" 10 | ) 11 | 12 | func BenchmarkCollectionGet(b *testing.B) { 13 | node := nano.New(config) 14 | db := node.Namespace("test").RegisterTypes(types...) 15 | defer node.Close() 16 | defer node.Clear() 17 | 18 | users := db.Collection("User") 19 | users.Set("1", newUser(1)) 20 | 21 | b.ReportAllocs() 22 | b.ResetTimer() 23 | b.RunParallel(func(pb *testing.PB) { 24 | for pb.Next() { 25 | _, err := users.Get("1") 26 | 27 | if err != nil { 28 | b.Fail() 29 | } 30 | } 31 | }) 32 | b.StopTimer() 33 | } 34 | 35 | func BenchmarkCollectionSet(b *testing.B) { 36 | node := nano.New(config) 37 | db := node.Namespace("test").RegisterTypes(types...) 38 | defer node.Close() 39 | defer node.Clear() 40 | 41 | users := db.Collection("User") 42 | example := newUser(1) 43 | 44 | b.ReportAllocs() 45 | b.ResetTimer() 46 | b.RunParallel(func(pb *testing.PB) { 47 | for pb.Next() { 48 | users.Set("1", example) 49 | } 50 | }) 51 | b.StopTimer() 52 | } 53 | 54 | func BenchmarkCollectionDelete(b *testing.B) { 55 | node := nano.New(config) 56 | db := node.Namespace("test").RegisterTypes(types...) 57 | defer node.Close() 58 | defer node.Clear() 59 | 60 | users := db.Collection("User") 61 | 62 | for i := 0; i < 10000; i++ { 63 | users.Set(strconv.Itoa(i), newUser(i)) 64 | } 65 | 66 | b.ReportAllocs() 67 | b.ResetTimer() 68 | b.RunParallel(func(pb *testing.PB) { 69 | for pb.Next() { 70 | users.Delete("42") 71 | } 72 | }) 73 | b.StopTimer() 74 | } 75 | 76 | func BenchmarkCollectionAll(b *testing.B) { 77 | node := nano.New(config) 78 | db := node.Namespace("test").RegisterTypes(types...) 79 | defer node.Close() 80 | defer node.Clear() 81 | 82 | users := db.Collection("User") 83 | 84 | for i := 0; i < 1; i++ { 85 | users.Set(strconv.Itoa(i), newUser(i)) 86 | } 87 | 88 | b.ReportAllocs() 89 | b.ResetTimer() 90 | 91 | for i := 0; i < b.N; i++ { 92 | for range users.All() { 93 | // ... 94 | } 95 | } 96 | 97 | b.StopTimer() 98 | } 99 | 100 | func BenchmarkClusterGet(b *testing.B) { 101 | // Create cluster 102 | nodes := make([]*nano.Node, nodeCount) 103 | 104 | for i := 0; i < nodeCount; i++ { 105 | nodes[i] = nano.New(config) 106 | nodes[i].Namespace("test").RegisterTypes(types...) 107 | } 108 | 109 | // Wait for clients to connect 110 | for nodes[0].Server().ClientCount() < nodeCount-1 { 111 | time.Sleep(10 * time.Millisecond) 112 | } 113 | 114 | i := int64(0) 115 | 116 | // Run benchmark 117 | b.ReportAllocs() 118 | b.ResetTimer() 119 | b.RunParallel(func(pb *testing.PB) { 120 | for pb.Next() { 121 | atomic.AddInt64(&i, 1) 122 | id := int(atomic.LoadInt64(&i)) 123 | _, err := nodes[id%nodeCount].Namespace("test").Get("User", strconv.Itoa(id)) 124 | 125 | if err != nil { 126 | b.Fail() 127 | } 128 | } 129 | }) 130 | b.StopTimer() 131 | 132 | // Cleanup 133 | for i := nodeCount - 1; i >= 0; i-- { 134 | nodes[i].Clear() 135 | nodes[i].Close() 136 | } 137 | } 138 | 139 | func BenchmarkClusterSet(b *testing.B) { 140 | // Create cluster 141 | nodes := make([]*nano.Node, nodeCount) 142 | 143 | for i := 0; i < nodeCount; i++ { 144 | nodes[i] = nano.New(config) 145 | nodes[i].Namespace("test").RegisterTypes(types...) 146 | } 147 | 148 | // Wait for clients to connect 149 | for nodes[0].Server().ClientCount() < nodeCount-1 { 150 | time.Sleep(10 * time.Millisecond) 151 | } 152 | 153 | counter := int64(0) 154 | 155 | // Run benchmark 156 | b.ReportAllocs() 157 | b.ResetTimer() 158 | b.RunParallel(func(pb *testing.PB) { 159 | for pb.Next() { 160 | atomic.AddInt64(&counter, 1) 161 | id := int(atomic.LoadInt64(&counter)) 162 | nodes[id%nodeCount].Namespace("test").Set("User", strconv.Itoa(id), newUser(id)) 163 | } 164 | }) 165 | b.StopTimer() 166 | 167 | // Cleanup 168 | for i := nodeCount - 1; i >= 0; i-- { 169 | nodes[i].Clear() 170 | nodes[i].Close() 171 | } 172 | } 173 | 174 | func BenchmarkClusterDelete(b *testing.B) { 175 | // Create cluster 176 | nodes := make([]*nano.Node, nodeCount) 177 | 178 | for i := 0; i < nodeCount; i++ { 179 | nodes[i] = nano.New(config) 180 | nodes[i].Namespace("test").RegisterTypes(types...) 181 | } 182 | 183 | // Wait for clients to connect 184 | for nodes[0].Server().ClientCount() < nodeCount-1 { 185 | time.Sleep(10 * time.Millisecond) 186 | } 187 | 188 | counter := int64(0) 189 | 190 | // Run benchmark 191 | b.ReportAllocs() 192 | b.ResetTimer() 193 | b.RunParallel(func(pb *testing.PB) { 194 | for pb.Next() { 195 | atomic.AddInt64(&counter, 1) 196 | id := int(atomic.LoadInt64(&counter)) 197 | nodes[id%nodeCount].Namespace("test").Delete("User", strconv.Itoa(id)) 198 | } 199 | }) 200 | b.StopTimer() 201 | 202 | // Cleanup 203 | for i := nodeCount - 1; i >= 0; i-- { 204 | nodes[i].Clear() 205 | nodes[i].Close() 206 | } 207 | } 208 | -------------------------------------------------------------------------------- /Cluster_test.go: -------------------------------------------------------------------------------- 1 | package nano_test 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/aerogo/flow" 8 | "github.com/aerogo/nano" 9 | "github.com/akyoto/assert" 10 | ) 11 | 12 | const ( 13 | nodeCount = 4 14 | parallelRequestCount = 8 15 | ) 16 | 17 | func TestClusterClose(t *testing.T) { 18 | nodes := make([]*nano.Node, nodeCount) 19 | 20 | for i := 0; i < nodeCount; i++ { 21 | nodes[i] = nano.New(config) 22 | 23 | if i == 0 { 24 | assert.True(t, nodes[0].IsServer()) 25 | } else { 26 | assert.False(t, nodes[i].IsServer()) 27 | } 28 | } 29 | 30 | // Wait for clients to connect 31 | for nodes[0].Server().ClientCount() < nodeCount-1 { 32 | time.Sleep(10 * time.Millisecond) 33 | } 34 | 35 | // Close clients 36 | for i := nodeCount - 1; i >= 0; i-- { 37 | assert.False(t, nodes[i].IsClosed()) 38 | nodes[i].Close() 39 | assert.True(t, nodes[i].IsClosed()) 40 | } 41 | } 42 | 43 | func TestClusterReconnect(t *testing.T) { 44 | nodes := make([]*nano.Node, nodeCount) 45 | 46 | for i := 0; i < nodeCount; i++ { 47 | nodes[i] = nano.New(config) 48 | nodes[i].Namespace("test").RegisterTypes(types...) 49 | 50 | if i == 0 { 51 | assert.True(t, nodes[0].IsServer()) 52 | nodes[0].Namespace("test").Set("User", "1", newUser(1)) 53 | } else { 54 | assert.False(t, nodes[i].IsServer()) 55 | } 56 | } 57 | 58 | // Wait for clients to connect 59 | for nodes[0].Server().ClientCount() < nodeCount-1 { 60 | time.Sleep(10 * time.Millisecond) 61 | } 62 | 63 | nodes[2].Namespace("test").Set("User", "2", newUser(2)) 64 | assert.True(t, nodes[2].Namespace("test").Exists("User", "2")) 65 | 66 | // Close server only 67 | nodes[0].Close() 68 | 69 | // Wait a bit, to test some real downtime 70 | time.Sleep(1500 * time.Millisecond) 71 | 72 | // Restart server 73 | nodes[0] = nano.New(config) 74 | nodes[0].Namespace("test").RegisterTypes(types...) 75 | assert.True(t, nodes[0].IsServer()) 76 | 77 | // Wait for clients to reconnect in the span of a few seconds 78 | start := time.Now() 79 | for nodes[0].Server().ClientCount() < nodeCount-1 { 80 | time.Sleep(10 * time.Millisecond) 81 | 82 | if time.Since(start) > 2*time.Second { 83 | panic("Not enough clients reconnected") 84 | } 85 | } 86 | 87 | for i := 0; i < nodeCount; i++ { 88 | obj, err := nodes[i].Namespace("test").Get("User", "1") 89 | assert.Nil(t, err) 90 | assert.NotNil(t, obj) 91 | assert.Equal(t, "1", obj.(*User).ID) 92 | 93 | obj, err = nodes[i].Namespace("test").Get("User", "2") 94 | assert.Nil(t, err) 95 | assert.NotNil(t, obj) 96 | assert.Equal(t, "2", obj.(*User).ID) 97 | } 98 | 99 | for i := nodeCount - 1; i >= 0; i-- { 100 | nodes[i].Close() 101 | } 102 | } 103 | 104 | func TestClusterDataSharing(t *testing.T) { 105 | // Create cluster where the server has initial data 106 | nodes := make([]*nano.Node, nodeCount) 107 | 108 | for i := 0; i < nodeCount; i++ { 109 | nodes[i] = nano.New(config) 110 | nodes[i].Namespace("test").RegisterTypes(types...) 111 | 112 | if i == 0 { 113 | assert.True(t, nodes[0].IsServer()) 114 | nodes[0].Namespace("test").Set("User", "100", newUser(100)) 115 | } else { 116 | assert.False(t, nodes[i].IsServer()) 117 | } 118 | } 119 | 120 | // Wait for clients to connect 121 | for nodes[0].Server().ClientCount() < nodeCount-1 { 122 | time.Sleep(10 * time.Millisecond) 123 | } 124 | 125 | // Check data on client nodes 126 | for i := 1; i < nodeCount; i++ { 127 | flow.ParallelRepeat(parallelRequestCount, func() { 128 | user, err := nodes[i].Namespace("test").Get("User", "100") 129 | assert.Nil(t, err) 130 | assert.NotNil(t, user) 131 | }) 132 | } 133 | 134 | for i := nodeCount - 1; i >= 0; i-- { 135 | nodes[i].Clear() 136 | nodes[i].Close() 137 | } 138 | } 139 | 140 | func TestClusterSet(t *testing.T) { 141 | // Create cluster 142 | nodes := make([]*nano.Node, nodeCount) 143 | 144 | for i := 0; i < nodeCount; i++ { 145 | nodes[i] = nano.New(config) 146 | nodes[i].Namespace("test").RegisterTypes(types...) 147 | 148 | if i == 0 { 149 | assert.True(t, nodes[i].IsServer()) 150 | } else { 151 | assert.False(t, nodes[i].IsServer()) 152 | } 153 | } 154 | 155 | // Wait for clients to connect 156 | for nodes[0].Server().ClientCount() < nodeCount-1 { 157 | time.Sleep(10 * time.Millisecond) 158 | } 159 | 160 | // Make sure that node #0 does not have the record 161 | flow.ParallelRepeat(parallelRequestCount, func() { 162 | nodes[0].Namespace("test").Delete("User", "42") 163 | }) 164 | 165 | // Set record on node #1 166 | flow.ParallelRepeat(parallelRequestCount, func() { 167 | nodes[1].Namespace("test").Set("User", "42", newUser(42)) 168 | }) 169 | 170 | // Wait until it propagates through the whole cluster 171 | time.Sleep(300 * time.Millisecond) 172 | 173 | // Confirm that all nodes have the record now 174 | for i := 0; i < nodeCount; i++ { 175 | user, err := nodes[i].Namespace("test").Get("User", "42") 176 | assert.Nil(t, err) 177 | assert.NotNil(t, user) 178 | } 179 | 180 | for i := nodeCount - 1; i >= 0; i-- { 181 | nodes[i].Clear() 182 | nodes[i].Close() 183 | } 184 | } 185 | 186 | func TestClusterDelete(t *testing.T) { 187 | // Create cluster 188 | nodes := make([]*nano.Node, nodeCount) 189 | 190 | for i := 0; i < nodeCount; i++ { 191 | nodes[i] = nano.New(config) 192 | nodes[i].Namespace("test").RegisterTypes(types...) 193 | 194 | if i == 0 { 195 | assert.True(t, nodes[i].IsServer()) 196 | } else { 197 | assert.False(t, nodes[i].IsServer()) 198 | } 199 | } 200 | 201 | // Wait for clients to connect 202 | for nodes[0].Server().ClientCount() < nodeCount-1 { 203 | time.Sleep(10 * time.Millisecond) 204 | } 205 | 206 | // Set record on node #1 207 | nodes[1].Namespace("test").Set("User", "42", newUser(42)) 208 | 209 | // Wait until it propagates through the whole cluster 210 | time.Sleep(150 * time.Millisecond) 211 | 212 | // Confirm that all nodes have the record now 213 | for i := 0; i < nodeCount; i++ { 214 | user, err := nodes[i].Namespace("test").Get("User", "42") 215 | assert.Nil(t, err) 216 | assert.NotNil(t, user) 217 | } 218 | 219 | // Delete on all nodes 220 | flow.ParallelRepeat(parallelRequestCount, func() { 221 | nodes[2].Namespace("test").Delete("User", "42") 222 | }) 223 | 224 | // Wait until it propagates through the whole cluster 225 | time.Sleep(150 * time.Millisecond) 226 | 227 | // Confirm that all nodes deleted the record now 228 | for i := 0; i < nodeCount; i++ { 229 | exists := nodes[i].Namespace("test").Exists("User", "42") 230 | assert.False(t, exists) 231 | } 232 | 233 | for i := nodeCount - 1; i >= 0; i-- { 234 | nodes[i].Clear() 235 | nodes[i].Close() 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /Collection.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "io" 9 | "os" 10 | "path" 11 | "reflect" 12 | "runtime" 13 | "sort" 14 | "sync" 15 | "sync/atomic" 16 | "time" 17 | 18 | "github.com/aerogo/packet" 19 | jsoniter "github.com/json-iterator/go" 20 | ) 21 | 22 | // ChannelBufferSize is the size of the channels used to iterate over a whole collection. 23 | const ChannelBufferSize = 128 24 | 25 | // Collection is a hash map of data of the same type that is synchronized across network and disk. 26 | type Collection struct { 27 | data sync.Map 28 | lastModification sync.Map 29 | ns *Namespace 30 | node *Node 31 | name string 32 | dirty chan bool 33 | close chan bool 34 | loaded chan bool 35 | count int64 36 | fileMutex sync.Mutex 37 | typ reflect.Type 38 | } 39 | 40 | // newCollection creates a new collection in the namespace with the given name. 41 | func newCollection(ns *Namespace, name string) *Collection { 42 | collection := &Collection{ 43 | ns: ns, 44 | node: ns.node, 45 | name: name, 46 | dirty: make(chan bool, runtime.NumCPU()), 47 | close: make(chan bool), 48 | loaded: make(chan bool), 49 | } 50 | 51 | t, exists := collection.ns.types.Load(collection.name) 52 | 53 | if !exists { 54 | panic("Type " + collection.name + " has not been defined") 55 | } 56 | 57 | collection.typ = t.(reflect.Type) 58 | collection.load() 59 | 60 | return collection 61 | } 62 | 63 | // load loads all collection data 64 | func (collection *Collection) load() { 65 | if collection.node.IsServer() { 66 | // Server loads the collection from disk 67 | err := collection.loadFromDisk() 68 | 69 | if err != nil { 70 | panic(err) 71 | } 72 | 73 | // Indicate that collection is loaded 74 | close(collection.loaded) 75 | 76 | go func() { 77 | for { 78 | select { 79 | case <-collection.dirty: 80 | for len(collection.dirty) > 0 { 81 | <-collection.dirty 82 | } 83 | 84 | err := collection.flush() 85 | 86 | if err != nil { 87 | fmt.Println("Error writing collection", collection.name, "to disk", err) 88 | } 89 | 90 | time.Sleep(collection.node.ioSleepTime) 91 | 92 | case <-collection.close: 93 | if len(collection.dirty) > 0 { 94 | err := collection.flush() 95 | 96 | if err != nil { 97 | fmt.Println("Error writing collection", collection.name, "to disk", err) 98 | } 99 | } 100 | 101 | close(collection.close) 102 | return 103 | } 104 | } 105 | }() 106 | } else { 107 | // Client asks the server to send the most recent collection data 108 | collection.ns.collectionsLoading.Store(collection.name, collection) 109 | packetData := bytes.Buffer{} 110 | fmt.Fprintf(&packetData, "%s\n%s\n", collection.ns.name, collection.name) 111 | collection.node.Client().Stream.Outgoing <- packet.New(packetCollectionRequest, packetData.Bytes()) 112 | <-collection.loaded 113 | } 114 | } 115 | 116 | // Get returns the value for the given key. 117 | func (collection *Collection) Get(key string) (interface{}, error) { 118 | val, ok := collection.data.Load(key) 119 | 120 | if !ok { 121 | return val, errors.New("Key not found: " + key) 122 | } 123 | 124 | return val, nil 125 | } 126 | 127 | // GetMany is the same as Get, except it looks up multiple keys at once. 128 | func (collection *Collection) GetMany(keys []string) []interface{} { 129 | values := make([]interface{}, len(keys)) 130 | 131 | for i := 0; i < len(keys); i++ { 132 | values[i], _ = collection.Get(keys[i]) 133 | } 134 | 135 | return values 136 | } 137 | 138 | // set is the internally used function to store a value for a key. 139 | func (collection *Collection) set(key string, value interface{}) { 140 | collection.data.Store(key, value) 141 | 142 | if collection.node.IsServer() && len(collection.dirty) == 0 { 143 | collection.dirty <- true 144 | } 145 | } 146 | 147 | // Set sets the value for the key. 148 | func (collection *Collection) Set(key string, value interface{}) { 149 | if value == nil { 150 | return 151 | } 152 | 153 | if collection.node.broadcastRequired() { 154 | // It's important to store the timestamp BEFORE the actual collection.set 155 | collection.lastModification.Store(key, time.Now().UnixNano()) 156 | 157 | // Serialize the value into JSON format 158 | jsonBytes, err := jsoniter.Marshal(value) 159 | 160 | if err != nil { 161 | panic(err) 162 | } 163 | 164 | // Create a network packet for the "set" command 165 | buffer := bytes.Buffer{} 166 | buffer.Write(packet.Int64ToBytes(time.Now().UnixNano())) 167 | buffer.WriteString(collection.ns.name) 168 | buffer.WriteByte('\n') 169 | buffer.WriteString(collection.name) 170 | buffer.WriteByte('\n') 171 | buffer.WriteString(key) 172 | buffer.WriteByte('\n') 173 | buffer.Write(jsonBytes) 174 | buffer.WriteByte('\n') 175 | 176 | msg := packet.New(packetSet, buffer.Bytes()) 177 | collection.node.Broadcast(msg) 178 | } 179 | 180 | collection.set(key, value) 181 | } 182 | 183 | // delete is the internally used command to delete a key. 184 | func (collection *Collection) delete(key string) { 185 | collection.data.Delete(key) 186 | 187 | if collection.node.IsServer() && len(collection.dirty) == 0 { 188 | collection.dirty <- true 189 | } 190 | } 191 | 192 | // Delete deletes a key from the collection. 193 | func (collection *Collection) Delete(key string) bool { 194 | if collection.node.broadcastRequired() { 195 | // It's important to store the timestamp BEFORE the actual collection.delete 196 | collection.lastModification.Store(key, time.Now().UnixNano()) 197 | 198 | buffer := bytes.Buffer{} 199 | buffer.Write(packet.Int64ToBytes(time.Now().UnixNano())) 200 | buffer.WriteString(collection.ns.name) 201 | buffer.WriteByte('\n') 202 | buffer.WriteString(collection.name) 203 | buffer.WriteByte('\n') 204 | buffer.WriteString(key) 205 | buffer.WriteByte('\n') 206 | 207 | msg := packet.New(packetDelete, buffer.Bytes()) 208 | collection.node.Broadcast(msg) 209 | } 210 | 211 | _, exists := collection.data.Load(key) 212 | collection.delete(key) 213 | 214 | return exists 215 | } 216 | 217 | // Clear deletes all objects from the collection. 218 | func (collection *Collection) Clear() { 219 | collection.data.Range(func(key, value interface{}) bool { 220 | collection.data.Delete(key) 221 | return true 222 | }) 223 | 224 | runtime.GC() 225 | 226 | if len(collection.dirty) == 0 { 227 | collection.dirty <- true 228 | } 229 | 230 | atomic.StoreInt64(&collection.count, 0) 231 | } 232 | 233 | // Exists returns whether or not the key exists. 234 | func (collection *Collection) Exists(key string) bool { 235 | _, exists := collection.data.Load(key) 236 | return exists 237 | } 238 | 239 | // All returns a channel of all objects in the collection. 240 | func (collection *Collection) All() chan interface{} { 241 | channel := make(chan interface{}, ChannelBufferSize) 242 | 243 | go func() { 244 | collection.data.Range(func(key, value interface{}) bool { 245 | channel <- value 246 | return true 247 | }) 248 | 249 | close(channel) 250 | }() 251 | 252 | return channel 253 | } 254 | 255 | // Count gives you a rough estimate of how many elements are in the collection. 256 | // It DOES NOT GUARANTEE that the returned number is the actual number of elements. 257 | // A good use for this function is to preallocate slices with the given capacity. 258 | // In the future, this function could possibly return the exact number of elements. 259 | func (collection *Collection) Count() int64 { 260 | return atomic.LoadInt64(&collection.count) 261 | } 262 | 263 | // flush writes all data to the file system. 264 | func (collection *Collection) flush() error { 265 | collection.fileMutex.Lock() 266 | defer collection.fileMutex.Unlock() 267 | 268 | newFilePath := path.Join(collection.ns.root, collection.name+".new") 269 | oldFilePath := path.Join(collection.ns.root, collection.name+".dat") 270 | tmpFilePath := path.Join(collection.ns.root, collection.name+".tmp") 271 | 272 | file, err := os.OpenFile(newFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) 273 | 274 | if err != nil { 275 | return err 276 | } 277 | 278 | bufferedWriter := bufio.NewWriter(file) 279 | err = collection.writeRecords(bufferedWriter, true) 280 | 281 | if err != nil { 282 | return err 283 | } 284 | 285 | err = bufferedWriter.Flush() 286 | 287 | if err != nil { 288 | return err 289 | } 290 | 291 | err = file.Sync() 292 | 293 | if err != nil { 294 | return err 295 | } 296 | 297 | err = file.Close() 298 | 299 | if err != nil { 300 | return err 301 | } 302 | 303 | // Swap .dat and .new files 304 | err = os.Rename(oldFilePath, tmpFilePath) 305 | 306 | if err != nil && !os.IsNotExist(err) { 307 | return err 308 | } 309 | 310 | err = os.Rename(newFilePath, oldFilePath) 311 | 312 | if err != nil { 313 | return err 314 | } 315 | 316 | err = os.Remove(tmpFilePath) 317 | 318 | if err != nil && !os.IsNotExist(err) { 319 | return err 320 | } 321 | 322 | return nil 323 | } 324 | 325 | // writeRecords writes the entire collection to the IO writer. 326 | func (collection *Collection) writeRecords(writer io.Writer, sorted bool) error { 327 | records := []keyValue{} 328 | stringWriter, ok := writer.(io.StringWriter) 329 | 330 | if !ok { 331 | return errors.New("The given io.Writer is not an io.StringWriter") 332 | } 333 | 334 | collection.data.Range(func(key, value interface{}) bool { 335 | records = append(records, keyValue{ 336 | key: key.(string), 337 | value: value, 338 | }) 339 | return true 340 | }) 341 | 342 | if sorted { 343 | sort.Slice(records, func(i, j int) bool { 344 | return records[i].key < records[j].key 345 | }) 346 | } 347 | 348 | atomic.StoreInt64(&collection.count, int64(len(records))) 349 | encoder := jsoniter.NewEncoder(writer) 350 | 351 | for _, record := range records { 352 | // Key in the first line 353 | _, err := stringWriter.WriteString(record.key) 354 | 355 | if err != nil { 356 | return err 357 | } 358 | 359 | _, err = stringWriter.WriteString("\n") 360 | 361 | if err != nil { 362 | return err 363 | } 364 | 365 | // Value in the second line 366 | err = encoder.Encode(record.value) 367 | 368 | if err != nil { 369 | return err 370 | } 371 | } 372 | 373 | return nil 374 | } 375 | 376 | // loadFromDisk loads the entire collection from disk. 377 | func (collection *Collection) loadFromDisk() error { 378 | filePath := path.Join(collection.ns.root, collection.name+".dat") 379 | stream, err := os.OpenFile(filePath, os.O_RDONLY|os.O_SYNC, 0644) 380 | 381 | if os.IsNotExist(err) { 382 | return nil 383 | } 384 | 385 | if err != nil { 386 | return err 387 | } 388 | 389 | return collection.readRecords(stream) 390 | } 391 | 392 | // readRecords reads the entire collection from an IO reader. 393 | func (collection *Collection) readRecords(stream io.Reader) error { 394 | var key string 395 | var value []byte 396 | 397 | reader := bufio.NewReader(stream) 398 | lineCount := 0 399 | 400 | defer func() { 401 | atomic.StoreInt64(&collection.count, int64(lineCount/2)) 402 | }() 403 | 404 | for { 405 | line, err := reader.ReadBytes('\n') 406 | 407 | if err == io.EOF { 408 | return nil 409 | } 410 | 411 | if err != nil { 412 | return err 413 | } 414 | 415 | // Remove delimiter 416 | if len(line) > 0 && line[len(line)-1] == '\n' { 417 | line = line[:len(line)-1] 418 | } 419 | 420 | if lineCount%2 == 0 { 421 | key = string(line) 422 | } else { 423 | value = line 424 | v := reflect.New(collection.typ) 425 | obj := v.Interface() 426 | err = jsoniter.Unmarshal(value, &obj) 427 | 428 | if err != nil { 429 | return err 430 | } 431 | 432 | collection.data.Store(key, obj) 433 | } 434 | 435 | lineCount++ 436 | } 437 | } 438 | -------------------------------------------------------------------------------- /Configuration.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | // Configuration represents the nano configuration 4 | // which is only read once at node creation time. 5 | type Configuration struct { 6 | // Port is the port used by the server and client nodes. 7 | Port int 8 | 9 | // Directory includes the path to the namespaces stored on the disk. 10 | Directory string 11 | 12 | // Hosts represents a list of node addresses that this node should connect to. 13 | Hosts []string 14 | } 15 | -------------------------------------------------------------------------------- /KeyValue.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | type keyValue struct { 4 | key string 5 | value interface{} 6 | } 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Sponsorship License 2 | 3 | Copyright (c) 2019 Eduard Urbach 4 | 5 | * You are permitted to use the software freely for non-commercial purposes. 6 | * You MUST be a sponsor donating at least 10 USD every month to me if you use the software for commercial purposes. 7 | 8 | To become a sponsor: 9 | 10 | 1. Visit the following page: https://github.com/users/akyoto/sponsorship 11 | 2. Register as a sponsor for the 10 USD per month tier. 12 | 13 | Additional notes: 14 | 15 | * Canceling your sponsorship will also revoke the rights to use the software for commercial purposes. 16 | * The software is provided "as is", without warranty of any kind. 17 | * The above copyright notice and this permission must be included in all copies of the software. 18 | 19 | -------------------------------------------------------------------------------- /Namespace.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | import ( 4 | "os" 5 | "path" 6 | "reflect" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | // Namespace combines multiple collections under a single name. 12 | type Namespace struct { 13 | collections sync.Map 14 | collectionsLoading sync.Map 15 | name string 16 | root string 17 | types sync.Map 18 | node *Node 19 | } 20 | 21 | // newNamespace is the internal function used to create a new namespace. 22 | func newNamespace(node *Node, name string) *Namespace { 23 | // Create namespace 24 | namespace := &Namespace{ 25 | node: node, 26 | name: name, 27 | root: path.Join(node.config.Directory, name), 28 | } 29 | 30 | // Create directory 31 | err := os.MkdirAll(namespace.root, 0777) 32 | 33 | if err != nil { 34 | panic(err) 35 | } 36 | 37 | return namespace 38 | } 39 | 40 | // RegisterTypes expects a list of pointers and will look up the types 41 | // of the given pointers. These types will be registered so that collections 42 | // can store data using the given type. Note that nil pointers are acceptable. 43 | func (ns *Namespace) RegisterTypes(types ...interface{}) *Namespace { 44 | // Convert example objects to their respective types 45 | for _, example := range types { 46 | typeInfo := reflect.TypeOf(example) 47 | 48 | if typeInfo.Kind() == reflect.Ptr { 49 | typeInfo = typeInfo.Elem() 50 | } 51 | 52 | ns.types.Store(typeInfo.Name(), typeInfo) 53 | } 54 | 55 | return ns 56 | } 57 | 58 | // Collection returns the collection with the given name. 59 | func (ns *Namespace) Collection(name string) *Collection { 60 | obj, loaded := ns.collections.LoadOrStore(name, nil) 61 | 62 | if !loaded { 63 | collection := newCollection(ns, name) 64 | ns.collections.Store(name, collection) 65 | return collection 66 | } 67 | 68 | // Wait for existing collection load 69 | for obj == nil { 70 | time.Sleep(1 * time.Millisecond) 71 | obj, _ = ns.collections.Load(name) 72 | } 73 | 74 | return obj.(*Collection) 75 | } 76 | 77 | // collectionLoading returns the collection that is currently being loaded. 78 | func (ns *Namespace) collectionLoading(name string) *Collection { 79 | obj, _ := ns.collectionsLoading.Load(name) 80 | return obj.(*Collection) 81 | } 82 | 83 | // Get returns the value for the given key. 84 | func (ns *Namespace) Get(collection string, key string) (interface{}, error) { 85 | return ns.Collection(collection).Get(key) 86 | } 87 | 88 | // GetMany is the same as Get, except it looks up multiple keys at once. 89 | func (ns *Namespace) GetMany(collection string, keys []string) []interface{} { 90 | return ns.Collection(collection).GetMany(keys) 91 | } 92 | 93 | // Set sets the value for the key. 94 | func (ns *Namespace) Set(collection string, key string, value interface{}) { 95 | ns.Collection(collection).Set(key, value) 96 | } 97 | 98 | // Delete deletes a key from the collection. 99 | func (ns *Namespace) Delete(collection string, key string) bool { 100 | return ns.Collection(collection).Delete(key) 101 | } 102 | 103 | // Exists returns whether or not the key exists. 104 | func (ns *Namespace) Exists(collection string, key string) bool { 105 | return ns.Collection(collection).Exists(key) 106 | } 107 | 108 | // All returns a channel of all objects in the collection. 109 | func (ns *Namespace) All(name string) chan interface{} { 110 | return ns.Collection(name).All() 111 | } 112 | 113 | // Clear deletes all objects from the collection. 114 | func (ns *Namespace) Clear(collection string) { 115 | ns.Collection(collection).Clear() 116 | } 117 | 118 | // ClearAll deletes all objects from all collections, 119 | // effectively resetting the entire database. 120 | func (ns *Namespace) ClearAll() { 121 | ns.collections.Range(func(key, value interface{}) bool { 122 | if value == nil { 123 | return true 124 | } 125 | 126 | collection := value.(*Collection) 127 | collection.Clear() 128 | 129 | return true 130 | }) 131 | } 132 | 133 | // Types returns a map of type names mapped to their reflection type. 134 | func (ns *Namespace) Types() map[string]reflect.Type { 135 | copied := make(map[string]reflect.Type) 136 | 137 | ns.types.Range(func(key, value interface{}) bool { 138 | copied[key.(string)] = value.(reflect.Type) 139 | return true 140 | }) 141 | 142 | return copied 143 | } 144 | 145 | // HasType returns true if the given type name has been registered. 146 | func (ns *Namespace) HasType(typeName string) bool { 147 | _, exists := ns.types.Load(typeName) 148 | return exists 149 | } 150 | 151 | // Node returns the cluster node used for this namespace. 152 | func (ns *Namespace) Node() *Node { 153 | return ns.node 154 | } 155 | 156 | // Close will close all collections in the namespace, 157 | // forcing them to sync all data to disk before shutting down. 158 | func (ns *Namespace) Close() { 159 | if !ns.node.node.IsServer() { 160 | return 161 | } 162 | 163 | ns.collections.Range(func(key, value interface{}) bool { 164 | collection := value.(*Collection) 165 | 166 | // Stop writing 167 | collection.close <- true 168 | <-collection.close 169 | 170 | return true 171 | }) 172 | } 173 | 174 | // Prefetch loads all the data for this namespace from disk into memory. 175 | func (ns *Namespace) Prefetch() { 176 | wg := sync.WaitGroup{} 177 | 178 | ns.types.Range(func(key, value interface{}) bool { 179 | typeName := key.(string) 180 | wg.Add(1) 181 | 182 | go func(name string) { 183 | ns.Collection(name) 184 | wg.Done() 185 | }(typeName) 186 | 187 | return true 188 | }) 189 | 190 | wg.Wait() 191 | } 192 | -------------------------------------------------------------------------------- /Namespace_test.go: -------------------------------------------------------------------------------- 1 | package nano_test 2 | 3 | import ( 4 | "reflect" 5 | "strconv" 6 | "testing" 7 | 8 | "github.com/aerogo/nano" 9 | "github.com/akyoto/assert" 10 | ) 11 | 12 | func TestNamespaceGet(t *testing.T) { 13 | node := nano.New(config) 14 | defer node.Close() 15 | defer node.Clear() 16 | 17 | db := node.Namespace("test").RegisterTypes(types...) 18 | assert.True(t, node.IsServer()) 19 | 20 | db.Set("User", "1", newUser(1)) 21 | db.Set("User", "2", newUser(2)) 22 | 23 | val, err := db.Get("User", "1") 24 | assert.Nil(t, err) 25 | 26 | user, ok := val.(*User) 27 | assert.True(t, ok) 28 | assert.Equal(t, "Test User", user.Name) 29 | 30 | assert.NotNil(t, val) 31 | } 32 | 33 | func TestNamespaceGetMany(t *testing.T) { 34 | node := nano.New(config) 35 | defer node.Close() 36 | defer node.Clear() 37 | 38 | db := node.Namespace("test").RegisterTypes(types...) 39 | assert.True(t, node.IsServer()) 40 | 41 | db.Set("User", "1", newUser(1)) 42 | db.Set("User", "2", newUser(2)) 43 | 44 | objects := db.GetMany("User", []string{ 45 | "1", 46 | "2", 47 | }) 48 | 49 | assert.Equal(t, len(objects), 2) 50 | 51 | for _, object := range objects { 52 | user, ok := object.(*User) 53 | assert.True(t, ok) 54 | assert.Equal(t, "Test User", user.Name) 55 | } 56 | } 57 | 58 | func TestNamespaceSet(t *testing.T) { 59 | node := nano.New(config) 60 | defer node.Close() 61 | defer node.Clear() 62 | 63 | db := node.Namespace("test").RegisterTypes(types...) 64 | assert.True(t, node.IsServer()) 65 | 66 | db.Set("User", "1", newUser(1)) 67 | db.Delete("User", "2") 68 | 69 | assert.True(t, db.Exists("User", "1")) 70 | assert.False(t, db.Exists("User", "2")) 71 | } 72 | 73 | func TestNamespaceClear(t *testing.T) { 74 | node := nano.New(config) 75 | defer node.Close() 76 | defer node.Clear() 77 | 78 | db := node.Namespace("test").RegisterTypes(types...) 79 | assert.True(t, node.IsServer()) 80 | 81 | db.Set("User", "1", newUser(1)) 82 | db.Set("User", "2", newUser(2)) 83 | db.Set("User", "3", newUser(3)) 84 | 85 | assert.True(t, db.Exists("User", "1")) 86 | assert.True(t, db.Exists("User", "2")) 87 | assert.True(t, db.Exists("User", "3")) 88 | 89 | db.Clear("User") 90 | 91 | assert.False(t, db.Exists("User", "1")) 92 | assert.False(t, db.Exists("User", "2")) 93 | assert.False(t, db.Exists("User", "3")) 94 | } 95 | 96 | func TestNamespaceAll(t *testing.T) { 97 | node := nano.New(config) 98 | db := node.Namespace("test").RegisterTypes(types...) 99 | assert.True(t, node.IsServer()) 100 | defer node.Close() 101 | defer node.Clear() 102 | 103 | db.Collection("User").Clear() 104 | recordCount := 10000 105 | 106 | for i := 0; i < recordCount; i++ { 107 | db.Set("User", strconv.Itoa(i), newUser(i)) 108 | } 109 | 110 | count := 0 111 | 112 | for user := range db.All("User") { 113 | assert.NotNil(t, user) 114 | count++ 115 | } 116 | 117 | assert.Equal(t, recordCount, count) 118 | assert.True(t, db.Collection("User").Count() >= 0) 119 | } 120 | 121 | func TestNamespaceClose(t *testing.T) { 122 | node := nano.New(config) 123 | 124 | assert.True(t, node.IsServer()) 125 | assert.False(t, node.IsClosed()) 126 | 127 | node.Close() 128 | 129 | assert.True(t, node.IsClosed()) 130 | } 131 | 132 | func TestNamespaceTypes(t *testing.T) { 133 | node := nano.New(config) 134 | defer node.Close() 135 | 136 | db := node.Namespace("test").RegisterTypes(types...) 137 | assert.Equal(t, reflect.TypeOf(User{}), db.Types()["User"]) 138 | } 139 | 140 | func TestNamespacePrefetch(t *testing.T) { 141 | node := nano.New(config) 142 | defer node.Close() 143 | 144 | db := node.Namespace("test").RegisterTypes(types...) 145 | db.Prefetch() 146 | } 147 | 148 | func TestNamespaceNode(t *testing.T) { 149 | node := nano.New(config) 150 | defer node.Close() 151 | 152 | db := node.Namespace("test").RegisterTypes(types...) 153 | assert.Equal(t, db.Node(), node) 154 | } 155 | 156 | // func TestNamespaceColdStart(t *testing.T) { 157 | // time.Sleep(500 * time.Millisecond) 158 | // db := nano.New(config).Namespace("test").RegisterTypes(types...) 159 | // assert.True(t, node.IsServer()) 160 | 161 | // for i := 0; i < 10000; i++ { 162 | // db.Set("User", strconv.Itoa(i), newUser(i)) 163 | // assert.True(t, db.Exists("User", strconv.Itoa(i))) 164 | // } 165 | 166 | // db.Close() 167 | 168 | // // Sync filesystem 169 | // exec.Command("sync").Run() 170 | 171 | // // Wait a little 172 | // time.Sleep(2000 * time.Millisecond) 173 | 174 | // // Cold start 175 | // newDB := nano.New(config).Namespace("test").RegisterTypes(types...) 176 | // assert.True(t, newnode.IsServer()) 177 | 178 | // defer newDB.Close() 179 | // defer newDB.ClearAll() 180 | 181 | // for i := 0; i < 10000; i++ { 182 | // if !newDB.Exists("User", strconv.Itoa(i)) { 183 | // assert.FailNow(t, fmt.Sprintf("User %d does not exist after cold start", i)) 184 | // } 185 | // } 186 | // } 187 | -------------------------------------------------------------------------------- /Network.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "errors" 7 | "fmt" 8 | "reflect" 9 | "strings" 10 | 11 | "github.com/aerogo/cluster/client" 12 | "github.com/aerogo/cluster/server" 13 | "github.com/aerogo/packet" 14 | jsoniter "github.com/json-iterator/go" 15 | ) 16 | 17 | // serverReadPacketsFromClient reads packets from clients on the server side. 18 | func serverReadPacketsFromClient(client *packet.Stream, node *Node) { 19 | for msg := range client.Incoming { 20 | switch msg.Type { 21 | case packetCollectionRequest: 22 | data := bytes.NewBuffer(msg.Data) 23 | 24 | namespaceName, _ := data.ReadString('\n') 25 | namespaceName = strings.TrimSuffix(namespaceName, "\n") 26 | 27 | namespace := node.Namespace(namespaceName) 28 | 29 | collectionName, _ := data.ReadString('\n') 30 | collectionName = strings.TrimSuffix(collectionName, "\n") 31 | 32 | if node.verbose { 33 | fmt.Println("COLLECTION REQUEST", client.Connection().RemoteAddr(), namespaceName+"."+collectionName) 34 | } 35 | 36 | collection := namespace.Collection(collectionName) 37 | buffer := bytes.Buffer{} 38 | 39 | buffer.WriteString(namespace.name) 40 | buffer.WriteByte('\n') 41 | 42 | buffer.WriteString(collection.name) 43 | buffer.WriteByte('\n') 44 | 45 | writer := bufio.NewWriter(&buffer) 46 | err := collection.writeRecords(writer, false) 47 | 48 | if err != nil { 49 | fmt.Println("Error answering collection request:", err) 50 | continue 51 | } 52 | 53 | err = writer.Flush() 54 | 55 | if err != nil { 56 | fmt.Println("Error calling writer.Flush() on collection request:", err) 57 | continue 58 | } 59 | 60 | client.Outgoing <- packet.New(packetCollectionResponse, buffer.Bytes()) 61 | 62 | if node.verbose { 63 | fmt.Println("COLLECTION REQUEST ANSWERED", client.Connection().RemoteAddr()) 64 | } 65 | 66 | case packetSet: 67 | if networkSet(msg, node) == nil { 68 | serverForwardPacket(node.Server(), client, msg) 69 | } 70 | 71 | case packetDelete: 72 | if networkDelete(msg, node) == nil { 73 | serverForwardPacket(node.Server(), client, msg) 74 | } 75 | 76 | default: 77 | fmt.Printf("Error: Unknown network packet type %d of length %d\n", msg.Type, msg.Length) 78 | } 79 | } 80 | 81 | if node.verbose { 82 | fmt.Println("[server] Client disconnected", client.Connection().RemoteAddr()) 83 | } 84 | } 85 | 86 | // clientReadPacketsFromServer reads packets from the server on the client side. 87 | func clientReadPacketsFromServer(client *client.Node, node *Node) { 88 | for msg := range client.Stream.Incoming { 89 | switch msg.Type { 90 | case packetCollectionResponse: 91 | data := bytes.NewBuffer(msg.Data) 92 | 93 | namespaceName, _ := data.ReadString('\n') 94 | namespaceName = strings.TrimSuffix(namespaceName, "\n") 95 | 96 | namespace := node.Namespace(namespaceName) 97 | 98 | collectionName, _ := data.ReadString('\n') 99 | collectionName = strings.TrimSuffix(collectionName, "\n") 100 | 101 | if node.verbose { 102 | fmt.Println("COLLECTION RESPONSE RECEIVED", client.Address(), namespaceName+"."+collectionName) 103 | } 104 | 105 | collection := namespace.collectionLoading(collectionName) 106 | err := collection.readRecords(data) 107 | 108 | if err != nil { 109 | panic(err) 110 | } 111 | 112 | namespace.collectionsLoading.Delete(collectionName) 113 | close(collection.loaded) 114 | 115 | case packetSet, packetDelete: 116 | node.networkWorkerQueue <- msg 117 | 118 | case packetServerClose: 119 | if node.verbose { 120 | fmt.Println("[client] Server closed!", client.Address()) 121 | } 122 | 123 | client.Close() 124 | 125 | if node.verbose { 126 | fmt.Println("[client] Reconnecting", client.Address()) 127 | } 128 | 129 | err := client.Connect() 130 | 131 | if err != nil { 132 | fmt.Println("Error re-connecting to server:", err.Error()) 133 | } 134 | 135 | if node.verbose { 136 | fmt.Println("[client] Reconnect finished!", client.Address()) 137 | } 138 | 139 | default: 140 | fmt.Printf("Error: Unknown network packet type %d of length %d\n", msg.Type, msg.Length) 141 | } 142 | } 143 | 144 | close(node.networkWorkerQueue) 145 | 146 | if node.verbose { 147 | fmt.Println(client.Address(), "clientReadPacketsFromServer goroutine stopped") 148 | } 149 | } 150 | 151 | // clientNetworkWorker runs in a separate goroutine and handles the set & delete packets. 152 | func clientNetworkWorker(node *Node) { 153 | for msg := range node.networkWorkerQueue { 154 | switch msg.Type { 155 | case packetSet: 156 | err := networkSet(msg, node) 157 | 158 | if err != nil { 159 | fmt.Printf("nano: networkSet failed: %s\n", err.Error()) 160 | } 161 | 162 | case packetDelete: 163 | err := networkDelete(msg, node) 164 | 165 | if err != nil { 166 | fmt.Printf("nano: networkDelete failed: %s\n", err.Error()) 167 | } 168 | } 169 | } 170 | } 171 | 172 | // networkSet performs a set operation based on the information in the network packet. 173 | func networkSet(msg *packet.Packet, db *Node) error { 174 | data := bytes.NewBuffer(msg.Data) 175 | 176 | packetTimeBuffer := make([]byte, 8) 177 | _, err := data.Read(packetTimeBuffer) 178 | 179 | if err != nil { 180 | return err 181 | } 182 | 183 | packetTime, err := packet.Int64FromBytes(packetTimeBuffer) 184 | 185 | if err != nil { 186 | return err 187 | } 188 | 189 | namespaceName := readLine(data) 190 | namespace := db.Namespace(namespaceName) 191 | 192 | collectionName := readLine(data) 193 | collectionObj, exists := namespace.collections.Load(collectionName) 194 | 195 | if !exists || collectionObj == nil { 196 | return nil //errors.New("Received networkSet command on non-existing collection") 197 | } 198 | 199 | collection := collectionObj.(*Collection) 200 | key := readLine(data) 201 | 202 | jsonBytes, _ := data.ReadBytes('\n') 203 | jsonBytes = bytes.TrimSuffix(jsonBytes, []byte("\n")) 204 | 205 | value := reflect.New(collection.typ).Interface() 206 | err = jsoniter.Unmarshal(jsonBytes, &value) 207 | 208 | if err != nil { 209 | return err 210 | } 211 | 212 | // Check timestamp 213 | lastModificationObj, exists := collection.lastModification.Load(key) 214 | 215 | if exists { 216 | lastModification := lastModificationObj.(int64) 217 | 218 | if packetTime < lastModification { 219 | return errors.New("Outdated packet") 220 | } 221 | } 222 | 223 | // Perform the actual set 224 | collection.set(key, value) 225 | 226 | // Update last modification time 227 | collection.lastModification.Store(key, packetTime) 228 | 229 | return nil 230 | } 231 | 232 | // networkDelete performs a delete operation based on the information in the network packet. 233 | func networkDelete(msg *packet.Packet, db *Node) error { 234 | data := bytes.NewBuffer(msg.Data) 235 | 236 | packetTimeBuffer := make([]byte, 8) 237 | _, err := data.Read(packetTimeBuffer) 238 | 239 | if err != nil { 240 | return err 241 | } 242 | 243 | packetTime, err := packet.Int64FromBytes(packetTimeBuffer) 244 | 245 | if err != nil { 246 | return err 247 | } 248 | 249 | namespaceName := readLine(data) 250 | namespace := db.Namespace(namespaceName) 251 | 252 | collectionName := readLine(data) 253 | collectionObj, exists := namespace.collections.Load(collectionName) 254 | 255 | if !exists || collectionObj == nil { 256 | return nil //errors.New("Received networkDelete command on non-existing collection") 257 | } 258 | 259 | collection := collectionObj.(*Collection) 260 | key := readLine(data) 261 | 262 | // Check timestamp 263 | obj, exists := collection.lastModification.Load(key) 264 | 265 | if exists { 266 | lastModification := obj.(int64) 267 | 268 | if packetTime < lastModification { 269 | return errors.New("Outdated packet") 270 | } 271 | } 272 | 273 | // Perform the actual deletion 274 | collection.delete(key) 275 | 276 | // Update last modification time 277 | collection.lastModification.Store(key, packetTime) 278 | 279 | return nil 280 | } 281 | 282 | // serverOnConnect returns a function that can be used as a parameter 283 | // for the OnConnect method. It is called every time a new client connects 284 | // to the node. 285 | func serverOnConnect(node *Node) func(*packet.Stream) { 286 | return func(stream *packet.Stream) { 287 | if node.verbose { 288 | fmt.Println("[server] New client", stream.Connection().RemoteAddr()) 289 | } 290 | 291 | // Start reading packets from the client 292 | go serverReadPacketsFromClient(stream, node) 293 | } 294 | } 295 | 296 | // serverForwardPacket forwards the packet from the given client to other clients. 297 | func serverForwardPacket(serverNode *server.Node, client *packet.Stream, msg *packet.Packet) { 298 | fromRemoteClient := serverNode.IsRemoteAddress(client.Connection().RemoteAddr()) 299 | 300 | for targetClient := range serverNode.AllClients() { 301 | // Ignore the client who sent us the packet in the first place 302 | if targetClient == client { 303 | continue 304 | } 305 | 306 | // Do not send packets from remote clients to other remote clients. 307 | // Every node is responsible for notifying other remote nodes about changes. 308 | if fromRemoteClient && serverNode.IsRemoteAddress(targetClient.Connection().RemoteAddr()) { 309 | continue 310 | } 311 | 312 | // Send packet 313 | select { 314 | case targetClient.Outgoing <- msg: 315 | // Send successful. 316 | default: 317 | // Discard packet. 318 | // TODO: Find a better solution to deal with this. 319 | } 320 | } 321 | } 322 | 323 | // readLine reads a single line from the byte buffer and will not include the line break character. 324 | func readLine(data *bytes.Buffer) string { 325 | line, _ := data.ReadString('\n') 326 | line = strings.TrimSuffix(line, "\n") 327 | return line 328 | } 329 | -------------------------------------------------------------------------------- /Node.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | import ( 4 | "fmt" 5 | "net" 6 | "os/user" 7 | "path" 8 | "runtime" 9 | "sync" 10 | "time" 11 | 12 | "github.com/aerogo/cluster" 13 | "github.com/aerogo/cluster/client" 14 | "github.com/aerogo/cluster/server" 15 | "github.com/aerogo/packet" 16 | ) 17 | 18 | // Force interface implementation 19 | var _ cluster.Node = (*Node)(nil) 20 | 21 | // Node represents a single database node in the cluster. 22 | type Node struct { 23 | namespaces sync.Map 24 | node cluster.Node 25 | server *server.Node 26 | client *client.Node 27 | config Configuration 28 | ioSleepTime time.Duration 29 | networkWorkerQueue chan *packet.Packet 30 | verbose bool 31 | } 32 | 33 | // New starts up a new database node. 34 | func New(config Configuration) *Node { 35 | // Get user info to access the home directory 36 | user, err := user.Current() 37 | 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | // Create Node 43 | node := &Node{ 44 | config: config, 45 | ioSleepTime: 100 * time.Millisecond, 46 | networkWorkerQueue: make(chan *packet.Packet, 8192), 47 | } 48 | 49 | if node.config.Directory == "" { 50 | node.config.Directory = path.Join(user.HomeDir, ".aero", "db") 51 | } 52 | 53 | node.connect() 54 | return node 55 | } 56 | 57 | // Namespace ... 58 | func (node *Node) Namespace(name string) *Namespace { 59 | obj, loaded := node.namespaces.LoadOrStore(name, nil) 60 | 61 | if !loaded { 62 | namespace := newNamespace(node, name) 63 | node.namespaces.Store(name, namespace) 64 | return namespace 65 | } 66 | 67 | // Wait for existing namespace load 68 | for obj == nil { 69 | time.Sleep(1 * time.Millisecond) 70 | obj, _ = node.namespaces.Load(name) 71 | } 72 | 73 | return obj.(*Namespace) 74 | } 75 | 76 | // IsServer ... 77 | func (node *Node) IsServer() bool { 78 | return node.node.IsServer() 79 | } 80 | 81 | // IsClosed ... 82 | func (node *Node) IsClosed() bool { 83 | return node.node.IsClosed() 84 | } 85 | 86 | // Broadcast ... 87 | func (node *Node) Broadcast(msg *packet.Packet) { 88 | node.node.Broadcast(msg) 89 | } 90 | 91 | // Server ... 92 | func (node *Node) Server() *server.Node { 93 | return node.server 94 | } 95 | 96 | // Client ... 97 | func (node *Node) Client() *client.Node { 98 | return node.client 99 | } 100 | 101 | // Address ... 102 | func (node *Node) Address() net.Addr { 103 | return node.node.Address() 104 | } 105 | 106 | // Clear deletes all data in the Node. 107 | func (node *Node) Clear() { 108 | node.namespaces.Range(func(key, value interface{}) bool { 109 | namespace := value.(*Namespace) 110 | namespace.ClearAll() 111 | return true 112 | }) 113 | } 114 | 115 | // Close frees up resources used by the node. 116 | func (node *Node) Close() { 117 | if node.IsServer() { 118 | if node.verbose { 119 | fmt.Println("[server] broadcast close") 120 | } 121 | 122 | node.Broadcast(packet.New(packetServerClose, nil)) 123 | } 124 | 125 | // Close cluster node 126 | node.node.Close() 127 | 128 | // Close namespaces 129 | node.namespaces.Range(func(key, value interface{}) bool { 130 | namespace := value.(*Namespace) 131 | namespace.Close() 132 | return true 133 | }) 134 | } 135 | 136 | // connect ... 137 | func (node *Node) connect() { 138 | node.node = cluster.New(node.config.Port, node.config.Hosts...) 139 | 140 | if node.node.IsServer() { 141 | node.server = node.node.(*server.Node) 142 | node.server.OnConnect(serverOnConnect(node)) 143 | } else { 144 | node.client = node.node.(*client.Node) 145 | go clientReadPacketsFromServer(node.client, node) 146 | 147 | for i := 0; i < runtime.NumCPU(); i++ { 148 | go clientNetworkWorker(node) 149 | } 150 | } 151 | } 152 | 153 | // broadcastRequired ... 154 | func (node *Node) broadcastRequired() bool { 155 | if !node.IsServer() { 156 | return true 157 | } 158 | 159 | return node.server.ClientCount() > 0 160 | } 161 | -------------------------------------------------------------------------------- /Node_test.go: -------------------------------------------------------------------------------- 1 | package nano_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/aerogo/nano" 7 | "github.com/akyoto/assert" 8 | ) 9 | 10 | func TestNodeAddress(t *testing.T) { 11 | node := nano.New(config) 12 | defer node.Close() 13 | assert.True(t, node.Address().String() != "") 14 | } 15 | -------------------------------------------------------------------------------- /PacketType.go: -------------------------------------------------------------------------------- 1 | package nano 2 | 3 | const ( 4 | packetCollectionRequest = iota 5 | packetCollectionResponse = iota 6 | packetSet = iota 7 | packetDelete = iota 8 | packetServerClose = iota 9 | ) 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # nano 2 | 3 | [![Godoc][godoc-image]][godoc-url] 4 | [![Report][report-image]][report-url] 5 | [![Tests][tests-image]][tests-url] 6 | [![Coverage][coverage-image]][coverage-url] 7 | [![Sponsor][sponsor-image]][sponsor-url] 8 | 9 | High-performance database. Basically network and disk synchronized hashmaps. 10 | 11 | ## Benchmarks 12 | 13 | ```text 14 | BenchmarkCollectionGet-12 317030264 3.75 ns/op 0 B/op 0 allocs/op 15 | BenchmarkCollectionSet-12 11678318 102 ns/op 32 B/op 2 allocs/op 16 | BenchmarkCollectionDelete-12 123748969 9.50 ns/op 5 B/op 0 allocs/op 17 | BenchmarkCollectionAll-12 1403905 859 ns/op 2144 B/op 2 allocs/op 18 | ``` 19 | 20 | ## Features 21 | 22 | * Low latency commands 23 | * Every command is "local first, sync later" 24 | * Data is stored in memory 25 | * Data is synchronized between all nodes in a cluster 26 | * Data is saved to disk persistently using JSON 27 | * Timestamp based conflict resolution 28 | * Uses the extremely fast `sync.Map` 29 | 30 | ## Terminology 31 | 32 | * **Namespace**: Contains multiple collections (e.g. "google") 33 | * **Collection**: Contains homogeneous data for a data type (e.g. "User") 34 | * **Key**: The string that lets you look up a single object in a collection 35 | 36 | All of the above require a unique name. Given namespace, collection and key, you can access the data stored for it. 37 | 38 | ## Style 39 | 40 | Please take a look at the [style guidelines](https://github.com/akyoto/quality/blob/master/STYLE.md) if you'd like to make a pull request. 41 | 42 | ## Sponsors 43 | 44 | | [![Cedric Fung](https://avatars3.githubusercontent.com/u/2269238?s=70&v=4)](https://github.com/cedricfung) | [![Scott Rayapoullé](https://avatars3.githubusercontent.com/u/11772084?s=70&v=4)](https://github.com/soulcramer) | [![Eduard Urbach](https://avatars3.githubusercontent.com/u/438936?s=70&v=4)](https://twitter.com/eduardurbach) | 45 | | --- | --- | --- | 46 | | [Cedric Fung](https://github.com/cedricfung) | [Scott Rayapoullé](https://github.com/soulcramer) | [Eduard Urbach](https://eduardurbach.com) | 47 | 48 | Want to see [your own name here?](https://github.com/users/akyoto/sponsorship) 49 | 50 | [godoc-image]: https://godoc.org/github.com/aerogo/nano?status.svg 51 | [godoc-url]: https://godoc.org/github.com/aerogo/nano 52 | [report-image]: https://goreportcard.com/badge/github.com/aerogo/nano 53 | [report-url]: https://goreportcard.com/report/github.com/aerogo/nano 54 | [tests-image]: https://cloud.drone.io/api/badges/aerogo/nano/status.svg 55 | [tests-url]: https://cloud.drone.io/aerogo/nano 56 | [coverage-image]: https://codecov.io/gh/aerogo/nano/graph/badge.svg 57 | [coverage-url]: https://codecov.io/gh/aerogo/nano 58 | [sponsor-image]: https://img.shields.io/badge/github-donate-green.svg 59 | [sponsor-url]: https://github.com/users/akyoto/sponsorship 60 | -------------------------------------------------------------------------------- /README.src.md: -------------------------------------------------------------------------------- 1 | # {name} 2 | 3 | {go:header} 4 | 5 | High-performance database. Basically network and disk synchronized hashmaps. 6 | 7 | ## Benchmarks 8 | 9 | ```text 10 | BenchmarkCollectionGet-12 317030264 3.75 ns/op 0 B/op 0 allocs/op 11 | BenchmarkCollectionSet-12 11678318 102 ns/op 32 B/op 2 allocs/op 12 | BenchmarkCollectionDelete-12 123748969 9.50 ns/op 5 B/op 0 allocs/op 13 | BenchmarkCollectionAll-12 1403905 859 ns/op 2144 B/op 2 allocs/op 14 | ``` 15 | 16 | ## Features 17 | 18 | * Low latency commands 19 | * Every command is "local first, sync later" 20 | * Data is stored in memory 21 | * Data is synchronized between all nodes in a cluster 22 | * Data is saved to disk persistently using JSON 23 | * Timestamp based conflict resolution 24 | * Uses the extremely fast `sync.Map` 25 | 26 | ## Terminology 27 | 28 | * **Namespace**: Contains multiple collections (e.g. "google") 29 | * **Collection**: Contains homogeneous data for a data type (e.g. "User") 30 | * **Key**: The string that lets you look up a single object in a collection 31 | 32 | All of the above require a unique name. Given namespace, collection and key, you can access the data stored for it. 33 | 34 | {go:footer} 35 | -------------------------------------------------------------------------------- /Utils_test.go: -------------------------------------------------------------------------------- 1 | package nano_test 2 | 3 | import ( 4 | "strconv" 5 | 6 | "github.com/aerogo/nano" 7 | ) 8 | 9 | type User struct { 10 | ID string 11 | Name string 12 | BirthYear string 13 | Text string 14 | Created string 15 | Edited string 16 | Following []string 17 | } 18 | 19 | func newUser(id int) *User { 20 | //nolint:misspell 21 | return &User{ 22 | ID: strconv.Itoa(id), 23 | Name: "Test User", 24 | BirthYear: "1991", 25 | Text: `Lorem ipsum dolor sit amet, consectetur adipiscing elit. Etiam sit amet ante interdum, congue est vel, gravida odio. Praesent consequat, sem id convallis tincidunt, turpis dolor varius justo, sed consequat ante urna ac tortor. Nullam a tellus ac velit condimentum semper. Nulla et dolor a justo dignissim consectetur vel eu urna. Quisque molestie tincidunt mi non consectetur. Nulla eget faucibus lacus. Suspendisse dui lacus, volutpat vel quam ac, vehicula egestas est. Quisque a malesuada velit, mollis ullamcorper neque. Cras lobortis vitae tortor eget vehicula. Sed dictum augue vel risus eleifend, non venenatis mi vulputate. Sed laoreet accumsan enim ac porttitor. Ut blandit nibh ut ipsum ullamcorper, ut congue massa eleifend. Vivamus condimentum pharetra lorem, eget bibendum nunc porta id. Ut nulla orci, commodo id odio ac, mollis molestie ipsum. 26 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas sit amet dolor sit amet sem volutpat iaculis. Nunc viverra est quis sodales dictum. Fusce elementum nunc ac aliquet efficitur. Morbi id nunc sed urna dictum mattis at vitae est. Orci varius natoque penatibus et magnis dis parturient montes, nascetur ridiculus mus. Donec vestibulum mauris non metus fermentum molestie. 27 | Integer eu tortor a tellus tincidunt pretium et a urna. Integer tortor felis, rutrum vitae ipsum rutrum, laoreet maximus purus. Aliquam diam ipsum, pulvinar a leo eu, convallis ultricies urna. In consequat et eros id porta. Sed congue quam eu turpis vestibulum hendrerit. Suspendisse massa arcu, placerat sit amet tempor lobortis, ornare ut magna. Nunc sit amet gravida mi, aliquam laoreet metus. Cras non dolor at sapien euismod pulvinar ultrices eget turpis. Morbi vitae enim venenatis lacus tincidunt mollis eu non lectus. Proin nec libero porttitor, gravida turpis sed, bibendum mi. In venenatis molestie dapibus. Phasellus molestie tincidunt arcu, vel vestibulum orci dapibus in. Aliquam maximus justo eros, eu efficitur leo porta eu. Lorem ipsum dolor sit amet, consectetur adipiscing elit. 28 | Vivamus efficitur, libero accumsan molestie blandit, velit augue mattis enim, ut elementum arcu nulla quis eros. Sed aliquet, nibh sed dapibus porta, nisi sem efficitur purus, in sollicitudin ipsum nisi sit amet nisl. In hac habitasse platea dictumst. Nulla eu odio sit amet turpis mollis mollis finibus vitae diam. Mauris vel lorem vitae erat accumsan rhoncus eget porta libero. Curabitur tincidunt id dolor ut consectetur. Donec ornare elit sed metus malesuada fringilla. Cras purus nisl, laoreet ac risus et, consectetur consequat neque. Aliquam porttitor viverra aliquam. 29 | Etiam condimentum justo mi, eu hendrerit mauris ornare eget. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Morbi ac metus diam. Proin pulvinar orci at ex commodo, a blandit metus mollis. Phasellus tincidunt vel purus feugiat consequat. Proin vel accumsan massa. Cras suscipit neque dolor. Sed sit amet aliquet metus, non fringilla libero. Donec sagittis neque vel purus euismod, in tempus libero dignissim. Cras scelerisque vehicula bibendum. Maecenas augue orci, blandit posuere metus at, consectetur consectetur urna. Duis consectetur posuere est, vitae rutrum augue vehicula vitae. Aliquam id ornare odio.`, 30 | Created: "2017-01-01", 31 | Edited: "2017-01-01", 32 | Following: []string{"Vy2Hk5yvx", "VJOK1ckvx", "VJCuoQevx", "41oBveZPx", "41w5sjZKg", "4y1WgNMDx", "NyQph5bFe", "NJ3kffzwl", "Vy2We3bKe", "VkVaI_MPl", "V1eSUNSYx", "BJdJDFgc", "r1nTQ8Ko", "BkXadrU5"}, 33 | } 34 | } 35 | 36 | const port = 3000 37 | 38 | var types = []interface{}{ 39 | (*User)(nil), 40 | } 41 | 42 | var config = nano.Configuration{Port: port} 43 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/aerogo/nano 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/aerogo/cluster v0.1.8 7 | github.com/aerogo/flow v0.1.5 8 | github.com/aerogo/packet v0.2.2 9 | github.com/akyoto/assert v0.2.0 10 | github.com/json-iterator/go v1.1.7 11 | github.com/kr/pretty v0.1.0 // indirect 12 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 13 | github.com/modern-go/reflect2 v1.0.1 // indirect 14 | github.com/stretchr/testify v1.4.0 // indirect 15 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/aerogo/cluster v0.1.8 h1:N/jU2t7kQfKjXzQIArBQvtNkJCiaP9+jgdEid3P2Omc= 2 | github.com/aerogo/cluster v0.1.8/go.mod h1:ldxff0R9VPbm1yoZdkCrVYWroflgmwh+g6EM8+jFXyE= 3 | github.com/aerogo/flow v0.1.5 h1:wmSzIpHKV63CUsQ/YaLBti/5csUj1toK8jGvM+0z/fg= 4 | github.com/aerogo/flow v0.1.5/go.mod h1:kG63T/cHB2uR0nu0SGvy8d49J6YuI6LP1IehkP7VtwM= 5 | github.com/aerogo/packet v0.2.0 h1:YipWaCqHLn73WP+fU85a6yl6GULlUHWyy+ATmcm7pog= 6 | github.com/aerogo/packet v0.2.0/go.mod h1:8+cOKIJ35ZJAi8Afd94ed6q8D0eq3KeJFxXUEgTxPY0= 7 | github.com/aerogo/packet v0.2.2 h1:Fxoeljvod5cO2xgiHzDFRR8nhoNcA8u3FBaUkwBVsPk= 8 | github.com/aerogo/packet v0.2.2/go.mod h1:8+cOKIJ35ZJAi8Afd94ed6q8D0eq3KeJFxXUEgTxPY0= 9 | github.com/akyoto/assert v0.2.0 h1:lR7OHrbbBNNZFmRVS8I5MzS0ShLH36ZQVZVyg1bvs6A= 10 | github.com/akyoto/assert v0.2.0/go.mod h1:g5e6ag+ksCEQENq/LnmU9z04wCAIFDr8KacBusVL0H8= 11 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 12 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 13 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 16 | github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= 17 | github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 18 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 19 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 20 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 21 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 22 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 23 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 24 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 25 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 26 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 27 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 28 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 29 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 30 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 31 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 32 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 33 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 34 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 35 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 36 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 37 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 38 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 39 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 40 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 41 | --------------------------------------------------------------------------------