├── .gitignore ├── gun ├── tests │ ├── package.json │ ├── util_test.go │ ├── js_test.go │ ├── ws_test.go │ ├── context_test.go │ └── gun_test.go ├── doc.go ├── message.go ├── util.go ├── server.go ├── scoped.go ├── state.go ├── storage.go ├── websocket.go ├── node.go ├── peer.go ├── scoped_put.go ├── scoped_fetch.go └── gun.go ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /gun/tests/node_modules 2 | /gun/tests/ossl 3 | /gun/tests/radata-server 4 | /gun/tests/package-lock.json 5 | -------------------------------------------------------------------------------- /gun/tests/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "esgopeta-tests", 3 | "version": "0.1.0", 4 | "private": true, 5 | "dependencies": { 6 | "gun": "^0.9.9999991" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /gun/doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | Package gun is an implementation of the Gun distributed graph database in Go. 3 | See https://gun.eco for more information on the Gun distributed graph database. 4 | 5 | For common use, create a Gun instance via New, use Scoped to arrive at the 6 | desired field, then use Fetch to get/listen to values and/or Put to write 7 | values. A simple example is in the README at https://github.com/cretz/esgopeta. 8 | 9 | WARNING: This is an early proof-of-concept alpha version. Many pieces are not 10 | implemented or don't work. 11 | */ 12 | package gun 13 | -------------------------------------------------------------------------------- /gun/tests/util_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "bytes" 5 | "crypto/rand" 6 | ) 7 | 8 | const randChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 9 | 10 | func randString(n int) (s string) { 11 | // We accept that a multiple of 64 is %'d on 62 potentially favoring 0 or 1 more, but we don't care 12 | byts := make([]byte, n) 13 | if _, err := rand.Read(byts); err != nil { 14 | panic(err) 15 | } 16 | for _, byt := range byts { 17 | s += string(randChars[int(byt)%len(randChars)]) 18 | } 19 | return s 20 | } 21 | 22 | func removeGunJSWelcome(b []byte) []byte { 23 | if bytes.Index(b, []byte("Hello wonderful person!")) == 0 { 24 | b = b[bytes.IndexByte(b, '\n')+1:] 25 | } 26 | return b 27 | } 28 | 29 | func skipGunJSWelcome(buf *bytes.Buffer) { 30 | if bytes.Index(buf.Bytes(), []byte("Hello wonderful person!")) == 0 { 31 | if _, err := buf.ReadBytes('\n'); err != nil { 32 | panic(err) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /gun/tests/js_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | ) 7 | 8 | func TestSimpleJS(t *testing.T) { 9 | ctx, cancelFn := newContext(t) 10 | defer cancelFn() 11 | ctx.Require.Equal("yay 3\n", string(ctx.runJS("console.log('yay', 1 + 2)"))) 12 | } 13 | 14 | func TestGunJS(t *testing.T) { 15 | // Run the server, put in one call, get in another, then check 16 | ctx, cancelFn := newContextWithGunJServer(t) 17 | defer cancelFn() 18 | randStr := randString(30) 19 | ctx.runJSWithGun(` 20 | gun.get('esgopeta-test').get('TestGunJS').get('some-field').put('` + randStr + `', ack => { 21 | if (ack.err) { 22 | console.error(ack.err) 23 | process.exit(1) 24 | } 25 | process.exit(0) 26 | }) 27 | `) 28 | out := ctx.runJSWithGun(` 29 | gun.get('esgopeta-test').get('TestGunJS').get('some-field').once(data => { 30 | console.log(data) 31 | process.exit(0) 32 | }) 33 | `) 34 | ctx.Require.Equal(randStr, strings.TrimSpace(string(out))) 35 | } 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Chad Retz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /gun/message.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import "encoding/json" 4 | 5 | // Message is the JSON-encodable message that Gun peers send to each other. 6 | type Message struct { 7 | Ack string `json:"@,omitempty"` 8 | ID string `json:"#,omitempty"` 9 | To string `json:"><,omitempty"` 10 | Hash json.Number `json:"##,omitempty"` 11 | How string `json:"how,omitempty"` 12 | Get *MessageGetRequest `json:"get,omitempty"` 13 | Put map[string]*Node `json:"put,omitempty"` 14 | DAM string `json:"dam,omitempty"` 15 | PID string `json:"pid,omitempty"` 16 | OK int `json:"ok,omitempty"` 17 | Err string `json:"err,omitempty"` 18 | } 19 | 20 | // MessageGetRequest is the format for Message.Get. 21 | type MessageGetRequest struct { 22 | Soul string `json:"#,omitempty"` 23 | Field string `json:".,omitempty"` 24 | } 25 | 26 | type messageReceived struct { 27 | *Message 28 | peer *Peer 29 | // storedPuts are the souls and their fields that have been stored by 30 | // another part of the code. This is useful if the main instance stores 31 | // something it sees, there's no need for the message listener to do so as 32 | // well. 33 | storedPuts map[string][]string 34 | } 35 | -------------------------------------------------------------------------------- /gun/util.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "crypto/rand" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | const randChars = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" 10 | 11 | func randString(n int) (s string) { 12 | // We accept that a multiple of 64 is %'d on 62 potentially favoring 0 or 1 more, but we don't care 13 | byts := make([]byte, n) 14 | if _, err := rand.Read(byts); err != nil { 15 | panic(err) 16 | } 17 | for _, byt := range byts { 18 | s += string(randChars[int(byt)%len(randChars)]) 19 | } 20 | return s 21 | } 22 | 23 | // timeFromUnixMs returns zero'd time if ms is 0 24 | func timeFromUnixMs(ms int64) time.Time { 25 | if ms == 0 { 26 | return time.Time{} 27 | } 28 | return time.Unix(0, ms*int64(time.Millisecond)) 29 | } 30 | 31 | // timeToUnixMs returns 0 if t.IsZero 32 | func timeToUnixMs(t time.Time) int64 { 33 | if t.IsZero() { 34 | return 0 35 | } 36 | return t.UnixNano() / int64(time.Millisecond) 37 | } 38 | 39 | func timeNowUnixMs() int64 { 40 | return timeToUnixMs(time.Now()) 41 | } 42 | 43 | var lastNano int64 44 | 45 | // uniqueNano is 0 if ms is first time seen, otherwise a unique num in combination with ms 46 | func timeNowUniqueUnix() (ms int64, uniqueNum int64) { 47 | now := time.Now() 48 | newNano := now.UnixNano() 49 | for { 50 | prevLastNano := lastNano 51 | if prevLastNano < newNano && atomic.CompareAndSwapInt64(&lastNano, prevLastNano, newNano) { 52 | ms = newNano / int64(time.Millisecond) 53 | // If was same ms as seen before, set uniqueNum to the nano part 54 | if prevLastNano/int64(time.Millisecond) == ms { 55 | uniqueNum = newNano%int64(time.Millisecond) + 1 56 | } 57 | return 58 | } 59 | newNano = prevLastNano + 1 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /gun/server.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | ) 6 | 7 | // Server is the interface implemented by servers. 8 | type Server interface { 9 | // Serve is called by Gun to start the server. It should not return until 10 | // an error occurs or Close is called. 11 | Serve() error 12 | // Accept is called to wait for the next peer connection or if an error 13 | // occurs while trying. 14 | Accept() (PeerConn, error) 15 | // Close is called by Gun to stop and close this server. 16 | Close() error 17 | } 18 | 19 | func (g *Gun) startServers(ctx context.Context, servers []Server) { 20 | ctx, g.serversCancelFn = context.WithCancel(ctx) 21 | for _, server := range servers { 22 | // TODO: log error? 23 | go g.serve(ctx, server) 24 | } 25 | } 26 | 27 | func (g *Gun) serve(ctx context.Context, s Server) error { 28 | errCh := make(chan error, 1) 29 | ctx, cancelFn := context.WithCancel(ctx) 30 | defer cancelFn() 31 | // Start the server 32 | go func() { errCh <- s.Serve() }() 33 | defer s.Close() 34 | // Accept connections and break off 35 | go func() { 36 | if conn, err := s.Accept(); err == nil { 37 | // TODO: log error (for accept and handle)? 38 | go g.onNewPeerConn(ctx, conn) 39 | } 40 | }() 41 | // Wait for server close or context close 42 | select { 43 | case err := <-errCh: 44 | return err 45 | case <-ctx.Done(): 46 | return ctx.Err() 47 | } 48 | } 49 | 50 | func (g *Gun) onNewPeerConn(ctx context.Context, conn PeerConn) error { 51 | ctx, cancelFn := context.WithCancel(ctx) 52 | defer cancelFn() 53 | defer conn.Close() 54 | // We always send a DAM req first 55 | if err := conn.Send(ctx, &Message{DAM: "?"}); err != nil { 56 | return err 57 | } 58 | // Now add the connection to Gun 59 | panic("TODO") 60 | } 61 | -------------------------------------------------------------------------------- /gun/scoped.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | ) 9 | 10 | // Scoped is a contextual, namespaced field to read or write. 11 | type Scoped struct { 12 | gun *Gun 13 | 14 | parent *Scoped 15 | field string 16 | cachedParentSoul string 17 | cachedParentSoulLock sync.RWMutex 18 | 19 | fetchResultListeners map[<-chan *FetchResult]*fetchResultListener 20 | fetchResultListenersLock sync.Mutex 21 | 22 | putResultListeners map[<-chan *PutResult]*putResultListener 23 | putResultListenersLock sync.Mutex 24 | } 25 | 26 | func newScoped(gun *Gun, parent *Scoped, field string) *Scoped { 27 | return &Scoped{ 28 | gun: gun, 29 | parent: parent, 30 | field: field, 31 | fetchResultListeners: map[<-chan *FetchResult]*fetchResultListener{}, 32 | putResultListeners: map[<-chan *PutResult]*putResultListener{}, 33 | } 34 | } 35 | 36 | // ErrNotObject occurs when a put or a fetch is attempted as a child of an 37 | // existing, non-relation value. 38 | var ErrNotObject = errors.New("Scoped value not an object") 39 | 40 | // ErrLookupOnTopLevel occurs when a put or remote fetch is attempted on 41 | // a top-level field. 42 | var ErrLookupOnTopLevel = errors.New("Cannot do put/lookup on top level") 43 | 44 | // Soul returns the current soul of this value relation. It returns a cached 45 | // value if called before. Otherwise, it does a FetchOne to get the value 46 | // and return its soul if a relation. If any parent is not a relation or this 47 | // value exists and is not a relation, ErrNotObject is returned. If this field 48 | // doesn't exist yet, an empty string is returned with no error. Otherwise, 49 | // the soul of the relation is returned. The context can be used to timeout the 50 | // fetch. 51 | func (s *Scoped) Soul(ctx context.Context) (string, error) { 52 | if cachedSoul := s.cachedSoul(); cachedSoul != "" { 53 | return cachedSoul, nil 54 | } else if r := s.FetchOne(ctx); r.Err != nil { 55 | return "", r.Err 56 | } else if !r.ValueExists { 57 | return "", nil 58 | } else if rel, ok := r.Value.(ValueRelation); !ok { 59 | return "", ErrNotObject 60 | } else if !s.setCachedSoul(rel) { 61 | return "", fmt.Errorf("Concurrent soul cache set") 62 | } else { 63 | return string(rel), nil 64 | } 65 | } 66 | 67 | func (s *Scoped) cachedSoul() string { 68 | s.cachedParentSoulLock.RLock() 69 | defer s.cachedParentSoulLock.RUnlock() 70 | return s.cachedParentSoul 71 | } 72 | 73 | func (s *Scoped) setCachedSoul(val ValueRelation) bool { 74 | s.cachedParentSoulLock.Lock() 75 | defer s.cachedParentSoulLock.Unlock() 76 | if s.cachedParentSoul != "" { 77 | return false 78 | } 79 | s.cachedParentSoul = string(val) 80 | return true 81 | } 82 | 83 | // Scoped returns a scoped instance to the given field and children for reading 84 | // and writing. This is the equivalent of calling the Gun JS API "get" function 85 | // (sans callback) for each field/child. 86 | func (s *Scoped) Scoped(ctx context.Context, field string, children ...string) *Scoped { 87 | ret := newScoped(s.gun, s, field) 88 | for _, child := range children { 89 | ret = newScoped(s.gun, ret, child) 90 | } 91 | return ret 92 | } 93 | -------------------------------------------------------------------------------- /gun/state.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "time" 7 | ) 8 | 9 | // State represents the conflict state of this value. It is usually the Unix 10 | // time in milliseconds. 11 | type State float64 // TODO: what if larger? 12 | 13 | // StateNow is the current machine state (i.e. current Unix time in ms). 14 | func StateNow() State { 15 | // TODO: Should I use timeNowUniqueUnix or otherwise set decimal spots to disambiguate? 16 | return State(timeNowUnixMs()) 17 | } 18 | 19 | // StateFromTime converts a time to a State (i.e. converts to Unix ms). 20 | func StateFromTime(t time.Time) State { return State(timeToUnixMs(t)) } 21 | 22 | // ConflictResolution is how to handle two values for the same field. 23 | type ConflictResolution int 24 | 25 | const ( 26 | // ConflictResolutionNeverSeenUpdate occurs when there is no existing value. 27 | // It means an update should always occur. 28 | ConflictResolutionNeverSeenUpdate ConflictResolution = iota 29 | // ConflictResolutionTooFutureDeferred occurs when the update is after our 30 | // current machine state. It means the update should be deferred. 31 | ConflictResolutionTooFutureDeferred 32 | // ConflictResolutionOlderHistorical occurs when the update happened before 33 | // the existing value's last update. It means it can be noted, but the 34 | // update should be discarded. 35 | ConflictResolutionOlderHistorical 36 | // ConflictResolutionNewerUpdate occurs when the update happened after last 37 | // update but is not beyond ur current machine state. It means the update 38 | // should overwrite. 39 | ConflictResolutionNewerUpdate 40 | // ConflictResolutionSameKeep occurs when the update happened at the same 41 | // time and it is lexically not the one chosen. It means the update should 42 | // be discarded. 43 | ConflictResolutionSameKeep 44 | // ConflictResolutionSameUpdate occurs when the update happened at the same 45 | // time and it is lexically the one chosen. It means the update should 46 | // overwrite. 47 | ConflictResolutionSameUpdate 48 | ) 49 | 50 | // IsImmediateUpdate returns true for ConflictResolutionNeverSeenUpdate, 51 | // ConflictResolutionNewerUpdate, and ConflictResolutionSameUpdate 52 | func (c ConflictResolution) IsImmediateUpdate() bool { 53 | return c == ConflictResolutionNeverSeenUpdate || c == ConflictResolutionNewerUpdate || c == ConflictResolutionSameUpdate 54 | } 55 | 56 | // ConflictResolve checks the existing val/state, new val/state, and the current 57 | // machine state to choose what to do with the update. Note, the existing val 58 | // should always exist meaning it will never return 59 | // ConflictResolutionNeverSeenUpdate. 60 | func ConflictResolve(existingVal Value, existingState State, newVal Value, newState State, sysState State) ConflictResolution { 61 | // Existing gunjs impl serializes to JSON first to do lexical comparisons, so we will too 62 | if sysState < newState { 63 | return ConflictResolutionTooFutureDeferred 64 | } else if newState < existingState { 65 | return ConflictResolutionOlderHistorical 66 | } else if existingState < newState { 67 | return ConflictResolutionNewerUpdate 68 | } else if existingVal == newVal { 69 | return ConflictResolutionSameKeep 70 | } else if existingJSON, err := json.Marshal(existingVal); err != nil { 71 | panic(err) 72 | } else if newJSON, err := json.Marshal(newVal); err != nil { 73 | panic(err) 74 | } else if bytes.Compare(existingJSON, newJSON) < 0 { 75 | return ConflictResolutionSameUpdate 76 | } else { 77 | return ConflictResolutionSameKeep 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Esgopeta [![GoDoc](https://godoc.org/github.com/cretz/esgopeta/gun?status.svg)](https://godoc.org/github.com/cretz/esgopeta/gun) 2 | 3 | Esgopeta is a Go implementation of the [Gun](https://github.com/amark/gun) distributed graph database. See the 4 | [Godoc](https://godoc.org/github.com/cretz/esgopeta/gun) for API details. 5 | 6 | **WARNING: This is an early proof-of-concept alpha version. Many pieces are not implemented or don't work.** 7 | 8 | Features: 9 | 10 | * Client for reading and writing w/ rudimentary conflict resolution 11 | * In-memory storage 12 | 13 | Not yet implemented: 14 | 15 | * Server 16 | * Alternative storage methods 17 | * SEA (i.e. encryption/auth) 18 | 19 | ### Usage 20 | 21 | The package is `github.com/cretz/esgopeta/gun` which can be fetched via `go get`. To listen to database changes for a 22 | value, use `Fetch`. The example below listens for updates on a key for a minute: 23 | 24 | ```go 25 | package main 26 | 27 | import ( 28 | "context" 29 | "log" 30 | "time" 31 | 32 | "github.com/cretz/esgopeta/gun" 33 | ) 34 | 35 | func main() { 36 | // Let's listen for a minute 37 | ctx, cancelFn := context.WithTimeout(context.Background(), 1*time.Minute) 38 | defer cancelFn() 39 | // Create the Gun client connecting to common Gun server 40 | g, err := gun.New(ctx, gun.Config{ 41 | PeerURLs: []string{"https://gunjs.herokuapp.com/gun"}, 42 | PeerErrorHandler: func(err *gun.ErrPeer) { log.Print(err) }, 43 | }) 44 | if err != nil { 45 | log.Panic(err) 46 | } 47 | defer g.Close() 48 | // Issue a fetch and get a channel for updates 49 | fetchCh := g.Scoped(ctx, "esgopeta-example", "sample-key").Fetch(ctx) 50 | // Log all updates and exit when context times out 51 | log.Print("Waiting for value") 52 | for { 53 | select { 54 | case <-ctx.Done(): 55 | log.Print("Time's up") 56 | return 57 | case fetchResult := <-fetchCh: 58 | if fetchResult.Err != nil { 59 | log.Printf("Error fetching: %v", fetchResult.Err) 60 | } else if fetchResult.ValueExists { 61 | log.Printf("Got value: %v", fetchResult.Value) 62 | } 63 | } 64 | } 65 | } 66 | ``` 67 | 68 | When that's running, we can send values via a `Put`. The example below sends two updates for that key: 69 | 70 | ```go 71 | package main 72 | 73 | import ( 74 | "context" 75 | "log" 76 | "time" 77 | 78 | "github.com/cretz/esgopeta/gun" 79 | ) 80 | 81 | func main() { 82 | // Give a 1 minute timeout, but shouldn't get hit 83 | ctx, cancelFn := context.WithTimeout(context.Background(), 1*time.Minute) 84 | defer cancelFn() 85 | // Create the Gun client connecting to common Gun server 86 | g, err := gun.New(ctx, gun.Config{ 87 | PeerURLs: []string{"https://gunjs.herokuapp.com/gun"}, 88 | PeerErrorHandler: func(err *gun.ErrPeer) { log.Print(err) }, 89 | }) 90 | if err != nil { 91 | log.Panic(err) 92 | } 93 | defer g.Close() 94 | // Issue a simple put and wait for a single peer ack 95 | putScope := g.Scoped(ctx, "esgopeta-example", "sample-key") 96 | log.Print("Sending first value") 97 | putCh := putScope.Put(ctx, gun.ValueString("first value")) 98 | for { 99 | if result := <-putCh; result.Err != nil { 100 | log.Printf("Error putting: %v", result.Err) 101 | } else if result.Peer != nil { 102 | log.Printf("Got ack from %v", result.Peer) 103 | break 104 | } 105 | } 106 | // Let's send another value 107 | log.Print("Sending second value") 108 | putCh = putScope.Put(ctx, gun.ValueString("second value")) 109 | for { 110 | if result := <-putCh; result.Err != nil { 111 | log.Printf("Error putting: %v", result.Err) 112 | } else if result.Peer != nil { 113 | log.Printf("Got ack from %v", result.Peer) 114 | break 115 | } 116 | } 117 | } 118 | ``` 119 | 120 | Note, these are just examples and you may want to control the lifetime of the channels better. See the 121 | [Godoc](https://godoc.org/github.com/cretz/esgopeta/gun) for more information. -------------------------------------------------------------------------------- /gun/tests/ws_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "net/http" 7 | "strconv" 8 | "testing" 9 | "time" 10 | 11 | "github.com/gorilla/websocket" 12 | ) 13 | 14 | func (t *testContext) startGunWebSocketProxyLogger(listenPort int, targetURL string) { 15 | fromGun, toGun := t.startGunWebSocketProxy(listenPort, targetURL) 16 | time.Sleep(time.Second) 17 | go func() { 18 | for { 19 | select { 20 | case msg, ok := <-fromGun: 21 | if !ok { 22 | return 23 | } 24 | if testing.Verbose() { 25 | t.debugf("From gun raw: %v", string(msg)) 26 | for _, s := range t.formattedGunJSONs(msg) { 27 | t.debugf("From gun: %v", s) 28 | } 29 | } 30 | case msg, ok := <-toGun: 31 | if !ok { 32 | return 33 | } 34 | if testing.Verbose() { 35 | t.debugf("To gun raw: %v", string(msg)) 36 | if len(msg) == 0 { 37 | t.debugf("To gun: empty message") 38 | } else { 39 | for _, s := range t.formattedGunJSONs(msg) { 40 | t.debugf("To gun: %v", s) 41 | } 42 | } 43 | } 44 | } 45 | } 46 | }() 47 | } 48 | 49 | func (t *testContext) formattedGunJSONs(msg []byte) []string { 50 | var objs []interface{} 51 | if msg[0] == '[' { 52 | arr := []string{} 53 | t.Require.NoError(json.Unmarshal(msg, &arr)) 54 | for _, v := range arr { 55 | var obj interface{} 56 | t.Require.NoError(json.Unmarshal([]byte(v), &obj)) 57 | objs = append(objs, obj) 58 | } 59 | } else { 60 | var obj interface{} 61 | t.Require.NoError(json.Unmarshal(msg, &obj)) 62 | objs = append(objs, obj) 63 | } 64 | ret := make([]string, len(objs)) 65 | for i, obj := range objs { 66 | b, err := json.MarshalIndent(obj, "", " ") 67 | t.Require.NoError(err) 68 | ret[i] = string(b) 69 | } 70 | return ret 71 | } 72 | 73 | func (t *testContext) startGunWebSocketProxy(listenPort int, targetURL string) (fromTarget <-chan []byte, toTarget <-chan []byte) { 74 | fromTargetCh := make(chan []byte) 75 | toTargetCh := make(chan []byte) 76 | server := &http.Server{ 77 | Addr: "127.0.0.1:" + strconv.Itoa(listenPort), 78 | Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 79 | t.debugf("New ws proxy connection") 80 | err := t.handleGunWebSocketProxy(targetURL, w, r, fromTargetCh, toTargetCh) 81 | if _, ok := err.(*websocket.CloseError); !ok { 82 | t.debugf("Unexpected web socket close error: %v", err) 83 | } 84 | }), 85 | } 86 | serverErrCh := make(chan error, 1) 87 | go func() { serverErrCh <- server.ListenAndServe() }() 88 | go func() { 89 | defer server.Close() 90 | select { 91 | case <-t.Done(): 92 | case err := <-serverErrCh: 93 | log.Printf("Server error: %v", err) 94 | } 95 | }() 96 | return fromTargetCh, toTargetCh 97 | } 98 | 99 | var wsDefaultUpgrader = websocket.Upgrader{} 100 | 101 | func (t *testContext) handleGunWebSocketProxy( 102 | targetURL string, 103 | w http.ResponseWriter, 104 | r *http.Request, 105 | fromOther chan<- []byte, 106 | toOther chan<- []byte, 107 | ) error { 108 | otherConn, _, err := websocket.DefaultDialer.DialContext(t, targetURL, nil) 109 | if err != nil { 110 | return err 111 | } 112 | defer otherConn.Close() 113 | // Upgrade 114 | c, err := wsDefaultUpgrader.Upgrade(w, r, nil) 115 | if err != nil { 116 | return err 117 | } 118 | defer c.Close() 119 | type readMsg struct { 120 | messageType int 121 | p []byte 122 | err error 123 | } 124 | 125 | readCh := make(chan *readMsg) 126 | go func() { 127 | for { 128 | msg := new(readMsg) 129 | msg.messageType, msg.p, msg.err = c.ReadMessage() 130 | readCh <- msg 131 | } 132 | }() 133 | otherReadCh := make(chan *readMsg) 134 | go func() { 135 | for { 136 | msg := new(readMsg) 137 | msg.messageType, msg.p, msg.err = otherConn.ReadMessage() 138 | otherReadCh <- msg 139 | } 140 | }() 141 | for { 142 | select { 143 | case msg := <-readCh: 144 | if msg.err != nil { 145 | return msg.err 146 | } 147 | toOther <- msg.p 148 | if err := otherConn.WriteMessage(msg.messageType, msg.p); err != nil { 149 | return err 150 | } 151 | case otherMsg := <-otherReadCh: 152 | if otherMsg.err != nil { 153 | return otherMsg.err 154 | } 155 | fromOther <- otherMsg.p 156 | if err := c.WriteMessage(otherMsg.messageType, otherMsg.p); err != nil { 157 | return err 158 | } 159 | case <-t.Done(): 160 | return t.Err() 161 | } 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /gun/tests/context_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "runtime" 11 | "strconv" 12 | "strings" 13 | "testing" 14 | "time" 15 | 16 | "github.com/cretz/esgopeta/gun" 17 | "github.com/stretchr/testify/require" 18 | ) 19 | 20 | type testContext struct { 21 | context.Context 22 | *testing.T 23 | Require *require.Assertions 24 | GunJSPort int 25 | } 26 | 27 | const defaultTestTimeout = 1 * time.Minute 28 | 29 | func newContext(t *testing.T) (*testContext, context.CancelFunc) { 30 | return withTestContext(context.Background(), t) 31 | } 32 | 33 | func newContextWithGunJServer(t *testing.T) (*testContext, context.CancelFunc) { 34 | ctx, cancelFn := newContext(t) 35 | serverCancelFn := ctx.startGunJSServer() 36 | return ctx, func() { 37 | serverCancelFn() 38 | cancelFn() 39 | } 40 | } 41 | 42 | const defaultGunJSPort = 8080 43 | const defaultRemoteGunServerURL = "https://gunjs.herokuapp.com/gun" 44 | 45 | func withTestContext(ctx context.Context, t *testing.T) (*testContext, context.CancelFunc) { 46 | ctx, cancelFn := context.WithTimeout(ctx, defaultTestTimeout) 47 | return &testContext{ 48 | Context: ctx, 49 | T: t, 50 | Require: require.New(t), 51 | GunJSPort: defaultGunJSPort, 52 | }, cancelFn 53 | } 54 | 55 | func (t *testContext) debugf(format string, args ...interface{}) { 56 | if testing.Verbose() { 57 | log.Printf(format, args...) 58 | } 59 | } 60 | 61 | func (t *testContext) runJS(script string) []byte { 62 | cmd := exec.CommandContext(t, "node") 63 | _, currFile, _, _ := runtime.Caller(0) 64 | cmd.Dir = filepath.Dir(currFile) 65 | cmd.Stdin = bytes.NewReader([]byte(script)) 66 | out, err := cmd.CombinedOutput() 67 | out = removeGunJSWelcome(out) 68 | t.Require.NoErrorf(err, "JS failure, output:\n%v", string(out)) 69 | return out 70 | } 71 | 72 | func (t *testContext) runJSWithGun(script string) []byte { 73 | return t.runJSWithGunURL("http://127.0.0.1:"+strconv.Itoa(t.GunJSPort)+"/gun", script) 74 | } 75 | 76 | func (t *testContext) runJSWithGunURL(url string, script string) []byte { 77 | return t.runJS(` 78 | var Gun = require('gun') 79 | const gun = Gun({ 80 | peers: ['` + url + `'], 81 | radisk: false 82 | }) 83 | ` + script) 84 | } 85 | 86 | func (t *testContext) startJS(script string) (*bytes.Buffer, *exec.Cmd, context.CancelFunc) { 87 | cmdCtx, cancelFn := context.WithCancel(t) 88 | cmd := exec.CommandContext(cmdCtx, "node") 89 | _, currFile, _, _ := runtime.Caller(0) 90 | cmd.Dir = filepath.Dir(currFile) 91 | cmd.Stdin = bytes.NewReader([]byte(script)) 92 | var buf bytes.Buffer 93 | cmd.Stdout, cmd.Stderr = &buf, &buf 94 | t.Require.NoError(cmd.Start()) 95 | return &buf, cmd, cancelFn 96 | } 97 | 98 | func (t *testContext) startGunJSServer() context.CancelFunc { 99 | // If we're logging, use a proxy 100 | port := t.GunJSPort 101 | if testing.Verbose() { 102 | t.startGunWebSocketProxyLogger(port, "ws://127.0.0.1:"+strconv.Itoa(port+1)+"/gun") 103 | port++ 104 | } 105 | // Remove entire data folder first just in case 106 | t.Require.NoError(os.RemoveAll("radata-server")) 107 | _, cmd, cancelFn := t.startJS(` 108 | var Gun = require('gun') 109 | const server = require('http').createServer().listen(` + strconv.Itoa(port) + `) 110 | const gun = Gun({web: server, file: 'radata-server'}) 111 | `) 112 | return func() { 113 | cancelFn() 114 | cmd.Wait() 115 | // Remove the data folder at the end 116 | os.RemoveAll("radata-server") 117 | } 118 | } 119 | 120 | func (t *testContext) prepareRemoteGunServer(origURL string) (newURL string) { 121 | // If we're verbose, use proxy, otherwise just use orig 122 | if !testing.Verbose() { 123 | return origURL 124 | } 125 | origURL = strings.Replace(origURL, "http://", "ws://", 1) 126 | origURL = strings.Replace(origURL, "https://", "wss://", 1) 127 | t.startGunWebSocketProxyLogger(t.GunJSPort, origURL) 128 | return "http://127.0.0.1:" + strconv.Itoa(t.GunJSPort) + "/gun" 129 | } 130 | 131 | func (t *testContext) newGunConnectedToGunJS() *gun.Gun { 132 | return t.newGunConnectedToGunServer("http://127.0.0.1:" + strconv.Itoa(t.GunJSPort) + "/gun") 133 | } 134 | 135 | func (t *testContext) newGunConnectedToGunServer(url string) *gun.Gun { 136 | config := gun.Config{ 137 | PeerURLs: []string{url}, 138 | PeerErrorHandler: func(errPeer *gun.ErrPeer) { t.debugf("Got peer error: %v", errPeer) }, 139 | } 140 | g, err := gun.New(t, config) 141 | t.Require.NoError(err) 142 | return g 143 | } 144 | -------------------------------------------------------------------------------- /gun/tests/gun_test.go: -------------------------------------------------------------------------------- 1 | package tests 2 | 3 | import ( 4 | "strings" 5 | "testing" 6 | 7 | "github.com/cretz/esgopeta/gun" 8 | ) 9 | 10 | func TestGunGetSimple(t *testing.T) { 11 | // Run the server, put in one call, get in another, then check 12 | ctx, cancelFn := newContext(t) 13 | defer cancelFn() 14 | serverCancelFn := ctx.startGunJSServer() 15 | defer serverCancelFn() 16 | randStr := randString(30) 17 | // Write w/ JS 18 | ctx.runJSWithGun(` 19 | gun.get('esgopeta-test').get('TestGunGetSimple').get('some-field').put('` + randStr + `', ack => { 20 | if (ack.err) { 21 | console.error(ack.err) 22 | process.exit(1) 23 | } 24 | process.exit(0) 25 | }) 26 | `) 27 | // Get 28 | g := ctx.newGunConnectedToGunJS() 29 | defer g.Close() 30 | // Make sure we got back the same value 31 | r := g.Scoped(ctx, "esgopeta-test", "TestGunGetSimple", "some-field").FetchOne(ctx) 32 | ctx.Require.NoError(r.Err) 33 | ctx.Require.Equal(gun.ValueString(randStr), r.Value.(gun.ValueString)) 34 | // Do it again with the JS server closed since it should fetch from memory 35 | serverCancelFn() 36 | ctx.debugf("Asking for field again") 37 | r = g.Scoped(ctx, "esgopeta-test", "TestGunGetSimple", "some-field").FetchOne(ctx) 38 | ctx.Require.NoError(r.Err) 39 | ctx.Require.Equal(gun.ValueString(randStr), r.Value.(gun.ValueString)) 40 | } 41 | 42 | func TestGunGetSimpleRemote(t *testing.T) { 43 | // Do the above but w/ remote server 44 | ctx, cancelFn := newContext(t) 45 | defer cancelFn() 46 | remoteURL := ctx.prepareRemoteGunServer(defaultRemoteGunServerURL) 47 | randField, randVal := "field-"+randString(30), gun.ValueString(randString(30)) 48 | // Write w/ JS 49 | ctx.debugf("Writing value") 50 | ctx.runJSWithGunURL(remoteURL, ` 51 | gun.get('esgopeta-test').get('TestGunGetSimpleRemote').get('`+randField+`').put('`+string(randVal)+`', ack => { 52 | if (ack.err) { 53 | console.error(ack.err) 54 | process.exit(1) 55 | } 56 | process.exit(0) 57 | }) 58 | `) 59 | // Get 60 | ctx.debugf("Reading value") 61 | g := ctx.newGunConnectedToGunServer(remoteURL) 62 | defer g.Close() 63 | // Make sure we got back the same value 64 | r := g.Scoped(ctx, "esgopeta-test", "TestGunGetSimpleRemote", randField).FetchOne(ctx) 65 | ctx.Require.NoError(r.Err) 66 | ctx.Require.Equal(randVal, r.Value) 67 | } 68 | 69 | func TestGunPutSimple(t *testing.T) { 70 | ctx, cancelFn := newContextWithGunJServer(t) 71 | defer cancelFn() 72 | randStr := randString(30) 73 | // Put 74 | g := ctx.newGunConnectedToGunJS() 75 | defer g.Close() 76 | // Just wait for two acks (one local, one remote) 77 | ch := g.Scoped(ctx, "esgopeta-test", "TestGunPutSimple", "some-field").Put(ctx, gun.ValueString(randStr)) 78 | // TODO: test local is null peer and remote is non-null 79 | r := <-ch 80 | ctx.Require.NoError(r.Err) 81 | r = <-ch 82 | ctx.Require.NoError(r.Err) 83 | // Get from JS 84 | out := ctx.runJSWithGun(` 85 | gun.get('esgopeta-test').get('TestGunPutSimple').get('some-field').once(data => { 86 | console.log(data) 87 | process.exit(0) 88 | }) 89 | `) 90 | ctx.Require.Equal(randStr, strings.TrimSpace(string(out))) 91 | } 92 | 93 | func TestGunPubSubSimpleRemote(t *testing.T) { 94 | ctx, cancelFn := newContext(t) 95 | defer cancelFn() 96 | remoteURL := ctx.prepareRemoteGunServer(defaultRemoteGunServerURL) 97 | randField, randVal := "field-"+randString(30), gun.ValueString(randString(30)) 98 | // Start a fetcher 99 | ctx.debugf("Starting fetcher") 100 | fetchGun := ctx.newGunConnectedToGunServer(remoteURL) 101 | defer fetchGun.Close() 102 | fetchCh := fetchGun.Scoped(ctx, "esgopeta-test", "TestGunPubSubSimpleRemote", randField).Fetch(ctx) 103 | // Now put it from another instance 104 | ctx.debugf("Putting data") 105 | putGun := ctx.newGunConnectedToGunServer(remoteURL) 106 | defer putGun.Close() 107 | putScope := putGun.Scoped(ctx, "esgopeta-test", "TestGunPubSubSimpleRemote", randField) 108 | putScope.Put(ctx, randVal) 109 | ctx.debugf("Checking fetcher") 110 | // See that the fetch got the value 111 | for { 112 | select { 113 | case <-ctx.Done(): 114 | ctx.Require.NoError(ctx.Err()) 115 | case result := <-fetchCh: 116 | ctx.Require.NoError(result.Err) 117 | if !result.ValueExists { 118 | ctx.debugf("No value, trying again (got %v)", result) 119 | continue 120 | } 121 | ctx.Require.Equal(randVal, result.Value) 122 | return 123 | } 124 | } 125 | } 126 | 127 | /* 128 | TODO Tests to write: 129 | * test put w/ future state happens then 130 | * test put w/ old state is discarded 131 | * test put w/ new state is persisted 132 | * test put w/ same state but greater is persisted 133 | * test put w/ same state but less is discarded 134 | */ 135 | -------------------------------------------------------------------------------- /gun/storage.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | // ErrStorageNotFound is returned by Storage.Get and sometimes Storage.Put when 12 | // the field doesn't exist. 13 | var ErrStorageNotFound = errors.New("Not found") 14 | 15 | // Storage is the interface that storage adapters must implement. 16 | type Storage interface { 17 | // Get obtains the value (which can be nil) and state from storage for the 18 | // given field. If the field does not exist, this errors with 19 | // ErrStorageNotFound. 20 | Get(ctx context.Context, parentSoul, field string) (Value, State, error) 21 | // Put sets the value (which can be nil) and state in storage for the given 22 | // field if the conflict resolution says it should (see ConflictResolve). It 23 | // also returns the conflict resolution. If onlyIfExists is true and the 24 | // field does not exist, this errors with ErrStorageNotFound. Otherwise, if 25 | // the resulting resolution is an immediate update, it is done. If the 26 | // resulting resolution is deferred for the future, it is scheduled for then 27 | // but is not even attempted if context is completed or storage is closed. 28 | Put(ctx context.Context, parentSoul, field string, val Value, state State, onlyIfExists bool) (ConflictResolution, error) 29 | // Close closes this storage and disallows future gets or puts. 30 | Close() error 31 | } 32 | 33 | type storageInMem struct { 34 | values map[parentSoulAndField]*valueWithState 35 | valueLock sync.RWMutex 36 | closed bool // Do not mutate outside of valueLock 37 | purgeCancelFn context.CancelFunc 38 | } 39 | 40 | type parentSoulAndField struct{ parentSoul, field string } 41 | 42 | type valueWithState struct { 43 | val Value 44 | state State 45 | } 46 | 47 | // NewStorageInMem creates an in-memory storage that automatically purges 48 | // values that are older than the given oldestAllowed. If oldestAllowed is 0, 49 | // it keeps all values forever. 50 | func NewStorageInMem(oldestAllowed time.Duration) Storage { 51 | s := &storageInMem{values: map[parentSoulAndField]*valueWithState{}} 52 | // Start the purger 53 | if oldestAllowed > 0 { 54 | var ctx context.Context 55 | ctx, s.purgeCancelFn = context.WithCancel(context.Background()) 56 | go func() { 57 | tick := time.NewTicker(5 * time.Second) 58 | defer tick.Stop() 59 | for { 60 | select { 61 | case <-ctx.Done(): 62 | return 63 | case t := <-tick.C: 64 | oldestStateAllowed := StateFromTime(t.Add(-oldestAllowed)) 65 | s.valueLock.Lock() 66 | s.valueLock.Unlock() 67 | for k, v := range s.values { 68 | if v.state < oldestStateAllowed { 69 | delete(s.values, k) 70 | } 71 | } 72 | } 73 | } 74 | }() 75 | } 76 | return s 77 | } 78 | 79 | func (s *storageInMem) Get(ctx context.Context, parentSoul, field string) (Value, State, error) { 80 | s.valueLock.RLock() 81 | defer s.valueLock.RUnlock() 82 | if s.closed { 83 | return nil, 0, fmt.Errorf("Storage closed") 84 | } else if vs := s.values[parentSoulAndField{parentSoul, field}]; vs == nil { 85 | return nil, 0, ErrStorageNotFound 86 | } else { 87 | return vs.val, vs.state, nil 88 | } 89 | } 90 | 91 | func (s *storageInMem) Put( 92 | ctx context.Context, parentSoul, field string, val Value, state State, onlyIfExists bool, 93 | ) (confRes ConflictResolution, err error) { 94 | s.valueLock.Lock() 95 | defer s.valueLock.Unlock() 96 | key, newVs := parentSoulAndField{parentSoul, field}, &valueWithState{val, state} 97 | sysState := StateNow() 98 | if s.closed { 99 | return 0, fmt.Errorf("Storage closed") 100 | } else if existingVs := s.values[key]; existingVs == nil && onlyIfExists { 101 | return 0, ErrStorageNotFound 102 | } else if existingVs == nil { 103 | confRes = ConflictResolutionNeverSeenUpdate 104 | } else { 105 | confRes = ConflictResolve(existingVs.val, existingVs.state, val, state, sysState) 106 | } 107 | if confRes == ConflictResolutionTooFutureDeferred { 108 | // Schedule for 100ms past when it's deferred to 109 | time.AfterFunc(time.Duration(state-sysState)*time.Millisecond+100, func() { 110 | s.valueLock.RLock() 111 | closed := s.closed 112 | s.valueLock.RUnlock() 113 | // TODO: what to do w/ error? 114 | if !closed && ctx.Err() == nil { 115 | s.Put(ctx, parentSoul, field, val, state, onlyIfExists) 116 | } 117 | }) 118 | } else if confRes.IsImmediateUpdate() { 119 | s.values[key] = newVs 120 | } 121 | return 122 | } 123 | 124 | func (s *storageInMem) Close() error { 125 | if s.purgeCancelFn != nil { 126 | s.purgeCancelFn() 127 | } 128 | s.valueLock.Lock() 129 | defer s.valueLock.Unlock() 130 | s.closed = true 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /gun/websocket.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "sync" 10 | 11 | "github.com/gorilla/websocket" 12 | ) 13 | 14 | type serverWebSocket struct { 15 | server *http.Server 16 | acceptCh chan *websocket.Conn 17 | acceptCtx context.Context 18 | acceptCancelFn context.CancelFunc 19 | serveErrCh chan error 20 | } 21 | 22 | // NewServerWebSocket creates a new Server for the given HTTP server and given upgrader. 23 | func NewServerWebSocket(server *http.Server, upgrader *websocket.Upgrader) Server { 24 | // TODO: wait, what if they already have a server and want to control serve, close, and handler? 25 | if upgrader == nil { 26 | upgrader = &websocket.Upgrader{} 27 | } 28 | s := &serverWebSocket{ 29 | server: server, 30 | acceptCh: make(chan *websocket.Conn), 31 | serveErrCh: make(chan error, 1), 32 | } 33 | // Setup the accepter 34 | s.acceptCtx, s.acceptCancelFn = context.WithCancel(context.Background()) 35 | server.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 36 | conn, err := upgrader.Upgrade(w, r, nil) 37 | if err != nil { 38 | if server.ErrorLog != nil { 39 | server.ErrorLog.Printf("Failed upgrading websocket: %v", err) 40 | } 41 | return 42 | } 43 | select { 44 | case <-s.acceptCtx.Done(): 45 | case s.acceptCh <- conn: 46 | } 47 | }) 48 | return s 49 | } 50 | 51 | func (s *serverWebSocket) Serve() error { 52 | return s.server.ListenAndServe() 53 | } 54 | 55 | func (s *serverWebSocket) Accept() (PeerConn, error) { 56 | select { 57 | case <-s.acceptCtx.Done(): 58 | return nil, http.ErrServerClosed 59 | case conn := <-s.acceptCh: 60 | return NewPeerConnWebSocket(conn), nil 61 | } 62 | } 63 | 64 | func (s *serverWebSocket) Close() error { 65 | s.acceptCancelFn() 66 | return s.server.Close() 67 | } 68 | 69 | // PeerConnWebSocket implements PeerConn for a websocket connection. 70 | type PeerConnWebSocket struct { 71 | Underlying *websocket.Conn 72 | WriteLock sync.Mutex 73 | } 74 | 75 | // DialPeerConnWebSocket opens a peer websocket connection. 76 | func DialPeerConnWebSocket(ctx context.Context, peerURL *url.URL) (*PeerConnWebSocket, error) { 77 | conn, _, err := websocket.DefaultDialer.DialContext(ctx, peerURL.String(), nil) 78 | if err != nil { 79 | return nil, err 80 | } 81 | return NewPeerConnWebSocket(conn), nil 82 | } 83 | 84 | // NewPeerConnWebSocket wraps an existing websocket connection. 85 | func NewPeerConnWebSocket(underlying *websocket.Conn) *PeerConnWebSocket { 86 | return &PeerConnWebSocket{Underlying: underlying} 87 | } 88 | 89 | // Send implements PeerConn.Send. 90 | func (p *PeerConnWebSocket) Send(ctx context.Context, msg *Message, moreMsgs ...*Message) error { 91 | // If there are more, send all as an array of JSON strings, otherwise just the msg 92 | var toWrite interface{} 93 | if len(moreMsgs) == 0 { 94 | toWrite = msg 95 | } else { 96 | b, err := json.Marshal(msg) 97 | if err != nil { 98 | return err 99 | } 100 | msgs := []string{string(b)} 101 | for _, nextMsg := range moreMsgs { 102 | if b, err = json.Marshal(nextMsg); err != nil { 103 | return err 104 | } 105 | msgs = append(msgs, string(b)) 106 | } 107 | toWrite = msgs 108 | } 109 | // Send async so we can wait on context 110 | errCh := make(chan error, 1) 111 | go func() { 112 | p.WriteLock.Lock() 113 | defer p.WriteLock.Unlock() 114 | errCh <- p.Underlying.WriteJSON(toWrite) 115 | }() 116 | select { 117 | case err := <-errCh: 118 | return err 119 | case <-ctx.Done(): 120 | return ctx.Err() 121 | } 122 | } 123 | 124 | // Receive implements PeerConn.Receive. 125 | func (p *PeerConnWebSocket) Receive(ctx context.Context) ([]*Message, error) { 126 | bytsCh := make(chan []byte, 1) 127 | errCh := make(chan error, 1) 128 | go func() { 129 | if _, b, err := p.Underlying.ReadMessage(); err != nil { 130 | errCh <- err 131 | } else { 132 | bytsCh <- b 133 | } 134 | }() 135 | select { 136 | case err := <-errCh: 137 | return nil, err 138 | case <-ctx.Done(): 139 | return nil, ctx.Err() 140 | case byts := <-bytsCh: 141 | // If it's a JSON array, it means it's an array of JSON strings, otherwise it's one message 142 | if byts[0] != '[' { 143 | var msg Message 144 | if err := json.Unmarshal(byts, &msg); err != nil { 145 | return nil, err 146 | } 147 | return []*Message{&msg}, nil 148 | } 149 | var jsonStrs []string 150 | if err := json.Unmarshal(byts, &jsonStrs); err != nil { 151 | return nil, err 152 | } 153 | msgs := make([]*Message, len(jsonStrs)) 154 | for i, jsonStr := range jsonStrs { 155 | if err := json.Unmarshal([]byte(jsonStr), &(msgs[i])); err != nil { 156 | return nil, err 157 | } 158 | } 159 | return msgs, nil 160 | } 161 | } 162 | 163 | // RemoteURL implements PeerConn.RemoteURL. 164 | func (p *PeerConnWebSocket) RemoteURL() string { 165 | return fmt.Sprintf("http://%v", p.Underlying.RemoteAddr()) 166 | } 167 | 168 | // Close implements PeerConn.Close. 169 | func (p *PeerConnWebSocket) Close() error { 170 | return p.Underlying.Close() 171 | } 172 | -------------------------------------------------------------------------------- /gun/node.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | ) 9 | 10 | // DefaultSoulGen is the default soul generator. It uses the current time in 11 | // MS as the first part of the string. However if that MS was already used in 12 | // this process, the a guaranteed-process-unique-nano-level time is added to 13 | // the first part. The second part is a random string. 14 | func DefaultSoulGen() string { 15 | ms, uniqueNum := timeNowUniqueUnix() 16 | s := strconv.FormatInt(ms, 36) 17 | if uniqueNum > 0 { 18 | s += strconv.FormatInt(uniqueNum, 36) 19 | } 20 | return s + randString(12) 21 | } 22 | 23 | // Node is a JSON-encodable representation of a Gun node which is a set of 24 | // scalar values by name and metadata about those values. 25 | type Node struct { 26 | // Metadata is the metadata for this node. 27 | Metadata 28 | // Values is the set of values (including null) keyed by the field name. 29 | Values map[string]Value 30 | } 31 | 32 | // MarshalJSON implements encoding/json.Marshaler for Node. 33 | func (n *Node) MarshalJSON() ([]byte, error) { 34 | // Just put it all in a map and then encode it 35 | toEnc := make(map[string]interface{}, len(n.Values)+1) 36 | toEnc["_"] = &n.Metadata 37 | for k, v := range n.Values { 38 | toEnc[k] = v 39 | } 40 | return json.Marshal(toEnc) 41 | } 42 | 43 | // UnmarshalJSON implements encoding/json.Unmarshaler for Node. 44 | func (n *Node) UnmarshalJSON(b []byte) error { 45 | dec := json.NewDecoder(bytes.NewReader(b)) 46 | dec.UseNumber() 47 | // We'll just go from start brace to end brace 48 | if t, err := dec.Token(); err != nil { 49 | return err 50 | } else if t != json.Delim('{') { 51 | return fmt.Errorf("Unexpected token %v", t) 52 | } 53 | n.Values = map[string]Value{} 54 | for { 55 | if key, err := dec.Token(); err != nil { 56 | return err 57 | } else if key == json.Delim('}') { 58 | return nil 59 | } else if keyStr, ok := key.(string); !ok { 60 | return fmt.Errorf("Unrecognized token %v", key) 61 | } else if keyStr == "_" { 62 | if err = dec.Decode(&n.Metadata); err != nil { 63 | return fmt.Errorf("Failed unmarshaling metadata: %v", err) 64 | } 65 | } else if val, err := dec.Token(); err != nil { 66 | return err 67 | } else if n.Values[keyStr], err = ValueDecodeJSON(val, dec); err != nil { 68 | return err 69 | } 70 | } 71 | } 72 | 73 | // Metadata is the soul of a node and the state of its values. 74 | type Metadata struct { 75 | // Soul is the unique identifier of this node. 76 | Soul string `json:"#,omitempty"` 77 | // State is the conflict state value for each node field. 78 | State map[string]State `json:">,omitempty"` 79 | } 80 | 81 | // Value is the common interface implemented by all possible Gun values. The 82 | // possible values of Value are nil and instances of ValueNumber, ValueString, 83 | // ValueBool, and ValueRelation. They can all be marshaled into JSON. 84 | type Value interface { 85 | nodeValue() 86 | } 87 | 88 | // ValueDecodeJSON decodes a single Value from JSON. For the given JSON decoder 89 | // and last read token, this decodes a Value. The decoder needs to have ran 90 | // UseNumber. Unrecognized values are errors. 91 | func ValueDecodeJSON(token json.Token, dec *json.Decoder) (Value, error) { 92 | switch token := token.(type) { 93 | case nil: 94 | return nil, nil 95 | case json.Number: 96 | return ValueNumber(token), nil 97 | case string: 98 | return ValueString(token), nil 99 | case bool: 100 | return ValueBool(token), nil 101 | case json.Delim: 102 | if token != json.Delim('{') { 103 | return nil, fmt.Errorf("Unrecognized token %v", token) 104 | } else if relKey, err := dec.Token(); err != nil { 105 | return nil, err 106 | } else if relKey != "#" { 107 | return nil, fmt.Errorf("Unrecognized token %v", relKey) 108 | } else if relVal, err := dec.Token(); err != nil { 109 | return nil, err 110 | } else if relValStr, ok := relVal.(string); !ok { 111 | return nil, fmt.Errorf("Unrecognized token %v", relVal) 112 | } else if endTok, err := dec.Token(); err != nil { 113 | return nil, err 114 | } else if endTok != json.Delim('}') { 115 | return nil, fmt.Errorf("Unrecognized token %v", endTok) 116 | } else { 117 | return ValueRelation(relValStr), nil 118 | } 119 | default: 120 | return nil, fmt.Errorf("Unrecognized token %v", token) 121 | } 122 | } 123 | 124 | // ValueString is a representation of a string Value. 125 | type ValueString string 126 | 127 | func (ValueString) nodeValue() {} 128 | func (v ValueString) String() string { return string(v) } 129 | 130 | // ValueNumber is a representation of a number Value. It is typed as a string 131 | // similar to how encoding/json.Number works since it can overflow numeric 132 | // types. 133 | type ValueNumber string 134 | 135 | func (ValueNumber) nodeValue() {} 136 | func (v ValueNumber) String() string { return string(v) } 137 | 138 | // Float64 returns the number as a float64. 139 | func (v ValueNumber) Float64() (float64, error) { return strconv.ParseFloat(string(v), 64) } 140 | 141 | // Int64 returns the number as an int64. 142 | func (v ValueNumber) Int64() (int64, error) { return strconv.ParseInt(string(v), 10, 64) } 143 | 144 | // ValueBool is a representation of a bool Value. 145 | type ValueBool bool 146 | 147 | func (ValueBool) nodeValue() {} 148 | 149 | // ValueRelation is a representation of a relation Value. The value is the soul 150 | // of the linked node. It has a custom JSON encoding. 151 | type ValueRelation string 152 | 153 | func (ValueRelation) nodeValue() {} 154 | func (v ValueRelation) String() string { return string(v) } 155 | 156 | // MarshalJSON implements encoding/json.Marshaler fr ValueRelation. 157 | func (v ValueRelation) MarshalJSON() ([]byte, error) { 158 | return json.Marshal(map[string]string{"#": string(v)}) 159 | } 160 | -------------------------------------------------------------------------------- /gun/peer.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/url" 7 | "sync" 8 | "time" 9 | ) 10 | 11 | // ErrPeer is an error specific to a peer. 12 | type ErrPeer struct { 13 | // Err is the error. 14 | Err error 15 | // Peer is the peer the error relates to. 16 | Peer *Peer 17 | } 18 | 19 | func (e *ErrPeer) Error() string { return fmt.Sprintf("Error on peer %v: %v", e.Peer, e.Err) } 20 | 21 | // Peer is a known peer to Gun. It has a single connection. Some peers are 22 | // "reconnectable" which means due to failure, they may be in a "bad" state 23 | // awaiting reconnection. 24 | type Peer struct { 25 | name string 26 | newConn func() (PeerConn, error) 27 | sleepOnErr time.Duration // TODO: would be better as backoff 28 | id string 29 | 30 | connCurrent PeerConn 31 | connBad bool // If true, don't try anything 32 | connLock sync.Mutex 33 | } 34 | 35 | func newPeer(name string, newConn func() (PeerConn, error), sleepOnErr time.Duration) (*Peer, error) { 36 | p := &Peer{name: name, newConn: newConn, sleepOnErr: sleepOnErr} 37 | var err error 38 | if p.connCurrent, err = newConn(); err != nil { 39 | return nil, err 40 | } 41 | return p, nil 42 | } 43 | 44 | // ID is the identifier of the peer as given by the peer. It is empty if the 45 | // peer wasn't asked for or didn't give an ID. 46 | func (p *Peer) ID() string { return p.id } 47 | 48 | // Name is the name of the peer which is usually the URL. 49 | func (p *Peer) Name() string { return p.name } 50 | 51 | // String is a string representation of the peer including whether it's 52 | // connected. 53 | func (p *Peer) String() string { 54 | id := "" 55 | if p.id != "" { 56 | id = "(id: " + p.id + ")" 57 | } 58 | connStatus := "disconnected" 59 | if conn := p.Conn(); conn != nil { 60 | connStatus = "connected to " + conn.RemoteURL() 61 | } 62 | return fmt.Sprintf("Peer%v %v (%v)", id, p.name, connStatus) 63 | } 64 | 65 | func (p *Peer) reconnectSupported() bool { 66 | return p.sleepOnErr > 0 67 | } 68 | 69 | func (p *Peer) reconnect() (err error) { 70 | if !p.reconnectSupported() { 71 | return fmt.Errorf("Reconnect not supported") 72 | } 73 | p.connLock.Lock() 74 | defer p.connLock.Unlock() 75 | if p.connCurrent == nil && p.connBad { 76 | p.connBad = false 77 | if p.connCurrent, err = p.newConn(); err != nil { 78 | p.connBad = true 79 | time.AfterFunc(p.sleepOnErr, func() { p.reconnect() }) 80 | } 81 | } 82 | return 83 | } 84 | 85 | // Conn is the underlying PeerConn. This can be nil if the peer is currently 86 | // "bad" or closed. 87 | func (p *Peer) Conn() PeerConn { 88 | p.connLock.Lock() 89 | defer p.connLock.Unlock() 90 | return p.connCurrent 91 | } 92 | 93 | func (p *Peer) markConnErrored(conn PeerConn) { 94 | if !p.reconnectSupported() { 95 | p.Close() 96 | return 97 | } 98 | p.connLock.Lock() 99 | defer p.connLock.Unlock() 100 | if conn == p.connCurrent { 101 | p.connCurrent = nil 102 | p.connBad = true 103 | conn.Close() 104 | time.AfterFunc(p.sleepOnErr, func() { p.reconnect() }) 105 | } 106 | } 107 | 108 | func (p *Peer) send(ctx context.Context, msg *Message, moreMsgs ...*Message) (ok bool, err error) { 109 | conn := p.Conn() 110 | if conn == nil { 111 | return false, nil 112 | } 113 | // Clone them with peer "to" 114 | updatedMsg := &Message{} 115 | *updatedMsg = *msg 116 | updatedMsg.To = conn.RemoteURL() 117 | updatedMoreMsgs := make([]*Message, len(moreMsgs)) 118 | for i, moreMsg := range moreMsgs { 119 | updatedMoreMsg := &Message{} 120 | *updatedMoreMsg = *moreMsg 121 | updatedMoreMsg.To = conn.RemoteURL() 122 | updatedMoreMsgs[i] = updatedMoreMsg 123 | } 124 | if err = conn.Send(ctx, updatedMsg, updatedMoreMsgs...); err != nil { 125 | p.markConnErrored(conn) 126 | return false, err 127 | } 128 | return true, nil 129 | } 130 | 131 | func (p *Peer) receive(ctx context.Context) (ok bool, msgs []*Message, err error) { 132 | if conn := p.Conn(); conn == nil { 133 | return false, nil, nil 134 | } else if msgs, err = conn.Receive(ctx); err != nil { 135 | p.markConnErrored(conn) 136 | return false, nil, err 137 | } else { 138 | return true, msgs, nil 139 | } 140 | } 141 | 142 | // Close closes the peer and the connection is connected. 143 | func (p *Peer) Close() error { 144 | p.connLock.Lock() 145 | defer p.connLock.Unlock() 146 | var err error 147 | if p.connCurrent != nil { 148 | err = p.connCurrent.Close() 149 | p.connCurrent = nil 150 | } 151 | p.connBad = false 152 | return err 153 | } 154 | 155 | // Closed is whether the peer is closed. 156 | func (p *Peer) Closed() bool { 157 | p.connLock.Lock() 158 | defer p.connLock.Unlock() 159 | return p.connCurrent == nil && !p.connBad 160 | } 161 | 162 | // PeerConn is a single peer connection. 163 | type PeerConn interface { 164 | // Send sends the given message (and maybe others) to the peer. The context 165 | // governs just this send. 166 | Send(ctx context.Context, msg *Message, moreMsgs ...*Message) error 167 | // Receive waits for the next message (or set of messages if sent at once) 168 | // from a peer. The context can be used to control a timeout. 169 | Receive(ctx context.Context) ([]*Message, error) 170 | // RemoteURL is the URL this peer is connected via. 171 | RemoteURL() string 172 | // Close closes this connection. 173 | Close() error 174 | } 175 | 176 | // PeerURLSchemes is the map that maps URL schemes to factory functions to 177 | // create the connection. Currently "http" and "https" simply defer to "ws" and 178 | // "wss" respectively. "ws" and "wss" use DialPeerConnWebSocket. 179 | var PeerURLSchemes map[string]func(context.Context, *url.URL) (PeerConn, error) 180 | 181 | func init() { 182 | PeerURLSchemes = map[string]func(context.Context, *url.URL) (PeerConn, error){ 183 | "http": func(ctx context.Context, peerURL *url.URL) (PeerConn, error) { 184 | schemeChangedURL := &url.URL{} 185 | *schemeChangedURL = *peerURL 186 | schemeChangedURL.Scheme = "ws" 187 | return DialPeerConnWebSocket(ctx, schemeChangedURL) 188 | }, 189 | "https": func(ctx context.Context, peerURL *url.URL) (PeerConn, error) { 190 | schemeChangedURL := &url.URL{} 191 | *schemeChangedURL = *peerURL 192 | schemeChangedURL.Scheme = "wss" 193 | return DialPeerConnWebSocket(ctx, schemeChangedURL) 194 | }, 195 | "ws": func(ctx context.Context, peerURL *url.URL) (PeerConn, error) { 196 | return DialPeerConnWebSocket(ctx, peerURL) 197 | }, 198 | "wss": func(ctx context.Context, peerURL *url.URL) (PeerConn, error) { 199 | return DialPeerConnWebSocket(ctx, peerURL) 200 | }, 201 | } 202 | } 203 | 204 | // NewPeerConn connects to a peer for the given URL. 205 | func NewPeerConn(ctx context.Context, peerURL string) (PeerConn, error) { 206 | if parsedURL, err := url.Parse(peerURL); err != nil { 207 | return nil, err 208 | } else if peerNew := PeerURLSchemes[parsedURL.Scheme]; peerNew == nil { 209 | return nil, fmt.Errorf("Unknown peer URL scheme %v", parsedURL.Scheme) 210 | } else { 211 | return peerNew(ctx, parsedURL) 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /gun/scoped_put.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | type putResultListener struct { 9 | id string 10 | results chan *PutResult 11 | receivedMessages chan *messageReceived 12 | } 13 | 14 | // PutResult is either an acknowledgement or an error for a put. 15 | type PutResult struct { 16 | // Err is any error on put, local or remote. This can be a context error 17 | // if the put's context completes. This is nil on successful put. 18 | // 19 | // This may be ErrLookupOnTopLevel for a remote fetch of a top-level field. 20 | // This may be ErrNotObject if the field is a child of a non-relation value. 21 | Err error 22 | // Field is the name of the field that was put. It is a convenience value 23 | // for the scope's field this was originally called on. 24 | Field string 25 | // Peer is the peer this result is for. This is nil for results from local 26 | // storage. This may be nil on error. 27 | Peer *Peer 28 | } 29 | 30 | // PutOption is the base interface for all options that can be passed to Put. 31 | type PutOption interface { 32 | putOption() 33 | } 34 | 35 | type putOptionStoreLocalOnly struct{} 36 | 37 | func (putOptionStoreLocalOnly) putOption() {} 38 | 39 | // PutOptionStoreLocalOnly makes Put only store locally and then be done. 40 | var PutOptionStoreLocalOnly PutOption = putOptionStoreLocalOnly{} 41 | 42 | type putOptionFailWithoutParent struct{} 43 | 44 | func (putOptionFailWithoutParent) putOption() {} 45 | 46 | // PutOptionFailWithoutParent makes Put fail if it would need to lazily create 47 | // parent relations. 48 | var PutOptionFailWithoutParent PutOption = putOptionFailWithoutParent{} 49 | 50 | // Put puts a value on the field in local storage. It also sends the put to all 51 | // peers unless the PutOptionStoreLocalOnly option is present. Each 52 | // acknowledgement or error will be sent to the resulting channel. Unless the 53 | // PutOptionFailWithoutParent option is present, this will lazily create all 54 | // parent relations that do not already exist. This will error if called for a 55 | // top-level field. The resulting channel is closed on Gun close, context 56 | // completion, or when PutDone is called with it. Users should ensure one of the 57 | // three happen in a reasonable timeframe to stop listening for acks and prevent 58 | // leaks. 59 | // 60 | // This is the equivalent of the Gun JS API "put" function. 61 | func (s *Scoped) Put(ctx context.Context, val Value, opts ...PutOption) <-chan *PutResult { 62 | // Collect the options 63 | storeLocalOnly := false 64 | failWithoutParent := false 65 | for _, opt := range opts { 66 | switch opt.(type) { 67 | case putOptionStoreLocalOnly: 68 | storeLocalOnly = true 69 | case putOptionFailWithoutParent: 70 | failWithoutParent = true 71 | } 72 | } 73 | ch := make(chan *PutResult, 1) 74 | // Get all the parents 75 | parents := []*Scoped{} 76 | for next := s.parent; next != nil; next = next.parent { 77 | parents = append([]*Scoped{next}, parents...) 78 | } 79 | if len(parents) == 0 { 80 | ch <- &PutResult{Err: ErrLookupOnTopLevel} 81 | return ch 82 | } 83 | // Ask for the soul on the last parent. What this will do is trigger 84 | // lazy soul fetch up the chain. Then we can go through and find who doesn't have a 85 | // cached soul, create one, and store locally. 86 | if soul, err := parents[len(parents)-1].Soul(ctx); err != nil { 87 | ch <- &PutResult{Err: ErrLookupOnTopLevel, Field: s.field} 88 | return ch 89 | } else if soul == "" && failWithoutParent { 90 | ch <- &PutResult{Err: fmt.Errorf("Parent not present but required"), Field: s.field} 91 | return ch 92 | } 93 | // Now for every parent that doesn't have a cached soul we create one and 94 | // put as part of the message. We accept fetching the cache this way is a bit 95 | // racy. 96 | req := &Message{ 97 | ID: randString(9), 98 | Put: make(map[string]*Node), 99 | } 100 | // We know that the first has a soul 101 | prevParentSoul := parents[0].cachedSoul() 102 | currState := StateNow() 103 | for _, parent := range parents[1:] { 104 | parentCachedSoul := parent.cachedSoul() 105 | if parentCachedSoul == "" { 106 | // Create the soul and make it as part of the next put 107 | parentCachedSoul = s.gun.soulGen() 108 | req.Put[prevParentSoul] = &Node{ 109 | Metadata: Metadata{ 110 | Soul: prevParentSoul, 111 | State: map[string]State{parent.field: currState}, 112 | }, 113 | Values: map[string]Value{parent.field: ValueRelation(parentCachedSoul)}, 114 | } 115 | // Also store locally and set the cached soul 116 | // TODO: Should I not store until the very end just in case it errors halfway 117 | // though? There are no standard cases where it should fail. 118 | if _, err := s.gun.storage.Put(ctx, prevParentSoul, parent.field, ValueRelation(parentCachedSoul), currState, false); err != nil { 119 | ch <- &PutResult{Err: err, Field: s.field} 120 | return ch 121 | } else if !parent.setCachedSoul(ValueRelation(parentCachedSoul)) { 122 | ch <- &PutResult{Err: fmt.Errorf("Concurrent cached soul set"), Field: s.field} 123 | return ch 124 | } 125 | } 126 | prevParentSoul = parentCachedSoul 127 | } 128 | // Now that we've setup all the parents, we can do this store locally 129 | if _, err := s.gun.storage.Put(ctx, prevParentSoul, s.field, val, currState, false); err != nil { 130 | ch <- &PutResult{Err: err, Field: s.field} 131 | return ch 132 | } 133 | // We need an ack for local store and stop if local only 134 | ch <- &PutResult{Field: s.field} 135 | if storeLocalOnly { 136 | return ch 137 | } 138 | // Now, we begin the remote storing 139 | req.Put[prevParentSoul] = &Node{ 140 | Metadata: Metadata{ 141 | Soul: prevParentSoul, 142 | State: map[string]State{s.field: currState}, 143 | }, 144 | Values: map[string]Value{s.field: val}, 145 | } 146 | // Make a msg chan and register it to listen for acks 147 | msgCh := make(chan *messageReceived) 148 | s.putResultListenersLock.Lock() 149 | s.putResultListeners[ch] = &putResultListener{req.ID, ch, msgCh} 150 | s.putResultListenersLock.Unlock() 151 | s.gun.registerMessageIDListener(req.ID, msgCh) 152 | // Start message listener 153 | go func() { 154 | for { 155 | select { 156 | case <-ctx.Done(): 157 | ch <- &PutResult{Err: ctx.Err(), Field: s.field} 158 | s.PutDone(ch) 159 | return 160 | case msg, ok := <-msgCh: 161 | if !ok { 162 | return 163 | } 164 | r := &PutResult{Field: s.field, Peer: msg.peer} 165 | if msg.Err != "" { 166 | r.Err = fmt.Errorf("Remote error: %v", msg.Err) 167 | } else if msg.OK != 1 { 168 | r.Err = fmt.Errorf("Unexpected remote ok value of %v", msg.OK) 169 | } 170 | safePutResultSend(ch, r) 171 | } 172 | } 173 | }() 174 | // Send async, sending back errors 175 | go func() { 176 | for peerErr := range s.gun.send(ctx, req, nil) { 177 | safePutResultSend(ch, &PutResult{ 178 | Err: peerErr.Err, 179 | Field: s.field, 180 | Peer: peerErr.Peer, 181 | }) 182 | } 183 | }() 184 | return ch 185 | } 186 | 187 | // PutDone is called with a channel returned from Put to stop 188 | // listening for acks and close the channel. It returns true if it actually 189 | // stopped listening or false if it wasn't listening. 190 | func (s *Scoped) PutDone(ch <-chan *PutResult) bool { 191 | s.putResultListenersLock.Lock() 192 | l := s.putResultListeners[ch] 193 | delete(s.putResultListeners, ch) 194 | s.putResultListenersLock.Unlock() 195 | if l != nil { 196 | // Unregister the chan 197 | s.gun.unregisterMessageIDListener(l.id) 198 | // Close the message chan and the result chan 199 | close(l.receivedMessages) 200 | close(l.results) 201 | } 202 | return l != nil 203 | } 204 | 205 | func safePutResultSend(ch chan<- *PutResult, r *PutResult) { 206 | // Due to the fact that we may send on a closed channel here, we ignore the panic 207 | defer func() { recover() }() 208 | ch <- r 209 | } 210 | -------------------------------------------------------------------------------- /gun/scoped_fetch.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | ) 7 | 8 | type fetchResultListener struct { 9 | id string 10 | parentSoul string 11 | results chan *FetchResult 12 | receivedMessages chan *messageReceived 13 | } 14 | 15 | // FetchResult is a result of a fetch. 16 | type FetchResult struct { 17 | // Err is any error on fetch, local or remote. This can be a context error 18 | // if the fetch's context completes. This is nil on successful fetch. 19 | // 20 | // This may be ErrLookupOnTopLevel for a remote fetch of a top-level field. 21 | // This may be ErrNotObject if the field is a child of a non-relation value. 22 | Err error 23 | // Field is the name of the field that was fetched. It is a convenience 24 | // value for the scope's field this was originally called on. 25 | Field string 26 | // Value is the fetched value. This will be nil if Err is not nil. This will 27 | // also be nil if a peer said the value does not exist. This will also be 28 | // nil if the value exists but is nil. Use ValueExists to distinguish 29 | // between the last two cases. 30 | Value Value 31 | // State is the conflict state of the value. It may be 0 for errors. It may 32 | // also be 0 if this is a top-level field. 33 | State State 34 | // ValueExists is true if there is no error and the fetched value, nil or 35 | // not, does exist. 36 | ValueExists bool 37 | // Peer is the peer this result is for. This is nil for results from local 38 | // storage. This may be nil on error. 39 | Peer *Peer 40 | } 41 | 42 | // FetchOne fetches a single result, trying local first. It is a shortcut for 43 | // calling FetchOneLocal and if it doesn't exist falling back to FetchOneRemote. 44 | // This should not be called on a top-level field. The context can be used to 45 | // timeout the wait. 46 | // 47 | // This is the equivalent of the Gun JS API "once" function. 48 | func (s *Scoped) FetchOne(ctx context.Context) *FetchResult { 49 | // Try local before remote 50 | if r := s.FetchOneLocal(ctx); r.Err != nil || r.ValueExists { 51 | return r 52 | } 53 | return s.FetchOneRemote(ctx) 54 | } 55 | 56 | // FetchOneLocal gets a local value from storage. For top-level fields, it 57 | // simply returns the field name as a relation. 58 | func (s *Scoped) FetchOneLocal(ctx context.Context) *FetchResult { 59 | // If there is no parent, this is just the relation 60 | if s.parent == nil { 61 | return &FetchResult{Field: s.field, Value: ValueRelation(s.field), ValueExists: true} 62 | } 63 | r := &FetchResult{Field: s.field} 64 | // Need parent soul for lookup 65 | var parentSoul string 66 | if parentSoul, r.Err = s.parent.Soul(ctx); r.Err == nil { 67 | if r.Value, r.State, r.Err = s.gun.storage.Get(ctx, parentSoul, s.field); r.Err == ErrStorageNotFound { 68 | r.Err = nil 69 | } else if r.Err == nil { 70 | r.ValueExists = true 71 | } 72 | } 73 | return r 74 | } 75 | 76 | // FetchOneRemote fetches a single result from the first peer that responds. It 77 | // will not look in storage first. This will error if called for a top-level 78 | // field. This is a shortcut for calling FetchRemote, waiting for a single 79 | // value, then calling FetchDone. The context can be used to timeout the wait. 80 | func (s *Scoped) FetchOneRemote(ctx context.Context) *FetchResult { 81 | if s.parent == nil { 82 | return &FetchResult{Err: ErrLookupOnTopLevel, Field: s.field} 83 | } 84 | ch := s.FetchRemote(ctx) 85 | defer s.FetchDone(ch) 86 | return <-ch 87 | } 88 | 89 | // Fetch fetches and listens for updates on the field and sends them to the 90 | // resulting channel. This will error if called for a top-level field. The 91 | // resulting channel is closed on Gun close, context completion, or when 92 | // FetchDone is called with it. Users should ensure one of the three happen in a 93 | // reasonable timeframe to stop listening and prevent leaks. This is a shortcut 94 | // for FetchOneLocal (but doesn't send to channel if doesn't exist) followed by 95 | // FetchRemote. 96 | // 97 | // This is the equivalent of the Gun JS API "on" function. 98 | func (s *Scoped) Fetch(ctx context.Context) <-chan *FetchResult { 99 | ch := make(chan *FetchResult, 1) 100 | if s.parent == nil { 101 | ch <- &FetchResult{Err: ErrLookupOnTopLevel, Field: s.field} 102 | close(ch) 103 | } else { 104 | if r := s.FetchOneLocal(ctx); r.Err != nil || r.ValueExists { 105 | ch <- r 106 | } 107 | go s.fetchRemote(ctx, ch) 108 | } 109 | return ch 110 | } 111 | 112 | // FetchRemote fetches and listens for updates on a field only from peers, not 113 | // via storage. This will error if called for a top-level field. The resulting 114 | // channel is closed on Gun close, context completion, or when FetchDone is 115 | // called with it. Users should ensure one of the three happen in a reasonable 116 | // timeframe to stop listening and prevent leaks. 117 | func (s *Scoped) FetchRemote(ctx context.Context) <-chan *FetchResult { 118 | ch := make(chan *FetchResult, 1) 119 | if s.parent == nil { 120 | ch <- &FetchResult{Err: ErrLookupOnTopLevel, Field: s.field} 121 | } else { 122 | go s.fetchRemote(ctx, ch) 123 | } 124 | return ch 125 | } 126 | 127 | func (s *Scoped) fetchRemote(ctx context.Context, ch chan *FetchResult) { 128 | if s.parent == nil { 129 | panic("No parent") 130 | } 131 | // We have to get the parent soul first 132 | parentSoul, err := s.parent.Soul(ctx) 133 | if err != nil { 134 | ch <- &FetchResult{Err: ErrLookupOnTopLevel, Field: s.field} 135 | return 136 | } 137 | // Create get request 138 | req := &Message{ 139 | ID: randString(9), 140 | Get: &MessageGetRequest{Soul: parentSoul, Field: s.field}, 141 | } 142 | // Make a chan to listen for received messages and link it to 143 | // the given one so we can turn it "off". Off will close this 144 | // chan. 145 | msgCh := make(chan *messageReceived) 146 | s.fetchResultListenersLock.Lock() 147 | s.fetchResultListeners[ch] = &fetchResultListener{req.ID, parentSoul, ch, msgCh} 148 | s.fetchResultListenersLock.Unlock() 149 | // Listen for responses to this get 150 | s.gun.registerMessageIDListener(req.ID, msgCh) 151 | s.gun.registerMessageSoulListener(parentSoul, msgCh) 152 | // TODO: Also listen for any changes to the value or just for specific requests? 153 | // Handle received messages turning them to value fetches 154 | var lastSeenValue Value 155 | var lastSeenState State 156 | go func() { 157 | for { 158 | select { 159 | case <-ctx.Done(): 160 | ch <- &FetchResult{Err: ctx.Err(), Field: s.field} 161 | s.FetchDone(ch) 162 | return 163 | case msg, ok := <-msgCh: 164 | if !ok { 165 | return 166 | } 167 | r := &FetchResult{Field: s.field, Peer: msg.peer} 168 | // We asked for a single field, should only get that field or it doesn't exist 169 | if msg.Err != "" { 170 | r.Err = fmt.Errorf("Remote error: %v", msg.Err) 171 | } else if n := msg.Put[parentSoul]; n != nil { 172 | if newVal, ok := n.Values[s.field]; ok { 173 | newState := n.State[s.field] 174 | // Dedupe the value 175 | if lastSeenValue == newVal && lastSeenState == newState { 176 | continue 177 | } 178 | // If we're storing only what we requested (we do "everything" at a higher level), do it here 179 | // and only send result if it was an update. Otherwise only do it if we would have done one. 180 | confRes := ConflictResolutionNeverSeenUpdate 181 | if s.gun.tracking == TrackingRequested { 182 | // Wait, wait, we may have already stored this 183 | alreadyStored := false 184 | for _, storedField := range msg.storedPuts[parentSoul] { 185 | if storedField == s.field { 186 | alreadyStored = true 187 | break 188 | } 189 | } 190 | if !alreadyStored { 191 | confRes, r.Err = s.gun.storage.Put(ctx, parentSoul, s.field, newVal, newState, false) 192 | } 193 | } else if lastSeenState > 0 { 194 | confRes = ConflictResolve(lastSeenValue, lastSeenState, newVal, newState, StateNow()) 195 | } 196 | // If there are no errors and it was an update, update the last seen and set the response vals 197 | if r.Err == nil && confRes.IsImmediateUpdate() { 198 | lastSeenValue, lastSeenState = newVal, newState 199 | r.Value, r.State, r.ValueExists = newVal, newState, true 200 | } 201 | } 202 | } 203 | safeFetchResultSend(ch, r) 204 | } 205 | } 206 | }() 207 | // Send async, sending back errors 208 | go func() { 209 | for peerErr := range s.gun.send(ctx, req, nil) { 210 | safeFetchResultSend(ch, &FetchResult{ 211 | Err: peerErr.Err, 212 | Field: s.field, 213 | Peer: peerErr.Peer, 214 | }) 215 | } 216 | }() 217 | } 218 | 219 | // FetchDone is called with a channel returned from Fetch or FetchRemote to stop 220 | // listening and close the channel. It returns true if it actually stopped 221 | // listening or false if it wasn't listening. 222 | func (s *Scoped) FetchDone(ch <-chan *FetchResult) bool { 223 | s.fetchResultListenersLock.Lock() 224 | l := s.fetchResultListeners[ch] 225 | delete(s.fetchResultListeners, ch) 226 | s.fetchResultListenersLock.Unlock() 227 | if l != nil { 228 | // Unregister the chan 229 | s.gun.unregisterMessageIDListener(l.id) 230 | s.gun.unregisterMessageSoulListener(l.parentSoul) 231 | // Close the message chan and the result chan 232 | close(l.receivedMessages) 233 | close(l.results) 234 | } 235 | return l != nil 236 | } 237 | 238 | func safeFetchResultSend(ch chan<- *FetchResult, r *FetchResult) { 239 | // Due to the fact that we may send on a closed channel here, we ignore the panic 240 | defer func() { recover() }() 241 | ch <- r 242 | } 243 | -------------------------------------------------------------------------------- /gun/gun.go: -------------------------------------------------------------------------------- 1 | package gun 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | ) 9 | 10 | // Gun is the main client/server instance for the database. 11 | type Gun struct { 12 | // currentPeers is the up-to-date peer list. Do not access this without 13 | // using the associated lock. This slice is never mutated, only replaced. 14 | // There are methods to get and change the list. 15 | currentPeers []*Peer 16 | currentPeersLock sync.RWMutex 17 | 18 | storage Storage 19 | soulGen func() string 20 | peerErrorHandler func(*ErrPeer) 21 | peerSleepOnError time.Duration 22 | myPeerID string 23 | tracking Tracking 24 | 25 | serversCancelFn context.CancelFunc 26 | 27 | messageIDListeners map[string]chan<- *messageReceived 28 | messageIDListenersLock sync.RWMutex 29 | 30 | messageSoulListeners map[string]chan<- *messageReceived 31 | messageSoulListenersLock sync.RWMutex 32 | } 33 | 34 | // Config is the configuration for a Gun instance. 35 | type Config struct { 36 | // PeerURLs is the initial set of peer URLs to connect to. This can be empty 37 | // to not connect to any peers. 38 | PeerURLs []string 39 | // Servers is the set of local servers to listen for new peers on. This can 40 | // be empty to not run any server. 41 | Servers []Server 42 | // Storage is the backend storage to locally store data. If not present, a 43 | // NewStorageInMem is created with a value expiration of 44 | // DefaultOldestAllowedStorageValue. 45 | Storage Storage 46 | // SoulGen is a generator for new soul values. If not present, 47 | // DefaultSoulGen is used. 48 | SoulGen func() string 49 | // PeerErrorHandler is called on every error from or concerning a peer. Not 50 | // required. 51 | PeerErrorHandler func(*ErrPeer) 52 | // PeerSleepOnError is the amount of time we will consider a reconnectable 53 | // peer "bad" on error before attempting reconnect. If 0 (i.e. not set), 54 | // DefaultPeerSleepOnError is used. 55 | PeerSleepOnError time.Duration 56 | // MyPeerID is the identifier given for this peer to whoever asks. If empty 57 | // it is set to a random string 58 | MyPeerID string 59 | // Tracking is how seen values are updated when they are seen on the wire. 60 | // When set to TrackingNothing, no seen values will be updated. When set to 61 | // TrackingRequested (the default), only values that are already in storage 62 | // will be updated when seen. When set to TrackingEverything, all values 63 | // seen on the wire will be stored. 64 | Tracking Tracking 65 | } 66 | 67 | // Tracking is what to do when a value update is seen on the wire. 68 | type Tracking int 69 | 70 | const ( 71 | // TrackingRequested means any value we have already stored will be updated 72 | // when seen. 73 | TrackingRequested Tracking = iota 74 | // TrackingNothing means no values will be updated. 75 | TrackingNothing 76 | // TrackingEverything means every value update is stored. 77 | TrackingEverything 78 | ) 79 | 80 | // DefaultPeerSleepOnError is the default amount of time that an errored 81 | // reconnectable peer will wait until trying to reconnect. 82 | const DefaultPeerSleepOnError = 30 * time.Second 83 | 84 | // DefaultOldestAllowedStorageValue is the default NewStorageInMem expiration. 85 | const DefaultOldestAllowedStorageValue = 7 * (60 * time.Minute) 86 | 87 | // New creates a new Gun instance for the given config. The given context is 88 | // used for all peer connection reconnects now/later, all server servings, and 89 | // peer message handling. Therefore it should be kept available at least until 90 | // Close. Callers must still call Close on complete, the completion of the 91 | // context does not automatically do it. 92 | // 93 | // This returns nil with an error on any initial peer connection failure (and 94 | // cleans up any other peers temporarily connected to). 95 | func New(ctx context.Context, config Config) (*Gun, error) { 96 | g := &Gun{ 97 | currentPeers: make([]*Peer, len(config.PeerURLs)), 98 | storage: config.Storage, 99 | soulGen: config.SoulGen, 100 | peerErrorHandler: config.PeerErrorHandler, 101 | peerSleepOnError: config.PeerSleepOnError, 102 | myPeerID: config.MyPeerID, 103 | tracking: config.Tracking, 104 | messageIDListeners: map[string]chan<- *messageReceived{}, 105 | messageSoulListeners: map[string]chan<- *messageReceived{}, 106 | } 107 | // Set config defaults 108 | if g.storage == nil { 109 | g.storage = NewStorageInMem(DefaultOldestAllowedStorageValue) 110 | } 111 | if g.soulGen == nil { 112 | g.soulGen = DefaultSoulGen 113 | } 114 | if g.peerSleepOnError == 0 { 115 | g.peerSleepOnError = DefaultPeerSleepOnError 116 | } 117 | if g.myPeerID == "" { 118 | g.myPeerID = randString(9) 119 | } 120 | // Create all the peers 121 | var err error 122 | for i := 0; i < len(config.PeerURLs) && err == nil; i++ { 123 | peerURL := config.PeerURLs[i] 124 | newConn := func() (PeerConn, error) { return NewPeerConn(ctx, peerURL) } 125 | if g.currentPeers[i], err = newPeer(peerURL, newConn, g.peerSleepOnError); err != nil { 126 | err = fmt.Errorf("Failed connecting to peer %v: %v", peerURL, err) 127 | } 128 | } 129 | // If there was an error, we need to close what we did create 130 | if err != nil { 131 | for _, peer := range g.currentPeers { 132 | if peer != nil { 133 | peer.Close() 134 | } 135 | } 136 | return nil, err 137 | } 138 | // Start receiving from peers 139 | for _, peer := range g.currentPeers { 140 | go g.startReceiving(ctx, peer) 141 | } 142 | // Start all the servers 143 | go g.startServers(ctx, config.Servers) 144 | return g, nil 145 | } 146 | 147 | // Scoped returns a scoped instance to the given field and children for reading 148 | // and writing. This is the equivalent of calling the Gun JS API "get" function 149 | // (sans callback) for each field/child. 150 | func (g *Gun) Scoped(ctx context.Context, field string, children ...string) *Scoped { 151 | s := newScoped(g, nil, field) 152 | if len(children) > 0 { 153 | s = s.Scoped(ctx, children[0], children[1:]...) 154 | } 155 | return s 156 | } 157 | 158 | // Close stops all peers and servers and closes the underlying storage. 159 | func (g *Gun) Close() error { 160 | var errs []error 161 | for _, p := range g.peers() { 162 | if err := p.Close(); err != nil { 163 | errs = append(errs, err) 164 | } 165 | } 166 | g.serversCancelFn() 167 | if err := g.storage.Close(); err != nil { 168 | errs = append(errs, err) 169 | } 170 | if len(errs) == 0 { 171 | return nil 172 | } else if len(errs) == 1 { 173 | return errs[0] 174 | } else { 175 | return fmt.Errorf("Multiple errors: %v", errs) 176 | } 177 | } 178 | 179 | func (g *Gun) peers() []*Peer { 180 | g.currentPeersLock.RLock() 181 | defer g.currentPeersLock.RUnlock() 182 | return g.currentPeers 183 | } 184 | 185 | func (g *Gun) addPeer(p *Peer) { 186 | g.currentPeersLock.Lock() 187 | defer g.currentPeersLock.Unlock() 188 | prev := g.currentPeers 189 | g.currentPeers = make([]*Peer, len(prev)+1) 190 | copy(g.currentPeers, prev) 191 | g.currentPeers[len(prev)] = p 192 | } 193 | 194 | func (g *Gun) removePeer(p *Peer) { 195 | g.currentPeersLock.Lock() 196 | defer g.currentPeersLock.Unlock() 197 | prev := g.currentPeers 198 | g.currentPeers = make([]*Peer, 0, len(prev)) 199 | for _, peer := range prev { 200 | if peer != p { 201 | g.currentPeers = append(g.currentPeers, peer) 202 | } 203 | } 204 | } 205 | 206 | func (g *Gun) send(ctx context.Context, msg *Message, ignorePeer *Peer) <-chan *ErrPeer { 207 | peers := g.peers() 208 | ch := make(chan *ErrPeer, len(peers)) 209 | // Everything async 210 | go func() { 211 | defer close(ch) 212 | var wg sync.WaitGroup 213 | for _, peer := range peers { 214 | if peer == ignorePeer { 215 | continue 216 | } 217 | wg.Add(1) 218 | go func(peer *Peer) { 219 | defer wg.Done() 220 | // Just do nothing if the peer is bad and we couldn't send 221 | if _, err := peer.send(ctx, msg); err != nil { 222 | if !peer.reconnectSupported() { 223 | g.removePeer(peer) 224 | } 225 | peerErr := &ErrPeer{err, peer} 226 | go g.onPeerError(peerErr) 227 | ch <- peerErr 228 | } 229 | }(peer) 230 | } 231 | wg.Wait() 232 | }() 233 | return ch 234 | } 235 | 236 | func (g *Gun) startReceiving(ctx context.Context, peer *Peer) { 237 | ctx, cancelFn := context.WithCancel(ctx) 238 | defer cancelFn() 239 | for !peer.Closed() && ctx.Err() == nil { 240 | // We might not be able receive because peer is sleeping from 241 | // an error happened within or a just-before send error. 242 | if ok, msgs, err := peer.receive(ctx); !ok { 243 | if err != nil { 244 | go g.onPeerError(&ErrPeer{err, peer}) 245 | } 246 | // If can reconnect, sleep at least the err duration, otherwise remove 247 | if peer.reconnectSupported() { 248 | time.Sleep(g.peerSleepOnError) 249 | } else { 250 | g.removePeer(peer) 251 | } 252 | } else { 253 | // Go over each message and see if it needs delivering or rebroadcasting 254 | for _, msg := range msgs { 255 | g.onPeerMessage(ctx, &messageReceived{Message: msg, peer: peer}) 256 | } 257 | } 258 | } 259 | } 260 | 261 | func (g *Gun) onPeerMessage(ctx context.Context, msg *messageReceived) { 262 | // TODO: 263 | // * if message-acks are not considered part of a store-all server's storage, then use msg.Ack 264 | // to determine whether we even put here instead of how we do it now. 265 | // * handle gets 266 | 267 | // If we're tracking anything, we try to put it (may only be if exists). 268 | if g.tracking != TrackingNothing { 269 | // If we're tracking everything, we persist everything. Otherwise if we're 270 | // only tracking requested, we persist only if it already exists. 271 | putOnlyIfExists := g.tracking == TrackingRequested 272 | for parentSoul, node := range msg.Put { 273 | for field, value := range node.Values { 274 | if state, ok := node.Metadata.State[field]; ok { 275 | // TODO: warn on other error or something 276 | _, err := g.storage.Put(ctx, parentSoul, field, value, state, putOnlyIfExists) 277 | if err == nil { 278 | if msg.storedPuts == nil { 279 | msg.storedPuts = map[string][]string{} 280 | } 281 | msg.storedPuts[parentSoul] = append(msg.storedPuts[parentSoul], field) 282 | } 283 | } 284 | } 285 | } 286 | } 287 | 288 | // If there is a listener for this message ID, use it and consider the message handled 289 | if msg.Ack != "" { 290 | g.messageIDListenersLock.RLock() 291 | l := g.messageIDListeners[msg.Ack] 292 | g.messageIDListenersLock.RUnlock() 293 | if l != nil { 294 | go safeReceivedMessageSend(l, msg) 295 | return 296 | } 297 | } 298 | 299 | // If there are listeners for any of the souls, use them but don't consider the message handled 300 | for parentSoul := range msg.Put { 301 | g.messageSoulListenersLock.RLock() 302 | l := g.messageSoulListeners[parentSoul] 303 | g.messageSoulListenersLock.RUnlock() 304 | if l != nil { 305 | go safeReceivedMessageSend(l, msg) 306 | } 307 | } 308 | 309 | // DAM messages are either requests for our ID or setting of theirs 310 | if msg.DAM != "" { 311 | if msg.PID == "" { 312 | // This is a request, set the PID and send it back 313 | msg.PID = g.myPeerID 314 | if _, err := msg.peer.send(ctx, msg.Message); err != nil { 315 | go g.onPeerError(&ErrPeer{err, msg.peer}) 316 | if !msg.peer.reconnectSupported() { 317 | g.removePeer(msg.peer) 318 | } 319 | } 320 | } else { 321 | // This is them telling us theirs 322 | msg.peer.id = msg.PID 323 | } 324 | return 325 | } 326 | 327 | // Unhandled message means rebroadcast 328 | g.send(ctx, msg.Message, msg.peer) 329 | } 330 | 331 | func (g *Gun) onPeerError(err *ErrPeer) { 332 | if g.peerErrorHandler != nil { 333 | g.peerErrorHandler(err) 334 | } 335 | } 336 | 337 | func (g *Gun) registerMessageIDListener(id string, ch chan<- *messageReceived) { 338 | g.messageIDListenersLock.Lock() 339 | defer g.messageIDListenersLock.Unlock() 340 | g.messageIDListeners[id] = ch 341 | } 342 | 343 | func (g *Gun) unregisterMessageIDListener(id string) { 344 | g.messageIDListenersLock.Lock() 345 | defer g.messageIDListenersLock.Unlock() 346 | delete(g.messageIDListeners, id) 347 | } 348 | 349 | func (g *Gun) registerMessageSoulListener(soul string, ch chan<- *messageReceived) { 350 | g.messageSoulListenersLock.Lock() 351 | defer g.messageSoulListenersLock.Unlock() 352 | g.messageSoulListeners[soul] = ch 353 | } 354 | 355 | func (g *Gun) unregisterMessageSoulListener(soul string) { 356 | g.messageSoulListenersLock.Lock() 357 | defer g.messageSoulListenersLock.Unlock() 358 | delete(g.messageSoulListeners, soul) 359 | } 360 | 361 | func safeReceivedMessageSend(ch chan<- *messageReceived, msg *messageReceived) { 362 | // Due to the fact that we may send on a closed channel here, we ignore the panic 363 | defer func() { recover() }() 364 | ch <- msg 365 | } 366 | --------------------------------------------------------------------------------