├── README.md ├── batch.go ├── buffer.go ├── buffer_test.go ├── client.go ├── client_test.go ├── cmap.go ├── go.mod ├── go.sum └── object.go /README.md: -------------------------------------------------------------------------------- 1 | # objects-go 2 | 3 | Segment objects client for Go. For additional documentation view the [godocs](http://godoc.org/github.com/segmentio/objects-go). For documentation about our Go analytics library, visit [here](https://segment.com/docs/libraries/go). 4 | 5 | ## Description 6 | Segment’s Objects API allows you to send stateful business objects right to Redshift and other Segment supported data warehouses. These objects can be anything that is relevant to your business: products in your product catalogs, partners on your platform, articles on your blog, etc. 7 | 8 | The Objects API lets you `set` custom objects in your own data warehouse. 9 | 10 | ```go 11 | // First call to Set 12 | Client.Set(&objects.Object{ 13 | ID: "room1000", 14 | Collection: "rooms" 15 | Properties: map[string]interface{}{ 16 | "name": "Charming Beach Room Facing Ocean", 17 | "location": "Lihue, HI", 18 | "review_count": 47, 19 | }) 20 | 21 | // Second call on the same object 22 | Client.Set(*objects.Object{ 23 | ID: "room1000", 24 | Collection: "rooms" 25 | Properties: map[string]interface{}{ 26 | "owner": "Calvin", 27 | "public_listing": true, 28 | }) 29 | 30 | // Make sure objects are flushed before your main goroutine exits 31 | Client.Close() 32 | ``` 33 | 34 | This call makes the objects available in your data warehouse… 35 | 36 | ```SQL 37 | select id, name, location, review_count, owner, public_listing from hotel.rooms 38 | ``` 39 | 40 | ..which will return… 41 | 42 | ```CSV 43 | 'room1000' | 'Charming Beach Room Facing Ocean' | 'Lihue, HI' | 47 | "Calvin" | true 44 | ``` 45 | 46 | > All objects will be flattened using the `go-tableize` library. Objects API doesn't allow nested objects, empty objects, and only allows strings, numeric types or booleans as values. 47 | 48 | ## HTTP API 49 | 50 | There is a single `.set` HTTP API endpoint that you'll use to send data to Segment. 51 | 52 | 53 | POST https://objects.segment.com/v1/set 54 | 55 | with the following payload: 56 | 57 | 58 | { 59 | "collection": "rooms", 60 | "objects": [ 61 | { 62 | "id": "2561341", 63 | "properties": { 64 | "name": "Charming Beach Room Facing Ocean", 65 | "location": "Lihue, HI", 66 | "review_count": 47 67 | } 68 | }, { 69 | "id": "2561342", 70 | "properties": { 71 | "name": "College town feel — plenty of bars nearby", 72 | "location": "Austin, TX", 73 | "review_count": 32 74 | } 75 | } 76 | ] 77 | } 78 | 79 | Here’s a `curl` example of how to get started: 80 | 81 | 82 | curl https://objects.segment.com/v1/set \ 83 | -u PROJECT_WRITE_KEY: \ 84 | -H 'Content-Type: application/json' \ 85 | -X POST -d '{"collection":"rooms","objects":[{"id": "2561341","properties": {"name": "Charming Beach Room Facing Ocean","location":"Lihue, HI","review_count":47}}]}' 86 | 87 | 88 | ## License 89 | 90 | MIT 91 | -------------------------------------------------------------------------------- /batch.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import "encoding/json" 4 | 5 | type batch struct { 6 | Source string `json:"source"` 7 | Collection string `json:"collection"` 8 | WriteKey string `json:"write_key"` 9 | Objects json.RawMessage `json:"objects"` 10 | } 11 | -------------------------------------------------------------------------------- /buffer.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | ) 7 | 8 | type buffer struct { 9 | Channel chan *Object 10 | Exit chan struct{} 11 | collection string 12 | buf [][]byte 13 | currentByteSize int 14 | } 15 | 16 | func newBuffer(collection string) *buffer { 17 | return &buffer{ 18 | collection: collection, 19 | Channel: make(chan *Object, 100), 20 | Exit: make(chan struct{}), 21 | buf: [][]byte{}, 22 | currentByteSize: 0, 23 | } 24 | } 25 | 26 | func (b *buffer) add(x []byte) { 27 | b.buf = append(b.buf, x) 28 | b.currentByteSize += len(x) 29 | } 30 | 31 | func (b *buffer) size() int { 32 | return b.currentByteSize 33 | } 34 | 35 | func (b *buffer) count() int { 36 | return len(b.buf) 37 | } 38 | 39 | func (b *buffer) reset() { 40 | b.buf = [][]byte{} 41 | b.currentByteSize = 0 42 | } 43 | 44 | func (b *buffer) marshalArray() json.RawMessage { 45 | rm := bytes.Join(b.buf, []byte{','}) 46 | rm = append([]byte{'['}, rm...) 47 | rm = append(rm, ']') 48 | return json.RawMessage(rm) 49 | } 50 | -------------------------------------------------------------------------------- /buffer_test.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/suite" 8 | ) 9 | 10 | func TestBuffer(t *testing.T) { 11 | suite.Run(t, &BufferTestSuite{}) 12 | } 13 | 14 | type BufferTestSuite struct { 15 | suite.Suite 16 | } 17 | 18 | func (b *BufferTestSuite) TestNewBuffer() { 19 | buf := newBuffer("collection") 20 | b.NotNil(buf) 21 | 22 | b.Equal("collection", buf.collection) 23 | b.Equal(0, buf.count()) 24 | b.Equal(0, buf.size()) 25 | b.Equal(0, buf.currentByteSize) 26 | b.Len(buf.buf, 0) 27 | b.NotNil(buf.Channel) 28 | b.NotNil(buf.Exit) 29 | } 30 | 31 | func (b *BufferTestSuite) TestAddNew() { 32 | buf := newBuffer("collection") 33 | b.NotNil(buf) 34 | json1 := []byte(`{"string": "test", "int": 1}`) 35 | buf.add(json1) 36 | 37 | b.Equal(1, buf.count()) 38 | b.Equal(len(json1), buf.size()) 39 | } 40 | 41 | func (b *BufferTestSuite) TestMarshalEmptyArray() { 42 | buf := newBuffer("collection") 43 | b.NotNil(buf) 44 | res := buf.marshalArray() 45 | b.Equal("[]", string(res)) 46 | 47 | v := []map[string]interface{}{} 48 | b.NoError(json.Unmarshal(res, &v)) 49 | b.Len(v, 0) 50 | } 51 | 52 | func (b *BufferTestSuite) TestMarshalSingleArray() { 53 | buf := newBuffer("collection") 54 | b.NotNil(buf) 55 | json1 := []byte(`{"string": "test", "int": 1}`) 56 | buf.add(json1) 57 | 58 | res := buf.marshalArray() 59 | b.Equal(`[`+string(json1)+`]`, string(res)) 60 | 61 | v := []map[string]interface{}{} 62 | b.NoError(json.Unmarshal(res, &v)) 63 | b.Len(v, 1) 64 | b.Equal(v[0]["string"], "test") 65 | b.Equal(v[0]["int"], float64(1)) // json package uses float64 for all json numbers by default 66 | } 67 | 68 | func (b *BufferTestSuite) TestAddMultiple() { 69 | buf := newBuffer("collection") 70 | b.NotNil(buf) 71 | json1 := []byte(`{"string": "test", "int": 1}`) 72 | buf.add(json1) 73 | 74 | b.Equal(1, buf.count()) 75 | b.Equal(len(json1), buf.size()) 76 | 77 | json2 := []byte(`{"string": "test", "int": 46}`) 78 | buf.add(json2) 79 | 80 | b.Equal(2, buf.count()) 81 | b.Equal(len(json1)+len(json2), buf.size()) 82 | 83 | json3 := []byte(`{"string": "test_3", "int": 1000}`) 84 | buf.add(json3) 85 | 86 | json4 := []byte(`{"string": "test_4", "float": -1.0}`) 87 | buf.add(json4) 88 | 89 | b.Equal(4, buf.count()) 90 | b.Equal(len(json1)+len(json2)+len(json3)+len(json4), buf.size()) 91 | } 92 | 93 | func (b *BufferTestSuite) TestAddMultipleReset() { 94 | buf := newBuffer("collection") 95 | b.NotNil(buf) 96 | json1 := []byte(`{"string": "test", "int": 1}`) 97 | buf.add(json1) 98 | 99 | b.Equal(1, buf.count()) 100 | b.Equal(len(json1), buf.size()) 101 | 102 | json2 := []byte(`{"string": "test", "int": 46}`) 103 | buf.add(json2) 104 | 105 | b.Equal(2, buf.count()) 106 | b.Equal(len(json1)+len(json2), buf.size()) 107 | 108 | json3 := []byte(`{"string": "test_3", "int": 1000}`) 109 | buf.add(json3) 110 | 111 | b.Equal(3, buf.count()) 112 | b.Equal(len(json1)+len(json2)+len(json3), buf.size()) 113 | 114 | buf.reset() 115 | b.Equal(0, buf.count()) 116 | b.Equal(0, buf.size()) 117 | b.Equal(0, buf.currentByteSize) 118 | } 119 | 120 | func (b *BufferTestSuite) TestAddMultipleMarshalReset() { 121 | buf := newBuffer("collection") 122 | b.NotNil(buf) 123 | json1 := []byte(`{"string": "test", "int": 1}`) 124 | buf.add(json1) 125 | 126 | b.Equal(1, buf.count()) 127 | b.Equal(len(json1), buf.size()) 128 | 129 | json2 := []byte(`{"string": "test", "int": 46}`) 130 | buf.add(json2) 131 | 132 | b.Equal(2, buf.count()) 133 | b.Equal(len(json1)+len(json2), buf.size()) 134 | 135 | json3 := []byte(`{"string": "test_3", "int": -1.0}`) 136 | buf.add(json3) 137 | 138 | b.Equal(3, buf.count()) 139 | b.Equal(len(json1)+len(json2)+len(json3), buf.size()) 140 | 141 | res := buf.marshalArray() 142 | b.Equal(`[`+string(json1)+`,`+string(json2)+`,`+string(json3)+`]`, string(res)) 143 | 144 | v := []map[string]interface{}{} 145 | b.NoError(json.Unmarshal(res, &v)) 146 | b.Len(v, 3) 147 | b.Equal(v[0]["string"], "test") 148 | b.Equal(v[0]["int"], float64(1)) 149 | b.Equal(v[1]["int"], float64(46)) 150 | b.Equal(v[2]["int"], float64(-1.0)) 151 | b.Equal(v[2]["string"], "test_3") 152 | 153 | buf.reset() 154 | b.Equal(0, buf.count()) 155 | b.Equal(0, buf.size()) 156 | b.Equal(0, buf.currentByteSize) 157 | } 158 | -------------------------------------------------------------------------------- /client.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "os" 11 | "sync" 12 | "sync/atomic" 13 | "time" 14 | 15 | "gopkg.in/validator.v2" 16 | 17 | "github.com/cenkalti/backoff" 18 | "github.com/segmentio/go-tableize" 19 | "github.com/tj/go-sync/semaphore" 20 | ) 21 | 22 | const ( 23 | // Version of the client library. 24 | Version = "0.0.1" 25 | 26 | // Endpoint for Segment Objects API. 27 | DefaultBaseEndpoint = "https://objects.segment.com" 28 | 29 | // Default source 30 | DefaultSource = "project" 31 | ) 32 | 33 | var ( 34 | ErrClientClosed = errors.New("Client is closed") 35 | ) 36 | 37 | type Config struct { 38 | BaseEndpoint string 39 | Logger *log.Logger 40 | Client *http.Client 41 | 42 | Source string 43 | 44 | MaxBatchBytes int 45 | MaxBatchCount int 46 | MaxBatchInterval time.Duration 47 | 48 | PrintErrors bool 49 | } 50 | 51 | type Client struct { 52 | Config 53 | writeKey string 54 | wg sync.WaitGroup 55 | semaphore semaphore.Semaphore 56 | closed int64 57 | cmap concurrentMap 58 | } 59 | 60 | func New(writeKey string) *Client { 61 | return NewWithConfig(writeKey, Config{}) 62 | } 63 | 64 | func NewWithConfig(writeKey string, config Config) *Client { 65 | return &Client{ 66 | Config: withDefaults(config), 67 | writeKey: writeKey, 68 | cmap: newConcurrentMap(), 69 | semaphore: make(semaphore.Semaphore, 10), 70 | } 71 | } 72 | 73 | func withDefaults(c Config) Config { 74 | if c.BaseEndpoint == "" { 75 | c.BaseEndpoint = DefaultBaseEndpoint 76 | } 77 | 78 | if c.Logger == nil { 79 | c.Logger = log.New(os.Stderr, "segment ", log.LstdFlags) 80 | } 81 | 82 | if c.Client == nil { 83 | c.Client = http.DefaultClient 84 | } 85 | 86 | if c.MaxBatchBytes <= 0 { 87 | c.MaxBatchBytes = 500 << 10 88 | } 89 | 90 | if c.MaxBatchCount <= 0 { 91 | c.MaxBatchCount = 100 92 | } 93 | 94 | if c.MaxBatchInterval <= 0 { 95 | c.MaxBatchInterval = 10 * time.Second 96 | } 97 | 98 | if c.Source == "" { 99 | c.Source = DefaultSource 100 | } 101 | 102 | return c 103 | } 104 | 105 | func (c *Client) fetchFunction(key string) *buffer { 106 | b := newBuffer(key) 107 | c.wg.Add(1) 108 | go c.buffer(b) 109 | return b 110 | } 111 | 112 | func (c *Client) flush(b *buffer) { 113 | if b.count() == 0 { 114 | return 115 | } 116 | 117 | rm := b.marshalArray() 118 | c.semaphore.Run(func() { 119 | batchRequest := &batch{ 120 | Source: c.Source, 121 | Collection: b.collection, 122 | WriteKey: c.writeKey, 123 | Objects: rm, 124 | } 125 | 126 | err := c.makeRequest(batchRequest) 127 | if c.PrintErrors { 128 | log.Printf("[ERROR] Batch failed making request: %v", err) 129 | } 130 | }) 131 | b.reset() 132 | } 133 | 134 | func (c *Client) buffer(b *buffer) { 135 | defer c.wg.Done() 136 | 137 | tick := time.NewTicker(c.MaxBatchInterval) 138 | defer tick.Stop() 139 | 140 | for { 141 | select { 142 | case req := <-b.Channel: 143 | req.Properties = tableize.Tableize(&tableize.Input{ 144 | Value: req.Properties, 145 | }) 146 | x, err := json.Marshal(req) 147 | if err != nil { 148 | if c.PrintErrors { 149 | log.Printf("[Error] Message `%s` excluded from batch: %v", req.ID, err) 150 | } 151 | continue 152 | } 153 | if b.size()+len(x) >= c.MaxBatchBytes || b.count()+1 >= c.MaxBatchCount { 154 | c.flush(b) 155 | } 156 | b.add(x) 157 | case <-tick.C: 158 | c.flush(b) 159 | case <-b.Exit: 160 | for req := range b.Channel { 161 | req.Properties = tableize.Tableize(&tableize.Input{ 162 | Value: req.Properties, 163 | }) 164 | x, err := json.Marshal(req) 165 | if err != nil { 166 | if c.PrintErrors { 167 | log.Printf("[Error] Exiting: Message `%s` excluded from batch: %v", req.ID, err) 168 | } 169 | continue 170 | } 171 | if b.size()+len(x) >= c.MaxBatchBytes || b.count()+1 >= c.MaxBatchCount { 172 | c.flush(b) 173 | } 174 | b.add(x) 175 | } 176 | c.flush(b) 177 | return 178 | } 179 | } 180 | 181 | } 182 | 183 | func (c *Client) Close() error { 184 | if !atomic.CompareAndSwapInt64(&c.closed, 0, 1) { 185 | return ErrClientClosed 186 | } 187 | 188 | for t := range c.cmap.Iter() { 189 | t.Val.Exit <- struct{}{} 190 | close(t.Val.Exit) 191 | close(t.Val.Channel) 192 | } 193 | 194 | c.wg.Wait() 195 | c.semaphore.Wait() 196 | 197 | return nil 198 | } 199 | 200 | func (c *Client) Set(v *Object) error { 201 | if atomic.LoadInt64(&c.closed) == 1 { 202 | return ErrClientClosed 203 | } 204 | 205 | if err := validator.Validate(v); err != nil { 206 | return err 207 | } 208 | 209 | c.cmap.Fetch(v.Collection, c.fetchFunction).Channel <- v 210 | return nil 211 | } 212 | 213 | func (c *Client) makeRequest(request *batch) error { 214 | payload, err := json.Marshal(request) 215 | if err != nil { 216 | return err 217 | } 218 | 219 | b := backoff.NewExponentialBackOff() 220 | b.MaxElapsedTime = 10 * time.Second 221 | err = backoff.Retry(func() error { 222 | bodyReader := bytes.NewReader(payload) 223 | resp, err := http.Post(c.BaseEndpoint+"/v1/set", "application/json", bodyReader) 224 | if err != nil { 225 | return err 226 | } 227 | defer resp.Body.Close() 228 | 229 | response := map[string]interface{}{} 230 | dec := json.NewDecoder(resp.Body) 231 | dec.Decode(&response) 232 | 233 | if resp.StatusCode != http.StatusOK { 234 | return fmt.Errorf("HTTP Post Request Failed, Status Code %d. \nResponse: %v", 235 | resp.StatusCode, response) 236 | } 237 | 238 | return nil 239 | }, b) 240 | 241 | return err 242 | } 243 | -------------------------------------------------------------------------------- /client_test.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | import ( 4 | "encoding/json" 5 | "net/http" 6 | "sync" 7 | "sync/atomic" 8 | "testing" 9 | "time" 10 | 11 | "github.com/jarcoal/httpmock" 12 | "github.com/stretchr/testify/suite" 13 | ) 14 | 15 | func TestClient(t *testing.T) { 16 | suite.Run(t, &ClientTestSuite{}) 17 | } 18 | 19 | type ClientTestSuite struct { 20 | suite.Suite 21 | 22 | httpRequestsMutex sync.Mutex 23 | httpRequests []*batch 24 | httpSuccess int64 25 | httpErrors int64 26 | } 27 | 28 | func (c *ClientTestSuite) SetupSuite() { 29 | httpmock.Activate() 30 | 31 | responder := func(req *http.Request) (*http.Response, error) { 32 | defer req.Body.Close() 33 | 34 | v := &batch{} 35 | dec := json.NewDecoder(req.Body) 36 | 37 | if err := dec.Decode(v); err != nil { 38 | atomic.AddInt64(&c.httpErrors, 1) 39 | return httpmock.NewStringResponse(500, ""), nil 40 | } 41 | 42 | c.httpRequestsMutex.Lock() 43 | c.httpRequests = append(c.httpRequests, v) 44 | c.httpRequestsMutex.Unlock() 45 | atomic.AddInt64(&c.httpSuccess, 1) 46 | 47 | return httpmock.NewStringResponse(200, `{"success": true}`), nil 48 | } 49 | 50 | httpmock.RegisterResponder("POST", "https://objects.segment.com/v1/set", responder) 51 | } 52 | 53 | func (c *ClientTestSuite) TestNewClient() { 54 | client := New("writeKey") 55 | c.NotNil(client) 56 | c.NotEmpty(client.BaseEndpoint) 57 | c.NotNil(client.Client) 58 | c.NotNil(client.Logger) 59 | c.NotNil(client.semaphore) 60 | c.NotNil(client.wg) 61 | c.NotNil(client.Source) 62 | c.Equal("writeKey", client.writeKey) 63 | c.Equal(0, client.cmap.Count()) 64 | } 65 | 66 | func (c *ClientTestSuite) TestSetOnce() { 67 | client := New("writeKey") 68 | c.NotNil(client) 69 | 70 | v := &Object{ID: "id", Collection: "c", Properties: map[string]interface{}{"p": "1"}} 71 | c.NoError(client.Set(v)) 72 | c.Equal(1, client.cmap.Count()) 73 | } 74 | 75 | func (c *ClientTestSuite) TestSetFull() { 76 | client := New("writeKey") 77 | c.NotNil(client) 78 | 79 | v1 := &Object{ID: "id", Collection: "c", Properties: map[string]interface{}{"p": "1"}} 80 | c.NoError(client.Set(v1)) 81 | v2 := &Object{ID: "id2", Collection: "c", Properties: map[string]interface{}{"p": "2"}} 82 | c.NoError(client.Set(v2)) 83 | 84 | c.Equal(1, client.cmap.Count()) 85 | 86 | c.NoError(client.Close()) 87 | 88 | c.Len(c.httpRequests, 1) 89 | c.Equal("c", c.httpRequests[0].Collection) 90 | 91 | received := []*Object{} 92 | c.NoError(json.Unmarshal(c.httpRequests[0].Objects, &received)) 93 | c.Len(received, 2) 94 | c.Equal("id", received[0].ID) 95 | c.Equal("id2", received[1].ID) 96 | } 97 | 98 | func (c *ClientTestSuite) TestChannelFlow() { 99 | client := New("writeKey") 100 | c.NotNil(client) 101 | 102 | v := &Object{ID: "id", Collection: "c", Properties: map[string]interface{}{"p": "1"}} 103 | 104 | buf := client.cmap.Fetch("c", client.fetchFunction) 105 | buf.Channel <- v 106 | 107 | // TODO(vince): Find a better solution to test this 108 | // Wait for the channel to add to buffer 109 | time.Sleep(250 * time.Millisecond) 110 | c.Equal(1, buf.count()) 111 | 112 | bt, err := json.Marshal(v) 113 | c.NoError(err) 114 | 115 | c.Equal(len(bt), buf.size()) 116 | } 117 | 118 | func (c *ClientTestSuite) TestSetErrors() { 119 | client := New("writeKey") 120 | c.NotNil(client) 121 | 122 | // Error with empty object 123 | c.Error(client.Set(&Object{})) 124 | 125 | // Error with empty Collection 126 | c.Error(client.Set(&Object{ID: "id", Collection: "", Properties: map[string]interface{}{"prop1": "1"}})) 127 | 128 | // Error without properties 129 | c.Error(client.Set(&Object{ID: "id", Collection: "collection"})) 130 | 131 | // Error with empty ID 132 | c.Error(client.Set(&Object{ID: "", Collection: "collection", Properties: map[string]interface{}{"prop1": "1"}})) 133 | } 134 | 135 | func (c *ClientTestSuite) TestClose() { 136 | client := New("writeKey") 137 | c.NotNil(client) 138 | client.Close() 139 | 140 | // Error already closed 141 | c.Error(client.Set(&Object{ID: "id", Collection: "collection", Properties: map[string]interface{}{"prop1": "1"}})) 142 | } 143 | -------------------------------------------------------------------------------- /cmap.go: -------------------------------------------------------------------------------- 1 | // Code generated by github.com/streamrail/concurrent-map. 2 | // DO NOT EDIT! 3 | 4 | package objects 5 | 6 | import ( 7 | "encoding/json" 8 | "hash/fnv" 9 | "sync" 10 | ) 11 | 12 | const shardCount = 32 13 | 14 | // A "thread" safe map of type string:*buffer. 15 | // To avoid lock bottlenecks this map is dived to several (shardCount) map shards. 16 | type concurrentMap []*concurrentMapShared 17 | type concurrentMapShared struct { 18 | items map[string]*buffer 19 | sync.RWMutex // Read Write mutex, guards access to internal map. 20 | } 21 | 22 | // Creates a new concurrent map. 23 | func newConcurrentMap() concurrentMap { 24 | m := make(concurrentMap, shardCount) 25 | for i := 0; i < shardCount; i++ { 26 | m[i] = &concurrentMapShared{items: make(map[string]*buffer)} 27 | } 28 | return m 29 | } 30 | 31 | // Returns shard under given key 32 | func (m concurrentMap) GetShard(key string) *concurrentMapShared { 33 | hasher := fnv.New32() 34 | hasher.Write([]byte(key)) 35 | return m[int(hasher.Sum32())%shardCount] 36 | } 37 | 38 | // Sets the given value under the specified key. 39 | func (m *concurrentMap) Set(key string, value *buffer) { 40 | // Get map shard. 41 | shard := m.GetShard(key) 42 | shard.Lock() 43 | defer shard.Unlock() 44 | shard.items[key] = value 45 | } 46 | 47 | // Retrieves an element from map under given key. 48 | func (m concurrentMap) Get(key string) (*buffer, bool) { 49 | // Get shard 50 | shard := m.GetShard(key) 51 | shard.RLock() 52 | defer shard.RUnlock() 53 | 54 | // Get item from shard. 55 | val, ok := shard.items[key] 56 | return val, ok 57 | } 58 | 59 | // Sets the given value under the specified key. 60 | func (m *concurrentMap) Fetch(key string, f func(key string) *buffer) (v *buffer) { 61 | // Get map shard. 62 | shard := m.GetShard(key) 63 | shard.Lock() 64 | defer shard.Unlock() 65 | v, ok := shard.items[key] 66 | if !ok { 67 | v = f(key) 68 | shard.items[key] = v 69 | } 70 | 71 | return 72 | } 73 | 74 | // Returns the number of elements within the map. 75 | func (m concurrentMap) Count() int { 76 | count := 0 77 | for i := 0; i < shardCount; i++ { 78 | shard := m[i] 79 | shard.RLock() 80 | count += len(shard.items) 81 | shard.RUnlock() 82 | } 83 | return count 84 | } 85 | 86 | // Looks up an item under specified key 87 | func (m *concurrentMap) Has(key string) bool { 88 | // Get shard 89 | shard := m.GetShard(key) 90 | shard.RLock() 91 | defer shard.RUnlock() 92 | 93 | // See if element is within shard. 94 | _, ok := shard.items[key] 95 | return ok 96 | } 97 | 98 | // Removes an element from the map. 99 | func (m *concurrentMap) Remove(key string) { 100 | // Try to get shard. 101 | shard := m.GetShard(key) 102 | shard.Lock() 103 | defer shard.Unlock() 104 | delete(shard.items, key) 105 | } 106 | 107 | // Checks if map is empty. 108 | func (m *concurrentMap) IsEmpty() bool { 109 | return m.Count() == 0 110 | } 111 | 112 | // Used by the Iter & IterBuffered functions to wrap two variables together over a channel, 113 | type Tuple struct { 114 | Key string 115 | Val *buffer 116 | } 117 | 118 | // Returns an iterator which could be used in a for range loop. 119 | func (m concurrentMap) Iter() <-chan Tuple { 120 | ch := make(chan Tuple) 121 | go func() { 122 | // Foreach shard. 123 | for _, shard := range m { 124 | // Foreach key, value pair. 125 | shard.RLock() 126 | for key, val := range shard.items { 127 | ch <- Tuple{key, val} 128 | } 129 | shard.RUnlock() 130 | } 131 | close(ch) 132 | }() 133 | return ch 134 | } 135 | 136 | // Returns a buffered iterator which could be used in a for range loop. 137 | func (m concurrentMap) IterBuffered() <-chan Tuple { 138 | ch := make(chan Tuple, m.Count()) 139 | go func() { 140 | // Foreach shard. 141 | for _, shard := range m { 142 | // Foreach key, value pair. 143 | shard.RLock() 144 | for key, val := range shard.items { 145 | ch <- Tuple{key, val} 146 | } 147 | shard.RUnlock() 148 | } 149 | close(ch) 150 | }() 151 | return ch 152 | } 153 | 154 | //Reviles concurrentMap "private" variables to json marshal. 155 | func (m concurrentMap) MarshalJSON() ([]byte, error) { 156 | // Create a temporary map, which will hold all item spread across shards. 157 | tmp := make(map[string]*buffer) 158 | 159 | // Insert items to temporary map. 160 | for item := range m.Iter() { 161 | tmp[item.Key] = item.Val 162 | } 163 | return json.Marshal(tmp) 164 | } 165 | 166 | func (m *concurrentMap) UnmarshalJSON(b []byte) (err error) { 167 | // Reverse process of Marshal. 168 | 169 | tmp := make(map[string]*buffer) 170 | 171 | // Unmarshal into a single map. 172 | if err := json.Unmarshal(b, &tmp); err != nil { 173 | return nil 174 | } 175 | 176 | // foreach key,value pair in temporary map insert into our concurrent map. 177 | for key, val := range tmp { 178 | m.Set(key, val) 179 | } 180 | return nil 181 | } 182 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/segmentio/objects-go 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect 7 | github.com/cenkalti/backoff v1.0.1-0.20160610100912-cdf48bbc1eb7 8 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2 // indirect 9 | github.com/jarcoal/httpmock v0.0.0-20150801143502-145b10d65926 10 | github.com/pmezard/go-difflib v1.0.0 // indirect 11 | github.com/segmentio/go-snakecase v0.0.0-20160726192916-45c439a7815b // indirect 12 | github.com/segmentio/go-tableize v1.0.1-0.20160728214455-e912c9fa0f24 13 | github.com/stretchr/testify v1.1.4-0.20160615092844-d77da356e56a 14 | github.com/tj/go-sync v0.0.0-20160119181431-8448302468e8 15 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 16 | gopkg.in/validator.v2 v2.0.0-20160201165114-3e4f037f12a1 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= 2 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= 3 | github.com/cenkalti/backoff v1.0.1-0.20160610100912-cdf48bbc1eb7 h1:3po9rsFzJqO2nE3wYulDHRSDx2x581FLA0z3sSQXSV4= 4 | github.com/cenkalti/backoff v1.0.1-0.20160610100912-cdf48bbc1eb7/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= 5 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2 h1:5zdDAMuB3gvbHB1m2BZT9+t9w+xaBmK3ehb7skDXcwM= 6 | github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/jarcoal/httpmock v0.0.0-20150801143502-145b10d65926 h1:xzEkqEt60DFJU4KfOatlQgnGq+ewG1oMNDki9xnp+98= 8 | github.com/jarcoal/httpmock v0.0.0-20150801143502-145b10d65926/go.mod h1:ks+b9deReOc7jgqp+e7LuFiCBH6Rm5hL32cLcEAArb4= 9 | github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= 10 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 11 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 12 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 13 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 14 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 15 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 16 | github.com/segmentio/go-snakecase v0.0.0-20160726192916-45c439a7815b h1:tQZxd64g1vPKGrSUErTkx8k5t6oZ0bNMqYn5kJm7xZQ= 17 | github.com/segmentio/go-snakecase v0.0.0-20160726192916-45c439a7815b/go.mod h1:jk1miR5MS7Na32PZUykG89Arm+1BUSYhuGR6b7+hJto= 18 | github.com/segmentio/go-tableize v1.0.1-0.20160728214455-e912c9fa0f24 h1:zTM1xnAqaWSqELHjeZ8HSHu5kvO4AralEzVEGcWICnc= 19 | github.com/segmentio/go-tableize v1.0.1-0.20160728214455-e912c9fa0f24/go.mod h1:igu8uNdIUFCr6TpLHo1ZOcWRCNK7jV+bemtqf4mGT3Y= 20 | github.com/stretchr/testify v1.1.4-0.20160615092844-d77da356e56a h1:UWu0XgfW9PCuyeZYNe2eGGkDZjooQKjVQqY/+d/jYmc= 21 | github.com/stretchr/testify v1.1.4-0.20160615092844-d77da356e56a/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 22 | github.com/tj/go-sync v0.0.0-20160119181431-8448302468e8 h1:AIlsdVcgic6+IcqnS9nubp+g+nMsQ6Be6a/6udAM0a0= 23 | github.com/tj/go-sync v0.0.0-20160119181431-8448302468e8/go.mod h1:Yc1ubQWqgwxY/yM0yekxzBPuxtZg4hDgWa2OjlKpi8M= 24 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 25 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 26 | gopkg.in/validator.v2 v2.0.0-20160201165114-3e4f037f12a1 h1:1IZMbdoz1SZAQ4HMRwAP0FPSyXt7ywsiJ4q7OPTEu4A= 27 | gopkg.in/validator.v2 v2.0.0-20160201165114-3e4f037f12a1/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= 28 | -------------------------------------------------------------------------------- /object.go: -------------------------------------------------------------------------------- 1 | package objects 2 | 3 | type Object struct { 4 | Collection string `json:"-" validate:"nonzero"` 5 | ID string `json:"id" validate:"nonzero"` 6 | Properties map[string]interface{} `json:"properties" validate:"min=1"` 7 | } 8 | --------------------------------------------------------------------------------