├── pkg.go ├── .gitignore ├── Gopkg.toml ├── README.md ├── LICENSE ├── Gopkg.lock └── stream └── stream.go /pkg.go: -------------------------------------------------------------------------------- 1 | package go_dynamodb_stream_subscriber 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | 26 | vendor/ 27 | -------------------------------------------------------------------------------- /Gopkg.toml: -------------------------------------------------------------------------------- 1 | # Gopkg.toml example 2 | # 3 | # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md 4 | # for detailed Gopkg.toml documentation. 5 | # 6 | # required = ["github.com/user/thing/cmd/thing"] 7 | # ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] 8 | # 9 | # [[constraint]] 10 | # name = "github.com/user/project" 11 | # version = "1.0.0" 12 | # 13 | # [[constraint]] 14 | # name = "github.com/user/project2" 15 | # branch = "dev" 16 | # source = "github.com/myfork/project2" 17 | # 18 | # [[override]] 19 | # name = "github.com/x/y" 20 | # version = "2.4.0" 21 | # 22 | # [prune] 23 | # non-go = false 24 | # go-tests = true 25 | # unused-packages = true 26 | 27 | 28 | [[constraint]] 29 | name = "github.com/aws/aws-sdk-go" 30 | version = "^1.15.4" 31 | 32 | [prune] 33 | go-tests = true 34 | unused-packages = true 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-dynamodb-stream-subscriber 2 | Go channel for streaming Dynamodb Updates 3 | 4 | ```go 5 | package main 6 | 7 | import ( 8 | "fmt" 9 | "github.com/aws/aws-sdk-go/aws" 10 | "github.com/aws/aws-sdk-go/aws/session" 11 | "github.com/aws/aws-sdk-go/service/dynamodb" 12 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 13 | "github.com/urakozz/go-dynamodb-stream-subscriber/stream" 14 | ) 15 | 16 | func main() { 17 | cfg := aws.NewConfig().WithRegion("eu-west-1") 18 | sess := session.New() 19 | streamSvc := dynamodbstreams.New(sess, cfg) 20 | dynamoSvc := dynamodb.New(sess, cfg) 21 | table := "tableName" 22 | 23 | streamSubscriber := stream.NewStreamSubscriber(dynamoSvc, streamSvc, table) 24 | ch, errCh := streamSubscriber.GetStreamDataAsync() 25 | 26 | go func(errCh <-chan error) { 27 | for err := range errCh { 28 | fmt.Println("Stream Subscriber error: ", err) 29 | } 30 | }(errCh) 31 | 32 | for record := range ch { 33 | fmt.Println("from channel:", record) 34 | } 35 | } 36 | ``` 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Yury Kozyrev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Gopkg.lock: -------------------------------------------------------------------------------- 1 | # This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. 2 | 3 | 4 | [[projects]] 5 | name = "github.com/aws/aws-sdk-go" 6 | packages = [ 7 | "aws", 8 | "aws/awserr", 9 | "aws/awsutil", 10 | "aws/client", 11 | "aws/client/metadata", 12 | "aws/credentials", 13 | "aws/endpoints", 14 | "aws/request", 15 | "aws/signer/v4", 16 | "internal/sdkio", 17 | "internal/sdkrand", 18 | "internal/shareddefaults", 19 | "private/protocol", 20 | "private/protocol/json/jsonutil", 21 | "private/protocol/jsonrpc", 22 | "private/protocol/rest", 23 | "service/dynamodb", 24 | "service/dynamodbstreams" 25 | ] 26 | revision = "d1f87361dea69f086010bc3d03c120a602320b97" 27 | version = "v1.15.4" 28 | 29 | [[projects]] 30 | name = "github.com/go-ini/ini" 31 | packages = ["."] 32 | revision = "358ee7663966325963d4e8b2e1fbd570c5195153" 33 | version = "v1.38.1" 34 | 35 | [[projects]] 36 | name = "github.com/jmespath/go-jmespath" 37 | packages = ["."] 38 | revision = "0b12d6b5" 39 | 40 | [solve-meta] 41 | analyzer-name = "dep" 42 | analyzer-version = 1 43 | inputs-digest = "f621df2f7cbd26efad990561b9cce3c2dec3435caaf87313ec088876a4827247" 44 | solver-name = "gps-cdcl" 45 | solver-version = 1 46 | -------------------------------------------------------------------------------- /stream/stream.go: -------------------------------------------------------------------------------- 1 | // Yury Kozyrev (urakozz) 2 | // MIT License 3 | package stream 4 | 5 | import ( 6 | "errors" 7 | "time" 8 | 9 | "sync" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/service/dynamodb" 14 | "github.com/aws/aws-sdk-go/service/dynamodb/dynamodbiface" 15 | "github.com/aws/aws-sdk-go/service/dynamodbstreams" 16 | "github.com/aws/aws-sdk-go/service/dynamodbstreams/dynamodbstreamsiface" 17 | ) 18 | 19 | type StreamSubscriber struct { 20 | dynamoSvc dynamodbiface.DynamoDBAPI 21 | streamSvc dynamodbstreamsiface.DynamoDBStreamsAPI 22 | table *string 23 | ShardIteratorType *string 24 | Limit *int64 25 | } 26 | 27 | func NewStreamSubscriber( 28 | dynamoSvc dynamodbiface.DynamoDBAPI, 29 | streamSvc dynamodbstreamsiface.DynamoDBStreamsAPI, 30 | table string) *StreamSubscriber { 31 | s := &StreamSubscriber{dynamoSvc: dynamoSvc, streamSvc: streamSvc, table: &table} 32 | s.applyDefaults() 33 | return s 34 | } 35 | 36 | func (r *StreamSubscriber) applyDefaults() { 37 | if r.ShardIteratorType == nil { 38 | r.ShardIteratorType = aws.String(dynamodbstreams.ShardIteratorTypeLatest) 39 | } 40 | } 41 | 42 | func (r *StreamSubscriber) SetLimit(v int64) { 43 | r.Limit = aws.Int64(v) 44 | } 45 | 46 | func (r *StreamSubscriber) SetShardIteratorType(s string) { 47 | r.ShardIteratorType = aws.String(s) 48 | } 49 | 50 | func (r *StreamSubscriber) GetStreamData() (<-chan *dynamodbstreams.Record, <-chan error) { 51 | 52 | ch := make(chan *dynamodbstreams.Record, 1) 53 | errCh := make(chan error, 1) 54 | 55 | go func(ch chan<- *dynamodbstreams.Record, errCh chan<- error) { 56 | var shardId *string 57 | var prevShardId *string 58 | var streamArn *string 59 | var err error 60 | 61 | for { 62 | prevShardId = shardId 63 | shardId, streamArn, err = r.findProperShardId(prevShardId) 64 | if err != nil { 65 | errCh <- err 66 | } 67 | if shardId != nil { 68 | err = r.processShardBackport(shardId, streamArn, ch) 69 | if err != nil { 70 | errCh <- err 71 | // reset shard id to process it again 72 | shardId = prevShardId 73 | } 74 | } 75 | if shardId == nil { 76 | time.Sleep(time.Second * 10) 77 | } 78 | 79 | } 80 | }(ch, errCh) 81 | 82 | return ch, errCh 83 | } 84 | 85 | func (r *StreamSubscriber) GetStreamDataAsync() (<-chan *dynamodbstreams.Record, <-chan error) { 86 | 87 | ch := make(chan *dynamodbstreams.Record, 1) 88 | errCh := make(chan error, 1) 89 | 90 | needUpdateChannel := make(chan struct{}, 1) 91 | needUpdateChannel <- struct{}{} 92 | 93 | allShards := make(map[string]struct{}) 94 | shardProcessingLimit := 5 95 | shardsCh := make(chan *dynamodbstreams.GetShardIteratorInput, shardProcessingLimit) 96 | lock := sync.Mutex{} 97 | 98 | go func() { 99 | tick := time.NewTicker(time.Minute) 100 | for { 101 | select { 102 | case <-tick.C: 103 | needUpdateChannel <- struct{}{} 104 | } 105 | } 106 | }() 107 | 108 | go func() { 109 | for { 110 | select { 111 | case <-needUpdateChannel: 112 | streamArn, err := r.getLatestStreamArn() 113 | if err != nil { 114 | errCh <- err 115 | return 116 | } 117 | ids, err := r.getShardIds(streamArn) 118 | if err != nil { 119 | errCh <- err 120 | return 121 | } 122 | for _, sObj := range ids { 123 | lock.Lock() 124 | if _, ok := allShards[*sObj.ShardId]; !ok { 125 | allShards[*sObj.ShardId] = struct{}{} 126 | shardsCh <- &dynamodbstreams.GetShardIteratorInput{ 127 | StreamArn: streamArn, 128 | ShardId: sObj.ShardId, 129 | ShardIteratorType: r.ShardIteratorType, 130 | } 131 | } 132 | lock.Unlock() 133 | } 134 | 135 | } 136 | } 137 | 138 | }() 139 | 140 | limit := make(chan struct{}, shardProcessingLimit) 141 | 142 | go func() { 143 | time.Sleep(time.Second * 10) 144 | for shardInput := range shardsCh { 145 | limit <- struct{}{} 146 | go func(sInput *dynamodbstreams.GetShardIteratorInput) { 147 | err := r.processShard(sInput, ch) 148 | if err != nil { 149 | errCh <- err 150 | } 151 | // TODO: think about cleaning list of shards: delete(allShards, *sInput.ShardId) 152 | <-limit 153 | }(shardInput) 154 | } 155 | }() 156 | return ch, errCh 157 | } 158 | 159 | func (r *StreamSubscriber) getShardIds(streamArn *string) (ids []*dynamodbstreams.Shard, err error) { 160 | des, err := r.streamSvc.DescribeStream(&dynamodbstreams.DescribeStreamInput{ 161 | StreamArn: streamArn, 162 | }) 163 | if err != nil { 164 | return nil, err 165 | } 166 | // No shards 167 | if 0 == len(des.StreamDescription.Shards) { 168 | return nil, nil 169 | } 170 | 171 | return des.StreamDescription.Shards, nil 172 | } 173 | 174 | func (r *StreamSubscriber) findProperShardId(previousShardId *string) (shadrId *string, streamArn *string, err error) { 175 | streamArn, err = r.getLatestStreamArn() 176 | if err != nil { 177 | return nil, nil, err 178 | } 179 | des, err := r.streamSvc.DescribeStream(&dynamodbstreams.DescribeStreamInput{ 180 | StreamArn: streamArn, 181 | }) 182 | if err != nil { 183 | return nil, nil, err 184 | } 185 | 186 | if 0 == len(des.StreamDescription.Shards) { 187 | return nil, nil, nil 188 | } 189 | 190 | if previousShardId == nil { 191 | shadrId = des.StreamDescription.Shards[0].ShardId 192 | return 193 | } 194 | 195 | for _, shard := range des.StreamDescription.Shards { 196 | shadrId = shard.ShardId 197 | if shard.ParentShardId != nil && *shard.ParentShardId == *previousShardId { 198 | return 199 | } 200 | } 201 | 202 | return 203 | } 204 | 205 | func (r *StreamSubscriber) getLatestStreamArn() (*string, error) { 206 | tableInfo, err := r.dynamoSvc.DescribeTable(&dynamodb.DescribeTableInput{TableName: r.table}) 207 | if err != nil { 208 | return nil, err 209 | } 210 | if nil == tableInfo.Table.LatestStreamArn { 211 | return nil, errors.New("empty table stream arn") 212 | } 213 | return tableInfo.Table.LatestStreamArn, nil 214 | } 215 | 216 | func (r *StreamSubscriber) processShardBackport(shardId, lastStreamArn *string, ch chan<- *dynamodbstreams.Record) error { 217 | return r.processShard(&dynamodbstreams.GetShardIteratorInput{ 218 | StreamArn: lastStreamArn, 219 | ShardId: shardId, 220 | ShardIteratorType: r.ShardIteratorType, 221 | }, ch) 222 | } 223 | 224 | func (r *StreamSubscriber) processShard(input *dynamodbstreams.GetShardIteratorInput, ch chan<- *dynamodbstreams.Record) error { 225 | iter, err := r.streamSvc.GetShardIterator(input) 226 | if err != nil { 227 | return err 228 | } 229 | if iter.ShardIterator == nil { 230 | return nil 231 | } 232 | 233 | nextIterator := iter.ShardIterator 234 | 235 | for nextIterator != nil { 236 | recs, err := r.streamSvc.GetRecords(&dynamodbstreams.GetRecordsInput{ 237 | ShardIterator: nextIterator, 238 | Limit: r.Limit, 239 | }) 240 | if awsErr, ok := err.(awserr.Error); ok && awsErr.Code() == "TrimmedDataAccessException" { 241 | //Trying to request data older than 24h, that's ok 242 | //http://docs.aws.amazon.com/dynamodbstreams/latest/APIReference/API_GetShardIterator.html -> Errors 243 | return nil 244 | } 245 | if err != nil { 246 | return err 247 | } 248 | 249 | for _, record := range recs.Records { 250 | ch <- record 251 | } 252 | 253 | nextIterator = recs.NextShardIterator 254 | 255 | sleepDuration := time.Second 256 | 257 | // Nil next itarator, shard is closed 258 | if nextIterator == nil { 259 | sleepDuration = time.Millisecond * 10 260 | } else if len(recs.Records) == 0 { 261 | // Empty set, but shard is not closed -> sleep a little 262 | sleepDuration = time.Second * 10 263 | } 264 | 265 | time.Sleep(sleepDuration) 266 | } 267 | return nil 268 | } 269 | --------------------------------------------------------------------------------