├── main.go └── README.md /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/aws/session" 8 | "github.com/aws/aws-sdk-go/service/kinesis" 9 | ) 10 | 11 | var ( 12 | stream = flag.String("stream", "your-stream", "your stream name") 13 | region = flag.String("region", "ap-northeast-1", "your AWS region") 14 | ) 15 | 16 | func main() { 17 | flag.Parse() 18 | 19 | s := session.New(&aws.Config{Region: aws.String(*region)}) 20 | kc := kinesis.New(s) 21 | 22 | streamName := aws.String(*stream) 23 | 24 | out, err := kc.CreateStream(&kinesis.CreateStreamInput{ 25 | ShardCount: aws.Int64(1), 26 | StreamName: streamName, 27 | }) 28 | if err != nil { 29 | panic(err) 30 | } 31 | fmt.Printf("%v\n", out) 32 | 33 | if err := kc.WaitUntilStreamExists(&kinesis.DescribeStreamInput{StreamName: streamName}); err != nil { 34 | panic(err) 35 | } 36 | 37 | streams, err := kc.DescribeStream(&kinesis.DescribeStreamInput{StreamName: streamName}) 38 | if err != nil { 39 | panic(err) 40 | } 41 | fmt.Printf("%v\n", streams) 42 | 43 | putOutput, err := kc.PutRecord(&kinesis.PutRecordInput{ 44 | Data: []byte("hoge"), 45 | StreamName: streamName, 46 | PartitionKey: aws.String("key1"), 47 | }) 48 | if err != nil { 49 | panic(err) 50 | } 51 | fmt.Printf("%v\n", putOutput) 52 | 53 | // put 10 records using PutRecords API 54 | entries := make([]*kinesis.PutRecordsRequestEntry, 10) 55 | for i := 0; i < len(entries); i++ { 56 | entries[i] = &kinesis.PutRecordsRequestEntry{ 57 | Data: []byte(fmt.Sprintf("hoge%d", i)), 58 | PartitionKey: aws.String("key2"), 59 | } 60 | } 61 | fmt.Printf("%v\n", entries) 62 | putsOutput, err := kc.PutRecords(&kinesis.PutRecordsInput{ 63 | Records: entries, 64 | StreamName: streamName, 65 | }) 66 | if err != nil { 67 | panic(err) 68 | } 69 | // putsOutput has Records, and its shard id and sequece enumber. 70 | fmt.Printf("%v\n", putsOutput) 71 | 72 | // retrieve iterator 73 | iteratorOutput, err := kc.GetShardIterator(&kinesis.GetShardIteratorInput{ 74 | // Shard Id is provided when making put record(s) request. 75 | ShardId: putOutput.ShardId, 76 | ShardIteratorType: aws.String("TRIM_HORIZON"), 77 | // ShardIteratorType: aws.String("AT_SEQUENCE_NUMBER"), 78 | // ShardIteratorType: aws.String("LATEST"), 79 | StreamName: streamName, 80 | }) 81 | if err != nil { 82 | panic(err) 83 | } 84 | fmt.Printf("%v\n", iteratorOutput) 85 | 86 | // get records use shard iterator for making request 87 | records, err := kc.GetRecords(&kinesis.GetRecordsInput{ 88 | ShardIterator: iteratorOutput.ShardIterator, 89 | }) 90 | if err != nil { 91 | panic(err) 92 | } 93 | fmt.Printf("%v\n", records) 94 | 95 | // and, you can iteratively make GetRecords request using records.NextShardIterator 96 | recordsSecond, err := kc.GetRecords(&kinesis.GetRecordsInput{ 97 | ShardIterator: records.NextShardIterator, 98 | }) 99 | if err != nil { 100 | panic(err) 101 | } 102 | fmt.Printf("%v\n", recordsSecond) 103 | 104 | // OK, finally delete your stream 105 | deleteOutput, err := kc.DeleteStream(&kinesis.DeleteStreamInput{ 106 | StreamName: streamName, 107 | }) 108 | if err != nil { 109 | panic(err) 110 | } 111 | fmt.Printf("%v\n", deleteOutput) 112 | } 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Amazon Kinesis Stream consumer and producer example using aws-sdk-go 2 | 3 | Demonstration for 4 | 5 | * create stream 6 | * wait until Kinesis stream available 7 | * describe stream 8 | * put record 9 | * put multiple records (using PutRecords API) 10 | * get records using shard iterator 11 | * delete stream 12 | 13 | Authorization is depends on aws-sdk-go. I use `~/.aws/config` for this purpose. 14 | 15 | Example output 16 | 17 | ``` 18 | -> % go run main.go -stream my-kinesis-stream -region ap-northeast-1 19 | { 20 | 21 | } 22 | { 23 | StreamDescription: { 24 | HasMoreShards: false, 25 | RetentionPeriodHours: 24, 26 | Shards: [{ 27 | HashKeyRange: { 28 | EndingHashKey: "340282366920938463463374607431768211455", 29 | StartingHashKey: "0" 30 | }, 31 | SequenceNumberRange: { 32 | StartingSequenceNumber: "49556707348180705259475312504965645807584792562676793346" 33 | }, 34 | ShardId: "shardId-000000000000" 35 | }], 36 | StreamARN: "arn:aws:kinesis:ap-northeast-1:xxxxxxxxxxxx:stream/my-kinesis-stream", 37 | StreamName: "my-kinesis-stream", 38 | StreamStatus: "ACTIVE" 39 | } 40 | } 41 | { 42 | SequenceNumber: "49556707348180705259475312505169954271099667229664346114", 43 | ShardId: "shardId-000000000000" 44 | } 45 | [{ 46 | Data: [ 47 | 104, 48 | 111, 49 | 103, 50 | 101, 51 | 48 52 | ], 53 | PartitionKey: "key2" 54 | } { 55 | Data: [ 56 | 104, 57 | 111, 58 | 103, 59 | 101, 60 | 49 61 | ], 62 | PartitionKey: "key2" 63 | } { 64 | Data: [ 65 | 104, 66 | 111, 67 | 103, 68 | 101, 69 | 50 70 | ], 71 | PartitionKey: "key2" 72 | } { 73 | Data: [ 74 | 104, 75 | 111, 76 | 103, 77 | 101, 78 | 51 79 | ], 80 | PartitionKey: "key2" 81 | } { 82 | Data: [ 83 | 104, 84 | 111, 85 | 103, 86 | 101, 87 | 52 88 | ], 89 | PartitionKey: "key2" 90 | } { 91 | Data: [ 92 | 104, 93 | 111, 94 | 103, 95 | 101, 96 | 53 97 | ], 98 | PartitionKey: "key2" 99 | } { 100 | Data: [ 101 | 104, 102 | 111, 103 | 103, 104 | 101, 105 | 54 106 | ], 107 | PartitionKey: "key2" 108 | } { 109 | Data: [ 110 | 104, 111 | 111, 112 | 103, 113 | 101, 114 | 55 115 | ], 116 | PartitionKey: "key2" 117 | } { 118 | Data: [ 119 | 104, 120 | 111, 121 | 103, 122 | 101, 123 | 56 124 | ], 125 | PartitionKey: "key2" 126 | } { 127 | Data: [ 128 | 104, 129 | 111, 130 | 103, 131 | 101, 132 | 57 133 | ], 134 | PartitionKey: "key2" 135 | }] 136 | { 137 | FailedRecordCount: 0, 138 | Records: [ 139 | { 140 | SequenceNumber: "49556707348180705259475312505171163196919281858839052290", 141 | ShardId: "shardId-000000000000" 142 | }, 143 | { 144 | SequenceNumber: "49556707348180705259475312505172372122738896488013758466", 145 | ShardId: "shardId-000000000000" 146 | }, 147 | { 148 | SequenceNumber: "49556707348180705259475312505173581048558511117188464642", 149 | ShardId: "shardId-000000000000" 150 | }, 151 | { 152 | SequenceNumber: "49556707348180705259475312505174789974378125746363170818", 153 | ShardId: "shardId-000000000000" 154 | }, 155 | { 156 | SequenceNumber: "49556707348180705259475312505175998900197740375537876994", 157 | ShardId: "shardId-000000000000" 158 | }, 159 | { 160 | SequenceNumber: "49556707348180705259475312505177207826017355004712583170", 161 | ShardId: "shardId-000000000000" 162 | }, 163 | { 164 | SequenceNumber: "49556707348180705259475312505178416751836969633887289346", 165 | ShardId: "shardId-000000000000" 166 | }, 167 | { 168 | SequenceNumber: "49556707348180705259475312505179625677656584263061995522", 169 | ShardId: "shardId-000000000000" 170 | }, 171 | { 172 | SequenceNumber: "49556707348180705259475312505180834603476198892236701698", 173 | ShardId: "shardId-000000000000" 174 | }, 175 | { 176 | SequenceNumber: "49556707348180705259475312505182043529295813521411407874", 177 | ShardId: "shardId-000000000000" 178 | } 179 | ] 180 | } 181 | { 182 | ShardIterator: "AAAAAAAAAAE8llzsV1gPXCWRLV7UcXSisncE5jyOHgs2yEDWRWT5q8VpFZ7vFW8qncqwvxHeqavSnkW+jdAUjBRYeNrDC0eBdgRMYHLnaIoIuolr78xS/ooobWxH8RZwG0IiyDYn/dOUDLv32PoB7skzAZ/1AYjFAYBnIVPp5AUq771NKh8WQ5nSABzI5jxJDJceH2ZYTZdwAfn95aOBeMXWwCvQoZMr" 183 | } 184 | { 185 | MillisBehindLatest: 0, 186 | NextShardIterator: "AAAAAAAAAAE9dau92rwQBbwfw+VFECaAlJsgOwdYE3ZXfQufLdXqsorcrvsFl0t+MqB+KwkROo8t1PaYwDBFH1uBR7R/4ZGb5m2W04p8MjdYbl9rkkyMkF5x356V2IyXIZqchzHBzYo34FM9C2+ndN+VNY/NRiKqtONp4q0W9hLcFT1WtbTkCJThq5E4RefhQuyX8D6p6A4c0wUpk2KadAZrZDmb0a44", 187 | Records: [ 188 | { 189 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 190 | Data: [ 191 | 104, 192 | 111, 193 | 103, 194 | 101 195 | ], 196 | PartitionKey: "key1", 197 | SequenceNumber: "49556707348180705259475312505169954271099667229664346114" 198 | }, 199 | { 200 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 201 | Data: [ 202 | 104, 203 | 111, 204 | 103, 205 | 101, 206 | 48 207 | ], 208 | PartitionKey: "key2", 209 | SequenceNumber: "49556707348180705259475312505171163196919281858839052290" 210 | }, 211 | { 212 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 213 | Data: [ 214 | 104, 215 | 111, 216 | 103, 217 | 101, 218 | 49 219 | ], 220 | PartitionKey: "key2", 221 | SequenceNumber: "49556707348180705259475312505172372122738896488013758466" 222 | }, 223 | { 224 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 225 | Data: [ 226 | 104, 227 | 111, 228 | 103, 229 | 101, 230 | 50 231 | ], 232 | PartitionKey: "key2", 233 | SequenceNumber: "49556707348180705259475312505173581048558511117188464642" 234 | }, 235 | { 236 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 237 | Data: [ 238 | 104, 239 | 111, 240 | 103, 241 | 101, 242 | 51 243 | ], 244 | PartitionKey: "key2", 245 | SequenceNumber: "49556707348180705259475312505174789974378125746363170818" 246 | }, 247 | { 248 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 249 | Data: [ 250 | 104, 251 | 111, 252 | 103, 253 | 101, 254 | 52 255 | ], 256 | PartitionKey: "key2", 257 | SequenceNumber: "49556707348180705259475312505175998900197740375537876994" 258 | }, 259 | { 260 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 261 | Data: [ 262 | 104, 263 | 111, 264 | 103, 265 | 101, 266 | 53 267 | ], 268 | PartitionKey: "key2", 269 | SequenceNumber: "49556707348180705259475312505177207826017355004712583170" 270 | }, 271 | { 272 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 273 | Data: [ 274 | 104, 275 | 111, 276 | 103, 277 | 101, 278 | 54 279 | ], 280 | PartitionKey: "key2", 281 | SequenceNumber: "49556707348180705259475312505178416751836969633887289346" 282 | }, 283 | { 284 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 285 | Data: [ 286 | 104, 287 | 111, 288 | 103, 289 | 101, 290 | 55 291 | ], 292 | PartitionKey: "key2", 293 | SequenceNumber: "49556707348180705259475312505179625677656584263061995522" 294 | }, 295 | { 296 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 297 | Data: [ 298 | 104, 299 | 111, 300 | 103, 301 | 101, 302 | 56 303 | ], 304 | PartitionKey: "key2", 305 | SequenceNumber: "49556707348180705259475312505180834603476198892236701698" 306 | }, 307 | { 308 | ApproximateArrivalTimestamp: 2015-11-26 07:19:31 +0000 UTC, 309 | Data: [ 310 | 104, 311 | 111, 312 | 103, 313 | 101, 314 | 57 315 | ], 316 | PartitionKey: "key2", 317 | SequenceNumber: "49556707348180705259475312505182043529295813521411407874" 318 | } 319 | ] 320 | } 321 | { 322 | MillisBehindLatest: 0, 323 | NextShardIterator: "AAAAAAAAAAH+BPEFSUYT2vcKcZBldMWp14lGxZbnLxF0zrbeLO3UOmN6Lc2NavT4hPVxivCzxUHY8bctYaj7l3F95y9z/zIs7dytvsVI5kvcYr23OFCJfRjhSYnXutz64GKPLwlxoN2sZg38LZSpcRUMqKROD/5AJHwx1OWA9RAIvI6n25gd6u0rX7SgGXbQHko/8CjoVMMUzzBofLNr7YBRyDl59ErA", 324 | Records: [] 325 | } 326 | { 327 | 328 | } 329 | ``` 330 | 331 | For handling errors, see [handling errors · aws/aws-sdk-go Wiki](https://github.com/aws/aws-sdk-go/wiki/handling-errors). 332 | 333 | ## LICENSE 334 | 335 | MIT 336 | 337 | ## Author 338 | 339 | Kenta Suzuki (a.k.a. suzuken) 340 | --------------------------------------------------------------------------------