├── pkg ├── command │ └── cwlogs │ │ └── cwlogs.go ├── rawdata │ └── rawdata.go ├── ktail │ ├── kinesis_test.go │ ├── ktail.go │ └── kinesis.go ├── matcher │ ├── matcher.go │ └── matcher_test.go ├── logdata │ └── logdata.go ├── sorter │ └── sorter.go └── streamer │ └── kinesis.go ├── .envrc ├── .gitignore ├── .goreleaser.yml ├── go.mod ├── LICENSE ├── Makefile ├── README.md ├── .golangci.yml ├── go.sum └── cmd └── kinesis-tail └── main.go /pkg/command/cwlogs/cwlogs.go: -------------------------------------------------------------------------------- 1 | package cwlogs 2 | -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | export AWS_PROFILE=saml 2 | export AWS_REGION=us-west-2 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | coverage.txt 3 | console.log 4 | trace.out 5 | dist 6 | node_modules 7 | /bin 8 | /.idea 9 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | 2 | build: 3 | main: cmd/kinesis-tail/main.go 4 | binary: /kinesis-tail 5 | goos: 6 | - darwin 7 | - linux 8 | - windows 9 | goarch: 10 | - amd64 -------------------------------------------------------------------------------- /pkg/rawdata/rawdata.go: -------------------------------------------------------------------------------- 1 | package rawdata 2 | 3 | import ( 4 | "strings" 5 | "time" 6 | 7 | "github.com/versent/kinesis-tail/pkg/ktail" 8 | ) 9 | 10 | // DecodeRawData format the raw data 11 | func DecodeRawData(ts *time.Time, data []byte) *ktail.LogMessage { 12 | return &ktail.LogMessage{ 13 | Timestamp: ts.Format(time.RFC3339), 14 | Message: strings.TrimSuffix(string(data), "\n"), 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /pkg/ktail/kinesis_test.go: -------------------------------------------------------------------------------- 1 | package ktail 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestTimestamp(t *testing.T) { 11 | timestamp := int64(1526626158315) 12 | 13 | ts := buildTimestamp(timestamp) 14 | 15 | require.Equal(t, int64(1526626158315000000), ts.UnixNano()) 16 | require.Equal(t, "2018-05-18T06:49:18.315Z", ts.UTC().Format(time.RFC3339Nano)) 17 | } 18 | -------------------------------------------------------------------------------- /pkg/matcher/matcher.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import "strings" 4 | 5 | // MatchesTokens checks if the includes list is matched in the supplied message string. If the includes 6 | // list is empty everything is matched. 7 | func MatchesTokens(includes []string, msg string, fallback bool) bool { 8 | // if there are no include patterns just let everything through 9 | if len(includes) == 0 { 10 | return fallback 11 | } 12 | 13 | for _, inc := range includes { 14 | if strings.Contains(msg, inc) { 15 | return true 16 | } 17 | } 18 | 19 | return false 20 | } 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/versent/kinesis-tail 2 | 3 | go 1.14 4 | 5 | require ( 6 | github.com/alecthomas/kingpin v2.2.6+incompatible 7 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect 8 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect 9 | github.com/aws/aws-sdk-go v1.30.26 10 | github.com/fatih/color v1.7.0 11 | github.com/mattn/go-colorable v0.0.9 // indirect 12 | github.com/mattn/go-isatty v0.0.4 // indirect 13 | github.com/pkg/errors v0.9.1 14 | github.com/sirupsen/logrus v1.3.0 15 | github.com/stretchr/testify v1.5.1 16 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017 Versent Pty. Ltd. 2 | 3 | Please consider promoting this project if you find it useful. 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without restriction, 8 | including without limitation the rights to use, copy, modify, merge, 9 | publish, distribute, sublicense, and/or sell copies of the Software, 10 | and to permit persons to whom the Software is furnished to do so, 11 | subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included 14 | in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 18 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, 20 | DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT 21 | OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE 22 | OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /pkg/ktail/ktail.go: -------------------------------------------------------------------------------- 1 | package ktail 2 | 3 | // LogEntry matches the cloudwatch log entry structure 4 | type LogEntry struct { 5 | ID string `json:"id,omitempty"` 6 | Timestamp int64 `json:"timestamp,omitempty"` 7 | Message string `json:"message,omitempty"` 8 | } 9 | 10 | // LogBatch matches the cloudwatch logs batch structure 11 | type LogBatch struct { 12 | MessageType string `json:"messageType,omitempty"` 13 | Owner string `json:"owner,omitempty"` 14 | LogGroup string `json:"logGroup,omitempty"` 15 | LogStream string `json:"logStream,omitempty"` 16 | SubscriptionFilters []string `json:"subscriptionFilters,omitempty"` 17 | LogEvents []*LogEntry `json:"logEvents,omitempty"` 18 | } 19 | 20 | // LogMessage log message after decompression and parsing 21 | type LogMessage struct { 22 | LogGroup string // optional log group 23 | Message string 24 | Timestamp string 25 | } 26 | 27 | // ByTimestamp used to sort log messages 28 | type ByTimestamp []*LogMessage 29 | 30 | func (a ByTimestamp) Len() int { return len(a) } 31 | func (a ByTimestamp) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 32 | func (a ByTimestamp) Less(i, j int) bool { return a[i].Timestamp < a[j].Timestamp } 33 | -------------------------------------------------------------------------------- /pkg/matcher/matcher_test.go: -------------------------------------------------------------------------------- 1 | package matcher 2 | 3 | import "testing" 4 | 5 | func Test_MatchesTokens(t *testing.T) { 6 | type args struct { 7 | includes []string 8 | msg string 9 | fallback bool 10 | } 11 | tests := []struct { 12 | name string 13 | args args 14 | want bool 15 | }{ 16 | { 17 | name: "check matching with positive performs contain check", 18 | args: args{ 19 | includes: []string{"test123"}, 20 | msg: "/var/log/something-test123", 21 | fallback: false, 22 | }, 23 | want: true, 24 | }, 25 | { 26 | name: "check matching with negative performs contain check", 27 | args: args{ 28 | includes: []string{"test123"}, 29 | msg: "/var/log/something-test23", 30 | fallback: false, 31 | }, 32 | want: false, 33 | }, 34 | { 35 | name: "check empty includes results in pass through", 36 | args: args{ 37 | includes: []string{}, 38 | msg: "/var/log/something-test23", 39 | fallback: true, 40 | }, 41 | want: true, 42 | }, 43 | } 44 | for _, tt := range tests { 45 | t.Run(tt.name, func(t *testing.T) { 46 | if got := MatchesTokens(tt.args.includes, tt.args.msg, tt.args.fallback); got != tt.want { 47 | t.Errorf("matchesTokens() = %v, want %v", got, tt.want) 48 | } 49 | }) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SOURCE_FILES?=$$(go list ./... | grep -v /vendor/) 2 | TEST_PATTERN?=. 3 | TEST_OPTIONS?= 4 | 5 | GOLANGCI_VERSION = 1.24.0 6 | 7 | GO ?= go 8 | 9 | bin/golangci-lint: bin/golangci-lint-${GOLANGCI_VERSION} 10 | @ln -sf golangci-lint-${GOLANGCI_VERSION} bin/golangci-lint 11 | bin/golangci-lint-${GOLANGCI_VERSION}: 12 | @curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | BINARY=golangci-lint bash -s -- v${GOLANGCI_VERSION} 13 | @mv bin/golangci-lint $@ 14 | 15 | # Install from source. 16 | install: 17 | @echo "==> Installing up ${GOPATH}/bin/kinesis-tail" 18 | @$(GO) install ./... 19 | .PHONY: install 20 | 21 | # Run all the tests 22 | test: 23 | @echo "==> Testing" 24 | @go test -v -covermode=count -coverprofile=coverage.txt ./pkg/... ./cmd/... 25 | .PHONY: test 26 | 27 | # Run all the linters 28 | lint: bin/golangci-lint 29 | @echo "==> Linting" 30 | @bin/golangci-lint run 31 | .PHONY: lint 32 | 33 | # Run all the tests and code checks 34 | ci: test lint 35 | .PHONY: ci 36 | 37 | # Release binaries to GitHub. 38 | release: 39 | @echo "==> Releasing" 40 | @goreleaser -p 1 --rm-dist -config .goreleaser.yml 41 | @echo "==> Complete" 42 | .PHONY: release 43 | 44 | generate-mocks: 45 | mockery -dir ../../aws/aws-sdk-go/service/lambda/lambdaiface --all 46 | .PHONY: generate-mocks 47 | -------------------------------------------------------------------------------- /pkg/logdata/logdata.go: -------------------------------------------------------------------------------- 1 | package logdata 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/json" 7 | "strings" 8 | "time" 9 | 10 | "github.com/pkg/errors" 11 | "github.com/versent/kinesis-tail/pkg/ktail" 12 | "github.com/versent/kinesis-tail/pkg/matcher" 13 | ) 14 | 15 | // UncompressLogs uncompress and parse cloudwatch log batch data 16 | func UncompressLogs(includes, excludes []string, ts *time.Time, data []byte) ([]*ktail.LogMessage, error) { 17 | dataReader := bytes.NewReader(data) 18 | 19 | gzipReader, err := gzip.NewReader(dataReader) 20 | if err != nil { 21 | return nil, errors.Wrap(err, "un gzip data failed") 22 | } 23 | 24 | var batch ktail.LogBatch 25 | 26 | err = json.NewDecoder(gzipReader).Decode(&batch) 27 | if err != nil { 28 | return nil, errors.Wrap(err, "json decode failed") 29 | } 30 | 31 | if !matcher.MatchesTokens(includes, batch.LogGroup, true) { 32 | return []*ktail.LogMessage{}, nil 33 | } 34 | 35 | if matcher.MatchesTokens(excludes, batch.LogGroup, false) { 36 | return []*ktail.LogMessage{}, nil 37 | } 38 | 39 | logEvents := make([]*ktail.LogMessage, len(batch.LogEvents)) 40 | 41 | for i, entry := range batch.LogEvents { 42 | logEvents[i] = &ktail.LogMessage{ 43 | LogGroup: batch.LogGroup, 44 | Timestamp: ts.Format(time.RFC3339), 45 | Message: strings.TrimSuffix(entry.Message, "\n"), 46 | } 47 | } 48 | 49 | return logEvents, nil 50 | } 51 | -------------------------------------------------------------------------------- /pkg/sorter/sorter.go: -------------------------------------------------------------------------------- 1 | package sorter 2 | 3 | import ( 4 | "io" 5 | "sort" 6 | 7 | "github.com/versent/kinesis-tail/pkg/ktail" 8 | ) 9 | 10 | // FormatFunc func which is passed to the message sorter and invoked for each line to format it 11 | type FormatFunc func(wr io.Writer, msg *ktail.LogMessage) 12 | 13 | // MessageSorter manages a cache of messages and sorts then formats them on each flush 14 | type MessageSorter struct { 15 | batchSize int 16 | current int 17 | cache []*ktail.LogMessage 18 | wr io.Writer 19 | format FormatFunc 20 | } 21 | 22 | // New create a new message sorter 23 | func New(wr io.Writer, batchSize int, formatFunc FormatFunc) *MessageSorter { 24 | return &MessageSorter{ 25 | batchSize: batchSize, 26 | wr: wr, 27 | format: formatFunc, 28 | } 29 | } 30 | 31 | // PushBatch this inserts a batch in the cache and checks whether to flush 32 | func (lms *MessageSorter) PushBatch(logMessageBatch []*ktail.LogMessage) bool { 33 | lms.cache = append(lms.cache, logMessageBatch...) 34 | return lms.flushCheck() 35 | } 36 | 37 | // Flush force a flush of messages 38 | func (lms *MessageSorter) Flush() { 39 | sort.Sort(ktail.ByTimestamp(lms.cache)) 40 | 41 | for _, msg := range lms.cache { 42 | lms.format(lms.wr, msg) 43 | } 44 | 45 | lms.cache = []*ktail.LogMessage{} 46 | lms.current = 0 47 | } 48 | 49 | func (lms *MessageSorter) flushCheck() bool { 50 | lms.current++ 51 | 52 | if lms.current != lms.batchSize { 53 | return false 54 | } 55 | 56 | lms.Flush() 57 | 58 | return true 59 | } 60 | -------------------------------------------------------------------------------- /pkg/streamer/kinesis.go: -------------------------------------------------------------------------------- 1 | package streamer 2 | 3 | import ( 4 | "sync" 5 | "time" 6 | 7 | "github.com/sirupsen/logrus" 8 | 9 | "github.com/aws/aws-sdk-go/service/kinesis" 10 | "github.com/aws/aws-sdk-go/service/kinesis/kinesisiface" 11 | "github.com/pkg/errors" 12 | ) 13 | 14 | // KinesisStreamer this manages streaming data from a number of shards asynchronously 15 | type KinesisStreamer struct { 16 | svc kinesisiface.KinesisAPI 17 | iterators map[string]*string 18 | iteratorMutex *sync.Mutex 19 | pollFreqMs int64 20 | logger *logrus.Logger 21 | } 22 | 23 | // GetRecordsEntry returns the results of the last get records request 24 | type GetRecordsEntry struct { 25 | Created time.Time 26 | Shard string 27 | Records []*kinesis.Record 28 | Err error 29 | } 30 | 31 | // New return a new configured streamer 32 | func New(svc kinesisiface.KinesisAPI, iterators map[string]*string, pollFreqMs int64, logger *logrus.Logger) *KinesisStreamer { 33 | return &KinesisStreamer{ 34 | svc: svc, 35 | iterators: iterators, 36 | pollFreqMs: pollFreqMs, 37 | iteratorMutex: &sync.Mutex{}, 38 | logger: logger, 39 | } 40 | } 41 | 42 | // StartGetRecords intiate the streaming of records using the configured iterators 43 | func (ks *KinesisStreamer) StartGetRecords() chan *GetRecordsEntry { 44 | ch := make(chan *GetRecordsEntry) 45 | 46 | for key := range ks.iterators { 47 | go ks.asyncGetRecords(key, ch) 48 | } 49 | 50 | return ch 51 | } 52 | 53 | func (ks *KinesisStreamer) asyncGetRecords(shard string, ch chan *GetRecordsEntry) { 54 | c := time.Tick(time.Duration(ks.pollFreqMs) * time.Millisecond) 55 | 56 | for now := range c { 57 | if ks.iterators[shard] == nil { 58 | ks.logger.Debugf("nil iterator for shard as it is CLOSED: %s", shard) 59 | continue 60 | } 61 | 62 | resp, err := ks.svc.GetRecords(&kinesis.GetRecordsInput{ 63 | ShardIterator: ks.iterators[shard], 64 | }) 65 | if err != nil { 66 | ch <- &GetRecordsEntry{Created: now, Shard: shard, Err: errors.Wrap(err, "get records failed")} 67 | } 68 | 69 | ks.logger.WithField("iterator", resp).Debug("get records shard") 70 | 71 | ch <- &GetRecordsEntry{Created: now, Shard: shard, Records: resp.Records} 72 | 73 | ks.iteratorMutex.Lock() 74 | ks.iterators[shard] = resp.NextShardIterator 75 | ks.iteratorMutex.Unlock() 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kinesis-tai 2 | 3 | Tool which provides tail for [Kinesis](https://aws.amazon.com/kinesis/streams/), it allows you to use one of two processors for the data returned, firstly one which decompresses and parses [CloudWatch Logs](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/WhatIsCloudWatchLogs.html) data, and secondly one which just returns the raw data. 4 | 5 | # background 6 | 7 | This cloudwatch logs reader works with a pattern used at Versent for log distribution and storage. 8 | 9 | For more information on the setup for `cwlogs` sub command to function it assumes the logs are gzipped batches of log JSON records in Kinesis see [Real-time Processing of Log Data with Subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CreateDestination.html) 10 | 11 | # installation 12 | 13 | You can download `kinesis-tail` from [Releases](https://github.com/Versent/kinesis-tail/releases) or install it using npm. 14 | 15 | # usage 16 | 17 | ``` 18 | usage: kinesis-tail [] [ ...] 19 | 20 | Flags: 21 | --help Show context-sensitive help (also try --help-long and --help-man). 22 | -t, --trace Enable trace mode. 23 | -r, --region=REGION Configure the aws region. 24 | --version Show application version. 25 | 26 | Commands: 27 | help [...] 28 | Show help. 29 | 30 | 31 | cwlogs [] 32 | Process cloudwatch logs data from kinesis. 33 | 34 | --include=INCLUDE ... Include anything in log group names which match the supplied string. 35 | --exclude=EXCLUDE ... Exclude anything in log group names which match the supplied string. 36 | 37 | raw [] 38 | Process raw data from kinesis. 39 | 40 | --timeout=3600000 How long to capture raw data for before exiting in ms. 41 | --count=0 How many records to capture raw data for before exiting. 42 | 43 | 44 | ``` 45 | 46 | # example 47 | 48 | List the kinesis streams in your account. 49 | 50 | ``` 51 | aws kinesis list-streams 52 | ``` 53 | 54 | To tail one of these streams and exit once you have captured 20 records. 55 | 56 | ``` 57 | kinesis-tail raw dev-1-stream --count 20 58 | ``` 59 | 60 | To tail one of these streams and exit after 30 seconds, and write the data to a file. 61 | 62 | ``` 63 | kinesis-tail raw dev-1-stream --timeout 30000 | tee data.log 64 | ``` 65 | 66 | # license 67 | 68 | This code is released under MIT License. 69 | 70 | -------------------------------------------------------------------------------- /pkg/ktail/kinesis.go: -------------------------------------------------------------------------------- 1 | package ktail 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/aws/aws-sdk-go/aws" 7 | "github.com/aws/aws-sdk-go/service/kinesis" 8 | "github.com/aws/aws-sdk-go/service/kinesis/kinesisiface" 9 | "github.com/pkg/errors" 10 | "github.com/sirupsen/logrus" 11 | ) 12 | 13 | const ( 14 | millisecondInNanoseconds = 1e6 15 | ) 16 | 17 | // KinesisHelper simple helper for general high level kinesis operations 18 | type KinesisHelper struct { 19 | svc kinesisiface.KinesisAPI 20 | logger *logrus.Logger 21 | } 22 | 23 | type iteratorResult struct { 24 | shardID string 25 | iterator *string 26 | } 27 | 28 | // New build a new configured kinesis helper 29 | func New(svc kinesisiface.KinesisAPI, logger *logrus.Logger) *KinesisHelper { 30 | return &KinesisHelper{ 31 | svc: svc, 32 | logger: logger, 33 | } 34 | } 35 | 36 | // GetStreamIterators build a list of iterators for the stream 37 | func (kh *KinesisHelper) GetStreamIterators(streamName string, timestamp int64) (map[string]*string, error) { 38 | ts := buildTimestamp(timestamp) 39 | 40 | kh.logger.WithField("ts", ts.UnixNano()).Info("starting stream") 41 | 42 | respDesc, err := kh.svc.DescribeStream(&kinesis.DescribeStreamInput{ 43 | StreamName: aws.String(streamName), 44 | }) 45 | if err != nil { 46 | return nil, errors.Wrap(err, "describe stream failed") 47 | } 48 | kh.logger.WithField("respDesc", respDesc).Debug("describe stream response") 49 | 50 | ch := make(chan *iteratorResult, len(respDesc.StreamDescription.Shards)) // buffered 51 | iterators := map[string]*string{} 52 | 53 | for _, shard := range respDesc.StreamDescription.Shards { 54 | go kh.asyncGetShardIterator(streamName, aws.StringValue(shard.ShardId), ts, ch) 55 | } 56 | 57 | for range respDesc.StreamDescription.Shards { 58 | res := <-ch 59 | 60 | iterators[res.shardID] = res.iterator 61 | } 62 | 63 | return iterators, nil 64 | } 65 | 66 | func (kh *KinesisHelper) asyncGetShardIterator(streamName, shardID string, ts time.Time, ch chan *iteratorResult) { 67 | kh.logger.WithField("shard", shardID).Debug("get shard iterator") 68 | 69 | respShard, err := kh.svc.GetShardIterator(&kinesis.GetShardIteratorInput{ 70 | StreamName: aws.String(streamName), 71 | ShardIteratorType: aws.String(kinesis.ShardIteratorTypeAtTimestamp), 72 | ShardId: aws.String(shardID), 73 | Timestamp: aws.Time(ts), 74 | }) 75 | if err != nil { 76 | kh.logger.WithError(err).Fatal("get shard iterator failed") 77 | } 78 | 79 | ch <- &iteratorResult{shardID: shardID, iterator: respShard.ShardIterator} 80 | } 81 | 82 | func buildTimestamp(timestamp int64) time.Time { 83 | ts := time.Now() 84 | 85 | if timestamp > 0 { 86 | ts = time.Unix(0, timestamp*millisecondInNanoseconds) 87 | } 88 | 89 | return ts 90 | } 91 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | dupl: 3 | threshold: 100 4 | funlen: 5 | lines: 100 6 | statements: 50 7 | goconst: 8 | min-len: 2 9 | min-occurrences: 2 10 | gocritic: 11 | enabled-tags: 12 | - diagnostic 13 | - experimental 14 | - opinionated 15 | - performance 16 | - style 17 | disabled-checks: 18 | - dupImport # https://github.com/go-critic/go-critic/issues/845 19 | - ifElseChain 20 | - octalLiteral 21 | - whyNoLint 22 | - wrapperFunc 23 | gocyclo: 24 | min-complexity: 15 25 | goimports: 26 | local-prefixes: github.com/golangci/golangci-lint 27 | golint: 28 | min-confidence: 0 29 | gomnd: 30 | settings: 31 | mnd: 32 | # don't include the "operation" and "assign" 33 | checks: argument,case,condition,return 34 | govet: 35 | check-shadowing: true 36 | lll: 37 | line-length: 140 38 | maligned: 39 | suggest-new: true 40 | misspell: 41 | locale: US 42 | 43 | linters: 44 | # please, do not use `enable-all`: it's deprecated and will be removed soon. 45 | # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint 46 | disable-all: true 47 | enable: 48 | - bodyclose 49 | - deadcode 50 | - depguard 51 | - dogsled 52 | - dupl 53 | - errcheck 54 | - funlen 55 | - gochecknoinits 56 | - goconst 57 | - gocritic 58 | - gocyclo 59 | - gofmt 60 | - goimports 61 | - golint 62 | - gomnd 63 | - goprintffuncname 64 | - gosec 65 | - gosimple 66 | - govet 67 | - ineffassign 68 | - interfacer 69 | # - lll 70 | - misspell 71 | - nakedret 72 | - rowserrcheck 73 | - scopelint 74 | - staticcheck 75 | - structcheck 76 | - stylecheck 77 | - typecheck 78 | - unconvert 79 | - unparam 80 | - unused 81 | - varcheck 82 | - whitespace 83 | 84 | # don't enable: 85 | # - gochecknoglobals 86 | # - gocognit 87 | # - godox 88 | # - maligned 89 | # - prealloc 90 | 91 | issues: 92 | # Excluding configuration per-path, per-linter, per-text and per-source 93 | exclude: 94 | - Using the variable on range scope `tt` in function literal # exclude table test variables 95 | exclude-rules: 96 | - path: _test\.go 97 | linters: 98 | - gomnd 99 | 100 | run: 101 | skip-dirs: 102 | - testdata 103 | - mocks 104 | 105 | # golangci.com configuration 106 | # https://github.com/golangci/golangci/wiki/Configuration 107 | service: 108 | golangci-lint-version: 1.23.x # use the fixed version to not introduce new linters unexpectedly 109 | prepare: 110 | - echo "here I can run custom commands, but no preparation needed for this repo" 111 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= 2 | github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= 3 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= 4 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 5 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= 6 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 7 | github.com/aws/aws-sdk-go v1.30.26 h1:wP0N6DBb/3EyHTtWNz4jzgGVi1l290zoFGfu4HFGeM0= 8 | github.com/aws/aws-sdk-go v1.30.26/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= 13 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 14 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 15 | github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= 16 | github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= 17 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 18 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 19 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 20 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 21 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 22 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 23 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 24 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 25 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 26 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 27 | github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= 28 | github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 29 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 30 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 31 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 32 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 33 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 34 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 35 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= 36 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 37 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 38 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y= 39 | golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= 40 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 41 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 42 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= 43 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 44 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 45 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 46 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 49 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 50 | -------------------------------------------------------------------------------- /cmd/kinesis-tail/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "runtime/trace" 8 | "time" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/versent/kinesis-tail/pkg/rawdata" 12 | 13 | "github.com/alecthomas/kingpin" 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/credentials" 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | "github.com/aws/aws-sdk-go/service/kinesis" 18 | "github.com/aws/aws-sdk-go/service/kinesis/kinesisiface" 19 | "github.com/fatih/color" 20 | "github.com/pkg/errors" 21 | "github.com/versent/kinesis-tail/pkg/ktail" 22 | "github.com/versent/kinesis-tail/pkg/logdata" 23 | "github.com/versent/kinesis-tail/pkg/sorter" 24 | "github.com/versent/kinesis-tail/pkg/streamer" 25 | ) 26 | 27 | var ( 28 | // Version program version which is updated via build flags 29 | version = "master" 30 | 31 | tracing = kingpin.Flag("trace", "Enable trace mode.").Short('t').Bool() 32 | debug = kingpin.Flag("debug", "Enable debug logging.").Short('d').Bool() 33 | region = kingpin.Flag("region", "Configure the aws region.").Short('r').String() 34 | profile = kingpin.Flag("profile", "Configure the aws profile.").Short('p').String() 35 | timestamp = kingpin.Flag("timestamp", "Start time in epoch milliseconds.").Short('T').Int64() 36 | cwlogsCommand = kingpin.Command("cwlogs", "Process cloudwatch logs data from kinesis.") 37 | includes = cwlogsCommand.Flag("include", "Include anything in log group names which match the supplied string.").Strings() 38 | excludes = cwlogsCommand.Flag("exclude", "Exclude anything in log group names which match the supplied string.").Strings() 39 | cwlogsStream = cwlogsCommand.Arg("stream", "Kinesis stream name.").Required().String() 40 | rawCommand = kingpin.Command("raw", "Process raw data from kinesis.") 41 | rawStream = rawCommand.Arg("stream", "Kinesis stream name.").Required().String() 42 | timeout = rawCommand.Flag("timeout", "How long to capture raw data for before exiting in ms.").Default("3600000").Int64() 43 | count = rawCommand.Flag("count", "How many records to capture raw data for before exiting.").Default("0").Int() 44 | 45 | logger = logrus.New() 46 | ) 47 | 48 | func main() { 49 | kingpin.Version(version) 50 | subCommand := kingpin.Parse() 51 | 52 | if *tracing { 53 | f, err := os.Create("trace.out") 54 | if err != nil { 55 | logger.WithError(err).Fatal("failed to create trace file") 56 | } 57 | 58 | err = trace.Start(f) 59 | if err != nil { 60 | logger.WithError(err).Fatal("failed to start trace") 61 | } 62 | 63 | defer trace.Stop() 64 | } 65 | 66 | if *debug { 67 | // set debug globally 68 | logrus.SetLevel(logrus.DebugLevel) 69 | // set debug in the logger we already created 70 | logger.SetLevel(logrus.DebugLevel) 71 | } 72 | 73 | svc := newKinesis(region, profile) 74 | 75 | logger.WithField("timestamp", *timestamp).Debug("built kinesis service") 76 | 77 | switch subCommand { 78 | case "cwlogs": 79 | err := processLogData(svc, *cwlogsStream, *timestamp, *includes, *excludes) 80 | if err != nil { 81 | logger.WithError(err).Fatal("failed to process log data") 82 | } 83 | case "raw": 84 | err := processRawData(svc, *rawStream, *timeout, *timestamp, *count) 85 | if err != nil { 86 | logger.WithError(err).Fatal("failed to process log data") 87 | } 88 | } 89 | } 90 | 91 | func processLogData(svc kinesisiface.KinesisAPI, stream string, timestamp int64, includes, excludes []string) error { 92 | helper := ktail.New(svc, logger) 93 | 94 | iterators, err := helper.GetStreamIterators(stream, timestamp) 95 | if err != nil { 96 | return errors.Wrap(err, "get iterators failed") 97 | } 98 | 99 | kstream := streamer.New(svc, iterators, 5000, logger) 100 | ch := kstream.StartGetRecords() 101 | 102 | messageSorter := sorter.New(os.Stdout, len(iterators), formatLogsMsg) 103 | 104 | for result := range ch { 105 | logger.WithField("count", len(result.Records)).WithField("shard", result.Shard).Debug("received records") 106 | 107 | if result.Err != nil { 108 | return errors.Wrap(result.Err, "get records failed") 109 | } 110 | 111 | msgResults := []*ktail.LogMessage{} 112 | 113 | for _, rec := range result.Records { 114 | msgs, err := logdata.UncompressLogs(includes, excludes, rec.ApproximateArrivalTimestamp, rec.Data) 115 | if err != nil { 116 | return errors.Wrap(err, "parse log records failed") 117 | } 118 | 119 | msgResults = append(msgResults, msgs...) 120 | } 121 | 122 | messageSorter.PushBatch(msgResults) 123 | } 124 | 125 | return nil 126 | } 127 | 128 | func processRawData(svc kinesisiface.KinesisAPI, stream string, timeout, timestamp int64, count int) error { 129 | helper := ktail.New(svc, logger) 130 | 131 | iterators, err := helper.GetStreamIterators(stream, timestamp) 132 | if err != nil { 133 | return errors.Wrap(err, "get iterators failed") 134 | } 135 | 136 | kstream := streamer.New(svc, iterators, 5000, logger) 137 | ch := kstream.StartGetRecords() 138 | 139 | messageSorter := sorter.New(os.Stdout, len(iterators), formatRawMsg) 140 | 141 | timer1 := time.NewTimer(time.Duration(timeout) * time.Millisecond) 142 | 143 | if count > 0 { 144 | logger.WithField("count", count).Debug("waiting for records") 145 | } 146 | 147 | var recordCount int 148 | 149 | LOOP: 150 | for { 151 | select { 152 | case result := <-ch: 153 | if result.Err != nil { 154 | return errors.Wrap(result.Err, "get records failed") 155 | } 156 | 157 | logger.WithFields(logrus.Fields{ 158 | "count": len(result.Records), 159 | "total": recordCount, 160 | "shard": result.Shard, 161 | }).Debug("received records") 162 | 163 | msgResults := []*ktail.LogMessage{} 164 | 165 | for _, rec := range result.Records { 166 | msg := rawdata.DecodeRawData(rec.ApproximateArrivalTimestamp, rec.Data) 167 | msgResults = append(msgResults, msg) 168 | } 169 | 170 | messageSorter.PushBatch(msgResults) 171 | 172 | recordCount += len(result.Records) 173 | 174 | if count != 0 { 175 | if recordCount >= count { 176 | messageSorter.Flush() 177 | 178 | logger.WithField("recordCount", recordCount).Info("reached count exit") 179 | break LOOP 180 | } 181 | } 182 | 183 | case <-timer1.C: 184 | logger.Info("timer expired exit") 185 | break LOOP 186 | } 187 | } 188 | 189 | return nil 190 | } 191 | 192 | func formatRawMsg(wr io.Writer, msg *ktail.LogMessage) { 193 | _, err := fmt.Fprintln(wr, msg.Message) 194 | if err != nil { 195 | logger.WithError(err).Fatal("failed to create trace file") 196 | } 197 | } 198 | 199 | func formatLogsMsg(wr io.Writer, msg *ktail.LogMessage) { 200 | c := color.New(color.FgBlue) 201 | _, err := fmt.Fprintf(wr, "%s %s\n", c.Sprintf("[%s %s]", msg.Timestamp, msg.LogGroup), msg.Message) 202 | if err != nil { 203 | logger.WithError(err).Fatal("failed to create trace file") 204 | } 205 | } 206 | 207 | func newKinesis(region, profile *string) kinesisiface.KinesisAPI { 208 | sess := session.Must(session.NewSession()) 209 | 210 | cfg := aws.NewConfig() 211 | 212 | if aws.StringValue(region) != "" { 213 | cfg = cfg.WithRegion(*region) 214 | } 215 | 216 | if aws.StringValue(profile) != "" { 217 | cfg = cfg.WithCredentials(credentials.NewSharedCredentials("", *profile)) 218 | } 219 | 220 | return kinesis.New(sess, cfg) 221 | } 222 | --------------------------------------------------------------------------------