├── .dockerignore ├── .gitattributes ├── .github ├── FUNDING.yml ├── dependabot.yml └── workflows │ ├── go.yml │ └── release.yml ├── doc └── flow.png ├── stash ├── filter │ ├── removefieldfilter.go │ ├── transferfilter.go │ ├── filters.go │ ├── dropfilter.go │ ├── addurifieldfilter.go │ ├── removefieldfilter_test.go │ └── transferfilter_test.go ├── Dockerfile ├── etc │ └── config.yaml ├── handler │ └── handler.go ├── es │ ├── index_test.go │ ├── writer.go │ └── index.go ├── config │ └── config.go └── stash.go ├── .gitignore ├── .golangci.yml ├── LICENSE ├── go.mod ├── readme-cn.md ├── readme.md └── go.sum /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | * text=auto eol=lf 2 | -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [kevwan] 2 | -------------------------------------------------------------------------------- /doc/flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kevwan/go-stash/HEAD/doc/flow.png -------------------------------------------------------------------------------- /stash/filter/removefieldfilter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | func RemoveFieldFilter(fields []string) FilterFunc { 4 | return func(m map[string]interface{}) map[string]interface{} { 5 | for _, field := range fields { 6 | delete(m, field) 7 | } 8 | return m 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore all 2 | * 3 | 4 | # Unignore all with extensions 5 | !*.* 6 | !**/Dockerfile 7 | 8 | # Unignore all dirs 9 | !*/ 10 | !api 11 | 12 | .idea 13 | **/.DS_Store 14 | **/logs 15 | !Makefile 16 | 17 | # gitlab ci 18 | .cache 19 | 20 | # vim auto backup file 21 | *~ 22 | !OWNERS 23 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "gomod" # See documentation for possible values 9 | directory: "/" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | branches: [ master ] 6 | pull_request: 7 | branches: [ master ] 8 | 9 | jobs: 10 | 11 | build: 12 | name: Build 13 | runs-on: ubuntu-latest 14 | steps: 15 | 16 | - name: Set up Go 1.x 17 | uses: actions/setup-go@v2 18 | with: 19 | go-version: ^1.25 20 | id: go 21 | 22 | - name: Check out code into the Go module directory 23 | uses: actions/checkout@v2 24 | 25 | - name: Get dependencies 26 | run: | 27 | go get -v -t -d ./... 28 | 29 | - name: Test 30 | run: go test -v -race ./... 31 | -------------------------------------------------------------------------------- /stash/filter/transferfilter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | jsoniter "github.com/json-iterator/go" 5 | ) 6 | 7 | func TransferFilter(field, target string) FilterFunc { 8 | return func(m map[string]interface{}) map[string]interface{} { 9 | val, ok := m[field] 10 | if !ok { 11 | return m 12 | } 13 | 14 | s, ok := val.(string) 15 | if !ok { 16 | return m 17 | } 18 | 19 | var nm map[string]interface{} 20 | if err := jsoniter.Unmarshal([]byte(s), &nm); err != nil { 21 | return m 22 | } 23 | 24 | delete(m, field) 25 | if len(target) > 0 { 26 | m[target] = nm 27 | } else { 28 | for k, v := range nm { 29 | m[k] = v 30 | } 31 | } 32 | 33 | return m 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # concurrency: 6 3 | timeout: 5m 4 | skip-dirs: 5 | - core 6 | - doc 7 | - example 8 | - rest 9 | - rpcx 10 | - tools 11 | 12 | 13 | linters: 14 | disable-all: true 15 | enable: 16 | - bodyclose 17 | - deadcode 18 | - errcheck 19 | - gosimple 20 | - govet 21 | - ineffassign 22 | - staticcheck 23 | - structcheck 24 | - typecheck 25 | - unused 26 | - varcheck 27 | # - dupl 28 | 29 | 30 | linters-settings: 31 | 32 | issues: 33 | exclude-rules: 34 | - linters: 35 | - staticcheck 36 | text: 'SA1019: (baseresponse.BoolResponse|oldresponse.FormatBadRequestResponse|oldresponse.FormatResponse)|SA5008: unknown JSON option ("optional"|"default=|"range=|"options=)' 37 | -------------------------------------------------------------------------------- /stash/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.25.1-alpine AS builder 2 | 3 | LABEL stage=gobuilder 4 | 5 | ENV CGO_ENABLED 0 6 | ENV GOPROXY https://goproxy.cn,direct 7 | 8 | RUN apk update --no-cache && apk add --no-cache tzdata 9 | 10 | WORKDIR /build 11 | 12 | ADD go.mod . 13 | ADD go.sum . 14 | RUN go mod download 15 | COPY . . 16 | COPY stash/etc /app/etc 17 | RUN go build -ldflags="-s -w" -o /app/stash stash/stash.go 18 | 19 | 20 | FROM scratch 21 | 22 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 23 | COPY --from=builder /usr/share/zoneinfo/Asia/Shanghai /usr/share/zoneinfo/Asia/Shanghai 24 | ENV TZ Asia/Shanghai 25 | 26 | WORKDIR /app 27 | COPY --from=builder /app/stash /app/stash 28 | COPY --from=builder /app/etc /app/etc 29 | 30 | CMD ["./stash", "-f", "etc/config.yaml"] 31 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | # .github/workflows/release.yaml 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | releases-matrix: 9 | name: Release Go Binary 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | # build and publish in parallel: linux/386, linux/amd64, linux/arm64, windows/386, windows/amd64, darwin/amd64, darwin/arm64 14 | goos: [linux] 15 | goarch: [amd64, arm64] 16 | steps: 17 | - uses: actions/checkout@v4 18 | - uses: wangyoucao577/go-release-action@v1.46 19 | with: 20 | github_token: ${{ secrets.GITHUB_TOKEN }} 21 | goos: ${{ matrix.goos }} 22 | goarch: ${{ matrix.goarch }} 23 | 24 | goversion: "https://go.dev/dl/go1.25.1.linux-amd64.tar.gz" 25 | project_path: "./stash" 26 | binary_name: "stash" 27 | -------------------------------------------------------------------------------- /stash/filter/filters.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "github.com/kevwan/go-stash/stash/config" 4 | 5 | const ( 6 | filterDrop = "drop" 7 | filterRemoveFields = "remove_field" 8 | filterTransfer = "transfer" 9 | opAnd = "and" 10 | opOr = "or" 11 | typeContains = "contains" 12 | typeMatch = "match" 13 | ) 14 | 15 | type FilterFunc func(map[string]interface{}) map[string]interface{} 16 | 17 | func CreateFilters(p config.Cluster) []FilterFunc { 18 | var filters []FilterFunc 19 | 20 | for _, f := range p.Filters { 21 | switch f.Action { 22 | case filterDrop: 23 | filters = append(filters, DropFilter(f.Conditions)) 24 | case filterRemoveFields: 25 | filters = append(filters, RemoveFieldFilter(f.Fields)) 26 | case filterTransfer: 27 | filters = append(filters, TransferFilter(f.Field, f.Target)) 28 | } 29 | } 30 | 31 | return filters 32 | } 33 | -------------------------------------------------------------------------------- /stash/filter/dropfilter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/kevwan/go-stash/stash/config" 7 | ) 8 | 9 | func DropFilter(conds []config.Condition) FilterFunc { 10 | return func(m map[string]interface{}) map[string]interface{} { 11 | var qualify bool 12 | for _, cond := range conds { 13 | var qualifyOnce bool 14 | switch cond.Type { 15 | case typeMatch: 16 | qualifyOnce = cond.Value == m[cond.Key] 17 | case typeContains: 18 | if val, ok := m[cond.Key].(string); ok { 19 | qualifyOnce = strings.Contains(val, cond.Value) 20 | } 21 | } 22 | 23 | switch cond.Op { 24 | case opAnd: 25 | if !qualifyOnce { 26 | return m 27 | } else { 28 | qualify = true 29 | } 30 | case opOr: 31 | if qualifyOnce { 32 | qualify = true 33 | } 34 | } 35 | } 36 | 37 | if qualify { 38 | return nil 39 | } else { 40 | return m 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /stash/filter/addurifieldfilter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/hex" 5 | "strings" 6 | ) 7 | 8 | func AddUriFieldFilter(inField, outField string) FilterFunc { 9 | return func(m map[string]interface{}) map[string]interface{} { 10 | if val, ok := m[inField].(string); ok { 11 | var datas []string 12 | idx := strings.Index(val, "?") 13 | if idx < 0 { 14 | datas = strings.Split(val, "/") 15 | } else { 16 | datas = strings.Split(val[:idx], "/") 17 | } 18 | 19 | for i, data := range datas { 20 | if IsObjectIdHex(data) { 21 | datas[i] = "*" 22 | } 23 | } 24 | 25 | m[outField] = strings.Join(datas, "/") 26 | } 27 | 28 | return m 29 | } 30 | } 31 | 32 | // IsObjectIdHex returns whether s is a valid hex representation of 33 | // an ObjectId. See the ObjectIdHex function. 34 | func IsObjectIdHex(s string) bool { 35 | if len(s) != 24 { 36 | return false 37 | } 38 | 39 | _, err := hex.DecodeString(s) 40 | return err == nil 41 | } 42 | -------------------------------------------------------------------------------- /stash/filter/removefieldfilter_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRemoveFieldFilter(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | input map[string]interface{} 13 | fields []string 14 | expect map[string]interface{} 15 | }{ 16 | { 17 | name: "remove field", 18 | input: map[string]interface{}{ 19 | "a": "aa", 20 | "b": `{"c":"cc"}`, 21 | }, 22 | fields: []string{"b"}, 23 | expect: map[string]interface{}{ 24 | "a": "aa", 25 | }, 26 | }, 27 | { 28 | name: "remove field", 29 | input: map[string]interface{}{ 30 | "a": "aa", 31 | "b": `{"c":"cc"}`, 32 | }, 33 | fields: []string{"c"}, 34 | expect: map[string]interface{}{ 35 | "a": "aa", 36 | "b": `{"c":"cc"}`, 37 | }, 38 | }, 39 | } 40 | 41 | for _, test := range tests { 42 | t.Run(test.name, func(t *testing.T) { 43 | actual := RemoveFieldFilter(test.fields)(test.input) 44 | assert.EqualValues(t, test.expect, actual) 45 | }) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /stash/etc/config.yaml: -------------------------------------------------------------------------------- 1 | Clusters: 2 | - Input: 3 | Kafka: 4 | Name: gostash 5 | Brokers: 6 | - "172.16.186.16:19092" 7 | - "172.16.186.17:19092" 8 | Topics: 9 | - k8slog 10 | Group: pro 11 | Consumers: 16 12 | Filters: 13 | - Action: drop 14 | Conditions: 15 | - Key: k8s_container_name 16 | Value: "-rpc" 17 | Type: contains 18 | - Key: level 19 | Value: info 20 | Type: match 21 | Op: and 22 | - Action: remove_field 23 | Fields: 24 | - message 25 | - _source 26 | - _type 27 | - _score 28 | - _id 29 | - "@version" 30 | - topic 31 | - index 32 | - beat 33 | - docker_container 34 | - offset 35 | - prospector 36 | - source 37 | - stream 38 | - Action: transfer 39 | Field: message 40 | Target: data 41 | Output: 42 | ElasticSearch: 43 | Hosts: 44 | - http://172.16.141.4:9200 45 | - http://172.16.141.5:9200 46 | Index: "{.event}-{{yyyy-MM-dd}}" 47 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 xiaoheiban_server_go 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /stash/handler/handler.go: -------------------------------------------------------------------------------- 1 | package handler 2 | 3 | import ( 4 | "context" 5 | 6 | jsoniter "github.com/json-iterator/go" 7 | "github.com/kevwan/go-stash/stash/es" 8 | "github.com/kevwan/go-stash/stash/filter" 9 | ) 10 | 11 | type MessageHandler struct { 12 | writer *es.Writer 13 | indexer *es.Index 14 | filters []filter.FilterFunc 15 | } 16 | 17 | func NewHandler(writer *es.Writer, indexer *es.Index) *MessageHandler { 18 | return &MessageHandler{ 19 | writer: writer, 20 | indexer: indexer, 21 | } 22 | } 23 | 24 | func (mh *MessageHandler) AddFilters(filters ...filter.FilterFunc) { 25 | for _, f := range filters { 26 | mh.filters = append(mh.filters, f) 27 | } 28 | } 29 | 30 | func (mh *MessageHandler) Consume(_ context.Context, _, val string) error { 31 | var m map[string]interface{} 32 | if err := jsoniter.Unmarshal([]byte(val), &m); err != nil { 33 | return err 34 | } 35 | 36 | index := mh.indexer.GetIndex(m) 37 | for _, proc := range mh.filters { 38 | if m = proc(m); m == nil { 39 | return nil 40 | } 41 | } 42 | 43 | bs, err := jsoniter.Marshal(m) 44 | if err != nil { 45 | return err 46 | } 47 | 48 | return mh.writer.Write(index, string(bs)) 49 | } 50 | -------------------------------------------------------------------------------- /stash/es/index_test.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | const testTime = "2020-09-13T08:22:29.294Z" 11 | 12 | func TestBuildIndexFormatter(t *testing.T) { 13 | tests := []struct { 14 | name string 15 | val string 16 | attrs map[string]interface{} 17 | expect string 18 | }{ 19 | { 20 | name: "plain text only", 21 | val: "yyyy/MM/dd", 22 | expect: "yyyy/MM/dd", 23 | }, 24 | { 25 | name: "time only", 26 | val: "{{yyyy/MM/dd}}", 27 | expect: time.Now().Format("2006/01/02"), 28 | }, 29 | { 30 | name: "attr without time", 31 | val: "{.event}", 32 | attrs: map[string]interface{}{ 33 | "event": "foo", 34 | }, 35 | expect: "foo", 36 | }, 37 | { 38 | name: "attr with time", 39 | val: "{.event}-{{yyyy/MM/dd}}", 40 | attrs: map[string]interface{}{ 41 | "event": "foo", 42 | timestampKey: testTime, 43 | }, 44 | expect: "foo-2020/09/13", 45 | }, 46 | { 47 | name: "attr with time, with missing", 48 | val: "{.event}-{.foo}-{{yyyy/MM/dd}}", 49 | attrs: map[string]interface{}{ 50 | "event": "foo", 51 | timestampKey: testTime, 52 | }, 53 | expect: "foo--2020/09/13", 54 | }, 55 | { 56 | name: "attr with time, leading alphas", 57 | val: "{the.event}-{{yyyy/MM/dd}}", 58 | attrs: map[string]interface{}{ 59 | "event": "foo", 60 | timestampKey: testTime, 61 | }, 62 | expect: "foo-2020/09/13", 63 | }, 64 | } 65 | 66 | for _, test := range tests { 67 | t.Run(test.name, func(t *testing.T) { 68 | formatter := buildIndexFormatter(test.val, time.Local) 69 | assert.Equal(t, test.expect, formatter(test.attrs)) 70 | }) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /stash/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/zeromicro/go-zero/core/service" 7 | ) 8 | 9 | type ( 10 | Condition struct { 11 | Key string 12 | Value string 13 | Type string `json:",default=match,options=match|contains"` 14 | Op string `json:",default=and,options=and|or"` 15 | } 16 | 17 | ElasticSearchConf struct { 18 | Hosts []string 19 | Index string 20 | DocType string `json:",default=doc"` 21 | TimeZone string `json:",optional"` 22 | MaxChunkBytes int `json:",default=15728640"` // default 15M 23 | Compress bool `json:",default=false"` 24 | Username string `json:",optional"` 25 | Password string `json:",optional"` 26 | } 27 | 28 | Filter struct { 29 | Action string `json:",options=drop|remove_field|transfer"` 30 | Conditions []Condition `json:",optional"` 31 | Fields []string `json:",optional"` 32 | Field string `json:",optional"` 33 | Target string `json:",optional"` 34 | } 35 | 36 | KafkaConf struct { 37 | service.ServiceConf 38 | Brokers []string 39 | Group string 40 | Topics []string 41 | Offset string `json:",options=first|last,default=last"` 42 | Conns int `json:",default=1"` 43 | Consumers int `json:",default=8"` 44 | Processors int `json:",default=8"` 45 | MinBytes int `json:",default=10240"` // 10K 46 | MaxBytes int `json:",default=10485760"` // 10M 47 | Username string `json:",optional"` 48 | Password string `json:",optional"` 49 | } 50 | 51 | Cluster struct { 52 | Input struct { 53 | Kafka KafkaConf 54 | } 55 | Filters []Filter `json:",optional"` 56 | Output struct { 57 | ElasticSearch ElasticSearchConf 58 | } 59 | } 60 | 61 | Config struct { 62 | Clusters []Cluster 63 | GracePeriod time.Duration `json:",default=10s"` 64 | } 65 | ) 66 | -------------------------------------------------------------------------------- /stash/filter/transferfilter_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestTransferFilter(t *testing.T) { 10 | tests := []struct { 11 | name string 12 | input map[string]interface{} 13 | field string 14 | target string 15 | expect map[string]interface{} 16 | }{ 17 | { 18 | name: "with target", 19 | input: map[string]interface{}{ 20 | "a": "aa", 21 | "b": `{"c":"cc"}`, 22 | }, 23 | field: "b", 24 | target: "data", 25 | expect: map[string]interface{}{ 26 | "a": "aa", 27 | "data": map[string]interface{}{ 28 | "c": "cc", 29 | }, 30 | }, 31 | }, 32 | { 33 | name: "without target", 34 | input: map[string]interface{}{ 35 | "a": "aa", 36 | "b": `{"c":"cc"}`, 37 | }, 38 | field: "b", 39 | expect: map[string]interface{}{ 40 | "a": "aa", 41 | "c": "cc", 42 | }, 43 | }, 44 | { 45 | name: "without field", 46 | input: map[string]interface{}{ 47 | "a": "aa", 48 | "b": `{"c":"cc"}`, 49 | }, 50 | field: "c", 51 | expect: map[string]interface{}{ 52 | "a": "aa", 53 | "b": `{"c":"cc"}`, 54 | }, 55 | }, 56 | { 57 | name: "with not json", 58 | input: map[string]interface{}{ 59 | "a": "aa", 60 | "b": `{"c":"cc"`, 61 | }, 62 | field: "b", 63 | expect: map[string]interface{}{ 64 | "a": "aa", 65 | "b": `{"c":"cc"`, 66 | }, 67 | }, 68 | { 69 | name: "with not string", 70 | input: map[string]interface{}{ 71 | "a": "aa", 72 | "b": map[string]interface{}{"c": "cc"}, 73 | }, 74 | field: "b", 75 | expect: map[string]interface{}{ 76 | "a": "aa", 77 | "b": map[string]interface{}{"c": "cc"}, 78 | }, 79 | }, 80 | } 81 | 82 | for _, test := range tests { 83 | t.Run(test.name, func(t *testing.T) { 84 | actual := TransferFilter(test.field, test.target)(test.input) 85 | assert.EqualValues(t, test.expect, actual) 86 | }) 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /stash/es/writer.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/kevwan/go-stash/stash/config" 7 | "github.com/olivere/elastic/v7" 8 | "github.com/rogpeppe/go-internal/semver" 9 | "github.com/zeromicro/go-zero/core/executors" 10 | "github.com/zeromicro/go-zero/core/logx" 11 | ) 12 | 13 | const es8Version = "8.0.0" 14 | 15 | type ( 16 | Writer struct { 17 | docType string 18 | esVersion string 19 | client *elastic.Client 20 | inserter *executors.ChunkExecutor 21 | } 22 | 23 | valueWithIndex struct { 24 | index string 25 | val string 26 | } 27 | ) 28 | 29 | func NewWriter(c config.ElasticSearchConf) (*Writer, error) { 30 | client, err := elastic.NewClient( 31 | elastic.SetSniff(false), 32 | elastic.SetURL(c.Hosts...), 33 | elastic.SetGzip(c.Compress), 34 | elastic.SetBasicAuth(c.Username, c.Password), 35 | ) 36 | if err != nil { 37 | return nil, err 38 | } 39 | 40 | version, err := client.ElasticsearchVersion(c.Hosts[0]) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | writer := Writer{ 46 | docType: c.DocType, 47 | client: client, 48 | esVersion: version, 49 | } 50 | writer.inserter = executors.NewChunkExecutor(writer.execute, executors.WithChunkBytes(c.MaxChunkBytes)) 51 | return &writer, nil 52 | } 53 | 54 | func (w *Writer) Write(index, val string) error { 55 | return w.inserter.Add(valueWithIndex{ 56 | index: index, 57 | val: val, 58 | }, len(val)) 59 | } 60 | 61 | func (w *Writer) execute(vals []interface{}) { 62 | var bulk = w.client.Bulk() 63 | for _, val := range vals { 64 | pair := val.(valueWithIndex) 65 | req := elastic.NewBulkIndexRequest().Index(pair.index) 66 | if isSupportType(w.esVersion) && len(w.docType) > 0 { 67 | req = req.Type(w.docType) 68 | } 69 | req = req.Doc(pair.val) 70 | bulk.Add(req) 71 | } 72 | resp, err := bulk.Do(context.Background()) 73 | if err != nil { 74 | logx.Error(err) 75 | return 76 | } 77 | 78 | // bulk error in docs will report in response items 79 | if !resp.Errors { 80 | return 81 | } 82 | 83 | for _, imap := range resp.Items { 84 | for _, item := range imap { 85 | if item.Error == nil { 86 | continue 87 | } 88 | 89 | logx.Error(item.Error) 90 | } 91 | } 92 | } 93 | 94 | func isSupportType(version string) bool { 95 | // es8.x not support type field 96 | return semver.Compare(version, es8Version) < 0 97 | } 98 | -------------------------------------------------------------------------------- /stash/stash.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "time" 6 | 7 | "github.com/kevwan/go-stash/stash/config" 8 | "github.com/kevwan/go-stash/stash/es" 9 | "github.com/kevwan/go-stash/stash/filter" 10 | "github.com/kevwan/go-stash/stash/handler" 11 | "github.com/olivere/elastic/v7" 12 | "github.com/zeromicro/go-queue/kq" 13 | "github.com/zeromicro/go-zero/core/conf" 14 | "github.com/zeromicro/go-zero/core/logx" 15 | "github.com/zeromicro/go-zero/core/proc" 16 | "github.com/zeromicro/go-zero/core/service" 17 | ) 18 | 19 | var configFile = flag.String("f", "etc/config.yaml", "Specify the config file") 20 | 21 | func toKqConf(c config.KafkaConf) []kq.KqConf { 22 | var ret []kq.KqConf 23 | 24 | for _, topic := range c.Topics { 25 | ret = append(ret, kq.KqConf{ 26 | ServiceConf: c.ServiceConf, 27 | Brokers: c.Brokers, 28 | Group: c.Group, 29 | Topic: topic, 30 | Offset: c.Offset, 31 | Conns: c.Conns, 32 | Consumers: c.Consumers, 33 | Processors: c.Processors, 34 | MinBytes: c.MinBytes, 35 | MaxBytes: c.MaxBytes, 36 | Username: c.Username, 37 | Password: c.Password, 38 | }) 39 | } 40 | 41 | return ret 42 | } 43 | 44 | func main() { 45 | flag.Parse() 46 | 47 | var c config.Config 48 | conf.MustLoad(*configFile, &c) 49 | proc.SetTimeToForceQuit(c.GracePeriod) 50 | 51 | group := service.NewServiceGroup() 52 | defer group.Stop() 53 | 54 | for _, processor := range c.Clusters { 55 | client, err := elastic.NewClient( 56 | elastic.SetSniff(false), 57 | elastic.SetURL(processor.Output.ElasticSearch.Hosts...), 58 | elastic.SetBasicAuth(processor.Output.ElasticSearch.Username, processor.Output.ElasticSearch.Password), 59 | ) 60 | logx.Must(err) 61 | 62 | filters := filter.CreateFilters(processor) 63 | writer, err := es.NewWriter(processor.Output.ElasticSearch) 64 | logx.Must(err) 65 | 66 | var loc *time.Location 67 | if len(processor.Output.ElasticSearch.TimeZone) > 0 { 68 | loc, err = time.LoadLocation(processor.Output.ElasticSearch.TimeZone) 69 | logx.Must(err) 70 | } else { 71 | loc = time.Local 72 | } 73 | indexer := es.NewIndex(client, processor.Output.ElasticSearch.Index, loc) 74 | handle := handler.NewHandler(writer, indexer) 75 | handle.AddFilters(filters...) 76 | handle.AddFilters(filter.AddUriFieldFilter("url", "uri")) 77 | for _, k := range toKqConf(processor.Input.Kafka) { 78 | group.Add(kq.MustNewQueue(k, handle)) 79 | } 80 | } 81 | 82 | group.Start() 83 | } 84 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kevwan/go-stash 2 | 3 | go 1.25 4 | 5 | toolchain go1.25.1 6 | 7 | require ( 8 | github.com/json-iterator/go v1.1.12 9 | github.com/olivere/elastic/v7 v7.0.32 10 | github.com/rogpeppe/go-internal v1.14.1 11 | github.com/stretchr/testify v1.11.1 12 | github.com/vjeantet/jodaTime v1.0.0 13 | github.com/zeromicro/go-queue v1.2.2 14 | github.com/zeromicro/go-zero v1.9.3 15 | ) 16 | 17 | require ( 18 | github.com/beorn7/perks v1.0.1 // indirect 19 | github.com/cenkalti/backoff/v4 v4.3.0 // indirect 20 | github.com/cespare/xxhash/v2 v2.3.0 // indirect 21 | github.com/davecgh/go-spew v1.1.1 // indirect 22 | github.com/fatih/color v1.18.0 // indirect 23 | github.com/go-logr/logr v1.4.2 // indirect 24 | github.com/go-logr/stdr v1.2.2 // indirect 25 | github.com/grafana/pyroscope-go v1.2.7 // indirect 26 | github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect 27 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect 28 | github.com/josharian/intern v1.0.0 // indirect 29 | github.com/klauspost/compress v1.17.11 // indirect 30 | github.com/mailru/easyjson v0.7.7 // indirect 31 | github.com/mattn/go-colorable v0.1.13 // indirect 32 | github.com/mattn/go-isatty v0.0.20 // indirect 33 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 34 | github.com/modern-go/reflect2 v1.0.2 // indirect 35 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 36 | github.com/openzipkin/zipkin-go v0.4.3 // indirect 37 | github.com/pelletier/go-toml/v2 v2.2.2 // indirect 38 | github.com/pierrec/lz4/v4 v4.1.21 // indirect 39 | github.com/pkg/errors v0.9.1 // indirect 40 | github.com/pmezard/go-difflib v1.0.0 // indirect 41 | github.com/prometheus/client_golang v1.21.1 // indirect 42 | github.com/prometheus/client_model v0.6.1 // indirect 43 | github.com/prometheus/common v0.62.0 // indirect 44 | github.com/prometheus/procfs v0.15.1 // indirect 45 | github.com/segmentio/kafka-go v0.4.47 // indirect 46 | github.com/spaolacci/murmur3 v1.1.0 // indirect 47 | go.opentelemetry.io/otel v1.24.0 // indirect 48 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect 49 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect 50 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect 51 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect 52 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect 53 | go.opentelemetry.io/otel/exporters/zipkin v1.24.0 // indirect 54 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 55 | go.opentelemetry.io/otel/sdk v1.24.0 // indirect 56 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 57 | go.opentelemetry.io/proto/otlp v1.3.1 // indirect 58 | go.uber.org/automaxprocs v1.6.0 // indirect 59 | golang.org/x/mod v0.21.0 // indirect 60 | golang.org/x/net v0.35.0 // indirect 61 | golang.org/x/sys v0.30.0 // indirect 62 | golang.org/x/text v0.22.0 // indirect 63 | google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d // indirect 64 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect 65 | google.golang.org/grpc v1.65.0 // indirect 66 | google.golang.org/protobuf v1.36.5 // indirect 67 | gopkg.in/yaml.v2 v2.4.0 // indirect 68 | gopkg.in/yaml.v3 v3.0.1 // indirect 69 | ) 70 | -------------------------------------------------------------------------------- /readme-cn.md: -------------------------------------------------------------------------------- 1 | [English](readme.md) | 简体中文 2 | 3 | # go-stash简介 4 | 5 | go-stash是一个高效的从Kafka获取，根据配置的规则进行处理，然后发送到ElasticSearch集群的工具。 6 | 7 | go-stash有大概logstash 5倍的吞吐性能，并且部署简单，一个可执行文件即可。 8 | 9 | ![go-stash](doc/flow.png) 10 | 11 | 12 | ### 安装 13 | 14 | ```shell 15 | cd stash && go build stash.go 16 | ``` 17 | 18 | ### Quick Start 19 | 20 | - 可执行文件方式 21 | 22 | ```shell 23 | ./stash -f etc/config.yaml 24 | ``` 25 | 26 | - docker 方式，确保配置文件路径正确 27 | 28 | ```shell 29 | docker run -d -v `pwd`/etc:/app/etc kevinwan/go-stash 30 | ``` 31 | 32 | config.yaml示例如下: 33 | 34 | ```yaml 35 | Clusters: 36 | - Input: 37 | Kafka: 38 | Name: go-stash 39 | Log: 40 | Mode: file 41 | Brokers: 42 | - "172.16.48.41:9092" 43 | - "172.16.48.42:9092" 44 | - "172.16.48.43:9092" 45 | Topic: ngapplog 46 | Group: stash 47 | Conns: 3 48 | Consumers: 10 49 | Processors: 60 50 | MinBytes: 1048576 51 | MaxBytes: 10485760 52 | Offset: first 53 | Filters: 54 | - Action: drop 55 | Conditions: 56 | - Key: status 57 | Value: 503 58 | Type: contains 59 | - Key: type 60 | Value: "app" 61 | Type: match 62 | Op: and 63 | - Action: remove_field 64 | Fields: 65 | - message 66 | - source 67 | - beat 68 | - fields 69 | - input_type 70 | - offset 71 | - "@version" 72 | - _score 73 | - _type 74 | - clientip 75 | - http_host 76 | - request_time 77 | Output: 78 | ElasticSearch: 79 | Hosts: 80 | - "http://172.16.188.73:9200" 81 | - "http://172.16.188.74:9200" 82 | - "http://172.16.188.75:9200" 83 | Index: "go-stash-{{yyyy.MM.dd}}" 84 | MaxChunkBytes: 5242880 85 | GracePeriod: 10s 86 | Compress: false 87 | TimeZone: UTC 88 | ``` 89 | 90 | ## 详细说明 91 | 92 | ### input 93 | 94 | ```shell 95 | Conns: 3 96 | Consumers: 10 97 | Processors: 60 98 | MinBytes: 1048576 99 | MaxBytes: 10485760 100 | Offset: first 101 | ``` 102 | #### Conns 103 | 链接kafka的链接数，链接数依据cpu的核数，一般<= CPU的核数； 104 | 105 | #### Consumers 106 | 每个连接数打开的线程数，计算规则为Conns * Consumers，不建议超过分片总数，比如topic分片为30，Conns *Consumers <= 30 107 | 108 | #### Processors 109 | 处理数据的线程数量，依据CPU的核数，可以适当增加，建议配置：Conns * Consumers * 2 或 Conns * Consumers * 3，例如：60 或 90 110 | 111 | #### MinBytes MaxBytes 112 | 每次从kafka获取数据块的区间大小，默认为1M~10M，网络和IO较好的情况下，可以适当调高 113 | 114 | #### Offset 115 | 可选last和first，默认为last，表示从头从kafka开始读取数据 116 | 117 | 118 | ### Filters 119 | 120 | ```shell 121 | - Action: drop 122 | Conditions: 123 | - Key: k8s_container_name 124 | Value: "-rpc" 125 | Type: contains 126 | - Key: level 127 | Value: info 128 | Type: match 129 | Op: and 130 | - Action: remove_field 131 | Fields: 132 | - message 133 | - _source 134 | - _type 135 | - _score 136 | - _id 137 | - "@version" 138 | - topic 139 | - index 140 | - beat 141 | - docker_container 142 | - offset 143 | - prospector 144 | - source 145 | - stream 146 | - Action: transfer 147 | Field: message 148 | Target: data 149 | 150 | ``` 151 | #### - Action: drop 152 | - 删除标识：满足此条件的数据，在处理时将被移除，不进入es 153 | - 按照删除条件，指定key字段及Value的值，Type字段可选contains(包含)或match(匹配) 154 | - 拼接条件Op: and，也可写or 155 | 156 | #### - Action: remove_field 157 | 移除字段标识：需要移除的字段，在下面列出即可 158 | 159 | #### - Action: transfer 160 | 转移字段标识：例如可以将message字段，重新定义为data字段 161 | 162 | 163 | ### Output 164 | 165 | #### Index 166 | 索引名称，indexname-{{yyyy.MM.dd}}表示年.月.日，也可以用{{yyyy-MM-dd}}，格式自己定义 167 | 168 | #### MaxChunkBytes 169 | 每次往ES提交的bulk大小，默认是5M，可依据ES的io情况，适当的调整 170 | 171 | #### GracePeriod 172 | 默认为10s，在程序关闭后，在10s内用于处理余下的消费和数据，优雅退出 173 | 174 | #### Compress 175 | 数据压缩，压缩会减少传输的数据量，但会增加一定的处理性能，可选值true/false，默认为false 176 | 177 | #### TimeZone 178 | 默认值为UTC，世界标准时间 179 | 180 | 181 | 182 | 183 | 184 | ## ES性能写入测试 185 | 186 | 187 | ### 测试环境 188 | - stash服务器：3台 4核 8G 189 | - es服务器： 15台 16核 64G 190 | 191 | ### 关键配置 192 | 193 | ```shell 194 | - Input: 195 | Conns: 3 196 | Consumers: 10 197 | Processors: 60 198 | MinBytes: 1048576 199 | MaxBytes: 10485760 200 | Filters: 201 | - Action: remove_field 202 | Fields: 203 | - message 204 | - source 205 | - beat 206 | - fields 207 | - input_type 208 | - offset 209 | - request_time 210 | Output: 211 | Index: "nginx_pro-{{yyyy.MM.d}}" 212 | Compress: false 213 | MaxChunkBytes: 5242880 214 | TimeZone: UTC 215 | ``` 216 | 217 | ### 写入速度平均在15W/S以上 218 | ![go-stash](https://pro-public.xiaoheiban.cn/icon/ee207a1cb094c0b3dcaa91ae75b118b8.png) 219 | 220 | 221 | ### 微信交流群 222 | 223 | 加群之前有劳给一个star，一个小小的star是作者们回答问题的动力。 224 | 225 | 如果文档中未能覆盖的任何疑问，欢迎您在群里提出，我们会尽快答复。 226 | 227 | 您可以在群内提出使用中需要改进的地方，我们会考虑合理性并尽快修改。 228 | 229 | 如果您发现bug请及时提issue，我们会尽快确认并修改。 230 | 231 | 添加我的微信：kevwan，请注明go-stash，我拉进go-stash社区群🤝 232 | -------------------------------------------------------------------------------- /stash/es/index.go: -------------------------------------------------------------------------------- 1 | package es 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | "sync" 8 | "time" 9 | 10 | "github.com/olivere/elastic/v7" 11 | "github.com/vjeantet/jodaTime" 12 | "github.com/zeromicro/go-zero/core/fx" 13 | "github.com/zeromicro/go-zero/core/lang" 14 | "github.com/zeromicro/go-zero/core/logx" 15 | "github.com/zeromicro/go-zero/core/syncx" 16 | ) 17 | 18 | const ( 19 | timestampFormat = "2006-01-02T15:04:05.000Z" 20 | timestampKey = "@timestamp" 21 | leftBrace = '{' 22 | rightBrace = '}' 23 | dot = '.' 24 | ) 25 | 26 | const ( 27 | stateNormal = iota 28 | stateWrap 29 | stateVar 30 | stateDot 31 | ) 32 | 33 | type ( 34 | IndexFormat func(m map[string]interface{}) string 35 | IndexFunc func() string 36 | 37 | Index struct { 38 | client *elastic.Client 39 | indexFormat IndexFormat 40 | indices map[string]lang.PlaceholderType 41 | lock sync.RWMutex 42 | singleFlight syncx.SingleFlight 43 | } 44 | ) 45 | 46 | func NewIndex(client *elastic.Client, indexFormat string, loc *time.Location) *Index { 47 | return &Index{ 48 | client: client, 49 | indexFormat: buildIndexFormatter(indexFormat, loc), 50 | indices: make(map[string]lang.PlaceholderType), 51 | singleFlight: syncx.NewSingleFlight(), 52 | } 53 | } 54 | 55 | func (idx *Index) GetIndex(m map[string]interface{}) string { 56 | index := idx.indexFormat(m) 57 | idx.lock.RLock() 58 | if _, ok := idx.indices[index]; ok { 59 | idx.lock.RUnlock() 60 | return index 61 | } 62 | 63 | idx.lock.RUnlock() 64 | if err := idx.ensureIndex(index); err != nil { 65 | logx.Error(err) 66 | } 67 | return index 68 | } 69 | 70 | func (idx *Index) ensureIndex(index string) error { 71 | _, err := idx.singleFlight.Do(index, func() (i interface{}, err error) { 72 | idx.lock.Lock() 73 | defer idx.lock.Unlock() 74 | 75 | if _, ok := idx.indices[index]; ok { 76 | return nil, nil 77 | } 78 | 79 | defer func() { 80 | if err == nil { 81 | idx.indices[index] = lang.Placeholder 82 | } 83 | }() 84 | 85 | existsService := elastic.NewIndicesExistsService(idx.client) 86 | existsService.Index([]string{index}) 87 | exist, err := existsService.Do(context.Background()) 88 | if err != nil { 89 | return nil, err 90 | } 91 | if exist { 92 | return nil, nil 93 | } 94 | 95 | createService := idx.client.CreateIndex(index) 96 | if err := fx.DoWithRetry(func() error { 97 | // is it necessary to check the result? 98 | _, err := createService.Do(context.Background()) 99 | return err 100 | }); err != nil { 101 | return nil, err 102 | } 103 | 104 | return nil, nil 105 | }) 106 | return err 107 | } 108 | 109 | func buildIndexFormatter(indexFormat string, loc *time.Location) func(map[string]interface{}) string { 110 | format, attrs, timePos := getFormat(indexFormat) 111 | if len(attrs) == 0 { 112 | return func(m map[string]interface{}) string { 113 | return format 114 | } 115 | } 116 | 117 | return func(m map[string]interface{}) string { 118 | var vals []interface{} 119 | for i, attr := range attrs { 120 | if i == timePos { 121 | vals = append(vals, formatTime(attr, getTime(m).In(loc))) 122 | continue 123 | } 124 | 125 | if val, ok := m[attr]; ok { 126 | vals = append(vals, val) 127 | } else { 128 | vals = append(vals, "") 129 | } 130 | } 131 | return fmt.Sprintf(format, vals...) 132 | } 133 | } 134 | 135 | func formatTime(format string, t time.Time) string { 136 | return jodaTime.Format(format, t) 137 | } 138 | 139 | func getTime(m map[string]interface{}) time.Time { 140 | if ti, ok := m[timestampKey]; ok { 141 | if ts, ok := ti.(string); ok { 142 | if t, err := time.Parse(timestampFormat, ts); err == nil { 143 | return t 144 | } 145 | } 146 | } 147 | 148 | return time.Now() 149 | } 150 | 151 | func getFormat(indexFormat string) (format string, attrs []string, timePos int) { 152 | var state = stateNormal 153 | var builder strings.Builder 154 | var keyBuf strings.Builder 155 | timePos = -1 156 | writeHolder := func() { 157 | if keyBuf.Len() > 0 { 158 | attrs = append(attrs, keyBuf.String()) 159 | keyBuf.Reset() 160 | builder.WriteString("%s") 161 | } 162 | } 163 | 164 | for _, ch := range indexFormat { 165 | switch state { 166 | case stateNormal: 167 | switch ch { 168 | case leftBrace: 169 | state = stateWrap 170 | default: 171 | builder.WriteRune(ch) 172 | } 173 | case stateWrap: 174 | switch ch { 175 | case leftBrace: 176 | state = stateVar 177 | case dot: 178 | state = stateDot 179 | keyBuf.Reset() 180 | case rightBrace: 181 | state = stateNormal 182 | timePos = len(attrs) 183 | writeHolder() 184 | default: 185 | keyBuf.WriteRune(ch) 186 | } 187 | case stateVar: 188 | switch ch { 189 | case rightBrace: 190 | state = stateWrap 191 | default: 192 | keyBuf.WriteRune(ch) 193 | } 194 | case stateDot: 195 | switch ch { 196 | case rightBrace: 197 | state = stateNormal 198 | writeHolder() 199 | default: 200 | keyBuf.WriteRune(ch) 201 | } 202 | default: 203 | builder.WriteRune(ch) 204 | } 205 | } 206 | 207 | format = builder.String() 208 | return 209 | } 210 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | English | [简体中文](readme-cn.md) 2 | 3 | 4 | 5 | # go-stash 6 | 7 | go-stash is a high performance, free and open source server-side data processing pipeline that ingests data from Kafka, processes it, and then sends it to ElasticSearch. 8 | 9 | go-stash is about 5x throughput more than logstash, and easy to deploy, only one executable file. 10 | 11 | ![go-stash](doc/flow.png) 12 | 13 | ## Quick Start 14 | 15 | ### Install 16 | 17 | ```shell 18 | cd stash && go build stash.go 19 | ``` 20 | 21 | ### Quick Start 22 | 23 | - With binary 24 | 25 | ```shell 26 | ./stash -f etc/config.yaml 27 | ``` 28 | 29 | - With docker, make sure the path of config file is correct. 30 | 31 | ```shell 32 | docker run -d -v `pwd`/etc:/app/etc kevinwan/go-stash 33 | ``` 34 | 35 | The config.yaml example is as follows: 36 | 37 | ```yaml 38 | Clusters: 39 | - Input: 40 | Kafka: 41 | Name: go-stash 42 | Log: 43 | Mode: file 44 | Brokers: 45 | - "172.16.48.41:9092" 46 | - "172.16.48.42:9092" 47 | - "172.16.48.43:9092" 48 | Topic: ngapplog 49 | Group: stash 50 | Conns: 3 51 | Consumers: 10 52 | Processors: 60 53 | MinBytes: 1048576 54 | MaxBytes: 10485760 55 | Offset: first 56 | Filters: 57 | - Action: drop 58 | Conditions: 59 | - Key: status 60 | Value: 503 61 | Type: contains 62 | - Key: type 63 | Value: "app" 64 | Type: match 65 | Op: and 66 | - Action: remove_field 67 | Fields: 68 | - message 69 | - source 70 | - beat 71 | - fields 72 | - input_type 73 | - offset 74 | - "@version" 75 | - _score 76 | - _type 77 | - clientip 78 | - http_host 79 | - request_time 80 | Output: 81 | ElasticSearch: 82 | Hosts: 83 | - "http://172.16.188.73:9200" 84 | - "http://172.16.188.74:9200" 85 | - "http://172.16.188.75:9200" 86 | Index: "go-stash-{{yyyy.MM.dd}}" 87 | MaxChunkBytes: 5242880 88 | GracePeriod: 10s 89 | Compress: false 90 | TimeZone: UTC 91 | ``` 92 | 93 | ## Details 94 | 95 | ### input 96 | 97 | ```yaml 98 | Conns: 3 99 | Consumers: 10 100 | Processors: 60 101 | MinBytes: 1048576 102 | MaxBytes: 10485760 103 | Offset: first 104 | ``` 105 | #### Conns 106 | * The number of links to kafka, the number of links is based on the number of cores of the CPU, usually <= the number of cores of the CPU. 107 | 108 | #### Consumers 109 | * The number of open threads per connection, the calculation rule is Conns * Consumers, not recommended to exceed the total number of slices, for example, if the topic slice is 30, Conns * Consumers <= 30 110 | 111 | #### Processors 112 | * The number of threads to process data, depending on the number of CPU cores, can be increased appropriately, the recommended configuration: Conns * Consumers * 2 or Conns * Consumers * 3, for example: 60 or 90 113 | 114 | #### MinBytes MaxBytes 115 | * The default size of the data block from kafka is 1M~10M. If the network and IO are better, you can adjust it higher. 116 | 117 | #### Offset 118 | * Optional last and first, the default is last, which means read data from kafka from the beginning 119 | 120 | 121 | ### Filters 122 | 123 | ```yaml 124 | - Action: drop 125 | Conditions: 126 | - Key: k8s_container_name 127 | Value: "-rpc" 128 | Type: contains 129 | - Key: level 130 | Value: info 131 | Type: match 132 | Op: and 133 | - Action: remove_field 134 | Fields: 135 | - message 136 | - _source 137 | - _type 138 | - _score 139 | - _id 140 | - "@version" 141 | - topic 142 | - index 143 | - beat 144 | - docker_container 145 | - offset 146 | - prospector 147 | - source 148 | - stream 149 | - Action: transfer 150 | Field: message 151 | Target: data 152 | ``` 153 | 154 | #### - Action: drop 155 | - Delete flag: The data that meets this condition will be removed when processing and will not be entered into es 156 | - According to the delete condition, specify the value of the key field and Value, the Type field can be contains (contains) or match (match) 157 | - Splice condition Op: and, can also write or 158 | 159 | #### - Action: remove_field 160 | Remove_field_id: the field to be removed, just list it below 161 | 162 | #### - Action: transfer 163 | Transfer field identifier: for example, the message field can be redefined as a data field 164 | 165 | 166 | ### Output 167 | 168 | #### Index 169 | * Index name, indexname-{{yyyy.MM.dd}} for year. Month. Day, or {{yyyy-MM-dd}}, in your own format 170 | 171 | #### MaxChunkBytes 172 | * The size of the bulk submitted to ES each time, default is 5M, can be adjusted according to the ES io situation. 173 | 174 | #### GracePeriod 175 | * The default is 10s, which is used to process the remaining consumption and data within 10s after the program closes and exits gracefully 176 | 177 | #### Compress 178 | * Data compression, compression will reduce the amount of data transferred, but will increase certain processing performance, optional value true/false, default is false 179 | 180 | #### TimeZone 181 | * Default value is UTC, Universal Standard Time 182 | 183 | ## ES performance write test 184 | 185 | 186 | ### Test environment 187 | - stash server: 3 units 4 cores 8G 188 | - es server: 15 units 16 cores 64G 189 | 190 | ### Key configuration 191 | 192 | ```yaml 193 | - Input: 194 | Conns: 3 195 | Consumers: 10 196 | Processors: 60 197 | MinBytes: 1048576 198 | MaxBytes: 10485760 199 | Filters: 200 | - Action: remove_field 201 | Fields: 202 | - Message 203 | - source 204 | - beat 205 | - fields 206 | - input_type 207 | - offset 208 | - request_time 209 | Output: 210 | Index: "nginx_pro-{{yyyy.MM.d}}" 211 | Compress: false 212 | MaxChunkBytes: 5242880 213 | TimeZone: UTC 214 | ``` 215 | 216 | ### Write speed is above 150k/s on average 217 | ![go-stash](https://pro-public.xiaoheiban.cn/icon/ee207a1cb094c0b3dcaa91ae75b118b8.png) 218 | 219 | ## Acknowledgements 220 | 221 | go-stash is powered by [**go-zero**](https://github.com/zeromicro/go-zero) ![GitHub Repo stars](https://img.shields.io/github/stars/zeromicro/go-zero?style=social) ![GitHub forks](https://img.shields.io/github/forks/zeromicro/go-zero?style=social) for great performance! 222 | 223 | ## Give a Star! ⭐ 224 | 225 | If you like or are using this project to learn or start your solution, please give it a star. Thanks! 226 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 2 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 3 | github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= 4 | github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= 5 | github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= 6 | github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 11 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 12 | github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= 13 | github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= 14 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 15 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 16 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 17 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 18 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 19 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 20 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 21 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 22 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 23 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 24 | github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= 25 | github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= 26 | github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= 27 | github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= 28 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= 29 | github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= 30 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= 31 | github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= 32 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 33 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 34 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 35 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 36 | github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= 37 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 38 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 39 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 40 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 41 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 42 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 43 | github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= 44 | github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= 45 | github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= 46 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 47 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 48 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 49 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 50 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 51 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 52 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 53 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 54 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 55 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 56 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 57 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= 58 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= 59 | github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= 60 | github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= 61 | github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7sjsSdg= 62 | github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= 63 | github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= 64 | github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= 65 | github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 66 | github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= 67 | github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 68 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 69 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 70 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 71 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 72 | github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= 73 | github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= 74 | github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= 75 | github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= 76 | github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= 77 | github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= 78 | github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= 79 | github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= 80 | github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= 81 | github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= 82 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= 83 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 84 | github.com/segmentio/kafka-go v0.4.47 h1:IqziR4pA3vrZq7YdRxaT3w1/5fvIH5qpCwstUanQQB0= 85 | github.com/segmentio/kafka-go v0.4.47/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= 86 | github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= 87 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 88 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 89 | github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= 90 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 91 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 92 | github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= 93 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 94 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 95 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 96 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 97 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 98 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 99 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 100 | github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= 101 | github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= 102 | github.com/vjeantet/jodaTime v1.0.0 h1:Fq2K9UCsbTFtKbHpe/L7C57XnSgbZ5z+gyGpn7cTE3s= 103 | github.com/vjeantet/jodaTime v1.0.0/go.mod h1:gA+i8InPfZxL1ToHaDpzi6QT/npjl3uPlcV4cxDNerI= 104 | github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= 105 | github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= 106 | github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= 107 | github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= 108 | github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= 109 | github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= 110 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 111 | github.com/zeromicro/go-queue v1.2.2 h1:3TMhRlI/8lZy13Sj6FBBWWRXlsQhGCchRxY2itfV1Is= 112 | github.com/zeromicro/go-queue v1.2.2/go.mod h1:5HiNTEw1tACi9itho0JYQ1+EpIGpSFM4tOQ4bit+yKM= 113 | github.com/zeromicro/go-zero v1.9.3 h1:dJ568uUoRJY0RUxo4aH4htSglbEUF60WiM1MZVkTK9A= 114 | github.com/zeromicro/go-zero v1.9.3/go.mod h1:JBAtfXQvErk+V7pxzcySR0mW6m2I4KPhNQZGASltDRQ= 115 | go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= 116 | go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= 117 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0 h1:D7UpUy2Xc2wsi1Ras6V40q806WM07rqoCWzXu7Sqy+4= 118 | go.opentelemetry.io/otel/exporters/jaeger v1.17.0/go.mod h1:nPCqOnEH9rNLKqH/+rrUjiMzHJdV1BlpKcTwRTyKkKI= 119 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= 120 | go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= 121 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= 122 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= 123 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= 124 | go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= 125 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDOpt2M8OTG92cWqUESvzh2MxiR5xY8= 126 | go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA= 127 | go.opentelemetry.io/otel/exporters/zipkin v1.24.0 h1:3evrL5poBuh1KF51D9gO/S+N/1msnm4DaBqs/rpXUqY= 128 | go.opentelemetry.io/otel/exporters/zipkin v1.24.0/go.mod h1:0EHgD8R0+8yRhUYJOGR8Hfg2dpiJQxDOszd5smVO9wM= 129 | go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= 130 | go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= 131 | go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= 132 | go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= 133 | go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= 134 | go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= 135 | go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= 136 | go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= 137 | go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= 138 | go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= 139 | go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= 140 | go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= 141 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 142 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 143 | golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= 144 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 145 | golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= 146 | golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= 147 | golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= 148 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 149 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 150 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 151 | golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 152 | golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= 153 | golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= 154 | golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= 155 | golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= 156 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 157 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 158 | golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 159 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 160 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 161 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 162 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 163 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 164 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 165 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 166 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 167 | golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 168 | golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 169 | golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= 170 | golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 171 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 172 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 173 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 174 | golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= 175 | golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= 176 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 177 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 178 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 179 | golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= 180 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 181 | golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= 182 | golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= 183 | golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= 184 | golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= 185 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 186 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 187 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 188 | golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= 189 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 190 | google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d h1:kHjw/5UfflP/L5EbledDrcG4C2597RtymmGRZvHiCuY= 191 | google.golang.org/genproto/googleapis/api v0.0.0-20240711142825-46eb208f015d/go.mod h1:mw8MG/Qz5wfgYr6VqVCiZcHe/GJEfI+oGGDCohaVgB0= 192 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 h1:BwIjyKYGsK9dMCBOorzRri8MQwmi7mT9rGHsCEinZkA= 193 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094/go.mod h1:Ue6ibwXGpU+dqIcODieyLOcgj7z8+IcskoNIgZxtrFY= 194 | google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= 195 | google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= 196 | google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= 197 | google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= 198 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 199 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 200 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 201 | gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= 202 | gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= 203 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 204 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 205 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 206 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 207 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 208 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 209 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 210 | k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= 211 | k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= 212 | --------------------------------------------------------------------------------