├── .github ├── ISSUE_TEMPLATE │ └── 数据处理问题-与预期不符.md └── workflows │ ├── ci.yml │ ├── create-github-release.yml │ └── docker-image.yml ├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README-EN.md ├── README.md ├── codec ├── coder.go ├── decoder.go ├── encoder.go ├── encoder_test.go ├── format_encoder.go ├── format_encoder_test.go ├── json_decoder.go ├── json_encoder.go └── plain_decoder.go ├── condition_filter ├── filter.go ├── filter_test.go └── parse.go ├── example.yml ├── field_deleter ├── field_deleter.go ├── multi_level_fields_deleter.go ├── multi_level_fields_deleter_test.go ├── one_level_fields_deleter.go └── one_level_fields_deleter_test.go ├── field_setter ├── field_setter.go ├── mfields_field_setter.go └── onelevel_field_setter.go ├── filter ├── add.go ├── add_test.go ├── convert.go ├── convert_test.go ├── date.go ├── date_test.go ├── drop.go ├── drop_test.go ├── filter.go ├── filters.go ├── grok.go ├── grok_test.go ├── gsub.go ├── gsub_test.go ├── ipip.go ├── json.go ├── json_test.go ├── kv.go ├── kv_test.go ├── link_metric.go ├── link_metric_test.go ├── link_stats_metric.go ├── link_stats_metric_test.go ├── lowercase.go ├── remove.go ├── rename.go ├── rename_test.go ├── replace_filter.go ├── replace_test.go ├── split_filter.go ├── split_test.go ├── translate.go ├── uppercase.go └── url_decode.go ├── go.mod ├── go.sum ├── gohangout.go ├── gohangout_test.go ├── input ├── input.go ├── input_box.go ├── kafka_input.go ├── random_input.go ├── stdin_input.go ├── tcp_input.go └── udp_input.go ├── internal ├── config │ ├── config_parser.go │ ├── config_watcher.go │ └── yaml_config_parser.go └── signal │ ├── signalhandle_unix.go │ └── signalhandle_windows.go ├── output ├── bulk_http.go ├── clickhouse_output.go ├── dot_output.go ├── elasticsearch_output.go ├── elasticsearch_output_test.go ├── host_selector.go ├── influxdb_output.go ├── kafka_output.go ├── output.go ├── stdout_output.go └── tcp_output.go ├── simplejson └── simple_json_encode.go ├── test ├── LinkMetricInFilters.yml ├── itest-1.yml ├── itest-2.yml ├── itest-3.yml ├── itest-4.yml ├── itest-5.yml ├── itest-6-2.yml ├── itest-6-3.yml ├── itest-6-4.yml ├── itest-6.yml ├── itest-7-1.yml ├── itest-es7.yml ├── itest-tcp.sh ├── itest-tcpinput.yml ├── itest-tcpoutput.yml ├── itest.sh └── t.yml ├── topology ├── filter.go ├── input.go ├── output.go ├── processor.go ├── prom_counter.go └── prom_counter_test.go └── value_render ├── index_render.go ├── index_render_test.go ├── jsonpath_render.go ├── jsonpath_render_test.go ├── literal_value_render.go ├── mfields_value_render.go ├── mfields_value_render_test.go ├── one_level_value_render.go ├── template_value_render.go └── value_render.go /.github/ISSUE_TEMPLATE/数据处理问题-与预期不符.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: 数据处理问题-与预期不符 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: cookbook 6 | assignees: childe 7 | 8 | --- 9 | 10 | 按以下配置测试一下并给出结果 11 | 12 | ``` 13 | inputs: 14 | - Stdin: {} 15 | filters: 16 | - XXX 17 | outputs: 18 | - Stdout: {} 19 | ``` 20 | 21 | - 源数据 22 | - 配置文件 23 | - 输出 24 | - 期望 25 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: test and build 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | 8 | jobs: 9 | make: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 1 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v3 19 | with: 20 | go-version: 1.23 21 | 22 | - name: Test 23 | run: make test 24 | 25 | - name: Build 26 | run: make gohangout 27 | -------------------------------------------------------------------------------- /.github/workflows/create-github-release.yml: -------------------------------------------------------------------------------- 1 | name: Create Release And Upload Binary To Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: 1.23 18 | 19 | - name: Test 20 | run: go test -v ./... 21 | 22 | - name: Create Release 23 | id: create_release 24 | uses: actions/create-release@v1.1.4 25 | env: 26 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 27 | with: 28 | tag_name: ${{ github.ref }} 29 | release_name: Release ${{ github.ref }} 30 | draft: false 31 | prerelease: false 32 | 33 | - name: Build 34 | run: make all -e tag=${GITHUB_REF_NAME} 35 | 36 | - name: copy # for upload release binary step 37 | run: | 38 | cp build/gohangout-windows-x64-${GITHUB_REF_NAME}.exe gohangout-windows-x64.exe 39 | cp build/gohangout-windows-386-${GITHUB_REF_NAME}.exe gohangout-windows-386.exe 40 | cp build/gohangout-linux-x64-${GITHUB_REF_NAME} gohangout-linux-x64 41 | cp build/gohangout-linux-386-${GITHUB_REF_NAME} gohangout-linux-386 42 | cp build/gohangout-linux-arm64-${GITHUB_REF_NAME} gohangout-linux-arm64 43 | cp build/gohangout-darwin-x64-${GITHUB_REF_NAME} gohangout-darwin-x64 44 | 45 | - name: LS 46 | run: ls -l ./build 47 | 48 | - name: Upload linux-x64 49 | id: upload-linux-x64 50 | uses: actions/upload-release-asset@v1.0.2 51 | env: 52 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 53 | with: 54 | upload_url: ${{ steps.create_release.outputs.upload_url }} 55 | asset_path: gohangout-linux-x64 56 | asset_name: gohangout-linux-x64 57 | asset_content_type: application/zip 58 | 59 | - name: Upload linux-386 60 | id: upload-linux-386 61 | uses: actions/upload-release-asset@v1.0.2 62 | env: 63 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 64 | with: 65 | upload_url: ${{ steps.create_release.outputs.upload_url }} 66 | asset_path: gohangout-linux-386 67 | asset_name: gohangout-linux-386 68 | asset_content_type: application/zip 69 | 70 | - name: Upload linux-arm64 71 | id: upload-linux-arm64 72 | uses: actions/upload-release-asset@v1.0.2 73 | env: 74 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 75 | with: 76 | upload_url: ${{ steps.create_release.outputs.upload_url }} 77 | asset_path: gohangout-linux-arm64 78 | asset_name: gohangout-linux-arm64 79 | asset_content_type: application/zip 80 | 81 | - name: Upload windows-x64 82 | id: upload-windows-x64 83 | uses: actions/upload-release-asset@v1.0.2 84 | env: 85 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 86 | with: 87 | upload_url: ${{ steps.create_release.outputs.upload_url }} 88 | asset_path: gohangout-windows-x64.exe 89 | asset_name: gohangout-windows-x64.exe 90 | asset_content_type: application/zip 91 | 92 | - name: Upload windows-386 93 | id: upload-windows-386 94 | uses: actions/upload-release-asset@v1.0.2 95 | env: 96 | GITHUB_TOKEN: ${{ secrets.GOHANGOUT_UPLOAD_ASSET_TOKEN }} 97 | with: 98 | upload_url: ${{ steps.create_release.outputs.upload_url }} 99 | asset_path: gohangout-windows-386.exe 100 | asset_name: gohangout-windows-386.exe 101 | asset_content_type: application/zip 102 | 103 | -------------------------------------------------------------------------------- /.github/workflows/docker-image.yml: -------------------------------------------------------------------------------- 1 | name: Build and Push Docker Image 2 | 3 | on: 4 | push: 5 | tags: 6 | - "v*" 7 | 8 | jobs: 9 | docker-build-push: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v3 14 | with: 15 | fetch-depth: 1 16 | 17 | - name: Set up Go 18 | uses: actions/setup-go@v3 19 | with: 20 | go-version: 1.23 21 | 22 | - name: Test 23 | run: make test 24 | 25 | - name: Set up QEMU 26 | uses: docker/setup-qemu-action@v2 27 | 28 | - name: Set up Docker Buildx 29 | uses: docker/setup-buildx-action@v2 30 | with: 31 | driver-opts: network=host 32 | 33 | - name: Login to docker hub 34 | uses: docker/login-action@v2 35 | with: 36 | registry: docker.io 37 | username: ${{ secrets.DOCKER_USERNAME }} 38 | password: ${{ secrets.DOCKER_PASSWORD }} 39 | 40 | - name: build and publish image 41 | env: 42 | DOCKER_REPO: docker.io/${{ secrets.DOCKER_USERNAME }}/gohangout 43 | run: | 44 | docker buildx build \ 45 | --platform linux/amd64,linux/arm64 \ 46 | --label "org.opencontainers.image.source=https://github.com/${{ github.repository_owner }}/gohangout" \ 47 | --label "org.opencontainers.image.description=gohangout image" \ 48 | --label "org.opencontainers.image.licenses=MIT" \ 49 | --push \ 50 | -t ${DOCKER_REPO}:${GITHUB_REF_NAME} \ 51 | -f Dockerfile \ 52 | . 53 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.yml 3 | *.exe 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, build with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 15 | .glide/ 16 | 17 | build/ 18 | /gohangout 19 | 20 | .ropeproject/ 21 | vendor/ 22 | 23 | !test/*.yml 24 | 25 | .DS_Store 26 | 27 | .idea/ 28 | .vscode/ 29 | 30 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.22.5-alpine3.20 as build 2 | 3 | RUN apk update && apk add make 4 | 5 | WORKDIR /gohangout 6 | COPY . . 7 | 8 | RUN make 9 | 10 | FROM alpine:3.20 11 | 12 | ARG TZ="Asia/Shanghai" 13 | ENV TZ ${TZ} 14 | 15 | COPY --from=build /gohangout/gohangout /usr/local/bin/ 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Childe, https://github.com/childe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | hash:=$(shell git describe --tags --always) 2 | buildTime:=$(shell git log -1 --format="%cI") 3 | tag:=$(shell git rev-parse --short HEAD) 4 | 5 | .PHONY: gohangout all clean check test docker 6 | 7 | gohangout: 8 | CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o gohangout 9 | 10 | docker: 11 | docker build -t gohangout . 12 | 13 | all: 14 | @echo $(hash) 15 | mkdir -p build/ 16 | 17 | GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-windows-x64-$(tag).exe 18 | GOOS=windows GOARCH=386 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-windows-386-$(tag).exe 19 | GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-linux-x64-$(tag) 20 | GOOS=linux GOARCH=386 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-linux-386-$(tag) 21 | GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-linux-arm64-$(tag) 22 | GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X main.version=$(hash) -X main.buildTime=$(buildTime)" -o build/gohangout-darwin-x64-$(tag) 23 | 24 | clean: 25 | rm -rf gohangout 26 | 27 | test: 28 | for _ in {1..5} ; do go test -v -count=1 -gcflags="all=-N -l" ./... && break ; done 29 | -------------------------------------------------------------------------------- /codec/coder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import jsoniter "github.com/json-iterator/go" 4 | 5 | var json = jsoniter.ConfigCompatibleWithStandardLibrary 6 | -------------------------------------------------------------------------------- /codec/decoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "plugin" 5 | 6 | "k8s.io/klog/v2" 7 | ) 8 | 9 | type Decoder interface { 10 | Decode([]byte) map[string]interface{} 11 | } 12 | 13 | func NewDecoder(t string) Decoder { 14 | switch t { 15 | case "plain": 16 | return &PlainDecoder{} 17 | case "json": 18 | return &JsonDecoder{useNumber: true} 19 | case "json:not_usenumber": 20 | return &JsonDecoder{useNumber: false} 21 | default: 22 | p, err := plugin.Open(t) 23 | if err != nil { 24 | klog.Fatalf("could not open %s: %s", t, err) 25 | } 26 | newFunc, err := p.Lookup("New") 27 | if err != nil { 28 | klog.Fatalf("could not find New function in %s: %s", t, err) 29 | } 30 | return newFunc.(func() interface{})().(Decoder) 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /codec/encoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "plugin" 5 | "strings" 6 | 7 | "github.com/childe/gohangout/simplejson" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | type Encoder interface { 12 | Encode(interface{}) ([]byte, error) 13 | } 14 | 15 | func NewEncoder(t string) Encoder { 16 | switch t { 17 | case "json": 18 | return &JsonEncoder{} 19 | case "simplejson": 20 | return &simplejson.SimpleJsonDecoder{} 21 | } 22 | 23 | // FormatEncoder 24 | if strings.HasPrefix(t, "format:") { 25 | splited := strings.SplitN(t, ":", 2) 26 | if len(splited) != 2 { 27 | klog.Fatalf("format of `%s` is incorrect", t) 28 | } 29 | format := splited[1] 30 | return NewFormatEncoder(format) 31 | } 32 | 33 | // try plugin 34 | p, err := plugin.Open(t) 35 | if err != nil { 36 | klog.Fatalf("could not open %s: %s", t, err) 37 | } 38 | newFunc, err := p.Lookup("New") 39 | if err != nil { 40 | klog.Fatalf("could not find New function in %s: %s", t, err) 41 | } 42 | return newFunc.(func() interface{})().(Encoder) 43 | } 44 | -------------------------------------------------------------------------------- /codec/encoder_test.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/childe/gohangout/simplejson" 8 | ) 9 | 10 | func TestNewEncoder(t *testing.T) { 11 | cases := []struct { 12 | codec string 13 | encoderType Encoder 14 | }{ 15 | { 16 | codec: "json", 17 | encoderType: &JsonEncoder{}, 18 | }, 19 | { 20 | codec: "simplejson", 21 | encoderType: &simplejson.SimpleJsonDecoder{}, 22 | }, 23 | { 24 | codec: "format:[msg]", 25 | encoderType: &FormatEncoder{}, 26 | }, 27 | } 28 | 29 | for _, c := range cases { 30 | t.Logf("test %v", c.codec) 31 | encoder := NewEncoder(c.codec) 32 | got := reflect.TypeOf(encoder).String() 33 | expectedEncoderType := reflect.TypeOf(c.encoderType).String() 34 | if got != expectedEncoderType { 35 | t.Errorf("expected `%s`, got `%s`", expectedEncoderType, got) 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /codec/format_encoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "errors" 5 | 6 | "github.com/childe/gohangout/value_render" 7 | ) 8 | 9 | type FormatEncoder struct { 10 | render value_render.ValueRender 11 | } 12 | 13 | var ErrNotString = errors.New("value returned by FormatEncoder is not a string type") 14 | 15 | func NewFormatEncoder(format string) *FormatEncoder { 16 | return &FormatEncoder{ 17 | render: value_render.GetValueRender(format), 18 | } 19 | } 20 | 21 | func (e *FormatEncoder) Encode(v interface{}) ([]byte, error) { 22 | rst, err := e.render.Render(v.(map[string]interface{})) 23 | if err != nil { 24 | return nil, err 25 | } 26 | 27 | if v, ok := rst.(string); ok { 28 | return []byte(v), nil 29 | } 30 | return nil, ErrNotString 31 | } 32 | -------------------------------------------------------------------------------- /codec/format_encoder_test.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestFormatEncoder(t *testing.T) { 8 | cases := []struct { 9 | codec string 10 | event map[string]interface{} 11 | expected string 12 | }{ 13 | { 14 | codec: "format:[msg]", 15 | event: map[string]interface{}{"msg": "this is a line"}, 16 | expected: "this is a line", 17 | }, 18 | { 19 | codec: "format:msg", 20 | event: map[string]interface{}{"msg": "this is a line"}, 21 | expected: "msg", 22 | }, 23 | { 24 | codec: "format:my name is %{name}", 25 | event: map[string]interface{}{"name": "childe"}, 26 | expected: "my name is childe", 27 | }, 28 | { 29 | codec: "format:my name is $.name.firstname", 30 | event: map[string]interface{}{"name": map[string]string{"firstname": "jia"}}, 31 | expected: "my name is $.name.firstname", 32 | }, 33 | { 34 | codec: "format:$.name.firstname", 35 | event: map[string]interface{}{"name": map[string]string{"firstname": "jia"}}, 36 | expected: "jia", 37 | }, 38 | { 39 | codec: "format:my name is {{.name.firstname}}", 40 | event: map[string]interface{}{"name": map[string]string{"firstname": "jia"}}, 41 | expected: "my name is jia", 42 | }, 43 | } 44 | 45 | for _, c := range cases { 46 | t.Logf("test %v", c.codec) 47 | encoder := NewEncoder(c.codec) 48 | got, err := encoder.Encode(c.event) 49 | if err != nil { 50 | t.Errorf("get error from `%s`: %v", c.codec, err) 51 | continue 52 | } 53 | if string(got) != c.expected { 54 | t.Errorf("expected `%s`, got `%s`", c.expected, got) 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /codec/json_decoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import ( 4 | "bytes" 5 | "time" 6 | ) 7 | 8 | type JsonDecoder struct { 9 | useNumber bool 10 | } 11 | 12 | func (jd *JsonDecoder) Decode(value []byte) map[string]interface{} { 13 | rst := make(map[string]interface{}) 14 | rst["@timestamp"] = time.Now() 15 | d := json.NewDecoder(bytes.NewReader(value)) 16 | 17 | if jd.useNumber { 18 | d.UseNumber() 19 | } 20 | err := d.Decode(&rst) 21 | if err != nil || d.More() { 22 | return map[string]interface{}{ 23 | "@timestamp": time.Now(), 24 | "message": string(value), 25 | } 26 | } 27 | 28 | return rst 29 | } 30 | -------------------------------------------------------------------------------- /codec/json_encoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | type JsonEncoder struct{} 4 | 5 | func (e *JsonEncoder) Encode(v interface{}) ([]byte, error) { 6 | return json.Marshal(v) 7 | } 8 | -------------------------------------------------------------------------------- /codec/plain_decoder.go: -------------------------------------------------------------------------------- 1 | package codec 2 | 3 | import "time" 4 | 5 | type PlainDecoder struct { 6 | } 7 | 8 | func (d *PlainDecoder) Decode(value []byte) map[string]interface{} { 9 | rst := make(map[string]interface{}) 10 | rst["@timestamp"] = time.Now() 11 | rst["message"] = string(value) 12 | return rst 13 | } 14 | -------------------------------------------------------------------------------- /condition_filter/parse.go: -------------------------------------------------------------------------------- 1 | package condition_filter 2 | 3 | import ( 4 | "errors" 5 | "strings" 6 | 7 | "k8s.io/klog/v2" 8 | ) 9 | 10 | const ( 11 | _op_sharp = iota 12 | _op_left 13 | _op_right 14 | _op_or 15 | _op_and 16 | _op_not 17 | ) 18 | 19 | const ( 20 | _OUTSIDES_CONDITION = iota 21 | _IN_CONDITION 22 | _IN_STRING 23 | ) 24 | 25 | var errorParse = errors.New("parse condition error") 26 | 27 | func parseBoolTree(c string) (node *OPNode, err error) { 28 | defer func() { 29 | if r := recover(); r != nil { 30 | klog.Errorf("parse `%s` error at `%s`", c, r) 31 | node = nil 32 | err = errorParse 33 | } 34 | }() 35 | 36 | //klog.Info(c) 37 | c = strings.Trim(c, " ") 38 | if c == "" { 39 | return nil, nil 40 | } 41 | 42 | s2, err := buildRPNStack(c) 43 | if err != nil { 44 | return nil, err 45 | } 46 | //klog.Info(s2) 47 | s := make([]interface{}, 0) 48 | 49 | for _, e := range s2 { 50 | if c, ok := e.(Condition); ok { 51 | s = append(s, c) 52 | } else { 53 | sLen := len(s) 54 | op := e.(int) 55 | if op == _op_not { 56 | right := s[sLen-1].(*OPNode) 57 | s = s[:sLen-1] 58 | node := &OPNode{ 59 | op: op, 60 | right: right, 61 | } 62 | s = append(s, node) 63 | } else { 64 | right := s[sLen-1].(*OPNode) 65 | left := s[sLen-2].(*OPNode) 66 | s = s[:sLen-2] 67 | node := &OPNode{ 68 | op: op, 69 | left: left, 70 | right: right, 71 | } 72 | s = append(s, node) 73 | } 74 | } 75 | } 76 | 77 | //klog.Info(s) 78 | if len(s) != 1 { 79 | return nil, errorParse 80 | } 81 | return s[0].(*OPNode), nil 82 | } 83 | 84 | func buildRPNStack(c string) ([]interface{}, error) { 85 | var ( 86 | state = _OUTSIDES_CONDITION 87 | i int 88 | length = len(c) 89 | parenthesis = 0 90 | condition_start_pos int 91 | 92 | s1 = []int{_op_sharp} 93 | s2 = make([]interface{}, 0) 94 | ) 95 | 96 | // 哪些导致状态变化?? 97 | 98 | for i < length { 99 | switch c[i] { 100 | case '(': 101 | switch state { 102 | case _OUTSIDES_CONDITION: // push s1 103 | s1 = append(s1, _op_left) 104 | case _IN_CONDITION: 105 | parenthesis++ 106 | } 107 | case ')': 108 | switch state { 109 | case _OUTSIDES_CONDITION: 110 | if !pushOp(_op_right, &s1, &s2) { 111 | panic(c[:i+1]) 112 | } 113 | 114 | case _IN_CONDITION: 115 | parenthesis-- 116 | if parenthesis == 0 { 117 | condition, err := NewSingleCondition(c[condition_start_pos : i+1]) 118 | if err != nil { 119 | klog.Error(err) 120 | panic(c[:i+1]) 121 | } 122 | n := &OPNode{ 123 | condition: condition, 124 | } 125 | s2 = append(s2, n) 126 | state = _OUTSIDES_CONDITION 127 | } 128 | } 129 | case '&': 130 | switch state { 131 | case _OUTSIDES_CONDITION: // push s1 132 | if c[i+1] != '&' { 133 | panic(c[:i+1]) 134 | } else { 135 | if !pushOp(_op_and, &s1, &s2) { 136 | panic(c[:i+1]) 137 | } 138 | i++ 139 | } 140 | } 141 | case '|': 142 | switch state { 143 | case _OUTSIDES_CONDITION: // push s1 144 | if c[i+1] != '|' { 145 | panic(c[:i+1]) 146 | } else { 147 | if !pushOp(_op_or, &s1, &s2) { 148 | panic(c[:i+1]) 149 | } 150 | i++ 151 | } 152 | } 153 | case '!': 154 | switch state { 155 | case _OUTSIDES_CONDITION: // push s1 156 | if n := c[i+1]; n == '|' || n == '&' || n == ' ' { 157 | panic(c[:i+1]) 158 | } 159 | if !pushOp(_op_not, &s1, &s2) { 160 | panic(c[:i+1]) 161 | } 162 | } 163 | case '"': 164 | switch state { 165 | case _OUTSIDES_CONDITION: // push s1 166 | panic(c[:i+1]) 167 | case _IN_STRING: 168 | state = _IN_CONDITION 169 | } 170 | case ' ': 171 | default: 172 | if state == _OUTSIDES_CONDITION { 173 | state = _IN_CONDITION 174 | condition_start_pos = i 175 | } 176 | 177 | } 178 | i++ 179 | } 180 | 181 | if state != _OUTSIDES_CONDITION { 182 | return nil, errorParse 183 | } 184 | 185 | for j := len(s1) - 1; j > 0; j-- { 186 | s2 = append(s2, s1[j]) 187 | } 188 | 189 | return s2, nil 190 | } 191 | 192 | func pushOp(op int, s1 *[]int, s2 *[]interface{}) bool { 193 | if op == _op_right { 194 | return findLeftInS1(s1, s2) 195 | } 196 | return compareOpWithS1(op, s1, s2) 197 | } 198 | 199 | // find ( in s1 200 | func findLeftInS1(s1 *[]int, s2 *[]interface{}) bool { 201 | var j int 202 | for j = len(*s1) - 1; j > 0 && (*s1)[j] != _op_left; j-- { 203 | *s2 = append(*s2, (*s1)[j]) 204 | } 205 | 206 | if j == 0 { 207 | return false 208 | } 209 | 210 | *s1 = (*s1)[:j] 211 | return true 212 | } 213 | 214 | // compare op with ops in s1, and put them to s2 215 | func compareOpWithS1(op int, s1 *[]int, s2 *[]interface{}) bool { 216 | var j int 217 | for j = len(*s1) - 1; j > 0; j-- { 218 | //if (*s1)[j] == _op_left || op > (*s1)[j] { 219 | n1 := (*s1)[j] 220 | b := true 221 | switch { 222 | case n1 == _op_left: 223 | break 224 | case op > n1: 225 | break 226 | case op == _op_not && n1 == _op_not: 227 | break 228 | default: 229 | b = false 230 | } 231 | if b { 232 | break 233 | } 234 | *s2 = append(*s2, n1) 235 | } 236 | 237 | *s1 = (*s1)[:j+1] 238 | *s1 = append(*s1, op) 239 | return true 240 | } 241 | -------------------------------------------------------------------------------- /example.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | - Kafka: 5 | topic: 6 | test: 1 7 | #assign: 8 | #test: [0, 9] 9 | codec: json 10 | consumer_settings: 11 | bootstrap.servers: "10.0.0.100:9092" 12 | group.id: hangout.test 13 | from.beginning: 'true' 14 | 15 | filters: 16 | - Add: 17 | fields: 18 | xxx: xxx 19 | yyy: '[client]' 20 | zzz: '[stored][message]' 21 | '[a][b]': '[stored][message]' 22 | - Grok: 23 | src: message 24 | pattern_paths: 25 | - '/opt/gohangout/patterns' 26 | match: 27 | - '^(?P\S+) (?P\w+) (?P\d+)$' 28 | - '^%{USER:user} %{INT:status} %{INT:request_time}$' 29 | remove_fields: ['message'] 30 | - Date: 31 | location: 'Asia/Shanghai' 32 | src: logtime 33 | formats: 34 | - 'RFC3339' 35 | - '2006-01-02T15:04:05' 36 | - '2006-01-02T15:04:05Z07:00' 37 | - '2006-01-02T15:04:05Z0700' 38 | - '2006-01-02' 39 | - 'UNIX' 40 | - 'UNIX_MS' 41 | remove_fields: ["logtime"] 42 | - Translate: 43 | source: user 44 | target: nick 45 | refresh_interval: 3600 46 | dictionary_path: http://corp.com/dict/user.yml 47 | - Drop: 48 | if: 49 | #- '{{if .name}}y{{end}}' 50 | #- '{{if eq .name "childe"}}y{{end}}' 51 | #- '{{if or (before . "-24h") (after . "24h")}}y{{end}}' 52 | - 'EQ(name,"childe")' 53 | - 'Before(-24h) || After(24h)' 54 | 55 | outputs: 56 | - Stdout: 57 | if: 58 | - '{{if .error}}y{{end}}' 59 | - Elasticsearch: 60 | hosts: 61 | - http://127.0.0.1:9200 62 | index: 'web-%{+2006-01-02}' #golang里面的渲染方式就是用数字, 而不是用YYMM. 63 | index_type: "logs" 64 | bulk_actions: 5000 65 | bulk_size: 20 66 | flush_interval: 60 67 | -------------------------------------------------------------------------------- /field_deleter/field_deleter.go: -------------------------------------------------------------------------------- 1 | package field_deleter 2 | 3 | import "regexp" 4 | 5 | type FieldDeleter interface { 6 | Delete(map[string]interface{}) 7 | } 8 | 9 | func NewFieldDeleter(template string) FieldDeleter { 10 | matchp, _ := regexp.Compile(`(\[.*?\])+`) 11 | findp, _ := regexp.Compile(`(\[(.*?)\])`) 12 | if matchp.Match([]byte(template)) { 13 | fields := make([]string, 0) 14 | for _, v := range findp.FindAllStringSubmatch(template, -1) { 15 | fields = append(fields, v[2]) 16 | } 17 | return NewMultiLevelFieldDeleter(fields) 18 | } else { 19 | return NewOneLevelFieldDeleter(template) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /field_deleter/multi_level_fields_deleter.go: -------------------------------------------------------------------------------- 1 | package field_deleter 2 | 3 | type MultiLevelFieldDeleter struct { 4 | preFields []string 5 | lastField string 6 | } 7 | 8 | func NewMultiLevelFieldDeleter(fields []string) *MultiLevelFieldDeleter { 9 | fieldsLength := len(fields) 10 | preFields := make([]string, fieldsLength-1) 11 | for i := range preFields { 12 | preFields[i] = fields[i] 13 | } 14 | 15 | return &MultiLevelFieldDeleter{ 16 | preFields: preFields, 17 | lastField: fields[fieldsLength-1], 18 | } 19 | } 20 | 21 | func (d *MultiLevelFieldDeleter) Delete(event map[string]interface{}) { 22 | current := event 23 | for _, field := range d.preFields { 24 | if v, ok := current[field]; ok { 25 | if current, ok = v.(map[string]interface{}); !ok { 26 | return 27 | } 28 | } else { 29 | return 30 | } 31 | } 32 | delete(current, d.lastField) 33 | } 34 | -------------------------------------------------------------------------------- /field_deleter/multi_level_fields_deleter_test.go: -------------------------------------------------------------------------------- 1 | package field_deleter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestMultiLevelDelete(t *testing.T) { 9 | for _, c := range []struct { 10 | fields []string 11 | event map[string]interface{} 12 | want map[string]interface{} 13 | }{ 14 | { 15 | event: map[string]interface{}{"hostname": "xxx"}, 16 | fields: []string{"hostname"}, 17 | want: map[string]interface{}{}, 18 | }, 19 | { 20 | event: map[string]interface{}{"hostname": "xxx"}, 21 | fields: []string{"metadata", "hostname"}, 22 | want: map[string]interface{}{"hostname": "xxx"}, 23 | }, 24 | { 25 | event: map[string]interface{}{"metadata": "xxx"}, 26 | fields: []string{"metadata", "hostname"}, 27 | want: map[string]interface{}{"metadata": "xxx"}, 28 | }, 29 | { 30 | event: map[string]interface{}{"metadata": map[string]interface{}{"hostname": "xxx"}}, 31 | fields: []string{"metadata", "hostname"}, 32 | want: map[string]interface{}{"metadata": map[string]interface{}{}}, 33 | }, 34 | } { 35 | deleter := NewMultiLevelFieldDeleter(c.fields) 36 | deleter.Delete(c.event) 37 | if !reflect.DeepEqual(c.event, c.want) { 38 | t.Errorf("got %v, want %v", c.event, c.want) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /field_deleter/one_level_fields_deleter.go: -------------------------------------------------------------------------------- 1 | package field_deleter 2 | 3 | type OneLevelFieldDeleter struct { 4 | field string 5 | } 6 | 7 | func NewOneLevelFieldDeleter(field string) *OneLevelFieldDeleter { 8 | return &OneLevelFieldDeleter{ 9 | field: field, 10 | } 11 | } 12 | 13 | func (d *OneLevelFieldDeleter) Delete(event map[string]interface{}) { 14 | delete(event, d.field) 15 | } 16 | -------------------------------------------------------------------------------- /field_deleter/one_level_fields_deleter_test.go: -------------------------------------------------------------------------------- 1 | package field_deleter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func TestOneLevelDelete(t *testing.T) { 9 | for _, c := range []struct { 10 | field string 11 | event map[string]interface{} 12 | want map[string]interface{} 13 | }{ 14 | { 15 | event: map[string]interface{}{"hostname": "xxx"}, 16 | field: "hostname", 17 | want: map[string]interface{}{}, 18 | }, 19 | { 20 | event: map[string]interface{}{"hostname": "xxx"}, 21 | field: "metadata", 22 | want: map[string]interface{}{"hostname": "xxx"}, 23 | }, 24 | { 25 | event: map[string]interface{}{"metadata": map[string]interface{}{"hostname": "xxx"}}, 26 | field: "metadata", 27 | want: map[string]interface{}{}, 28 | }, 29 | } { 30 | deleter := NewOneLevelFieldDeleter(c.field) 31 | deleter.Delete(c.event) 32 | if !reflect.DeepEqual(c.event, c.want) { 33 | t.Errorf("got %v, want %v", c.event, c.want) 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /field_setter/field_setter.go: -------------------------------------------------------------------------------- 1 | package field_setter 2 | 3 | import "regexp" 4 | 5 | // FieldSetter is the interface that wraps the SetField method. 6 | type FieldSetter interface { 7 | SetField(event map[string]interface{}, value interface{}, fieldName string, overwrite bool) map[string]interface{} 8 | } 9 | 10 | // NewFieldSetter creates a new FieldSetter. 11 | // It returns OneLevelFieldSetter if [xxx] passed 12 | // It returns MultiLevelFieldSetter if [xxx][yyy] passed 13 | func NewFieldSetter(template string) FieldSetter { 14 | matchp, _ := regexp.Compile(`(\[.*?\])+`) 15 | findp, _ := regexp.Compile(`(\[(.*?)\])`) 16 | if matchp.Match([]byte(template)) { 17 | fields := make([]string, 0) 18 | for _, v := range findp.FindAllStringSubmatch(template, -1) { 19 | fields = append(fields, v[2]) 20 | } 21 | if len(fields) == 1 { 22 | return NewOneLevelFieldSetter(fields[0]) 23 | } 24 | return NewMultiLevelFieldSetter(fields) 25 | } else { 26 | return NewOneLevelFieldSetter(template) 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /field_setter/mfields_field_setter.go: -------------------------------------------------------------------------------- 1 | package field_setter 2 | 3 | import "reflect" 4 | 5 | type MultiLevelFieldSetter struct { 6 | preFields []string 7 | lastField string 8 | } 9 | 10 | func NewMultiLevelFieldSetter(fields []string) *MultiLevelFieldSetter { 11 | fieldsLength := len(fields) 12 | preFields := make([]string, fieldsLength-1) 13 | for i := range preFields { 14 | preFields[i] = fields[i] 15 | } 16 | 17 | return &MultiLevelFieldSetter{ 18 | preFields: preFields, 19 | lastField: fields[fieldsLength-1], 20 | } 21 | } 22 | 23 | func (fs *MultiLevelFieldSetter) SetField(event map[string]interface{}, value interface{}, field string, overwrite bool) map[string]interface{} { 24 | current := event 25 | for _, field := range fs.preFields { 26 | if value, ok := current[field]; ok { 27 | if reflect.TypeOf(value).Kind() == reflect.Map { 28 | current = value.(map[string]interface{}) 29 | } 30 | } else { 31 | a := make(map[string]interface{}) 32 | current[field] = a 33 | current = a 34 | } 35 | } 36 | current[fs.lastField] = value 37 | return event 38 | } 39 | -------------------------------------------------------------------------------- /field_setter/onelevel_field_setter.go: -------------------------------------------------------------------------------- 1 | package field_setter 2 | 3 | type OneLevelFieldSetter struct { 4 | field string 5 | } 6 | 7 | func NewOneLevelFieldSetter(field string) *OneLevelFieldSetter { 8 | r := &OneLevelFieldSetter{ 9 | field: field, 10 | } 11 | return r 12 | } 13 | 14 | func (fs *OneLevelFieldSetter) SetField(event map[string]interface{}, value interface{}, field string, overwrite bool) map[string]interface{} { 15 | if _, ok := event[fs.field]; !ok || overwrite { 16 | event[fs.field] = value 17 | } 18 | return event 19 | } 20 | -------------------------------------------------------------------------------- /filter/add.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/childe/gohangout/field_setter" 5 | "github.com/childe/gohangout/topology" 6 | "github.com/childe/gohangout/value_render" 7 | "k8s.io/klog/v2" 8 | ) 9 | 10 | type AddFilter struct { 11 | config map[interface{}]interface{} 12 | fields map[field_setter.FieldSetter]value_render.ValueRender 13 | overwrite bool 14 | } 15 | 16 | func init() { 17 | Register("Add", newAddFilter) 18 | } 19 | 20 | func newAddFilter(config map[interface{}]interface{}) topology.Filter { 21 | plugin := &AddFilter{ 22 | config: config, 23 | fields: make(map[field_setter.FieldSetter]value_render.ValueRender), 24 | overwrite: true, 25 | } 26 | 27 | if overwrite, ok := config["overwrite"]; ok { 28 | plugin.overwrite = overwrite.(bool) 29 | } 30 | 31 | if fieldsValue, ok := config["fields"]; ok { 32 | for f, v := range fieldsValue.(map[interface{}]interface{}) { 33 | fieldSetter := field_setter.NewFieldSetter(f.(string)) 34 | if fieldSetter == nil { 35 | klog.Fatalf("could build field setter from %s", f.(string)) 36 | } 37 | plugin.fields[fieldSetter] = value_render.GetValueRender(v.(string)) 38 | } 39 | } else { 40 | klog.Fatal("fields must be set in add filter plugin") 41 | } 42 | return plugin 43 | } 44 | 45 | func (plugin *AddFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 46 | for fs, r := range plugin.fields { 47 | v, _ := r.Render(event) 48 | event = fs.SetField(event, v, "", plugin.overwrite) 49 | } 50 | return event, true 51 | } 52 | -------------------------------------------------------------------------------- /filter/add_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestAddFilter(t *testing.T) { 9 | config := make(map[interface{}]interface{}) 10 | fields := make(map[interface{}]interface{}) 11 | fields["name"] = `{{.first}} {{.last}}` 12 | fields["firstname"] = `$.first` 13 | config["fields"] = fields 14 | f := BuildFilter("Add", config) 15 | 16 | event := make(map[string]interface{}) 17 | event["@timestamp"] = time.Now().Unix() 18 | event["first"] = "dehua" 19 | event["last"] = "liu" 20 | t.Log(event) 21 | 22 | event, ok := f.Filter(event) 23 | t.Log(event) 24 | 25 | if ok == false { 26 | t.Error("add filter fail") 27 | } 28 | 29 | name, ok := event["name"] 30 | if ok == false { 31 | t.Error("add filter should add `name` field") 32 | } 33 | if name != "dehua liu" { 34 | t.Error("name field should be `dehua liu`") 35 | } 36 | 37 | firstname, ok := event["firstname"] 38 | if ok == false { 39 | t.Error("add filter should add `firstname` field") 40 | } 41 | if firstname != "dehua" { 42 | t.Error("firstname field should be `dehua`") 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /filter/convert.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "strconv" 7 | 8 | "github.com/childe/cast" 9 | "github.com/childe/gohangout/field_setter" 10 | "github.com/childe/gohangout/topology" 11 | "github.com/childe/gohangout/value_render" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | type Converter interface { 16 | convert(v interface{}) (interface{}, error) 17 | } 18 | 19 | var ErrConvertUnknownFormat error = errors.New("unknown format") 20 | 21 | type IntConverter struct{} 22 | 23 | func (c *IntConverter) convert(v interface{}) (interface{}, error) { 24 | return cast.ToInt64E(v) 25 | } 26 | 27 | type UIntConverter struct{} 28 | 29 | func (c *UIntConverter) convert(v interface{}) (interface{}, error) { 30 | return cast.ToUint64E(v) 31 | } 32 | 33 | type FloatConverter struct{} 34 | 35 | func (c *FloatConverter) convert(v interface{}) (interface{}, error) { 36 | return cast.ToFloat64E(v) 37 | } 38 | 39 | type BoolConverter struct{} 40 | 41 | func (c *BoolConverter) convert(v interface{}) (interface{}, error) { 42 | if v, ok := v.(string); ok { 43 | rst, err := strconv.ParseBool(v) 44 | if err != nil { 45 | return nil, err 46 | } else { 47 | return rst, err 48 | } 49 | } 50 | return nil, ErrConvertUnknownFormat 51 | } 52 | 53 | type StringConverter struct{} 54 | 55 | func (c *StringConverter) convert(v interface{}) (interface{}, error) { 56 | if r, ok := v.(json.Number); ok { 57 | return r.String(), nil 58 | } 59 | 60 | if r, ok := v.(string); ok { 61 | return r, nil 62 | } 63 | 64 | jsonString, err := json.Marshal(v) 65 | if err != nil { 66 | return nil, err 67 | } 68 | return string(jsonString), nil 69 | } 70 | 71 | type ArrayIntConverter struct{} 72 | 73 | func (c *ArrayIntConverter) convert(v interface{}) (interface{}, error) { 74 | if v1, ok1 := v.([]interface{}); ok1 { 75 | var t2 = []int{} 76 | for _, i := range v1 { 77 | j, err := i.(json.Number).Int64() 78 | // j, err := strconv.ParseInt(i.String(), 0, 64) 79 | if err != nil { 80 | return nil, ErrConvertUnknownFormat 81 | } 82 | t2 = append(t2, (int)(j)) 83 | } 84 | return t2, nil 85 | } 86 | return nil, ErrConvertUnknownFormat 87 | } 88 | 89 | type ArrayFloatConverter struct{} 90 | 91 | func (c *ArrayFloatConverter) convert(v interface{}) (interface{}, error) { 92 | if v1, ok1 := v.([]interface{}); ok1 { 93 | var t2 = []float64{} 94 | for _, i := range v1 { 95 | j, err := i.(json.Number).Float64() 96 | if err != nil { 97 | return nil, ErrConvertUnknownFormat 98 | } 99 | t2 = append(t2, (float64)(j)) 100 | } 101 | return t2, nil 102 | } 103 | return nil, ErrConvertUnknownFormat 104 | } 105 | 106 | type ConveterAndRender struct { 107 | converter Converter 108 | valueRender value_render.ValueRender 109 | removeIfFail bool 110 | settoIfFail interface{} 111 | settoIfNil interface{} 112 | } 113 | 114 | type ConvertFilter struct { 115 | config map[interface{}]interface{} 116 | fields map[field_setter.FieldSetter]ConveterAndRender 117 | } 118 | 119 | func init() { 120 | Register("Convert", newConvertFilter) 121 | } 122 | 123 | func newConvertFilter(config map[interface{}]interface{}) topology.Filter { 124 | plugin := &ConvertFilter{ 125 | config: config, 126 | fields: make(map[field_setter.FieldSetter]ConveterAndRender), 127 | } 128 | 129 | if fieldsValue, ok := config["fields"]; ok { 130 | for f, vI := range fieldsValue.(map[interface{}]interface{}) { 131 | v := vI.(map[interface{}]interface{}) 132 | fieldSetter := field_setter.NewFieldSetter(f.(string)) 133 | if fieldSetter == nil { 134 | klog.Fatalf("could build field setter from %s", f.(string)) 135 | } 136 | 137 | to := v["to"].(string) 138 | remove_if_fail := false 139 | if I, ok := v["remove_if_fail"]; ok { 140 | remove_if_fail = I.(bool) 141 | } 142 | setto_if_fail := v["setto_if_fail"] 143 | setto_if_nil := v["setto_if_nil"] 144 | 145 | var converter Converter 146 | if to == "float" { 147 | converter = &FloatConverter{} 148 | } else if to == "int" { 149 | converter = &IntConverter{} 150 | } else if to == "uint" { 151 | converter = &UIntConverter{} 152 | } else if to == "bool" { 153 | converter = &BoolConverter{} 154 | } else if to == "string" { 155 | converter = &StringConverter{} 156 | } else if to == "array(int)" { 157 | converter = &ArrayIntConverter{} 158 | } else if to == "array(float)" { 159 | converter = &ArrayFloatConverter{} 160 | } else { 161 | klog.Fatal("can only convert to int/float/bool/array(int)/array(float)") 162 | } 163 | 164 | plugin.fields[fieldSetter] = ConveterAndRender{ 165 | converter: converter, 166 | valueRender: value_render.GetValueRender2(f.(string)), 167 | removeIfFail: remove_if_fail, 168 | settoIfFail: setto_if_fail, 169 | settoIfNil: setto_if_nil, 170 | } 171 | } 172 | } else { 173 | klog.Fatal("fields must be set in convert filter plugin") 174 | } 175 | return plugin 176 | } 177 | 178 | func (plugin *ConvertFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 179 | for fs, conveterAndRender := range plugin.fields { 180 | originanV, err := conveterAndRender.valueRender.Render(event) 181 | if err != nil || originanV == nil { 182 | if conveterAndRender.settoIfNil != nil { 183 | event = fs.SetField(event, conveterAndRender.settoIfNil, "", true) 184 | } 185 | continue 186 | } 187 | v, err := conveterAndRender.converter.convert(originanV) 188 | if err == nil { 189 | event = fs.SetField(event, v, "", true) 190 | } else { 191 | klog.V(10).Infof("convert error: %s", err) 192 | if conveterAndRender.removeIfFail { 193 | event = fs.SetField(event, nil, "", true) 194 | } else if conveterAndRender.settoIfFail != nil { 195 | event = fs.SetField(event, conveterAndRender.settoIfFail, "", true) 196 | } 197 | } 198 | } 199 | return event, true 200 | } 201 | -------------------------------------------------------------------------------- /filter/convert_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | func TestIntConverter(t *testing.T) { 9 | type testCase struct { 10 | v interface{} 11 | want interface{} 12 | err bool 13 | } 14 | 15 | convert := &IntConverter{} 16 | 17 | cases := []testCase{ 18 | { 19 | json.Number("1"), int64(1), false, 20 | }, 21 | { 22 | "1", int64(1), false, 23 | }, 24 | { 25 | 1, int64(1), false, 26 | }, 27 | { 28 | -1, int64(-1), false, 29 | }, 30 | { 31 | "-1", int64(-1), false, 32 | }, 33 | { 34 | "12345678901234567890", int64(0), true, 35 | }, 36 | } 37 | 38 | for _, c := range cases { 39 | ans, err := convert.convert(c.v) 40 | if ans != c.want { 41 | t.Errorf("convert %v: want %v, got %v", c.v, c.want, ans) 42 | } 43 | 44 | if c.err != (err != nil) { 45 | t.Errorf("convert %v: want %v, got %v", c.v, c.err, err) 46 | } 47 | } 48 | } 49 | 50 | func TestUIntConverter(t *testing.T) { 51 | type testCase struct { 52 | v interface{} 53 | want interface{} 54 | err bool 55 | } 56 | 57 | convert := &UIntConverter{} 58 | 59 | cases := []testCase{ 60 | { 61 | json.Number("1"), uint64(1), false, 62 | }, 63 | { 64 | "1", uint64(1), false, 65 | }, 66 | { 67 | 1, uint64(1), false, 68 | }, 69 | { 70 | -1, uint64(0), true, 71 | }, 72 | { 73 | "-1", uint64(0), true, 74 | }, 75 | { 76 | "12345678901234567890", uint64(12345678901234567890), false, 77 | }, 78 | } 79 | 80 | for _, c := range cases { 81 | ans, err := convert.convert(c.v) 82 | if ans != c.want { 83 | t.Errorf("convert %v: want %v, got %v", c.v, c.want, ans) 84 | } 85 | 86 | if c.err != (err != nil) { 87 | t.Errorf("convert %v: want %v, got %v", c.v, c.err, err) 88 | } 89 | } 90 | } 91 | 92 | func TestFloatConverter(t *testing.T) { 93 | type testCase struct { 94 | v interface{} 95 | want interface{} 96 | err bool 97 | } 98 | 99 | convert := &FloatConverter{} 100 | 101 | cases := []testCase{ 102 | { 103 | json.Number("1.1"), float64(1.1), false, 104 | }, 105 | { 106 | "1.2", float64(1.2), false, 107 | }, 108 | { 109 | 1.3, float64(1.3), false, 110 | }, 111 | { 112 | -1.4, float64(-1.4), false, 113 | }, 114 | { 115 | "-1.5", float64(-1.5), false, 116 | }, 117 | { 118 | "abcd", 0.0, true, 119 | }, 120 | { 121 | "", 0.0, true, 122 | }, 123 | } 124 | 125 | for _, c := range cases { 126 | ans, err := convert.convert(c.v) 127 | if ans != c.want { 128 | t.Errorf("convert %v: want %v, got %v", c.v, c.want, ans) 129 | } 130 | 131 | if c.err != (err != nil) { 132 | t.Errorf("convert %v: want %v, got %v", c.v, c.err, err) 133 | } 134 | } 135 | } 136 | 137 | func TestBoolConverter(t *testing.T) { 138 | type testCase struct { 139 | v interface{} 140 | want interface{} 141 | err bool 142 | } 143 | 144 | convert := &BoolConverter{} 145 | 146 | cases := []testCase{ 147 | { 148 | "abcd", nil, true, 149 | }, 150 | { 151 | "True", true, false, 152 | }, 153 | { 154 | "false", false, false, 155 | }, 156 | { 157 | json.Number("1"), nil, true, 158 | }, 159 | { 160 | 1234, nil, true, 161 | }, 162 | } 163 | 164 | for _, c := range cases { 165 | ans, err := convert.convert(c.v) 166 | if ans != c.want { 167 | t.Errorf("convert %v: want %v, got %v", c.v, c.want, ans) 168 | } 169 | 170 | if c.err != (err != nil) { 171 | t.Errorf("convert %v: want %v, got %v", c.v, c.err, err) 172 | } 173 | } 174 | } 175 | 176 | func TestSettoIfNil(t *testing.T) { 177 | config := make(map[interface{}]interface{}) 178 | fields := make(map[interface{}]interface{}) 179 | fields["timeTaken"] = map[interface{}]interface{}{ 180 | "to": "float", 181 | "setto_if_nil": 0.0, 182 | } 183 | config["fields"] = fields 184 | f := BuildFilter("Convert", config) 185 | event := map[string]interface{}{} 186 | 187 | event, ok := f.Filter(event) 188 | t.Log(event) 189 | 190 | if ok == false { 191 | t.Error("ConvertFilter fail") 192 | } 193 | 194 | if event["timeTaken"].(float64) != 0.0 { 195 | t.Error("timeTaken convert error") 196 | } 197 | } 198 | 199 | func TestConvertFilter(t *testing.T) { 200 | config := make(map[interface{}]interface{}) 201 | fields := make(map[interface{}]interface{}) 202 | fields["id"] = map[interface{}]interface{}{ 203 | "to": "uint", 204 | "setto_if_fail": 0, 205 | } 206 | fields["responseSize"] = map[interface{}]interface{}{ 207 | "to": "int", 208 | "setto_if_fail": 0, 209 | } 210 | fields["timeTaken"] = map[interface{}]interface{}{ 211 | "to": "float", 212 | "remove_if_fail": true, 213 | } 214 | // add to string test case 215 | fields["toString"] = map[interface{}]interface{}{ 216 | "to": "string", 217 | "remove_if_fail": true, 218 | } 219 | config["fields"] = fields 220 | f := BuildFilter("Convert", config) 221 | 222 | case1 := map[string]int{"a": 5, "b": 7} 223 | event := map[string]interface{}{ 224 | "id": "12345678901234567890", 225 | "responseSize": "10", 226 | "timeTaken": "0.010", 227 | "toString": case1, 228 | } 229 | t.Log(event) 230 | 231 | event, ok := f.Filter(event) 232 | t.Log(event) 233 | 234 | if ok == false { 235 | t.Error("ConvertFilter fail") 236 | } 237 | 238 | if event["id"].(uint64) != 12345678901234567890 { 239 | t.Error("id should be 12345678901234567890") 240 | } 241 | if event["responseSize"].(int64) != 10 { 242 | t.Error("responseSize should be 10") 243 | } 244 | if event["timeTaken"].(float64) != 0.01 { 245 | t.Error("timeTaken should be 0.01") 246 | } 247 | if event["toString"].(string) != "{\"a\":5,\"b\":7}" { 248 | t.Error("toString is unexpected") 249 | } 250 | event = map[string]interface{}{ 251 | "responseSize": "10.1", 252 | "timeTaken": "abcd", 253 | "toString": "huangjacky", 254 | } 255 | t.Log(event) 256 | 257 | event, ok = f.Filter(event) 258 | t.Log(event) 259 | 260 | if ok == false { 261 | t.Error("ConvertFilter fail") 262 | } 263 | 264 | if event["responseSize"].(int) != 0 { 265 | t.Error("responseSize should be 0") 266 | } 267 | if event["timeTaken"] != nil { 268 | t.Error("timeTaken should be nil") 269 | } 270 | if event["toString"].(string) != "huangjacky" { 271 | t.Error("toString should be huangjacky") 272 | } 273 | } 274 | -------------------------------------------------------------------------------- /filter/date.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "math" 8 | "reflect" 9 | "strconv" 10 | "time" 11 | 12 | "github.com/childe/gohangout/field_setter" 13 | "github.com/childe/gohangout/topology" 14 | "github.com/childe/gohangout/value_render" 15 | "github.com/relvacode/iso8601" // https://pkg.go.dev/github.com/relvacode/iso8601#section-readme 16 | "k8s.io/klog/v2" 17 | ) 18 | 19 | type DateParser interface { 20 | Parse(interface{}) (time.Time, error) 21 | } 22 | 23 | type FormatParser struct { 24 | format string 25 | location *time.Location 26 | addYear bool 27 | } 28 | 29 | var MustStringTypeError = errors.New("timestamp field must be string") 30 | 31 | func (dp *FormatParser) Parse(t interface{}) (time.Time, error) { 32 | var ( 33 | rst time.Time 34 | err error 35 | ) 36 | value, ok := t.(string) 37 | 38 | if !ok { 39 | return rst, MustStringTypeError 40 | } 41 | 42 | if dp.addYear { 43 | value = fmt.Sprintf("%d%s", time.Now().Year(), value) 44 | } 45 | if dp.location == nil { 46 | return time.Parse(dp.format, value) 47 | } 48 | rst, err = time.ParseInLocation(dp.format, value, dp.location) 49 | if err != nil { 50 | return rst, err 51 | } 52 | return rst.UTC(), nil 53 | } 54 | 55 | type UnixParser struct{} 56 | 57 | func (p *UnixParser) Parse(t interface{}) (time.Time, error) { 58 | var ( 59 | rst time.Time 60 | ) 61 | if v, ok := t.(json.Number); ok { 62 | t1, err := v.Int64() 63 | if err != nil { 64 | return rst, err 65 | } 66 | return time.Unix(t1, 0), nil 67 | } 68 | 69 | if v, ok := t.(string); ok { 70 | t1, err := strconv.Atoi(v) 71 | if err != nil { 72 | f, err := strconv.ParseFloat(v, 64) 73 | if err != nil { 74 | return rst, err 75 | } 76 | t1 := math.Floor(f) 77 | return time.Unix(int64(t1), int64(1000000000*(f-t1))), nil 78 | } 79 | return time.Unix(int64(t1), 0), nil 80 | } 81 | 82 | if t1, ok := t.(int); ok { 83 | return time.Unix(int64(t1), 0), nil 84 | } 85 | if t1, ok := t.(int64); ok { 86 | return time.Unix(t1, 0), nil 87 | } 88 | return rst, fmt.Errorf("%s unknown type:%s", t, reflect.TypeOf(t).String()) 89 | } 90 | 91 | type UnixMSParser struct{} 92 | 93 | func (p *UnixMSParser) Parse(t interface{}) (time.Time, error) { 94 | var ( 95 | rst time.Time 96 | ) 97 | if v, ok := t.(json.Number); ok { 98 | t1, err := v.Int64() 99 | if err != nil { 100 | return rst, err 101 | } 102 | return time.Unix(t1/1000, t1%1000*1000000), nil 103 | } 104 | if v, ok := t.(string); ok { 105 | t1, err := strconv.Atoi(v) 106 | if err != nil { 107 | return rst, err 108 | } 109 | t2 := int64(t1) 110 | return time.Unix(t2/1000, t2%1000*1000000), nil 111 | } 112 | if v, ok := t.(int); ok { 113 | t1 := int64(v) 114 | return time.Unix(t1/1000, t1%1000*1000000), nil 115 | } 116 | if v, ok := t.(int64); ok { 117 | return time.Unix(v/1000, v%1000*1000000), nil 118 | } 119 | return rst, fmt.Errorf("%s unknown type:%s", t, reflect.TypeOf(t).String()) 120 | } 121 | 122 | type ISO8601Parser struct { 123 | location *time.Location // If the input does not have timezone information, it will use the given location. 124 | } 125 | 126 | func (p *ISO8601Parser) Parse(t interface{}) (time.Time, error) { 127 | var ( 128 | rst time.Time 129 | ) 130 | if v, ok := t.(string); ok { 131 | if p.location == nil { 132 | return iso8601.ParseString(v) 133 | } 134 | return iso8601.ParseStringInLocation(v, p.location) 135 | } 136 | 137 | return rst, fmt.Errorf("%s unknown type:%s", t, reflect.TypeOf(t).String()) 138 | } 139 | 140 | func getDateParser(format string, l *time.Location, addYear bool) DateParser { 141 | if format == "UNIX" { 142 | return &UnixParser{} 143 | } 144 | if format == "UNIX_MS" { 145 | return &UnixMSParser{} 146 | } 147 | if format == "RFC3339" { 148 | return &FormatParser{time.RFC3339, l, addYear} 149 | } 150 | if format == "ISO8601" { 151 | return &ISO8601Parser{l} 152 | } 153 | return &FormatParser{format, l, addYear} 154 | } 155 | 156 | type DateFilter struct { 157 | config map[interface{}]interface{} 158 | dateParsers []DateParser 159 | overwrite bool 160 | src string 161 | srcVR value_render.ValueRender 162 | target string 163 | targetFS field_setter.FieldSetter 164 | } 165 | 166 | func init() { 167 | Register("Date", newDateFilter) 168 | } 169 | 170 | func newDateFilter(config map[interface{}]interface{}) topology.Filter { 171 | plugin := &DateFilter{ 172 | config: config, 173 | overwrite: true, 174 | dateParsers: make([]DateParser, 0), 175 | } 176 | 177 | if overwrite, ok := config["overwrite"]; ok { 178 | plugin.overwrite = overwrite.(bool) 179 | } 180 | 181 | if srcValue, ok := config["src"]; ok { 182 | plugin.src = srcValue.(string) 183 | } else { 184 | klog.Fatal("src must be set in date filter plugin") 185 | } 186 | plugin.srcVR = value_render.GetValueRender2(plugin.src) 187 | 188 | if targetI, ok := config["target"]; ok { 189 | plugin.target = targetI.(string) 190 | } else { 191 | plugin.target = "@timestamp" 192 | } 193 | plugin.targetFS = field_setter.NewFieldSetter(plugin.target) 194 | 195 | var ( 196 | location *time.Location 197 | addYear bool = false 198 | err error 199 | ) 200 | if locationI, ok := config["location"]; ok { 201 | location, err = time.LoadLocation(locationI.(string)) 202 | if err != nil { 203 | klog.Fatalf("load location error:%s", err) 204 | } 205 | } else { 206 | location = nil 207 | } 208 | if addYearI, ok := config["add_year"]; ok { 209 | addYear = addYearI.(bool) 210 | } 211 | if formats, ok := config["formats"]; ok { 212 | for _, formatI := range formats.([]interface{}) { 213 | plugin.dateParsers = append(plugin.dateParsers, getDateParser(formatI.(string), location, addYear)) 214 | } 215 | } else { 216 | klog.Fatal("formats must be set in date filter plugin") 217 | } 218 | 219 | return plugin 220 | } 221 | 222 | func (plugin *DateFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 223 | inputI, err := plugin.srcVR.Render(event) 224 | if err != nil || inputI == nil { 225 | return event, false 226 | } 227 | 228 | for _, dp := range plugin.dateParsers { 229 | t, err := dp.Parse(inputI) 230 | if err == nil { 231 | event = plugin.targetFS.SetField(event, t, "", plugin.overwrite) 232 | return event, true 233 | } 234 | } 235 | return event, false 236 | } 237 | -------------------------------------------------------------------------------- /filter/date_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strconv" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | var ts int64 = 1580212332 10 | var tsMs int64 = 1580212332123 11 | 12 | func TestUnix(t *testing.T) { 13 | for _, v := range []interface{}{ts, int(ts), "1580212332"} { 14 | p := &UnixParser{} 15 | r, err := p.Parse(v) 16 | 17 | if err != nil { 18 | t.Error(err) 19 | } 20 | 21 | t.Log(r) 22 | if r.Unix() != ts { 23 | t.Errorf("%v %d", v, r.Unix()) 24 | } 25 | } 26 | } 27 | 28 | func TestUnixMs(t *testing.T) { 29 | for _, v := range []interface{}{tsMs, int(tsMs), "1580212332123"} { 30 | p := &UnixMSParser{} 31 | r, err := p.Parse(v) 32 | 33 | if err != nil { 34 | t.Fatalf("%v %s", v, err) 35 | } 36 | 37 | t.Log(r) 38 | t.Log(r.Unix()) 39 | rr := r.UnixNano() / 1000000 40 | if rr != tsMs { 41 | t.Fatalf("%#v %d", v, rr) 42 | } 43 | } 44 | } 45 | 46 | func TestFormatParser(t *testing.T) { 47 | var tStr = "2020-01-28T19:52:12.123+08:00" 48 | p := &FormatParser{time.RFC3339, nil, false} 49 | r, err := p.Parse(tStr) 50 | 51 | if err != nil { 52 | t.Fatalf("%s", err) 53 | } 54 | 55 | rr := r.UnixNano() / 1000000 56 | if rr != tsMs { 57 | t.Fatalf("%v %d", tStr, rr) 58 | } 59 | } 60 | 61 | func TestFormatParserWithLocation(t *testing.T) { 62 | var tStr = "2020-01-28T19:52:12.123456" 63 | location, _ := time.LoadLocation("Asia/Shanghai") 64 | p := &FormatParser{"2006-01-02T15:04:05.9", location, false} 65 | r, err := p.Parse(tStr) 66 | 67 | if err != nil { 68 | t.Fatalf("%s", err) 69 | } 70 | 71 | rr := r.UnixNano() / 1000 72 | if rr != tsMs*1000+456 { 73 | t.Fatalf("%v %d", tStr, rr) 74 | } 75 | } 76 | 77 | func TestISO8601Parser(t *testing.T) { 78 | var tArr = [5]string{ 79 | "2020-01-28T19:52:12", 80 | "2020-01-28T19:52:12.000", 81 | "2020-01-28T11:52:12Z", 82 | "2020-01-28T19:52:12+0800", 83 | "2020-01-28T19:52:12+08:00", 84 | } 85 | // further cases please see: https://github.com/relvacode/iso8601/blob/master/iso8601_test.go 86 | 87 | location, _ := time.LoadLocation("Asia/Shanghai") 88 | p := &ISO8601Parser{location} 89 | for _, tStr := range tArr { 90 | r, err := p.Parse(tStr) 91 | 92 | if err != nil { 93 | t.Fatalf("%s", err) 94 | } 95 | 96 | rr := r.UnixNano() / 1000 97 | if rr != ts * 1000000 { 98 | t.Fatalf("%v %d", tStr, rr) 99 | } 100 | } 101 | } 102 | 103 | func TestDateFilter(t *testing.T) { 104 | config := make(map[interface{}]interface{}) 105 | config["location"] = "Asia/Shanghai" 106 | config["src"] = "@timestamp" 107 | config["formats"] = []interface{}{"RFC3339", "UNIX"} 108 | f := BuildFilter("Date", config) 109 | 110 | event := make(map[string]interface{}) 111 | event["@timestamp"] = time.Now().Unix() 112 | t.Log(event) 113 | 114 | event, ok := f.Filter(event) 115 | t.Log(event) 116 | 117 | if ok == false { 118 | t.Error("fail") 119 | } 120 | 121 | event["@timestamp"] = strconv.Itoa((int)(time.Now().Unix())) 122 | t.Log(event) 123 | 124 | event, ok = f.Filter(event) 125 | t.Log(event) 126 | 127 | if ok == false { 128 | t.Error("fail") 129 | } 130 | 131 | event["@timestamp"] = "2018-01-23T17:06:05+08:00" 132 | t.Log(event) 133 | 134 | event, ok = f.Filter(event) 135 | t.Log(event) 136 | 137 | if ok == false { 138 | t.Error("fail") 139 | } 140 | 141 | config["location"] = "Etc/UTC" 142 | config["formats"] = []interface{}{"2006-01-02T15:04:05"} 143 | f = BuildFilter("Date", config) 144 | event["@timestamp"] = "2018-01-23T17:06:05" 145 | t.Log(event) 146 | 147 | event, ok = f.Filter(event) 148 | t.Log(event) 149 | 150 | if ok == false { 151 | t.Error("fail") 152 | } 153 | 154 | } 155 | -------------------------------------------------------------------------------- /filter/drop.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "github.com/childe/gohangout/topology" 4 | 5 | type dropFilter struct { 6 | config map[interface{}]interface{} 7 | } 8 | 9 | func init() { 10 | Register("Drop", newDropFilter) 11 | } 12 | 13 | func newDropFilter(config map[interface{}]interface{}) topology.Filter { 14 | plugin := &dropFilter{ 15 | config: config, 16 | } 17 | return plugin 18 | } 19 | 20 | func (plugin *dropFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 21 | return nil, true 22 | } 23 | -------------------------------------------------------------------------------- /filter/drop_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestDropFilter(t *testing.T) { 9 | var ( 10 | config map[interface{}]interface{} 11 | event map[string]interface{} 12 | ok bool 13 | ) 14 | 15 | // test DropFilter without any condition 16 | config = make(map[interface{}]interface{}) 17 | f := BuildFilter("Drop", config) 18 | 19 | event = make(map[string]interface{}) 20 | event["@timestamp"] = time.Now().Unix() 21 | event["first"] = "dehua" 22 | event["last"] = "liu" 23 | 24 | event, ok = f.Filter(event) 25 | 26 | if ok == false { 27 | t.Error("drop filter fail") 28 | } 29 | 30 | if event != nil { 31 | t.Error("event should be nil after being dropped") 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /filter/filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "fmt" 5 | "plugin" 6 | 7 | "github.com/childe/gohangout/topology" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | type BuildFilterFunc func(map[interface{}]interface{}) topology.Filter 12 | 13 | var registeredFilter map[string]BuildFilterFunc = make(map[string]BuildFilterFunc) 14 | 15 | // Register is used by input plugins to register themselves 16 | func Register(filterType string, bf BuildFilterFunc) { 17 | if _, ok := registeredFilter[filterType]; ok { 18 | klog.Errorf("%s has been registered, ignore %T", filterType, bf) 19 | return 20 | } 21 | registeredFilter[filterType] = bf 22 | } 23 | 24 | // BuildFilter builds Filter from filter type and config. it firstly tries built-in filter, and then try 3rd party plugin 25 | func BuildFilter(filterType string, config map[interface{}]interface{}) topology.Filter { 26 | if v, ok := registeredFilter[filterType]; ok { 27 | return v(config) 28 | } 29 | klog.Infof("could not load %s filter plugin, try third party plugin", filterType) 30 | 31 | pluginPath := filterType 32 | filter, err := getFilterFromPlugin(pluginPath, config) 33 | if err != nil { 34 | klog.Errorf("could not open %s: %s", pluginPath, err) 35 | return nil 36 | } 37 | return filter 38 | } 39 | 40 | func getFilterFromPlugin(pluginPath string, config map[interface{}]interface{}) (topology.Filter, error) { 41 | p, err := plugin.Open(pluginPath) 42 | if err != nil { 43 | return nil, fmt.Errorf("could not open %s: %s", pluginPath, err) 44 | } 45 | newFunc, err := p.Lookup("New") 46 | if err != nil { 47 | return nil, fmt.Errorf("could not find `New` function in %s: %s", pluginPath, err) 48 | } 49 | 50 | f, ok := newFunc.(func(map[interface{}]interface{}) interface{}) 51 | if !ok { 52 | return nil, fmt.Errorf("`New` func in %s format error", pluginPath) 53 | } 54 | 55 | rst := f(config) 56 | filter, ok := rst.(topology.Filter) 57 | if !ok { 58 | return nil, fmt.Errorf("`New` func in %s dose not return Filter Interface", pluginPath) 59 | } 60 | return filter, nil 61 | } 62 | -------------------------------------------------------------------------------- /filter/filters.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/childe/gohangout/topology" 7 | "k8s.io/klog/v2" 8 | ) 9 | 10 | type FiltersFilter struct { 11 | config map[interface{}]interface{} 12 | processorNode *topology.ProcessorNode 13 | filterBoxes []*topology.FilterBox 14 | } 15 | 16 | func init() { 17 | Register("Filters", newFiltersFilter) 18 | } 19 | 20 | func newFiltersFilter(config map[interface{}]interface{}) topology.Filter { 21 | f := &FiltersFilter{ 22 | config: config, 23 | } 24 | 25 | _config := make(map[string]interface{}) 26 | for k, v := range config { 27 | _config[k.(string)] = v 28 | } 29 | 30 | f.filterBoxes = topology.BuildFilterBoxes(_config, BuildFilter) 31 | if len(f.filterBoxes) == 0 { 32 | klog.Fatal("no filters configured in Filters") 33 | } 34 | 35 | for _, b := range f.filterBoxes { 36 | f.processorNode = topology.AppendProcessorsToLink(f.processorNode, b) 37 | } 38 | 39 | return f 40 | } 41 | 42 | func (f *FiltersFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 43 | return f.processorNode.Process(event), true 44 | } 45 | 46 | func (f *FiltersFilter) SetBelongTo(next topology.Processor) { 47 | var b *topology.FilterBox = f.filterBoxes[len(f.filterBoxes)-1] 48 | v := reflect.ValueOf(b.Filter) 49 | fun := v.MethodByName("SetBelongTo") 50 | if fun.IsValid() { 51 | fun.Call([]reflect.Value{reflect.ValueOf(next)}) 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /filter/grok_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "testing" 4 | 5 | func TestGetPath(t *testing.T) { 6 | filepath := "https://raw.githubusercontent.com/vjeantet/grok/master/patterns/grok-patterns" 7 | _, err := getFiles(filepath) 8 | if err != nil { 9 | t.Errorf("getFiles error:%s", err) 10 | } 11 | } 12 | 13 | func TestGrokFilter(t *testing.T) { 14 | config := make(map[interface{}]interface{}) 15 | match := make([]interface{}, 2) 16 | match[0] = `(?P\S+ \S+) \[(?P\w+)\] (?P.*)$` 17 | match[1] = `(?P\S+ \S+)` 18 | config["match"] = match 19 | config["src"] = "message" 20 | 21 | f := BuildFilter("Grok", config) 22 | 23 | event := make(map[string]interface{}) 24 | event["message"] = "2018-07-12T14:45:00 +0800 [info] message" 25 | 26 | event, ok := f.Filter(event) 27 | t.Log(event) 28 | 29 | if ok == false { 30 | t.Error("grok filter fail") 31 | } 32 | 33 | if v, ok := event["msg"]; !ok { 34 | t.Error("msg field should exist") 35 | } else { 36 | if v != "message" { 37 | t.Error("msg field do not match") 38 | } 39 | } 40 | } 41 | 42 | func TestTarget(t *testing.T) { 43 | config := make(map[interface{}]interface{}) 44 | match := make([]interface{}, 2) 45 | match[0] = `(?P\S+ \S+) \[(?P\w+)\] (?P.*)$` 46 | match[1] = `(?P\S+ \S+)` 47 | config["match"] = match 48 | config["src"] = "message" 49 | config["target"] = "grok" 50 | 51 | f := BuildFilter("Grok", config) 52 | 53 | event := make(map[string]interface{}) 54 | event["message"] = "2018-07-12T14:45:00 +0800 [info] message" 55 | 56 | event, ok := f.Filter(event) 57 | t.Log(event) 58 | 59 | if ok == false { 60 | t.Error("grok filter fail") 61 | } 62 | 63 | if grok, ok := event["grok"]; ok { 64 | if msg, ok := grok.(map[string]interface{})["msg"]; !ok || msg.(string) != "message" { 65 | t.Error("msg field do not match") 66 | } 67 | } else { 68 | t.Error("grok field should exist") 69 | } 70 | } 71 | 72 | func TestPattern(t *testing.T) { 73 | grok := &Grok{} 74 | grok.patterns = map[string]string{ 75 | "USERNAME": "[a-zA-Z0-9._-]+", 76 | "USER": "%{USERNAME}", 77 | "IPV6": `((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?`, 78 | "IPV4": `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`, 79 | "IP": "(?:%{IPV6}|%{IPV4})", 80 | } 81 | 82 | p := grok.translateMatchPattern(`%{USER:user}`) 83 | if p != `(?P([a-zA-Z0-9._-]+))` { 84 | t.Error(p) 85 | } 86 | 87 | config := make(map[interface{}]interface{}) 88 | match := make([]interface{}, 1) 89 | match[0] = grok.translateMatchPattern(`^%{IP:ip} %{USER:user} \[(?P\w+)\] (?P.*)`) 90 | config["match"] = match 91 | config["src"] = "message" 92 | 93 | f := BuildFilter("Grok", config) 94 | 95 | event := make(map[string]interface{}) 96 | event["message"] = "10.10.10.255 childe [info] message" 97 | 98 | event, ok := f.Filter(event) 99 | t.Log(event) 100 | 101 | if ok == false { 102 | t.Error("grok filter fail") 103 | } 104 | 105 | if v, ok := event["loglevel"]; !ok { 106 | t.Error("loglevel field should exist") 107 | } else { 108 | if v != "info" { 109 | t.Error("loglevel field do not match") 110 | } 111 | } 112 | 113 | if v, ok := event["user"]; !ok { 114 | t.Error("user field should exist") 115 | } else { 116 | if v != "childe" { 117 | t.Error("user field do not match") 118 | } 119 | } 120 | 121 | if v, ok := event["ip"]; !ok { 122 | t.Error("ip field should exist") 123 | } else { 124 | if v != "10.10.10.255" { 125 | t.Error("ip field do not match") 126 | } 127 | } 128 | 129 | if v, ok := event["msg"]; !ok { 130 | t.Error("msg field should exist") 131 | } else { 132 | if v != "message" { 133 | t.Error("msg field do not match") 134 | } 135 | } 136 | } 137 | 138 | func TestOverwrite(t *testing.T) { 139 | config := make(map[interface{}]interface{}) 140 | match := make([]interface{}, 1) 141 | match[0] = `(?P\S+ \S+) \[(?P\w+)\] (?P.*)$` 142 | config["match"] = match 143 | config["src"] = "message" 144 | config["overwrite"] = false 145 | 146 | f := BuildFilter("Grok", config) 147 | 148 | event := make(map[string]interface{}) 149 | event["message"] = "2018-07-12T14:45:00 +0800 [info] message" 150 | event["level"] = "warning" 151 | event, ok := f.Filter(event) 152 | t.Log(event) 153 | 154 | if ok == false { 155 | t.Error("grok filter fail") 156 | } 157 | if level, ok := event["level"]; ok { 158 | if level != "warning" { 159 | t.Error("msg field do not match") 160 | } 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /filter/gsub.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "regexp" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "github.com/mitchellh/mapstructure" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | type rs struct { 14 | r value_render.ValueRender 15 | s field_setter.FieldSetter 16 | } 17 | 18 | type oneFieldConfig struct { 19 | rs rs 20 | Field string 21 | 22 | Src string 23 | srcRegexp *regexp.Regexp 24 | 25 | Repl string 26 | } 27 | 28 | func init() { 29 | Register("Gsub", newGsubFilter) 30 | } 31 | 32 | // GsubFilter implements topology.Filter. 33 | type GsubFilter struct { 34 | fields []*oneFieldConfig 35 | } 36 | 37 | func newGsubFilter(config map[interface{}]interface{}) topology.Filter { 38 | gsubFilter := &GsubFilter{} 39 | fields, ok := config["fields"] 40 | if !ok { 41 | klog.Fatal("fields must be set in gsub filter") 42 | } 43 | 44 | err := mapstructure.Decode(fields, &gsubFilter.fields) 45 | if err != nil { 46 | klog.Fatal("decode fields config in gusb error:", err) 47 | } 48 | 49 | for _, config := range gsubFilter.fields { 50 | config.rs.r = value_render.GetValueRender2(config.Field) 51 | config.rs.s = field_setter.NewFieldSetter(config.Field) 52 | 53 | config.srcRegexp = regexp.MustCompile(config.Src) 54 | } 55 | 56 | return gsubFilter 57 | } 58 | 59 | // Filter implements topology.Filter. 60 | // One field config fails if could not get src or src is not string. 61 | // Filter returns false if either field config fails. 62 | func (f *GsubFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 63 | rst := true 64 | for _, config := range f.fields { 65 | v, err := config.rs.r.Render(event) 66 | if err != nil || v == nil { 67 | rst = false 68 | continue 69 | } 70 | if v, ok := v.(string); !ok { 71 | rst = false 72 | continue 73 | } else { 74 | rst := config.srcRegexp.ReplaceAllString(v, config.Repl) 75 | config.rs.s.SetField(event, rst, config.Field, true) 76 | } 77 | } 78 | return event, rst 79 | } 80 | -------------------------------------------------------------------------------- /filter/gsub_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | type gsubTestCase struct { 9 | event map[string]interface{} 10 | config map[interface{}]interface{} 11 | 12 | wantEvent map[string]interface{} 13 | wantOK bool 14 | } 15 | 16 | func TestGsub(t *testing.T) { 17 | for _, c := range []gsubTestCase{ 18 | { 19 | event: map[string]interface{}{ 20 | "msg1": "corp/com", 21 | "msg2": `trip#corp?com\cn`, 22 | }, 23 | config: map[interface{}]interface{}{ 24 | "fields": []map[string]string{ 25 | {"field": "msg1", "src": "/", "repl": "_"}, 26 | {"field": "msg2", "src": `[\\?#-]`, "repl": "."}, 27 | }, 28 | }, 29 | wantEvent: map[string]interface{}{ 30 | "msg1": "corp_com", 31 | "msg2": "trip.corp.com.cn", 32 | }, 33 | wantOK: true, 34 | }, 35 | { 36 | event: map[string]interface{}{ 37 | "msg": "corp.com", 38 | }, 39 | config: map[interface{}]interface{}{ 40 | "fields": []map[string]string{ 41 | {"field": "msg", "src": "(^\\w+)", "repl": "xxx-$1-yyy"}, 42 | }, 43 | }, 44 | wantEvent: map[string]interface{}{ 45 | "msg": "xxx-corp-yyy.com", 46 | }, 47 | wantOK: true, 48 | }, 49 | { 50 | event: map[string]interface{}{ 51 | "msg": map[string]interface{}{ 52 | "data": "corp.com", 53 | }, 54 | }, 55 | config: map[interface{}]interface{}{ 56 | "fields": []map[string]string{ 57 | {"field": "[msg][data]", "src": "(^\\w+)", "repl": "xxx-$1-yyy"}, 58 | }, 59 | }, 60 | wantEvent: map[string]interface{}{ 61 | "msg": map[string]interface{}{ 62 | "data": "xxx-corp-yyy.com", 63 | }, 64 | }, 65 | wantOK: true, 66 | }, 67 | } { 68 | filter := newGsubFilter(c.config) 69 | event, ok := filter.Filter(c.event) 70 | if !reflect.DeepEqual(event, c.wantEvent) { 71 | t.Errorf("case %v error want %+v, got %+v", c, c.wantEvent, event) 72 | } 73 | if ok != c.wantOK { 74 | t.Errorf("case %v error. want %v, got %v", c, c.wantOK, ok) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /filter/ipip.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strconv" 5 | "unsafe" 6 | 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | datx "github.com/ipipdotnet/datx-go" 10 | ipdb "github.com/ipipdotnet/ipdb-go" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | type IPIPFilter struct { 15 | config map[interface{}]interface{} 16 | src string 17 | srcVR value_render.ValueRender 18 | target string 19 | data_type string 20 | language string 21 | database string 22 | city unsafe.Pointer 23 | overwrite bool 24 | } 25 | 26 | const bitSize int = 64 27 | 28 | func init() { 29 | Register("IPIP", newIPIPFilter) 30 | } 31 | 32 | func newIPIPFilter(config map[interface{}]interface{}) topology.Filter { 33 | plugin := &IPIPFilter{ 34 | config: config, 35 | target: "geoip", 36 | data_type: "datx", 37 | language: "CN", 38 | overwrite: true, 39 | } 40 | 41 | if overwrite, ok := config["overwrite"]; ok { 42 | plugin.overwrite = overwrite.(bool) 43 | } 44 | if data_type, ok := config["type"]; ok { 45 | plugin.data_type = data_type.(string) 46 | } 47 | if language, ok := config["language"]; ok { 48 | plugin.language = language.(string) 49 | } 50 | if database, ok := config["database"]; ok { 51 | plugin.database = database.(string) 52 | var ( 53 | c1 *datx.City 54 | c2 *ipdb.City 55 | err error 56 | ) 57 | if plugin.data_type == "datx" { 58 | c1, err = datx.NewCity(plugin.database) 59 | plugin.city = unsafe.Pointer(c1) 60 | } else { 61 | c2, err = ipdb.NewCity(plugin.database) 62 | plugin.city = unsafe.Pointer(c2) 63 | } 64 | if err != nil { 65 | klog.Fatalf("could not load %s: %s", plugin.database, err) 66 | } 67 | } else { 68 | klog.Fatal("database must be set in IPIP filter plugin") 69 | } 70 | 71 | if src, ok := config["src"]; ok { 72 | plugin.src = src.(string) 73 | plugin.srcVR = value_render.GetValueRender2(plugin.src) 74 | } else { 75 | klog.Fatal("src must be set in IPIP filter plugin") 76 | } 77 | 78 | if target, ok := config["target"]; ok { 79 | plugin.target = target.(string) 80 | } 81 | return plugin 82 | } 83 | 84 | func (plugin *IPIPFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 85 | inputI, err := plugin.srcVR.Render(event) 86 | if err != nil || inputI == nil { 87 | return event, false 88 | } 89 | var a []string 90 | if plugin.data_type == "datx" { 91 | city := (*datx.City)(plugin.city) 92 | a, err = city.Find(inputI.(string)) 93 | } else { 94 | city := (*ipdb.City)(plugin.city) 95 | a, err = city.Find(inputI.(string), plugin.language) 96 | } 97 | if err != nil { 98 | klog.V(10).Infof("failed to find %s: %s", inputI.(string), err) 99 | return event, false 100 | } 101 | if plugin.target == "" { 102 | event["country_name"] = a[0] 103 | event["province_name"] = a[1] 104 | event["city_name"] = a[2] 105 | if len(a) >= 5 { 106 | event["isp"] = a[4] 107 | } 108 | if len(a) >= 10 { 109 | latitude, _ := strconv.ParseFloat(a[5], bitSize) 110 | longitude, _ := strconv.ParseFloat(a[6], bitSize) 111 | event["latitude"] = latitude 112 | event["longitude"] = longitude 113 | event["location"] = []interface{}{longitude, latitude} 114 | event["country_code"] = a[11] 115 | } 116 | } else { 117 | target := make(map[string]interface{}) 118 | target["country_name"] = a[0] 119 | target["province_name"] = a[1] 120 | target["city_name"] = a[2] 121 | if len(a) >= 5 { 122 | target["isp"] = a[4] 123 | } 124 | if len(a) >= 10 { 125 | latitude, _ := strconv.ParseFloat(a[5], bitSize) 126 | longitude, _ := strconv.ParseFloat(a[6], bitSize) 127 | target["latitude"] = latitude 128 | target["longitude"] = longitude 129 | target["location"] = []interface{}{longitude, latitude} 130 | target["country_code"] = a[11] 131 | } 132 | event[plugin.target] = target 133 | } 134 | return event, true 135 | } 136 | -------------------------------------------------------------------------------- /filter/json.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/childe/gohangout/topology" 9 | "github.com/childe/gohangout/value_render" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | // JSONFilter will parse json string in `field` and put the result into `target` field 14 | type JSONFilter struct { 15 | field string 16 | vr value_render.ValueRender 17 | target string 18 | overwrite bool 19 | include []string 20 | exclude []string 21 | } 22 | 23 | func init() { 24 | Register("Json", newJSONFilter) 25 | } 26 | 27 | func newJSONFilter(config map[interface{}]interface{}) topology.Filter { 28 | plugin := &JSONFilter{ 29 | overwrite: true, 30 | target: "", 31 | } 32 | 33 | if field, ok := config["field"]; ok { 34 | plugin.field = field.(string) 35 | plugin.vr = value_render.GetValueRender2(plugin.field) 36 | } else { 37 | klog.Fatal("field must be set in Json filter") 38 | } 39 | 40 | if overwrite, ok := config["overwrite"]; ok { 41 | plugin.overwrite = overwrite.(bool) 42 | } 43 | 44 | if target, ok := config["target"]; ok { 45 | plugin.target = target.(string) 46 | } 47 | 48 | if include, ok := config["include"]; ok { 49 | for _, i := range include.([]interface{}) { 50 | plugin.include = append(plugin.include, i.(string)) 51 | } 52 | } 53 | if exclude, ok := config["exclude"]; ok { 54 | for _, i := range exclude.([]interface{}) { 55 | plugin.exclude = append(plugin.exclude, i.(string)) 56 | } 57 | } 58 | 59 | return plugin 60 | } 61 | 62 | // Filter will parse json string in `field` and put the result into `target` field 63 | func (plugin *JSONFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 64 | f, err := plugin.vr.Render(event) 65 | if err != nil || f == nil { 66 | return event, false 67 | } 68 | 69 | ss, ok := f.(string) 70 | if !ok { 71 | return event, false 72 | } 73 | 74 | var o interface{} = nil 75 | d := json.NewDecoder(strings.NewReader(ss)) 76 | d.UseNumber() 77 | err = d.Decode(&o) 78 | if err != nil || o == nil { 79 | return event, false 80 | } 81 | 82 | if len(plugin.include) > 0 { 83 | oo := map[string]interface{}{} 84 | if o, ok := o.(map[string]interface{}); ok { 85 | for _, k := range plugin.include { 86 | oo[k] = o[k] 87 | } 88 | } else { 89 | klog.V(5).Infof("%s field is not map type, could not get `include` fields from it", plugin.field) 90 | return event, false 91 | } 92 | o = oo 93 | } else if len(plugin.exclude) > 0 { 94 | if o, ok := o.(map[string]interface{}); ok { 95 | for _, k := range plugin.exclude { 96 | delete(o, k) 97 | } 98 | } else { 99 | klog.V(5).Infof("%s field is not map type, could not get `include` fields from it", plugin.field) 100 | return event, false 101 | } 102 | } 103 | 104 | if plugin.target == "" { 105 | if reflect.TypeOf(o).Kind() != reflect.Map { 106 | klog.V(5).Infof("%s field is not map type, `target` must be set in config file", plugin.field) 107 | return event, false 108 | } 109 | if plugin.overwrite { 110 | for k, v := range o.(map[string]interface{}) { 111 | event[k] = v 112 | } 113 | } else { 114 | for k, v := range o.(map[string]interface{}) { 115 | if _, ok := event[k]; !ok { 116 | event[k] = v 117 | } 118 | } 119 | } 120 | } else { 121 | event[plugin.target] = o 122 | } 123 | return event, true 124 | } 125 | -------------------------------------------------------------------------------- /filter/json_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "encoding/json" 5 | "reflect" 6 | "testing" 7 | ) 8 | 9 | func TestJson(t *testing.T) { 10 | type testCase struct { 11 | event map[string]interface{} 12 | config map[interface{}]interface{} 13 | want map[string]interface{} 14 | success bool 15 | } 16 | 17 | cases := []testCase{ 18 | { 19 | map[string]interface{}{ 20 | "message": `{"a":1,"b":2}`, 21 | "a": 10, 22 | }, 23 | map[interface{}]interface{}{ 24 | "field": "message", 25 | "overwrite": true, 26 | }, 27 | map[string]interface{}{ 28 | "message": `{"a":1,"b":2}`, 29 | "a": json.Number("1"), 30 | "b": json.Number("2"), 31 | }, 32 | true, 33 | }, 34 | { 35 | map[string]interface{}{ 36 | "message": `{"a":1,"b":2}`, 37 | "a": 10, 38 | }, 39 | map[interface{}]interface{}{ 40 | "field": "message", 41 | "overwrite": false, 42 | }, 43 | map[string]interface{}{ 44 | "message": `{"a":1,"b":2}`, 45 | "a": 10, 46 | "b": json.Number("2"), 47 | }, 48 | true, 49 | }, 50 | { 51 | map[string]interface{}{ 52 | "message": `{"a":1,"b":2}`, 53 | "a": 10, 54 | }, 55 | map[interface{}]interface{}{ 56 | "field": "message", 57 | "overwrite": false, 58 | "target": "c", 59 | }, 60 | map[string]interface{}{ 61 | "message": `{"a":1,"b":2}`, 62 | "a": 10, 63 | "c": map[string]interface{}{ 64 | "a": json.Number("1"), 65 | "b": json.Number("2"), 66 | }, 67 | }, 68 | true, 69 | }, 70 | { 71 | map[string]interface{}{ 72 | "message": `{"message":"hello","b":2}`, 73 | "a": 10, 74 | }, 75 | map[interface{}]interface{}{ 76 | "field": "message", 77 | "overwrite": true, 78 | }, 79 | map[string]interface{}{ 80 | "message": "hello", 81 | "a": 10, 82 | "b": json.Number("2"), 83 | }, 84 | true, 85 | }, 86 | { 87 | map[string]interface{}{ 88 | "message": `{"message":"hello","b":2}`, 89 | "a": 10, 90 | }, 91 | map[interface{}]interface{}{ 92 | "field": "$.message", 93 | "overwrite": false, 94 | }, 95 | map[string]interface{}{ 96 | "message": `{"message":"hello","b":2}`, 97 | "a": 10, 98 | "b": json.Number("2"), 99 | }, 100 | true, 101 | }, 102 | } 103 | 104 | for _, c := range cases { 105 | f := newJSONFilter(c.config) 106 | got, ok := f.Filter(c.event) 107 | if !reflect.DeepEqual(got, c.want) { 108 | t.Errorf("config: %#v event: %v: want %#v, got %#v", c.config, c.event, c.want, got) 109 | } 110 | 111 | if ok != c.success { 112 | t.Errorf("config: %#v event: %v: want %v, got %v", c.config, c.event, c.success, ok) 113 | } 114 | } 115 | } 116 | 117 | func TestIncludeExclude(t *testing.T) { 118 | type testCase struct { 119 | event map[string]interface{} 120 | config map[interface{}]interface{} 121 | want map[string]interface{} 122 | success bool 123 | } 124 | 125 | cases := []testCase{ 126 | { 127 | map[string]interface{}{ 128 | "message": `{"a":1,"b":2, "c": 3}`, 129 | }, 130 | map[interface{}]interface{}{ 131 | "field": "message", 132 | "overwrite": true, 133 | "include": []interface{}{"a", "b"}, 134 | }, 135 | map[string]interface{}{ 136 | "message": `{"a":1,"b":2, "c": 3}`, 137 | "a": json.Number("1"), 138 | "b": json.Number("2"), 139 | }, 140 | true, 141 | }, 142 | { 143 | map[string]interface{}{ 144 | "message": `{"a":1,"b":2, "c": 3}`, 145 | }, 146 | map[interface{}]interface{}{ 147 | "field": "message", 148 | "overwrite": true, 149 | "exclude": []interface{}{"a", "b"}, 150 | }, 151 | map[string]interface{}{ 152 | "message": `{"a":1,"b":2, "c": 3}`, 153 | "c": json.Number("3"), 154 | }, 155 | true, 156 | }, 157 | { 158 | map[string]interface{}{ 159 | "message": `{"a":1,"b":2, "c": 3}`, 160 | }, 161 | map[interface{}]interface{}{ 162 | "field": "message", 163 | "overwrite": true, 164 | "include": []interface{}{"a", "b"}, 165 | "exclude": []interface{}{"a", "b"}, 166 | }, 167 | map[string]interface{}{ 168 | "message": `{"a":1,"b":2, "c": 3}`, 169 | "a": json.Number("1"), 170 | "b": json.Number("2"), 171 | }, 172 | true, 173 | }, 174 | } 175 | 176 | for _, c := range cases { 177 | f := newJSONFilter(c.config) 178 | got, ok := f.Filter(c.event) 179 | if !reflect.DeepEqual(got, c.want) { 180 | t.Errorf("config: %#v event: %v: want %#v, got %#v", c.config, c.event, c.want, got) 181 | } 182 | 183 | if ok != c.success { 184 | t.Errorf("config: %#v event: %v: want %v, got %v", c.config, c.event, c.success, ok) 185 | } 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /filter/kv.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type KVFilter struct { 13 | config map[interface{}]interface{} 14 | fields map[field_setter.FieldSetter]value_render.ValueRender 15 | src value_render.ValueRender 16 | target string 17 | field_split string 18 | value_split string 19 | trim string 20 | trim_key string 21 | include_keys map[string]bool 22 | exclude_keys map[string]bool 23 | } 24 | 25 | func init() { 26 | Register("KV", newKVFilter) 27 | } 28 | 29 | func newKVFilter(config map[interface{}]interface{}) topology.Filter { 30 | plugin := &KVFilter{ 31 | config: config, 32 | fields: make(map[field_setter.FieldSetter]value_render.ValueRender), 33 | } 34 | 35 | if src, ok := config["src"]; ok { 36 | plugin.src = value_render.GetValueRender2(src.(string)) 37 | } else { 38 | klog.Fatal("src must be set in kv filter") 39 | } 40 | 41 | if target, ok := config["target"]; ok { 42 | plugin.target = target.(string) 43 | } else { 44 | plugin.target = "" 45 | } 46 | 47 | if field_split, ok := config["field_split"]; ok { 48 | plugin.field_split = field_split.(string) 49 | } else { 50 | klog.Fatal("field_split must be set in kv filter") 51 | } 52 | 53 | if value_split, ok := config["value_split"]; ok { 54 | plugin.value_split = value_split.(string) 55 | } else { 56 | klog.Fatal("value_split must be set in kv filter") 57 | } 58 | 59 | if trim, ok := config["trim"]; ok { 60 | plugin.trim = trim.(string) 61 | } else { 62 | plugin.trim = "" 63 | } 64 | 65 | if trim_key, ok := config["trim_key"]; ok { 66 | plugin.trim_key = trim_key.(string) 67 | } else { 68 | plugin.trim_key = "" 69 | } 70 | 71 | plugin.include_keys = make(map[string]bool) 72 | if include_keys, ok := config["include_keys"]; ok { 73 | for _, k := range include_keys.([]interface{}) { 74 | plugin.include_keys[k.(string)] = true 75 | } 76 | } 77 | 78 | plugin.exclude_keys = make(map[string]bool) 79 | if exclude_keys, ok := config["exclude_keys"]; ok { 80 | for _, k := range exclude_keys.([]interface{}) { 81 | plugin.exclude_keys[k.(string)] = true 82 | } 83 | } 84 | 85 | return plugin 86 | } 87 | 88 | func (p *KVFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 89 | msg, err := p.src.Render(event) 90 | if err != nil || msg == nil { 91 | return event, false 92 | } 93 | A := strings.Split(msg.(string), p.field_split) 94 | 95 | var o map[string]interface{} = event 96 | if p.target != "" { 97 | o = make(map[string]interface{}) 98 | event[p.target] = o 99 | } 100 | 101 | var success bool = true 102 | var key string 103 | for _, kv := range A { 104 | a := strings.SplitN(kv, p.value_split, 2) 105 | if len(a) != 2 { 106 | success = false 107 | continue 108 | } 109 | 110 | key = strings.Trim(a[0], p.trim_key) 111 | 112 | if _, ok := p.exclude_keys[key]; ok { 113 | continue 114 | } 115 | 116 | if _, ok := p.include_keys[key]; len(p.include_keys) == 0 || ok { 117 | o[key] = strings.Trim(a[1], p.trim) 118 | } 119 | } 120 | return event, success 121 | } 122 | -------------------------------------------------------------------------------- /filter/kv_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "testing" 4 | 5 | func TestIncludeKeys(t *testing.T) { 6 | config := make(map[interface{}]interface{}) 7 | config["field_split"] = " " 8 | config["value_split"] = "=" 9 | config["src"] = "message" 10 | config["include_keys"] = []interface{}{"a", "b", "c", "xyz"} 11 | config["exclude_keys"] = []interface{}{"c"} 12 | f := BuildFilter("KV", config) 13 | 14 | event := make(map[string]interface{}) 15 | event["message"] = "a=aaa b=bbb c=ccc xyz=\txyzxyz\t d=ddd" 16 | t.Log(event) 17 | 18 | event, ok := f.Filter(event) 19 | if !ok { 20 | t.Error("kv failed") 21 | } 22 | t.Log(event) 23 | 24 | if event["a"] != "aaa" { 25 | t.Error("kv failed") 26 | } 27 | if event["b"] != "bbb" { 28 | t.Error("kv failed") 29 | } 30 | if _, ok := event["c"]; ok { 31 | t.Error("c is excluded") 32 | } 33 | if event["xyz"] != "\txyzxyz\t" { 34 | t.Error("kv failed") 35 | } 36 | if _, ok := event["d"]; ok { 37 | t.Error("d is excluded") 38 | } 39 | } 40 | 41 | func TestKVFilter(t *testing.T) { 42 | config := make(map[interface{}]interface{}) 43 | config["field_split"] = " " 44 | config["value_split"] = "=" 45 | config["src"] = "message" 46 | f := BuildFilter("KV", config) 47 | 48 | event := make(map[string]interface{}) 49 | event["message"] = "a=aaa b=bbb c=ccc xyz=\txyzxyz\t d=ddd" 50 | t.Log(event) 51 | 52 | event, ok := f.Filter(event) 53 | if !ok { 54 | t.Error("kv failed") 55 | } 56 | t.Log(event) 57 | 58 | if event["a"] != "aaa" { 59 | t.Error("kv failed") 60 | } 61 | if event["b"] != "bbb" { 62 | t.Error("kv failed") 63 | } 64 | if event["c"] != "ccc" { 65 | t.Error("kv failed") 66 | } 67 | if event["xyz"] != "\txyzxyz\t" { 68 | t.Error("kv failed") 69 | } 70 | if event["d"] != "ddd" { 71 | t.Error("kv failed") 72 | } 73 | 74 | // trim 75 | config = make(map[interface{}]interface{}) 76 | config["field_split"] = " " 77 | config["value_split"] = "=" 78 | config["trim"] = "\t \"" 79 | config["trim_key"] = `"` 80 | config["src"] = "message" 81 | f = BuildFilter("KV", config) 82 | 83 | event = make(map[string]interface{}) 84 | event["message"] = "a=aaa b=bbb xyz=\"\txyzxyz\t\" d=ddd" 85 | t.Log(event) 86 | 87 | event, ok = f.Filter(event) 88 | if !ok { 89 | t.Error("kv failed") 90 | } 91 | t.Log(event) 92 | 93 | if event["a"] != "aaa" { 94 | t.Error("kv failed") 95 | } 96 | if event["b"] != "bbb" { 97 | t.Error("kv failed") 98 | } 99 | if event["xyz"] != "xyzxyz" { 100 | t.Error("kv failed") 101 | } 102 | if event["d"] != "ddd" { 103 | t.Error("kv failed") 104 | } 105 | } 106 | func TestKVFilterOneField(t *testing.T) { 107 | config := make(map[interface{}]interface{}) 108 | config["field_split"] = " " 109 | config["value_split"] = "=" 110 | config["src"] = "message" 111 | f := BuildFilter("KV", config) 112 | 113 | event := make(map[string]interface{}) 114 | event["message"] = "a=aaa" 115 | t.Log(event) 116 | 117 | event, ok := f.Filter(event) 118 | if !ok { 119 | t.Error("kv failed") 120 | } 121 | t.Log(event) 122 | 123 | if event["a"] != "aaa" { 124 | t.Error("kv failed") 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /filter/lowercase.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "reflect" 5 | "strings" 6 | 7 | "github.com/childe/gohangout/field_setter" 8 | "github.com/childe/gohangout/topology" 9 | "github.com/childe/gohangout/value_render" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | type LowercaseFilter struct { 14 | config map[interface{}]interface{} 15 | fields map[field_setter.FieldSetter]value_render.ValueRender 16 | } 17 | 18 | func init() { 19 | Register("Lowercase", newLowercaseFilter) 20 | } 21 | 22 | func newLowercaseFilter(config map[interface{}]interface{}) topology.Filter { 23 | plugin := &LowercaseFilter{ 24 | config: config, 25 | fields: make(map[field_setter.FieldSetter]value_render.ValueRender), 26 | } 27 | 28 | if fieldsValue, ok := config["fields"]; ok { 29 | for _, field := range fieldsValue.([]interface{}) { 30 | fieldSetter := field_setter.NewFieldSetter(field.(string)) 31 | if fieldSetter == nil { 32 | klog.Fatalf("could build field setter from %s", field.(string)) 33 | } 34 | plugin.fields[fieldSetter] = value_render.GetValueRender2(field.(string)) 35 | } 36 | } else { 37 | klog.Fatal("fields must be set in remove filter plugin") 38 | } 39 | return plugin 40 | } 41 | 42 | // 如果字段不是字符串, 返回false, 其它返回true 43 | func (plugin *LowercaseFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 44 | success := true 45 | for s, v := range plugin.fields { 46 | value, err := v.Render(event) 47 | if err != nil || value != nil { 48 | if reflect.TypeOf(value).Kind() != reflect.String { 49 | success = false 50 | continue 51 | } 52 | s.SetField(event, strings.ToLower(value.(string)), "", true) 53 | } 54 | } 55 | return event, success 56 | } 57 | -------------------------------------------------------------------------------- /filter/remove.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/childe/gohangout/field_deleter" 5 | "github.com/childe/gohangout/topology" 6 | "k8s.io/klog/v2" 7 | ) 8 | 9 | type RemoveFilter struct { 10 | config map[interface{}]interface{} 11 | fieldsDeleters []field_deleter.FieldDeleter 12 | } 13 | 14 | func init() { 15 | Register("Remove", newRemoveFilter) 16 | } 17 | 18 | func newRemoveFilter(config map[interface{}]interface{}) topology.Filter { 19 | plugin := &RemoveFilter{ 20 | config: config, 21 | fieldsDeleters: make([]field_deleter.FieldDeleter, 0), 22 | } 23 | 24 | if fieldsValue, ok := config["fields"]; ok { 25 | for _, field := range fieldsValue.([]interface{}) { 26 | plugin.fieldsDeleters = append(plugin.fieldsDeleters, field_deleter.NewFieldDeleter(field.(string))) 27 | } 28 | } else { 29 | klog.Fatal("fields must be set in remove filter plugin") 30 | } 31 | return plugin 32 | } 33 | 34 | func (plugin *RemoveFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 35 | for _, d := range plugin.fieldsDeleters { 36 | d.Delete(event) 37 | } 38 | return event, true 39 | } 40 | -------------------------------------------------------------------------------- /filter/rename.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/childe/gohangout/field_deleter" 5 | "github.com/childe/gohangout/field_setter" 6 | "github.com/childe/gohangout/topology" 7 | "github.com/childe/gohangout/value_render" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | type gsd struct { 12 | g value_render.ValueRender 13 | s field_setter.FieldSetter 14 | d field_deleter.FieldDeleter 15 | } 16 | 17 | type RenameFilter struct { 18 | config map[interface{}]interface{} 19 | fields map[string]gsd 20 | } 21 | 22 | func init() { 23 | Register("Rename", newRenameFilter) 24 | } 25 | 26 | func newRenameFilter(config map[interface{}]interface{}) topology.Filter { 27 | plugin := &RenameFilter{ 28 | config: config, 29 | fields: make(map[string]gsd), 30 | } 31 | 32 | if fieldsValue, ok := config["fields"]; ok { 33 | for src, dst := range fieldsValue.(map[interface{}]interface{}) { 34 | g := value_render.GetValueRender2(src.(string)) 35 | s := field_setter.NewFieldSetter(dst.(string)) 36 | d := field_deleter.NewFieldDeleter(src.(string)) 37 | plugin.fields[src.(string)] = gsd{g, s, d} 38 | } 39 | } else { 40 | klog.Fatal("fields must be set in rename filter plugin") 41 | } 42 | return plugin 43 | } 44 | 45 | func (plugin *RenameFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 46 | for _, _gsd := range plugin.fields { 47 | v, err := _gsd.g.Render(event) 48 | if err == nil { 49 | _gsd.s.SetField(event, v, "", true) 50 | _gsd.d.Delete(event) 51 | } 52 | } 53 | return event, true 54 | } 55 | -------------------------------------------------------------------------------- /filter/rename_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/smartystreets/goconvey/convey" 7 | ) 8 | 9 | func TestRenameFilter(t *testing.T) { 10 | type testcase struct { 11 | config map[interface{}]interface{} 12 | event map[string]interface{} 13 | expected map[string]interface{} 14 | } 15 | 16 | testcases := []testcase{ 17 | { 18 | config: map[interface{}]interface{}{ 19 | "fields": map[interface{}]interface{}{ 20 | "name1": "n1", 21 | "name2": "n2", 22 | }, 23 | }, 24 | event: map[string]interface{}{ 25 | "name1": "liu", 26 | "name2": "dehua", 27 | }, 28 | expected: map[string]interface{}{ 29 | "n1": "liu", 30 | "n2": "dehua", 31 | }, 32 | }, 33 | { 34 | config: map[interface{}]interface{}{ 35 | "fields": map[interface{}]interface{}{ 36 | "[name][last]": "[name][first]", 37 | }, 38 | }, 39 | event: map[string]interface{}{ 40 | "name": map[string]interface{}{ 41 | "last": "liu", 42 | }, 43 | }, 44 | expected: map[string]interface{}{ 45 | "name": map[string]interface{}{ 46 | "first": "liu", 47 | }, 48 | }, 49 | }, 50 | { 51 | config: map[interface{}]interface{}{ 52 | "fields": map[interface{}]interface{}{ 53 | "[name][last]": "[name][first]", 54 | }, 55 | }, 56 | event: map[string]interface{}{ 57 | "name": map[string]interface{}{ 58 | "last": nil, 59 | }, 60 | }, 61 | expected: map[string]interface{}{ 62 | "name": map[string]interface{}{ 63 | "first": nil, 64 | }, 65 | }, 66 | }, 67 | { 68 | config: map[interface{}]interface{}{ 69 | "fields": map[interface{}]interface{}{ 70 | "[name][last]": "[name][first]", 71 | }, 72 | }, 73 | event: map[string]interface{}{ 74 | "name": map[string]interface{}{ 75 | "full": "dehua liu", 76 | }, 77 | }, 78 | expected: map[string]interface{}{ 79 | "name": map[string]interface{}{ 80 | "full": "dehua liu", 81 | }, 82 | }, 83 | }, 84 | } 85 | convey.Convey("RenameFilter", t, func() { 86 | for _, tc := range testcases { 87 | f := BuildFilter("Rename", tc.config) 88 | event, ok := f.Filter(tc.event) 89 | if !ok { 90 | t.Error("RenameFilter error") 91 | } 92 | convey.So(event, convey.ShouldResemble, tc.expected) 93 | } 94 | }) 95 | } 96 | -------------------------------------------------------------------------------- /filter/replace_filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type replaceConfig struct { 13 | s field_setter.FieldSetter 14 | v value_render.ValueRender 15 | old string 16 | new string 17 | n int 18 | } 19 | 20 | type ReplaceFilter struct { 21 | config map[interface{}]interface{} 22 | fields []replaceConfig 23 | } 24 | 25 | func init() { 26 | Register("Replace", newReplaceFilter) 27 | } 28 | 29 | func newReplaceFilter(config map[interface{}]interface{}) topology.Filter { 30 | p := &ReplaceFilter{ 31 | config: config, 32 | fields: make([]replaceConfig, 0), 33 | } 34 | 35 | if fieldsI, ok := config["fields"]; ok { 36 | for fieldI, configI := range fieldsI.(map[interface{}]interface{}) { 37 | fieldSetter := field_setter.NewFieldSetter(fieldI.(string)) 38 | if fieldSetter == nil { 39 | klog.Fatalf("could build field setter from %s", fieldI.(string)) 40 | } 41 | 42 | v := value_render.GetValueRender2(fieldI.(string)) 43 | 44 | rConfig := configI.([]interface{}) 45 | if len(rConfig) == 2 { 46 | t := replaceConfig{ 47 | fieldSetter, 48 | v, 49 | rConfig[0].(string), 50 | rConfig[1].(string), 51 | -1, 52 | } 53 | p.fields = append(p.fields, t) 54 | } else if len(rConfig) == 3 { 55 | t := replaceConfig{ 56 | fieldSetter, 57 | v, 58 | rConfig[0].(string), 59 | rConfig[1].(string), 60 | rConfig[2].(int), 61 | } 62 | p.fields = append(p.fields, t) 63 | } else { 64 | klog.Fatal("invalid fields config in replace filter") 65 | } 66 | } 67 | } else { 68 | klog.Fatal("fields must be set in replace filter plugin") 69 | } 70 | return p 71 | } 72 | 73 | // 如果字段不是字符串, 返回false, 其它返回true 74 | func (p *ReplaceFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 75 | success := true 76 | for _, f := range p.fields { 77 | value, err := f.v.Render(event) 78 | if err != nil || value == nil { 79 | continue 80 | } 81 | if s, ok := value.(string); ok { 82 | new := strings.Replace(s, f.old, f.new, f.n) 83 | f.s.SetField(event, new, "", true) 84 | } else { 85 | success = false 86 | } 87 | } 88 | return event, success 89 | } 90 | -------------------------------------------------------------------------------- /filter/replace_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "testing" 4 | 5 | func TestReplaceFilter(t *testing.T) { 6 | config := make(map[interface{}]interface{}) 7 | fields := make(map[interface{}]interface{}) 8 | fields["msg"] = []interface{}{"'", `"`} 9 | config["fields"] = fields 10 | f := BuildFilter("Replace", config) 11 | 12 | event := make(map[string]interface{}) 13 | event["msg"] = `this is 'cat'` 14 | 15 | event, ok := f.Filter(event) 16 | t.Log(event) 17 | if !ok { 18 | t.Error("ReplaceFilter error") 19 | } 20 | 21 | if event["msg"] != `this is "cat"` { 22 | t.Error(event["msg"]) 23 | } 24 | 25 | config = make(map[interface{}]interface{}) 26 | fields = make(map[interface{}]interface{}) 27 | fields["name1"] = []interface{}{"wang", "Wang", 1} 28 | fields["name2"] = []interface{}{"en", "eng"} 29 | config["fields"] = fields 30 | f = BuildFilter("Replace", config) 31 | 32 | event = make(map[string]interface{}) 33 | event["name1"] = "wang wangwang" 34 | event["name2"] = "wang henhen" 35 | 36 | event, ok = f.Filter(event) 37 | t.Log(event) 38 | if !ok { 39 | t.Error("ReplaceFilter error") 40 | } 41 | 42 | if event["name1"] != "Wang wangwang" { 43 | t.Error(event["name1"]) 44 | } 45 | 46 | if event["name2"] != "wang hengheng" { 47 | t.Error(event["name2"]) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /filter/split_filter.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type SplitFilter struct { 13 | config map[interface{}]interface{} 14 | fields []field_setter.FieldSetter 15 | fieldsLength int 16 | sep string 17 | sepRender value_render.ValueRender 18 | maxSplit int 19 | trim string 20 | src value_render.ValueRender 21 | overwrite bool 22 | ignoreBlank bool 23 | dynamicSep bool 24 | } 25 | 26 | func init() { 27 | Register("Split", newSplitFilter) 28 | } 29 | 30 | func newSplitFilter(config map[interface{}]interface{}) topology.Filter { 31 | plugin := &SplitFilter{ 32 | config: config, 33 | fields: make([]field_setter.FieldSetter, 0), 34 | overwrite: true, 35 | sep: "", 36 | trim: "", 37 | ignoreBlank: true, 38 | dynamicSep: false, 39 | maxSplit: -1, 40 | } 41 | 42 | if ignoreBlank, ok := config["ignore_blank"]; ok { 43 | plugin.ignoreBlank = ignoreBlank.(bool) 44 | } 45 | 46 | if overwrite, ok := config["overwrite"]; ok { 47 | plugin.overwrite = overwrite.(bool) 48 | } 49 | 50 | if maxSplit, ok := config["maxSplit"]; ok { 51 | plugin.maxSplit = maxSplit.(int) 52 | } 53 | 54 | if src, ok := config["src"]; ok { 55 | plugin.src = value_render.GetValueRender2(src.(string)) 56 | } else { 57 | plugin.src = value_render.GetValueRender2("message") 58 | } 59 | 60 | if sep, ok := config["sep"]; ok { 61 | plugin.sep = sep.(string) 62 | } 63 | if plugin.sep == "" { 64 | klog.Fatal("sep must be set in split filter plugin") 65 | } 66 | 67 | if dynamicSep, ok := config["dynamicSep"]; ok { 68 | plugin.dynamicSep = dynamicSep.(bool) 69 | } 70 | if plugin.dynamicSep { 71 | plugin.sepRender = value_render.GetValueRender(plugin.sep) 72 | } 73 | 74 | if fieldsI, ok := config["fields"]; ok { 75 | for _, f := range fieldsI.([]interface{}) { 76 | plugin.fields = append(plugin.fields, field_setter.NewFieldSetter(f.(string))) 77 | } 78 | } else { 79 | klog.Fatal("fields must be set in split filter plugin") 80 | } 81 | plugin.fieldsLength = len(plugin.fields) 82 | 83 | if trim, ok := config["trim"]; ok { 84 | plugin.trim = trim.(string) 85 | } 86 | 87 | return plugin 88 | } 89 | 90 | func (plugin *SplitFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 91 | src, err := plugin.src.Render(event) 92 | if err != nil || src == nil { 93 | return event, false 94 | } 95 | 96 | var sep string 97 | if plugin.dynamicSep { 98 | s, err := plugin.sepRender.Render(event) 99 | if err != nil { 100 | return event, false 101 | } 102 | var ok bool 103 | if sep, ok = s.(string); !ok { 104 | return event, false 105 | } 106 | } else { 107 | sep = plugin.sep 108 | } 109 | values := strings.SplitN(src.(string), sep, plugin.maxSplit) 110 | 111 | if len(values) < plugin.fieldsLength { 112 | return event, false 113 | } 114 | 115 | for i, f := range plugin.fields { 116 | if values[i] == "" && plugin.ignoreBlank { 117 | continue 118 | } 119 | if plugin.trim == "" { 120 | event = f.SetField(event, values[i], "", plugin.overwrite) 121 | } else { 122 | event = f.SetField(event, strings.Trim(values[i], plugin.trim), "", plugin.overwrite) 123 | } 124 | } 125 | return event, true 126 | } 127 | -------------------------------------------------------------------------------- /filter/split_test.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import "testing" 4 | 5 | func TestSplitFilter1(t *testing.T) { 6 | config := make(map[interface{}]interface{}) 7 | fields := []interface{}{"loglevel", "date", "time", "message"} 8 | config["src"] = "message" 9 | config["fields"] = fields 10 | config["sep"] = " " 11 | config["maxSplit"] = 4 12 | config["trim"] = "[]" 13 | f := BuildFilter("Split", config) 14 | 15 | event := make(map[string]interface{}) 16 | event["message"] = `[INFO] [2019-03-21 23:59:59,998] messages ...` 17 | 18 | event, ok := f.Filter(event) 19 | t.Log(event) 20 | if !ok { 21 | t.Error("SplitFilter error") 22 | } 23 | 24 | if event["loglevel"] != "INFO" { 25 | t.Errorf("loglevel error: %#v", event) 26 | } 27 | 28 | if event["date"] != "2019-03-21" { 29 | t.Errorf("date error: %#v", event) 30 | } 31 | 32 | if event["time"] != "23:59:59,998" { 33 | t.Errorf("time error: %#v", event) 34 | } 35 | 36 | if event["message"] != "messages ..." { 37 | t.Errorf("message error: %#v", event) 38 | } 39 | } 40 | 41 | func TestSplitFilter2(t *testing.T) { 42 | config := make(map[interface{}]interface{}) 43 | fields := []interface{}{"loglevel", "logtime", "message"} 44 | config["src"] = "message" 45 | config["fields"] = fields 46 | config["sep"] = "] " 47 | config["maxSplit"] = 3 48 | config["trim"] = "[]" 49 | f := BuildFilter("Split", config) 50 | 51 | event := make(map[string]interface{}) 52 | event["message"] = `[INFO] [2019-03-21 23:59:59,998] messages ...` 53 | 54 | event, ok := f.Filter(event) 55 | t.Log(event) 56 | if !ok { 57 | t.Error("SplitFilter error") 58 | } 59 | 60 | if event["loglevel"] != "INFO" { 61 | t.Errorf("loglevel error: %#v", event) 62 | } 63 | 64 | if event["logtime"] != "2019-03-21 23:59:59,998" { 65 | t.Errorf("logtime error: %#v", event) 66 | } 67 | 68 | if event["message"] != "messages ..." { 69 | t.Errorf("message error: %#v", event) 70 | } 71 | } 72 | 73 | // dynamic sep 74 | func TestSplitFilter3(t *testing.T) { 75 | config := make(map[interface{}]interface{}) 76 | fields := []interface{}{"loglevel", "date", "time", "message"} 77 | config["src"] = "message" 78 | config["fields"] = fields 79 | config["sep"] = "[sep]" 80 | config["dynamicSep"] = true 81 | config["maxSplit"] = 4 82 | config["trim"] = "[]" 83 | f := BuildFilter("Split", config) 84 | 85 | event := make(map[string]interface{}) 86 | event["message"] = `[INFO] [2019-03-21 23:59:59,998] messages ...` 87 | event["sep"] = " " 88 | 89 | event, ok := f.Filter(event) 90 | t.Log(event) 91 | if !ok { 92 | t.Error("SplitFilter error") 93 | } 94 | 95 | if event["loglevel"] != "INFO" { 96 | t.Errorf("loglevel error: %#v", event) 97 | } 98 | 99 | if event["date"] != "2019-03-21" { 100 | t.Errorf("logtime error: %#v", event) 101 | } 102 | 103 | if event["time"] != "23:59:59,998" { 104 | t.Errorf("logtime error: %#v", event) 105 | } 106 | 107 | if event["message"] != "messages ..." { 108 | t.Errorf("message error: %#v", event) 109 | } 110 | } 111 | 112 | // length of fields do not match length of splited 113 | func TestSplitFilter4(t *testing.T) { 114 | config := make(map[interface{}]interface{}) 115 | fields := []interface{}{"loglevel", "date", "time"} 116 | config["src"] = "message" 117 | config["fields"] = fields 118 | config["sep"] = "[sep]" 119 | config["dynamicSep"] = true 120 | config["maxSplit"] = 4 121 | config["trim"] = "[]" 122 | f := BuildFilter("Split", config) 123 | 124 | event := make(map[string]interface{}) 125 | event["message"] = `[INFO] [2019-03-21 23:59:59,998] messages ...` 126 | event["sep"] = " " 127 | 128 | event, ok := f.Filter(event) 129 | t.Log(event) 130 | if !ok { 131 | t.Error("SplitFilter error") 132 | } 133 | 134 | if event["loglevel"] != "INFO" { 135 | t.Errorf("loglevel error: %#v", event) 136 | } 137 | 138 | if event["date"] != "2019-03-21" { 139 | t.Errorf("logtime error: %#v", event) 140 | } 141 | 142 | if event["time"] != "23:59:59,998" { 143 | t.Errorf("logtime error: %#v", event) 144 | } 145 | 146 | if event["message"] != `[INFO] [2019-03-21 23:59:59,998] messages ...` { 147 | t.Errorf("message error: %#v", event) 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /filter/translate.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "os" 7 | "strings" 8 | "time" 9 | 10 | yaml "gopkg.in/yaml.v2" 11 | 12 | "github.com/childe/gohangout/topology" 13 | "github.com/childe/gohangout/value_render" 14 | "k8s.io/klog/v2" 15 | ) 16 | 17 | type TranslateFilter struct { 18 | config map[interface{}]interface{} 19 | refreshInterval int 20 | source string 21 | target string 22 | sourceVR value_render.ValueRender 23 | dictionaryPath string 24 | 25 | // TODO put code to utils 26 | dict map[interface{}]interface{} 27 | } 28 | 29 | func (plugin *TranslateFilter) parseDict() error { 30 | var ( 31 | buffer []byte 32 | err error 33 | ) 34 | if strings.HasPrefix(plugin.dictionaryPath, "http://") || strings.HasPrefix(plugin.dictionaryPath, "https://") { 35 | resp, err := http.Get(plugin.dictionaryPath) 36 | if err != nil { 37 | return err 38 | } 39 | defer resp.Body.Close() 40 | buffer, err = ioutil.ReadAll(resp.Body) 41 | if err != nil { 42 | return err 43 | } 44 | } else { 45 | configFile, err := os.Open(plugin.dictionaryPath) 46 | if err != nil { 47 | return err 48 | } 49 | fi, _ := configFile.Stat() 50 | 51 | buffer = make([]byte, fi.Size()) 52 | _, err = configFile.Read(buffer) 53 | if err != nil { 54 | return err 55 | } 56 | } 57 | 58 | dict := make(map[interface{}]interface{}) 59 | err = yaml.Unmarshal(buffer, &dict) 60 | if err != nil { 61 | return err 62 | } 63 | plugin.dict = dict 64 | return nil 65 | } 66 | 67 | func init() { 68 | Register("Translate", newTranslateFilter) 69 | } 70 | 71 | func newTranslateFilter(config map[interface{}]interface{}) topology.Filter { 72 | plugin := &TranslateFilter{ 73 | config: config, 74 | } 75 | 76 | if source, ok := config["source"]; ok { 77 | plugin.source = source.(string) 78 | } else { 79 | klog.Fatal("source must be set in translate filter plugin") 80 | } 81 | plugin.sourceVR = value_render.GetValueRender2(plugin.source) 82 | 83 | if target, ok := config["target"]; ok { 84 | plugin.target = target.(string) 85 | } else { 86 | klog.Fatal("target must be set in translate filter plugin") 87 | } 88 | 89 | if dictionaryPath, ok := config["dictionary_path"]; ok { 90 | plugin.dictionaryPath = dictionaryPath.(string) 91 | } else { 92 | klog.Fatal("dictionary_path must be set in translate filter plugin") 93 | } 94 | 95 | if refreshInterval, ok := config["refresh_interval"]; ok { 96 | plugin.refreshInterval = refreshInterval.(int) 97 | } else { 98 | klog.Fatal("refresh_interval must be set in translate filter plugin") 99 | } 100 | 101 | err := plugin.parseDict() 102 | if err != nil { 103 | klog.Fatalf("could not parse %s:%s", plugin.dictionaryPath, err) 104 | } 105 | 106 | ticker := time.NewTicker(time.Second * time.Duration(plugin.refreshInterval)) 107 | go func() { 108 | for range ticker.C { 109 | err := plugin.parseDict() 110 | if err != nil { 111 | klog.Errorf("could not parse %s:%s", plugin.dictionaryPath, err) 112 | } 113 | } 114 | }() 115 | 116 | return plugin 117 | } 118 | 119 | func (plugin *TranslateFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 120 | o, err := plugin.sourceVR.Render(event) 121 | if err != nil || o == nil { 122 | return event, false 123 | } 124 | if targetValue, ok := plugin.dict[o]; ok { 125 | event[plugin.target] = targetValue 126 | return event, true 127 | } 128 | return event, false 129 | } 130 | -------------------------------------------------------------------------------- /filter/uppercase.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type UppercaseFilter struct { 13 | config map[interface{}]interface{} 14 | fields map[field_setter.FieldSetter]value_render.ValueRender 15 | } 16 | 17 | func init() { 18 | Register("Uppercase", newUppercaseFilter) 19 | } 20 | 21 | func newUppercaseFilter(config map[interface{}]interface{}) topology.Filter { 22 | plugin := &UppercaseFilter{ 23 | config: config, 24 | fields: make(map[field_setter.FieldSetter]value_render.ValueRender), 25 | } 26 | 27 | if fieldsValue, ok := config["fields"]; ok { 28 | for _, field := range fieldsValue.([]interface{}) { 29 | fieldSetter := field_setter.NewFieldSetter(field.(string)) 30 | if fieldSetter == nil { 31 | klog.Fatalf("could build field setter from %s", field.(string)) 32 | } 33 | plugin.fields[fieldSetter] = value_render.GetValueRender2(field.(string)) 34 | } 35 | } else { 36 | klog.Fatal("fields must be set in remove filter plugin") 37 | } 38 | return plugin 39 | } 40 | 41 | // 如果字段不是字符串, 返回false, 其它返回true 42 | func (plugin *UppercaseFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 43 | success := true 44 | for s, v := range plugin.fields { 45 | value, err := v.Render(event) 46 | if err != nil || value == nil { 47 | success = false 48 | continue 49 | } 50 | if t, ok := value.(string); !ok { 51 | success = false 52 | continue 53 | } else { 54 | s.SetField(event, strings.ToUpper(t), "", true) 55 | } 56 | } 57 | return event, success 58 | } 59 | -------------------------------------------------------------------------------- /filter/url_decode.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "net/url" 5 | 6 | "github.com/childe/gohangout/field_setter" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type URLDecodeFilter struct { 13 | config map[interface{}]interface{} 14 | fields map[field_setter.FieldSetter]value_render.ValueRender 15 | } 16 | 17 | func init() { 18 | Register("URLDecode", newURLDecodeFilter) 19 | } 20 | 21 | func newURLDecodeFilter(config map[interface{}]interface{}) topology.Filter { 22 | plugin := &URLDecodeFilter{ 23 | config: config, 24 | fields: make(map[field_setter.FieldSetter]value_render.ValueRender), 25 | } 26 | 27 | if fieldsValue, ok := config["fields"]; ok { 28 | for _, field := range fieldsValue.([]interface{}) { 29 | fieldSetter := field_setter.NewFieldSetter(field.(string)) 30 | if fieldSetter == nil { 31 | klog.Fatalf("could build field setter from %s", field.(string)) 32 | } 33 | plugin.fields[fieldSetter] = value_render.GetValueRender2(field.(string)) 34 | } 35 | } else { 36 | klog.Fatal("fields must be set in URLDecode filter plugin") 37 | } 38 | return plugin 39 | } 40 | 41 | // 如果字段不是字符串, 返回false, 其它返回true 42 | func (plugin *URLDecodeFilter) Filter(event map[string]interface{}) (map[string]interface{}, bool) { 43 | success := true 44 | for s, v := range plugin.fields { 45 | value, err := v.Render(event) 46 | if err != nil || value == nil { 47 | success = false 48 | continue 49 | } 50 | if t, ok := value.(string); !ok { 51 | success = false 52 | continue 53 | } else { 54 | rst, err := url.QueryUnescape(t) 55 | if err != nil { 56 | success = false 57 | continue 58 | } 59 | s.SetField(event, rst, "", true) 60 | } 61 | } 62 | return event, success 63 | } 64 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/childe/gohangout 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/ClickHouse/clickhouse-go v1.5.4 7 | github.com/Masterminds/sprig/v3 v3.2.2 8 | github.com/bytedance/mockey v1.2.14 9 | github.com/childe/cast v1.5.4 10 | github.com/childe/healer v0.6.22 11 | github.com/fsnotify/fsnotify v1.5.1 12 | github.com/ipipdotnet/datx-go v0.0.0-20181123035258-af996d4701a0 13 | github.com/ipipdotnet/ipdb-go v1.3.1 14 | github.com/json-iterator/go v1.1.12 15 | github.com/magiconair/properties v1.8.6 16 | github.com/mitchellh/mapstructure v1.5.0 17 | github.com/oliveagle/jsonpath v0.0.0-20180606110733-2e52cf6e6852 18 | github.com/prometheus/client_golang v1.12.1 19 | github.com/relvacode/iso8601 v1.6.0 20 | github.com/smartystreets/goconvey v1.7.2 21 | github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 22 | go.uber.org/automaxprocs v1.6.0 23 | gopkg.in/yaml.v2 v2.4.0 24 | k8s.io/klog/v2 v2.120.1 25 | ) 26 | 27 | require ( 28 | github.com/Masterminds/goutils v1.1.1 // indirect 29 | github.com/Masterminds/semver/v3 v3.1.1 // indirect 30 | github.com/aviddiviner/go-murmur v0.0.0-20150519214947-b9740d71e571 // indirect 31 | github.com/beorn7/perks v1.0.1 // indirect 32 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 33 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 // indirect 34 | github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect 35 | github.com/go-logr/logr v1.4.1 // indirect 36 | github.com/golang/protobuf v1.5.2 // indirect 37 | github.com/golang/snappy v0.0.2 // indirect 38 | github.com/google/uuid v1.1.1 // indirect 39 | github.com/gopherjs/gopherjs v1.12.80 // indirect 40 | github.com/huandu/xstrings v1.3.1 // indirect 41 | github.com/imdario/mergo v0.3.11 // indirect 42 | github.com/jtolds/gls v4.20.0+incompatible // indirect 43 | github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect 44 | github.com/mitchellh/copystructure v1.0.0 // indirect 45 | github.com/mitchellh/reflectwalk v1.0.0 // indirect 46 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 47 | github.com/modern-go/reflect2 v1.0.2 // indirect 48 | github.com/pierrec/lz4 v2.6.1+incompatible // indirect 49 | github.com/pierrec/lz4/v4 v4.1.22 // indirect 50 | github.com/prometheus/client_model v0.2.0 // indirect 51 | github.com/prometheus/common v0.32.1 // indirect 52 | github.com/prometheus/procfs v0.7.3 // indirect 53 | github.com/shopspring/decimal v1.2.0 // indirect 54 | github.com/smartystreets/assertions v1.2.0 // indirect 55 | github.com/spf13/cast v1.3.1 // indirect 56 | golang.org/x/arch v0.11.0 // indirect 57 | golang.org/x/crypto v0.35.0 // indirect 58 | golang.org/x/sys v0.30.0 // indirect 59 | google.golang.org/protobuf v1.33.0 // indirect 60 | ) 61 | -------------------------------------------------------------------------------- /gohangout.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "flag" 6 | "fmt" 7 | "net/http" 8 | _ "net/http/pprof" 9 | "os" 10 | "runtime" 11 | "runtime/pprof" 12 | "sync" 13 | _ "time/tzdata" 14 | 15 | _ "go.uber.org/automaxprocs" 16 | 17 | "github.com/childe/gohangout/input" 18 | "github.com/childe/gohangout/internal/config" 19 | "github.com/childe/gohangout/internal/signal" 20 | "github.com/childe/gohangout/topology" 21 | "github.com/prometheus/client_golang/prometheus/promhttp" 22 | "k8s.io/klog/v2" 23 | ) 24 | 25 | var ( 26 | version string 27 | buildTime string 28 | ) 29 | 30 | var options = &struct { 31 | config string 32 | autoReload bool // 配置文件更新自动重启 33 | pprof bool 34 | pprofAddr string 35 | cpuprofile string 36 | memprofile string 37 | version bool 38 | 39 | prometheus string 40 | 41 | exitWhenNil bool 42 | }{} 43 | 44 | var ( 45 | worker = flag.Int("worker", 1, "worker thread count") 46 | ) 47 | 48 | type gohangoutInputs []*input.InputBox 49 | 50 | var inputs gohangoutInputs 51 | 52 | var ( 53 | ctx context.Context 54 | cancel context.CancelFunc 55 | ) 56 | 57 | // start all workers in all inputboxes, and wait until stop is called (stop will shutdown all inputboxes) 58 | func (inputs gohangoutInputs) start() { 59 | boxes := ([]*input.InputBox)(inputs) 60 | var wg sync.WaitGroup 61 | wg.Add(len(boxes)) 62 | 63 | for i := range boxes { 64 | go func(i int) { 65 | defer wg.Done() 66 | boxes[i].Beat(*worker) 67 | }(i) 68 | } 69 | 70 | wg.Wait() 71 | } 72 | 73 | func (inputs gohangoutInputs) stop() { 74 | boxes := ([]*input.InputBox)(inputs) 75 | for _, box := range boxes { 76 | box.Shutdown() 77 | } 78 | } 79 | 80 | func buildPluginLink(config map[string]interface{}) (boxes []*input.InputBox, err error) { 81 | boxes = make([]*input.InputBox, 0) 82 | 83 | for inputIdx, inputI := range config["inputs"].([]interface{}) { 84 | var inputPlugin topology.Input 85 | 86 | i := inputI.(map[interface{}]interface{}) 87 | klog.Infof("input[%d] %v", inputIdx+1, i) 88 | 89 | // len(i) is 1 90 | for inputTypeI, inputConfigI := range i { 91 | inputType := inputTypeI.(string) 92 | inputConfig := inputConfigI.(map[interface{}]interface{}) 93 | 94 | inputPlugin = input.GetInput(inputType, inputConfig) 95 | if inputPlugin == nil { 96 | err = fmt.Errorf("invalid input plugin") 97 | return 98 | } 99 | 100 | box := input.NewInputBox(inputPlugin, inputConfig, config, exit) 101 | if box == nil { 102 | err = fmt.Errorf("new input box fail") 103 | return 104 | } 105 | box.SetShutdownWhenNil(options.exitWhenNil) 106 | boxes = append(boxes, box) 107 | } 108 | } 109 | 110 | return 111 | } 112 | 113 | // reload config file. stop inputs and start new inputs 114 | func reload() { 115 | gohangoutConfig, err := config.ParseConfig(options.config) 116 | if err != nil { 117 | klog.Errorf("could not parse config, ignore reload: %v", err) 118 | return 119 | } 120 | klog.Info("stop old inputs") 121 | inputs.stop() 122 | 123 | boxes, err := buildPluginLink(gohangoutConfig) 124 | if err != nil { 125 | klog.Errorf("build plugin link error, ignore reload: %v", err) 126 | return 127 | } 128 | inputs = gohangoutInputs(boxes) 129 | klog.Info("start new inputs") 130 | go inputs.start() 131 | } 132 | 133 | func _main() { 134 | gohangoutConfig, err := config.ParseConfig(options.config) 135 | if err != nil { 136 | klog.Fatalf("could not parse config: %v", err) 137 | } 138 | 139 | boxes, err := buildPluginLink(gohangoutConfig) 140 | if err != nil { 141 | klog.Fatalf("build plugin link error: %v", err) 142 | } 143 | ctx, cancel = context.WithCancel(context.Background()) 144 | defer cancel() 145 | 146 | inputs = gohangoutInputs(boxes) 147 | go inputs.start() 148 | 149 | if options.autoReload { 150 | if err := config.WatchConfig(options.config, reload); err != nil { 151 | klog.Fatalf("watch config fail: %s", err) 152 | } 153 | } 154 | 155 | go signal.ListenSignal(exit, reload) 156 | 157 | <-ctx.Done() 158 | inputs.stop() 159 | } 160 | 161 | func main() { 162 | flag.StringVar(&options.config, "config", options.config, "path to configuration file or directory") 163 | flag.BoolVar(&options.autoReload, "reload", options.autoReload, "if auto reload while config file changed") 164 | 165 | flag.BoolVar(&options.pprof, "pprof", false, "pprof or not") 166 | flag.StringVar(&options.pprofAddr, "pprof-address", "127.0.0.1:8899", "default: 127.0.0.1:8899") 167 | flag.StringVar(&options.cpuprofile, "cpuprofile", "", "write cpu profile to `file`") 168 | flag.StringVar(&options.memprofile, "memprofile", "", "write mem profile to `file`") 169 | 170 | flag.BoolVar(&options.version, "version", false, "print version and exit") 171 | 172 | flag.StringVar(&options.prometheus, "prometheus", "", "address to expose prometheus metrics") 173 | 174 | flag.BoolVar(&options.exitWhenNil, "exit-when-nil", false, "triger gohangout to exit when receive a nil event") 175 | 176 | klog.InitFlags(nil) 177 | flag.Parse() 178 | 179 | if options.version { 180 | fmt.Printf("gohangout: %s compiled at %s with %v on %v/%v\n", version, buildTime, runtime.Version(), runtime.GOOS, runtime.GOARCH) 181 | return 182 | } 183 | 184 | if *worker <= 0 { 185 | klog.Fatalf("worker must be greater than 0") 186 | } 187 | 188 | klog.Infof("gohangout version: %s", version) 189 | defer klog.Flush() 190 | 191 | if options.prometheus != "" { 192 | go func() { 193 | http.Handle("/metrics", promhttp.Handler()) 194 | http.ListenAndServe(options.prometheus, nil) 195 | }() 196 | } 197 | 198 | if options.pprof { 199 | go func() { 200 | http.ListenAndServe(options.pprofAddr, nil) 201 | }() 202 | } 203 | if options.cpuprofile != "" { 204 | f, err := os.Create(options.cpuprofile) 205 | if err != nil { 206 | klog.Fatalf("could not create CPU profile: %s", err) 207 | } 208 | if err := pprof.StartCPUProfile(f); err != nil { 209 | klog.Fatalf("could not start CPU profile: %s", err) 210 | } 211 | defer pprof.StopCPUProfile() 212 | } 213 | 214 | if options.memprofile != "" { 215 | defer func() { 216 | f, err := os.Create(options.memprofile) 217 | if err != nil { 218 | klog.Fatalf("could not create memory profile: %s", err) 219 | } 220 | defer f.Close() 221 | runtime.GC() // get up-to-date statistics 222 | if err := pprof.WriteHeapProfile(f); err != nil { 223 | klog.Fatalf("could not write memory profile: %s", err) 224 | } 225 | }() 226 | } 227 | 228 | _main() 229 | 230 | } 231 | 232 | func exit() { 233 | cancel() 234 | } 235 | -------------------------------------------------------------------------------- /input/input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "fmt" 5 | "plugin" 6 | 7 | "github.com/childe/gohangout/topology" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | type BuildInputFunc func(map[interface{}]interface{}) topology.Input 12 | 13 | var registeredInput map[string]BuildInputFunc = make(map[string]BuildInputFunc) 14 | 15 | // Register is used by input plugins to register themselves 16 | func Register(inputType string, bf BuildInputFunc) { 17 | if _, ok := registeredInput[inputType]; ok { 18 | klog.Errorf("%s has been registered, ignore %T", inputType, bf) 19 | return 20 | } 21 | registeredInput[inputType] = bf 22 | } 23 | 24 | // GetInput return topoloty.Input from builtin plugins or from a 3rd party plugin 25 | func GetInput(inputType string, config map[interface{}]interface{}) topology.Input { 26 | if v, ok := registeredInput[inputType]; ok { 27 | return v(config) 28 | } 29 | klog.Infof("could not load %s input plugin, try third party plugin", inputType) 30 | 31 | pluginPath := inputType 32 | output, err := getInputFromPlugin(pluginPath, config) 33 | if err != nil { 34 | klog.Errorf("could not load %s: %v", pluginPath, err) 35 | return nil 36 | } 37 | return output 38 | } 39 | 40 | func getInputFromPlugin(pluginPath string, config map[interface{}]interface{}) (topology.Input, error) { 41 | p, err := plugin.Open(pluginPath) 42 | if err != nil { 43 | return nil, fmt.Errorf("could not open %s: %v", pluginPath, err) 44 | } 45 | newFunc, err := p.Lookup("New") 46 | if err != nil { 47 | return nil, fmt.Errorf("could not find `New` function in %s: %s", pluginPath, err) 48 | } 49 | f, ok := newFunc.(func(map[interface{}]interface{}) interface{}) 50 | if !ok { 51 | return nil, fmt.Errorf("`New` func in %s format error", pluginPath) 52 | } 53 | rst := f(config) 54 | input, ok := rst.(topology.Input) 55 | if !ok { 56 | return nil, fmt.Errorf("`New` func in %s dose not return Input Interface", pluginPath) 57 | } 58 | return input, nil 59 | } 60 | -------------------------------------------------------------------------------- /input/input_box.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "reflect" 5 | "sync" 6 | 7 | "github.com/childe/gohangout/field_setter" 8 | "github.com/childe/gohangout/filter" 9 | "github.com/childe/gohangout/output" 10 | "github.com/childe/gohangout/topology" 11 | "github.com/childe/gohangout/value_render" 12 | "github.com/prometheus/client_golang/prometheus" 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | type InputBox struct { 17 | config map[string]interface{} // whole config 18 | input topology.Input 19 | outputsInAllWorker [][]*topology.OutputBox 20 | stop bool 21 | once sync.Once 22 | shutdownChan chan bool 23 | 24 | promCounter prometheus.Counter 25 | 26 | shutdownWhenNil bool 27 | exit func() 28 | 29 | addFields map[field_setter.FieldSetter]value_render.ValueRender 30 | } 31 | 32 | // SetShutdownWhenNil is used for benchmark. 33 | // Gohangout main thread would exit when one input box receive a nil message, such as Ctrl-D in Stdin input 34 | func (box *InputBox) SetShutdownWhenNil(shutdownWhenNil bool) { 35 | box.shutdownWhenNil = shutdownWhenNil 36 | } 37 | 38 | func NewInputBox(input topology.Input, inputConfig map[interface{}]interface{}, config map[string]interface{}, exit func()) *InputBox { 39 | b := &InputBox{ 40 | input: input, 41 | config: config, 42 | stop: false, 43 | shutdownChan: make(chan bool, 1), 44 | 45 | promCounter: topology.GetPromCounter(inputConfig), 46 | 47 | exit: exit, 48 | } 49 | if add_fields, ok := inputConfig["add_fields"]; ok { 50 | b.addFields = make(map[field_setter.FieldSetter]value_render.ValueRender) 51 | for k, v := range add_fields.(map[interface{}]interface{}) { 52 | fieldSetter := field_setter.NewFieldSetter(k.(string)) 53 | if fieldSetter == nil { 54 | klog.Errorf("could build field setter from %s", k.(string)) 55 | return nil 56 | } 57 | b.addFields[fieldSetter] = value_render.GetValueRender(v.(string)) 58 | } 59 | } else { 60 | b.addFields = nil 61 | } 62 | return b 63 | } 64 | 65 | func (box *InputBox) beat(workerIdx int) { 66 | var firstNode *topology.ProcessorNode = box.buildTopology(workerIdx) 67 | 68 | var ( 69 | event map[string]interface{} 70 | ) 71 | 72 | for !box.stop { 73 | event = box.input.ReadOneEvent() 74 | if box.promCounter != nil { 75 | box.promCounter.Inc() 76 | } 77 | if event == nil { 78 | klog.V(5).Info("received nil message.") 79 | if box.stop { 80 | break 81 | } 82 | if box.shutdownWhenNil { 83 | klog.Info("received nil message. shutdown...") 84 | box.exit() 85 | break 86 | } else { 87 | continue 88 | } 89 | } 90 | for fs, r := range box.addFields { 91 | v, _ := r.Render(event) 92 | event = fs.SetField(event, v, "", false) 93 | } 94 | firstNode.Process(event) 95 | } 96 | } 97 | 98 | func (box *InputBox) buildTopology(workerIdx int) *topology.ProcessorNode { 99 | outputs := topology.BuildOutputs(box.config, output.BuildOutput) 100 | box.outputsInAllWorker[workerIdx] = outputs 101 | 102 | var outputProcessor topology.Processor 103 | if len(outputs) == 1 { 104 | outputProcessor = outputs[0] 105 | } else { 106 | outputProcessor = (topology.OutputsProcessor)(outputs) 107 | } 108 | 109 | filterBoxes := topology.BuildFilterBoxes(box.config, filter.BuildFilter) 110 | 111 | var firstNode *topology.ProcessorNode 112 | for _, b := range filterBoxes { 113 | firstNode = topology.AppendProcessorsToLink(firstNode, b) 114 | } 115 | firstNode = topology.AppendProcessorsToLink(firstNode, outputProcessor) 116 | 117 | // Set BelongTo 118 | var node *topology.ProcessorNode 119 | node = firstNode 120 | for _, b := range filterBoxes { 121 | node = node.Next 122 | v := reflect.ValueOf(b.Filter) 123 | f := v.MethodByName("SetBelongTo") 124 | if f.IsValid() { 125 | f.Call([]reflect.Value{reflect.ValueOf(node)}) 126 | } 127 | } 128 | 129 | return firstNode 130 | } 131 | 132 | // Beat starts the processors and wait until shutdown 133 | func (box *InputBox) Beat(worker int) { 134 | box.outputsInAllWorker = make([][]*topology.OutputBox, worker) 135 | for i := 0; i < worker; i++ { 136 | go box.beat(i) 137 | } 138 | 139 | <-box.shutdownChan 140 | } 141 | 142 | func (box *InputBox) shutdown() { 143 | box.once.Do(func() { 144 | 145 | klog.Infof("try to shutdown input %T", box.input) 146 | box.input.Shutdown() 147 | 148 | for i, outputs := range box.outputsInAllWorker { 149 | for _, o := range outputs { 150 | klog.Infof("try to shutdown output %T in worker %d", o, i) 151 | o.Output.Shutdown() 152 | } 153 | } 154 | }) 155 | 156 | box.shutdownChan <- true 157 | } 158 | 159 | // Shutdown shutdowns the inputs and outputs 160 | func (box *InputBox) Shutdown() { 161 | box.stop = true 162 | box.shutdown() 163 | } 164 | -------------------------------------------------------------------------------- /input/kafka_input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/childe/gohangout/codec" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/healer" 9 | jsoniter "github.com/json-iterator/go" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | type KafkaInput struct { 14 | config map[interface{}]interface{} 15 | decorateEvents bool 16 | 17 | messages chan *healer.FullMessage 18 | 19 | decoder codec.Decoder 20 | 21 | groupConsumers []*healer.GroupConsumer 22 | consumers []*healer.Consumer 23 | } 24 | 25 | func init() { 26 | Register("Kafka", newKafkaInput) 27 | } 28 | 29 | func newKafkaInput(config map[interface{}]interface{}) topology.Input { 30 | var ( 31 | codertype string = "plain" 32 | decorateEvents = false 33 | topics map[interface{}]interface{} 34 | assign map[string][]int 35 | ) 36 | 37 | consumer_settings := make(map[string]interface{}) 38 | if v, ok := config["consumer_settings"]; !ok { 39 | klog.Fatal("kafka input must have consumer_settings") 40 | } else { 41 | // official json marshal: unsupported type: map[interface {}]interface {} 42 | json := jsoniter.ConfigCompatibleWithStandardLibrary 43 | if b, err := json.Marshal(v); err != nil { 44 | klog.Fatalf("marshal consumer settings error: %v", err) 45 | } else { 46 | json.Unmarshal(b, &consumer_settings) 47 | } 48 | } 49 | if v, ok := config["topic"]; ok { 50 | topics = v.(map[interface{}]interface{}) 51 | } else { 52 | topics = nil 53 | } 54 | if v, ok := config["assign"]; ok { 55 | assign = make(map[string][]int) 56 | for topicName, partitions := range v.(map[interface{}]interface{}) { 57 | assign[topicName.(string)] = make([]int, len(partitions.([]interface{}))) 58 | for i, p := range partitions.([]interface{}) { 59 | assign[topicName.(string)][i] = p.(int) 60 | } 61 | } 62 | } else { 63 | assign = nil 64 | } 65 | 66 | if topics == nil && assign == nil { 67 | klog.Fatal("either topic or assign should be set") 68 | } 69 | if topics != nil && assign != nil { 70 | klog.Fatal("topic and assign can not be both set") 71 | } 72 | 73 | if codecV, ok := config["codec"]; ok { 74 | codertype = codecV.(string) 75 | } 76 | 77 | if decorateEventsV, ok := config["decorate_events"]; ok { 78 | decorateEvents = decorateEventsV.(bool) 79 | } 80 | 81 | messagesLength := 10 82 | if v, ok := config["messages_queue_length"]; ok { 83 | messagesLength = v.(int) 84 | } 85 | 86 | kafkaInput := &KafkaInput{ 87 | config: config, 88 | decorateEvents: decorateEvents, 89 | messages: make(chan *healer.FullMessage, messagesLength), 90 | 91 | decoder: codec.NewDecoder(codertype), 92 | } 93 | 94 | // GroupConsumer 95 | if topics != nil { 96 | for topic, threadCount := range topics { 97 | for i := 0; i < threadCount.(int); i++ { 98 | c, err := healer.NewGroupConsumer(topic.(string), consumer_settings) 99 | if err != nil { 100 | klog.Fatalf("could not create kafka GroupConsumer: %s", err) 101 | } 102 | kafkaInput.groupConsumers = append(kafkaInput.groupConsumers, c) 103 | 104 | go func() { 105 | _, err = c.Consume(kafkaInput.messages) 106 | if err != nil { 107 | klog.Fatalf("try to consumer error: %s", err) 108 | } 109 | }() 110 | } 111 | } 112 | } else { 113 | c, err := healer.NewConsumer(consumer_settings) 114 | if err != nil { 115 | klog.Fatalf("could not create kafka Consumer: %s", err) 116 | } 117 | kafkaInput.consumers = append(kafkaInput.consumers, c) 118 | 119 | c.Assign(assign) 120 | 121 | go func() { 122 | _, err = c.Consume(kafkaInput.messages) 123 | if err != nil { 124 | klog.Fatalf("try to consume error: %s", err) 125 | } 126 | }() 127 | } 128 | 129 | return kafkaInput 130 | } 131 | 132 | // ReadOneEvent implement method in topology.Input. 133 | // gohangout call this method to get one event and pass it to filter or output 134 | func (p *KafkaInput) ReadOneEvent() map[string]interface{} { 135 | message, more := <-p.messages 136 | if !more { 137 | return nil 138 | } 139 | 140 | if message.Error != nil { 141 | klog.Error("kafka message carries error: ", message.Error) 142 | return nil 143 | } 144 | event := p.decoder.Decode(message.Message.Value) 145 | if p.decorateEvents { 146 | kafkaMeta := make(map[string]interface{}) 147 | kafkaMeta["topic"] = message.TopicName 148 | kafkaMeta["partition"] = message.PartitionID 149 | kafkaMeta["offset"] = message.Message.Offset 150 | event["@metadata"] = map[string]interface{}{"kafka": kafkaMeta} 151 | } 152 | return event 153 | } 154 | 155 | // Shutdown implement method in topology.Input. It closes all consumers 156 | func (p *KafkaInput) Shutdown() { 157 | if len(p.groupConsumers) > 0 { 158 | for _, c := range p.groupConsumers { 159 | c.AwaitClose(30 * time.Second) 160 | } 161 | } 162 | if len(p.consumers) > 0 { 163 | for _, c := range p.consumers { 164 | c.AwaitClose(30 * time.Second) 165 | } 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /input/random_input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "math/rand" 5 | "strconv" 6 | 7 | "github.com/childe/gohangout/codec" 8 | "github.com/childe/gohangout/topology" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type RandomInput struct { 13 | config map[interface{}]interface{} 14 | decoder codec.Decoder 15 | 16 | from int 17 | to int 18 | 19 | maxMessages int 20 | count int 21 | } 22 | 23 | func init() { 24 | Register("Random", newRandomInput) 25 | } 26 | 27 | func newRandomInput(config map[interface{}]interface{}) topology.Input { 28 | var codertype string = "plain" 29 | 30 | p := &RandomInput{ 31 | config: config, 32 | decoder: codec.NewDecoder(codertype), 33 | count: 0, 34 | maxMessages: -1, 35 | } 36 | 37 | if v, ok := config["from"]; ok { 38 | p.from = v.(int) 39 | } else { 40 | klog.Fatal("from must be configured in Random Input") 41 | } 42 | 43 | if v, ok := config["to"]; ok { 44 | p.to = v.(int) 45 | } else { 46 | klog.Fatal("to must be configured in Random Input") 47 | } 48 | 49 | if v, ok := config["max_messages"]; ok { 50 | p.maxMessages = v.(int) 51 | } 52 | 53 | return p 54 | } 55 | 56 | func (p *RandomInput) ReadOneEvent() map[string]interface{} { 57 | if p.maxMessages != -1 && p.count >= p.maxMessages { 58 | return nil 59 | } 60 | n := p.from + rand.Intn(1+p.to-p.from) 61 | p.count++ 62 | return p.decoder.Decode([]byte(strconv.Itoa(n))) 63 | } 64 | 65 | func (p *RandomInput) Shutdown() {} 66 | -------------------------------------------------------------------------------- /input/stdin_input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "bufio" 5 | "os" 6 | "sync" 7 | "time" 8 | 9 | "github.com/childe/gohangout/codec" 10 | "github.com/childe/gohangout/topology" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | type StdinInput struct { 15 | config map[interface{}]interface{} 16 | decoder codec.Decoder 17 | 18 | scanner *bufio.Scanner 19 | scanLock sync.Mutex 20 | 21 | stop bool 22 | } 23 | 24 | func init() { 25 | Register("Stdin", newStdinInput) 26 | } 27 | 28 | func newStdinInput(config map[interface{}]interface{}) topology.Input { 29 | var codertype string = "plain" 30 | if v, ok := config["codec"]; ok { 31 | codertype = v.(string) 32 | } 33 | p := &StdinInput{ 34 | config: config, 35 | decoder: codec.NewDecoder(codertype), 36 | scanner: bufio.NewScanner(os.Stdin), 37 | } 38 | 39 | return p 40 | } 41 | 42 | func (p *StdinInput) ReadOneEvent() map[string]interface{} { 43 | p.scanLock.Lock() 44 | defer p.scanLock.Unlock() 45 | 46 | if p.scanner.Scan() { 47 | t := p.scanner.Bytes() 48 | msg := make([]byte, len(t)) 49 | copy(msg, t) 50 | return p.decoder.Decode(msg) 51 | } 52 | if err := p.scanner.Err(); err != nil { 53 | klog.Errorf("stdin scan error: %v", err) 54 | } else { 55 | // EOF here. when stdin is closed by C-D, cpu will raise up to 100% if not sleep 56 | time.Sleep(time.Millisecond * 1000) 57 | } 58 | return nil 59 | } 60 | 61 | func (p *StdinInput) Shutdown() { 62 | // what we need is to stop emit new event; close messages or not is not important 63 | p.stop = true 64 | } 65 | -------------------------------------------------------------------------------- /input/tcp_input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "bufio" 5 | "net" 6 | 7 | "github.com/childe/gohangout/codec" 8 | "github.com/childe/gohangout/topology" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type TCPInput struct { 13 | config map[interface{}]interface{} 14 | network string 15 | address string 16 | 17 | decoder codec.Decoder 18 | 19 | l net.Listener 20 | messages chan []byte 21 | stop bool 22 | 23 | connections []net.Conn 24 | } 25 | 26 | func readLine(scanner *bufio.Scanner, c net.Conn, messages chan<- []byte) { 27 | for scanner.Scan() { 28 | t := scanner.Bytes() 29 | buf := make([]byte, len(t)) 30 | copy(buf, t) 31 | messages <- buf 32 | } 33 | 34 | if err := scanner.Err(); err != nil { 35 | klog.Errorf("read from %v->%v error: %v", c.RemoteAddr(), c.LocalAddr(), err) 36 | } 37 | c.Close() 38 | } 39 | 40 | func init() { 41 | Register("TCP", newTCPInput) 42 | } 43 | func newTCPInput(config map[interface{}]interface{}) topology.Input { 44 | var codertype string = "plain" 45 | if v, ok := config["codec"]; ok { 46 | codertype = v.(string) 47 | } 48 | 49 | p := &TCPInput{ 50 | config: config, 51 | decoder: codec.NewDecoder(codertype), 52 | messages: make(chan []byte, 10), 53 | } 54 | 55 | if v, ok := config["max_length"]; ok { 56 | if max, ok := v.(int); ok { 57 | if max <= 0 { 58 | klog.Fatal("max_length must be bigger than zero") 59 | } 60 | } else { 61 | klog.Fatal("max_length must be int") 62 | } 63 | } 64 | 65 | p.network = "tcp" 66 | if network, ok := config["network"]; ok { 67 | p.network = network.(string) 68 | } 69 | 70 | if addr, ok := config["address"]; ok { 71 | p.address = addr.(string) 72 | } else { 73 | klog.Fatal("address must be set in TCP input") 74 | } 75 | 76 | l, err := net.Listen(p.network, p.address) 77 | if err != nil { 78 | klog.Fatal(err) 79 | } 80 | p.l = l 81 | 82 | go func() { 83 | for !p.stop { 84 | conn, err := l.Accept() 85 | if err != nil { 86 | if p.stop { 87 | return 88 | } 89 | klog.Error(err) 90 | } else { 91 | p.connections = append(p.connections, conn) 92 | scanner := bufio.NewScanner(conn) 93 | if v, ok := config["max_length"]; ok { 94 | max := v.(int) 95 | scanner.Buffer(make([]byte, 0, max), max) 96 | } 97 | go readLine(scanner, conn, p.messages) 98 | } 99 | } 100 | }() 101 | return p 102 | } 103 | 104 | func (p *TCPInput) ReadOneEvent() map[string]interface{} { 105 | text, more := <-p.messages 106 | if !more || text == nil { 107 | return nil 108 | } 109 | return p.decoder.Decode(text) 110 | } 111 | 112 | func (p *TCPInput) Shutdown() { 113 | p.stop = true 114 | p.l.Close() 115 | for _, conn := range p.connections { 116 | conn.Close() 117 | } 118 | close(p.messages) 119 | } 120 | -------------------------------------------------------------------------------- /input/udp_input.go: -------------------------------------------------------------------------------- 1 | package input 2 | 3 | import ( 4 | "net" 5 | 6 | "github.com/childe/gohangout/codec" 7 | "github.com/childe/gohangout/topology" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | type msg struct { 12 | message []byte 13 | addr *net.UDPAddr 14 | } 15 | type UDPInput struct { 16 | config map[interface{}]interface{} 17 | network string 18 | address string 19 | addRemoteAddr string 20 | 21 | decoder codec.Decoder 22 | 23 | conn *net.UDPConn 24 | messages chan msg 25 | stop bool 26 | } 27 | 28 | func init() { 29 | Register("UDP", newUDPInput) 30 | } 31 | 32 | func newUDPInput(config map[interface{}]interface{}) topology.Input { 33 | var codertype string = "plain" 34 | if v, ok := config["codec"]; ok { 35 | codertype = v.(string) 36 | } 37 | 38 | p := &UDPInput{ 39 | config: config, 40 | decoder: codec.NewDecoder(codertype), 41 | messages: make(chan msg, 10), 42 | } 43 | 44 | if v, ok := config["max_length"]; ok { 45 | if max, ok := v.(int); ok { 46 | if max <= 0 { 47 | klog.Fatal("max_length must be bigger than zero") 48 | } 49 | } else { 50 | klog.Fatal("max_length must be int") 51 | } 52 | } 53 | 54 | p.network = "udp" 55 | if network, ok := config["network"]; ok { 56 | p.network = network.(string) 57 | } 58 | 59 | if addr, ok := config["address"]; ok { 60 | p.address = addr.(string) 61 | } else { 62 | klog.Fatal("address must be set in UDP input") 63 | } 64 | 65 | udpAddr, err := net.ResolveUDPAddr(p.network, p.address) 66 | if err != nil { 67 | klog.Fatalf("resolve udp addr error: %v", err) 68 | } 69 | 70 | conn, err := net.ListenUDP(p.network, udpAddr) 71 | if err != nil { 72 | klog.Fatalf("listen udp error: %v", err) 73 | } 74 | p.conn = conn 75 | 76 | if v, ok := config["add_remote_addr"]; ok { 77 | p.addRemoteAddr = v.(string) 78 | } 79 | 80 | var max int = 65535 81 | if v, ok := config["max_length"]; ok { 82 | max = v.(int) 83 | } 84 | 85 | go func() { 86 | for !p.stop { 87 | buf := make([]byte, max) 88 | n, addr, err := p.conn.ReadFromUDP(buf) 89 | if err != nil { 90 | if p.stop { 91 | return 92 | } 93 | klog.Errorf("read from UDP error: %v", err) 94 | } 95 | p.messages <- msg{ 96 | message: buf[:n], 97 | addr: addr, 98 | } 99 | } 100 | }() 101 | return p 102 | } 103 | 104 | func (p *UDPInput) ReadOneEvent() map[string]interface{} { 105 | msg, more := <-p.messages 106 | if !more { 107 | return nil 108 | } 109 | event := p.decoder.Decode(msg.message) 110 | 111 | if p.addRemoteAddr != "" && msg.addr != nil { 112 | event[p.addRemoteAddr] = msg.addr.IP.String() 113 | } 114 | 115 | return event 116 | } 117 | 118 | func (p *UDPInput) Shutdown() { 119 | p.stop = true 120 | p.conn.Close() 121 | close(p.messages) 122 | } 123 | -------------------------------------------------------------------------------- /internal/config/config_parser.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "errors" 5 | "regexp" 6 | "strings" 7 | 8 | yaml "gopkg.in/yaml.v2" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type Config map[string]interface{} 13 | 14 | type Parser interface { 15 | parse(filename string) (map[string]interface{}, error) 16 | } 17 | 18 | func ParseConfig(filename string) (map[string]interface{}, error) { 19 | lowerFilename := strings.ToLower(filename) 20 | if strings.HasSuffix(lowerFilename, ".yaml") || strings.HasSuffix(lowerFilename, ".yml") { 21 | yp := &YamlParser{} 22 | return yp.parse(filename) 23 | } 24 | return nil, errors.New("unknown config format. config filename should ends with yaml|yml") 25 | } 26 | 27 | // remove sensitive info before output 28 | func RemoveSensitiveInfo(config map[string]interface{}) string { 29 | re := regexp.MustCompile(`(.*password:\s+)(.*)`) 30 | re2 := regexp.MustCompile(`(http(s)?://\w+:)\w+`) 31 | 32 | b, err := yaml.Marshal(config) 33 | if err != nil { 34 | klog.Errorf("marshal config error: %s", err) 35 | return "" 36 | } 37 | 38 | output := make([]string, 0) 39 | for _, l := range strings.Split(string(b), "\n") { 40 | if re.MatchString(l) { 41 | output = append(output, re.ReplaceAllString(l, "${1}xxxxxx")) 42 | continue 43 | } 44 | if re2.MatchString(l) { 45 | output = append(output, re2.ReplaceAllString(l, "${1}xxxxxx")) 46 | continue 47 | } 48 | output = append(output, l) 49 | } 50 | 51 | return strings.Join(output, "\n") 52 | } 53 | -------------------------------------------------------------------------------- /internal/config/config_watcher.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/fsnotify/fsnotify" 5 | "k8s.io/klog/v2" 6 | ) 7 | 8 | // Watcher watches the config file and callback f 9 | func WatchConfig(filename string, reloadFunc func()) error { 10 | watcher, err := fsnotify.NewWatcher() 11 | if err != nil { 12 | return err 13 | } 14 | watcher.Add(filename) 15 | 16 | go func() { 17 | defer watcher.Close() 18 | for { 19 | select { 20 | case event, more := <-watcher.Events: 21 | if !more { 22 | klog.Info("config file watcher closed") 23 | return 24 | } 25 | klog.Infof("capture file watch event: %s", event) 26 | reloadFunc() 27 | 28 | // filename may be renamed, so add it again 29 | watcher.Add(filename) 30 | case err, more := <-watcher.Errors: 31 | if !more { 32 | klog.Info("error channel of config file watcher closed") 33 | return 34 | } 35 | klog.Errorf("error from config file watcher: %v", err) 36 | } 37 | } 38 | }() 39 | 40 | return nil 41 | } 42 | -------------------------------------------------------------------------------- /internal/config/yaml_config_parser.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | 10 | yaml "gopkg.in/yaml.v2" 11 | ) 12 | 13 | type YamlParser struct{} 14 | 15 | func (yp *YamlParser) parse(filepath string) (map[string]interface{}, error) { 16 | var ( 17 | buffer []byte 18 | err error 19 | ) 20 | if strings.HasPrefix(filepath, "http://") || strings.HasPrefix(filepath, "https://") { 21 | resp, err := http.Get(filepath) 22 | if err != nil { 23 | return nil, err 24 | } 25 | defer resp.Body.Close() 26 | buffer, err = io.ReadAll(resp.Body) 27 | if err != nil { 28 | return nil, err 29 | } 30 | } else { 31 | configFile, err := os.Open(filepath) 32 | if err != nil { 33 | return nil, err 34 | } 35 | fi, _ := configFile.Stat() 36 | 37 | if fi.Size() == 0 { 38 | return nil, fmt.Errorf("config file (%s) is empty", filepath) 39 | } 40 | 41 | buffer = make([]byte, fi.Size()) 42 | _, err = configFile.Read(buffer) 43 | if err != nil { 44 | return nil, err 45 | } 46 | } 47 | 48 | buffer = []byte(os.ExpandEnv(string(buffer))) 49 | 50 | config := make(map[string]interface{}) 51 | err = yaml.Unmarshal(buffer, &config) 52 | if err != nil { 53 | return nil, err 54 | } 55 | return config, nil 56 | } 57 | -------------------------------------------------------------------------------- /internal/signal/signalhandle_unix.go: -------------------------------------------------------------------------------- 1 | //go:build linux || darwin 2 | // +build linux darwin 3 | 4 | package signal 5 | 6 | import ( 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | func ListenSignal(termFunc func(), reloadFunc func()) { 15 | c := make(chan os.Signal, 1) 16 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR1) 17 | 18 | for sig := range c { 19 | klog.Infof("capture signal: %v", sig) 20 | switch sig { 21 | case syscall.SIGINT, syscall.SIGTERM: 22 | termFunc() 23 | case syscall.SIGUSR1: 24 | reloadFunc() 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /internal/signal/signalhandle_windows.go: -------------------------------------------------------------------------------- 1 | //go:build windows 2 | // +build windows 3 | 4 | package signal 5 | 6 | import ( 7 | "os" 8 | "os/signal" 9 | "syscall" 10 | 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | func ListenSignal(termFunc func(), reloadFunc func()) { 15 | c := make(chan os.Signal, 1) 16 | signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) 17 | 18 | for sig := range c { 19 | klog.Infof("capture signal: %v", sig) 20 | switch sig { 21 | case syscall.SIGINT, syscall.SIGTERM: 22 | termFunc() 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /output/dot_output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/childe/gohangout/topology" 7 | ) 8 | 9 | type DotOutput struct { 10 | config map[interface{}]interface{} 11 | } 12 | 13 | func newDotOutput(config map[interface{}]interface{}) topology.Output { 14 | return &DotOutput{ 15 | config: config, 16 | } 17 | } 18 | 19 | func init() { 20 | Register("Dot", newDotOutput) 21 | } 22 | 23 | func (outputPlugin *DotOutput) Emit(event map[string]interface{}) { 24 | fmt.Print(".") 25 | } 26 | 27 | func (outputPlugin *DotOutput) Shutdown() {} 28 | -------------------------------------------------------------------------------- /output/host_selector.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | type HostSelector interface { 9 | Next() interface{} 10 | ReduceWeight() 11 | AddWeight() 12 | Size() int 13 | } 14 | 15 | type RRHostSelector struct { 16 | hosts []interface{} 17 | initWeight int 18 | weight []int 19 | index int 20 | hostsCount int 21 | } 22 | 23 | func NewRRHostSelector(hosts []interface{}, weight int) *RRHostSelector { 24 | rand.Seed(time.Now().UnixNano()) 25 | hostsCount := len(hosts) 26 | rst := &RRHostSelector{ 27 | hosts: hosts, 28 | index: int(rand.Int31n(int32(hostsCount))), 29 | hostsCount: hostsCount, 30 | initWeight: weight, 31 | } 32 | rst.weight = make([]int, hostsCount) 33 | for i := 0; i < hostsCount; i++ { 34 | rst.weight[i] = weight 35 | } 36 | 37 | return rst 38 | } 39 | 40 | func (s *RRHostSelector) Next() interface{} { 41 | for i := 1; i <= s.hostsCount; i++ { 42 | idx := (s.index + i) % s.hostsCount 43 | if s.weight[idx] > 0 { 44 | s.index = idx 45 | return s.hosts[idx] 46 | } 47 | } 48 | 49 | s.resetWeight(s.initWeight) 50 | // allow client wait for some time and then get Next 51 | return nil 52 | } 53 | 54 | func (s *RRHostSelector) resetWeight(weight int) { 55 | for i := range s.weight { 56 | s.weight[i] = weight 57 | } 58 | } 59 | 60 | func (s *RRHostSelector) ReduceWeight() { 61 | s.weight[s.index]-- 62 | if s.weight[s.index] <= 0 { 63 | i := s.index 64 | time.AfterFunc(time.Minute*30, func() { 65 | s.weight[i] = 1 66 | }) 67 | } 68 | } 69 | 70 | func (s *RRHostSelector) AddWeight() { 71 | s.weight[s.index] = s.weight[s.index] + 1 72 | if s.weight[s.index] > s.initWeight { 73 | s.weight[s.index] = s.initWeight 74 | } 75 | } 76 | 77 | func (s *RRHostSelector) Size() int { 78 | return len(s.hosts) 79 | } 80 | -------------------------------------------------------------------------------- /output/influxdb_output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "reflect" 7 | "strings" 8 | "time" 9 | 10 | "github.com/childe/gohangout/topology" 11 | "github.com/childe/gohangout/value_render" 12 | "k8s.io/klog/v2" 13 | ) 14 | 15 | const () 16 | 17 | type InAction struct { 18 | measurement string 19 | event map[string]interface{} 20 | tags []string 21 | fields []string 22 | timestamp string 23 | } 24 | 25 | func (action *InAction) Encode() []byte { 26 | bulk_buf := []byte(action.measurement) 27 | 28 | //tag set 29 | tag_set := make([]string, 0) 30 | for _, tag := range action.tags { 31 | if v, ok := action.event[tag]; ok { 32 | tag_set = append(tag_set, fmt.Sprintf("%s=%v", tag, v)) 33 | } 34 | } 35 | if len(tag_set) > 0 { 36 | bulk_buf = append(bulk_buf, ',') 37 | bulk_buf = append(bulk_buf, strings.Join(tag_set, ",")...) 38 | } 39 | 40 | //field set 41 | field_set := make([]string, 0) 42 | for _, field := range action.fields { 43 | if v, ok := action.event[field]; ok { 44 | field_set = append(field_set, fmt.Sprintf("%s=%v", field, v)) 45 | } 46 | } 47 | if len(field_set) <= 0 { 48 | klog.V(20).Infof("field set is nil. fields: %v. event: %v", action.fields, action.event) 49 | return nil 50 | } else { 51 | bulk_buf = append(bulk_buf, ' ') 52 | bulk_buf = append(bulk_buf, strings.Join(field_set, ",")...) 53 | } 54 | 55 | //timestamp 56 | t := action.event[action.timestamp] 57 | if t != nil && reflect.TypeOf(t).String() == "time.Time" { 58 | bulk_buf = append(bulk_buf, fmt.Sprintf(" %d", t.(time.Time).UnixNano())...) 59 | } else { 60 | klog.V(20).Infof("%s is not time.Time", action.timestamp) 61 | } 62 | 63 | return bulk_buf 64 | } 65 | 66 | type InfluxdbBulkRequest struct { 67 | events []Event 68 | bulk_buf []byte 69 | } 70 | 71 | func (br *InfluxdbBulkRequest) add(event Event) { 72 | br.bulk_buf = append(br.bulk_buf, event.Encode()...) 73 | br.bulk_buf = append(br.bulk_buf, '\n') 74 | br.events = append(br.events, event) 75 | } 76 | 77 | func (br *InfluxdbBulkRequest) bufSizeByte() int { 78 | return len(br.bulk_buf) 79 | } 80 | func (br *InfluxdbBulkRequest) eventCount() int { 81 | return len(br.events) 82 | } 83 | func (br *InfluxdbBulkRequest) readBuf() []byte { 84 | return br.bulk_buf 85 | } 86 | 87 | type InfluxdbOutput struct { 88 | config map[interface{}]interface{} 89 | 90 | db string 91 | measurement value_render.ValueRender 92 | tags []string 93 | fields []string 94 | timestamp string 95 | 96 | bulkProcessor BulkProcessor 97 | } 98 | 99 | func influxdbGetRetryEvents(resp *http.Response, respBody []byte, bulkRequest *BulkRequest) ([]int, []int, BulkRequest) { 100 | return nil, nil, nil 101 | } 102 | 103 | func init() { 104 | Register("Influxdb", newInfluxdbOutput) 105 | } 106 | 107 | func newInfluxdbOutput(config map[interface{}]interface{}) topology.Output { 108 | rst := &InfluxdbOutput{ 109 | config: config, 110 | } 111 | 112 | if v, ok := config["db"]; ok { 113 | rst.db = v.(string) 114 | } else { 115 | klog.Fatal("db must be set in elasticsearch output") 116 | } 117 | 118 | if v, ok := config["measurement"]; ok { 119 | rst.measurement = value_render.GetValueRender(v.(string)) 120 | } else { 121 | klog.Fatal("measurement must be set in elasticsearch output") 122 | } 123 | 124 | if v, ok := config["tags"]; ok { 125 | for _, t := range v.([]interface{}) { 126 | rst.tags = append(rst.tags, t.(string)) 127 | } 128 | } 129 | if v, ok := config["fields"]; ok { 130 | for _, f := range v.([]interface{}) { 131 | rst.fields = append(rst.fields, f.(string)) 132 | } 133 | } 134 | if v, ok := config["timestamp"]; ok { 135 | rst.timestamp = v.(string) 136 | } else { 137 | rst.timestamp = "@timestamp" 138 | } 139 | 140 | var ( 141 | bulk_size, bulk_actions, flush_interval, concurrent int 142 | compress bool 143 | ) 144 | if v, ok := config["bulk_size"]; ok { 145 | bulk_size = v.(int) * 1024 * 1024 146 | } else { 147 | bulk_size = DEFAULT_BULK_SIZE 148 | } 149 | 150 | if v, ok := config["bulk_actions"]; ok { 151 | bulk_actions = v.(int) 152 | } else { 153 | bulk_actions = DEFAULT_BULK_ACTIONS 154 | } 155 | if v, ok := config["flush_interval"]; ok { 156 | flush_interval = v.(int) 157 | } else { 158 | flush_interval = DEFAULT_FLUSH_INTERVAL 159 | } 160 | if v, ok := config["concurrent"]; ok { 161 | concurrent = v.(int) 162 | } else { 163 | concurrent = DEFAULT_CONCURRENT 164 | } 165 | if concurrent <= 0 { 166 | klog.Fatal("concurrent must > 0") 167 | } 168 | if v, ok := config["compress"]; ok { 169 | compress = v.(bool) 170 | } else { 171 | compress = true 172 | } 173 | 174 | var hosts []string 175 | if v, ok := config["hosts"]; ok { 176 | for _, h := range v.([]interface{}) { 177 | hosts = append(hosts, h.(string)+"/write?db="+rst.db) 178 | } 179 | } else { 180 | klog.Fatal("hosts must be set in elasticsearch output") 181 | } 182 | 183 | headers := make(map[string]string) 184 | if v, ok := config["headers"]; ok { 185 | for keyI, valueI := range v.(map[interface{}]interface{}) { 186 | headers[keyI.(string)] = valueI.(string) 187 | } 188 | } 189 | var requestMethod string = "POST" 190 | 191 | retryResponseCode := make(map[int]bool) 192 | if v, ok := config["retry_response_code"]; ok { 193 | for _, cI := range v.([]interface{}) { 194 | retryResponseCode[cI.(int)] = true 195 | } 196 | } 197 | 198 | byte_size_applied_in_advance := bulk_size + 1024*1024 199 | if byte_size_applied_in_advance > MAX_BYTE_SIZE_APPLIED_IN_ADVANCE { 200 | byte_size_applied_in_advance = MAX_BYTE_SIZE_APPLIED_IN_ADVANCE 201 | } 202 | var f = func() BulkRequest { 203 | return &InfluxdbBulkRequest{ 204 | bulk_buf: make([]byte, 0, byte_size_applied_in_advance), 205 | } 206 | } 207 | 208 | rst.bulkProcessor = NewHTTPBulkProcessor(headers, hosts, requestMethod, retryResponseCode, bulk_size, bulk_actions, flush_interval, concurrent, compress, f, influxdbGetRetryEvents) 209 | return rst 210 | } 211 | 212 | func (p *InfluxdbOutput) Emit(event map[string]interface{}) { 213 | measurement, err := p.measurement.Render(event) 214 | if err != nil { 215 | klog.V(20).Infof("measurement render error: %v", err) 216 | return 217 | } 218 | p.bulkProcessor.add(&InAction{measurement.(string), event, p.tags, p.fields, p.timestamp}) 219 | } 220 | 221 | func (outputPlugin *InfluxdbOutput) Shutdown() { 222 | outputPlugin.bulkProcessor.awaitclose(30 * time.Second) 223 | } 224 | -------------------------------------------------------------------------------- /output/kafka_output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/childe/gohangout/codec" 7 | "github.com/childe/gohangout/topology" 8 | "github.com/childe/gohangout/value_render" 9 | "github.com/childe/healer" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | func init() { 14 | Register("Kafka", newKafkaOutput) 15 | } 16 | 17 | type KafkaOutput struct { 18 | config map[interface{}]interface{} 19 | 20 | encoder codec.Encoder 21 | 22 | producer *healer.Producer 23 | key value_render.ValueRender 24 | } 25 | 26 | func newKafkaOutput(config map[interface{}]interface{}) topology.Output { 27 | p := &KafkaOutput{ 28 | config: config, 29 | } 30 | 31 | if v, ok := config["codec"]; ok { 32 | p.encoder = codec.NewEncoder(v.(string)) 33 | } else { 34 | p.encoder = codec.NewEncoder("json") 35 | } 36 | 37 | pc, ok := config["producer_settings"] 38 | if !ok { 39 | klog.Fatal("kafka output must have producer_settings") 40 | } 41 | newPc := make(map[string]interface{}) 42 | for k, v := range pc.(map[interface{}]interface{}) { 43 | newPc[k.(string)] = v 44 | } 45 | producer_settings := make(map[string]interface{}) 46 | if b, err := json.Marshal(newPc); err != nil { 47 | klog.Fatalf("could not init kafka producer config: %v", err) 48 | } else { 49 | json.Unmarshal(b, &producer_settings) 50 | } 51 | 52 | klog.Info(producer_settings) 53 | 54 | var topic string 55 | if v, ok := config["topic"]; !ok { 56 | klog.Fatal("kafka output must have topic setting") 57 | } else { 58 | topic = v.(string) 59 | } 60 | 61 | producer, err := healer.NewProducer(topic, producer_settings) 62 | if err != nil { 63 | klog.Fatalf("could not create kafka producer: %v", err) 64 | } 65 | p.producer = producer 66 | 67 | if v, ok := config["key"]; ok { 68 | p.key = value_render.GetValueRender(v.(string)) 69 | } else { 70 | p.key = nil 71 | } 72 | 73 | return p 74 | } 75 | 76 | func (p *KafkaOutput) Emit(event map[string]interface{}) { 77 | buf, err := p.encoder.Encode(event) 78 | if err != nil { 79 | klog.Errorf("marshal %v error: %s", event, err) 80 | return 81 | } 82 | if p.key == nil { 83 | p.producer.AddMessage(nil, buf) 84 | } else { 85 | key, _ := p.key.Render(event) 86 | p.producer.AddMessage([]byte(key.(string)), buf) 87 | } 88 | } 89 | 90 | func (p *KafkaOutput) Shutdown() { 91 | p.producer.Close() 92 | } 93 | -------------------------------------------------------------------------------- /output/output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | "plugin" 6 | 7 | "github.com/childe/gohangout/condition_filter" 8 | "github.com/childe/gohangout/topology" 9 | "k8s.io/klog/v2" 10 | ) 11 | 12 | type BuildOutputFunc func(map[interface{}]interface{}) topology.Output 13 | 14 | var registeredOutput map[string]BuildOutputFunc = make(map[string]BuildOutputFunc) 15 | 16 | // Register is used by output plugins to register themselves 17 | func Register(outputType string, bf BuildOutputFunc) { 18 | if _, ok := registeredOutput[outputType]; ok { 19 | klog.Errorf("%s has been registered, ignore %T", outputType, bf) 20 | return 21 | } 22 | registeredOutput[outputType] = bf 23 | } 24 | 25 | // BuildOutput builds OutputBox. it firstly tries built-in plugin, and then try 3rd party plugin 26 | func BuildOutput(outputType string, config map[interface{}]interface{}) *topology.OutputBox { 27 | var output topology.Output 28 | var err error 29 | if v, ok := registeredOutput[outputType]; ok { 30 | output = v(config) 31 | } else { 32 | klog.Info("use third party plugin") 33 | output, err = getOutputFromPlugin(outputType, config) 34 | if err != nil { 35 | klog.Errorf("could not load %s: %v", outputType, err) 36 | return nil 37 | } 38 | } 39 | 40 | return &topology.OutputBox{ 41 | Output: output, 42 | ConditionFilter: condition_filter.NewConditionFilter(config), 43 | } 44 | } 45 | 46 | func getOutputFromPlugin(pluginPath string, config map[interface{}]interface{}) (topology.Output, error) { 47 | p, err := plugin.Open(pluginPath) 48 | if err != nil { 49 | return nil, fmt.Errorf("could not open %s: %v", pluginPath, err) 50 | } 51 | newFunc, err := p.Lookup("New") 52 | if err != nil { 53 | return nil, fmt.Errorf("could not find `New` function in %s: %s", pluginPath, err) 54 | } 55 | 56 | f, ok := newFunc.(func(map[interface{}]interface{}) interface{}) 57 | if !ok { 58 | return nil, fmt.Errorf("`New` func in %s format error", pluginPath) 59 | } 60 | 61 | rst := f(config) 62 | filter, ok := rst.(topology.Output) 63 | if !ok { 64 | return nil, fmt.Errorf("`New` func in %s dose not return Output Interface", pluginPath) 65 | } 66 | return filter, nil 67 | } 68 | -------------------------------------------------------------------------------- /output/stdout_output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/childe/gohangout/codec" 7 | "github.com/childe/gohangout/topology" 8 | "k8s.io/klog/v2" 9 | ) 10 | 11 | func init() { 12 | Register("Stdout", newStdoutOutput) 13 | } 14 | 15 | type StdoutOutput struct { 16 | config map[interface{}]interface{} 17 | encoder codec.Encoder 18 | } 19 | 20 | func newStdoutOutput(config map[interface{}]interface{}) topology.Output { 21 | p := &StdoutOutput{ 22 | config: config, 23 | } 24 | 25 | if v, ok := config["codec"]; ok { 26 | p.encoder = codec.NewEncoder(v.(string)) 27 | } else { 28 | p.encoder = codec.NewEncoder("json") 29 | } 30 | 31 | return p 32 | 33 | } 34 | 35 | func (p *StdoutOutput) Emit(event map[string]interface{}) { 36 | buf, err := p.encoder.Encode(event) 37 | if err != nil { 38 | klog.Errorf("marshal %v error:%s", event, err) 39 | } 40 | fmt.Println(string(buf)) 41 | } 42 | 43 | func (p *StdoutOutput) Shutdown() {} 44 | -------------------------------------------------------------------------------- /output/tcp_output.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "io" 5 | "net" 6 | "sync" 7 | "time" 8 | 9 | "github.com/childe/gohangout/simplejson" 10 | "github.com/childe/gohangout/topology" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | func init() { 15 | Register("TCP", newTCPOutput) 16 | } 17 | 18 | type TCPOutput struct { 19 | config map[interface{}]interface{} 20 | network string 21 | address string 22 | timeout time.Duration 23 | keepalive time.Duration 24 | 25 | concurrent int 26 | messages chan map[string]interface{} 27 | conn []net.Conn 28 | //writer *bufio.Writer 29 | 30 | dialLock sync.Mutex 31 | } 32 | 33 | func newTCPOutput(config map[interface{}]interface{}) topology.Output { 34 | p := &TCPOutput{ 35 | config: config, 36 | concurrent: 1, 37 | } 38 | 39 | p.network = "tcp" 40 | if network, ok := config["network"]; ok { 41 | p.network = network.(string) 42 | } 43 | 44 | if addr, ok := config["address"]; ok { 45 | p.address, ok = addr.(string) 46 | } else { 47 | klog.Fatal("address must be set in TCP output") 48 | } 49 | 50 | if timeoutI, ok := config["dial.timeout"]; ok { 51 | timeout := timeoutI.(int) 52 | p.timeout = time.Second * time.Duration(timeout) 53 | } 54 | 55 | if keepaliveI, ok := config["keepalive"]; ok { 56 | keepalive, ok := keepaliveI.(int) 57 | if !ok { 58 | klog.Fatal("keepalive must be integer") 59 | } 60 | p.keepalive = time.Second * time.Duration(keepalive) 61 | } 62 | 63 | if v, ok := config["concurrent"]; ok { 64 | p.concurrent = v.(int) 65 | } 66 | p.messages = make(chan map[string]interface{}, p.concurrent) 67 | p.conn = make([]net.Conn, p.concurrent) 68 | 69 | for i := 0; i < p.concurrent; i++ { 70 | go func(i int) { 71 | p.conn[i] = p.loopDial() 72 | for { 73 | event := <-p.messages 74 | d := &simplejson.SimpleJsonDecoder{} 75 | buf, err := d.Encode(event) 76 | if err != nil { 77 | klog.Errorf("marshal %v error:%s", event, err) 78 | return 79 | } 80 | 81 | buf = append(buf, '\n') 82 | for { 83 | if err = write(p.conn[i], buf); err != nil { 84 | klog.Error(err) 85 | p.conn[i].Close() 86 | p.conn[i] = p.loopDial() 87 | } else { 88 | break 89 | } 90 | } 91 | } 92 | }(i) 93 | } 94 | 95 | return p 96 | } 97 | 98 | func (p *TCPOutput) loopDial() net.Conn { 99 | for { 100 | if conn, err := p.dial(); err != nil { 101 | klog.Errorf("dial error: %s. sleep 1s", err) 102 | time.Sleep(1 * time.Second) 103 | } else { 104 | klog.Infof("conn built to %s", conn.RemoteAddr()) 105 | return conn 106 | } 107 | } 108 | } 109 | 110 | func (p *TCPOutput) dial() (net.Conn, error) { 111 | var d net.Dialer 112 | d.Timeout = p.timeout 113 | d.KeepAlive = p.keepalive 114 | 115 | conn, err := net.Dial(p.network, p.address) 116 | if err != nil { 117 | return conn, err 118 | } 119 | // *TcpConn is net.Conn interface, so we can pass conn instead of &conn 120 | go probe(conn) 121 | //p.writer = bufio.NewWriter(conn) 122 | 123 | return conn, nil 124 | } 125 | 126 | func probe(conn net.Conn) { 127 | var b = make([]byte, 1) 128 | 129 | conn.SetDeadline(time.Time{}) 130 | conn.SetReadDeadline(time.Time{}) 131 | _, err := conn.Read(b) // should block here 132 | if err != nil && err == io.EOF { 133 | klog.Infof("conn [%s] is closed by the server, close the conn.", conn.RemoteAddr()) 134 | conn.Close() 135 | } 136 | } 137 | 138 | func (p *TCPOutput) Emit(event map[string]interface{}) { 139 | p.messages <- event 140 | //buf = append(buf, '\n') 141 | //n, err := p.writer.Write(buf) 142 | //if n != len(buf) { 143 | //klog.Errorf("write to %s[%s] error: %s", p.address, p.conn.RemoteAddr(), err) 144 | //} 145 | //p.writer.Flush() 146 | } 147 | 148 | func write(conn net.Conn, buf []byte) error { 149 | for len(buf) > 0 { 150 | n, err := conn.Write(buf) 151 | if err != nil { 152 | return err 153 | //klog.Errorf("write to %s[%s] error: %s", p.address, conn.RemoteAddr(), err) 154 | //switch { 155 | //case strings.Contains(str, "use of closed network connection"): 156 | //conn = loopDial() 157 | //return err 158 | //case strings.Contains(str, "write: broken pipe"): 159 | //conn.Close() 160 | //conn = loopDial() 161 | //return err 162 | //} 163 | } 164 | buf = buf[n:] 165 | } 166 | return nil 167 | } 168 | 169 | func (p *TCPOutput) Shutdown() { 170 | //p.writer.Flush() 171 | //p.conn.Close() 172 | } 173 | -------------------------------------------------------------------------------- /test/LinkMetricInFilters.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | 5 | filters: 6 | - Date: 7 | src: '@timestamp' 8 | formats: 9 | - 'RFC3339' 10 | - Convert: 11 | fields: 12 | message: 13 | to: int 14 | - Rename: 15 | fields: 16 | message: size 17 | - Add: 18 | fields: 19 | name: test1 20 | - Add: 21 | if: 22 | - Random(2) 23 | fields: 24 | name: test2 25 | 26 | - Filters: 27 | filters: 28 | - LinkMetric: 29 | fieldsLink: 'name->size' 30 | timestamp: '@timestamp' 31 | batchWindow: 1 32 | reserveWindow: 1 33 | windowOffset: 0 34 | drop_original_event: false 35 | accumulateMode: separate 36 | 37 | outputs: 38 | - Stdout: {} 39 | -------------------------------------------------------------------------------- /test/itest-1.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 1 4 | to: 100 5 | max_messages: 1000 6 | - Random: 7 | from: 1 8 | to: 100 9 | max_messages: 1000 10 | 11 | outputs: 12 | - Stdout: {} 13 | - Stdout: {} 14 | -------------------------------------------------------------------------------- /test/itest-2.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 300 4 | to: 1000 5 | max_messages: 1000 6 | - Random: 7 | from: 200 8 | to: 299 9 | max_messages: 1000 10 | 11 | filters: 12 | - Drop: 13 | if: 14 | - 'HasPrefix(message,2)' 15 | 16 | outputs: 17 | - Stdout: {} 18 | - Stdout: {} 19 | -------------------------------------------------------------------------------- /test/itest-3.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 400 4 | to: 1000 5 | max_messages: 1000 6 | - Random: 7 | from: 200 8 | to: 299 9 | max_messages: 1000 10 | - Random: 11 | from: 300 12 | to: 399 13 | max_messages: 1000 14 | 15 | filters: 16 | - Drop: 17 | if: 18 | - 'HasPrefix(message,2)' 19 | 20 | outputs: 21 | - Stdout: 22 | if: 23 | - 'HasPrefix(message,3)' 24 | - Stdout: {} 25 | -------------------------------------------------------------------------------- /test/itest-4.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 400 4 | to: 1000 5 | max_messages: 1000 6 | - Random: 7 | from: 200 8 | to: 299 9 | max_messages: 1000 10 | - Random: 11 | from: 300 12 | to: 399 13 | max_messages: 1000 14 | 15 | filters: 16 | - Drop: 17 | if: 18 | - 'HasPrefix(message,2)' 19 | - Filters: 20 | filters: 21 | - Add: 22 | fields: 23 | tag1: add 24 | - Add: 25 | fields: 26 | tag2: add 27 | 28 | outputs: 29 | - Stdout: 30 | if: 31 | - 'HasPrefix(message,3)' 32 | - Stdout: {} 33 | -------------------------------------------------------------------------------- /test/itest-5.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 400 4 | to: 1000 5 | max_messages: 1000 6 | - Random: 7 | from: 200 8 | to: 299 9 | max_messages: 1000 10 | - Random: 11 | from: 300 12 | to: 399 13 | max_messages: 1000 14 | 15 | filters: 16 | - Drop: 17 | if: 18 | - 'HasPrefix(message,2)' 19 | - Filters: 20 | filters: 21 | - Add: 22 | fields: 23 | tag1: add 24 | - Add: 25 | fields: 26 | tag2: add 27 | 28 | outputs: 29 | - Stdout: 30 | if: 31 | - 'HasPrefix(message,3)' 32 | - Stdout: {} 33 | -------------------------------------------------------------------------------- /test/itest-6-2.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | 5 | filters: 6 | - Date: 7 | src: '@timestamp' 8 | formats: 9 | - 'RFC3339' 10 | - Convert: 11 | fields: 12 | message: 13 | to: int 14 | - Rename: 15 | fields: 16 | message: size 17 | - Add: 18 | fields: 19 | name: test1 20 | - Add: 21 | if: 22 | - Random(2) 23 | fields: 24 | name: test2 25 | 26 | - LinkMetric: 27 | fieldsLink: 'name->size' 28 | timestamp: '@timestamp' 29 | batchWindow: 1 30 | reserveWindow: 1 31 | windowOffset: 0 32 | drop_original_event: false 33 | accumulateMode: separate 34 | 35 | outputs: 36 | - Stdout: {} 37 | -------------------------------------------------------------------------------- /test/itest-6-3.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | 5 | filters: 6 | - Date: 7 | src: '@timestamp' 8 | formats: 9 | - 'RFC3339' 10 | - Convert: 11 | fields: 12 | message: 13 | to: int 14 | - Rename: 15 | fields: 16 | message: size 17 | - Add: 18 | fields: 19 | name: test1 20 | - Add: 21 | if: 22 | - Random(2) 23 | fields: 24 | name: test2 25 | 26 | - LinkMetric: 27 | fieldsLink: 'name->size' 28 | timestamp: '@timestamp' 29 | batchWindow: 1 30 | reserveWindow: 1 31 | windowOffset: 0 32 | drop_original_event: false 33 | accumulateMode: cumulative 34 | 35 | outputs: 36 | - Stdout: {} 37 | -------------------------------------------------------------------------------- /test/itest-6-4.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | 5 | filters: 6 | - Date: 7 | src: '@timestamp' 8 | formats: 9 | - 'RFC3339' 10 | - Convert: 11 | fields: 12 | message: 13 | to: int 14 | - Rename: 15 | fields: 16 | message: size 17 | - Add: 18 | fields: 19 | name: test1 20 | - Add: 21 | if: 22 | - Random(2) 23 | fields: 24 | name: test2 25 | 26 | - LinkMetric: 27 | fieldsLink: 'name->size' 28 | timestamp: '@timestamp' 29 | batchWindow: 1 30 | reserveWindow: 10 31 | windowOffset: 0 32 | drop_original_event: false 33 | accumulateMode: cumulative 34 | 35 | outputs: 36 | - Stdout: {} 37 | -------------------------------------------------------------------------------- /test/itest-6.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 1 4 | to: 2 5 | max_messages: 1000 6 | outputs: 7 | - Stdout: {} 8 | -------------------------------------------------------------------------------- /test/itest-7-1.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Stdin: 3 | codec: json 4 | 5 | filters: 6 | - Date: 7 | src: '@timestamp' 8 | formats: 9 | - 'RFC3339' 10 | - Convert: 11 | fields: 12 | message: 13 | to: float 14 | - Rename: 15 | fields: 16 | message: size 17 | - Add: 18 | fields: 19 | name: test1 20 | age: '10' 21 | - Add: 22 | if: 23 | - Random(2) 24 | fields: 25 | name: test2 26 | - Add: 27 | if: 28 | - Random(2) 29 | fields: 30 | age: '20' 31 | 32 | - LinkStatsMetric: 33 | fieldsLink: 'name->age->size' 34 | timestamp: '@timestamp' 35 | batchWindow: 1 36 | reserveWindow: 10 37 | windowOffset: 1 38 | drop_original_event: false 39 | accumulateMode: cumulative 40 | 41 | outputs: 42 | - Stdout: {} 43 | -------------------------------------------------------------------------------- /test/itest-es7.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 1 4 | to: 100 5 | max_messages: 1000 6 | outputs: 7 | # - Stdout: {} 8 | - Stdout: {} 9 | - Elasticsearch: 10 | hosts: 11 | - 'http://username:password@10.0.0.2:9200' 12 | index: 'web-%{+2006.01.02}' 13 | index_time_location: 'UTC' 14 | index_type: "logs" 15 | bulk_actions: 10 16 | bulk_size: 5 17 | flush_interval: 5 18 | concurrent: 1 19 | compress: false 20 | es_version: 7 21 | # es_version: 6 22 | retry_response_code: [401, 502] -------------------------------------------------------------------------------- /test/itest-tcp.sh: -------------------------------------------------------------------------------- 1 | go build -o build/gohangout-test || exit 255 2 | gohangout='build/gohangout-test' 3 | 4 | em_print() { 5 | echo "\n=======" 6 | echo $1 7 | echo "=======\n" 8 | } 9 | 10 | em_print 'test tcp input/output' 11 | 12 | tmpfile=$(mktemp) 13 | echo "$tmpfile" 14 | 15 | nohup $gohangout --config test/itest-tcpinput.yml > $tmpfile & 16 | sleep 1 17 | 18 | $gohangout --config test/itest-tcpoutput.yml 19 | sleep 2 20 | 21 | ps -ef | grep "$gohangout --config test/itest-tcpinput.yml" | grep -v grep | awk '{print $2}' | xargs kill 22 | 23 | wcl=$(wc -l $tmpfile | awk '{print $1}') 24 | 25 | 26 | if [ "$wcl" != "200000" ] 27 | then 28 | echo $wcl 29 | em_print 'tcp input/output should create 200000 docs!' 30 | exit 255 31 | else 32 | em_print 'tcp plugin passes' 33 | fi 34 | 35 | #rm $tmpfile 36 | -------------------------------------------------------------------------------- /test/itest-tcpinput.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - TCP: 3 | address: 0.0.0.0:10000 4 | codec: json 5 | outputs: 6 | - Stdout: {} 7 | -------------------------------------------------------------------------------- /test/itest-tcpoutput.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | - Random: 3 | from: 1 4 | to: 100 5 | max_messages: 100000 6 | - Random: 7 | from: 1 8 | to: 100 9 | max_messages: 100000 10 | outputs: 11 | - TCP: 12 | address: 127.0.0.1:10000 13 | -------------------------------------------------------------------------------- /test/itest.sh: -------------------------------------------------------------------------------- 1 | go build -o build/gohangout-test || exit 255 2 | gohangout='build/gohangout-test' 3 | 4 | em_print() { 5 | echo "\n=======" 6 | echo $1 7 | echo "=======\n" 8 | } 9 | 10 | em_print 'test multi inputs&outputs' 11 | tmpfile=$(mktemp) 12 | 13 | $gohangout --config test/itest-1.yml > $tmpfile 14 | 15 | wcl=`wc -l $tmpfile | awk '{print $1}'` 16 | echo "$wcl lines in output" 17 | 18 | if [ "$wcl" != "4000" ] 19 | then 20 | em_print 'should output 4000 docs!' 21 | exit 255 22 | fi 23 | 24 | em_print 'test simple filters with if condition' 25 | $gohangout --config test/itest-2.yml > $tmpfile 26 | 27 | wcl=`wc -l $tmpfile | awk '{print $1}'` 28 | echo "$wcl lines in output" 29 | 30 | if [ "$wcl" != "2000" ] 31 | then 32 | em_print 'should output 2000 docs!' 33 | exit 255 34 | fi 35 | 36 | em_print 'test output with if condition' 37 | $gohangout --config test/itest-3.yml > $tmpfile 38 | 39 | wcl=`wc -l $tmpfile | awk '{print $1}'` 40 | echo "$wcl lines in output" 41 | 42 | if [ "$wcl" != "3000" ] 43 | then 44 | em_print 'should output 3000 docs!' 45 | exit 255 46 | fi 47 | 48 | em_print 'test filtersFilter' 49 | $gohangout --config test/itest-4.yml > $tmpfile 50 | 51 | wcl=`wc -l $tmpfile | awk '{print $1}'` 52 | echo "$wcl lines in output" 53 | 54 | if [ "$wcl" != "3000" ] 55 | then 56 | em_print 'should output 3000 docs!' 57 | exit 255 58 | fi 59 | 60 | wcl=`grep tag1 $tmpfile | wc -l | awk '{print $1}'` 61 | echo "$wcl tag1 lines in output" 62 | 63 | if [ "$wcl" != "3000" ] 64 | then 65 | em_print 'tag1 should output 3000 docs!' 66 | exit 255 67 | fi 68 | 69 | wcl=`grep tag2 $tmpfile | wc -l | awk '{print $1}'` 70 | echo "$wcl tag2 lines in output" 71 | 72 | if [ "$wcl" != "3000" ] 73 | then 74 | em_print 'tag2 should output 3000 docs!' 75 | exit 255 76 | fi 77 | 78 | em_print 'test LinkMetricInFilters' 79 | 80 | ($gohangout --config test/itest-6.yml && sleep 1) | $gohangout --config test/LinkMetricInFilters.yml > $tmpfile 81 | 82 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 83 | echo "$wcl metric lines in output" 84 | 85 | if [ "$wcl" != "4" ] 86 | then 87 | em_print 'metric should output 8 docs!' 88 | exit 255 89 | fi 90 | 91 | em_print 'test LinkMetric Filter 1: seperate' 92 | 93 | ($gohangout --config test/itest-6.yml && sleep 1) | $gohangout --config test/itest-6-2.yml > $tmpfile 94 | 95 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 96 | echo "$wcl metric lines in output" 97 | 98 | if [ "$wcl" != "4" ] 99 | then 100 | em_print 'metric should output 8 docs!' 101 | exit 255 102 | fi 103 | 104 | ($gohangout --config test/itest-6.yml && sleep 2) | $gohangout --config test/itest-6-2.yml > $tmpfile 105 | 106 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 107 | echo "$wcl metric lines in output" 108 | 109 | if [ "$wcl" != "4" ] 110 | then 111 | em_print 'metric should output 3 docs!' 112 | exit 255 113 | fi 114 | 115 | wcl=`grep -v count $tmpfile | wc -l | awk '{print $1}'` 116 | echo "$wcl raw lines in output" 117 | 118 | if [ "$wcl" != "1000" ] 119 | then 120 | em_print 'raw should output 1000 docs!' 121 | exit 255 122 | fi 123 | 124 | em_print 'test LinkMetric Filter 2: cumulative' 125 | 126 | ($gohangout --config test/itest-6.yml && sleep 1) | $gohangout --config test/itest-6-3.yml > $tmpfile 127 | 128 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 129 | echo "$wcl metric lines in output" 130 | 131 | if [ "$wcl" != "4" ] 132 | then 133 | em_print 'metric should output 3 docs!' 134 | exit 255 135 | fi 136 | 137 | ($gohangout --config test/itest-6.yml && sleep 2) | $gohangout --config test/itest-6-3.yml > $tmpfile 138 | 139 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 140 | echo "$wcl metric lines in output" 141 | 142 | if [ "$wcl" != "8" ] 143 | then 144 | em_print 'metric should output 8 docs!' 145 | exit 255 146 | fi 147 | 148 | wcl=`grep -v count $tmpfile | wc -l | awk '{print $1}'` 149 | echo "$wcl raw lines in output" 150 | 151 | if [ "$wcl" != "1000" ] 152 | then 153 | em_print 'raw should output 1000 docs!' 154 | exit 255 155 | fi 156 | 157 | em_print 'test LinkMetric Filter 3: seperate' 158 | 159 | ($gohangout --config test/itest-6.yml && sleep 2 && $gohangout --config test/itest-6.yml && sleep 2) | $gohangout --config test/itest-6-2.yml > $tmpfile 160 | 161 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 162 | echo "$wcl metric lines in output" 163 | 164 | if [ "$wcl" != "8" ] 165 | then 166 | em_print 'metric should output 8 docs!' 167 | exit 255 168 | fi 169 | 170 | wcl=`grep -v count $tmpfile | wc -l | awk '{print $1}'` 171 | echo "$wcl raw lines in output" 172 | 173 | if [ "$wcl" != "2000" ] 174 | then 175 | em_print 'raw should output 2000 docs!' 176 | exit 255 177 | fi 178 | 179 | em_print 'test LinkMetric Filter 4: cumulative' 180 | 181 | ($gohangout --config test/itest-6.yml && sleep 1 && $gohangout --config test/itest-6.yml && sleep 2) | $gohangout --config test/itest-6-4.yml > $tmpfile 182 | 183 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 184 | echo "$wcl metric lines in output" 185 | 186 | if [ "$wcl" != "20" ] 187 | then 188 | em_print 'metric should output 20 docs!' 189 | exit 255 190 | fi 191 | 192 | wcl=`grep -v count $tmpfile | wc -l | awk '{print $1}'` 193 | echo "$wcl raw lines in output" 194 | 195 | if [ "$wcl" != "2000" ] 196 | then 197 | em_print 'raw should output 2000 docs!' 198 | exit 255 199 | fi 200 | 201 | em_print 'test LinkStatsMetrcik Filter' 202 | 203 | ($gohangout --config test/itest-6.yml && sleep 1 && $gohangout --config test/itest-6.yml && sleep 2) | $gohangout --config test/itest-7-1.yml > $tmpfile 204 | 205 | wcl=`grep count $tmpfile | wc -l | awk '{print $1}'` 206 | echo "$wcl metric lines in output" 207 | 208 | if [ "$wcl" != "20" ] 209 | then 210 | em_print 'metric should output 20 docs!' 211 | exit 255 212 | fi 213 | 214 | # test tcp input/output 215 | test/itest-tcp.sh 216 | if [ "$?" != "0" ] 217 | then 218 | exit $? 219 | fi 220 | 221 | em_print 'pass :)' 222 | -------------------------------------------------------------------------------- /test/t.yml: -------------------------------------------------------------------------------- 1 | inputs: 2 | #- Stdin: 3 | #codec: json 4 | - Kafka: 5 | assign: 6 | healer.test: [0] 7 | codec: json 8 | consumer_settings: 9 | bootstrap.servers: "10.1.1.100:9092" 10 | group.id: gohangout.test 11 | filters: 12 | - Add: 13 | fields: 14 | xxx: xxx 15 | yyy: '[client]' 16 | zzz: '[stored][message]' 17 | '[a][b]': '[stored][message]' 18 | - Grok: 19 | src: message 20 | match: 21 | - '^(?P\S+) (?P\w+) (?P\d+)$' 22 | - '^(?P\S+) (?P\d+) (?P\w+)$' 23 | remove_fields: ['message'] 24 | - Date: 25 | location: 'Asia/Shanghai' 26 | src: logtime 27 | formats: 28 | - 'RFC3339' 29 | - '2006-01-02T15:04:05' 30 | - '2006-01-02T15:04:05Z07:00' 31 | - '2006-01-02T15:04:05Z0700' 32 | - '2006-01-02' 33 | - 'UNIX' 34 | - 'UNIX_MS' 35 | remove_fields: ["logtime"] 36 | - Filters: 37 | if: 38 | - '{{if eq .name "childe"}}y{{end}}' 39 | filters: 40 | - Add: 41 | fields: 42 | a: 'xyZ' 43 | - Lowercase: 44 | fields: 45 | - a 46 | - Drop: 47 | if: 48 | - '{{if .name}}y{{end}}' 49 | - '{{if eq .name "null"}}y{{end}}' 50 | - '{{if or (before . "-24h") (after . "24h")}}y{{end}}' 51 | - Filters: 52 | if: 53 | - '{{if eq .name "liujia"}}y{{end}}' 54 | filters: 55 | - Add: 56 | fields: 57 | b: 'xYz' 58 | - Lowercase: 59 | fields: 60 | - b 61 | 62 | outputs: 63 | - Stdout: {} 64 | -------------------------------------------------------------------------------- /topology/filter.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | import ( 4 | "reflect" 5 | 6 | "github.com/childe/gohangout/condition_filter" 7 | "github.com/childe/gohangout/field_deleter" 8 | "github.com/childe/gohangout/field_setter" 9 | "github.com/childe/gohangout/value_render" 10 | "github.com/prometheus/client_golang/prometheus" 11 | "k8s.io/klog/v2" 12 | ) 13 | 14 | type Filter interface { 15 | Filter(map[string]interface{}) (map[string]interface{}, bool) 16 | } 17 | 18 | type FilterBox struct { 19 | Filter Filter 20 | 21 | conditionFilter *condition_filter.ConditionFilter 22 | 23 | promCounter prometheus.Counter 24 | 25 | config map[interface{}]interface{} 26 | 27 | failTag string 28 | removeFields []field_deleter.FieldDeleter 29 | addFields map[field_setter.FieldSetter]value_render.ValueRender 30 | } 31 | 32 | func NewFilterBox(config map[interface{}]interface{}) *FilterBox { 33 | f := FilterBox{ 34 | config: config, 35 | conditionFilter: condition_filter.NewConditionFilter(config), 36 | promCounter: GetPromCounter(config), 37 | } 38 | 39 | if v, ok := config["failTag"]; ok { 40 | f.failTag = v.(string) 41 | } else { 42 | f.failTag = "" 43 | } 44 | 45 | if remove_fields, ok := config["remove_fields"]; ok { 46 | f.removeFields = make([]field_deleter.FieldDeleter, 0) 47 | for _, field := range remove_fields.([]interface{}) { 48 | f.removeFields = append(f.removeFields, field_deleter.NewFieldDeleter(field.(string))) 49 | } 50 | } else { 51 | f.removeFields = nil 52 | } 53 | 54 | if add_fields, ok := config["add_fields"]; ok { 55 | f.addFields = make(map[field_setter.FieldSetter]value_render.ValueRender) 56 | for k, v := range add_fields.(map[interface{}]interface{}) { 57 | fieldSetter := field_setter.NewFieldSetter(k.(string)) 58 | if fieldSetter == nil { 59 | klog.Fatalf("could build field setter from %s", k.(string)) 60 | } 61 | f.addFields[fieldSetter] = value_render.GetValueRender(v.(string)) 62 | } 63 | } else { 64 | f.addFields = nil 65 | } 66 | return &f 67 | } 68 | 69 | func (f *FilterBox) PostProcess(event map[string]interface{}, success bool) map[string]interface{} { 70 | if success { 71 | for fs, r := range f.addFields { 72 | v, _ := r.Render(event) 73 | event = fs.SetField(event, v, "", false) 74 | } 75 | if f.removeFields != nil { 76 | for _, d := range f.removeFields { 77 | d.Delete(event) 78 | } 79 | } 80 | } else { 81 | if f.failTag != "" { 82 | if tags, ok := event["tags"]; ok { 83 | if reflect.TypeOf(tags).Kind() == reflect.String { 84 | event["tags"] = []interface{}{tags.(string), f.failTag} 85 | } else if reflect.TypeOf(tags).Kind() == reflect.Array { 86 | event["tags"] = append(tags.([]interface{}), f.failTag) 87 | } 88 | } else { 89 | event["tags"] = f.failTag 90 | } 91 | } 92 | } 93 | return event 94 | } 95 | 96 | func (b *FilterBox) Process(event map[string]interface{}) map[string]interface{} { 97 | var rst bool 98 | 99 | if b.conditionFilter.Pass(event) { 100 | if b.promCounter != nil { 101 | b.promCounter.Inc() 102 | } 103 | event, rst = b.Filter.Filter(event) 104 | if event == nil { 105 | return nil 106 | } 107 | event = b.PostProcess(event, rst) 108 | } 109 | return event 110 | } 111 | 112 | type buildFilterFunc func(filterType string, config map[interface{}]interface{}) Filter 113 | 114 | func BuildFilterBoxes(config map[string]interface{}, buildFilter buildFilterFunc) []*FilterBox { 115 | if _, ok := config["filters"]; !ok { 116 | return nil 117 | } 118 | 119 | filtersI := config["filters"].([]interface{}) 120 | filters := make([]Filter, len(filtersI)) 121 | 122 | for i := 0; i < len(filters); i++ { 123 | for filterTypeI, filterConfigI := range filtersI[i].(map[interface{}]interface{}) { 124 | filterType := filterTypeI.(string) 125 | klog.Infof("filter type: %s", filterType) 126 | filterConfig := filterConfigI.(map[interface{}]interface{}) 127 | klog.Infof("filter config: %v", filterConfig) 128 | 129 | filterPlugin := buildFilter(filterType, filterConfig) 130 | 131 | filters[i] = filterPlugin 132 | } 133 | } 134 | 135 | boxes := make([]*FilterBox, len(filters)) 136 | for i := 0; i < len(filters); i++ { 137 | for _, cfg := range filtersI[i].(map[interface{}]interface{}) { 138 | boxes[i] = NewFilterBox(cfg.(map[interface{}]interface{})) 139 | boxes[i].Filter = filters[i] 140 | } 141 | } 142 | 143 | return boxes 144 | } 145 | -------------------------------------------------------------------------------- /topology/input.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | type Input interface { 4 | ReadOneEvent() map[string]interface{} 5 | Shutdown() 6 | } 7 | -------------------------------------------------------------------------------- /topology/output.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | import ( 4 | "github.com/childe/gohangout/condition_filter" 5 | "github.com/prometheus/client_golang/prometheus" 6 | "k8s.io/klog/v2" 7 | ) 8 | 9 | type Output interface { 10 | Emit(map[string]interface{}) 11 | Shutdown() 12 | } 13 | 14 | type OutputBox struct { 15 | Output 16 | *condition_filter.ConditionFilter 17 | promCounter prometheus.Counter 18 | } 19 | 20 | type buildOutputFunc func(outputType string, config map[interface{}]interface{}) *OutputBox 21 | 22 | func BuildOutputs(config map[string]interface{}, buildOutput buildOutputFunc) []*OutputBox { 23 | rst := make([]*OutputBox, 0) 24 | 25 | for _, outputs := range config["outputs"].([]interface{}) { 26 | for outputType, outputConfig := range outputs.(map[interface{}]interface{}) { 27 | outputType := outputType.(string) 28 | klog.Infof("output type: %s", outputType) 29 | outputConfig := outputConfig.(map[interface{}]interface{}) 30 | output := buildOutput(outputType, outputConfig) 31 | 32 | output.promCounter = GetPromCounter(outputConfig) 33 | 34 | rst = append(rst, output) 35 | } 36 | } 37 | return rst 38 | } 39 | 40 | // Process implement Processor interface 41 | func (p *OutputBox) Process(event map[string]interface{}) map[string]interface{} { 42 | if p.Pass(event) { 43 | if p.promCounter != nil { 44 | p.promCounter.Inc() 45 | } 46 | p.Emit(event) 47 | } 48 | return nil 49 | } 50 | 51 | type OutputsProcessor []*OutputBox 52 | 53 | // Process implement Processor interface 54 | func (p OutputsProcessor) Process(event map[string]interface{}) map[string]interface{} { 55 | for _, o := range ([]*OutputBox)(p) { 56 | if o.Pass(event) { 57 | if o.promCounter != nil { 58 | o.promCounter.Inc() 59 | } 60 | o.Emit(event) 61 | } 62 | } 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /topology/processor.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | // FilterBox and OutputBox is Processor 4 | type Processor interface { 5 | Process(map[string]interface{}) map[string]interface{} 6 | } 7 | 8 | type NilProcessorInLink struct{} 9 | 10 | func (n *NilProcessorInLink) Process(event map[string]interface{}) map[string]interface{} { 11 | return event 12 | } 13 | 14 | // ProcessorNode is a node in the filter/output link 15 | type ProcessorNode struct { 16 | Processor Processor 17 | Next *ProcessorNode 18 | } 19 | 20 | // Processor will process event , and pass it to next, and then next , until last one(generally output) 21 | func (node *ProcessorNode) Process(event map[string]interface{}) map[string]interface{} { 22 | event = node.Processor.Process(event) 23 | if event == nil || node.Next == nil { 24 | return event 25 | } 26 | 27 | return node.Next.Process(event) 28 | } 29 | 30 | // AppendProcessorsToLink add new processors to tail, return head node 31 | func AppendProcessorsToLink(head *ProcessorNode, processors ...Processor) *ProcessorNode { 32 | preHead := &ProcessorNode{nil, head} 33 | n := preHead 34 | 35 | // look for tail 36 | for n.Next != nil { 37 | n = n.Next 38 | } 39 | 40 | for _, processor := range processors { 41 | n.Next = &ProcessorNode{processor, nil} 42 | n = n.Next 43 | } 44 | 45 | return preHead.Next 46 | } 47 | -------------------------------------------------------------------------------- /topology/prom_counter.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | import ( 4 | "encoding/json" 5 | "sync" 6 | 7 | "github.com/mitchellh/mapstructure" 8 | "github.com/prometheus/client_golang/prometheus" 9 | "github.com/prometheus/client_golang/prometheus/promauto" 10 | "k8s.io/klog/v2" 11 | ) 12 | 13 | var lock = sync.Mutex{} 14 | var counterManager map[string]prometheus.Counter = make(map[string]prometheus.Counter) 15 | 16 | func hashValue(opts prometheus.CounterOpts) string { 17 | opts.Help = "" 18 | b, _ := json.Marshal(opts) 19 | return string(b) 20 | } 21 | 22 | // GetPromCounter creates a prometheus.Counter from config. 23 | // if same config exsits before, GetPromCounter would return the counter created before. Because tow counters with the same config leads to panic. 24 | // Better practice maybe to let it panic, so owner can fix the config when program fails to start. 25 | // But if user use multi workers to run gohangout, panic are bound to happen, this is bad. So we use a manager to return one counter with save config. 26 | // Better way is to add {worker: idx} to ConstLabels, but it is too hard to implement it by code. 27 | func GetPromCounter(config map[interface{}]interface{}) prometheus.Counter { 28 | lock.Lock() 29 | defer lock.Unlock() 30 | if promConf, ok := config["prometheus_counter"]; ok { 31 | // promConf := promConf.(map[interface{}]interface{}) 32 | 33 | var opts prometheus.CounterOpts = prometheus.CounterOpts{} 34 | err := mapstructure.Decode(promConf, &opts) 35 | if err != nil { 36 | klog.Errorf("marshal prometheus counter config error: %v", err) 37 | return nil 38 | } 39 | 40 | key := hashValue(opts) 41 | 42 | if v, ok := counterManager[key]; ok { 43 | return v 44 | } 45 | c := promauto.NewCounter(opts) 46 | counterManager[key] = c 47 | return c 48 | } 49 | 50 | return nil 51 | } 52 | -------------------------------------------------------------------------------- /topology/prom_counter_test.go: -------------------------------------------------------------------------------- 1 | package topology 2 | 3 | import "testing" 4 | 5 | func TestGetPromCounter(t *testing.T) { 6 | type TestCase struct { 7 | config map[interface{}]interface{} 8 | want bool 9 | } 10 | 11 | for _, c := range []TestCase{ 12 | { 13 | config: nil, 14 | want: false, 15 | }, 16 | { 17 | config: map[interface{}]interface{}{"prometheus_counter": "test"}, 18 | want: false, 19 | }, 20 | { 21 | config: map[interface{}]interface{}{"prometheus_counter": map[string]string{ 22 | "name": "gohangout_add_filter", 23 | "namespace": "rack_a", 24 | "help": "rack_a gohangout add filter counter", 25 | }}, 26 | want: true, 27 | }, 28 | { 29 | config: map[interface{}]interface{}{"prometheus_counter": map[string]string{ 30 | "name": "gohangout_add_filter", 31 | "namespace": "rack_a", 32 | "help": "rack_a gohangout add filter counter", 33 | }}, 34 | want: true, 35 | }, 36 | { 37 | config: map[interface{}]interface{}{"prometheus_counter": map[string]string{ 38 | "name": "gohangout_add_filter", 39 | "namespace": "rack_a", 40 | "help": "xxxxxxxxxxx", 41 | }}, 42 | want: true, 43 | }, 44 | { 45 | config: map[interface{}]interface{}{"prometheus_counter": map[string]string{ 46 | "name": "gohangout_raname_filter", 47 | "namespace": "rack_a", 48 | "help": "rack_a gohangout add filter counter", 49 | }}, 50 | want: true, 51 | }, 52 | } { 53 | counter := GetPromCounter(c.config) 54 | if (counter != nil) != c.want { 55 | t.Errorf("GetPromCounter(%v) = %v, want %v", c.config, counter != nil, c.want) 56 | } 57 | } 58 | 59 | if len(counterManager) != 2 { 60 | t.Errorf("len(counterManager) = %v, want %v", len(counterManager), 2) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /value_render/index_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | // used for ES indexname template 4 | 5 | import ( 6 | "encoding/json" 7 | "errors" 8 | "reflect" 9 | "regexp" 10 | "strings" 11 | "time" 12 | 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | type field struct { 17 | literal bool 18 | date bool 19 | value string // used in datetime %{+} and literal 20 | location *time.Location 21 | 22 | mv *MultiLevelValueRender 23 | } 24 | 25 | var errNotString = errors.New("field is not string") 26 | 27 | // render returns error , but it is not used in the caller 28 | // always use "null" as a result when err is not nil 29 | // for compatibility 30 | func (f *field) render(event map[string]interface{}, location *time.Location) (string, error) { 31 | if f.literal { 32 | return f.value, nil 33 | } 34 | 35 | if f.date { 36 | if t, ok := event["@timestamp"]; ok { 37 | return dateFormat(t, f.value, location) 38 | } else { 39 | return dateFormat(time.Now(), f.value, location) 40 | } 41 | } 42 | v, err := f.mv.Render(event) 43 | if err != nil { 44 | return "null", err 45 | } 46 | 47 | if v, ok := v.(string); ok { 48 | return v, nil 49 | } 50 | return "null", errNotString 51 | } 52 | 53 | type IndexRender struct { 54 | fields []*field 55 | location *time.Location 56 | } 57 | 58 | // getAllFields("%{@metadata}{kafka}{topic}") => ["@metadata","kafka","topic"] 59 | func getAllFields(s string) []string { 60 | fields := make([]string, 0) 61 | r, _ := regexp.Compile(`{(.*?)}`) 62 | for _, v := range r.FindAll([]byte(s), -1) { 63 | fields = append(fields, string(v[1:len(v)-1])) 64 | } 65 | return fields 66 | } 67 | 68 | func NewIndexRender(t string) *IndexRender { 69 | r, _ := regexp.Compile(`%({.*?})+`) 70 | fields := make([]*field, 0) 71 | lastPos := 0 72 | for _, loc := range r.FindAllStringIndex(t, -1) { 73 | s, e := loc[0], loc[1] 74 | fields = append(fields, &field{ 75 | literal: true, 76 | value: t[lastPos:s], 77 | }) 78 | 79 | if t[s+2] == '+' { 80 | fields = append(fields, &field{ 81 | literal: false, 82 | date: true, 83 | value: t[s+3 : e-1], 84 | }) 85 | } else { 86 | fields = append(fields, &field{ 87 | literal: false, 88 | date: false, 89 | mv: NewMultiLevelValueRender(getAllFields(t[s+1 : e])), 90 | }) 91 | } 92 | 93 | lastPos = e 94 | } 95 | 96 | if lastPos < len(t) { 97 | fields = append(fields, &field{ 98 | literal: true, 99 | date: false, 100 | value: t[lastPos:], 101 | }) 102 | } 103 | return &IndexRender{fields, time.UTC} 104 | } 105 | 106 | // SetTimeLocation parse `location` to time.Location ans set it as its member. 107 | // use this location to format time string 108 | func (r *IndexRender) SetTimeLocation(loc string) { 109 | location, err := time.LoadLocation(loc) 110 | if err != nil { 111 | klog.Fatalf("invalid localtion: %s", loc) 112 | } 113 | r.location = location 114 | } 115 | 116 | func dateFormat(t interface{}, format string, location *time.Location) (string, error) { 117 | if t1, ok := t.(time.Time); ok { 118 | return t1.In(location).Format(format), nil 119 | } 120 | if reflect.TypeOf(t).String() == "json.Number" { 121 | t1, err := t.(json.Number).Int64() 122 | if err != nil { 123 | return format, err 124 | } 125 | return time.Unix(t1/1000, t1%1000*1000000).In(location).Format(format), nil 126 | } 127 | if reflect.TypeOf(t).Kind() == reflect.Int { 128 | t1 := int64(t.(int)) 129 | return time.Unix(t1/1000, t1%1000*1000000).In(location).Format(format), nil 130 | } 131 | if reflect.TypeOf(t).Kind() == reflect.Int64 { 132 | t1 := t.(int64) 133 | return time.Unix(t1/1000, t1%1000*1000000).In(location).Format(format), nil 134 | } 135 | if reflect.TypeOf(t).Kind() == reflect.String { 136 | t1, e := time.Parse(time.RFC3339, t.(string)) 137 | if e != nil { 138 | return format, e 139 | } 140 | return t1.In(location).Format(format), nil 141 | } 142 | return format, errors.New("could not tell the type timestamp field belongs to") 143 | } 144 | 145 | // Render implements ValueRender. note: it's field use "null" as result when error occurs for compatibility 146 | func (r *IndexRender) Render(event map[string]interface{}) (value interface{}, err error) { 147 | fields := make([]string, len(r.fields)) 148 | for i, f := range r.fields { 149 | v, _ := f.render(event, r.location) 150 | fields[i] = v 151 | } 152 | return strings.Join(fields, ""), nil 153 | } 154 | -------------------------------------------------------------------------------- /value_render/index_render_test.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestIndexRender(t *testing.T) { 9 | ts, _ := time.Parse("2006-01-02", "2022-03-04") 10 | for _, c := range []struct { 11 | event map[string]interface{} 12 | template string 13 | want string 14 | }{ 15 | { 16 | event: map[string]interface{}{ 17 | "@timestamp": ts, 18 | }, 19 | template: "%{+2006.01.02}", 20 | want: "2022.03.04", 21 | }, 22 | { 23 | event: map[string]interface{}{ 24 | "@timestamp": ts, 25 | }, 26 | template: "app-%{+2006.01.02}", 27 | want: "app-2022.03.04", 28 | }, 29 | { 30 | event: map[string]interface{}{ 31 | "@timestamp": ts, 32 | }, 33 | template: "%{+2006.01.02}-log", 34 | want: "2022.03.04-log", 35 | }, 36 | { 37 | event: map[string]interface{}{ 38 | "@timestamp": ts, 39 | }, 40 | template: "app-%{+2006.01.02}-log", 41 | want: "app-2022.03.04-log", 42 | }, 43 | { 44 | event: map[string]interface{}{ 45 | "topic": "topic-one", 46 | "@timestamp": ts, 47 | }, 48 | template: "app-%{topic}-%{+2006.01.02}-log", 49 | want: "app-topic-one-2022.03.04-log", 50 | }, 51 | { 52 | event: map[string]interface{}{ 53 | "@timestamp": ts, 54 | }, 55 | template: "app-%{topic}-%{+2006.01.02}-log", 56 | want: "app-null-2022.03.04-log", 57 | }, 58 | { 59 | event: map[string]interface{}{ 60 | "@timestamp": ts, 61 | }, 62 | template: "app-%{@metadata}{kafka}{topic}-%{+2006.01.02}-log", 63 | want: "app-null-2022.03.04-log", 64 | }, 65 | { 66 | event: map[string]interface{}{ 67 | "@metadata": nil, 68 | "@timestamp": ts, 69 | }, 70 | template: "app-%{@metadata}{kafka}{topic}-%{+2006.01.02}-log", 71 | want: "app-null-2022.03.04-log", 72 | }, 73 | { 74 | event: map[string]interface{}{ 75 | "@metadata": map[string]interface{}{ 76 | "topic": "topic-one", 77 | }, 78 | "@timestamp": ts, 79 | }, 80 | template: "app-%{@metadata}{kafka}{topic}-%{+2006.01.02}-log", 81 | want: "app-null-2022.03.04-log", 82 | }, 83 | { 84 | event: map[string]interface{}{ 85 | "@metadata": map[string]interface{}{ 86 | "kafka": map[string]interface{}{ 87 | "topic": "topic-one", 88 | }, 89 | }, 90 | "@timestamp": ts, 91 | }, 92 | template: "app-%{@metadata}{kafka}{topic}-%{+2006.01.02}-log", 93 | want: "app-topic-one-2022.03.04-log", 94 | }, 95 | } { 96 | vr := NewIndexRender(c.template) 97 | got, err := vr.Render(c.event) 98 | if err != nil { 99 | t.Errorf("err:%s\n", err) 100 | } 101 | if c.want != got { 102 | t.Errorf("render %q, want %s, got %s", c.template, c.want, got) 103 | } 104 | } 105 | var event map[string]interface{} 106 | var template string 107 | var vr ValueRender 108 | 109 | // timestamp exists, appid missing 110 | event = make(map[string]interface{}) 111 | event["@timestamp"], _ = time.Parse("2006-01-02T15:04:05", "2019-03-04T14:21:00") 112 | 113 | template = "nginx-%{appid}-%{+2006.01.02}" 114 | 115 | vr = NewIndexRender(template) 116 | indexname, err := vr.Render(event) 117 | 118 | if err != nil { 119 | t.Errorf("err:%s\n", err) 120 | } 121 | 122 | if indexname != "nginx-null-2019.03.04" { 123 | t.Errorf("%s != nginx-null-2019.03.04\n", indexname) 124 | } 125 | 126 | } 127 | -------------------------------------------------------------------------------- /value_render/jsonpath_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import "github.com/oliveagle/jsonpath" 4 | 5 | type JsonpathRender struct { 6 | Pat *jsonpath.Compiled 7 | } 8 | 9 | func (r *JsonpathRender) Render(event map[string]interface{}) (value interface{}, err error) { 10 | return r.Pat.Lookup(event) 11 | } 12 | -------------------------------------------------------------------------------- /value_render/jsonpath_render_test.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func TestJsonpathRender(t *testing.T) { 8 | var event map[string]interface{} 9 | var template string 10 | var vr ValueRender 11 | 12 | event = make(map[string]interface{}) 13 | event["msg"] = "this is msg line" 14 | 15 | template = "$.msg" 16 | 17 | vr = GetValueRender(template) 18 | value, err := vr.Render(event) 19 | t.Log(value) 20 | 21 | if err != nil { 22 | t.Errorf("err != nil") 23 | } 24 | 25 | if value != "this is msg line" { 26 | t.Errorf("%q != %q", value, "this is msg line") 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /value_render/literal_value_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | type LiteralValueRender struct { 4 | value string 5 | } 6 | 7 | func NewLiteralValueRender(template string) *LiteralValueRender { 8 | return &LiteralValueRender{template} 9 | } 10 | 11 | func (r *LiteralValueRender) Render(event map[string]interface{}) (value interface{}, err error) { 12 | return r.value, nil 13 | } 14 | -------------------------------------------------------------------------------- /value_render/mfields_value_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | // MultiLevelValueRender is a ValueRender that can render [xxx][yyy][zzz] 4 | type MultiLevelValueRender struct { 5 | preFields []string 6 | lastField string 7 | } 8 | 9 | // NewMultiLevelValueRender create a MultiLevelValueRender 10 | func NewMultiLevelValueRender(fields []string) *MultiLevelValueRender { 11 | fieldsLength := len(fields) 12 | preFields := make([]string, fieldsLength-1) 13 | for i := range preFields { 14 | preFields[i] = fields[i] 15 | } 16 | 17 | return &MultiLevelValueRender{ 18 | preFields: preFields, 19 | lastField: fields[fieldsLength-1], 20 | } 21 | } 22 | 23 | // Render implements ValueRender 24 | func (vr *MultiLevelValueRender) Render(event map[string]interface{}) (value interface{}, err error) { 25 | var current map[string]interface{} = event 26 | for _, field := range vr.preFields { 27 | v, ok := current[field] 28 | if !ok { 29 | return nil, ErrNotExist 30 | } 31 | if current, ok = v.(map[string]interface{}); !ok { 32 | return nil, ErrInvalidType 33 | } 34 | } 35 | 36 | if v, ok := current[vr.lastField]; ok { 37 | return v, nil 38 | } else { 39 | return nil, ErrNotExist 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /value_render/mfields_value_render_test.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | type mfieldsTestCase struct { 9 | event map[string]interface{} 10 | fields []string 11 | 12 | hasError bool 13 | want interface{} 14 | } 15 | 16 | func TestMultiLevelValueRender(t *testing.T) { 17 | for _, c := range []mfieldsTestCase{ 18 | { 19 | event: map[string]interface{}{ 20 | "a": map[string]interface{}{ 21 | "b": map[string]interface{}{ 22 | "c": "c", 23 | }, 24 | }, 25 | }, 26 | fields: []string{"a", "b", "c"}, 27 | hasError: false, 28 | want: "c", 29 | }, 30 | { 31 | event: map[string]interface{}{ 32 | "a": map[string]interface{}{ 33 | "b": map[string]interface{}{ 34 | "c": "c", 35 | }, 36 | }, 37 | }, 38 | fields: []string{"a", "b", "d"}, 39 | hasError: true, 40 | want: nil, 41 | }, 42 | { 43 | event: map[string]interface{}{ 44 | "a": map[string]interface{}{ 45 | "b": map[string]interface{}{ 46 | "c": "c", 47 | }, 48 | }, 49 | }, 50 | fields: []string{"a", "b", "c", "d"}, 51 | hasError: true, 52 | want: nil, 53 | }, 54 | { 55 | event: map[string]interface{}{ 56 | "a": map[string]interface{}{ 57 | "b": map[string]interface{}{ 58 | "c": "c", 59 | }, 60 | }, 61 | }, 62 | fields: []string{"a", "b", "c", "d", "e"}, 63 | hasError: true, 64 | want: nil, 65 | }, 66 | { 67 | event: map[string]interface{}{ 68 | "a": map[string]interface{}{ 69 | "b": map[string]interface{}{ 70 | "c": 10, 71 | }, 72 | }, 73 | }, 74 | fields: []string{"a", "b", "c"}, 75 | hasError: false, 76 | want: 10, 77 | }, 78 | { 79 | event: map[string]interface{}{ 80 | "a": map[string]interface{}{ 81 | "b": 10, 82 | }, 83 | }, 84 | fields: []string{"a", "b", "c"}, 85 | hasError: true, 86 | want: nil, 87 | }, 88 | } { 89 | v := NewMultiLevelValueRender(c.fields) 90 | got, err := v.Render(c.event) 91 | 92 | if c.hasError != (err != nil) { 93 | t.Errorf("if has error, case: %v, want %v, got %v", c, c.hasError, err != nil) 94 | } 95 | 96 | if !reflect.DeepEqual(got, c.want) { 97 | t.Errorf("case: %v, want %q, got %q", c, c.want, got) 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /value_render/one_level_value_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | type OneLevelValueRender struct { 4 | field string 5 | } 6 | 7 | func NewOneLevelValueRender(template string) *OneLevelValueRender { 8 | return &OneLevelValueRender{ 9 | field: template, 10 | } 11 | } 12 | 13 | func (vr *OneLevelValueRender) Render(event map[string]interface{}) (value interface{}, err error) { 14 | if value, ok := event[vr.field]; ok { 15 | return value, nil 16 | } 17 | return false, ErrNotExist 18 | } 19 | -------------------------------------------------------------------------------- /value_render/template_value_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "errors" 7 | "reflect" 8 | "strings" 9 | "text/template" 10 | "time" 11 | 12 | "github.com/Masterminds/sprig/v3" 13 | "k8s.io/klog/v2" 14 | ) 15 | 16 | type TemplateValueRender struct { 17 | tmpl *template.Template 18 | } 19 | 20 | var GOHANGOUT_TYPE_UNKNOWN_ERROR error = errors.New("field type unknown, it must be of json.Number|Int64|Int|int8") 21 | 22 | var ErrNotFloat64 error = errors.New("Only float64 type value could be calculated") 23 | var ErrNotInt64 error = errors.New("Only int64 type value could be calculated") 24 | 25 | var funcMap = template.FuncMap{} 26 | 27 | func convertToInt(x interface{}) (int, error) { 28 | if reflect.TypeOf(x).String() == "json.Number" { 29 | b, _ := x.(json.Number).Int64() 30 | return int(b), nil 31 | } else if reflect.TypeOf(x).Kind() == reflect.Int64 { 32 | return int(x.(int64)), nil 33 | } else if reflect.TypeOf(x).Kind() == reflect.Int { 34 | return x.(int), nil 35 | } else if reflect.TypeOf(x).Kind() == reflect.Int8 { 36 | return int(x.(int8)), nil 37 | } 38 | return 0, GOHANGOUT_TYPE_UNKNOWN_ERROR 39 | } 40 | 41 | func init() { 42 | for k, v := range sprig.FuncMap() { 43 | funcMap[k] = v 44 | } 45 | 46 | funcMap["compare"] = strings.Compare 47 | funcMap["contains"] = strings.Contains 48 | funcMap["containsAny"] = strings.ContainsAny 49 | funcMap["hasprefix"] = strings.HasPrefix 50 | funcMap["hassuffix"] = strings.HasSuffix 51 | funcMap["replace"] = strings.Replace 52 | 53 | funcMap["timeFormat"] = func(t time.Time, format string) string { 54 | return t.Format(format) 55 | } 56 | 57 | funcMap["now"] = func() int64 { return time.Now().UnixNano() / 1000000 } 58 | funcMap["timestamp"] = func(event map[string]interface{}) int64 { 59 | timestamp := event["@timestamp"] 60 | if timestamp == nil { 61 | return 0 62 | } 63 | if reflect.TypeOf(timestamp).String() == "time.Time" { 64 | return timestamp.(time.Time).UnixNano() / 1000000 65 | } 66 | return 0 67 | } 68 | 69 | funcMap["before"] = func(event map[string]interface{}, s string) bool { 70 | timestamp := event["@timestamp"] 71 | if timestamp == nil || reflect.TypeOf(timestamp).String() != "time.Time" { 72 | return false 73 | } 74 | d, err := time.ParseDuration(s) 75 | if err != nil { 76 | klog.Error(err) 77 | return false 78 | } 79 | dst := time.Now().Add(d) 80 | return timestamp.(time.Time).Before(dst) 81 | } 82 | 83 | funcMap["after"] = func(event map[string]interface{}, s string) bool { 84 | timestamp := event["@timestamp"] 85 | if timestamp == nil || reflect.TypeOf(timestamp).String() != "time.Time" { 86 | return false 87 | } 88 | d, err := time.ParseDuration(s) 89 | if err != nil { 90 | klog.Error(err) 91 | return false 92 | } 93 | dst := time.Now().Add(d) 94 | return timestamp.(time.Time).After(dst) 95 | } 96 | 97 | funcMap["plus"] = func(x, y interface{}) (float64, error) { 98 | if xf, ok := x.(float64); ok { 99 | if yf, ok := y.(float64); ok { 100 | return xf + yf, nil 101 | } 102 | } 103 | return 0, ErrNotFloat64 104 | } 105 | 106 | funcMap["minus"] = func(x, y interface{}) (float64, error) { 107 | if xf, ok := x.(float64); ok { 108 | if yf, ok := y.(float64); ok { 109 | return xf - yf, nil 110 | } 111 | } 112 | return 0, ErrNotFloat64 113 | } 114 | funcMap["multiply"] = func(x, y interface{}) (float64, error) { 115 | if xf, ok := x.(float64); ok { 116 | if yf, ok := y.(float64); ok { 117 | return xf * yf, nil 118 | } 119 | } 120 | return 0, ErrNotFloat64 121 | } 122 | funcMap["divide"] = func(x, y interface{}) (float64, error) { 123 | if xf, ok := x.(float64); ok { 124 | if yf, ok := y.(float64); ok { 125 | return xf / yf, nil 126 | } 127 | } 128 | return 0, ErrNotFloat64 129 | } 130 | funcMap["mod"] = func(x, y interface{}) (int64, error) { 131 | if xf, ok := x.(int64); ok { 132 | if yf, ok := y.(int64); ok { 133 | return xf % yf, nil 134 | } 135 | } 136 | return 0, ErrNotInt64 137 | } 138 | } 139 | 140 | func NewTemplateValueRender(t string) *TemplateValueRender { 141 | tmpl, err := template.New(t).Funcs(funcMap).Parse(t) 142 | if err != nil { 143 | klog.Fatalf("could not parse template %s:%s", t, err) 144 | } 145 | return &TemplateValueRender{ 146 | tmpl: tmpl, 147 | } 148 | } 149 | 150 | // Render return "exist" and value. 151 | // But the returned "exist" is meaningless; the user needs to see if the "value" is nil. 152 | func (r *TemplateValueRender) Render(event map[string]interface{}) (value interface{}, err error) { 153 | b := bytes.NewBuffer(nil) 154 | if err := r.tmpl.Execute(b, event); err != nil { 155 | return nil, err 156 | } 157 | return b.String(), nil 158 | } 159 | -------------------------------------------------------------------------------- /value_render/value_render.go: -------------------------------------------------------------------------------- 1 | package value_render 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | 7 | "github.com/oliveagle/jsonpath" 8 | ) 9 | 10 | var matchp, matchGoTemp, matchESIndex, jsonPath *regexp.Regexp 11 | 12 | func init() { 13 | matchp, _ = regexp.Compile(`^(\[.*?\])+$`) 14 | matchGoTemp, _ = regexp.Compile(`{{.*}}`) 15 | matchESIndex, _ = regexp.Compile(`%{.*?}`) //%{+YYYY.MM.dd} 16 | jsonPath, _ = regexp.Compile(`^\$\.`) 17 | } 18 | 19 | var ErrNotExist = fmt.Errorf("field does not exist") 20 | var ErrInvalidType = fmt.Errorf("field is not a valid type") 21 | 22 | type ValueRender interface { 23 | Render(map[string]interface{}) (value interface{}, err error) 24 | } 25 | 26 | // getValueRender matches all regexp pattern and return a ValueRender 27 | // return nil if no pattern matched 28 | func getValueRender(template string) ValueRender { 29 | if matchp.Match([]byte(template)) { 30 | findp, _ := regexp.Compile(`(\[(.*?)\])`) 31 | fields := make([]string, 0) 32 | for _, v := range findp.FindAllStringSubmatch(template, -1) { 33 | fields = append(fields, v[2]) 34 | } 35 | 36 | if len(fields) == 1 { 37 | return NewOneLevelValueRender(fields[0]) 38 | } 39 | return NewMultiLevelValueRender(fields) 40 | } 41 | if matchGoTemp.Match([]byte(template)) { 42 | return NewTemplateValueRender(template) 43 | } 44 | if matchESIndex.Match([]byte(template)) { 45 | return NewIndexRender(template) 46 | } 47 | if jsonPath.Match([]byte(template)) { 48 | pat, err := jsonpath.Compile(template) 49 | if err != nil { 50 | panic(fmt.Sprintf("json path compile `%s` error: %s", template, err)) 51 | } 52 | return &JsonpathRender{pat} 53 | } 54 | 55 | return nil 56 | } 57 | 58 | // GetValueRender return a ValueRender, and return LiteralValueRender if no pattern matched 59 | func GetValueRender(template string) ValueRender { 60 | r := getValueRender(template) 61 | if r != nil { 62 | return r 63 | } 64 | return NewLiteralValueRender(template) 65 | } 66 | 67 | // GetValueRender2 return a ValueRender, and return OneLevelValueRender("message") if no pattern matched 68 | func GetValueRender2(template string) ValueRender { 69 | r := getValueRender(template) 70 | if r != nil { 71 | return r 72 | } 73 | return NewOneLevelValueRender(template) 74 | } 75 | --------------------------------------------------------------------------------