├── .gitignore ├── .golangci.yml ├── .travis.yml ├── LICENSE.md ├── Makefile ├── README.md ├── cmd ├── bm │ ├── README.md │ ├── fluent_forward_go │ │ └── bm_test.go │ ├── fluent_logger_golang │ │ └── bm_test.go │ └── helpers.go ├── forward │ └── main.go └── ws │ ├── listener.go │ └── main.go ├── fixtures └── fluent.conf ├── fluent ├── client │ ├── client.go │ ├── client_suite_test.go │ ├── client_test.go │ ├── clientfakes │ │ ├── cert.pem │ │ ├── fake_client_factory.go │ │ ├── fake_connection_factory.go │ │ ├── fake_message_client.go │ │ ├── fake_wsconnection_factory.go │ │ └── key.pem │ ├── conn_factory.go │ ├── conn_factory_test.go │ ├── errors.go │ ├── ws │ │ ├── connection.go │ │ ├── connection_test.go │ │ ├── ext │ │ │ ├── conn.go │ │ │ └── extfakes │ │ │ │ └── fake_conn.go │ │ ├── ws_suite_test.go │ │ └── wsfakes │ │ │ └── fake_connection.go │ ├── ws_client.go │ └── ws_client_test.go ├── fluent_suite_test.go └── protocol │ ├── .message_test.go.swp │ ├── chunk.go │ ├── chunk_test.go │ ├── forward_message.go │ ├── forward_message_gen.go │ ├── forward_message_gen_test.go │ ├── forward_message_test.go │ ├── handshake.go │ ├── handshake_gen.go │ ├── handshake_gen_test.go │ ├── message.go │ ├── message_gen.go │ ├── message_gen_test.go │ ├── message_test.go │ ├── packed_forward_message.go │ ├── packed_forward_message_gen.go │ ├── packed_forward_message_gen_test.go │ ├── packed_forward_message_test.go │ ├── protocol_suite_test.go │ ├── protocolfakes │ └── forwarded_records.msgpack.bin │ ├── transport.go │ ├── transport_gen.go │ ├── transport_gen_test.go │ └── transport_test.go ├── go.mod ├── go.sum ├── scripts └── lint.sh └── test └── run-tests.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | .vscode/* 3 | !.vscode/settings.json 4 | !.vscode/tasks.json 5 | !.vscode/launch.json 6 | !.vscode/extensions.json 7 | *.code-workspace 8 | vendor/ 9 | # Local History for Visual Studio Code 10 | .history/ 11 | test.message* 12 | foo* 13 | bar* 14 | myapp* 15 | *.out 16 | *.test 17 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | run: 2 | # timeout for analysis, e.g. 30s, 5m, default is 1m 3 | timeout: 3m 4 | linters: 5 | # enable some additional (non-default) linters 6 | enable: 7 | - revive 8 | - bodyclose 9 | - exportloopref 10 | - gocognit 11 | - goconst 12 | - gofmt 13 | - gosec 14 | - misspell 15 | - nakedret 16 | - unconvert 17 | - unparam 18 | - wsl 19 | issues: 20 | exclude-rules: 21 | # exclude some linters from running on tests files. 22 | - path: _test\.go 23 | linters: 24 | - errcheck 25 | - exportloopref 26 | - gocognit 27 | - goconst 28 | - gosec 29 | - unparam 30 | - wsl 31 | - revive 32 | - path: '(.+)_test.go' 33 | linters: 34 | - revive 35 | text: "dot-imports: should not use dot imports" 36 | # maximum count of issues with the same text. set to 0 to disable. default is 3. 37 | max-same-issues: 0 38 | linters-settings: 39 | goconst: 40 | min-len: 2 41 | min-occurrences: 5 42 | misspell: 43 | locale: US 44 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | --- 2 | language: go 3 | os: linux 4 | dist: focal 5 | go: 6 | - "1.20.14" 7 | 8 | before_install: 9 | - pip --quiet install yamllint 10 | - go get github.com/sonatype-nexus-community/nancy 11 | - go mod tidy 12 | 13 | cache: 14 | directories: 15 | - $HOME/.cache/go-build 16 | - $HOME/gopath/pkg/mod 17 | 18 | jobs: 19 | include: 20 | - stage: unit tests 21 | script: 22 | - test/run-tests.sh ; [[ "$?" == "0" ]] || (travis_terminate 1) 23 | if: type = pull_request 24 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 International Business Machines 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | GOFILES=$(shell find . -type f -name '*.go' -not -path "./vendor/*") 2 | 3 | .PHONY: lintall 4 | lintall: fmt lint 5 | 6 | .PHONY: 7 | lint: 8 | go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.0 9 | golangci-lint run ./... 10 | 11 | .PHONY: fmt 12 | fmt: 13 | @gofmt -d ${GOFILES}; \ 14 | if [ -n "$$(gofmt -l ${GOFILES})" ]; then \ 15 | echo "Please run 'make dofmt'" && exit 1; \ 16 | fi 17 | 18 | .PHONY: travis-lint 19 | travis-lint: 20 | yamllint .travis.yml 21 | 22 | .PHONY: gosec 23 | gosec: 24 | go get github.com/securego/gosec/cmd/gosec 25 | gosec -quiet --exclude=G104 ./... 26 | 27 | .PHONY: scan-nancy 28 | scan-nancy: 29 | go mod tidy 30 | go list -json -m all | nancy sleuth --skip-update-check 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-forward-go 2 | 3 | `fluent-forward-go` is a fast, memory-efficient implementation of the [Fluent Forward v1 specification](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1). It allows you to send events to [Fluentd](https://www.fluentd.org/), [Fluent Bit](https://fluentbit.io/), and other endpoints supporting the Fluent protocol. It also includes a websocket client for high-speed proxying of Fluent events over ports such as `80` and `443`. 4 | 5 | Features include: 6 | 7 | - TCP, TLS, mTLS, and unix socket transport 8 | - shared-key authentication 9 | - support for all [Fluent message modes](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#message-modes) 10 | - [`gzip` compression](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#compressedpackedforward-mode) 11 | - ability to send byte-encoded messages 12 | - `ack` support 13 | - a websocket client for proxying Fluent messages 14 | 15 | 16 | ## Installation 17 | 18 | ```shell 19 | go get github.com/IBM/fluent-forward-go 20 | ``` 21 | 22 | ## Examples 23 | 24 | ### Create a TCP client 25 | 26 | ```go 27 | c := client.New(client.ConnectionOptions{ 28 | Factory: &client.ConnFactory{ 29 | Address: "localhost:24224", 30 | }, 31 | }) 32 | if err := c.Connect(); err != nil { 33 | // ... 34 | } 35 | defer c.Disconnect() 36 | ``` 37 | 38 | ### Create a TLS client 39 | 40 | ```go 41 | c := client.New(client.ConnectionOptions{ 42 | Factory: &client.ConnFactory{ 43 | Address: "localhost:24224", 44 | TLSConfig: &tls.Config{InsecureSkipVerify: true}, 45 | }, 46 | }) 47 | if err := c.Connect(); err != nil { 48 | // ... 49 | } 50 | defer c.Disconnect() 51 | ``` 52 | 53 | ### Send a new log message 54 | 55 | The `record` object must be a `map` or `struct`. Objects that implement the [`msgp.Encodable`](https://pkg.go.dev/github.com/tinylib/msgp/msgp#Encodable) interface will the be most performant. 56 | 57 | ```go 58 | record := map[string]interface{}{ 59 | "Hello": "World", 60 | } 61 | err := c.SendMessage("tag", record) 62 | ``` 63 | 64 | ### Send a byte-encoded message 65 | 66 | ```go 67 | err := c.SendRaw(myMessageBytes) 68 | ``` 69 | 70 | ### Message confirmation 71 | 72 | The client supports `ack` confirmations as specified by the Fluent protocol. When enabled, `Send` returns once the acknowledgement is received or the timeout is reached. 73 | 74 | Note: For types other than `RawMessage`, the `Send` function sets the "chunk" option before sending. A `RawMessage` is immutable and must already contain a "chunk" value. The behavior is otherwise identical. 75 | 76 | ```go 77 | c := client.New(client.ConnectionOptions{ 78 | RequireAck: true, 79 | }) 80 | //... 81 | err := c.Send(myMsg) 82 | ``` 83 | 84 | ## Performance 85 | 86 | **tl;dr** `fluent-forward-go` is fast and memory efficient. 87 | 88 | You can read more about the benchmarks [here](cmd/bm/README.md). 89 | 90 | ### Send 91 | 92 | Run on `localhost`. Does not include message creation. 93 | 94 | ```shell 95 | Benchmark_Fluent_Forward_Go_SendOnly-16 10000 10847 ns/op 0 B/op 0 allocs/op 96 | ``` 97 | 98 | ### Comparisons with `fluent-logger-golang` 99 | 100 | The benchmarks below compare `fluent-forward-go` with the official package, [`fluent-logger-golang`](https://github.com/fluent/fluent-logger-golang). The message is a simple map with twelve keys. 101 | 102 | The differences in execution times can vary from one test run to another. The differences in memory allocations, however, are constant. 103 | 104 | #### Send a single message 105 | 106 | ```shell 107 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 11355 ns/op 48 B/op 1 allocs/op 108 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19687 ns/op 2169 B/op 33 allocs/op 109 | ``` 110 | 111 | #### Send a single message with confirmation 112 | 113 | ```shell 114 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 768743 ns/op 185 B/op 6 allocs/op 115 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 793360 ns/op 6015 B/op 47 allocs/op 116 | ``` 117 | 118 | ## Developing 119 | 120 | ### Installation instructions 121 | 122 | Before running the generate tool, you must have msgp installed. To install run: 123 | 124 | ```shell 125 | go get github.com/tinylib/msgp 126 | ``` 127 | 128 | Afterwards, generate the msgp packets with: 129 | 130 | ```shell 131 | go generate ./... 132 | ``` 133 | 134 | ### Testing 135 | 136 | To test against fluent-bit, start up fluent-bit in a docker container with: 137 | 138 | ```shell 139 | docker pull fluent/fluent-bit:1.8.2 140 | docker run -p 127.0.0.1:24224:24224/tcp -v `pwd`:`pwd` -w `pwd` \ 141 | -ti fluent/fluent-bit:1.8.2 /fluent-bit/bin/fluent-bit \ 142 | -c $(pwd)/fixtures/fluent.conf 143 | ``` 144 | 145 | You can then build and run with: 146 | 147 | ```shell 148 | go run ./cmd/forward -t foo.bar 149 | ``` 150 | 151 | This will send two regular `Message`s, one with the timestamp as seconds since 152 | the epoch, the other with the timestamp as seconds.nanoseconds. Those will 153 | then be written to a file with the same name as the tag supplied as the argument 154 | to the `-t` flag (`foo.bar` in the above example). 155 | 156 | It will also send a `ForwardMessage` containing a pair of events - these will be 157 | written to the same file. 158 | 159 | It will then send a `PackedForwardMessage` containing a pair of events - these 160 | will be written to `$TAG.packed`. 161 | 162 | Last, it will send a `CompressedPackedForwardMessage` with the same pair of events, which should then be written to `$TAG.compressed`. 163 | -------------------------------------------------------------------------------- /cmd/bm/README.md: -------------------------------------------------------------------------------- 1 | # Benchmarks 2 | 3 | Very helpful information on Go benchmarking: https://dave.cheney.net/high-performance-go-workshop/gopherchina-2019.html 4 | 5 | ## Recent results 6 | 7 | ### `fluent-forward-go` vs `fluent-logger-golang` v1.8.0 8 | 9 | #### Running the comparisons 10 | 11 | The benchmark packages must be run separately. Running them together generates an error because `fluent-forward-go` and `fluent-logger-golang` each tries to register the same extension with `msgp`, which results in an error. 12 | 13 | 1) Start Fluent Bit 14 | 15 | ```shell 16 | ❱❱ docker run -p 127.0.0.1:24224:24224/tcp -v `pwd`:`pwd` -w `pwd` -ti fluent/fluent-bit:1.8.2 /fluent-bit/bin/fluent-bit -c $(pwd)/fixtures/fluent.conf 17 | ``` 18 | 19 | 2) Run the benchmarks 20 | 21 | ```shell 22 | # no ack 23 | go test -benchmem -run=^$ -bench ^.*Message$ -benchtime=10000x -count=10 github.com/IBM/fluent-forward-go/cmd/bm/fluent_forward_go 24 | go test -benchmem -run=^$ -bench ^.*Message$ -benchtime=10000x -count=10 github.com/IBM/fluent-forward-go/cmd/bm/fluent_logger_golang 25 | 26 | # with ack 27 | go test -benchmem -run=^$ -bench ^.*MessageAck$ -benchtime=10000x -count=10 github.com/IBM/fluent-forward-go/cmd/bm/fluent_forward_go 28 | go test -benchmem -run=^$ -bench ^.*MessageAck$ -benchtime=10000x -count=10 github.com/IBM/fluent-forward-go/cmd/bm/fluent_logger_golang 29 | ``` 30 | 31 | #### Best of 10: create and send single message 32 | 33 | ```shell 34 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 11355 ns/op 48 B/op 1 allocs/op 35 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19687 ns/op 2169 B/op 33 allocs/op 36 | ``` 37 | 38 | #### Best of 10: create and send single message with `ack` 39 | 40 | ```shell 41 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 768743 ns/op 185 B/op 6 allocs/op 42 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 793360 ns/op 6015 B/op 47 allocs/op 43 | ``` 44 | 45 | #### Full results 46 | 47 | ##### `fluent-forward-go` 48 | 49 | ```shell 50 | pkg: github.com/IBM/fluent-forward-go/cmd/bm/fluent_forward_go 51 | cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz 52 | 53 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 13153 ns/op 48 B/op 1 allocs/op 54 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12776 ns/op 48 B/op 1 allocs/op 55 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12710 ns/op 48 B/op 1 allocs/op 56 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 13048 ns/op 48 B/op 1 allocs/op 57 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12228 ns/op 48 B/op 1 allocs/op 58 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12250 ns/op 48 B/op 1 allocs/op 59 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 11355 ns/op 48 B/op 1 allocs/op 60 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12445 ns/op 48 B/op 1 allocs/op 61 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 12959 ns/op 48 B/op 1 allocs/op 62 | Benchmark_Fluent_Forward_Go_SingleMessage-16 10000 11597 ns/op 48 B/op 1 allocs/op 63 | 64 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 777020 ns/op 184 B/op 6 allocs/op 65 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 768743 ns/op 185 B/op 6 allocs/op 66 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 787335 ns/op 185 B/op 6 allocs/op 67 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 786457 ns/op 185 B/op 6 allocs/op 68 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 796123 ns/op 185 B/op 6 allocs/op 69 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 781143 ns/op 185 B/op 6 allocs/op 70 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 819758 ns/op 185 B/op 6 allocs/op 71 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 811781 ns/op 185 B/op 6 allocs/op 72 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 800595 ns/op 185 B/op 6 allocs/op 73 | Benchmark_Fluent_Forward_Go_SingleMessageAck-16 10000 885662 ns/op 185 B/op 6 allocs/op 74 | ``` 75 | 76 | #### `fluent-logger-golang` 77 | 78 | ```shell 79 | goos: darwin 80 | goarch: amd64 81 | pkg: github.com/IBM/fluent-forward-go/cmd/bm/fluent_logger_golang 82 | cpu: Intel(R) Core(TM) i9-9980HK CPU @ 2.40GHz 83 | 84 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 20002 ns/op 2171 B/op 33 allocs/op 85 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 20167 ns/op 2170 B/op 33 allocs/op 86 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 20983 ns/op 2169 B/op 33 allocs/op 87 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19779 ns/op 2170 B/op 33 allocs/op 88 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19687 ns/op 2169 B/op 33 allocs/op 89 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19893 ns/op 2169 B/op 33 allocs/op 90 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 20014 ns/op 2170 B/op 33 allocs/op 91 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 20163 ns/op 2170 B/op 33 allocs/op 92 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19819 ns/op 2170 B/op 33 allocs/op 93 | Benchmark_Fluent_Logger_Golang_SingleMessage-16 10000 19796 ns/op 2169 B/op 33 allocs/op 94 | 95 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 823867 ns/op 6015 B/op 47 allocs/op 96 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 891730 ns/op 6013 B/op 47 allocs/op 97 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 800438 ns/op 6012 B/op 47 allocs/op 98 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 793360 ns/op 6015 B/op 47 allocs/op 99 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 843148 ns/op 6014 B/op 47 allocs/op 100 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 816468 ns/op 6011 B/op 47 allocs/op 101 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 833102 ns/op 6013 B/op 47 allocs/op 102 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 809983 ns/op 6014 B/op 47 allocs/op 103 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 848345 ns/op 6015 B/op 47 allocs/op 104 | Benchmark_Fluent_Logger_Golang_SingleMessageAck-16 10000 846259 ns/op 6013 B/op 47 allocs/op 105 | ``` 106 | -------------------------------------------------------------------------------- /cmd/bm/fluent_forward_go/bm_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go test -benchmem -benchtime=100x -run=^$ -bench ^Benchmark.*$ github.com/IBM/fluent-forward-go/cmd/bm/fluent_forward_go 4 | 5 | import ( 6 | "testing" 7 | "time" 8 | 9 | "github.com/IBM/fluent-forward-go/cmd/bm" 10 | "github.com/IBM/fluent-forward-go/fluent/client" 11 | "github.com/IBM/fluent-forward-go/fluent/protocol" 12 | ) 13 | 14 | func Benchmark_Fluent_Forward_Go_SendOnly(b *testing.B) { 15 | tagVar := "bar" 16 | 17 | c := client.New(client.ConnectionOptions{ 18 | ConnectionTimeout: 3 * time.Second, 19 | }) 20 | 21 | err := c.Connect() 22 | if err != nil { 23 | b.Fatal(err) 24 | } 25 | 26 | defer c.Disconnect() 27 | 28 | record := bm.MakeRecord(12) 29 | mne := protocol.NewMessage(tagVar, record) 30 | 31 | b.ReportAllocs() 32 | b.ResetTimer() 33 | 34 | for i := 0; i < b.N; i++ { 35 | err = c.Send(mne) 36 | if err != nil { 37 | b.Fatal(err) 38 | } 39 | } 40 | } 41 | 42 | func Benchmark_Fluent_Forward_Go_SingleMessage(b *testing.B) { 43 | tagVar := "bar" 44 | 45 | c := client.New(client.ConnectionOptions{ 46 | ConnectionTimeout: 3 * time.Second, 47 | }) 48 | 49 | err := c.Connect() 50 | if err != nil { 51 | b.Fatal(err) 52 | } 53 | 54 | defer c.Disconnect() 55 | 56 | record := bm.MakeRecord(12) 57 | 58 | b.ReportAllocs() 59 | b.ResetTimer() 60 | 61 | for i := 0; i < b.N; i++ { 62 | mne := protocol.NewMessage(tagVar, record) 63 | err = c.Send(mne) 64 | if err != nil { 65 | b.Fatal(err) 66 | } 67 | } 68 | } 69 | 70 | func Benchmark_Fluent_Forward_Go_SingleMessageAck(b *testing.B) { 71 | tagVar := "foo" 72 | 73 | c := client.New(client.ConnectionOptions{ 74 | RequireAck: true, 75 | ConnectionTimeout: 3 * time.Second, 76 | }) 77 | 78 | err := c.Connect() 79 | if err != nil { 80 | b.Fatal(err) 81 | } 82 | 83 | record := bm.MakeRecord(12) 84 | 85 | defer c.Disconnect() 86 | 87 | b.ReportAllocs() 88 | b.ResetTimer() 89 | 90 | for i := 0; i < b.N; i++ { 91 | mne := protocol.NewMessage(tagVar, record) 92 | err = c.Send(mne) 93 | if err != nil { 94 | b.Fatal(err) 95 | } 96 | } 97 | } 98 | 99 | func Benchmark_Fluent_Forward_Go_Bytes(b *testing.B) { 100 | tagVar := "foo" 101 | 102 | c := client.New(client.ConnectionOptions{ 103 | ConnectionTimeout: 3 * time.Second, 104 | }) 105 | 106 | err := c.Connect() 107 | if err != nil { 108 | b.Fatal(err) 109 | } 110 | 111 | defer c.Disconnect() 112 | 113 | record := bm.MakeRecord(12) 114 | mne := protocol.NewMessage(tagVar, record) 115 | 116 | bits, _ := mne.MarshalMsg(nil) 117 | 118 | b.ReportAllocs() 119 | b.ResetTimer() 120 | 121 | for i := 0; i < b.N; i++ { 122 | err = c.SendRaw(bits) 123 | if err != nil { 124 | b.Fatal(err) 125 | } 126 | } 127 | } 128 | 129 | func Benchmark_Fluent_Forward_Go_BytesAck(b *testing.B) { 130 | tagVar := "foo" 131 | 132 | c := client.New(client.ConnectionOptions{ 133 | RequireAck: true, 134 | ConnectionTimeout: 3 * time.Second, 135 | }) 136 | 137 | err := c.Connect() 138 | if err != nil { 139 | b.Fatal(err) 140 | } 141 | 142 | defer c.Disconnect() 143 | 144 | record := bm.MakeRecord(12) 145 | mne := protocol.NewMessage(tagVar, record) 146 | 147 | mne.Chunk() 148 | bits, _ := mne.MarshalMsg(nil) 149 | 150 | b.ReportAllocs() 151 | b.ResetTimer() 152 | 153 | for i := 0; i < b.N; i++ { 154 | err = c.SendRaw(bits) 155 | if err != nil { 156 | b.Fatal(err) 157 | } 158 | } 159 | } 160 | 161 | func Benchmark_Fluent_Forward_Go_RawMessage(b *testing.B) { 162 | tagVar := "foo" 163 | 164 | c := client.New(client.ConnectionOptions{ 165 | ConnectionTimeout: 3 * time.Second, 166 | }) 167 | 168 | err := c.Connect() 169 | if err != nil { 170 | b.Fatal(err) 171 | } 172 | 173 | defer c.Disconnect() 174 | 175 | record := bm.MakeRecord(12) 176 | mne := protocol.NewMessage(tagVar, record) 177 | 178 | bits, _ := mne.MarshalMsg(nil) 179 | 180 | b.ReportAllocs() 181 | b.ResetTimer() 182 | 183 | for i := 0; i < b.N; i++ { 184 | rbits := protocol.RawMessage(bits) 185 | err = c.Send(rbits) 186 | if err != nil { 187 | b.Fatal(err) 188 | } 189 | } 190 | } 191 | 192 | func Benchmark_Fluent_Forward_Go_RawMessageAck(b *testing.B) { 193 | tagVar := "foo" 194 | 195 | c := client.New(client.ConnectionOptions{ 196 | RequireAck: true, 197 | ConnectionTimeout: 3 * time.Second, 198 | }) 199 | 200 | err := c.Connect() 201 | if err != nil { 202 | b.Fatal(err) 203 | } 204 | 205 | defer c.Disconnect() 206 | 207 | record := bm.MakeRecord(12) 208 | mne := protocol.NewMessage(tagVar, record) 209 | 210 | mne.Chunk() 211 | bits, _ := mne.MarshalMsg(nil) 212 | 213 | b.ReportAllocs() 214 | b.ResetTimer() 215 | 216 | for i := 0; i < b.N; i++ { 217 | rbits := protocol.RawMessage(bits) 218 | err = c.Send(rbits) 219 | if err != nil { 220 | b.Fatal(err) 221 | } 222 | } 223 | } 224 | 225 | func Benchmark_Fluent_Forward_Go_CompressedMessage(b *testing.B) { 226 | tagVar := "foo" 227 | 228 | c := client.New(client.ConnectionOptions{ 229 | ConnectionTimeout: 3 * time.Second, 230 | }) 231 | 232 | err := c.Connect() 233 | if err != nil { 234 | b.Fatal(err) 235 | } 236 | 237 | defer c.Disconnect() 238 | 239 | record := bm.MakeRecord(12) 240 | entries := []protocol.EntryExt{ 241 | { 242 | Timestamp: protocol.EventTimeNow(), 243 | Record: record, 244 | }, 245 | { 246 | Timestamp: protocol.EventTimeNow(), 247 | Record: record, 248 | }, 249 | { 250 | Timestamp: protocol.EventTimeNow(), 251 | Record: record, 252 | }, 253 | { 254 | Timestamp: protocol.EventTimeNow(), 255 | Record: record, 256 | }, 257 | { 258 | Timestamp: protocol.EventTimeNow(), 259 | Record: record, 260 | }, 261 | { 262 | Timestamp: protocol.EventTimeNow(), 263 | Record: record, 264 | }, 265 | } 266 | 267 | b.ReportAllocs() 268 | b.ResetTimer() 269 | 270 | for i := 0; i < b.N; i++ { 271 | mne, _ := protocol.NewCompressedPackedForwardMessage(tagVar, entries) 272 | err = c.Send(mne) 273 | if err != nil { 274 | b.Fatal(err) 275 | } 276 | } 277 | } 278 | 279 | func Benchmark_Fluent_Forward_Go_CompressedMessageAck(b *testing.B) { 280 | tagVar := "foo" 281 | 282 | c := client.New(client.ConnectionOptions{ 283 | RequireAck: true, 284 | ConnectionTimeout: 3 * time.Second, 285 | }) 286 | 287 | err := c.Connect() 288 | if err != nil { 289 | b.Fatal(err) 290 | } 291 | 292 | defer c.Disconnect() 293 | 294 | record := bm.MakeRecord(12) 295 | entries := []protocol.EntryExt{ 296 | { 297 | Timestamp: protocol.EventTimeNow(), 298 | Record: record, 299 | }, 300 | { 301 | Timestamp: protocol.EventTimeNow(), 302 | Record: record, 303 | }, 304 | { 305 | Timestamp: protocol.EventTimeNow(), 306 | Record: record, 307 | }, 308 | { 309 | Timestamp: protocol.EventTimeNow(), 310 | Record: record, 311 | }, 312 | { 313 | Timestamp: protocol.EventTimeNow(), 314 | Record: record, 315 | }, 316 | { 317 | Timestamp: protocol.EventTimeNow(), 318 | Record: record, 319 | }, 320 | } 321 | 322 | b.ReportAllocs() 323 | b.ResetTimer() 324 | 325 | for i := 0; i < b.N; i++ { 326 | mne, _ := protocol.NewCompressedPackedForwardMessage(tagVar, entries) 327 | err = c.Send(mne) 328 | if err != nil { 329 | b.Fatal(err) 330 | } 331 | } 332 | } 333 | -------------------------------------------------------------------------------- /cmd/bm/fluent_logger_golang/bm_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | //go test -benchmem -run=^$ -bench ^Benchmark.*$ github.com/IBM/fluent-forward-go/cmd/bm/fluent_logger_golang 4 | 5 | import ( 6 | "testing" 7 | "time" 8 | 9 | "github.com/IBM/fluent-forward-go/cmd/bm" 10 | "github.com/fluent/fluent-logger-golang/fluent" 11 | ) 12 | 13 | func Benchmark_Fluent_Logger_Golang_SingleMessage(b *testing.B) { 14 | logger, err := fluent.New(fluent.Config{ 15 | SubSecondPrecision: false, 16 | }) 17 | 18 | if err != nil { 19 | b.Fatal(err) 20 | } 21 | defer logger.Close() 22 | 23 | tag := "foo" 24 | data := bm.MakeRecord(12) 25 | 26 | b.ReportAllocs() 27 | b.ResetTimer() 28 | 29 | for i := 0; i < b.N; i++ { 30 | err = logger.Post(tag, data) 31 | if err != nil { 32 | b.Fatal(err) 33 | } 34 | } 35 | } 36 | 37 | func Benchmark_Fluent_Logger_Golang_SingleMessageAck(b *testing.B) { 38 | logger, err := fluent.New(fluent.Config{ 39 | Timeout: 3 * time.Second, 40 | RequestAck: true, 41 | SubSecondPrecision: false, 42 | }) 43 | 44 | if err != nil { 45 | b.Fatal(err) 46 | } 47 | defer logger.Close() 48 | 49 | tag := "foo" 50 | data := bm.MakeRecord(12) 51 | 52 | b.ReportAllocs() 53 | b.ResetTimer() 54 | 55 | for i := 0; i < b.N; i++ { 56 | err = logger.Post(tag, data) 57 | if err != nil { 58 | b.Fatal(err) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /cmd/bm/helpers.go: -------------------------------------------------------------------------------- 1 | package bm 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | const lorem = "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore magna aliqua Ut enim ad minim veniam quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat" 8 | 9 | var l = strings.Split(lorem, " ") 10 | 11 | func MakeRecord(numKeys int) map[string]interface{} { 12 | r := make(map[string]interface{}, numKeys) 13 | for i := 0; i < (2*numKeys)-1; i += 2 { 14 | r[l[i]] = l[i+1] 15 | } 16 | 17 | if len(r) == 0 { 18 | panic("empty record") 19 | } 20 | 21 | return r 22 | } 23 | -------------------------------------------------------------------------------- /cmd/forward/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "flag" 29 | "fmt" 30 | "os" 31 | 32 | "github.com/IBM/fluent-forward-go/fluent/client" 33 | "github.com/IBM/fluent-forward-go/fluent/protocol" 34 | ) 35 | 36 | var ( 37 | tagVar string 38 | ) 39 | 40 | func init() { 41 | flag.StringVar(&tagVar, "tag", "test.message", "-tag ") 42 | flag.StringVar(&tagVar, "t", "test.message", "-t (shorthand for -tag)") 43 | } 44 | 45 | func main() { 46 | flag.Parse() 47 | 48 | c := client.New(client.ConnectionOptions{ 49 | RequireAck: true, 50 | }) 51 | 52 | err := c.Connect() 53 | if err != nil { 54 | fmt.Fprintln(os.Stderr, "Unable to connect, exiting", err) 55 | os.Exit(-1) 56 | } 57 | 58 | record := map[string]interface{}{ 59 | "first": "Sir", 60 | "last": "Gawain", 61 | "enemy": "Green Knight", 62 | "equipment": []string{ 63 | "sword", 64 | "lance", 65 | "full plate", 66 | }, 67 | } 68 | 69 | entries := []protocol.EntryExt{ 70 | { 71 | Timestamp: protocol.EventTimeNow(), 72 | Record: map[string]interface{}{ 73 | "first": "Edgar", 74 | "last": "Winter", 75 | "enemy": "wimpy music", 76 | }, 77 | }, 78 | { 79 | Timestamp: protocol.EventTimeNow(), 80 | Record: map[string]interface{}{ 81 | "first": "George", 82 | "last": "Clinton", 83 | "enemy": "Sir Nose D Voidoffunk", 84 | }, 85 | }, 86 | } 87 | 88 | err = c.SendMessage(tagVar, record) 89 | if err != nil { 90 | fmt.Println(err) 91 | os.Exit(1) 92 | } 93 | 94 | err = c.SendMessageExt(tagVar, record) 95 | if err != nil { 96 | fmt.Println(err) 97 | os.Exit(1) 98 | } 99 | 100 | err = c.SendForward(tagVar, entries) 101 | if err != nil { 102 | fmt.Println(err) 103 | os.Exit(1) 104 | } 105 | 106 | err = c.SendPacked(tagVar+".packed", entries) 107 | if err != nil { 108 | fmt.Println(err) 109 | os.Exit(1) 110 | } 111 | 112 | err = c.SendCompressed(tagVar+".compressed", entries) 113 | if err != nil { 114 | fmt.Println(err) 115 | os.Exit(1) 116 | } 117 | 118 | fmt.Println("Messages sent") 119 | 120 | os.Exit(0) 121 | } 122 | -------------------------------------------------------------------------------- /cmd/ws/listener.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "context" 29 | "crypto/tls" 30 | "log" 31 | "net/http" 32 | "sync" 33 | "time" 34 | 35 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 36 | "github.com/gorilla/mux" 37 | "github.com/gorilla/websocket" 38 | ) 39 | 40 | type Listener struct { 41 | upgrader *websocket.Upgrader 42 | server *http.Server 43 | shutdown chan struct{} 44 | exited chan struct{} 45 | connectionLock sync.Mutex 46 | wsopts ws.ConnectionOptions 47 | } 48 | 49 | func NewListener(server *http.Server, wsopts ws.ConnectionOptions) *Listener { 50 | return &Listener{ 51 | &websocket.Upgrader{}, 52 | server, 53 | make(chan struct{}, 1), 54 | make(chan struct{}, 1), 55 | sync.Mutex{}, 56 | wsopts, 57 | } 58 | } 59 | 60 | func (s *Listener) Connect(w http.ResponseWriter, r *http.Request) { 61 | var ( 62 | c *websocket.Conn 63 | err error 64 | ) 65 | 66 | if c, err = s.upgrader.Upgrade(w, r, nil); err == nil { 67 | connection, _ := ws.NewConnection(c, s.wsopts) 68 | 69 | s.server.RegisterOnShutdown(func() { 70 | if !connection.Closed() { 71 | if err := connection.Close(); err != nil && err != websocket.ErrCloseSent { 72 | log.Println("server conn close error:", err) 73 | } 74 | } 75 | 76 | log.Println("server conn closed") 77 | }) 78 | 79 | if err = connection.Listen(); err != nil && 80 | !websocket.IsCloseError(err, websocket.CloseNormalClosure) { 81 | log.Println("server listen error:", err) 82 | } 83 | } 84 | } 85 | 86 | var ( 87 | once sync.Once 88 | router *mux.Router 89 | ) 90 | 91 | func (s *Listener) ListenAndServe() error { 92 | defer func() { s.exited <- struct{}{} }() 93 | 94 | once.Do(func() { 95 | // TODO comment explaining this weirdness 96 | router = mux.NewRouter() 97 | http.Handle("/", router) 98 | }) 99 | 100 | router.HandleFunc("/", s.Connect) 101 | 102 | if useTLS { 103 | config := &tls.Config{ 104 | CipherSuites: []uint16{ 105 | tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, 106 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, 107 | tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, 108 | tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, 109 | }, 110 | PreferServerCipherSuites: true, 111 | MinVersion: tls.VersionTLS12, 112 | } 113 | 114 | s.server.TLSConfig = config 115 | 116 | go func() { 117 | if err := s.server.ListenAndServeTLS( 118 | "../../fluent/client/clientfakes/cert.pem", "../../fluent/client/clientfakes/key.pem", 119 | ); err != nil && err != http.ErrServerClosed { 120 | log.Fatal("ListenAndServe error: " + err.Error()) 121 | } 122 | }() 123 | } else { 124 | go func() { 125 | if err := s.server.ListenAndServe(); err != nil && err != http.ErrServerClosed { 126 | log.Fatal("ListenAndServe error: " + err.Error()) 127 | } 128 | }() 129 | } 130 | 131 | <-s.shutdown 132 | log.Println("shutting down") 133 | 134 | ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) 135 | defer cancel() 136 | 137 | var err error 138 | if err = s.server.Shutdown(ctx); err != nil && err != http.ErrServerClosed { 139 | log.Println("shutdown server error:", err) 140 | 141 | if err = s.server.Close(); err != nil { 142 | log.Println("close server error:", err) 143 | } 144 | } 145 | 146 | if ctx.Err() != nil { 147 | log.Println("close context error:", ctx.Err()) 148 | } 149 | 150 | s.exited <- struct{}{} 151 | 152 | return err 153 | } 154 | 155 | func (s *Listener) Shutdown() { 156 | s.shutdown <- struct{}{} 157 | <-s.exited 158 | } 159 | -------------------------------------------------------------------------------- /cmd/ws/main.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package main 26 | 27 | import ( 28 | "crypto/tls" 29 | "flag" 30 | "fmt" 31 | "log" 32 | "net/http" 33 | "os" 34 | "os/signal" 35 | "time" 36 | 37 | "github.com/IBM/fluent-forward-go/fluent/client" 38 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 39 | "github.com/IBM/fluent-forward-go/fluent/protocol" 40 | ) 41 | 42 | var ( 43 | tagVar string 44 | useTLS bool 45 | ) 46 | 47 | func init() { 48 | flag.StringVar(&tagVar, "tag", "test.message", "-tag ") 49 | flag.StringVar(&tagVar, "t", "test.message", "-t (shorthand for -tag)") 50 | flag.BoolVar(&useTLS, "s", false, "specify to use tls") 51 | } 52 | 53 | // nolint 54 | func listen() *Listener { 55 | 56 | log.Println("Starting server on port 8083") 57 | 58 | s := &http.Server{Addr: ":8083"} 59 | wo := ws.ConnectionOptions{ 60 | ReadHandler: func(conn ws.Connection, _ int, p []byte, err error) error { 61 | msg := protocol.Message{} 62 | msg.UnmarshalMsg(p) 63 | 64 | log.Println("server got a message", msg, err) 65 | 66 | return err 67 | }, 68 | } 69 | 70 | wsSvr := NewListener(s, wo) 71 | 72 | go func() { 73 | if err := wsSvr.ListenAndServe(); err != nil { 74 | panic("ListenAndServe: " + err.Error()) 75 | } 76 | }() 77 | 78 | return wsSvr 79 | } 80 | 81 | func main() { 82 | flag.Parse() 83 | 84 | var tlsCfg *tls.Config 85 | 86 | url := "ws://127.0.0.1:8083" 87 | if useTLS { 88 | url = "wss://127.0.0.1:8083" 89 | tlsCfg = &tls.Config{InsecureSkipVerify: true} //#nosec 90 | } 91 | 92 | fmt.Fprintln(os.Stderr, "Connecting to - ", url) 93 | 94 | c := client.NewWS(client.WSConnectionOptions{ 95 | Factory: &client.DefaultWSConnectionFactory{ 96 | URL: url, 97 | TLSConfig: tlsCfg, 98 | }, 99 | }) 100 | 101 | wsSvr := listen() 102 | 103 | time.Sleep(time.Second) 104 | 105 | err := c.Connect() 106 | if err != nil { 107 | fmt.Fprintln(os.Stderr, "Unable to connect, exiting", err) 108 | os.Exit(1) 109 | } 110 | 111 | fmt.Println("Creating message") 112 | 113 | msg := protocol.Message{ 114 | Tag: tagVar, 115 | Timestamp: time.Now().UTC().Unix(), 116 | Record: map[string]interface{}{ 117 | "first": "Sir", 118 | "last": "Gawain", 119 | "enemy": "Green Knight", 120 | }, 121 | } 122 | 123 | if err := c.Send(&msg); err != nil { 124 | log.Fatal(err) 125 | } 126 | 127 | msg = protocol.Message{ 128 | Tag: tagVar, 129 | Timestamp: time.Now().UTC().Unix(), 130 | Record: map[string]interface{}{ 131 | "first": "Sir", 132 | "last": "Lancelot", 133 | "enemy": "Himself", 134 | }, 135 | } 136 | 137 | if err := c.Send(&msg); err != nil { 138 | log.Fatal(err) 139 | } 140 | 141 | fmt.Println("Messages sent") 142 | 143 | interrupt := make(chan os.Signal, 1) 144 | signal.Notify(interrupt, os.Interrupt) 145 | 146 | go func() { 147 | if err := c.Disconnect(); err != nil { 148 | log.Fatal(err) 149 | } 150 | 151 | wsSvr.Shutdown() 152 | interrupt <- os.Interrupt 153 | }() 154 | 155 | <-interrupt 156 | 157 | os.Exit(0) 158 | } 159 | -------------------------------------------------------------------------------- /fixtures/fluent.conf: -------------------------------------------------------------------------------- 1 | [SERVICE] 2 | Log_Level debug 3 | 4 | [INPUT] 5 | Name forward 6 | Listen 0.0.0.0 7 | Port 24224 8 | 9 | [OUTPUT] 10 | Name file 11 | Match * 12 | Workers 4 13 | -------------------------------------------------------------------------------- /fluent/client/client.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package client 26 | 27 | import ( 28 | "errors" 29 | "fmt" 30 | "sync" 31 | 32 | "crypto/rand" 33 | "net" 34 | "time" 35 | 36 | "github.com/IBM/fluent-forward-go/fluent/protocol" 37 | 38 | "github.com/tinylib/msgp/msgp" 39 | ) 40 | 41 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate 42 | 43 | const ( 44 | DefaultConnectionTimeout time.Duration = 60 * time.Second 45 | ) 46 | 47 | // MessageClient implementations send MessagePack messages to a peer 48 | // 49 | //counterfeiter:generate . MessageClient 50 | type MessageClient interface { 51 | Connect() error 52 | Disconnect() (err error) 53 | Reconnect() error 54 | Send(e protocol.ChunkEncoder) error 55 | SendCompressed(tag string, entries protocol.EntryList) error 56 | SendCompressedFromBytes(tag string, entries []byte) error 57 | SendForward(tag string, entries protocol.EntryList) error 58 | SendMessage(tag string, record interface{}) error 59 | SendMessageExt(tag string, record interface{}) error 60 | SendPacked(tag string, entries protocol.EntryList) error 61 | SendPackedFromBytes(tag string, entries []byte) error 62 | SendRaw(raw []byte) error 63 | } 64 | 65 | // ConnectionFactory implementations create new connections 66 | // 67 | //counterfeiter:generate . ConnectionFactory 68 | type ConnectionFactory interface { 69 | New() (net.Conn, error) 70 | } 71 | 72 | type Client struct { 73 | ConnectionFactory 74 | RequireAck bool 75 | Timeout time.Duration 76 | AuthInfo AuthInfo 77 | Hostname string 78 | session *Session 79 | ackLock sync.Mutex 80 | sessionLock sync.RWMutex 81 | } 82 | 83 | type ConnectionOptions struct { 84 | Factory ConnectionFactory 85 | RequireAck bool 86 | ConnectionTimeout time.Duration 87 | // TODO: 88 | // ReadTimeout time.Duration 89 | // WriteTimeout time.Duration 90 | AuthInfo AuthInfo 91 | } 92 | 93 | type AuthInfo struct { 94 | SharedKey []byte 95 | Username string 96 | Password string 97 | } 98 | 99 | type Session struct { 100 | Connection net.Conn 101 | TransportPhase bool 102 | } 103 | 104 | func New(opts ConnectionOptions) *Client { 105 | factory := opts.Factory 106 | if factory == nil { 107 | factory = &ConnFactory{ 108 | Network: "tcp", 109 | Address: "localhost:24224", 110 | } 111 | } 112 | 113 | if opts.ConnectionTimeout == 0 { 114 | opts.ConnectionTimeout = DefaultConnectionTimeout 115 | } 116 | 117 | return &Client{ 118 | ConnectionFactory: factory, 119 | AuthInfo: opts.AuthInfo, 120 | RequireAck: opts.RequireAck, 121 | Timeout: opts.ConnectionTimeout, 122 | } 123 | } 124 | 125 | // TransportPhase indicates if the client has completed the 126 | // initial connection handshake. 127 | func (c *Client) TransportPhase() bool { 128 | c.sessionLock.RLock() 129 | defer c.sessionLock.RUnlock() 130 | 131 | return c.session != nil && c.session.TransportPhase 132 | } 133 | 134 | func (c *Client) connect() error { 135 | conn, err := c.New() 136 | if err != nil { 137 | return err 138 | } 139 | 140 | c.session = &Session{ 141 | Connection: conn, 142 | } 143 | 144 | // If no shared key, handshake mode is not required 145 | if c.AuthInfo.SharedKey == nil { 146 | c.session.TransportPhase = true 147 | } 148 | 149 | return nil 150 | } 151 | 152 | // Handshake initiates handshake mode. Users must call this before attempting 153 | // to send any messages when the server is configured with a shared key, otherwise 154 | // the server will reject any message events. Successful completion of the 155 | // handshake puts the connection into message (or forward) mode, at which time 156 | // the client is free to send event messages. 157 | func (c *Client) Handshake() error { 158 | c.sessionLock.RLock() 159 | defer c.sessionLock.RUnlock() 160 | 161 | if c.session == nil { 162 | return errors.New("not connected") 163 | } 164 | 165 | var helo protocol.Helo 166 | 167 | r := msgp.NewReader(c.session.Connection) 168 | err := helo.DecodeMsg(r) 169 | 170 | if err != nil { 171 | return err 172 | } 173 | 174 | salt := make([]byte, 16) 175 | 176 | _, err = rand.Read(salt) 177 | if err != nil { 178 | return err 179 | } 180 | 181 | ping, err := protocol.NewPing(c.Hostname, c.AuthInfo.SharedKey, salt, helo.Options.Nonce) 182 | if err != nil { 183 | return err 184 | } 185 | 186 | err = msgp.Encode(c.session.Connection, ping) 187 | if err != nil { 188 | return err 189 | } 190 | 191 | var pong protocol.Pong 192 | 193 | err = pong.DecodeMsg(r) 194 | if err != nil { 195 | return err 196 | } 197 | 198 | if err := protocol.ValidatePongDigest(&pong, c.AuthInfo.SharedKey, 199 | helo.Options.Nonce, salt); err != nil { 200 | return err 201 | } 202 | 203 | c.session.TransportPhase = true 204 | 205 | return nil 206 | } 207 | 208 | // Connect initializes the Session and Connection objects by opening 209 | // a client connect to the target configured in the ConnectionFactory 210 | func (c *Client) Connect() error { 211 | c.sessionLock.Lock() 212 | defer c.sessionLock.Unlock() 213 | 214 | if c.session != nil { 215 | return errors.New("a session is already active") 216 | } 217 | 218 | return c.connect() 219 | } 220 | 221 | func (c *Client) disconnect() (err error) { 222 | if c.session != nil { 223 | err = c.session.Connection.Close() 224 | } 225 | 226 | c.session = nil 227 | 228 | return 229 | } 230 | 231 | // Disconnect terminates a client connection 232 | func (c *Client) Disconnect() error { 233 | c.sessionLock.Lock() 234 | defer c.sessionLock.Unlock() 235 | 236 | return c.disconnect() 237 | } 238 | 239 | func (c *Client) Reconnect() error { 240 | c.sessionLock.Lock() 241 | defer c.sessionLock.Unlock() 242 | 243 | _ = c.disconnect() 244 | 245 | return c.connect() 246 | } 247 | 248 | func (c *Client) checkAck(chunk string) error { 249 | if c.Timeout != 0 { 250 | if err := c.session.Connection.SetReadDeadline(time.Now().Add(c.Timeout)); err != nil { 251 | return err 252 | } 253 | } 254 | 255 | var ack protocol.AckMessage 256 | if err := msgp.Decode(c.session.Connection, &ack); err != nil { 257 | return err 258 | } 259 | 260 | if ack.Ack != chunk { 261 | return fmt.Errorf("Expected chunk %s, but got %s", chunk, ack.Ack) 262 | } 263 | 264 | return nil 265 | } 266 | 267 | // Send sends a single protocol.ChunkEncoder across the wire. If the session 268 | // is not yet in transport phase, an error is returned, and no message is sent. 269 | func (c *Client) Send(e protocol.ChunkEncoder) error { 270 | c.sessionLock.RLock() 271 | defer c.sessionLock.RUnlock() 272 | 273 | if c.session == nil { 274 | return errors.New("no active session") 275 | } 276 | 277 | if !c.session.TransportPhase { 278 | return errors.New("session handshake not completed") 279 | } 280 | 281 | var ( 282 | chunk string 283 | err error 284 | ) 285 | 286 | if c.RequireAck { 287 | if chunk, err = e.Chunk(); err != nil { 288 | return err 289 | } 290 | 291 | c.ackLock.Lock() 292 | defer c.ackLock.Unlock() 293 | } 294 | 295 | err = msgp.Encode(c.session.Connection, e) 296 | if err != nil || !c.RequireAck { 297 | return err 298 | } 299 | 300 | return c.checkAck(chunk) 301 | } 302 | 303 | // SendRaw sends bytes across the wire. If the session 304 | // is not yet in transport phase, an error is returned, 305 | // and no message is sent. 306 | func (c *Client) SendRaw(m []byte) error { 307 | c.sessionLock.RLock() 308 | defer c.sessionLock.RUnlock() 309 | 310 | if c.session == nil { 311 | return errors.New("no active session") 312 | } 313 | 314 | if !c.session.TransportPhase { 315 | return errors.New("session handshake not completed") 316 | } 317 | 318 | _, err := c.session.Connection.Write(m) 319 | 320 | return err 321 | } 322 | 323 | func (c *Client) SendPacked(tag string, entries protocol.EntryList) error { 324 | msg, err := protocol.NewPackedForwardMessage(tag, entries) 325 | if err == nil { 326 | err = c.Send(msg) 327 | } 328 | 329 | return err 330 | } 331 | 332 | func (c *Client) SendPackedFromBytes(tag string, entries []byte) error { 333 | msg := protocol.NewPackedForwardMessageFromBytes(tag, entries) 334 | 335 | return c.Send(msg) 336 | } 337 | 338 | func (c *Client) SendMessage(tag string, record interface{}) error { 339 | msg := protocol.NewMessage(tag, record) 340 | 341 | return c.Send(msg) 342 | } 343 | 344 | func (c *Client) SendMessageExt(tag string, record interface{}) error { 345 | msg := protocol.NewMessageExt(tag, record) 346 | 347 | return c.Send(msg) 348 | } 349 | 350 | func (c *Client) SendForward(tag string, entries protocol.EntryList) error { 351 | msg := protocol.NewForwardMessage(tag, entries) 352 | 353 | return c.Send(msg) 354 | } 355 | 356 | func (c *Client) SendCompressed(tag string, entries protocol.EntryList) error { 357 | msg, err := protocol.NewCompressedPackedForwardMessage(tag, entries) 358 | if err == nil { 359 | err = c.Send(msg) 360 | } 361 | 362 | return err 363 | } 364 | 365 | func (c *Client) SendCompressedFromBytes(tag string, entries []byte) error { 366 | msg, err := protocol.NewCompressedPackedForwardMessageFromBytes(tag, entries) 367 | if err == nil { 368 | err = c.Send(msg) 369 | } 370 | 371 | return err 372 | } 373 | -------------------------------------------------------------------------------- /fluent/client/client_suite_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestClient(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Client Suite") 13 | } 14 | -------------------------------------------------------------------------------- /fluent/client/clientfakes/cert.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIICMjCCAZugAwIBAgIQb+b13AtP/mmU8NrBioV2wzANBgkqhkiG9w0BAQsFADAS 3 | MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw 4 | MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB 5 | iQKBgQDDUCzf4qeF187ipTDjrJFak85aURvHBahIk/fffnZUJYcAWMV56qaaIq7j 6 | cACz02zIU0JEff+ybnieO4w1qOVoEbue/ZE+IKl8gGMxTMLTuSSCBsoaK4g1sLdp 7 | 2VeEqxYWAdsoSDWt597IH4q3QU4PQlqjr9vFOgBac2ROV/iwRQIDAQABo4GGMIGD 8 | MA4GA1UdDwEB/wQEAwICpDATBgNVHSUEDDAKBggrBgEFBQcDATAPBgNVHRMBAf8E 9 | BTADAQH/MB0GA1UdDgQWBBQWQMDUQuh1bYAgt+c/dMKAGlt6PTAsBgNVHREEJTAj 10 | gglsb2NhbGhvc3SHBH8AAAGHEAAAAAAAAAAAAAAAAAAAAAEwDQYJKoZIhvcNAQEL 11 | BQADgYEAlzaPza825diQRxwWXAom+sipDe8N9dKJpgSqUoVsxHebWARW5uvqtsDo 12 | EVoIKuNQEOWFcDoMg9U2uZWteRAxk/X5RNmH22xKn+FqQW8QorCuirBadHg9yfy7 13 | RoC6XgEU78Y95iiQ5Bg7eXGmfkEPkI/0woxSV2tpnVWhhs9RkYk= 14 | -----END CERTIFICATE----- 15 | -------------------------------------------------------------------------------- /fluent/client/clientfakes/fake_client_factory.go: -------------------------------------------------------------------------------- 1 | // Code generated by counterfeiter. DO NOT EDIT. 2 | package clientfakes 3 | 4 | import ( 5 | "sync" 6 | 7 | "github.com/IBM/fluent-forward-go/fluent/client" 8 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 9 | "github.com/IBM/fluent-forward-go/fluent/client/ws/ext" 10 | ) 11 | 12 | type FakeClientFactory struct { 13 | NewStub func() (ext.Conn, error) 14 | newMutex sync.RWMutex 15 | newArgsForCall []struct { 16 | } 17 | newReturns struct { 18 | result1 ext.Conn 19 | result2 error 20 | } 21 | newReturnsOnCall map[int]struct { 22 | result1 ext.Conn 23 | result2 error 24 | } 25 | NewSessionStub func(ws.Connection) *client.WSSession 26 | newSessionMutex sync.RWMutex 27 | newSessionArgsForCall []struct { 28 | arg1 ws.Connection 29 | } 30 | newSessionReturns struct { 31 | result1 *client.WSSession 32 | } 33 | newSessionReturnsOnCall map[int]struct { 34 | result1 *client.WSSession 35 | } 36 | invocations map[string][][]interface{} 37 | invocationsMutex sync.RWMutex 38 | } 39 | 40 | func (fake *FakeClientFactory) New() (ext.Conn, error) { 41 | fake.newMutex.Lock() 42 | ret, specificReturn := fake.newReturnsOnCall[len(fake.newArgsForCall)] 43 | fake.newArgsForCall = append(fake.newArgsForCall, struct { 44 | }{}) 45 | stub := fake.NewStub 46 | fakeReturns := fake.newReturns 47 | fake.recordInvocation("New", []interface{}{}) 48 | fake.newMutex.Unlock() 49 | if stub != nil { 50 | return stub() 51 | } 52 | if specificReturn { 53 | return ret.result1, ret.result2 54 | } 55 | return fakeReturns.result1, fakeReturns.result2 56 | } 57 | 58 | func (fake *FakeClientFactory) NewCallCount() int { 59 | fake.newMutex.RLock() 60 | defer fake.newMutex.RUnlock() 61 | return len(fake.newArgsForCall) 62 | } 63 | 64 | func (fake *FakeClientFactory) NewCalls(stub func() (ext.Conn, error)) { 65 | fake.newMutex.Lock() 66 | defer fake.newMutex.Unlock() 67 | fake.NewStub = stub 68 | } 69 | 70 | func (fake *FakeClientFactory) NewReturns(result1 ext.Conn, result2 error) { 71 | fake.newMutex.Lock() 72 | defer fake.newMutex.Unlock() 73 | fake.NewStub = nil 74 | fake.newReturns = struct { 75 | result1 ext.Conn 76 | result2 error 77 | }{result1, result2} 78 | } 79 | 80 | func (fake *FakeClientFactory) NewReturnsOnCall(i int, result1 ext.Conn, result2 error) { 81 | fake.newMutex.Lock() 82 | defer fake.newMutex.Unlock() 83 | fake.NewStub = nil 84 | if fake.newReturnsOnCall == nil { 85 | fake.newReturnsOnCall = make(map[int]struct { 86 | result1 ext.Conn 87 | result2 error 88 | }) 89 | } 90 | fake.newReturnsOnCall[i] = struct { 91 | result1 ext.Conn 92 | result2 error 93 | }{result1, result2} 94 | } 95 | 96 | func (fake *FakeClientFactory) NewSession(arg1 ws.Connection) *client.WSSession { 97 | fake.newSessionMutex.Lock() 98 | ret, specificReturn := fake.newSessionReturnsOnCall[len(fake.newSessionArgsForCall)] 99 | fake.newSessionArgsForCall = append(fake.newSessionArgsForCall, struct { 100 | arg1 ws.Connection 101 | }{arg1}) 102 | stub := fake.NewSessionStub 103 | fakeReturns := fake.newSessionReturns 104 | fake.recordInvocation("NewSession", []interface{}{arg1}) 105 | fake.newSessionMutex.Unlock() 106 | if stub != nil { 107 | return stub(arg1) 108 | } 109 | if specificReturn { 110 | return ret.result1 111 | } 112 | return fakeReturns.result1 113 | } 114 | 115 | func (fake *FakeClientFactory) NewSessionCallCount() int { 116 | fake.newSessionMutex.RLock() 117 | defer fake.newSessionMutex.RUnlock() 118 | return len(fake.newSessionArgsForCall) 119 | } 120 | 121 | func (fake *FakeClientFactory) NewSessionCalls(stub func(ws.Connection) *client.WSSession) { 122 | fake.newSessionMutex.Lock() 123 | defer fake.newSessionMutex.Unlock() 124 | fake.NewSessionStub = stub 125 | } 126 | 127 | func (fake *FakeClientFactory) NewSessionArgsForCall(i int) ws.Connection { 128 | fake.newSessionMutex.RLock() 129 | defer fake.newSessionMutex.RUnlock() 130 | argsForCall := fake.newSessionArgsForCall[i] 131 | return argsForCall.arg1 132 | } 133 | 134 | func (fake *FakeClientFactory) NewSessionReturns(result1 *client.WSSession) { 135 | fake.newSessionMutex.Lock() 136 | defer fake.newSessionMutex.Unlock() 137 | fake.NewSessionStub = nil 138 | fake.newSessionReturns = struct { 139 | result1 *client.WSSession 140 | }{result1} 141 | } 142 | 143 | func (fake *FakeClientFactory) NewSessionReturnsOnCall(i int, result1 *client.WSSession) { 144 | fake.newSessionMutex.Lock() 145 | defer fake.newSessionMutex.Unlock() 146 | fake.NewSessionStub = nil 147 | if fake.newSessionReturnsOnCall == nil { 148 | fake.newSessionReturnsOnCall = make(map[int]struct { 149 | result1 *client.WSSession 150 | }) 151 | } 152 | fake.newSessionReturnsOnCall[i] = struct { 153 | result1 *client.WSSession 154 | }{result1} 155 | } 156 | 157 | func (fake *FakeClientFactory) Invocations() map[string][][]interface{} { 158 | fake.invocationsMutex.RLock() 159 | defer fake.invocationsMutex.RUnlock() 160 | fake.newMutex.RLock() 161 | defer fake.newMutex.RUnlock() 162 | fake.newSessionMutex.RLock() 163 | defer fake.newSessionMutex.RUnlock() 164 | copiedInvocations := map[string][][]interface{}{} 165 | for key, value := range fake.invocations { 166 | copiedInvocations[key] = value 167 | } 168 | return copiedInvocations 169 | } 170 | 171 | func (fake *FakeClientFactory) recordInvocation(key string, args []interface{}) { 172 | fake.invocationsMutex.Lock() 173 | defer fake.invocationsMutex.Unlock() 174 | if fake.invocations == nil { 175 | fake.invocations = map[string][][]interface{}{} 176 | } 177 | if fake.invocations[key] == nil { 178 | fake.invocations[key] = [][]interface{}{} 179 | } 180 | fake.invocations[key] = append(fake.invocations[key], args) 181 | } 182 | 183 | var _ client.WSConnectionFactory = new(FakeClientFactory) 184 | -------------------------------------------------------------------------------- /fluent/client/clientfakes/fake_connection_factory.go: -------------------------------------------------------------------------------- 1 | // Code generated by counterfeiter. DO NOT EDIT. 2 | package clientfakes 3 | 4 | import ( 5 | "net" 6 | "sync" 7 | 8 | "github.com/IBM/fluent-forward-go/fluent/client" 9 | ) 10 | 11 | type FakeConnectionFactory struct { 12 | NewStub func() (net.Conn, error) 13 | newMutex sync.RWMutex 14 | newArgsForCall []struct { 15 | } 16 | newReturns struct { 17 | result1 net.Conn 18 | result2 error 19 | } 20 | newReturnsOnCall map[int]struct { 21 | result1 net.Conn 22 | result2 error 23 | } 24 | invocations map[string][][]interface{} 25 | invocationsMutex sync.RWMutex 26 | } 27 | 28 | func (fake *FakeConnectionFactory) New() (net.Conn, error) { 29 | fake.newMutex.Lock() 30 | ret, specificReturn := fake.newReturnsOnCall[len(fake.newArgsForCall)] 31 | fake.newArgsForCall = append(fake.newArgsForCall, struct { 32 | }{}) 33 | stub := fake.NewStub 34 | fakeReturns := fake.newReturns 35 | fake.recordInvocation("New", []interface{}{}) 36 | fake.newMutex.Unlock() 37 | if stub != nil { 38 | return stub() 39 | } 40 | if specificReturn { 41 | return ret.result1, ret.result2 42 | } 43 | return fakeReturns.result1, fakeReturns.result2 44 | } 45 | 46 | func (fake *FakeConnectionFactory) NewCallCount() int { 47 | fake.newMutex.RLock() 48 | defer fake.newMutex.RUnlock() 49 | return len(fake.newArgsForCall) 50 | } 51 | 52 | func (fake *FakeConnectionFactory) NewCalls(stub func() (net.Conn, error)) { 53 | fake.newMutex.Lock() 54 | defer fake.newMutex.Unlock() 55 | fake.NewStub = stub 56 | } 57 | 58 | func (fake *FakeConnectionFactory) NewReturns(result1 net.Conn, result2 error) { 59 | fake.newMutex.Lock() 60 | defer fake.newMutex.Unlock() 61 | fake.NewStub = nil 62 | fake.newReturns = struct { 63 | result1 net.Conn 64 | result2 error 65 | }{result1, result2} 66 | } 67 | 68 | func (fake *FakeConnectionFactory) NewReturnsOnCall(i int, result1 net.Conn, result2 error) { 69 | fake.newMutex.Lock() 70 | defer fake.newMutex.Unlock() 71 | fake.NewStub = nil 72 | if fake.newReturnsOnCall == nil { 73 | fake.newReturnsOnCall = make(map[int]struct { 74 | result1 net.Conn 75 | result2 error 76 | }) 77 | } 78 | fake.newReturnsOnCall[i] = struct { 79 | result1 net.Conn 80 | result2 error 81 | }{result1, result2} 82 | } 83 | 84 | func (fake *FakeConnectionFactory) Invocations() map[string][][]interface{} { 85 | fake.invocationsMutex.RLock() 86 | defer fake.invocationsMutex.RUnlock() 87 | fake.newMutex.RLock() 88 | defer fake.newMutex.RUnlock() 89 | copiedInvocations := map[string][][]interface{}{} 90 | for key, value := range fake.invocations { 91 | copiedInvocations[key] = value 92 | } 93 | return copiedInvocations 94 | } 95 | 96 | func (fake *FakeConnectionFactory) recordInvocation(key string, args []interface{}) { 97 | fake.invocationsMutex.Lock() 98 | defer fake.invocationsMutex.Unlock() 99 | if fake.invocations == nil { 100 | fake.invocations = map[string][][]interface{}{} 101 | } 102 | if fake.invocations[key] == nil { 103 | fake.invocations[key] = [][]interface{}{} 104 | } 105 | fake.invocations[key] = append(fake.invocations[key], args) 106 | } 107 | 108 | var _ client.ConnectionFactory = new(FakeConnectionFactory) 109 | -------------------------------------------------------------------------------- /fluent/client/clientfakes/fake_wsconnection_factory.go: -------------------------------------------------------------------------------- 1 | // Code generated by counterfeiter. DO NOT EDIT. 2 | package clientfakes 3 | 4 | import ( 5 | "sync" 6 | 7 | "github.com/IBM/fluent-forward-go/fluent/client" 8 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 9 | "github.com/IBM/fluent-forward-go/fluent/client/ws/ext" 10 | ) 11 | 12 | type FakeWSConnectionFactory struct { 13 | NewStub func() (ext.Conn, error) 14 | newMutex sync.RWMutex 15 | newArgsForCall []struct { 16 | } 17 | newReturns struct { 18 | result1 ext.Conn 19 | result2 error 20 | } 21 | newReturnsOnCall map[int]struct { 22 | result1 ext.Conn 23 | result2 error 24 | } 25 | NewSessionStub func(ws.Connection) *client.WSSession 26 | newSessionMutex sync.RWMutex 27 | newSessionArgsForCall []struct { 28 | arg1 ws.Connection 29 | } 30 | newSessionReturns struct { 31 | result1 *client.WSSession 32 | } 33 | newSessionReturnsOnCall map[int]struct { 34 | result1 *client.WSSession 35 | } 36 | invocations map[string][][]interface{} 37 | invocationsMutex sync.RWMutex 38 | } 39 | 40 | func (fake *FakeWSConnectionFactory) New() (ext.Conn, error) { 41 | fake.newMutex.Lock() 42 | ret, specificReturn := fake.newReturnsOnCall[len(fake.newArgsForCall)] 43 | fake.newArgsForCall = append(fake.newArgsForCall, struct { 44 | }{}) 45 | stub := fake.NewStub 46 | fakeReturns := fake.newReturns 47 | fake.recordInvocation("New", []interface{}{}) 48 | fake.newMutex.Unlock() 49 | if stub != nil { 50 | return stub() 51 | } 52 | if specificReturn { 53 | return ret.result1, ret.result2 54 | } 55 | return fakeReturns.result1, fakeReturns.result2 56 | } 57 | 58 | func (fake *FakeWSConnectionFactory) NewCallCount() int { 59 | fake.newMutex.RLock() 60 | defer fake.newMutex.RUnlock() 61 | return len(fake.newArgsForCall) 62 | } 63 | 64 | func (fake *FakeWSConnectionFactory) NewCalls(stub func() (ext.Conn, error)) { 65 | fake.newMutex.Lock() 66 | defer fake.newMutex.Unlock() 67 | fake.NewStub = stub 68 | } 69 | 70 | func (fake *FakeWSConnectionFactory) NewReturns(result1 ext.Conn, result2 error) { 71 | fake.newMutex.Lock() 72 | defer fake.newMutex.Unlock() 73 | fake.NewStub = nil 74 | fake.newReturns = struct { 75 | result1 ext.Conn 76 | result2 error 77 | }{result1, result2} 78 | } 79 | 80 | func (fake *FakeWSConnectionFactory) NewReturnsOnCall(i int, result1 ext.Conn, result2 error) { 81 | fake.newMutex.Lock() 82 | defer fake.newMutex.Unlock() 83 | fake.NewStub = nil 84 | if fake.newReturnsOnCall == nil { 85 | fake.newReturnsOnCall = make(map[int]struct { 86 | result1 ext.Conn 87 | result2 error 88 | }) 89 | } 90 | fake.newReturnsOnCall[i] = struct { 91 | result1 ext.Conn 92 | result2 error 93 | }{result1, result2} 94 | } 95 | 96 | func (fake *FakeWSConnectionFactory) NewSession(arg1 ws.Connection) *client.WSSession { 97 | fake.newSessionMutex.Lock() 98 | ret, specificReturn := fake.newSessionReturnsOnCall[len(fake.newSessionArgsForCall)] 99 | fake.newSessionArgsForCall = append(fake.newSessionArgsForCall, struct { 100 | arg1 ws.Connection 101 | }{arg1}) 102 | stub := fake.NewSessionStub 103 | fakeReturns := fake.newSessionReturns 104 | fake.recordInvocation("NewSession", []interface{}{arg1}) 105 | fake.newSessionMutex.Unlock() 106 | if stub != nil { 107 | return stub(arg1) 108 | } 109 | if specificReturn { 110 | return ret.result1 111 | } 112 | return fakeReturns.result1 113 | } 114 | 115 | func (fake *FakeWSConnectionFactory) NewSessionCallCount() int { 116 | fake.newSessionMutex.RLock() 117 | defer fake.newSessionMutex.RUnlock() 118 | return len(fake.newSessionArgsForCall) 119 | } 120 | 121 | func (fake *FakeWSConnectionFactory) NewSessionCalls(stub func(ws.Connection) *client.WSSession) { 122 | fake.newSessionMutex.Lock() 123 | defer fake.newSessionMutex.Unlock() 124 | fake.NewSessionStub = stub 125 | } 126 | 127 | func (fake *FakeWSConnectionFactory) NewSessionArgsForCall(i int) ws.Connection { 128 | fake.newSessionMutex.RLock() 129 | defer fake.newSessionMutex.RUnlock() 130 | argsForCall := fake.newSessionArgsForCall[i] 131 | return argsForCall.arg1 132 | } 133 | 134 | func (fake *FakeWSConnectionFactory) NewSessionReturns(result1 *client.WSSession) { 135 | fake.newSessionMutex.Lock() 136 | defer fake.newSessionMutex.Unlock() 137 | fake.NewSessionStub = nil 138 | fake.newSessionReturns = struct { 139 | result1 *client.WSSession 140 | }{result1} 141 | } 142 | 143 | func (fake *FakeWSConnectionFactory) NewSessionReturnsOnCall(i int, result1 *client.WSSession) { 144 | fake.newSessionMutex.Lock() 145 | defer fake.newSessionMutex.Unlock() 146 | fake.NewSessionStub = nil 147 | if fake.newSessionReturnsOnCall == nil { 148 | fake.newSessionReturnsOnCall = make(map[int]struct { 149 | result1 *client.WSSession 150 | }) 151 | } 152 | fake.newSessionReturnsOnCall[i] = struct { 153 | result1 *client.WSSession 154 | }{result1} 155 | } 156 | 157 | func (fake *FakeWSConnectionFactory) Invocations() map[string][][]interface{} { 158 | fake.invocationsMutex.RLock() 159 | defer fake.invocationsMutex.RUnlock() 160 | fake.newMutex.RLock() 161 | defer fake.newMutex.RUnlock() 162 | fake.newSessionMutex.RLock() 163 | defer fake.newSessionMutex.RUnlock() 164 | copiedInvocations := map[string][][]interface{}{} 165 | for key, value := range fake.invocations { 166 | copiedInvocations[key] = value 167 | } 168 | return copiedInvocations 169 | } 170 | 171 | func (fake *FakeWSConnectionFactory) recordInvocation(key string, args []interface{}) { 172 | fake.invocationsMutex.Lock() 173 | defer fake.invocationsMutex.Unlock() 174 | if fake.invocations == nil { 175 | fake.invocations = map[string][][]interface{}{} 176 | } 177 | if fake.invocations[key] == nil { 178 | fake.invocations[key] = [][]interface{}{} 179 | } 180 | fake.invocations[key] = append(fake.invocations[key], args) 181 | } 182 | 183 | var _ client.WSConnectionFactory = new(FakeWSConnectionFactory) 184 | -------------------------------------------------------------------------------- /fluent/client/clientfakes/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBAMNQLN/ip4XXzuKl 3 | MOOskVqTzlpRG8cFqEiT999+dlQlhwBYxXnqppoiruNwALPTbMhTQkR9/7JueJ47 4 | jDWo5WgRu579kT4gqXyAYzFMwtO5JIIGyhoriDWwt2nZV4SrFhYB2yhINa3n3sgf 5 | irdBTg9CWqOv28U6AFpzZE5X+LBFAgMBAAECgYBCfb35LilH2HNXF1OwfqQxSNZc 6 | SvaA570lkDI3hM710JzIEnCQE8FKAfq19QBYN+b1v9p0hMySyycG/1C5R9lo3tns 7 | yTRZ20K49m1Lmdpy0wyo+iA6FAN+WDugB0CM9yQ/4sQsNDh4iodHNxSwnLPRfQyr 8 | n0gqhejixIz7tejcAQJBANCKBhDgGhBdhBABXc8BIoBeg79dueCjpn4jhkBB7Y6L 9 | ZZKxbuPHIDk3DDSfEPz3gq5bjzfXUmI331Dlk4b0i0UCQQDvw5NnbfKOnuFUDKoi 10 | ++SV++MZYvzpflrbB7qrysPIr6Wr2WHopv9hNOpB9u6eViJFguYdHxr2IKnjDqib 11 | s2EBAkEAgEhXwOvKvMR6J8pfZabCDYuWqhLC9FBXbAL4Y/DqbKSNoDasC0yEIw4c 12 | 4QX00liDuDQ0ntpDII5UPpRrMzqdrQJABUd2IGngVFmVNoeM23ZWYyMHrhBq5y08 13 | DOSPrKarRujvKeiAamH674X9vnQUbvIQkR0/udZsfozbkL9K4RrDAQJBAJwbuFCM 14 | 8flvBRh6vtcsGEvPe66tO/iSsEdYJd5llW9UGFPOPalr99L7C5hLmBpWc1KpuRo9 15 | g2TGdsmSxXTG/F0= 16 | -----END PRIVATE KEY----- 17 | -------------------------------------------------------------------------------- /fluent/client/conn_factory.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package client 26 | 27 | import ( 28 | "crypto/tls" 29 | "net" 30 | "time" 31 | ) 32 | 33 | // ConnFactory is a light wrapper for net.Dial and tls.Dial. When TLSConfig 34 | // is not nil, tls.Dial is called. Otherwise, net.Dial is used. See Go's 35 | // net.Dial documentation for more information. 36 | type ConnFactory struct { 37 | // Network indicates the type of connection. The default value is "tcp". 38 | Network string 39 | Address string 40 | TLSConfig *tls.Config 41 | Timeout time.Duration 42 | } 43 | 44 | func (f *ConnFactory) New() (net.Conn, error) { 45 | if len(f.Network) == 0 { 46 | f.Network = "tcp" 47 | } 48 | 49 | dialer := &net.Dialer{Timeout: f.Timeout} 50 | 51 | if f.TLSConfig != nil { 52 | return tls.DialWithDialer(dialer, f.Network, f.Address, f.TLSConfig) 53 | } 54 | 55 | return dialer.Dial(f.Network, f.Address) 56 | } 57 | -------------------------------------------------------------------------------- /fluent/client/conn_factory_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package client_test 26 | 27 | import ( 28 | "crypto/tls" 29 | "net" 30 | "os" 31 | "time" 32 | 33 | . "github.com/onsi/ginkgo/v2" 34 | . "github.com/onsi/gomega" 35 | 36 | . "github.com/IBM/fluent-forward-go/fluent/client" 37 | ) 38 | 39 | var _ = Describe("ConnFactory", func() { 40 | 41 | var ( 42 | network, address string 43 | server net.Listener 44 | serverErr error 45 | factory *ConnFactory 46 | tlsConfig *tls.Config 47 | ) 48 | 49 | BeforeEach(func() { 50 | network = "tcp" 51 | address = ":0" 52 | tlsConfig = nil 53 | }) 54 | 55 | JustBeforeEach(func() { 56 | svrNetwork := network 57 | if svrNetwork == "" { 58 | svrNetwork = "tcp" 59 | } 60 | 61 | var clientTLSCfg *tls.Config 62 | if tlsConfig != nil { 63 | clientTLSCfg = &tls.Config{InsecureSkipVerify: true} 64 | server, serverErr = tls.Listen(svrNetwork, address, tlsConfig) 65 | } else { 66 | server, serverErr = net.Listen(svrNetwork, address) 67 | } 68 | Expect(serverErr).NotTo(HaveOccurred()) 69 | 70 | if svrNetwork == "tcp" { 71 | address = server.Addr().(*net.TCPAddr).String() 72 | } 73 | 74 | factory = &ConnFactory{ 75 | Network: network, 76 | Address: address, 77 | TLSConfig: clientTLSCfg, 78 | Timeout: 100 * time.Millisecond, 79 | } 80 | }) 81 | 82 | AfterEach(func() { 83 | err := server.Close() 84 | Expect(err).NotTo(HaveOccurred()) 85 | }) 86 | 87 | Describe("New", func() { 88 | testConnection := func(testTls bool) { 89 | socketConn, err := factory.New() 90 | Expect(err).NotTo(HaveOccurred()) 91 | Expect(socketConn).NotTo(BeNil()) 92 | time.Sleep(time.Millisecond) 93 | 94 | if testTls { 95 | tconn := socketConn.(*tls.Conn) 96 | state := tconn.ConnectionState() 97 | Expect(state.PeerCertificates).ToNot(BeEmpty()) 98 | } 99 | 100 | tmp := make([]byte, 256) 101 | n, err := socketConn.Read(tmp) 102 | Expect(err).NotTo(HaveOccurred()) 103 | Expect(n).To(Equal(1)) 104 | 105 | Expect(socketConn.Close()).ToNot(HaveOccurred()) 106 | } 107 | 108 | JustBeforeEach(func() { 109 | svr := server 110 | go func() { 111 | defer GinkgoRecover() 112 | conn, err := svr.Accept() 113 | Expect(err).NotTo(HaveOccurred()) 114 | Expect(conn).NotTo(BeNil()) 115 | defer conn.Close() 116 | 117 | n, err := conn.Write([]byte{0x00}) 118 | Expect(err).NotTo(HaveOccurred()) 119 | Expect(n).To(Equal(1)) 120 | }() 121 | 122 | time.Sleep(3 * time.Millisecond) 123 | }) 124 | 125 | When("connecting with tcp", func() { 126 | It("returns an established connection", func() { 127 | testConnection(false) 128 | }) 129 | 130 | When("Network is empty", func() { 131 | BeforeEach(func() { 132 | network = "" 133 | }) 134 | 135 | It("defaults to tcp", func() { 136 | Expect(factory.Network).To(BeEmpty()) 137 | testConnection(false) 138 | Expect(factory.Network).To(Equal("tcp")) 139 | }) 140 | }) 141 | 142 | When("using tls", func() { 143 | BeforeEach(func() { 144 | // go run $GOROOT/src/crypto/tls/generate_cert.go --rsa-bits 1024 \ 145 | // --host 127.0.0.1,::1,localhost --ca \ 146 | // --start-date "Jan 1 00:00:00 1970" --duration=1000000h 147 | cer, err := tls.LoadX509KeyPair("clientfakes/cert.pem", "clientfakes/key.pem") 148 | Expect(err).ToNot(HaveOccurred()) 149 | 150 | tlsConfig = &tls.Config{ 151 | Certificates: []tls.Certificate{cer}, 152 | } 153 | }) 154 | 155 | It("returns an established connection", func() { 156 | testConnection(true) 157 | }) 158 | }) 159 | }) 160 | 161 | When("using unix socket", func() { 162 | BeforeEach(func() { 163 | network = "unix" 164 | address = "/tmp/test.sock" 165 | }) 166 | 167 | AfterEach(func() { 168 | if err := os.RemoveAll(address); err != nil { 169 | Fail(err.Error()) 170 | } 171 | }) 172 | 173 | It("returns an established connection", func() { 174 | testConnection(false) 175 | }) 176 | }) 177 | }) 178 | }) 179 | -------------------------------------------------------------------------------- /fluent/client/errors.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | ) 7 | 8 | type WSConnError struct { 9 | StatusCode int 10 | ResponseBody string 11 | ConnErr error 12 | retryable bool 13 | } 14 | 15 | var nonRetryableStatusCodes = []int{ 16 | http.StatusBadRequest, 17 | http.StatusUnauthorized, 18 | http.StatusForbidden, 19 | http.StatusNotFound, 20 | http.StatusMethodNotAllowed, 21 | http.StatusNotImplemented, 22 | http.StatusHTTPVersionNotSupported, 23 | } 24 | 25 | func (e *WSConnError) Error() string { 26 | if e.ConnErr != nil { 27 | return fmt.Sprintf("Connection Error %s. Status Code: %d. Response: %s", e.ConnErr.Error(), e.StatusCode, e.ResponseBody) 28 | } 29 | 30 | return fmt.Sprintf("Connection Error. Status Code: %d. Response: %s", e.StatusCode, e.ResponseBody) 31 | } 32 | 33 | func (e *WSConnError) IsRetryable() bool { 34 | return e.retryable 35 | } 36 | 37 | func NewWSConnError(err error, statusCode int, respBody string) *WSConnError { 38 | return &WSConnError{ConnErr: err, 39 | StatusCode: statusCode, 40 | ResponseBody: respBody, 41 | retryable: isRetryableStatusCode(statusCode), 42 | } 43 | } 44 | 45 | // isRetryableStatusCode checks if the provided HTTP status code is retryable 46 | func isRetryableStatusCode(statusCode int) bool { 47 | for _, nonRetryableCode := range nonRetryableStatusCodes { 48 | if statusCode == nonRetryableCode { 49 | return false 50 | } 51 | } 52 | 53 | return true 54 | } 55 | -------------------------------------------------------------------------------- /fluent/client/ws/connection.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package ws 26 | 27 | import ( 28 | "errors" 29 | "io" 30 | "net" 31 | "sync" 32 | "time" 33 | 34 | ext "github.com/IBM/fluent-forward-go/fluent/client/ws/ext" 35 | "github.com/gorilla/websocket" 36 | ) 37 | 38 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate 39 | 40 | const ( 41 | DefaultCloseDeadline = 5 * time.Second 42 | ) 43 | 44 | type Logger interface { 45 | Println(v ...interface{}) 46 | Printf(format string, v ...interface{}) 47 | } 48 | 49 | type noopLogger struct{} 50 | 51 | func (l *noopLogger) Println(_ ...interface{}) {} 52 | 53 | func (l *noopLogger) Printf(_ string, _ ...interface{}) {} 54 | 55 | type ReadHandler func(conn Connection, messageType int, p []byte, err error) error 56 | 57 | type ConnectionOptions struct { 58 | CloseDeadline time.Duration 59 | CloseHandler func(conn Connection, code int, text string) error 60 | PingHandler func(conn Connection, appData string) error 61 | PongHandler func(conn Connection, appData string) error 62 | // TODO: should be a duration and added to `now` before every operation 63 | ReadDeadline time.Time 64 | // ReadHandler handles new messages received on the websocket. If an error 65 | // is received the client MUST call `Close`. An error returned by ReadHandler 66 | // will be retured by `Listen`. 67 | ReadHandler ReadHandler 68 | // TODO: should be a duration and added to `now` before every operation 69 | WriteDeadline time.Time 70 | // Logger is an optional debug log writer. 71 | Logger Logger 72 | } 73 | 74 | type ConnState uint8 75 | 76 | const ( 77 | ConnStateOpen ConnState = 1 << iota 78 | ConnStateListening 79 | ConnStateCloseReceived 80 | ConnStateCloseSent 81 | ConnStateClosed 82 | ConnStateError 83 | ) 84 | 85 | //counterfeiter:generate . Connection 86 | type Connection interface { 87 | ext.Conn 88 | CloseWithMsg(closeCode int, msg string) error 89 | Closed() bool 90 | ConnState() ConnState 91 | Listen() error 92 | ReadHandler() ReadHandler 93 | SetReadHandler(rh ReadHandler) 94 | Write(data []byte) (int, error) 95 | } 96 | 97 | type connection struct { 98 | ext.Conn 99 | logger Logger 100 | closeLock sync.Mutex 101 | listenLock sync.Mutex 102 | writeLock sync.Mutex 103 | stateLock sync.RWMutex 104 | readHandler ReadHandler 105 | done chan struct{} 106 | connState ConnState 107 | closeDeadline time.Duration 108 | } 109 | 110 | func NewConnection(conn ext.Conn, opts ConnectionOptions) (Connection, error) { 111 | wsc := &connection{ 112 | Conn: conn, 113 | done: make(chan struct{}), 114 | connState: ConnStateOpen, 115 | logger: opts.Logger, 116 | } 117 | 118 | if wsc.logger == nil { 119 | wsc.logger = &noopLogger{} 120 | } 121 | 122 | if opts.CloseHandler == nil { 123 | opts.CloseHandler = wsc.handleClose 124 | } 125 | 126 | wsc.SetCloseHandler(func(code int, text string) error { 127 | return opts.CloseHandler(wsc, code, text) 128 | }) 129 | 130 | if opts.PingHandler != nil { 131 | wsc.SetPingHandler(func(appData string) error { 132 | return opts.PingHandler(wsc, appData) 133 | }) 134 | } 135 | 136 | if opts.PongHandler != nil { 137 | wsc.SetPongHandler(func(appData string) error { 138 | return opts.PongHandler(wsc, appData) 139 | }) 140 | } 141 | 142 | if opts.ReadHandler == nil { 143 | opts.ReadHandler = func(c Connection, _ int, _ []byte, err error) error { 144 | if err != nil { 145 | wsc.logger.Println("Default ReadHandler error:", err) 146 | 147 | _ = c.Close() 148 | } 149 | 150 | return err 151 | } 152 | } 153 | 154 | wsc.SetReadHandler(opts.ReadHandler) 155 | 156 | if opts.CloseDeadline == 0 { 157 | opts.CloseDeadline = DefaultCloseDeadline 158 | } 159 | 160 | wsc.closeDeadline = opts.CloseDeadline 161 | 162 | if err := wsc.SetReadDeadline(opts.ReadDeadline); err != nil { 163 | return nil, err 164 | } 165 | 166 | if err := wsc.SetWriteDeadline(opts.WriteDeadline); err != nil { 167 | return nil, err 168 | } 169 | 170 | return wsc, nil 171 | } 172 | 173 | func (wsc *connection) ConnState() ConnState { 174 | wsc.stateLock.RLock() 175 | defer wsc.stateLock.RUnlock() 176 | 177 | return wsc.connState 178 | } 179 | 180 | func (wsc *connection) hasConnState(cs ConnState) bool { 181 | wsc.stateLock.RLock() 182 | defer wsc.stateLock.RUnlock() 183 | 184 | return wsc.connState&cs != 0 185 | } 186 | 187 | func (wsc *connection) setConnState(cs ConnState) { 188 | wsc.stateLock.Lock() 189 | defer wsc.stateLock.Unlock() 190 | 191 | wsc.connState |= cs 192 | } 193 | 194 | func (wsc *connection) unsetConnState(cs ConnState) { 195 | wsc.stateLock.Lock() 196 | defer wsc.stateLock.Unlock() 197 | 198 | if wsc.connState&cs == 0 { 199 | return 200 | } 201 | 202 | wsc.connState ^= cs 203 | } 204 | 205 | // CloseWithMsg sends a close message to the peer 206 | func (wsc *connection) CloseWithMsg(closeCode int, msg string) error { 207 | wsc.closeLock.Lock() 208 | 209 | if wsc.Closed() { 210 | wsc.closeLock.Unlock() 211 | return errors.New("multiple close calls") 212 | } 213 | 214 | wsc.unsetConnState(ConnStateOpen) 215 | wsc.closeLock.Unlock() 216 | 217 | var err error 218 | 219 | if !wsc.hasConnState(ConnStateError) { 220 | wsc.logger.Printf("sending close message: code %d; msg '%s'", closeCode, msg) 221 | 222 | // TODO: currently only the tests check this state to confirm handshake; 223 | // need to refactor it out 224 | wsc.setConnState(ConnStateCloseSent) 225 | 226 | err = wsc.WriteMessage( 227 | websocket.CloseMessage, 228 | websocket.FormatCloseMessage( 229 | closeCode, msg, 230 | ), 231 | ) 232 | 233 | // if the close message was sent and the connection is listening for incoming 234 | // messages, wait N seconds for a response. 235 | if err == nil && wsc.hasConnState(ConnStateListening) { 236 | wsc.logger.Println("awaiting peer response") 237 | 238 | select { 239 | case <-time.After(wsc.closeDeadline): 240 | // sent a close, but never heard back, close anyway 241 | err = errors.New("close deadline expired") 242 | case <-wsc.done: 243 | } 244 | } 245 | } 246 | 247 | wsc.setConnState(ConnStateClosed) 248 | 249 | wsc.logger.Println("closing the connection") 250 | 251 | if cerr := wsc.Conn.Close(); cerr != nil { 252 | err = cerr 253 | } 254 | 255 | return err 256 | } 257 | 258 | func (wsc *connection) Close() error { 259 | return wsc.CloseWithMsg(websocket.CloseNormalClosure, "closing connection") 260 | } 261 | 262 | func (wsc *connection) Closed() bool { 263 | return !wsc.hasConnState(ConnStateOpen) 264 | } 265 | 266 | func (wsc *connection) handleClose(_ Connection, code int, msg string) error { 267 | wsc.logger.Printf("close received: code %d; msg '%s'", code, msg) 268 | 269 | // TODO: currently only the tests check this state to confirm handshake; 270 | // need to refactor it out 271 | wsc.setConnState(ConnStateCloseReceived) 272 | 273 | return nil 274 | } 275 | 276 | type connMsg struct { 277 | mt int 278 | message []byte 279 | err error 280 | } 281 | 282 | func (wsc *connection) runReadLoop(nextMsg chan connMsg) { 283 | defer func() { 284 | wsc.logger.Println("exiting read loop") 285 | 286 | close(nextMsg) 287 | wsc.unsetConnState(ConnStateListening) 288 | close(wsc.done) 289 | }() 290 | 291 | msg := connMsg{} 292 | 293 | for { 294 | msg.mt, msg.message, msg.err = wsc.Conn.ReadMessage() 295 | 296 | if msg.err != nil { 297 | if wsc.hasConnState(ConnStateClosed) && errors.Is(msg.err, net.ErrClosed) { 298 | // healthy close 299 | break 300 | } 301 | 302 | var err net.Error 303 | if errors.As(msg.err, &err) || errors.Is(msg.err, net.ErrClosed) || 304 | websocket.IsCloseError(msg.err, websocket.CloseAbnormalClosure) { 305 | // mark the connection with error state so Close doesn't attempt to 306 | // send closing message to peer 307 | wsc.setConnState(ConnStateError) 308 | } 309 | } 310 | 311 | nextMsg <- msg 312 | 313 | if msg.err != nil { 314 | break 315 | } 316 | } 317 | } 318 | 319 | func (wsc *connection) Listen() error { 320 | wsc.listenLock.Lock() 321 | 322 | if wsc.hasConnState(ConnStateListening) { 323 | wsc.listenLock.Unlock() 324 | return errors.New("already listening on this connection") 325 | } 326 | 327 | wsc.logger.Println("listening") 328 | wsc.setConnState(ConnStateListening) 329 | wsc.listenLock.Unlock() 330 | 331 | nextMsg := make(chan connMsg) 332 | go wsc.runReadLoop(nextMsg) 333 | 334 | var err error 335 | 336 | for msg := range nextMsg { 337 | if rerr := wsc.readHandler(wsc, msg.mt, msg.message, msg.err); rerr != nil { 338 | if msg.err != nil { 339 | wsc.logger.Println("handler returned error: ", msg.err.Error()) 340 | } 341 | 342 | // set error only if it is something other than a normal close 343 | if !websocket.IsCloseError(rerr, websocket.CloseNormalClosure) { 344 | err = rerr 345 | } 346 | } 347 | } 348 | 349 | return err 350 | } 351 | 352 | func (wsc *connection) NextReader() (messageType int, r io.Reader, err error) { 353 | panic("use ReadHandler instead") 354 | } 355 | 356 | func (wsc *connection) ReadMessage() (messageType int, p []byte, err error) { 357 | panic("use ReadHandler instead") 358 | } 359 | 360 | func (wsc *connection) SetReadHandler(rh ReadHandler) { 361 | wsc.readHandler = rh 362 | } 363 | 364 | func (wsc *connection) ReadHandler() ReadHandler { 365 | return wsc.readHandler 366 | } 367 | 368 | func (wsc *connection) WriteMessage(messageType int, data []byte) error { 369 | wsc.writeLock.Lock() 370 | defer wsc.writeLock.Unlock() 371 | 372 | return wsc.Conn.WriteMessage(messageType, data) 373 | } 374 | 375 | func (wsc *connection) Write(data []byte) (int, error) { 376 | if err := wsc.WriteMessage(websocket.BinaryMessage, data); err != nil { 377 | return 0, err 378 | } 379 | 380 | return len(data), nil 381 | } 382 | -------------------------------------------------------------------------------- /fluent/client/ws/connection_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package ws_test 26 | 27 | import ( 28 | "bytes" 29 | "fmt" 30 | "log" 31 | "net/http" 32 | "net/http/httptest" 33 | "strings" 34 | "time" 35 | 36 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 37 | "github.com/gorilla/websocket" 38 | . "github.com/onsi/ginkgo/v2" 39 | . "github.com/onsi/gomega" 40 | "github.com/onsi/gomega/gbytes" 41 | ) 42 | 43 | type message struct { 44 | mt int 45 | msg []byte 46 | err error 47 | } 48 | 49 | var _ = Describe("Connection", func() { 50 | var ( 51 | checkClose, checkSvrClose bool 52 | connection, svrConnection ws.Connection 53 | svr *httptest.Server 54 | opts ws.ConnectionOptions 55 | svrRcvdMsgs chan message 56 | listenErrs chan error 57 | exitConnState, svrExitConnState ws.ConnState 58 | logBuffer *gbytes.Buffer 59 | ) 60 | 61 | var makeOpts = func(logBuffer *gbytes.Buffer, msgChan chan message, name string) ws.ConnectionOptions { 62 | logFlags := log.Lshortfile | log.LUTC | log.Lmicroseconds 63 | logger := log.New(logBuffer, name+"> ", logFlags) 64 | 65 | return ws.ConnectionOptions{ 66 | CloseDeadline: 500 * time.Millisecond, 67 | ReadHandler: func(conn ws.Connection, msgType int, p []byte, err error) error { 68 | if msgChan != nil { 69 | msgChan <- message{ 70 | mt: msgType, 71 | msg: p, 72 | err: err, 73 | } 74 | } 75 | 76 | if err != nil { 77 | logger.Println("ReadHandler received error:", err) 78 | _ = conn.Close() 79 | } 80 | 81 | return err 82 | }, 83 | Logger: logger, 84 | } 85 | } 86 | 87 | newHandler := func(logBuffer *gbytes.Buffer, svrRcvdMsgs chan message) http.Handler { 88 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 89 | defer GinkgoRecover() 90 | svrOpts := makeOpts(logBuffer, svrRcvdMsgs, "server") 91 | 92 | var upgrader websocket.Upgrader 93 | wc, _ := upgrader.Upgrade(w, r, nil) 94 | 95 | var err error 96 | svrConnection, err = ws.NewConnection(wc, svrOpts) 97 | if err != nil { 98 | return 99 | } 100 | 101 | svrConnection.Listen() 102 | log.Println("exit server handler") 103 | }) 104 | } 105 | 106 | BeforeEach(func() { 107 | logBuffer = gbytes.NewBuffer() 108 | 109 | exitConnState = ws.ConnStateCloseReceived | ws.ConnStateCloseSent | ws.ConnStateClosed 110 | svrExitConnState = ws.ConnStateCloseReceived | ws.ConnStateCloseSent | ws.ConnStateClosed 111 | 112 | checkSvrClose = true 113 | svrRcvdMsgs = make(chan message, 1) 114 | svr = httptest.NewServer(newHandler(logBuffer, svrRcvdMsgs)) 115 | 116 | checkClose = true 117 | opts = makeOpts(logBuffer, nil, "client") 118 | }) 119 | 120 | JustBeforeEach(func() { 121 | u := "ws" + strings.TrimPrefix(svr.URL, "http") 122 | conn, _, err := websocket.DefaultDialer.Dial(u, nil) 123 | Expect(err).ToNot(HaveOccurred()) 124 | 125 | connection, err = ws.NewConnection(conn, opts) 126 | Expect(err).ToNot(HaveOccurred()) 127 | 128 | listenErrs = make(chan error, 1) 129 | 130 | go func() { 131 | defer GinkgoRecover() 132 | 133 | Expect(connection.ConnState()).To(Equal(ws.ConnStateOpen)) 134 | 135 | if err := connection.Listen(); err != nil { 136 | listenErrs <- err 137 | } 138 | }() 139 | 140 | // wait for Listen loop to start 141 | time.Sleep(10 * time.Millisecond) 142 | Expect(connection.Closed()).To(BeFalse()) 143 | }) 144 | 145 | AfterEach(func() { 146 | defer func() { 147 | fmt.Print(string(logBuffer.Contents())) 148 | logBuffer.Close() 149 | }() 150 | 151 | if !connection.Closed() { 152 | err := connection.Close() 153 | if checkClose { 154 | Expect(err).ToNot(HaveOccurred()) 155 | } 156 | Eventually(connection.Closed).Should(BeTrue()) 157 | } 158 | 159 | if !svrConnection.Closed() { 160 | err := svrConnection.Close() 161 | if checkSvrClose { 162 | Expect(err).ToNot(HaveOccurred()) 163 | } 164 | Eventually(svrConnection.Closed).Should(BeTrue()) 165 | } 166 | 167 | svr.Close() 168 | 169 | if checkClose { 170 | Eventually(connection.ConnState).Should(Equal(exitConnState)) 171 | } 172 | if checkSvrClose { 173 | Eventually(svrConnection.ConnState).Should(Equal(svrExitConnState)) 174 | } 175 | }) 176 | 177 | Describe("NewConnection", func() { 178 | BeforeEach(func() { 179 | opts.ReadHandler = nil 180 | }) 181 | 182 | When("no ReadHandler is set", func() { 183 | It("sets a default handler that handles the closing handshake", func() { 184 | Expect(connection.Close()).ToNot(HaveOccurred()) 185 | closeMsg := <-svrRcvdMsgs 186 | Expect(closeMsg.err.Error()).To(MatchRegexp("closing connection")) 187 | Eventually(logBuffer.Contents).Should(MatchRegexp("Default ReadHandler error.+closing connection")) 188 | }) 189 | }) 190 | }) 191 | 192 | Describe("WriteMessage", func() { 193 | When("everything is copacetic", func() { 194 | It("writes messages to the connection", func() { 195 | err := connection.WriteMessage(1, []byte("oi")) 196 | Expect(err).ToNot(HaveOccurred()) 197 | err = connection.WriteMessage(1, []byte("koi")) 198 | Expect(err).ToNot(HaveOccurred()) 199 | 200 | m := <-svrRcvdMsgs 201 | Expect(m.msg).To(Equal([]byte("oi"))) 202 | m = <-svrRcvdMsgs 203 | Expect(m.msg).To(Equal([]byte("koi"))) 204 | 205 | Consistently(svrRcvdMsgs).ShouldNot(Receive()) 206 | }) 207 | }) 208 | 209 | When("an error occurs", func() { 210 | It("returns an error", func() { 211 | Expect(connection.Close()).ToNot(HaveOccurred()) 212 | Expect(connection.WriteMessage(1, nil).Error()).To(MatchRegexp("close sent")) 213 | }) 214 | }) 215 | }) 216 | 217 | Describe("Listen", func() { 218 | When("everything is copacetic", func() { 219 | It("reads a message from the connection and calls the read handler", func() { 220 | Expect(len(svrRcvdMsgs)).To(Equal(0)) 221 | 222 | err := connection.WriteMessage(1, []byte("oi")) 223 | Expect(err).ToNot(HaveOccurred()) 224 | 225 | m := <-svrRcvdMsgs 226 | Expect(m.err).ToNot(HaveOccurred()) 227 | Expect(bytes.Equal(m.msg, []byte("oi"))).To(BeTrue()) 228 | }) 229 | }) 230 | 231 | When("already listening", func() { 232 | It("errors", func() { 233 | Expect(connection.Listen().Error()).To(MatchRegexp("already listening on this connection")) 234 | Expect(connection.Listen().Error()).To(MatchRegexp("already listening on this connection")) 235 | }) 236 | }) 237 | 238 | When("a network error occurs", func() { 239 | JustBeforeEach(func() { 240 | checkClose = false 241 | checkSvrClose = false 242 | exitConnState = ws.ConnStateClosed | ws.ConnStateError 243 | svrExitConnState = exitConnState 244 | connection.UnderlyingConn().Close() 245 | }) 246 | 247 | It("returns a network error", func() { 248 | err := <-listenErrs 249 | Expect(err.Error()).To(MatchRegexp("use of closed network connection")) 250 | }) 251 | }) 252 | 253 | When("a close error occurs", func() { 254 | It("returns abnormal closures", func() { 255 | err := svrConnection.CloseWithMsg(websocket.ClosePolicyViolation, "meh") 256 | Expect(err).ToNot(HaveOccurred()) 257 | err = <-listenErrs 258 | Expect(err.Error()).To(MatchRegexp("meh")) 259 | }) 260 | 261 | It("does not return normal closures", func() { 262 | Expect(svrConnection.Close()).ToNot(HaveOccurred()) 263 | Consistently(listenErrs).ShouldNot(Receive()) 264 | }) 265 | }) 266 | }) 267 | 268 | Describe("CloseWithMsg", func() { 269 | When("everything is copacetic", func() { 270 | It("sends a signal", func() { 271 | Expect(connection.CloseWithMsg(1000, "oi")).ToNot(HaveOccurred()) 272 | Expect(connection.Closed()).To(BeTrue()) 273 | 274 | closeMsg := <-svrRcvdMsgs 275 | Expect(closeMsg.err.Error()).To(MatchRegexp("oi")) 276 | }) 277 | }) 278 | }) 279 | 280 | Describe("Close and Closed", func() { 281 | JustBeforeEach(func() { 282 | Expect(connection.Closed()).To(BeFalse()) 283 | }) 284 | 285 | AfterEach(func() { 286 | Expect(connection.Closed()).To(BeTrue()) 287 | }) 288 | 289 | When("everything is copacetic", func() { 290 | It("signals close", func() { 291 | Expect(connection.Close()).ToNot(HaveOccurred()) 292 | closeMsg := <-svrRcvdMsgs 293 | Expect(closeMsg.err.Error()).To(MatchRegexp("closing connection")) 294 | }) 295 | }) 296 | 297 | When("called multiple times", func() { 298 | It("errors", func() { 299 | Expect(connection.Close()).ToNot(HaveOccurred()) 300 | Expect(connection.Close().Error()).To(MatchRegexp("multiple close calls")) 301 | }) 302 | }) 303 | 304 | When("the connection errors on close", func() { 305 | BeforeEach(func() { 306 | opts.ReadHandler = func(conn ws.Connection, msgType int, p []byte, err error) error { 307 | // This is kinda cheating, but there is a race condition where the default test 308 | // read handler occasionally calls Close before the test does. In that case, 309 | // `Close` returns the "multiple close calls" error. 310 | return nil 311 | } 312 | }) 313 | 314 | JustBeforeEach(func() { 315 | checkClose = false 316 | checkSvrClose = false 317 | exitConnState = ws.ConnStateClosed | ws.ConnStateError 318 | svrExitConnState = exitConnState 319 | connection.UnderlyingConn().Close() 320 | }) 321 | 322 | AfterEach(func() { 323 | Expect(connection.ConnState() & exitConnState).To(BeNumerically(">", 1)) 324 | Expect(connection.ConnState() & svrExitConnState).To(BeNumerically(">", 1)) 325 | }) 326 | 327 | It("returns an error", func() { 328 | Expect(connection.Close().Error()).To(MatchRegexp("use of closed network connection")) 329 | }) 330 | }) 331 | }) 332 | }) 333 | -------------------------------------------------------------------------------- /fluent/client/ws/ext/conn.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package ext 26 | 27 | import ( 28 | "io" 29 | "net" 30 | "time" 31 | 32 | "github.com/gorilla/websocket" 33 | ) 34 | 35 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate 36 | 37 | // Conn is an interface for websocket.Conn. Generated with 38 | // ifacemaker (https://github.com/vburenin/ifacemaker). 39 | // 40 | //counterfeiter:generate . Conn 41 | type Conn interface { 42 | // Subprotocol returns the negotiated protocol for the connection. 43 | Subprotocol() string 44 | // Close closes the underlying network connection without sending or waiting 45 | // for a close message. 46 | Close() error 47 | // LocalAddr returns the local network address. 48 | LocalAddr() net.Addr 49 | // RemoteAddr returns the remote network address. 50 | RemoteAddr() net.Addr 51 | // WriteControl writes a control message with the given deadline. The allowed 52 | // message types are CloseMessage, PingMessage and PongMessage. 53 | WriteControl(messageType int, data []byte, deadline time.Time) error 54 | // NextWriter returns a writer for the next message to send. The writer's Close 55 | // method flushes the complete message to the network. 56 | // 57 | // There can be at most one open writer on a connection. NextWriter closes the 58 | // previous writer if the application has not already done so. 59 | // 60 | // All message types (TextMessage, BinaryMessage, CloseMessage, PingMessage and 61 | // PongMessage) are supported. 62 | NextWriter(messageType int) (io.WriteCloser, error) 63 | // WritePreparedMessage writes prepared message into connection. 64 | WritePreparedMessage(pm *websocket.PreparedMessage) error 65 | // WriteMessage is a helper method for getting a writer using NextWriter, 66 | // writing the message and closing the writer. 67 | WriteMessage(messageType int, data []byte) error 68 | // SetWriteDeadline sets the write deadline on the underlying network 69 | // connection. After a write has timed out, the websocket state is corrupt and 70 | // all future writes will return an error. A zero value for t means writes will 71 | // not time out. 72 | SetWriteDeadline(t time.Time) error 73 | // NextReader returns the next data message received from the peer. The 74 | // returned messageType is either TextMessage or BinaryMessage. 75 | // 76 | // There can be at most one open reader on a connection. NextReader discards 77 | // the previous message if the application has not already consumed it. 78 | // 79 | // Applications must break out of the application's read loop when this method 80 | // returns a non-nil error value. Errors returned from this method are 81 | // permanent. Once this method returns a non-nil error, all subsequent calls to 82 | // this method return the same error. 83 | NextReader() (messageType int, r io.Reader, err error) 84 | // ReadMessage is a helper method for getting a reader using NextReader and 85 | // reading from that reader to a buffer. 86 | ReadMessage() (messageType int, p []byte, err error) 87 | // SetReadDeadline sets the read deadline on the underlying network connection. 88 | // After a read has timed out, the websocket connection state is corrupt and 89 | // all future reads will return an error. A zero value for t means reads will 90 | // not time out. 91 | SetReadDeadline(t time.Time) error 92 | // SetReadLimit sets the maximum size in bytes for a message read from the peer. If a 93 | // message exceeds the limit, the connection sends a close message to the peer 94 | // and returns ErrReadLimit to the application. 95 | SetReadLimit(limit int64) 96 | // CloseHandler returns the current close handler 97 | CloseHandler() func(code int, text string) error 98 | // SetCloseHandler sets the handler for close messages received from the peer. 99 | // The code argument to h is the received close code or CloseNoStatusReceived 100 | // if the close message is empty. The default close handler sends a close 101 | // message back to the peer. 102 | // 103 | // The handler function is called from the NextReader, ReadMessage and message 104 | // reader Read methods. The application must read the connection to process 105 | // close messages as described in the section on Control Messages above. 106 | // 107 | // The connection read methods return a CloseError when a close message is 108 | // received. Most applications should handle close messages as part of their 109 | // normal error handling. Applications should only set a close handler when the 110 | // application must perform some action before sending a close message back to 111 | // the peer. 112 | SetCloseHandler(h func(code int, text string) error) 113 | // PingHandler returns the current ping handler 114 | PingHandler() func(appData string) error 115 | // SetPingHandler sets the handler for ping messages received from the peer. 116 | // The appData argument to h is the PING message application data. The default 117 | // ping handler sends a pong to the peer. 118 | // 119 | // The handler function is called from the NextReader, ReadMessage and message 120 | // reader Read methods. The application must read the connection to process 121 | // ping messages as described in the section on Control Messages above. 122 | SetPingHandler(h func(appData string) error) 123 | // PongHandler returns the current pong handler 124 | PongHandler() func(appData string) error 125 | // SetPongHandler sets the handler for pong messages received from the peer. 126 | // The appData argument to h is the PONG message application data. The default 127 | // pong handler does nothing. 128 | // 129 | // The handler function is called from the NextReader, ReadMessage and message 130 | // reader Read methods. The application must read the connection to process 131 | // pong messages as described in the section on Control Messages above. 132 | SetPongHandler(h func(appData string) error) 133 | // UnderlyingConn returns the internal net.Conn. This can be used to further 134 | // modifications to connection specific flags. 135 | UnderlyingConn() net.Conn 136 | // EnableWriteCompression enables and disables write compression of 137 | // subsequent text and binary messages. This function is a noop if 138 | // compression was not negotiated with the peer. 139 | EnableWriteCompression(enable bool) 140 | // SetCompressionLevel sets the flate compression level for subsequent text and 141 | // binary messages. This function is a noop if compression was not negotiated 142 | // with the peer. See the compress/flate package for a description of 143 | // compression levels. 144 | SetCompressionLevel(level int) error 145 | } 146 | -------------------------------------------------------------------------------- /fluent/client/ws/ws_suite_test.go: -------------------------------------------------------------------------------- 1 | package ws_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestWs(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Ws Suite") 13 | } 14 | -------------------------------------------------------------------------------- /fluent/client/ws_client.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package client 26 | 27 | import ( 28 | "bytes" 29 | "crypto/tls" 30 | "errors" 31 | "io" 32 | "net/http" 33 | "sync" 34 | 35 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 36 | "github.com/IBM/fluent-forward-go/fluent/client/ws/ext" 37 | "github.com/IBM/fluent-forward-go/fluent/protocol" 38 | "github.com/gorilla/websocket" 39 | 40 | "github.com/tinylib/msgp/msgp" 41 | ) 42 | 43 | //go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -generate 44 | 45 | const ( 46 | AuthorizationHeader = "Authorization" 47 | ) 48 | 49 | // Expose message types as defined in underlying websocket library 50 | const ( 51 | TextMessage = websocket.TextMessage 52 | BinaryMessage = websocket.BinaryMessage 53 | ) 54 | 55 | //counterfeiter:generate . WSConnectionFactory 56 | type WSConnectionFactory interface { 57 | New() (ext.Conn, error) 58 | NewSession(ws.Connection) *WSSession 59 | } 60 | 61 | type IAMAuthInfo struct { 62 | token string 63 | mutex sync.RWMutex 64 | } 65 | 66 | // IAMToken returns the current token value. It is thread safe. 67 | func (ai *IAMAuthInfo) IAMToken() string { 68 | ai.mutex.RLock() 69 | defer ai.mutex.RUnlock() 70 | 71 | return ai.token 72 | } 73 | 74 | // SetIAMToken updates the token returned by IAMToken(). It is thread safe. 75 | func (ai *IAMAuthInfo) SetIAMToken(token string) { 76 | ai.mutex.Lock() 77 | defer ai.mutex.Unlock() 78 | 79 | ai.token = token 80 | } 81 | 82 | func NewIAMAuthInfo(token string) *IAMAuthInfo { 83 | return &IAMAuthInfo{token: token} 84 | } 85 | 86 | // WSSession represents a single websocket connection. 87 | type WSSession struct { 88 | URL string 89 | Connection ws.Connection 90 | } 91 | 92 | // DefaultWSConnectionFactory is used by the client if no other 93 | // ConnectionFactory is provided. 94 | type DefaultWSConnectionFactory struct { 95 | URL string 96 | AuthInfo *IAMAuthInfo 97 | TLSConfig *tls.Config 98 | Header http.Header 99 | } 100 | 101 | func (wcf *DefaultWSConnectionFactory) New() (ext.Conn, error) { 102 | var ( 103 | dialer websocket.Dialer 104 | header = http.Header{} 105 | ) 106 | 107 | // set additional custom headers. here we do not validate 108 | // header names and values. Caller should make sure the 109 | // headers provided are not conflict with protocols 110 | if wcf.Header != nil { 111 | header = wcf.Header 112 | } 113 | 114 | if wcf.AuthInfo != nil && len(wcf.AuthInfo.IAMToken()) > 0 { 115 | header.Set(AuthorizationHeader, wcf.AuthInfo.IAMToken()) 116 | } 117 | 118 | if wcf.TLSConfig != nil { 119 | dialer.TLSClientConfig = wcf.TLSConfig 120 | } 121 | 122 | conn, resp, err := dialer.Dial(wcf.URL, header) 123 | if resp != nil && resp.Body != nil { 124 | defer resp.Body.Close() 125 | 126 | bodyBytes, readErr := io.ReadAll(resp.Body) 127 | if readErr != nil { 128 | err = readErr 129 | } 130 | 131 | if resp.StatusCode >= 300 { 132 | err = NewWSConnError(err, resp.StatusCode, string(bodyBytes)) 133 | } 134 | } 135 | 136 | return conn, err 137 | } 138 | 139 | func (wcf *DefaultWSConnectionFactory) NewSession(connection ws.Connection) *WSSession { 140 | return &WSSession{ 141 | URL: wcf.URL, 142 | Connection: connection, 143 | } 144 | } 145 | 146 | type WSConnectionOptions struct { 147 | ws.ConnectionOptions 148 | Factory WSConnectionFactory 149 | } 150 | 151 | // WSClient manages the lifetime of a single websocket connection. 152 | type WSClient struct { 153 | ConnectionFactory WSConnectionFactory 154 | ConnectionOptions ws.ConnectionOptions 155 | session *WSSession 156 | errLock sync.RWMutex 157 | sessionLock sync.RWMutex 158 | err error 159 | } 160 | 161 | func NewWS(opts WSConnectionOptions) *WSClient { 162 | if opts.Factory == nil { 163 | opts.Factory = &DefaultWSConnectionFactory{ 164 | URL: "127.0.0.1:8083", 165 | } 166 | } 167 | 168 | return &WSClient{ 169 | ConnectionOptions: opts.ConnectionOptions, 170 | ConnectionFactory: opts.Factory, 171 | } 172 | } 173 | 174 | func (c *WSClient) setErr(err error) { 175 | c.errLock.Lock() 176 | defer c.errLock.Unlock() 177 | 178 | c.err = err 179 | } 180 | 181 | func (c *WSClient) getErr() error { 182 | c.errLock.RLock() 183 | defer c.errLock.RUnlock() 184 | 185 | return c.err 186 | } 187 | 188 | // Session provides the web socket session instance 189 | func (c *WSClient) Session() *WSSession { 190 | c.sessionLock.RLock() 191 | defer c.sessionLock.RUnlock() 192 | 193 | return c.session 194 | } 195 | 196 | // connect is for internal use and should be called within 197 | // the scope of an acquired 'c.sessionLock.Lock()' 198 | // 199 | // extracted for internal re-use. 200 | func (c *WSClient) connect() error { 201 | conn, err := c.ConnectionFactory.New() 202 | if err != nil { 203 | return err 204 | } 205 | 206 | connection, err := ws.NewConnection(conn, c.ConnectionOptions) 207 | if err != nil { 208 | return err 209 | } 210 | 211 | c.session = c.ConnectionFactory.NewSession(connection) 212 | 213 | go func() { 214 | // There is a race condition where session is set to nil before 215 | // Listen is called. This check resolves segfaults during tests, 216 | // but there's still a gap where session can be nullified before 217 | // Listen is invoked. The odds of that happening outside of tests 218 | // is extremely small; e.g., who will call Dis/Reconnect immediately 219 | // after calling Connect? 220 | if c.Session() == nil { 221 | return 222 | } 223 | 224 | // Starts the async read. If there is a read error, it is set so that 225 | // it is returned the next time Send is called. That should be 226 | // sufficient for most cases where the client cares only about sending. 227 | // If the client really cares about handling reads, they will define a 228 | // custom ReadHandler that will receive the error synchronously. 229 | if err := c.session.Connection.Listen(); err != nil { 230 | c.setErr(err) 231 | } 232 | }() 233 | 234 | return nil 235 | } 236 | 237 | // Connect initializes the Session and Connection objects by opening 238 | // a websocket connection. If AuthInfo is not nil, the token it returns 239 | // will be passed via the "Authentication" header during the initial 240 | // HTTP call. 241 | func (c *WSClient) Connect() error { 242 | c.sessionLock.Lock() 243 | defer c.sessionLock.Unlock() 244 | 245 | if c.session != nil { 246 | return errors.New("a session is already active") 247 | } 248 | 249 | return c.connect() 250 | } 251 | 252 | // Disconnect ends the current Session and terminates its websocket connection. 253 | func (c *WSClient) Disconnect() (err error) { 254 | c.sessionLock.Lock() 255 | defer c.sessionLock.Unlock() 256 | 257 | if c.session != nil && !c.session.Connection.Closed() { 258 | err = c.session.Connection.Close() 259 | } 260 | 261 | c.session = nil 262 | 263 | return 264 | } 265 | 266 | // Reconnect terminates the existing Session and creates a new one. 267 | func (c *WSClient) Reconnect() (err error) { 268 | c.sessionLock.Lock() 269 | defer c.sessionLock.Unlock() 270 | 271 | if c.session != nil && !c.session.Connection.Closed() { 272 | _ = c.session.Connection.Close() 273 | } 274 | 275 | if err = c.connect(); err != nil { 276 | c.session = nil 277 | } 278 | 279 | c.setErr(err) 280 | 281 | return 282 | } 283 | 284 | // Send sends a single msgp.Encodable across the wire. 285 | func (c *WSClient) Send(e protocol.ChunkEncoder) error { 286 | var ( 287 | err error 288 | rawMessageData bytes.Buffer 289 | ) 290 | // Check for an async connection error and return it here. 291 | // In most cases, the client will not care about reading from 292 | // the connection, so checking for the error here is sufficient. 293 | if err = c.getErr(); err != nil { 294 | return err // TODO: wrap this 295 | } 296 | 297 | // prevent this from raise conditions by copy the session pointer 298 | session := c.Session() 299 | if session == nil || session.Connection.Closed() { 300 | return errors.New("no active session") 301 | } 302 | 303 | err = msgp.Encode(&rawMessageData, e) 304 | if err != nil { 305 | return err 306 | } 307 | 308 | bytesData := rawMessageData.Bytes() 309 | // Write function does not accurately return the number of bytes written 310 | // so it would be ineffective to compare 311 | _, err = c.session.Connection.Write(bytesData) 312 | 313 | return err 314 | } 315 | 316 | // SendRaw sends an array of bytes across the wire. 317 | func (c *WSClient) SendRaw(m []byte) error { 318 | // Check for an async connection error and return it here. 319 | // In most cases, the client will not care about reading from 320 | // the connection, so checking for the error here is sufficient. 321 | if err := c.getErr(); err != nil { 322 | return err // TODO: wrap this 323 | } 324 | 325 | // prevent this from raise conditions by copy the session pointer 326 | session := c.Session() 327 | if session == nil || session.Connection.Closed() { 328 | return errors.New("no active session") 329 | } 330 | 331 | _, err := session.Connection.Write(m) 332 | 333 | return err 334 | } 335 | -------------------------------------------------------------------------------- /fluent/client/ws_client_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package client_test 26 | 27 | import ( 28 | "bytes" 29 | "crypto/tls" 30 | "errors" 31 | "math/rand" 32 | "net/http" 33 | "net/http/httptest" 34 | "strings" 35 | 36 | "time" 37 | 38 | "github.com/IBM/fluent-forward-go/fluent/client" 39 | . "github.com/IBM/fluent-forward-go/fluent/client" 40 | fclient "github.com/IBM/fluent-forward-go/fluent/client" 41 | "github.com/IBM/fluent-forward-go/fluent/client/clientfakes" 42 | "github.com/IBM/fluent-forward-go/fluent/client/ws" 43 | "github.com/IBM/fluent-forward-go/fluent/client/ws/ext" 44 | "github.com/IBM/fluent-forward-go/fluent/client/ws/ext/extfakes" 45 | "github.com/IBM/fluent-forward-go/fluent/client/ws/wsfakes" 46 | "github.com/IBM/fluent-forward-go/fluent/protocol" 47 | "github.com/gorilla/websocket" 48 | . "github.com/onsi/ginkgo/v2" 49 | . "github.com/onsi/gomega" 50 | "github.com/tinylib/msgp/msgp" 51 | ) 52 | 53 | var _ = Describe("IAMAuthInfo", func() { 54 | It("gets and sets an IAM token", func() { 55 | iai := NewIAMAuthInfo("a") 56 | Expect(iai.IAMToken()).To(Equal("a")) 57 | iai.SetIAMToken("b") 58 | Expect(iai.IAMToken()).To(Equal("b")) 59 | }) 60 | }) 61 | 62 | var _ = Describe("DefaultWSConnectionFactory", func() { 63 | var ( 64 | svr *httptest.Server 65 | ch chan struct{} 66 | useTLS, testError bool 67 | testHeaders http.Header 68 | customErr *client.WSConnError 69 | ) 70 | 71 | happyHandler := func(ch chan struct{}) http.Handler { 72 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 73 | defer GinkgoRecover() 74 | svrOpts := ws.ConnectionOptions{} 75 | 76 | var upgrader websocket.Upgrader 77 | wc, _ := upgrader.Upgrade(w, r, nil) 78 | 79 | header := r.Header.Get(fclient.AuthorizationHeader) 80 | Expect(header).To(Equal("oi")) 81 | 82 | for k := range testHeaders { 83 | v := r.Header.Get(k) 84 | Expect(v).To(Equal(testHeaders[k][0])) 85 | } 86 | 87 | svrConnection, err := ws.NewConnection(wc, svrOpts) 88 | if err != nil { 89 | Fail("broke") 90 | } 91 | 92 | ch <- struct{}{} 93 | 94 | svrConnection.Close() 95 | }) 96 | } 97 | 98 | sadHandler := func(ch chan struct{}) http.Handler { 99 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 100 | http.Error(w, "broken test", http.StatusInternalServerError) 101 | }) 102 | } 103 | 104 | JustBeforeEach(func() { 105 | ch = make(chan struct{}) 106 | 107 | if useTLS { 108 | svr = httptest.NewTLSServer(happyHandler(ch)) 109 | } else if testError { 110 | svr = httptest.NewTLSServer(sadHandler(ch)) 111 | } else { 112 | svr = httptest.NewServer(happyHandler(ch)) 113 | } 114 | 115 | time.Sleep(5 * time.Millisecond) 116 | }) 117 | 118 | AfterEach(func() { 119 | svr.Close() 120 | testHeaders = nil 121 | }) 122 | 123 | It("sends auth headers", func() { 124 | u := "ws" + strings.TrimPrefix(svr.URL, "http") 125 | 126 | cli := fclient.NewWS(client.WSConnectionOptions{ 127 | Factory: &client.DefaultWSConnectionFactory{ 128 | URL: u, 129 | TLSConfig: &tls.Config{ 130 | InsecureSkipVerify: true, 131 | }, 132 | AuthInfo: NewIAMAuthInfo("oi"), 133 | }, 134 | }) 135 | 136 | Expect(cli.Connect()).ToNot(HaveOccurred()) 137 | Eventually(ch).Should(Receive()) 138 | Expect(cli.Disconnect()).ToNot(HaveOccurred()) 139 | }) 140 | 141 | It("sends auth headers with additional header", func() { 142 | u := "ws" + strings.TrimPrefix(svr.URL, "http") 143 | 144 | testHeaders = http.Header{ 145 | "User-Agent": []string{"xxxx:1.0.5"}, // user agent 146 | "X-a": []string{""}, // empty value 147 | "X-b": []string{"value"}, // some string value 148 | } 149 | 150 | cli := fclient.NewWS(client.WSConnectionOptions{ 151 | Factory: &client.DefaultWSConnectionFactory{ 152 | URL: u, 153 | TLSConfig: &tls.Config{ 154 | InsecureSkipVerify: true, 155 | }, 156 | AuthInfo: NewIAMAuthInfo("oi"), 157 | Header: testHeaders, 158 | }, 159 | }) 160 | 161 | Expect(cli.Connect()).ToNot(HaveOccurred()) 162 | Eventually(ch).Should(Receive()) 163 | Expect(cli.Disconnect()).ToNot(HaveOccurred()) 164 | }) 165 | 166 | When("sends wrong url, expects error", func() { 167 | 168 | BeforeEach(func() { 169 | testError = true 170 | }) 171 | 172 | It("returns error", func() { 173 | u := "ws" + strings.TrimPrefix(svr.URL, "http") 174 | 175 | cli := fclient.NewWS(client.WSConnectionOptions{ 176 | Factory: &client.DefaultWSConnectionFactory{ 177 | URL: u + "/test", 178 | TLSConfig: &tls.Config{ 179 | InsecureSkipVerify: true, 180 | }, 181 | AuthInfo: NewIAMAuthInfo("oi"), 182 | }, 183 | }) 184 | 185 | err := cli.Connect() 186 | Expect(err).To(HaveOccurred()) 187 | 188 | Expect(errors.As(err, &customErr)).To(BeTrue()) 189 | Expect(customErr.StatusCode).To(Equal(http.StatusInternalServerError)) 190 | Expect(customErr.ConnErr.Error()).To(ContainSubstring("websocket: bad handshake")) 191 | Expect(customErr.ResponseBody).To(ContainSubstring("broken test")) 192 | Expect(customErr.IsRetryable()).To(BeTrue()) 193 | }) 194 | 195 | }) 196 | 197 | When("the factory is configured for TLS", func() { 198 | BeforeEach(func() { 199 | useTLS = true 200 | }) 201 | 202 | It("works", func() { 203 | u := "wss" + strings.TrimPrefix(svr.URL, "https") 204 | 205 | cli := fclient.NewWS(client.WSConnectionOptions{ 206 | Factory: &client.DefaultWSConnectionFactory{ 207 | URL: u, 208 | TLSConfig: &tls.Config{ 209 | InsecureSkipVerify: true, 210 | }, 211 | AuthInfo: NewIAMAuthInfo("oi"), 212 | }, 213 | }) 214 | 215 | err := cli.Connect() 216 | Expect(err).NotTo(HaveOccurred()) 217 | }) 218 | }) 219 | }) 220 | 221 | var _ = Describe("WSClient", func() { 222 | var ( 223 | factory *clientfakes.FakeWSConnectionFactory 224 | client *WSClient 225 | clientSide ext.Conn 226 | conn *wsfakes.FakeConnection 227 | session *WSSession 228 | ) 229 | 230 | BeforeEach(func() { 231 | factory = &clientfakes.FakeWSConnectionFactory{} 232 | client = fclient.NewWS(fclient.WSConnectionOptions{ 233 | Factory: factory, 234 | }) 235 | clientSide = &extfakes.FakeConn{} 236 | conn = &wsfakes.FakeConnection{} 237 | session = &WSSession{Connection: conn} 238 | 239 | Expect(factory.NewCallCount()).To(Equal(0)) 240 | Expect(client.Session()).To(BeNil()) 241 | }) 242 | 243 | JustBeforeEach(func() { 244 | factory.NewReturns(clientSide, nil) 245 | factory.NewSessionReturns(session) 246 | }) 247 | 248 | Describe("Connect", func() { 249 | It("Does not return an error", func() { 250 | Expect(client.Connect()).ToNot(HaveOccurred()) 251 | }) 252 | 253 | It("Gets the connection from the ConnectionFactory", func() { 254 | err := client.Connect() 255 | Expect(err).NotTo(HaveOccurred()) 256 | Expect(factory.NewCallCount()).To(Equal(1)) 257 | Expect(factory.NewSessionCallCount()).To(Equal(1)) 258 | Expect(client.Session()).To(Equal(session)) 259 | Expect(client.Session().Connection).To(Equal(conn)) 260 | }) 261 | 262 | When("the factory returns an error", func() { 263 | var ( 264 | connectionError error 265 | ) 266 | 267 | JustBeforeEach(func() { 268 | connectionError = errors.New("Nope") 269 | factory.NewReturns(nil, connectionError) 270 | }) 271 | 272 | It("Returns an error", func() { 273 | err := client.Connect() 274 | Expect(err).To(HaveOccurred()) 275 | Expect(err).To(BeIdenticalTo(connectionError)) 276 | }) 277 | }) 278 | 279 | }) 280 | 281 | Describe("Disconnect", func() { 282 | When("the session is not nil", func() { 283 | JustBeforeEach(func() { 284 | err := client.Connect() 285 | Expect(err).NotTo(HaveOccurred()) 286 | time.Sleep(100 * time.Millisecond) 287 | }) 288 | 289 | It("closes the connection", func() { 290 | Expect(client.Disconnect()).ToNot(HaveOccurred()) 291 | Expect(conn.CloseCallCount()).To(Equal(1)) 292 | }) 293 | }) 294 | 295 | // When("the session is nil", func() { 296 | // JustBeforeEach(func() { 297 | // client.Session = nil 298 | // }) 299 | 300 | // It("does not error or panic", func() { 301 | // Expect(func() { 302 | // Expect(client.Disconnect()).ToNot(HaveOccurred()) 303 | // }).ToNot(Panic()) 304 | // }) 305 | // }) 306 | }) 307 | 308 | Describe("Reconnect", func() { 309 | JustBeforeEach(func() { 310 | err := client.Connect() 311 | Expect(err).NotTo(HaveOccurred()) 312 | time.Sleep(100 * time.Millisecond) 313 | }) 314 | 315 | It("calls Disconnect and creates a new Session", func() { 316 | Expect(client.Reconnect()).ToNot(HaveOccurred()) 317 | 318 | Expect(conn.CloseCallCount()).To(Equal(1)) 319 | 320 | Expect(factory.NewSessionCallCount()).To(Equal(2)) 321 | Expect(client.Session().Connection).ToNot(BeNil()) 322 | }) 323 | 324 | It("works if no active session", func() { 325 | Expect(client.Disconnect()).ToNot(HaveOccurred()) 326 | Expect(conn.CloseCallCount()).To(Equal(1)) 327 | 328 | Expect(client.Reconnect()).ToNot(HaveOccurred()) 329 | Expect(conn.CloseCallCount()).To(Equal(1)) 330 | 331 | Expect(factory.NewSessionCallCount()).To(Equal(2)) 332 | Expect(client.Session().Connection).ToNot(BeNil()) 333 | }) 334 | }) 335 | 336 | Describe("Send", func() { 337 | var ( 338 | msg protocol.MessageExt 339 | ) 340 | 341 | BeforeEach(func() { 342 | msg = protocol.MessageExt{ 343 | Tag: "foo.bar", 344 | Timestamp: protocol.EventTime{time.Now()}, //nolint 345 | Record: map[string]interface{}{}, 346 | Options: &protocol.MessageOptions{}, 347 | } 348 | }) 349 | 350 | JustBeforeEach(func() { 351 | err := client.Connect() 352 | Expect(err).ToNot(HaveOccurred()) 353 | time.Sleep(100 * time.Millisecond) 354 | }) 355 | 356 | It("Sends the message", func() { 357 | bits, _ := msg.MarshalMsg(nil) 358 | Expect(client.Send(&msg)).ToNot(HaveOccurred()) 359 | 360 | writtenbits := conn.WriteArgsForCall(0) 361 | Expect(bytes.Equal(bits, writtenbits)).To(BeTrue()) 362 | }) 363 | 364 | It("Sends the message", func() { 365 | msgBytes, _ := msg.MarshalMsg(nil) 366 | Expect(client.Send(&msg)).ToNot(HaveOccurred()) 367 | 368 | writtenBytes := conn.WriteArgsForCall(0) 369 | Expect(bytes.Equal(msgBytes, writtenBytes)).To(BeTrue()) 370 | }) 371 | 372 | When("The message is large", func() { 373 | const charset = "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" 374 | 375 | var ( 376 | expectedBytes int 377 | messageSize = 65536 378 | ) 379 | 380 | JustBeforeEach(func() { 381 | seededRand := rand.New( 382 | rand.NewSource(time.Now().UnixNano())) 383 | m := make([]byte, messageSize) 384 | for i := range m { 385 | m[i] = charset[seededRand.Intn(len(charset))] 386 | } 387 | msg.Record = m 388 | 389 | var b bytes.Buffer 390 | Expect(msgp.Encode(&b, &msg)).ToNot(HaveOccurred()) 391 | expectedBytes = len(b.Bytes()) 392 | }) 393 | 394 | It("Sends the correct number of bits", func() { 395 | Expect(client.Send(&msg)).ToNot(HaveOccurred()) 396 | Expect(conn.WriteCallCount()).To(Equal(1)) 397 | writtenBytes := len(conn.WriteArgsForCall(0)) 398 | Expect(writtenBytes).To(Equal(expectedBytes)) 399 | }) 400 | }) 401 | 402 | When("the connection is disconnected", func() { 403 | JustBeforeEach(func() { 404 | err := client.Disconnect() 405 | Expect(err).ToNot(HaveOccurred()) 406 | }) 407 | 408 | It("returns an error", func() { 409 | Expect(client.Send(&msg)).To(MatchError("no active session")) 410 | }) 411 | }) 412 | 413 | When("the connection is closed with an error", func() { 414 | BeforeEach(func() { 415 | conn.ListenReturns(errors.New("BOOM")) 416 | }) 417 | 418 | It("returns the error", func() { 419 | Expect(client.Send(&msg)).To(MatchError("BOOM")) 420 | }) 421 | }) 422 | }) 423 | 424 | Describe("SendRaw", func() { 425 | var ( 426 | bits []byte 427 | ) 428 | 429 | BeforeEach(func() { 430 | bits = []byte("oi") 431 | }) 432 | 433 | JustBeforeEach(func() { 434 | err := client.Connect() 435 | Expect(err).ToNot(HaveOccurred()) 436 | time.Sleep(100 * time.Millisecond) 437 | }) 438 | 439 | It("Sends the message", func() { 440 | Expect(client.SendRaw(bits)).ToNot(HaveOccurred()) 441 | 442 | writtenbits := conn.WriteArgsForCall(0) 443 | Expect(bytes.Equal(bits, writtenbits)).To(BeTrue()) 444 | }) 445 | 446 | When("the connection is disconnected", func() { 447 | JustBeforeEach(func() { 448 | err := client.Disconnect() 449 | Expect(err).ToNot(HaveOccurred()) 450 | }) 451 | 452 | It("returns an error", func() { 453 | Expect(client.SendRaw(bits)).To(MatchError("no active session")) 454 | }) 455 | }) 456 | 457 | When("the connection is closed with an error", func() { 458 | BeforeEach(func() { 459 | conn.ListenReturns(errors.New("BOOM")) 460 | }) 461 | 462 | It("returns the error", func() { 463 | Expect(client.SendRaw(bits)).To(MatchError("BOOM")) 464 | }) 465 | }) 466 | }) 467 | }) 468 | -------------------------------------------------------------------------------- /fluent/fluent_suite_test.go: -------------------------------------------------------------------------------- 1 | package fluent_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestFluent(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Fluent Suite") 13 | } 14 | -------------------------------------------------------------------------------- /fluent/protocol/.message_test.go.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/fluent-forward-go/52a98c7ca3af3f48776bf436c51412adb8ede305/fluent/protocol/.message_test.go.swp -------------------------------------------------------------------------------- /fluent/protocol/chunk.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import ( 28 | "bytes" 29 | "encoding/base64" 30 | "errors" 31 | "fmt" 32 | 33 | "github.com/google/uuid" 34 | "github.com/tinylib/msgp/msgp" 35 | ) 36 | 37 | // ChunkEncoder wraps methods to encode a message and generate 38 | // "chunk" IDs for use with Fluent's chunk-ack protocol. See 39 | // https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#response 40 | // for more information. 41 | type ChunkEncoder interface { 42 | Chunk() (string, error) 43 | EncodeMsg(*msgp.Writer) error 44 | } 45 | 46 | func makeChunkID() (string, error) { 47 | b, err := uuid.New().MarshalBinary() 48 | if err != nil { 49 | return "", err 50 | } 51 | 52 | return base64.StdEncoding.EncodeToString(b), nil 53 | } 54 | 55 | //msgp:ignore GzipCompressor 56 | type ChunkReader struct { 57 | br *bytes.Reader 58 | R *msgp.Reader 59 | } 60 | 61 | func (cr *ChunkReader) Reset(b []byte) { 62 | if cr.br == nil { 63 | cr.br = bytes.NewReader(b) 64 | cr.R = msgp.NewReader(cr.br) 65 | 66 | return 67 | } 68 | 69 | cr.br.Reset(b) 70 | cr.R.Reset(cr.br) 71 | } 72 | 73 | var chunkKeyBits = []byte("chunk") 74 | 75 | // GetChunk searches a marshaled message for the "chunk" 76 | // option value and returns it. The chunk can be used for 77 | // ack checks without the overhead of unmarshalling. 78 | // GetChunk returns an error if no value is found. 79 | func GetChunk(b []byte) (string, error) { 80 | chunkReader := chunkReaderPool.Get().(*ChunkReader) 81 | chunkReader.Reset(b) 82 | reader := chunkReader.R 83 | 84 | defer func() { 85 | chunkReaderPool.Put(chunkReader) 86 | }() 87 | 88 | sz, err := reader.ReadArrayHeader() 89 | if err != nil { 90 | return "", fmt.Errorf("read array header: %w", err) 91 | } 92 | 93 | if sz == 2 { 94 | return "", errors.New("chunk not found") 95 | } 96 | 97 | if err = reader.Skip(); err != nil { 98 | return "", fmt.Errorf("skip tag: %w", err) 99 | } 100 | 101 | t, err := reader.NextType() 102 | if err != nil { 103 | return "", fmt.Errorf("next type: %w", err) 104 | } 105 | 106 | if t == msgp.ExtensionType || t == msgp.IntType { 107 | // this is Message or MessageExt, which is sz 3 108 | // when there are no options 109 | if sz == 3 { 110 | return "", errors.New("chunk not found") 111 | } 112 | 113 | if err = reader.Skip(); err != nil { 114 | return "", fmt.Errorf("skip timestamp: %w", err) 115 | } 116 | } 117 | 118 | if err = reader.Skip(); err != nil { 119 | return "", fmt.Errorf("skip records: %w", err) 120 | } 121 | 122 | if t, err = reader.NextType(); t != msgp.MapType || err != nil { 123 | return "", fmt.Errorf("chunk not found: %w", err) 124 | } 125 | 126 | sz, err = reader.ReadMapHeader() 127 | if err != nil { 128 | return "", fmt.Errorf("read map header: %w", err) 129 | } 130 | 131 | for i := uint32(0); i < sz; i++ { 132 | keyBits, err := reader.ReadMapKeyPtr() 133 | if err != nil { 134 | return "", fmt.Errorf("read map key: %w", err) 135 | } 136 | 137 | if bytes.Equal(keyBits, chunkKeyBits) { 138 | v, err := reader.ReadMapKey(nil) 139 | return string(v), err 140 | } 141 | 142 | // didn't find "chunk", so skip to next key 143 | if err = reader.Skip(); err != nil { 144 | return "", fmt.Errorf("skip value: %w", err) 145 | } 146 | } 147 | 148 | return "", errors.New("chunk not found") 149 | } 150 | -------------------------------------------------------------------------------- /fluent/protocol/chunk_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol_test 26 | 27 | import ( 28 | "encoding/base64" 29 | 30 | . "github.com/onsi/ginkgo/v2" 31 | . "github.com/onsi/gomega" 32 | 33 | "github.com/IBM/fluent-forward-go/fluent/protocol" 34 | ) 35 | 36 | var _ = Describe("Chunk", func() { 37 | Describe("GetChunk", func() { 38 | 39 | It("returns the chunk ID for a byte-encoded message", func() { 40 | msg := protocol.Message{} 41 | expected, err := msg.Chunk() 42 | Expect(err).ToNot(HaveOccurred()) 43 | 44 | bits, err := msg.MarshalMsg(nil) 45 | Expect(err).ToNot(HaveOccurred()) 46 | 47 | chunk, err := protocol.GetChunk(bits) 48 | Expect(err).ToNot(HaveOccurred()) 49 | Expect(chunk).To(Equal(expected)) 50 | }) 51 | 52 | It("returns an error when chunk is not found", func() { 53 | msg := protocol.Message{} 54 | 55 | bits, err := msg.MarshalMsg(nil) 56 | Expect(err).ToNot(HaveOccurred()) 57 | 58 | _, err = protocol.GetChunk(bits) 59 | Expect(err.Error()).To(ContainSubstring("chunk not found")) 60 | }) 61 | }) 62 | 63 | Describe("Messages", func() { 64 | When("Chunk is called", func() { 65 | It("works as expected", func() { 66 | msg := &protocol.Message{} 67 | msg.Chunk() 68 | bits, _ := msg.MarshalMsg(nil) 69 | raw := protocol.RawMessage(bits) 70 | 71 | for _, ce := range []protocol.ChunkEncoder{raw, &protocol.Message{}, &protocol.MessageExt{}, &protocol.ForwardMessage{}, &protocol.PackedForwardMessage{}} { 72 | chunk, err := ce.Chunk() 73 | Expect(err).ToNot(HaveOccurred()) 74 | Expect(chunk).ToNot(BeEmpty()) 75 | chunk2, err := ce.Chunk() 76 | Expect(err).ToNot(HaveOccurred()) 77 | Expect(chunk).To(Equal(chunk2)) 78 | 79 | b, err := base64.StdEncoding.DecodeString(chunk) 80 | Expect(err).ToNot(HaveOccurred()) 81 | Expect(b).ToNot(BeEmpty()) 82 | } 83 | }) 84 | }) 85 | }) 86 | }) 87 | -------------------------------------------------------------------------------- /fluent/protocol/forward_message.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import "github.com/tinylib/msgp/msgp" 28 | 29 | //go:generate msgp 30 | 31 | // ForwardMessage is used in Forward mode to send multiple events in a single 32 | // msgpack array within a single request. 33 | // 34 | //msgp:tuple ForwardMessage 35 | //mgsp:test ignore ForwardMessage 36 | //msgp:encode ignore ForwardMessage 37 | //msgp:decode ignore ForwardMessage 38 | //msgp:marshal ignore ForwardMessage 39 | //msgp:unmarshal ignore ForwardMessage 40 | //msgp:size ignore ForwardMessage 41 | //msgp:test ignore ForwardMessage 42 | type ForwardMessage struct { 43 | // Tag is a dot-delimted string used to categorize events 44 | Tag string 45 | // Entries is the set of event objects to be carried in this message 46 | Entries EntryList 47 | // Options - used to control server behavior. Same as above, may need to 48 | // switch to interface{} or similar at some point. 49 | Options *MessageOptions 50 | } 51 | 52 | // NewForwardMessage creates a ForwardMessage from the supplied 53 | // tag, EntryList, and MessageOptions. this function will set 54 | // Options.Size to the length of the entry list. 55 | func NewForwardMessage( 56 | tag string, 57 | entries EntryList, 58 | ) *ForwardMessage { 59 | lenEntries := len(entries) 60 | 61 | pfm := &ForwardMessage{ 62 | Tag: tag, 63 | Entries: entries, 64 | } 65 | 66 | pfm.Options = &MessageOptions{ 67 | Size: &lenEntries, 68 | } 69 | 70 | return pfm 71 | } 72 | 73 | func (fm *ForwardMessage) EncodeMsg(dc *msgp.Writer) error { 74 | size := 2 75 | if fm.Options != nil { 76 | size = 3 77 | } 78 | 79 | err := dc.WriteArrayHeader(uint32(size)) 80 | if err != nil { 81 | return msgp.WrapError(err, "Array Header") 82 | } 83 | 84 | err = dc.WriteString(fm.Tag) 85 | if err != nil { 86 | return msgp.WrapError(err, "Tag") 87 | } 88 | 89 | err = fm.Entries.EncodeMsg(dc) 90 | if err != nil { 91 | return err 92 | } 93 | 94 | // if the options were included, inlcude them in our encoded message 95 | if size == 3 { 96 | err = fm.Options.EncodeMsg(dc) 97 | if err != nil { 98 | return err 99 | } 100 | } 101 | 102 | return nil 103 | } 104 | 105 | func (fm *ForwardMessage) DecodeMsg(dc *msgp.Reader) error { 106 | sz, err := dc.ReadArrayHeader() 107 | if err != nil { 108 | return msgp.WrapError(err, "Array Header") 109 | } 110 | 111 | if fm.Tag, err = dc.ReadString(); err != nil { 112 | return msgp.WrapError(err, "Tag") 113 | } 114 | 115 | fm.Entries = EntryList{} 116 | if err = fm.Entries.DecodeMsg(dc); err != nil { 117 | return msgp.WrapError(err, "Entries") 118 | } 119 | 120 | // has three elements only when options are included 121 | if sz == 3 { 122 | if t, err := dc.NextType(); t == msgp.NilType || err != nil { 123 | if err != nil { 124 | return msgp.WrapError(err, "Options") 125 | } 126 | 127 | return dc.ReadNil() 128 | } 129 | 130 | fm.Options = &MessageOptions{} 131 | if err = fm.Options.DecodeMsg(dc); err != nil { 132 | return msgp.WrapError(err, "Options") 133 | } 134 | } 135 | 136 | return nil 137 | } 138 | 139 | func (fm *ForwardMessage) MarshalMsg(bits []byte) ([]byte, error) { 140 | var ( 141 | sz uint32 142 | err error 143 | ) 144 | 145 | if fm.Options != nil { 146 | sz = 3 147 | } else { 148 | sz = 2 149 | } 150 | 151 | bits = msgp.AppendArrayHeader(bits, sz) 152 | bits = msgp.AppendString(bits, fm.Tag) 153 | 154 | bits, err = fm.Entries.MarshalMsg(bits) 155 | if err != nil { 156 | return bits, err 157 | } 158 | 159 | if sz == 3 { 160 | bits, err = fm.Options.MarshalMsg(bits) 161 | } 162 | 163 | return bits, err 164 | } 165 | 166 | func (fm *ForwardMessage) UnmarshalMsg(bits []byte) ([]byte, error) { 167 | var ( 168 | sz uint32 169 | err error 170 | ) 171 | 172 | if sz, bits, err = msgp.ReadArrayHeaderBytes(bits); err != nil { 173 | return bits, msgp.WrapError(err, "Array Header") 174 | } 175 | 176 | if fm.Tag, bits, err = msgp.ReadStringBytes(bits); err != nil { 177 | return bits, msgp.WrapError(err, "Tag") 178 | } 179 | 180 | fm.Entries = EntryList{} 181 | if bits, err = fm.Entries.UnmarshalMsg(bits); err != nil { 182 | return bits, err 183 | } 184 | 185 | // has three elements only when options are included 186 | if sz == 3 { 187 | if t := msgp.NextType(bits); t == msgp.NilType { 188 | return msgp.ReadNilBytes(bits) 189 | } 190 | 191 | fm.Options = &MessageOptions{} 192 | if bits, err = fm.Options.UnmarshalMsg(bits); err != nil { 193 | return bits, msgp.WrapError(err, "Options") 194 | } 195 | } 196 | 197 | return bits, err 198 | } 199 | 200 | // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message 201 | func (fm *ForwardMessage) Msgsize() (s int) { 202 | s = 1 + msgp.StringPrefixSize + len(fm.Tag) + fm.Entries.Msgsize() 203 | if fm.Options != nil { 204 | s += fm.Options.Msgsize() 205 | } 206 | 207 | return 208 | } 209 | 210 | func (fm *ForwardMessage) Chunk() (string, error) { 211 | if fm.Options == nil { 212 | fm.Options = &MessageOptions{} 213 | } 214 | 215 | if fm.Options.Chunk != "" { 216 | return fm.Options.Chunk, nil 217 | } 218 | 219 | chunk, err := makeChunkID() 220 | fm.Options.Chunk = chunk 221 | 222 | return chunk, err 223 | } 224 | -------------------------------------------------------------------------------- /fluent/protocol/forward_message_gen.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | -------------------------------------------------------------------------------- /fluent/protocol/forward_message_gen_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | -------------------------------------------------------------------------------- /fluent/protocol/forward_message_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol_test 26 | 27 | import ( 28 | "bytes" 29 | "io/ioutil" 30 | "time" 31 | 32 | . "github.com/onsi/ginkgo/v2" 33 | . "github.com/onsi/gomega" 34 | "github.com/tinylib/msgp/msgp" 35 | 36 | "github.com/IBM/fluent-forward-go/fluent/protocol" 37 | ) 38 | 39 | var _ = Describe("ForwardMessage", func() { 40 | var ( 41 | fwdmsg *protocol.ForwardMessage 42 | ) 43 | 44 | BeforeEach(func() { 45 | bits, err := ioutil.ReadFile("protocolfakes/forwarded_records.msgpack.bin") 46 | Expect(err).ToNot(HaveOccurred()) 47 | fwdmsg = &protocol.ForwardMessage{} 48 | _, err = fwdmsg.UnmarshalMsg(bits) 49 | Expect(err).NotTo(HaveOccurred()) 50 | entries := []protocol.EntryExt{ 51 | { 52 | Timestamp: protocol.EventTime{time.Now()}, 53 | Record: map[string]interface{}{ 54 | "foo": "bar", 55 | "george": "jungle", 56 | }, 57 | }, 58 | { 59 | Timestamp: protocol.EventTime{time.Now()}, 60 | Record: map[string]interface{}{ 61 | "foo": "kablooie", 62 | "george": "frank", 63 | }, 64 | }, 65 | } 66 | 67 | fwdmsg = protocol.NewForwardMessage("foo", entries) 68 | Expect(*fwdmsg.Options.Size).To(Equal(len(entries))) 69 | }) 70 | 71 | Describe("Unmarshaling", func() { 72 | testMarshalling := func(msg *protocol.ForwardMessage, opts *protocol.MessageOptions) { 73 | msg.Options = opts 74 | b, err := msg.MarshalMsg(nil) 75 | Expect(err).NotTo(HaveOccurred()) 76 | 77 | var unmfwd protocol.ForwardMessage 78 | _, err = unmfwd.UnmarshalMsg(b) 79 | Expect(err).NotTo(HaveOccurred()) 80 | 81 | if opts == nil { 82 | Expect(unmfwd.Options).To(BeNil()) 83 | } else { 84 | Expect(unmfwd.Options).ToNot(BeNil()) 85 | } 86 | Expect(unmfwd.Tag).To(Equal("foo")) 87 | Expect(unmfwd.Entries[0].Timestamp.Time.Equal(msg.Entries[0].Timestamp.Time)).To(BeTrue()) 88 | Expect(unmfwd.Entries[0].Record).To(HaveKeyWithValue("foo", "bar")) 89 | Expect(unmfwd.Entries[0].Record).To(HaveKeyWithValue("george", "jungle")) 90 | Expect(unmfwd.Entries[1].Timestamp.Time.Equal(msg.Entries[1].Timestamp.Time)).To(BeTrue()) 91 | Expect(unmfwd.Entries[1].Record).To(HaveKeyWithValue("foo", "kablooie")) 92 | Expect(unmfwd.Entries[1].Record).To(HaveKeyWithValue("george", "frank")) 93 | } 94 | 95 | It("Marshals and unmarshals correctly", func() { 96 | testMarshalling(fwdmsg, nil) 97 | testMarshalling(fwdmsg, &protocol.MessageOptions{}) 98 | }) 99 | 100 | testEncodingDecoding := func(msg *protocol.ForwardMessage, opts *protocol.MessageOptions) { 101 | var buf bytes.Buffer 102 | en := msgp.NewWriter(&buf) 103 | 104 | msg.Options = opts 105 | err := msg.EncodeMsg(en) 106 | Expect(err).NotTo(HaveOccurred()) 107 | en.Flush() 108 | 109 | var unmfwd protocol.ForwardMessage 110 | re := msgp.NewReader(&buf) 111 | err = unmfwd.DecodeMsg(re) 112 | Expect(err).NotTo(HaveOccurred()) 113 | 114 | if opts == nil { 115 | Expect(unmfwd.Options).To(BeNil()) 116 | } else { 117 | Expect(unmfwd.Options).ToNot(BeNil()) 118 | } 119 | Expect(unmfwd.Tag).To(Equal("foo")) 120 | Expect(unmfwd.Entries[0].Timestamp.Time.Equal(msg.Entries[0].Timestamp.Time)).To(BeTrue()) 121 | Expect(unmfwd.Entries[0].Record).To(HaveKeyWithValue("foo", "bar")) 122 | Expect(unmfwd.Entries[0].Record).To(HaveKeyWithValue("george", "jungle")) 123 | Expect(unmfwd.Entries[1].Timestamp.Time.Equal(msg.Entries[1].Timestamp.Time)).To(BeTrue()) 124 | Expect(unmfwd.Entries[1].Record).To(HaveKeyWithValue("foo", "kablooie")) 125 | Expect(unmfwd.Entries[1].Record).To(HaveKeyWithValue("george", "frank")) 126 | } 127 | 128 | It("Encodes and decodes correctly", func() { 129 | testEncodingDecoding(fwdmsg, nil) 130 | testEncodingDecoding(fwdmsg, &protocol.MessageOptions{}) 131 | }) 132 | 133 | It("Properly deserializes real fluentbit messages with no options", func() { 134 | bits, err := ioutil.ReadFile("protocolfakes/forwarded_records.msgpack.bin") 135 | Expect(err).ToNot(HaveOccurred()) 136 | 137 | fwdmsg := protocol.ForwardMessage{} 138 | _, err = fwdmsg.UnmarshalMsg(bits) 139 | Expect(err).NotTo(HaveOccurred()) 140 | }) 141 | }) 142 | }) 143 | -------------------------------------------------------------------------------- /fluent/protocol/handshake.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import ( 28 | "crypto/sha512" 29 | "encoding/hex" 30 | "errors" 31 | "io" 32 | ) 33 | 34 | //go:generate msgp 35 | 36 | // ========= 37 | // HANDSHAKE 38 | // ========= 39 | 40 | const ( 41 | MsgTypeHelo = "HELO" 42 | MsgTypePing = "PING" 43 | MsgTypePong = "PONG" 44 | ) 45 | 46 | // Remember that the handshake flow is like this: 47 | // 1. Server -> HELO -> Client 48 | // 2. Client -> PING -> Server 49 | // 3. Server -> PONG -> Client 50 | 51 | // NewHelo returns a Helo message with the specified options. 52 | // if opts is nil, then a nonce is generated, auth is left empty, and 53 | // keepalive is true. 54 | func NewHelo(opts *HeloOpts) *Helo { 55 | h := Helo{MessageType: MsgTypeHelo} 56 | if opts == nil { 57 | h.Options = &HeloOpts{ 58 | Keepalive: true, 59 | } 60 | } else { 61 | h.Options = opts 62 | } 63 | 64 | return &h 65 | } 66 | 67 | // Helo is the initial handshake message, sent by the server and received 68 | // by the client. Client will respond with a Ping. 69 | // 70 | //msgp:tuple Helo 71 | type Helo struct { 72 | MessageType string 73 | Options *HeloOpts 74 | } 75 | 76 | type HeloOpts struct { 77 | Nonce []byte `msg:"nonce"` 78 | Auth []byte `msg:"auth"` 79 | Keepalive bool `msg:"keepalive"` 80 | } 81 | 82 | // NewPing returns a PING message. The digest is computed 83 | // from the hostname, key, salt, and nonce using SHA512. 84 | func NewPing(hostname string, sharedKey, salt, nonce []byte) (*Ping, error) { 85 | return makePing(hostname, sharedKey, salt, nonce) 86 | } 87 | 88 | // NewPingWithAuth returns a PING message containing the username and password 89 | // to be used for authentication. The digest is computed 90 | // from the hostname, key, salt, and nonce using SHA512. 91 | func NewPingWithAuth(hostname string, sharedKey, salt, nonce []byte, username, password string) (*Ping, error) { 92 | return makePing(hostname, sharedKey, salt, nonce, username, password) 93 | } 94 | 95 | func makePing(hostname string, sharedKey, salt, nonce []byte, creds ...string) (*Ping, error) { 96 | hexDigest, err := computeHexDigest(salt, hostname, nonce, sharedKey) 97 | 98 | p := Ping{ 99 | MessageType: MsgTypePing, 100 | ClientHostname: hostname, 101 | SharedKeySalt: salt, 102 | SharedKeyHexDigest: hexDigest, 103 | } 104 | 105 | if len(creds) >= 2 { 106 | p.Username = creds[0] 107 | p.Password = creds[1] 108 | } 109 | 110 | return &p, err 111 | } 112 | 113 | // Ping is the response message sent by the client after receiving a 114 | // Helo from the server. Server will respond with a Pong. 115 | // 116 | //msgp:tuple Ping 117 | type Ping struct { 118 | MessageType string 119 | ClientHostname string 120 | SharedKeySalt []byte 121 | SharedKeyHexDigest string 122 | Username string 123 | Password string 124 | } 125 | 126 | // NewPong returns a PONG message. AuthResult indicates 127 | // whether the credentials presented by the client were accepted and therefore 128 | // whether the client can continue using the connection, switching from 129 | // handshake mode to sending events. 130 | // As with the PING, the digest is computed 131 | // from the hostname, key, salt, and nonce using SHA512. 132 | // Server implementations must use the nonce created for the initial Helo and 133 | // the salt sent by the client in the Ping. 134 | func NewPong(authResult bool, reason string, hostname string, sharedKey []byte, 135 | helo *Helo, ping *Ping) (*Pong, error) { 136 | if helo == nil || ping == nil { 137 | return nil, errors.New("Either helo or ping is nil") 138 | } 139 | 140 | if helo.Options == nil { 141 | return nil, errors.New("Helo has a nil options field") 142 | } 143 | 144 | hexDigest, err := computeHexDigest(ping.SharedKeySalt, hostname, helo.Options.Nonce, sharedKey) 145 | 146 | p := Pong{ 147 | MessageType: MsgTypePong, 148 | AuthResult: authResult, 149 | Reason: reason, 150 | ServerHostname: hostname, 151 | SharedKeyHexDigest: hexDigest, 152 | } 153 | 154 | return &p, err 155 | } 156 | 157 | // Pong is the response message sent by the server after receiving a 158 | // Ping from the client. A Pong concludes the handshake. 159 | // 160 | //msgp:tuple Pong 161 | type Pong struct { 162 | MessageType string 163 | AuthResult bool 164 | Reason string 165 | ServerHostname string 166 | SharedKeyHexDigest string 167 | } 168 | 169 | // ValidatePingDigest validates that the digest contained in the PING message 170 | // is valid for the client hostname (as contained in the PING). 171 | // Returns a non-nil error if validation fails, nil otherwise. 172 | func ValidatePingDigest(p *Ping, key, nonce []byte) error { 173 | return validateDigest(p.SharedKeyHexDigest, key, nonce, p.SharedKeySalt, p.ClientHostname) 174 | } 175 | 176 | // ValidatePongDigest validates that the digest contained in the PONG message 177 | // is valid for the server hostname (as contained in the PONG). 178 | // Returns a non-nil error if validation fails, nil otherwise. 179 | func ValidatePongDigest(p *Pong, key, nonce, salt []byte) error { 180 | return validateDigest(p.SharedKeyHexDigest, key, nonce, salt, p.ServerHostname) 181 | } 182 | 183 | func validateDigest(received string, key, nonce, salt []byte, hostname string) error { 184 | expected, err := computeHexDigest(salt, hostname, nonce, key) 185 | if err != nil { 186 | return err 187 | } 188 | 189 | if received != expected { 190 | return errors.New("No match") 191 | } 192 | 193 | return nil 194 | } 195 | 196 | func computeHexDigest(salt []byte, hostname string, nonce, sharedKey []byte) (string, error) { 197 | h := sha512.New() 198 | h.Write(salt) 199 | 200 | _, err := io.WriteString(h, hostname) 201 | if err != nil { 202 | return "", err 203 | } 204 | 205 | h.Write(nonce) 206 | h.Write(sharedKey) 207 | sum := h.Sum(nil) 208 | hexOut := make([]byte, hex.EncodedLen(len(sum))) 209 | hex.Encode(hexOut, sum) 210 | stringValue := string(hexOut[:]) 211 | 212 | return stringValue, err 213 | } 214 | -------------------------------------------------------------------------------- /fluent/protocol/handshake_gen_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | 5 | import ( 6 | "bytes" 7 | "testing" 8 | 9 | "github.com/tinylib/msgp/msgp" 10 | ) 11 | 12 | func TestMarshalUnmarshalHelo(t *testing.T) { 13 | v := Helo{} 14 | bts, err := v.MarshalMsg(nil) 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | left, err := v.UnmarshalMsg(bts) 19 | if err != nil { 20 | t.Fatal(err) 21 | } 22 | if len(left) > 0 { 23 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 24 | } 25 | 26 | left, err = msgp.Skip(bts) 27 | if err != nil { 28 | t.Fatal(err) 29 | } 30 | if len(left) > 0 { 31 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 32 | } 33 | } 34 | 35 | func BenchmarkMarshalMsgHelo(b *testing.B) { 36 | v := Helo{} 37 | b.ReportAllocs() 38 | b.ResetTimer() 39 | for i := 0; i < b.N; i++ { 40 | v.MarshalMsg(nil) 41 | } 42 | } 43 | 44 | func BenchmarkAppendMsgHelo(b *testing.B) { 45 | v := Helo{} 46 | bts := make([]byte, 0, v.Msgsize()) 47 | bts, _ = v.MarshalMsg(bts[0:0]) 48 | b.SetBytes(int64(len(bts))) 49 | b.ReportAllocs() 50 | b.ResetTimer() 51 | for i := 0; i < b.N; i++ { 52 | bts, _ = v.MarshalMsg(bts[0:0]) 53 | } 54 | } 55 | 56 | func BenchmarkUnmarshalHelo(b *testing.B) { 57 | v := Helo{} 58 | bts, _ := v.MarshalMsg(nil) 59 | b.ReportAllocs() 60 | b.SetBytes(int64(len(bts))) 61 | b.ResetTimer() 62 | for i := 0; i < b.N; i++ { 63 | _, err := v.UnmarshalMsg(bts) 64 | if err != nil { 65 | b.Fatal(err) 66 | } 67 | } 68 | } 69 | 70 | func TestEncodeDecodeHelo(t *testing.T) { 71 | v := Helo{} 72 | var buf bytes.Buffer 73 | msgp.Encode(&buf, &v) 74 | 75 | m := v.Msgsize() 76 | if buf.Len() > m { 77 | t.Log("WARNING: TestEncodeDecodeHelo Msgsize() is inaccurate") 78 | } 79 | 80 | vn := Helo{} 81 | err := msgp.Decode(&buf, &vn) 82 | if err != nil { 83 | t.Error(err) 84 | } 85 | 86 | buf.Reset() 87 | msgp.Encode(&buf, &v) 88 | err = msgp.NewReader(&buf).Skip() 89 | if err != nil { 90 | t.Error(err) 91 | } 92 | } 93 | 94 | func BenchmarkEncodeHelo(b *testing.B) { 95 | v := Helo{} 96 | var buf bytes.Buffer 97 | msgp.Encode(&buf, &v) 98 | b.SetBytes(int64(buf.Len())) 99 | en := msgp.NewWriter(msgp.Nowhere) 100 | b.ReportAllocs() 101 | b.ResetTimer() 102 | for i := 0; i < b.N; i++ { 103 | v.EncodeMsg(en) 104 | } 105 | en.Flush() 106 | } 107 | 108 | func BenchmarkDecodeHelo(b *testing.B) { 109 | v := Helo{} 110 | var buf bytes.Buffer 111 | msgp.Encode(&buf, &v) 112 | b.SetBytes(int64(buf.Len())) 113 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 114 | dc := msgp.NewReader(rd) 115 | b.ReportAllocs() 116 | b.ResetTimer() 117 | for i := 0; i < b.N; i++ { 118 | err := v.DecodeMsg(dc) 119 | if err != nil { 120 | b.Fatal(err) 121 | } 122 | } 123 | } 124 | 125 | func TestMarshalUnmarshalHeloOpts(t *testing.T) { 126 | v := HeloOpts{} 127 | bts, err := v.MarshalMsg(nil) 128 | if err != nil { 129 | t.Fatal(err) 130 | } 131 | left, err := v.UnmarshalMsg(bts) 132 | if err != nil { 133 | t.Fatal(err) 134 | } 135 | if len(left) > 0 { 136 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 137 | } 138 | 139 | left, err = msgp.Skip(bts) 140 | if err != nil { 141 | t.Fatal(err) 142 | } 143 | if len(left) > 0 { 144 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 145 | } 146 | } 147 | 148 | func BenchmarkMarshalMsgHeloOpts(b *testing.B) { 149 | v := HeloOpts{} 150 | b.ReportAllocs() 151 | b.ResetTimer() 152 | for i := 0; i < b.N; i++ { 153 | v.MarshalMsg(nil) 154 | } 155 | } 156 | 157 | func BenchmarkAppendMsgHeloOpts(b *testing.B) { 158 | v := HeloOpts{} 159 | bts := make([]byte, 0, v.Msgsize()) 160 | bts, _ = v.MarshalMsg(bts[0:0]) 161 | b.SetBytes(int64(len(bts))) 162 | b.ReportAllocs() 163 | b.ResetTimer() 164 | for i := 0; i < b.N; i++ { 165 | bts, _ = v.MarshalMsg(bts[0:0]) 166 | } 167 | } 168 | 169 | func BenchmarkUnmarshalHeloOpts(b *testing.B) { 170 | v := HeloOpts{} 171 | bts, _ := v.MarshalMsg(nil) 172 | b.ReportAllocs() 173 | b.SetBytes(int64(len(bts))) 174 | b.ResetTimer() 175 | for i := 0; i < b.N; i++ { 176 | _, err := v.UnmarshalMsg(bts) 177 | if err != nil { 178 | b.Fatal(err) 179 | } 180 | } 181 | } 182 | 183 | func TestEncodeDecodeHeloOpts(t *testing.T) { 184 | v := HeloOpts{} 185 | var buf bytes.Buffer 186 | msgp.Encode(&buf, &v) 187 | 188 | m := v.Msgsize() 189 | if buf.Len() > m { 190 | t.Log("WARNING: TestEncodeDecodeHeloOpts Msgsize() is inaccurate") 191 | } 192 | 193 | vn := HeloOpts{} 194 | err := msgp.Decode(&buf, &vn) 195 | if err != nil { 196 | t.Error(err) 197 | } 198 | 199 | buf.Reset() 200 | msgp.Encode(&buf, &v) 201 | err = msgp.NewReader(&buf).Skip() 202 | if err != nil { 203 | t.Error(err) 204 | } 205 | } 206 | 207 | func BenchmarkEncodeHeloOpts(b *testing.B) { 208 | v := HeloOpts{} 209 | var buf bytes.Buffer 210 | msgp.Encode(&buf, &v) 211 | b.SetBytes(int64(buf.Len())) 212 | en := msgp.NewWriter(msgp.Nowhere) 213 | b.ReportAllocs() 214 | b.ResetTimer() 215 | for i := 0; i < b.N; i++ { 216 | v.EncodeMsg(en) 217 | } 218 | en.Flush() 219 | } 220 | 221 | func BenchmarkDecodeHeloOpts(b *testing.B) { 222 | v := HeloOpts{} 223 | var buf bytes.Buffer 224 | msgp.Encode(&buf, &v) 225 | b.SetBytes(int64(buf.Len())) 226 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 227 | dc := msgp.NewReader(rd) 228 | b.ReportAllocs() 229 | b.ResetTimer() 230 | for i := 0; i < b.N; i++ { 231 | err := v.DecodeMsg(dc) 232 | if err != nil { 233 | b.Fatal(err) 234 | } 235 | } 236 | } 237 | 238 | func TestMarshalUnmarshalPing(t *testing.T) { 239 | v := Ping{} 240 | bts, err := v.MarshalMsg(nil) 241 | if err != nil { 242 | t.Fatal(err) 243 | } 244 | left, err := v.UnmarshalMsg(bts) 245 | if err != nil { 246 | t.Fatal(err) 247 | } 248 | if len(left) > 0 { 249 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 250 | } 251 | 252 | left, err = msgp.Skip(bts) 253 | if err != nil { 254 | t.Fatal(err) 255 | } 256 | if len(left) > 0 { 257 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 258 | } 259 | } 260 | 261 | func BenchmarkMarshalMsgPing(b *testing.B) { 262 | v := Ping{} 263 | b.ReportAllocs() 264 | b.ResetTimer() 265 | for i := 0; i < b.N; i++ { 266 | v.MarshalMsg(nil) 267 | } 268 | } 269 | 270 | func BenchmarkAppendMsgPing(b *testing.B) { 271 | v := Ping{} 272 | bts := make([]byte, 0, v.Msgsize()) 273 | bts, _ = v.MarshalMsg(bts[0:0]) 274 | b.SetBytes(int64(len(bts))) 275 | b.ReportAllocs() 276 | b.ResetTimer() 277 | for i := 0; i < b.N; i++ { 278 | bts, _ = v.MarshalMsg(bts[0:0]) 279 | } 280 | } 281 | 282 | func BenchmarkUnmarshalPing(b *testing.B) { 283 | v := Ping{} 284 | bts, _ := v.MarshalMsg(nil) 285 | b.ReportAllocs() 286 | b.SetBytes(int64(len(bts))) 287 | b.ResetTimer() 288 | for i := 0; i < b.N; i++ { 289 | _, err := v.UnmarshalMsg(bts) 290 | if err != nil { 291 | b.Fatal(err) 292 | } 293 | } 294 | } 295 | 296 | func TestEncodeDecodePing(t *testing.T) { 297 | v := Ping{} 298 | var buf bytes.Buffer 299 | msgp.Encode(&buf, &v) 300 | 301 | m := v.Msgsize() 302 | if buf.Len() > m { 303 | t.Log("WARNING: TestEncodeDecodePing Msgsize() is inaccurate") 304 | } 305 | 306 | vn := Ping{} 307 | err := msgp.Decode(&buf, &vn) 308 | if err != nil { 309 | t.Error(err) 310 | } 311 | 312 | buf.Reset() 313 | msgp.Encode(&buf, &v) 314 | err = msgp.NewReader(&buf).Skip() 315 | if err != nil { 316 | t.Error(err) 317 | } 318 | } 319 | 320 | func BenchmarkEncodePing(b *testing.B) { 321 | v := Ping{} 322 | var buf bytes.Buffer 323 | msgp.Encode(&buf, &v) 324 | b.SetBytes(int64(buf.Len())) 325 | en := msgp.NewWriter(msgp.Nowhere) 326 | b.ReportAllocs() 327 | b.ResetTimer() 328 | for i := 0; i < b.N; i++ { 329 | v.EncodeMsg(en) 330 | } 331 | en.Flush() 332 | } 333 | 334 | func BenchmarkDecodePing(b *testing.B) { 335 | v := Ping{} 336 | var buf bytes.Buffer 337 | msgp.Encode(&buf, &v) 338 | b.SetBytes(int64(buf.Len())) 339 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 340 | dc := msgp.NewReader(rd) 341 | b.ReportAllocs() 342 | b.ResetTimer() 343 | for i := 0; i < b.N; i++ { 344 | err := v.DecodeMsg(dc) 345 | if err != nil { 346 | b.Fatal(err) 347 | } 348 | } 349 | } 350 | 351 | func TestMarshalUnmarshalPong(t *testing.T) { 352 | v := Pong{} 353 | bts, err := v.MarshalMsg(nil) 354 | if err != nil { 355 | t.Fatal(err) 356 | } 357 | left, err := v.UnmarshalMsg(bts) 358 | if err != nil { 359 | t.Fatal(err) 360 | } 361 | if len(left) > 0 { 362 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 363 | } 364 | 365 | left, err = msgp.Skip(bts) 366 | if err != nil { 367 | t.Fatal(err) 368 | } 369 | if len(left) > 0 { 370 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 371 | } 372 | } 373 | 374 | func BenchmarkMarshalMsgPong(b *testing.B) { 375 | v := Pong{} 376 | b.ReportAllocs() 377 | b.ResetTimer() 378 | for i := 0; i < b.N; i++ { 379 | v.MarshalMsg(nil) 380 | } 381 | } 382 | 383 | func BenchmarkAppendMsgPong(b *testing.B) { 384 | v := Pong{} 385 | bts := make([]byte, 0, v.Msgsize()) 386 | bts, _ = v.MarshalMsg(bts[0:0]) 387 | b.SetBytes(int64(len(bts))) 388 | b.ReportAllocs() 389 | b.ResetTimer() 390 | for i := 0; i < b.N; i++ { 391 | bts, _ = v.MarshalMsg(bts[0:0]) 392 | } 393 | } 394 | 395 | func BenchmarkUnmarshalPong(b *testing.B) { 396 | v := Pong{} 397 | bts, _ := v.MarshalMsg(nil) 398 | b.ReportAllocs() 399 | b.SetBytes(int64(len(bts))) 400 | b.ResetTimer() 401 | for i := 0; i < b.N; i++ { 402 | _, err := v.UnmarshalMsg(bts) 403 | if err != nil { 404 | b.Fatal(err) 405 | } 406 | } 407 | } 408 | 409 | func TestEncodeDecodePong(t *testing.T) { 410 | v := Pong{} 411 | var buf bytes.Buffer 412 | msgp.Encode(&buf, &v) 413 | 414 | m := v.Msgsize() 415 | if buf.Len() > m { 416 | t.Log("WARNING: TestEncodeDecodePong Msgsize() is inaccurate") 417 | } 418 | 419 | vn := Pong{} 420 | err := msgp.Decode(&buf, &vn) 421 | if err != nil { 422 | t.Error(err) 423 | } 424 | 425 | buf.Reset() 426 | msgp.Encode(&buf, &v) 427 | err = msgp.NewReader(&buf).Skip() 428 | if err != nil { 429 | t.Error(err) 430 | } 431 | } 432 | 433 | func BenchmarkEncodePong(b *testing.B) { 434 | v := Pong{} 435 | var buf bytes.Buffer 436 | msgp.Encode(&buf, &v) 437 | b.SetBytes(int64(buf.Len())) 438 | en := msgp.NewWriter(msgp.Nowhere) 439 | b.ReportAllocs() 440 | b.ResetTimer() 441 | for i := 0; i < b.N; i++ { 442 | v.EncodeMsg(en) 443 | } 444 | en.Flush() 445 | } 446 | 447 | func BenchmarkDecodePong(b *testing.B) { 448 | v := Pong{} 449 | var buf bytes.Buffer 450 | msgp.Encode(&buf, &v) 451 | b.SetBytes(int64(buf.Len())) 452 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 453 | dc := msgp.NewReader(rd) 454 | b.ReportAllocs() 455 | b.ResetTimer() 456 | for i := 0; i < b.N; i++ { 457 | err := v.DecodeMsg(dc) 458 | if err != nil { 459 | b.Fatal(err) 460 | } 461 | } 462 | } 463 | -------------------------------------------------------------------------------- /fluent/protocol/message.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import ( 28 | "time" 29 | 30 | "github.com/tinylib/msgp/msgp" 31 | ) 32 | 33 | //go:generate msgp 34 | 35 | // Message is used to send a single event at a time 36 | // 37 | //msgp:tuple Message 38 | //msgp:decode ignore Message 39 | //msgp:unmarshal ignore Message 40 | //msgp:size ignore Message 41 | //msgp:test ignore Message 42 | type Message struct { 43 | // Tag is a dot-delimited string used to categorize events 44 | Tag string 45 | Timestamp int64 46 | Record interface{} 47 | // Options - used to control server behavior. 48 | Options *MessageOptions 49 | } 50 | 51 | // NewMessage creates a Message from the supplied 52 | // tag and record. The record object must be a map or 53 | // struct. Objects that implement the msgp.Encodable 54 | // interface will be the most performant. Timestamp is 55 | // set to time.Now().UTC() and marshaled with second 56 | // precision. 57 | func NewMessage( 58 | tag string, 59 | record interface{}, 60 | ) *Message { 61 | msg := &Message{ 62 | Tag: tag, 63 | Timestamp: time.Now().UTC().Unix(), 64 | Record: record, 65 | } 66 | 67 | return msg 68 | } 69 | 70 | func (msg *Message) DecodeMsg(dc *msgp.Reader) error { 71 | sz, err := dc.ReadArrayHeader() 72 | if err != nil { 73 | return msgp.WrapError(err, "Array Header") 74 | } 75 | 76 | if msg.Tag, err = dc.ReadString(); err != nil { 77 | return msgp.WrapError(err, "Tag") 78 | } 79 | 80 | if msg.Timestamp, err = dc.ReadInt64(); err != nil { 81 | return msgp.WrapError(err, "Timestamp") 82 | } 83 | 84 | if msg.Record, err = dc.ReadIntf(); err != nil { 85 | return msgp.WrapError(err, "Record") 86 | } 87 | 88 | // has four elements only when options are included 89 | if sz == 4 { 90 | if t, err := dc.NextType(); t == msgp.NilType || err != nil { 91 | if err != nil { 92 | return msgp.WrapError(err, "Options") 93 | } 94 | 95 | return dc.ReadNil() 96 | } 97 | 98 | msg.Options = &MessageOptions{} 99 | if err = msg.Options.DecodeMsg(dc); err != nil { 100 | return msgp.WrapError(err, "Options") 101 | } 102 | } 103 | 104 | return nil 105 | } 106 | 107 | func (msg *Message) UnmarshalMsg(bits []byte) ([]byte, error) { 108 | var ( 109 | sz uint32 110 | err error 111 | ) 112 | 113 | if sz, bits, err = msgp.ReadArrayHeaderBytes(bits); err != nil { 114 | return bits, msgp.WrapError(err, "Array Header") 115 | } 116 | 117 | if msg.Tag, bits, err = msgp.ReadStringBytes(bits); err != nil { 118 | return bits, msgp.WrapError(err, "Tag") 119 | } 120 | 121 | if msg.Timestamp, bits, err = msgp.ReadInt64Bytes(bits); err != nil { 122 | return bits, msgp.WrapError(err, "Timestamp") 123 | } 124 | 125 | if msg.Record, bits, err = msgp.ReadIntfBytes(bits); err != nil { 126 | return bits, msgp.WrapError(err, "Record") 127 | } 128 | 129 | // has four elements only when options are included 130 | if sz == 4 { 131 | if t := msgp.NextType(bits); t == msgp.NilType { 132 | return msgp.ReadNilBytes(bits) 133 | } 134 | 135 | msg.Options = &MessageOptions{} 136 | if bits, err = msg.Options.UnmarshalMsg(bits); err != nil { 137 | return bits, msgp.WrapError(err, "Options") 138 | } 139 | } 140 | 141 | return bits, err 142 | } 143 | 144 | // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message 145 | func (msg *Message) Msgsize() (s int) { 146 | s = 1 + msgp.StringPrefixSize + len(msg.Tag) + msgp.Int64Size + msgp.GuessSize(msg.Record) 147 | if msg.Options != nil { 148 | s += msg.Options.Msgsize() 149 | } 150 | 151 | return s 152 | } 153 | 154 | func (msg *Message) Chunk() (string, error) { 155 | if msg.Options == nil { 156 | msg.Options = &MessageOptions{} 157 | } 158 | 159 | if msg.Options.Chunk != "" { 160 | return msg.Options.Chunk, nil 161 | } 162 | 163 | chunk, err := makeChunkID() 164 | msg.Options.Chunk = chunk 165 | 166 | return chunk, err 167 | } 168 | 169 | // MessageExt 170 | // 171 | //msgp:tuple MessageExt 172 | //msgp:decode ignore MessageExt 173 | //msgp:unmarshal ignore MessageExt 174 | //msgp:size ignore MessageExt 175 | //msgp:test ignore MessageExt 176 | type MessageExt struct { 177 | Tag string 178 | Timestamp EventTime `msg:"eventTime,extension"` 179 | Record interface{} 180 | Options *MessageOptions 181 | } 182 | 183 | // NewMessageExt creates a MessageExt from the supplied 184 | // tag and record. The record object must be a map or 185 | // struct. Objects that implement the msgp.Encodable 186 | // interface will be the most performant. Timestamp is 187 | // set to time.Now().UTC() and marshaled with subsecond 188 | // precision. 189 | func NewMessageExt( 190 | tag string, 191 | record interface{}, 192 | ) *MessageExt { 193 | msg := &MessageExt{ 194 | Tag: tag, 195 | Timestamp: EventTimeNow(), 196 | Record: record, 197 | } 198 | 199 | return msg 200 | } 201 | 202 | func (msg *MessageExt) DecodeMsg(dc *msgp.Reader) error { 203 | sz, err := dc.ReadArrayHeader() 204 | if err != nil { 205 | return msgp.WrapError(err, "Array Header") 206 | } 207 | 208 | if msg.Tag, err = dc.ReadString(); err != nil { 209 | return msgp.WrapError(err, "Tag") 210 | } 211 | 212 | if err = dc.ReadExtension(&msg.Timestamp); err != nil { 213 | return msgp.WrapError(err, "Timestamp") 214 | } 215 | 216 | if msg.Record, err = dc.ReadIntf(); err != nil { 217 | return msgp.WrapError(err, "Record") 218 | } 219 | 220 | // has four elements only when options are included 221 | if sz == 4 { 222 | if t, err := dc.NextType(); t == msgp.NilType || err != nil { 223 | if err != nil { 224 | return msgp.WrapError(err, "Options") 225 | } 226 | 227 | return dc.ReadNil() 228 | } 229 | 230 | msg.Options = &MessageOptions{} 231 | if err = msg.Options.DecodeMsg(dc); err != nil { 232 | return msgp.WrapError(err, "Options") 233 | } 234 | } 235 | 236 | return nil 237 | } 238 | 239 | func (msg *MessageExt) UnmarshalMsg(bits []byte) ([]byte, error) { 240 | var ( 241 | sz uint32 242 | err error 243 | ) 244 | 245 | if sz, bits, err = msgp.ReadArrayHeaderBytes(bits); err != nil { 246 | return bits, msgp.WrapError(err, "Array Header") 247 | } 248 | 249 | if msg.Tag, bits, err = msgp.ReadStringBytes(bits); err != nil { 250 | return bits, msgp.WrapError(err, "Tag") 251 | } 252 | 253 | if bits, err = msgp.ReadExtensionBytes(bits, &msg.Timestamp); err != nil { 254 | return bits, msgp.WrapError(err, "Timestamp") 255 | } 256 | 257 | if msg.Record, bits, err = msgp.ReadIntfBytes(bits); err != nil { 258 | return bits, msgp.WrapError(err, "Record") 259 | } 260 | 261 | // has four elements only when options are included 262 | if sz == 4 { 263 | if t := msgp.NextType(bits); t == msgp.NilType { 264 | return msgp.ReadNilBytes(bits) 265 | } 266 | 267 | msg.Options = &MessageOptions{} 268 | if bits, err = msg.Options.UnmarshalMsg(bits); err != nil { 269 | return bits, msgp.WrapError(err, "Options") 270 | } 271 | } 272 | 273 | return bits, err 274 | } 275 | 276 | // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message 277 | func (msg *MessageExt) Msgsize() (s int) { 278 | s = 1 + msgp.StringPrefixSize + len(msg.Tag) + msgp.ExtensionPrefixSize + msg.Timestamp.Len() + msgp.GuessSize(msg.Record) 279 | if msg.Options != nil { 280 | s += msg.Options.Msgsize() 281 | } 282 | 283 | return 284 | } 285 | 286 | func (msg *MessageExt) Chunk() (string, error) { 287 | if msg.Options == nil { 288 | msg.Options = &MessageOptions{} 289 | } 290 | 291 | if msg.Options.Chunk != "" { 292 | return msg.Options.Chunk, nil 293 | } 294 | 295 | chunk, err := makeChunkID() 296 | msg.Options.Chunk = chunk 297 | 298 | return chunk, err 299 | } 300 | -------------------------------------------------------------------------------- /fluent/protocol/message_gen.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | 5 | import ( 6 | "github.com/tinylib/msgp/msgp" 7 | ) 8 | 9 | // EncodeMsg implements msgp.Encodable 10 | func (z *Message) EncodeMsg(en *msgp.Writer) (err error) { 11 | // array header, size 4 12 | err = en.Append(0x94) 13 | if err != nil { 14 | return 15 | } 16 | err = en.WriteString(z.Tag) 17 | if err != nil { 18 | err = msgp.WrapError(err, "Tag") 19 | return 20 | } 21 | err = en.WriteInt64(z.Timestamp) 22 | if err != nil { 23 | err = msgp.WrapError(err, "Timestamp") 24 | return 25 | } 26 | err = en.WriteIntf(z.Record) 27 | if err != nil { 28 | err = msgp.WrapError(err, "Record") 29 | return 30 | } 31 | if z.Options == nil { 32 | err = en.WriteNil() 33 | if err != nil { 34 | return 35 | } 36 | } else { 37 | err = z.Options.EncodeMsg(en) 38 | if err != nil { 39 | err = msgp.WrapError(err, "Options") 40 | return 41 | } 42 | } 43 | return 44 | } 45 | 46 | // MarshalMsg implements msgp.Marshaler 47 | func (z *Message) MarshalMsg(b []byte) (o []byte, err error) { 48 | o = msgp.Require(b, z.Msgsize()) 49 | // array header, size 4 50 | o = append(o, 0x94) 51 | o = msgp.AppendString(o, z.Tag) 52 | o = msgp.AppendInt64(o, z.Timestamp) 53 | o, err = msgp.AppendIntf(o, z.Record) 54 | if err != nil { 55 | err = msgp.WrapError(err, "Record") 56 | return 57 | } 58 | if z.Options == nil { 59 | o = msgp.AppendNil(o) 60 | } else { 61 | o, err = z.Options.MarshalMsg(o) 62 | if err != nil { 63 | err = msgp.WrapError(err, "Options") 64 | return 65 | } 66 | } 67 | return 68 | } 69 | 70 | // EncodeMsg implements msgp.Encodable 71 | func (z *MessageExt) EncodeMsg(en *msgp.Writer) (err error) { 72 | // array header, size 4 73 | err = en.Append(0x94) 74 | if err != nil { 75 | return 76 | } 77 | err = en.WriteString(z.Tag) 78 | if err != nil { 79 | err = msgp.WrapError(err, "Tag") 80 | return 81 | } 82 | err = en.WriteExtension(&z.Timestamp) 83 | if err != nil { 84 | err = msgp.WrapError(err, "Timestamp") 85 | return 86 | } 87 | err = en.WriteIntf(z.Record) 88 | if err != nil { 89 | err = msgp.WrapError(err, "Record") 90 | return 91 | } 92 | if z.Options == nil { 93 | err = en.WriteNil() 94 | if err != nil { 95 | return 96 | } 97 | } else { 98 | err = z.Options.EncodeMsg(en) 99 | if err != nil { 100 | err = msgp.WrapError(err, "Options") 101 | return 102 | } 103 | } 104 | return 105 | } 106 | 107 | // MarshalMsg implements msgp.Marshaler 108 | func (z *MessageExt) MarshalMsg(b []byte) (o []byte, err error) { 109 | o = msgp.Require(b, z.Msgsize()) 110 | // array header, size 4 111 | o = append(o, 0x94) 112 | o = msgp.AppendString(o, z.Tag) 113 | o, err = msgp.AppendExtension(o, &z.Timestamp) 114 | if err != nil { 115 | err = msgp.WrapError(err, "Timestamp") 116 | return 117 | } 118 | o, err = msgp.AppendIntf(o, z.Record) 119 | if err != nil { 120 | err = msgp.WrapError(err, "Record") 121 | return 122 | } 123 | if z.Options == nil { 124 | o = msgp.AppendNil(o) 125 | } else { 126 | o, err = z.Options.MarshalMsg(o) 127 | if err != nil { 128 | err = msgp.WrapError(err, "Options") 129 | return 130 | } 131 | } 132 | return 133 | } 134 | -------------------------------------------------------------------------------- /fluent/protocol/message_gen_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | -------------------------------------------------------------------------------- /fluent/protocol/message_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol_test 26 | 27 | import ( 28 | "bytes" 29 | "testing" 30 | 31 | "github.com/stretchr/testify/assert" 32 | "github.com/tinylib/msgp/msgp" 33 | 34 | . "github.com/IBM/fluent-forward-go/fluent/protocol" 35 | ) 36 | 37 | func TestMarshalNewMessage(t *testing.T) { 38 | record := map[string]interface{}{ 39 | "first": "Sir", 40 | "last": "Gawain", 41 | "equipment": []string{ 42 | "sword", 43 | }, 44 | } 45 | msg := NewMessage("tag", record) 46 | assert.Equal(t, msg.Tag, "tag") 47 | assert.Equal(t, msg.Record, record) 48 | assert.Greater(t, msg.Timestamp, int64(0)) 49 | 50 | msgext := NewMessageExt("tag", record) 51 | assert.Equal(t, msgext.Tag, "tag") 52 | assert.Equal(t, msgext.Record, record) 53 | assert.Greater(t, msgext.Timestamp.Time.UTC().Nanosecond(), 0) 54 | } 55 | 56 | func TestMarshalUnmarshalMessage(t *testing.T) { 57 | v := Message{ 58 | Options: &MessageOptions{}, 59 | } 60 | bts, err := v.MarshalMsg(nil) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | left, err := v.UnmarshalMsg(bts) 65 | if err != nil { 66 | t.Fatal(err) 67 | } 68 | if len(left) > 0 { 69 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 70 | } 71 | 72 | left, err = msgp.Skip(bts) 73 | if err != nil { 74 | t.Fatal(err) 75 | } 76 | if len(left) > 0 { 77 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 78 | } 79 | } 80 | 81 | func BenchmarkMarshalMsgMessage(b *testing.B) { 82 | v := Message{ 83 | Options: &MessageOptions{}, 84 | } 85 | b.ReportAllocs() 86 | b.ResetTimer() 87 | for i := 0; i < b.N; i++ { 88 | _, err := v.MarshalMsg(nil) 89 | if err != nil { 90 | b.Error(err) 91 | } 92 | } 93 | } 94 | 95 | func BenchmarkAppendMsgMessage(b *testing.B) { 96 | v := Message{ 97 | Options: &MessageOptions{}, 98 | } 99 | bts := make([]byte, 0, v.Msgsize()) 100 | bts, _ = v.MarshalMsg(bts[0:0]) 101 | b.SetBytes(int64(len(bts))) 102 | b.ReportAllocs() 103 | b.ResetTimer() 104 | for i := 0; i < b.N; i++ { 105 | bts, _ = v.MarshalMsg(bts[0:0]) 106 | } 107 | } 108 | 109 | func BenchmarkUnmarshalMessage(b *testing.B) { 110 | v := Message{ 111 | Options: &MessageOptions{}, 112 | } 113 | bts, _ := v.MarshalMsg(nil) 114 | b.ReportAllocs() 115 | b.SetBytes(int64(len(bts))) 116 | b.ResetTimer() 117 | for i := 0; i < b.N; i++ { 118 | _, err := v.UnmarshalMsg(bts) 119 | if err != nil { 120 | b.Fatal(err) 121 | } 122 | } 123 | } 124 | 125 | func TestEncodeDecodeMessage(t *testing.T) { 126 | v := Message{ 127 | Options: &MessageOptions{}, 128 | } 129 | var buf bytes.Buffer 130 | err := msgp.Encode(&buf, &v) 131 | if err != nil { 132 | t.Error(err) 133 | } 134 | 135 | m := v.Msgsize() 136 | if buf.Len() > m { 137 | t.Log("WARNING: TestEncodeDecodeMessage Msgsize() is inaccurate") 138 | } 139 | 140 | vn := Message{} 141 | err = msgp.Decode(&buf, &vn) 142 | if err != nil { 143 | t.Error(err) 144 | } 145 | 146 | buf.Reset() 147 | err = msgp.Encode(&buf, &v) 148 | if err != nil { 149 | t.Error(err) 150 | } 151 | err = msgp.NewReader(&buf).Skip() 152 | if err != nil { 153 | t.Error(err) 154 | } 155 | } 156 | 157 | func BenchmarkEncodeMessage(b *testing.B) { 158 | v := Message{} 159 | var buf bytes.Buffer 160 | err := msgp.Encode(&buf, &v) 161 | if err != nil { 162 | b.Error(err) 163 | } 164 | b.SetBytes(int64(buf.Len())) 165 | en := msgp.NewWriter(msgp.Nowhere) 166 | b.ReportAllocs() 167 | b.ResetTimer() 168 | for i := 0; i < b.N; i++ { 169 | err := v.EncodeMsg(en) 170 | if err != nil { 171 | b.Error(err) 172 | } 173 | } 174 | en.Flush() 175 | } 176 | 177 | func BenchmarkDecodeMessage(b *testing.B) { 178 | v := Message{} 179 | var buf bytes.Buffer 180 | err := msgp.Encode(&buf, &v) 181 | if err != nil { 182 | b.Error(err) 183 | } 184 | b.SetBytes(int64(buf.Len())) 185 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 186 | dc := msgp.NewReader(rd) 187 | b.ReportAllocs() 188 | b.ResetTimer() 189 | for i := 0; i < b.N; i++ { 190 | err := v.DecodeMsg(dc) 191 | if err != nil { 192 | b.Fatal(err) 193 | } 194 | } 195 | } 196 | 197 | func TestMarshalUnmarshalMessageExt(t *testing.T) { 198 | v := MessageExt{ 199 | Options: &MessageOptions{}, 200 | } 201 | bts, err := v.MarshalMsg(nil) 202 | if err != nil { 203 | t.Fatal(err) 204 | } 205 | left, err := v.UnmarshalMsg(bts) 206 | if err != nil { 207 | t.Fatal(err) 208 | } 209 | if len(left) > 0 { 210 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 211 | } 212 | 213 | left, err = msgp.Skip(bts) 214 | if err != nil { 215 | t.Fatal(err) 216 | } 217 | if len(left) > 0 { 218 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 219 | } 220 | } 221 | 222 | func BenchmarkMarshalMsgMessageExt(b *testing.B) { 223 | v := MessageExt{ 224 | Options: &MessageOptions{}, 225 | } 226 | b.ReportAllocs() 227 | b.ResetTimer() 228 | for i := 0; i < b.N; i++ { 229 | _, err := v.MarshalMsg(nil) 230 | if err != nil { 231 | b.Error(err) 232 | } 233 | } 234 | } 235 | 236 | func BenchmarkAppendMsgMessageExt(b *testing.B) { 237 | v := MessageExt{ 238 | Options: &MessageOptions{}, 239 | } 240 | bts := make([]byte, 0, v.Msgsize()) 241 | bts, _ = v.MarshalMsg(bts[0:0]) 242 | b.SetBytes(int64(len(bts))) 243 | b.ReportAllocs() 244 | b.ResetTimer() 245 | for i := 0; i < b.N; i++ { 246 | bts, _ = v.MarshalMsg(bts[0:0]) 247 | } 248 | } 249 | 250 | func BenchmarkUnmarshalMessageExt(b *testing.B) { 251 | v := MessageExt{ 252 | Options: &MessageOptions{}, 253 | } 254 | bts, _ := v.MarshalMsg(nil) 255 | b.ReportAllocs() 256 | b.SetBytes(int64(len(bts))) 257 | b.ResetTimer() 258 | for i := 0; i < b.N; i++ { 259 | _, err := v.UnmarshalMsg(bts) 260 | if err != nil { 261 | b.Fatal(err) 262 | } 263 | } 264 | } 265 | 266 | func TestEncodeDecodeMessageExt(t *testing.T) { 267 | v := MessageExt{ 268 | Options: &MessageOptions{}, 269 | } 270 | var buf bytes.Buffer 271 | err := msgp.Encode(&buf, &v) 272 | if err != nil { 273 | t.Error(err) 274 | } 275 | 276 | m := v.Msgsize() 277 | if buf.Len() > m { 278 | t.Log("WARNING: TestEncodeDecodeMessageExt Msgsize() is inaccurate") 279 | } 280 | 281 | vn := MessageExt{} 282 | err = msgp.Decode(&buf, &vn) 283 | if err != nil { 284 | t.Error(err) 285 | } 286 | 287 | buf.Reset() 288 | err = msgp.Encode(&buf, &v) 289 | if err != nil { 290 | t.Error(err) 291 | } 292 | err = msgp.NewReader(&buf).Skip() 293 | if err != nil { 294 | t.Error(err) 295 | } 296 | } 297 | 298 | func BenchmarkEncodeMessageExt(b *testing.B) { 299 | v := MessageExt{ 300 | Options: &MessageOptions{}, 301 | } 302 | var buf bytes.Buffer 303 | err := msgp.Encode(&buf, &v) 304 | if err != nil { 305 | b.Error(err) 306 | } 307 | b.SetBytes(int64(buf.Len())) 308 | en := msgp.NewWriter(msgp.Nowhere) 309 | b.ReportAllocs() 310 | b.ResetTimer() 311 | for i := 0; i < b.N; i++ { 312 | err := v.EncodeMsg(en) 313 | if err != nil { 314 | b.Error(err) 315 | } 316 | } 317 | en.Flush() 318 | } 319 | 320 | func BenchmarkDecodeMessageExt(b *testing.B) { 321 | v := MessageExt{ 322 | Options: &MessageOptions{}, 323 | } 324 | var buf bytes.Buffer 325 | err := msgp.Encode(&buf, &v) 326 | if err != nil { 327 | b.Error(err) 328 | } 329 | b.SetBytes(int64(buf.Len())) 330 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 331 | dc := msgp.NewReader(rd) 332 | b.ReportAllocs() 333 | b.ResetTimer() 334 | for i := 0; i < b.N; i++ { 335 | err := v.DecodeMsg(dc) 336 | if err != nil { 337 | b.Fatal(err) 338 | } 339 | } 340 | } 341 | -------------------------------------------------------------------------------- /fluent/protocol/packed_forward_message.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import ( 28 | "bytes" 29 | "compress/gzip" 30 | 31 | "github.com/tinylib/msgp/msgp" 32 | ) 33 | 34 | //go:generate msgp 35 | 36 | // PackedForwardMessage is just like ForwardMessage, except that the events 37 | // are carried as a msgpack binary stream 38 | // 39 | //msgp:tuple PackedForwardMessage 40 | //msgp:decode ignore PackedForwardMessage 41 | //msgp:unmarshal ignore PackedForwardMessage 42 | //msgp:size ignore PackedForwardMessage 43 | //msgp:test ignore PackedForwardMessage 44 | type PackedForwardMessage struct { 45 | // Tag is a dot-delimited string used to categorize events 46 | Tag string 47 | // EventStream is the set of events (entries in Fluent-speak) serialized 48 | // into a msgpack byte stream 49 | EventStream []byte 50 | // Options - used to control server behavior. Same as above, may need to 51 | // switch to interface{} or similar at some point. 52 | Options *MessageOptions 53 | } 54 | 55 | // NewPackedForwardMessage creates a PackedForwardMessage from the supplied 56 | // tag, EntryList, and MessageOptions. Regardless of the options supplied, 57 | // this function will set Options.Size to the length of the entry list. 58 | func NewPackedForwardMessage( 59 | tag string, 60 | entries EntryList, 61 | ) (*PackedForwardMessage, error) { 62 | el := EntryList(entries) //nolint 63 | 64 | bits, err := el.MarshalPacked() 65 | if err != nil { 66 | return nil, err 67 | } 68 | 69 | lenEntries := len(entries) 70 | 71 | pfm := NewPackedForwardMessageFromBytes(tag, bits) 72 | pfm.Options = &MessageOptions{ 73 | Size: &lenEntries, 74 | } 75 | 76 | return pfm, nil 77 | } 78 | 79 | // NewPackedForwardMessageFromBytes creates a PackedForwardMessage from the 80 | // supplied tag, bytes, and MessageOptions. This function does not set 81 | // Options.Size to the length of the entry list. 82 | func NewPackedForwardMessageFromBytes( 83 | tag string, 84 | entries []byte, 85 | ) *PackedForwardMessage { 86 | return &PackedForwardMessage{ 87 | Tag: tag, 88 | EventStream: entries, 89 | } 90 | } 91 | 92 | func (msg *PackedForwardMessage) DecodeMsg(dc *msgp.Reader) error { 93 | sz, err := dc.ReadArrayHeader() 94 | if err != nil { 95 | return msgp.WrapError(err, "Array Header") 96 | } 97 | 98 | if msg.Tag, err = dc.ReadString(); err != nil { 99 | return msgp.WrapError(err, "Tag") 100 | } 101 | 102 | if msg.EventStream, err = dc.ReadBytes(msg.EventStream); err != nil { 103 | return msgp.WrapError(err, "EventStream") 104 | } 105 | 106 | // has three elements only when options are included 107 | if sz == 3 { 108 | if t, _ := dc.NextType(); t == msgp.NilType { 109 | return dc.ReadNil() 110 | } 111 | 112 | msg.Options = &MessageOptions{} 113 | if err = msg.Options.DecodeMsg(dc); err != nil { 114 | return msgp.WrapError(err, "Options") 115 | } 116 | } 117 | 118 | return nil 119 | } 120 | 121 | func (msg *PackedForwardMessage) UnmarshalMsg(bits []byte) ([]byte, error) { 122 | var ( 123 | sz uint32 124 | err error 125 | ) 126 | 127 | if sz, bits, err = msgp.ReadArrayHeaderBytes(bits); err != nil { 128 | return bits, msgp.WrapError(err, "Array Header") 129 | } 130 | 131 | if msg.Tag, bits, err = msgp.ReadStringBytes(bits); err != nil { 132 | return bits, msgp.WrapError(err, "Tag") 133 | } 134 | 135 | if msg.EventStream, bits, err = msgp.ReadBytesBytes(bits, msg.EventStream); err != nil { 136 | return bits, msgp.WrapError(err, "EventStream") 137 | } 138 | 139 | // has three elements only when options are included 140 | if sz == 3 { 141 | if t := msgp.NextType(bits); t == msgp.NilType { 142 | return msgp.ReadNilBytes(bits) 143 | } 144 | 145 | msg.Options = &MessageOptions{} 146 | if bits, err = msg.Options.UnmarshalMsg(bits); err != nil { 147 | return bits, msgp.WrapError(err, "Options") 148 | } 149 | } 150 | 151 | return bits, err 152 | } 153 | 154 | func (msg *PackedForwardMessage) Msgsize() (s int) { 155 | s = 1 + msgp.StringPrefixSize + len(msg.Tag) + msgp.BytesPrefixSize + len(msg.EventStream) 156 | if msg.Options != nil { 157 | s += msg.Options.Msgsize() 158 | } 159 | 160 | return 161 | } 162 | 163 | func (msg *PackedForwardMessage) Chunk() (string, error) { 164 | if msg.Options == nil { 165 | msg.Options = &MessageOptions{} 166 | } 167 | 168 | if msg.Options.Chunk != "" { 169 | return msg.Options.Chunk, nil 170 | } 171 | 172 | chunk, err := makeChunkID() 173 | msg.Options.Chunk = chunk 174 | 175 | return chunk, err 176 | } 177 | 178 | //msgp:ignore GzipCompressor 179 | type GzipCompressor struct { 180 | Buffer *bytes.Buffer 181 | GzipWriter *gzip.Writer 182 | } 183 | 184 | // Write writes to the compression stream. 185 | func (mc *GzipCompressor) Write(bits []byte) error { 186 | _, err := mc.GzipWriter.Write(bits) 187 | 188 | if cerr := mc.GzipWriter.Close(); err == nil { 189 | err = cerr 190 | } 191 | 192 | return err 193 | } 194 | 195 | // Reset resets the buffer to be empty, but it retains the 196 | // underlying storage for use by future writes. 197 | func (mc *GzipCompressor) Reset() { 198 | if mc.Buffer == nil { 199 | mc.Buffer = new(bytes.Buffer) 200 | mc.GzipWriter = gzip.NewWriter(mc.Buffer) 201 | 202 | return 203 | } 204 | 205 | mc.Buffer.Reset() 206 | mc.GzipWriter.Reset(mc.Buffer) 207 | } 208 | 209 | // Bytes returns the gzip-compressed byte stream. 210 | func (mc *GzipCompressor) Bytes() []byte { 211 | return mc.Buffer.Bytes() 212 | } 213 | 214 | // NewCompressedPackedForwardMessage returns a PackedForwardMessage with a 215 | // gzip-compressed byte stream. 216 | func NewCompressedPackedForwardMessage( 217 | tag string, entries []EntryExt, 218 | ) (*PackedForwardMessage, error) { 219 | el := EntryList(entries) //nolint 220 | 221 | bits, err := el.MarshalPacked() 222 | if err != nil { 223 | return nil, err 224 | } 225 | 226 | lenEntries := len(entries) 227 | 228 | msg, err := NewCompressedPackedForwardMessageFromBytes(tag, bits) 229 | if err == nil { 230 | msg.Options.Size = &lenEntries 231 | } 232 | 233 | return msg, err 234 | } 235 | 236 | // NewCompressedPackedForwardMessageFromBytes returns a PackedForwardMessage 237 | // with a gzip-compressed byte stream. 238 | func NewCompressedPackedForwardMessageFromBytes( 239 | tag string, entries []byte, 240 | ) (*PackedForwardMessage, error) { 241 | mc := compressorPool.Get().(*GzipCompressor) 242 | mc.Reset() 243 | 244 | defer func() { 245 | compressorPool.Put(mc) 246 | }() 247 | 248 | if err := mc.Write(entries); err != nil { 249 | return nil, err 250 | } 251 | 252 | pfm := NewPackedForwardMessageFromBytes(tag, mc.Bytes()) 253 | pfm.Options = &MessageOptions{Compressed: "gzip"} 254 | 255 | return pfm, nil 256 | } 257 | -------------------------------------------------------------------------------- /fluent/protocol/packed_forward_message_gen.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | 5 | import ( 6 | "github.com/tinylib/msgp/msgp" 7 | ) 8 | 9 | // EncodeMsg implements msgp.Encodable 10 | func (z *PackedForwardMessage) EncodeMsg(en *msgp.Writer) (err error) { 11 | // array header, size 3 12 | err = en.Append(0x93) 13 | if err != nil { 14 | return 15 | } 16 | err = en.WriteString(z.Tag) 17 | if err != nil { 18 | err = msgp.WrapError(err, "Tag") 19 | return 20 | } 21 | err = en.WriteBytes(z.EventStream) 22 | if err != nil { 23 | err = msgp.WrapError(err, "EventStream") 24 | return 25 | } 26 | if z.Options == nil { 27 | err = en.WriteNil() 28 | if err != nil { 29 | return 30 | } 31 | } else { 32 | err = z.Options.EncodeMsg(en) 33 | if err != nil { 34 | err = msgp.WrapError(err, "Options") 35 | return 36 | } 37 | } 38 | return 39 | } 40 | 41 | // MarshalMsg implements msgp.Marshaler 42 | func (z *PackedForwardMessage) MarshalMsg(b []byte) (o []byte, err error) { 43 | o = msgp.Require(b, z.Msgsize()) 44 | // array header, size 3 45 | o = append(o, 0x93) 46 | o = msgp.AppendString(o, z.Tag) 47 | o = msgp.AppendBytes(o, z.EventStream) 48 | if z.Options == nil { 49 | o = msgp.AppendNil(o) 50 | } else { 51 | o, err = z.Options.MarshalMsg(o) 52 | if err != nil { 53 | err = msgp.WrapError(err, "Options") 54 | return 55 | } 56 | } 57 | return 58 | } 59 | -------------------------------------------------------------------------------- /fluent/protocol/packed_forward_message_gen_test.go: -------------------------------------------------------------------------------- 1 | package protocol 2 | 3 | // Code generated by github.com/tinylib/msgp DO NOT EDIT. 4 | -------------------------------------------------------------------------------- /fluent/protocol/packed_forward_message_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol_test 26 | 27 | import ( 28 | "bytes" 29 | "testing" 30 | 31 | "github.com/tinylib/msgp/msgp" 32 | 33 | . "github.com/IBM/fluent-forward-go/fluent/protocol" 34 | ) 35 | 36 | func TestMarshalUnmarshalPackedForwardMessage(t *testing.T) { 37 | v := PackedForwardMessage{} 38 | bts, err := v.MarshalMsg(nil) 39 | if err != nil { 40 | t.Fatal(err) 41 | } 42 | left, err := v.UnmarshalMsg(bts) 43 | if err != nil { 44 | t.Fatal(err) 45 | } 46 | if len(left) > 0 { 47 | t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) 48 | } 49 | 50 | left, err = msgp.Skip(bts) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | if len(left) > 0 { 55 | t.Errorf("%d bytes left over after Skip(): %q", len(left), left) 56 | } 57 | } 58 | 59 | func BenchmarkMarshalMsgPackedForwardMessage(b *testing.B) { 60 | v := PackedForwardMessage{} 61 | b.ReportAllocs() 62 | b.ResetTimer() 63 | for i := 0; i < b.N; i++ { 64 | _, err := v.MarshalMsg(nil) 65 | if err != nil { 66 | b.Error(err) 67 | } 68 | } 69 | } 70 | 71 | func BenchmarkAppendMsgPackedForwardMessage(b *testing.B) { 72 | v := PackedForwardMessage{} 73 | bts := make([]byte, 0, v.Msgsize()) 74 | bts, _ = v.MarshalMsg(bts[0:0]) 75 | b.SetBytes(int64(len(bts))) 76 | b.ReportAllocs() 77 | b.ResetTimer() 78 | for i := 0; i < b.N; i++ { 79 | bts, _ = v.MarshalMsg(bts[0:0]) 80 | } 81 | } 82 | 83 | func BenchmarkUnmarshalPackedForwardMessage(b *testing.B) { 84 | v := PackedForwardMessage{} 85 | bts, _ := v.MarshalMsg(nil) 86 | b.ReportAllocs() 87 | b.SetBytes(int64(len(bts))) 88 | b.ResetTimer() 89 | for i := 0; i < b.N; i++ { 90 | _, err := v.UnmarshalMsg(bts) 91 | if err != nil { 92 | b.Fatal(err) 93 | } 94 | } 95 | } 96 | 97 | func TestEncodeDecodePackedForwardMessage(t *testing.T) { 98 | v := PackedForwardMessage{} 99 | var buf bytes.Buffer 100 | err := msgp.Encode(&buf, &v) 101 | if err != nil { 102 | t.Error(err) 103 | } 104 | 105 | m := v.Msgsize() 106 | if buf.Len() > m { 107 | t.Log("WARNING: TestEncodeDecodePackedForwardMessage Msgsize() is inaccurate") 108 | } 109 | 110 | vn := PackedForwardMessage{} 111 | err = msgp.Decode(&buf, &vn) 112 | if err != nil { 113 | t.Error(err) 114 | } 115 | 116 | buf.Reset() 117 | err = msgp.Encode(&buf, &v) 118 | if err != nil { 119 | t.Error(err) 120 | } 121 | err = msgp.NewReader(&buf).Skip() 122 | if err != nil { 123 | t.Error(err) 124 | } 125 | } 126 | 127 | func BenchmarkEncodePackedForwardMessage(b *testing.B) { 128 | v := PackedForwardMessage{} 129 | var buf bytes.Buffer 130 | err := msgp.Encode(&buf, &v) 131 | if err != nil { 132 | b.Error(err) 133 | } 134 | b.SetBytes(int64(buf.Len())) 135 | en := msgp.NewWriter(msgp.Nowhere) 136 | b.ReportAllocs() 137 | b.ResetTimer() 138 | for i := 0; i < b.N; i++ { 139 | err := v.EncodeMsg(en) 140 | if err != nil { 141 | b.Error(err) 142 | } 143 | } 144 | en.Flush() 145 | } 146 | 147 | func BenchmarkDecodePackedForwardMessage(b *testing.B) { 148 | v := PackedForwardMessage{} 149 | var buf bytes.Buffer 150 | err := msgp.Encode(&buf, &v) 151 | if err != nil { 152 | b.Error(err) 153 | } 154 | b.SetBytes(int64(buf.Len())) 155 | rd := msgp.NewEndlessReader(buf.Bytes(), b) 156 | dc := msgp.NewReader(rd) 157 | b.ReportAllocs() 158 | b.ResetTimer() 159 | for i := 0; i < b.N; i++ { 160 | err := v.DecodeMsg(dc) 161 | if err != nil { 162 | b.Fatal(err) 163 | } 164 | } 165 | } 166 | 167 | func TestEncodeDecodeCompressedPackedForwardMessage(t *testing.T) { 168 | bits := make([]byte, 1028) 169 | v, err := NewCompressedPackedForwardMessageFromBytes("foo", bits) 170 | if err != nil { 171 | t.Error(err) 172 | } 173 | 174 | var buf bytes.Buffer 175 | err = msgp.Encode(&buf, v) 176 | if err != nil { 177 | t.Error(err) 178 | } 179 | 180 | m := v.Msgsize() 181 | if buf.Len() > m { 182 | t.Log("WARNING: TestEncodeDecodePackedForwardMessage Msgsize() is inaccurate") 183 | } 184 | 185 | vn := PackedForwardMessage{} 186 | err = msgp.Decode(&buf, &vn) 187 | if err != nil { 188 | t.Error(err) 189 | } 190 | 191 | buf.Reset() 192 | err = msgp.Encode(&buf, v) 193 | if err != nil { 194 | t.Error(err) 195 | } 196 | err = msgp.NewReader(&buf).Skip() 197 | if err != nil { 198 | t.Error(err) 199 | } 200 | } 201 | 202 | func BenchmarkNCFMFB(b *testing.B) { 203 | bits := make([]byte, 1024) 204 | b.ReportAllocs() 205 | b.ResetTimer() 206 | for i := 0; i < b.N; i++ { 207 | _, err := NewCompressedPackedForwardMessageFromBytes("foo", bits) 208 | if err != nil { 209 | b.Error(err) 210 | } 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /fluent/protocol/protocol_suite_test.go: -------------------------------------------------------------------------------- 1 | package protocol_test 2 | 3 | import ( 4 | "testing" 5 | 6 | . "github.com/onsi/ginkgo/v2" 7 | . "github.com/onsi/gomega" 8 | ) 9 | 10 | func TestProtocol(t *testing.T) { 11 | RegisterFailHandler(Fail) 12 | RunSpecs(t, "Protocol Suite") 13 | } 14 | -------------------------------------------------------------------------------- /fluent/protocol/protocolfakes/forwarded_records.msgpack.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/IBM/fluent-forward-go/52a98c7ca3af3f48776bf436c51412adb8ede305/fluent/protocol/protocolfakes/forwarded_records.msgpack.bin -------------------------------------------------------------------------------- /fluent/protocol/transport.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol 26 | 27 | import ( 28 | "bytes" 29 | "encoding/binary" 30 | "errors" 31 | "reflect" 32 | "sync" 33 | "time" 34 | 35 | "github.com/google/uuid" 36 | "github.com/tinylib/msgp/msgp" 37 | ) 38 | 39 | //go:generate msgp 40 | 41 | // ========= 42 | // TRANSPORT 43 | // ========= 44 | 45 | const ( 46 | OptSize string = "size" 47 | OptChunk string = "chunk" 48 | OptCompressed string = "compressed" 49 | OptValGZIP string = "gzip" 50 | 51 | extensionType int8 = 0 52 | eventTimeLen int = 8 53 | ) 54 | 55 | var ( 56 | compressorPool sync.Pool 57 | chunkReaderPool sync.Pool 58 | bufferPool sync.Pool 59 | ) 60 | 61 | func init() { 62 | uuid.EnableRandPool() 63 | 64 | msgp.RegisterExtension(extensionType, func() msgp.Extension { 65 | return new(EventTime) 66 | }) 67 | 68 | compressorPool.New = func() interface{} { 69 | return new(GzipCompressor) 70 | } 71 | 72 | chunkReaderPool.New = func() interface{} { 73 | return new(ChunkReader) 74 | } 75 | 76 | bufferPool.New = func() interface{} { 77 | return new(bytes.Buffer) 78 | } 79 | } 80 | 81 | // EventTime is the fluent-forward representation of a timestamp 82 | type EventTime struct { 83 | time.Time 84 | } 85 | 86 | // EventTimeNow returns an EventTime set to time.Now().UTC(). 87 | func EventTimeNow() EventTime { 88 | return EventTime{ 89 | Time: time.Now().UTC(), 90 | } 91 | } 92 | 93 | func (et *EventTime) ExtensionType() int8 { 94 | return extensionType 95 | } 96 | 97 | func (et *EventTime) Len() int { 98 | return eventTimeLen 99 | } 100 | 101 | // MarshalBinaryTo implements the Extension interface for marshaling an 102 | // EventTime into a byte slice. 103 | func (et *EventTime) MarshalBinaryTo(b []byte) error { 104 | utc := et.UTC() 105 | 106 | // b[0] = 0xD7 107 | // b[1] = 0x00 108 | binary.BigEndian.PutUint32(b, uint32(utc.Unix())) 109 | binary.BigEndian.PutUint32(b[4:], uint32(utc.Nanosecond())) 110 | 111 | return nil 112 | } 113 | 114 | // UnmarshalBinary implements the Extension interface for unmarshaling 115 | // into an EventTime object. 116 | func (et *EventTime) UnmarshalBinary(timeBytes []byte) error { 117 | if len(timeBytes) != eventTimeLen { 118 | return errors.New("Invalid length") 119 | } 120 | 121 | seconds := binary.BigEndian.Uint32(timeBytes) 122 | nanoseconds := binary.BigEndian.Uint32(timeBytes[4:]) 123 | 124 | et.Time = time.Unix(int64(seconds), int64(nanoseconds)) 125 | 126 | return nil 127 | } 128 | 129 | // EntryExt is the basic representation of an individual event, but using the 130 | // msgpack extension format for the timestamp. 131 | // 132 | //msgp:tuple EntryExt 133 | type EntryExt struct { 134 | // Timestamp can contain the timestamp in either seconds or nanoseconds 135 | Timestamp EventTime `msg:"eventTime,extension"` 136 | // Record is the actual event record. The object must be a map or 137 | // struct. Objects that implement the msgp.Encodable interface will 138 | // be the most performant. 139 | Record interface{} 140 | } 141 | 142 | type EntryList []EntryExt 143 | 144 | func (el *EntryList) UnmarshalPacked(bits []byte) ([]byte, error) { 145 | var ( 146 | entry EntryExt 147 | err error 148 | ) 149 | 150 | *el = (*el)[:0] 151 | 152 | for len(bits) > 0 { 153 | if bits, err = entry.UnmarshalMsg(bits); err != nil { 154 | break 155 | } 156 | 157 | *el = append(*el, entry) 158 | } 159 | 160 | return bits, err 161 | } 162 | 163 | func (el EntryList) MarshalPacked() ([]byte, error) { 164 | buf := bufferPool.Get().(*bytes.Buffer) 165 | buf.Reset() 166 | 167 | defer func() { 168 | bufferPool.Put(buf) 169 | }() 170 | 171 | for _, e := range el { 172 | if err := msgp.Encode(buf, e); err != nil { 173 | return nil, err 174 | } 175 | } 176 | 177 | return buf.Bytes(), nil 178 | } 179 | 180 | // Equal compares two EntryList objects and returns true if they have 181 | // exactly the same elements, false otherwise. 182 | func (el EntryList) Equal(e2 EntryList) bool { 183 | if len(el) != len(e2) { 184 | return false 185 | } 186 | 187 | first := make(EntryList, len(el)) 188 | 189 | copy(first, el) 190 | 191 | second := make(EntryList, len(e2)) 192 | 193 | copy(second, e2) 194 | 195 | matches := 0 196 | 197 | for _, ea := range first { 198 | for _, eb := range second { 199 | if ea.Timestamp.Equal(eb.Timestamp.Time) { 200 | // Timestamps equal, check the record 201 | if reflect.DeepEqual(ea.Record, eb.Record) { 202 | matches++ 203 | } 204 | } 205 | } 206 | } 207 | 208 | return matches == len(el) 209 | } 210 | 211 | // EntryExt is the basic representation of an individual event. The timestamp 212 | // is an int64 representing seconds since the epoch (UTC). The initial creator 213 | // of the entry is responsible for converting to UTC. 214 | // 215 | //msgp:tuple Entry 216 | type Entry struct { 217 | // Timestamp can contain the timestamp in either seconds or nanoseconds 218 | Timestamp int64 219 | // Record is the actual event record. 220 | Record interface{} 221 | } 222 | 223 | type MessageOptions struct { 224 | Size *int `msg:"size,omitempty"` 225 | Chunk string `msg:"chunk,omitempty"` 226 | Compressed string `msg:"compressed,omitempty"` 227 | } 228 | 229 | type AckMessage struct { 230 | Ack string `msg:"ack"` 231 | } 232 | 233 | // RawMessage is a ChunkEncoder wrapper for []byte. 234 | // 235 | //msgp:encode ignore RawMessage 236 | type RawMessage []byte 237 | 238 | func (rm RawMessage) EncodeMsg(w *msgp.Writer) error { 239 | if len(rm) == 0 { 240 | return w.WriteNil() 241 | } 242 | 243 | _, err := w.Write([]byte(rm)) 244 | 245 | return err 246 | } 247 | 248 | // Chunk searches the message for the chunk ID. In the case of RawMessage, 249 | // Chunk is read-only. It returns an error if the chunk is not found. 250 | func (rm RawMessage) Chunk() (string, error) { 251 | return GetChunk([]byte(rm)) 252 | } 253 | -------------------------------------------------------------------------------- /fluent/protocol/transport_test.go: -------------------------------------------------------------------------------- 1 | /* 2 | MIT License 3 | 4 | Copyright contributors to the fluent-forward-go project 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | */ 24 | 25 | package protocol_test 26 | 27 | import ( 28 | "fmt" 29 | "strings" 30 | "time" 31 | 32 | . "github.com/onsi/ginkgo/v2" 33 | . "github.com/onsi/gomega" 34 | 35 | "github.com/IBM/fluent-forward-go/fluent/protocol" 36 | ) 37 | 38 | var _ = Describe("Transport", func() { 39 | Describe("EventTime", func() { 40 | var ( 41 | ent protocol.EntryExt 42 | ) 43 | 44 | BeforeEach(func() { 45 | ent = protocol.EntryExt{ 46 | Timestamp: protocol.EventTime{ 47 | Time: time.Unix(int64(1257894000), int64(12340000)), 48 | }, 49 | } 50 | }) 51 | 52 | // This covers both MarshalBinaryTo() and UnmarshalBinary() 53 | It("Marshals and unmarshals correctly", func() { 54 | b, err := ent.MarshalMsg(nil) 55 | 56 | Expect(err).NotTo(HaveOccurred()) 57 | 58 | // This is the msgpack fixext8 encoding for the timestamp 59 | // per the fluent-forward spec: 60 | // D7 == fixext8 61 | // 00 == type 0 62 | // 4AF9F070 == 1257894000 63 | // 00BC4B20 == 12340000 64 | Expect( 65 | strings.Contains(fmt.Sprintf("%X", b), "D7004AF9F07000BC4B20"), 66 | ).To(BeTrue()) 67 | 68 | var unment protocol.EntryExt 69 | _, err = unment.UnmarshalMsg(b) 70 | Expect(err).NotTo(HaveOccurred()) 71 | 72 | Expect(unment.Timestamp.Time.Equal(ent.Timestamp.Time)).To(BeTrue()) 73 | }) 74 | }) 75 | 76 | Describe("EntryList", func() { 77 | var ( 78 | e1 protocol.EntryList 79 | et time.Time 80 | ) 81 | 82 | BeforeEach(func() { 83 | et = time.Now() 84 | e1 = protocol.EntryList{ 85 | { 86 | Timestamp: protocol.EventTime{et}, 87 | Record: map[string]interface{}{ 88 | "foo": "bar", 89 | "george": "jungle", 90 | }, 91 | }, 92 | { 93 | Timestamp: protocol.EventTime{et}, 94 | Record: map[string]interface{}{ 95 | "foo": "kablooie", 96 | "george": "frank", 97 | }, 98 | }, 99 | } 100 | }) 101 | 102 | Describe("Un/MarshalPacked", func() { 103 | var ( 104 | e2 protocol.EntryList 105 | ) 106 | 107 | BeforeEach(func() { 108 | e2 = protocol.EntryList{ 109 | { 110 | Timestamp: protocol.EventTime{et}, 111 | Record: map[string]interface{}{ 112 | "foo": "bar", 113 | "george": "jungle", 114 | }, 115 | }, 116 | { 117 | Timestamp: protocol.EventTime{et}, 118 | Record: map[string]interface{}{ 119 | "foo": "kablooie", 120 | "george": "frank", 121 | }, 122 | }, 123 | } 124 | }) 125 | 126 | It("Can marshal and unmarshal packed entries", func() { 127 | b, err := e2.MarshalPacked() 128 | Expect(err).ToNot(HaveOccurred()) 129 | 130 | el := protocol.EntryList{} 131 | _, err = el.UnmarshalPacked(b) 132 | Expect(err).ToNot(HaveOccurred()) 133 | Expect(el.Equal(e2)).To(BeTrue()) 134 | }) 135 | }) 136 | 137 | Describe("Equal", func() { 138 | var ( 139 | e2 protocol.EntryList 140 | ) 141 | 142 | BeforeEach(func() { 143 | e2 = protocol.EntryList{ 144 | { 145 | Timestamp: protocol.EventTime{et}, 146 | Record: map[string]interface{}{ 147 | "foo": "bar", 148 | "george": "jungle", 149 | }, 150 | }, 151 | { 152 | Timestamp: protocol.EventTime{et}, 153 | Record: map[string]interface{}{ 154 | "foo": "kablooie", 155 | "george": "frank", 156 | }, 157 | }, 158 | } 159 | }) 160 | 161 | It("Returns true", func() { 162 | Expect(e1.Equal(e2)).To(BeTrue()) 163 | }) 164 | 165 | Context("When the lists have different element counts", func() { 166 | BeforeEach(func() { 167 | e2 = e2[:1] 168 | }) 169 | 170 | It("Returns false", func() { 171 | Expect(e1.Equal(e2)).To(BeFalse()) 172 | }) 173 | }) 174 | 175 | Context("When the lists have differing elements", func() { 176 | BeforeEach(func() { 177 | e2[0].Timestamp = protocol.EventTime{et.Add(5 * time.Second)} 178 | }) 179 | 180 | It("Returns false", func() { 181 | Expect(e1.Equal(e2)).To(BeFalse()) 182 | }) 183 | }) 184 | }) 185 | }) 186 | 187 | Describe("NewPackedForwardMessage", func() { 188 | var ( 189 | tag string 190 | entries protocol.EntryList 191 | ) 192 | 193 | BeforeEach(func() { 194 | tag = "foo.bar" 195 | entries = protocol.EntryList{ 196 | { 197 | Timestamp: protocol.EventTime{time.Now()}, 198 | Record: map[string]interface{}{ 199 | "foo": "bar", 200 | "george": "jungle", 201 | }, 202 | }, 203 | { 204 | Timestamp: protocol.EventTime{time.Now()}, 205 | Record: map[string]interface{}{ 206 | "foo": "kablooie", 207 | "george": "frank", 208 | }, 209 | }, 210 | } 211 | }) 212 | 213 | It("Returns a PackedForwardMessage", func() { 214 | msg, err := protocol.NewPackedForwardMessage(tag, entries) 215 | Expect(err).NotTo(HaveOccurred()) 216 | Expect(msg).NotTo(BeNil()) 217 | Expect(*msg.Options.Size).To(Equal(len(entries))) 218 | Expect(msg.Options.Compressed).To(BeEmpty()) 219 | }) 220 | 221 | It("Correctly encodes the entries into a bytestream", func() { 222 | msg, err := protocol.NewPackedForwardMessage(tag, entries) 223 | Expect(err).NotTo(HaveOccurred()) 224 | elist := make(protocol.EntryList, 2) 225 | _, err = elist.UnmarshalPacked(msg.EventStream) 226 | Expect(err).NotTo(HaveOccurred()) 227 | Expect(elist.Equal(entries)).To(BeTrue()) 228 | }) 229 | }) 230 | 231 | Describe("NewCompressedPackedForwardMessage", func() { 232 | var ( 233 | tag string 234 | entries []protocol.EntryExt 235 | ) 236 | 237 | BeforeEach(func() { 238 | tag = "foo.bar" 239 | entries = []protocol.EntryExt{ 240 | { 241 | Timestamp: protocol.EventTime{time.Now()}, 242 | Record: map[string]interface{}{ 243 | "foo": "bar", 244 | "george": "jungle", 245 | }, 246 | }, 247 | { 248 | Timestamp: protocol.EventTime{time.Now()}, 249 | Record: map[string]interface{}{ 250 | "foo": "kablooie", 251 | "george": "frank", 252 | }, 253 | }, 254 | } 255 | }) 256 | 257 | It("Returns a message with a gzip-compressed event stream", func() { 258 | msg, err := protocol.NewCompressedPackedForwardMessage(tag, entries) 259 | Expect(err).ToNot(HaveOccurred()) 260 | Expect(msg).NotTo(BeNil()) 261 | Expect(*msg.Options.Size).To(Equal(len(entries))) 262 | Expect(msg.Options.Compressed).To(Equal("gzip")) 263 | }) 264 | }) 265 | }) 266 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/IBM/fluent-forward-go 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/fluent/fluent-logger-golang v1.8.0 7 | github.com/google/uuid v1.3.0 8 | github.com/gorilla/mux v1.8.0 9 | github.com/gorilla/websocket v1.4.2 10 | github.com/onsi/ginkgo/v2 v2.9.7 11 | github.com/onsi/gomega v1.27.8 12 | github.com/stretchr/testify v1.7.0 13 | github.com/tinylib/msgp v1.1.9 14 | ) 15 | 16 | require ( 17 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect 18 | github.com/davecgh/go-spew v1.1.1 // indirect 19 | github.com/go-logr/logr v1.2.4 // indirect 20 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect 21 | github.com/google/go-cmp v0.6.0 // indirect 22 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect 23 | github.com/kr/pretty v0.3.1 // indirect 24 | github.com/philhofer/fwd v1.1.2 // indirect 25 | github.com/pmezard/go-difflib v1.0.0 // indirect 26 | golang.org/x/net v0.21.0 // indirect 27 | golang.org/x/sys v0.17.0 // indirect 28 | golang.org/x/text v0.14.0 // indirect 29 | golang.org/x/tools v0.18.0 // indirect 30 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 31 | gopkg.in/yaml.v3 v3.0.1 // indirect 32 | ) 33 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= 2 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= 3 | github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= 4 | github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= 5 | github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= 6 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/fluent/fluent-logger-golang v1.8.0 h1:K/fUDqUAItNcdf/Rq7aA2d1apwqsNgNzzInlXZTwK28= 11 | github.com/fluent/fluent-logger-golang v1.8.0/go.mod h1:2/HCT/jTy78yGyeNGQLGQsjF3zzzAuy6Xlk6FCMV5eU= 12 | github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= 13 | github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 14 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= 15 | github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= 16 | github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= 17 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 18 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 19 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= 20 | github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= 21 | github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= 22 | github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 23 | github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= 24 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 25 | github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= 26 | github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 27 | github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= 28 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 29 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 30 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 31 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 32 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 33 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 34 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 35 | github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= 36 | github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= 37 | github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= 38 | github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= 39 | github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= 40 | github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= 41 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 42 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 43 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 44 | github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= 45 | github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= 46 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 47 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 48 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 49 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 50 | github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= 51 | github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= 52 | golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= 53 | golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= 54 | golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 55 | golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= 56 | golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 57 | golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= 58 | golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= 59 | golang.org/x/tools v0.18.0 h1:k8NLag8AGHnn+PHbl7g43CtqZAwG60vZkLqgyZgIHgQ= 60 | golang.org/x/tools v0.18.0/go.mod h1:GL7B4CwcLLeo59yx/9UWWuNOW1n3VZ4f5axWfML7Lcg= 61 | google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= 62 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 63 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 64 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 65 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 66 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 67 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 68 | -------------------------------------------------------------------------------- /scripts/lint.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | golangci-lint run 4 | -------------------------------------------------------------------------------- /test/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | go mod download 4 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin 5 | ${GOPATH}/bin/golangci-lint run ./... 6 | 7 | if [[ "$?" != "0" ]]; then 8 | echo "golangci-lint run ./... failed..." 9 | exit 1 10 | fi 11 | 12 | go test ./... 13 | if [[ "$?" != "0" ]]; then 14 | echo "go test ./... failed..." 15 | exit 1 16 | fi 17 | --------------------------------------------------------------------------------